From 741d12324fec0de05d849a09fd01be9681f66da4 Mon Sep 17 00:00:00 2001
From: Ward Vandewege <ward@jhvc.com>
Date: Tue, 26 May 2020 17:53:03 -0400
Subject: [PATCH] 16430: Refactor to share code between the GKE and Minikube
 tests.

Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward@jhvc.com>
---
 tests/GKE.sh          | 103 ++++------------------------------
 tests/minikube.sh     |  82 +++++----------------------
 tests/test_library.sh | 126 ++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 150 insertions(+), 161 deletions(-)
 create mode 100755 tests/test_library.sh

diff --git a/tests/GKE.sh b/tests/GKE.sh
index e0b7d2b..07ee117 100755
--- a/tests/GKE.sh
+++ b/tests/GKE.sh
@@ -3,56 +3,30 @@
 set -e
 
 DEBUG=$1
+CLUSTERTYPE="GKE"
 
-if [[ "$DEBUG" == "--debug" ]]; then
-  set -x
-fi
+. test_library.sh
 
-testReady() {
+loadK8sIP() {
   set +e
-  ready=0
-  apiReady=0
-  keepProxyReady=0
-  curl --connect-timeout 1 -k -s -H "Authorization: Bearer $MANAGEMENTTOKEN" https://$GKE_IP:444/rails/_health/ping |grep -q OK
-  if [[ $? -eq 0 ]]; then
-    apiReady=1
-  else
-    return
-  fi
-  curl --connect-timeout 1 -k -s -H "Authorization: Bearer $MANAGEMENTTOKEN" https://$GKE_IP:25107/_health/ping |grep -q OK
-  if [[ $? -eq 0 ]]; then
-    keepProxyReady=1
-  fi
-  if [[ $apiReady -eq 1 ]] && [[ $keepProxyReady -eq 1 ]]; then
-    ready=1
+  K8S_IP=`gcloud compute addresses describe arvados-k8s-ip --region us-central1 --format="value(address)" 2>/dev/null`
+  if [[ $? -ne 0 ]]; then
+    K8S_IP=
   fi
   set -e
 }
 
-stopCluster() {
-  echo "Stopping Arvados cluster..."
-  cd $MY_PATH/../charts/arvados
-  helm delete arvados
-
+stopK8s() {
   echo "Stopping k8s cluster on GKE"
   gcloud container clusters delete arvados --zone us-central1-a --quiet
   gcloud compute addresses delete arvados-k8s-ip --region us-central1 --quiet
 }
 
-kubectlStatus() {
-  echo "Current k8s status:"
-  echo "services:"
-  kubectl get svc
-  echo "pods:"
-  kubectl get pods
-  echo
-}
-
-startCluster() {
+startK8s() {
   echo "Starting k8s cluster on GKE"
-  if [[ -z "$GKE_IP" ]]; then
+  if [[ -z "$K8S_IP" ]]; then
     gcloud compute addresses create arvados-k8s-ip --region us-central1
-    GKE_IP=`gcloud compute addresses describe arvados-k8s-ip --region us-central1 --format="value(address)"`
+    loadK8sIP
   fi
   set +e
   CLUSTER=`gcloud container clusters describe arvados --zone us-central1-a 2>/dev/null`
@@ -74,61 +48,6 @@ startCluster() {
     done
   fi
   set -e
-
-  echo "Starting Arvados cluster..."
-  cd $MY_PATH/../charts/arvados
-  ./cert-gen.sh "$GKE_IP"
-
-  helm install arvados . --set externalIP="$GKE_IP"
-
-  echo "Waiting for cluster health OK..."
-  while [ $ready -ne 1 ]; do
-    testReady
-    kubectlStatus
-    sleep 10
-  done
-}
-
-main() {
-  MY_PATH=`pwd`
-  MANAGEMENTTOKEN=`cat $MY_PATH/../charts/arvados/config/config.yml |grep Management |cut -f2 -d ':' |sed -e 's/ //'`
-  set +e
-  GKE_IP=`gcloud compute addresses describe arvados-k8s-ip --region us-central1 --format="value(address)" 2>/dev/null`
-  set -e
-  date
-  # testReady needs $GKE_IP
-  testReady
-
-  if [[ $ready -ne 1 ]]; then
-    startCluster
-  else
-    # create the necessary kubectl context for the running cluster
-    gcloud container clusters get-credentials arvados --zone us-central1-a
-    kubectlStatus
-  fi
-  date
-  echo "cluster health OK"
-
-  export ARVADOS_API_HOST=$GKE_IP:444
-  export ARVADOS_API_HOST_INSECURE=true
-  export ARVADOS_API_TOKEN=`grep superUserSecret $MY_PATH/../charts/arvados/values.yaml |cut -f2 -d\"`
-
-  cd $MY_PATH/cwl-diagnostics-hasher/
-
-  echo "uploading requirements for CWL hasher"
-  arv-put 4xphq-8i9sb-fmwod1qn74cemdp.log.txt  --no-resume
-  echo "uploading Arvados jobs image for CWL hasher"
-  # just in case, clear the arv-put cache first, arv-keepdocker doesn't pass through --no-resume
-  rm -rf ~/.cache/arvados/arv-put
-  echo "running CWL hasher"
-  cwl-runner hasher-workflow.cwl hasher-workflow-job.yml
-  if [[ $? -eq 0 ]]; then
-    echo "Success!"
-  else
-    echo "Test failed!"
-  fi
-
-  stopCluster
 }
 
-main
+run
diff --git a/tests/minikube.sh b/tests/minikube.sh
index bcb8013..a19ce8c 100755
--- a/tests/minikube.sh
+++ b/tests/minikube.sh
@@ -2,85 +2,29 @@
 
 set -e
 
-testReady() {
+DEBUG=$1
+CLUSTERTYPE="minikube"
+
+. test_library.sh
+
+loadK8sIP() {
   set +e
-  ready=0
-  apiReady=0
-  keepProxyReady=0
-  curl -k -s -H "Authorization: Bearer $MANAGEMENTTOKEN" https://`minikube ip`:444/rails/_health/ping |grep -q OK
-  if [[ $? -eq 0 ]]; then
-    apiReady=1
-  fi
-  curl -k -s -H "Authorization: Bearer $MANAGEMENTTOKEN" https://`minikube ip`:25107/_health/ping |grep -q OK
-  if [[ $? -eq 0 ]]; then
-    keepProxyReady=1
-  fi
-  if [[ $apiReady -eq 1 ]] && [[ $keepProxyReady -eq 1 ]]; then
-    ready=1
+  K8S_IP=`minikube ip 2>/dev/null`
+  if [[ $? -ne 0 ]]; then
+    K8S_IP=
   fi
   set -e
 }
 
-stopCluster() {
-  echo "Stopping Arvados cluster..."
-  cd $MY_PATH/../charts/arvados
-  helm delete arvados
-
+stopK8s() {
   echo "Stopping Minikube"
   minikube stop
 }
 
-startCluster() {
+startK8s() {
   echo "Starting Minikube"
   minikube start
-
-  echo "Starting Arvados cluster..."
-  cd $MY_PATH/../charts/arvados
-  ./cert-gen.sh `minikube ip`
-
-  helm install arvados . --set externalIP=`minikube ip`
-
-  ./minikube-external-ip.sh
-
-  echo "Waiting for cluster health OK..."
-  while [ $ready -ne 1 ]; do
-    testReady
-    sleep 1
-  done
-}
-
-main() {
-  MY_PATH=`pwd`
-  MANAGEMENTTOKEN=`cat $MY_PATH/../charts/arvados/config/config.yml |grep Management |cut -f2 -d ':' |sed -e 's/ //'`
-  date
-  testReady
-
-  if [[ $ready -ne 1 ]]; then
-    startCluster
-  fi
-  date
-  echo "cluster health OK"
- 
-  export ARVADOS_API_HOST=`minikube ip`:444
-  export ARVADOS_API_HOST_INSECURE=true
-  export ARVADOS_API_TOKEN=`grep superUserSecret $MY_PATH/../charts/arvados/values.yaml |cut -f2 -d\"`
-
-  cd $MY_PATH/cwl-diagnostics-hasher/
-
-  echo "uploading requirements for CWL hasher"
-  arv-put 4xphq-8i9sb-fmwod1qn74cemdp.log.txt  --no-resume
-  echo "uploading Arvados jobs image for CWL hasher"
-  # just in case, clear the arv-put cache first, arv-keepdocker doesn't pass through --no-resume
-  rm -rf ~/.cache/arvados/arv-put
-  echo "running CWL hasher"
-  cwl-runner hasher-workflow.cwl hasher-workflow-job.yml
-  if [[ $? -eq 0 ]]; then
-    echo "Success!"
-  else
-    echo "Test failed!"
-  fi
-
-  stopCluster
+  loadK8sIP
 }
 
-main
+run
diff --git a/tests/test_library.sh b/tests/test_library.sh
new file mode 100755
index 0000000..f72f289
--- /dev/null
+++ b/tests/test_library.sh
@@ -0,0 +1,126 @@
+#!/bin/bash
+
+if [[ "$DEBUG" == "--debug" ]]; then
+  set -x
+fi
+
+testReady() {
+  loadK8sIP
+  set +e
+  ready=0
+  k8sReady=0
+  apiReady=0
+  keepProxyReady=0
+
+  # Is k8s ready?
+  if [[ -n "$K8S_IP" ]]; then
+    k8sReady=1
+  else
+    set -e
+    return
+  fi
+
+  # Is the Arvados API server ready?
+  curl --connect-timeout 1 -k -s -H "Authorization: Bearer $MANAGEMENTTOKEN" https://$K8S_IP:444/rails/_health/ping |grep -q OK
+  if [[ $? -eq 0 ]]; then
+    apiReady=1
+  else
+    set -e
+    return
+  fi
+
+  # Is the Arvados Keep proxy ready?
+  curl --connect-timeout 1 -k -s -H "Authorization: Bearer $MANAGEMENTTOKEN" https://$K8S_IP:25107/_health/ping |grep -q OK
+  if [[ $? -eq 0 ]]; then
+    keepProxyReady=1
+  else
+    set -e
+    return
+  fi
+
+  # Everything is working
+  ready=1
+  set -e
+}
+
+stopCluster() {
+  echo "Stopping Arvados cluster..."
+  cd $MY_PATH/../charts/arvados
+  helm delete arvados
+
+  stopK8s
+}
+
+kubectlStatus() {
+  echo "Current k8s status:"
+  echo "services:"
+  kubectl get svc
+  echo "pods:"
+  kubectl get pods
+  echo
+}
+
+startCluster() {
+  startK8s
+
+  echo "Starting Arvados cluster..."
+  cd $MY_PATH/../charts/arvados
+  ./cert-gen.sh "$K8S_IP"
+
+  helm install arvados . --set externalIP="$K8S_IP"
+
+  if [[ "$CLUSTERTYPE" == "minikube" ]]; then
+    ./minikube-external-ip.sh
+  fi
+
+  awaitHealthOK
+}
+
+awaitHealthOK() {
+  echo "Waiting for cluster health OK..."
+  while [ $ready -ne 1 ]; do
+    testReady
+    kubectlStatus
+    sleep 2
+  done
+}
+
+run() {
+  MY_PATH=`pwd`
+  MANAGEMENTTOKEN=`cat $MY_PATH/../charts/arvados/config/config.yml |grep Management |cut -f2 -d ':' |sed -e 's/ //'`
+  date
+  testReady
+  if [[ $ready -ne 1 ]]; then
+    startCluster
+  else
+    if [[ "$CLUSTERTYPE" == "GKE" ]]; then
+      # create the necessary kubectl context for the running cluster
+      gcloud container clusters get-credentials arvados --zone us-central1-a
+      kubectlStatus
+    fi
+  fi
+
+  date
+  echo "cluster health OK"
+
+  export ARVADOS_API_HOST=$K8S_IP:444
+  export ARVADOS_API_HOST_INSECURE=true
+  export ARVADOS_API_TOKEN=`grep superUserSecret $MY_PATH/../charts/arvados/values.yaml |cut -f2 -d\"`
+
+  cd $MY_PATH/cwl-diagnostics-hasher/
+
+  echo "uploading requirements for CWL hasher"
+  arv-put 4xphq-8i9sb-fmwod1qn74cemdp.log.txt  --no-resume
+  echo "uploading Arvados jobs image for CWL hasher"
+  # just in case, clear the arv-put cache first, arv-keepdocker doesn't pass through --no-resume
+  rm -rf ~/.cache/arvados/arv-put
+  echo "running CWL hasher"
+  cwl-runner hasher-workflow.cwl hasher-workflow-job.yml
+  if [[ $? -eq 0 ]]; then
+    echo "Success!"
+  else
+    echo "Test failed!"
+  fi
+
+  stopCluster
+}
-- 
2.30.2