sdk/cwl/test_with_arvbox.sh

   1 #!/bin/sh
   2 # Copyright (C) The Arvados Authors. All rights reserved.
   3 #
   4 # SPDX-License-Identifier: Apache-2.0
   5
   6 set -x
   7
   8 if ! which arvbox >/dev/null ; then
   9     export PATH=$PATH:$(readlink -f $(dirname $0)/../../tools/arvbox/bin)
  10 fi
  11
  12 reset_container=1
  13 leave_running=0
  14 config=dev
  15 devcwl=0
  16 tag="latest"
  17 pythoncmd=python3
  18 suite=conformance
  19 runapi=containers
  20
  21 while test -n "$1" ; do
  22     arg="$1"
  23     case "$arg" in
  24         --no-reset-container)
  25             reset_container=0
  26             shift
  27             ;;
  28         --leave-running)
  29             leave_running=1
  30             shift
  31             ;;
  32         --config)
  33             config=$2
  34             shift ; shift
  35             ;;
  36         --tag)
  37             tag=$2
  38             shift ; shift
  39             ;;
  40         --build)
  41             build=1
  42             shift
  43             ;;
  44         --devcwl)
  45             devcwl=1
  46             shift
  47             ;;
  48         --pythoncmd)
  49             pythoncmd=$2
  50             shift ; shift
  51             ;;
  52         --suite)
  53             suite=$2
  54             shift ; shift
  55             ;;
  56         --api)
  57             runapi=$2
  58             shift ; shift
  59             ;;
  60         -h|--help)
  61             echo "$0 [--no-reset-container] [--leave-running] [--config dev|localdemo] [--tag docker_tag] [--build] [--pythoncmd python(2|3)] [--suite (integration|conformance-v1.0|conformance-*)]"
  62             exit
  63             ;;
  64         *)
  65             break
  66             ;;
  67     esac
  68 done
  69
  70 if test -z "$ARVBOX_CONTAINER" ; then
  71    export ARVBOX_CONTAINER=cwltest
  72 fi
  73
  74 if test "$suite" = "conformance" ; then
  75   suite=conformance-v1.0
  76 fi
  77
  78 if test $reset_container = 1 ; then
  79     arvbox stop
  80     docker rm $ARVBOX_CONTAINER
  81     arvbox reset -f
  82 fi
  83
  84 arvbox start $config $tag
  85
  86 arvbox pipe <<EOF
  87 set -eu -o pipefail
  88
  89 . /usr/local/lib/arvbox/common.sh
  90
  91 export PYCMD=$pythoncmd
  92
  93 if test $config = dev ; then
  94   cd /usr/src/arvados/sdk/cwl
  95   \$PYCMD setup.py sdist
  96   pip_install \$(ls -r dist/arvados-cwl-runner-*.tar.gz | head -n1)
  97 fi
  98
  99 set -x
 100
 101 if [ "\$PYCMD" = "python3" ]; then
 102     pip3 install cwltest
 103 else
 104     pip install cwltest
 105 fi
 106
 107 mkdir -p /tmp/cwltest
 108 cd /tmp/cwltest
 109
 110 if [[ "$suite" = "conformance-v1.0" ]] ; then
 111    if ! test -d common-workflow-language ; then
 112      git clone https://github.com/common-workflow-language/common-workflow-language.git
 113    fi
 114    cd common-workflow-language
 115 elif [[ "$suite" =~ conformance-(.*) ]] ; then
 116    version=\${BASH_REMATCH[1]}
 117    if ! test -d cwl-\${version} ; then
 118      git clone https://github.com/common-workflow-language/cwl-\${version}.git
 119    fi
 120    cd cwl-\${version}
 121 elif [[ "$suite" != "integration" ]] ; then
 122    echo "ERROR: unknown suite '$suite'"
 123    exit 1
 124 fi
 125
 126 if [[ "$suite" != "integration" ]] ; then
 127   git pull
 128 fi
 129
 130 export ARVADOS_API_HOST=localhost:8000
 131 export ARVADOS_API_HOST_INSECURE=1
 132 export ARVADOS_API_TOKEN=\$(cat /var/lib/arvados-arvbox/superuser_token)
 133
 134 if test -n "$build" ; then
 135   /usr/src/arvados/build/build-dev-docker-jobs-image.sh
 136 elif test "$tag" = "latest" ; then
 137   arv-keepdocker --pull arvados/jobs $tag
 138 else
 139   set +u
 140   export WORKSPACE=/usr/src/arvados
 141   . /usr/src/arvados/build/run-library.sh
 142   TMPHERE=\$(pwd)
 143   cd /usr/src/arvados
 144
 145   # This defines python_sdk_version and cwl_runner_version with python-style
 146   # package suffixes (.dev/rc)
 147   calculate_python_sdk_cwl_package_versions
 148
 149   cd \$TMPHERE
 150   set -u
 151
 152   arv-keepdocker --pull arvados/jobs \$cwl_runner_version
 153   docker tag arvados/jobs:\$cwl_runner_version arvados/jobs:latest
 154   arv-keepdocker arvados/jobs latest
 155 fi
 156
 157 cat >/tmp/cwltest/arv-cwl-jobs <<EOF2
 158 #!/bin/sh
 159 exec arvados-cwl-runner --api=jobs \\\$@
 160 EOF2
 161 chmod +x /tmp/cwltest/arv-cwl-jobs
 162
 163 cat >/tmp/cwltest/arv-cwl-containers <<EOF2
 164 #!/bin/sh
 165 exec arvados-cwl-runner --api=containers \\\$@
 166 EOF2
 167 chmod +x /tmp/cwltest/arv-cwl-containers
 168
 169 EXTRA=--compute-checksum
 170
 171 if [[ $devcwl -eq 1 ]] ; then
 172    EXTRA="\$EXTRA --enable-dev"
 173 fi
 174
 175 env
 176 if [[ "$suite" = "integration" ]] ; then
 177    cd /usr/src/arvados/sdk/cwl/tests
 178    exec ./arvados-tests.sh $@
 179 else
 180    exec ./run_test.sh RUNNER=/tmp/cwltest/arv-cwl-${runapi} EXTRA="\$EXTRA" $@
 181 fi
 182 EOF
 183
 184 CODE=$?
 185
 186 if test $leave_running = 0 ; then
 187     arvbox stop
 188 fi
 189
 190 exit $CODE