X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/e17f20cd7d10d81b0dcb5522a3e93305b318da8f..427b630d31d8427b09d5a022f4c699111497915f:/sdk/cwl/test_with_arvbox.sh

diff --git a/sdk/cwl/test_with_arvbox.sh b/sdk/cwl/test_with_arvbox.sh
index 37eb5171eb..51d64b3f84 100755
--- a/sdk/cwl/test_with_arvbox.sh
+++ b/sdk/cwl/test_with_arvbox.sh
@@ -5,17 +5,20 @@
 
 set -x
 
+cwldir=$(readlink -f $(dirname $0))
+
 if ! which arvbox >/dev/null ; then
-    export PATH=$PATH:$(readlink -f $(dirname $0)/../../tools/arvbox/bin)
+    export PATH=$PATH:$cwldir/../../tools/arvbox/bin
 fi
 
 reset_container=1
 leave_running=0
 config=dev
+devcwl=0
 tag="latest"
-pythoncmd=python
 suite=conformance
 runapi=containers
+reinstall=0
 
 while test -n "$1" ; do
     arg="$1"
@@ -40,8 +43,16 @@ while test -n "$1" ; do
             build=1
             shift
             ;;
+        --devcwl)
+            devcwl=1
+            shift
+            ;;
+        --reinstall)
+            reinstall=1
+            shift
+            ;;
         --pythoncmd)
-            pythoncmd=$2
+            echo "warning: --pythoncmd option is no longer supported; ignored" >&2
             shift ; shift
             ;;
         --suite)
@@ -53,7 +64,7 @@ while test -n "$1" ; do
             shift ; shift
             ;;
         -h|--help)
-            echo "$0 [--no-reset-container] [--leave-running] [--config dev|localdemo] [--tag docker_tag] [--build] [--pythoncmd python(2|3)] [--suite (integration|conformance)]"
+            echo "$0 [--no-reset-container] [--leave-running] [--config dev|localdemo] [--tag docker_tag] [--build] [--suite (integration|conformance-v1.0|conformance-*)]"
             exit
             ;;
         *)
@@ -66,6 +77,10 @@ if test -z "$ARVBOX_CONTAINER" ; then
    export ARVBOX_CONTAINER=cwltest
 fi
 
+if test "$suite" = "conformance" ; then
+  suite=conformance-v1.0
+fi
+
 if test $reset_container = 1 ; then
     arvbox stop
     docker rm $ARVBOX_CONTAINER
@@ -74,73 +89,106 @@ fi
 
 arvbox start $config $tag
 
+# Copy the integration test suite from our local arvados clone instead
+# of using the one inside the container, so we can make changes to the
+# integration tests without necessarily having to rebuilding the
+# container image.
+docker cp -L $cwldir/tests $ARVBOX_CONTAINER:/usr/src/arvados/sdk/cwl
+
 arvbox pipe <<EOF
 set -eu -o pipefail
 
 . /usr/local/lib/arvbox/common.sh
 
-export PYCMD=$pythoncmd
-
-if test $config = dev ; then
-  cd /usr/src/arvados/sdk/cwl
-  \$PYCMD setup.py sdist
-  pip_install \$(ls -r dist/arvados-cwl-runner-*.tar.gz | head -n1)
+if test $config = dev -o $reinstall = 1; then
+  pip_install_sdist sdk/python sdk/cwl
 fi
 
 set -x
 
-if [ \$PYCMD = "python3" ]; then
-    pip3 install cwltest
-else
-    pip install cwltest
-fi
+# 2.3.20230527113600 release of cwltest confirms that files exist on disk, since
+# our files are in Keep, all the tests fail.
+# We should add [optional] Arvados support to cwltest so it can access
+# Keep but for the time being just install the last working version.
+/opt/arvados-py/bin/pip install 'cwltest<2.3.20230527113600'
 
 mkdir -p /tmp/cwltest
 cd /tmp/cwltest
-if ! test -d common-workflow-language ; then
-  git clone https://github.com/common-workflow-language/common-workflow-language.git
+
+if [[ "$suite" = "conformance-v1.0" ]] ; then
+   if ! test -d common-workflow-language ; then
+     git clone https://github.com/common-workflow-language/common-workflow-language.git
+   fi
+   cd common-workflow-language
+elif [[ "$suite" =~ conformance-(.*) ]] ; then
+   version=\${BASH_REMATCH[1]}
+   if ! test -d cwl-\${version} ; then
+     git clone https://github.com/common-workflow-language/cwl-\${version}.git
+   fi
+   cd cwl-\${version}
+   git checkout \${version}.0
+elif [[ "$suite" != "integration" ]] ; then
+   echo "ERROR: unknown suite '$suite'"
+   exit 1
 fi
-cd common-workflow-language
-git pull
+
+if [[ "$suite" = "conformance-v1.1" ]] ; then
+   git checkout main
+fi
+
+if [[ "$suite" = "conformance-v1.2" ]] ; then
+   git checkout v1.2.1
+fi
+
+#if [[ "$suite" != "integration" ]] ; then
+#  git pull
+#fi
+
 export ARVADOS_API_HOST=localhost:8000
 export ARVADOS_API_HOST_INSECURE=1
-export ARVADOS_API_TOKEN=\$(cat /var/lib/arvados/superuser_token)
-
+export ARVADOS_API_TOKEN=\$(cat /var/lib/arvados-arvbox/superuser_token)
 
 if test -n "$build" ; then
-   /usr/src/arvados/build/build-dev-docker-jobs-image.sh
-elif test "$tag" = "latest" ; then
-  arv-keepdocker --pull arvados/jobs $tag
-else
-  jobsimg=\$(curl https://versions.arvados.org/v1/commit/$tag | python -c "import json; import sys; sys.stdout.write(json.load(sys.stdin)['Versions']['Docker']['arvados/jobs'])")
-  arv-keepdocker --pull arvados/jobs \$jobsimg
-  docker tag arvados/jobs:\$jobsimg arvados/jobs:latest
-  arv-keepdocker arvados/jobs latest
+  /usr/src/arvados/build/build-dev-docker-jobs-image.sh
 fi
 
-cat >/tmp/cwltest/arv-cwl-jobs <<EOF2
-#!/bin/sh
-exec arvados-cwl-runner --api=jobs \\\$@
-EOF2
-chmod +x /tmp/cwltest/arv-cwl-jobs
+EXTRA=--compute-checksum
 
-cat >/tmp/cwltest/arv-cwl-containers <<EOF2
-#!/bin/sh
-exec arvados-cwl-runner --api=containers \\\$@
-EOF2
-chmod +x /tmp/cwltest/arv-cwl-containers
+if [[ $devcwl -eq 1 ]] ; then
+   EXTRA="\$EXTRA --enable-dev"
+fi
 
 env
-if [[ "$suite" = "conformance" ]] ; then
-   exec ./run_test.sh RUNNER=/tmp/cwltest/arv-cwl-${runapi} EXTRA=--compute-checksum $@
-elif [[ "$suite" = "integration" ]] ; then
+
+arvados-cwl-runner --version
+cwltest --version
+
+# Skip docker_entrypoint test because it fails on singularity
+#
+# Skip timelimit_invalid_wf test because the timeout is very short
+# (5s) and singularity containers loading off an arv-mount take too long
+# to start and get incorrectly terminated
+#
+# Skip test 199 in the v1.1 suite because it has different output
+# depending on whether there is a pty associated with stdout (fixed in
+# the v1.2 suite)
+
+if [[ "$suite" = "integration" ]] ; then
    cd /usr/src/arvados/sdk/cwl/tests
    exec ./arvados-tests.sh $@
+elif [[ "$suite" = "conformance-v1.2" ]] ; then
+   exec cwltest --tool arvados-cwl-runner --test conformance_tests.yaml -Sdocker_entrypoint --badgedir /tmp/badges $@ -- \$EXTRA
+elif [[ "$suite" = "conformance-v1.1" ]] ; then
+   exec cwltest --tool arvados-cwl-runner --test conformance_tests.yaml -Sdocker_entrypoint,timelimit_invalid_wf -N199 --badgedir /tmp/badges $@ -- \$EXTRA
+elif [[ "$suite" = "conformance-v1.0" ]] ; then
+   exec cwltest --tool arvados-cwl-runner --test v1.0/conformance_test_v1.0.yaml -Sdocker_entrypoint --badgedir /tmp/badges $@ -- \$EXTRA
 fi
 EOF
 
 CODE=$?
 
+docker cp -L $ARVBOX_CONTAINER:/tmp/badges $cwldir/badges
+
 if test $leave_running = 0 ; then
     arvbox stop
 fi