X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/970254fba5b9297884c521987d081c232004eb77..9f1850a385ee8e0a011474de19ee6507b0b168f3:/sdk/cwl/test_with_arvbox.sh

diff --git a/sdk/cwl/test_with_arvbox.sh b/sdk/cwl/test_with_arvbox.sh
index d38414fc81..55099afdf7 100755
--- a/sdk/cwl/test_with_arvbox.sh
+++ b/sdk/cwl/test_with_arvbox.sh
@@ -17,6 +17,7 @@ tag="latest"
 pythoncmd=python3
 suite=conformance
 runapi=containers
+reinstall=0
 
 while test -n "$1" ; do
     arg="$1"
@@ -45,6 +46,10 @@ while test -n "$1" ; do
             devcwl=1
             shift
             ;;
+        --reinstall)
+            reinstall=1
+            shift
+            ;;
         --pythoncmd)
             pythoncmd=$2
             shift ; shift
@@ -83,6 +88,12 @@ fi
 
 arvbox start $config $tag
 
+# Copy the integration test suite from our local arvados clone instead
+# of using the one inside the container, so we can make changes to the
+# integration tests without necessarily having to rebuilding the
+# container image.
+docker cp -L $(readlink -f $(dirname $0)/tests) $ARVBOX_CONTAINER:/usr/src/arvados/sdk/cwl
+
 arvbox pipe <<EOF
 set -eu -o pipefail
 
@@ -90,7 +101,11 @@ set -eu -o pipefail
 
 export PYCMD=$pythoncmd
 
-if test $config = dev ; then
+if test $config = dev -o $reinstall = 1; then
+  cd /usr/src/arvados/sdk/python
+  \$PYCMD setup.py sdist
+  pip_install \$(ls -r dist/arvados-python-client-*.tar.gz | head -n1)
+
   cd /usr/src/arvados/sdk/cwl
   \$PYCMD setup.py sdist
   pip_install \$(ls -r dist/arvados-cwl-runner-*.tar.gz | head -n1)
@@ -98,10 +113,14 @@ fi
 
 set -x
 
+# 2.3.20230527113600 release of cwltest confirms that files exist on disk, since
+# our files are in Keep, all the tests fail.
+# We should add [optional] Arvados support to cwltest so it can access
+# Keep but for the time being just install the last working version.
 if [ "\$PYCMD" = "python3" ]; then
-    pip3 install cwltest
+    pip3 install 'cwltest<2.3.20230527113600'
 else
-    pip install cwltest
+    pip install 'cwltest<2.3.20230527113600'
 fi
 
 mkdir -p /tmp/cwltest
@@ -124,6 +143,14 @@ elif [[ "$suite" != "integration" ]] ; then
    exit 1
 fi
 
+if [[ "$suite" = "conformance-v1.1" ]] ; then
+   git checkout main
+fi
+
+if [[ "$suite" = "conformance-v1.2" ]] ; then
+   git checkout 1.2.1_proposed
+fi
+
 #if [[ "$suite" != "integration" ]] ; then
 #  git pull
 #fi
@@ -134,25 +161,6 @@ export ARVADOS_API_TOKEN=\$(cat /var/lib/arvados-arvbox/superuser_token)
 
 if test -n "$build" ; then
   /usr/src/arvados/build/build-dev-docker-jobs-image.sh
-elif test "$tag" = "latest" ; then
-  arv-keepdocker --pull arvados/jobs $tag
-else
-  set +u
-  export WORKSPACE=/usr/src/arvados
-  . /usr/src/arvados/build/run-library.sh
-  TMPHERE=\$(pwd)
-  cd /usr/src/arvados
-
-  # This defines python_sdk_version and cwl_runner_version with python-style
-  # package suffixes (.dev/rc)
-  calculate_python_sdk_cwl_package_versions
-
-  cd \$TMPHERE
-  set -u
-
-  arv-keepdocker --pull arvados/jobs \$cwl_runner_version
-  docker tag arvados/jobs:\$cwl_runner_version arvados/jobs:latest
-  arv-keepdocker arvados/jobs latest
 fi
 
 EXTRA=--compute-checksum
@@ -162,13 +170,32 @@ if [[ $devcwl -eq 1 ]] ; then
 fi
 
 env
+
+arvados-cwl-runner --version
+cwltest --version
+
+# Skip docker_entrypoint test because it fails on singularity
+#
+# Skip timelimit_invalid_wf test because the timeout is very short
+# (5s) and singularity containers loading off an arv-mount take too long
+# to start and get incorrectly terminated
+#
+# Skip test 199 in the v1.1 suite because it has different output
+# depending on whether there is a pty associated with stdout (fixed in
+# the v1.2 suite)
+#
+# Skip test 307 in the v1.2 suite because the test relied on
+# secondary file behavior of cwltool that wasn't actually correct to specification
+
 if [[ "$suite" = "integration" ]] ; then
    cd /usr/src/arvados/sdk/cwl/tests
    exec ./arvados-tests.sh $@
 elif [[ "$suite" = "conformance-v1.2" ]] ; then
-   exec cwltest --tool arvados-cwl-runner --test conformance_tests.yaml -N307 $@ -- \$EXTRA
-else
-   exec ./run_test.sh RUNNER=arvados-cwl-runner EXTRA="\$EXTRA" $@
+   exec cwltest --tool arvados-cwl-runner --test conformance_tests.yaml -Sdocker_entrypoint,timelimit_invalid_wf -N307 $@ -- \$EXTRA
+elif [[ "$suite" = "conformance-v1.1" ]] ; then
+   exec cwltest --tool arvados-cwl-runner --test conformance_tests.yaml -Sdocker_entrypoint,timelimit_invalid_wf -N199 $@ -- \$EXTRA
+elif [[ "$suite" = "conformance-v1.0" ]] ; then
+   exec cwltest --tool arvados-cwl-runner --test v1.0/conformance_test_v1.0.yaml -Sdocker_entrypoint $@ -- \$EXTRA
 fi
 EOF