13306: test_with_arvbox.sh can test either python2 or python3

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz@veritasgenetics.com>

diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index b65e2c58e180a00f6bb8bca6688498880c3d5e00..ad8e903ba65f28bec56710a3ed15915482362d72 100644 (file)
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -467,7 +467,7 @@ class Runner(Process):
             if "cwl.output.json" in outc:
                 with outc.open("cwl.output.json", "rb") as f:
                     if f.size() > 0:
-                        outputs = json.load(f)
+                        outputs = json.loads(f.read().decode())
             def keepify(fileobj):
                 path = fileobj["location"]
                 if not path.startswith("keep:"):

diff --git a/sdk/cwl/test_with_arvbox.sh b/sdk/cwl/test_with_arvbox.sh
index a6f4022cc3559adb40f7a5891495de2fbfd78c2a..aa98252c0c9520fdc6e00fa3b400cd15db28de02 100755 (executable)
--- a/sdk/cwl/test_with_arvbox.sh
+++ b/sdk/cwl/test_with_arvbox.sh
@@ -13,6 +13,7 @@ reset_container=1
 leave_running=0
 config=dev
 tag="latest"
+pythoncmd=python
 
 while test -n "$1" ; do
     arg="$1"
@@ -33,8 +34,16 @@ while test -n "$1" ; do
             tag=$2
             shift ; shift
             ;;
+        --build)
+            build=1
+            shift
+            ;;
+        --pythoncmd)
+            pythoncmd=$2
+            shift ; shift
+            ;;
         -h|--help)
-            echo "$0 [--no-reset-container] [--leave-running] [--config dev|localdemo] [--tag docker_tag]"
+            echo "$0 [--no-reset-container] [--leave-running] [--config dev|localdemo] [--tag docker_tag] [--build] [--pythoncmd python[23]]"
             exit
             ;;
         *)
@@ -60,13 +69,19 @@ set -eu -o pipefail
 
 . /usr/local/lib/arvbox/common.sh
 
+export PYCMD=$pythoncmd
+
 if test $config = dev ; then
   cd /usr/src/arvados/sdk/cwl
-  python setup.py sdist
+  \$PYCMD setup.py sdist
   pip_install \$(ls -r dist/arvados-cwl-runner-*.tar.gz | head -n1)
 fi
 
-pip install cwltest
+if [ \$PYCMD = "python3" ]; then
+    pip3 install cwltest
+else
+    pip3 install cwltest
+fi
 
 mkdir -p /tmp/cwltest
 cd /tmp/cwltest
@@ -80,7 +95,9 @@ export ARVADOS_API_HOST_INSECURE=1
 export ARVADOS_API_TOKEN=\$(cat /var/lib/arvados/superuser_token)
 
 
-if test "$tag" = "latest" ; then
+if test -n "$build" ; then
+   /usr/src/arvados/build/build-dev-docker-jobs-image.sh
+elif test "$tag" = "latest" ; then
   arv-keepdocker --pull arvados/jobs $tag
 else
   jobsimg=\$(curl https://versions.arvados.org/v1/commit/$tag | python -c "import json; import sys; sys.stdout.write(json.load(sys.stdin)['Versions']['Docker']['arvados/jobs'])")

diff --git a/sdk/dev-jobs.dockerfile b/sdk/dev-jobs.dockerfile
index 9c4a5aa3345c9a900a87e04884f057250d497f12..f9e370794d4e90d87a12deb54167c1d700ba16f1 100644 (file)
--- a/sdk/dev-jobs.dockerfile
+++ b/sdk/dev-jobs.dockerfile
@@ -20,8 +20,15 @@ ENV DEBIAN_FRONTEND noninteractive
 
 ARG pythoncmd=python
 
-RUN apt-get update -q && apt-get install -qy git ${pythoncmd}-pip python-virtualenv ${pythoncmd}-dev libcurl4-gnutls-dev libgnutls28-dev nodejs python-pyasn1-modules
-RUN if [ "$pythoncmd" = "python3" ]; then pip3 install -U setuptools six requests; else pip install -U setuptools six requests; fi
+RUN apt-get update -q && apt-get install -qy --no-install-recommends \
+    git ${pythoncmd}-pip ${pythoncmd}-virtualenv ${pythoncmd}-dev libcurl4-gnutls-dev \
+    libgnutls28-dev nodejs ${pythoncmd}-pyasn1-modules build-essential
+
+RUN if [ "$pythoncmd" = "python3" ]; then \
+       pip3 install -U setuptools six requests ; \
+    else \
+       pip install -U setuptools six requests ; \
+    fi
 
 ARG sdk
 ARG runner

diff --git a/sdk/python/arvados/commands/keepdocker.py b/sdk/python/arvados/commands/keepdocker.py
index 2d306d516a29652d7a0942e47acc7ef3572414ff..811f096c60808b2033c8123d5330909ee88db816 100644 (file)
--- a/sdk/python/arvados/commands/keepdocker.py
+++ b/sdk/python/arvados/commands/keepdocker.py
@@ -504,7 +504,7 @@ def main(arguments=None, stdout=sys.stdout, install_sig_handlers=True, api=None)
         else:
             json_filename = raw_image_hash + '/json'
         json_file = image_tar.extractfile(image_tar.getmember(json_filename))
-        image_metadata = json.load(json_file)
+        image_metadata = json.loads(json_file.read().decode())
         json_file.close()
         image_tar.close()
         link_base = {'head_uuid': coll_uuid, 'properties': {}}