19070: Still trying to fix test_with_arvbox

[arvados.git] / build / build-dev-docker-jobs-image.sh

diff --git a/build/build-dev-docker-jobs-image.sh b/build/build-dev-docker-jobs-image.sh
index fb392de234bee80871735d46fc77c71db633e5f7..af838d68e8c7e33ac5f7d1d0f10e52fa7b95b47f 100755 (executable)
--- a/build/build-dev-docker-jobs-image.sh
+++ b/build/build-dev-docker-jobs-image.sh
@@ -1,4 +1,7 @@
 #!/bin/bash
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
 
 read -rd "\000" helpmessage <<EOF
 Build an arvados/jobs Docker image from local git tree.
@@ -12,16 +15,16 @@ Syntax:
 
 WORKSPACE=path         Path to the Arvados source tree to build packages from
 CWLTOOL=path           (optional) Path to cwltool git repository.
+SALAD=path             (optional) Path to schema_salad git repository.
+PYCMD=pythonexec       (optional) Specify the python3 executable to use in the docker image. Defaults to "python3".
 
 EOF
 
 set -e
 
 if [[ -z "$WORKSPACE" ]] ; then
-    echo "$helpmessage"
-    echo
-    echo "Must set WORKSPACE"
-    exit 1
+    export WORKSPACE=$(readlink -f $(dirname $0)/..)
+    echo "Using WORKSPACE $WORKSPACE"
 fi
 
 if [[ -z "$ARVADOS_API_HOST" || -z "$ARVADOS_API_TOKEN" ]] ; then
@@ -31,21 +34,46 @@ if [[ -z "$ARVADOS_API_HOST" || -z "$ARVADOS_API_TOKEN" ]] ; then
     exit 1
 fi
 
-(cd "$WORKSPACE/sdk/python" && python setup.py sdist)
-sdk=$(cd "$WORKSPACE/sdk/python/dist" && ls -t arvados-python-client-*.tar.gz | head -n1)
+cd "$WORKSPACE"
+
+py=python3
+pipcmd=pip
+if [[ -n "$PYCMD" ]] ; then
+    py="$PYCMD"
+fi
+if [[ $py = python3 ]] ; then
+    pipcmd=pip3
+fi
 
-(cd "$WORKSPACE/sdk/cwl" && python setup.py sdist)
-runner=$(cd "$WORKSPACE/sdk/cwl/dist" && ls -t arvados-cwl-runner-*.tar.gz | head -n1)
+(cd sdk/python && python3 setup.py sdist)
+sdk=$(cd sdk/python/dist && ls -t arvados-python-client-*.tar.gz | head -n1)
 
-rm -rf "$WORKSPACE/sdk/cwl/cwltool_dist"
-mkdir -p "$WORKSPACE/sdk/cwl/cwltool_dist"
+(cd sdk/cwl && python3 setup.py sdist)
+runner=$(cd sdk/cwl/dist && ls -t arvados-cwl-runner-*.tar.gz | head -n1)
+
+rm -rf sdk/cwl/salad_dist
+mkdir -p sdk/cwl/salad_dist
+if [[ -n "$SALAD" ]] ; then
+    (cd "$SALAD" && python3 setup.py sdist)
+    salad=$(cd "$SALAD/dist" && ls -t schema-salad-*.tar.gz | head -n1)
+    cp "$SALAD/dist/$salad" $WORKSPACE/sdk/cwl/salad_dist
+fi
+
+rm -rf sdk/cwl/cwltool_dist
+mkdir -p sdk/cwl/cwltool_dist
 if [[ -n "$CWLTOOL" ]] ; then
-    (cd "$CWLTOOL" && python setup.py sdist)
+    (cd "$CWLTOOL" && python3 setup.py sdist)
     cwltool=$(cd "$CWLTOOL/dist" && ls -t cwltool-*.tar.gz | head -n1)
     cp "$CWLTOOL/dist/$cwltool" $WORKSPACE/sdk/cwl/cwltool_dist
 fi
 
-gittag=$(cd "$WORKSPACE/sdk/cwl" && git log --first-parent --max-count=1 --format=format:%H)
-docker build --build-arg sdk=$sdk --build-arg runner=$runner --build-arg cwltool=$cwltool -f "$WORKSPACE/sdk/dev-jobs.dockerfile" -t arvados/jobs:$gittag "$WORKSPACE/sdk"
-echo arv-keepdocker arvados/jobs $gittag
-arv-keepdocker arvados/jobs $gittag
+. build/run-library.sh
+
+# This defines python_sdk_version and cwl_runner_version with python-style
+# package suffixes (.dev/rc)
+calculate_python_sdk_cwl_package_versions
+
+set -x
+docker build --no-cache --build-arg sdk=$sdk --build-arg runner=$runner --build-arg salad=$salad --build-arg cwltool=$cwltool --build-arg pythoncmd=$py --build-arg pipcmd=$pipcmd -f "$WORKSPACE/sdk/dev-jobs.dockerfile" -t arvados/jobs:$cwl_runner_version "$WORKSPACE/sdk"
+echo arv-keepdocker arvados/jobs $cwl_runner_version
+arv-keepdocker arvados/jobs $cwl_runner_version