build/build-dev-docker-jobs-image.sh

   1 #!/bin/bash
   2 # Copyright (C) The Arvados Authors. All rights reserved.
   3 #
   4 # SPDX-License-Identifier: AGPL-3.0
   5
   6 read -rd "\000" helpmessage <<EOF
   7 Build an arvados/jobs Docker image from local git tree.
   8
   9 Intended for use by developers working on arvados-python-client or
  10 arvados-cwl-runner and need to run a crunch job with a custom package
  11 version.  Also supports building custom cwltool if CWLTOOL is set.
  12
  13 Syntax:
  14         WORKSPACE=/path/to/arvados $(basename $0)
  15
  16 WORKSPACE=path         Path to the Arvados source tree to build packages from
  17 CWLTOOL=path           (optional) Path to cwltool git repository.
  18 SALAD=path             (optional) Path to schema_salad git repository.
  19 CWL_UTILS=path         (optional) Path to cwl-utils git repository.
  20 PYCMD=pythonexec       (optional) Specify the python3 executable to use in the docker image. Defaults to "python3".
  21
  22 EOF
  23
  24 set -e
  25
  26 if [[ -z "$WORKSPACE" ]] ; then
  27     export WORKSPACE=$(readlink -f $(dirname $0)/..)
  28     echo "Using WORKSPACE $WORKSPACE"
  29 fi
  30
  31 if [[ -z "$ARVADOS_API_HOST" || -z "$ARVADOS_API_TOKEN" ]] ; then
  32     echo "$helpmessage"
  33     echo
  34     echo "Must set ARVADOS_API_HOST and ARVADOS_API_TOKEN"
  35     exit 1
  36 fi
  37
  38 cd "$WORKSPACE"
  39
  40 py=python3
  41 pipcmd=pip
  42 if [[ -n "$PYCMD" ]] ; then
  43     py="$PYCMD"
  44 fi
  45 if [[ $py = python3 ]] ; then
  46     pipcmd=pip3
  47 fi
  48
  49 (cd sdk/python && python3 setup.py sdist)
  50 sdk=$(cd sdk/python/dist && ls -t arvados-python-client-*.tar.gz | head -n1)
  51
  52 (cd sdk/cwl && python3 setup.py sdist)
  53 runner=$(cd sdk/cwl/dist && ls -t arvados-cwl-runner-*.tar.gz | head -n1)
  54
  55 rm -rf sdk/cwl/salad_dist
  56 mkdir -p sdk/cwl/salad_dist
  57 if [[ -n "$SALAD" ]] ; then
  58     (cd "$SALAD" && python3 setup.py sdist)
  59     salad=$(cd "$SALAD/dist" && ls -t schema-salad-*.tar.gz | head -n1)
  60     cp "$SALAD/dist/$salad" $WORKSPACE/sdk/cwl/salad_dist
  61 fi
  62
  63 rm -rf sdk/cwl/cwltool_dist
  64 mkdir -p sdk/cwl/cwltool_dist
  65 if [[ -n "$CWLTOOL" ]] ; then
  66     (cd "$CWLTOOL" && python3 setup.py sdist)
  67     cwltool=$(cd "$CWLTOOL/dist" && ls -t cwltool-*.tar.gz | head -n1)
  68     cp "$CWLTOOL/dist/$cwltool" $WORKSPACE/sdk/cwl/cwltool_dist
  69 fi
  70
  71 rm -rf sdk/cwl/cwlutils_dist
  72 mkdir -p sdk/cwl/cwlutils_dist
  73 if [[ -n "$CWL_UTILS" ]] ; then
  74     (cd "$CWL_UTILS" && python3 setup.py sdist)
  75     cwlutils=$(cd "$CWL_UTILS/dist" && ls -t cwl-utils-*.tar.gz | head -n1)
  76     cp "$CWL_UTILS/dist/$cwlutils" $WORKSPACE/sdk/cwl/cwlutils_dist
  77 fi
  78
  79 . build/run-library.sh
  80
  81 # This defines python_sdk_version and cwl_runner_version with python-style
  82 # package suffixes (.dev/rc)
  83 calculate_python_sdk_cwl_package_versions
  84
  85 set -x
  86 docker build --no-cache \
  87        --build-arg sdk=$sdk \
  88        --build-arg runner=$runner \
  89        --build-arg salad=$salad \
  90        --build-arg cwltool=$cwltool \
  91        --build-arg pythoncmd=$py \
  92        --build-arg pipcmd=$pipcmd \
  93        --build-arg cwlutils=$cwlutils \
  94        -f "$WORKSPACE/sdk/dev-jobs.dockerfile" \
  95        -t arvados/jobs:$cwl_runner_version \
  96        "$WORKSPACE/sdk"
  97
  98 echo arv-keepdocker arvados/jobs $cwl_runner_version
  99 arv-keepdocker arvados/jobs $cwl_runner_version