build/run-build-docker-jobs-image.sh

   1 #!/bin/bash
   2 # Copyright (C) The Arvados Authors. All rights reserved.
   3 #
   4 # SPDX-License-Identifier: AGPL-3.0
   5
   6 function usage {
   7     echo >&2
   8     echo >&2 "usage: WORKSPACE=/path/to/arvados $0 [options]"
   9     echo >&2
  10     echo >&2 "$0 options:"
  11     echo >&2 "  -t, --tags                    version tag for docker"
  12     echo >&2 "  -r, --repo                    Arvados package repo to use: dev (default), testing, stable"
  13     echo >&2 "  -u, --upload                  Upload the images (docker push)"
  14     echo >&2 "  --no-cache                    Don't use build cache"
  15     echo >&2 "  -h, --help                    Display this help and exit"
  16     echo >&2
  17     echo >&2 "  WORKSPACE=path                Path to the Arvados source tree to build from"
  18     echo >&2
  19 }
  20 upload=false
  21 REPO=dev
  22
  23 # NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
  24 TEMP=`getopt -o hut:r: \
  25     --long help,upload,no-cache,tags:,repo: \
  26     -n "$0" -- "$@"`
  27
  28 if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi
  29 # Note the quotes around `$TEMP': they are essential!
  30 eval set -- "$TEMP"
  31
  32 while [ $# -ge 1 ]
  33 do
  34     case $1 in
  35         -u | --upload)
  36             upload=true
  37             shift
  38             ;;
  39         --no-cache)
  40             NOCACHE=--no-cache
  41             shift
  42             ;;
  43         -t | --tags)
  44             case "$2" in
  45                 "")
  46                   echo "ERROR: --tags needs a parameter";
  47                   usage;
  48                   exit 1
  49                   ;;
  50                 *)
  51                   version_tag="$2";
  52                   shift 2
  53                   ;;
  54             esac
  55             ;;
  56         -r | --repo)
  57             case "$2" in
  58                 "")
  59                   echo "ERROR: --repo needs a parameter";
  60                   usage;
  61                   exit 1
  62                   ;;
  63                 *)
  64                   REPO="$2";
  65                   shift 2
  66                   ;;
  67             esac
  68             ;;
  69         --)
  70             shift
  71             break
  72             ;;
  73         *)
  74             usage
  75             exit 1
  76             ;;
  77     esac
  78 done
  79
  80 EXITCODE=0
  81
  82 exit_cleanly() {
  83     trap - INT
  84     report_outcomes
  85     exit $EXITCODE
  86 }
  87
  88 # Sanity check
  89 if ! [[ -n "$WORKSPACE" ]]; then
  90     usage;
  91     echo >&2 "Error: WORKSPACE environment variable not set"
  92     echo >&2
  93     exit 1
  94 fi
  95
  96 echo $WORKSPACE
  97
  98 COLUMNS=80
  99 . $WORKSPACE/build/run-library.sh
 100
 101 docker_push () {
 102     # Sometimes docker push fails; retry it a few times if necessary.
 103     for i in `seq 1 5`; do
 104         $DOCKER push $*
 105         ECODE=$?
 106         if [[ "$ECODE" == "0" ]]; then
 107             break
 108         fi
 109     done
 110
 111     if [[ "$ECODE" != "0" ]]; then
 112         EXITCODE=$(($EXITCODE + $ECODE))
 113     fi
 114     checkexit $ECODE "docker push $*"
 115 }
 116
 117 # find the docker binary
 118 DOCKER=`which docker.io`
 119
 120 if [[ "$DOCKER" == "" ]]; then
 121     DOCKER=`which docker`
 122 fi
 123
 124 if [[ "$DOCKER" == "" ]]; then
 125     title "Error: you need to have docker installed. Could not find the docker executable."
 126     exit 1
 127 fi
 128
 129 # DOCKER
 130 title "Starting docker build"
 131
 132 timer_reset
 133
 134 # clean up the docker build environment
 135 cd "$WORKSPACE"
 136
 137 if [[ -z "$ARVADOS_BUILDING_VERSION" ]] && ! [[ -z "$version_tag" ]]; then
 138         ARVADOS_BUILDING_VERSION="$version_tag"
 139         ARVADOS_BUILDING_ITERATION="1"
 140 fi
 141
 142 calculate_python_sdk_cwl_package_versions
 143
 144 echo cwl_runner_version $cwl_runner_version python_sdk_version $python_sdk_version
 145
 146 if [[ "${python_sdk_version}" != "${ARVADOS_BUILDING_VERSION}" ]]; then
 147         python_sdk_version="${python_sdk_version}-1"
 148 else
 149         python_sdk_version="${ARVADOS_BUILDING_VERSION}-${ARVADOS_BUILDING_ITERATION}"
 150 fi
 151
 152 # What we use to tag the Docker image.  For release candidate
 153 # packages, the OS package has a "~rc" suffix, but Python requires
 154 # just an "rc" suffix.  Arvados-cwl-runner will be expecting the
 155 # Python-compatible version string when it tries to pull the Docker
 156 # image, but --build-arg is expecting the OS package version.
 157 cwl_runner_version_tag=$(echo -n $cwl_runner_version | sed s/~rc/rc/g)
 158
 159 if [[ "${cwl_runner_version}" != "${ARVADOS_BUILDING_VERSION}" ]]; then
 160         cwl_runner_version="${cwl_runner_version}-1"
 161 else
 162         cwl_runner_version="${ARVADOS_BUILDING_VERSION}-${ARVADOS_BUILDING_ITERATION}"
 163 fi
 164
 165 cd docker/jobs
 166 docker build $NOCACHE \
 167        --build-arg python_sdk_version=${python_sdk_version} \
 168        --build-arg cwl_runner_version=${cwl_runner_version} \
 169        --build-arg repo_version=${REPO} \
 170        -t arvados/jobs:$cwl_runner_version_tag .
 171
 172 ECODE=$?
 173
 174 if [[ "$ECODE" != "0" ]]; then
 175     EXITCODE=$(($EXITCODE + $ECODE))
 176 fi
 177
 178 checkexit $ECODE "docker build"
 179 title "docker build complete (`timer`)"
 180
 181 if [[ "$ECODE" != "0" ]]; then
 182   exit_cleanly
 183 fi
 184
 185 timer_reset
 186
 187 if docker --version |grep " 1\.[0-9]\." ; then
 188     # Docker version prior 1.10 require -f flag
 189     # -f flag removed in Docker 1.12
 190     FORCE=-f
 191 fi
 192
 193 title "uploading images"
 194
 195 timer_reset
 196
 197 if [[ "$EXITCODE" != "0" ]]; then
 198     title "upload arvados images SKIPPED because build or tag failed"
 199 else
 200     if [[ $upload == true ]]; then
 201         ## 20150526 nico -- *sometimes* dockerhub needs re-login
 202         ## even though credentials are already in .dockercfg
 203         docker login -u arvados
 204         docker_push arvados/jobs:$cwl_runner_version_tag
 205         title "upload arvados images finished (`timer`)"
 206     else
 207         title "upload arvados images SKIPPED because no --upload option set (`timer`)"
 208     fi
 209 fi
 210
 211 exit_cleanly