build/run-build-docker-jobs-image.sh

   1 #!/bin/bash
   2 # Copyright (C) The Arvados Authors. All rights reserved.
   3 #
   4 # SPDX-License-Identifier: AGPL-3.0
   5
   6 function usage {
   7     echo >&2
   8     echo >&2 "usage: WORKSPACE=/path/to/arvados $0 [options]"
   9     echo >&2
  10     echo >&2 "$0 options:"
  11     echo >&2 "  -t, --tags                    version tag for docker"
  12     echo >&2 "  -r, --repo                    Arvados package repo to use: dev (default), testing, stable"
  13     echo >&2 "  -u, --upload                  Upload the images (docker push)"
  14     echo >&2 "  --no-cache                    Don't use build cache"
  15     echo >&2 "  -h, --help                    Display this help and exit"
  16     echo >&2
  17     echo >&2 "  WORKSPACE=path                Path to the Arvados source tree to build from"
  18     echo >&2
  19 }
  20 upload=false
  21 REPO=dev
  22
  23 # NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
  24 TEMP=`getopt -o hut:r: \
  25     --long help,upload,no-cache,tags:,repo: \
  26     -n "$0" -- "$@"`
  27
  28 if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi
  29 # Note the quotes around `$TEMP': they are essential!
  30 eval set -- "$TEMP"
  31
  32 while [ $# -ge 1 ]
  33 do
  34     case $1 in
  35         -u | --upload)
  36             upload=true
  37             shift
  38             ;;
  39         --no-cache)
  40             NOCACHE=--no-cache
  41             shift
  42             ;;
  43         -t | --tags)
  44             case "$2" in
  45                 "")
  46                   echo "ERROR: --tags needs a parameter";
  47                   usage;
  48                   exit 1
  49                   ;;
  50                 *)
  51                   version_tag="$2";
  52                   shift 2
  53                   ;;
  54             esac
  55             ;;
  56         -r | --repo)
  57             case "$2" in
  58                 "")
  59                   echo "ERROR: --repo needs a parameter";
  60                   usage;
  61                   exit 1
  62                   ;;
  63                 *)
  64                   REPO="$2";
  65                   shift 2
  66                   ;;
  67             esac
  68             ;;
  69         --)
  70             shift
  71             break
  72             ;;
  73         *)
  74             usage
  75             exit 1
  76             ;;
  77     esac
  78 done
  79
  80 EXITCODE=0
  81
  82 exit_cleanly() {
  83     trap - INT
  84     report_outcomes
  85     exit $EXITCODE
  86 }
  87
  88 # Sanity check
  89 if ! [[ -n "$WORKSPACE" ]]; then
  90     usage;
  91     echo >&2 "Error: WORKSPACE environment variable not set"
  92     echo >&2
  93     exit 1
  94 fi
  95
  96 echo $WORKSPACE
  97
  98 COLUMNS=80
  99 . $WORKSPACE/build/run-library.sh
 100
 101 docker_push () {
 102     # Sometimes docker push fails; retry it a few times if necessary.
 103     for i in `seq 1 5`; do
 104         $DOCKER push $*
 105         ECODE=$?
 106         if [[ "$ECODE" == "0" ]]; then
 107             break
 108         fi
 109     done
 110
 111     if [[ "$ECODE" != "0" ]]; then
 112         EXITCODE=$(($EXITCODE + $ECODE))
 113     fi
 114     checkexit $ECODE "docker push $*"
 115 }
 116
 117 # find the docker binary
 118 DOCKER=`which docker.io`
 119
 120 if [[ "$DOCKER" == "" ]]; then
 121     DOCKER=`which docker`
 122 fi
 123
 124 if [[ "$DOCKER" == "" ]]; then
 125     title "Error: you need to have docker installed. Could not find the docker executable."
 126     exit 1
 127 fi
 128
 129 # DOCKER
 130 title "Starting docker build"
 131
 132 timer_reset
 133
 134 # clean up the docker build environment
 135 cd "$WORKSPACE"
 136
 137 if [[ -z "$ARVADOS_BUILDING_VERSION" ]] && ! [[ -z "$version_tag" ]]; then
 138         ARVADOS_BUILDING_VERSION="$version_tag"
 139         ARVADOS_BUILDING_ITERATION="1"
 140 fi
 141
 142 python_sdk_ts=$(cd sdk/python && timestamp_from_git)
 143 cwl_runner_ts=$(cd sdk/cwl && timestamp_from_git)
 144
 145 python_sdk_version=$(cd sdk/python && nohash_version_from_git 0.1)
 146 cwl_runner_version=$(cd sdk/cwl && nohash_version_from_git 1.0)
 147
 148 if [[ $python_sdk_ts -gt $cwl_runner_ts ]]; then
 149     cwl_runner_version=$(cd sdk/python && nohash_version_from_git 1.0)
 150 fi
 151
 152 echo cwl_runner_version $cwl_runner_version python_sdk_version $python_sdk_version
 153
 154 if [[ "${python_sdk_version}" != "${ARVADOS_BUILDING_VERSION}" ]]; then
 155         python_sdk_version="${python_sdk_version}-1"
 156 else
 157         python_sdk_version="${ARVADOS_BUILDING_VERSION}-${ARVADOS_BUILDING_ITERATION}"
 158 fi
 159
 160 cwl_runner_version_orig=$cwl_runner_version
 161
 162 if [[ "${cwl_runner_version}" != "${ARVADOS_BUILDING_VERSION}" ]]; then
 163         cwl_runner_version="${cwl_runner_version}-1"
 164 else
 165         cwl_runner_version="${ARVADOS_BUILDING_VERSION}-${ARVADOS_BUILDING_ITERATION}"
 166 fi
 167
 168 cd docker/jobs
 169 docker build $NOCACHE \
 170        --build-arg python_sdk_version=${python_sdk_version} \
 171        --build-arg cwl_runner_version=${cwl_runner_version} \
 172        --build-arg repo_version=${REPO} \
 173        -t arvados/jobs:$cwl_runner_version_orig .
 174
 175 ECODE=$?
 176
 177 if [[ "$ECODE" != "0" ]]; then
 178     EXITCODE=$(($EXITCODE + $ECODE))
 179 fi
 180
 181 checkexit $ECODE "docker build"
 182 title "docker build complete (`timer`)"
 183
 184 if [[ "$ECODE" != "0" ]]; then
 185   exit_cleanly
 186 fi
 187
 188 timer_reset
 189
 190 if docker --version |grep " 1\.[0-9]\." ; then
 191     # Docker version prior 1.10 require -f flag
 192     # -f flag removed in Docker 1.12
 193     FORCE=-f
 194 fi
 195
 196 #docker export arvados/jobs:$cwl_runner_version_orig | docker import - arvados/jobs:$cwl_runner_version_orig
 197
 198 if ! [[ -z "$version_tag" ]]; then
 199     docker tag $FORCE arvados/jobs:$cwl_runner_version_orig arvados/jobs:"$version_tag"
 200 else
 201     docker tag $FORCE arvados/jobs:$cwl_runner_version_orig arvados/jobs:latest
 202 fi
 203
 204 ECODE=$?
 205
 206 if [[ "$ECODE" != "0" ]]; then
 207     EXITCODE=$(($EXITCODE + $ECODE))
 208 fi
 209
 210 checkexit $ECODE "docker tag"
 211 title "docker tag complete (`timer`)"
 212
 213 title "uploading images"
 214
 215 timer_reset
 216
 217 if [[ "$ECODE" != "0" ]]; then
 218     title "upload arvados images SKIPPED because build or tag failed"
 219 else
 220     if [[ $upload == true ]]; then
 221         ## 20150526 nico -- *sometimes* dockerhub needs re-login
 222         ## even though credentials are already in .dockercfg
 223         docker login -u arvados
 224         if ! [[ -z "$version_tag" ]]; then
 225             docker_push arvados/jobs:"$version_tag"
 226         else
 227            docker_push arvados/jobs:$cwl_runner_version_orig
 228            docker_push arvados/jobs:latest
 229         fi
 230         title "upload arvados images finished (`timer`)"
 231     else
 232         title "upload arvados images SKIPPED because no --upload option set (`timer`)"
 233     fi
 234 fi
 235
 236 exit_cleanly