build/run-build-docker-jobs-image.sh

   1 #!/bin/bash
   2 # Copyright (C) The Arvados Authors. All rights reserved.
   3 #
   4 # SPDX-License-Identifier: AGPL-3.0
   5
   6 function usage {
   7     echo >&2
   8     echo >&2 "usage: WORKSPACE=/path/to/arvados $0 [options]"
   9     echo >&2
  10     echo >&2 "$0 options:"
  11     echo >&2 "  -t, --tags                    version tag for docker"
  12     echo >&2 "  -r, --repo                    Arvados package repo to use: dev (default), testing, stable"
  13     echo >&2 "  -u, --upload                  Upload the images (docker push)"
  14     echo >&2 "  --no-cache                    Don't use build cache"
  15     echo >&2 "  -h, --help                    Display this help and exit"
  16     echo >&2
  17     echo >&2 "  WORKSPACE=path                Path to the Arvados source tree to build from"
  18     echo >&2
  19 }
  20 upload=false
  21 REPO=dev
  22
  23 # NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
  24 TEMP=`getopt -o hut:r: \
  25     --long help,upload,no-cache,tags:,repo: \
  26     -n "$0" -- "$@"`
  27
  28 if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi
  29 # Note the quotes around `$TEMP': they are essential!
  30 eval set -- "$TEMP"
  31
  32 while [ $# -ge 1 ]
  33 do
  34     case $1 in
  35         -u | --upload)
  36             upload=true
  37             shift
  38             ;;
  39         --no-cache)
  40             NOCACHE=--no-cache
  41             shift
  42             ;;
  43         -t | --tags)
  44             case "$2" in
  45                 "")
  46                   echo "ERROR: --tags needs a parameter";
  47                   usage;
  48                   exit 1
  49                   ;;
  50                 *)
  51                   version_tag="$2";
  52                   shift 2
  53                   ;;
  54             esac
  55             ;;
  56         -r | --repo)
  57             case "$2" in
  58                 "")
  59                   echo "ERROR: --repo needs a parameter";
  60                   usage;
  61                   exit 1
  62                   ;;
  63                 *)
  64                   REPO="$2";
  65                   shift 2
  66                   ;;
  67             esac
  68             ;;
  69         --)
  70             shift
  71             break
  72             ;;
  73         *)
  74             usage
  75             exit 1
  76             ;;
  77     esac
  78 done
  79
  80 EXITCODE=0
  81
  82 exit_cleanly() {
  83     trap - INT
  84     report_outcomes
  85     exit $EXITCODE
  86 }
  87
  88 # Sanity check
  89 if ! [[ -n "$WORKSPACE" ]]; then
  90     usage;
  91     echo >&2 "Error: WORKSPACE environment variable not set"
  92     echo >&2
  93     exit 1
  94 fi
  95
  96 echo $WORKSPACE
  97
  98 COLUMNS=80
  99 . $WORKSPACE/build/run-library.sh
 100
 101 docker_push () {
 102     # Sometimes docker push fails; retry it a few times if necessary.
 103     for i in `seq 1 5`; do
 104         $DOCKER push $*
 105         ECODE=$?
 106         if [[ "$ECODE" == "0" ]]; then
 107             break
 108         fi
 109     done
 110
 111     if [[ "$ECODE" != "0" ]]; then
 112         EXITCODE=$(($EXITCODE + $ECODE))
 113     fi
 114     checkexit $ECODE "docker push $*"
 115 }
 116
 117 # find the docker binary
 118 DOCKER=`which docker.io`
 119
 120 if [[ "$DOCKER" == "" ]]; then
 121     DOCKER=`which docker`
 122 fi
 123
 124 if [[ "$DOCKER" == "" ]]; then
 125     title "Error: you need to have docker installed. Could not find the docker executable."
 126     exit 1
 127 fi
 128
 129 # DOCKER
 130 title "Starting docker build"
 131
 132 timer_reset
 133
 134 # clean up the docker build environment
 135 cd "$WORKSPACE"
 136
 137 if [[ -z "$ARVADOS_BUILDING_VERSION" ]] && ! [[ -z "$version_tag" ]]; then
 138         ARVADOS_BUILDING_VERSION="$version_tag"
 139         ARVADOS_BUILDING_ITERATION="1"
 140 fi
 141
 142 calculate_python_sdk_cwl_package_versions
 143
 144 echo cwl_runner_version $cwl_runner_version python_sdk_version $python_sdk_version
 145
 146 if [[ "${python_sdk_version}" != "${ARVADOS_BUILDING_VERSION}" ]]; then
 147         python_sdk_version="${python_sdk_version}-1"
 148 else
 149         python_sdk_version="${ARVADOS_BUILDING_VERSION}-${ARVADOS_BUILDING_ITERATION}"
 150 fi
 151
 152 cwl_runner_version_orig=$cwl_runner_version
 153
 154 if [[ "${cwl_runner_version}" != "${ARVADOS_BUILDING_VERSION}" ]]; then
 155         cwl_runner_version="${cwl_runner_version}-1"
 156 else
 157         cwl_runner_version="${ARVADOS_BUILDING_VERSION}-${ARVADOS_BUILDING_ITERATION}"
 158 fi
 159
 160 cd docker/jobs
 161 docker build $NOCACHE \
 162        --build-arg python_sdk_version=${python_sdk_version} \
 163        --build-arg cwl_runner_version=${cwl_runner_version} \
 164        --build-arg repo_version=${REPO} \
 165        -t arvados/jobs:$cwl_runner_version_orig .
 166
 167 ECODE=$?
 168
 169 if [[ "$ECODE" != "0" ]]; then
 170     EXITCODE=$(($EXITCODE + $ECODE))
 171 fi
 172
 173 checkexit $ECODE "docker build"
 174 title "docker build complete (`timer`)"
 175
 176 if [[ "$ECODE" != "0" ]]; then
 177   exit_cleanly
 178 fi
 179
 180 timer_reset
 181
 182 if docker --version |grep " 1\.[0-9]\." ; then
 183     # Docker version prior 1.10 require -f flag
 184     # -f flag removed in Docker 1.12
 185     FORCE=-f
 186 fi
 187
 188 #docker export arvados/jobs:$cwl_runner_version_orig | docker import - arvados/jobs:$cwl_runner_version_orig
 189
 190 if ! [[ -z "$version_tag" ]]; then
 191     docker tag $FORCE arvados/jobs:$cwl_runner_version_orig arvados/jobs:"$version_tag"
 192 else
 193     docker tag $FORCE arvados/jobs:$cwl_runner_version_orig arvados/jobs:latest
 194 fi
 195
 196 ECODE=$?
 197
 198 if [[ "$ECODE" != "0" ]]; then
 199     EXITCODE=$(($EXITCODE + $ECODE))
 200 fi
 201
 202 checkexit $ECODE "docker tag"
 203 title "docker tag complete (`timer`)"
 204
 205 title "uploading images"
 206
 207 timer_reset
 208
 209 if [[ "$ECODE" != "0" ]]; then
 210     title "upload arvados images SKIPPED because build or tag failed"
 211 else
 212     if [[ $upload == true ]]; then
 213         ## 20150526 nico -- *sometimes* dockerhub needs re-login
 214         ## even though credentials are already in .dockercfg
 215         docker login -u arvados
 216         if ! [[ -z "$version_tag" ]]; then
 217             docker_push arvados/jobs:"$version_tag"
 218         else
 219            docker_push arvados/jobs:$cwl_runner_version_orig
 220            docker_push arvados/jobs:latest
 221         fi
 222         title "upload arvados images finished (`timer`)"
 223     else
 224         title "upload arvados images SKIPPED because no --upload option set (`timer`)"
 225     fi
 226 fi
 227
 228 exit_cleanly