X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/07ad9aa37c3b907adf85821a192041b5bb478fec..8a27fe370239ecb8e50d53f46b45ed61203a35ca:/build/run-build-docker-jobs-image.sh diff --git a/build/run-build-docker-jobs-image.sh b/build/run-build-docker-jobs-image.sh index 22f6f54288..a7e3ee5466 100755 --- a/build/run-build-docker-jobs-image.sh +++ b/build/run-build-docker-jobs-image.sh @@ -1,22 +1,28 @@ #!/bin/bash +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 function usage { echo >&2 - echo >&2 "usage: $0 [options]" + echo >&2 "usage: WORKSPACE=/path/to/arvados $0 [options]" echo >&2 echo >&2 "$0 options:" - echo >&2 " -t, --tags [csv_tags] comma separated tags" + echo >&2 " -t, --tags version tag for docker" + echo >&2 " -r, --repo Arvados package repo to use: dev (default), testing, stable" echo >&2 " -u, --upload Upload the images (docker push)" + echo >&2 " --no-cache Don't use build cache" echo >&2 " -h, --help Display this help and exit" echo >&2 - echo >&2 " If no options are given, just builds the images." + echo >&2 " WORKSPACE=path Path to the Arvados source tree to build from" + echo >&2 } - upload=false +REPO=dev # NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros). -TEMP=`getopt -o hut: \ - --long help,upload,tags: \ +TEMP=`getopt -o hut:r: \ + --long help,upload,no-cache,tags:,repo: \ -n "$0" -- "$@"` if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi @@ -30,6 +36,10 @@ do upload=true shift ;; + --no-cache) + NOCACHE=--no-cache + shift + ;; -t | --tags) case "$2" in "") @@ -38,7 +48,20 @@ do exit 1 ;; *) - tags=$2; + version_tag="$2"; + shift 2 + ;; + esac + ;; + -r | --repo) + case "$2" in + "") + echo "ERROR: --repo needs a parameter"; + usage; + exit 1 + ;; + *) + REPO="$2"; shift 2 ;; esac @@ -62,18 +85,20 @@ exit_cleanly() { exit $EXITCODE } +# Sanity check +if ! [[ -n "$WORKSPACE" ]]; then + usage; + echo >&2 "Error: WORKSPACE environment variable not set" + echo >&2 + exit 1 +fi + +echo $WORKSPACE + COLUMNS=80 . $WORKSPACE/build/run-library.sh docker_push () { - if [[ ! -z "$tags" ]] - then - for tag in $( echo $tags|tr "," " " ) - do - $DOCKER tag $1 $1:$tag - done - fi - # Sometimes docker push fails; retry it a few times if necessary. for i in `seq 1 5`; do $DOCKER push $* @@ -89,16 +114,6 @@ docker_push () { checkexit $ECODE "docker push $*" } -# Sanity check -if ! [[ -n "$WORKSPACE" ]]; then - echo >&2 - echo >&2 "Error: WORKSPACE environment variable not set" - echo >&2 - exit 1 -fi - -echo $WORKSPACE - # find the docker binary DOCKER=`which docker.io` @@ -118,13 +133,54 @@ timer_reset # clean up the docker build environment cd "$WORKSPACE" -cd docker/jobs -if [[ ! -z "$tags" ]]; then - docker build --build-arg COMMIT=${tags/,*/} -t arvados/jobs . + +if [[ -z "$ARVADOS_BUILDING_VERSION" ]] && ! [[ -z "$version_tag" ]]; then + export ARVADOS_BUILDING_VERSION="$version_tag" + export ARVADOS_BUILDING_ITERATION="1" +fi + +# This defines python_sdk_version and cwl_runner_version with python-style +# package suffixes (.dev/rc) +calculate_python_sdk_cwl_package_versions + +if [[ -z "$cwl_runner_version" ]]; then + echo "ERROR: cwl_runner_version is empty"; + exit 1 +fi + +echo cwl_runner_version $cwl_runner_version python_sdk_version $python_sdk_version + +# For development and release candidate packages, the OS package has a "~dev" +# or "~rc" suffix, but Python requires a ".dev" or "rc" suffix. +# +# Arvados-cwl-runner will be expecting the Python-compatible version string +# when it tries to pull the Docker image, so we use that to tag the Docker +# image. +# +# The --build-arg docker invocation arguments are expecting the OS package +# version. +python_sdk_version_os=$(echo -n $python_sdk_version | sed s/.dev/~dev/g | sed s/rc/~rc/g) +cwl_runner_version_os=$(echo -n $cwl_runner_version | sed s/.dev/~dev/g | sed s/rc/~rc/g) + +if [[ "${python_sdk_version}" != "${ARVADOS_BUILDING_VERSION}" ]]; then + python_sdk_version_os="${python_sdk_version_os}-1" else - docker build -t arvados/jobs . + python_sdk_version_os="${ARVADOS_BUILDING_VERSION}-${ARVADOS_BUILDING_ITERATION}" fi +if [[ "${cwl_runner_version_os}" != "${ARVADOS_BUILDING_VERSION}" ]]; then + cwl_runner_version_os="${cwl_runner_version_os}-1" +else + cwl_runner_version_os="${ARVADOS_BUILDING_VERSION}-${ARVADOS_BUILDING_ITERATION}" +fi + +cd docker/jobs +docker build $NOCACHE \ + --build-arg python_sdk_version=${python_sdk_version_os} \ + --build-arg cwl_runner_version=${cwl_runner_version_os} \ + --build-arg repo_version=${REPO} \ + -t arvados/jobs:$cwl_runner_version . + ECODE=$? if [[ "$ECODE" != "0" ]]; then @@ -134,19 +190,30 @@ fi checkexit $ECODE "docker build" title "docker build complete (`timer`)" +if [[ "$ECODE" != "0" ]]; then + exit_cleanly +fi + +timer_reset + +if docker --version |grep " 1\.[0-9]\." ; then + # Docker version prior 1.10 require -f flag + # -f flag removed in Docker 1.12 + FORCE=-f +fi + title "uploading images" timer_reset -if [[ "$ECODE" != "0" ]]; then - title "upload arvados images SKIPPED because build failed" +if [[ "$EXITCODE" != "0" ]]; then + title "upload arvados images SKIPPED because build or tag failed" else if [[ $upload == true ]]; then ## 20150526 nico -- *sometimes* dockerhub needs re-login ## even though credentials are already in .dockercfg docker login -u arvados - - docker_push arvados/jobs + docker_push arvados/jobs:$cwl_runner_version title "upload arvados images finished (`timer`)" else title "upload arvados images SKIPPED because no --upload option set (`timer`)"