From: Peter Amstutz Date: Fri, 21 Oct 2016 20:35:09 +0000 (-0400) Subject: 10194: Improve versioning of arvados/jobs image. X-Git-Tag: 1.1.0~638^2 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/4c86081e1ee5149f09f05cd3b77e59fc652d4ab3 10194: Improve versioning of arvados/jobs image. arvados-cwl-runner sets version to max(cwl-runner, python-sdk). build/run-build-docker-jobs-image.sh determines package versions corresponding to current commit and supplies explicit package versions as Docker build arg. --- diff --git a/build/run-build-docker-jobs-image.sh b/build/run-build-docker-jobs-image.sh index 22f6f54288..7b5ea4ecec 100755 --- a/build/run-build-docker-jobs-image.sh +++ b/build/run-build-docker-jobs-image.sh @@ -5,8 +5,8 @@ function usage { echo >&2 "usage: $0 [options]" echo >&2 echo >&2 "$0 options:" - echo >&2 " -t, --tags [csv_tags] comma separated tags" echo >&2 " -u, --upload Upload the images (docker push)" + echo >&2 " --no-cache Don't use build cache" echo >&2 " -h, --help Display this help and exit" echo >&2 echo >&2 " If no options are given, just builds the images." @@ -16,7 +16,7 @@ upload=false # NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros). TEMP=`getopt -o hut: \ - --long help,upload,tags: \ + --long help,upload,no-cache,tags: \ -n "$0" -- "$@"` if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi @@ -30,6 +30,10 @@ do upload=true shift ;; + --no-cache) + NOCACHE=--no-cache + shift + ;; -t | --tags) case "$2" in "") @@ -38,7 +42,7 @@ do exit 1 ;; *) - tags=$2; + echo "WARNING: --tags is deprecated and doesn't do anything"; shift 2 ;; esac @@ -66,14 +70,6 @@ COLUMNS=80 . $WORKSPACE/build/run-library.sh docker_push () { - if [[ ! -z "$tags" ]] - then - for tag in $( echo $tags|tr "," " " ) - do - $DOCKER tag $1 $1:$tag - done - fi - # Sometimes docker push fails; retry it a few times if necessary. for i in `seq 1 5`; do $DOCKER push $* @@ -118,13 +114,28 @@ timer_reset # clean up the docker build environment cd "$WORKSPACE" -cd docker/jobs -if [[ ! -z "$tags" ]]; then - docker build --build-arg COMMIT=${tags/,*/} -t arvados/jobs . + +python_sdk_ts=$(cd sdk/python && timestamp_from_git) +cwl_runner_ts=$(cd sdk/cwl && timestamp_from_git) + +python_sdk_version=$(cd sdk/python && nohash_version_from_git 0.1)-2 +cwl_runner_version=$(cd sdk/cwl && nohash_version_from_git 1.0)-3 + +if [[ $python_sdk_ts -gt $cwl_runner_ts ]]; then + cwl_runner_version=$python_sdk_version + gittag=$(cd sdk/python && git log --first-parent --max-count=1 --format=format:%H) else - docker build -t arvados/jobs . + gittag=$(cd sdk/cwl && git log --first-parent --max-count=1 --format=format:%H) fi +echo cwl_runner_version $cwl_runner_version python_sdk_version $python_sdk_version + +cd docker/jobs +docker build $NOCACHE \ + --build-arg python_sdk_version=$python_sdk_version \ + --build-arg cwl_runner_version=$cwl_runner_version \ + -t arvados/jobs:$gittag . + ECODE=$? if [[ "$ECODE" != "0" ]]; then @@ -134,19 +145,43 @@ fi checkexit $ECODE "docker build" title "docker build complete (`timer`)" +if [[ "$ECODE" != "0" ]]; then + exit_cleanly +fi + +timer_reset + +if docker --version |grep " 1\.[0-9]\." ; then + # Docker version prior 1.10 require -f flag + # -f flag removed in Docker 1.12 + FORCE=-f +fi + +docker tag $FORCE arvados/jobs:$gittag arvados/jobs:latest + +ECODE=$? + +if [[ "$ECODE" != "0" ]]; then + EXITCODE=$(($EXITCODE + $ECODE)) +fi + +checkexit $ECODE "docker tag" +title "docker tag complete (`timer`)" + title "uploading images" timer_reset if [[ "$ECODE" != "0" ]]; then - title "upload arvados images SKIPPED because build failed" + title "upload arvados images SKIPPED because build or tag failed" else if [[ $upload == true ]]; then ## 20150526 nico -- *sometimes* dockerhub needs re-login ## even though credentials are already in .dockercfg docker login -u arvados - docker_push arvados/jobs + docker_push arvados/jobs:$gittag + docker_push arvados/jobs:latest title "upload arvados images finished (`timer`)" else title "upload arvados images SKIPPED because no --upload option set (`timer`)" diff --git a/build/run-library.sh b/build/run-library.sh index 73a99dabd7..f0b120f6bf 100755 --- a/build/run-library.sh +++ b/build/run-library.sh @@ -35,13 +35,19 @@ format_last_commit_here() { version_from_git() { # Generates a version number from the git log for the current working # directory, and writes it to stdout. - local git_ts git_hash + local git_ts git_hash prefix + if [[ -n "$1" ]] ; then + prefix="$1" + else + prefix="0.1" + fi + declare $(format_last_commit_here "git_ts=%ct git_hash=%h") - echo "0.1.$(date -ud "@$git_ts" +%Y%m%d%H%M%S).$git_hash" + echo "${prefix}.$(date -ud "@$git_ts" +%Y%m%d%H%M%S).$git_hash" } nohash_version_from_git() { - version_from_git | cut -d. -f1-3 + version_from_git $1 | cut -d. -f1-3 } timestamp_from_git() { diff --git a/docker/jobs/Dockerfile b/docker/jobs/Dockerfile index e1e7e87c5e..9b1be1e74d 100644 --- a/docker/jobs/Dockerfile +++ b/docker/jobs/Dockerfile @@ -8,10 +8,16 @@ ADD apt.arvados.org.list /etc/apt/sources.list.d/ RUN apt-key adv --keyserver pool.sks-keyservers.net --recv 1078ECD7 RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 -ARG COMMIT=latest -RUN echo $COMMIT && apt-get update -q +ARG python_sdk_version +ARG cwl_runner_version +RUN echo cwl_runner_version $cwl_runner_version python_sdk_version $python_sdk_version -RUN apt-get install -qy git python-pip python-virtualenv python-arvados-python-client python-dev libgnutls28-dev libcurl4-gnutls-dev nodejs python-arvados-cwl-runner +RUN apt-get update -q +RUN apt-get install -yq --no-install-recommends \ + git python-pip python-virtualenv \ + python-dev libgnutls28-dev libcurl4-gnutls-dev nodejs \ + python-arvados-python-client=$python_sdk_version \ + python-arvados-cwl-runner=$cwl_runner_version # Install dependencies and set up system. RUN /usr/sbin/adduser --disabled-password \ diff --git a/sdk/cwl/gittaggers.py b/sdk/cwl/gittaggers.py deleted file mode 120000 index d59c02ca17..0000000000 --- a/sdk/cwl/gittaggers.py +++ /dev/null @@ -1 +0,0 @@ -../python/gittaggers.py \ No newline at end of file diff --git a/sdk/cwl/gittaggers.py b/sdk/cwl/gittaggers.py new file mode 100644 index 0000000000..2a292ee061 --- /dev/null +++ b/sdk/cwl/gittaggers.py @@ -0,0 +1,37 @@ +from setuptools.command.egg_info import egg_info +import subprocess +import time +import os + +SETUP_DIR = os.path.dirname(__file__) or '.' + +def choose_version_from(): + sdk_ts = subprocess.check_output( + ['git', 'log', '--first-parent', '--max-count=1', + '--format=format:%ct', os.path.join(SETUP_DIR, "../python")]).strip() + cwl_ts = subprocess.check_output( + ['git', 'log', '--first-parent', '--max-count=1', + '--format=format:%ct', SETUP_DIR]).strip() + if int(sdk_ts) > int(cwl_ts): + getver = os.path.join(SETUP_DIR, "../python") + else: + getver = SETUP_DIR + return getver + +class EggInfoFromGit(egg_info): + """Tag the build with git commit timestamp. + + If a build tag has already been set (e.g., "egg_info -b", building + from source package), leave it alone. + """ + + def git_timestamp_tag(self): + gitinfo = subprocess.check_output( + ['git', 'log', '--first-parent', '--max-count=1', + '--format=format:%ct', choose_version_from()]).strip() + return time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo))) + + def tags(self): + if self.tag_build is None: + self.tag_build = self.git_timestamp_tag() + return egg_info.tags(self) diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py index 4adcac4daf..d1c8f9b567 100644 --- a/sdk/cwl/setup.py +++ b/sdk/cwl/setup.py @@ -20,7 +20,7 @@ versionfile = os.path.join(SETUP_DIR, "arvados_cwl/_version.py") try: gitinfo = subprocess.check_output( ['git', 'log', '--first-parent', '--max-count=1', - '--format=format:%H', SETUP_DIR]).strip() + '--format=format:%H', gittaggers.choose_version_from()]).strip() with open(versionfile, "w") as f: f.write("__version__ = '%s'\n" % gitinfo) except Exception as e: