10194: Improve versioning of arvados/jobs image.

arvados-cwl-runner sets version to max(cwl-runner, python-sdk).

build/run-build-docker-jobs-image.sh determines package versions corresponding
to current commit and supplies explicit package versions as Docker build arg.

diff --git a/build/run-build-docker-jobs-image.sh b/build/run-build-docker-jobs-image.sh
index 22f6f54288741d2ed4723e9260234b469fb24591..7b5ea4ecec6bcda873ed8a98ea09d862e8dd86f4 100755 (executable)
--- a/build/run-build-docker-jobs-image.sh
+++ b/build/run-build-docker-jobs-image.sh
@@ -5,8 +5,8 @@ function usage {
     echo >&2 "usage: $0 [options]"
     echo >&2
     echo >&2 "$0 options:"
-    echo >&2 "  -t, --tags [csv_tags]         comma separated tags"
     echo >&2 "  -u, --upload                  Upload the images (docker push)"
+    echo >&2 "  --no-cache                    Don't use build cache"
     echo >&2 "  -h, --help                    Display this help and exit"
     echo >&2
     echo >&2 "  If no options are given, just builds the images."
@@ -16,7 +16,7 @@ upload=false
 
 # NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
 TEMP=`getopt -o hut: \
-    --long help,upload,tags: \
+    --long help,upload,no-cache,tags: \
     -n "$0" -- "$@"`
 
 if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi
@@ -30,6 +30,10 @@ do
             upload=true
             shift
             ;;
+        --no-cache)
+            NOCACHE=--no-cache
+            shift
+            ;;
         -t | --tags)
             case "$2" in
                 "")
@@ -38,7 +42,7 @@ do
                   exit 1
                   ;;
                 *)
-                  tags=$2;
+                  echo "WARNING: --tags is deprecated and doesn't do anything";
                   shift 2
                   ;;
             esac
@@ -66,14 +70,6 @@ COLUMNS=80
 . $WORKSPACE/build/run-library.sh
 
 docker_push () {
-    if [[ ! -z "$tags" ]]
-    then
-        for tag in $( echo $tags|tr "," " " )
-        do
-             $DOCKER tag $1 $1:$tag
-        done
-    fi
-
     # Sometimes docker push fails; retry it a few times if necessary.
     for i in `seq 1 5`; do
         $DOCKER push $*
@@ -118,13 +114,28 @@ timer_reset
 
 # clean up the docker build environment
 cd "$WORKSPACE"
-cd docker/jobs
-if [[ ! -z "$tags" ]]; then
-    docker build --build-arg COMMIT=${tags/,*/} -t arvados/jobs .
+
+python_sdk_ts=$(cd sdk/python && timestamp_from_git)
+cwl_runner_ts=$(cd sdk/cwl && timestamp_from_git)
+
+python_sdk_version=$(cd sdk/python && nohash_version_from_git 0.1)-2
+cwl_runner_version=$(cd sdk/cwl && nohash_version_from_git 1.0)-3
+
+if [[ $python_sdk_ts -gt $cwl_runner_ts ]]; then
+    cwl_runner_version=$python_sdk_version
+    gittag=$(cd sdk/python && git log --first-parent --max-count=1 --format=format:%H)
 else
-    docker build -t arvados/jobs .
+    gittag=$(cd sdk/cwl && git log --first-parent --max-count=1 --format=format:%H)
 fi
 
+echo cwl_runner_version $cwl_runner_version python_sdk_version $python_sdk_version
+
+cd docker/jobs
+docker build $NOCACHE \
+       --build-arg python_sdk_version=$python_sdk_version \
+       --build-arg cwl_runner_version=$cwl_runner_version \
+       -t arvados/jobs:$gittag .
+
 ECODE=$?
 
 if [[ "$ECODE" != "0" ]]; then
@@ -134,19 +145,43 @@ fi
 checkexit $ECODE "docker build"
 title "docker build complete (`timer`)"
 
+if [[ "$ECODE" != "0" ]]; then
+  exit_cleanly
+fi
+
+timer_reset
+
+if docker --version |grep " 1\.[0-9]\." ; then
+    # Docker version prior 1.10 require -f flag
+    # -f flag removed in Docker 1.12
+    FORCE=-f
+fi
+
+docker tag $FORCE arvados/jobs:$gittag arvados/jobs:latest
+
+ECODE=$?
+
+if [[ "$ECODE" != "0" ]]; then
+    EXITCODE=$(($EXITCODE + $ECODE))
+fi
+
+checkexit $ECODE "docker tag"
+title "docker tag complete (`timer`)"
+
 title "uploading images"
 
 timer_reset
 
 if [[ "$ECODE" != "0" ]]; then
-    title "upload arvados images SKIPPED because build failed"
+    title "upload arvados images SKIPPED because build or tag failed"
 else
     if [[ $upload == true ]]; then
         ## 20150526 nico -- *sometimes* dockerhub needs re-login
         ## even though credentials are already in .dockercfg
         docker login -u arvados
 
-        docker_push arvados/jobs
+        docker_push arvados/jobs:$gittag
+        docker_push arvados/jobs:latest
         title "upload arvados images finished (`timer`)"
     else
         title "upload arvados images SKIPPED because no --upload option set (`timer`)"

diff --git a/build/run-library.sh b/build/run-library.sh
index 73a99dabd7b3626582a418040e1ce6713a68ada9..f0b120f6bf1e4e011a69f9f811ee67ad55624938 100755 (executable)
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -35,13 +35,19 @@ format_last_commit_here() {
 version_from_git() {
   # Generates a version number from the git log for the current working
   # directory, and writes it to stdout.
-  local git_ts git_hash
+  local git_ts git_hash prefix
+  if [[ -n "$1" ]] ; then
+      prefix="$1"
+  else
+      prefix="0.1"
+  fi
+
   declare $(format_last_commit_here "git_ts=%ct git_hash=%h")
-  echo "0.1.$(date -ud "@$git_ts" +%Y%m%d%H%M%S).$git_hash"
+  echo "${prefix}.$(date -ud "@$git_ts" +%Y%m%d%H%M%S).$git_hash"
 }
 
 nohash_version_from_git() {
-    version_from_git | cut -d. -f1-3
+    version_from_git $1 | cut -d. -f1-3
 }
 
 timestamp_from_git() {

diff --git a/docker/jobs/Dockerfile b/docker/jobs/Dockerfile
index e1e7e87c5e53d0c297ec6d2e3ad0870890f402ff..9b1be1e74df9e36501ed0bd8def2d70cd7e1bfea 100644 (file)
--- a/docker/jobs/Dockerfile
+++ b/docker/jobs/Dockerfile
@@ -8,10 +8,16 @@ ADD apt.arvados.org.list /etc/apt/sources.list.d/
 RUN apt-key adv --keyserver pool.sks-keyservers.net --recv 1078ECD7
 RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3
 
-ARG COMMIT=latest
-RUN echo $COMMIT && apt-get update -q
+ARG python_sdk_version
+ARG cwl_runner_version
+RUN echo cwl_runner_version $cwl_runner_version python_sdk_version $python_sdk_version
 
-RUN apt-get install -qy git python-pip python-virtualenv python-arvados-python-client python-dev libgnutls28-dev libcurl4-gnutls-dev nodejs python-arvados-cwl-runner
+RUN apt-get update -q
+RUN apt-get install -yq --no-install-recommends \
+    git python-pip python-virtualenv \
+    python-dev libgnutls28-dev libcurl4-gnutls-dev nodejs \
+    python-arvados-python-client=$python_sdk_version \
+    python-arvados-cwl-runner=$cwl_runner_version
 
 # Install dependencies and set up system.
 RUN /usr/sbin/adduser --disabled-password \

diff --git a/sdk/cwl/gittaggers.py b/sdk/cwl/gittaggers.py
deleted file mode 120000 (symlink)
index d59c02ca1758d4b5e484cb6e3280ba0ae2b7c5ea..0000000000000000000000000000000000000000
--- a/sdk/cwl/gittaggers.py
+++ /dev/null
@@ -1 +0,0 @@
-../python/gittaggers.py
\ No newline at end of file

diff --git a/sdk/cwl/gittaggers.py b/sdk/cwl/gittaggers.py
new file mode 100644 (file)
index 0000000000000000000000000000000000000000..2a292ee061b412d85216c68b81e2d0acd9557765
--- /dev/null
+++ b/sdk/cwl/gittaggers.py
@@ -0,0 +1,37 @@
+from setuptools.command.egg_info import egg_info
+import subprocess
+import time
+import os
+
+SETUP_DIR = os.path.dirname(__file__) or '.'
+
+def choose_version_from():
+    sdk_ts = subprocess.check_output(
+        ['git', 'log', '--first-parent', '--max-count=1',
+         '--format=format:%ct', os.path.join(SETUP_DIR, "../python")]).strip()
+    cwl_ts = subprocess.check_output(
+        ['git', 'log', '--first-parent', '--max-count=1',
+         '--format=format:%ct', SETUP_DIR]).strip()
+    if int(sdk_ts) > int(cwl_ts):
+        getver = os.path.join(SETUP_DIR, "../python")
+    else:
+        getver = SETUP_DIR
+    return getver
+
+class EggInfoFromGit(egg_info):
+    """Tag the build with git commit timestamp.
+
+    If a build tag has already been set (e.g., "egg_info -b", building
+    from source package), leave it alone.
+    """
+
+    def git_timestamp_tag(self):
+        gitinfo = subprocess.check_output(
+            ['git', 'log', '--first-parent', '--max-count=1',
+             '--format=format:%ct', choose_version_from()]).strip()
+        return time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo)))
+
+    def tags(self):
+        if self.tag_build is None:
+            self.tag_build = self.git_timestamp_tag()
+        return egg_info.tags(self)

diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 4adcac4daf87b4f4899fb883bb6fe499e4ee53d0..d1c8f9b567839bb6aaf1e78db2d6855b9a6038c2 100644 (file)
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -20,7 +20,7 @@ versionfile = os.path.join(SETUP_DIR, "arvados_cwl/_version.py")
 try:
     gitinfo = subprocess.check_output(
         ['git', 'log', '--first-parent', '--max-count=1',
-         '--format=format:%H', SETUP_DIR]).strip()
+         '--format=format:%H', gittaggers.choose_version_from()]).strip()
     with open(versionfile, "w") as f:
         f.write("__version__ = '%s'\n" % gitinfo)
 except Exception as e: