Merge branch '16976-python-rc-packages' refs #16976

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz@curii.com>

diff --git a/build/package-build-dockerfiles/Makefile b/build/package-build-dockerfiles/Makefile
index 818f2575254f91ab81cacdb04f9db055ad68e1b8..e116cb7fc3ffb2688abcf5a5c6fe10d6e32d4718 100644 (file)
--- a/build/package-build-dockerfiles/Makefile
+++ b/build/package-build-dockerfiles/Makefile
@@ -6,24 +6,24 @@ all: centos7/generated debian9/generated debian10/generated ubuntu1604/generated
 
 centos7/generated: common-generated-all
        test -d centos7/generated || mkdir centos7/generated
-       cp -rlt centos7/generated common-generated/*
+       cp -f -rlt centos7/generated common-generated/*
 
 debian9/generated: common-generated-all
        test -d debian9/generated || mkdir debian9/generated
-       cp -rlt debian9/generated common-generated/*
+       cp -f -rlt debian9/generated common-generated/*
 
 debian10/generated: common-generated-all
        test -d debian10/generated || mkdir debian10/generated
-       cp -rlt debian10/generated common-generated/*
+       cp -f -rlt debian10/generated common-generated/*
 
 
 ubuntu1604/generated: common-generated-all
        test -d ubuntu1604/generated || mkdir ubuntu1604/generated
-       cp -rlt ubuntu1604/generated common-generated/*
+       cp -f -rlt ubuntu1604/generated common-generated/*
 
 ubuntu1804/generated: common-generated-all
        test -d ubuntu1804/generated || mkdir ubuntu1804/generated
-       cp -rlt ubuntu1804/generated common-generated/*
+       cp -f -rlt ubuntu1804/generated common-generated/*
 
 GOTARBALL=go1.13.4.linux-amd64.tar.gz
 NODETARBALL=node-v6.11.2-linux-x64.tar.xz

diff --git a/build/package-test-dockerfiles/Makefile b/build/package-test-dockerfiles/Makefile
index 1066750fe551c583edc1059a0fcb750f98799e8b..4bc13b0f34a205db65bc8afbaa384454f39cb8c6 100644 (file)
--- a/build/package-test-dockerfiles/Makefile
+++ b/build/package-test-dockerfiles/Makefile
@@ -6,23 +6,23 @@ all: centos7/generated debian9/generated  debian10/generated ubuntu1604/generate
 
 centos7/generated: common-generated-all
        test -d centos7/generated || mkdir centos7/generated
-       cp -rlt centos7/generated common-generated/*
+       cp -f -rlt centos7/generated common-generated/*
 
 debian9/generated: common-generated-all
        test -d debian9/generated || mkdir debian9/generated
-       cp -rlt debian9/generated common-generated/*
+       cp -f -rlt debian9/generated common-generated/*
 
 debian10/generated: common-generated-all
        test -d debian10/generated || mkdir debian10/generated
-       cp -rlt debian10/generated common-generated/*
+       cp -f -rlt debian10/generated common-generated/*
 
 ubuntu1604/generated: common-generated-all
        test -d ubuntu1604/generated || mkdir ubuntu1604/generated
-       cp -rlt ubuntu1604/generated common-generated/*
+       cp -f -rlt ubuntu1604/generated common-generated/*
 
 ubuntu1804/generated: common-generated-all
        test -d ubuntu1804/generated || mkdir ubuntu1804/generated
-       cp -rlt ubuntu1804/generated common-generated/*
+       cp -f -rlt ubuntu1804/generated common-generated/*
 
 RVMKEY1=mpapis.asc
 RVMKEY2=pkuczynski.asc

diff --git a/build/run-build-docker-jobs-image.sh b/build/run-build-docker-jobs-image.sh
index d1fb2ac67054dfdc31ce8a31401747c3a55aefbf..7ec82c4401d708ffb36606b008a3bdf479a50883 100755 (executable)
--- a/build/run-build-docker-jobs-image.sh
+++ b/build/run-build-docker-jobs-image.sh
@@ -149,7 +149,12 @@ else
        python_sdk_version="${ARVADOS_BUILDING_VERSION}-${ARVADOS_BUILDING_ITERATION}"
 fi
 
-cwl_runner_version_orig=$cwl_runner_version
+# What we use to tag the Docker image.  For release candidate
+# packages, the OS package has a "~rc" suffix, but Python requires
+# just an "rc" suffix.  Arvados-cwl-runner will be expecting the
+# Python-compatible version string when it tries to pull the Docker
+# image, but --build-arg is expecting the OS package version.
+cwl_runner_version_tag=$(echo -n $cwl_runner_version | sed s/~rc/rc/g)
 
 if [[ "${cwl_runner_version}" != "${ARVADOS_BUILDING_VERSION}" ]]; then
        cwl_runner_version="${cwl_runner_version}-1"
@@ -162,7 +167,7 @@ docker build $NOCACHE \
        --build-arg python_sdk_version=${python_sdk_version} \
        --build-arg cwl_runner_version=${cwl_runner_version} \
        --build-arg repo_version=${REPO} \
-       -t arvados/jobs:$cwl_runner_version_orig .
+       -t arvados/jobs:$cwl_runner_version_tag .
 
 ECODE=$?
 
@@ -185,18 +190,6 @@ if docker --version |grep " 1\.[0-9]\." ; then
     FORCE=-f
 fi
 
-if ! [[ -z "$version_tag" ]]; then
-    docker tag $FORCE arvados/jobs:$cwl_runner_version_orig arvados/jobs:"$version_tag"
-    ECODE=$?
-
-    if [[ "$ECODE" != "0" ]]; then
-        EXITCODE=$(($EXITCODE + $ECODE))
-    fi
-
-    checkexit $ECODE "docker tag"
-    title "docker tag complete (`timer`)"
-fi
-
 title "uploading images"
 
 timer_reset
@@ -208,11 +201,7 @@ else
         ## 20150526 nico -- *sometimes* dockerhub needs re-login
         ## even though credentials are already in .dockercfg
         docker login -u arvados
-        if ! [[ -z "$version_tag" ]]; then
-            docker_push arvados/jobs:"$version_tag"
-        else
-           docker_push arvados/jobs:$cwl_runner_version_orig
-        fi
+        docker_push arvados/jobs:$cwl_runner_version_tag
         title "upload arvados images finished (`timer`)"
     else
         title "upload arvados images SKIPPED because no --upload option set (`timer`)"

diff --git a/build/run-library.sh b/build/run-library.sh
index bd150e6a9b36ae96ed57792f2c7f79c51d21fcae..a085843d29de42f621a221fb73faf9edc345e9d1 100755 (executable)
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -525,13 +525,19 @@ fpm_build_virtualenv () {
   fi
 
   # Determine the package version from the generated sdist archive
-  PYTHON_VERSION=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' *.egg-info/PKG-INFO)}
+  if [[ -n "$ARVADOS_BUILDING_VERSION" ]] ; then
+      UNFILTERED_PYTHON_VERSION=$(echo -n $ARVADOS_BUILDING_VERSION)
+      PYTHON_VERSION=$(echo -n $ARVADOS_BUILDING_VERSION | sed s/~rc/rc/g)
+  else
+      UNFILTERED_PYTHON_VERSION=$(awk '($1 == "Version:"){print $2}' *.egg-info/PKG-INFO)
+      PYTHON_VERSION=$(awk '($1 == "Version:"){print $2}' *.egg-info/PKG-INFO)
+  fi
 
   # See if we actually need to build this package; does it exist already?
   # We can't do this earlier than here, because we need PYTHON_VERSION...
   # This isn't so bad; the sdist call above is pretty quick compared to
   # the invocation of virtualenv and fpm, below.
-  if ! test_package_presence "$PYTHON_PKG" $PYTHON_VERSION $PACKAGE_TYPE $ARVADOS_BUILDING_ITERATION; then
+  if ! test_package_presence "$PYTHON_PKG" $UNFILTERED_PYTHON_VERSION $PACKAGE_TYPE $ARVADOS_BUILDING_ITERATION; then
     return 0
   fi
 
@@ -642,7 +648,7 @@ fpm_build_virtualenv () {
     COMMAND_ARR+=('--verbose' '--log' 'info')
   fi
 
-  COMMAND_ARR+=('-v' "$PYTHON_VERSION")
+  COMMAND_ARR+=('-v' $(echo "$PYTHON_VERSION" | sed s/rc/~rc/g))
   COMMAND_ARR+=('--iteration' "$ARVADOS_BUILDING_ITERATION")
   COMMAND_ARR+=('-n' "$PYTHON_PKG")
   COMMAND_ARR+=('-C' "build")

diff --git a/sdk/cwl/arvados_version.py b/sdk/cwl/arvados_version.py
index d9ce12487a1ce3c4f281296125e8cf81bc6b48fe..64c2fa833c173c30a06ce50d76fb505492da7081 100644 (file)
--- a/sdk/cwl/arvados_version.py
+++ b/sdk/cwl/arvados_version.py
@@ -41,6 +41,7 @@ def get_version(setup_dir, module):
     env_version = os.environ.get("ARVADOS_BUILDING_VERSION")
 
     if env_version:
+        env_version = env_version.replace("~rc", "rc")
         save_version(setup_dir, module, env_version)
     else:
         try:

diff --git a/sdk/python/arvados_version.py b/sdk/python/arvados_version.py
index 9aabff42929838a1f9748362a63eeed003775a64..21a6c3c6d238199838ddc9b02f259e7a71acbc7a 100644 (file)
--- a/sdk/python/arvados_version.py
+++ b/sdk/python/arvados_version.py
@@ -26,6 +26,7 @@ def get_version(setup_dir, module):
     env_version = os.environ.get("ARVADOS_BUILDING_VERSION")
 
     if env_version:
+        env_version = env_version.replace("~rc", "rc")
         save_version(setup_dir, module, env_version)
     else:
         try:

diff --git a/services/dockercleaner/arvados_version.py b/services/dockercleaner/arvados_version.py
index 9aabff42929838a1f9748362a63eeed003775a64..21a6c3c6d238199838ddc9b02f259e7a71acbc7a 100644 (file)
--- a/services/dockercleaner/arvados_version.py
+++ b/services/dockercleaner/arvados_version.py
@@ -26,6 +26,7 @@ def get_version(setup_dir, module):
     env_version = os.environ.get("ARVADOS_BUILDING_VERSION")
 
     if env_version:
+        env_version = env_version.replace("~rc", "rc")
         save_version(setup_dir, module, env_version)
     else:
         try:

diff --git a/services/fuse/arvados_version.py b/services/fuse/arvados_version.py
index 0c653694f566b3883ccd2682b05d446eff849bd0..5669342189cb777558b6410be75702cff24e5872 100644 (file)
--- a/services/fuse/arvados_version.py
+++ b/services/fuse/arvados_version.py
@@ -41,6 +41,7 @@ def get_version(setup_dir, module):
     env_version = os.environ.get("ARVADOS_BUILDING_VERSION")
 
     if env_version:
+        env_version = env_version.replace("~rc", "rc")
         save_version(setup_dir, module, env_version)
     else:
         try:

diff --git a/tools/crunchstat-summary/arvados_version.py b/tools/crunchstat-summary/arvados_version.py
index 0c653694f566b3883ccd2682b05d446eff849bd0..5669342189cb777558b6410be75702cff24e5872 100644 (file)
--- a/tools/crunchstat-summary/arvados_version.py
+++ b/tools/crunchstat-summary/arvados_version.py
@@ -41,6 +41,7 @@ def get_version(setup_dir, module):
     env_version = os.environ.get("ARVADOS_BUILDING_VERSION")
 
     if env_version:
+        env_version = env_version.replace("~rc", "rc")
         save_version(setup_dir, module, env_version)
     else:
         try: