From: Peter Amstutz Date: Thu, 8 Oct 2020 16:55:31 +0000 (-0400) Subject: 16976: Support release candidates for our Python packages X-Git-Tag: 2.1.0~6^2~2 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/2af7b0336b2b92e38f6966b8bbc233c05704815d 16976: Support release candidates for our Python packages Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- diff --git a/build/package-build-dockerfiles/Makefile b/build/package-build-dockerfiles/Makefile index 818f257525..e116cb7fc3 100644 --- a/build/package-build-dockerfiles/Makefile +++ b/build/package-build-dockerfiles/Makefile @@ -6,24 +6,24 @@ all: centos7/generated debian9/generated debian10/generated ubuntu1604/generated centos7/generated: common-generated-all test -d centos7/generated || mkdir centos7/generated - cp -rlt centos7/generated common-generated/* + cp -f -rlt centos7/generated common-generated/* debian9/generated: common-generated-all test -d debian9/generated || mkdir debian9/generated - cp -rlt debian9/generated common-generated/* + cp -f -rlt debian9/generated common-generated/* debian10/generated: common-generated-all test -d debian10/generated || mkdir debian10/generated - cp -rlt debian10/generated common-generated/* + cp -f -rlt debian10/generated common-generated/* ubuntu1604/generated: common-generated-all test -d ubuntu1604/generated || mkdir ubuntu1604/generated - cp -rlt ubuntu1604/generated common-generated/* + cp -f -rlt ubuntu1604/generated common-generated/* ubuntu1804/generated: common-generated-all test -d ubuntu1804/generated || mkdir ubuntu1804/generated - cp -rlt ubuntu1804/generated common-generated/* + cp -f -rlt ubuntu1804/generated common-generated/* GOTARBALL=go1.13.4.linux-amd64.tar.gz NODETARBALL=node-v6.11.2-linux-x64.tar.xz diff --git a/build/package-test-dockerfiles/Makefile b/build/package-test-dockerfiles/Makefile index 1066750fe5..4bc13b0f34 100644 --- a/build/package-test-dockerfiles/Makefile +++ b/build/package-test-dockerfiles/Makefile @@ -6,23 +6,23 @@ all: centos7/generated debian9/generated debian10/generated ubuntu1604/generate centos7/generated: common-generated-all test -d centos7/generated || mkdir centos7/generated - cp -rlt centos7/generated common-generated/* + cp -f -rlt centos7/generated common-generated/* debian9/generated: common-generated-all test -d debian9/generated || mkdir debian9/generated - cp -rlt debian9/generated common-generated/* + cp -f -rlt debian9/generated common-generated/* debian10/generated: common-generated-all test -d debian10/generated || mkdir debian10/generated - cp -rlt debian10/generated common-generated/* + cp -f -rlt debian10/generated common-generated/* ubuntu1604/generated: common-generated-all test -d ubuntu1604/generated || mkdir ubuntu1604/generated - cp -rlt ubuntu1604/generated common-generated/* + cp -f -rlt ubuntu1604/generated common-generated/* ubuntu1804/generated: common-generated-all test -d ubuntu1804/generated || mkdir ubuntu1804/generated - cp -rlt ubuntu1804/generated common-generated/* + cp -f -rlt ubuntu1804/generated common-generated/* RVMKEY1=mpapis.asc RVMKEY2=pkuczynski.asc diff --git a/build/run-build-docker-jobs-image.sh b/build/run-build-docker-jobs-image.sh index d1fb2ac670..9d7e3777cd 100755 --- a/build/run-build-docker-jobs-image.sh +++ b/build/run-build-docker-jobs-image.sh @@ -149,7 +149,12 @@ else python_sdk_version="${ARVADOS_BUILDING_VERSION}-${ARVADOS_BUILDING_ITERATION}" fi -cwl_runner_version_orig=$cwl_runner_version +# What we use to tag the Docker image. For release candidate +# packages, the OS package has a "~rc" suffix, but Python requires +# just an "rc" suffix. Arvados-cwl-runner will be expecting the +# Python-compatible version string when it tries to pull the Docker +# image, but --build-arg is expecting the OS package version. +cwl_runner_version_tag=$(echo -n $cwl_runner_version | sed s/~rc/rc/g) if [[ "${cwl_runner_version}" != "${ARVADOS_BUILDING_VERSION}" ]]; then cwl_runner_version="${cwl_runner_version}-1" @@ -162,7 +167,7 @@ docker build $NOCACHE \ --build-arg python_sdk_version=${python_sdk_version} \ --build-arg cwl_runner_version=${cwl_runner_version} \ --build-arg repo_version=${REPO} \ - -t arvados/jobs:$cwl_runner_version_orig . + -t arvados/jobs:$cwl_runner_version_tag . ECODE=$? @@ -186,7 +191,7 @@ if docker --version |grep " 1\.[0-9]\." ; then fi if ! [[ -z "$version_tag" ]]; then - docker tag $FORCE arvados/jobs:$cwl_runner_version_orig arvados/jobs:"$version_tag" + docker tag $FORCE arvados/jobs:$cwl_runner_version_tag arvados/jobs:"$version_tag" ECODE=$? if [[ "$ECODE" != "0" ]]; then @@ -211,7 +216,7 @@ else if ! [[ -z "$version_tag" ]]; then docker_push arvados/jobs:"$version_tag" else - docker_push arvados/jobs:$cwl_runner_version_orig + docker_push arvados/jobs:$cwl_runner_version_tag fi title "upload arvados images finished (`timer`)" else diff --git a/build/run-library.sh b/build/run-library.sh index bd150e6a9b..216b5e5b0f 100755 --- a/build/run-library.sh +++ b/build/run-library.sh @@ -525,7 +525,11 @@ fpm_build_virtualenv () { fi # Determine the package version from the generated sdist archive - PYTHON_VERSION=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' *.egg-info/PKG-INFO)} + if [[ -n "$ARVADOS_BUILDING_VERSION" ]] ; then + PYTHON_VERSION=$(echo -n $ARVADOS_BUILDING_VERSION | sed s/~rc/rc/g) + else + PYTHON_VERSION=$(awk '($1 == "Version:"){print $2}' *.egg-info/PKG-INFO) + fi # See if we actually need to build this package; does it exist already? # We can't do this earlier than here, because we need PYTHON_VERSION... @@ -642,7 +646,7 @@ fpm_build_virtualenv () { COMMAND_ARR+=('--verbose' '--log' 'info') fi - COMMAND_ARR+=('-v' "$PYTHON_VERSION") + COMMAND_ARR+=('-v' $(echo "$PYTHON_VERSION" | sed s/rc/~rc/g)) COMMAND_ARR+=('--iteration' "$ARVADOS_BUILDING_ITERATION") COMMAND_ARR+=('-n' "$PYTHON_PKG") COMMAND_ARR+=('-C' "build") diff --git a/sdk/cwl/arvados_version.py b/sdk/cwl/arvados_version.py index d9ce12487a..64c2fa833c 100644 --- a/sdk/cwl/arvados_version.py +++ b/sdk/cwl/arvados_version.py @@ -41,6 +41,7 @@ def get_version(setup_dir, module): env_version = os.environ.get("ARVADOS_BUILDING_VERSION") if env_version: + env_version = env_version.replace("~rc", "rc") save_version(setup_dir, module, env_version) else: try: diff --git a/sdk/python/arvados_version.py b/sdk/python/arvados_version.py index 9aabff4292..21a6c3c6d2 100644 --- a/sdk/python/arvados_version.py +++ b/sdk/python/arvados_version.py @@ -26,6 +26,7 @@ def get_version(setup_dir, module): env_version = os.environ.get("ARVADOS_BUILDING_VERSION") if env_version: + env_version = env_version.replace("~rc", "rc") save_version(setup_dir, module, env_version) else: try: diff --git a/services/dockercleaner/arvados_version.py b/services/dockercleaner/arvados_version.py index 9aabff4292..21a6c3c6d2 100644 --- a/services/dockercleaner/arvados_version.py +++ b/services/dockercleaner/arvados_version.py @@ -26,6 +26,7 @@ def get_version(setup_dir, module): env_version = os.environ.get("ARVADOS_BUILDING_VERSION") if env_version: + env_version = env_version.replace("~rc", "rc") save_version(setup_dir, module, env_version) else: try: diff --git a/services/fuse/arvados_version.py b/services/fuse/arvados_version.py index 0c653694f5..5669342189 100644 --- a/services/fuse/arvados_version.py +++ b/services/fuse/arvados_version.py @@ -41,6 +41,7 @@ def get_version(setup_dir, module): env_version = os.environ.get("ARVADOS_BUILDING_VERSION") if env_version: + env_version = env_version.replace("~rc", "rc") save_version(setup_dir, module, env_version) else: try: diff --git a/tools/crunchstat-summary/arvados_version.py b/tools/crunchstat-summary/arvados_version.py index 0c653694f5..5669342189 100644 --- a/tools/crunchstat-summary/arvados_version.py +++ b/tools/crunchstat-summary/arvados_version.py @@ -41,6 +41,7 @@ def get_version(setup_dir, module): env_version = os.environ.get("ARVADOS_BUILDING_VERSION") if env_version: + env_version = env_version.replace("~rc", "rc") save_version(setup_dir, module, env_version) else: try: