From: Peter Amstutz Date: Wed, 15 Jan 2020 20:26:54 +0000 (-0500) Subject: Update python packages to use version_at_commit script X-Git-Tag: 2.0.0~53 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/a344ad38e66d12b94aa7bc3a53f64c343e6fc0e6 Update python packages to use version_at_commit script Update test images to use Bundler 2.0.2 no issue # Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- diff --git a/build/package-test-dockerfiles/centos7/Dockerfile b/build/package-test-dockerfiles/centos7/Dockerfile index 49d04aa744..6508c86be0 100644 --- a/build/package-test-dockerfiles/centos7/Dockerfile +++ b/build/package-test-dockerfiles/centos7/Dockerfile @@ -16,7 +16,8 @@ RUN touch /var/lib/rpm/* && \ gpg --import --no-tty /tmp/pkuczynski.asc && \ curl -L https://get.rvm.io | bash -s stable && \ /usr/local/rvm/bin/rvm install 2.3 && \ - /usr/local/rvm/bin/rvm alias create default ruby-2.3 + /usr/local/rvm/bin/rvm alias create default ruby-2.3 && \ + /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.0.2 # Install Bash 4.4.12 // see https://dev.arvados.org/issues/15612 RUN cd /usr/local/src \ diff --git a/build/package-test-dockerfiles/debian10/Dockerfile b/build/package-test-dockerfiles/debian10/Dockerfile index 3aa6fdcce1..48e4984d58 100644 --- a/build/package-test-dockerfiles/debian10/Dockerfile +++ b/build/package-test-dockerfiles/debian10/Dockerfile @@ -18,7 +18,8 @@ RUN gpg --import --no-tty /tmp/mpapis.asc && \ gpg --import --no-tty /tmp/pkuczynski.asc && \ curl -L https://get.rvm.io | bash -s stable && \ /usr/local/rvm/bin/rvm install 2.5 && \ - /usr/local/rvm/bin/rvm alias create default ruby-2.5 + /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \ + /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.0.2 # udev daemon can't start in a container, so don't try. RUN mkdir -p /etc/udev/disabled diff --git a/build/package-test-dockerfiles/debian9/Dockerfile b/build/package-test-dockerfiles/debian9/Dockerfile index e759c9ce1d..470845ae97 100644 --- a/build/package-test-dockerfiles/debian9/Dockerfile +++ b/build/package-test-dockerfiles/debian9/Dockerfile @@ -18,7 +18,8 @@ RUN gpg --import --no-tty /tmp/mpapis.asc && \ gpg --import --no-tty /tmp/pkuczynski.asc && \ curl -L https://get.rvm.io | bash -s stable && \ /usr/local/rvm/bin/rvm install 2.5 && \ - /usr/local/rvm/bin/rvm alias create default ruby-2.5 + /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \ + /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.0.2 # udev daemon can't start in a container, so don't try. RUN mkdir -p /etc/udev/disabled diff --git a/build/package-test-dockerfiles/ubuntu1604/Dockerfile b/build/package-test-dockerfiles/ubuntu1604/Dockerfile index 422ee43e06..c35deebf10 100644 --- a/build/package-test-dockerfiles/ubuntu1604/Dockerfile +++ b/build/package-test-dockerfiles/ubuntu1604/Dockerfile @@ -18,7 +18,8 @@ RUN gpg --import --no-tty /tmp/mpapis.asc && \ gpg --import --no-tty /tmp/pkuczynski.asc && \ curl -L https://get.rvm.io | bash -s stable && \ /usr/local/rvm/bin/rvm install 2.5 && \ - /usr/local/rvm/bin/rvm alias create default ruby-2.5 + /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \ + /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.0.2 # udev daemon can't start in a container, so don't try. RUN mkdir -p /etc/udev/disabled diff --git a/build/package-test-dockerfiles/ubuntu1804/Dockerfile b/build/package-test-dockerfiles/ubuntu1804/Dockerfile index 68babe3fd5..60296e6be1 100644 --- a/build/package-test-dockerfiles/ubuntu1804/Dockerfile +++ b/build/package-test-dockerfiles/ubuntu1804/Dockerfile @@ -18,7 +18,8 @@ RUN gpg --import --no-tty /tmp/mpapis.asc && \ gpg --import --no-tty /tmp/pkuczynski.asc && \ curl -L https://get.rvm.io | bash -s stable && \ /usr/local/rvm/bin/rvm install 2.5 && \ - /usr/local/rvm/bin/rvm alias create default ruby-2.5 + /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \ + /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.0.2 # udev daemon can't start in a container, so don't try. RUN mkdir -p /etc/udev/disabled diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh index 3173aa7057..f00d0c0310 100755 --- a/build/run-build-packages.sh +++ b/build/run-build-packages.sh @@ -349,6 +349,8 @@ if [[ -e "$WORKSPACE/cwltest" ]]; then rm -rf "$WORKSPACE/cwltest" fi git clone https://github.com/common-workflow-language/cwltest.git +# last release to support python 2.7 +(cd cwltest && git checkout 1.0.20190906212748) # signal to our build script that we want a cwltest executable installed in /usr/bin/ mkdir cwltest/bin && touch cwltest/bin/cwltest fpm_build_virtualenv "cwltest" "cwltest" diff --git a/build/run-library.sh b/build/run-library.sh index 46b4dc6045..f09071cee7 100755 --- a/build/run-library.sh +++ b/build/run-library.sh @@ -10,6 +10,8 @@ # older packages. LICENSE_PACKAGE_TS=20151208015500 +librarydir=`dirname "$(readlink -f "$0")"` + if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then RAILS_PACKAGE_ITERATION=1 else @@ -57,7 +59,7 @@ version_from_git() { local git_ts git_hash declare $(format_last_commit_here "git_ts=%ct git_hash=%h" "$subdir") - ARVADOS_BUILDING_VERSION="$(git tag -l |sort -V -r |head -n1).$(date -ud "@$git_ts" +%Y%m%d%H%M%S)" + ARVADOS_BUILDING_VERSION="$($librarydir/version-at-commit.sh $git_hash)" echo "$ARVADOS_BUILDING_VERSION" } diff --git a/sdk/cwl/arvados_version.py b/sdk/cwl/arvados_version.py index d13dd5ec53..9aabff4292 100644 --- a/sdk/cwl/arvados_version.py +++ b/sdk/cwl/arvados_version.py @@ -7,38 +7,19 @@ import time import os import re -SETUP_DIR = os.path.dirname(__file__) or '.' - -def git_latest_tag(): - gittags = subprocess.check_output(['git', 'tag', '-l']).split() - gittags.sort(key=lambda s: [int(u) for u in s.split(b'.')],reverse=True) - return str(next(iter(gittags)).decode('utf-8')) - -def choose_version_from(): - sdk_ts = subprocess.check_output( - ['git', 'log', '--first-parent', '--max-count=1', - '--format=format:%ct', os.path.join(SETUP_DIR, "../python")]).strip() - cwl_ts = subprocess.check_output( - ['git', 'log', '--first-parent', '--max-count=1', - '--format=format:%ct', SETUP_DIR]).strip() - if int(sdk_ts) > int(cwl_ts): - getver = os.path.join(SETUP_DIR, "../python") - else: - getver = SETUP_DIR - return getver - -def git_timestamp_tag(): - gitinfo = subprocess.check_output( - ['git', 'log', '--first-parent', '--max-count=1', - '--format=format:%ct', choose_version_from()]).strip() - return str(time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo)))) +def git_version_at_commit(): + curdir = os.path.dirname(os.path.abspath(__file__)) + myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent', + '--format=%H', curdir]).strip() + myversion = subprocess.check_output([curdir+'/../../build/version-at-commit.sh', myhash]).strip().decode() + return myversion def save_version(setup_dir, module, v): - with open(os.path.join(setup_dir, module, "_version.py"), 'w') as fp: + with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp: return fp.write("__version__ = '%s'\n" % v) def read_version(setup_dir, module): - with open(os.path.join(setup_dir, module, "_version.py"), 'r') as fp: + with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp: return re.match("__version__ = '(.*)'$", fp.read()).groups()[0] def get_version(setup_dir, module): @@ -48,7 +29,7 @@ def get_version(setup_dir, module): save_version(setup_dir, module, env_version) else: try: - save_version(setup_dir, module, git_latest_tag() + git_timestamp_tag()) + save_version(setup_dir, module, git_version_at_commit()) except (subprocess.CalledProcessError, OSError): pass diff --git a/sdk/pam/arvados_version.py b/sdk/pam/arvados_version.py index 2e6484cabd..9aabff4292 100644 --- a/sdk/pam/arvados_version.py +++ b/sdk/pam/arvados_version.py @@ -7,23 +7,19 @@ import time import os import re -def git_latest_tag(): - gittags = subprocess.check_output(['git', 'tag', '-l']).split() - gittags.sort(key=lambda s: [int(u) for u in s.split(b'.')],reverse=True) - return str(next(iter(gittags)).decode('utf-8')) - -def git_timestamp_tag(): - gitinfo = subprocess.check_output( - ['git', 'log', '--first-parent', '--max-count=1', - '--format=format:%ct', '.']).strip() - return str(time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo)))) +def git_version_at_commit(): + curdir = os.path.dirname(os.path.abspath(__file__)) + myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent', + '--format=%H', curdir]).strip() + myversion = subprocess.check_output([curdir+'/../../build/version-at-commit.sh', myhash]).strip().decode() + return myversion def save_version(setup_dir, module, v): - with open(os.path.join(setup_dir, module, "_version.py"), 'w') as fp: + with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp: return fp.write("__version__ = '%s'\n" % v) def read_version(setup_dir, module): - with open(os.path.join(setup_dir, module, "_version.py"), 'r') as fp: + with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp: return re.match("__version__ = '(.*)'$", fp.read()).groups()[0] def get_version(setup_dir, module): @@ -33,8 +29,8 @@ def get_version(setup_dir, module): save_version(setup_dir, module, env_version) else: try: - save_version(setup_dir, module, git_latest_tag() + git_timestamp_tag()) - except subprocess.CalledProcessError: + save_version(setup_dir, module, git_version_at_commit()) + except (subprocess.CalledProcessError, OSError): pass return read_version(setup_dir, module) diff --git a/sdk/python/arvados_version.py b/sdk/python/arvados_version.py index 6921464652..9aabff4292 100644 --- a/sdk/python/arvados_version.py +++ b/sdk/python/arvados_version.py @@ -7,23 +7,19 @@ import time import os import re -def git_latest_tag(): - gittags = subprocess.check_output(['git', 'tag', '-l']).split() - gittags.sort(key=lambda s: [int(u) for u in s.split(b'.')],reverse=True) - return str(next(iter(gittags)).decode('utf-8')) - -def git_timestamp_tag(): - gitinfo = subprocess.check_output( - ['git', 'log', '--first-parent', '--max-count=1', - '--format=format:%ct', '.']).strip() - return str(time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo)))) +def git_version_at_commit(): + curdir = os.path.dirname(os.path.abspath(__file__)) + myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent', + '--format=%H', curdir]).strip() + myversion = subprocess.check_output([curdir+'/../../build/version-at-commit.sh', myhash]).strip().decode() + return myversion def save_version(setup_dir, module, v): - with open(os.path.join(setup_dir, module, "_version.py"), 'w') as fp: + with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp: return fp.write("__version__ = '%s'\n" % v) def read_version(setup_dir, module): - with open(os.path.join(setup_dir, module, "_version.py"), 'r') as fp: + with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp: return re.match("__version__ = '(.*)'$", fp.read()).groups()[0] def get_version(setup_dir, module): @@ -33,7 +29,7 @@ def get_version(setup_dir, module): save_version(setup_dir, module, env_version) else: try: - save_version(setup_dir, module, git_latest_tag() + git_timestamp_tag()) + save_version(setup_dir, module, git_version_at_commit()) except (subprocess.CalledProcessError, OSError): pass diff --git a/services/dockercleaner/arvados_version.py b/services/dockercleaner/arvados_version.py index 2e6484cabd..9aabff4292 100644 --- a/services/dockercleaner/arvados_version.py +++ b/services/dockercleaner/arvados_version.py @@ -7,23 +7,19 @@ import time import os import re -def git_latest_tag(): - gittags = subprocess.check_output(['git', 'tag', '-l']).split() - gittags.sort(key=lambda s: [int(u) for u in s.split(b'.')],reverse=True) - return str(next(iter(gittags)).decode('utf-8')) - -def git_timestamp_tag(): - gitinfo = subprocess.check_output( - ['git', 'log', '--first-parent', '--max-count=1', - '--format=format:%ct', '.']).strip() - return str(time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo)))) +def git_version_at_commit(): + curdir = os.path.dirname(os.path.abspath(__file__)) + myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent', + '--format=%H', curdir]).strip() + myversion = subprocess.check_output([curdir+'/../../build/version-at-commit.sh', myhash]).strip().decode() + return myversion def save_version(setup_dir, module, v): - with open(os.path.join(setup_dir, module, "_version.py"), 'w') as fp: + with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp: return fp.write("__version__ = '%s'\n" % v) def read_version(setup_dir, module): - with open(os.path.join(setup_dir, module, "_version.py"), 'r') as fp: + with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp: return re.match("__version__ = '(.*)'$", fp.read()).groups()[0] def get_version(setup_dir, module): @@ -33,8 +29,8 @@ def get_version(setup_dir, module): save_version(setup_dir, module, env_version) else: try: - save_version(setup_dir, module, git_latest_tag() + git_timestamp_tag()) - except subprocess.CalledProcessError: + save_version(setup_dir, module, git_version_at_commit()) + except (subprocess.CalledProcessError, OSError): pass return read_version(setup_dir, module) diff --git a/services/fuse/arvados_version.py b/services/fuse/arvados_version.py index 2e6484cabd..9aabff4292 100644 --- a/services/fuse/arvados_version.py +++ b/services/fuse/arvados_version.py @@ -7,23 +7,19 @@ import time import os import re -def git_latest_tag(): - gittags = subprocess.check_output(['git', 'tag', '-l']).split() - gittags.sort(key=lambda s: [int(u) for u in s.split(b'.')],reverse=True) - return str(next(iter(gittags)).decode('utf-8')) - -def git_timestamp_tag(): - gitinfo = subprocess.check_output( - ['git', 'log', '--first-parent', '--max-count=1', - '--format=format:%ct', '.']).strip() - return str(time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo)))) +def git_version_at_commit(): + curdir = os.path.dirname(os.path.abspath(__file__)) + myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent', + '--format=%H', curdir]).strip() + myversion = subprocess.check_output([curdir+'/../../build/version-at-commit.sh', myhash]).strip().decode() + return myversion def save_version(setup_dir, module, v): - with open(os.path.join(setup_dir, module, "_version.py"), 'w') as fp: + with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp: return fp.write("__version__ = '%s'\n" % v) def read_version(setup_dir, module): - with open(os.path.join(setup_dir, module, "_version.py"), 'r') as fp: + with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp: return re.match("__version__ = '(.*)'$", fp.read()).groups()[0] def get_version(setup_dir, module): @@ -33,8 +29,8 @@ def get_version(setup_dir, module): save_version(setup_dir, module, env_version) else: try: - save_version(setup_dir, module, git_latest_tag() + git_timestamp_tag()) - except subprocess.CalledProcessError: + save_version(setup_dir, module, git_version_at_commit()) + except (subprocess.CalledProcessError, OSError): pass return read_version(setup_dir, module) diff --git a/services/nodemanager/arvados_version.py b/services/nodemanager/arvados_version.py index 2e6484cabd..9aabff4292 100644 --- a/services/nodemanager/arvados_version.py +++ b/services/nodemanager/arvados_version.py @@ -7,23 +7,19 @@ import time import os import re -def git_latest_tag(): - gittags = subprocess.check_output(['git', 'tag', '-l']).split() - gittags.sort(key=lambda s: [int(u) for u in s.split(b'.')],reverse=True) - return str(next(iter(gittags)).decode('utf-8')) - -def git_timestamp_tag(): - gitinfo = subprocess.check_output( - ['git', 'log', '--first-parent', '--max-count=1', - '--format=format:%ct', '.']).strip() - return str(time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo)))) +def git_version_at_commit(): + curdir = os.path.dirname(os.path.abspath(__file__)) + myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent', + '--format=%H', curdir]).strip() + myversion = subprocess.check_output([curdir+'/../../build/version-at-commit.sh', myhash]).strip().decode() + return myversion def save_version(setup_dir, module, v): - with open(os.path.join(setup_dir, module, "_version.py"), 'w') as fp: + with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp: return fp.write("__version__ = '%s'\n" % v) def read_version(setup_dir, module): - with open(os.path.join(setup_dir, module, "_version.py"), 'r') as fp: + with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp: return re.match("__version__ = '(.*)'$", fp.read()).groups()[0] def get_version(setup_dir, module): @@ -33,8 +29,8 @@ def get_version(setup_dir, module): save_version(setup_dir, module, env_version) else: try: - save_version(setup_dir, module, git_latest_tag() + git_timestamp_tag()) - except subprocess.CalledProcessError: + save_version(setup_dir, module, git_version_at_commit()) + except (subprocess.CalledProcessError, OSError): pass return read_version(setup_dir, module) diff --git a/tools/crunchstat-summary/arvados_version.py b/tools/crunchstat-summary/arvados_version.py index 2e6484cabd..9aabff4292 100644 --- a/tools/crunchstat-summary/arvados_version.py +++ b/tools/crunchstat-summary/arvados_version.py @@ -7,23 +7,19 @@ import time import os import re -def git_latest_tag(): - gittags = subprocess.check_output(['git', 'tag', '-l']).split() - gittags.sort(key=lambda s: [int(u) for u in s.split(b'.')],reverse=True) - return str(next(iter(gittags)).decode('utf-8')) - -def git_timestamp_tag(): - gitinfo = subprocess.check_output( - ['git', 'log', '--first-parent', '--max-count=1', - '--format=format:%ct', '.']).strip() - return str(time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo)))) +def git_version_at_commit(): + curdir = os.path.dirname(os.path.abspath(__file__)) + myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent', + '--format=%H', curdir]).strip() + myversion = subprocess.check_output([curdir+'/../../build/version-at-commit.sh', myhash]).strip().decode() + return myversion def save_version(setup_dir, module, v): - with open(os.path.join(setup_dir, module, "_version.py"), 'w') as fp: + with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp: return fp.write("__version__ = '%s'\n" % v) def read_version(setup_dir, module): - with open(os.path.join(setup_dir, module, "_version.py"), 'r') as fp: + with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp: return re.match("__version__ = '(.*)'$", fp.read()).groups()[0] def get_version(setup_dir, module): @@ -33,8 +29,8 @@ def get_version(setup_dir, module): save_version(setup_dir, module, env_version) else: try: - save_version(setup_dir, module, git_latest_tag() + git_timestamp_tag()) - except subprocess.CalledProcessError: + save_version(setup_dir, module, git_version_at_commit()) + except (subprocess.CalledProcessError, OSError): pass return read_version(setup_dir, module)