Update python packages to use version_at_commit script

Update test images to use Bundler 2.0.2

no issue #

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz@curii.com>

diff --git a/build/package-test-dockerfiles/centos7/Dockerfile b/build/package-test-dockerfiles/centos7/Dockerfile
index 49d04aa7444a89f03cea6e838d2703eba5319ad3..6508c86be02013c12144be186a33c6f065872b95 100644 (file)
--- a/build/package-test-dockerfiles/centos7/Dockerfile
+++ b/build/package-test-dockerfiles/centos7/Dockerfile
@@ -16,7 +16,8 @@ RUN touch /var/lib/rpm/* && \
     gpg --import --no-tty /tmp/pkuczynski.asc && \
     curl -L https://get.rvm.io | bash -s stable && \
     /usr/local/rvm/bin/rvm install 2.3 && \
-    /usr/local/rvm/bin/rvm alias create default ruby-2.3
+    /usr/local/rvm/bin/rvm alias create default ruby-2.3 && \
+    /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.0.2
 
 # Install Bash 4.4.12  // see https://dev.arvados.org/issues/15612
 RUN cd /usr/local/src \

diff --git a/build/package-test-dockerfiles/debian10/Dockerfile b/build/package-test-dockerfiles/debian10/Dockerfile
index 3aa6fdcce18a1a6afe84398b6641103f53c3aaa0..48e4984d58181789b7d35519baadea6dc4fc265c 100644 (file)
--- a/build/package-test-dockerfiles/debian10/Dockerfile
+++ b/build/package-test-dockerfiles/debian10/Dockerfile
@@ -18,7 +18,8 @@ RUN gpg --import --no-tty /tmp/mpapis.asc && \
     gpg --import --no-tty /tmp/pkuczynski.asc && \
     curl -L https://get.rvm.io | bash -s stable && \
     /usr/local/rvm/bin/rvm install 2.5 && \
-    /usr/local/rvm/bin/rvm alias create default ruby-2.5
+    /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+    /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.0.2
 
 # udev daemon can't start in a container, so don't try.
 RUN mkdir -p /etc/udev/disabled

diff --git a/build/package-test-dockerfiles/debian9/Dockerfile b/build/package-test-dockerfiles/debian9/Dockerfile
index e759c9ce1dc1966e846f9bb6899f87b7ab52089c..470845ae975453f2fbc6c01ec15055e66088913e 100644 (file)
--- a/build/package-test-dockerfiles/debian9/Dockerfile
+++ b/build/package-test-dockerfiles/debian9/Dockerfile
@@ -18,7 +18,8 @@ RUN gpg --import --no-tty /tmp/mpapis.asc && \
     gpg --import --no-tty /tmp/pkuczynski.asc && \
     curl -L https://get.rvm.io | bash -s stable && \
     /usr/local/rvm/bin/rvm install 2.5 && \
-    /usr/local/rvm/bin/rvm alias create default ruby-2.5
+    /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+    /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.0.2
 
 # udev daemon can't start in a container, so don't try.
 RUN mkdir -p /etc/udev/disabled

diff --git a/build/package-test-dockerfiles/ubuntu1604/Dockerfile b/build/package-test-dockerfiles/ubuntu1604/Dockerfile
index 422ee43e061eb80382368e7bb5c6ce9c850aac7f..c35deebf10df6af5eba67977d9b7445d338820be 100644 (file)
--- a/build/package-test-dockerfiles/ubuntu1604/Dockerfile
+++ b/build/package-test-dockerfiles/ubuntu1604/Dockerfile
@@ -18,7 +18,8 @@ RUN gpg --import --no-tty /tmp/mpapis.asc && \
     gpg --import --no-tty /tmp/pkuczynski.asc && \
     curl -L https://get.rvm.io | bash -s stable && \
     /usr/local/rvm/bin/rvm install 2.5 && \
-    /usr/local/rvm/bin/rvm alias create default ruby-2.5
+    /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+    /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.0.2
 
 # udev daemon can't start in a container, so don't try.
 RUN mkdir -p /etc/udev/disabled

diff --git a/build/package-test-dockerfiles/ubuntu1804/Dockerfile b/build/package-test-dockerfiles/ubuntu1804/Dockerfile
index 68babe3fd58c3fd8f308b639347ca7c0b2d12ff0..60296e6be1618bcfd335c13a35ce2486eb336322 100644 (file)
--- a/build/package-test-dockerfiles/ubuntu1804/Dockerfile
+++ b/build/package-test-dockerfiles/ubuntu1804/Dockerfile
@@ -18,7 +18,8 @@ RUN gpg --import --no-tty /tmp/mpapis.asc && \
     gpg --import --no-tty /tmp/pkuczynski.asc && \
     curl -L https://get.rvm.io | bash -s stable && \
     /usr/local/rvm/bin/rvm install 2.5 && \
-    /usr/local/rvm/bin/rvm alias create default ruby-2.5
+    /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+    /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.0.2
 
 # udev daemon can't start in a container, so don't try.
 RUN mkdir -p /etc/udev/disabled

diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index 3173aa7057eed439e7827aec3f70cc384226f869..f00d0c03101210da37bd8cf06d9becfddcd21766 100755 (executable)
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -349,6 +349,8 @@ if [[ -e "$WORKSPACE/cwltest" ]]; then
        rm -rf "$WORKSPACE/cwltest"
 fi
 git clone https://github.com/common-workflow-language/cwltest.git
+# last release to support python 2.7
+(cd cwltest && git checkout 1.0.20190906212748)
 # signal to our build script that we want a cwltest executable installed in /usr/bin/
 mkdir cwltest/bin && touch cwltest/bin/cwltest
 fpm_build_virtualenv "cwltest" "cwltest"

diff --git a/build/run-library.sh b/build/run-library.sh
index 46b4dc60458c4fce6249ed4c6acf71259d7f933e..f09071cee7cec7cc2355b5f3f8e2736367524961 100755 (executable)
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -10,6 +10,8 @@
 # older packages.
 LICENSE_PACKAGE_TS=20151208015500
 
+librarydir=`dirname "$(readlink -f "$0")"`
+
 if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
     RAILS_PACKAGE_ITERATION=1
 else
@@ -57,7 +59,7 @@ version_from_git() {
 
     local git_ts git_hash
     declare $(format_last_commit_here "git_ts=%ct git_hash=%h" "$subdir")
-    ARVADOS_BUILDING_VERSION="$(git tag -l |sort -V -r |head -n1).$(date -ud "@$git_ts" +%Y%m%d%H%M%S)"
+    ARVADOS_BUILDING_VERSION="$($librarydir/version-at-commit.sh $git_hash)"
     echo "$ARVADOS_BUILDING_VERSION"
 }
 

diff --git a/sdk/cwl/arvados_version.py b/sdk/cwl/arvados_version.py
index d13dd5ec538e678268d7b79836d745ba89d46047..9aabff42929838a1f9748362a63eeed003775a64 100644 (file)
--- a/sdk/cwl/arvados_version.py
+++ b/sdk/cwl/arvados_version.py
@@ -7,38 +7,19 @@ import time
 import os
 import re
 
-SETUP_DIR = os.path.dirname(__file__) or '.'
-
-def git_latest_tag():
-    gittags = subprocess.check_output(['git', 'tag', '-l']).split()
-    gittags.sort(key=lambda s: [int(u) for u in s.split(b'.')],reverse=True)
-    return str(next(iter(gittags)).decode('utf-8'))
-
-def choose_version_from():
-    sdk_ts = subprocess.check_output(
-        ['git', 'log', '--first-parent', '--max-count=1',
-         '--format=format:%ct', os.path.join(SETUP_DIR, "../python")]).strip()
-    cwl_ts = subprocess.check_output(
-        ['git', 'log', '--first-parent', '--max-count=1',
-         '--format=format:%ct', SETUP_DIR]).strip()
-    if int(sdk_ts) > int(cwl_ts):
-        getver = os.path.join(SETUP_DIR, "../python")
-    else:
-        getver = SETUP_DIR
-    return getver
-
-def git_timestamp_tag():
-    gitinfo = subprocess.check_output(
-        ['git', 'log', '--first-parent', '--max-count=1',
-         '--format=format:%ct', choose_version_from()]).strip()
-    return str(time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo))))
+def git_version_at_commit():
+    curdir = os.path.dirname(os.path.abspath(__file__))
+    myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
+                                       '--format=%H', curdir]).strip()
+    myversion = subprocess.check_output([curdir+'/../../build/version-at-commit.sh', myhash]).strip().decode()
+    return myversion
 
 def save_version(setup_dir, module, v):
-  with open(os.path.join(setup_dir, module, "_version.py"), 'w') as fp:
+  with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp:
       return fp.write("__version__ = '%s'\n" % v)
 
 def read_version(setup_dir, module):
-  with open(os.path.join(setup_dir, module, "_version.py"), 'r') as fp:
+  with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp:
       return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
 
 def get_version(setup_dir, module):
@@ -48,7 +29,7 @@ def get_version(setup_dir, module):
         save_version(setup_dir, module, env_version)
     else:
         try:
-            save_version(setup_dir, module, git_latest_tag() + git_timestamp_tag())
+            save_version(setup_dir, module, git_version_at_commit())
         except (subprocess.CalledProcessError, OSError):
             pass
 

diff --git a/sdk/pam/arvados_version.py b/sdk/pam/arvados_version.py
index 2e6484cabdf1e71d39f5fe21139b29c2ce09ad93..9aabff42929838a1f9748362a63eeed003775a64 100644 (file)
--- a/sdk/pam/arvados_version.py
+++ b/sdk/pam/arvados_version.py
@@ -7,23 +7,19 @@ import time
 import os
 import re
 
-def git_latest_tag():
-    gittags = subprocess.check_output(['git', 'tag', '-l']).split()
-    gittags.sort(key=lambda s: [int(u) for u in s.split(b'.')],reverse=True)
-    return str(next(iter(gittags)).decode('utf-8'))
-
-def git_timestamp_tag():
-    gitinfo = subprocess.check_output(
-        ['git', 'log', '--first-parent', '--max-count=1',
-         '--format=format:%ct', '.']).strip()
-    return str(time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo))))
+def git_version_at_commit():
+    curdir = os.path.dirname(os.path.abspath(__file__))
+    myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
+                                       '--format=%H', curdir]).strip()
+    myversion = subprocess.check_output([curdir+'/../../build/version-at-commit.sh', myhash]).strip().decode()
+    return myversion
 
 def save_version(setup_dir, module, v):
-  with open(os.path.join(setup_dir, module, "_version.py"), 'w') as fp:
+  with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp:
       return fp.write("__version__ = '%s'\n" % v)
 
 def read_version(setup_dir, module):
-  with open(os.path.join(setup_dir, module, "_version.py"), 'r') as fp:
+  with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp:
       return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
 
 def get_version(setup_dir, module):
@@ -33,8 +29,8 @@ def get_version(setup_dir, module):
         save_version(setup_dir, module, env_version)
     else:
         try:
-            save_version(setup_dir, module, git_latest_tag() + git_timestamp_tag())
-        except subprocess.CalledProcessError:
+            save_version(setup_dir, module, git_version_at_commit())
+        except (subprocess.CalledProcessError, OSError):
             pass
 
     return read_version(setup_dir, module)

diff --git a/sdk/python/arvados_version.py b/sdk/python/arvados_version.py
index 692146465216448b7cb3f95fa8ce20bf8beff83c..9aabff42929838a1f9748362a63eeed003775a64 100644 (file)
--- a/sdk/python/arvados_version.py
+++ b/sdk/python/arvados_version.py
@@ -7,23 +7,19 @@ import time
 import os
 import re
 
-def git_latest_tag():
-    gittags = subprocess.check_output(['git', 'tag', '-l']).split()
-    gittags.sort(key=lambda s: [int(u) for u in s.split(b'.')],reverse=True)
-    return str(next(iter(gittags)).decode('utf-8'))
-
-def git_timestamp_tag():
-    gitinfo = subprocess.check_output(
-        ['git', 'log', '--first-parent', '--max-count=1',
-         '--format=format:%ct', '.']).strip()
-    return str(time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo))))
+def git_version_at_commit():
+    curdir = os.path.dirname(os.path.abspath(__file__))
+    myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
+                                       '--format=%H', curdir]).strip()
+    myversion = subprocess.check_output([curdir+'/../../build/version-at-commit.sh', myhash]).strip().decode()
+    return myversion
 
 def save_version(setup_dir, module, v):
-  with open(os.path.join(setup_dir, module, "_version.py"), 'w') as fp:
+  with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp:
       return fp.write("__version__ = '%s'\n" % v)
 
 def read_version(setup_dir, module):
-  with open(os.path.join(setup_dir, module, "_version.py"), 'r') as fp:
+  with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp:
       return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
 
 def get_version(setup_dir, module):
@@ -33,7 +29,7 @@ def get_version(setup_dir, module):
         save_version(setup_dir, module, env_version)
     else:
         try:
-            save_version(setup_dir, module, git_latest_tag() + git_timestamp_tag())
+            save_version(setup_dir, module, git_version_at_commit())
         except (subprocess.CalledProcessError, OSError):
             pass
 

diff --git a/services/dockercleaner/arvados_version.py b/services/dockercleaner/arvados_version.py
index 2e6484cabdf1e71d39f5fe21139b29c2ce09ad93..9aabff42929838a1f9748362a63eeed003775a64 100644 (file)
--- a/services/dockercleaner/arvados_version.py
+++ b/services/dockercleaner/arvados_version.py
@@ -7,23 +7,19 @@ import time
 import os
 import re
 
-def git_latest_tag():
-    gittags = subprocess.check_output(['git', 'tag', '-l']).split()
-    gittags.sort(key=lambda s: [int(u) for u in s.split(b'.')],reverse=True)
-    return str(next(iter(gittags)).decode('utf-8'))
-
-def git_timestamp_tag():
-    gitinfo = subprocess.check_output(
-        ['git', 'log', '--first-parent', '--max-count=1',
-         '--format=format:%ct', '.']).strip()
-    return str(time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo))))
+def git_version_at_commit():
+    curdir = os.path.dirname(os.path.abspath(__file__))
+    myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
+                                       '--format=%H', curdir]).strip()
+    myversion = subprocess.check_output([curdir+'/../../build/version-at-commit.sh', myhash]).strip().decode()
+    return myversion
 
 def save_version(setup_dir, module, v):
-  with open(os.path.join(setup_dir, module, "_version.py"), 'w') as fp:
+  with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp:
       return fp.write("__version__ = '%s'\n" % v)
 
 def read_version(setup_dir, module):
-  with open(os.path.join(setup_dir, module, "_version.py"), 'r') as fp:
+  with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp:
       return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
 
 def get_version(setup_dir, module):
@@ -33,8 +29,8 @@ def get_version(setup_dir, module):
         save_version(setup_dir, module, env_version)
     else:
         try:
-            save_version(setup_dir, module, git_latest_tag() + git_timestamp_tag())
-        except subprocess.CalledProcessError:
+            save_version(setup_dir, module, git_version_at_commit())
+        except (subprocess.CalledProcessError, OSError):
             pass
 
     return read_version(setup_dir, module)

diff --git a/services/fuse/arvados_version.py b/services/fuse/arvados_version.py
index 2e6484cabdf1e71d39f5fe21139b29c2ce09ad93..9aabff42929838a1f9748362a63eeed003775a64 100644 (file)
--- a/services/fuse/arvados_version.py
+++ b/services/fuse/arvados_version.py
@@ -7,23 +7,19 @@ import time
 import os
 import re
 
-def git_latest_tag():
-    gittags = subprocess.check_output(['git', 'tag', '-l']).split()
-    gittags.sort(key=lambda s: [int(u) for u in s.split(b'.')],reverse=True)
-    return str(next(iter(gittags)).decode('utf-8'))
-
-def git_timestamp_tag():
-    gitinfo = subprocess.check_output(
-        ['git', 'log', '--first-parent', '--max-count=1',
-         '--format=format:%ct', '.']).strip()
-    return str(time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo))))
+def git_version_at_commit():
+    curdir = os.path.dirname(os.path.abspath(__file__))
+    myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
+                                       '--format=%H', curdir]).strip()
+    myversion = subprocess.check_output([curdir+'/../../build/version-at-commit.sh', myhash]).strip().decode()
+    return myversion
 
 def save_version(setup_dir, module, v):
-  with open(os.path.join(setup_dir, module, "_version.py"), 'w') as fp:
+  with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp:
       return fp.write("__version__ = '%s'\n" % v)
 
 def read_version(setup_dir, module):
-  with open(os.path.join(setup_dir, module, "_version.py"), 'r') as fp:
+  with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp:
       return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
 
 def get_version(setup_dir, module):
@@ -33,8 +29,8 @@ def get_version(setup_dir, module):
         save_version(setup_dir, module, env_version)
     else:
         try:
-            save_version(setup_dir, module, git_latest_tag() + git_timestamp_tag())
-        except subprocess.CalledProcessError:
+            save_version(setup_dir, module, git_version_at_commit())
+        except (subprocess.CalledProcessError, OSError):
             pass
 
     return read_version(setup_dir, module)

diff --git a/services/nodemanager/arvados_version.py b/services/nodemanager/arvados_version.py
index 2e6484cabdf1e71d39f5fe21139b29c2ce09ad93..9aabff42929838a1f9748362a63eeed003775a64 100644 (file)
--- a/services/nodemanager/arvados_version.py
+++ b/services/nodemanager/arvados_version.py
@@ -7,23 +7,19 @@ import time
 import os
 import re
 
-def git_latest_tag():
-    gittags = subprocess.check_output(['git', 'tag', '-l']).split()
-    gittags.sort(key=lambda s: [int(u) for u in s.split(b'.')],reverse=True)
-    return str(next(iter(gittags)).decode('utf-8'))
-
-def git_timestamp_tag():
-    gitinfo = subprocess.check_output(
-        ['git', 'log', '--first-parent', '--max-count=1',
-         '--format=format:%ct', '.']).strip()
-    return str(time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo))))
+def git_version_at_commit():
+    curdir = os.path.dirname(os.path.abspath(__file__))
+    myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
+                                       '--format=%H', curdir]).strip()
+    myversion = subprocess.check_output([curdir+'/../../build/version-at-commit.sh', myhash]).strip().decode()
+    return myversion
 
 def save_version(setup_dir, module, v):
-  with open(os.path.join(setup_dir, module, "_version.py"), 'w') as fp:
+  with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp:
       return fp.write("__version__ = '%s'\n" % v)
 
 def read_version(setup_dir, module):
-  with open(os.path.join(setup_dir, module, "_version.py"), 'r') as fp:
+  with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp:
       return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
 
 def get_version(setup_dir, module):
@@ -33,8 +29,8 @@ def get_version(setup_dir, module):
         save_version(setup_dir, module, env_version)
     else:
         try:
-            save_version(setup_dir, module, git_latest_tag() + git_timestamp_tag())
-        except subprocess.CalledProcessError:
+            save_version(setup_dir, module, git_version_at_commit())
+        except (subprocess.CalledProcessError, OSError):
             pass
 
     return read_version(setup_dir, module)

diff --git a/tools/crunchstat-summary/arvados_version.py b/tools/crunchstat-summary/arvados_version.py
index 2e6484cabdf1e71d39f5fe21139b29c2ce09ad93..9aabff42929838a1f9748362a63eeed003775a64 100644 (file)
--- a/tools/crunchstat-summary/arvados_version.py
+++ b/tools/crunchstat-summary/arvados_version.py
@@ -7,23 +7,19 @@ import time
 import os
 import re
 
-def git_latest_tag():
-    gittags = subprocess.check_output(['git', 'tag', '-l']).split()
-    gittags.sort(key=lambda s: [int(u) for u in s.split(b'.')],reverse=True)
-    return str(next(iter(gittags)).decode('utf-8'))
-
-def git_timestamp_tag():
-    gitinfo = subprocess.check_output(
-        ['git', 'log', '--first-parent', '--max-count=1',
-         '--format=format:%ct', '.']).strip()
-    return str(time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo))))
+def git_version_at_commit():
+    curdir = os.path.dirname(os.path.abspath(__file__))
+    myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
+                                       '--format=%H', curdir]).strip()
+    myversion = subprocess.check_output([curdir+'/../../build/version-at-commit.sh', myhash]).strip().decode()
+    return myversion
 
 def save_version(setup_dir, module, v):
-  with open(os.path.join(setup_dir, module, "_version.py"), 'w') as fp:
+  with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp:
       return fp.write("__version__ = '%s'\n" % v)
 
 def read_version(setup_dir, module):
-  with open(os.path.join(setup_dir, module, "_version.py"), 'r') as fp:
+  with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp:
       return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
 
 def get_version(setup_dir, module):
@@ -33,8 +29,8 @@ def get_version(setup_dir, module):
         save_version(setup_dir, module, env_version)
     else:
         try:
-            save_version(setup_dir, module, git_latest_tag() + git_timestamp_tag())
-        except subprocess.CalledProcessError:
+            save_version(setup_dir, module, git_version_at_commit())
+        except (subprocess.CalledProcessError, OSError):
             pass
 
     return read_version(setup_dir, module)