Merge branch 'master' into 17012-fix-python-version-calculation

refs #17012

Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward@curii.com>

diff --git a/build/run-build-docker-jobs-image.sh b/build/run-build-docker-jobs-image.sh
index d2c9f7274b3e4a006bd335229fbbbd16c905ac13..59914a2ee9dcdeb78a7de4eb9d59c7716342ff05 100755 (executable)
--- a/build/run-build-docker-jobs-image.sh
+++ b/build/run-build-docker-jobs-image.sh
@@ -157,6 +157,11 @@ fi
 # version.
 cwl_runner_version_tag=$(echo -n $cwl_runner_version | sed s/~dev/.dev/g | sed s/~rc/rc/g)
 
+if [[ -z "$cwl_runner_version_tag" ]]; then
+  echo "ERROR: cwl_runner_version_tag is empty";
+  exit 1
+fi
+
 if [[ "${cwl_runner_version}" != "${ARVADOS_BUILDING_VERSION}" ]]; then
        cwl_runner_version="${cwl_runner_version}-1"
 else

diff --git a/build/run-library.sh b/build/run-library.sh
index c598270ceba2beb4af967c9093e405b2531e57c0..1716cf3706240323ef96486398634cd6084d449a 100755 (executable)
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -61,11 +61,12 @@ version_from_git() {
 }
 
 nohash_version_from_git() {
+    local subdir="$1"; shift
     if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
         echo "$ARVADOS_BUILDING_VERSION"
         return
     fi
-    version_from_git | cut -d. -f1-4
+    version_from_git $subdir | cut -d. -f1-4
 }
 
 timestamp_from_git() {
@@ -74,15 +75,8 @@ timestamp_from_git() {
 }
 
 calculate_python_sdk_cwl_package_versions() {
-  python_sdk_ts=$(cd sdk/python && timestamp_from_git)
-  cwl_runner_ts=$(cd sdk/cwl && timestamp_from_git)
-
-  python_sdk_version=$(cd sdk/python && nohash_version_from_git)
-  cwl_runner_version=$(cd sdk/cwl && nohash_version_from_git)
-
-  if [[ $python_sdk_ts -gt $cwl_runner_ts ]]; then
-    cwl_runner_version=$python_sdk_version
-  fi
+  python_sdk_version=$(cd sdk/python && python3 arvados_version.py)
+  cwl_runner_version=$(cd sdk/cwl && python3 arvados_version.py)
 }
 
 handle_python_package () {

diff --git a/sdk/cwl/arvados_version.py b/sdk/cwl/arvados_version.py
index d5f48c066bd485262b0226e60b09a7d1d857a520..c3936617f09aa46e11a6822aa2cb868608d20c53 100644 (file)
--- a/sdk/cwl/arvados_version.py
+++ b/sdk/cwl/arvados_version.py
@@ -6,27 +6,32 @@ import subprocess
 import time
 import os
 import re
+import sys
 
 SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
+VERSION_PATHS = {
+        SETUP_DIR,
+        os.path.abspath(os.path.join(SETUP_DIR, "../python")),
+        os.path.abspath(os.path.join(SETUP_DIR, "../../build/version-at-commit.sh"))
+        }
 
 def choose_version_from():
-    sdk_ts = subprocess.check_output(
-        ['git', 'log', '--first-parent', '--max-count=1',
-         '--format=format:%ct', os.path.join(SETUP_DIR, "../python")]).strip()
-    cwl_ts = subprocess.check_output(
-        ['git', 'log', '--first-parent', '--max-count=1',
-         '--format=format:%ct', SETUP_DIR]).strip()
-    if int(sdk_ts) > int(cwl_ts):
-        getver = os.path.join(SETUP_DIR, "../python")
-    else:
-        getver = SETUP_DIR
+    ts = {}
+    for path in VERSION_PATHS:
+        ts[subprocess.check_output(
+            ['git', 'log', '--first-parent', '--max-count=1',
+             '--format=format:%ct', path]).strip()] = path
+
+    sorted_ts = sorted(ts.items())
+    getver = sorted_ts[-1][1]
+    print("Using "+getver+" for version number calculation of "+SETUP_DIR, file=sys.stderr)
     return getver
 
 def git_version_at_commit():
     curdir = choose_version_from()
     myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
                                        '--format=%H', curdir]).strip()
-    myversion = subprocess.check_output([curdir+'/../../build/version-at-commit.sh', myhash]).strip().decode()
+    myversion = subprocess.check_output([SETUP_DIR+'/../../build/version-at-commit.sh', myhash]).strip().decode()
     return myversion
 
 def save_version(setup_dir, module, v):
@@ -46,7 +51,12 @@ def get_version(setup_dir, module):
     else:
         try:
             save_version(setup_dir, module, git_version_at_commit())
-        except (subprocess.CalledProcessError, OSError):
+        except (subprocess.CalledProcessError, OSError) as err:
+            print("ERROR: {0}".format(err), file=sys.stderr)
             pass
 
     return read_version(setup_dir, module)
+
+# Called from calculate_python_sdk_cwl_package_versions() in run-library.sh
+if __name__ == '__main__':
+    print(get_version(SETUP_DIR, "arvados_cwl"))

diff --git a/sdk/cwl/gittaggers.py b/sdk/cwl/gittaggers.py
deleted file mode 100644 (file)
index d6a4c24..0000000
--- a/sdk/cwl/gittaggers.py
+++ /dev/null
@@ -1,48 +0,0 @@
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: Apache-2.0
-
-from builtins import str
-from builtins import next
-
-from setuptools.command.egg_info import egg_info
-import subprocess
-import time
-import os
-
-SETUP_DIR = os.path.dirname(__file__) or '.'
-
-def choose_version_from():
-    sdk_ts = subprocess.check_output(
-        ['git', 'log', '--first-parent', '--max-count=1',
-         '--format=format:%ct', os.path.join(SETUP_DIR, "../python")]).strip()
-    cwl_ts = subprocess.check_output(
-        ['git', 'log', '--first-parent', '--max-count=1',
-         '--format=format:%ct', SETUP_DIR]).strip()
-    if int(sdk_ts) > int(cwl_ts):
-        getver = os.path.join(SETUP_DIR, "../python")
-    else:
-        getver = SETUP_DIR
-    return getver
-
-class EggInfoFromGit(egg_info):
-    """Tag the build with git commit timestamp.
-
-    If a build tag has already been set (e.g., "egg_info -b", building
-    from source package), leave it alone.
-    """
-    def git_latest_tag(self):
-        gittags = subprocess.check_output(['git', 'tag', '-l']).split()
-        gittags.sort(key=lambda s: [int(u) for u in s.split(b'.')],reverse=True)
-        return str(next(iter(gittags)).decode('utf-8'))
-
-    def git_timestamp_tag(self):
-        gitinfo = subprocess.check_output(
-            ['git', 'log', '--first-parent', '--max-count=1',
-             '--format=format:%ct', choose_version_from()]).strip()
-        return time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo)))
-
-    def tags(self):
-        if self.tag_build is None:
-            self.tag_build = self.git_latest_tag() + self.git_timestamp_tag()
-        return egg_info.tags(self)

diff --git a/sdk/python/arvados_version.py b/sdk/python/arvados_version.py
index 36804bf5b36601e6e12b9bf1c86a979abc6dd40e..092131d930aeddf880eae21a521d59f4122b7404 100644 (file)
--- a/sdk/python/arvados_version.py
+++ b/sdk/python/arvados_version.py
@@ -6,12 +6,31 @@ import subprocess
 import time
 import os
 import re
+import sys
+
+SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
+VERSION_PATHS = {
+        SETUP_DIR,
+        os.path.abspath(os.path.join(SETUP_DIR, "../../build/version-at-commit.sh"))
+        }
+
+def choose_version_from():
+    ts = {}
+    for path in VERSION_PATHS:
+        ts[subprocess.check_output(
+            ['git', 'log', '--first-parent', '--max-count=1',
+             '--format=format:%ct', path]).strip()] = path
+
+    sorted_ts = sorted(ts.items())
+    getver = sorted_ts[-1][1]
+    print("Using "+getver+" for version number calculation of "+SETUP_DIR, file=sys.stderr)
+    return getver
 
 def git_version_at_commit():
-    curdir = os.path.dirname(os.path.abspath(__file__))
+    curdir = choose_version_from()
     myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
                                        '--format=%H', curdir]).strip()
-    myversion = subprocess.check_output([curdir+'/../../build/version-at-commit.sh', myhash]).strip().decode()
+    myversion = subprocess.check_output([SETUP_DIR+'/../../build/version-at-commit.sh', myhash]).strip().decode()
     return myversion
 
 def save_version(setup_dir, module, v):
@@ -31,7 +50,12 @@ def get_version(setup_dir, module):
     else:
         try:
             save_version(setup_dir, module, git_version_at_commit())
-        except (subprocess.CalledProcessError, OSError):
+        except (subprocess.CalledProcessError, OSError) as err:
+            print("ERROR: {0}".format(err), file=sys.stderr)
             pass
 
     return read_version(setup_dir, module)
+
+# Called from calculate_python_sdk_cwl_package_versions() in run-library.sh
+if __name__ == '__main__':
+    print(get_version(SETUP_DIR, "arvados"))

diff --git a/sdk/python/gittaggers.py b/sdk/python/gittaggers.py
deleted file mode 100644 (file)
index f3278fc..0000000
--- a/sdk/python/gittaggers.py
+++ /dev/null
@@ -1,29 +0,0 @@
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: Apache-2.0
-
-from setuptools.command.egg_info import egg_info
-import subprocess
-import time
-
-class EggInfoFromGit(egg_info):
-    """Tag the build with git commit timestamp.
-
-    If a build tag has already been set (e.g., "egg_info -b", building
-    from source package), leave it alone.
-    """
-    def git_latest_tag(self):
-        gittags = subprocess.check_output(['git', 'tag', '-l']).split()
-        gittags.sort(key=lambda s: [int(u) for u in s.split(b'.')],reverse=True)
-        return str(next(iter(gittags)).decode('utf-8'))
-
-    def git_timestamp_tag(self):
-        gitinfo = subprocess.check_output(
-            ['git', 'log', '--first-parent', '--max-count=1',
-             '--format=format:%ct', '.']).strip()
-        return time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo)))
-
-    def tags(self):
-        if self.tag_build is None:
-            self.tag_build = self.git_latest_tag()+self.git_timestamp_tag()
-        return egg_info.tags(self)

diff --git a/services/dockercleaner/arvados_version.py b/services/dockercleaner/arvados_version.py
index 36804bf5b36601e6e12b9bf1c86a979abc6dd40e..38e6f564e717d23dc217d66f59465ad584deb4b7 100644 (file)
--- a/services/dockercleaner/arvados_version.py
+++ b/services/dockercleaner/arvados_version.py
@@ -6,12 +6,31 @@ import subprocess
 import time
 import os
 import re
+import sys
+
+SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
+VERSION_PATHS = {
+        SETUP_DIR,
+        os.path.abspath(os.path.join(SETUP_DIR, "../../build/version-at-commit.sh"))
+        }
+
+def choose_version_from():
+    ts = {}
+    for path in VERSION_PATHS:
+        ts[subprocess.check_output(
+            ['git', 'log', '--first-parent', '--max-count=1',
+             '--format=format:%ct', path]).strip()] = path
+
+    sorted_ts = sorted(ts.items())
+    getver = sorted_ts[-1][1]
+    print("Using "+getver+" for version number calculation of "+SETUP_DIR, file=sys.stderr)
+    return getver
 
 def git_version_at_commit():
-    curdir = os.path.dirname(os.path.abspath(__file__))
+    curdir = choose_version_from()
     myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
                                        '--format=%H', curdir]).strip()
-    myversion = subprocess.check_output([curdir+'/../../build/version-at-commit.sh', myhash]).strip().decode()
+    myversion = subprocess.check_output([SETUP_DIR+'/../../build/version-at-commit.sh', myhash]).strip().decode()
     return myversion
 
 def save_version(setup_dir, module, v):
@@ -31,7 +50,8 @@ def get_version(setup_dir, module):
     else:
         try:
             save_version(setup_dir, module, git_version_at_commit())
-        except (subprocess.CalledProcessError, OSError):
+        except (subprocess.CalledProcessError, OSError) as err:
+            print("ERROR: {0}".format(err), file=sys.stderr)
             pass
 
     return read_version(setup_dir, module)

diff --git a/services/dockercleaner/gittaggers.py b/services/dockercleaner/gittaggers.py
deleted file mode 120000 (symlink)
index a9ad861..0000000
--- a/services/dockercleaner/gittaggers.py
+++ /dev/null
@@ -1 +0,0 @@
-../../sdk/python/gittaggers.py
\ No newline at end of file

diff --git a/services/fuse/arvados_version.py b/services/fuse/arvados_version.py
index 0d307c1be88f16aa88089e803013ba01f5a4fa04..d8eec3d9ee98bcdf1bd2ea603d237c5265c1750d 100644 (file)
--- a/services/fuse/arvados_version.py
+++ b/services/fuse/arvados_version.py
@@ -6,27 +6,32 @@ import subprocess
 import time
 import os
 import re
+import sys
 
 SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
+VERSION_PATHS = {
+        SETUP_DIR,
+        os.path.abspath(os.path.join(SETUP_DIR, "../../sdk/python")),
+        os.path.abspath(os.path.join(SETUP_DIR, "../../build/version-at-commit.sh"))
+        }
 
 def choose_version_from():
-    sdk_ts = subprocess.check_output(
-        ['git', 'log', '--first-parent', '--max-count=1',
-         '--format=format:%ct', os.path.join(SETUP_DIR, "../../sdk/python")]).strip()
-    cwl_ts = subprocess.check_output(
-        ['git', 'log', '--first-parent', '--max-count=1',
-         '--format=format:%ct', SETUP_DIR]).strip()
-    if int(sdk_ts) > int(cwl_ts):
-        getver = os.path.join(SETUP_DIR, "../../sdk/python")
-    else:
-        getver = SETUP_DIR
+    ts = {}
+    for path in VERSION_PATHS:
+        ts[subprocess.check_output(
+            ['git', 'log', '--first-parent', '--max-count=1',
+             '--format=format:%ct', path]).strip()] = path
+
+    sorted_ts = sorted(ts.items())
+    getver = sorted_ts[-1][1]
+    print("Using "+getver+" for version number calculation of "+SETUP_DIR, file=sys.stderr)
     return getver
 
 def git_version_at_commit():
     curdir = choose_version_from()
     myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
                                        '--format=%H', curdir]).strip()
-    myversion = subprocess.check_output([curdir+'/../../build/version-at-commit.sh', myhash]).strip().decode()
+    myversion = subprocess.check_output([SETUP_DIR+'/../../build/version-at-commit.sh', myhash]).strip().decode()
     return myversion
 
 def save_version(setup_dir, module, v):
@@ -46,7 +51,8 @@ def get_version(setup_dir, module):
     else:
         try:
             save_version(setup_dir, module, git_version_at_commit())
-        except (subprocess.CalledProcessError, OSError):
+        except (subprocess.CalledProcessError, OSError) as err:
+            print("ERROR: {0}".format(err), file=sys.stderr)
             pass
 
     return read_version(setup_dir, module)

diff --git a/services/fuse/gittaggers.py b/services/fuse/gittaggers.py
deleted file mode 120000 (symlink)
index a9ad861..0000000
--- a/services/fuse/gittaggers.py
+++ /dev/null
@@ -1 +0,0 @@
-../../sdk/python/gittaggers.py
\ No newline at end of file

diff --git a/tools/crunchstat-summary/arvados_version.py b/tools/crunchstat-summary/arvados_version.py
index 0d307c1be88f16aa88089e803013ba01f5a4fa04..d8eec3d9ee98bcdf1bd2ea603d237c5265c1750d 100644 (file)
--- a/tools/crunchstat-summary/arvados_version.py
+++ b/tools/crunchstat-summary/arvados_version.py
@@ -6,27 +6,32 @@ import subprocess
 import time
 import os
 import re
+import sys
 
 SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
+VERSION_PATHS = {
+        SETUP_DIR,
+        os.path.abspath(os.path.join(SETUP_DIR, "../../sdk/python")),
+        os.path.abspath(os.path.join(SETUP_DIR, "../../build/version-at-commit.sh"))
+        }
 
 def choose_version_from():
-    sdk_ts = subprocess.check_output(
-        ['git', 'log', '--first-parent', '--max-count=1',
-         '--format=format:%ct', os.path.join(SETUP_DIR, "../../sdk/python")]).strip()
-    cwl_ts = subprocess.check_output(
-        ['git', 'log', '--first-parent', '--max-count=1',
-         '--format=format:%ct', SETUP_DIR]).strip()
-    if int(sdk_ts) > int(cwl_ts):
-        getver = os.path.join(SETUP_DIR, "../../sdk/python")
-    else:
-        getver = SETUP_DIR
+    ts = {}
+    for path in VERSION_PATHS:
+        ts[subprocess.check_output(
+            ['git', 'log', '--first-parent', '--max-count=1',
+             '--format=format:%ct', path]).strip()] = path
+
+    sorted_ts = sorted(ts.items())
+    getver = sorted_ts[-1][1]
+    print("Using "+getver+" for version number calculation of "+SETUP_DIR, file=sys.stderr)
     return getver
 
 def git_version_at_commit():
     curdir = choose_version_from()
     myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
                                        '--format=%H', curdir]).strip()
-    myversion = subprocess.check_output([curdir+'/../../build/version-at-commit.sh', myhash]).strip().decode()
+    myversion = subprocess.check_output([SETUP_DIR+'/../../build/version-at-commit.sh', myhash]).strip().decode()
     return myversion
 
 def save_version(setup_dir, module, v):
@@ -46,7 +51,8 @@ def get_version(setup_dir, module):
     else:
         try:
             save_version(setup_dir, module, git_version_at_commit())
-        except (subprocess.CalledProcessError, OSError):
+        except (subprocess.CalledProcessError, OSError) as err:
+            print("ERROR: {0}".format(err), file=sys.stderr)
             pass
 
     return read_version(setup_dir, module)

diff --git a/tools/crunchstat-summary/gittaggers.py b/tools/crunchstat-summary/gittaggers.py
deleted file mode 120000 (symlink)
index a9ad861..0000000
--- a/tools/crunchstat-summary/gittaggers.py
+++ /dev/null
@@ -1 +0,0 @@
-../../sdk/python/gittaggers.py
\ No newline at end of file