Merge branch '13301-cwl-resource-scatter' closes #13301

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz@veritasgenetics.com>

diff --git a/build/run-build-test-packages-one-target.sh b/build/run-build-test-packages-one-target.sh
index 744b5834d852aaa77035cd2460d8802db4b0d466..7bdaacac52268a49b6fe929f0f467da061682662 100755 (executable)
--- a/build/run-build-test-packages-one-target.sh
+++ b/build/run-build-test-packages-one-target.sh
@@ -115,6 +115,7 @@ if [[ "$UPLOAD" != 0 ]]; then
   timer_reset
 
   if [ ${#failures[@]} -eq 0 ]; then
+    echo "/usr/local/arvados-dev/jenkins/run_upload_packages.py -H jenkinsapt@apt.arvados.org -o Port=2222 --workspace $WORKSPACE $TARGET"
     /usr/local/arvados-dev/jenkins/run_upload_packages.py -H jenkinsapt@apt.arvados.org -o Port=2222 --workspace $WORKSPACE $TARGET
   else
     echo "Skipping package upload, there were errors building and/or testing the packages"

diff --git a/build/run-library.sh b/build/run-library.sh
index ba946882ee49c2a3beb840d841732e87f28bffe3..fb4df6a79215ea3cfa86f0bd5cfc6c9233fa8233 100755 (executable)
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -58,15 +58,16 @@ version_from_git() {
     fi
 
     declare $(format_last_commit_here "git_ts=%ct git_hash=%h")
-    echo "${prefix}.$(date -ud "@$git_ts" +%Y%m%d%H%M%S).$git_hash"
-}
+    ARVADOS_BUILDING_VERSION="$(git describe --abbrev=0).$(date -ud "@$git_ts" +%Y%m%d%H%M%S)"
+    echo "$ARVADOS_BUILDING_VERSION"
+} 
 
 nohash_version_from_git() {
     if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
         echo "$ARVADOS_BUILDING_VERSION"
         return
     fi
-    version_from_git $1 | cut -d. -f1-3
+    version_from_git $1 | cut -d. -f1-4
 }
 
 timestamp_from_git() {
@@ -263,13 +264,42 @@ test_package_presence() {
     # See if we can skip building the package, only if it already exists in the
     # processed/ directory. If so, move it back to the packages directory to make
     # sure it gets picked up by the test and/or upload steps.
-    if [[ -e "processed/$complete_pkgname" ]]; then
-      echo "Package $complete_pkgname exists, not rebuilding!"
-      mv processed/$complete_pkgname .
-      return 1
+    # Get the list of packages from the repos
+
+    if [[ "$FORMAT" == "deb" ]]; then
+      debian_distros="jessie precise stretch trusty wheezy xenial"
+
+      for D in ${debian_distros}; do
+        if [ ${pkgname:0:3} = "lib" ]; then
+          repo_subdir=${pkgname:0:4}
+        else
+          repo_subdir=${pkgname:0:1}
+        fi
+
+        repo_pkg_list=$(curl -o - http://apt.arvados.org/pool/${D}/main/${repo_subdir}/)
+        echo ${repo_pkg_list} |grep -q ${complete_pkgname}
+        if [ $? -eq 0 ]; then
+          echo "Package $complete_pkgname exists, not rebuilding!"
+          curl -o ./${complete_pkgname} http://apt.arvados.org/pool/${D}/main/${repo_subdir}/${complete_pkgname}
+          return 1
+        else
+          echo "Package $complete_pkgname not found, building"
+          return 0
+        fi
+      done
     else
-      echo "Package $complete_pkgname not found, building"
-      return 0
+      centos_repo="http://rpm.arvados.org/CentOS/7/dev/x86_64/"
+
+      repo_pkg_list=$(curl -o - ${centos_repo})
+      echo ${repo_pkg_list} |grep -q ${complete_pkgname}
+      if [ $? -eq 0 ]; then
+        echo "Package $complete_pkgname exists, not rebuilding!"
+        curl -o ./${complete_pkgname} ${centos_repo}${complete_pkgname}
+        return 1
+      else
+        echo "Package $complete_pkgname not found, building"
+        return 0
+      fi
     fi
 }
 

diff --git a/sdk/cwl/arvados_version.py b/sdk/cwl/arvados_version.py
new file mode 100644 (file)
index 0000000..db46417
--- /dev/null
+++ b/sdk/cwl/arvados_version.py
@@ -0,0 +1,20 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import subprocess
+import time
+
+class VersionInfoFromGit():
+    """Return arvados version from git
+    """
+    def git_latest_tag(self):
+        gitinfo = subprocess.check_output(
+            ['git', 'describe', '--abbrev=0']).strip()
+        return str(gitinfo.decode('utf-8'))
+
+    def git_timestamp_tag(self):
+        gitinfo = subprocess.check_output(
+            ['git', 'log', '--first-parent', '--max-count=1',
+             '--format=format:%ct', '.']).strip()
+        return str(time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo))))

diff --git a/sdk/cwl/gittaggers.py b/sdk/cwl/gittaggers.py
index 3fb0fdecd8fbb646b2a8175169b944e126147a86..8ccb6645de8c78ccf77d3e049fa1b1e6257e5c91 100644 (file)
--- a/sdk/cwl/gittaggers.py
+++ b/sdk/cwl/gittaggers.py
@@ -28,6 +28,10 @@ class EggInfoFromGit(egg_info):
     If a build tag has already been set (e.g., "egg_info -b", building
     from source package), leave it alone.
     """
+    def git_latest_tag(self):
+        gitinfo = subprocess.check_output(
+            ['git', 'describe', '--abbrev=0']).strip()
+        return str(gitinfo.decode('utf-8'))
 
     def git_timestamp_tag(self):
         gitinfo = subprocess.check_output(
@@ -37,5 +41,5 @@ class EggInfoFromGit(egg_info):
 
     def tags(self):
         if self.tag_build is None:
-            self.tag_build = self.git_timestamp_tag()
+            self.tag_build = self.git_latest_tag() + self.git_timestamp_tag()
         return egg_info.tags(self)

diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 0e9ee945fbcddd304005b09e2cd5683702ba966a..d1df898cd295feace460c2c0cf090cb222775980 100644 (file)
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -16,10 +16,10 @@ README = os.path.join(SETUP_DIR, 'README.rst')
 tagger = egg_info_cmd.egg_info
 version = os.environ.get("ARVADOS_BUILDING_VERSION")
 if not version:
-    version = "1.0"
     try:
-        import gittaggers
-        tagger = gittaggers.EggInfoFromGit
+        import arvados_version
+        vtag = arvados_version.VersionInfoFromGit()
+        version = vtag.git_latest_tag() + vtag.git_timestamp_tag()
     except ImportError:
         pass
 

diff --git a/sdk/python/arvados_version.py b/sdk/python/arvados_version.py
new file mode 100644 (file)
index 0000000..db46417
--- /dev/null
+++ b/sdk/python/arvados_version.py
@@ -0,0 +1,20 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import subprocess
+import time
+
+class VersionInfoFromGit():
+    """Return arvados version from git
+    """
+    def git_latest_tag(self):
+        gitinfo = subprocess.check_output(
+            ['git', 'describe', '--abbrev=0']).strip()
+        return str(gitinfo.decode('utf-8'))
+
+    def git_timestamp_tag(self):
+        gitinfo = subprocess.check_output(
+            ['git', 'log', '--first-parent', '--max-count=1',
+             '--format=format:%ct', '.']).strip()
+        return str(time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo))))

diff --git a/sdk/python/gittaggers.py b/sdk/python/gittaggers.py
index da73634cbf0f3518e56b10894323b4cf0996cb48..ccf25c422e62085e1edf3829459f5cdb8a8710ff 100644 (file)
--- a/sdk/python/gittaggers.py
+++ b/sdk/python/gittaggers.py
@@ -12,6 +12,11 @@ class EggInfoFromGit(egg_info):
     If a build tag has already been set (e.g., "egg_info -b", building
     from source package), leave it alone.
     """
+    def git_latest_tag(self):
+        gitinfo = subprocess.check_output(
+            ['git', 'describe', '--abbrev=0']).strip()
+        return str(gitinfo.decode('utf-8'))
+
     def git_timestamp_tag(self):
         gitinfo = subprocess.check_output(
             ['git', 'log', '--first-parent', '--max-count=1',
@@ -20,5 +25,5 @@ class EggInfoFromGit(egg_info):
 
     def tags(self):
         if self.tag_build is None:
-            self.tag_build = self.git_timestamp_tag()
+            self.tag_build = self.git_latest_tag()+self.git_timestamp_tag()
         return egg_info.tags(self)

diff --git a/sdk/python/setup.py b/sdk/python/setup.py
index 88bf51e8a8971181ea0f45a50e2b44ec27a9f275..1adc19f997ef880a777e662090de9f4882aa93fe 100644 (file)
--- a/sdk/python/setup.py
+++ b/sdk/python/setup.py
@@ -15,10 +15,10 @@ README = os.path.join(SETUP_DIR, 'README.rst')
 tagger = egg_info_cmd.egg_info
 version = os.environ.get("ARVADOS_BUILDING_VERSION")
 if not version:
-    version = "0.1"
     try:
-        import gittaggers
-        tagger = gittaggers.EggInfoFromGit
+        import arvados_version
+        vtag = arvados_version.VersionInfoFromGit()
+        version = vtag.git_latest_tag() + vtag.git_timestamp_tag()
     except ImportError:
         pass
 

diff --git a/services/dockercleaner/arvados_version.py b/services/dockercleaner/arvados_version.py
new file mode 100644 (file)
index 0000000..db46417
--- /dev/null
+++ b/services/dockercleaner/arvados_version.py
@@ -0,0 +1,20 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import subprocess
+import time
+
+class VersionInfoFromGit():
+    """Return arvados version from git
+    """
+    def git_latest_tag(self):
+        gitinfo = subprocess.check_output(
+            ['git', 'describe', '--abbrev=0']).strip()
+        return str(gitinfo.decode('utf-8'))
+
+    def git_timestamp_tag(self):
+        gitinfo = subprocess.check_output(
+            ['git', 'log', '--first-parent', '--max-count=1',
+             '--format=format:%ct', '.']).strip()
+        return str(time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo))))

diff --git a/services/dockercleaner/setup.py b/services/dockercleaner/setup.py
index 6a6a96a0455eee17d5f5bb1b725b69f767805cae..b287bd00d27514af6604aafb80062602d40a9161 100644 (file)
--- a/services/dockercleaner/setup.py
+++ b/services/dockercleaner/setup.py
@@ -12,10 +12,10 @@ from setuptools import setup, find_packages
 tagger = egg_info_cmd.egg_info
 version = os.environ.get("ARVADOS_BUILDING_VERSION")
 if not version:
-    version = "0.1"
     try:
-        import gittaggers
-        tagger = gittaggers.EggInfoFromGit
+        import arvados_version
+        vtag = arvados_version.VersionInfoFromGit()
+        version = vtag.git_latest_tag() + vtag.git_timestamp_tag()
     except ImportError:
         pass
 

diff --git a/services/fuse/arvados_version.py b/services/fuse/arvados_version.py
new file mode 100644 (file)
index 0000000..db46417
--- /dev/null
+++ b/services/fuse/arvados_version.py
@@ -0,0 +1,20 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import subprocess
+import time
+
+class VersionInfoFromGit():
+    """Return arvados version from git
+    """
+    def git_latest_tag(self):
+        gitinfo = subprocess.check_output(
+            ['git', 'describe', '--abbrev=0']).strip()
+        return str(gitinfo.decode('utf-8'))
+
+    def git_timestamp_tag(self):
+        gitinfo = subprocess.check_output(
+            ['git', 'log', '--first-parent', '--max-count=1',
+             '--format=format:%ct', '.']).strip()
+        return str(time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo))))

diff --git a/services/fuse/setup.py b/services/fuse/setup.py
index 2358eb928fd6b2cccda04c7f7e08bf2657b42d00..8bc11315fa20834e3d1222044fd24155cb76fdcb 100644 (file)
--- a/services/fuse/setup.py
+++ b/services/fuse/setup.py
@@ -15,13 +15,13 @@ README = os.path.join(SETUP_DIR, 'README.rst')
 tagger = egg_info_cmd.egg_info
 version = os.environ.get("ARVADOS_BUILDING_VERSION")
 if not version:
-    version = "0.1"
     try:
-        import gittaggers
-        tagger = gittaggers.EggInfoFromGit
+        import arvados_version
+        vtag = arvados_version.VersionInfoFromGit()
+        version = vtag.git_latest_tag() + vtag.git_timestamp_tag()
     except ImportError:
         pass
-
+        
 short_tests_only = False
 if '--short-tests-only' in sys.argv:
     short_tests_only = True

diff --git a/services/nodemanager/arvados_version.py b/services/nodemanager/arvados_version.py
new file mode 100644 (file)
index 0000000..db46417
--- /dev/null
+++ b/services/nodemanager/arvados_version.py
@@ -0,0 +1,20 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import subprocess
+import time
+
+class VersionInfoFromGit():
+    """Return arvados version from git
+    """
+    def git_latest_tag(self):
+        gitinfo = subprocess.check_output(
+            ['git', 'describe', '--abbrev=0']).strip()
+        return str(gitinfo.decode('utf-8'))
+
+    def git_timestamp_tag(self):
+        gitinfo = subprocess.check_output(
+            ['git', 'log', '--first-parent', '--max-count=1',
+             '--format=format:%ct', '.']).strip()
+        return str(time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo))))

diff --git a/services/nodemanager/setup.py b/services/nodemanager/setup.py
index 454c24edbf115aa594b3e89cf810a89d2dbf7fdc..9ae1135b88cd1d2b2eb4f6260a694eb5728972f1 100644 (file)
--- a/services/nodemanager/setup.py
+++ b/services/nodemanager/setup.py
@@ -15,13 +15,14 @@ README = os.path.join(SETUP_DIR, 'README.rst')
 tagger = egg_info_cmd.egg_info
 version = os.environ.get("ARVADOS_BUILDING_VERSION")
 if not version:
-    version = "0.1"
     try:
-        import gittaggers
-        tagger = gittaggers.EggInfoFromGit
+        import arvados_version
+        vtag = arvados_version.VersionInfoFromGit()
+        version = vtag.git_latest_tag() + vtag.git_timestamp_tag()
     except ImportError:
         pass
 
+
 setup(name='arvados-node-manager',
       version=version,
       description='Arvados compute node manager',

diff --git a/tools/crunchstat-summary/arvados_version.py b/tools/crunchstat-summary/arvados_version.py
new file mode 100644 (file)
index 0000000..db46417
--- /dev/null
+++ b/tools/crunchstat-summary/arvados_version.py
@@ -0,0 +1,20 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import subprocess
+import time
+
+class VersionInfoFromGit():
+    """Return arvados version from git
+    """
+    def git_latest_tag(self):
+        gitinfo = subprocess.check_output(
+            ['git', 'describe', '--abbrev=0']).strip()
+        return str(gitinfo.decode('utf-8'))
+
+    def git_timestamp_tag(self):
+        gitinfo = subprocess.check_output(
+            ['git', 'log', '--first-parent', '--max-count=1',
+             '--format=format:%ct', '.']).strip()
+        return str(time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo))))

diff --git a/tools/crunchstat-summary/setup.py b/tools/crunchstat-summary/setup.py
index ce1467b058259ce7bb19e9e46cdae847e53bdc97..0965faea83ad4757a54630d51c6889a692d99290 100755 (executable)
--- a/tools/crunchstat-summary/setup.py
+++ b/tools/crunchstat-summary/setup.py
@@ -14,13 +14,14 @@ SETUP_DIR = os.path.dirname(__file__) or '.'
 tagger = egg_info_cmd.egg_info
 version = os.environ.get("ARVADOS_BUILDING_VERSION")
 if not version:
-    version = "0.1"
     try:
-        import gittaggers
-        tagger = gittaggers.EggInfoFromGit
+        import arvados_version
+        vtag = arvados_version.VersionInfoFromGit()
+        version = vtag.git_latest_tag() + vtag.git_timestamp_tag()
     except ImportError:
         pass
 
+
 setup(name='crunchstat_summary',
       version=version,
       description='read crunch log files and summarize resource usage',