15209: Makes python components packages ask for current PySDK version.

This make arvados-cwl-runner, arvados-nodemanager, arvados-fuse and arvados-pam
to ask for the current arvados-python-client version when building packages.
Ask pip to look for packages on the PySDK sdist directory, so that current and
not yet uploaded versions can be asked as dependency.

Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima@veritasgenetics.com>

diff --git a/build/run-build-packages-python-and-ruby.sh b/build/run-build-packages-python-and-ruby.sh
index 4c5f39a373e66cdf160ac71aaa7edf7fc47cd2e3..f9b61179cae7d21f9e0129bb6d4b00c3d9f64a32 100755 (executable)
--- a/build/run-build-packages-python-and-ruby.sh
+++ b/build/run-build-packages-python-and-ruby.sh
@@ -159,8 +159,8 @@ if [ ${#failures[@]} -ne 0 ]; then
   GEM_BUILD_FAILURES=${#failures[@]}
 fi
 
-python_wrapper arvados-pam "$WORKSPACE/sdk/pam"
 python_wrapper arvados-python-client "$WORKSPACE/sdk/python"
+python_wrapper arvados-pam "$WORKSPACE/sdk/pam"
 python_wrapper arvados-cwl-runner "$WORKSPACE/sdk/cwl"
 python_wrapper arvados_fuse "$WORKSPACE/services/fuse"
 python_wrapper arvados-node-manager "$WORKSPACE/services/nodemanager"

diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index b800d43e08a060e1231cf1bfdfee5facdca02ed9..bd55fb9518cf2e3b31f2c6c07472d9c06fd7d45f 100755 (executable)
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -323,9 +323,8 @@ package_go_binary tools/keep-rsync keep-rsync \
 package_go_binary tools/keep-exercise keep-exercise \
     "Performance testing tool for Arvados Keep"
 
-# The Python SDK
+# The Python SDK - Should be built first because it's needed by others
 fpm_build_virtualenv "arvados-python-client" "sdk/python"
-fpm_build_virtualenv "arvados-python-client" "sdk/python" "python3"
 
 # Arvados cwl runner
 fpm_build_virtualenv "arvados-cwl-runner" "sdk/cwl"
@@ -339,12 +338,15 @@ fpm_build_virtualenv "arvados-fuse" "services/fuse"
 # The node manager
 fpm_build_virtualenv "arvados-node-manager" "services/nodemanager"
 
-# The Docker image cleaner
-fpm_build_virtualenv "arvados-docker-cleaner" "services/dockercleaner" "python3"
-
 # The Arvados crunchstat-summary tool
 fpm_build_virtualenv "crunchstat-summary" "tools/crunchstat-summary"
 
+# The Python SDK - Python3 package
+fpm_build_virtualenv "arvados-python-client" "sdk/python" "python3"
+
+# The Docker image cleaner
+fpm_build_virtualenv "arvados-docker-cleaner" "services/dockercleaner" "python3"
+
 # The cwltest package, which lives out of tree
 cd "$WORKSPACE"
 if [[ -e "$WORKSPACE/cwltest" ]]; then

diff --git a/build/run-library.sh b/build/run-library.sh
index 1aa3e3cfd1147ebe15b0c041637a055b015f3c93..b0068d2112a9ee14d422bb3320126063456f50f8 100755 (executable)
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -427,7 +427,9 @@ fpm_build_virtualenv () {
     PYTHON_PKG=$PKG
   fi
 
-  if [[ -n "$ONLY_BUILD" ]] && [[ "$PYTHON_PKG" != "$ONLY_BUILD" ]] && [[ "$PKG" != "$ONLY_BUILD" ]]; then
+  # arvados-python-client sdist should always be built, to be available
+  # for other dependant packages.
+  if [[ -n "$ONLY_BUILD" ]] && [[ "arvados-python-client" != "$PKG" ]] && [[ "$PYTHON_PKG" != "$ONLY_BUILD" ]] && [[ "$PKG" != "$ONLY_BUILD" ]]; then
     return 0
   fi
 
@@ -449,6 +451,14 @@ fpm_build_virtualenv () {
 
   PACKAGE_PATH=`(cd dist; ls *tar.gz)`
 
+  if [[ "arvados-python-client" == "$PKG" ]]; then
+    PYSDK_PATH=`pwd`/dist/
+  fi
+
+  if [[ -n "$ONLY_BUILD" ]] && [[ "$PYTHON_PKG" != "$ONLY_BUILD" ]] && [[ "$PKG" != "$ONLY_BUILD" ]]; then
+    return 0
+  fi
+
   # Determine the package version from the generated sdist archive
   PYTHON_VERSION=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' *.egg-info/PKG-INFO)}
 
@@ -500,16 +510,16 @@ fpm_build_virtualenv () {
   echo "wheel version:      `build/usr/share/$python/dist/$PYTHON_PKG/bin/wheel version`"
 
   if [[ "$TARGET" != "centos7" ]] || [[ "$PYTHON_PKG" != "python-arvados-fuse" ]]; then
-    build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG $PACKAGE_PATH
+    build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -f $PYSDK_PATH $PACKAGE_PATH
   else
     # centos7 needs these special tweaks to install python-arvados-fuse
     build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG docutils
-    PYCURL_SSL_LIBRARY=nss build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG $PACKAGE_PATH
+    PYCURL_SSL_LIBRARY=nss build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -f $PYSDK_PATH $PACKAGE_PATH
   fi
 
   if [[ "$?" != "0" ]]; then
     echo "Error, unable to run"
-    echo "  build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG $PACKAGE_PATH"
+    echo "  build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -f $PYSDK_PATH $PACKAGE_PATH"
     exit 1
   fi
 

diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index ceea719efe3f3f8a539dd9c9079d67bb93b588e4..b59df35c777ee1da5d125ca8504742f7fbaff04b 100644 (file)
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -14,6 +14,11 @@ README = os.path.join(SETUP_DIR, 'README.rst')
 
 import arvados_version
 version = arvados_version.get_version(SETUP_DIR, "arvados_cwl")
+if os.environ.get('ARVADOS_BUILDING_VERSION', False):
+    pysdk_dep = "=={}".format(version)
+else:
+    # On dev releases, arvados-python-client may have a different timestamp
+    pysdk_dep = "<={}".format(version)
 
 setup(name='arvados-cwl-runner',
       version=version,
@@ -37,7 +42,7 @@ setup(name='arvados-cwl-runner',
           'schema-salad==4.2.20190417121603',
           'typing >= 3.6.4',
           'ruamel.yaml >=0.15.54, <=0.15.77',
-          'arvados-python-client>=1.3.0.20190205182514',
+          'arvados-python-client{}'.format(pysdk_dep),
           'setuptools',
           'ciso8601 >= 2.0.0',
           'networkx < 2.3'

diff --git a/sdk/pam/setup.py b/sdk/pam/setup.py
index 9bc2cef60169b674a51d6ae3cffc24662aaa78df..807c948f1b3db651476456d349289ace101f9b4c 100755 (executable)
--- a/sdk/pam/setup.py
+++ b/sdk/pam/setup.py
@@ -17,6 +17,11 @@ README = os.path.join(SETUP_DIR, 'README.rst')
 
 import arvados_version
 version = arvados_version.get_version(SETUP_DIR, "arvados_pam")
+if os.environ.get('ARVADOS_BUILDING_VERSION', False):
+    pysdk_dep = "=={}".format(version)
+else:
+    # On dev releases, arvados-python-client may have a different timestamp
+    pysdk_dep = "<={}".format(version)
 
 short_tests_only = False
 if '--short-tests-only' in sys.argv:
@@ -44,7 +49,7 @@ setup(name='arvados-pam',
           ('share/doc/arvados-pam/examples', glob.glob('examples/*')),
       ],
       install_requires=[
-          'arvados-python-client>=0.1.20150801000000',
+          'arvados-python-client=={}'.format(pysdk_dep),
       ],
       test_suite='tests',
       tests_require=['pbr<1.7.0', 'mock>=1.0', 'python-pam'],

diff --git a/services/fuse/setup.py b/services/fuse/setup.py
index cbc9cb23edf5f36f0c6a83120d8c21cc0f582241..d95fc1d8114912b0b787ebd7bb4e7bc87aecffb0 100644 (file)
--- a/services/fuse/setup.py
+++ b/services/fuse/setup.py
@@ -15,6 +15,11 @@ README = os.path.join(SETUP_DIR, 'README.rst')
 
 import arvados_version
 version = arvados_version.get_version(SETUP_DIR, "arvados_fuse")
+if os.environ.get('ARVADOS_BUILDING_VERSION', False):
+    pysdk_dep = "=={}".format(version)
+else:
+    # On dev releases, arvados-python-client may have a different timestamp
+    pysdk_dep = "<={}".format(version)
 
 short_tests_only = False
 if '--short-tests-only' in sys.argv:
@@ -38,7 +43,7 @@ setup(name='arvados_fuse',
           ('share/doc/arvados_fuse', ['agpl-3.0.txt', 'README.rst']),
       ],
       install_requires=[
-        'arvados-python-client >= 0.1.20151118035730',
+        'arvados-python-client=={}'.format(pysdk_dep),
         # llfuse 1.3.4 fails to install via pip
         'llfuse >=1.2, <1.3.4',
         'python-daemon',

diff --git a/services/nodemanager/setup.py b/services/nodemanager/setup.py
index ef05467810a79a88e1e4f27149fd30531804d318..20d9f2cc5874d5cbf4559025a4226f84cd27fbd7 100644 (file)
--- a/services/nodemanager/setup.py
+++ b/services/nodemanager/setup.py
@@ -15,6 +15,11 @@ README = os.path.join(SETUP_DIR, 'README.rst')
 
 import arvados_version
 version = arvados_version.get_version(SETUP_DIR, "arvnodeman")
+if os.environ.get('ARVADOS_BUILDING_VERSION', False):
+    pysdk_dep = "=={}".format(version)
+else:
+    # On dev releases, arvados-python-client may have a different timestamp
+    pysdk_dep = "<={}".format(version)
 
 short_tests_only = False
 if '--short-tests-only' in sys.argv:
@@ -36,7 +41,7 @@ setup(name='arvados-node-manager',
       ],
       install_requires=[
           'apache-libcloud>=2.3.1.dev1',
-          'arvados-python-client>=0.1.20170731145219',
+          'arvados-python-client=={}'.format(pysdk_dep),
           'future',
           'pykka < 2',
           'python-daemon',