From 3bc30ad82b45d729b40a4e1209e160e679f7b851 Mon Sep 17 00:00:00 2001 From: Lucas Di Pentima Date: Thu, 27 Jun 2019 11:54:21 -0300 Subject: [PATCH] 15209: Makes python components packages ask for current PySDK version. This make arvados-cwl-runner, arvados-nodemanager, arvados-fuse and arvados-pam to ask for the current arvados-python-client version when building packages. Ask pip to look for packages on the PySDK sdist directory, so that current and not yet uploaded versions can be asked as dependency. Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima --- build/run-build-packages-python-and-ruby.sh | 2 +- build/run-build-packages.sh | 12 +++++++----- build/run-library.sh | 18 ++++++++++++++---- sdk/cwl/setup.py | 7 ++++++- sdk/pam/setup.py | 7 ++++++- services/fuse/setup.py | 7 ++++++- services/nodemanager/setup.py | 7 ++++++- 7 files changed, 46 insertions(+), 14 deletions(-) diff --git a/build/run-build-packages-python-and-ruby.sh b/build/run-build-packages-python-and-ruby.sh index 4c5f39a373..f9b61179ca 100755 --- a/build/run-build-packages-python-and-ruby.sh +++ b/build/run-build-packages-python-and-ruby.sh @@ -159,8 +159,8 @@ if [ ${#failures[@]} -ne 0 ]; then GEM_BUILD_FAILURES=${#failures[@]} fi -python_wrapper arvados-pam "$WORKSPACE/sdk/pam" python_wrapper arvados-python-client "$WORKSPACE/sdk/python" +python_wrapper arvados-pam "$WORKSPACE/sdk/pam" python_wrapper arvados-cwl-runner "$WORKSPACE/sdk/cwl" python_wrapper arvados_fuse "$WORKSPACE/services/fuse" python_wrapper arvados-node-manager "$WORKSPACE/services/nodemanager" diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh index b800d43e08..bd55fb9518 100755 --- a/build/run-build-packages.sh +++ b/build/run-build-packages.sh @@ -323,9 +323,8 @@ package_go_binary tools/keep-rsync keep-rsync \ package_go_binary tools/keep-exercise keep-exercise \ "Performance testing tool for Arvados Keep" -# The Python SDK +# The Python SDK - Should be built first because it's needed by others fpm_build_virtualenv "arvados-python-client" "sdk/python" -fpm_build_virtualenv "arvados-python-client" "sdk/python" "python3" # Arvados cwl runner fpm_build_virtualenv "arvados-cwl-runner" "sdk/cwl" @@ -339,12 +338,15 @@ fpm_build_virtualenv "arvados-fuse" "services/fuse" # The node manager fpm_build_virtualenv "arvados-node-manager" "services/nodemanager" -# The Docker image cleaner -fpm_build_virtualenv "arvados-docker-cleaner" "services/dockercleaner" "python3" - # The Arvados crunchstat-summary tool fpm_build_virtualenv "crunchstat-summary" "tools/crunchstat-summary" +# The Python SDK - Python3 package +fpm_build_virtualenv "arvados-python-client" "sdk/python" "python3" + +# The Docker image cleaner +fpm_build_virtualenv "arvados-docker-cleaner" "services/dockercleaner" "python3" + # The cwltest package, which lives out of tree cd "$WORKSPACE" if [[ -e "$WORKSPACE/cwltest" ]]; then diff --git a/build/run-library.sh b/build/run-library.sh index 1aa3e3cfd1..b0068d2112 100755 --- a/build/run-library.sh +++ b/build/run-library.sh @@ -427,7 +427,9 @@ fpm_build_virtualenv () { PYTHON_PKG=$PKG fi - if [[ -n "$ONLY_BUILD" ]] && [[ "$PYTHON_PKG" != "$ONLY_BUILD" ]] && [[ "$PKG" != "$ONLY_BUILD" ]]; then + # arvados-python-client sdist should always be built, to be available + # for other dependant packages. + if [[ -n "$ONLY_BUILD" ]] && [[ "arvados-python-client" != "$PKG" ]] && [[ "$PYTHON_PKG" != "$ONLY_BUILD" ]] && [[ "$PKG" != "$ONLY_BUILD" ]]; then return 0 fi @@ -449,6 +451,14 @@ fpm_build_virtualenv () { PACKAGE_PATH=`(cd dist; ls *tar.gz)` + if [[ "arvados-python-client" == "$PKG" ]]; then + PYSDK_PATH=`pwd`/dist/ + fi + + if [[ -n "$ONLY_BUILD" ]] && [[ "$PYTHON_PKG" != "$ONLY_BUILD" ]] && [[ "$PKG" != "$ONLY_BUILD" ]]; then + return 0 + fi + # Determine the package version from the generated sdist archive PYTHON_VERSION=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' *.egg-info/PKG-INFO)} @@ -500,16 +510,16 @@ fpm_build_virtualenv () { echo "wheel version: `build/usr/share/$python/dist/$PYTHON_PKG/bin/wheel version`" if [[ "$TARGET" != "centos7" ]] || [[ "$PYTHON_PKG" != "python-arvados-fuse" ]]; then - build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG $PACKAGE_PATH + build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -f $PYSDK_PATH $PACKAGE_PATH else # centos7 needs these special tweaks to install python-arvados-fuse build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG docutils - PYCURL_SSL_LIBRARY=nss build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG $PACKAGE_PATH + PYCURL_SSL_LIBRARY=nss build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -f $PYSDK_PATH $PACKAGE_PATH fi if [[ "$?" != "0" ]]; then echo "Error, unable to run" - echo " build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG $PACKAGE_PATH" + echo " build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -f $PYSDK_PATH $PACKAGE_PATH" exit 1 fi diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py index ceea719efe..b59df35c77 100644 --- a/sdk/cwl/setup.py +++ b/sdk/cwl/setup.py @@ -14,6 +14,11 @@ README = os.path.join(SETUP_DIR, 'README.rst') import arvados_version version = arvados_version.get_version(SETUP_DIR, "arvados_cwl") +if os.environ.get('ARVADOS_BUILDING_VERSION', False): + pysdk_dep = "=={}".format(version) +else: + # On dev releases, arvados-python-client may have a different timestamp + pysdk_dep = "<={}".format(version) setup(name='arvados-cwl-runner', version=version, @@ -37,7 +42,7 @@ setup(name='arvados-cwl-runner', 'schema-salad==4.2.20190417121603', 'typing >= 3.6.4', 'ruamel.yaml >=0.15.54, <=0.15.77', - 'arvados-python-client>=1.3.0.20190205182514', + 'arvados-python-client{}'.format(pysdk_dep), 'setuptools', 'ciso8601 >= 2.0.0', 'networkx < 2.3' diff --git a/sdk/pam/setup.py b/sdk/pam/setup.py index 9bc2cef601..807c948f1b 100755 --- a/sdk/pam/setup.py +++ b/sdk/pam/setup.py @@ -17,6 +17,11 @@ README = os.path.join(SETUP_DIR, 'README.rst') import arvados_version version = arvados_version.get_version(SETUP_DIR, "arvados_pam") +if os.environ.get('ARVADOS_BUILDING_VERSION', False): + pysdk_dep = "=={}".format(version) +else: + # On dev releases, arvados-python-client may have a different timestamp + pysdk_dep = "<={}".format(version) short_tests_only = False if '--short-tests-only' in sys.argv: @@ -44,7 +49,7 @@ setup(name='arvados-pam', ('share/doc/arvados-pam/examples', glob.glob('examples/*')), ], install_requires=[ - 'arvados-python-client>=0.1.20150801000000', + 'arvados-python-client=={}'.format(pysdk_dep), ], test_suite='tests', tests_require=['pbr<1.7.0', 'mock>=1.0', 'python-pam'], diff --git a/services/fuse/setup.py b/services/fuse/setup.py index cbc9cb23ed..d95fc1d811 100644 --- a/services/fuse/setup.py +++ b/services/fuse/setup.py @@ -15,6 +15,11 @@ README = os.path.join(SETUP_DIR, 'README.rst') import arvados_version version = arvados_version.get_version(SETUP_DIR, "arvados_fuse") +if os.environ.get('ARVADOS_BUILDING_VERSION', False): + pysdk_dep = "=={}".format(version) +else: + # On dev releases, arvados-python-client may have a different timestamp + pysdk_dep = "<={}".format(version) short_tests_only = False if '--short-tests-only' in sys.argv: @@ -38,7 +43,7 @@ setup(name='arvados_fuse', ('share/doc/arvados_fuse', ['agpl-3.0.txt', 'README.rst']), ], install_requires=[ - 'arvados-python-client >= 0.1.20151118035730', + 'arvados-python-client=={}'.format(pysdk_dep), # llfuse 1.3.4 fails to install via pip 'llfuse >=1.2, <1.3.4', 'python-daemon', diff --git a/services/nodemanager/setup.py b/services/nodemanager/setup.py index ef05467810..20d9f2cc58 100644 --- a/services/nodemanager/setup.py +++ b/services/nodemanager/setup.py @@ -15,6 +15,11 @@ README = os.path.join(SETUP_DIR, 'README.rst') import arvados_version version = arvados_version.get_version(SETUP_DIR, "arvnodeman") +if os.environ.get('ARVADOS_BUILDING_VERSION', False): + pysdk_dep = "=={}".format(version) +else: + # On dev releases, arvados-python-client may have a different timestamp + pysdk_dep = "<={}".format(version) short_tests_only = False if '--short-tests-only' in sys.argv: @@ -36,7 +41,7 @@ setup(name='arvados-node-manager', ], install_requires=[ 'apache-libcloud>=2.3.1.dev1', - 'arvados-python-client>=0.1.20170731145219', + 'arvados-python-client=={}'.format(pysdk_dep), 'future', 'pykka < 2', 'python-daemon', -- 2.30.2