21601: Specify Python interdependencies with ~=

This has the same rationale as using <= before, but it's stricter. It
should prevent pip from using release versions to satisfy development
dependencies in the future, and help root out bugs in our build
processes.

DRY up this logic in arvados_version.py.

Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith@curii.com>

diff --git a/sdk/cwl/arvados_version.py b/sdk/cwl/arvados_version.py
index b594f88a7c87b6c5a021131602a088e80833a4a4..794b6afe4261cba9c6bfc4c5dd3fee9d6bb6c19b 100644 (file)
--- a/sdk/cwl/arvados_version.py
+++ b/sdk/cwl/arvados_version.py
@@ -12,6 +12,7 @@
 #    it reads _version.py and generates dependencies from it.
 
 import os
+import re
 import runpy
 import subprocess
 import sys
@@ -123,6 +124,22 @@ def get_version(setup_dir=SETUP_DIR, module=MODULE_NAME):
     save_version(setup_dir, module, version)
     return version
 
+def iter_dependencies(version=None):
+    if version is None:
+        version = get_version()
+    # A packaged development release should be installed with other
+    # development packages built from the same source, but those
+    # dependencies may have earlier "dev" versions (read: less recent
+    # Git commit timestamps). This compatible version dependency
+    # expresses that as closely as possible. Allowing versions
+    # compatible with .dev0 allows any development release.
+    # Regular expression borrowed partially from
+    # <https://packaging.python.org/en/latest/specifications/version-specifiers/#version-specifiers-regex>
+    dep_ver, match_count = re.subn(r'\.dev(0|[1-9][0-9]*)$', '.dev0', version, 1)
+    dep_op = '~=' if match_count else '=='
+    for dep_pkg in PACKAGE_DEPENDENCY_MAP.get(PACKAGE_NAME, ()):
+        yield f'{dep_pkg}{dep_op}{dep_ver}'
+
 # Called from calculate_python_sdk_cwl_package_versions() in run-library.sh
 if __name__ == '__main__':
     print(get_version())

diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 043b52cb814067f573423044a88d34b823f72d20..551bd964b1dd152b1c26f073a97c42eaf50d614c 100644 (file)
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -9,16 +9,9 @@ import sys
 
 from setuptools import setup, find_packages
 
-SETUP_DIR = os.path.dirname(__file__) or '.'
-README = os.path.join(SETUP_DIR, 'README.rst')
-
 import arvados_version
-version = arvados_version.get_version(SETUP_DIR, "arvados_cwl")
-if os.environ.get('ARVADOS_BUILDING_VERSION', False):
-    pysdk_dep = "=={}".format(version)
-else:
-    # On dev releases, arvados-python-client may have a different timestamp
-    pysdk_dep = "<={}".format(version)
+version = arvados_version.get_version()
+README = os.path.join(arvados_version.SETUP_DIR, 'README.rst')
 
 setup(name='arvados-cwl-runner',
       version=version,
@@ -36,10 +29,9 @@ setup(name='arvados-cwl-runner',
       # file to determine what version of cwltool and schema-salad to
       # build.
       install_requires=[
+          *arvados_version.iter_dependencies(version),
           'cwltool==3.1.20230601100705',
           'schema-salad==8.4.20230601112322',
-          'arvados-python-client{}'.format(pysdk_dep),
-          'crunchstat-summary{}'.format(pysdk_dep),
           'ciso8601 >= 2.0.0',
           'networkx < 2.6',
           'msgpack==1.0.3',

diff --git a/sdk/python/arvados_version.py b/sdk/python/arvados_version.py
index b594f88a7c87b6c5a021131602a088e80833a4a4..794b6afe4261cba9c6bfc4c5dd3fee9d6bb6c19b 100644 (file)
--- a/sdk/python/arvados_version.py
+++ b/sdk/python/arvados_version.py
@@ -12,6 +12,7 @@
 #    it reads _version.py and generates dependencies from it.
 
 import os
+import re
 import runpy
 import subprocess
 import sys
@@ -123,6 +124,22 @@ def get_version(setup_dir=SETUP_DIR, module=MODULE_NAME):
     save_version(setup_dir, module, version)
     return version
 
+def iter_dependencies(version=None):
+    if version is None:
+        version = get_version()
+    # A packaged development release should be installed with other
+    # development packages built from the same source, but those
+    # dependencies may have earlier "dev" versions (read: less recent
+    # Git commit timestamps). This compatible version dependency
+    # expresses that as closely as possible. Allowing versions
+    # compatible with .dev0 allows any development release.
+    # Regular expression borrowed partially from
+    # <https://packaging.python.org/en/latest/specifications/version-specifiers/#version-specifiers-regex>
+    dep_ver, match_count = re.subn(r'\.dev(0|[1-9][0-9]*)$', '.dev0', version, 1)
+    dep_op = '~=' if match_count else '=='
+    for dep_pkg in PACKAGE_DEPENDENCY_MAP.get(PACKAGE_NAME, ()):
+        yield f'{dep_pkg}{dep_op}{dep_ver}'
+
 # Called from calculate_python_sdk_cwl_package_versions() in run-library.sh
 if __name__ == '__main__':
     print(get_version())

diff --git a/sdk/python/setup.py b/sdk/python/setup.py
index 00429de844d3aeb384336166cdc3ac3cfe8da1e9..e13e51609a56d6fcf811716e062d346ee9ceac8c 100644 (file)
--- a/sdk/python/setup.py
+++ b/sdk/python/setup.py
@@ -12,12 +12,10 @@ from pathlib import Path
 from setuptools import setup, find_packages
 from setuptools.command import build_py
 
-SETUP_DIR = os.path.dirname(__file__) or '.'
-README = os.path.join(SETUP_DIR, 'README.rst')
-
 import arvados_version
-version = arvados_version.get_version(SETUP_DIR, "arvados")
+version = arvados_version.get_version()
 short_tests_only = arvados_version.short_tests_only()
+README = os.path.join(arvados_version.SETUP_DIR, 'README.rst')
 
 class BuildPython(build_py.build_py):
     """Extend setuptools `build_py` to generate API documentation
@@ -111,6 +109,7 @@ setup(name='arvados-python-client',
           ('share/doc/arvados-python-client', ['LICENSE-2.0.txt', 'README.rst']),
       ],
       install_requires=[
+          *arvados_version.iter_dependencies(version),
           'ciso8601 >=2.0.0',
           'future',
           'google-api-core <2.11.0', # 2.11.0rc1 is incompatible with google-auth<2

diff --git a/services/dockercleaner/arvados_version.py b/services/dockercleaner/arvados_version.py
index b594f88a7c87b6c5a021131602a088e80833a4a4..794b6afe4261cba9c6bfc4c5dd3fee9d6bb6c19b 100644 (file)
--- a/services/dockercleaner/arvados_version.py
+++ b/services/dockercleaner/arvados_version.py
@@ -12,6 +12,7 @@
 #    it reads _version.py and generates dependencies from it.
 
 import os
+import re
 import runpy
 import subprocess
 import sys
@@ -123,6 +124,22 @@ def get_version(setup_dir=SETUP_DIR, module=MODULE_NAME):
     save_version(setup_dir, module, version)
     return version
 
+def iter_dependencies(version=None):
+    if version is None:
+        version = get_version()
+    # A packaged development release should be installed with other
+    # development packages built from the same source, but those
+    # dependencies may have earlier "dev" versions (read: less recent
+    # Git commit timestamps). This compatible version dependency
+    # expresses that as closely as possible. Allowing versions
+    # compatible with .dev0 allows any development release.
+    # Regular expression borrowed partially from
+    # <https://packaging.python.org/en/latest/specifications/version-specifiers/#version-specifiers-regex>
+    dep_ver, match_count = re.subn(r'\.dev(0|[1-9][0-9]*)$', '.dev0', version, 1)
+    dep_op = '~=' if match_count else '=='
+    for dep_pkg in PACKAGE_DEPENDENCY_MAP.get(PACKAGE_NAME, ()):
+        yield f'{dep_pkg}{dep_op}{dep_ver}'
+
 # Called from calculate_python_sdk_cwl_package_versions() in run-library.sh
 if __name__ == '__main__':
     print(get_version())

diff --git a/services/dockercleaner/setup.py b/services/dockercleaner/setup.py
index 565fdcd1ea3c582057bf70d9fdf8633c3c45dd94..9c69879b45b581a7c5ab49f64ef0045a8f7177e6 100644 (file)
--- a/services/dockercleaner/setup.py
+++ b/services/dockercleaner/setup.py
@@ -10,12 +10,10 @@ import re
 
 from setuptools import setup, find_packages
 
-SETUP_DIR = os.path.dirname(__file__) or '.'
-README = os.path.join(SETUP_DIR, 'README.rst')
-
 import arvados_version
-version = arvados_version.get_version(SETUP_DIR, "arvados_docker")
+version = arvados_version.get_version()
 short_tests_only = arvados_version.short_tests_only()
+README = os.path.join(arvados_version.SETUP_DIR, 'README.rst')
 
 setup(name="arvados-docker-cleaner",
       version=version,
@@ -33,6 +31,7 @@ setup(name="arvados-docker-cleaner",
           ('share/doc/arvados-docker-cleaner', ['agpl-3.0.txt', 'arvados-docker-cleaner.service']),
       ],
       install_requires=[
+          *arvados_version.iter_dependencies(version),
           'docker>=6.1.0',
           'setuptools',
       ],

diff --git a/services/fuse/arvados_version.py b/services/fuse/arvados_version.py
index b594f88a7c87b6c5a021131602a088e80833a4a4..794b6afe4261cba9c6bfc4c5dd3fee9d6bb6c19b 100644 (file)
--- a/services/fuse/arvados_version.py
+++ b/services/fuse/arvados_version.py
@@ -12,6 +12,7 @@
 #    it reads _version.py and generates dependencies from it.
 
 import os
+import re
 import runpy
 import subprocess
 import sys
@@ -123,6 +124,22 @@ def get_version(setup_dir=SETUP_DIR, module=MODULE_NAME):
     save_version(setup_dir, module, version)
     return version
 
+def iter_dependencies(version=None):
+    if version is None:
+        version = get_version()
+    # A packaged development release should be installed with other
+    # development packages built from the same source, but those
+    # dependencies may have earlier "dev" versions (read: less recent
+    # Git commit timestamps). This compatible version dependency
+    # expresses that as closely as possible. Allowing versions
+    # compatible with .dev0 allows any development release.
+    # Regular expression borrowed partially from
+    # <https://packaging.python.org/en/latest/specifications/version-specifiers/#version-specifiers-regex>
+    dep_ver, match_count = re.subn(r'\.dev(0|[1-9][0-9]*)$', '.dev0', version, 1)
+    dep_op = '~=' if match_count else '=='
+    for dep_pkg in PACKAGE_DEPENDENCY_MAP.get(PACKAGE_NAME, ()):
+        yield f'{dep_pkg}{dep_op}{dep_ver}'
+
 # Called from calculate_python_sdk_cwl_package_versions() in run-library.sh
 if __name__ == '__main__':
     print(get_version())

diff --git a/services/fuse/setup.py b/services/fuse/setup.py
index 80a26980f178d763820cf2bbe1a8d8c72de95a94..07c9dcf5e1a945fce0a034f322b0374c6500ae03 100644 (file)
--- a/services/fuse/setup.py
+++ b/services/fuse/setup.py
@@ -10,17 +10,10 @@ import re
 
 from setuptools import setup, find_packages
 
-SETUP_DIR = os.path.dirname(__file__) or '.'
-README = os.path.join(SETUP_DIR, 'README.rst')
-
 import arvados_version
-version = arvados_version.get_version(SETUP_DIR, "arvados_fuse")
-if os.environ.get('ARVADOS_BUILDING_VERSION', False):
-    pysdk_dep = "=={}".format(version)
-else:
-    # On dev releases, arvados-python-client may have a different timestamp
-    pysdk_dep = "<={}".format(version)
+version = arvados_version.get_version()
 short_tests_only = arvados_version.short_tests_only()
+README = os.path.join(arvados_version.SETUP_DIR, 'README.rst')
 
 setup(name='arvados_fuse',
       version=version,
@@ -39,7 +32,7 @@ setup(name='arvados_fuse',
           ('share/doc/arvados_fuse', ['agpl-3.0.txt', 'README.rst']),
       ],
       install_requires=[
-        'arvados-python-client{}'.format(pysdk_dep),
+        *arvados_version.iter_dependencies(version),
         'llfuse >= 1.3.6',
         'future',
         'python-daemon',

diff --git a/tools/crunchstat-summary/arvados_version.py b/tools/crunchstat-summary/arvados_version.py
index b594f88a7c87b6c5a021131602a088e80833a4a4..794b6afe4261cba9c6bfc4c5dd3fee9d6bb6c19b 100644 (file)
--- a/tools/crunchstat-summary/arvados_version.py
+++ b/tools/crunchstat-summary/arvados_version.py
@@ -12,6 +12,7 @@
 #    it reads _version.py and generates dependencies from it.
 
 import os
+import re
 import runpy
 import subprocess
 import sys
@@ -123,6 +124,22 @@ def get_version(setup_dir=SETUP_DIR, module=MODULE_NAME):
     save_version(setup_dir, module, version)
     return version
 
+def iter_dependencies(version=None):
+    if version is None:
+        version = get_version()
+    # A packaged development release should be installed with other
+    # development packages built from the same source, but those
+    # dependencies may have earlier "dev" versions (read: less recent
+    # Git commit timestamps). This compatible version dependency
+    # expresses that as closely as possible. Allowing versions
+    # compatible with .dev0 allows any development release.
+    # Regular expression borrowed partially from
+    # <https://packaging.python.org/en/latest/specifications/version-specifiers/#version-specifiers-regex>
+    dep_ver, match_count = re.subn(r'\.dev(0|[1-9][0-9]*)$', '.dev0', version, 1)
+    dep_op = '~=' if match_count else '=='
+    for dep_pkg in PACKAGE_DEPENDENCY_MAP.get(PACKAGE_NAME, ()):
+        yield f'{dep_pkg}{dep_op}{dep_ver}'
+
 # Called from calculate_python_sdk_cwl_package_versions() in run-library.sh
 if __name__ == '__main__':
     print(get_version())

diff --git a/tools/crunchstat-summary/setup.py b/tools/crunchstat-summary/setup.py
index 4bb9df2ba940b6a9c9e1fe445a55b481dad61aa5..24a6bf5e4f9155ddf446738b7f4a157c62273213 100755 (executable)
--- a/tools/crunchstat-summary/setup.py
+++ b/tools/crunchstat-summary/setup.py
@@ -10,17 +10,10 @@ import re
 
 from setuptools import setup, find_packages
 
-SETUP_DIR = os.path.dirname(__file__) or '.'
-README = os.path.join(SETUP_DIR, 'README.rst')
-
 import arvados_version
-version = arvados_version.get_version(SETUP_DIR, "crunchstat_summary")
-if os.environ.get('ARVADOS_BUILDING_VERSION', False):
-    pysdk_dep = "=={}".format(version)
-else:
-    # On dev releases, arvados-python-client may have a different timestamp
-    pysdk_dep = "<={}".format(version)
+version = arvados_version.get_version()
 short_tests_only = arvados_version.short_tests_only()
+README = os.path.join(arvados_version.SETUP_DIR, 'README.rst')
 
 setup(name='crunchstat_summary',
       version=version,
@@ -39,7 +32,7 @@ setup(name='crunchstat_summary',
           ('share/doc/crunchstat_summary', ['agpl-3.0.txt']),
       ],
       install_requires=[
-          'arvados-python-client{}'.format(pysdk_dep),
+          *arvados_version.iter_dependencies(version),
       ],
       python_requires="~=3.8",
       test_suite='tests',

diff --git a/tools/user-activity/arvados_version.py b/tools/user-activity/arvados_version.py
index b594f88a7c87b6c5a021131602a088e80833a4a4..794b6afe4261cba9c6bfc4c5dd3fee9d6bb6c19b 100644 (file)
--- a/tools/user-activity/arvados_version.py
+++ b/tools/user-activity/arvados_version.py
@@ -12,6 +12,7 @@
 #    it reads _version.py and generates dependencies from it.
 
 import os
+import re
 import runpy
 import subprocess
 import sys
@@ -123,6 +124,22 @@ def get_version(setup_dir=SETUP_DIR, module=MODULE_NAME):
     save_version(setup_dir, module, version)
     return version
 
+def iter_dependencies(version=None):
+    if version is None:
+        version = get_version()
+    # A packaged development release should be installed with other
+    # development packages built from the same source, but those
+    # dependencies may have earlier "dev" versions (read: less recent
+    # Git commit timestamps). This compatible version dependency
+    # expresses that as closely as possible. Allowing versions
+    # compatible with .dev0 allows any development release.
+    # Regular expression borrowed partially from
+    # <https://packaging.python.org/en/latest/specifications/version-specifiers/#version-specifiers-regex>
+    dep_ver, match_count = re.subn(r'\.dev(0|[1-9][0-9]*)$', '.dev0', version, 1)
+    dep_op = '~=' if match_count else '=='
+    for dep_pkg in PACKAGE_DEPENDENCY_MAP.get(PACKAGE_NAME, ()):
+        yield f'{dep_pkg}{dep_op}{dep_ver}'
+
 # Called from calculate_python_sdk_cwl_package_versions() in run-library.sh
 if __name__ == '__main__':
     print(get_version())

diff --git a/tools/user-activity/setup.py b/tools/user-activity/setup.py
index 4b7ec16b934881540e45081e77f9c67ba01519c5..8611fa47a131fc26d90408413353dcc5bb16db93 100755 (executable)
--- a/tools/user-activity/setup.py
+++ b/tools/user-activity/setup.py
@@ -10,11 +10,9 @@ import re
 
 from setuptools import setup, find_packages
 
-SETUP_DIR = os.path.dirname(__file__) or '.'
-README = os.path.join(SETUP_DIR, 'README.rst')
-
 import arvados_version
-version = arvados_version.get_version(SETUP_DIR, "arvados_user_activity")
+version = arvados_version.get_version()
+README = os.path.join(arvados_version.SETUP_DIR, 'README.rst')
 
 setup(name='arvados-user-activity',
       version=version,
@@ -31,7 +29,7 @@ setup(name='arvados-user-activity',
           ('share/doc/arvados_user_activity', ['agpl-3.0.txt']),
       ],
       install_requires=[
-          'arvados-python-client >= 2.2.0.dev20201118185221',
+          *arvados_version.iter_dependencies(version),
       ],
       python_requires="~=3.8",
       zip_safe=True,