services/dockercleaner/arvados_version.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import dataclasses
   6 import os
   7 import re
   8 import runpy
   9 import subprocess
  10 import typing as t
  11
  12 from pathlib import Path, PurePath, PurePosixPath
  13
  14 import setuptools
  15 import setuptools.command.build
  16
  17 SETUP_DIR = Path(__file__).absolute().parent
  18 VERSION_SCRIPT_PATH = PurePath('build', 'version-at-commit.sh')
  19
  20 ### Metadata generation
  21
  22 @dataclasses.dataclass
  23 class ArvadosPythonPackage:
  24     package_name: str
  25     module_name: str
  26     src_path: PurePath
  27     dependencies: t.Sequence['ArvadosPythonPackage']
  28
  29     _VERSION_SUBS = {
  30         'development-': '',
  31         '~dev': '.dev',
  32         '~rc': 'rc',
  33     }
  34
  35     def version_file_path(self):
  36         return PurePath(self.module_name, '_version.py')
  37
  38     def _workspace_path(self, workdir):
  39         try:
  40             workspace = Path(os.environ['WORKSPACE'])
  41             # This will raise ValueError if they're not related,
  42             # in which case we don't want to use this $WORKSPACE.
  43             workdir.relative_to(workspace)
  44         except (KeyError, ValueError):
  45             return None
  46         if (workspace / VERSION_SCRIPT_PATH).exists():
  47             return workspace
  48         else:
  49             return None
  50
  51     def _git_version(self, workdir):
  52         workspace = self._workspace_path(workdir)
  53         if workspace is None:
  54             return None
  55         git_log_cmd = [
  56             'git', 'log', '-n1', '--format=%H', '--',
  57             str(VERSION_SCRIPT_PATH), str(self.src_path),
  58         ]
  59         git_log_cmd.extend(str(dep.src_path) for dep in self.dependencies)
  60         git_log_proc = subprocess.run(
  61             git_log_cmd,
  62             check=True,
  63             cwd=workspace,
  64             stdout=subprocess.PIPE,
  65             text=True,
  66         )
  67         version_proc = subprocess.run(
  68             [str(VERSION_SCRIPT_PATH), git_log_proc.stdout.rstrip('\n')],
  69             check=True,
  70             cwd=workspace,
  71             stdout=subprocess.PIPE,
  72             text=True,
  73         )
  74         return version_proc.stdout.rstrip('\n')
  75
  76     def _sdist_version(self, workdir):
  77         try:
  78             pkg_info = (workdir / 'PKG-INFO').open()
  79         except FileNotFoundError:
  80             return None
  81         with pkg_info:
  82             for line in pkg_info:
  83                 key, _, val = line.partition(': ')
  84                 if key == 'Version':
  85                     return val.rstrip('\n')
  86         raise Exception("found PKG-INFO file but not Version metadata in it")
  87
  88     def get_version(self, workdir=SETUP_DIR):
  89         version = (
  90             # If we're building out of a distribution, we should pass that
  91             # version through unchanged.
  92             self._sdist_version(workdir)
  93             # Otherwise follow the usual Arvados versioning rules.
  94             or os.environ.get('ARVADOS_BUILDING_VERSION')
  95             or self._git_version(workdir)
  96         )
  97         if not version:
  98             raise Exception(f"no version information available for {self.package_name}")
  99         else:
 100             return re.sub(
 101                 r'(^development-|~dev|~rc)',
 102                 lambda match: self._VERSION_SUBS[match.group(0)],
 103                 version,
 104             )
 105
 106     def get_dependencies_version(self, workdir=SETUP_DIR, version=None):
 107         if version is None:
 108             version = self.get_version(workdir)
 109         # A packaged development release should be installed with other
 110         # development packages built from the same source, but those
 111         # dependencies may have earlier "dev" versions (read: less recent
 112         # Git commit timestamps). This compatible version dependency
 113         # expresses that as closely as possible. Allowing versions
 114         # compatible with .dev0 allows any development release.
 115         # Regular expression borrowed partially from
 116         # <https://packaging.python.org/en/latest/specifications/version-specifiers/#version-specifiers-regex>
 117         dep_ver, match_count = re.subn(r'\.dev(0|[1-9][0-9]*)$', '.dev0', version, 1)
 118         return ('~=' if match_count else '==', dep_ver)
 119
 120     def iter_dependencies(self, workdir=SETUP_DIR, version=None):
 121         dep_op, dep_ver = self.get_dependencies_version(workdir, version)
 122         for dep in self.dependencies:
 123             yield f'{dep.package_name} {dep_op} {dep_ver}'
 124
 125
 126 ### Package database
 127
 128 _PYSDK = ArvadosPythonPackage(
 129     'arvados-python-client',
 130     'arvados',
 131     PurePath('sdk', 'python'),
 132     [],
 133 )
 134 _CRUNCHSTAT_SUMMARY = ArvadosPythonPackage(
 135     'crunchstat_summary',
 136     'crunchstat_summary',
 137     PurePath('tools', 'crunchstat-summary'),
 138     [_PYSDK],
 139 )
 140 ARVADOS_PYTHON_MODULES = {mod.package_name: mod for mod in [
 141     _PYSDK,
 142     _CRUNCHSTAT_SUMMARY,
 143     ArvadosPythonPackage(
 144         'arvados-cluster-activity',
 145         'arvados_cluster_activity',
 146         PurePath('tools', 'cluster-activity'),
 147         [_PYSDK],
 148     ),
 149     ArvadosPythonPackage(
 150         'arvados-cwl-runner',
 151         'arvados_cwl',
 152         PurePath('sdk', 'cwl'),
 153         [_PYSDK, _CRUNCHSTAT_SUMMARY],
 154     ),
 155     ArvadosPythonPackage(
 156         'arvados-docker-cleaner',
 157         'arvados_docker',
 158         PurePath('services', 'dockercleaner'),
 159         [],
 160     ),
 161     ArvadosPythonPackage(
 162         'arvados_fuse',
 163         'arvados_fuse',
 164         PurePath('services', 'fuse'),
 165         [_PYSDK],
 166     ),
 167     ArvadosPythonPackage(
 168         'arvados-user-activity',
 169         'arvados_user_activity',
 170         PurePath('tools', 'user-activity'),
 171         [_PYSDK],
 172     ),
 173 ]}
 174
 175 ### setuptools integration
 176
 177 class BuildArvadosVersion(setuptools.Command):
 178     """Write _version.py for an Arvados module"""
 179     def initialize_options(self):
 180         self.build_lib = None
 181
 182     def finalize_options(self):
 183         self.set_undefined_options("build_py", ("build_lib", "build_lib"))
 184         arv_mod = ARVADOS_PYTHON_MODULES[self.distribution.get_name()]
 185         self.out_path = Path(self.build_lib, arv_mod.version_file_path())
 186
 187     def run(self):
 188         with self.out_path.open('w') as out_file:
 189             print(f'__version__ = {self.distribution.get_version()!r}', file=out_file)
 190
 191     def get_outputs(self):
 192         return [str(self.out_path)]
 193
 194     def get_source_files(self):
 195         return []
 196
 197     def get_output_mapping(self):
 198         return {}
 199
 200
 201 class ArvadosBuildCommand(setuptools.command.build.build):
 202     sub_commands = [
 203         *setuptools.command.build.build.sub_commands,
 204         ('build_arvados_version', None),
 205     ]
 206
 207
 208 CMDCLASS = {
 209     'build': ArvadosBuildCommand,
 210     'build_arvados_version': BuildArvadosVersion,
 211 }