projects / arvados-dev.git / blobdiff
summary | shortlog | log | commit | commitdiff | tree
raw | inline | side by side
Merge branch '21840-twine-upload'
[arvados-dev.git] / jenkins / run_upload_packages.py
diff --git a/jenkins/run_upload_packages.py b/jenkins/run_upload_packages.py
index bc4f4ffc6ae11096fcb742333ab6aa9c077941a4..a299c2ad74e0b8ae3fa459a84a0e273cb3d75bcd 100755 (executable)
--- a/jenkins/run_upload_packages.py
+++ b/jenkins/run_upload_packages.py
@@ -11,8 +11,8 @@ import glob
 import locale
 import logging
 import os
 import locale
 import logging
 import os
-import pipes
 import re
 import re
+import shlex
 import shutil
 import subprocess
 import sys
 import shutil
 import subprocess
 import sys
@@ -117,40 +117,15 @@ class PackageSuite:
 
 class PythonPackageSuite(PackageSuite):
     LOGGER_PART = 'python'
 
 class PythonPackageSuite(PackageSuite):
     LOGGER_PART = 'python'
-    REUPLOAD_REGEXPS = [
-        re.compile(
-            r'^error: Upload failed \(400\): A file named "[^"]+" already exists\b'),
-        re.compile(
-            r'^error: Upload failed \(400\): File already exists\b'),
-        re.compile(
-            r'^error: Upload failed \(400\): Only one sdist may be uploaded per release\b'),
-    ]
-
-    def __init__(self, glob_root, rel_globs):
-        super().__init__(glob_root, rel_globs)
-        self.seen_packages = set()
 
     def upload_file(self, path):
 
     def upload_file(self, path):
-        src_dir = os.path.dirname(os.path.dirname(path))
-        if src_dir in self.seen_packages:
-            return
-        self.seen_packages.add(src_dir)
-        # We also must run `sdist` before `upload`: `upload` uploads any
-        # distributions previously generated in the command.  It doesn't
-        # know how to upload distributions already on disk.  We write the
-        # result to a dedicated directory to avoid interfering with our
-        # timestamp tracking.
-        cmd = ['python3', 'setup.py']
-        if not self.logger.isEnabledFor(logging.INFO):
-            cmd.append('--quiet')
-        cmd.extend(['bdist_wheel', '--dist-dir', '.upload_dist'])
-        cmd.extend(['sdist', '--dist-dir', '.upload_dist'])
-        cmd.extend(['upload'])
-        upload_returncode, repushed = run_and_grep(
-            cmd, 'stderr', *self.REUPLOAD_REGEXPS, cwd=src_dir)
-        if (upload_returncode != 0) and not repushed:
-            raise subprocess.CalledProcessError(upload_returncode, cmd)
-        shutil.rmtree(os.path.join(src_dir, '.upload_dist'))
+        subprocess.run([
+            'twine', 'upload',
+            '--disable-progress-bar',
+            '--non-interactive',
+            '--skip-existing',
+            path,
+        ], stdin=subprocess.DEVNULL, check=True)
 
 
 class GemPackageSuite(PackageSuite):
 
 
 class GemPackageSuite(PackageSuite):
@@ -191,8 +166,8 @@ class DistroPackageSuite(PackageSuite):
         # self.__class__.__name__ provides $0 for the script, which makes a
         # nicer message if there's an error.
         subprocess.check_call(self._build_cmd(
         # self.__class__.__name__ provides $0 for the script, which makes a
         # nicer message if there's an error.
         subprocess.check_call(self._build_cmd(
-                'ssh', self.ssh_host, 'bash', '-ec', pipes.quote(script),
-                self.__class__.__name__, *(pipes.quote(s) for s in args)))
+                'ssh', self.ssh_host, 'bash', '-ec', shlex.quote(script),
+                self.__class__.__name__, *(shlex.quote(s) for s in args)))
 
     def upload_files(self, paths):
         dest_dir = os.path.join(self.REMOTE_DEST_DIR, self.target)
 
     def upload_files(self, paths):
         dest_dir = os.path.join(self.REMOTE_DEST_DIR, self.target)
@@ -205,14 +180,17 @@ class DistroPackageSuite(PackageSuite):
 
 class DebianPackageSuite(DistroPackageSuite):
     APT_SCRIPT = """
 
 class DebianPackageSuite(DistroPackageSuite):
     APT_SCRIPT = """
+set -e
 cd "$1"; shift
 DISTNAME=$1; shift
 cd "$1"; shift
 DISTNAME=$1; shift
+# aptly implements its own locking, but its wait strategy as of April 2024 is
+# not patient enough to accommodate multiple simultaneous uploads.
+APTLY_LOCK="${XDG_RUNTIME_DIR:-/tmp}/aptly-upload.lock"
+aptly() {
+  flock --wait=300 "$APTLY_LOCK" aptly "$@"
+}
 for package in "$@"; do
 for package in "$@"; do
-  set +e
-  aptly repo search "$DISTNAME" "${package%.deb}" >/dev/null 2>&1
-  RET=$?
-  set -e
-  if [[ $RET -eq 0 ]]; then
+  if aptly repo search "$DISTNAME" "${package%.deb}" >/dev/null 2>&1; then
     echo "Not adding $package, it is already present in repo $DISTNAME"
     rm "$package"
   else
     echo "Not adding $package, it is already present in repo $DISTNAME"
     rm "$package"
   else
@@ -225,13 +203,12 @@ aptly publish update "$DISTNAME" filesystem:"${DISTNAME%-*}":
     def __init__(self, glob_root, rel_globs, target, ssh_host, ssh_opts, repo):
         super().__init__(glob_root, rel_globs, target, ssh_host, ssh_opts)
         self.TARGET_DISTNAMES = {
     def __init__(self, glob_root, rel_globs, target, ssh_host, ssh_opts, repo):
         super().__init__(glob_root, rel_globs, target, ssh_host, ssh_opts)
         self.TARGET_DISTNAMES = {
-            'debian8': 'jessie-'+repo,
-            'debian9': 'stretch-'+repo,
             'debian10': 'buster-'+repo,
             'debian10': 'buster-'+repo,
-            'ubuntu1404': 'trusty-'+repo,
-            'ubuntu1604': 'xenial-'+repo,
+            'debian11': 'bullseye-'+repo,
+            'debian12': 'bookworm-'+repo,
             'ubuntu1804': 'bionic-'+repo,
             'ubuntu2004': 'focal-'+repo,
             'ubuntu1804': 'bionic-'+repo,
             'ubuntu2004': 'focal-'+repo,
+            'ubuntu2204': 'jammy-'+repo,
             }
 
     def post_uploads(self, paths):
             }
 
     def post_uploads(self, paths):
@@ -246,7 +223,7 @@ cd "$1"; shift
 REPODIR=$1; shift
 rpmsign --addsign "$@" </dev/null
 mv "$@" "$REPODIR"
 REPODIR=$1; shift
 rpmsign --addsign "$@" </dev/null
 mv "$@" "$REPODIR"
-createrepo -c ~/.createrepo-cache --update "$REPODIR"
+createrepo_c -c ~/.createrepo-cache --update "$REPODIR"
 """
     REPO_ROOT = '/var/www/rpm.arvados.org/'
 
 """
     REPO_ROOT = '/var/www/rpm.arvados.org/'
 
@@ -254,6 +231,7 @@ createrepo -c ~/.createrepo-cache --update "$REPODIR"
         super().__init__(glob_root, rel_globs, target, ssh_host, ssh_opts)
         self.TARGET_REPODIRS = {
             'centos7': 'CentOS/7/%s/x86_64/' % repo,
         super().__init__(glob_root, rel_globs, target, ssh_host, ssh_opts)
         self.TARGET_REPODIRS = {
             'centos7': 'CentOS/7/%s/x86_64/' % repo,
+            'rocky8': 'CentOS/8/%s/x86_64/' % repo,
         }
 
     def post_uploads(self, paths):
         }
 
     def post_uploads(self, paths):
@@ -268,9 +246,14 @@ def _define_suite(suite_class, *rel_globs, **kwargs):
 
 PACKAGE_SUITES = {
     'python': _define_suite(PythonPackageSuite,
 
 PACKAGE_SUITES = {
     'python': _define_suite(PythonPackageSuite,
-                            'sdk/python/dist/*.tar.gz',
                             'sdk/cwl/dist/*.tar.gz',
                             'sdk/cwl/dist/*.tar.gz',
+                            'sdk/cwl/dist/*.whl',
+                            'sdk/python/dist/*.tar.gz',
+                            'sdk/python/dist/*.whl',
                             'services/fuse/dist/*.tar.gz',
                             'services/fuse/dist/*.tar.gz',
+                            'services/fuse/dist/*.whl',
+                            'tools/crunchstat-summary/dist/*.tar.gz',
+                            'tools/crunchstat-summary/dist/*.whl',
                         ),
     'gems': _define_suite(GemPackageSuite,
                           'sdk/ruby/*.gem',
                         ),
     'gems': _define_suite(GemPackageSuite,
                           'sdk/ruby/*.gem',
@@ -307,11 +290,14 @@ def parse_arguments(arguments):
     if args.workspace is None:
         parser.error("workspace not set from command line or environment")
 
     if args.workspace is None:
         parser.error("workspace not set from command line or environment")
 
-    for target in ['debian8', 'debian9', 'debian10', 'ubuntu1404', 'ubuntu1604', 'ubuntu1804', 'ubuntu2004']:
+    for target in [
+            'debian10', 'debian11', 'debian12',
+            'ubuntu1804', 'ubuntu2004', 'ubuntu2204',
+    ]:
         PACKAGE_SUITES[target] = _define_suite(
             DebianPackageSuite, os.path.join('packages', target, '*.deb'),
             target=target, repo=args.repo)
         PACKAGE_SUITES[target] = _define_suite(
             DebianPackageSuite, os.path.join('packages', target, '*.deb'),
             target=target, repo=args.repo)
-    for target in ['centos7']:
+    for target in ['centos7', 'rocky8']:
         PACKAGE_SUITES[target] = _define_suite(
             RedHatPackageSuite, os.path.join('packages', target, '*.rpm'),
             target=target, repo=args.repo)
         PACKAGE_SUITES[target] = _define_suite(
             RedHatPackageSuite, os.path.join('packages', target, '*.rpm'),
             target=target, repo=args.repo)