jenkins/run_upload_packages.py

   1 #!/usr/bin/env python3
   2
   3 # Copyright (C) The Arvados Authors. All rights reserved.
   4 #
   5 # SPDX-License-Identifier: AGPL-3.0
   6
   7 import argparse
   8 import functools
   9 import glob
  10 import locale
  11 import logging
  12 import os
  13 import pipes
  14 import re
  15 import shutil
  16 import subprocess
  17 import sys
  18 import time
  19
  20 def run_and_grep(cmd, read_output, *regexps,
  21                  encoding=locale.getpreferredencoding(), **popen_kwargs):
  22     """Run a subprocess and capture output lines matching regexps.
  23
  24     Arguments:
  25     * cmd: The command to run, as a list or string, as for subprocess.Popen.
  26     * read_output: 'stdout' or 'stderr', the name of the output stream to read.
  27     Remaining arguments are regexps to match output, as strings or compiled
  28     regexp objects.  Output lines matching any regexp will be captured.
  29
  30     Keyword arguments:
  31     * encoding: The encoding used to decode the subprocess output.
  32     Remaining keyword arguments are passed directly to subprocess.Popen.
  33
  34     Returns 2-tuple (subprocess returncode, list of matched output lines).
  35     """
  36     regexps = [regexp if hasattr(regexp, 'search') else re.compile(regexp)
  37                for regexp in regexps]
  38     popen_kwargs[read_output] = subprocess.PIPE
  39     proc = subprocess.Popen(cmd, **popen_kwargs)
  40     with open(getattr(proc, read_output).fileno(), encoding=encoding) as output:
  41         matched_lines = [line for line in output
  42                          if any(regexp.search(line) for regexp in regexps)]
  43     return proc.wait(), matched_lines
  44
  45
  46 class TimestampFile:
  47     def __init__(self, path):
  48         self.path = path
  49         self.start_time = time.time()
  50
  51     def last_upload(self):
  52         try:
  53             return os.path.getmtime(self.path)
  54         except EnvironmentError:
  55             return -1
  56
  57     def update(self):
  58         os.close(os.open(self.path, os.O_CREAT | os.O_APPEND))
  59         os.utime(self.path, (time.time(), self.start_time))
  60
  61
  62 class PackageSuite:
  63     NEED_SSH = False
  64
  65     def __init__(self, glob_root, rel_globs):
  66         logger_part = getattr(self, 'LOGGER_PART', os.path.basename(glob_root))
  67         self.logger = logging.getLogger('arvados-dev.upload.' + logger_part)
  68         self.globs = [os.path.join(glob_root, rel_glob)
  69                       for rel_glob in rel_globs]
  70
  71     def files_to_upload(self, since_timestamp):
  72         for abs_glob in self.globs:
  73             for path in glob.glob(abs_glob):
  74                 if os.path.getmtime(path) >= since_timestamp:
  75                     yield path
  76
  77     def upload_file(self, path):
  78         raise NotImplementedError("PackageSuite.upload_file")
  79
  80     def upload_files(self, paths):
  81         for path in paths:
  82             self.logger.info("Uploading %s", path)
  83             self.upload_file(path)
  84
  85     def post_uploads(self, paths):
  86         pass
  87
  88     def update_packages(self, since_timestamp):
  89         upload_paths = list(self.files_to_upload(since_timestamp))
  90         if upload_paths:
  91             self.upload_files(upload_paths)
  92             self.post_uploads(upload_paths)
  93
  94
  95 class PythonPackageSuite(PackageSuite):
  96     LOGGER_PART = 'python'
  97     REUPLOAD_REGEXP = re.compile(
  98         r'^error: Upload failed \(400\): A file named "[^"]+" already exists\b')
  99
 100     def __init__(self, glob_root, rel_globs):
 101         super().__init__(glob_root, rel_globs)
 102         self.seen_packages = set()
 103
 104     def upload_file(self, path):
 105         src_dir = os.path.dirname(os.path.dirname(path))
 106         if src_dir in self.seen_packages:
 107             return
 108         self.seen_packages.add(src_dir)
 109         # NOTE: If we ever start uploading Python 3 packages, we'll need to
 110         # figure out some way to adapt cmd to match.  It might be easiest
 111         # to give all our setup.py files the executable bit, and run that
 112         # directly.
 113         # We also must run `sdist` before `upload`: `upload` uploads any
 114         # distributions previously generated in the command.  It doesn't
 115         # know how to upload distributions already on disk.  We write the
 116         # result to a dedicated directory to avoid interfering with our
 117         # timestamp tracking.
 118         cmd = ['python2.7', 'setup.py']
 119         if not self.logger.isEnabledFor(logging.INFO):
 120             cmd.append('--quiet')
 121         cmd.extend(['sdist', '--dist-dir', '.upload_dist', 'upload'])
 122         upload_returncode, repushed = run_and_grep(
 123             cmd, 'stderr', self.REUPLOAD_REGEXP, cwd=src_dir)
 124         if (upload_returncode != 0) and not repushed:
 125             raise subprocess.CalledProcessError(upload_returncode, cmd)
 126         shutil.rmtree(os.path.join(src_dir, '.upload_dist'))
 127
 128
 129 class GemPackageSuite(PackageSuite):
 130     LOGGER_PART = 'gems'
 131     REUPLOAD_REGEXP = re.compile(r'^Repushing of gem versions is not allowed\.$')
 132
 133     def upload_file(self, path):
 134         cmd = ['gem', 'push', path]
 135         push_returncode, repushed = run_and_grep(cmd, 'stdout', self.REUPLOAD_REGEXP)
 136         if (push_returncode != 0) and not repushed:
 137             raise subprocess.CalledProcessError(push_returncode, cmd)
 138
 139
 140 class DistroPackageSuite(PackageSuite):
 141     NEED_SSH = True
 142     REMOTE_DEST_DIR = 'tmp'
 143
 144     def __init__(self, glob_root, rel_globs, target, ssh_host, ssh_opts):
 145         super().__init__(glob_root, rel_globs)
 146         self.target = target
 147         self.ssh_host = ssh_host
 148         self.ssh_opts = ['-o' + opt for opt in ssh_opts]
 149         if not self.logger.isEnabledFor(logging.INFO):
 150             self.ssh_opts.append('-q')
 151
 152     def _build_cmd(self, base_cmd, *args):
 153         cmd = [base_cmd]
 154         cmd.extend(self.ssh_opts)
 155         cmd.extend(args)
 156         return cmd
 157
 158     def _paths_basenames(self, paths):
 159         return (os.path.basename(path) for path in paths)
 160
 161     def _run_script(self, script, *args):
 162         # SSH will use a shell to run our bash command, so we have to
 163         # quote our arguments.
 164         # self.__class__.__name__ provides $0 for the script, which makes a
 165         # nicer message if there's an error.
 166         subprocess.check_call(self._build_cmd(
 167                 'ssh', self.ssh_host, 'bash', '-ec', pipes.quote(script),
 168                 self.__class__.__name__, *(pipes.quote(s) for s in args)))
 169
 170     def upload_files(self, paths):
 171         dest_dir = os.path.join(self.REMOTE_DEST_DIR, self.target)
 172         mkdir = self._build_cmd('ssh', self.ssh_host, 'install', '-d', dest_dir)
 173         subprocess.check_call(mkdir)
 174         cmd = self._build_cmd('scp', *paths)
 175         cmd.append('{}:{}'.format(self.ssh_host, dest_dir))
 176         subprocess.check_call(cmd)
 177
 178
 179 class DebianPackageSuite(DistroPackageSuite):
 180     FREIGHT_SCRIPT = """
 181 cd "$1"; shift
 182 DISTNAME=$1; shift
 183 freight add "$@" "apt/$DISTNAME"
 184 freight cache "apt/$DISTNAME"
 185 rm "$@"
 186 """
 187     TARGET_DISTNAMES = {
 188         'debian8': 'jessie',
 189         'debian9': 'stretch',
 190         'ubuntu1204': 'precise',
 191         'ubuntu1404': 'trusty',
 192         'ubuntu1604': 'xenial',
 193         }
 194
 195     def post_uploads(self, paths):
 196         self._run_script(self.FREIGHT_SCRIPT, self.REMOTE_DEST_DIR + '/' + self.target,
 197                          self.TARGET_DISTNAMES[self.target],
 198                          *self._paths_basenames(paths))
 199
 200
 201 class RedHatPackageSuite(DistroPackageSuite):
 202     CREATEREPO_SCRIPT = """
 203 cd "$1"; shift
 204 REPODIR=$1; shift
 205 rpmsign --addsign "$@" </dev/null
 206 mv "$@" "$REPODIR"
 207 createrepo "$REPODIR"
 208 """
 209     REPO_ROOT = '/var/www/rpm.arvados.org/'
 210     TARGET_REPODIRS = {
 211         'centos7': 'CentOS/7/os/x86_64/',
 212         }
 213
 214     def post_uploads(self, paths):
 215         repo_dir = os.path.join(self.REPO_ROOT,
 216                                 self.TARGET_REPODIRS[self.target])
 217         self._run_script(self.CREATEREPO_SCRIPT, self.REMOTE_DEST_DIR + '/' + self.target,
 218                          repo_dir, *self._paths_basenames(paths))
 219
 220
 221 def _define_suite(suite_class, *rel_globs, **kwargs):
 222     return functools.partial(suite_class, rel_globs=rel_globs, **kwargs)
 223
 224 PACKAGE_SUITES = {
 225     'python': _define_suite(PythonPackageSuite,
 226                             'sdk/pam/dist/*.tar.gz',
 227                             'sdk/python/dist/*.tar.gz',
 228                             'sdk/cwl/dist/*.tar.gz',
 229                             'services/nodemanager/dist/*.tar.gz',
 230                             'services/fuse/dist/*.tar.gz',
 231                         ),
 232     'gems': _define_suite(GemPackageSuite,
 233                           'sdk/ruby/*.gem',
 234                           'sdk/cli/*.gem',
 235                           'services/login-sync/*.gem',
 236                       ),
 237     }
 238 for target in ['debian8', 'debian9', 'ubuntu1204', 'ubuntu1404', 'ubuntu1604']:
 239     PACKAGE_SUITES[target] = _define_suite(
 240         DebianPackageSuite, os.path.join('packages', target, '*.deb'),
 241         target=target)
 242 for target in ['centos7']:
 243     PACKAGE_SUITES[target] = _define_suite(
 244         RedHatPackageSuite, os.path.join('packages', target, '*.rpm'),
 245         target=target)
 246
 247 def parse_arguments(arguments):
 248     parser = argparse.ArgumentParser(
 249         prog="run_upload_packages.py",
 250         description="Upload Arvados packages to various repositories")
 251     parser.add_argument(
 252         '--workspace', '-W', default=os.environ.get('WORKSPACE'),
 253         help="Arvados source directory with built packages to upload")
 254     parser.add_argument(
 255         '--ssh-host', '-H',
 256         help="Host specification for distribution repository server")
 257     parser.add_argument('-o', action='append', default=[], dest='ssh_opts',
 258                          metavar='OPTION', help="Pass option to `ssh -o`")
 259     parser.add_argument('--verbose', '-v', action='count', default=0,
 260                         help="Log more information and subcommand output")
 261     parser.add_argument(
 262         'targets', nargs='*', default=['all'], metavar='target',
 263         help="Upload packages to these targets (default all)\nAvailable targets: " +
 264         ', '.join(sorted(PACKAGE_SUITES.keys())))
 265     args = parser.parse_args(arguments)
 266     if 'all' in args.targets:
 267         args.targets = list(PACKAGE_SUITES.keys())
 268
 269     if args.workspace is None:
 270         parser.error("workspace not set from command line or environment")
 271     for target in args.targets:
 272         try:
 273             suite_class = PACKAGE_SUITES[target].func
 274         except KeyError:
 275             parser.error("unrecognized target {!r}".format(target))
 276         if suite_class.NEED_SSH and (args.ssh_host is None):
 277             parser.error(
 278                 "--ssh-host must be specified to upload distribution packages")
 279     return args
 280
 281 def setup_logger(stream_dest, args):
 282     log_handler = logging.StreamHandler(stream_dest)
 283     log_handler.setFormatter(logging.Formatter(
 284             '%(asctime)s %(name)s[%(process)d] %(levelname)s: %(message)s',
 285             '%Y-%m-%d %H:%M:%S'))
 286     logger = logging.getLogger('arvados-dev.upload')
 287     logger.addHandler(log_handler)
 288     logger.setLevel(max(1, logging.WARNING - (10 * args.verbose)))
 289
 290 def build_suite_and_upload(target, since_timestamp, args):
 291     suite_def = PACKAGE_SUITES[target]
 292     kwargs = {}
 293     if suite_def.func.NEED_SSH:
 294         kwargs.update(ssh_host=args.ssh_host, ssh_opts=args.ssh_opts)
 295     suite = suite_def(args.workspace, **kwargs)
 296     suite.update_packages(since_timestamp)
 297
 298 def main(arguments, stdout=sys.stdout, stderr=sys.stderr):
 299     args = parse_arguments(arguments)
 300     setup_logger(stderr, args)
 301     for target in args.targets:
 302         ts_file = TimestampFile(os.path.join(args.workspace, 'packages',
 303                                              '.last_upload_%s' % target))
 304         last_upload_ts = ts_file.last_upload()
 305         build_suite_and_upload(target, last_upload_ts, args)
 306         ts_file.update()
 307
 308 if __name__ == '__main__':
 309     main(sys.argv[1:])