jenkins/run_upload_packages.py

   1 #!/usr/bin/env python3
   2
   3 # Copyright (C) The Arvados Authors. All rights reserved.
   4 #
   5 # SPDX-License-Identifier: AGPL-3.0
   6
   7 import argparse
   8 import functools
   9 import glob
  10 import locale
  11 import logging
  12 import os
  13 import pipes
  14 import re
  15 import shutil
  16 import subprocess
  17 import sys
  18 import time
  19
  20 def run_and_grep(cmd, read_output, *regexps,
  21                  encoding=locale.getpreferredencoding(), **popen_kwargs):
  22     """Run a subprocess and capture output lines matching regexps.
  23
  24     Arguments:
  25     * cmd: The command to run, as a list or string, as for subprocess.Popen.
  26     * read_output: 'stdout' or 'stderr', the name of the output stream to read.
  27     Remaining arguments are regexps to match output, as strings or compiled
  28     regexp objects.  Output lines matching any regexp will be captured.
  29
  30     Keyword arguments:
  31     * encoding: The encoding used to decode the subprocess output.
  32     Remaining keyword arguments are passed directly to subprocess.Popen.
  33
  34     Returns 2-tuple (subprocess returncode, list of matched output lines).
  35     """
  36     regexps = [regexp if hasattr(regexp, 'search') else re.compile(regexp)
  37                for regexp in regexps]
  38     popen_kwargs[read_output] = subprocess.PIPE
  39     proc = subprocess.Popen(cmd, **popen_kwargs)
  40     with open(getattr(proc, read_output).fileno(), encoding=encoding) as output:
  41         matched_lines = []
  42         for line in output:
  43             if any(regexp.search(line) for regexp in regexps):
  44                 matched_lines.append(line)
  45             if read_output == 'stderr':
  46                 print(line, file=sys.stderr, end='')
  47     return proc.wait(), matched_lines
  48
  49
  50 class TimestampFile:
  51     def __init__(self, path):
  52         self.path = path
  53         self.start_time = time.time()
  54
  55     def last_upload(self):
  56         try:
  57             return os.path.getmtime(self.path)
  58         except EnvironmentError:
  59             return -1
  60
  61     def update(self):
  62         os.close(os.open(self.path, os.O_CREAT | os.O_APPEND))
  63         os.utime(self.path, (time.time(), self.start_time))
  64
  65
  66 class PackageSuite:
  67     NEED_SSH = False
  68
  69     def __init__(self, glob_root, rel_globs):
  70         logger_part = getattr(self, 'LOGGER_PART', os.path.basename(glob_root))
  71         self.logger = logging.getLogger('arvados-dev.upload.' + logger_part)
  72         self.globs = [os.path.join(glob_root, rel_glob)
  73                       for rel_glob in rel_globs]
  74
  75     def files_to_upload(self, since_timestamp):
  76         for abs_glob in self.globs:
  77             for path in glob.glob(abs_glob):
  78                 if os.path.getmtime(path) >= since_timestamp:
  79                     yield path
  80
  81     def upload_file(self, path):
  82         raise NotImplementedError("PackageSuite.upload_file")
  83
  84     def upload_files(self, paths):
  85         for path in paths:
  86             self.logger.info("Uploading %s", path)
  87             self.upload_file(path)
  88
  89     def post_uploads(self, paths):
  90         pass
  91
  92     def update_packages(self, since_timestamp):
  93         upload_paths = list(self.files_to_upload(since_timestamp))
  94         if upload_paths:
  95             self.upload_files(upload_paths)
  96             self.post_uploads(upload_paths)
  97
  98
  99 class PythonPackageSuite(PackageSuite):
 100     LOGGER_PART = 'python'
 101     REUPLOAD_REGEXP = re.compile(
 102         r'^error: Upload failed \(400\): A file named "[^"]+" already exists\b')
 103
 104     def __init__(self, glob_root, rel_globs):
 105         super().__init__(glob_root, rel_globs)
 106         self.seen_packages = set()
 107
 108     def upload_file(self, path):
 109         src_dir = os.path.dirname(os.path.dirname(path))
 110         if src_dir in self.seen_packages:
 111             return
 112         self.seen_packages.add(src_dir)
 113         # NOTE: If we ever start uploading Python 3 packages, we'll need to
 114         # figure out some way to adapt cmd to match.  It might be easiest
 115         # to give all our setup.py files the executable bit, and run that
 116         # directly.
 117         # We also must run `sdist` before `upload`: `upload` uploads any
 118         # distributions previously generated in the command.  It doesn't
 119         # know how to upload distributions already on disk.  We write the
 120         # result to a dedicated directory to avoid interfering with our
 121         # timestamp tracking.
 122         cmd = ['python2.7', 'setup.py']
 123         if not self.logger.isEnabledFor(logging.INFO):
 124             cmd.append('--quiet')
 125         cmd.extend(['sdist', '--dist-dir', '.upload_dist', 'upload'])
 126         upload_returncode, repushed = run_and_grep(
 127             cmd, 'stderr', self.REUPLOAD_REGEXP, cwd=src_dir)
 128         if (upload_returncode != 0) and not repushed:
 129             raise subprocess.CalledProcessError(upload_returncode, cmd)
 130         shutil.rmtree(os.path.join(src_dir, '.upload_dist'))
 131
 132
 133 class GemPackageSuite(PackageSuite):
 134     LOGGER_PART = 'gems'
 135     REUPLOAD_REGEXP = re.compile(r'^Repushing of gem versions is not allowed\.$')
 136
 137     def upload_file(self, path):
 138         cmd = ['gem', 'push', path]
 139         push_returncode, repushed = run_and_grep(cmd, 'stdout', self.REUPLOAD_REGEXP)
 140         if (push_returncode != 0) and not repushed:
 141             raise subprocess.CalledProcessError(push_returncode, cmd)
 142
 143
 144 class DistroPackageSuite(PackageSuite):
 145     NEED_SSH = True
 146     REMOTE_DEST_DIR = 'tmp'
 147
 148     def __init__(self, glob_root, rel_globs, target, ssh_host, ssh_opts):
 149         super().__init__(glob_root, rel_globs)
 150         self.target = target
 151         self.ssh_host = ssh_host
 152         self.ssh_opts = ['-o' + opt for opt in ssh_opts]
 153         if not self.logger.isEnabledFor(logging.INFO):
 154             self.ssh_opts.append('-q')
 155
 156     def _build_cmd(self, base_cmd, *args):
 157         cmd = [base_cmd]
 158         cmd.extend(self.ssh_opts)
 159         cmd.extend(args)
 160         return cmd
 161
 162     def _paths_basenames(self, paths):
 163         return (os.path.basename(path) for path in paths)
 164
 165     def _run_script(self, script, *args):
 166         # SSH will use a shell to run our bash command, so we have to
 167         # quote our arguments.
 168         # self.__class__.__name__ provides $0 for the script, which makes a
 169         # nicer message if there's an error.
 170         subprocess.check_call(self._build_cmd(
 171                 'ssh', self.ssh_host, 'bash', '-ec', pipes.quote(script),
 172                 self.__class__.__name__, *(pipes.quote(s) for s in args)))
 173
 174     def upload_files(self, paths):
 175         dest_dir = os.path.join(self.REMOTE_DEST_DIR, self.target)
 176         mkdir = self._build_cmd('ssh', self.ssh_host, 'install', '-d', dest_dir)
 177         subprocess.check_call(mkdir)
 178         cmd = self._build_cmd('scp', *paths)
 179         cmd.append('{}:{}'.format(self.ssh_host, dest_dir))
 180         subprocess.check_call(cmd)
 181
 182
 183 class DebianPackageSuite(DistroPackageSuite):
 184     FREIGHT_SCRIPT = """
 185 cd "$1"; shift
 186 DISTNAME=$1; shift
 187 freight add "$@" "apt/$DISTNAME"
 188 freight cache "apt/$DISTNAME"
 189 rm "$@"
 190 """
 191     TARGET_DISTNAMES = {
 192         'debian8': 'jessie',
 193         'debian9': 'stretch',
 194         'ubuntu1204': 'precise',
 195         'ubuntu1404': 'trusty',
 196         'ubuntu1604': 'xenial',
 197         }
 198
 199     def post_uploads(self, paths):
 200         self._run_script(self.FREIGHT_SCRIPT, self.REMOTE_DEST_DIR + '/' + self.target,
 201                          self.TARGET_DISTNAMES[self.target],
 202                          *self._paths_basenames(paths))
 203
 204
 205 class RedHatPackageSuite(DistroPackageSuite):
 206     CREATEREPO_SCRIPT = """
 207 cd "$1"; shift
 208 REPODIR=$1; shift
 209 rpmsign --addsign "$@" </dev/null
 210 mv "$@" "$REPODIR"
 211 createrepo "$REPODIR"
 212 """
 213     REPO_ROOT = '/var/www/rpm.arvados.org/'
 214     TARGET_REPODIRS = {
 215         'centos7': 'CentOS/7/os/x86_64/',
 216         }
 217
 218     def post_uploads(self, paths):
 219         repo_dir = os.path.join(self.REPO_ROOT,
 220                                 self.TARGET_REPODIRS[self.target])
 221         self._run_script(self.CREATEREPO_SCRIPT, self.REMOTE_DEST_DIR + '/' + self.target,
 222                          repo_dir, *self._paths_basenames(paths))
 223
 224
 225 def _define_suite(suite_class, *rel_globs, **kwargs):
 226     return functools.partial(suite_class, rel_globs=rel_globs, **kwargs)
 227
 228 PACKAGE_SUITES = {
 229     'python': _define_suite(PythonPackageSuite,
 230                             'sdk/pam/dist/*.tar.gz',
 231                             'sdk/python/dist/*.tar.gz',
 232                             'sdk/cwl/dist/*.tar.gz',
 233                             'services/nodemanager/dist/*.tar.gz',
 234                             'services/fuse/dist/*.tar.gz',
 235                         ),
 236     'gems': _define_suite(GemPackageSuite,
 237                           'sdk/ruby/*.gem',
 238                           'sdk/cli/*.gem',
 239                           'services/login-sync/*.gem',
 240                       ),
 241     }
 242 for target in ['debian8', 'debian9', 'ubuntu1204', 'ubuntu1404', 'ubuntu1604']:
 243     PACKAGE_SUITES[target] = _define_suite(
 244         DebianPackageSuite, os.path.join('packages', target, '*.deb'),
 245         target=target)
 246 for target in ['centos7']:
 247     PACKAGE_SUITES[target] = _define_suite(
 248         RedHatPackageSuite, os.path.join('packages', target, '*.rpm'),
 249         target=target)
 250
 251 def parse_arguments(arguments):
 252     parser = argparse.ArgumentParser(
 253         prog="run_upload_packages.py",
 254         description="Upload Arvados packages to various repositories")
 255     parser.add_argument(
 256         '--workspace', '-W', default=os.environ.get('WORKSPACE'),
 257         help="Arvados source directory with built packages to upload")
 258     parser.add_argument(
 259         '--ssh-host', '-H',
 260         help="Host specification for distribution repository server")
 261     parser.add_argument('-o', action='append', default=[], dest='ssh_opts',
 262                          metavar='OPTION', help="Pass option to `ssh -o`")
 263     parser.add_argument('--verbose', '-v', action='count', default=0,
 264                         help="Log more information and subcommand output")
 265     parser.add_argument(
 266         'targets', nargs='*', default=['all'], metavar='target',
 267         help="Upload packages to these targets (default all)\nAvailable targets: " +
 268         ', '.join(sorted(PACKAGE_SUITES.keys())))
 269     args = parser.parse_args(arguments)
 270     if 'all' in args.targets:
 271         args.targets = list(PACKAGE_SUITES.keys())
 272
 273     if args.workspace is None:
 274         parser.error("workspace not set from command line or environment")
 275     for target in args.targets:
 276         try:
 277             suite_class = PACKAGE_SUITES[target].func
 278         except KeyError:
 279             parser.error("unrecognized target {!r}".format(target))
 280         if suite_class.NEED_SSH and (args.ssh_host is None):
 281             parser.error(
 282                 "--ssh-host must be specified to upload distribution packages")
 283     return args
 284
 285 def setup_logger(stream_dest, args):
 286     log_handler = logging.StreamHandler(stream_dest)
 287     log_handler.setFormatter(logging.Formatter(
 288             '%(asctime)s %(name)s[%(process)d] %(levelname)s: %(message)s',
 289             '%Y-%m-%d %H:%M:%S'))
 290     logger = logging.getLogger('arvados-dev.upload')
 291     logger.addHandler(log_handler)
 292     logger.setLevel(max(1, logging.WARNING - (10 * args.verbose)))
 293
 294 def build_suite_and_upload(target, since_timestamp, args):
 295     suite_def = PACKAGE_SUITES[target]
 296     kwargs = {}
 297     if suite_def.func.NEED_SSH:
 298         kwargs.update(ssh_host=args.ssh_host, ssh_opts=args.ssh_opts)
 299     suite = suite_def(args.workspace, **kwargs)
 300     suite.update_packages(since_timestamp)
 301
 302 def main(arguments, stdout=sys.stdout, stderr=sys.stderr):
 303     args = parse_arguments(arguments)
 304     setup_logger(stderr, args)
 305     for target in args.targets:
 306         ts_file = TimestampFile(os.path.join(args.workspace, 'packages',
 307                                              '.last_upload_%s' % target))
 308         last_upload_ts = ts_file.last_upload()
 309         build_suite_and_upload(target, last_upload_ts, args)
 310         ts_file.update()
 311
 312 if __name__ == '__main__':
 313     main(sys.argv[1:])