jenkins/run_upload_packages.py

   1 #!/usr/bin/env python3
   2
   3 # Copyright (C) The Arvados Authors. All rights reserved.
   4 #
   5 # SPDX-License-Identifier: AGPL-3.0
   6
   7 import argparse
   8 import functools
   9 import glob
  10 import locale
  11 import logging
  12 import os
  13 import pipes
  14 import re
  15 import shutil
  16 import subprocess
  17 import sys
  18 import time
  19
  20 def run_and_grep(cmd, read_output, *regexps,
  21                  encoding=locale.getpreferredencoding(), **popen_kwargs):
  22     """Run a subprocess and capture output lines matching regexps.
  23
  24     Arguments:
  25     * cmd: The command to run, as a list or string, as for subprocess.Popen.
  26     * read_output: 'stdout' or 'stderr', the name of the output stream to read.
  27     Remaining arguments are regexps to match output, as strings or compiled
  28     regexp objects.  Output lines matching any regexp will be captured.
  29
  30     Keyword arguments:
  31     * encoding: The encoding used to decode the subprocess output.
  32     Remaining keyword arguments are passed directly to subprocess.Popen.
  33
  34     Returns 2-tuple (subprocess returncode, list of matched output lines).
  35     """
  36     regexps = [regexp if hasattr(regexp, 'search') else re.compile(regexp)
  37                for regexp in regexps]
  38     popen_kwargs[read_output] = subprocess.PIPE
  39     proc = subprocess.Popen(cmd, **popen_kwargs)
  40     with open(getattr(proc, read_output).fileno(), encoding=encoding) as output:
  41         matched_lines = [line for line in output
  42                          if any(regexp.search(line) for regexp in regexps)]
  43     return proc.wait(), matched_lines
  44
  45
  46 class TimestampFile:
  47     def __init__(self, path):
  48         self.path = path
  49         self.start_time = time.time()
  50
  51     def last_upload(self):
  52         try:
  53             return os.path.getmtime(self.path)
  54         except EnvironmentError:
  55             return -1
  56
  57     def update(self):
  58         os.close(os.open(self.path, os.O_CREAT | os.O_APPEND))
  59         os.utime(self.path, (time.time(), self.start_time))
  60
  61
  62 class PackageSuite:
  63     NEED_SSH = False
  64
  65     def __init__(self, glob_root, rel_globs):
  66         logger_part = getattr(self, 'LOGGER_PART', os.path.basename(glob_root))
  67         self.logger = logging.getLogger('arvados-dev.upload.' + logger_part)
  68         self.globs = [os.path.join(glob_root, rel_glob)
  69                       for rel_glob in rel_globs]
  70
  71     def files_to_upload(self, since_timestamp):
  72         for abs_glob in self.globs:
  73             for path in glob.glob(abs_glob):
  74                 if os.path.getmtime(path) >= since_timestamp:
  75                     yield path
  76
  77     def upload_file(self, path):
  78         raise NotImplementedError("PackageSuite.upload_file")
  79
  80     def upload_files(self, paths):
  81         for path in paths:
  82             self.logger.info("Uploading %s", path)
  83             self.upload_file(path)
  84
  85     def post_uploads(self, paths):
  86         pass
  87
  88     def update_packages(self, since_timestamp):
  89         upload_paths = list(self.files_to_upload(since_timestamp))
  90         if upload_paths:
  91             self.upload_files(upload_paths)
  92             self.post_uploads(upload_paths)
  93
  94
  95 class PythonPackageSuite(PackageSuite):
  96     LOGGER_PART = 'python'
  97     REUPLOAD_REGEXP = re.compile(
  98         r'^error: Upload failed \(400\): A file named "[^"]+" already exists\b')
  99
 100     def __init__(self, glob_root, rel_globs):
 101         super().__init__(glob_root, rel_globs)
 102         self.seen_packages = set()
 103
 104     def upload_file(self, path):
 105         src_dir = os.path.dirname(os.path.dirname(path))
 106         if src_dir in self.seen_packages:
 107             return
 108         self.seen_packages.add(src_dir)
 109         # NOTE: If we ever start uploading Python 3 packages, we'll need to
 110         # figure out some way to adapt cmd to match.  It might be easiest
 111         # to give all our setup.py files the executable bit, and run that
 112         # directly.
 113         # We also must run `sdist` before `upload`: `upload` uploads any
 114         # distributions previously generated in the command.  It doesn't
 115         # know how to upload distributions already on disk.  We write the
 116         # result to a dedicated directory to avoid interfering with our
 117         # timestamp tracking.
 118         cmd = ['python2.7', 'setup.py']
 119         if not self.logger.isEnabledFor(logging.INFO):
 120             cmd.append('--quiet')
 121         cmd.extend(['sdist', '--dist-dir', '.upload_dist', 'upload'])
 122         upload_returncode, repushed = run_and_grep(
 123             cmd, 'stderr', self.REUPLOAD_REGEXP, cwd=src_dir)
 124         if (upload_returncode != 0) and not repushed:
 125             raise subprocess.CalledProcessError(upload_returncode, cmd)
 126         shutil.rmtree(os.path.join(src_dir, '.upload_dist'))
 127
 128
 129 class GemPackageSuite(PackageSuite):
 130     LOGGER_PART = 'gems'
 131     REUPLOAD_REGEXP = re.compile(r'^Repushing of gem versions is not allowed\.$')
 132
 133     def upload_file(self, path):
 134         cmd = ['gem', 'push', path]
 135         push_returncode, repushed = run_and_grep(cmd, 'stdout', self.REUPLOAD_REGEXP)
 136         if (push_returncode != 0) and not repushed:
 137             raise subprocess.CalledProcessError(push_returncode, cmd)
 138
 139
 140 class DistroPackageSuite(PackageSuite):
 141     NEED_SSH = True
 142     REMOTE_DEST_DIR = 'tmp'
 143
 144     def __init__(self, glob_root, rel_globs, target, ssh_host, ssh_opts):
 145         super().__init__(glob_root, rel_globs)
 146         self.target = target
 147         self.ssh_host = ssh_host
 148         self.ssh_opts = ['-o' + opt for opt in ssh_opts]
 149         if not self.logger.isEnabledFor(logging.INFO):
 150             self.ssh_opts.append('-q')
 151
 152     def _build_cmd(self, base_cmd, *args):
 153         cmd = [base_cmd]
 154         cmd.extend(self.ssh_opts)
 155         cmd.extend(args)
 156         return cmd
 157
 158     def _paths_basenames(self, paths):
 159         return (os.path.basename(path) for path in paths)
 160
 161     def _run_script(self, script, *args):
 162         # SSH will use a shell to run our bash command, so we have to
 163         # quote our arguments.
 164         # self.__class__.__name__ provides $0 for the script, which makes a
 165         # nicer message if there's an error.
 166         subprocess.check_call(self._build_cmd(
 167                 'ssh', self.ssh_host, 'bash', '-ec', pipes.quote(script),
 168                 self.__class__.__name__, *(pipes.quote(s) for s in args)))
 169
 170     def upload_files(self, paths):
 171         dest_dir = os.path.join(self.REMOTE_DEST_DIR, self.target)
 172         mkdir = self._build_cmd('ssh', self.ssh_host, 'install', '-d', dest_dir)
 173         subprocess.check_call(mkdir)
 174         cmd = self._build_cmd('scp', *paths)
 175         cmd.append('{}:{}'.format(self.ssh_host, dest_dir))
 176         subprocess.check_call(cmd)
 177
 178
 179 class DebianPackageSuite(DistroPackageSuite):
 180     FREIGHT_SCRIPT = """
 181 cd "$1"; shift
 182 DISTNAME=$1; shift
 183 freight add "$@" "apt/$DISTNAME"
 184 freight cache "apt/$DISTNAME"
 185 rm "$@"
 186 """
 187     TARGET_DISTNAMES = {
 188         'debian8': 'jessie',
 189         'ubuntu1204': 'precise',
 190         'ubuntu1404': 'trusty',
 191         'ubuntu1604': 'xenial',
 192         }
 193
 194     def post_uploads(self, paths):
 195         self._run_script(self.FREIGHT_SCRIPT, self.REMOTE_DEST_DIR + '/' + self.target,
 196                          self.TARGET_DISTNAMES[self.target],
 197                          *self._paths_basenames(paths))
 198
 199
 200 class RedHatPackageSuite(DistroPackageSuite):
 201     CREATEREPO_SCRIPT = """
 202 cd "$1"; shift
 203 REPODIR=$1; shift
 204 rpmsign --addsign "$@" </dev/null
 205 mv "$@" "$REPODIR"
 206 createrepo "$REPODIR"
 207 """
 208     REPO_ROOT = '/var/www/rpm.arvados.org/'
 209     TARGET_REPODIRS = {
 210         'centos7': 'CentOS/7/os/x86_64/',
 211         }
 212
 213     def post_uploads(self, paths):
 214         repo_dir = os.path.join(self.REPO_ROOT,
 215                                 self.TARGET_REPODIRS[self.target])
 216         self._run_script(self.CREATEREPO_SCRIPT, self.REMOTE_DEST_DIR + '/' + self.target,
 217                          repo_dir, *self._paths_basenames(paths))
 218
 219
 220 def _define_suite(suite_class, *rel_globs, **kwargs):
 221     return functools.partial(suite_class, rel_globs=rel_globs, **kwargs)
 222
 223 PACKAGE_SUITES = {
 224     'python': _define_suite(PythonPackageSuite,
 225                             'sdk/pam/dist/*.tar.gz',
 226                             'sdk/python/dist/*.tar.gz',
 227                             'sdk/cwl/dist/*.tar.gz',
 228                             'services/nodemanager/dist/*.tar.gz',
 229                             'services/fuse/dist/*.tar.gz',
 230                         ),
 231     'gems': _define_suite(GemPackageSuite,
 232                           'sdk/ruby/*.gem',
 233                           'sdk/cli/*.gem',
 234                           'services/login-sync/*.gem',
 235                       ),
 236     }
 237 for target in ['debian8', 'ubuntu1204', 'ubuntu1404', 'ubuntu1604']:
 238     PACKAGE_SUITES[target] = _define_suite(
 239         DebianPackageSuite, os.path.join('packages', target, '*.deb'),
 240         target=target)
 241 for target in ['centos7']:
 242     PACKAGE_SUITES[target] = _define_suite(
 243         RedHatPackageSuite, os.path.join('packages', target, '*.rpm'),
 244         target=target)
 245
 246 def parse_arguments(arguments):
 247     parser = argparse.ArgumentParser(
 248         prog="run_upload_packages.py",
 249         description="Upload Arvados packages to various repositories")
 250     parser.add_argument(
 251         '--workspace', '-W', default=os.environ.get('WORKSPACE'),
 252         help="Arvados source directory with built packages to upload")
 253     parser.add_argument(
 254         '--ssh-host', '-H',
 255         help="Host specification for distribution repository server")
 256     parser.add_argument('-o', action='append', default=[], dest='ssh_opts',
 257                          metavar='OPTION', help="Pass option to `ssh -o`")
 258     parser.add_argument('--verbose', '-v', action='count', default=0,
 259                         help="Log more information and subcommand output")
 260     parser.add_argument(
 261         'targets', nargs='*', default=['all'], metavar='target',
 262         help="Upload packages to these targets (default all)\nAvailable targets: " +
 263         ', '.join(sorted(PACKAGE_SUITES.keys())))
 264     args = parser.parse_args(arguments)
 265     if 'all' in args.targets:
 266         args.targets = list(PACKAGE_SUITES.keys())
 267
 268     if args.workspace is None:
 269         parser.error("workspace not set from command line or environment")
 270     for target in args.targets:
 271         try:
 272             suite_class = PACKAGE_SUITES[target].func
 273         except KeyError:
 274             parser.error("unrecognized target {!r}".format(target))
 275         if suite_class.NEED_SSH and (args.ssh_host is None):
 276             parser.error(
 277                 "--ssh-host must be specified to upload distribution packages")
 278     return args
 279
 280 def setup_logger(stream_dest, args):
 281     log_handler = logging.StreamHandler(stream_dest)
 282     log_handler.setFormatter(logging.Formatter(
 283             '%(asctime)s %(name)s[%(process)d] %(levelname)s: %(message)s',
 284             '%Y-%m-%d %H:%M:%S'))
 285     logger = logging.getLogger('arvados-dev.upload')
 286     logger.addHandler(log_handler)
 287     logger.setLevel(max(1, logging.WARNING - (10 * args.verbose)))
 288
 289 def build_suite_and_upload(target, since_timestamp, args):
 290     suite_def = PACKAGE_SUITES[target]
 291     kwargs = {}
 292     if suite_def.func.NEED_SSH:
 293         kwargs.update(ssh_host=args.ssh_host, ssh_opts=args.ssh_opts)
 294     suite = suite_def(args.workspace, **kwargs)
 295     suite.update_packages(since_timestamp)
 296
 297 def main(arguments, stdout=sys.stdout, stderr=sys.stderr):
 298     args = parse_arguments(arguments)
 299     setup_logger(stderr, args)
 300     for target in args.targets:
 301         ts_file = TimestampFile(os.path.join(args.workspace, 'packages',
 302                                              '.last_upload_%s' % target))
 303         last_upload_ts = ts_file.last_upload()
 304         build_suite_and_upload(target, last_upload_ts, args)
 305         ts_file.update()
 306
 307 if __name__ == '__main__':
 308     main(sys.argv[1:])