jenkins/run_upload_packages.py

   1 #!/usr/bin/env python3
   2
   3 import argparse
   4 import functools
   5 import glob
   6 import locale
   7 import logging
   8 import os
   9 import pipes
  10 import re
  11 import shutil
  12 import subprocess
  13 import sys
  14 import time
  15
  16 def run_and_grep(cmd, read_output, *regexps,
  17                  encoding=locale.getpreferredencoding(), **popen_kwargs):
  18     """Run a subprocess and capture output lines matching regexps.
  19
  20     Arguments:
  21     * cmd: The command to run, as a list or string, as for subprocess.Popen.
  22     * read_output: 'stdout' or 'stderr', the name of the output stream to read.
  23     Remaining arguments are regexps to match output, as strings or compiled
  24     regexp objects.  Output lines matching any regexp will be captured.
  25
  26     Keyword arguments:
  27     * encoding: The encoding used to decode the subprocess output.
  28     Remaining keyword arguments are passed directly to subprocess.Popen.
  29
  30     Returns 2-tuple (subprocess returncode, list of matched output lines).
  31     """
  32     regexps = [regexp if hasattr(regexp, 'search') else re.compile(regexp)
  33                for regexp in regexps]
  34     popen_kwargs[read_output] = subprocess.PIPE
  35     proc = subprocess.Popen(cmd, **popen_kwargs)
  36     with open(getattr(proc, read_output).fileno(), encoding=encoding) as output:
  37         matched_lines = [line for line in output
  38                          if any(regexp.search(line) for regexp in regexps)]
  39     return proc.wait(), matched_lines
  40
  41
  42 class TimestampFile:
  43     def __init__(self, path):
  44         self.path = path
  45         self.start_time = time.time()
  46
  47     def last_upload(self):
  48         try:
  49             return os.path.getmtime(self.path)
  50         except EnvironmentError:
  51             return -1
  52
  53     def update(self):
  54         os.close(os.open(self.path, os.O_CREAT | os.O_APPEND))
  55         os.utime(self.path, (time.time(), self.start_time))
  56
  57
  58 class PackageSuite:
  59     NEED_SSH = False
  60
  61     def __init__(self, glob_root, rel_globs):
  62         logger_part = getattr(self, 'LOGGER_PART', os.path.basename(glob_root))
  63         self.logger = logging.getLogger('arvados-dev.upload.' + logger_part)
  64         self.globs = [os.path.join(glob_root, rel_glob)
  65                       for rel_glob in rel_globs]
  66
  67     def files_to_upload(self, since_timestamp):
  68         for abs_glob in self.globs:
  69             for path in glob.glob(abs_glob):
  70                 if os.path.getmtime(path) >= since_timestamp:
  71                     yield path
  72
  73     def upload_file(self, path):
  74         raise NotImplementedError("PackageSuite.upload_file")
  75
  76     def upload_files(self, paths):
  77         for path in paths:
  78             self.logger.info("Uploading %s", path)
  79             self.upload_file(path)
  80
  81     def post_uploads(self, paths):
  82         pass
  83
  84     def update_packages(self, since_timestamp):
  85         upload_paths = list(self.files_to_upload(since_timestamp))
  86         if upload_paths:
  87             self.upload_files(upload_paths)
  88             self.post_uploads(upload_paths)
  89
  90
  91 class PythonPackageSuite(PackageSuite):
  92     LOGGER_PART = 'python'
  93     REUPLOAD_REGEXP = re.compile(
  94         r'^error: Upload failed \(400\): A file named "[^"]+" already exists\b')
  95
  96     def __init__(self, glob_root, rel_globs):
  97         super().__init__(glob_root, rel_globs)
  98         self.seen_packages = set()
  99
 100     def upload_file(self, path):
 101         src_dir = os.path.dirname(os.path.dirname(path))
 102         if src_dir in self.seen_packages:
 103             return
 104         self.seen_packages.add(src_dir)
 105         # NOTE: If we ever start uploading Python 3 packages, we'll need to
 106         # figure out some way to adapt cmd to match.  It might be easiest
 107         # to give all our setup.py files the executable bit, and run that
 108         # directly.
 109         # We also must run `sdist` before `upload`: `upload` uploads any
 110         # distributions previously generated in the command.  It doesn't
 111         # know how to upload distributions already on disk.  We write the
 112         # result to a dedicated directory to avoid interfering with our
 113         # timestamp tracking.
 114         cmd = ['python2.7', 'setup.py']
 115         if not self.logger.isEnabledFor(logging.INFO):
 116             cmd.append('--quiet')
 117         cmd.extend(['sdist', '--dist-dir', '.upload_dist', 'upload'])
 118         upload_returncode, repushed = run_and_grep(
 119             cmd, 'stderr', self.REUPLOAD_REGEXP, cwd=src_dir)
 120         if (upload_returncode != 0) and not repushed:
 121             raise subprocess.CalledProcessError(upload_returncode, cmd)
 122         shutil.rmtree(os.path.join(src_dir, '.upload_dist'))
 123
 124
 125 class GemPackageSuite(PackageSuite):
 126     LOGGER_PART = 'gems'
 127     REUPLOAD_REGEXP = re.compile(r'^Repushing of gem versions is not allowed\.$')
 128
 129     def upload_file(self, path):
 130         cmd = ['gem', 'push', path]
 131         push_returncode, repushed = run_and_grep(cmd, 'stdout', self.REUPLOAD_REGEXP)
 132         if (push_returncode != 0) and not repushed:
 133             raise subprocess.CalledProcessError(push_returncode, cmd)
 134
 135
 136 class DistroPackageSuite(PackageSuite):
 137     NEED_SSH = True
 138     REMOTE_DEST_DIR = 'tmp'
 139
 140     def __init__(self, glob_root, rel_globs, target, ssh_host, ssh_opts):
 141         super().__init__(glob_root, rel_globs)
 142         self.target = target
 143         self.ssh_host = ssh_host
 144         self.ssh_opts = ['-o' + opt for opt in ssh_opts]
 145         if not self.logger.isEnabledFor(logging.INFO):
 146             self.ssh_opts.append('-q')
 147
 148     def _build_cmd(self, base_cmd, *args):
 149         cmd = [base_cmd]
 150         cmd.extend(self.ssh_opts)
 151         cmd.extend(args)
 152         return cmd
 153
 154     def _paths_basenames(self, paths):
 155         return (os.path.basename(path) for path in paths)
 156
 157     def _run_script(self, script, *args):
 158         # SSH will use a shell to run our bash command, so we have to
 159         # quote our arguments.
 160         # self.__class__.__name__ provides $0 for the script, which makes a
 161         # nicer message if there's an error.
 162         subprocess.check_call(self._build_cmd(
 163                 'ssh', self.ssh_host, 'bash', '-ec', pipes.quote(script),
 164                 self.__class__.__name__, *(pipes.quote(s) for s in args)))
 165
 166     def upload_files(self, paths):
 167         dest_dir = os.path.join(self.REMOTE_DEST_DIR, self.target)
 168         mkdir = self._build_cmd('ssh', self.ssh_host, 'install', '-d', dest_dir)
 169         subprocess.check_call(mkdir)
 170         cmd = self._build_cmd('scp', *paths)
 171         cmd.append('{}:{}'.format(self.ssh_host, dest_dir))
 172         subprocess.check_call(cmd)
 173
 174
 175 class DebianPackageSuite(DistroPackageSuite):
 176     FREIGHT_SCRIPT = """
 177 cd "$1"; shift
 178 DISTNAME=$1; shift
 179 freight add "$@" "apt/$DISTNAME"
 180 freight cache "apt/$DISTNAME"
 181 rm "$@"
 182 """
 183     TARGET_DISTNAMES = {
 184         'debian7': 'wheezy',
 185         'debian8': 'jessie',
 186         'ubuntu1204': 'precise',
 187         'ubuntu1404': 'trusty',
 188         'ubuntu1604': 'xenial',
 189         }
 190
 191     def post_uploads(self, paths):
 192         self._run_script(self.FREIGHT_SCRIPT, self.REMOTE_DEST_DIR + '/' + self.target,
 193                          self.TARGET_DISTNAMES[self.target],
 194                          *self._paths_basenames(paths))
 195
 196
 197 class RedHatPackageSuite(DistroPackageSuite):
 198     CREATEREPO_SCRIPT = """
 199 cd "$1"; shift
 200 REPODIR=$1; shift
 201 rpmsign --addsign "$@" </dev/null
 202 mv "$@" "$REPODIR"
 203 createrepo "$REPODIR"
 204 """
 205     REPO_ROOT = '/var/www/rpm.arvados.org/'
 206     TARGET_REPODIRS = {
 207         'centos6': 'CentOS/6/os/x86_64/',
 208         'centos7': 'CentOS/7/os/x86_64/',
 209         }
 210
 211     def post_uploads(self, paths):
 212         repo_dir = os.path.join(self.REPO_ROOT,
 213                                 self.TARGET_REPODIRS[self.target])
 214         self._run_script(self.CREATEREPO_SCRIPT, self.REMOTE_DEST_DIR + '/' + self.target,
 215                          repo_dir, *self._paths_basenames(paths))
 216
 217
 218 def _define_suite(suite_class, *rel_globs, **kwargs):
 219     return functools.partial(suite_class, rel_globs=rel_globs, **kwargs)
 220
 221 PACKAGE_SUITES = {
 222     'python': _define_suite(PythonPackageSuite,
 223                             'sdk/pam/dist/*.tar.gz',
 224                             'sdk/python/dist/*.tar.gz',
 225                             'sdk/cwl/dist/*.tar.gz',
 226                             'services/nodemanager/dist/*.tar.gz',
 227                             'services/fuse/dist/*.tar.gz',
 228                         ),
 229     'gems': _define_suite(GemPackageSuite,
 230                           'sdk/ruby/*.gem',
 231                           'sdk/cli/*.gem',
 232                           'services/login-sync/*.gem',
 233                       ),
 234     }
 235 for target in ['debian7', 'debian8', 'ubuntu1204', 'ubuntu1404', 'ubuntu1604']:
 236     PACKAGE_SUITES[target] = _define_suite(
 237         DebianPackageSuite, os.path.join('packages', target, '*.deb'),
 238         target=target)
 239 for target in ['centos6', 'centos7']:
 240     PACKAGE_SUITES[target] = _define_suite(
 241         RedHatPackageSuite, os.path.join('packages', target, '*.rpm'),
 242         target=target)
 243
 244 def parse_arguments(arguments):
 245     parser = argparse.ArgumentParser(
 246         prog="run_upload_packages.py",
 247         description="Upload Arvados packages to various repositories")
 248     parser.add_argument(
 249         '--workspace', '-W', default=os.environ.get('WORKSPACE'),
 250         help="Arvados source directory with built packages to upload")
 251     parser.add_argument(
 252         '--ssh-host', '-H',
 253         help="Host specification for distribution repository server")
 254     parser.add_argument('-o', action='append', default=[], dest='ssh_opts',
 255                          metavar='OPTION', help="Pass option to `ssh -o`")
 256     parser.add_argument('--verbose', '-v', action='count', default=0,
 257                         help="Log more information and subcommand output")
 258     parser.add_argument(
 259         'targets', nargs='*', default=['all'], metavar='target',
 260         help="Upload packages to these targets (default all)\nAvailable targets: " +
 261         ', '.join(sorted(PACKAGE_SUITES.keys())))
 262     args = parser.parse_args(arguments)
 263     if 'all' in args.targets:
 264         args.targets = list(PACKAGE_SUITES.keys())
 265
 266     if args.workspace is None:
 267         parser.error("workspace not set from command line or environment")
 268     for target in args.targets:
 269         try:
 270             suite_class = PACKAGE_SUITES[target].func
 271         except KeyError:
 272             parser.error("unrecognized target {!r}".format(target))
 273         if suite_class.NEED_SSH and (args.ssh_host is None):
 274             parser.error(
 275                 "--ssh-host must be specified to upload distribution packages")
 276     return args
 277
 278 def setup_logger(stream_dest, args):
 279     log_handler = logging.StreamHandler(stream_dest)
 280     log_handler.setFormatter(logging.Formatter(
 281             '%(asctime)s %(name)s[%(process)d] %(levelname)s: %(message)s',
 282             '%Y-%m-%d %H:%M:%S'))
 283     logger = logging.getLogger('arvados-dev.upload')
 284     logger.addHandler(log_handler)
 285     logger.setLevel(max(1, logging.WARNING - (10 * args.verbose)))
 286
 287 def build_suite_and_upload(target, since_timestamp, args):
 288     suite_def = PACKAGE_SUITES[target]
 289     kwargs = {}
 290     if suite_def.func.NEED_SSH:
 291         kwargs.update(ssh_host=args.ssh_host, ssh_opts=args.ssh_opts)
 292     suite = suite_def(args.workspace, **kwargs)
 293     suite.update_packages(since_timestamp)
 294
 295 def main(arguments, stdout=sys.stdout, stderr=sys.stderr):
 296     args = parse_arguments(arguments)
 297     setup_logger(stderr, args)
 298     for target in args.targets:
 299         ts_file = TimestampFile(os.path.join(args.workspace, 'packages',
 300                                              '.last_upload_%s' % target))
 301         last_upload_ts = ts_file.last_upload()
 302         build_suite_and_upload(target, last_upload_ts, args)
 303         ts_file.update()
 304
 305 if __name__ == '__main__':
 306     main(sys.argv[1:])