jenkins/run_upload_packages.py

   1 #!/usr/bin/env python3
   2
   3 import argparse
   4 import functools
   5 import glob
   6 import locale
   7 import logging
   8 import os
   9 import pipes
  10 import re
  11 import shutil
  12 import subprocess
  13 import sys
  14 import time
  15
  16 def run_and_grep(cmd, read_output, *regexps,
  17                  encoding=locale.getpreferredencoding(), **popen_kwargs):
  18     """Run a subprocess and capture output lines matching regexps.
  19
  20     Arguments:
  21     * cmd: The command to run, as a list or string, as for subprocess.Popen.
  22     * read_output: 'stdout' or 'stderr', the name of the output stream to read.
  23     Remaining arguments are regexps to match output, as strings or compiled
  24     regexp objects.  Output lines matching any regexp will be captured.
  25
  26     Keyword arguments:
  27     * encoding: The encoding used to decode the subprocess output.
  28     Remaining keyword arguments are passed directly to subprocess.Popen.
  29
  30     Returns 2-tuple (subprocess returncode, list of matched output lines).
  31     """
  32     regexps = [regexp if hasattr(regexp, 'search') else re.compile(regexp)
  33                for regexp in regexps]
  34     popen_kwargs[read_output] = subprocess.PIPE
  35     proc = subprocess.Popen(cmd, **popen_kwargs)
  36     with open(getattr(proc, read_output).fileno(), encoding=encoding) as output:
  37         matched_lines = [line for line in output
  38                          if any(regexp.search(line) for regexp in regexps)]
  39     return proc.wait(), matched_lines
  40
  41
  42 class TimestampFile:
  43     def __init__(self, path):
  44         self.path = path
  45         self.start_time = time.time()
  46
  47     def last_upload(self):
  48         try:
  49             return os.path.getmtime(self.path)
  50         except EnvironmentError:
  51             return -1
  52
  53     def update(self):
  54         os.close(os.open(self.path, os.O_CREAT | os.O_APPEND))
  55         os.utime(self.path, (time.time(), self.start_time))
  56
  57
  58 class PackageSuite:
  59     NEED_SSH = False
  60
  61     def __init__(self, glob_root, rel_globs):
  62         logger_part = getattr(self, 'LOGGER_PART', os.path.basename(glob_root))
  63         self.logger = logging.getLogger('arvados-dev.upload.' + logger_part)
  64         self.globs = [os.path.join(glob_root, rel_glob)
  65                       for rel_glob in rel_globs]
  66
  67     def files_to_upload(self, since_timestamp):
  68         for abs_glob in self.globs:
  69             for path in glob.glob(abs_glob):
  70                 if os.path.getmtime(path) >= since_timestamp:
  71                     yield path
  72
  73     def upload_file(self, path):
  74         raise NotImplementedError("PackageSuite.upload_file")
  75
  76     def upload_files(self, paths):
  77         for path in paths:
  78             self.logger.info("Uploading %s", path)
  79             self.upload_file(path)
  80
  81     def post_uploads(self, paths):
  82         pass
  83
  84     def update_packages(self, since_timestamp):
  85         upload_paths = list(self.files_to_upload(since_timestamp))
  86         if upload_paths:
  87             self.upload_files(upload_paths)
  88             self.post_uploads(upload_paths)
  89
  90
  91 class PythonPackageSuite(PackageSuite):
  92     LOGGER_PART = 'python'
  93     REUPLOAD_REGEXP = re.compile(
  94         r'^error: Upload failed \(400\): A file named "[^"]+" already exists\b')
  95
  96     def __init__(self, glob_root, rel_globs):
  97         super().__init__(glob_root, rel_globs)
  98         self.seen_packages = set()
  99
 100     def upload_file(self, path):
 101         src_dir = os.path.dirname(os.path.dirname(path))
 102         if src_dir in self.seen_packages:
 103             return
 104         self.seen_packages.add(src_dir)
 105         # NOTE: If we ever start uploading Python 3 packages, we'll need to
 106         # figure out some way to adapt cmd to match.  It might be easiest
 107         # to give all our setup.py files the executable bit, and run that
 108         # directly.
 109         # We also must run `sdist` before `upload`: `upload` uploads any
 110         # distributions previously generated in the command.  It doesn't
 111         # know how to upload distributions already on disk.  We write the
 112         # result to a dedicated directory to avoid interfering with our
 113         # timestamp tracking.
 114         cmd = ['python2.7', 'setup.py']
 115         if not self.logger.isEnabledFor(logging.INFO):
 116             cmd.append('--quiet')
 117         cmd.extend(['sdist', '--dist-dir', '.upload_dist', 'upload'])
 118         upload_returncode, repushed = run_and_grep(
 119             cmd, 'stderr', self.REUPLOAD_REGEXP, cwd=src_dir)
 120         if (upload_returncode != 0) and not repushed:
 121             raise subprocess.CalledProcessError(upload_returncode, cmd)
 122         shutil.rmtree(os.path.join(src_dir, '.upload_dist'))
 123
 124
 125 class GemPackageSuite(PackageSuite):
 126     LOGGER_PART = 'gems'
 127     REUPLOAD_REGEXP = re.compile(r'^Repushing of gem versions is not allowed\.$')
 128
 129     def upload_file(self, path):
 130         cmd = ['gem', 'push', path]
 131         push_returncode, repushed = run_and_grep(cmd, 'stdout', self.REUPLOAD_REGEXP)
 132         if (push_returncode != 0) and not repushed:
 133             raise subprocess.CalledProcessError(push_returncode, cmd)
 134
 135
 136 class DistroPackageSuite(PackageSuite):
 137     NEED_SSH = True
 138     REMOTE_DEST_DIR = 'tmp'
 139
 140     def __init__(self, glob_root, rel_globs, target, ssh_host, ssh_opts):
 141         super().__init__(glob_root, rel_globs)
 142         self.target = target
 143         self.ssh_host = ssh_host
 144         self.ssh_opts = ['-o' + opt for opt in ssh_opts]
 145         if not self.logger.isEnabledFor(logging.INFO):
 146             self.ssh_opts.append('-q')
 147
 148     def _build_cmd(self, base_cmd, *args):
 149         cmd = [base_cmd]
 150         cmd.extend(self.ssh_opts)
 151         cmd.extend(args)
 152         return cmd
 153
 154     def _paths_basenames(self, paths):
 155         return (os.path.basename(path) for path in paths)
 156
 157     def _run_script(self, script, *args):
 158         # SSH will use a shell to run our bash command, so we have to
 159         # quote our arguments.
 160         # self.__class__.__name__ provides $0 for the script, which makes a
 161         # nicer message if there's an error.
 162         subprocess.check_call(self._build_cmd(
 163                 'ssh', self.ssh_host, 'bash', '-ec', pipes.quote(script),
 164                 self.__class__.__name__, *(pipes.quote(s) for s in args)))
 165
 166     def upload_files(self, paths):
 167         dest_dir = os.path.join(self.REMOTE_DEST_DIR, self.target)
 168         mkdir = self._build_cmd('ssh', self.ssh_host, 'install', '-d', dest_dir)
 169         subprocess.check_call(mkdir)
 170         cmd = self._build_cmd('scp', *paths)
 171         cmd.append('{}:{}'.format(self.ssh_host, dest_dir))
 172         subprocess.check_call(cmd)
 173
 174
 175 class DebianPackageSuite(DistroPackageSuite):
 176     FREIGHT_SCRIPT = """
 177 cd "$1"; shift
 178 DISTNAME=$1; shift
 179 freight add "$@" "apt/$DISTNAME"
 180 freight cache "apt/$DISTNAME"
 181 rm "$@"
 182 """
 183     TARGET_DISTNAMES = {
 184         'debian8': 'jessie',
 185         'ubuntu1204': 'precise',
 186         'ubuntu1404': 'trusty',
 187         'ubuntu1604': 'xenial',
 188         }
 189
 190     def post_uploads(self, paths):
 191         self._run_script(self.FREIGHT_SCRIPT, self.REMOTE_DEST_DIR + '/' + self.target,
 192                          self.TARGET_DISTNAMES[self.target],
 193                          *self._paths_basenames(paths))
 194
 195
 196 class RedHatPackageSuite(DistroPackageSuite):
 197     CREATEREPO_SCRIPT = """
 198 cd "$1"; shift
 199 REPODIR=$1; shift
 200 rpmsign --addsign "$@" </dev/null
 201 mv "$@" "$REPODIR"
 202 createrepo "$REPODIR"
 203 """
 204     REPO_ROOT = '/var/www/rpm.arvados.org/'
 205     TARGET_REPODIRS = {
 206         'centos7': 'CentOS/7/os/x86_64/',
 207         }
 208
 209     def post_uploads(self, paths):
 210         repo_dir = os.path.join(self.REPO_ROOT,
 211                                 self.TARGET_REPODIRS[self.target])
 212         self._run_script(self.CREATEREPO_SCRIPT, self.REMOTE_DEST_DIR + '/' + self.target,
 213                          repo_dir, *self._paths_basenames(paths))
 214
 215
 216 def _define_suite(suite_class, *rel_globs, **kwargs):
 217     return functools.partial(suite_class, rel_globs=rel_globs, **kwargs)
 218
 219 PACKAGE_SUITES = {
 220     'python': _define_suite(PythonPackageSuite,
 221                             'sdk/pam/dist/*.tar.gz',
 222                             'sdk/python/dist/*.tar.gz',
 223                             'sdk/cwl/dist/*.tar.gz',
 224                             'services/nodemanager/dist/*.tar.gz',
 225                             'services/fuse/dist/*.tar.gz',
 226                         ),
 227     'gems': _define_suite(GemPackageSuite,
 228                           'sdk/ruby/*.gem',
 229                           'sdk/cli/*.gem',
 230                           'services/login-sync/*.gem',
 231                       ),
 232     }
 233 for target in ['debian8', 'ubuntu1204', 'ubuntu1404', 'ubuntu1604']:
 234     PACKAGE_SUITES[target] = _define_suite(
 235         DebianPackageSuite, os.path.join('packages', target, '*.deb'),
 236         target=target)
 237 for target in ['centos7']:
 238     PACKAGE_SUITES[target] = _define_suite(
 239         RedHatPackageSuite, os.path.join('packages', target, '*.rpm'),
 240         target=target)
 241
 242 def parse_arguments(arguments):
 243     parser = argparse.ArgumentParser(
 244         prog="run_upload_packages.py",
 245         description="Upload Arvados packages to various repositories")
 246     parser.add_argument(
 247         '--workspace', '-W', default=os.environ.get('WORKSPACE'),
 248         help="Arvados source directory with built packages to upload")
 249     parser.add_argument(
 250         '--ssh-host', '-H',
 251         help="Host specification for distribution repository server")
 252     parser.add_argument('-o', action='append', default=[], dest='ssh_opts',
 253                          metavar='OPTION', help="Pass option to `ssh -o`")
 254     parser.add_argument('--verbose', '-v', action='count', default=0,
 255                         help="Log more information and subcommand output")
 256     parser.add_argument(
 257         'targets', nargs='*', default=['all'], metavar='target',
 258         help="Upload packages to these targets (default all)\nAvailable targets: " +
 259         ', '.join(sorted(PACKAGE_SUITES.keys())))
 260     args = parser.parse_args(arguments)
 261     if 'all' in args.targets:
 262         args.targets = list(PACKAGE_SUITES.keys())
 263
 264     if args.workspace is None:
 265         parser.error("workspace not set from command line or environment")
 266     for target in args.targets:
 267         try:
 268             suite_class = PACKAGE_SUITES[target].func
 269         except KeyError:
 270             parser.error("unrecognized target {!r}".format(target))
 271         if suite_class.NEED_SSH and (args.ssh_host is None):
 272             parser.error(
 273                 "--ssh-host must be specified to upload distribution packages")
 274     return args
 275
 276 def setup_logger(stream_dest, args):
 277     log_handler = logging.StreamHandler(stream_dest)
 278     log_handler.setFormatter(logging.Formatter(
 279             '%(asctime)s %(name)s[%(process)d] %(levelname)s: %(message)s',
 280             '%Y-%m-%d %H:%M:%S'))
 281     logger = logging.getLogger('arvados-dev.upload')
 282     logger.addHandler(log_handler)
 283     logger.setLevel(max(1, logging.WARNING - (10 * args.verbose)))
 284
 285 def build_suite_and_upload(target, since_timestamp, args):
 286     suite_def = PACKAGE_SUITES[target]
 287     kwargs = {}
 288     if suite_def.func.NEED_SSH:
 289         kwargs.update(ssh_host=args.ssh_host, ssh_opts=args.ssh_opts)
 290     suite = suite_def(args.workspace, **kwargs)
 291     suite.update_packages(since_timestamp)
 292
 293 def main(arguments, stdout=sys.stdout, stderr=sys.stderr):
 294     args = parse_arguments(arguments)
 295     setup_logger(stderr, args)
 296     for target in args.targets:
 297         ts_file = TimestampFile(os.path.join(args.workspace, 'packages',
 298                                              '.last_upload_%s' % target))
 299         last_upload_ts = ts_file.last_upload()
 300         build_suite_and_upload(target, last_upload_ts, args)
 301         ts_file.update()
 302
 303 if __name__ == '__main__':
 304     main(sys.argv[1:])