build/jenkins/run_upload_packages.py

   1 #!/usr/bin/env python3
   2
   3 import argparse
   4 import functools
   5 import glob
   6 import logging
   7 import os
   8 import pipes
   9 import shutil
  10 import subprocess
  11 import sys
  12 import time
  13
  14 class TimestampFile:
  15     def __init__(self, path):
  16         self.path = path
  17         self.start_time = time.time()
  18
  19     def last_upload(self):
  20         try:
  21             return os.path.getmtime(self.path)
  22         except EnvironmentError:
  23             return -1
  24
  25     def update(self):
  26         os.close(os.open(self.path, os.O_CREAT | os.O_APPEND))
  27         os.utime(self.path, (time.time(), self.start_time))
  28
  29
  30 class PackageSuite:
  31     NEED_SSH = False
  32
  33     def __init__(self, glob_root, rel_globs):
  34         logger_part = getattr(self, 'LOGGER_PART', os.path.basename(glob_root))
  35         self.logger = logging.getLogger('arvados-dev.upload.' + logger_part)
  36         self.globs = [os.path.join(glob_root, rel_glob)
  37                       for rel_glob in rel_globs]
  38
  39     def files_to_upload(self, since_timestamp):
  40         for abs_glob in self.globs:
  41             for path in glob.glob(abs_glob):
  42                 if os.path.getmtime(path) >= since_timestamp:
  43                     yield path
  44
  45     def upload_file(self, path):
  46         raise NotImplementedError("PackageSuite.upload_file")
  47
  48     def upload_files(self, paths):
  49         for path in paths:
  50             self.logger.info("Uploading %s", path)
  51             self.upload_file(path)
  52
  53     def post_uploads(self, paths):
  54         pass
  55
  56     def update_packages(self, since_timestamp):
  57         upload_paths = list(self.files_to_upload(since_timestamp))
  58         if upload_paths:
  59             self.upload_files(upload_paths)
  60             self.post_uploads(upload_paths)
  61
  62
  63 class PythonPackageSuite(PackageSuite):
  64     LOGGER_PART = 'python'
  65
  66     def __init__(self, glob_root, rel_globs):
  67         super().__init__(glob_root, rel_globs)
  68         self.seen_packages = set()
  69
  70     def upload_file(self, path):
  71         src_dir = os.path.dirname(os.path.dirname(path))
  72         if src_dir in self.seen_packages:
  73             return
  74         self.seen_packages.add(src_dir)
  75         # NOTE: If we ever start uploading Python 3 packages, we'll need to
  76         # figure out some way to adapt cmd to match.  It might be easiest
  77         # to give all our setup.py files the executable bit, and run that
  78         # directly.
  79         # We also must run `sdist` before `upload`: `upload` uploads any
  80         # distributions previously generated in the command.  It doesn't
  81         # know how to upload distributions already on disk.  We write the
  82         # result to a dedicated directory to avoid interfering with our
  83         # timestamp tracking.
  84         cmd = ['python2.7', 'setup.py']
  85         if not self.logger.isEnabledFor(logging.INFO):
  86             cmd.append('--quiet')
  87         cmd.extend(['sdist', '--dist-dir', '.upload_dist', 'upload'])
  88         subprocess.check_call(cmd, cwd=src_dir)
  89         shutil.rmtree(os.path.join(src_dir, '.upload_dist'))
  90
  91
  92 class GemPackageSuite(PackageSuite):
  93     LOGGER_PART = 'gems'
  94
  95     def upload_file(self, path):
  96         cmd = ['gem', 'push', path]
  97         push_proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
  98         repushed = any(line == b'Repushing of gem versions is not allowed.\n'
  99                        for line in push_proc.stdout)
 100         # Read any remaining stdout before closing.
 101         for line in push_proc.stdout:
 102             pass
 103         push_proc.stdout.close()
 104         if (push_proc.wait() != 0) and not repushed:
 105             raise subprocess.CalledProcessError(push_proc.returncode, cmd)
 106
 107
 108 class DistroPackageSuite(PackageSuite):
 109     NEED_SSH = True
 110     REMOTE_DEST_DIR = 'tmp'
 111
 112     def __init__(self, glob_root, rel_globs, target, ssh_host, ssh_opts):
 113         super().__init__(glob_root, rel_globs)
 114         self.target = target
 115         self.ssh_host = ssh_host
 116         self.ssh_opts = ['-o' + opt for opt in ssh_opts]
 117         if not self.logger.isEnabledFor(logging.INFO):
 118             self.ssh_opts.append('-q')
 119
 120     def _build_cmd(self, base_cmd, *args):
 121         cmd = [base_cmd]
 122         cmd.extend(self.ssh_opts)
 123         cmd.extend(args)
 124         return cmd
 125
 126     def _paths_basenames(self, paths):
 127         return (os.path.basename(path) for path in paths)
 128
 129     def _run_script(self, script, *args):
 130         # SSH will use a shell to run our bash command, so we have to
 131         # quote our arguments.
 132         # self.__class__.__name__ provides $0 for the script, which makes a
 133         # nicer message if there's an error.
 134         subprocess.check_call(self._build_cmd(
 135                 'ssh', self.ssh_host, 'bash', '-ec', pipes.quote(script),
 136                 self.__class__.__name__, *(pipes.quote(s) for s in args)))
 137
 138     def upload_files(self, paths):
 139         cmd = self._build_cmd('scp', *paths)
 140         cmd.append('{self.ssh_host}:{self.REMOTE_DEST_DIR}'.format(self=self))
 141         subprocess.check_call(cmd)
 142
 143
 144 class DebianPackageSuite(DistroPackageSuite):
 145     FREIGHT_SCRIPT = """
 146 cd "$1"; shift
 147 DISTNAME=$1; shift
 148 freight add "$@" "apt/$DISTNAME"
 149 freight cache "apt/$DISTNAME"
 150 rm "$@"
 151 """
 152     TARGET_DISTNAMES = {
 153         'debian7': 'wheezy',
 154         'debian8': 'jessie',
 155         'ubuntu1204': 'precise',
 156         'ubuntu1404': 'trusty',
 157         }
 158
 159     def post_uploads(self, paths):
 160         self._run_script(self.FREIGHT_SCRIPT, self.REMOTE_DEST_DIR,
 161                          self.TARGET_DISTNAMES[self.target],
 162                          *self._paths_basenames(paths))
 163
 164
 165 class RedHatPackageSuite(DistroPackageSuite):
 166     CREATEREPO_SCRIPT = """
 167 cd "$1"; shift
 168 REPODIR=$1; shift
 169 rpmsign --addsign "$@" </dev/null
 170 mv "$@" "$REPODIR"
 171 createrepo "$REPODIR"
 172 """
 173     REPO_ROOT = '/var/www/rpm.arvados.org/'
 174     TARGET_REPODIRS = {
 175         'centos6': 'CentOS/6/os/x86_64/'
 176         }
 177
 178     def post_uploads(self, paths):
 179         repo_dir = os.path.join(self.REPO_ROOT,
 180                                 self.TARGET_REPODIRS[self.target])
 181         self._run_script(self.CREATEREPO_SCRIPT, self.REMOTE_DEST_DIR,
 182                          repo_dir, *self._paths_basenames(paths))
 183
 184
 185 def _define_suite(suite_class, *rel_globs, **kwargs):
 186     return functools.partial(suite_class, rel_globs=rel_globs, **kwargs)
 187
 188 PACKAGE_SUITES = {
 189     'python': _define_suite(PythonPackageSuite,
 190                             'sdk/pam/dist/*.tar.gz',
 191                             'sdk/python/dist/*.tar.gz',
 192                             'sdk/cwl/dist/*.tar.gz',
 193                             'services/nodemanager/dist/*.tar.gz',
 194                             'services/fuse/dist/*.tar.gz',
 195                         ),
 196     'gems': _define_suite(GemPackageSuite,
 197                           'sdk/ruby/*.gem',
 198                           'sdk/cli/*.gem',
 199                           'services/login-sync/*.gem',
 200                       ),
 201     }
 202 for target in ['debian7', 'debian8', 'ubuntu1204', 'ubuntu1404']:
 203     PACKAGE_SUITES[target] = _define_suite(
 204         DebianPackageSuite, os.path.join('packages', target, '*.deb'),
 205         target=target)
 206 for target in ['centos6']:
 207     PACKAGE_SUITES[target] = _define_suite(
 208         RedHatPackageSuite, os.path.join('packages', target, '*.rpm'),
 209         target=target)
 210
 211 def parse_arguments(arguments):
 212     parser = argparse.ArgumentParser(
 213         prog="run_upload_packages.py",
 214         description="Upload Arvados packages to various repositories")
 215     parser.add_argument(
 216         '--workspace', '-W', default=os.environ.get('WORKSPACE'),
 217         help="Arvados source directory with built packages to upload")
 218     parser.add_argument(
 219         '--ssh-host', '-H',
 220         help="Host specification for distribution repository server")
 221     parser.add_argument('-o', action='append', default=[], dest='ssh_opts',
 222                          metavar='OPTION', help="Pass option to `ssh -o`")
 223     parser.add_argument('--verbose', '-v', action='count', default=0,
 224                         help="Log more information and subcommand output")
 225     parser.add_argument(
 226         'targets', nargs='*', default=['all'], metavar='target',
 227         help="Upload packages to these targets (default all)\nAvailable targets: " +
 228         ', '.join(sorted(PACKAGE_SUITES.keys())))
 229     args = parser.parse_args(arguments)
 230     if 'all' in args.targets:
 231         args.targets = list(PACKAGE_SUITES.keys())
 232
 233     if args.workspace is None:
 234         parser.error("workspace not set from command line or environment")
 235     for target in args.targets:
 236         try:
 237             suite_class = PACKAGE_SUITES[target].func
 238         except KeyError:
 239             parser.error("unrecognized target {!r}".format(target))
 240         if suite_class.NEED_SSH and (args.ssh_host is None):
 241             parser.error(
 242                 "--ssh-host must be specified to upload distribution packages")
 243     return args
 244
 245 def setup_logger(stream_dest, args):
 246     log_handler = logging.StreamHandler(stream_dest)
 247     log_handler.setFormatter(logging.Formatter(
 248             '%(asctime)s %(name)s[%(process)d] %(levelname)s: %(message)s',
 249             '%Y-%m-%d %H:%M:%S'))
 250     logger = logging.getLogger('arvados-dev.upload')
 251     logger.addHandler(log_handler)
 252     logger.setLevel(max(1, logging.WARNING - (10 * args.verbose)))
 253
 254 def build_suite_and_upload(target, since_timestamp, args):
 255     suite_def = PACKAGE_SUITES[target]
 256     kwargs = {}
 257     if suite_def.func.NEED_SSH:
 258         kwargs.update(ssh_host=args.ssh_host, ssh_opts=args.ssh_opts)
 259     suite = suite_def(args.workspace, **kwargs)
 260     suite.update_packages(since_timestamp)
 261
 262 def main(arguments, stdout=sys.stdout, stderr=sys.stderr):
 263     args = parse_arguments(arguments)
 264     setup_logger(stderr, args)
 265     ts_file = TimestampFile(os.path.join(args.workspace, 'packages',
 266                                          '.last_upload'))
 267     last_upload_ts = ts_file.last_upload()
 268     for target in args.targets:
 269         build_suite_and_upload(target, last_upload_ts, args)
 270     ts_file.update()
 271
 272 if __name__ == '__main__':
 273     main(sys.argv[1:])