jenkins/run_upload_packages.py

   1 #!/usr/bin/env python3
   2
   3 import argparse
   4 import functools
   5 import glob
   6 import logging
   7 import os
   8 import pipes
   9 import shutil
  10 import subprocess
  11 import sys
  12 import time
  13
  14 class TimestampFile:
  15     def __init__(self, path):
  16         self.path = path
  17         self.start_time = time.time()
  18
  19     def last_upload(self):
  20         try:
  21             return os.path.getmtime(self.path)
  22         except EnvironmentError:
  23             return -1
  24
  25     def update(self):
  26         os.close(os.open(self.path, os.O_CREAT | os.O_APPEND))
  27         os.utime(self.path, (time.time(), self.start_time))
  28
  29
  30 class PackageSuite:
  31     NEED_SSH = False
  32
  33     def __init__(self, glob_root, rel_globs):
  34         logger_part = getattr(self, 'LOGGER_PART', os.path.basename(glob_root))
  35         self.logger = logging.getLogger('arvados-dev.upload.' + logger_part)
  36         self.globs = [os.path.join(glob_root, rel_glob)
  37                       for rel_glob in rel_globs]
  38
  39     def files_to_upload(self, since_timestamp):
  40         for abs_glob in self.globs:
  41             for path in glob.glob(abs_glob):
  42                 if os.path.getmtime(path) >= since_timestamp:
  43                     yield path
  44
  45     def upload_file(self, path):
  46         raise NotImplementedError("PackageSuite.upload_file")
  47
  48     def upload_files(self, paths):
  49         for path in paths:
  50             self.logger.info("Uploading %s", path)
  51             self.upload_file(path)
  52
  53     def post_uploads(self, paths):
  54         pass
  55
  56     def update_packages(self, since_timestamp):
  57         upload_paths = list(self.files_to_upload(since_timestamp))
  58         if upload_paths:
  59             self.upload_files(upload_paths)
  60             self.post_uploads(upload_paths)
  61
  62
  63 class PythonPackageSuite(PackageSuite):
  64     LOGGER_PART = 'python'
  65
  66     def __init__(self, glob_root, rel_globs):
  67         super().__init__(glob_root, rel_globs)
  68         self.seen_packages = set()
  69
  70     def upload_file(self, path):
  71         src_dir = os.path.dirname(os.path.dirname(path))
  72         if src_dir in self.seen_packages:
  73             return
  74         self.seen_packages.add(src_dir)
  75         # NOTE: If we ever start uploading Python 3 packages, we'll need to
  76         # figure out some way to adapt cmd to match.  It might be easiest
  77         # to give all our setup.py files the executable bit, and run that
  78         # directly.
  79         # We also must run `sdist` before `upload`: `upload` uploads any
  80         # distributions previously generated in the command.  It doesn't
  81         # know how to upload distributions already on disk.  We write the
  82         # result to a dedicated directory to avoid interfering with our
  83         # timestamp tracking.
  84         cmd = ['python2.7', 'setup.py']
  85         if not self.logger.isEnabledFor(logging.INFO):
  86             cmd.append('--quiet')
  87         cmd.extend(['sdist', '--dist-dir', '.upload_dist', 'upload'])
  88         subprocess.check_call(cmd, cwd=src_dir)
  89         shutil.rmtree(os.path.join(src_dir, '.upload_dist'))
  90
  91
  92 class GemPackageSuite(PackageSuite):
  93     LOGGER_PART = 'gems'
  94
  95     def upload_file(self, path):
  96         cmd = ['gem', 'push', path]
  97         push_proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
  98         repushed = any(line == b'Repushing of gem versions is not allowed.\n'
  99                        for line in push_proc.stdout)
 100         # Read any remaining stdout before closing.
 101         for line in push_proc.stdout:
 102             pass
 103         push_proc.stdout.close()
 104         if (push_proc.wait() != 0) and not repushed:
 105             raise subprocess.CalledProcessError(push_proc.returncode, cmd)
 106
 107
 108 class DistroPackageSuite(PackageSuite):
 109     NEED_SSH = True
 110     REMOTE_DEST_DIR = 'tmp'
 111
 112     def __init__(self, glob_root, rel_globs, target, ssh_host, ssh_opts):
 113         super().__init__(glob_root, rel_globs)
 114         self.target = target
 115         self.ssh_host = ssh_host
 116         self.ssh_opts = ['-o' + opt for opt in ssh_opts]
 117         if not self.logger.isEnabledFor(logging.INFO):
 118             self.ssh_opts.append('-q')
 119
 120     def _build_cmd(self, base_cmd, *args):
 121         cmd = [base_cmd]
 122         cmd.extend(self.ssh_opts)
 123         cmd.extend(args)
 124         return cmd
 125
 126     def _paths_basenames(self, paths):
 127         return (os.path.basename(path) for path in paths)
 128
 129     def _run_script(self, script, *args):
 130         # SSH will use a shell to run our bash command, so we have to
 131         # quote our arguments.
 132         # self.__class__.__name__ provides $0 for the script, which makes a
 133         # nicer message if there's an error.
 134         subprocess.check_call(self._build_cmd(
 135                 'ssh', self.ssh_host, 'bash', '-ec', pipes.quote(script),
 136                 self.__class__.__name__, *(pipes.quote(s) for s in args)))
 137
 138     def upload_files(self, paths):
 139         cmd = self._build_cmd('scp', *paths)
 140         cmd.append('{self.ssh_host}:{self.REMOTE_DEST_DIR}'.format(self=self))
 141         subprocess.check_call(cmd)
 142
 143
 144 class DebianPackageSuite(DistroPackageSuite):
 145     FREIGHT_SCRIPT = """
 146 cd "$1"; shift
 147 DISTNAME=$1; shift
 148 freight add "$@" "apt/$DISTNAME"
 149 freight cache
 150 rm "$@"
 151 """
 152     TARGET_DISTNAMES = {
 153         'debian7': 'wheezy',
 154         'debian8': 'jessie',
 155         'ubuntu1204': 'precise',
 156         }
 157
 158     def post_uploads(self, paths):
 159         self._run_script(self.FREIGHT_SCRIPT, self.REMOTE_DEST_DIR,
 160                          self.TARGET_DISTNAMES[self.target],
 161                          *self._paths_basenames(paths))
 162
 163
 164 class RedHatPackageSuite(DistroPackageSuite):
 165     CREATEREPO_SCRIPT = """
 166 cd "$1"; shift
 167 REPODIR=$1; shift
 168 rpmsign --addsign "$@" </dev/null
 169 mv "$@" "$REPODIR"
 170 createrepo "$REPODIR"
 171 """
 172     REPO_ROOT = '/var/www/rpm.arvados.org/'
 173     TARGET_REPODIRS = {
 174         'centos6': 'CentOS/6/os/x86_64/'
 175         }
 176
 177     def post_uploads(self, paths):
 178         repo_dir = os.path.join(self.REPO_ROOT,
 179                                 self.TARGET_REPODIRS[self.target])
 180         self._run_script(self.CREATEREPO_SCRIPT, self.REMOTE_DEST_DIR,
 181                          repo_dir, *self._paths_basenames(paths))
 182
 183
 184 def _define_suite(suite_class, *rel_globs, **kwargs):
 185     return functools.partial(suite_class, rel_globs=rel_globs, **kwargs)
 186
 187 PACKAGE_SUITES = {
 188     'python': _define_suite(PythonPackageSuite,
 189                             'sdk/python/dist/*.tar.gz',
 190                             'services/fuse/dist/*.tar.gz',
 191                             'services/nodemanager/dist/*.tar.gz'),
 192     'gems': _define_suite(GemPackageSuite, 'sdk/ruby/*.gem', 'sdk/cli/*.gem'),
 193     }
 194 for target in ['debian7', 'debian8', 'ubuntu1204']:
 195     PACKAGE_SUITES[target] = _define_suite(
 196         DebianPackageSuite, os.path.join('packages', target, '*.deb'),
 197         target=target)
 198 for target in ['centos6']:
 199     PACKAGE_SUITES[target] = _define_suite(
 200         RedHatPackageSuite, os.path.join('packages', target, '*.rpm'),
 201         target=target)
 202
 203 def parse_arguments(arguments):
 204     parser = argparse.ArgumentParser(
 205         prog="run_upload_packages.py",
 206         description="Upload Arvados packages to various repositories")
 207     parser.add_argument(
 208         '--workspace', '-W', default=os.environ.get('WORKSPACE'),
 209         help="Arvados source directory with built packages to upload")
 210     parser.add_argument(
 211         '--ssh-host', '-H',
 212         help="Host specification for distribution repository server")
 213     parser.add_argument('-o', action='append', default=[], dest='ssh_opts',
 214                          metavar='OPTION', help="Pass option to `ssh -o`")
 215     parser.add_argument('--verbose', '-v', action='count', default=0,
 216                         help="Log more information and subcommand output")
 217     parser.add_argument(
 218         'targets', nargs='*', default=['all'], metavar='target',
 219         help="Upload packages to these targets (default all)\nAvailable targets: " +
 220         ', '.join(sorted(PACKAGE_SUITES.keys())))
 221     args = parser.parse_args(arguments)
 222     if 'all' in args.targets:
 223         args.targets = list(PACKAGE_SUITES.keys())
 224
 225     if args.workspace is None:
 226         parser.error("workspace not set from command line or environment")
 227     for target in args.targets:
 228         try:
 229             suite_class = PACKAGE_SUITES[target].func
 230         except KeyError:
 231             parser.error("unrecognized target {!r}".format(target))
 232         if suite_class.NEED_SSH and (args.ssh_host is None):
 233             parser.error(
 234                 "--ssh-host must be specified to upload distribution packages")
 235     return args
 236
 237 def setup_logger(stream_dest, args):
 238     log_handler = logging.StreamHandler(stream_dest)
 239     log_handler.setFormatter(logging.Formatter(
 240             '%(asctime)s %(name)s[%(process)d] %(levelname)s: %(message)s',
 241             '%Y-%m-%d %H:%M:%S'))
 242     logger = logging.getLogger('arvados-dev.upload')
 243     logger.addHandler(log_handler)
 244     logger.setLevel(max(1, logging.WARNING - (10 * args.verbose)))
 245
 246 def build_suite_and_upload(target, since_timestamp, args):
 247     suite_def = PACKAGE_SUITES[target]
 248     kwargs = {}
 249     if suite_def.func.NEED_SSH:
 250         kwargs.update(ssh_host=args.ssh_host, ssh_opts=args.ssh_opts)
 251     suite = suite_def(args.workspace, **kwargs)
 252     suite.update_packages(since_timestamp)
 253
 254 def main(arguments, stdout=sys.stdout, stderr=sys.stderr):
 255     args = parse_arguments(arguments)
 256     setup_logger(stderr, args)
 257     ts_file = TimestampFile(os.path.join(args.workspace, 'packages',
 258                                          '.last_upload'))
 259     last_upload_ts = ts_file.last_upload()
 260     for target in args.targets:
 261         build_suite_and_upload(target, last_upload_ts, args)
 262     ts_file.update()
 263
 264 if __name__ == '__main__':
 265     main(sys.argv[1:])