16 def run_and_grep(cmd, read_output, *regexps,
17 encoding=locale.getpreferredencoding(), **popen_kwargs):
18 """Run a subprocess and capture output lines matching regexps.
21 * cmd: The command to run, as a list or string, as for subprocess.Popen.
22 * read_output: 'stdout' or 'stderr', the name of the output stream to read.
23 Remaining arguments are regexps to match output, as strings or compiled
24 regexp objects. Output lines matching any regexp will be captured.
27 * encoding: The encoding used to decode the subprocess output.
28 Remaining keyword arguments are passed directly to subprocess.Popen.
30 Returns 2-tuple (subprocess returncode, list of matched output lines).
32 regexps = [regexp if hasattr(regexp, 'search') else re.compile(regexp)
33 for regexp in regexps]
34 popen_kwargs[read_output] = subprocess.PIPE
35 proc = subprocess.Popen(cmd, **popen_kwargs)
36 with open(getattr(proc, read_output).fileno(), encoding=encoding) as output:
37 matched_lines = [line for line in output
38 if any(regexp.search(line) for regexp in regexps)]
39 return proc.wait(), matched_lines
43 def __init__(self, path):
45 self.start_time = time.time()
47 def last_upload(self):
49 return os.path.getmtime(self.path)
50 except EnvironmentError:
54 os.close(os.open(self.path, os.O_CREAT | os.O_APPEND))
55 os.utime(self.path, (time.time(), self.start_time))
61 def __init__(self, glob_root, rel_globs):
62 logger_part = getattr(self, 'LOGGER_PART', os.path.basename(glob_root))
63 self.logger = logging.getLogger('arvados-dev.upload.' + logger_part)
64 self.globs = [os.path.join(glob_root, rel_glob)
65 for rel_glob in rel_globs]
67 def files_to_upload(self, since_timestamp):
68 for abs_glob in self.globs:
69 for path in glob.glob(abs_glob):
70 if os.path.getmtime(path) >= since_timestamp:
73 def upload_file(self, path):
74 raise NotImplementedError("PackageSuite.upload_file")
76 def upload_files(self, paths):
78 self.logger.info("Uploading %s", path)
79 self.upload_file(path)
81 def post_uploads(self, paths):
84 def update_packages(self, since_timestamp):
85 upload_paths = list(self.files_to_upload(since_timestamp))
87 self.upload_files(upload_paths)
88 self.post_uploads(upload_paths)
91 class PythonPackageSuite(PackageSuite):
92 LOGGER_PART = 'python'
93 REUPLOAD_REGEXP = re.compile(
94 r'^error: Upload failed \(400\): A file named "[^"]+" already exists\b')
96 def __init__(self, glob_root, rel_globs):
97 super().__init__(glob_root, rel_globs)
98 self.seen_packages = set()
100 def upload_file(self, path):
101 src_dir = os.path.dirname(os.path.dirname(path))
102 if src_dir in self.seen_packages:
104 self.seen_packages.add(src_dir)
105 # NOTE: If we ever start uploading Python 3 packages, we'll need to
106 # figure out some way to adapt cmd to match. It might be easiest
107 # to give all our setup.py files the executable bit, and run that
109 # We also must run `sdist` before `upload`: `upload` uploads any
110 # distributions previously generated in the command. It doesn't
111 # know how to upload distributions already on disk. We write the
112 # result to a dedicated directory to avoid interfering with our
113 # timestamp tracking.
114 cmd = ['python2.7', 'setup.py']
115 if not self.logger.isEnabledFor(logging.INFO):
116 cmd.append('--quiet')
117 cmd.extend(['sdist', '--dist-dir', '.upload_dist', 'upload'])
118 upload_returncode, repushed = run_and_grep(
119 cmd, 'stderr', self.REUPLOAD_REGEXP, cwd=src_dir)
120 if (upload_returncode != 0) and not repushed:
121 raise subprocess.CalledProcessError(upload_returncode, cmd)
122 shutil.rmtree(os.path.join(src_dir, '.upload_dist'))
125 class GemPackageSuite(PackageSuite):
127 REUPLOAD_REGEXP = re.compile(r'^Repushing of gem versions is not allowed\.$')
129 def upload_file(self, path):
130 cmd = ['gem', 'push', path]
131 push_returncode, repushed = run_and_grep(cmd, 'stdout', self.REUPLOAD_REGEXP)
132 if (push_returncode != 0) and not repushed:
133 raise subprocess.CalledProcessError(push_returncode, cmd)
136 class DistroPackageSuite(PackageSuite):
138 REMOTE_DEST_DIR = 'tmp'
140 def __init__(self, glob_root, rel_globs, target, ssh_host, ssh_opts):
141 super().__init__(glob_root, rel_globs)
143 self.ssh_host = ssh_host
144 self.ssh_opts = ['-o' + opt for opt in ssh_opts]
145 if not self.logger.isEnabledFor(logging.INFO):
146 self.ssh_opts.append('-q')
148 def _build_cmd(self, base_cmd, *args):
150 cmd.extend(self.ssh_opts)
154 def _paths_basenames(self, paths):
155 return (os.path.basename(path) for path in paths)
157 def _run_script(self, script, *args):
158 # SSH will use a shell to run our bash command, so we have to
159 # quote our arguments.
160 # self.__class__.__name__ provides $0 for the script, which makes a
161 # nicer message if there's an error.
162 subprocess.check_call(self._build_cmd(
163 'ssh', self.ssh_host, 'bash', '-ec', pipes.quote(script),
164 self.__class__.__name__, *(pipes.quote(s) for s in args)))
166 def upload_files(self, paths):
167 dest_dir = os.path.join(self.REMOTE_DEST_DIR, self.target)
168 mkdir = self._build_cmd('ssh', self.ssh_host, 'install', '-d', dest_dir)
169 subprocess.check_call(mkdir)
170 cmd = self._build_cmd('scp', *paths)
171 cmd.append('{}:{}'.format(self.ssh_host, dest_dir))
172 subprocess.check_call(cmd)
175 class DebianPackageSuite(DistroPackageSuite):
179 freight add "$@" "apt/$DISTNAME"
180 freight cache "apt/$DISTNAME"
185 'ubuntu1204': 'precise',
186 'ubuntu1404': 'trusty',
187 'ubuntu1604': 'xenial',
190 def post_uploads(self, paths):
191 self._run_script(self.FREIGHT_SCRIPT, self.REMOTE_DEST_DIR + '/' + self.target,
192 self.TARGET_DISTNAMES[self.target],
193 *self._paths_basenames(paths))
196 class RedHatPackageSuite(DistroPackageSuite):
197 CREATEREPO_SCRIPT = """
200 rpmsign --addsign "$@" </dev/null
202 createrepo "$REPODIR"
204 REPO_ROOT = '/var/www/rpm.arvados.org/'
206 'centos6': 'CentOS/6/os/x86_64/',
207 'centos7': 'CentOS/7/os/x86_64/',
210 def post_uploads(self, paths):
211 repo_dir = os.path.join(self.REPO_ROOT,
212 self.TARGET_REPODIRS[self.target])
213 self._run_script(self.CREATEREPO_SCRIPT, self.REMOTE_DEST_DIR + '/' + self.target,
214 repo_dir, *self._paths_basenames(paths))
217 def _define_suite(suite_class, *rel_globs, **kwargs):
218 return functools.partial(suite_class, rel_globs=rel_globs, **kwargs)
221 'python': _define_suite(PythonPackageSuite,
222 'sdk/pam/dist/*.tar.gz',
223 'sdk/python/dist/*.tar.gz',
224 'sdk/cwl/dist/*.tar.gz',
225 'services/nodemanager/dist/*.tar.gz',
226 'services/fuse/dist/*.tar.gz',
228 'gems': _define_suite(GemPackageSuite,
231 'services/login-sync/*.gem',
234 for target in ['debian8', 'ubuntu1204', 'ubuntu1404', 'ubuntu1604']:
235 PACKAGE_SUITES[target] = _define_suite(
236 DebianPackageSuite, os.path.join('packages', target, '*.deb'),
238 for target in ['centos6', 'centos7']:
239 PACKAGE_SUITES[target] = _define_suite(
240 RedHatPackageSuite, os.path.join('packages', target, '*.rpm'),
243 def parse_arguments(arguments):
244 parser = argparse.ArgumentParser(
245 prog="run_upload_packages.py",
246 description="Upload Arvados packages to various repositories")
248 '--workspace', '-W', default=os.environ.get('WORKSPACE'),
249 help="Arvados source directory with built packages to upload")
252 help="Host specification for distribution repository server")
253 parser.add_argument('-o', action='append', default=[], dest='ssh_opts',
254 metavar='OPTION', help="Pass option to `ssh -o`")
255 parser.add_argument('--verbose', '-v', action='count', default=0,
256 help="Log more information and subcommand output")
258 'targets', nargs='*', default=['all'], metavar='target',
259 help="Upload packages to these targets (default all)\nAvailable targets: " +
260 ', '.join(sorted(PACKAGE_SUITES.keys())))
261 args = parser.parse_args(arguments)
262 if 'all' in args.targets:
263 args.targets = list(PACKAGE_SUITES.keys())
265 if args.workspace is None:
266 parser.error("workspace not set from command line or environment")
267 for target in args.targets:
269 suite_class = PACKAGE_SUITES[target].func
271 parser.error("unrecognized target {!r}".format(target))
272 if suite_class.NEED_SSH and (args.ssh_host is None):
274 "--ssh-host must be specified to upload distribution packages")
277 def setup_logger(stream_dest, args):
278 log_handler = logging.StreamHandler(stream_dest)
279 log_handler.setFormatter(logging.Formatter(
280 '%(asctime)s %(name)s[%(process)d] %(levelname)s: %(message)s',
281 '%Y-%m-%d %H:%M:%S'))
282 logger = logging.getLogger('arvados-dev.upload')
283 logger.addHandler(log_handler)
284 logger.setLevel(max(1, logging.WARNING - (10 * args.verbose)))
286 def build_suite_and_upload(target, since_timestamp, args):
287 suite_def = PACKAGE_SUITES[target]
289 if suite_def.func.NEED_SSH:
290 kwargs.update(ssh_host=args.ssh_host, ssh_opts=args.ssh_opts)
291 suite = suite_def(args.workspace, **kwargs)
292 suite.update_packages(since_timestamp)
294 def main(arguments, stdout=sys.stdout, stderr=sys.stderr):
295 args = parse_arguments(arguments)
296 setup_logger(stderr, args)
297 for target in args.targets:
298 ts_file = TimestampFile(os.path.join(args.workspace, 'packages',
299 '.last_upload_%s' % target))
300 last_upload_ts = ts_file.last_upload()
301 build_suite_and_upload(target, last_upload_ts, args)
304 if __name__ == '__main__':