3 # Copyright (C) The Arvados Authors. All rights reserved.
5 # SPDX-License-Identifier: AGPL-3.0
20 def run_and_grep(cmd, read_output, *regexps,
21 encoding=locale.getpreferredencoding(), **popen_kwargs):
22 """Run a subprocess and capture output lines matching regexps.
25 * cmd: The command to run, as a list or string, as for subprocess.Popen.
26 * read_output: 'stdout' or 'stderr', the name of the output stream to read.
27 Remaining arguments are regexps to match output, as strings or compiled
28 regexp objects. Output lines matching any regexp will be captured.
31 * encoding: The encoding used to decode the subprocess output.
32 Remaining keyword arguments are passed directly to subprocess.Popen.
34 Returns 2-tuple (subprocess returncode, list of matched output lines).
36 regexps = [regexp if hasattr(regexp, 'search') else re.compile(regexp)
37 for regexp in regexps]
38 popen_kwargs[read_output] = subprocess.PIPE
39 proc = subprocess.Popen(cmd, **popen_kwargs)
40 with open(getattr(proc, read_output).fileno(), encoding=encoding) as output:
43 if any(regexp.search(line) for regexp in regexps):
44 matched_lines.append(line)
45 if read_output == 'stderr':
46 print(line, file=sys.stderr, end='')
47 return proc.wait(), matched_lines
51 def __init__(self, path):
53 self.start_time = time.time()
55 def last_upload(self):
57 return os.path.getmtime(self.path)
58 except EnvironmentError:
62 os.close(os.open(self.path, os.O_CREAT | os.O_APPEND))
63 os.utime(self.path, (time.time(), self.start_time))
69 def __init__(self, glob_root, rel_globs):
70 logger_part = getattr(self, 'LOGGER_PART', os.path.basename(glob_root))
71 self.logger = logging.getLogger('arvados-dev.upload.' + logger_part)
72 self.globs = [os.path.join(glob_root, rel_glob)
73 for rel_glob in rel_globs]
75 def files_to_upload(self, since_timestamp):
76 for abs_glob in self.globs:
77 for path in glob.glob(abs_glob):
78 if os.path.getmtime(path) >= since_timestamp:
81 def upload_file(self, path):
82 raise NotImplementedError("PackageSuite.upload_file")
84 def upload_files(self, paths):
86 self.logger.info("Uploading %s", path)
87 self.upload_file(path)
89 def post_uploads(self, paths):
92 def update_packages(self, since_timestamp):
93 upload_paths = list(self.files_to_upload(since_timestamp))
95 self.upload_files(upload_paths)
96 self.post_uploads(upload_paths)
99 class PythonPackageSuite(PackageSuite):
100 LOGGER_PART = 'python'
103 r'^error: Upload failed \(400\): A file named "[^"]+" already exists\b'),
105 r'^error: Upload failed \(400\): File already exists\b'),
107 r'^error: Upload failed \(400\): Only one sdist may be uploaded per release\b'),
110 def __init__(self, glob_root, rel_globs):
111 super().__init__(glob_root, rel_globs)
112 self.seen_packages = set()
114 def upload_file(self, path):
115 src_dir = os.path.dirname(os.path.dirname(path))
116 if src_dir in self.seen_packages:
118 self.seen_packages.add(src_dir)
119 # NOTE: If we ever start uploading Python 3 packages, we'll need to
120 # figure out some way to adapt cmd to match. It might be easiest
121 # to give all our setup.py files the executable bit, and run that
123 # We also must run `sdist` before `upload`: `upload` uploads any
124 # distributions previously generated in the command. It doesn't
125 # know how to upload distributions already on disk. We write the
126 # result to a dedicated directory to avoid interfering with our
127 # timestamp tracking.
128 cmd = ['python2.7', 'setup.py']
129 if not self.logger.isEnabledFor(logging.INFO):
130 cmd.append('--quiet')
131 cmd.extend(['sdist', '--dist-dir', '.upload_dist', 'upload'])
132 upload_returncode, repushed = run_and_grep(
133 cmd, 'stderr', *self.REUPLOAD_REGEXPS, cwd=src_dir)
134 if (upload_returncode != 0) and not repushed:
135 raise subprocess.CalledProcessError(upload_returncode, cmd)
136 shutil.rmtree(os.path.join(src_dir, '.upload_dist'))
139 class GemPackageSuite(PackageSuite):
141 REUPLOAD_REGEXP = re.compile(r'^Repushing of gem versions is not allowed\.$')
143 def upload_file(self, path):
144 cmd = ['gem', 'push', path]
145 push_returncode, repushed = run_and_grep(cmd, 'stdout', self.REUPLOAD_REGEXP)
146 if (push_returncode != 0) and not repushed:
147 raise subprocess.CalledProcessError(push_returncode, cmd)
150 class DistroPackageSuite(PackageSuite):
152 REMOTE_DEST_DIR = 'tmp'
154 def __init__(self, glob_root, rel_globs, target, ssh_host, ssh_opts):
155 super().__init__(glob_root, rel_globs)
157 self.ssh_host = ssh_host
158 self.ssh_opts = ['-o' + opt for opt in ssh_opts]
159 if not self.logger.isEnabledFor(logging.INFO):
160 self.ssh_opts.append('-q')
162 def _build_cmd(self, base_cmd, *args):
164 cmd.extend(self.ssh_opts)
168 def _paths_basenames(self, paths):
169 return (os.path.basename(path) for path in paths)
171 def _run_script(self, script, *args):
172 # SSH will use a shell to run our bash command, so we have to
173 # quote our arguments.
174 # self.__class__.__name__ provides $0 for the script, which makes a
175 # nicer message if there's an error.
176 subprocess.check_call(self._build_cmd(
177 'ssh', self.ssh_host, 'bash', '-ec', pipes.quote(script),
178 self.__class__.__name__, *(pipes.quote(s) for s in args)))
180 def upload_files(self, paths):
181 dest_dir = os.path.join(self.REMOTE_DEST_DIR, self.target)
182 mkdir = self._build_cmd('ssh', self.ssh_host, 'install', '-d', dest_dir)
183 subprocess.check_call(mkdir)
184 cmd = self._build_cmd('scp', *paths)
185 cmd.append('{}:{}'.format(self.ssh_host, dest_dir))
186 subprocess.check_call(cmd)
189 class DebianPackageSuite(DistroPackageSuite):
193 freight add "$@" "apt/$DISTNAME"
194 freight cache "apt/$DISTNAME"
198 'debian8': 'jessie-testing',
199 'debian9': 'stretch-testing',
200 'debian10': 'buster-testing',
201 'ubuntu1404': 'trusty-testing',
202 'ubuntu1604': 'xenial-testing',
203 'ubuntu1804': 'bionic-testing',
206 def post_uploads(self, paths):
207 self._run_script(self.FREIGHT_SCRIPT, self.REMOTE_DEST_DIR + '/' + self.target,
208 self.TARGET_DISTNAMES[self.target],
209 *self._paths_basenames(paths))
212 class RedHatPackageSuite(DistroPackageSuite):
213 CREATEREPO_SCRIPT = """
216 rpmsign --addsign "$@" </dev/null
218 createrepo "$REPODIR"
220 REPO_ROOT = '/var/www/rpm.arvados.org/'
222 'centos7': 'CentOS/7/testing/x86_64/',
225 def post_uploads(self, paths):
226 repo_dir = os.path.join(self.REPO_ROOT,
227 self.TARGET_REPODIRS[self.target])
228 self._run_script(self.CREATEREPO_SCRIPT, self.REMOTE_DEST_DIR + '/' + self.target,
229 repo_dir, *self._paths_basenames(paths))
232 def _define_suite(suite_class, *rel_globs, **kwargs):
233 return functools.partial(suite_class, rel_globs=rel_globs, **kwargs)
236 'python': _define_suite(PythonPackageSuite,
237 'sdk/pam/dist/*.tar.gz',
238 'sdk/python/dist/*.tar.gz',
239 'sdk/cwl/dist/*.tar.gz',
240 'services/nodemanager/dist/*.tar.gz',
241 'services/fuse/dist/*.tar.gz',
243 'gems': _define_suite(GemPackageSuite,
246 'services/login-sync/*.gem',
249 for target in ['debian8', 'debian9', 'debian10', 'ubuntu1404', 'ubuntu1604', 'ubuntu1804']:
250 PACKAGE_SUITES[target] = _define_suite(
251 DebianPackageSuite, os.path.join('packages', target, '*.deb'),
253 for target in ['centos7']:
254 PACKAGE_SUITES[target] = _define_suite(
255 RedHatPackageSuite, os.path.join('packages', target, '*.rpm'),
258 def parse_arguments(arguments):
259 parser = argparse.ArgumentParser(
260 prog="run_upload_packages_testing.py",
261 description="Upload Arvados packages to various repositories")
263 '--workspace', '-W', default=os.environ.get('WORKSPACE'),
264 help="Arvados source directory with built packages to upload")
267 help="Host specification for distribution repository server")
268 parser.add_argument('-o', action='append', default=[], dest='ssh_opts',
269 metavar='OPTION', help="Pass option to `ssh -o`")
270 parser.add_argument('--verbose', '-v', action='count', default=0,
271 help="Log more information and subcommand output")
273 'targets', nargs='*', default=['all'], metavar='target',
274 help="Upload packages to these targets (default all)\nAvailable targets: " +
275 ', '.join(sorted(PACKAGE_SUITES.keys())))
276 args = parser.parse_args(arguments)
277 if 'all' in args.targets:
278 args.targets = list(PACKAGE_SUITES.keys())
280 if args.workspace is None:
281 parser.error("workspace not set from command line or environment")
282 for target in args.targets:
284 suite_class = PACKAGE_SUITES[target].func
286 parser.error("unrecognized target {!r}".format(target))
287 if suite_class.NEED_SSH and (args.ssh_host is None):
289 "--ssh-host must be specified to upload distribution packages")
292 def setup_logger(stream_dest, args):
293 log_handler = logging.StreamHandler(stream_dest)
294 log_handler.setFormatter(logging.Formatter(
295 '%(asctime)s %(name)s[%(process)d] %(levelname)s: %(message)s',
296 '%Y-%m-%d %H:%M:%S'))
297 logger = logging.getLogger('arvados-dev.upload')
298 logger.addHandler(log_handler)
299 logger.setLevel(max(1, logging.WARNING - (10 * args.verbose)))
301 def build_suite_and_upload(target, since_timestamp, args):
302 suite_def = PACKAGE_SUITES[target]
304 if suite_def.func.NEED_SSH:
305 kwargs.update(ssh_host=args.ssh_host, ssh_opts=args.ssh_opts)
306 suite = suite_def(args.workspace, **kwargs)
307 suite.update_packages(since_timestamp)
309 def main(arguments, stdout=sys.stdout, stderr=sys.stderr):
310 args = parse_arguments(arguments)
311 setup_logger(stderr, args)
312 for target in args.targets:
313 ts_file = TimestampFile(os.path.join(args.workspace, 'packages',
314 '.last_upload_%s' % target))
315 last_upload_ts = ts_file.last_upload()
316 build_suite_and_upload(target, last_upload_ts, args)
319 if __name__ == '__main__':