3 # Copyright (C) The Arvados Authors. All rights reserved.
5 # SPDX-License-Identifier: AGPL-3.0
20 def run_and_grep(cmd, read_output, *regexps,
21 encoding=locale.getpreferredencoding(), **popen_kwargs):
22 """Run a subprocess and capture output lines matching regexps.
25 * cmd: The command to run, as a list or string, as for subprocess.Popen.
26 * read_output: 'stdout' or 'stderr', the name of the output stream to read.
27 Remaining arguments are regexps to match output, as strings or compiled
28 regexp objects. Output lines matching any regexp will be captured.
31 * encoding: The encoding used to decode the subprocess output.
32 Remaining keyword arguments are passed directly to subprocess.Popen.
34 Returns 2-tuple (subprocess returncode, list of matched output lines).
36 regexps = [regexp if hasattr(regexp, 'search') else re.compile(regexp)
37 for regexp in regexps]
38 popen_kwargs[read_output] = subprocess.PIPE
39 proc = subprocess.Popen(cmd, **popen_kwargs)
40 with open(getattr(proc, read_output).fileno(), encoding=encoding) as output:
43 if any(regexp.search(line) for regexp in regexps):
44 matched_lines.append(line)
45 if read_output == 'stderr':
46 print(line, file=sys.stderr, end='')
47 return proc.wait(), matched_lines
51 def __init__(self, path):
53 self.start_time = time.time()
55 def last_upload(self):
57 return os.path.getmtime(self.path)
58 except EnvironmentError:
62 os.close(os.open(self.path, os.O_CREAT | os.O_APPEND))
63 os.utime(self.path, (time.time(), self.start_time))
69 def __init__(self, glob_root, rel_globs):
70 logger_part = getattr(self, 'LOGGER_PART', os.path.basename(glob_root))
71 self.logger = logging.getLogger('arvados-dev.upload.' + logger_part)
72 self.globs = [os.path.join(glob_root, rel_glob)
73 for rel_glob in rel_globs]
75 def files_to_upload(self, since_timestamp):
76 for abs_glob in self.globs:
77 for path in glob.glob(abs_glob):
78 if os.path.getmtime(path) >= since_timestamp:
81 def upload_file(self, path):
82 raise NotImplementedError("PackageSuite.upload_file")
84 def upload_files(self, paths):
86 self.logger.info("Uploading %s", path)
87 self.upload_file(path)
89 def post_uploads(self, paths):
92 def update_packages(self, since_timestamp):
93 upload_paths = list(self.files_to_upload(since_timestamp))
95 self.upload_files(upload_paths)
96 self.post_uploads(upload_paths)
99 class PythonPackageSuite(PackageSuite):
100 LOGGER_PART = 'python'
103 r'^error: Upload failed \(400\): A file named "[^"]+" already exists\b'),
105 r'^error: Upload failed \(400\): File already exists\b'),
107 r'^error: Upload failed \(400\): Only one sdist may be uploaded per release\b'),
110 def __init__(self, glob_root, rel_globs):
111 super().__init__(glob_root, rel_globs)
112 self.seen_packages = set()
114 def upload_file(self, path):
115 src_dir = os.path.dirname(os.path.dirname(path))
116 if src_dir in self.seen_packages:
118 self.seen_packages.add(src_dir)
119 # NOTE: If we ever start uploading Python 3 packages, we'll need to
120 # figure out some way to adapt cmd to match. It might be easiest
121 # to give all our setup.py files the executable bit, and run that
123 # We also must run `sdist` before `upload`: `upload` uploads any
124 # distributions previously generated in the command. It doesn't
125 # know how to upload distributions already on disk. We write the
126 # result to a dedicated directory to avoid interfering with our
127 # timestamp tracking.
128 cmd = ['python2.7', 'setup.py']
129 if not self.logger.isEnabledFor(logging.INFO):
130 cmd.append('--quiet')
131 cmd.extend(['sdist', '--dist-dir', '.upload_dist', 'upload'])
132 upload_returncode, repushed = run_and_grep(
133 cmd, 'stderr', *self.REUPLOAD_REGEXPS, cwd=src_dir)
134 if (upload_returncode != 0) and not repushed:
135 raise subprocess.CalledProcessError(upload_returncode, cmd)
136 shutil.rmtree(os.path.join(src_dir, '.upload_dist'))
139 class GemPackageSuite(PackageSuite):
141 REUPLOAD_REGEXP = re.compile(r'^Repushing of gem versions is not allowed\.$')
143 def upload_file(self, path):
144 cmd = ['gem', 'push', path]
145 push_returncode, repushed = run_and_grep(cmd, 'stdout', self.REUPLOAD_REGEXP)
146 if (push_returncode != 0) and not repushed:
147 raise subprocess.CalledProcessError(push_returncode, cmd)
150 class DistroPackageSuite(PackageSuite):
152 REMOTE_DEST_DIR = 'tmp'
154 def __init__(self, glob_root, rel_globs, target, ssh_host, ssh_opts):
155 super().__init__(glob_root, rel_globs)
157 self.ssh_host = ssh_host
158 self.ssh_opts = ['-o' + opt for opt in ssh_opts]
159 if not self.logger.isEnabledFor(logging.INFO):
160 self.ssh_opts.append('-q')
162 def _build_cmd(self, base_cmd, *args):
164 cmd.extend(self.ssh_opts)
168 def _paths_basenames(self, paths):
169 return (os.path.basename(path) for path in paths)
171 def _run_script(self, script, *args):
172 # SSH will use a shell to run our bash command, so we have to
173 # quote our arguments.
174 # self.__class__.__name__ provides $0 for the script, which makes a
175 # nicer message if there's an error.
176 subprocess.check_call(self._build_cmd(
177 'ssh', self.ssh_host, 'bash', '-ec', pipes.quote(script),
178 self.__class__.__name__, *(pipes.quote(s) for s in args)))
180 def upload_files(self, paths):
181 dest_dir = os.path.join(self.REMOTE_DEST_DIR, self.target)
182 mkdir = self._build_cmd('ssh', self.ssh_host, 'install', '-d', dest_dir)
183 subprocess.check_call(mkdir)
184 cmd = self._build_cmd('scp', *paths)
185 cmd.append('{}:{}'.format(self.ssh_host, dest_dir))
186 subprocess.check_call(cmd)
189 class DebianPackageSuite(DistroPackageSuite):
193 freight add "$@" "apt/$DISTNAME"
194 freight cache "apt/$DISTNAME"
198 def __init__(self, glob_root, rel_globs, target, ssh_host, ssh_opts, repo):
199 super().__init__(glob_root, rel_globs, target, ssh_host, ssh_opts)
200 self.TARGET_DISTNAMES = {
201 'debian8': 'jessie-'+repo,
202 'debian9': 'stretch-'+repo,
203 'debian10': 'buster-'+repo,
204 'ubuntu1404': 'trusty-'+repo,
205 'ubuntu1604': 'xenial-'+repo,
206 'ubuntu1804': 'bionic-'+repo,
209 def post_uploads(self, paths):
210 self._run_script(self.FREIGHT_SCRIPT, self.REMOTE_DEST_DIR + '/' + self.target,
211 self.TARGET_DISTNAMES[self.target],
212 *self._paths_basenames(paths))
215 class RedHatPackageSuite(DistroPackageSuite):
216 CREATEREPO_SCRIPT = """
219 rpmsign --addsign "$@" </dev/null
221 createrepo "$REPODIR"
223 REPO_ROOT = '/var/www/rpm.arvados.org/'
225 def __init__(self, glob_root, rel_globs, target, ssh_host, ssh_opts, repo):
226 super().__init__(glob_root, rel_globs, target, ssh_host, ssh_opts)
227 self.TARGET_REPODIRS = {
228 'centos7': 'CentOS/7/%s/x86_64/' % repo,
231 def post_uploads(self, paths):
232 repo_dir = os.path.join(self.REPO_ROOT,
233 self.TARGET_REPODIRS[self.target])
234 self._run_script(self.CREATEREPO_SCRIPT, self.REMOTE_DEST_DIR + '/' + self.target,
235 repo_dir, *self._paths_basenames(paths))
238 def _define_suite(suite_class, *rel_globs, **kwargs):
239 return functools.partial(suite_class, rel_globs=rel_globs, **kwargs)
242 'python': _define_suite(PythonPackageSuite,
243 'sdk/python/dist/*.tar.gz',
244 'sdk/cwl/dist/*.tar.gz',
245 'services/fuse/dist/*.tar.gz',
247 'gems': _define_suite(GemPackageSuite,
250 'services/login-sync/*.gem',
254 def parse_arguments(arguments):
255 parser = argparse.ArgumentParser(
256 description="Upload Arvados packages to various repositories")
258 '--workspace', '-W', default=os.environ.get('WORKSPACE'),
259 help="Arvados source directory with built packages to upload")
262 help="Host specification for distribution repository server")
263 parser.add_argument('-o', action='append', default=[], dest='ssh_opts',
264 metavar='OPTION', help="Pass option to `ssh -o`")
265 parser.add_argument('--verbose', '-v', action='count', default=0,
266 help="Log more information and subcommand output")
268 '--repo', choices=['dev', 'testing'],
269 help="Whether to upload to dev (nightly) or testing (release candidate) repository")
272 'targets', nargs='*', default=['all'], metavar='target',
273 help="Upload packages to these targets (default all)\nAvailable targets: " +
274 ', '.join(sorted(PACKAGE_SUITES.keys())))
275 args = parser.parse_args(arguments)
276 if 'all' in args.targets:
277 args.targets = list(PACKAGE_SUITES.keys())
279 if args.workspace is None:
280 parser.error("workspace not set from command line or environment")
282 for target in ['debian8', 'debian9', 'debian10', 'ubuntu1404', 'ubuntu1604', 'ubuntu1804']:
283 PACKAGE_SUITES[target] = _define_suite(
284 DebianPackageSuite, os.path.join('packages', target, '*.deb'),
285 target=target, repo=args.repo)
286 for target in ['centos7']:
287 PACKAGE_SUITES[target] = _define_suite(
288 RedHatPackageSuite, os.path.join('packages', target, '*.rpm'),
289 target=target, repo=args.repo)
291 for target in args.targets:
293 suite_class = PACKAGE_SUITES[target].func
295 parser.error("unrecognized target {!r}".format(target))
296 if suite_class.NEED_SSH and (args.ssh_host is None):
298 "--ssh-host must be specified to upload distribution packages")
301 def setup_logger(stream_dest, args):
302 log_handler = logging.StreamHandler(stream_dest)
303 log_handler.setFormatter(logging.Formatter(
304 '%(asctime)s %(name)s[%(process)d] %(levelname)s: %(message)s',
305 '%Y-%m-%d %H:%M:%S'))
306 logger = logging.getLogger('arvados-dev.upload')
307 logger.addHandler(log_handler)
308 logger.setLevel(max(1, logging.WARNING - (10 * args.verbose)))
310 def build_suite_and_upload(target, since_timestamp, args):
311 suite_def = PACKAGE_SUITES[target]
313 if suite_def.func.NEED_SSH:
314 kwargs.update(ssh_host=args.ssh_host, ssh_opts=args.ssh_opts)
315 suite = suite_def(args.workspace, **kwargs)
316 suite.update_packages(since_timestamp)
318 def main(arguments, stdout=sys.stdout, stderr=sys.stderr):
319 args = parse_arguments(arguments)
320 setup_logger(stderr, args)
322 for target in args.targets:
323 ts_file = TimestampFile(os.path.join(args.workspace, 'packages',
324 '.last_upload_%s' % target))
325 last_upload_ts = ts_file.last_upload()
326 build_suite_and_upload(target, last_upload_ts, args)
329 if __name__ == '__main__':