3 # Copyright (C) The Arvados Authors. All rights reserved.
5 # SPDX-License-Identifier: AGPL-3.0
20 def run_and_grep(cmd, read_output, *regexps,
21 encoding=locale.getpreferredencoding(), **popen_kwargs):
22 """Run a subprocess and capture output lines matching regexps.
25 * cmd: The command to run, as a list or string, as for subprocess.Popen.
26 * read_output: 'stdout' or 'stderr', the name of the output stream to read.
27 Remaining arguments are regexps to match output, as strings or compiled
28 regexp objects. Output lines matching any regexp will be captured.
31 * encoding: The encoding used to decode the subprocess output.
32 Remaining keyword arguments are passed directly to subprocess.Popen.
34 Returns 2-tuple (subprocess returncode, list of matched output lines).
36 regexps = [regexp if hasattr(regexp, 'search') else re.compile(regexp)
37 for regexp in regexps]
38 popen_kwargs[read_output] = subprocess.PIPE
39 proc = subprocess.Popen(cmd, **popen_kwargs)
40 with open(getattr(proc, read_output).fileno(), encoding=encoding) as output:
41 matched_lines = [line for line in output
42 if any(regexp.search(line) for regexp in regexps)]
43 return proc.wait(), matched_lines
47 def __init__(self, path):
49 self.start_time = time.time()
51 def last_upload(self):
53 return os.path.getmtime(self.path)
54 except EnvironmentError:
58 os.close(os.open(self.path, os.O_CREAT | os.O_APPEND))
59 os.utime(self.path, (time.time(), self.start_time))
65 def __init__(self, glob_root, rel_globs):
66 logger_part = getattr(self, 'LOGGER_PART', os.path.basename(glob_root))
67 self.logger = logging.getLogger('arvados-dev.upload.' + logger_part)
68 self.globs = [os.path.join(glob_root, rel_glob)
69 for rel_glob in rel_globs]
71 def files_to_upload(self, since_timestamp):
72 for abs_glob in self.globs:
73 for path in glob.glob(abs_glob):
74 if os.path.getmtime(path) >= since_timestamp:
77 def upload_file(self, path):
78 raise NotImplementedError("PackageSuite.upload_file")
80 def upload_files(self, paths):
82 self.logger.info("Uploading %s", path)
83 self.upload_file(path)
85 def post_uploads(self, paths):
88 def update_packages(self, since_timestamp):
89 upload_paths = list(self.files_to_upload(since_timestamp))
91 self.upload_files(upload_paths)
92 self.post_uploads(upload_paths)
95 class PythonPackageSuite(PackageSuite):
96 LOGGER_PART = 'python'
97 REUPLOAD_REGEXP = re.compile(
98 r'^error: Upload failed \(400\): A file named "[^"]+" already exists\b')
100 def __init__(self, glob_root, rel_globs):
101 super().__init__(glob_root, rel_globs)
102 self.seen_packages = set()
104 def upload_file(self, path):
105 src_dir = os.path.dirname(os.path.dirname(path))
106 if src_dir in self.seen_packages:
108 self.seen_packages.add(src_dir)
109 # NOTE: If we ever start uploading Python 3 packages, we'll need to
110 # figure out some way to adapt cmd to match. It might be easiest
111 # to give all our setup.py files the executable bit, and run that
113 # We also must run `sdist` before `upload`: `upload` uploads any
114 # distributions previously generated in the command. It doesn't
115 # know how to upload distributions already on disk. We write the
116 # result to a dedicated directory to avoid interfering with our
117 # timestamp tracking.
118 cmd = ['python2.7', 'setup.py']
119 if not self.logger.isEnabledFor(logging.INFO):
120 cmd.append('--quiet')
121 cmd.extend(['sdist', '--dist-dir', '.upload_dist', 'upload'])
122 upload_returncode, repushed = run_and_grep(
123 cmd, 'stderr', self.REUPLOAD_REGEXP, cwd=src_dir)
124 if (upload_returncode != 0) and not repushed:
125 raise subprocess.CalledProcessError(upload_returncode, cmd)
126 shutil.rmtree(os.path.join(src_dir, '.upload_dist'))
129 class GemPackageSuite(PackageSuite):
131 REUPLOAD_REGEXP = re.compile(r'^Repushing of gem versions is not allowed\.$')
133 def upload_file(self, path):
134 cmd = ['gem', 'push', path]
135 push_returncode, repushed = run_and_grep(cmd, 'stdout', self.REUPLOAD_REGEXP)
136 if (push_returncode != 0) and not repushed:
137 raise subprocess.CalledProcessError(push_returncode, cmd)
140 class DistroPackageSuite(PackageSuite):
142 REMOTE_DEST_DIR = 'tmp'
144 def __init__(self, glob_root, rel_globs, target, ssh_host, ssh_opts):
145 super().__init__(glob_root, rel_globs)
147 self.ssh_host = ssh_host
148 self.ssh_opts = ['-o' + opt for opt in ssh_opts]
149 if not self.logger.isEnabledFor(logging.INFO):
150 self.ssh_opts.append('-q')
152 def _build_cmd(self, base_cmd, *args):
154 cmd.extend(self.ssh_opts)
158 def _paths_basenames(self, paths):
159 return (os.path.basename(path) for path in paths)
161 def _run_script(self, script, *args):
162 # SSH will use a shell to run our bash command, so we have to
163 # quote our arguments.
164 # self.__class__.__name__ provides $0 for the script, which makes a
165 # nicer message if there's an error.
166 subprocess.check_call(self._build_cmd(
167 'ssh', self.ssh_host, 'bash', '-ec', pipes.quote(script),
168 self.__class__.__name__, *(pipes.quote(s) for s in args)))
170 def upload_files(self, paths):
171 dest_dir = os.path.join(self.REMOTE_DEST_DIR, self.target)
172 mkdir = self._build_cmd('ssh', self.ssh_host, 'install', '-d', dest_dir)
173 subprocess.check_call(mkdir)
174 cmd = self._build_cmd('scp', *paths)
175 cmd.append('{}:{}'.format(self.ssh_host, dest_dir))
176 subprocess.check_call(cmd)
179 class DebianPackageSuite(DistroPackageSuite):
183 freight add "$@" "apt/$DISTNAME"
184 freight cache "apt/$DISTNAME"
188 'debian8': 'jessie-dev',
189 'debian9': 'stretch-dev',
190 'ubuntu1204': 'precise-dev',
191 'ubuntu1404': 'trusty-dev',
192 'ubuntu1604': 'xenial-dev',
195 def post_uploads(self, paths):
196 self._run_script(self.FREIGHT_SCRIPT, self.REMOTE_DEST_DIR + '/' + self.target,
197 self.TARGET_DISTNAMES[self.target],
198 *self._paths_basenames(paths))
201 class RedHatPackageSuite(DistroPackageSuite):
202 CREATEREPO_SCRIPT = """
205 rpmsign --addsign "$@" </dev/null
207 createrepo "$REPODIR"
209 REPO_ROOT = '/var/www/rpm.arvados.org/'
211 'centos7': 'CentOS/7/os/x86_64/',
214 def post_uploads(self, paths):
215 repo_dir = os.path.join(self.REPO_ROOT,
216 self.TARGET_REPODIRS[self.target])
217 self._run_script(self.CREATEREPO_SCRIPT, self.REMOTE_DEST_DIR + '/' + self.target,
218 repo_dir, *self._paths_basenames(paths))
221 def _define_suite(suite_class, *rel_globs, **kwargs):
222 return functools.partial(suite_class, rel_globs=rel_globs, **kwargs)
225 'python': _define_suite(PythonPackageSuite,
226 'sdk/pam/dist/*.tar.gz',
227 'sdk/python/dist/*.tar.gz',
228 'sdk/cwl/dist/*.tar.gz',
229 'services/nodemanager/dist/*.tar.gz',
230 'services/fuse/dist/*.tar.gz',
232 'gems': _define_suite(GemPackageSuite,
235 'services/login-sync/*.gem',
238 for target in ['debian8', 'ubuntu1204', 'ubuntu1404', 'ubuntu1604']:
239 PACKAGE_SUITES[target] = _define_suite(
240 DebianPackageSuite, os.path.join('packages', target, '*.deb'),
242 for target in ['centos7']:
243 PACKAGE_SUITES[target] = _define_suite(
244 RedHatPackageSuite, os.path.join('packages', target, '*.rpm'),
247 def parse_arguments(arguments):
248 parser = argparse.ArgumentParser(
249 prog="run_upload_packages.py",
250 description="Upload Arvados packages to various repositories")
252 '--workspace', '-W', default=os.environ.get('WORKSPACE'),
253 help="Arvados source directory with built packages to upload")
256 help="Host specification for distribution repository server")
257 parser.add_argument('-o', action='append', default=[], dest='ssh_opts',
258 metavar='OPTION', help="Pass option to `ssh -o`")
259 parser.add_argument('--verbose', '-v', action='count', default=0,
260 help="Log more information and subcommand output")
262 'targets', nargs='*', default=['all'], metavar='target',
263 help="Upload packages to these targets (default all)\nAvailable targets: " +
264 ', '.join(sorted(PACKAGE_SUITES.keys())))
265 args = parser.parse_args(arguments)
266 if 'all' in args.targets:
267 args.targets = list(PACKAGE_SUITES.keys())
269 if args.workspace is None:
270 parser.error("workspace not set from command line or environment")
271 for target in args.targets:
273 suite_class = PACKAGE_SUITES[target].func
275 parser.error("unrecognized target {!r}".format(target))
276 if suite_class.NEED_SSH and (args.ssh_host is None):
278 "--ssh-host must be specified to upload distribution packages")
281 def setup_logger(stream_dest, args):
282 log_handler = logging.StreamHandler(stream_dest)
283 log_handler.setFormatter(logging.Formatter(
284 '%(asctime)s %(name)s[%(process)d] %(levelname)s: %(message)s',
285 '%Y-%m-%d %H:%M:%S'))
286 logger = logging.getLogger('arvados-dev.upload')
287 logger.addHandler(log_handler)
288 logger.setLevel(max(1, logging.WARNING - (10 * args.verbose)))
290 def build_suite_and_upload(target, since_timestamp, args):
291 suite_def = PACKAGE_SUITES[target]
293 if suite_def.func.NEED_SSH:
294 kwargs.update(ssh_host=args.ssh_host, ssh_opts=args.ssh_opts)
295 suite = suite_def(args.workspace, **kwargs)
296 suite.update_packages(since_timestamp)
298 def main(arguments, stdout=sys.stdout, stderr=sys.stderr):
299 args = parse_arguments(arguments)
300 setup_logger(stderr, args)
301 for target in args.targets:
302 ts_file = TimestampFile(os.path.join(args.workspace, 'packages',
303 '.last_upload_%s' % target))
304 last_upload_ts = ts_file.last_upload()
305 build_suite_and_upload(target, last_upload_ts, args)
308 if __name__ == '__main__':