3 # Copyright (C) The Arvados Authors. All rights reserved.
5 # SPDX-License-Identifier: AGPL-3.0
20 def run_and_grep(cmd, read_output, *regexps,
21 encoding=locale.getpreferredencoding(), **popen_kwargs):
22 """Run a subprocess and capture output lines matching regexps.
25 * cmd: The command to run, as a list or string, as for subprocess.Popen.
26 * read_output: 'stdout' or 'stderr', the name of the output stream to read.
27 Remaining arguments are regexps to match output, as strings or compiled
28 regexp objects. Output lines matching any regexp will be captured.
31 * encoding: The encoding used to decode the subprocess output.
32 Remaining keyword arguments are passed directly to subprocess.Popen.
34 Returns 2-tuple (subprocess returncode, list of matched output lines).
36 regexps = [regexp if hasattr(regexp, 'search') else re.compile(regexp)
37 for regexp in regexps]
38 popen_kwargs[read_output] = subprocess.PIPE
39 proc = subprocess.Popen(cmd, **popen_kwargs)
40 with open(getattr(proc, read_output).fileno(), encoding=encoding) as output:
41 matched_lines = [line for line in output
42 if any(regexp.search(line) for regexp in regexps)]
43 return proc.wait(), matched_lines
47 def __init__(self, path):
49 self.start_time = time.time()
51 def last_upload(self):
53 return os.path.getmtime(self.path)
54 except EnvironmentError:
58 os.close(os.open(self.path, os.O_CREAT | os.O_APPEND))
59 os.utime(self.path, (time.time(), self.start_time))
65 def __init__(self, glob_root, rel_globs):
66 logger_part = getattr(self, 'LOGGER_PART', os.path.basename(glob_root))
67 self.logger = logging.getLogger('arvados-dev.upload.' + logger_part)
68 self.globs = [os.path.join(glob_root, rel_glob)
69 for rel_glob in rel_globs]
71 def files_to_upload(self, since_timestamp):
72 for abs_glob in self.globs:
73 for path in glob.glob(abs_glob):
74 if os.path.getmtime(path) >= since_timestamp:
77 def upload_file(self, path):
78 raise NotImplementedError("PackageSuite.upload_file")
80 def upload_files(self, paths):
82 self.logger.info("Uploading %s", path)
83 self.upload_file(path)
85 def post_uploads(self, paths):
88 def update_packages(self, since_timestamp):
89 upload_paths = list(self.files_to_upload(since_timestamp))
91 self.upload_files(upload_paths)
92 self.post_uploads(upload_paths)
95 class PythonPackageSuite(PackageSuite):
96 LOGGER_PART = 'python'
97 REUPLOAD_REGEXP = re.compile(
98 r'^error: Upload failed \(400\): A file named "[^"]+" already exists\b')
100 def __init__(self, glob_root, rel_globs):
101 super().__init__(glob_root, rel_globs)
102 self.seen_packages = set()
104 def upload_file(self, path):
105 src_dir = os.path.dirname(os.path.dirname(path))
106 if src_dir in self.seen_packages:
108 self.seen_packages.add(src_dir)
109 # NOTE: If we ever start uploading Python 3 packages, we'll need to
110 # figure out some way to adapt cmd to match. It might be easiest
111 # to give all our setup.py files the executable bit, and run that
113 # We also must run `sdist` before `upload`: `upload` uploads any
114 # distributions previously generated in the command. It doesn't
115 # know how to upload distributions already on disk. We write the
116 # result to a dedicated directory to avoid interfering with our
117 # timestamp tracking.
118 cmd = ['python2.7', 'setup.py']
119 if not self.logger.isEnabledFor(logging.INFO):
120 cmd.append('--quiet')
121 cmd.extend(['sdist', '--dist-dir', '.upload_dist', 'upload'])
122 upload_returncode, repushed = run_and_grep(
123 cmd, 'stderr', self.REUPLOAD_REGEXP, cwd=src_dir)
124 if (upload_returncode != 0) and not repushed:
125 raise subprocess.CalledProcessError(upload_returncode, cmd)
126 shutil.rmtree(os.path.join(src_dir, '.upload_dist'))
129 class GemPackageSuite(PackageSuite):
131 REUPLOAD_REGEXP = re.compile(r'^Repushing of gem versions is not allowed\.$')
133 def upload_file(self, path):
134 cmd = ['gem', 'push', path]
135 push_returncode, repushed = run_and_grep(cmd, 'stdout', self.REUPLOAD_REGEXP)
136 if (push_returncode != 0) and not repushed:
137 raise subprocess.CalledProcessError(push_returncode, cmd)
140 class DistroPackageSuite(PackageSuite):
142 REMOTE_DEST_DIR = 'tmp'
144 def __init__(self, glob_root, rel_globs, target, ssh_host, ssh_opts):
145 super().__init__(glob_root, rel_globs)
147 self.ssh_host = ssh_host
148 self.ssh_opts = ['-o' + opt for opt in ssh_opts]
149 if not self.logger.isEnabledFor(logging.INFO):
150 self.ssh_opts.append('-q')
152 def _build_cmd(self, base_cmd, *args):
154 cmd.extend(self.ssh_opts)
158 def _paths_basenames(self, paths):
159 return (os.path.basename(path) for path in paths)
161 def _run_script(self, script, *args):
162 # SSH will use a shell to run our bash command, so we have to
163 # quote our arguments.
164 # self.__class__.__name__ provides $0 for the script, which makes a
165 # nicer message if there's an error.
166 subprocess.check_call(self._build_cmd(
167 'ssh', self.ssh_host, 'bash', '-ec', pipes.quote(script),
168 self.__class__.__name__, *(pipes.quote(s) for s in args)))
170 def upload_files(self, paths):
171 dest_dir = os.path.join(self.REMOTE_DEST_DIR, self.target)
172 mkdir = self._build_cmd('ssh', self.ssh_host, 'install', '-d', dest_dir)
173 subprocess.check_call(mkdir)
174 cmd = self._build_cmd('scp', *paths)
175 cmd.append('{}:{}'.format(self.ssh_host, dest_dir))
176 subprocess.check_call(cmd)
179 class DebianPackageSuite(DistroPackageSuite):
183 freight add "$@" "apt/$DISTNAME"
184 freight cache "apt/$DISTNAME"
188 'debian8': 'jessie-dev',
189 'ubuntu1204': 'precise-dev',
190 'ubuntu1404': 'trusty-dev',
191 'ubuntu1604': 'xenial-dev',
194 def post_uploads(self, paths):
195 self._run_script(self.FREIGHT_SCRIPT, self.REMOTE_DEST_DIR + '/' + self.target,
196 self.TARGET_DISTNAMES[self.target],
197 *self._paths_basenames(paths))
200 class RedHatPackageSuite(DistroPackageSuite):
201 CREATEREPO_SCRIPT = """
204 rpmsign --addsign "$@" </dev/null
206 createrepo "$REPODIR"
208 REPO_ROOT = '/var/www/rpm.arvados.org/'
210 'centos7-dev': 'CentOS/7/dev/x86_64/',
213 def post_uploads(self, paths):
214 repo_dir = os.path.join(self.REPO_ROOT,
215 self.TARGET_REPODIRS[self.target])
216 self._run_script(self.CREATEREPO_SCRIPT, self.REMOTE_DEST_DIR + '/' + self.target,
217 repo_dir, *self._paths_basenames(paths))
220 def _define_suite(suite_class, *rel_globs, **kwargs):
221 return functools.partial(suite_class, rel_globs=rel_globs, **kwargs)
224 'python': _define_suite(PythonPackageSuite,
225 'sdk/pam/dist/*.tar.gz',
226 'sdk/python/dist/*.tar.gz',
227 'sdk/cwl/dist/*.tar.gz',
228 'services/nodemanager/dist/*.tar.gz',
229 'services/fuse/dist/*.tar.gz',
231 'gems': _define_suite(GemPackageSuite,
234 'services/login-sync/*.gem',
237 for target in ['debian8', 'ubuntu1204', 'ubuntu1404', 'ubuntu1604']:
238 PACKAGE_SUITES[target] = _define_suite(
239 DebianPackageSuite, os.path.join('packages', target, '*.deb'),
241 for target in ['centos7']:
242 PACKAGE_SUITES[target] = _define_suite(
243 RedHatPackageSuite, os.path.join('packages', target, '*.rpm'),
246 def parse_arguments(arguments):
247 parser = argparse.ArgumentParser(
248 prog="run_upload_packages.py",
249 description="Upload Arvados packages to various repositories")
251 '--workspace', '-W', default=os.environ.get('WORKSPACE'),
252 help="Arvados source directory with built packages to upload")
255 help="Host specification for distribution repository server")
256 parser.add_argument('-o', action='append', default=[], dest='ssh_opts',
257 metavar='OPTION', help="Pass option to `ssh -o`")
258 parser.add_argument('--verbose', '-v', action='count', default=0,
259 help="Log more information and subcommand output")
261 'targets', nargs='*', default=['all'], metavar='target',
262 help="Upload packages to these targets (default all)\nAvailable targets: " +
263 ', '.join(sorted(PACKAGE_SUITES.keys())))
264 args = parser.parse_args(arguments)
265 if 'all' in args.targets:
266 args.targets = list(PACKAGE_SUITES.keys())
268 if args.workspace is None:
269 parser.error("workspace not set from command line or environment")
270 for target in args.targets:
272 suite_class = PACKAGE_SUITES[target].func
274 parser.error("unrecognized target {!r}".format(target))
275 if suite_class.NEED_SSH and (args.ssh_host is None):
277 "--ssh-host must be specified to upload distribution packages")
280 def setup_logger(stream_dest, args):
281 log_handler = logging.StreamHandler(stream_dest)
282 log_handler.setFormatter(logging.Formatter(
283 '%(asctime)s %(name)s[%(process)d] %(levelname)s: %(message)s',
284 '%Y-%m-%d %H:%M:%S'))
285 logger = logging.getLogger('arvados-dev.upload')
286 logger.addHandler(log_handler)
287 logger.setLevel(max(1, logging.WARNING - (10 * args.verbose)))
289 def build_suite_and_upload(target, since_timestamp, args):
290 suite_def = PACKAGE_SUITES[target]
292 if suite_def.func.NEED_SSH:
293 kwargs.update(ssh_host=args.ssh_host, ssh_opts=args.ssh_opts)
294 suite = suite_def(args.workspace, **kwargs)
295 suite.update_packages(since_timestamp)
297 def main(arguments, stdout=sys.stdout, stderr=sys.stderr):
298 args = parse_arguments(arguments)
299 setup_logger(stderr, args)
300 for target in args.targets:
301 ts_file = TimestampFile(os.path.join(args.workspace, 'packages',
302 '.last_upload_%s' % target))
303 last_upload_ts = ts_file.last_upload()
304 build_suite_and_upload(target, last_upload_ts, args)
307 if __name__ == '__main__':