# SPDX-License-Identifier: AGPL-3.0
import argparse
+import errno
import functools
import glob
import locale
class TimestampFile:
def __init__(self, path):
self.path = path
+ # Make sure the dirname for `path` exists
+ p = os.path.dirname(path)
+ try:
+ os.makedirs(p)
+ except OSError as exc:
+ if exc.errno == errno.EEXIST and os.path.isdir(p):
+ pass
+ else:
+ raise
self.start_time = time.time()
def last_upload(self):
return -1
def update(self):
- os.close(os.open(self.path, os.O_CREAT | os.O_APPEND))
- os.utime(self.path, (time.time(), self.start_time))
-
+ try:
+ os.close(os.open(self.path, os.O_CREAT | os.O_APPEND))
+ os.utime(self.path, (time.time(), self.start_time))
+ except:
+ # when the packages directory is created/populated by a build in a
+ # docker container, as root, the script that runs the upload
+ # doesn't always have permission to touch a timestamp file there.
+ # In production, we build/upload from ephemeral machines, which
+ # means that the timestamp mechanism is not used. We print a
+ # warning and move on without erroring out.
+ print("Warning: unable to update timestamp file",self.path,"permission problem?")
+ pass
class PackageSuite:
NEED_SSH = False
if src_dir in self.seen_packages:
return
self.seen_packages.add(src_dir)
- # NOTE: If we ever start uploading Python 3 packages, we'll need to
- # figure out some way to adapt cmd to match. It might be easiest
- # to give all our setup.py files the executable bit, and run that
- # directly.
# We also must run `sdist` before `upload`: `upload` uploads any
# distributions previously generated in the command. It doesn't
# know how to upload distributions already on disk. We write the
# result to a dedicated directory to avoid interfering with our
# timestamp tracking.
- cmd = ['python2.7', 'setup.py']
+ cmd = ['python3', 'setup.py']
if not self.logger.isEnabledFor(logging.INFO):
cmd.append('--quiet')
- cmd.extend(['sdist', '--dist-dir', '.upload_dist', 'upload'])
+ cmd.extend(['bdist_wheel', '--dist-dir', '.upload_dist'])
+ cmd.extend(['sdist', '--dist-dir', '.upload_dist'])
+ cmd.extend(['upload'])
upload_returncode, repushed = run_and_grep(
cmd, 'stderr', *self.REUPLOAD_REGEXPS, cwd=src_dir)
if (upload_returncode != 0) and not repushed:
class DebianPackageSuite(DistroPackageSuite):
- FREIGHT_SCRIPT = """
+ APT_SCRIPT = """
cd "$1"; shift
DISTNAME=$1; shift
-freight add "$@" "apt/$DISTNAME"
-freight cache "apt/$DISTNAME"
-rm "$@"
+for package in "$@"; do
+ set +e
+ aptly repo search "$DISTNAME" "${package%.deb}" >/dev/null 2>&1
+ RET=$?
+ set -e
+ if [[ $RET -eq 0 ]]; then
+ echo "Not adding $package, it is already present in repo $DISTNAME"
+ rm "$package"
+ else
+ aptly repo add -remove-files "$DISTNAME" "$package"
+ fi
+done
+aptly publish update "$DISTNAME" filesystem:"${DISTNAME%-*}":
"""
def __init__(self, glob_root, rel_globs, target, ssh_host, ssh_opts, repo):
}
def post_uploads(self, paths):
- self._run_script(self.FREIGHT_SCRIPT, self.REMOTE_DEST_DIR + '/' + self.target,
+ self._run_script(self.APT_SCRIPT, self.REMOTE_DEST_DIR + '/' + self.target,
self.TARGET_DISTNAMES[self.target],
*self._paths_basenames(paths))
REPODIR=$1; shift
rpmsign --addsign "$@" </dev/null
mv "$@" "$REPODIR"
-createrepo "$REPODIR"
+createrepo -c ~/.createrepo-cache --update "$REPODIR"
"""
REPO_ROOT = '/var/www/rpm.arvados.org/'