3 # Copyright (C) The Arvados Authors. All rights reserved.
5 # SPDX-License-Identifier: AGPL-3.0
15 def __init__(self, package_list):
16 self.package_list = package_list
19 for a in self.package_list:
23 def __init__(self, distro, package_list):
24 self.package_list = package_list
28 for a in self.package_list:
31 destination = source.replace('dev','attic')
32 os.makedirs(os.path.dirname(destination), exist_ok=True)
33 print ("Moving " + a[0] + " to " + destination)
34 f = os.path.basename(os.path.splitext(a[0])[0])
35 output = subprocess.getoutput("aptly repo move " + source + " " + destination)
39 distroBase = re.sub('-.*$', '', self.distro)
40 output = subprocess.getoutput("aptly publish update " + distroBase + "-dev filesystem:" + distroBase + ":")
42 output = subprocess.getoutput("aptly publish update " + distroBase + "-attic filesystem:" + distroBase + ":")
45 class CollectPackageName:
46 def __init__(self, cache_dir, distro, min_packages, cutoff_date):
47 self.cache_dir = cache_dir
49 self.min_packages = min_packages
50 self.cutoff_date_unixepoch = int(cutoff_date.strftime('%s'))
52 def collect_packages(self):
53 distroBase = re.sub('-.*$', '', self.distro)
54 directory=os.path.join(self.cache_dir,distroBase,'pool/main')
56 ## rtn will have 4 element tuple: package_name, the path, the creation time for sorting, and if it's a candidate for deletion
59 # Get the list of packages in the repo
60 output = subprocess.getoutput("aptly repo search " + self.distro)
61 for f in output.splitlines():
63 # This is nasty and slow, but aptly doesn't seem to have a way to provide
64 # the on-disk path for a package in its repository. We also can't query
65 # for the list of packages that fit the cutoff date constraint with a
66 # 'package-query' parameter: the 'Date' field would be appropriate for
67 # that, but it's not populated for our packages because we don't ship a
68 # changelog with them (that's where the 'Date' field comes from, as per
69 # the Debian policy manual, cf.
70 # https://www.debian.org/doc/debian-policy/ch-controlfields.html#s-f-date).
71 the_file = subprocess.getoutput("find " + directory + " -name " + f + ".deb")
73 print("WARNING: skipping package, could not find file for package " + f + " under directory " + directory)
75 rtn.append ( (pkg, the_file,
76 os.path.getmtime(the_file),
77 os.path.getmtime(the_file) < self.cutoff_date_unixepoch) )
78 return self.collect_candidates_excluding_N_last(rtn)
80 def collect_candidates_excluding_N_last(self, tuples_with_packages):
83 ## separate all file into packages. (use the first element in the tuple for this)
84 dictionary_per_package = {}
85 for x in tuples_with_packages:
86 dictionary_per_package.setdefault(x[0], []).append(x[1:])
88 for pkg_name, metadata in dictionary_per_package.items():
89 candidates_local_copy = metadata[:]
92 candidates_local_copy.sort(key=lambda tup: tup[1])
94 return_value.extend(candidates_local_copy[:-self.min_packages])
99 if re.fullmatch(r'.*-dev', astring) == None:
103 parser = argparse.ArgumentParser(description='List the packages to delete.')
104 parser.add_argument('distro',
106 help='distro to process, must be a dev repository, e.g. buster-dev')
107 parser.add_argument('--repo_dir',
108 default='/var/www/aptly_public/',
109 help='parent directory of the aptly repositories (default: %(default)s)')
110 parser.add_argument('--min_packages', type=int,
112 help='minimum amount of packages to leave in the repo (default: %(default)s)')
113 parser.add_argument('--cutoff_date', type=lambda s: datetime.datetime.strptime(s, '%Y-%m-%d'),
114 default='2017-06-31',
115 help='date to cut-off in format YYYY-MM-DD (default: %(default)s)')
117 args = parser.parse_args()
120 p = CollectPackageName(args.repo_dir, args.distro, args.min_packages, args.cutoff_date)
122 #executor = DebugExecutor(p.collect_packages())
123 executor = MoveExecutor(args.distro, p.collect_packages())