From: Ward Vandewege <ward@curii.com>
Date: Mon, 4 Jan 2021 18:11:33 +0000 (-0500)
Subject: Add packaging-move-dev-to-attic.py, which now supports aptly. This is a
X-Git-Url: https://git.arvados.org/arvados-dev.git/commitdiff_plain/ab8f5649992b09908290ef624f3414023ded5c30

Add packaging-move-dev-to-attic.py, which now supports aptly. This is a
modified version of an old internal script that supported freight.

refs #17219

Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward@curii.com>
---

diff --git a/jenkins/packaging-move-dev-to-attic.py b/jenkins/packaging-move-dev-to-attic.py
new file mode 100755
index 0000000..5dac3c0
--- /dev/null
+++ b/jenkins/packaging-move-dev-to-attic.py
@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+import argparse
+import datetime
+import os
+import subprocess
+import pprint
+import re
+
+class DebugExecutor:
+  def __init__(self, package_list):
+    self.package_list = package_list
+
+  def do_it(self):
+    for a in self.package_list:
+      print (a[2])
+
+class MoveExecutor:
+  def __init__(self, distro, package_list):
+    self.package_list = package_list
+    self.distro = distro
+
+  def move_it(self):
+    for a in self.package_list:
+      if a[2]:
+        source = a[0]
+        destination = source.replace('dev','attic')
+        os.makedirs(os.path.dirname(destination), exist_ok=True)
+        print ("Moving " + a[0] + " to " + destination)
+        f = os.path.basename(os.path.splitext(a[0])[0])
+        output = subprocess.getoutput("aptly repo move " + source + " " + destination)
+        print(output)
+
+  def update_it(self):
+    distroBase = re.sub('-.*$', '', self.distro)
+    output = subprocess.getoutput("aptly publish update " + distroBase + "-dev filesystem:" + distroBase + ":")
+    print(output)
+    output = subprocess.getoutput("aptly publish update " + distroBase + "-attic filesystem:" + distroBase + ":")
+    print(output)
+
+class CollectPackageName:
+  def __init__(self, cache_dir, distro, min_packages,  cutoff_date):
+    self.cache_dir = cache_dir
+    self.distro = distro
+    self.min_packages = min_packages
+    self.cutoff_date_unixepoch = int(cutoff_date.strftime('%s'))
+
+  def collect_packages(self):
+    distroBase = re.sub('-.*$', '', self.distro)
+    directory=os.path.join(self.cache_dir,distroBase,'pool/main')
+
+    ## rtn will have 4 element tuple: package_name, the path, the creation time for sorting, and if it's a candidate for deletion
+    rtn = []
+
+    # Get the list of packages in the repo
+    output = subprocess.getoutput("aptly repo search " + self.distro)
+    for f in output.splitlines():
+      pkg = f.split('_')[0]
+      # This is nasty and slow, but aptly doesn't seem to have a way to provide
+      # the on-disk path for a package in its repository. We also can't query
+      # for the list of packages that fit the cutoff date constraint with a
+      # 'package-query' parameter: the 'Date' field would be appropriate for
+      # that, but it's not populated for our packages because we don't ship a
+      # changelog with them (that's where the 'Date' field comes from, as per
+      # the Debian policy manual, cf.
+      # https://www.debian.org/doc/debian-policy/ch-controlfields.html#s-f-date).
+      the_file = subprocess.getoutput("find " + directory + " -name " + f + ".deb")
+      if the_file == "":
+          print("WARNING: skipping package, could not find file for package " + f + " under directory " + directory)
+          continue
+      rtn.append ( (pkg, the_file,
+                    os.path.getmtime(the_file),
+                    os.path.getmtime(the_file) < self.cutoff_date_unixepoch) )
+    return self.collect_candidates_excluding_N_last(rtn)
+
+  def collect_candidates_excluding_N_last(self, tuples_with_packages):
+    return_value = []
+
+    ## separate all file into packages. (use the first element in the tuple for this)
+    dictionary_per_package  = {}
+    for x in tuples_with_packages:
+      dictionary_per_package.setdefault(x[0], []).append(x[1:])
+
+    for pkg_name, metadata in dictionary_per_package.items():
+      candidates_local_copy = metadata[:]
+
+      ## order them by date
+      candidates_local_copy.sort(key=lambda tup: tup[1])
+
+      return_value.extend(candidates_local_copy[:-self.min_packages])
+
+    return return_value
+
+parser = argparse.ArgumentParser(description='List the packages to delete.')
+parser.add_argument('distro', choices=['bionic-dev','jessie-dev','precise-dev','stretch-dev','trusty-dev','wheezy-dev','xenial-dev','buster-dev'],
+                    help='distro to do the clean up')
+parser.add_argument('--repo_dir',
+                    default='/var/www/aptly_public/',
+                    help='parent directory of the aptly repositories (default:  %(default)s)')
+parser.add_argument('--min_packages', type=int,
+                    default=5,
+                    help='minimum amount of packages to leave in the repo (default:  %(default)s)')
+parser.add_argument('--cutoff_date', type=lambda s: datetime.datetime.strptime(s, '%Y-%m-%d'),
+                    default='2017-06-31',
+                    help='date to cut-off in format YYYY-MM-DD (default:  %(default)s)')
+
+args = parser.parse_args()
+
+
+p = CollectPackageName(args.repo_dir, args.distro, args.min_packages,  args.cutoff_date)
+
+#executor = DebugExecutor(p.collect_packages())
+executor = MoveExecutor(args.distro, p.collect_packages())
+
+executor.move_it()
+executor.update_it()
+