tools/keep-xref/keep-xref.py

   1 #!/usr/bin/env python3
   2 #
   3 # Copyright (C) The Arvados Authors. All rights reserved.
   4 #
   5 # SPDX-License-Identifier: AGPL-3.0
   6 #
   7
   8 import argparse
   9 import arvados
  10 import arvados.util
  11 import csv
  12 import sys
  13 import logging
  14
  15 lglvl = logging.INFO+1
  16 logging.basicConfig(level=lglvl, format='%(message)s')
  17
  18 """
  19  Given a list of collections missing blocks (as produced by
  20 keep-balance), produce a report listing affected collections and
  21 container requests.
  22 """
  23
  24 def rerun_request(arv, container_requests_to_rerun, ct):
  25     requests = arvados.util.list_all(arv.container_requests().list, filters=[["container_uuid", "=", ct["uuid"]]])
  26     for cr in requests:
  27         if cr["requesting_container_uuid"]:
  28             rerun_request(arv, container_requests_to_rerun, arv.containers().get(uuid=cr["requesting_container_uuid"]).execute())
  29         else:
  30             container_requests_to_rerun[cr["uuid"]] = cr
  31
  32 def get_owner(arv, owners, record):
  33     uuid = record["owner_uuid"]
  34     if uuid not in owners:
  35         if uuid[6:11] == "tpzed":
  36             owners[uuid] = (arv.users().get(uuid=uuid).execute()["full_name"], uuid)
  37         else:
  38             grp = arv.groups().get(uuid=uuid).execute()
  39             _, ou = get_owner(arv, owners, grp)
  40             owners[uuid] = (grp["name"], ou)
  41     return owners[uuid]
  42
  43 def main():
  44     parser = argparse.ArgumentParser(description='Re-run containers associated with missing blocks')
  45     parser.add_argument('inp')
  46     args = parser.parse_args()
  47
  48     arv = arvados.api('v1')
  49
  50     busted_collections = set()
  51
  52     logging.log(lglvl, "Reading %s", args.inp)
  53
  54     # Get the list of bad collection PDHs
  55     blocksfile = open(args.inp, "rt")
  56     for line in blocksfile:
  57         # Ignore the first item, that's the block id
  58         collections = line.rstrip().split(" ")[1:]
  59         for c in collections:
  60             busted_collections.add(c)
  61
  62     out = csv.writer(sys.stdout)
  63
  64     out.writerow(("collection uuid", "container request uuid", "record name", "modified at", "owner uuid", "owner name", "root owner uuid", "root owner name", "notes"))
  65
  66     logging.log(lglvl, "Finding collections")
  67
  68     owners = {}
  69     collections_to_delete = {}
  70     container_requests_to_rerun = {}
  71     # Get containers that produced these collections
  72     i = 0
  73     for b in busted_collections:
  74         if (i % 100) == 0:
  75             logging.log(lglvl, "%d/%d", i, len(busted_collections))
  76         i += 1
  77         collections_to_delete = arvados.util.list_all(arv.collections().list, filters=[["portable_data_hash", "=", b]])
  78         for d in collections_to_delete:
  79             t = ""
  80             if d["properties"].get("type") not in ("output", "log"):
  81                 t = "\"type\" was '%s', expected one of 'output' or 'log'" % d["properties"].get("type")
  82             ou = get_owner(arv, owners, d)
  83             out.writerow((d["uuid"], "", d["name"], d["modified_at"], d["owner_uuid"], ou[0], ou[1], owners[ou[1]][0], t))
  84
  85         maybe_containers_to_rerun = arvados.util.list_all(arv.containers().list, filters=[["output", "=", b]])
  86         for ct in maybe_containers_to_rerun:
  87             rerun_request(arv, container_requests_to_rerun, ct)
  88
  89     logging.log(lglvl, "%d/%d", i, len(busted_collections))
  90     logging.log(lglvl, "Finding container requests")
  91
  92     i = 0
  93     for _, cr in container_requests_to_rerun.items():
  94         if (i % 100) == 0:
  95             logging.log(lglvl, "%d/%d", i, len(container_requests_to_rerun))
  96         i += 1
  97         ou = get_owner(arv, owners, cr)
  98         out.writerow(("", cr["uuid"], cr["name"], cr["modified_at"], cr["owner_uuid"], ou[0], ou[1], owners[ou[1]][0], ""))
  99
 100     logging.log(lglvl, "%d/%d", i, len(container_requests_to_rerun))
 101
 102 if __name__ == "__main__":
 103     main()