tools/keep-xref/keep-xref.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (C) The Arvados Authors. All rights reserved.
   4 #
   5 # SPDX-License-Identifier: AGPL-3.0
   6 #
   7
   8 from __future__ import print_function, absolute_import
   9 import argparse
  10 import arvados
  11 import arvados.util
  12 import csv
  13 import sys
  14
  15 """
  16  Given a list of collections missing blocks (as produced by
  17 keep-balance), produce a report listing affected collections and
  18 container requests.
  19 """
  20
  21 def rerun_request(arv, container_requests_to_rerun, ct):
  22     requests = arvados.util.list_all(arv.container_requests().list, filters=[["container_uuid", "=", ct["uuid"]]])
  23     for cr in requests:
  24         if cr["requesting_container_uuid"]:
  25             rerun_request(arv, container_requests_to_rerun, arv.containers().get(uuid=cr["requesting_container_uuid"]).execute())
  26         else:
  27             container_requests_to_rerun[cr["uuid"]] = cr
  28
  29 def get_owner(arv, owners, uuid):
  30     if uuid not in owners:
  31         if uuid[6:11] == "tpzed":
  32             owners[uuid] = arv.users().get(uuid=uuid).execute()["full_name"]
  33         else:
  34             owners[uuid] = arv.groups().get(uuid=uuid).execute()["name"]
  35     return owners[uuid]
  36
  37 def main():
  38     parser = argparse.ArgumentParser(description='Re-run containers associated with missing blocks')
  39     parser.add_argument('inp')
  40     args = parser.parse_args()
  41
  42     arv = arvados.api('v1')
  43
  44     busted_collections = set()
  45
  46     # Get the list of bad collection PDHs
  47     blocksfile = open(args.inp, "rt")
  48     for line in blocksfile:
  49         # Ignore the first item, that's the block id
  50         collections = line.rstrip().split(" ")[1:]
  51         for c in collections:
  52             busted_collections.add(c)
  53
  54     out = csv.writer(sys.stdout)
  55
  56     out.writerow(("collection uuid", "container request uuid", "record name", "modified at", "owner uuid", "owner name", "notes"))
  57
  58     owners = {}
  59     collections_to_delete = {}
  60     container_requests_to_rerun = {}
  61     # Get containers that produced these collections
  62     i = 0
  63     for b in busted_collections:
  64         i += 1
  65         collections_to_delete = arvados.util.list_all(arv.collections().list, filters=[["portable_data_hash", "=", b]])
  66         for d in collections_to_delete:
  67             t = ""
  68             if d["properties"].get("type") not in ("output", "log"):
  69                 t = "\"type\" was '%s', expected one of 'output' or 'log'" % d["properties"].get("type")
  70             out.writerow((d["uuid"], "", d["name"], d["modified_at"], d["owner_uuid"], get_owner(arv, owners, d["owner_uuid"]), t))
  71
  72         maybe_containers_to_rerun = arvados.util.list_all(arv.containers().list, filters=[["output", "=", b]])
  73         for ct in maybe_containers_to_rerun:
  74             rerun_request(arv, container_requests_to_rerun, ct)
  75
  76     i = 0
  77     for _, cr in container_requests_to_rerun.items():
  78         i += 1
  79         out.writerow(("", cr["uuid"], cr["name"], cr["modified_at"], cr["owner_uuid"], get_owner(arv, owners, cr["owner_uuid"]), ""))
  80
  81
  82 if __name__ == "__main__":
  83     main()