3 # Copyright (C) The Arvados Authors. All rights reserved.
5 # SPDX-License-Identifier: AGPL-3.0
15 lglvl = logging.INFO+1
16 logging.basicConfig(level=lglvl, format='%(message)s')
19 Given a list of collections missing blocks (as produced by
20 keep-balance), produce a report listing affected collections and
24 def rerun_request(arv, container_requests_to_rerun, ct):
25 requests = arvados.util.list_all(arv.container_requests().list, filters=[["container_uuid", "=", ct["uuid"]]])
27 if cr["requesting_container_uuid"]:
28 rerun_request(arv, container_requests_to_rerun, arv.containers().get(uuid=cr["requesting_container_uuid"]).execute())
30 container_requests_to_rerun[cr["uuid"]] = cr
32 def get_owner(arv, owners, record):
33 uuid = record["owner_uuid"]
34 if uuid not in owners:
35 if uuid[6:11] == "tpzed":
36 owners[uuid] = (arv.users().get(uuid=uuid).execute()["full_name"], uuid)
38 grp = arv.groups().get(uuid=uuid).execute()
39 _, ou = get_owner(arv, owners, grp)
40 owners[uuid] = (grp["name"], ou)
44 parser = argparse.ArgumentParser(description='Re-run containers associated with missing blocks')
45 parser.add_argument('inp')
46 args = parser.parse_args()
48 arv = arvados.api('v1')
50 busted_collections = set()
52 logging.log(lglvl, "Reading %s", args.inp)
54 # Get the list of bad collection PDHs
55 blocksfile = open(args.inp, "rt")
56 for line in blocksfile:
57 # Ignore the first item, that's the block id
58 collections = line.rstrip().split(" ")[1:]
60 busted_collections.add(c)
62 out = csv.writer(sys.stdout)
64 out.writerow(("collection uuid", "container request uuid", "record name", "modified at", "owner uuid", "owner name", "root owner uuid", "root owner name", "notes"))
66 logging.log(lglvl, "Finding collections")
69 collections_to_delete = {}
70 container_requests_to_rerun = {}
71 # Get containers that produced these collections
73 for b in busted_collections:
75 logging.log(lglvl, "%d/%d", i, len(busted_collections))
77 collections_to_delete = arvados.util.list_all(arv.collections().list, filters=[["portable_data_hash", "=", b]])
78 for d in collections_to_delete:
80 if d["properties"].get("type") not in ("output", "log"):
81 t = "\"type\" was '%s', expected one of 'output' or 'log'" % d["properties"].get("type")
82 ou = get_owner(arv, owners, d)
83 out.writerow((d["uuid"], "", d["name"], d["modified_at"], d["owner_uuid"], ou[0], ou[1], owners[ou[1]][0], t))
85 maybe_containers_to_rerun = arvados.util.list_all(arv.containers().list, filters=[["output", "=", b]])
86 for ct in maybe_containers_to_rerun:
87 rerun_request(arv, container_requests_to_rerun, ct)
89 logging.log(lglvl, "%d/%d", i, len(busted_collections))
90 logging.log(lglvl, "Finding container requests")
93 for _, cr in container_requests_to_rerun.items():
95 logging.log(lglvl, "%d/%d", i, len(container_requests_to_rerun))
97 ou = get_owner(arv, owners, cr)
98 out.writerow(("", cr["uuid"], cr["name"], cr["modified_at"], cr["owner_uuid"], ou[0], ou[1], owners[ou[1]][0], ""))
100 logging.log(lglvl, "%d/%d", i, len(container_requests_to_rerun))
102 if __name__ == "__main__":