From 972272c42f7ad8daaef29d3d981968e566f3f087 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Wed, 17 Nov 2021 16:19:14 -0500 Subject: [PATCH] 17962: Improve error handling when copying projects and workflows Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- sdk/python/arvados/commands/arv_copy.py | 34 ++++++++++++++++++++----- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/sdk/python/arvados/commands/arv_copy.py b/sdk/python/arvados/commands/arv_copy.py index d10234ca60..f041ec6ea4 100755 --- a/sdk/python/arvados/commands/arv_copy.py +++ b/sdk/python/arvados/commands/arv_copy.py @@ -161,7 +161,12 @@ def main(): logger.error("API server returned an error result: {}".format(result)) exit(1) - logger.info("") + print(result['uuid']) + + if result.get('partial_error'): + logger.warning("Warning: created copy with uuid {} but failed to copy some items: {}".format(result['uuid'], result['partial_error'])) + exit(1) + logger.info("Success: created copy with uuid {}".format(result['uuid'])) exit(0) @@ -292,8 +297,11 @@ def copy_workflow(wf_uuid, src, dst, args): # fetch the workflow from the source instance wf = src.workflows().get(uuid=wf_uuid).execute(num_retries=args.retries) + if not wf["definition"]: + logger.warning("Workflow object {} has an empty or null definition, it won't do anything.".format(wf_uuid)) + # copy collections and docker images - if args.recursive: + if args.recursive and wf["definition"]: wf_def = yaml.safe_load(wf["definition"]) if wf_def is not None: locations = [] @@ -683,17 +691,31 @@ def copy_project(obj_uuid, src, dst, owner_uuid, args): logger.debug('Copying %s to %s', obj_uuid, project_record["uuid"]) + + partial_error = "" + # Copy collections - copy_collections([col["uuid"] for col in arvados.util.list_all(src.collections().list, filters=[["owner_uuid", "=", obj_uuid]])], - src, dst, args) + try: + copy_collections([col["uuid"] for col in arvados.util.list_all(src.collections().list, filters=[["owner_uuid", "=", obj_uuid]])], + src, dst, args) + except Exception as e: + partial_error += "\n" + str(e) # Copy workflows for w in arvados.util.list_all(src.workflows().list, filters=[["owner_uuid", "=", obj_uuid]]): - copy_workflow(w["uuid"], src, dst, args) + try: + copy_workflow(w["uuid"], src, dst, args) + except Exception as e: + partial_error += "\n" + "Error while copying %s: %s" % (w["uuid"], e) if args.recursive: for g in arvados.util.list_all(src.groups().list, filters=[["owner_uuid", "=", obj_uuid]]): - copy_project(g["uuid"], src, dst, project_record["uuid"], args) + try: + copy_project(g["uuid"], src, dst, project_record["uuid"], args) + except Exception as e: + partial_error += "\n" + "Error while copying %s: %s" % (g["uuid"], e) + + project_record["partial_error"] = partial_error return project_record -- 2.39.5