Merge branch 'master' into 4232-slow-pipes-n-jobs

[arvados.git] / sdk / python / arvados / commands / arv_copy.py

diff --git a/sdk/python/arvados/commands/arv_copy.py b/sdk/python/arvados/commands/arv_copy.py
index 57f79b804797a0e25f3c023fb91f6417213b15dd..308bd7802821ec5f8d2433f077835229a8b66ead 100755 (executable)
--- a/sdk/python/arvados/commands/arv_copy.py
+++ b/sdk/python/arvados/commands/arv_copy.py
@@ -99,6 +99,9 @@ def main():
     src_arv = api_for_instance(args.source_arvados)
     dst_arv = api_for_instance(args.destination_arvados)
 
+    if not args.project_uuid:
+        args.project_uuid = dst_arv.users().current().execute(num_retries=args.retries)["uuid"]
+
     # Identify the kind of object we have been given, and begin copying.
     t = uuid_type(src_arv, args.object_uuid)
     if t == 'Collection':
@@ -221,10 +224,7 @@ def copy_pipeline_instance(pi_uuid, src, dst, args):
         pi_uuid,
         pi['description'] if pi.get('description', None) else '')
 
-    if args.project_uuid:
-        pi['owner_uuid'] = args.project_uuid
-    else:
-        del pi['owner_uuid']
+    pi['owner_uuid'] = args.project_uuid
 
     del pi['uuid']
 
@@ -261,10 +261,7 @@ def copy_pipeline_template(pt_uuid, src, dst, args):
     pt['name'] = "{} copied from {}".format(pt.get('name', ''), pt_uuid)
     del pt['uuid']
 
-    if args.project_uuid:
-        pt['owner_uuid'] = args.project_uuid
-    elif 'owner_uuid' in pt:
-        del pt['owner_uuid']
+    pt['owner_uuid'] = args.project_uuid
 
     return dst.pipeline_templates().create(body=pt, ensure_unique_name=True).execute()
 
@@ -375,6 +372,39 @@ def total_collection_size(manifest_text):
 
     return total_bytes
 
+def create_collection_from(c, src, dst, args):
+    """Create a new collection record on dst, and copy Docker metadata if
+    available."""
+
+    collection_uuid = c['uuid']
+
+    del c['uuid']
+
+    if 'properties' in c:
+        del c['properties']
+
+    c['owner_uuid'] = args.project_uuid
+
+    dst_collection = dst.collections().create(body=c, ensure_unique_name=True).execute(num_retries=args.retries)
+
+    # Create docker_image_repo+tag and docker_image_hash links
+    # at the destination.
+    for link_class in ("docker_image_repo+tag", "docker_image_hash"):
+        docker_links = src.links().list(filters=[["head_uuid", "=", collection_uuid], ["link_class", "=", link_class]]).execute(num_retries=args.retries)['items']
+
+        for d in docker_links:
+            body={
+                'head_uuid': dst_collection['uuid'],
+                'link_class': link_class,
+                'name': d['name'],
+            }
+            body['owner_uuid'] = args.project_uuid
+
+            lk = dst.links().create(body=body).execute(num_retries=args.retries)
+            logger.debug('created dst link {}'.format(lk))
+
+    return dst_collection
+
 # copy_collection(obj_uuid, src, dst, args)
 #
 #    Copies the collection identified by obj_uuid from src to dst.
@@ -412,15 +442,13 @@ def copy_collection(obj_uuid, src, dst, args):
             filters=[['portable_data_hash', '=', colhash]]
         ).execute()
         if dstcol['items_available'] > 0:
-            if args.project_uuid:
-                c['owner_uuid'] = args.project_uuid
-                del c['uuid']
-                if 'properties' in c:
-                    del c['properties']
-                return dst.collections().create(body=c, ensure_unique_name=True).execute()
-            else:
-                logger.info("Skipping collection %s (already at dst)", obj_uuid)
-            return dstcol['items'][0]
+            for d in dstcol['items']:
+                if ((args.project_uuid == d['owner_uuid']) and
+                    (c['name'] == d['name']) and
+                    (c['portable_data_hash'] == d['portable_data_hash'])):
+                    return d
+
+            return create_collection_from(c, src, dst, args)
 
     # Fetch the collection's manifest.
     manifest = c['manifest_text']
@@ -472,20 +500,10 @@ def copy_collection(obj_uuid, src, dst, args):
 
     # Copy the manifest and save the collection.
     logger.debug('saving %s with manifest: <%s>', obj_uuid, dst_manifest)
-    dst_keep.put(dst_manifest)
-
-    del c['uuid']
-    if 'properties' in c:
-        del c['properties']
-
-    if args.project_uuid:
-        c['owner_uuid'] = args.project_uuid
-    elif 'owner_uuid' in c:
-        del c['owner_uuid']
 
+    dst_keep.put(dst_manifest.encode('utf-8'))
     c['manifest_text'] = dst_manifest
-
-    return dst.collections().create(body=c, ensure_unique_name=True).execute()
+    return create_collection_from(c, src, dst, args)
 
 # copy_git_repo(src_git_repo, src, dst, dst_git_repo, script_version)
 #
@@ -582,30 +600,6 @@ def copy_docker_image(docker_image, docker_image_tag, src, dst, args):
     # Copy the collection it refers to.
     dst_image_col = copy_collection(image_uuid, src, dst, args)
 
-    # Create docker_image_repo+tag and docker_image_hash links
-    # at the destination.
-    body={
-        'head_uuid': dst_image_col['uuid'],
-        'link_class': 'docker_image_repo+tag',
-        'name': "{}:{}".format(docker_image, docker_image_tag),
-    }
-    if args.project_uuid:
-        body['owner_uuid'] = args.project_uuid
-
-    lk = dst.links().create(body=body).execute(num_retries=args.retries)
-    logger.debug('created dst link {}'.format(lk))
-
-    body={
-        'head_uuid': dst_image_col['uuid'],
-        'link_class': 'docker_image_hash',
-        'name': dst_image_col['portable_data_hash'],
-    }
-    if args.project_uuid:
-        body['owner_uuid'] = args.project_uuid
-
-    lk = dst.links().create(body=body).execute(num_retries=args.retries)
-    logger.debug('created dst link {}'.format(lk))
-
 
 # git_rev_parse(rev, repo)
 #