Merge branch 'master' into 6827-no-passwords-in-logs

[arvados.git] / sdk / python / arvados / commands / arv_copy.py

diff --git a/sdk/python/arvados/commands/arv_copy.py b/sdk/python/arvados/commands/arv_copy.py
index 1d10b042729f1712a896a4d1d17c1a01d1cef29c..f728b2c59142359bf66855ec530dd82bfb483507 100755 (executable)
--- a/sdk/python/arvados/commands/arv_copy.py
+++ b/sdk/python/arvados/commands/arv_copy.py
@@ -188,6 +188,13 @@ def api_for_instance(instance_name):
         abort('need ARVADOS_API_HOST and ARVADOS_API_TOKEN for {}'.format(instance_name))
     return client
 
+# Check if git is available
+def check_git_availability():
+    try:
+        arvados.util.run_command(['git', '--help'])
+    except Exception:
+        abort('git command is not available. Please ensure git is installed.')
+
 # copy_pipeline_instance(pi_uuid, src, dst, args)
 #
 #    Copies a pipeline instance identified by pi_uuid from src to dst.
@@ -212,6 +219,8 @@ def copy_pipeline_instance(pi_uuid, src, dst, args):
     pi = src.pipeline_instances().get(uuid=pi_uuid).execute(num_retries=args.retries)
 
     if args.recursive:
+        check_git_availability()
+
         if not args.dst_git_repo:
             abort('--dst-git-repo is required when copying a pipeline recursively.')
         # Copy the pipeline template and save the copied template.
@@ -265,6 +274,8 @@ def copy_pipeline_template(pt_uuid, src, dst, args):
     pt = src.pipeline_templates().get(uuid=pt_uuid).execute(num_retries=args.retries)
 
     if args.recursive:
+        check_git_availability()
+
         if not args.dst_git_repo:
             abort('--dst-git-repo is required when copying a pipeline recursively.')
         # Copy input collections, docker images and git repos.
@@ -318,9 +329,9 @@ def copy_collections(obj, src, dst, args):
         obj = arvados.util.portable_data_hash_pattern.sub(copy_collection_fn, obj)
         obj = arvados.util.collection_uuid_pattern.sub(copy_collection_fn, obj)
         return obj
-    elif type(obj) == dict:
+    elif isinstance(obj, dict):
         return {v: copy_collections(obj[v], src, dst, args) for v in obj}
-    elif type(obj) == list:
+    elif isinstance(obj, list):
         return [copy_collections(v, src, dst, args) for v in obj]
     return obj
 
@@ -536,31 +547,30 @@ def copy_collection(obj_uuid, src, dst, args):
     else:
         progress_writer = None
 
-    for line in manifest.splitlines(True):
+    for line in manifest.splitlines():
         words = line.split()
-        dst_manifest_line = words[0]
+        dst_manifest += words[0]
         for word in words[1:]:
             try:
                 loc = arvados.KeepLocator(word)
-                blockhash = loc.md5sum
-                # copy this block if we haven't seen it before
-                # (otherwise, just reuse the existing dst_locator)
-                if blockhash not in dst_locators:
-                    logger.debug("Copying block %s (%s bytes)", blockhash, loc.size)
-                    if progress_writer:
-                        progress_writer.report(obj_uuid, bytes_written, bytes_expected)
-                    data = src_keep.get(word)
-                    dst_locator = dst_keep.put(data)
-                    dst_locators[blockhash] = dst_locator
-                    bytes_written += loc.size
-                dst_manifest_line += ' ' + dst_locators[blockhash]
             except ValueError:
                 # If 'word' can't be parsed as a locator,
                 # presume it's a filename.
-                dst_manifest_line += ' ' + word
-        dst_manifest += dst_manifest_line
-        if line.endswith("\n"):
-            dst_manifest += "\n"
+                dst_manifest += ' ' + word
+                continue
+            blockhash = loc.md5sum
+            # copy this block if we haven't seen it before
+            # (otherwise, just reuse the existing dst_locator)
+            if blockhash not in dst_locators:
+                logger.debug("Copying block %s (%s bytes)", blockhash, loc.size)
+                if progress_writer:
+                    progress_writer.report(obj_uuid, bytes_written, bytes_expected)
+                data = src_keep.get(word)
+                dst_locator = dst_keep.put(data)
+                dst_locators[blockhash] = dst_locator
+                bytes_written += loc.size
+            dst_manifest += ' ' + dst_locators[blockhash]
+        dst_manifest += "\n"
 
     if progress_writer:
         progress_writer.report(obj_uuid, bytes_written, bytes_expected)
@@ -569,7 +579,6 @@ def copy_collection(obj_uuid, src, dst, args):
     # Copy the manifest and save the collection.
     logger.debug('saving %s with manifest: <%s>', obj_uuid, dst_manifest)
 
-    dst_keep.put(dst_manifest.encode('utf-8'))
     c['manifest_text'] = dst_manifest
     return create_collection_from(c, src, dst, args)