arv-copy: Make the help text for the --src and --dst options more easily

[arvados.git] / sdk / python / arvados / commands / arv_copy.py

diff --git a/sdk/python/arvados/commands/arv_copy.py b/sdk/python/arvados/commands/arv_copy.py
index e7eb5a9e2802ae8cede8ec2987f75cbcc8fba933..a10eb2b348aff4499648d6738b8042e5a9d4fa11 100755 (executable)
--- a/sdk/python/arvados/commands/arv_copy.py
+++ b/sdk/python/arvados/commands/arv_copy.py
@@ -71,10 +71,10 @@ def main():
         help='Perform copy even if the object appears to exist at the remote destination.')
     copy_opts.add_argument(
         '--src', dest='source_arvados', required=True,
-        help='The name of the source Arvados instance (required). May be either a pathname to a config file, or the basename of a file in $HOME/.config/arvados/instance_name.conf.')
+        help='The name of the source Arvados instance (required) - points at an Arvados config file. May be either a pathname to a config file, or (for example) "foo" as shorthand for $HOME/.config/arvados/foo.conf.')
     copy_opts.add_argument(
         '--dst', dest='destination_arvados', required=True,
-        help='The name of the destination Arvados instance (required). May be either a pathname to a config file, or the basename of a file in $HOME/.config/arvados/instance_name.conf.')
+        help='The name of the destination Arvados instance (required) - points at an Arvados config file. May be either a pathname to a config file, or (for example) "foo" as shorthand for $HOME/.config/arvados/foo.conf.')
     copy_opts.add_argument(
         '--recursive', dest='recursive', action='store_true',
         help='Recursively copy any dependencies for this object. (default)')
@@ -188,6 +188,13 @@ def api_for_instance(instance_name):
         abort('need ARVADOS_API_HOST and ARVADOS_API_TOKEN for {}'.format(instance_name))
     return client
 
+# Check if git is available
+def check_git_availability():
+    try:
+        arvados.util.run_command(['git', '--help'])
+    except Exception:
+        abort('git command is not available. Please ensure git is installed.')
+
 # copy_pipeline_instance(pi_uuid, src, dst, args)
 #
 #    Copies a pipeline instance identified by pi_uuid from src to dst.
@@ -212,11 +219,7 @@ def copy_pipeline_instance(pi_uuid, src, dst, args):
     pi = src.pipeline_instances().get(uuid=pi_uuid).execute(num_retries=args.retries)
 
     if args.recursive:
-        # Check if git is available
-        try:
-            arvados.util.run_command(['git', '--help'])
-        except:
-            abort('git command is not available. Please ensure git is installed.')
+        check_git_availability()
 
         if not args.dst_git_repo:
             abort('--dst-git-repo is required when copying a pipeline recursively.')
@@ -271,6 +274,8 @@ def copy_pipeline_template(pt_uuid, src, dst, args):
     pt = src.pipeline_templates().get(uuid=pt_uuid).execute(num_retries=args.retries)
 
     if args.recursive:
+        check_git_availability()
+
         if not args.dst_git_repo:
             abort('--dst-git-repo is required when copying a pipeline recursively.')
         # Copy input collections, docker images and git repos.
@@ -324,10 +329,11 @@ def copy_collections(obj, src, dst, args):
         obj = arvados.util.portable_data_hash_pattern.sub(copy_collection_fn, obj)
         obj = arvados.util.collection_uuid_pattern.sub(copy_collection_fn, obj)
         return obj
-    elif type(obj) == dict:
-        return {v: copy_collections(obj[v], src, dst, args) for v in obj}
-    elif type(obj) == list:
-        return [copy_collections(v, src, dst, args) for v in obj]
+    elif isinstance(obj, dict):
+        return type(obj)((v, copy_collections(obj[v], src, dst, args))
+                         for v in obj)
+    elif isinstance(obj, list):
+        return type(obj)(copy_collections(v, src, dst, args) for v in obj)
     return obj
 
 def migrate_jobspec(jobspec, src, dst, dst_repo, args):
@@ -542,31 +548,30 @@ def copy_collection(obj_uuid, src, dst, args):
     else:
         progress_writer = None
 
-    for line in manifest.splitlines(True):
+    for line in manifest.splitlines():
         words = line.split()
-        dst_manifest_line = words[0]
+        dst_manifest += words[0]
         for word in words[1:]:
             try:
                 loc = arvados.KeepLocator(word)
-                blockhash = loc.md5sum
-                # copy this block if we haven't seen it before
-                # (otherwise, just reuse the existing dst_locator)
-                if blockhash not in dst_locators:
-                    logger.debug("Copying block %s (%s bytes)", blockhash, loc.size)
-                    if progress_writer:
-                        progress_writer.report(obj_uuid, bytes_written, bytes_expected)
-                    data = src_keep.get(word)
-                    dst_locator = dst_keep.put(data)
-                    dst_locators[blockhash] = dst_locator
-                    bytes_written += loc.size
-                dst_manifest_line += ' ' + dst_locators[blockhash]
             except ValueError:
                 # If 'word' can't be parsed as a locator,
                 # presume it's a filename.
-                dst_manifest_line += ' ' + word
-        dst_manifest += dst_manifest_line
-        if line.endswith("\n"):
-            dst_manifest += "\n"
+                dst_manifest += ' ' + word
+                continue
+            blockhash = loc.md5sum
+            # copy this block if we haven't seen it before
+            # (otherwise, just reuse the existing dst_locator)
+            if blockhash not in dst_locators:
+                logger.debug("Copying block %s (%s bytes)", blockhash, loc.size)
+                if progress_writer:
+                    progress_writer.report(obj_uuid, bytes_written, bytes_expected)
+                data = src_keep.get(word)
+                dst_locator = dst_keep.put(data)
+                dst_locators[blockhash] = dst_locator
+                bytes_written += loc.size
+            dst_manifest += ' ' + dst_locators[blockhash]
+        dst_manifest += "\n"
 
     if progress_writer:
         progress_writer.report(obj_uuid, bytes_written, bytes_expected)
@@ -575,7 +580,6 @@ def copy_collection(obj_uuid, src, dst, args):
     # Copy the manifest and save the collection.
     logger.debug('saving %s with manifest: <%s>', obj_uuid, dst_manifest)
 
-    dst_keep.put(dst_manifest.encode('utf-8'))
     c['manifest_text'] = dst_manifest
     return create_collection_from(c, src, dst, args)