Merge branch '8784-dir-listings'

[arvados.git] / sdk / python / arvados / commands / get.py

diff --git a/sdk/python/arvados/commands/get.py b/sdk/python/arvados/commands/get.py
index f39e092135aa6234c6318326dd8db654aa84aa04..888fd390f0ebf1a550a1bfb758160f52aa2b7ea6 100755 (executable)
--- a/sdk/python/arvados/commands/get.py
+++ b/sdk/python/arvados/commands/get.py
@@ -10,6 +10,7 @@ import logging
 
 import arvados
 import arvados.commands._util as arv_cmd
 
 import arvados
 import arvados.commands._util as arv_cmd

+import arvados.util as util

 
 from arvados._version import __version__
 
 
 from arvados._version import __version__
 


@@ -84,6 +85,11 @@ write *anything* if any files exist that would have to be
 overwritten. This option causes even devices, sockets, and fifos to be
 skipped.
 """)
 overwritten. This option causes even devices, sockets, and fifos to be
 skipped.
 """)

+group.add_argument('--strip-manifest', action='store_true', default=False,
+                   help="""
+When getting a collection manifest, strip its access tokens before writing
+it.
+""")

 
 def parse_arguments(arguments, stdout, stderr):
     args = parser.parse_args(arguments)
 
 def parse_arguments(arguments, stdout, stderr):
     args = parser.parse_args(arguments)


@@ -125,18 +131,22 @@ def parse_arguments(arguments, stdout, stderr):
 
 def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr):
     global api_client
 
 def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr):
     global api_client

-    
+
+    if stdout is sys.stdout and hasattr(stdout, 'buffer'):
+        # in Python 3, write to stdout as binary
+        stdout = stdout.buffer
+

     args = parse_arguments(arguments, stdout, stderr)
     if api_client is None:
         api_client = arvados.api('v1')
 
     r = re.search(r'^(.*?)(/.*)?$', args.locator)
     args = parse_arguments(arguments, stdout, stderr)
     if api_client is None:
         api_client = arvados.api('v1')
 
     r = re.search(r'^(.*?)(/.*)?$', args.locator)

-    collection = r.group(1)
+    col_loc = r.group(1)

     get_prefix = r.group(2)
     if args.r and not get_prefix:
         get_prefix = os.sep
     try:
     get_prefix = r.group(2)
     if args.r and not get_prefix:
         get_prefix = os.sep
     try:

-        reader = arvados.CollectionReader(collection, num_retries=args.retries)
+        reader = arvados.CollectionReader(col_loc, num_retries=args.retries)

     except Exception as error:
         logger.error("failed to read collection: {}".format(error))
         return 1
     except Exception as error:
         logger.error("failed to read collection: {}".format(error))
         return 1


@@ -149,16 +159,16 @@ def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr):
                 open_flags |= os.O_EXCL
             try:
                 if args.destination == "-":
                 open_flags |= os.O_EXCL
             try:
                 if args.destination == "-":

-                    stdout.write(reader.manifest_text())
+                    stdout.write(reader.manifest_text(strip=args.strip_manifest).encode())

                 else:
                     out_fd = os.open(args.destination, open_flags)
                     with os.fdopen(out_fd, 'wb') as out_file:
                 else:
                     out_fd = os.open(args.destination, open_flags)
                     with os.fdopen(out_fd, 'wb') as out_file:

-                        out_file.write(reader.manifest_text())
+                        out_file.write(reader.manifest_text(strip=args.strip_manifest).encode())

             except (IOError, OSError) as error:
                 logger.error("can't write to '{}': {}".format(args.destination, error))
                 return 1
             except (arvados.errors.ApiError, arvados.errors.KeepReadError) as error:
             except (IOError, OSError) as error:
                 logger.error("can't write to '{}': {}".format(args.destination, error))
                 return 1
             except (arvados.errors.ApiError, arvados.errors.KeepReadError) as error:

-                logger.error("failed to download '{}': {}".format(collection, error))
+                logger.error("failed to download '{}': {}".format(col_loc, error))

                 return 1
         return 0
 
                 return 1
         return 0
 


@@ -230,7 +240,7 @@ def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr):
         if args.hash:
             digestor = hashlib.new(args.hash)
         try:
         if args.hash:
             digestor = hashlib.new(args.hash)
         try:

-            with s.open(f.name, 'r') as file_reader:
+            with s.open(f.name, 'rb') as file_reader:

                 for data in file_reader.readall():
                     if outfile:
                         outfile.write(data)
                 for data in file_reader.readall():
                     if outfile:
                         outfile.write(data)


@@ -265,7 +275,7 @@ def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr):
 
 def files_in_collection(c):
     # Sort first by file type, then alphabetically by file path.
 
 def files_in_collection(c):
     # Sort first by file type, then alphabetically by file path.

-    for i in sorted(c.keys(),
+    for i in sorted(list(c.keys()),

                     key=lambda k: (
                         isinstance(c[k], arvados.collection.Subcollection),
                         k.upper())):
                     key=lambda k: (
                         isinstance(c[k], arvados.collection.Subcollection),
                         k.upper())):