Merge branch '12134-llfuse-patch'

[arvados.git] / sdk / python / arvados / commands / run.py

diff --git a/sdk/python/arvados/commands/run.py b/sdk/python/arvados/commands/run.py
index 46a56ca5589cf07a28054114cb0ccd75e580f44a..831e496a29786d58d5272e4cffda0e4d24838bdd 100644 (file)
--- a/sdk/python/arvados/commands/run.py
+++ b/sdk/python/arvados/commands/run.py
@@ -1,5 +1,12 @@
-#!/usr/bin/env python
-
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import print_function
+from __future__ import absolute_import
+from builtins import range
+from past.builtins import basestring
+from builtins import object
 import arvados
 import arvados.commands.ws as ws
 import argparse
@@ -7,7 +14,7 @@ import json
 import re
 import os
 import stat
-import put
+from . import put
 import time
 import subprocess
 import logging
@@ -105,7 +112,7 @@ def determine_project(root, current_user):
 # ArvFile() (file already exists in a collection), UploadFile() (file needs to
 # be uploaded to a collection), or simply returns prefix+fn (which yields the
 # original parameter string).
-def statfile(prefix, fn, fnPattern="$(file %s/%s)", dirPattern="$(dir %s/%s/)"):
+def statfile(prefix, fn, fnPattern="$(file %s/%s)", dirPattern="$(dir %s/%s/)", raiseOSError=False):
     absfn = os.path.abspath(fn)
     try:
         st = os.stat(absfn)
@@ -122,7 +129,7 @@ def statfile(prefix, fn, fnPattern="$(file %s/%s)", dirPattern="$(dir %s/%s/)"):
             # trim leading '/' for path prefix test later
             return UploadFile(prefix, absfn[1:])
     except OSError as e:
-        if e.errno == errno.ENOENT:
+        if e.errno == errno.ENOENT and not raiseOSError:
             pass
         else:
             raise
@@ -141,43 +148,46 @@ def write_file(collection, pathprefix, fn):
 def uploadfiles(files, api, dry_run=False, num_retries=0,
                 project=None,
                 fnPattern="$(file %s/%s)",
-                name=None):
+                name=None,
+                collection=None):
     # Find the smallest path prefix that includes all the files that need to be uploaded.
     # This starts at the root and iteratively removes common parent directory prefixes
     # until all file paths no longer have a common parent.
-    n = True
-    pathprefix = "/"
-    while n:
-        pathstep = None
-        for c in files:
-            if pathstep is None:
-                sp = c.fn.split('/')
-                if len(sp) < 2:
-                    # no parent directories left
-                    n = False
-                    break
-                # path step takes next directory
-                pathstep = sp[0] + "/"
-            else:
-                # check if pathstep is common prefix for all files
-                if not c.fn.startswith(pathstep):
-                    n = False
-                    break
-        if n:
-            # pathstep is common parent directory for all files, so remove the prefix
-            # from each path
-            pathprefix += pathstep
+    if files:
+        n = True
+        pathprefix = "/"
+        while n:
+            pathstep = None
             for c in files:
-                c.fn = c.fn[len(pathstep):]
-
-    logger.info("Upload local files: \"%s\"", '" "'.join([c.fn for c in files]))
+                if pathstep is None:
+                    sp = c.fn.split('/')
+                    if len(sp) < 2:
+                        # no parent directories left
+                        n = False
+                        break
+                    # path step takes next directory
+                    pathstep = sp[0] + "/"
+                else:
+                    # check if pathstep is common prefix for all files
+                    if not c.fn.startswith(pathstep):
+                        n = False
+                        break
+            if n:
+                # pathstep is common parent directory for all files, so remove the prefix
+                # from each path
+                pathprefix += pathstep
+                for c in files:
+                    c.fn = c.fn[len(pathstep):]
+
+        logger.info("Upload local files: \"%s\"", '" "'.join([c.fn for c in files]))
 
     if dry_run:
         logger.info("$(input) is %s", pathprefix.rstrip('/'))
         pdh = "$(input)"
     else:
         files = sorted(files, key=lambda x: x.fn)
-        collection = arvados.collection.Collection(api_client=api, num_retries=num_retries)
+        if collection is None:
+            collection = arvados.collection.Collection(api_client=api, num_retries=num_retries)
         prev = ""
         for f in files:
             localpath = os.path.join(pathprefix, f.fn)
@@ -196,8 +206,9 @@ def uploadfiles(files, api, dry_run=False, num_retries=0,
                     for src in iterfiles:
                         write_file(collection, pathprefix, os.path.join(root, src))
 
-        filters=[["portable_data_hash", "=", collection.portable_data_hash()],
-                 ["name", "like", name+"%"]]
+        filters=[["portable_data_hash", "=", collection.portable_data_hash()]]
+        if name:
+            filters.append(["name", "like", name+"%"])
         if project:
             filters.append(["owner_uuid", "=", project])
 
@@ -207,7 +218,7 @@ def uploadfiles(files, api, dry_run=False, num_retries=0,
             item = exists["items"][0]
             pdh = item["portable_data_hash"]
             logger.info("Using collection %s (%s)", pdh, item["uuid"])
-        else:
+        elif len(collection) > 0:
             collection.save_new(name=name, owner_uuid=project, ensure_unique_name=True)
             pdh = collection.portable_data_hash()
             logger.info("Uploaded to %s (%s)", pdh, collection.manifest_locator())
@@ -299,7 +310,7 @@ def main(arguments=None):
     if files:
         uploadfiles(files, api, dry_run=args.dry_run, num_retries=args.retries, project=project)
 
-    for i in xrange(1, len(slots)):
+    for i in range(1, len(slots)):
         slots[i] = [("%s%s" % (c.prefix, c.fn)) if isinstance(c, ArvFile) else c for c in slots[i]]
 
     component = {
@@ -319,8 +330,8 @@ def main(arguments=None):
     group_parser.add_argument('-b', '--batch-size', type=int)
     group_parser.add_argument('args', nargs=argparse.REMAINDER)
 
-    for s in xrange(2, len(slots)):
-        for i in xrange(0, len(slots[s])):
+    for s in range(2, len(slots)):
+        for i in range(0, len(slots[s])):
             if slots[s][i] == '--':
                 inp = "input%i" % (s-2)
                 groupargs = group_parser.parse_args(slots[2][i+1:])