Merge branch 'master' into 10645-cr-mounts-display

[arvados.git] / sdk / cwl / arvados_cwl / runner.py

diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 49047fa14328271106608721c9fa22099dfcb5ae..69e4f5bd7b628e89349822e45ae01a587320c605 100644 (file)
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -3,7 +3,6 @@ import urlparse
 from functools import partial
 import logging
 import json
-import re
 import subprocess
 
 from StringIO import StringIO
@@ -24,26 +23,22 @@ import arvados.collection
 import ruamel.yaml as yaml
 
 from .arvdocker import arv_docker_get_image
-from .pathmapper import ArvPathMapper
+from .pathmapper import ArvPathMapper, trim_listing
 from ._version import __version__
 from . import done
 
 logger = logging.getLogger('arvados.cwl-runner')
 
-cwltool.draft2tool.ACCEPTLIST_RE = re.compile(r".*")
+def trim_anonymous_location(obj):
+    """Remove 'location' field from File and Directory literals.
 
-def trim_listing(obj):
-    """Remove 'listing' field from Directory objects that are keep references.
+    To make internal handling easier, literals are assigned a random id for
+    'location'.  However, when writing the record back out, this can break
+    reproducibility.  Since it is valid for literals not have a 'location'
+    field, remove it.
 
-    When Directory objects represent Keep references, it redundant and
-    potentially very expensive to pass fully enumerated Directory objects
-    between instances of cwl-runner (e.g. a submitting a job, or using the
-    RunInSingleContainer feature), so delete the 'listing' field when it is
-    safe to do so.
     """
 
-    if obj.get("location", "").startswith("keep:") and "listing" in obj:
-        del obj["listing"]
     if obj.get("location", "").startswith("_:"):
         del obj["location"]
 
@@ -268,7 +263,7 @@ class Runner(object):
         if submit_runner_ram:
             self.submit_runner_ram = submit_runner_ram
         else:
-            self.submit_runner_ram = 1024
+            self.submit_runner_ram = 3000
 
         if self.submit_runner_ram <= 0:
             raise Exception("Value of --submit-runner-ram must be greater than zero")