12186: Filter nameroot from runner-submitted input object.

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz@veritasgenetics.com>

diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 769a63bce3f56763e7fa1767317d5af9828a03d0..78086ed0cc07c4c75258d439cd0e978f12f828e9 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -14,14 +14,14 @@ import ruamel.yaml as yaml
 
 from cwltool.errors import WorkflowException
 from cwltool.process import get_feature, UnsupportedRequirement, shortname
-from cwltool.pathmapper import adjustFileObjs, adjustDirObjs
+from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
 from cwltool.utils import aslist
 
 import arvados.collection
 
 from .arvdocker import arv_docker_get_image
 from . import done
-from .runner import Runner, arvados_jobs_image, packed_workflow, trim_anonymous_location
+from .runner import Runner, arvados_jobs_image, packed_workflow, trim_anonymous_location, remove_redundant_fields
 from .fsaccess import CollectionFetcher
 from .pathmapper import NoFollowPathMapper, trim_listing
 from .perf import Perf
@@ -286,8 +286,8 @@ class RunnerContainer(Runner):
         """
 
         adjustDirObjs(self.job_order, trim_listing)
-        adjustFileObjs(self.job_order, trim_anonymous_location)
-        adjustDirObjs(self.job_order, trim_anonymous_location)
+        visit_class(self.job_order, ("File", "Directory"), trim_anonymous_location)
+        visit_class(self.job_order, ("File", "Directory"), remove_redundant_fields)
 
         container_req = {
             "owner_uuid": self.arvrunner.project_uuid,

diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index 64cd2aa04e562ded1d081ab71c56088cae54edb3..d104d56e91e7453b5605a0222e17818184a42cb2 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -13,7 +13,7 @@ from cwltool.errors import WorkflowException
 from cwltool.draft2tool import revmap_file, CommandLineTool
 from cwltool.load_tool import fetch_document
 from cwltool.builder import Builder
-from cwltool.pathmapper import adjustFileObjs, adjustDirObjs
+from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
 
 from schema_salad.sourceline import SourceLine
 
@@ -23,7 +23,7 @@ import arvados.collection
 from arvados.errors import ApiError
 
 from .arvdocker import arv_docker_get_image
-from .runner import Runner, arvados_jobs_image, packed_workflow, upload_workflow_collection, trim_anonymous_location
+from .runner import Runner, arvados_jobs_image, packed_workflow, upload_workflow_collection, trim_anonymous_location, remove_redundant_fields
 from .pathmapper import VwdPathMapper, trim_listing
 from .perf import Perf
 from . import done
@@ -282,8 +282,8 @@ class RunnerJob(Runner):
             self.job_order["cwl:tool"] = "%s/workflow.cwl#main" % wf_pdh
 
         adjustDirObjs(self.job_order, trim_listing)
-        adjustFileObjs(self.job_order, trim_anonymous_location)
-        adjustDirObjs(self.job_order, trim_anonymous_location)
+        visit_class(self.job_order, ("File", "Directory"), trim_anonymous_location)
+        visit_class(self.job_order, ("File", "Directory"), remove_redundant_fields)
 
         if self.output_name:
             self.job_order["arv:output_name"] = self.output_name

diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 687beaddeb59196125af81405612910bae1d7e8a..20cb4677c5935b5011a66b0a958518ce42a285c3 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -17,7 +17,7 @@ from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
 
 import ruamel.yaml as yaml
 
-from .runner import upload_dependencies, packed_workflow, upload_workflow_collection, trim_anonymous_location
+from .runner import upload_dependencies, packed_workflow, upload_workflow_collection, trim_anonymous_location, remove_redundant_fields
 from .pathmapper import trim_listing
 from .arvtool import ArvadosCommandTool
 from .perf import Perf
@@ -113,9 +113,7 @@ class ArvadosWorkflow(Workflow):
                 joborder_keepmount = copy.deepcopy(joborder)
 
                 def keepmount(obj):
-                    for field in ("path", "nameext", "nameroot", "dirname"):
-                        if field in obj:
-                            del obj[field]
+                    remove_redundant_fields(obj)
                     with SourceLine(obj, None, WorkflowException):
                         if "location" not in obj:
                             raise WorkflowException("%s object is missing required 'location' field: %s" % (obj["class"], obj))

diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index bb4fac2ae9541a55872ff6fd371b380659a0f15e..39497098498792d1a4bb8595a77f6dcf9d1f6d47 100644 (file)
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -46,6 +46,11 @@ def trim_anonymous_location(obj):
     if obj.get("location", "").startswith("_:"):
         del obj["location"]
 
+def remove_redundant_fields(obj):
+    for field in ("path", "nameext", "nameroot", "dirname"):
+        if field in obj:
+            del obj[field]
+
 def find_defaults(d, op):
     if isinstance(d, list):
         for i in d:

diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 49545a83dc7ac34eea9acc11dc3e022f2839f22c..03147206e467cb85311eb8641dbe4efc4e570f5e 100644 (file)
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -157,8 +157,6 @@ def stubs(func):
                             'basename': 'blorp.txt',
                             'class': 'File',
                             'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
-                            "nameext": ".txt",
-                            "nameroot": "blorp",
                             "size": 16
                         }},
                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
@@ -166,9 +164,7 @@ def stubs(func):
                                   {
                                       'basename': 'renamed.txt',
                                       'class': 'File', 'location':
-                                      'keep:99999999999999999999999999999998+99/file1.txt',
-                                      "nameext": ".txt",
-                                      "nameroot": "renamed"
+                                      'keep:99999999999999999999999999999998+99/file1.txt'
                                   }
                               ]}},
                         'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main',
@@ -223,16 +219,12 @@ def stubs(func):
                             'basename': u'blorp.txt',
                             'class': 'File',
                             'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
-                            "nameext": ".txt",
-                            "nameroot": "blorp",
                             "size": 16
                         },
                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
                             {'basename': 'renamed.txt',
                              'class': 'File',
-                             'location': 'keep:99999999999999999999999999999998+99/file1.txt',
-                             "nameext": ".txt",
-                             "nameroot": "renamed"
+                             'location': 'keep:99999999999999999999999999999998+99/file1.txt'
                             }
                         ]}
                     },