From ecf3cf1ec8544cffacc620919e35404e086b931a Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Tue, 29 Aug 2017 12:56:55 -0400 Subject: [PATCH] 12186: Filter nameroot from runner-submitted input object. Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- sdk/cwl/arvados_cwl/arvcontainer.py | 8 ++++---- sdk/cwl/arvados_cwl/arvjob.py | 8 ++++---- sdk/cwl/arvados_cwl/arvworkflow.py | 6 ++---- sdk/cwl/arvados_cwl/runner.py | 5 +++++ sdk/cwl/tests/test_submit.py | 12 ++---------- 5 files changed, 17 insertions(+), 22 deletions(-) diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py index 769a63bce3..78086ed0cc 100644 --- a/sdk/cwl/arvados_cwl/arvcontainer.py +++ b/sdk/cwl/arvados_cwl/arvcontainer.py @@ -14,14 +14,14 @@ import ruamel.yaml as yaml from cwltool.errors import WorkflowException from cwltool.process import get_feature, UnsupportedRequirement, shortname -from cwltool.pathmapper import adjustFileObjs, adjustDirObjs +from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class from cwltool.utils import aslist import arvados.collection from .arvdocker import arv_docker_get_image from . import done -from .runner import Runner, arvados_jobs_image, packed_workflow, trim_anonymous_location +from .runner import Runner, arvados_jobs_image, packed_workflow, trim_anonymous_location, remove_redundant_fields from .fsaccess import CollectionFetcher from .pathmapper import NoFollowPathMapper, trim_listing from .perf import Perf @@ -286,8 +286,8 @@ class RunnerContainer(Runner): """ adjustDirObjs(self.job_order, trim_listing) - adjustFileObjs(self.job_order, trim_anonymous_location) - adjustDirObjs(self.job_order, trim_anonymous_location) + visit_class(self.job_order, ("File", "Directory"), trim_anonymous_location) + visit_class(self.job_order, ("File", "Directory"), remove_redundant_fields) container_req = { "owner_uuid": self.arvrunner.project_uuid, diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py index 64cd2aa04e..d104d56e91 100644 --- a/sdk/cwl/arvados_cwl/arvjob.py +++ b/sdk/cwl/arvados_cwl/arvjob.py @@ -13,7 +13,7 @@ from cwltool.errors import WorkflowException from cwltool.draft2tool import revmap_file, CommandLineTool from cwltool.load_tool import fetch_document from cwltool.builder import Builder -from cwltool.pathmapper import adjustFileObjs, adjustDirObjs +from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class from schema_salad.sourceline import SourceLine @@ -23,7 +23,7 @@ import arvados.collection from arvados.errors import ApiError from .arvdocker import arv_docker_get_image -from .runner import Runner, arvados_jobs_image, packed_workflow, upload_workflow_collection, trim_anonymous_location +from .runner import Runner, arvados_jobs_image, packed_workflow, upload_workflow_collection, trim_anonymous_location, remove_redundant_fields from .pathmapper import VwdPathMapper, trim_listing from .perf import Perf from . import done @@ -282,8 +282,8 @@ class RunnerJob(Runner): self.job_order["cwl:tool"] = "%s/workflow.cwl#main" % wf_pdh adjustDirObjs(self.job_order, trim_listing) - adjustFileObjs(self.job_order, trim_anonymous_location) - adjustDirObjs(self.job_order, trim_anonymous_location) + visit_class(self.job_order, ("File", "Directory"), trim_anonymous_location) + visit_class(self.job_order, ("File", "Directory"), remove_redundant_fields) if self.output_name: self.job_order["arv:output_name"] = self.output_name diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py index 687beaddeb..20cb4677c5 100644 --- a/sdk/cwl/arvados_cwl/arvworkflow.py +++ b/sdk/cwl/arvados_cwl/arvworkflow.py @@ -17,7 +17,7 @@ from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class import ruamel.yaml as yaml -from .runner import upload_dependencies, packed_workflow, upload_workflow_collection, trim_anonymous_location +from .runner import upload_dependencies, packed_workflow, upload_workflow_collection, trim_anonymous_location, remove_redundant_fields from .pathmapper import trim_listing from .arvtool import ArvadosCommandTool from .perf import Perf @@ -113,9 +113,7 @@ class ArvadosWorkflow(Workflow): joborder_keepmount = copy.deepcopy(joborder) def keepmount(obj): - for field in ("path", "nameext", "nameroot", "dirname"): - if field in obj: - del obj[field] + remove_redundant_fields(obj) with SourceLine(obj, None, WorkflowException): if "location" not in obj: raise WorkflowException("%s object is missing required 'location' field: %s" % (obj["class"], obj)) diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py index bb4fac2ae9..3949709849 100644 --- a/sdk/cwl/arvados_cwl/runner.py +++ b/sdk/cwl/arvados_cwl/runner.py @@ -46,6 +46,11 @@ def trim_anonymous_location(obj): if obj.get("location", "").startswith("_:"): del obj["location"] +def remove_redundant_fields(obj): + for field in ("path", "nameext", "nameroot", "dirname"): + if field in obj: + del obj[field] + def find_defaults(d, op): if isinstance(d, list): for i in d: diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py index 49545a83dc..03147206e4 100644 --- a/sdk/cwl/tests/test_submit.py +++ b/sdk/cwl/tests/test_submit.py @@ -157,8 +157,6 @@ def stubs(func): 'basename': 'blorp.txt', 'class': 'File', 'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt', - "nameext": ".txt", - "nameroot": "blorp", "size": 16 }}, 'z': {"value": {'basename': 'anonymous', 'class': 'Directory', @@ -166,9 +164,7 @@ def stubs(func): { 'basename': 'renamed.txt', 'class': 'File', 'location': - 'keep:99999999999999999999999999999998+99/file1.txt', - "nameext": ".txt", - "nameroot": "renamed" + 'keep:99999999999999999999999999999998+99/file1.txt' } ]}}, 'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main', @@ -223,16 +219,12 @@ def stubs(func): 'basename': u'blorp.txt', 'class': 'File', 'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt', - "nameext": ".txt", - "nameroot": "blorp", "size": 16 }, 'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [ {'basename': 'renamed.txt', 'class': 'File', - 'location': 'keep:99999999999999999999999999999998+99/file1.txt', - "nameext": ".txt", - "nameroot": "renamed" + 'location': 'keep:99999999999999999999999999999998+99/file1.txt' } ]} }, -- 2.30.2