12256: Only scan for referenced files first time after packing

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz@veritasgenetics.com>

diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index ba16b866305f49e1fc4765a02982468aa18d3d52..05d4648d153a43a10b5ad0f5fb26bce4600dac41 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -116,6 +116,7 @@ class ArvadosWorkflow(Workflow):
         self.wf_pdh = None
         self.dynamic_resource_req = []
         self.static_resource_req = []
+        self.wf_reffiles = []
 
     def job(self, joborder, output_callback, **kwargs):
         kwargs["work_api"] = self.work_api
@@ -181,6 +182,9 @@ class ArvadosWorkflow(Workflow):
                                         uri,
                                         False)
 
+                    visit_class(packed, ("File", "Directory"), self.wf_reffiles.append)
+
+
             if self.dynamic_resource_req:
                 builder = Builder()
                 builder.job = joborder
@@ -206,15 +210,13 @@ class ArvadosWorkflow(Workflow):
 
                 reffiles = []
                 visit_class(joborder_keepmount, ("File", "Directory"), reffiles.append)
-                reffiles2 = []
-                visit_class(packed, ("File", "Directory"), reffiles2.append)
 
-                mapper = ArvPathMapper(self.arvrunner, reffiles+reffiles2, kwargs["basedir"],
+                mapper = ArvPathMapper(self.arvrunner, reffiles+self.wf_reffiles, kwargs["basedir"],
                                  "/keep/%s",
                                  "/keep/%s/%s",
                                  **kwargs)
 
-                kwargs["extra_reffiles"] = copy.deepcopy(reffiles2)
+                kwargs["extra_reffiles"] = copy.deepcopy(self.wf_reffiles)
 
                 def keepmount(obj):
                     remove_redundant_fields(obj)

diff --git a/sdk/cwl/tests/arvados-tests.yml b/sdk/cwl/tests/arvados-tests.yml
index 9106426159029ea7765f5e4b15eab12070194bb3..8eac71886cbf643ca97db1e033b9ba2808b40137 100644 (file)
--- a/sdk/cwl/tests/arvados-tests.yml
+++ b/sdk/cwl/tests/arvados-tests.yml
@@ -206,7 +206,8 @@
   output:
     out: null
   tool: wf-defaults/wf4.cwl
-  doc: default in embedded subworkflow
+  doc: default in embedded subworkflow missing 'id' field
+  should_fail: true
 
 - job: null
   output: