12256: Handle default references in RunInSingleContainer

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz@veritasgenetics.com>

diff --git a/sdk/cwl/arvados_cwl/arvtool.py b/sdk/cwl/arvados_cwl/arvtool.py
index ddc270d7375a6719e2117766d954df294e50e7e9..8268300e75b66d6f82b999f649003ec3a0615bbf 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvtool.py
+++ b/sdk/cwl/arvados_cwl/arvtool.py
@@ -24,7 +24,7 @@ class ArvadosCommandTool(CommandLineTool):
     def makePathMapper(self, reffiles, stagedir, **kwargs):
         # type: (List[Any], unicode, **Any) -> PathMapper
         if self.work_api == "containers":
-            return ArvPathMapper(self.arvrunner, reffiles, kwargs["basedir"],
+            return ArvPathMapper(self.arvrunner, reffiles+kwargs.get("extra_reffiles", []), kwargs["basedir"],
                                  "/keep/%s",
                                  "/keep/%s/%s",
                                  **kwargs)

diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index bdc2e274b0daa695e9b0f2cdcfe698f53f502730..ba16b866305f49e1fc4765a02982468aa18d3d52 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -205,13 +205,17 @@ class ArvadosWorkflow(Workflow):
                 joborder_keepmount = copy.deepcopy(joborder)
 
                 reffiles = []
-                visit_class(joborder_keepmount, ("File", "Directory"), lambda x: reffiles.append(x))
+                visit_class(joborder_keepmount, ("File", "Directory"), reffiles.append)
+                reffiles2 = []
+                visit_class(packed, ("File", "Directory"), reffiles2.append)
 
-                mapper = ArvPathMapper(self.arvrunner, reffiles, kwargs["basedir"],
+                mapper = ArvPathMapper(self.arvrunner, reffiles+reffiles2, kwargs["basedir"],
                                  "/keep/%s",
                                  "/keep/%s/%s",
                                  **kwargs)
 
+                kwargs["extra_reffiles"] = copy.deepcopy(reffiles2)
+
                 def keepmount(obj):
                     remove_redundant_fields(obj)
                     with SourceLine(obj, None, WorkflowException, logger.isEnabledFor(logging.DEBUG)):

diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index fe1ef223a098f038ed4153e18489ccccf04fe16d..3ce08f6cc7971973f7e6925bbc351d65b3492592 100644 (file)
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -217,13 +217,13 @@ def packed_workflow(arvrunner, tool, merged_map):
     packed = pack(tool.doc_loader, tool.doc_loader.fetch(tool.tool["id"]),
                   tool.tool["id"], tool.metadata, rewrite_out=rewrites)
 
-    rewrite_to_orig = {}
-    for k,v in rewrites.items():
-        rewrite_to_orig[v] = k
+    rewrite_to_orig = {v: k for k,v in rewrites.items()}
 
     def visit(v, cur_id):
         if isinstance(v, dict):
             if v.get("class") in ("CommandLineTool", "Workflow"):
+                if "id" not in v:
+                    raise SourceLine(v, None, Exception).makeError("Embedded process object is missing required 'id' field")
                 cur_id = rewrite_to_orig.get(v["id"], v["id"])
             if "location" in v and not v["location"].startswith("keep:"):
                 v["location"] = merged_map[cur_id].resolved[v["location"]]

diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 563015c2cee122c80922e5043c9cf75a8f9558b2..c636bb88b7eea61515ea595d312f019ae9b98219 100644 (file)
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -33,7 +33,7 @@ setup(name='arvados-cwl-runner',
       # Note that arvados/build/run-build-packages.sh looks at this
       # file to determine what version of cwltool and schema-salad to build.
       install_requires=[
-          'cwltool==1.0.20180502225535',
+          'cwltool==1.0.20180504184252',
           'schema-salad==2.7.20180501211602',
           'typing==3.5.3.0',
           'ruamel.yaml >=0.13.7, <0.15',

diff --git a/sdk/cwl/tests/arvados-tests.yml b/sdk/cwl/tests/arvados-tests.yml
index 172e6a97b19b91f7e33edd90e7a34acb2775a985..9106426159029ea7765f5e4b15eab12070194bb3 100644 (file)
--- a/sdk/cwl/tests/arvados-tests.yml
+++ b/sdk/cwl/tests/arvados-tests.yml
@@ -199,7 +199,7 @@
   output:
     out: null
   tool: wf-defaults/wf3.cwl
-  doc: "Do not accept a directory literal without a basename
+  doc: "Do not accept a directory literal without a basename"
   should_fail: true
 
 - job: null