17004: Work on CWL side

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz@curii.com>

diff --git a/doc/user/cwl/cwl-extensions.html.textile.liquid b/doc/user/cwl/cwl-extensions.html.textile.liquid
index 0e97e07da3864faa821173536836ef0b6d54a88d..431027ba5d490683258741d75c58c9047cece41e 100644 (file)
--- a/doc/user/cwl/cwl-extensions.html.textile.liquid
+++ b/doc/user/cwl/cwl-extensions.html.textile.liquid
@@ -58,6 +58,11 @@ hints:
       property1: value1
       property2: $(inputs.value2)
 
+  arv:OutputCollectionProperties:
+    outputProperties:
+      property1: value1
+      property2: $(inputs.value2)
+
   cwltool:CUDARequirement:
     cudaVersionMin: "11.0"
     cudaComputeCapability: "9.0"
@@ -156,6 +161,14 @@ table(table table-bordered table-condensed).
 |_. Field |_. Type |_. Description |
 |processProperties|key-value map, or list of objects with the fields {propertyName, propertyValue}|The properties that will be set on the container request.  May include expressions that reference `$(inputs)` of the current workflow or tool.|
 
+h2(#ProcessProperties). arv:OutputCollectionProperties
+
+Specify custom "properties":{{site.baseurl}}/api/methods.html#subpropertyfilters that will be set on the output collection of the workflow step.
+
+table(table table-bordered table-condensed).
+|_. Field |_. Type |_. Description |
+|outputProperties|key-value map, or list of objects with the fields {propertyName, propertyValue}|The properties that will be set on the output collection.  May include expressions that reference `$(inputs)` of the current workflow or tool.|
+
 h2(#CUDARequirement). cwltool:CUDARequirement
 
 Request support for Nvidia CUDA GPU acceleration in the container.  Assumes that the CUDA runtime (SDK) is installed in the container, and the host will inject the CUDA driver libraries into the container (equal or later to the version requested).

diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml
index 73b8120973b4aea9a9aabac06d6f3d67178a3396..54e0fc5122edad8cfd00708f27def0f34a9bc8b5 100644 (file)
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml
@@ -415,7 +415,7 @@ $graph:
       jsonldPredicate:
         _id: "@type"
         _type: "@vocab"
-    processProperties:
+    outputProperties:
       type: PropertyDef[]
       jsonldPredicate:
         mapSubject: propertyName

diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml
index 2044dac2a1227e4ed977840ab5bbe32f69dad9bd..b60d0ab1c9fd7e3144e413b49a109c805d036657 100644 (file)
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml
@@ -358,7 +358,7 @@ $graph:
       jsonldPredicate:
         _id: "@type"
         _type: "@vocab"
-    processProperties:
+    outputProperties:
       type: PropertyDef[]
       jsonldPredicate:
         mapSubject: propertyName

diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
index c969eef2523385acc8187988f0aa20b041306131..2769244a5df658de661dc7a80c3506de20299b21 100644 (file)
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
@@ -360,7 +360,7 @@ $graph:
       jsonldPredicate:
         _id: "@type"
         _type: "@vocab"
-    processProperties:
+    outputProperties:
       type: PropertyDef[]
       jsonldPredicate:
         mapSubject: propertyName

diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 0b8f2ca707bda7e58da8cfc306f071ed13ac980b..b1784f8f3ff15928da1cbe598c0baa2a4d2fffc1 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -342,12 +342,13 @@ class ArvadosContainer(JobBase):
             for pr in properties_req["processProperties"]:
                 container_request["properties"][pr["propertyName"]] = self.builder.do_eval(pr["propertyValue"])
 
+        container_request["output_properties"] = {}
         output_properties_req, _ = self.get_requirement("http://arvados.org/cwl#OutputCollectionProperties")
         if output_properties_req:
-            for pr in output_properties_req["processProperties"]:
+            for pr in output_properties_req["outputProperties"]:
                 container_request["output_properties"][pr["propertyName"]] = self.builder.do_eval(pr["propertyValue"])
 
-        output_properties.update(intermediate_collection_info["properties"])
+        container_request["output_properties"].update(intermediate_collection_info["properties"])
 
         if runtimeContext.runnerjob.startswith("arvwf:"):
             wfuuid = runtimeContext.runnerjob[6:runtimeContext.runnerjob.index("#")]

diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index 381b88543145c9ae7f98c5ec36aaab783b8c24ab..1a4bf65a57b943a2aa214c50e0ebbe10538863c2 100644 (file)
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -794,7 +794,7 @@ The 'jobs' API is no longer supported.
             output_properties = {}
             output_properties_req, _ = self.get_requirement("http://arvados.org/cwl#OutputCollectionProperties")
             if output_properties_req:
-                for pr in output_properties_req["processProperties"]:
+                for pr in output_properties_req["outputProperties"]:
                     output_properties[pr["propertyName"]] = self.builder.do_eval(pr["propertyValue"])
 
             self.final_output, self.final_output_collection = self.make_output_collection(self.output_name, storage_classes,

diff --git a/sdk/cwl/arvados_cwl/util.py b/sdk/cwl/arvados_cwl/util.py
index 85ae65ecf18c327aed6d9b3f2ddb5182dcc05b08..a0dfb290c1b2dd9a87e67151f8885c5ee70bd3f8 100644 (file)
--- a/sdk/cwl/arvados_cwl/util.py
+++ b/sdk/cwl/arvados_cwl/util.py
@@ -16,9 +16,9 @@ def get_intermediate_collection_info(workflow_step_name, current_container, inte
         if intermediate_output_ttl > 0:
             trash_time = datetime.datetime.utcnow() + datetime.timedelta(seconds=intermediate_output_ttl)
         container_uuid = None
+        props = {"type": "intermediate"}
         if current_container:
-            container_uuid = current_container['uuid']
-        props = {"type": "intermediate", "container": container_uuid}
+            props["container"] = current_container['uuid']
 
         return {"name" : name, "trash_at" : trash_time, "properties" : props}
 

diff --git a/services/api/app/models/container_request.rb b/services/api/app/models/container_request.rb
index 74fc0802d3fd264e4ea3395a6352d6c6a90d9c8a..605b376cf55c917848f08f07df7e3c3c2e1ddd99 100644 (file)
--- a/services/api/app/models/container_request.rb
+++ b/services/api/app/models/container_request.rb
@@ -242,7 +242,11 @@ class ContainerRequest < ArvadosModel
         merged_properties.update(self.output_properties)
       end
 
-      merged_properties['type'] = out_type
+      if out_type == 'output' and !requesting_container_uuid.nil?
+          merged_properties['type'] = 'intermediate'
+      else
+        merged_properties['type'] = out_type
+      end
       merged_properties['container_request'] = uuid
 
       coll.assign_attributes(