From: Peter Amstutz Date: Fri, 6 May 2022 21:12:04 +0000 (-0400) Subject: 17004: Work on CWL side X-Git-Tag: 2.5.0~165^2~7 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/a24e4872dc015304b76c33b1246e4a3fc5163711 17004: Work on CWL side Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- diff --git a/doc/user/cwl/cwl-extensions.html.textile.liquid b/doc/user/cwl/cwl-extensions.html.textile.liquid index 0e97e07da3..431027ba5d 100644 --- a/doc/user/cwl/cwl-extensions.html.textile.liquid +++ b/doc/user/cwl/cwl-extensions.html.textile.liquid @@ -58,6 +58,11 @@ hints: property1: value1 property2: $(inputs.value2) + arv:OutputCollectionProperties: + outputProperties: + property1: value1 + property2: $(inputs.value2) + cwltool:CUDARequirement: cudaVersionMin: "11.0" cudaComputeCapability: "9.0" @@ -156,6 +161,14 @@ table(table table-bordered table-condensed). |_. Field |_. Type |_. Description | |processProperties|key-value map, or list of objects with the fields {propertyName, propertyValue}|The properties that will be set on the container request. May include expressions that reference `$(inputs)` of the current workflow or tool.| +h2(#ProcessProperties). arv:OutputCollectionProperties + +Specify custom "properties":{{site.baseurl}}/api/methods.html#subpropertyfilters that will be set on the output collection of the workflow step. + +table(table table-bordered table-condensed). +|_. Field |_. Type |_. Description | +|outputProperties|key-value map, or list of objects with the fields {propertyName, propertyValue}|The properties that will be set on the output collection. May include expressions that reference `$(inputs)` of the current workflow or tool.| + h2(#CUDARequirement). cwltool:CUDARequirement Request support for Nvidia CUDA GPU acceleration in the container. Assumes that the CUDA runtime (SDK) is installed in the container, and the host will inject the CUDA driver libraries into the container (equal or later to the version requested). diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml index 73b8120973..54e0fc5122 100644 --- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml +++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml @@ -415,7 +415,7 @@ $graph: jsonldPredicate: _id: "@type" _type: "@vocab" - processProperties: + outputProperties: type: PropertyDef[] jsonldPredicate: mapSubject: propertyName diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml index 2044dac2a1..b60d0ab1c9 100644 --- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml +++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml @@ -358,7 +358,7 @@ $graph: jsonldPredicate: _id: "@type" _type: "@vocab" - processProperties: + outputProperties: type: PropertyDef[] jsonldPredicate: mapSubject: propertyName diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml index c969eef252..2769244a5d 100644 --- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml +++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml @@ -360,7 +360,7 @@ $graph: jsonldPredicate: _id: "@type" _type: "@vocab" - processProperties: + outputProperties: type: PropertyDef[] jsonldPredicate: mapSubject: propertyName diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py index 0b8f2ca707..b1784f8f3f 100644 --- a/sdk/cwl/arvados_cwl/arvcontainer.py +++ b/sdk/cwl/arvados_cwl/arvcontainer.py @@ -342,12 +342,13 @@ class ArvadosContainer(JobBase): for pr in properties_req["processProperties"]: container_request["properties"][pr["propertyName"]] = self.builder.do_eval(pr["propertyValue"]) + container_request["output_properties"] = {} output_properties_req, _ = self.get_requirement("http://arvados.org/cwl#OutputCollectionProperties") if output_properties_req: - for pr in output_properties_req["processProperties"]: + for pr in output_properties_req["outputProperties"]: container_request["output_properties"][pr["propertyName"]] = self.builder.do_eval(pr["propertyValue"]) - output_properties.update(intermediate_collection_info["properties"]) + container_request["output_properties"].update(intermediate_collection_info["properties"]) if runtimeContext.runnerjob.startswith("arvwf:"): wfuuid = runtimeContext.runnerjob[6:runtimeContext.runnerjob.index("#")] diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py index 381b885431..1a4bf65a57 100644 --- a/sdk/cwl/arvados_cwl/executor.py +++ b/sdk/cwl/arvados_cwl/executor.py @@ -794,7 +794,7 @@ The 'jobs' API is no longer supported. output_properties = {} output_properties_req, _ = self.get_requirement("http://arvados.org/cwl#OutputCollectionProperties") if output_properties_req: - for pr in output_properties_req["processProperties"]: + for pr in output_properties_req["outputProperties"]: output_properties[pr["propertyName"]] = self.builder.do_eval(pr["propertyValue"]) self.final_output, self.final_output_collection = self.make_output_collection(self.output_name, storage_classes, diff --git a/sdk/cwl/arvados_cwl/util.py b/sdk/cwl/arvados_cwl/util.py index 85ae65ecf1..a0dfb290c1 100644 --- a/sdk/cwl/arvados_cwl/util.py +++ b/sdk/cwl/arvados_cwl/util.py @@ -16,9 +16,9 @@ def get_intermediate_collection_info(workflow_step_name, current_container, inte if intermediate_output_ttl > 0: trash_time = datetime.datetime.utcnow() + datetime.timedelta(seconds=intermediate_output_ttl) container_uuid = None + props = {"type": "intermediate"} if current_container: - container_uuid = current_container['uuid'] - props = {"type": "intermediate", "container": container_uuid} + props["container"] = current_container['uuid'] return {"name" : name, "trash_at" : trash_time, "properties" : props} diff --git a/services/api/app/models/container_request.rb b/services/api/app/models/container_request.rb index 74fc0802d3..605b376cf5 100644 --- a/services/api/app/models/container_request.rb +++ b/services/api/app/models/container_request.rb @@ -242,7 +242,11 @@ class ContainerRequest < ArvadosModel merged_properties.update(self.output_properties) end - merged_properties['type'] = out_type + if out_type == 'output' and !requesting_container_uuid.nil? + merged_properties['type'] = 'intermediate' + else + merged_properties['type'] = out_type + end merged_properties['container_request'] = uuid coll.assign_attributes(