From: Peter Amstutz Date: Thu, 5 May 2022 20:00:37 +0000 (-0400) Subject: 17004: Initial CWL support for setting output properties X-Git-Tag: 2.5.0~165^2~11 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/60e38e35f49de11b5752c2bfb7f9f10443d82c49 17004: Initial CWL support for setting output properties Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- diff --git a/doc/api/methods/container_requests.html.textile.liquid b/doc/api/methods/container_requests.html.textile.liquid index 8704701105..cc5c19d792 100644 --- a/doc/api/methods/container_requests.html.textile.liquid +++ b/doc/api/methods/container_requests.html.textile.liquid @@ -61,6 +61,7 @@ table(table table-bordered table-condensed). |runtime_user_uuid|string|The user permission that will be granted to this container.|| |runtime_auth_scopes|array of string|The scopes associated with the auth token used to run this container.|| |output_storage_classes|array of strings|The storage classes that will be used for the log and output collections of this container request|default is ["default"]| +|output_properties|hash|User metadata properties to set on the output collection.| h2(#priority). Priority diff --git a/doc/api/methods/containers.html.textile.liquid b/doc/api/methods/containers.html.textile.liquid index 18fb4f0133..76e5730c9f 100644 --- a/doc/api/methods/containers.html.textile.liquid +++ b/doc/api/methods/containers.html.textile.liquid @@ -60,6 +60,7 @@ Generally this will contain additional keys that are not present in any correspo |gateway_address|string|Address (host:port) of gateway server.|Internal use only.| |interactive_session_started|boolean|Indicates whether @arvados-client shell@ has been used to run commands in the container, which may have altered the container's behavior and output.|| |output_storage_classes|array of strings|The storage classes that will be used for the log and output collections of this container|| +|output_properties|hash|User metadata properties to set on the output collection.| h2(#container_states). Container states diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py index 21b629f37a..08a05d571c 100644 --- a/sdk/cwl/arvados_cwl/__init__.py +++ b/sdk/cwl/arvados_cwl/__init__.py @@ -265,6 +265,7 @@ def add_arv_hints(): "http://arvados.org/cwl#ProcessProperties", "http://commonwl.org/cwltool#CUDARequirement", "http://arvados.org/cwl#UsePreemptible", + "http://arvados.org/cwl#OutputCollectionProperties", ]) def exit_signal_handler(sigcode, frame): diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml index af75481431..73b8120973 100644 --- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml +++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml @@ -299,8 +299,8 @@ $graph: - type: record name: PropertyDef doc: | - Define a property that will be set on the submitted container - request associated with this workflow or step. + Define an arvados metadata property that will be set on a + container request or output collection. fields: - name: propertyName type: string @@ -400,3 +400,23 @@ $graph: _id: "@type" _type: "@vocab" usePreemptible: boolean + +- name: OutputCollectionProperties + type: record + extends: cwl:ProcessRequirement + inVocab: false + doc: | + Specify metadata properties that will be set on the output + collection associated with this workflow or step. + fields: + class: + type: string + doc: "Always 'arv:OutputCollectionProperties" + jsonldPredicate: + _id: "@type" + _type: "@vocab" + processProperties: + type: PropertyDef[] + jsonldPredicate: + mapSubject: propertyName + mapPredicate: propertyValue diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml index 0ae451ccaa..2044dac2a1 100644 --- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml +++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml @@ -343,3 +343,23 @@ $graph: _id: "@type" _type: "@vocab" usePreemptible: boolean + +- name: OutputCollectionProperties + type: record + extends: cwl:ProcessRequirement + inVocab: false + doc: | + Specify metadata properties that will be set on the output + collection associated with this workflow or step. + fields: + class: + type: string + doc: "Always 'arv:OutputCollectionProperties" + jsonldPredicate: + _id: "@type" + _type: "@vocab" + processProperties: + type: PropertyDef[] + jsonldPredicate: + mapSubject: propertyName + mapPredicate: propertyValue diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml index de5e55ca01..c969eef252 100644 --- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml +++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml @@ -345,3 +345,23 @@ $graph: _id: "@type" _type: "@vocab" usePreemptible: boolean + +- name: OutputCollectionProperties + type: record + extends: cwl:ProcessRequirement + inVocab: false + doc: | + Specify metadata properties that will be set on the output + collection associated with this workflow or step. + fields: + class: + type: string + doc: "Always 'arv:OutputCollectionProperties" + jsonldPredicate: + _id: "@type" + _type: "@vocab" + processProperties: + type: PropertyDef[] + jsonldPredicate: + mapSubject: propertyName + mapPredicate: propertyValue diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py index 5082cc2f4b..7964486ff2 100644 --- a/sdk/cwl/arvados_cwl/arvcontainer.py +++ b/sdk/cwl/arvados_cwl/arvcontainer.py @@ -341,6 +341,11 @@ class ArvadosContainer(JobBase): for pr in properties_req["processProperties"]: container_request["properties"][pr["propertyName"]] = self.builder.do_eval(pr["propertyValue"]) + output_properties_req, _ = self.get_requirement("http://arvados.org/cwl#OutputCollectionProperties") + if output_properties_req: + for pr in output_properties_req["processProperties"]: + container_request["output_properties"][pr["propertyName"]] = self.builder.do_eval(pr["propertyValue"]) + if runtimeContext.runnerjob.startswith("arvwf:"): wfuuid = runtimeContext.runnerjob[6:runtimeContext.runnerjob.index("#")] wfrecord = self.arvrunner.api.workflows().get(uuid=wfuuid).execute(num_retries=self.arvrunner.num_retries) diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py index 1759e4ac28..381b885431 100644 --- a/sdk/cwl/arvados_cwl/executor.py +++ b/sdk/cwl/arvados_cwl/executor.py @@ -404,7 +404,7 @@ The 'jobs' API is no longer supported. with SourceLine(obj, i, UnsupportedRequirement, logger.isEnabledFor(logging.DEBUG)): self.check_features(v, parentfield=parentfield) - def make_output_collection(self, name, storage_classes, tagsString, outputObj): + def make_output_collection(self, name, storage_classes, tagsString, output_properties, outputObj): outputObj = copy.deepcopy(outputObj) files = [] @@ -456,7 +456,9 @@ The 'jobs' API is no longer supported. res = str(json.dumps(outputObj, sort_keys=True, indent=4, separators=(',',': '), ensure_ascii=False)) f.write(res) - final.save_new(name=name, owner_uuid=self.project_uuid, storage_classes=storage_classes, ensure_unique_name=True) + + final.save_new(name=name, owner_uuid=self.project_uuid, storage_classes=storage_classes, + ensure_unique_name=True, properties=output_properties) logger.info("Final output collection %s \"%s\" (%s)", final.portable_data_hash(), final.api_response()["name"], @@ -486,6 +488,7 @@ The 'jobs' API is no longer supported. self.api.containers().update(uuid=current['uuid'], body={ 'output': self.final_output_collection.portable_data_hash(), + 'output_properties': self.final_output_collection.get_properties(), }).execute(num_retries=self.num_retries) self.api.collections().update(uuid=self.final_output_collection.manifest_locator(), body={ @@ -788,7 +791,15 @@ The 'jobs' API is no longer supported. else: storage_classes = runtimeContext.storage_classes.strip().split(",") - self.final_output, self.final_output_collection = self.make_output_collection(self.output_name, storage_classes, self.output_tags, self.final_output) + output_properties = {} + output_properties_req, _ = self.get_requirement("http://arvados.org/cwl#OutputCollectionProperties") + if output_properties_req: + for pr in output_properties_req["processProperties"]: + output_properties[pr["propertyName"]] = self.builder.do_eval(pr["propertyValue"]) + + self.final_output, self.final_output_collection = self.make_output_collection(self.output_name, storage_classes, + self.output_tags, output_properties, + self.final_output) self.set_crunch_output() if runtimeContext.compute_checksum: