17004: Initial CWL support for setting output properties

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz@curii.com>

diff --git a/doc/api/methods/container_requests.html.textile.liquid b/doc/api/methods/container_requests.html.textile.liquid
index 8704701105bf052b8ac89561075631542051d719..cc5c19d7929e660fb0105ee9e11c27f464438328 100644 (file)
--- a/doc/api/methods/container_requests.html.textile.liquid
+++ b/doc/api/methods/container_requests.html.textile.liquid
@@ -61,6 +61,7 @@ table(table table-bordered table-condensed).
 |runtime_user_uuid|string|The user permission that will be granted to this container.||
 |runtime_auth_scopes|array of string|The scopes associated with the auth token used to run this container.||
 |output_storage_classes|array of strings|The storage classes that will be used for the log and output collections of this container request|default is ["default"]|
 |runtime_user_uuid|string|The user permission that will be granted to this container.||
 |runtime_auth_scopes|array of string|The scopes associated with the auth token used to run this container.||
 |output_storage_classes|array of strings|The storage classes that will be used for the log and output collections of this container request|default is ["default"]|

+|output_properties|hash|User metadata properties to set on the output collection.|

 
 h2(#priority). Priority
 
 
 h2(#priority). Priority
 

diff --git a/doc/api/methods/containers.html.textile.liquid b/doc/api/methods/containers.html.textile.liquid
index 18fb4f01330e033f06086f3125d8d01680955ee3..76e5730c9f14605989cb9f179a3997e1990c06a3 100644 (file)
--- a/doc/api/methods/containers.html.textile.liquid
+++ b/doc/api/methods/containers.html.textile.liquid
@@ -60,6 +60,7 @@ Generally this will contain additional keys that are not present in any correspo
 |gateway_address|string|Address (host:port) of gateway server.|Internal use only.|
 |interactive_session_started|boolean|Indicates whether @arvados-client shell@ has been used to run commands in the container, which may have altered the container's behavior and output.||
 |output_storage_classes|array of strings|The storage classes that will be used for the log and output collections of this container||
 |gateway_address|string|Address (host:port) of gateway server.|Internal use only.|
 |interactive_session_started|boolean|Indicates whether @arvados-client shell@ has been used to run commands in the container, which may have altered the container's behavior and output.||
 |output_storage_classes|array of strings|The storage classes that will be used for the log and output collections of this container||

+|output_properties|hash|User metadata properties to set on the output collection.|

 
 h2(#container_states). Container states
 
 
 h2(#container_states). Container states
 

diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 21b629f37ab9b50812415ecdad4a766345b28f32..08a05d571cb8e41bb48265489fcec9f13b1e6100 100644 (file)
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -265,6 +265,7 @@ def add_arv_hints():
         "http://arvados.org/cwl#ProcessProperties",
         "http://commonwl.org/cwltool#CUDARequirement",
         "http://arvados.org/cwl#UsePreemptible",
         "http://arvados.org/cwl#ProcessProperties",
         "http://commonwl.org/cwltool#CUDARequirement",
         "http://arvados.org/cwl#UsePreemptible",

+        "http://arvados.org/cwl#OutputCollectionProperties",

     ])
 
 def exit_signal_handler(sigcode, frame):
     ])
 
 def exit_signal_handler(sigcode, frame):

diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml
index af75481431b6cad88a3b346f180544617654045a..73b8120973b4aea9a9aabac06d6f3d67178a3396 100644 (file)
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml
@@ -299,8 +299,8 @@ $graph:
 - type: record
   name: PropertyDef
   doc: |
 - type: record
   name: PropertyDef
   doc: |

-    Define a property that will be set on the submitted container
-    request associated with this workflow or step.
+    Define an arvados metadata property that will be set on a
+    container request or output collection.

   fields:
     - name: propertyName
       type: string
   fields:
     - name: propertyName
       type: string


@@ -400,3 +400,23 @@ $graph:
         _id: "@type"
         _type: "@vocab"
     usePreemptible: boolean
         _id: "@type"
         _type: "@vocab"
     usePreemptible: boolean

+
+- name: OutputCollectionProperties
+  type: record
+  extends: cwl:ProcessRequirement
+  inVocab: false
+  doc: |
+    Specify metadata properties that will be set on the output
+    collection associated with this workflow or step.
+  fields:
+    class:
+      type: string
+      doc: "Always 'arv:OutputCollectionProperties"
+      jsonldPredicate:
+        _id: "@type"
+        _type: "@vocab"
+    processProperties:
+      type: PropertyDef[]
+      jsonldPredicate:
+        mapSubject: propertyName
+        mapPredicate: propertyValue

diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml
index 0ae451ccaac78e8173e0278d9745038d66b47b0e..2044dac2a1227e4ed977840ab5bbe32f69dad9bd 100644 (file)
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml
@@ -343,3 +343,23 @@ $graph:
         _id: "@type"
         _type: "@vocab"
     usePreemptible: boolean
         _id: "@type"
         _type: "@vocab"
     usePreemptible: boolean

+
+- name: OutputCollectionProperties
+  type: record
+  extends: cwl:ProcessRequirement
+  inVocab: false
+  doc: |
+    Specify metadata properties that will be set on the output
+    collection associated with this workflow or step.
+  fields:
+    class:
+      type: string
+      doc: "Always 'arv:OutputCollectionProperties"
+      jsonldPredicate:
+        _id: "@type"
+        _type: "@vocab"
+    processProperties:
+      type: PropertyDef[]
+      jsonldPredicate:
+        mapSubject: propertyName
+        mapPredicate: propertyValue

diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
index de5e55ca01164fa3d86454cdcf1d249a66018e2e..c969eef2523385acc8187988f0aa20b041306131 100644 (file)
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
@@ -345,3 +345,23 @@ $graph:
         _id: "@type"
         _type: "@vocab"
     usePreemptible: boolean
         _id: "@type"
         _type: "@vocab"
     usePreemptible: boolean

+
+- name: OutputCollectionProperties
+  type: record
+  extends: cwl:ProcessRequirement
+  inVocab: false
+  doc: |
+    Specify metadata properties that will be set on the output
+    collection associated with this workflow or step.
+  fields:
+    class:
+      type: string
+      doc: "Always 'arv:OutputCollectionProperties"
+      jsonldPredicate:
+        _id: "@type"
+        _type: "@vocab"
+    processProperties:
+      type: PropertyDef[]
+      jsonldPredicate:
+        mapSubject: propertyName
+        mapPredicate: propertyValue

diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 5082cc2f4b57eacd0934019099509c2f42c7493b..7964486ff25b2584af54ef84da8b4327207daabb 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -341,6 +341,11 @@ class ArvadosContainer(JobBase):
             for pr in properties_req["processProperties"]:
                 container_request["properties"][pr["propertyName"]] = self.builder.do_eval(pr["propertyValue"])
 
             for pr in properties_req["processProperties"]:
                 container_request["properties"][pr["propertyName"]] = self.builder.do_eval(pr["propertyValue"])
 

+        output_properties_req, _ = self.get_requirement("http://arvados.org/cwl#OutputCollectionProperties")
+        if output_properties_req:
+            for pr in output_properties_req["processProperties"]:
+                container_request["output_properties"][pr["propertyName"]] = self.builder.do_eval(pr["propertyValue"])
+

         if runtimeContext.runnerjob.startswith("arvwf:"):
             wfuuid = runtimeContext.runnerjob[6:runtimeContext.runnerjob.index("#")]
             wfrecord = self.arvrunner.api.workflows().get(uuid=wfuuid).execute(num_retries=self.arvrunner.num_retries)
         if runtimeContext.runnerjob.startswith("arvwf:"):
             wfuuid = runtimeContext.runnerjob[6:runtimeContext.runnerjob.index("#")]
             wfrecord = self.arvrunner.api.workflows().get(uuid=wfuuid).execute(num_retries=self.arvrunner.num_retries)

diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index 1759e4ac2829a4840895d47e465fdfcad6a2bf1d..381b88543145c9ae7f98c5ec36aaab783b8c24ab 100644 (file)
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -404,7 +404,7 @@ The 'jobs' API is no longer supported.
                 with SourceLine(obj, i, UnsupportedRequirement, logger.isEnabledFor(logging.DEBUG)):
                     self.check_features(v, parentfield=parentfield)
 
                 with SourceLine(obj, i, UnsupportedRequirement, logger.isEnabledFor(logging.DEBUG)):
                     self.check_features(v, parentfield=parentfield)
 

-    def make_output_collection(self, name, storage_classes, tagsString, outputObj):
+    def make_output_collection(self, name, storage_classes, tagsString, output_properties, outputObj):

         outputObj = copy.deepcopy(outputObj)
 
         files = []
         outputObj = copy.deepcopy(outputObj)
 
         files = []


@@ -456,7 +456,9 @@ The 'jobs' API is no longer supported.
             res = str(json.dumps(outputObj, sort_keys=True, indent=4, separators=(',',': '), ensure_ascii=False))
             f.write(res)
 
             res = str(json.dumps(outputObj, sort_keys=True, indent=4, separators=(',',': '), ensure_ascii=False))
             f.write(res)
 

-        final.save_new(name=name, owner_uuid=self.project_uuid, storage_classes=storage_classes, ensure_unique_name=True)
+
+        final.save_new(name=name, owner_uuid=self.project_uuid, storage_classes=storage_classes,
+                       ensure_unique_name=True, properties=output_properties)

 
         logger.info("Final output collection %s \"%s\" (%s)", final.portable_data_hash(),
                     final.api_response()["name"],
 
         logger.info("Final output collection %s \"%s\" (%s)", final.portable_data_hash(),
                     final.api_response()["name"],


@@ -486,6 +488,7 @@ The 'jobs' API is no longer supported.
                 self.api.containers().update(uuid=current['uuid'],
                                              body={
                                                  'output': self.final_output_collection.portable_data_hash(),
                 self.api.containers().update(uuid=current['uuid'],
                                              body={
                                                  'output': self.final_output_collection.portable_data_hash(),

+                                                 'output_properties': self.final_output_collection.get_properties(),

                                              }).execute(num_retries=self.num_retries)
                 self.api.collections().update(uuid=self.final_output_collection.manifest_locator(),
                                               body={
                                              }).execute(num_retries=self.num_retries)
                 self.api.collections().update(uuid=self.final_output_collection.manifest_locator(),
                                               body={


@@ -788,7 +791,15 @@ The 'jobs' API is no longer supported.
             else:
                 storage_classes = runtimeContext.storage_classes.strip().split(",")
 
             else:
                 storage_classes = runtimeContext.storage_classes.strip().split(",")
 

-            self.final_output, self.final_output_collection = self.make_output_collection(self.output_name, storage_classes, self.output_tags, self.final_output)
+            output_properties = {}
+            output_properties_req, _ = self.get_requirement("http://arvados.org/cwl#OutputCollectionProperties")
+            if output_properties_req:
+                for pr in output_properties_req["processProperties"]:
+                    output_properties[pr["propertyName"]] = self.builder.do_eval(pr["propertyValue"])
+
+            self.final_output, self.final_output_collection = self.make_output_collection(self.output_name, storage_classes,
+                                                                                          self.output_tags, output_properties,
+                                                                                          self.final_output)

             self.set_crunch_output()
 
         if runtimeContext.compute_checksum:
             self.set_crunch_output()
 
         if runtimeContext.compute_checksum: