14018: Setting process properties WIP

[arvados.git] / sdk / cwl / arvados_cwl / arvcontainer.py

diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 72ef14f6731baf83de87df28534a5c76b5a7dc42..c6e4577d03bcd8d36af869d8b6ec63cc2531bc47 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -67,15 +67,21 @@ class ArvadosContainer(JobBase):
 
         runtimeContext = self.job_runtime
 
-        container_request = {
-            "command": self.command_line,
-            "name": self.name,
-            "output_path": self.outdir,
-            "cwd": self.outdir,
-            "priority": runtimeContext.priority,
-            "state": "Committed",
-            "properties": {},
-        }
+        if runtimeContext.submit_request_uuid:
+            container_request = self.arvrunner.api.container_requests().get(
+                uuid=runtimeContext.submit_request_uuid
+            ).execute(num_retries=self.arvrunner.num_retries)
+        else:
+            container_request = {}
+
+        container_request["command"] = self.command_line
+        container_request["name"] = self.name
+        container_request["output_path"] = self.outdir
+        container_request["cwd"] = self.outdir
+        container_request["priority"] = runtimeContext.priority
+        container_request["state"] = "Committed"
+        container_request.setdefault("properties", {})
+
         runtime_constraints = {}
 
         if runtimeContext.project_uuid:
@@ -273,6 +279,12 @@ class ArvadosContainer(JobBase):
         if self.output_ttl < 0:
             raise WorkflowException("Invalid value %d for output_ttl, cannot be less than zero" % container_request["output_ttl"])
 
+        storage_class_req, _ = self.get_requirement("http://arvados.org/cwl#OutputStorageClass")
+        if storage_class_req and storage_class_req.get("intermediateStorageClass"):
+            container_request["output_storage_classes"] = aslist(storage_class_req["intermediateStorageClass"])
+        else:
+            container_request["output_storage_classes"] = runtimeContext.intermediate_storage_classes.strip().split(",")
+
         if self.timelimit is not None and self.timelimit > 0:
             scheduling_parameters["max_run_time"] = self.timelimit
 
@@ -297,6 +309,11 @@ class ArvadosContainer(JobBase):
                 enable_reuse = reuse_req["enableReuse"]
         container_request["use_existing"] = enable_reuse
 
+        properties_req, _ = self.get_requirement("http://arvados.org/cwl#ProcessProperties")
+        if properties_req:
+            for pr in properties_req["processProperties"]:
+                container_request["properties"][pr["propertyName"]] = self.builder.do_eval(pr["propertyValue"])
+
         if runtimeContext.runnerjob.startswith("arvwf:"):
             wfuuid = runtimeContext.runnerjob[6:runtimeContext.runnerjob.index("#")]
             wfrecord = self.arvrunner.api.workflows().get(uuid=wfuuid).execute(num_retries=self.arvrunner.num_retries)
@@ -462,6 +479,11 @@ class RunnerContainer(Runner):
             if self.embedded_tool.tool.get("id", "").startswith("arvwf:"):
                 container_req["properties"]["template_uuid"] = self.embedded_tool.tool["id"][6:33]
 
+        properties_req, _ = self.embedded_tool.get_requirement("http://arvados.org/cwl#ProcessProperties")
+        if properties_req:
+            builder = make_builder(self.job_order, self.embedded_tool.hints, self.embedded_tool.requirements, runtimeContext, self.embedded_tool.metadata)
+            for pr in properties_req["processProperties"]:
+                container_request["properties"][pr["propertyName"]] = self.builder.do_eval(pr["propertyValue"])
 
         # --local means execute the workflow instead of submitting a container request
         # --api=containers means use the containers API
@@ -495,6 +517,9 @@ class RunnerContainer(Runner):
         if runtimeContext.storage_classes != "default":
             command.append("--storage-classes=" + runtimeContext.storage_classes)
 
+        if runtimeContext.intermediate_storage_classes != "default":
+            command.append("--intermediate-storage-classes=" + runtimeContext.intermediate_storage_classes)
+
         if self.on_error:
             command.append("--on-error=" + self.on_error)