Merge branch 'master' of git.curoverse.com:arvados into 13330-cwl-intermediate-collec...

[arvados.git] / sdk / cwl / arvados_cwl / arvcontainer.py

diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 8e98e9604225f57b308a095a23e03dd7b2382d22..03f5a5eb678201f9610cf5fd76e085260695b742 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -21,6 +21,7 @@ from cwltool.job import JobBase
 
 import arvados.collection
 
+from arvados.errors import ApiError
 from .arvdocker import arv_docker_get_image
 from . import done
 from .runner import Runner, arvados_jobs_image, packed_workflow, trim_anonymous_location, remove_redundant_fields
@@ -165,8 +166,11 @@ class ArvadosContainer(JobBase):
 
                 keepemptydirs(vwd)
 
-                with Perf(metrics, "generatefiles.save_new %s" % self.name):
-                    vwd.save_new()
+                info = self._get_intermediate_collection_info()
+                vwd.save_new(name=info["name"],
+                             ensure_unique_name=True,
+                             trash_at=info["trash_at"],
+                             properties=info["properties"])
 
                 prev = None
                 for f, p in sorteditems:
@@ -239,6 +243,9 @@ class ArvadosContainer(JobBase):
         if self.output_ttl < 0:
             raise WorkflowException("Invalid value %d for output_ttl, cannot be less than zero" % container_request["output_ttl"])
 
+        if self.timelimit is not None:
+            scheduling_parameters["max_run_time"] = self.timelimit
+
         container_request["output_ttl"] = self.output_ttl
         container_request["mounts"] = mounts
         container_request["secret_mounts"] = secret_mounts
@@ -334,6 +341,26 @@ class ArvadosContainer(JobBase):
         finally:
             self.output_callback(outputs, processStatus)
 
+    def _get_intermediate_collection_info(self):
+            trash_time = None
+            if self.arvrunner.intermediate_output_ttl > 0:
+                trash_time = datetime.datetime.now() + datetime.timedelta(seconds=self.arvrunner.intermediate_output_ttl)
+
+            current_container_uuid = None
+            try:
+                current_container = self.arvrunner.api.containers().current().execute(num_retries=self.arvrunner.num_retries)
+                current_container_uuid = current_container['uuid']
+            except ApiError as e:
+                # Status code 404 just means we're not running in a container.
+                if e.resp.status != 404:
+                    logger.info("Getting current container: %s", e)
+            props = {"type": "Intermediate",
+                          "container": current_container_uuid}
+
+            return {"name" : "Intermediate collection",
+                    "trash_at" : trash_time,
+                    "properties" : props}
+
 
 class RunnerContainer(Runner):
     """Submit and manage a container that runs arvados-cwl-runner."""
@@ -437,7 +464,7 @@ class RunnerContainer(Runner):
             command.append("--debug")
 
         if runtimeContext.storage_classes != "default":
-            command.append("--storage-classes=" + runtimeContext.storage_classes
+            command.append("--storage-classes=" + runtimeContext.storage_classes)
 
         if self.on_error:
             command.append("--on-error=" + self.on_error)