Merge branch 'master' of git.curoverse.com:arvados into 13330-clean-up-collection...

[arvados.git] / sdk / cwl / arvados_cwl / arvjob.py

diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index 04256c68f8b10f47ede2fefcabb0172948c2ff00..8d1617146881ab9a0a3b8d208e40c328d886235f 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -7,6 +7,7 @@ import re
 import copy
 import json
 import time
+import datetime
 
 from cwltool.process import get_feature, shortname, UnsupportedRequirement
 from cwltool.errors import WorkflowException
@@ -66,8 +67,34 @@ class ArvadosJob(object):
                                 n.write(p.resolved.encode("utf-8"))
 
                 if vwd:
+                    trash_time = None
+                    if self.arvrunner.intermediate_output_ttl > 0:
+                        t = datetime.now() + datetime.timedelta(seconds=self.arvrunner.intermediate_output_ttl)
+                        trash_time = t.strftime("%Y-%m-%dT%H:%M:%S.%f000Z")
+
+                    current_container_uuid = None
+                    try:
+                        current_container = self.arvrunner.api.containers().current().execute(num_retries=self.arvrunner.num_retries)
+                        current_container_uuid = current_container['uuid']
+                    except ApiError as e:
+                        # Status code 404 just means we're not running in a container.
+                        if e.resp.status != 404:
+                            logger.info("Getting current container: %s", e)
+
+                    props = {"type": "Intermediate",
+                             "container": current_container_uuid}
+
                     with Perf(metrics, "generatefiles.save_new %s" % self.name):
-                        vwd.save_new()
+                        vwd.save_new(name="Intermediate collection",
+                                     ensure_unique_name=True,
+                                     trash_at=trash_time,
+                                     properties=props)
+
+                    logger.debug("Intermediate: arvjob")
+                    logger.debug("Intermediate: uuid = %s" % vwd._api_response["uuid"])
+                    logger.debug("Intermediate: name = %s" % vwd._api_response["name"])
+                    logger.debug("Intermediate: trash_at = %s" % vwd._api_response["trash_at"])
+                    logger.debug("Intermediate: props = " + str(vwd._api_response["properties"]))
 
                 for f, p in generatemapper.items():
                     if p.type == "File":