Merge branch 'master' of git.curoverse.com:arvados into 13330-clean-up-collection...

[arvados.git] / sdk / cwl / arvados_cwl / arvcontainer.py

diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 4e7811d2e8f5b0b477b82334b79385618c3456b9..04893cc455873d385aca9bad3e37efdbf4fba1af 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -5,14 +5,16 @@
 import logging
 import json
 import os
+import datetime
 import urllib
 import time
-import datetime
 import ciso8601
 import uuid
 
+
 import ruamel.yaml as yaml
 
+from arvados.errors import ApiError
 from cwltool.errors import WorkflowException
 from cwltool.process import get_feature, UnsupportedRequirement, shortname
 from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
@@ -154,8 +156,33 @@ class ArvadosContainer(object):
 
                 keepemptydirs(vwd)
 
+                trash_time = None
+                if self.arvrunner.intermediate_output_ttl > 0:
+                    t = datetime.datetime.now() + datetime.timedelta(seconds=self.arvrunner.intermediate_output_ttl)
+                    trash_time = t.strftime("%Y-%m-%dT%H:%M:%S.%f000Z")
+
+                current_container_uuid = None
+                try:
+                    current_container = self.arvrunner.api.containers().current().execute(num_retries=self.arvrunner.num_retries)
+                    current_container_uuid = current_container['uuid']
+                except ApiError as e:
+                    # Status code 404 just means we're not running in a container.
+                    if e.resp.status != 404:
+                        logger.info("Getting current container: %s", e)
+                props = {"type": "Intermediate",
+                         "container": current_container_uuid}
+
                 with Perf(metrics, "generatefiles.save_new %s" % self.name):
-                    vwd.save_new()
+                    vwd.save_new(name="Intermediate collection",
+                                 ensure_unique_name=True,
+                                 trash_at=trash_time,
+                                 properties=props)
+
+                logger.debug("Intermediate: arvcontainer")
+                logger.debug("Intermediate: uuid = %s" % vwd._api_response["uuid"])
+                logger.debug("Intermediate: name = %s" % vwd._api_response["name"])
+                logger.debug("Intermediate: trash_at = %s" % vwd._api_response["trash_at"])
+                logger.debug("Intermediate: props = " + str(vwd._api_response["properties"]))
 
                 prev = None
                 for f, p in sorteditems: