Add tests for intermediate collections

Arvados-DCO-1.1-Signed-off-by: Fuad Muhic <fmuhic@capeannenterprises.com>

diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 1ad276e67a6810575ec570e4cd1b771098687d78..014e8e58cfb74cd7564422c76d85fb99f94911f4 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -157,26 +157,11 @@ class ArvadosContainer(object):
 
                 keepemptydirs(vwd)
 
-                trash_time = None 
-                if self.arvrunner.intermediate_output_ttl > 0: 
-                    trash_time = datetime.datetime.now() + datetime.timedelta(seconds=self.arvrunner.intermediate_output_ttl) 
- 
-                current_container_uuid = None 
-                try: 
-                    current_container = self.arvrunner.api.containers().current().execute(num_retries=self.arvrunner.num_retries) 
-                    current_container_uuid = current_container['uuid'] 
-                except ApiError as e: 
-                    # Status code 404 just means we're not running in a container. 
-                    if e.resp.status != 404: 
-                        logger.info("Getting current container: %s", e) 
-                props = {"type": "Intermediate", 
-                         "container": current_container_uuid}
-
-                with Perf(metrics, "generatefiles.save_new %s" % self.name):
-                    vwd.save_new(name="Intermediate collection", 
-                                 ensure_unique_name=True, 
-                                 trash_at=trash_time, 
-                                 properties=props)
+                info = self._get_intermediate_collection_info()
+                vwd.save_new(name=info["name"], 
+                             ensure_unique_name=True, 
+                             trash_at=info["trash_at"], 
+                             properties=info["properties"])
 
                 prev = None
                 for f, p in sorteditems:
@@ -344,6 +329,26 @@ class ArvadosContainer(object):
         finally:
             self.output_callback(outputs, processStatus)
 
+    def _get_intermediate_collection_info(self):
+            trash_time = None 
+            if self.arvrunner.intermediate_output_ttl > 0: 
+                trash_time = datetime.datetime.now() + datetime.timedelta(seconds=self.arvrunner.intermediate_output_ttl) 
+
+            current_container_uuid = None 
+            try: 
+                current_container = self.arvrunner.api.containers().current().execute(num_retries=self.arvrunner.num_retries) 
+                current_container_uuid = current_container['uuid'] 
+            except ApiError as e: 
+                # Status code 404 just means we're not running in a container. 
+                if e.resp.status != 404: 
+                    logger.info("Getting current container: %s", e)
+            props = {"type": "Intermediate", 
+                          "container": current_container_uuid}
+
+            return {"name" : "Intermediate collection",
+                    "trash_at" : trash_time,
+                    "properties" : props}
+
 
 class RunnerContainer(Runner):
     """Submit and manage a container that runs arvados-cwl-runner."""

diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index 2d98a53b33f3f2a7835a4f73392f9c2cb7e5ab1d..ea599ea8371caaba0cae30a8f97a010ff13f1425 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -67,26 +67,12 @@ class ArvadosJob(object):
                                 n.write(p.resolved.encode("utf-8"))
 
                 if vwd:
-                    trash_time = None 
-                    if self.arvrunner.intermediate_output_ttl > 0: 
-                        trash_time = datetime.datetime.now() + datetime.timedelta(seconds=self.arvrunner.intermediate_output_ttl) 
- 
-                    current_container_uuid = None 
-                    try: 
-                        current_container = self.arvrunner.api.containers().current().execute(num_retries=self.arvrunner.num_retries) 
-                        current_container_uuid = current_container['uuid'] 
-                    except ApiError as e: 
-                        # Status code 404 just means we're not running in a container. 
-                        if e.resp.status != 404: 
-                            logger.info("Getting current container: %s", e) 
- 
-                    props = {"type": "Intermediate", 
-                             "container": current_container_uuid}
                     with Perf(metrics, "generatefiles.save_new %s" % self.name):
-                        vwd.save_new(name="Intermediate collection", 
+                        info = self._get_intermediate_collection_info()
+                        vwd.save_new(name=info["name"], 
                                      ensure_unique_name=True, 
-                                     trash_at=trash_time, 
-                                     properties=props)
+                                     trash_at=info["trash_at"], 
+                                     properties=info["properties"])
 
                 for f, p in generatemapper.items():
                     if p.type == "File":
@@ -284,6 +270,26 @@ class ArvadosJob(object):
         finally:
             self.output_callback(outputs, processStatus)
 
+    def _get_intermediate_collection_info(self):
+            trash_time = None 
+            if self.arvrunner.intermediate_output_ttl > 0: 
+                trash_time = datetime.datetime.now() + datetime.timedelta(seconds=self.arvrunner.intermediate_output_ttl) 
+
+            current_container_uuid = None 
+            try: 
+                current_container = self.arvrunner.api.containers().current().execute(num_retries=self.arvrunner.num_retries) 
+                current_container_uuid = current_container['uuid'] 
+            except ApiError as e: 
+                # Status code 404 just means we're not running in a container. 
+                if e.resp.status != 404: 
+                    logger.info("Getting current container: %s", e)
+            props = {"type": "Intermediate", 
+                          "container": current_container_uuid}
+
+            return {"name" : "Intermediate collection",
+                    "trash_at" : trash_time,
+                    "properties" : props}
+
 
 class RunnerJob(Runner):
     """Submit and manage a Crunch job that runs crunch_scripts/cwl-runner."""

diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index a749b51625b3af008e18ac4a727ca461c78a8712..b54fcdcc7b68f2d78c9fa8414271a8e298339a45 100644 (file)
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -155,13 +155,13 @@ class ArvPathMapper(PathMapper):
                 for l in srcobj.get("listing", []):
                     self.addentry(l, c, ".", remap)
 
-                trash_time, props = self.__get_collection_attributes()
+                info = self._get_intermediate_collection_info()
 
-                c.save_new(name="Intermediate collection", 
+                c.save_new(name=info["name"],
                            owner_uuid=self.arvrunner.project_uuid, 
                            ensure_unique_name=True, 
-                           trash_at=trash_time, 
-                           properties=props)
+                           trash_at=info["trash_at"], 
+                           properties=info["properties"])
 
                 ab = self.collection_pattern % c.portable_data_hash()
                 self._pathmap[srcobj["location"]] = MapperEnt("keep:"+c.portable_data_hash(), ab, "Directory", True)
@@ -173,13 +173,13 @@ class ArvPathMapper(PathMapper):
                                                   num_retries=self.arvrunner.num_retries                                                  )
                 self.addentry(srcobj, c, ".", remap)
 
-                trash_time, props = self.__get_collection_attributes()
+                info = self._get_intermediate_collection_info()
 
-                c.save_new(name="Intermediate collection", 
+                c.save_new(name=info["name"],
                            owner_uuid=self.arvrunner.project_uuid, 
                            ensure_unique_name=True, 
-                           trash_at=trash_time, 
-                           properties=props)
+                           trash_at=info["trash_at"], 
+                           properties=info["properties"])
 
                 ab = self.file_pattern % (c.portable_data_hash(), srcobj["basename"])
                 self._pathmap[srcobj["location"]] = MapperEnt("keep:%s/%s" % (c.portable_data_hash(), srcobj["basename"]),
@@ -212,7 +212,7 @@ class ArvPathMapper(PathMapper):
         else:
             return None
 
-    def __get_collection_attributes(self):
+    def _get_intermediate_collection_info(self):
             trash_time = None 
             if self.arvrunner.intermediate_output_ttl > 0: 
                 trash_time = datetime.datetime.now() + datetime.timedelta(seconds=self.arvrunner.intermediate_output_ttl) 
@@ -225,10 +225,12 @@ class ArvPathMapper(PathMapper):
                 # Status code 404 just means we're not running in a container. 
                 if e.resp.status != 404: 
                     logger.info("Getting current container: %s", e)
-            properties = {"type": "Intermediate", 
+            props = {"type": "Intermediate", 
                           "container": current_container_uuid}
 
-            return (trash_time, properties)
+            return {"name" : "Intermediate collection",
+                    "trash_at" : trash_time,
+                    "properties" : props}
 
 
 class StagingPathMapper(PathMapper):

diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index 2295e934ac77de76182d04749715a57f730874b4..a8a91efd3ad01432a281a9b14a107ebc2e7a88d2 100644 (file)
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -8,6 +8,7 @@ import logging
 import mock
 import unittest
 import os
+import datetime
 import functools
 import cwltool.process
 import cwltool.secrets
@@ -20,6 +21,12 @@ if not os.getenv('ARVADOS_DEBUG'):
     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
 
+class MockDateTime(datetime.datetime):
+    @classmethod
+    def now(cls):
+        return datetime.datetime(2018, 1, 1, 0, 0, 0, 0)
+
+datetime.datetime = MockDateTime
 
 class TestContainer(unittest.TestCase):
 
@@ -633,3 +640,16 @@ class TestContainer(unittest.TestCase):
                         }
                     }
                 }))
+
+    def test_get_intermediate_collection_info(self):
+        arvrunner = mock.MagicMock()
+        arvrunner.intermediate_output_ttl = 60
+        arvrunner.api.containers().current().execute.return_value = {"uuid" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"}
+
+        container = arvados_cwl.ArvadosContainer(arvrunner)
+
+        info = container._get_intermediate_collection_info()
+
+        self.assertEqual(info["name"], "Intermediate collection")
+        self.assertEqual(info["trash_at"], datetime.datetime(2018, 1, 1, 0, 1))
+        self.assertEqual(info["properties"], {"type" : "Intermediate", "container" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"})

diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py
index 30930dd49abb1a91fe371cc291cd916feac735df..1841fd3f89ea7a89aa1ae246f4f0dd5d305c9532 100644 (file)
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@ -10,6 +10,7 @@ import os
 import unittest
 import copy
 import StringIO
+import datetime
 
 import arvados
 import arvados_cwl
@@ -24,6 +25,13 @@ if not os.getenv('ARVADOS_DEBUG'):
     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
 
+class MockDateTime(datetime.datetime):
+    @classmethod
+    def now(cls):
+        return datetime.datetime(2018, 1, 1, 0, 0, 0, 0)
+
+datetime.datetime = MockDateTime
+
 class TestJob(unittest.TestCase):
 
     # The test passes no builder.resources
@@ -307,6 +315,20 @@ class TestJob(unittest.TestCase):
 
         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 
+    def test_get_intermediate_collection_info(self):
+        arvrunner = mock.MagicMock()
+        arvrunner.intermediate_output_ttl = 60
+        arvrunner.api.containers().current().execute.return_value = {"uuid" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"}
+
+        job = arvados_cwl.ArvadosJob(arvrunner)
+
+        info = job._get_intermediate_collection_info()
+
+        self.assertEqual(info["name"], "Intermediate collection")
+        self.assertEqual(info["trash_at"], datetime.datetime(2018, 1, 1, 0, 1))
+        self.assertEqual(info["properties"], {"type" : "Intermediate", "container" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"})
+
+
 
 class TestWorkflow(unittest.TestCase):
     # The test passes no builder.resources

diff --git a/sdk/cwl/tests/test_pathmapper.py b/sdk/cwl/tests/test_pathmapper.py
index eaa57114222233d6bcbd02ff2674c89f5169b168..e5326147d7737d9f0f0d2e52fda8120a54770ae0 100644 (file)
--- a/sdk/cwl/tests/test_pathmapper.py
+++ b/sdk/cwl/tests/test_pathmapper.py
@@ -9,6 +9,7 @@ import unittest
 import json
 import logging
 import os
+import datetime
 
 import arvados
 import arvados.keep
@@ -26,6 +27,13 @@ def upload_mock(files, api, dry_run=False, num_retries=0, project=None, fnPatter
         c.keepref = "%s/%s" % (pdh, os.path.basename(c.fn))
         c.fn = fnPattern % (pdh, os.path.basename(c.fn))
 
+class MockDateTime(datetime.datetime):
+    @classmethod
+    def now(cls):
+        return datetime.datetime(2018, 1, 1, 0, 0, 0, 0)
+
+datetime.datetime = MockDateTime
+
 class TestPathmap(unittest.TestCase):
     def setUp(self):
         self.api = mock.MagicMock()
@@ -101,3 +109,19 @@ class TestPathmap(unittest.TestCase):
                 "class": "File",
                 "location": "file:tests/hw.py"
             }], "", "/test/%s", "/test/%s/%s")
+
+    def test_get_intermediate_collection_info(self):
+        self.api.containers().current().execute.return_value = {"uuid" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"}
+        arvrunner = arvados_cwl.ArvCwlRunner(self.api)
+        arvrunner.intermediate_output_ttl = 60
+
+        path_mapper = ArvPathMapper(arvrunner, [{
+            "class": "File",
+            "location": "keep:99999999999999999999999999999991+99/hw.py"
+        }], "", "/test/%s", "/test/%s/%s")
+
+        info = path_mapper._get_intermediate_collection_info()
+
+        self.assertEqual(info["name"], "Intermediate collection")
+        self.assertEqual(info["trash_at"], datetime.datetime(2018, 1, 1, 0, 1))
+        self.assertEqual(info["properties"], {"type" : "Intermediate", "container" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"})