Update get_intermediate_collection_info and add owner_uuid to save_new

method in arvcontainer.py and arvjob.py

Arvados-DCO-1.1-Signed-off-by: Fuad Muhic <fmuhic@capeannenterprises.com>

diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 2618ddecacac0db113e857cc0bd5440f5cd46472..a2bd53c9fc2a5e713598791e814a286f926fe963 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -168,8 +168,9 @@ class ArvadosContainer(JobBase):
 
                 if not runtimeContext.current_container:
                     runtimeContext.current_container = get_current_container(self.arvrunner.api, self.arvrunner.num_retries, logger)
-                info = get_intermediate_collection_info(runtimeContext.current_container, runtimeContext.intermediate_output_ttl)
+                info = get_intermediate_collection_info(self.name, runtimeContext.current_container, runtimeContext.intermediate_output_ttl)
                 vwd.save_new(name=info["name"],
+                             owner_uuid=self.arvrunner.project_uuid,
                              ensure_unique_name=True,
                              trash_at=info["trash_at"],
                              properties=info["properties"])
@@ -248,6 +249,7 @@ class ArvadosContainer(JobBase):
         if self.timelimit is not None:
             scheduling_parameters["max_run_time"] = self.timelimit
 
+        container_request["output_name"] = "Output for step %s" % (self.name)
         container_request["output_ttl"] = self.output_ttl
         container_request["mounts"] = mounts
         container_request["secret_mounts"] = secret_mounts

diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index 36a047213ac66339088239068a8fb829124a7cc9..1287fbb6eaf7b8387ca3fe700c7c97cf0678b867 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -79,8 +79,9 @@ class ArvadosJob(JobBase):
                     with Perf(metrics, "generatefiles.save_new %s" % self.name):
                         if not runtimeContext.current_container:
                             runtimeContext.current_container = get_current_container(self.arvrunner.api, self.arvrunner.num_retries, logger)
-                        info = get_intermediate_collection_info(runtimeContext.current_container, runtimeContext.intermediate_output_ttl)
+                        info = get_intermediate_collection_info(self.name, runtimeContext.current_container, runtimeContext.intermediate_output_ttl)
                         vwd.save_new(name=info["name"],
+                                     owner_uuid=self.arvrunner.project_uuid,
                                      ensure_unique_name=True,
                                      trash_at=info["trash_at"],
                                      properties=info["properties"])

diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index a3b6a4dd9b956abd58c963230cee937c6ed8d110..d083b78f5a061906164a5978530af9230e767473 100644 (file)
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -156,7 +156,7 @@ class ArvPathMapper(PathMapper):
                     self.addentry(l, c, ".", remap)
 
                 container = get_current_container(self.arvrunner.api, self.arvrunner.num_retries, logger)
-                info = get_intermediate_collection_info(container, self.arvrunner.intermediate_output_ttl)
+                info = get_intermediate_collection_info(None, container, self.arvrunner.intermediate_output_ttl)
 
                 c.save_new(name=info["name"],
                            owner_uuid=self.arvrunner.project_uuid,
@@ -175,7 +175,7 @@ class ArvPathMapper(PathMapper):
                 self.addentry(srcobj, c, ".", remap)
 
                 container = get_current_container(self.arvrunner.api, self.arvrunner.num_retries, logger)
-                info = get_intermediate_collection_info(container, self.arvrunner.intermediate_output_ttl)
+                info = get_intermediate_collection_info(None, container, self.arvrunner.intermediate_output_ttl)
 
                 c.save_new(name=info["name"],
                            owner_uuid=self.arvrunner.project_uuid,

diff --git a/sdk/cwl/arvados_cwl/util.py b/sdk/cwl/arvados_cwl/util.py
index 7dafedac28ff811a625dcff40fc9031b36784c0e..98a2a89a1d281e056ee3067752e6094349123115 100644 (file)
--- a/sdk/cwl/arvados_cwl/util.py
+++ b/sdk/cwl/arvados_cwl/util.py
@@ -5,8 +5,11 @@
 import datetime
 from arvados.errors import ApiError
 
-def get_intermediate_collection_info(current_container, intermediate_output_ttl):
-        name = "Intermediate collection"
+def get_intermediate_collection_info(workflow_step_name, current_container, intermediate_output_ttl):
+        if workflow_step_name:
+            name = "Intermediate collection for step %s" % (workflow_step_name)
+        else:
+            name = "Intermediate collection"
         trash_time = None
         if intermediate_output_ttl > 0:
             trash_time = datetime.datetime.utcnow() + datetime.timedelta(seconds=intermediate_output_ttl)
@@ -25,4 +28,4 @@ def get_current_container(api, num_retries=0, logger=None):
         # Status code 404 just means we're not running in a container.
         if e.resp.status != 404 and logger:
             logger.info("Getting current container: %s", e)
-    return current_container;
+    return current_container

diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index e7ce9c37092084a171b8e0f0010d9cb9fa44277a..ae234414a3df90888cfbe9028c06aa5efbba9f55 100644 (file)
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -99,6 +99,7 @@ class TestContainer(unittest.TestCase):
                                                "capacity": 1073741824 }
                         },
                         'state': 'Committed',
+                        'output_name': 'Output for step test_run_'+str(enable_reuse),
                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
                         'output_path': '/var/spool/cwl',
                         'output_ttl': 0,
@@ -185,6 +186,7 @@ class TestContainer(unittest.TestCase):
                                    "capacity": 5242880000 }
             },
             'state': 'Committed',
+            'output_name': 'Output for step test_resource_requirements',
             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
             'output_path': '/var/spool/cwl',
             'output_ttl': 7200,
@@ -317,6 +319,7 @@ class TestContainer(unittest.TestCase):
                 }
             },
             'state': 'Committed',
+            'output_name': 'Output for step test_initial_work_dir',
             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
             'output_path': '/var/spool/cwl',
             'output_ttl': 0,
@@ -404,6 +407,7 @@ class TestContainer(unittest.TestCase):
                         },
                     },
                     'state': 'Committed',
+                    "output_name": "Output for step test_run_redirect",
                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
                     'output_path': '/var/spool/cwl',
                     'output_ttl': 0,
@@ -540,6 +544,7 @@ class TestContainer(unittest.TestCase):
                                            "capacity": 1073741824 }
                     },
                     'state': 'Committed',
+                    'output_name': 'Output for step test_run_mounts',
                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
                     'output_path': '/var/spool/cwl',
                     'output_ttl': 0,
@@ -632,6 +637,7 @@ class TestContainer(unittest.TestCase):
                                            "capacity": 1073741824 }
                     },
                     'state': 'Committed',
+                    'output_name': 'Output for step test_secrets',
                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
                     'output_path': '/var/spool/cwl',
                     'output_ttl': 0,

diff --git a/sdk/cwl/tests/test_util.py b/sdk/cwl/tests/test_util.py
index e42197e21cfb84625ed7469bd5614e35856671bf..2532bd596c0f3dfac3554d3b5fe31ffb9d7e50d4 100644 (file)
--- a/sdk/cwl/tests/test_util.py
+++ b/sdk/cwl/tests/test_util.py
@@ -19,12 +19,13 @@ datetime.datetime = MockDateTime
 
 class TestUtil(unittest.TestCase):
     def test_get_intermediate_collection_info(self):
+        name = "one"
         current_container = {"uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"}
         intermediate_output_ttl = 120
 
-        info = get_intermediate_collection_info(current_container, intermediate_output_ttl)
+        info = get_intermediate_collection_info(name, current_container, intermediate_output_ttl)
 
-        self.assertEqual(info["name"], "Intermediate collection")
+        self.assertEqual(info["name"], "Intermediate collection for step one")
         self.assertEqual(info["trash_at"], datetime.datetime(2018, 1, 1, 0, 2, 0, 0))
         self.assertEqual(info["properties"], {"type" : "intermediate", "container" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"})