From c405f9adab08daa3d4edef9cddd5453a79446c7b Mon Sep 17 00:00:00 2001 From: Fuad Muhic Date: Tue, 10 Jul 2018 12:33:43 +0200 Subject: [PATCH] Update get_intermediate_collection_info and add owner_uuid to save_new method in arvcontainer.py and arvjob.py Arvados-DCO-1.1-Signed-off-by: Fuad Muhic --- sdk/cwl/arvados_cwl/arvcontainer.py | 4 +++- sdk/cwl/arvados_cwl/arvjob.py | 3 ++- sdk/cwl/arvados_cwl/pathmapper.py | 4 ++-- sdk/cwl/arvados_cwl/util.py | 9 ++++++--- sdk/cwl/tests/test_container.py | 6 ++++++ sdk/cwl/tests/test_util.py | 5 +++-- 6 files changed, 22 insertions(+), 9 deletions(-) diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py index 2618ddecac..a2bd53c9fc 100644 --- a/sdk/cwl/arvados_cwl/arvcontainer.py +++ b/sdk/cwl/arvados_cwl/arvcontainer.py @@ -168,8 +168,9 @@ class ArvadosContainer(JobBase): if not runtimeContext.current_container: runtimeContext.current_container = get_current_container(self.arvrunner.api, self.arvrunner.num_retries, logger) - info = get_intermediate_collection_info(runtimeContext.current_container, runtimeContext.intermediate_output_ttl) + info = get_intermediate_collection_info(self.name, runtimeContext.current_container, runtimeContext.intermediate_output_ttl) vwd.save_new(name=info["name"], + owner_uuid=self.arvrunner.project_uuid, ensure_unique_name=True, trash_at=info["trash_at"], properties=info["properties"]) @@ -248,6 +249,7 @@ class ArvadosContainer(JobBase): if self.timelimit is not None: scheduling_parameters["max_run_time"] = self.timelimit + container_request["output_name"] = "Output for step %s" % (self.name) container_request["output_ttl"] = self.output_ttl container_request["mounts"] = mounts container_request["secret_mounts"] = secret_mounts diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py index 36a047213a..1287fbb6ea 100644 --- a/sdk/cwl/arvados_cwl/arvjob.py +++ b/sdk/cwl/arvados_cwl/arvjob.py @@ -79,8 +79,9 @@ class ArvadosJob(JobBase): with Perf(metrics, "generatefiles.save_new %s" % self.name): if not runtimeContext.current_container: runtimeContext.current_container = get_current_container(self.arvrunner.api, self.arvrunner.num_retries, logger) - info = get_intermediate_collection_info(runtimeContext.current_container, runtimeContext.intermediate_output_ttl) + info = get_intermediate_collection_info(self.name, runtimeContext.current_container, runtimeContext.intermediate_output_ttl) vwd.save_new(name=info["name"], + owner_uuid=self.arvrunner.project_uuid, ensure_unique_name=True, trash_at=info["trash_at"], properties=info["properties"]) diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py index a3b6a4dd9b..d083b78f5a 100644 --- a/sdk/cwl/arvados_cwl/pathmapper.py +++ b/sdk/cwl/arvados_cwl/pathmapper.py @@ -156,7 +156,7 @@ class ArvPathMapper(PathMapper): self.addentry(l, c, ".", remap) container = get_current_container(self.arvrunner.api, self.arvrunner.num_retries, logger) - info = get_intermediate_collection_info(container, self.arvrunner.intermediate_output_ttl) + info = get_intermediate_collection_info(None, container, self.arvrunner.intermediate_output_ttl) c.save_new(name=info["name"], owner_uuid=self.arvrunner.project_uuid, @@ -175,7 +175,7 @@ class ArvPathMapper(PathMapper): self.addentry(srcobj, c, ".", remap) container = get_current_container(self.arvrunner.api, self.arvrunner.num_retries, logger) - info = get_intermediate_collection_info(container, self.arvrunner.intermediate_output_ttl) + info = get_intermediate_collection_info(None, container, self.arvrunner.intermediate_output_ttl) c.save_new(name=info["name"], owner_uuid=self.arvrunner.project_uuid, diff --git a/sdk/cwl/arvados_cwl/util.py b/sdk/cwl/arvados_cwl/util.py index 7dafedac28..98a2a89a1d 100644 --- a/sdk/cwl/arvados_cwl/util.py +++ b/sdk/cwl/arvados_cwl/util.py @@ -5,8 +5,11 @@ import datetime from arvados.errors import ApiError -def get_intermediate_collection_info(current_container, intermediate_output_ttl): - name = "Intermediate collection" +def get_intermediate_collection_info(workflow_step_name, current_container, intermediate_output_ttl): + if workflow_step_name: + name = "Intermediate collection for step %s" % (workflow_step_name) + else: + name = "Intermediate collection" trash_time = None if intermediate_output_ttl > 0: trash_time = datetime.datetime.utcnow() + datetime.timedelta(seconds=intermediate_output_ttl) @@ -25,4 +28,4 @@ def get_current_container(api, num_retries=0, logger=None): # Status code 404 just means we're not running in a container. if e.resp.status != 404 and logger: logger.info("Getting current container: %s", e) - return current_container; + return current_container diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py index e7ce9c3709..ae234414a3 100644 --- a/sdk/cwl/tests/test_container.py +++ b/sdk/cwl/tests/test_container.py @@ -99,6 +99,7 @@ class TestContainer(unittest.TestCase): "capacity": 1073741824 } }, 'state': 'Committed', + 'output_name': 'Output for step test_run_'+str(enable_reuse), 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'output_path': '/var/spool/cwl', 'output_ttl': 0, @@ -185,6 +186,7 @@ class TestContainer(unittest.TestCase): "capacity": 5242880000 } }, 'state': 'Committed', + 'output_name': 'Output for step test_resource_requirements', 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'output_path': '/var/spool/cwl', 'output_ttl': 7200, @@ -317,6 +319,7 @@ class TestContainer(unittest.TestCase): } }, 'state': 'Committed', + 'output_name': 'Output for step test_initial_work_dir', 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'output_path': '/var/spool/cwl', 'output_ttl': 0, @@ -404,6 +407,7 @@ class TestContainer(unittest.TestCase): }, }, 'state': 'Committed', + "output_name": "Output for step test_run_redirect", 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'output_path': '/var/spool/cwl', 'output_ttl': 0, @@ -540,6 +544,7 @@ class TestContainer(unittest.TestCase): "capacity": 1073741824 } }, 'state': 'Committed', + 'output_name': 'Output for step test_run_mounts', 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'output_path': '/var/spool/cwl', 'output_ttl': 0, @@ -632,6 +637,7 @@ class TestContainer(unittest.TestCase): "capacity": 1073741824 } }, 'state': 'Committed', + 'output_name': 'Output for step test_secrets', 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'output_path': '/var/spool/cwl', 'output_ttl': 0, diff --git a/sdk/cwl/tests/test_util.py b/sdk/cwl/tests/test_util.py index e42197e21c..2532bd596c 100644 --- a/sdk/cwl/tests/test_util.py +++ b/sdk/cwl/tests/test_util.py @@ -19,12 +19,13 @@ datetime.datetime = MockDateTime class TestUtil(unittest.TestCase): def test_get_intermediate_collection_info(self): + name = "one" current_container = {"uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"} intermediate_output_ttl = 120 - info = get_intermediate_collection_info(current_container, intermediate_output_ttl) + info = get_intermediate_collection_info(name, current_container, intermediate_output_ttl) - self.assertEqual(info["name"], "Intermediate collection") + self.assertEqual(info["name"], "Intermediate collection for step one") self.assertEqual(info["trash_at"], datetime.datetime(2018, 1, 1, 0, 2, 0, 0)) self.assertEqual(info["properties"], {"type" : "intermediate", "container" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"}) -- 2.30.2