From 7f9839be2facf7e5bf3106dab0859da5d971f31e Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Tue, 2 May 2017 23:06:28 -0400 Subject: [PATCH] 11369: Containers API: Add capacity to tmp and output mounts. Also support {outputDirType: keep_output_dir} of RuntimeConstraints. --- sdk/cwl/arvados_cwl/arvcontainer.py | 26 ++++++++++++++++------ sdk/cwl/tests/test_container.py | 34 +++++++++++++++++++---------- 2 files changed, 42 insertions(+), 18 deletions(-) diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py index 657d5927d0..0b302b6280 100644 --- a/sdk/cwl/arvados_cwl/arvcontainer.py +++ b/sdk/cwl/arvados_cwl/arvcontainer.py @@ -45,12 +45,20 @@ class ArvadosContainer(object): "properties": {} } runtime_constraints = {} + + resources = self.builder.resources + if resources is not None: + runtime_constraints["vcpus"] = resources.get("cores", 1) + runtime_constraints["ram"] = resources.get("ram") * 2**20 + mounts = { self.outdir: { - "kind": "tmp" + "kind": "tmp", + "capacity": resources.get("outdirSize", 0) * 2**20 }, self.tmpdir: { - "kind": "tmp" + "kind": "tmp", + "capacity": resources.get("tmpdirSize", 0) * 2**20 } } scheduling_parameters = {} @@ -139,11 +147,6 @@ class ArvadosContainer(object): pull_image, self.arvrunner.project_uuid) - resources = self.builder.resources - if resources is not None: - runtime_constraints["vcpus"] = resources.get("cores", 1) - runtime_constraints["ram"] = resources.get("ram") * 2**20 - api_req, _ = get_feature(self, "http://arvados.org/cwl#APIRequirement") if api_req: runtime_constraints["API"] = True @@ -152,6 +155,15 @@ class ArvadosContainer(object): if runtime_req: if "keep_cache" in runtime_req: runtime_constraints["keep_cache_ram"] = runtime_req["keep_cache"] * 2**20 + if "outputDirType" in runtime_req: + if runtime_req["outputDirType"] == "local_output_dir": + # Currently the default behavior. + pass + elif runtime_req["outputDirType"] == "keep_output_dir": + mounts[self.outdir]= { + "kind": "collection", + "writable": True + } partition_req, _ = get_feature(self, "http://arvados.org/cwl#PartitionRequirement") if partition_req: diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py index b06eae8105..af05773e0c 100644 --- a/sdk/cwl/tests/test_container.py +++ b/sdk/cwl/tests/test_container.py @@ -63,8 +63,11 @@ class TestContainer(unittest.TestCase): 'use_existing': enable_reuse, 'priority': 1, 'mounts': { - '/tmp': {'kind': 'tmp'}, - '/var/spool/cwl': {'kind': 'tmp'} + '/tmp': {'kind': 'tmp', + "capacity": 1073741824 + }, + '/var/spool/cwl': {'kind': 'tmp', + "capacity": 1073741824 } }, 'state': 'Committed', 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', @@ -97,7 +100,8 @@ class TestContainer(unittest.TestCase): "class": "ResourceRequirement", "coresMin": 3, "ramMin": 3000, - "tmpdirMin": 4000 + "tmpdirMin": 4000, + "outdirMin": 5000 }, { "class": "http://arvados.org/cwl#RuntimeConstraints", "keep_cache": 512 @@ -136,8 +140,10 @@ class TestContainer(unittest.TestCase): 'use_existing': True, 'priority': 1, 'mounts': { - '/tmp': {'kind': 'tmp'}, - '/var/spool/cwl': {'kind': 'tmp'} + '/tmp': {'kind': 'tmp', + "capacity": 4194304000 }, + '/var/spool/cwl': {'kind': 'tmp', + "capacity": 5242880000 } }, 'state': 'Committed', 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', @@ -242,8 +248,10 @@ class TestContainer(unittest.TestCase): 'use_existing': True, 'priority': 1, 'mounts': { - '/tmp': {'kind': 'tmp'}, - '/var/spool/cwl': {'kind': 'tmp'}, + '/tmp': {'kind': 'tmp', + "capacity": 1073741824 }, + '/var/spool/cwl': {'kind': 'tmp', + "capacity": 1073741824 }, '/var/spool/cwl/foo': { 'kind': 'collection', 'path': 'foo', @@ -328,8 +336,10 @@ class TestContainer(unittest.TestCase): 'use_existing': True, 'priority': 1, 'mounts': { - '/tmp': {'kind': 'tmp'}, - '/var/spool/cwl': {'kind': 'tmp'}, + '/tmp': {'kind': 'tmp', + "capacity": 1073741824 }, + '/var/spool/cwl': {'kind': 'tmp', + "capacity": 1073741824 }, "stderr": { "kind": "file", "path": "/var/spool/cwl/stderr.txt" @@ -460,8 +470,10 @@ class TestContainer(unittest.TestCase): "kind": "collection", "portable_data_hash": "99999999999999999999999999999994+44" }, - '/tmp': {'kind': 'tmp'}, - '/var/spool/cwl': {'kind': 'tmp'} + '/tmp': {'kind': 'tmp', + "capacity": 1073741824 }, + '/var/spool/cwl': {'kind': 'tmp', + "capacity": 1073741824 } }, 'state': 'Committed', 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', -- 2.30.2