From 298fa8a8436596348086b42c8d31eba22609145a Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Fri, 2 Mar 2018 15:05:47 -0500 Subject: [PATCH] 12526: Fix tests Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- sdk/cwl/arvados_cwl/__init__.py | 5 ++++- sdk/cwl/arvados_cwl/arvcontainer.py | 2 +- sdk/cwl/tests/test_container.py | 20 ++++++++++---------- sdk/cwl/tests/test_submit.py | 26 +++++++++++++++++++++++--- 4 files changed, 38 insertions(+), 15 deletions(-) diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py index 4dbc659ca3..ee0a73de14 100644 --- a/sdk/cwl/arvados_cwl/__init__.py +++ b/sdk/cwl/arvados_cwl/__init__.py @@ -23,6 +23,7 @@ import cwltool.main import cwltool.workflow import cwltool.process from schema_salad.sourceline import SourceLine +import schema_salad.validate as validate import arvados import arvados.config @@ -415,12 +416,14 @@ class ArvCwlRunner(object): if self.work_api == "containers": if self.ignore_docker_for_reuse: - raise validate.ValidationException("--ignore-docker-for-reuse not supported with containers API.") + raise Exception("--ignore-docker-for-reuse not supported with containers API.") kwargs["outdir"] = "/var/spool/cwl" kwargs["docker_outdir"] = "/var/spool/cwl" kwargs["tmpdir"] = "/tmp" kwargs["docker_tmpdir"] = "/tmp" elif self.work_api == "jobs": + if kwargs["priority"] != 500: + raise Exception("--priority not implemented for jobs API.") kwargs["outdir"] = "$(task.outdir)" kwargs["docker_outdir"] = "$(task.outdir)" kwargs["tmpdir"] = "$(task.tmpdir)" diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py index 5c52d2828d..9f8382291c 100644 --- a/sdk/cwl/arvados_cwl/arvcontainer.py +++ b/sdk/cwl/arvados_cwl/arvcontainer.py @@ -47,7 +47,7 @@ class ArvadosContainer(object): "name": self.name, "output_path": self.outdir, "cwd": self.outdir, - "priority": 1, + "priority": kwargs.get("priority"), "state": "Committed", "properties": {}, } diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py index fea21e9e0f..cd555a72ca 100644 --- a/sdk/cwl/tests/test_container.py +++ b/sdk/cwl/tests/test_container.py @@ -55,7 +55,7 @@ class TestContainer(unittest.TestCase): arvtool.formatgraph = None for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_"+str(enable_reuse), make_fs_access=make_fs_access, tmpdir="/tmp"): - j.run(enable_reuse=enable_reuse) + j.run(enable_reuse=enable_reuse, priority=500) runner.api.container_requests().create.assert_called_with( body=JsonDiffMatcher({ 'environment': { @@ -68,7 +68,7 @@ class TestContainer(unittest.TestCase): 'ram': 1073741824 }, 'use_existing': enable_reuse, - 'priority': 1, + 'priority': 500, 'mounts': { '/tmp': {'kind': 'tmp', "capacity": 1073741824 @@ -138,7 +138,7 @@ class TestContainer(unittest.TestCase): arvtool.formatgraph = None for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_resource_requirements", make_fs_access=make_fs_access, tmpdir="/tmp"): - j.run(enable_reuse=True) + j.run(enable_reuse=True, priority=500) call_args, call_kwargs = runner.api.container_requests().create.call_args @@ -155,7 +155,7 @@ class TestContainer(unittest.TestCase): 'API': True }, 'use_existing': False, - 'priority': 1, + 'priority': 500, 'mounts': { '/tmp': {'kind': 'tmp', "capacity": 4194304000 }, @@ -247,7 +247,7 @@ class TestContainer(unittest.TestCase): arvtool.formatgraph = None for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_initial_work_dir", make_fs_access=make_fs_access, tmpdir="/tmp"): - j.run() + j.run(priority=500) call_args, call_kwargs = runner.api.container_requests().create.call_args @@ -267,7 +267,7 @@ class TestContainer(unittest.TestCase): 'ram': 1073741824 }, 'use_existing': True, - 'priority': 1, + 'priority': 500, 'mounts': { '/tmp': {'kind': 'tmp', "capacity": 1073741824 }, @@ -346,7 +346,7 @@ class TestContainer(unittest.TestCase): arvtool.formatgraph = None for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_redirect", make_fs_access=make_fs_access, tmpdir="/tmp"): - j.run() + j.run(priority=500) runner.api.container_requests().create.assert_called_with( body=JsonDiffMatcher({ 'environment': { @@ -359,7 +359,7 @@ class TestContainer(unittest.TestCase): 'ram': 1073741824 }, 'use_existing': True, - 'priority': 1, + 'priority': 500, 'mounts': { '/tmp': {'kind': 'tmp', "capacity": 1073741824 }, @@ -484,7 +484,7 @@ class TestContainer(unittest.TestCase): } for j in arvtool.job(job_order, mock.MagicMock(), basedir="", name="test_run_mounts", make_fs_access=make_fs_access, tmpdir="/tmp"): - j.run() + j.run(priority=500) runner.api.container_requests().create.assert_called_with( body=JsonDiffMatcher({ 'environment': { @@ -497,7 +497,7 @@ class TestContainer(unittest.TestCase): 'ram': 1073741824 }, 'use_existing': True, - 'priority': 1, + 'priority': 500, 'mounts': { "/keep/99999999999999999999999999999994+44": { "kind": "collection", diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py index c0b74fe4d3..298d6aa3cd 100644 --- a/sdk/cwl/tests/test_submit.py +++ b/sdk/cwl/tests/test_submit.py @@ -194,7 +194,7 @@ def stubs(func): expect_packed_workflow = yaml.round_trip_load(f) stubs.expect_container_spec = { - 'priority': 1, + 'priority': 500, 'mounts': { '/var/spool/cwl': { 'writable': True, @@ -709,7 +709,7 @@ class TestSubmit(unittest.TestCase): self.assertEqual(exited, 0) expect_container = { - 'priority': 1, + 'priority': 500, 'mounts': { '/var/spool/cwl': { 'writable': True, @@ -792,7 +792,7 @@ class TestSubmit(unittest.TestCase): self.assertEqual(exited, 0) expect_container = { - 'priority': 1, + 'priority': 500, 'mounts': { '/var/spool/cwl': { 'writable': True, @@ -980,6 +980,26 @@ class TestSubmit(unittest.TestCase): self.assertEqual(capture_stdout.getvalue(), stubs.expect_container_request_uuid + '\n') + @stubs + def test_submit_priority(self, stubs): + capture_stdout = cStringIO.StringIO() + try: + exited = arvados_cwl.main( + ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669", + "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"], + capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client) + self.assertEqual(exited, 0) + except: + logging.exception("") + + stubs.expect_container_spec["priority"] = 669 + + expect_container = copy.deepcopy(stubs.expect_container_spec) + stubs.api.container_requests().create.assert_called_with( + body=JsonDiffMatcher(expect_container)) + self.assertEqual(capture_stdout.getvalue(), + stubs.expect_container_request_uuid + '\n') + @mock.patch("arvados.commands.keepdocker.find_one_image_hash") @mock.patch("cwltool.docker.DockerCommandLineJob.get_image") -- 2.30.2