X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/a31816b03c8249178d51621524f05437f5f9478a..6b2ee0dd59e39307649b72940824ceafada35fac:/sdk/cwl/tests/test_submit.py diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py index 38741ebf02..2dff3e8f9d 100644 --- a/sdk/cwl/tests/test_submit.py +++ b/sdk/cwl/tests/test_submit.py @@ -30,7 +30,7 @@ def stubs(func): return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p)) stubs.KeepClient().put.side_effect = putstub - stubs.keepdocker.return_value = True + stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz" stubs.api = mock.MagicMock() @@ -41,15 +41,38 @@ def stubs(func): stubs.api.collections().create().execute.side_effect = ({ "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1", "portable_data_hash": "99999999999999999999999999999991+99", + "manifest_text": "" }, { "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2", "portable_data_hash": "99999999999999999999999999999992+99", - }) + "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt" + }, + { + "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz4", + "portable_data_hash": "99999999999999999999999999999994+99", + "manifest_text": "" + }, + { + "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz5", + "portable_data_hash": "99999999999999999999999999999995+99", + "manifest_text": "" + } ) + stubs.api.collections().get().execute.return_value = { + "portable_data_hash": "99999999999999999999999999999993+99", "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"} + stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" stubs.api.jobs().create().execute.return_value = { "uuid": stubs.expect_job_uuid, "state": "Queued", } + + stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz" + stubs.api.container_requests().create().execute.return_value = { + "uuid": stubs.expect_container_request_uuid, + "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz", + "state": "Queued" + } + stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz" stubs.api.pipeline_templates().create().execute.return_value = { "uuid": stubs.expect_pipeline_template_uuid, @@ -60,7 +83,8 @@ def stubs(func): }, 'script_parameters': { 'x': { - 'path': '99999999999999999999999999999992+99/blorp.txt', + 'basename': 'blorp.txt', + 'location': '99999999999999999999999999999994+99/blorp.txt', 'class': 'File' }, 'cwl:tool': @@ -70,6 +94,40 @@ def stubs(func): 'script_version': 'master', 'script': 'cwl-runner' } + + stubs.expect_container_spec = { + 'priority': 1, + 'mounts': { + '/var/spool/cwl': { + 'writable': True, + 'kind': 'collection' + }, + '/var/lib/cwl/workflow': { + 'portable_data_hash': '99999999999999999999999999999991+99', + 'kind': 'collection' + }, + 'stdout': { + 'path': '/var/spool/cwl/cwl.output.json', + 'kind': 'file' + }, + '/var/lib/cwl/job/cwl.input.json': { + 'portable_data_hash': '765fda0d9897729ff467a4609879c00a+60/cwl.input.json', + 'kind': 'collection' + } + }, + 'state': 'Committed', + 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz', + 'command': ['arvados-cwl-runner', '--local', '--api=containers', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/job/cwl.input.json'], + 'name': 'submit_wf.cwl', + 'container_image': '99999999999999999999999999999993+99', + 'output_path': '/var/spool/cwl', + 'cwd': '/var/spool/cwl', + 'runtime_constraints': { + 'API': True, + 'vcpus': 1, + 'ram': 268435456 + } + } return func(self, stubs, *args, **kwargs) return wrapped @@ -88,13 +146,19 @@ class TestSubmit(unittest.TestCase): mock.call(), mock.call(body={ 'manifest_text': - './tool a3954c369b8924d40547ec8cf5f6a7f4+449 ' - '0:16:blub.txt 16:433:submit_tool.cwl\n./wf ' - 'e046cace0b1a0a6ee645f6ea8688f7e2+364 0:364:submit_wf.cwl\n', + './tool d51232d96b6116d964a69bfb7e0c73bf+450 ' + '0:16:blub.txt 16:434:submit_tool.cwl\n./wf ' + '4d31c5fefd087faf67ca8db0111af36c+353 0:353:submit_wf.cwl\n', 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz', 'name': 'submit_wf.cwl', }, ensure_unique_name=True), mock.call().execute(), + mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 ' + '0:0:blub.txt 0:0:submit_tool.cwl\n', + 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz', + 'name': 'New collection'}, + ensure_unique_name=True), + mock.call().execute(num_retries=4), mock.call(body={ 'manifest_text': '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n', @@ -128,6 +192,47 @@ class TestSubmit(unittest.TestCase): body=expect_body, find_or_create=True) + @stubs + def test_submit_container(self, stubs): + capture_stdout = cStringIO.StringIO() + exited = arvados_cwl.main( + ["--submit", "--no-wait", "--api=containers", "--debug", + "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"], + capture_stdout, sys.stderr, api_client=stubs.api) + self.assertEqual(exited, 0) + + stubs.api.collections().create.assert_has_calls([ + mock.call(), + mock.call(body={ + 'manifest_text': + './tool d51232d96b6116d964a69bfb7e0c73bf+450 ' + '0:16:blub.txt 16:434:submit_tool.cwl\n./wf ' + '4d31c5fefd087faf67ca8db0111af36c+353 0:353:submit_wf.cwl\n', + 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz', + 'name': 'submit_wf.cwl', + }, ensure_unique_name=True), + mock.call().execute(), + mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 ' + '0:0:blub.txt 0:0:submit_tool.cwl\n', + 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz', + 'name': 'New collection'}, + ensure_unique_name=True), + mock.call().execute(num_retries=4), + mock.call(body={ + 'manifest_text': + '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n', + 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz', + 'name': '#', + }, ensure_unique_name=True), + mock.call().execute()]) + + expect_container = copy.deepcopy(stubs.expect_container_spec) + expect_container["owner_uuid"] = stubs.fake_user_uuid + stubs.api.container_requests().create.assert_called_with( + body=expect_container) + self.assertEqual(capture_stdout.getvalue(), + stubs.expect_container_request_uuid + '\n') + class TestCreateTemplate(unittest.TestCase): @stubs @@ -137,7 +242,7 @@ class TestCreateTemplate(unittest.TestCase): capture_stdout = cStringIO.StringIO() exited = arvados_cwl.main( - ["--create-template", "--no-wait", + ["--create-template", "--no-wait", "--debug", "--project-uuid", project_uuid, "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"], capture_stdout, sys.stderr, api_client=stubs.api) @@ -151,7 +256,7 @@ class TestCreateTemplate(unittest.TestCase): 'dataclass': 'File', 'required': True, 'type': 'File', - 'value': '99999999999999999999999999999992+99/blorp.txt', + 'value': '99999999999999999999999999999994+99/blorp.txt', } expect_template = { "components": { @@ -242,7 +347,7 @@ class TestTemplateInputs(unittest.TestCase): expect_template["owner_uuid"] = stubs.fake_user_uuid params = expect_template[ "components"]["inputs_test.cwl"]["script_parameters"] - params["fileInput"]["value"] = '99999999999999999999999999999992+99/blorp.txt' + params["fileInput"]["value"] = '99999999999999999999999999999994+99/blorp.txt' params["floatInput"]["value"] = 1.234 params["boolInput"]["value"] = True