X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/d4ed3e6460469f2766e1f1676c538d6c86e000b6..88bb9b9fc4392f4a3514ae59e3ffd454d3ce90a8:/sdk/cwl/tests/test_submit.py diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py index d215cba7fc..7aaa27fd63 100644 --- a/sdk/cwl/tests/test_submit.py +++ b/sdk/cwl/tests/test_submit.py @@ -18,6 +18,7 @@ import mock import sys import unittest import cwltool.process +import re from io import BytesIO @@ -67,7 +68,7 @@ def stubs(func): stubs.keep_client = keep_client2 stubs.docker_images = { "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", "")], - "debian:8": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", "")], + "debian:buster-slim": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", "")], "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", "")], "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", "")], } @@ -301,8 +302,8 @@ def stubs(func): 'secret_mounts': {}, 'state': 'Committed', 'command': ['arvados-cwl-runner', '--local', '--api=containers', - '--no-log-timestamps', '--disable-validate', - '--eval-timeout=20', '--thread-count=1', + '--no-log-timestamps', '--disable-validate', '--disable-color', + '--eval-timeout=20', '--thread-count=4', '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue', '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'], 'name': 'submit_wf.cwl', @@ -314,7 +315,7 @@ def stubs(func): 'vcpus': 1, 'ram': (1024+256)*1024*1024 }, - 'use_existing': True, + 'use_existing': False, 'properties': {}, 'secret_mounts': {} } @@ -388,6 +389,19 @@ class TestSubmit(unittest.TestCase): stubs.expect_container_request_uuid + '\n') self.assertEqual(exited, 0) + + @stubs + def test_submit_container_tool(self, stubs): + # test for issue #16139 + exited = arvados_cwl.main( + ["--submit", "--no-wait", "--api=containers", "--debug", + "tests/tool/tool_with_sf.cwl", "tests/tool/tool_with_sf.yml"], + stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client) + + self.assertEqual(stubs.capture_stdout.getvalue(), + stubs.expect_container_request_uuid + '\n') + self.assertEqual(exited, 0) + @stubs def test_submit_container_no_reuse(self, stubs): exited = arvados_cwl.main( @@ -398,8 +412,8 @@ class TestSubmit(unittest.TestCase): expect_container = copy.deepcopy(stubs.expect_container_spec) expect_container["command"] = [ 'arvados-cwl-runner', '--local', '--api=containers', - '--no-log-timestamps', '--disable-validate', - '--eval-timeout=20', '--thread-count=1', + '--no-log-timestamps', '--disable-validate', '--disable-color', + '--eval-timeout=20', '--thread-count=4', '--disable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue', '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'] @@ -422,8 +436,8 @@ class TestSubmit(unittest.TestCase): expect_container = copy.deepcopy(stubs.expect_container_spec) expect_container["command"] = [ 'arvados-cwl-runner', '--local', '--api=containers', - '--no-log-timestamps', '--disable-validate', - '--eval-timeout=20', '--thread-count=1', + '--no-log-timestamps', '--disable-validate', '--disable-color', + '--eval-timeout=20', '--thread-count=4', '--disable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue', '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'] expect_container["use_existing"] = False @@ -454,8 +468,8 @@ class TestSubmit(unittest.TestCase): expect_container = copy.deepcopy(stubs.expect_container_spec) expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', - '--no-log-timestamps', '--disable-validate', - '--eval-timeout=20', '--thread-count=1', + '--no-log-timestamps', '--disable-validate', '--disable-color', + '--eval-timeout=20', '--thread-count=4', '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=stop', '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'] @@ -477,8 +491,8 @@ class TestSubmit(unittest.TestCase): expect_container = copy.deepcopy(stubs.expect_container_spec) expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', - '--no-log-timestamps', '--disable-validate', - '--eval-timeout=20', '--thread-count=1', + '--no-log-timestamps', '--disable-validate', '--disable-color', + '--eval-timeout=20', '--thread-count=4', '--enable-reuse', "--collection-cache-size=256", "--output-name="+output_name, '--debug', '--on-error=continue', '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'] @@ -499,8 +513,8 @@ class TestSubmit(unittest.TestCase): expect_container = copy.deepcopy(stubs.expect_container_spec) expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', - '--no-log-timestamps', '--disable-validate', - '--eval-timeout=20', '--thread-count=1', + '--no-log-timestamps', '--disable-validate', '--disable-color', + '--eval-timeout=20', '--thread-count=4', '--enable-reuse', "--collection-cache-size=256", "--debug", "--storage-classes=foo", '--on-error=continue', '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'] @@ -511,11 +525,14 @@ class TestSubmit(unittest.TestCase): stubs.expect_container_request_uuid + '\n') self.assertEqual(exited, 0) - @mock.patch("arvados_cwl.task_queue.TaskQueue") + @mock.patch("cwltool.task_queue.TaskQueue") @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job") - @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection", return_value = (None, None)) + @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection") @stubs def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq): + final_output_c = arvados.collection.Collection() + make_output.return_value = ({},final_output_c) + def set_final_output(job_order, output_callback, runtimeContext): output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success") return [] @@ -524,16 +541,19 @@ class TestSubmit(unittest.TestCase): exited = arvados_cwl.main( ["--debug", "--local", "--storage-classes=foo", "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"], - sys.stdin, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client) + stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client) make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz') self.assertEqual(exited, 0) - @mock.patch("arvados_cwl.task_queue.TaskQueue") + @mock.patch("cwltool.task_queue.TaskQueue") @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job") - @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection", return_value = (None, None)) + @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection") @stubs def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq): + final_output_c = arvados.collection.Collection() + make_output.return_value = ({},final_output_c) + def set_final_output(job_order, output_callback, runtimeContext): output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success") return [] @@ -542,7 +562,7 @@ class TestSubmit(unittest.TestCase): exited = arvados_cwl.main( ["--debug", "--local", "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"], - sys.stdin, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client) + stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client) make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz') self.assertEqual(exited, 0) @@ -556,8 +576,8 @@ class TestSubmit(unittest.TestCase): expect_container = copy.deepcopy(stubs.expect_container_spec) expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', - '--no-log-timestamps', '--disable-validate', - '--eval-timeout=20', '--thread-count=1', + '--no-log-timestamps', '--disable-validate', '--disable-color', + '--eval-timeout=20', '--thread-count=4', '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue', "--intermediate-output-ttl=3600", @@ -579,8 +599,8 @@ class TestSubmit(unittest.TestCase): expect_container = copy.deepcopy(stubs.expect_container_spec) expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', - '--no-log-timestamps', '--disable-validate', - '--eval-timeout=20', '--thread-count=1', + '--no-log-timestamps', '--disable-validate', '--disable-color', + '--eval-timeout=20', '--thread-count=4', '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue', "--trash-intermediate", @@ -603,8 +623,8 @@ class TestSubmit(unittest.TestCase): expect_container = copy.deepcopy(stubs.expect_container_spec) expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', - '--no-log-timestamps', '--disable-validate', - '--eval-timeout=20', '--thread-count=1', + '--no-log-timestamps', '--disable-validate', '--disable-color', + '--eval-timeout=20', '--thread-count=4', '--enable-reuse', "--collection-cache-size=256", "--output-tags="+output_tags, '--debug', '--on-error=continue', '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'] @@ -680,8 +700,8 @@ class TestSubmit(unittest.TestCase): 'name': 'expect_arvworkflow.cwl#main', 'container_image': '999999999999999999999999999999d3+99', 'command': ['arvados-cwl-runner', '--local', '--api=containers', - '--no-log-timestamps', '--disable-validate', - '--eval-timeout=20', '--thread-count=1', + '--no-log-timestamps', '--disable-validate', '--disable-color', + '--eval-timeout=20', '--thread-count=4', '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue', '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'], 'cwd': '/var/spool/cwl', @@ -690,7 +710,7 @@ class TestSubmit(unittest.TestCase): 'vcpus': 1, 'ram': 1342177280 }, - 'use_existing': True, + 'use_existing': False, 'properties': {}, 'secret_mounts': {} } @@ -751,7 +771,7 @@ class TestSubmit(unittest.TestCase): ], 'requirements': [ { - 'dockerPull': 'debian:8', + 'dockerPull': 'debian:buster-slim', 'class': 'DockerRequirement', "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99" } @@ -775,8 +795,8 @@ class TestSubmit(unittest.TestCase): 'name': 'a test workflow', 'container_image': "999999999999999999999999999999d3+99", 'command': ['arvados-cwl-runner', '--local', '--api=containers', - '--no-log-timestamps', '--disable-validate', - '--eval-timeout=20', '--thread-count=1', + '--no-log-timestamps', '--disable-validate', '--disable-color', + '--eval-timeout=20', '--thread-count=4', '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue', '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'], 'cwd': '/var/spool/cwl', @@ -785,7 +805,7 @@ class TestSubmit(unittest.TestCase): 'vcpus': 1, 'ram': 1342177280 }, - 'use_existing': True, + 'use_existing': False, 'properties': { "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37" }, @@ -839,8 +859,8 @@ class TestSubmit(unittest.TestCase): expect_container = copy.deepcopy(stubs.expect_container_spec) expect_container["owner_uuid"] = project_uuid expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', - '--no-log-timestamps', '--disable-validate', - "--eval-timeout=20", "--thread-count=1", + '--no-log-timestamps', '--disable-validate', '--disable-color', + "--eval-timeout=20", "--thread-count=4", '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue', '--project-uuid='+project_uuid, @@ -861,8 +881,8 @@ class TestSubmit(unittest.TestCase): expect_container = copy.deepcopy(stubs.expect_container_spec) expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', - '--no-log-timestamps', '--disable-validate', - '--eval-timeout=60.0', '--thread-count=1', + '--no-log-timestamps', '--disable-validate', '--disable-color', + '--eval-timeout=60.0', '--thread-count=4', '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue', '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'] @@ -882,8 +902,8 @@ class TestSubmit(unittest.TestCase): expect_container = copy.deepcopy(stubs.expect_container_spec) expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', - '--no-log-timestamps', '--disable-validate', - '--eval-timeout=20', '--thread-count=1', + '--no-log-timestamps', '--disable-validate', '--disable-color', + '--eval-timeout=20', '--thread-count=4', '--enable-reuse', "--collection-cache-size=500", '--debug', '--on-error=continue', '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'] @@ -904,7 +924,7 @@ class TestSubmit(unittest.TestCase): expect_container = copy.deepcopy(stubs.expect_container_spec) expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', - '--no-log-timestamps', '--disable-validate', + '--no-log-timestamps', '--disable-validate', '--disable-color', '--eval-timeout=20', '--thread-count=20', '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue', @@ -974,8 +994,8 @@ class TestSubmit(unittest.TestCase): "arv": "http://arvados.org/cwl#", } expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers', - '--no-log-timestamps', '--disable-validate', - '--eval-timeout=20', '--thread-count=1', + '--no-log-timestamps', '--disable-validate', '--disable-color', + '--eval-timeout=20', '--thread-count=4', '--enable-reuse', "--collection-cache-size=512", '--debug', '--on-error=continue', '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'] @@ -1039,8 +1059,9 @@ class TestSubmit(unittest.TestCase): "--api=containers", "--no-log-timestamps", "--disable-validate", + "--disable-color", "--eval-timeout=20", - '--thread-count=1', + '--thread-count=4', "--enable-reuse", "--collection-cache-size=256", '--debug', @@ -1089,7 +1110,10 @@ class TestSubmit(unittest.TestCase): "outputs": [ { "id": "#secret_job.cwl/out", - "type": "stdout" + "type": "File", + "outputBinding": { + "glob": "hashed_example.txt" + } } ], "stdout": "hashed_example.txt", @@ -1110,7 +1134,7 @@ class TestSubmit(unittest.TestCase): "hints": [ { "class": "DockerRequirement", - "dockerPull": "debian:8", + "dockerPull": "debian:buster-slim", "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99" }, { @@ -1180,7 +1204,7 @@ class TestSubmit(unittest.TestCase): } }, "state": "Committed", - "use_existing": True + "use_existing": False } stubs.api.container_requests().create.assert_called_with( @@ -1298,8 +1322,8 @@ class TestSubmit(unittest.TestCase): stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client) self.assertEqual(exited, 1) - self.assertRegexpMatches( - capture_stderr.getvalue(), + self.assertRegex( + re.sub(r'[ \n]+', ' ', capture_stderr.getvalue()), r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99") finally: cwltool_logger.removeHandler(stderr_logger) @@ -1321,7 +1345,7 @@ class TestSubmit(unittest.TestCase): try: self.assertEqual(exited, 1) - self.assertRegexpMatches( + self.assertRegex( capture_stderr.getvalue(), r"Collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz not found") finally: @@ -1331,7 +1355,7 @@ class TestSubmit(unittest.TestCase): class TestCreateWorkflow(unittest.TestCase): existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml" expect_workflow = StripYAMLComments( - open("tests/wf/expect_packed.cwl").read()) + open("tests/wf/expect_upload_packed.cwl").read().rstrip()) @stubs def test_create(self, stubs): @@ -1449,7 +1473,7 @@ class TestCreateWorkflow(unittest.TestCase): stubs.capture_stdout, sys.stderr, api_client=stubs.api) toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl" - expect_workflow = StripYAMLComments(open(toolfile).read()) + expect_workflow = StripYAMLComments(open(toolfile).read().rstrip()) body = { "workflow": { @@ -1465,3 +1489,39 @@ class TestCreateWorkflow(unittest.TestCase): self.assertEqual(stubs.capture_stdout.getvalue(), stubs.expect_workflow_uuid + '\n') self.assertEqual(exited, 0) + + @stubs + def test_create_with_imports(self, stubs): + project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz' + + exited = arvados_cwl.main( + ["--create-workflow", "--debug", + "--api=containers", + "--project-uuid", project_uuid, + "tests/wf/feddemo/feddemo.cwl"], + stubs.capture_stdout, sys.stderr, api_client=stubs.api) + + stubs.api.pipeline_templates().create.refute_called() + stubs.api.container_requests().create.refute_called() + + self.assertEqual(stubs.capture_stdout.getvalue(), + stubs.expect_workflow_uuid + '\n') + self.assertEqual(exited, 0) + + @stubs + def test_create_with_no_input(self, stubs): + project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz' + + exited = arvados_cwl.main( + ["--create-workflow", "--debug", + "--api=containers", + "--project-uuid", project_uuid, + "tests/wf/revsort/revsort.cwl"], + stubs.capture_stdout, sys.stderr, api_client=stubs.api) + + stubs.api.pipeline_templates().create.refute_called() + stubs.api.container_requests().create.refute_called() + + self.assertEqual(stubs.capture_stdout.getvalue(), + stubs.expect_workflow_uuid + '\n') + self.assertEqual(exited, 0)