X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/1c95615a417b6435dc86f80e3f02563196449611..a3d2b8e1de5b8c785846ddc57ae9a4c02bc51adc:/sdk/cwl/tests/test_submit.py diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py index 77f70851e8..2b74435f7e 100644 --- a/sdk/cwl/tests/test_submit.py +++ b/sdk/cwl/tests/test_submit.py @@ -47,17 +47,26 @@ _rootDesc = None def stubs(func): @functools.wraps(func) + @mock.patch("arvados_cwl.arvdocker.determine_image_id") + @mock.patch("uuid.uuid4") @mock.patch("arvados.commands.keepdocker.list_images_in_arv") @mock.patch("arvados.collection.KeepClient") @mock.patch("arvados.keep.KeepClient") @mock.patch("arvados.events.subscribe") - def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs): + def wrapped(self, events, keep_client1, keep_client2, keepdocker, + uuid4, determine_image_id, *args, **kwargs): class Stubs(object): pass stubs = Stubs() stubs.events = events stubs.keepdocker = keepdocker + uuid4.side_effect = ["df80736f-f14d-4b10-b2e3-03aa27f034bb", "df80736f-f14d-4b10-b2e3-03aa27f034b1", + "df80736f-f14d-4b10-b2e3-03aa27f034b2", "df80736f-f14d-4b10-b2e3-03aa27f034b3", + "df80736f-f14d-4b10-b2e3-03aa27f034b4", "df80736f-f14d-4b10-b2e3-03aa27f034b5"] + + determine_image_id.return_value = None + def putstub(p, **kwargs): return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p)) keep_client1().put.side_effect = putstub @@ -67,12 +76,12 @@ def stubs(func): stubs.keep_client = keep_client2 stubs.docker_images = { - "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", "")], - "debian:buster-slim": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", "")], - "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", "")], - "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", "")], + "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", {})], + "debian:buster-slim": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", {})], + "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", {})], + "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", {})], } - def kd(a, b, image_name=None, image_tag=None): + def kd(a, b, image_name=None, image_tag=None, project_uuid=None): return stubs.docker_images.get("%s:%s" % (image_name, image_tag), []) stubs.keepdocker.side_effect = kd @@ -348,6 +357,12 @@ class TestSubmit(unittest.TestCase): cwltool.process._names = set() arvados_cwl.arvdocker.arv_docker_clear_cache() + def tearDown(self): + root_logger = logging.getLogger('') + + # Remove existing RuntimeStatusLoggingHandlers if they exist + handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)] + root_logger.handlers = handlers @mock.patch("time.sleep") @stubs @@ -554,7 +569,7 @@ class TestSubmit(unittest.TestCase): make_output.return_value = ({},final_output_c) def set_final_output(job_order, output_callback, runtimeContext): - output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success") + output_callback({"out": "zzzzz"}, "success") return [] job.side_effect = set_final_output @@ -563,7 +578,7 @@ class TestSubmit(unittest.TestCase): "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"], stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client) - make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz') + make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', {}, {"out": "zzzzz"}) self.assertEqual(exited, 0) @mock.patch("cwltool.task_queue.TaskQueue") @@ -576,7 +591,7 @@ class TestSubmit(unittest.TestCase): stubs.api.config().get.return_value = {"default": {"Default": True}} def set_final_output(job_order, output_callback, runtimeContext): - output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success") + output_callback({"out": "zzzzz"}, "success") return [] job.side_effect = set_final_output @@ -585,7 +600,7 @@ class TestSubmit(unittest.TestCase): "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"], stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client) - make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz') + make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], '', {}, {"out": "zzzzz"}) self.assertEqual(exited, 0) @mock.patch("cwltool.task_queue.TaskQueue") @@ -597,7 +612,7 @@ class TestSubmit(unittest.TestCase): make_output.return_value = ({},final_output_c) def set_final_output(job_order, output_callback, runtimeContext): - output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success") + output_callback({"out": "zzzzz"}, "success") return [] job.side_effect = set_final_output @@ -606,7 +621,7 @@ class TestSubmit(unittest.TestCase): "tests/wf/submit_storage_class_wf.cwl", "tests/submit_test_job.json"], stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client) - make_output.assert_called_with(u'Output of submit_storage_class_wf.cwl', ['foo', 'bar'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz') + make_output.assert_called_with(u'Output of submit_storage_class_wf.cwl', ['foo', 'bar'], '', {}, {"out": "zzzzz"}) self.assertEqual(exited, 0) @stubs @@ -1049,9 +1064,6 @@ class TestSubmit(unittest.TestCase): stubs.expect_container_request_uuid + '\n') self.assertEqual(exited, 0) - def tearDown(self): - arvados_cwl.arvdocker.arv_docker_clear_cache() - @mock.patch("arvados.commands.keepdocker.find_one_image_hash") @mock.patch("cwltool.docker.DockerCommandLineJob.get_image") @mock.patch("arvados.api") @@ -1062,12 +1074,25 @@ class TestSubmit(unittest.TestCase): arvrunner.project_uuid = "" api.return_value = mock.MagicMock() arvrunner.api = api.return_value + arvrunner.runtimeContext.match_local_docker = False arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "", "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb", "link_class": "docker_image_repo+tag", "name": "arvados/jobs:"+arvados_cwl.__version__, "owner_uuid": "", "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}, + {"items": [{"created_at": "", + "head_uuid": "", + "link_class": "docker_image_hash", + "name": "123456", + "owner_uuid": "", + "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}, + {"items": [{"created_at": "", + "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb", + "link_class": "docker_image_repo+tag", + "name": "arvados/jobs:"+arvados_cwl.__version__, + "owner_uuid": "", + "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}, {"items": [{"created_at": "", "head_uuid": "", "link_class": "docker_image_hash", @@ -1081,12 +1106,18 @@ class TestSubmit(unittest.TestCase): "owner_uuid": "", "manifest_text": "", "properties": "" - }], "items_available": 1, "offset": 0},) + }], "items_available": 1, "offset": 0}, + {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb", + "owner_uuid": "", + "manifest_text": "", + "properties": "" + }], "items_available": 1, "offset": 0}) arvrunner.api.collections().create().execute.return_value = {"uuid": ""} arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb", "portable_data_hash": "9999999999999999999999999999999b+99"} + self.assertEqual("9999999999999999999999999999999b+99", - arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__)) + arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__, arvrunner.runtimeContext)) @stubs @@ -1417,7 +1448,7 @@ class TestSubmit(unittest.TestCase): self.assertEqual(exited, 1) self.assertRegex( capture_stderr.getvalue(), - r"Collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz not found") + r"Collection\s*uuid\s*zzzzz-4zz18-zzzzzzzzzzzzzzz\s*not\s*found") finally: cwltool_logger.removeHandler(stderr_logger) @@ -1467,6 +1498,49 @@ class TestSubmit(unittest.TestCase): self.assertEqual(exited, 0) + @stubs + def test_submit_enable_preemptible(self, stubs): + exited = arvados_cwl.main( + ["--submit", "--no-wait", "--api=containers", "--debug", "--enable-preemptible", + "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"], + stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client) + + expect_container = copy.deepcopy(stubs.expect_container_spec) + expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers', + '--no-log-timestamps', '--disable-validate', '--disable-color', + '--eval-timeout=20', '--thread-count=0', + '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue', + '--enable-preemptible', + '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'] + + stubs.api.container_requests().create.assert_called_with( + body=JsonDiffMatcher(expect_container)) + self.assertEqual(stubs.capture_stdout.getvalue(), + stubs.expect_container_request_uuid + '\n') + self.assertEqual(exited, 0) + + @stubs + def test_submit_disable_preemptible(self, stubs): + exited = arvados_cwl.main( + ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-preemptible", + "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"], + stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client) + + expect_container = copy.deepcopy(stubs.expect_container_spec) + expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers', + '--no-log-timestamps', '--disable-validate', '--disable-color', + '--eval-timeout=20', '--thread-count=0', + '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue', + '--disable-preemptible', + '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'] + + stubs.api.container_requests().create.assert_called_with( + body=JsonDiffMatcher(expect_container)) + self.assertEqual(stubs.capture_stdout.getvalue(), + stubs.expect_container_request_uuid + '\n') + self.assertEqual(exited, 0) + + class TestCreateWorkflow(unittest.TestCase): existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml" expect_workflow = StripYAMLComments( @@ -1476,6 +1550,13 @@ class TestCreateWorkflow(unittest.TestCase): cwltool.process._names = set() arvados_cwl.arvdocker.arv_docker_clear_cache() + def tearDown(self): + root_logger = logging.getLogger('') + + # Remove existing RuntimeStatusLoggingHandlers if they exist + handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)] + root_logger.handlers = handlers + @stubs def test_create(self, stubs): project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz' @@ -1540,6 +1621,9 @@ class TestCreateWorkflow(unittest.TestCase): @stubs def test_update(self, stubs): + project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz' + stubs.api.workflows().get().execute.return_value = {"owner_uuid": project_uuid} + exited = arvados_cwl.main( ["--update-workflow", self.existing_workflow_uuid, "--debug", @@ -1551,6 +1635,7 @@ class TestCreateWorkflow(unittest.TestCase): "name": "submit_wf.cwl", "description": "", "definition": self.expect_workflow, + "owner_uuid": project_uuid } } stubs.api.workflows().update.assert_called_with( @@ -1560,8 +1645,12 @@ class TestCreateWorkflow(unittest.TestCase): self.existing_workflow_uuid + '\n') self.assertEqual(exited, 0) + @stubs def test_update_name(self, stubs): + project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz' + stubs.api.workflows().get().execute.return_value = {"owner_uuid": project_uuid} + exited = arvados_cwl.main( ["--update-workflow", self.existing_workflow_uuid, "--debug", "--name", "testing 123", @@ -1573,6 +1662,7 @@ class TestCreateWorkflow(unittest.TestCase): "name": "testing 123", "description": "", "definition": self.expect_workflow, + "owner_uuid": project_uuid } } stubs.api.workflows().update.assert_called_with(