X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/97e0090ec440c3322053d7635d883cf8af6633ca..f6a29cc714dd3409f865ceb799a886dd0b5d8152:/sdk/cwl/tests/test_submit.py diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py index 2889ce331e..305d51e144 100644 --- a/sdk/cwl/tests/test_submit.py +++ b/sdk/cwl/tests/test_submit.py @@ -47,17 +47,26 @@ _rootDesc = None def stubs(func): @functools.wraps(func) + @mock.patch("arvados_cwl.arvdocker.determine_image_id") + @mock.patch("uuid.uuid4") @mock.patch("arvados.commands.keepdocker.list_images_in_arv") @mock.patch("arvados.collection.KeepClient") @mock.patch("arvados.keep.KeepClient") @mock.patch("arvados.events.subscribe") - def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs): + def wrapped(self, events, keep_client1, keep_client2, keepdocker, + uuid4, determine_image_id, *args, **kwargs): class Stubs(object): pass stubs = Stubs() stubs.events = events stubs.keepdocker = keepdocker + uuid4.side_effect = ["df80736f-f14d-4b10-b2e3-03aa27f034bb", "df80736f-f14d-4b10-b2e3-03aa27f034b1", + "df80736f-f14d-4b10-b2e3-03aa27f034b2", "df80736f-f14d-4b10-b2e3-03aa27f034b3", + "df80736f-f14d-4b10-b2e3-03aa27f034b4", "df80736f-f14d-4b10-b2e3-03aa27f034b5"] + + determine_image_id.return_value = None + def putstub(p, **kwargs): return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p)) keep_client1().put.side_effect = putstub @@ -67,12 +76,12 @@ def stubs(func): stubs.keep_client = keep_client2 stubs.docker_images = { - "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", "")], - "debian:buster-slim": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", "")], - "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", "")], - "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", "")], + "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", {})], + "debian:buster-slim": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", {})], + "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", {})], + "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", {})], } - def kd(a, b, image_name=None, image_tag=None): + def kd(a, b, image_name=None, image_tag=None, project_uuid=None): return stubs.docker_images.get("%s:%s" % (image_name, image_tag), []) stubs.keepdocker.side_effect = kd @@ -87,6 +96,7 @@ def stubs(func): stubs.api = mock.MagicMock() stubs.api._rootDesc = get_rootDesc() stubs.api._rootDesc["uuidPrefix"] = "zzzzz" + stubs.api._rootDesc["revision"] = "20210628" stubs.api.users().current().execute.return_value = { "uuid": stubs.fake_user_uuid, @@ -347,6 +357,12 @@ class TestSubmit(unittest.TestCase): cwltool.process._names = set() arvados_cwl.arvdocker.arv_docker_clear_cache() + def tearDown(self): + root_logger = logging.getLogger('') + + # Remove existing RuntimeStatusLoggingHandlers if they exist + handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)] + root_logger.handlers = handlers @mock.patch("time.sleep") @stubs @@ -446,7 +462,7 @@ class TestSubmit(unittest.TestCase): "enableReuse": False, }, ] - expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = { + expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = { "arv": "http://arvados.org/cwl#", "cwltool": "http://commonwl.org/cwltool#" } @@ -572,6 +588,7 @@ class TestSubmit(unittest.TestCase): def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq): final_output_c = arvados.collection.Collection() make_output.return_value = ({},final_output_c) + stubs.api.config().get.return_value = {"default": {"Default": True}} def set_final_output(job_order, output_callback, runtimeContext): output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success") @@ -1032,7 +1049,7 @@ class TestSubmit(unittest.TestCase): "keep_cache": 512 } ] - expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = { + expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = { "arv": "http://arvados.org/cwl#", } expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers', @@ -1047,9 +1064,6 @@ class TestSubmit(unittest.TestCase): stubs.expect_container_request_uuid + '\n') self.assertEqual(exited, 0) - def tearDown(self): - arvados_cwl.arvdocker.arv_docker_clear_cache() - @mock.patch("arvados.commands.keepdocker.find_one_image_hash") @mock.patch("cwltool.docker.DockerCommandLineJob.get_image") @mock.patch("arvados.api") @@ -1060,12 +1074,25 @@ class TestSubmit(unittest.TestCase): arvrunner.project_uuid = "" api.return_value = mock.MagicMock() arvrunner.api = api.return_value + arvrunner.runtimeContext.match_local_docker = False arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "", "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb", "link_class": "docker_image_repo+tag", "name": "arvados/jobs:"+arvados_cwl.__version__, "owner_uuid": "", "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}, + {"items": [{"created_at": "", + "head_uuid": "", + "link_class": "docker_image_hash", + "name": "123456", + "owner_uuid": "", + "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}, + {"items": [{"created_at": "", + "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb", + "link_class": "docker_image_repo+tag", + "name": "arvados/jobs:"+arvados_cwl.__version__, + "owner_uuid": "", + "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}, {"items": [{"created_at": "", "head_uuid": "", "link_class": "docker_image_hash", @@ -1079,12 +1106,18 @@ class TestSubmit(unittest.TestCase): "owner_uuid": "", "manifest_text": "", "properties": "" - }], "items_available": 1, "offset": 0},) + }], "items_available": 1, "offset": 0}, + {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb", + "owner_uuid": "", + "manifest_text": "", + "properties": "" + }], "items_available": 1, "offset": 0}) arvrunner.api.collections().create().execute.return_value = {"uuid": ""} arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb", "portable_data_hash": "9999999999999999999999999999999b+99"} + self.assertEqual("9999999999999999999999999999999b+99", - arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__)) + arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__, arvrunner.runtimeContext)) @stubs @@ -1126,9 +1159,6 @@ class TestSubmit(unittest.TestCase): "content": { "$graph": [ { - "$namespaces": { - "cwltool": "http://commonwl.org/cwltool#" - }, "arguments": [ "md5sum", "example.conf" @@ -1217,6 +1247,9 @@ class TestSubmit(unittest.TestCase): ] } ], + "$namespaces": { + "cwltool": "http://commonwl.org/cwltool#" + }, "cwlVersion": "v1.0" }, "kind": "json" @@ -1415,7 +1448,7 @@ class TestSubmit(unittest.TestCase): self.assertEqual(exited, 1) self.assertRegex( capture_stderr.getvalue(), - r"Collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz not found") + r"Collection\s*uuid\s*zzzzz-4zz18-zzzzzzzzzzzzzzz\s*not\s*found") finally: cwltool_logger.removeHandler(stderr_logger) @@ -1432,10 +1465,10 @@ class TestSubmit(unittest.TestCase): { "class": "http://arvados.org/cwl#ProcessProperties", "processProperties": [ - {"propertyName": "foo", - "propertyValue": "bar"}, {"propertyName": "baz", "propertyValue": "$(inputs.x.basename)"}, + {"propertyName": "foo", + "propertyValue": "bar"}, {"propertyName": "quux", "propertyValue": { "q1": 1, @@ -1445,7 +1478,7 @@ class TestSubmit(unittest.TestCase): ], } ] - expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = { + expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = { "arv": "http://arvados.org/cwl#" } @@ -1465,6 +1498,49 @@ class TestSubmit(unittest.TestCase): self.assertEqual(exited, 0) + @stubs + def test_submit_enable_preemptible(self, stubs): + exited = arvados_cwl.main( + ["--submit", "--no-wait", "--api=containers", "--debug", "--enable-preemptible", + "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"], + stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client) + + expect_container = copy.deepcopy(stubs.expect_container_spec) + expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers', + '--no-log-timestamps', '--disable-validate', '--disable-color', + '--eval-timeout=20', '--thread-count=0', + '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue', + '--enable-preemptible', + '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'] + + stubs.api.container_requests().create.assert_called_with( + body=JsonDiffMatcher(expect_container)) + self.assertEqual(stubs.capture_stdout.getvalue(), + stubs.expect_container_request_uuid + '\n') + self.assertEqual(exited, 0) + + @stubs + def test_submit_disable_preemptible(self, stubs): + exited = arvados_cwl.main( + ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-preemptible", + "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"], + stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client) + + expect_container = copy.deepcopy(stubs.expect_container_spec) + expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers', + '--no-log-timestamps', '--disable-validate', '--disable-color', + '--eval-timeout=20', '--thread-count=0', + '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue', + '--disable-preemptible', + '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'] + + stubs.api.container_requests().create.assert_called_with( + body=JsonDiffMatcher(expect_container)) + self.assertEqual(stubs.capture_stdout.getvalue(), + stubs.expect_container_request_uuid + '\n') + self.assertEqual(exited, 0) + + class TestCreateWorkflow(unittest.TestCase): existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml" expect_workflow = StripYAMLComments( @@ -1474,6 +1550,13 @@ class TestCreateWorkflow(unittest.TestCase): cwltool.process._names = set() arvados_cwl.arvdocker.arv_docker_clear_cache() + def tearDown(self): + root_logger = logging.getLogger('') + + # Remove existing RuntimeStatusLoggingHandlers if they exist + handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)] + root_logger.handlers = handlers + @stubs def test_create(self, stubs): project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz' @@ -1538,6 +1621,9 @@ class TestCreateWorkflow(unittest.TestCase): @stubs def test_update(self, stubs): + project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz' + stubs.api.workflows().get().execute.return_value = {"owner_uuid": project_uuid} + exited = arvados_cwl.main( ["--update-workflow", self.existing_workflow_uuid, "--debug", @@ -1549,6 +1635,7 @@ class TestCreateWorkflow(unittest.TestCase): "name": "submit_wf.cwl", "description": "", "definition": self.expect_workflow, + "owner_uuid": project_uuid } } stubs.api.workflows().update.assert_called_with( @@ -1558,8 +1645,12 @@ class TestCreateWorkflow(unittest.TestCase): self.existing_workflow_uuid + '\n') self.assertEqual(exited, 0) + @stubs def test_update_name(self, stubs): + project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz' + stubs.api.workflows().get().execute.return_value = {"owner_uuid": project_uuid} + exited = arvados_cwl.main( ["--update-workflow", self.existing_workflow_uuid, "--debug", "--name", "testing 123", @@ -1571,6 +1662,7 @@ class TestCreateWorkflow(unittest.TestCase): "name": "testing 123", "description": "", "definition": self.expect_workflow, + "owner_uuid": project_uuid } } stubs.api.workflows().update.assert_called_with(