X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/37eb070f55b5ae0c622fb4bf0a946c9dd49b2752..37516bc14fdfe634c78764c15f3a8eb3a09b403c:/sdk/cwl/tests/test_submit.py?ds=sidebyside diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py index bb0485d8b0..c8bf127951 100644 --- a/sdk/cwl/tests/test_submit.py +++ b/sdk/cwl/tests/test_submit.py @@ -10,6 +10,7 @@ from future.utils import viewvalues import copy import io +import itertools import functools import hashlib import json @@ -70,7 +71,12 @@ def stubs(wfdetails=('submit_wf.cwl', None)): uuid4.side_effect = ["df80736f-f14d-4b10-b2e3-03aa27f034bb", "df80736f-f14d-4b10-b2e3-03aa27f034b1", "df80736f-f14d-4b10-b2e3-03aa27f034b2", "df80736f-f14d-4b10-b2e3-03aa27f034b3", "df80736f-f14d-4b10-b2e3-03aa27f034b4", "df80736f-f14d-4b10-b2e3-03aa27f034b5", - "df80736f-f14d-4b10-b2e3-03aa27f034b6"] + "df80736f-f14d-4b10-b2e3-03aa27f034b6", "df80736f-f14d-4b10-b2e3-03aa27f034b7", + "df80736f-f14d-4b10-b2e3-03aa27f034b8", "df80736f-f14d-4b10-b2e3-03aa27f034b9", + "df80736f-f14d-4b10-b2e3-03aa27f034c0", "df80736f-f14d-4b10-b2e3-03aa27f034c1", + "df80736f-f14d-4b10-b2e3-03aa27f034c2", "df80736f-f14d-4b10-b2e3-03aa27f034c3", + "df80736f-f14d-4b10-b2e3-03aa27f034c4", "df80736f-f14d-4b10-b2e3-03aa27f034c5", + "df80736f-f14d-4b10-b2e3-03aa27f034c6", "df80736f-f14d-4b10-b2e3-03aa27f034c7"] determine_image_id.return_value = None @@ -403,7 +409,7 @@ class TestSubmit(unittest.TestCase): "class": "http://arvados.org/cwl#WorkflowRunnerResources" } ] - expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:af64a47741bcc401f230eee99c6e80ff+137/wf/submit_wf_no_reuse.cwl" + expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:fa5fbf21deb74f9f239daa3f5bb4b902+292/wf/submit_wf_no_reuse.cwl" stubs.api.container_requests().create.assert_called_with( body=JsonDiffMatcher(expect_container)) @@ -1023,7 +1029,7 @@ class TestSubmit(unittest.TestCase): # "arv": "http://arvados.org/cwl#", #} expect_container["command"] = ["--collection-cache-size=512" if v == "--collection-cache-size=256" else v for v in expect_container["command"]] - expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:10f1d17b8c4aad888e1dc2a93a95ceab+145/wf/submit_wf_runner_resources.cwl" + expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:80b60e39456505b91d3989a1f5058b98+308/wf/submit_wf_runner_resources.cwl" stubs.api.container_requests().create.assert_called_with( body=JsonDiffMatcher(expect_container)) @@ -1042,43 +1048,37 @@ class TestSubmit(unittest.TestCase): api.return_value = mock.MagicMock() arvrunner.api = api.return_value arvrunner.runtimeContext.match_local_docker = False - arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "", - "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb", - "link_class": "docker_image_repo+tag", - "name": "arvados/jobs:"+arvados_cwl.__version__, - "owner_uuid": "", - "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}, - {"items": [{"created_at": "", - "head_uuid": "", - "link_class": "docker_image_hash", - "name": "123456", - "owner_uuid": "", - "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}, - {"items": [{"created_at": "", - "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb", - "link_class": "docker_image_repo+tag", - "name": "arvados/jobs:"+arvados_cwl.__version__, - "owner_uuid": "", - "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}, - {"items": [{"created_at": "", - "head_uuid": "", - "link_class": "docker_image_hash", - "name": "123456", - "owner_uuid": "", - "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0} - ) + arvrunner.api.links().list().execute.side_effect = itertools.cycle([ + {"items": [{"created_at": "2023-08-25T12:34:56.123456Z", + "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb", + "link_class": "docker_image_repo+tag", + "name": "arvados/jobs:"+arvados_cwl.__version__, + "owner_uuid": "", + "uuid": "zzzzz-o0j2j-arvadosjobsrepo", + "properties": {"image_timestamp": ""}}]}, + {"items": []}, + {"items": []}, + {"items": [{"created_at": "2023-08-25T12:34:57.234567Z", + "head_uuid": "", + "link_class": "docker_image_hash", + "name": "123456", + "owner_uuid": "", + "uuid": "zzzzz-o0j2j-arvadosjobshash", + "properties": {"image_timestamp": ""}}]}, + {"items": []}, + {"items": []}, + ]) find_one_image_hash.return_value = "123456" - arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb", - "owner_uuid": "", - "manifest_text": "", - "properties": "" - }], "items_available": 1, "offset": 0}, - {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb", - "owner_uuid": "", - "manifest_text": "", - "properties": "" - }], "items_available": 1, "offset": 0}) + arvrunner.api.collections().list().execute.side_effect = itertools.cycle([ + {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb", + "owner_uuid": "", + "manifest_text": "", + "created_at": "2023-08-25T12:34:55.012345Z", + "properties": {}}]}, + {"items": []}, + {"items": []}, + ]) arvrunner.api.collections().create().execute.return_value = {"uuid": ""} arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb", "portable_data_hash": "9999999999999999999999999999999b+99"} @@ -1180,7 +1180,7 @@ class TestSubmit(unittest.TestCase): "out": [ {"id": "#main/step/out"} ], - "run": "keep:702edbfca88e907b62234556089dbce7+119/secret_wf.cwl" + "run": "keep:991302581d01db470345a131480e623b+247/secret_wf.cwl" } ] } @@ -1423,7 +1423,7 @@ class TestSubmit(unittest.TestCase): # "arv": "http://arvados.org/cwl#" #} - expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:743a5bcaef0604899e4f4706ac525d83+147/wf/submit_wf_process_properties.cwl" + expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:df44f9dd4b9467159f210f967e45417f+312/wf/submit_wf_process_properties.cwl" expect_container["properties"].update({ "baz": "blorp.txt", @@ -1737,3 +1737,55 @@ class TestCreateWorkflow(unittest.TestCase): self.assertEqual(stubs.capture_stdout.getvalue(), stubs.expect_workflow_uuid + '\n') self.assertEqual(exited, 0) + + @stubs() + def test_create_map(self, stubs): + # test uploading a document that uses objects instead of arrays + # for certain fields like inputs and requirements. + + project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz' + stubs.api.groups().get().execute.return_value = {"group_class": "project"} + + exited = arvados_cwl.main( + ["--create-workflow", "--debug", + "--api=containers", + "--project-uuid", project_uuid, + "--disable-git", + "tests/wf/submit_wf_map.cwl", "tests/submit_test_job.json"], + stubs.capture_stdout, sys.stderr, api_client=stubs.api) + + stubs.api.pipeline_templates().create.refute_called() + stubs.api.container_requests().create.refute_called() + + expect_workflow = StripYAMLComments( + open("tests/wf/expect_upload_wrapper_map.cwl").read().rstrip()) + + body = { + "workflow": { + "owner_uuid": project_uuid, + "name": "submit_wf_map.cwl", + "description": "", + "definition": expect_workflow, + } + } + stubs.api.workflows().create.assert_called_with( + body=JsonDiffMatcher(body)) + + self.assertEqual(stubs.capture_stdout.getvalue(), + stubs.expect_workflow_uuid + '\n') + self.assertEqual(exited, 0) + + +class TestPrintKeepDeps(unittest.TestCase): + @stubs() + def test_print_keep_deps(self, stubs): + # test --print-keep-deps which is used by arv-copy + + exited = arvados_cwl.main( + ["--print-keep-deps", "--debug", + "tests/wf/submit_wf_map.cwl"], + stubs.capture_stdout, sys.stderr, api_client=stubs.api) + + self.assertEqual(stubs.capture_stdout.getvalue(), + '["5d373e7629203ce39e7c22af98a0f881+52", "999999999999999999999999999999d4+99"]' + '\n') + self.assertEqual(exited, 0)