projects / arvados.git / blobdiff
summary | shortlog | log | commit | commitdiff | tree
raw | inline | side by side
Merge branch 'main' into 21357-favorites-names
[arvados.git] / sdk / cwl / tests / test_submit.py
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index bb0485d8b0685a9fd7f774340d939b971e1dd257..c8bf1279511cd8591104af5b196b4938dd71eb88 100644 (file)
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -10,6 +10,7 @@ from future.utils import viewvalues
 
 import copy
 import io
+import itertools
 import functools
 import hashlib
 import json
@@ -70,7 +71,12 @@ def stubs(wfdetails=('submit_wf.cwl', None)):
             uuid4.side_effect = ["df80736f-f14d-4b10-b2e3-03aa27f034bb", "df80736f-f14d-4b10-b2e3-03aa27f034b1",
                                  "df80736f-f14d-4b10-b2e3-03aa27f034b2", "df80736f-f14d-4b10-b2e3-03aa27f034b3",
                                  "df80736f-f14d-4b10-b2e3-03aa27f034b4", "df80736f-f14d-4b10-b2e3-03aa27f034b5",
-                                 "df80736f-f14d-4b10-b2e3-03aa27f034b6"]
+                                 "df80736f-f14d-4b10-b2e3-03aa27f034b6", "df80736f-f14d-4b10-b2e3-03aa27f034b7",
+                                 "df80736f-f14d-4b10-b2e3-03aa27f034b8", "df80736f-f14d-4b10-b2e3-03aa27f034b9",
+                                 "df80736f-f14d-4b10-b2e3-03aa27f034c0", "df80736f-f14d-4b10-b2e3-03aa27f034c1",
+                                 "df80736f-f14d-4b10-b2e3-03aa27f034c2", "df80736f-f14d-4b10-b2e3-03aa27f034c3",
+                                 "df80736f-f14d-4b10-b2e3-03aa27f034c4", "df80736f-f14d-4b10-b2e3-03aa27f034c5",
+                                 "df80736f-f14d-4b10-b2e3-03aa27f034c6", "df80736f-f14d-4b10-b2e3-03aa27f034c7"]
 
             determine_image_id.return_value = None
 
@@ -403,7 +409,7 @@ class TestSubmit(unittest.TestCase):
                     "class": "http://arvados.org/cwl#WorkflowRunnerResources"
                 }
         ]
-        expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:af64a47741bcc401f230eee99c6e80ff+137/wf/submit_wf_no_reuse.cwl"
+        expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:fa5fbf21deb74f9f239daa3f5bb4b902+292/wf/submit_wf_no_reuse.cwl"
 
         stubs.api.container_requests().create.assert_called_with(
             body=JsonDiffMatcher(expect_container))
@@ -1023,7 +1029,7 @@ class TestSubmit(unittest.TestCase):
         #    "arv": "http://arvados.org/cwl#",
         #}
         expect_container["command"] = ["--collection-cache-size=512" if v == "--collection-cache-size=256" else v for v in expect_container["command"]]
-        expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:10f1d17b8c4aad888e1dc2a93a95ceab+145/wf/submit_wf_runner_resources.cwl"
+        expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:80b60e39456505b91d3989a1f5058b98+308/wf/submit_wf_runner_resources.cwl"
 
         stubs.api.container_requests().create.assert_called_with(
             body=JsonDiffMatcher(expect_container))
@@ -1042,43 +1048,37 @@ class TestSubmit(unittest.TestCase):
         api.return_value = mock.MagicMock()
         arvrunner.api = api.return_value
         arvrunner.runtimeContext.match_local_docker = False
-        arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
-                                                                        "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
-                                                                        "link_class": "docker_image_repo+tag",
-                                                                        "name": "arvados/jobs:"+arvados_cwl.__version__,
-                                                                        "owner_uuid": "",
-                                                                        "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
-                                                            {"items": [{"created_at": "",
-                                                                        "head_uuid": "",
-                                                                        "link_class": "docker_image_hash",
-                                                                        "name": "123456",
-                                                                        "owner_uuid": "",
-                                                                        "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
-                                                            {"items": [{"created_at": "",
-                                                                        "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
-                                                                        "link_class": "docker_image_repo+tag",
-                                                                        "name": "arvados/jobs:"+arvados_cwl.__version__,
-                                                                        "owner_uuid": "",
-                                                                        "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
-                                                            {"items": [{"created_at": "",
-                                                                        "head_uuid": "",
-                                                                        "link_class": "docker_image_hash",
-                                                                        "name": "123456",
-                                                                        "owner_uuid": "",
-                                                                        "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
-        )
+        arvrunner.api.links().list().execute.side_effect = itertools.cycle([
+            {"items": [{"created_at": "2023-08-25T12:34:56.123456Z",
+                        "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
+                        "link_class": "docker_image_repo+tag",
+                        "name": "arvados/jobs:"+arvados_cwl.__version__,
+                        "owner_uuid": "",
+                        "uuid": "zzzzz-o0j2j-arvadosjobsrepo",
+                        "properties": {"image_timestamp": ""}}]},
+            {"items": []},
+            {"items": []},
+            {"items": [{"created_at": "2023-08-25T12:34:57.234567Z",
+                        "head_uuid": "",
+                        "link_class": "docker_image_hash",
+                        "name": "123456",
+                        "owner_uuid": "",
+                        "uuid": "zzzzz-o0j2j-arvadosjobshash",
+                        "properties": {"image_timestamp": ""}}]},
+            {"items": []},
+            {"items": []},
+        ])
         find_one_image_hash.return_value = "123456"
 
-        arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
-                                                                              "owner_uuid": "",
-                                                                              "manifest_text": "",
-                                                                              "properties": ""
-                                                                              }], "items_available": 1, "offset": 0},
-                                                                  {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
-                                                                              "owner_uuid": "",
-                                                                              "manifest_text": "",
-                                                                              "properties": ""
-                                                                          }], "items_available": 1, "offset": 0})
+        arvrunner.api.collections().list().execute.side_effect = itertools.cycle([
+            {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
+                        "owner_uuid": "",
+                        "manifest_text": "",
+                        "created_at": "2023-08-25T12:34:55.012345Z",
+                        "properties": {}}]},
+            {"items": []},
+            {"items": []},
+        ])
         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
         arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
                                                                   "portable_data_hash": "9999999999999999999999999999999b+99"}
@@ -1180,7 +1180,7 @@ class TestSubmit(unittest.TestCase):
                                         "out": [
                                             {"id": "#main/step/out"}
                                         ],
-                                        "run": "keep:702edbfca88e907b62234556089dbce7+119/secret_wf.cwl"
+                                        "run": "keep:991302581d01db470345a131480e623b+247/secret_wf.cwl"
                                     }
                                 ]
                             }
@@ -1423,7 +1423,7 @@ class TestSubmit(unittest.TestCase):
         #    "arv": "http://arvados.org/cwl#"
         #}
 
-        expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:743a5bcaef0604899e4f4706ac525d83+147/wf/submit_wf_process_properties.cwl"
+        expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:df44f9dd4b9467159f210f967e45417f+312/wf/submit_wf_process_properties.cwl"
 
         expect_container["properties"].update({
             "baz": "blorp.txt",
@@ -1737,3 +1737,55 @@ class TestCreateWorkflow(unittest.TestCase):
         self.assertEqual(stubs.capture_stdout.getvalue(),
                          stubs.expect_workflow_uuid + '\n')
         self.assertEqual(exited, 0)
+
+    @stubs()
+    def test_create_map(self, stubs):
+        # test uploading a document that uses objects instead of arrays
+        # for certain fields like inputs and requirements.
+
+        project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
+        stubs.api.groups().get().execute.return_value = {"group_class": "project"}
+
+        exited = arvados_cwl.main(
+            ["--create-workflow", "--debug",
+             "--api=containers",
+             "--project-uuid", project_uuid,
+             "--disable-git",
+             "tests/wf/submit_wf_map.cwl", "tests/submit_test_job.json"],
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api)
+
+        stubs.api.pipeline_templates().create.refute_called()
+        stubs.api.container_requests().create.refute_called()
+
+        expect_workflow = StripYAMLComments(
+            open("tests/wf/expect_upload_wrapper_map.cwl").read().rstrip())
+
+        body = {
+            "workflow": {
+                "owner_uuid": project_uuid,
+                "name": "submit_wf_map.cwl",
+                "description": "",
+                "definition": expect_workflow,
+            }
+        }
+        stubs.api.workflows().create.assert_called_with(
+            body=JsonDiffMatcher(body))
+
+        self.assertEqual(stubs.capture_stdout.getvalue(),
+                         stubs.expect_workflow_uuid + '\n')
+        self.assertEqual(exited, 0)
+
+
+class TestPrintKeepDeps(unittest.TestCase):
+    @stubs()
+    def test_print_keep_deps(self, stubs):
+        # test --print-keep-deps which is used by arv-copy
+
+        exited = arvados_cwl.main(
+            ["--print-keep-deps", "--debug",
+             "tests/wf/submit_wf_map.cwl"],
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api)
+
+        self.assertEqual(stubs.capture_stdout.getvalue(),
+                         '["5d373e7629203ce39e7c22af98a0f881+52", "999999999999999999999999999999d4+99"]' + '\n')
+        self.assertEqual(exited, 0)