17778: Merge branch 'master' into 17778-doc-update

[arvados.git] / sdk / cwl / tests / test_submit.py

diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index d6ef665fb8fb3b89325ef022697c2f74cda05a0f..c448f218bc5d7d0029c9b1ce359e44887053a8a6 100644 (file)
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -18,6 +18,7 @@ import mock
 import sys
 import unittest
 import cwltool.process
+import re
 
 from io import BytesIO
 
@@ -67,7 +68,7 @@ def stubs(func):
         stubs.keep_client = keep_client2
         stubs.docker_images = {
             "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", "")],
-            "debian:8": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", "")],
+            "debian:buster-slim": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", "")],
             "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", "")],
             "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", "")],
         }
@@ -301,8 +302,8 @@ def stubs(func):
             'secret_mounts': {},
             'state': 'Committed',
             'command': ['arvados-cwl-runner', '--local', '--api=containers',
-                        '--no-log-timestamps', '--disable-validate',
-                        '--eval-timeout=20', '--thread-count=1',
+                        '--no-log-timestamps', '--disable-validate', '--disable-color',
+                        '--eval-timeout=20', '--thread-count=4',
                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
             'name': 'submit_wf.cwl',
@@ -314,7 +315,7 @@ def stubs(func):
                 'vcpus': 1,
                 'ram': (1024+256)*1024*1024
             },
-            'use_existing': True,
+            'use_existing': False,
             'properties': {},
             'secret_mounts': {}
         }
@@ -339,6 +340,7 @@ class TestSubmit(unittest.TestCase):
 
     def setUp(self):
         cwltool.process._names = set()
+        arvados_cwl.arvdocker.arv_docker_clear_cache()
 
     @stubs
     def test_error_when_multiple_storage_classes_specified(self, stubs):
@@ -388,6 +390,19 @@ class TestSubmit(unittest.TestCase):
                          stubs.expect_container_request_uuid + '\n')
         self.assertEqual(exited, 0)
 
+
+    @stubs
+    def test_submit_container_tool(self, stubs):
+        # test for issue #16139
+        exited = arvados_cwl.main(
+            ["--submit", "--no-wait", "--api=containers", "--debug",
+                "tests/tool/tool_with_sf.cwl", "tests/tool/tool_with_sf.yml"],
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+
+        self.assertEqual(stubs.capture_stdout.getvalue(),
+                         stubs.expect_container_request_uuid + '\n')
+        self.assertEqual(exited, 0)
+
     @stubs
     def test_submit_container_no_reuse(self, stubs):
         exited = arvados_cwl.main(
@@ -398,8 +413,8 @@ class TestSubmit(unittest.TestCase):
         expect_container = copy.deepcopy(stubs.expect_container_spec)
         expect_container["command"] = [
             'arvados-cwl-runner', '--local', '--api=containers',
-            '--no-log-timestamps', '--disable-validate',
-            '--eval-timeout=20', '--thread-count=1',
+            '--no-log-timestamps', '--disable-validate', '--disable-color',
+            '--eval-timeout=20', '--thread-count=4',
             '--disable-reuse', "--collection-cache-size=256",
             '--debug', '--on-error=continue',
             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
@@ -422,8 +437,8 @@ class TestSubmit(unittest.TestCase):
         expect_container = copy.deepcopy(stubs.expect_container_spec)
         expect_container["command"] = [
             'arvados-cwl-runner', '--local', '--api=containers',
-            '--no-log-timestamps', '--disable-validate',
-            '--eval-timeout=20', '--thread-count=1',
+            '--no-log-timestamps', '--disable-validate', '--disable-color',
+            '--eval-timeout=20', '--thread-count=4',
             '--disable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
         expect_container["use_existing"] = False
@@ -454,8 +469,8 @@ class TestSubmit(unittest.TestCase):
 
         expect_container = copy.deepcopy(stubs.expect_container_spec)
         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
-                                       '--no-log-timestamps', '--disable-validate',
-                                       '--eval-timeout=20', '--thread-count=1',
+                                       '--no-log-timestamps', '--disable-validate', '--disable-color',
+                                       '--eval-timeout=20', '--thread-count=4',
                                        '--enable-reuse', "--collection-cache-size=256",
                                        '--debug', '--on-error=stop',
                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
@@ -477,8 +492,8 @@ class TestSubmit(unittest.TestCase):
 
         expect_container = copy.deepcopy(stubs.expect_container_spec)
         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
-                                       '--no-log-timestamps', '--disable-validate',
-                                       '--eval-timeout=20', '--thread-count=1',
+                                       '--no-log-timestamps', '--disable-validate', '--disable-color',
+                                       '--eval-timeout=20', '--thread-count=4',
                                        '--enable-reuse', "--collection-cache-size=256",
                                        "--output-name="+output_name, '--debug', '--on-error=continue',
                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
@@ -499,8 +514,8 @@ class TestSubmit(unittest.TestCase):
 
         expect_container = copy.deepcopy(stubs.expect_container_spec)
         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
-                                       '--no-log-timestamps', '--disable-validate',
-                                       '--eval-timeout=20', '--thread-count=1',
+                                       '--no-log-timestamps', '--disable-validate', '--disable-color',
+                                       '--eval-timeout=20', '--thread-count=4',
                                        '--enable-reuse', "--collection-cache-size=256", "--debug",
                                        "--storage-classes=foo", '--on-error=continue',
                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
@@ -511,11 +526,14 @@ class TestSubmit(unittest.TestCase):
                          stubs.expect_container_request_uuid + '\n')
         self.assertEqual(exited, 0)
 
-    @mock.patch("arvados_cwl.task_queue.TaskQueue")
+    @mock.patch("cwltool.task_queue.TaskQueue")
     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
-    @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection", return_value = (None, None))
+    @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
     @stubs
     def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
+        final_output_c = arvados.collection.Collection()
+        make_output.return_value = ({},final_output_c)
+
         def set_final_output(job_order, output_callback, runtimeContext):
             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
             return []
@@ -524,16 +542,19 @@ class TestSubmit(unittest.TestCase):
         exited = arvados_cwl.main(
             ["--debug", "--local", "--storage-classes=foo",
                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-            sys.stdin, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 
         make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
         self.assertEqual(exited, 0)
 
-    @mock.patch("arvados_cwl.task_queue.TaskQueue")
+    @mock.patch("cwltool.task_queue.TaskQueue")
     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
-    @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection", return_value = (None, None))
+    @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
     @stubs
     def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
+        final_output_c = arvados.collection.Collection()
+        make_output.return_value = ({},final_output_c)
+
         def set_final_output(job_order, output_callback, runtimeContext):
             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
             return []
@@ -542,7 +563,7 @@ class TestSubmit(unittest.TestCase):
         exited = arvados_cwl.main(
             ["--debug", "--local",
                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-            sys.stdin, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 
         make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
         self.assertEqual(exited, 0)
@@ -556,8 +577,8 @@ class TestSubmit(unittest.TestCase):
 
         expect_container = copy.deepcopy(stubs.expect_container_spec)
         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
-                                       '--no-log-timestamps', '--disable-validate',
-                                       '--eval-timeout=20', '--thread-count=1',
+                                       '--no-log-timestamps', '--disable-validate', '--disable-color',
+                                       '--eval-timeout=20', '--thread-count=4',
                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
                                        '--on-error=continue',
                                        "--intermediate-output-ttl=3600",
@@ -579,8 +600,8 @@ class TestSubmit(unittest.TestCase):
 
         expect_container = copy.deepcopy(stubs.expect_container_spec)
         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
-                                       '--no-log-timestamps', '--disable-validate',
-                                       '--eval-timeout=20', '--thread-count=1',
+                                       '--no-log-timestamps', '--disable-validate', '--disable-color',
+                                       '--eval-timeout=20', '--thread-count=4',
                                        '--enable-reuse', "--collection-cache-size=256",
                                        '--debug', '--on-error=continue',
                                        "--trash-intermediate",
@@ -603,8 +624,8 @@ class TestSubmit(unittest.TestCase):
 
         expect_container = copy.deepcopy(stubs.expect_container_spec)
         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
-                                       '--no-log-timestamps', '--disable-validate',
-                                       '--eval-timeout=20', '--thread-count=1',
+                                       '--no-log-timestamps', '--disable-validate', '--disable-color',
+                                       '--eval-timeout=20', '--thread-count=4',
                                        '--enable-reuse', "--collection-cache-size=256",
                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
@@ -635,6 +656,7 @@ class TestSubmit(unittest.TestCase):
     @mock.patch("time.sleep")
     @stubs
     def test_submit_file_keepref(self, stubs, tm, collectionReader):
+        collectionReader().exists.return_value = True
         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
         exited = arvados_cwl.main(
             ["--submit", "--no-wait", "--api=containers", "--debug",
@@ -680,8 +702,8 @@ class TestSubmit(unittest.TestCase):
             'name': 'expect_arvworkflow.cwl#main',
             'container_image': '999999999999999999999999999999d3+99',
             'command': ['arvados-cwl-runner', '--local', '--api=containers',
-                        '--no-log-timestamps', '--disable-validate',
-                        '--eval-timeout=20', '--thread-count=1',
+                        '--no-log-timestamps', '--disable-validate', '--disable-color',
+                        '--eval-timeout=20', '--thread-count=4',
                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
             'cwd': '/var/spool/cwl',
@@ -690,7 +712,7 @@ class TestSubmit(unittest.TestCase):
                 'vcpus': 1,
                 'ram': 1342177280
             },
-            'use_existing': True,
+            'use_existing': False,
             'properties': {},
             'secret_mounts': {}
         }
@@ -751,7 +773,7 @@ class TestSubmit(unittest.TestCase):
                                 ],
                                 'requirements': [
                                     {
-                                        'dockerPull': 'debian:8',
+                                        'dockerPull': 'debian:buster-slim',
                                         'class': 'DockerRequirement',
                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
                                     }
@@ -775,8 +797,8 @@ class TestSubmit(unittest.TestCase):
             'name': 'a test workflow',
             'container_image': "999999999999999999999999999999d3+99",
             'command': ['arvados-cwl-runner', '--local', '--api=containers',
-                        '--no-log-timestamps', '--disable-validate',
-                        '--eval-timeout=20', '--thread-count=1',
+                        '--no-log-timestamps', '--disable-validate', '--disable-color',
+                        '--eval-timeout=20', '--thread-count=4',
                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
             'cwd': '/var/spool/cwl',
@@ -785,7 +807,7 @@ class TestSubmit(unittest.TestCase):
                 'vcpus': 1,
                 'ram': 1342177280
             },
-            'use_existing': True,
+            'use_existing': False,
             'properties': {
                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
             },
@@ -839,8 +861,8 @@ class TestSubmit(unittest.TestCase):
         expect_container = copy.deepcopy(stubs.expect_container_spec)
         expect_container["owner_uuid"] = project_uuid
         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
-                                       '--no-log-timestamps', '--disable-validate',
-                                       "--eval-timeout=20", "--thread-count=1",
+                                       '--no-log-timestamps', '--disable-validate', '--disable-color',
+                                       "--eval-timeout=20", "--thread-count=4",
                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
                                        '--on-error=continue',
                                        '--project-uuid='+project_uuid,
@@ -861,8 +883,8 @@ class TestSubmit(unittest.TestCase):
 
         expect_container = copy.deepcopy(stubs.expect_container_spec)
         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
-                                       '--no-log-timestamps', '--disable-validate',
-                                       '--eval-timeout=60.0', '--thread-count=1',
+                                       '--no-log-timestamps', '--disable-validate', '--disable-color',
+                                       '--eval-timeout=60.0', '--thread-count=4',
                                        '--enable-reuse', "--collection-cache-size=256",
                                        '--debug', '--on-error=continue',
                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
@@ -882,8 +904,8 @@ class TestSubmit(unittest.TestCase):
 
         expect_container = copy.deepcopy(stubs.expect_container_spec)
         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
-                                       '--no-log-timestamps', '--disable-validate',
-                                       '--eval-timeout=20', '--thread-count=1',
+                                       '--no-log-timestamps', '--disable-validate', '--disable-color',
+                                       '--eval-timeout=20', '--thread-count=4',
                                        '--enable-reuse', "--collection-cache-size=500",
                                        '--debug', '--on-error=continue',
                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
@@ -904,7 +926,7 @@ class TestSubmit(unittest.TestCase):
 
         expect_container = copy.deepcopy(stubs.expect_container_spec)
         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
-                                       '--no-log-timestamps', '--disable-validate',
+                                       '--no-log-timestamps', '--disable-validate', '--disable-color',
                                        '--eval-timeout=20', '--thread-count=20',
                                        '--enable-reuse', "--collection-cache-size=256",
                                        '--debug', '--on-error=continue',
@@ -974,8 +996,8 @@ class TestSubmit(unittest.TestCase):
             "arv": "http://arvados.org/cwl#",
         }
         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
-                        '--no-log-timestamps', '--disable-validate',
-                        '--eval-timeout=20', '--thread-count=1',
+                        '--no-log-timestamps', '--disable-validate', '--disable-color',
+                        '--eval-timeout=20', '--thread-count=4',
                         '--enable-reuse', "--collection-cache-size=512", '--debug', '--on-error=continue',
                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 
@@ -1039,8 +1061,9 @@ class TestSubmit(unittest.TestCase):
                 "--api=containers",
                 "--no-log-timestamps",
                 "--disable-validate",
+                "--disable-color",
                 "--eval-timeout=20",
-                '--thread-count=1',
+                '--thread-count=4',
                 "--enable-reuse",
                 "--collection-cache-size=256",
                 '--debug',
@@ -1089,7 +1112,10 @@ class TestSubmit(unittest.TestCase):
                                 "outputs": [
                                     {
                                         "id": "#secret_job.cwl/out",
-                                        "type": "stdout"
+                                        "type": "File",
+                                        "outputBinding": {
+                                              "glob": "hashed_example.txt"
+                                        }
                                     }
                                 ],
                                 "stdout": "hashed_example.txt",
@@ -1110,7 +1136,7 @@ class TestSubmit(unittest.TestCase):
                                 "hints": [
                                     {
                                         "class": "DockerRequirement",
-                                        "dockerPull": "debian:8",
+                                        "dockerPull": "debian:buster-slim",
                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
                                     },
                                     {
@@ -1180,7 +1206,7 @@ class TestSubmit(unittest.TestCase):
                 }
             },
             "state": "Committed",
-            "use_existing": True
+            "use_existing": False
         }
 
         stubs.api.container_requests().create.assert_called_with(
@@ -1237,9 +1263,25 @@ class TestSubmit(unittest.TestCase):
             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
         self.assertEqual(exited, 1)
 
+    @stubs
+    def test_submit_validate_project_uuid(self, stubs):
+        exited = arvados_cwl.main(
+            ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzb-j7d0g-zzzzzzzzzzzzzzz",
+             "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+        self.assertEqual(exited, 1)
+
+        stubs.api.groups().get().execute.side_effect = Exception("Bad project")
+        exited = arvados_cwl.main(
+            ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-j7d0g-zzzzzzzzzzzzzzx",
+             "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+        self.assertEqual(exited, 1)
+
     @mock.patch("arvados.collection.CollectionReader")
     @stubs
     def test_submit_uuid_inputs(self, stubs, collectionReader):
+        collectionReader().exists.return_value = True
         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
         def list_side_effect(**kwargs):
             m = mock.MagicMock()
@@ -1298,8 +1340,8 @@ class TestSubmit(unittest.TestCase):
                     stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 
                 self.assertEqual(exited, 1)
-                self.assertRegexpMatches(
-                    capture_stderr.getvalue(),
+                self.assertRegex(
+                    re.sub(r'[ \n]+', ' ', capture_stderr.getvalue()),
                     r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
             finally:
                 cwltool_logger.removeHandler(stderr_logger)
@@ -1321,7 +1363,7 @@ class TestSubmit(unittest.TestCase):
 
         try:
             self.assertEqual(exited, 1)
-            self.assertRegexpMatches(
+            self.assertRegex(
                 capture_stderr.getvalue(),
                 r"Collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz not found")
         finally:
@@ -1331,7 +1373,11 @@ class TestSubmit(unittest.TestCase):
 class TestCreateWorkflow(unittest.TestCase):
     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
     expect_workflow = StripYAMLComments(
-        open("tests/wf/expect_packed.cwl").read())
+        open("tests/wf/expect_upload_packed.cwl").read().rstrip())
+
+    def setUp(self):
+        cwltool.process._names = set()
+        arvados_cwl.arvdocker.arv_docker_clear_cache()
 
     @stubs
     def test_create(self, stubs):
@@ -1449,7 +1495,7 @@ class TestCreateWorkflow(unittest.TestCase):
             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 
         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
-        expect_workflow = StripYAMLComments(open(toolfile).read())
+        expect_workflow = StripYAMLComments(open(toolfile).read().rstrip())
 
         body = {
             "workflow": {