X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/1d77dec0f7bc1cca34d52288bd39ecd0a79be250..caf56e922b9caaa9a65332dc6bf3a36bd8fea48f:/sdk/cwl/tests/test_submit.py

diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 47844c1698..9cabea0794 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -1,3 +1,7 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
 import copy
 import cStringIO
 import functools
@@ -14,7 +18,7 @@ import arvados_cwl
 import arvados_cwl.runner
 import arvados.keep
 
-from .matcher import JsonDiffMatcher
+from .matcher import JsonDiffMatcher, StripYAMLComments
 from .mock_discovery import get_rootDesc
 
 import ruamel.yaml as yaml
@@ -75,7 +79,18 @@ def stubs(func):
                 if uuid in (v["uuid"], v["portable_data_hash"]):
                     return CollectionExecute(v)
 
-        created_collections = {}
+        created_collections = {
+            "99999999999999999999999999999998+99": {
+                "uuid": "",
+                "portable_data_hash": "99999999999999999999999999999998+99",
+                "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
+            },
+            "99999999999999999999999999999994+99": {
+                "uuid": "",
+                "portable_data_hash": "99999999999999999999999999999994+99",
+                "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
+            }
+        }
         stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
         stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
 
@@ -138,10 +153,19 @@ def stubs(func):
                     'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__, 'min_ram_mb_per_node': 1024},
                     'script_parameters': {
                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
-                        'x': {"value": {'basename': 'blorp.txt', 'class': 'File', 'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt'}},
+                        'x': {"value": {
+                            'basename': 'blorp.txt',
+                            'class': 'File',
+                            'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
+                            "size": 16
+                        }},
                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
                               'listing': [
-                                  {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
+                                  {
+                                      'basename': 'renamed.txt',
+                                      'class': 'File', 'location':
+                                      'keep:99999999999999999999999999999998+99/file1.txt'
+                                  }
                               ]}},
                         'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main',
                         'arv:enable_reuse': True,
@@ -187,10 +211,21 @@ def stubs(func):
                 '/var/lib/cwl/cwl.input.json': {
                     'kind': 'json',
                     'content': {
-                        'y': {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'},
-                        'x': {'basename': u'blorp.txt', 'class': 'File', 'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt'},
+                        'y': {
+                            'basename': '99999999999999999999999999999998+99',
+                            'location': 'keep:99999999999999999999999999999998+99',
+                            'class': 'Directory'},
+                        'x': {
+                            'basename': u'blorp.txt',
+                            'class': 'File',
+                            'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
+                            "size": 16
+                        },
                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
-                            {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
+                            {'basename': 'renamed.txt',
+                             'class': 'File',
+                             'location': 'keep:99999999999999999999999999999998+99/file1.txt'
+                            }
                         ]}
                     },
                     'kind': 'json'
@@ -210,7 +245,8 @@ def stubs(func):
                 'vcpus': 1,
                 'ram': 1024*1024*1024
             },
-            "properties": {}
+            'use_existing': True,
+            'properties': {}
         }
 
         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
@@ -285,6 +321,7 @@ class TestSubmit(unittest.TestCase):
 
         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:enable_reuse"] = {"value": False}
+        expect_pipeline["properties"] = {"run_options": {"enable_job_reuse": False}}
 
         stubs.api.pipeline_instances().create.assert_called_with(
             body=JsonDiffMatcher(expect_pipeline))
@@ -460,9 +497,45 @@ class TestSubmit(unittest.TestCase):
             logging.exception("")
 
         expect_container = copy.deepcopy(stubs.expect_container_spec)
-        expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
-                                                  '--disable-reuse', '--on-error=continue',
-                                                  '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
+        expect_container["command"] = [
+            'arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
+            '--disable-reuse', '--on-error=continue',
+            '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
+        expect_container["use_existing"] = False
+
+        stubs.api.container_requests().create.assert_called_with(
+            body=JsonDiffMatcher(expect_container))
+        self.assertEqual(capture_stdout.getvalue(),
+                         stubs.expect_container_request_uuid + '\n')
+
+
+    @stubs
+    def test_submit_container_reuse_disabled_by_workflow(self, stubs):
+        capture_stdout = cStringIO.StringIO()
+
+        exited = arvados_cwl.main(
+            ["--submit", "--no-wait", "--api=containers", "--debug",
+             "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
+            capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+        self.assertEqual(exited, 0)
+
+        expect_container = copy.deepcopy(stubs.expect_container_spec)
+        expect_container["command"] = [
+            'arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
+            '--disable-reuse', '--on-error=continue',
+            '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
+        expect_container["use_existing"] = False
+        expect_container["name"] = "submit_wf_no_reuse.cwl"
+        expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
+            {
+                "class": "http://arvados.org/cwl#ReuseRequirement",
+                "enableReuse": False,
+            },
+        ]
+        expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
+            "arv": "http://arvados.org/cwl#",
+            "cwltool": "http://commonwl.org/cwltool#"
+        }
 
         stubs.api.container_requests().create.assert_called_with(
             body=JsonDiffMatcher(expect_container))
@@ -517,6 +590,53 @@ class TestSubmit(unittest.TestCase):
         self.assertEqual(capture_stdout.getvalue(),
                          stubs.expect_container_request_uuid + '\n')
 
+
+    @stubs
+    def test_submit_container_output_ttl(self, stubs):
+        capture_stdout = cStringIO.StringIO()
+        try:
+            exited = arvados_cwl.main(
+                ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
+                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+                capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+            self.assertEqual(exited, 0)
+        except:
+            logging.exception("")
+
+        expect_container = copy.deepcopy(stubs.expect_container_spec)
+        expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
+                                       '--enable-reuse', '--on-error=continue',
+                                       "--intermediate-output-ttl=3600",
+                                       '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
+
+        stubs.api.container_requests().create.assert_called_with(
+            body=JsonDiffMatcher(expect_container))
+        self.assertEqual(capture_stdout.getvalue(),
+                         stubs.expect_container_request_uuid + '\n')
+
+    @stubs
+    def test_submit_container_trash_intermediate(self, stubs):
+        capture_stdout = cStringIO.StringIO()
+        try:
+            exited = arvados_cwl.main(
+                ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
+                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+                capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+            self.assertEqual(exited, 0)
+        except:
+            logging.exception("")
+
+        expect_container = copy.deepcopy(stubs.expect_container_spec)
+        expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
+                                       '--enable-reuse', '--on-error=continue',
+                                       "--trash-intermediate",
+                                       '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
+
+        stubs.api.container_requests().create.assert_called_with(
+            body=JsonDiffMatcher(expect_container))
+        self.assertEqual(capture_stdout.getvalue(),
+                         stubs.expect_container_request_uuid + '\n')
+
     @stubs
     def test_submit_container_output_tags(self, stubs):
         output_tags = "tag0,tag1,tag2"
@@ -623,7 +743,8 @@ class TestSubmit(unittest.TestCase):
                 'vcpus': 1,
                 'ram': 1073741824
             },
-            "properties": {}
+            'use_existing': True,
+            'properties': {}
         }
 
         stubs.api.container_requests().create.assert_called_with(
@@ -738,7 +859,8 @@ class TestSubmit(unittest.TestCase):
                 'vcpus': 1,
                 'ram': 1073741824
             },
-            "properties": {
+            'use_existing': True,
+            'properties': {
                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
             }
         }
@@ -770,6 +892,31 @@ class TestSubmit(unittest.TestCase):
                          stubs.expect_container_request_uuid + '\n')
 
 
+    @stubs
+    def test_submit_container_project(self, stubs):
+        project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
+        capture_stdout = cStringIO.StringIO()
+        try:
+            exited = arvados_cwl.main(
+                ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
+                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+                capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+            self.assertEqual(exited, 0)
+        except:
+            logging.exception("")
+
+        expect_container = copy.deepcopy(stubs.expect_container_spec)
+        expect_container["owner_uuid"] = project_uuid
+        expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
+                                       '--enable-reuse', '--on-error=continue', '--project-uuid='+project_uuid,
+                                       '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
+
+        stubs.api.container_requests().create.assert_called_with(
+            body=JsonDiffMatcher(expect_container))
+        self.assertEqual(capture_stdout.getvalue(),
+                         stubs.expect_container_request_uuid + '\n')
+
+
     @stubs
     def test_submit_job_runner_image(self, stubs):
         capture_stdout = cStringIO.StringIO()
@@ -970,7 +1117,8 @@ class TestCreateTemplate(unittest.TestCase):
 
 class TestCreateWorkflow(unittest.TestCase):
     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
-    expect_workflow = open("tests/wf/expect_packed.cwl").read()
+    expect_workflow = StripYAMLComments(
+        open("tests/wf/expect_packed.cwl").read())
 
     @stubs
     def test_create(self, stubs):
@@ -1117,7 +1265,8 @@ class TestCreateWorkflow(unittest.TestCase):
             capture_stdout, sys.stderr, api_client=stubs.api)
         self.assertEqual(exited, 0)
 
-        expect_workflow = open("tests/collection_per_tool/collection_per_tool_packed.cwl").read()
+        toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
+        expect_workflow = StripYAMLComments(open(toolfile).read())
 
         body = {
             "workflow": {