Merge branch '11590-log-reuse'

[arvados.git] / sdk / cwl / tests / test_job.py

diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py
index c8813adf7ea48d71f5b5b8cb9d7c091bfafa09c0..99dd3cb669729f09d6c45d914178a5e56a57ffd6 100644 (file)
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@ -11,7 +11,9 @@ import arvados
 import arvados_cwl
 import cwltool.process
 from schema_salad.ref_resolver import Loader
+from schema_salad.sourceline import cmap
 from .mock_discovery import get_rootDesc
+from .matcher import JsonDiffMatcher
 
 if not os.getenv('ARVADOS_DEBUG'):
     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
@@ -33,20 +35,21 @@ class TestJob(unittest.TestCase):
             list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
             runner.api.collections().get().execute.return_vaulue = {"portable_data_hash": "99999999999999999999999999999993+99"}
 
-            tool = {
+            tool = cmap({
                 "inputs": [],
                 "outputs": [],
                 "baseCommand": "ls",
                 "arguments": [{"valueFrom": "$(runtime.outdir)"}]
-            }
-            make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
+            })
+            make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
+                                         collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
             arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="jobs", avsc_names=avsc_names,
                                                      basedir="", make_fs_access=make_fs_access, loader=Loader({}))
             arvtool.formatgraph = None
             for j in arvtool.job({}, mock.MagicMock(), basedir="", make_fs_access=make_fs_access):
                 j.run(enable_reuse=enable_reuse)
                 runner.api.jobs().create.assert_called_with(
-                    body={
+                    body=JsonDiffMatcher({
                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
                         'runtime_constraints': {},
                         'script_parameters': {
@@ -56,21 +59,21 @@ class TestJob(unittest.TestCase):
                             }],
                         },
                         'script_version': 'master',
-                        'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6',
+                        'minimum_script_version': 'a3f2cb186e437bfce0031b024b2157b73ed2717d',
                         'repository': 'arvados',
                         'script': 'crunchrunner',
                         'runtime_constraints': {
-                            'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
+                            'docker_image': 'arvados/jobs',
                             'min_cores_per_node': 1,
                             'min_ram_mb_per_node': 1024,
                             'min_scratch_mb_per_node': 2048 # tmpdirSize + outdirSize
                         }
-                    },
+                    }),
                     find_or_create=enable_reuse,
                     filters=[['repository', '=', 'arvados'],
                              ['script', '=', 'crunchrunner'],
-                             ['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'],
-                             ['docker_image_locator', 'in docker', 'arvados/jobs:'+arvados_cwl.__version__]]
+                             ['script_version', 'in git', 'a3f2cb186e437bfce0031b024b2157b73ed2717d'],
+                             ['docker_image_locator', 'in docker', 'arvados/jobs']]
                 )
 
     # The test passes some fields in builder.resources
@@ -106,14 +109,15 @@ class TestJob(unittest.TestCase):
             }],
             "baseCommand": "ls"
         }
-        make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
+        make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
+                                         collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="jobs", avsc_names=avsc_names,
                                                  make_fs_access=make_fs_access, loader=Loader({}))
         arvtool.formatgraph = None
         for j in arvtool.job({}, mock.MagicMock(), basedir="", make_fs_access=make_fs_access):
             j.run()
         runner.api.jobs().create.assert_called_with(
-            body={
+            body=JsonDiffMatcher({
                 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
                 'runtime_constraints': {},
                 'script_parameters': {
@@ -124,22 +128,22 @@ class TestJob(unittest.TestCase):
                     }]
             },
             'script_version': 'master',
-                'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6',
+                'minimum_script_version': 'a3f2cb186e437bfce0031b024b2157b73ed2717d',
                 'repository': 'arvados',
                 'script': 'crunchrunner',
                 'runtime_constraints': {
-                    'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
+                    'docker_image': 'arvados/jobs',
                     'min_cores_per_node': 3,
-                    'min_ram_mb_per_node': 3000,
+                    'min_ram_mb_per_node': 3512,     # ramMin + keep_cache
                     'min_scratch_mb_per_node': 5024, # tmpdirSize + outdirSize
                     'keep_cache_mb_per_task': 512
                 }
-            },
+            }),
             find_or_create=True,
             filters=[['repository', '=', 'arvados'],
                      ['script', '=', 'crunchrunner'],
-                     ['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'],
-                     ['docker_image_locator', 'in docker', 'arvados/jobs:'+arvados_cwl.__version__]])
+                     ['script_version', 'in git', 'a3f2cb186e437bfce0031b024b2157b73ed2717d'],
+                     ['docker_image_locator', 'in docker', 'arvados/jobs']])
 
     @mock.patch("arvados.collection.CollectionReader")
     def test_done(self, reader):
@@ -164,6 +168,7 @@ class TestJob(unittest.TestCase):
         arvjob.builder = mock.MagicMock()
         arvjob.output_callback = mock.MagicMock()
         arvjob.collect_outputs = mock.MagicMock()
+        arvjob.collect_outputs.return_value = {"out": "stuff"}
 
         arvjob.done({
             "state": "Complete",
@@ -189,6 +194,8 @@ class TestJob(unittest.TestCase):
                   'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
                   'name': 'Output 9999999 of testjob'})
 
+        arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
+
     @mock.patch("arvados.collection.CollectionReader")
     def test_done_use_existing_collection(self, reader):
         api = mock.MagicMock()
@@ -211,6 +218,7 @@ class TestJob(unittest.TestCase):
         arvjob.builder = mock.MagicMock()
         arvjob.output_callback = mock.MagicMock()
         arvjob.collect_outputs = mock.MagicMock()
+        arvjob.collect_outputs.return_value = {"out": "stuff"}
 
         arvjob.done({
             "state": "Complete",
@@ -228,6 +236,8 @@ class TestJob(unittest.TestCase):
 
         self.assertFalse(api.collections().create.called)
 
+        arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
+
 
 class TestWorkflow(unittest.TestCase):
     # The test passes no builder.resources
@@ -256,7 +266,8 @@ class TestWorkflow(unittest.TestCase):
 
         mockcollection().portable_data_hash.return_value = "99999999999999999999999999999999+118"
 
-        make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
+        make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
+                                         collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, work_api="jobs", avsc_names=avsc_names,
                                               basedir="", make_fs_access=make_fs_access, loader=document_loader,
                                               makeTool=runner.arv_make_tool, metadata=metadata)
@@ -269,8 +280,8 @@ class TestWorkflow(unittest.TestCase):
             subwf = f.read()
 
         runner.api.jobs().create.assert_called_with(
-            body={
-                'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6',
+            body=JsonDiffMatcher({
+                'minimum_script_version': 'a3f2cb186e437bfce0031b024b2157b73ed2717d',
                 'repository': 'arvados',
                 'script_version': 'master',
                 'script': 'crunchrunner',
@@ -287,18 +298,21 @@ class TestWorkflow(unittest.TestCase):
                 'runtime_constraints': {
                     'min_scratch_mb_per_node': 2048,
                     'min_cores_per_node': 1,
-                    'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
+                    'docker_image': 'arvados/jobs',
                     'min_ram_mb_per_node': 1024
                 },
-                'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'},
+                'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}),
             filters=[['repository', '=', 'arvados'],
                      ['script', '=', 'crunchrunner'],
-                     ['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'],
-                     ['docker_image_locator', 'in docker', 'arvados/jobs:'+arvados_cwl.__version__]],
+                     ['script_version', 'in git', 'a3f2cb186e437bfce0031b024b2157b73ed2717d'],
+                     ['docker_image_locator', 'in docker', 'arvados/jobs']],
             find_or_create=True)
 
         mockcollection().open().__enter__().write.assert_has_calls([mock.call(subwf)])
-        mockcollection().open().__enter__().write.assert_has_calls([mock.call('{sleeptime: 5}')])
+        mockcollection().open().__enter__().write.assert_has_calls([mock.call(
+'''{
+  "sleeptime": 5
+}''')])
 
     def test_default_work_api(self):
         arvados_cwl.add_arv_hints()