+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from future import standard_library
+standard_library.install_aliases()
+
+import functools
+import json
import logging
import mock
-import unittest
import os
-import functools
-import json
-import StringIO
+import io
+import unittest
+import arvados
import arvados_cwl
+import arvados_cwl.executor
+from .mock_discovery import get_rootDesc
class TestMakeOutput(unittest.TestCase):
+ def setUp(self):
+ self.api = mock.MagicMock()
+ self.api._rootDesc = get_rootDesc()
+
+ def tearDown(self):
+ root_logger = logging.getLogger('')
+
+ # Remove existing RuntimeStatusLoggingHandlers if they exist
+ handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
+ root_logger.handlers = handlers
+
@mock.patch("arvados.collection.Collection")
@mock.patch("arvados.collection.CollectionReader")
def test_make_output_collection(self, reader, col):
- api = mock.MagicMock()
keep_client = mock.MagicMock()
- runner = arvados_cwl.ArvCwlRunner(api, keep_client=keep_client)
+ runner = arvados_cwl.executor.ArvCwlExecutor(self.api, keep_client=keep_client)
runner.project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
final = mock.MagicMock()
readermock = mock.MagicMock()
reader.return_value = readermock
- cwlout = StringIO.StringIO()
+ final_uuid = final.manifest_locator()
+ num_retries = runner.num_retries
+
+ cwlout = io.StringIO()
openmock = mock.MagicMock()
final.open.return_value = openmock
openmock.__enter__.return_value = cwlout
- runner.make_output_collection("Test output", {
+ _, runner.final_output_collection = runner.make_output_collection("Test output", ["foo"], "tag0,tag1,tag2", {
"foo": {
"class": "File",
"location": "keep:99999999999999999999999999999991+99/foo.txt",
"bar": {
"class": "File",
"location": "keep:99999999999999999999999999999992+99/bar.txt",
- "basename": "baz.txt"
+ "basename": "baz.txt",
+ "size": 4
}
})
final.copy.assert_has_calls([mock.call('bar.txt', 'baz.txt', overwrite=False, source_collection=readermock)])
final.copy.assert_has_calls([mock.call('foo.txt', 'foo.txt', overwrite=False, source_collection=readermock)])
- final.save_new.assert_has_calls([mock.call(ensure_unique_name=True, name='Test output', owner_uuid='zzzzz-j7d0g-zzzzzzzzzzzzzzz')])
+ final.save_new.assert_has_calls([mock.call(ensure_unique_name=True, name='Test output', owner_uuid='zzzzz-j7d0g-zzzzzzzzzzzzzzz', storage_classes=['foo'])])
self.assertEqual("""{
"bar": {
+ "basename": "baz.txt",
"class": "File",
- "location": "baz.txt"
+ "location": "baz.txt",
+ "size": 4
},
"foo": {
+ "basename": "foo.txt",
"class": "File",
- "location": "foo.txt"
+ "location": "foo.txt",
+ "size": 3
}
}""", cwlout.getvalue())
self.assertIs(final, runner.final_output_collection)
+ self.assertIs(final_uuid, runner.final_output_collection.manifest_locator())
+ self.api.links().create.assert_has_calls([mock.call(body={"head_uuid": final_uuid, "link_class": "tag", "name": "tag0"}), mock.call().execute(num_retries=num_retries)])
+ self.api.links().create.assert_has_calls([mock.call(body={"head_uuid": final_uuid, "link_class": "tag", "name": "tag1"}), mock.call().execute(num_retries=num_retries)])
+ self.api.links().create.assert_has_calls([mock.call(body={"head_uuid": final_uuid, "link_class": "tag", "name": "tag2"}), mock.call().execute(num_retries=num_retries)])
+
+ @mock.patch("arvados.collection.Collection")
+ @mock.patch("arvados.collection.CollectionReader")
+ def test_make_output_for_multiple_file_targets(self, reader, col):
+ keep_client = mock.MagicMock()
+ runner = arvados_cwl.executor.ArvCwlExecutor(self.api, keep_client=keep_client)
+ runner.project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
+
+ final = mock.MagicMock()
+ col.return_value = final
+ readermock = mock.MagicMock()
+ reader.return_value = readermock
+
+ # This output describes a single file listed in 2 different directories
+ _, runner.final_output_collection = runner.make_output_collection("Test output", ["foo"], "", { 'out': [
+ {
+ 'basename': 'testdir1',
+ 'listing': [
+ {
+ 'basename': 'test.txt',
+ 'nameroot': 'test',
+ 'nameext': '.txt',
+ 'location': 'keep:99999999999999999999999999999991+99/test.txt',
+ 'class': 'File',
+ 'size': 16
+ }
+ ],
+ 'location': '_:99999999999999999999999999999992+99',
+ 'class': 'Directory'
+ },
+ {
+ 'basename': 'testdir2',
+ 'listing': [
+ {
+ 'basename': 'test.txt',
+ 'nameroot': 'test',
+ 'nameext': '.txt',
+ 'location': 'keep:99999999999999999999999999999991+99/test.txt',
+ 'class':
+ 'File',
+ 'size': 16
+ }
+ ],
+ 'location': '_:99999999999999999999999999999993+99',
+ 'class': 'Directory'
+ }]})
+
+ # Check that copy is called on the collection for both locations
+ final.copy.assert_any_call("test.txt", "testdir1/test.txt", source_collection=mock.ANY, overwrite=mock.ANY)
+ final.copy.assert_any_call("test.txt", "testdir2/test.txt", source_collection=mock.ANY, overwrite=mock.ANY)
+
+ @mock.patch("arvados.collection.Collection")
+ @mock.patch("arvados.collection.CollectionReader")
+ def test_make_output_for_literal_name_conflicts(self, reader, col):
+ keep_client = mock.MagicMock()
+ runner = arvados_cwl.executor.ArvCwlExecutor(self.api, keep_client=keep_client)
+ runner.project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
+
+ final = mock.MagicMock()
+ col.return_value = final
+ readermock = mock.MagicMock()
+ reader.return_value = readermock
+
+ # This output describes two literals with the same basename
+ _, runner.final_output_collection = runner.make_output_collection("Test output", ["foo"], "", [
+ {
+ 'lit':
+ {
+ 'basename': 'a_file',
+ 'nameext': '',
+ 'nameroot': 'a_file',
+ 'location': '_:f168fc0c-4291-40aa-a04e-366d57390560',
+ 'class': 'File',
+ 'contents': 'Hello file literal.'
+ }
+ },
+ {
+ 'lit':
+ {
+ 'basename': 'a_file',
+ 'nameext': '',
+ 'nameroot': 'a_file',
+ 'location': '_:1728da8f-c64e-4a3e-b2e2-1ee356be7bc8',
+ 'class': 'File',
+ 'contents': 'Hello file literal.'
+ }
+ }])
+
+ # Check that the file name conflict is resolved and open is called for both
+ final.open.assert_any_call("a_file", "wb")
+ final.open.assert_any_call("a_file_2", "wb")
projects / arvados.git / blobdiff