sdk/cwl/tests/test_make_output.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from future import standard_library
   6 standard_library.install_aliases()
   7
   8 import functools
   9 import json
  10 import logging
  11 import os
  12 import io
  13 import unittest
  14
  15 from unittest import mock
  16
  17 import arvados
  18 import arvados_cwl
  19 import arvados_cwl.executor
  20 from .mock_discovery import get_rootDesc
  21
  22 class TestMakeOutput(unittest.TestCase):
  23     def setUp(self):
  24         self.api = mock.MagicMock()
  25         self.api._rootDesc = get_rootDesc()
  26
  27     def tearDown(self):
  28         root_logger = logging.getLogger('')
  29
  30         # Remove existing RuntimeStatusLoggingHandlers if they exist
  31         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
  32         root_logger.handlers = handlers
  33
  34     @mock.patch("arvados.collection.Collection")
  35     @mock.patch("arvados.collection.CollectionReader")
  36     def test_make_output_collection(self, reader, col):
  37         keep_client = mock.MagicMock()
  38         runner = arvados_cwl.executor.ArvCwlExecutor(self.api, keep_client=keep_client)
  39         runner.project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
  40
  41         final = mock.MagicMock()
  42         col.return_value = final
  43         readermock = mock.MagicMock()
  44         reader.return_value = readermock
  45
  46         final_uuid = final.manifest_locator()
  47         num_retries = runner.num_retries
  48
  49         cwlout = io.StringIO()
  50         openmock = mock.MagicMock()
  51         final.open.return_value = openmock
  52         openmock.__enter__.return_value = cwlout
  53
  54         _, runner.final_output_collection = runner.make_output_collection("Test output", ["foo"], "tag0,tag1,tag2", {}, {
  55             "foo": {
  56                 "class": "File",
  57                 "location": "keep:99999999999999999999999999999991+99/foo.txt",
  58                 "size": 3,
  59                 "basename": "foo.txt"
  60             },
  61             "bar": {
  62                 "class": "File",
  63                 "location": "keep:99999999999999999999999999999992+99/bar.txt",
  64                 "basename": "baz.txt",
  65                 "size": 4
  66             }
  67         })
  68
  69         final.copy.assert_has_calls([mock.call('bar.txt', 'baz.txt', overwrite=False, source_collection=readermock)])
  70         final.copy.assert_has_calls([mock.call('foo.txt', 'foo.txt', overwrite=False, source_collection=readermock)])
  71         final.save_new.assert_has_calls([mock.call(ensure_unique_name=True, name='Test output', owner_uuid='zzzzz-j7d0g-zzzzzzzzzzzzzzz', properties={}, storage_classes=['foo'])])
  72         self.assertEqual("""{
  73     "bar": {
  74         "basename": "baz.txt",
  75         "class": "File",
  76         "location": "baz.txt",
  77         "size": 4
  78     },
  79     "foo": {
  80         "basename": "foo.txt",
  81         "class": "File",
  82         "location": "foo.txt",
  83         "size": 3
  84     }
  85 }""", cwlout.getvalue())
  86
  87         self.assertIs(final, runner.final_output_collection)
  88         self.assertIs(final_uuid, runner.final_output_collection.manifest_locator())
  89         self.api.links().create.assert_has_calls([mock.call(body={"head_uuid": final_uuid, "link_class": "tag", "name": "tag0"}), mock.call().execute(num_retries=num_retries)])
  90         self.api.links().create.assert_has_calls([mock.call(body={"head_uuid": final_uuid, "link_class": "tag", "name": "tag1"}), mock.call().execute(num_retries=num_retries)])
  91         self.api.links().create.assert_has_calls([mock.call(body={"head_uuid": final_uuid, "link_class": "tag", "name": "tag2"}), mock.call().execute(num_retries=num_retries)])
  92
  93     @mock.patch("arvados.collection.Collection")
  94     @mock.patch("arvados.collection.CollectionReader")
  95     def test_make_output_for_multiple_file_targets(self, reader, col):
  96         keep_client = mock.MagicMock()
  97         runner = arvados_cwl.executor.ArvCwlExecutor(self.api, keep_client=keep_client)
  98         runner.project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
  99
 100         final = mock.MagicMock()
 101         col.return_value = final
 102         readermock = mock.MagicMock()
 103         reader.return_value = readermock
 104
 105         # This output describes a single file listed in 2 different directories
 106         _, runner.final_output_collection = runner.make_output_collection("Test output", ["foo"], "", {}, { 'out': [
 107         {
 108             'basename': 'testdir1',
 109             'listing': [
 110                 {
 111                     'basename': 'test.txt',
 112                     'nameroot': 'test',
 113                     'nameext': '.txt',
 114                     'location': 'keep:99999999999999999999999999999991+99/test.txt',
 115                     'class': 'File',
 116                     'size': 16
 117                 }
 118             ],
 119             'location': '_:99999999999999999999999999999992+99',
 120             'class': 'Directory'
 121         },
 122         {
 123             'basename': 'testdir2',
 124             'listing': [
 125                 {
 126                     'basename': 'test.txt',
 127                     'nameroot': 'test',
 128                     'nameext': '.txt',
 129                     'location': 'keep:99999999999999999999999999999991+99/test.txt',
 130                     'class':
 131                     'File',
 132                     'size': 16
 133                 }
 134             ],
 135             'location': '_:99999999999999999999999999999993+99',
 136             'class': 'Directory'
 137         }]})
 138
 139         # Check that copy is called on the collection for both locations
 140         final.copy.assert_any_call("test.txt", "testdir1/test.txt", source_collection=mock.ANY, overwrite=mock.ANY)
 141         final.copy.assert_any_call("test.txt", "testdir2/test.txt", source_collection=mock.ANY, overwrite=mock.ANY)
 142
 143     @mock.patch("arvados.collection.Collection")
 144     @mock.patch("arvados.collection.CollectionReader")
 145     def test_make_output_for_literal_name_conflicts(self, reader, col):
 146         keep_client = mock.MagicMock()
 147         runner = arvados_cwl.executor.ArvCwlExecutor(self.api, keep_client=keep_client)
 148         runner.project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 149
 150         final = mock.MagicMock()
 151         col.return_value = final
 152         readermock = mock.MagicMock()
 153         reader.return_value = readermock
 154
 155         # This output describes two literals with the same basename
 156         _, runner.final_output_collection = runner.make_output_collection("Test output", ["foo"], "",  {}, [
 157         {
 158             'lit':
 159             {
 160                 'basename': 'a_file',
 161                 'nameext': '',
 162                 'nameroot': 'a_file',
 163                 'location': '_:f168fc0c-4291-40aa-a04e-366d57390560',
 164                 'class': 'File',
 165                 'contents': 'Hello file literal.'
 166             }
 167         },
 168         {
 169             'lit':
 170             {
 171                 'basename': 'a_file',
 172                 'nameext': '',
 173                 'nameroot': 'a_file',
 174                 'location': '_:1728da8f-c64e-4a3e-b2e2-1ee356be7bc8',
 175                 'class': 'File',
 176                 'contents': 'Hello file literal.'
 177             }
 178         }])
 179
 180         # Check that the file name conflict is resolved and open is called for both
 181         final.open.assert_any_call("a_file", "wb")
 182         final.open.assert_any_call("a_file_2", "wb")