sdk/cwl/tests/test_make_output.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import functools
   6 import json
   7 import logging
   8 import os
   9 import io
  10 import unittest
  11
  12 from unittest import mock
  13
  14 import arvados
  15 import arvados_cwl
  16 import arvados_cwl.executor
  17 from .mock_discovery import get_rootDesc
  18
  19 class TestMakeOutput(unittest.TestCase):
  20     def setUp(self):
  21         self.api = mock.MagicMock()
  22         self.api._rootDesc = get_rootDesc()
  23
  24     def tearDown(self):
  25         root_logger = logging.getLogger('')
  26
  27         # Remove existing RuntimeStatusLoggingHandlers if they exist
  28         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
  29         root_logger.handlers = handlers
  30
  31     @mock.patch("arvados.collection.Collection")
  32     @mock.patch("arvados.collection.CollectionReader")
  33     def test_make_output_collection(self, reader, col):
  34         keep_client = mock.MagicMock()
  35         runner = arvados_cwl.executor.ArvCwlExecutor(self.api, keep_client=keep_client)
  36         runner.project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
  37
  38         final = mock.MagicMock()
  39         col.return_value = final
  40         readermock = mock.MagicMock()
  41         reader.return_value = readermock
  42
  43         final_uuid = final.manifest_locator()
  44         num_retries = runner.num_retries
  45
  46         cwlout = io.StringIO()
  47         openmock = mock.MagicMock()
  48         final.open.return_value = openmock
  49         openmock.__enter__.return_value = cwlout
  50
  51         _, runner.final_output_collection = runner.make_output_collection("Test output", ["foo"], "tag0,tag1,tag2", {}, {
  52             "foo": {
  53                 "class": "File",
  54                 "location": "keep:99999999999999999999999999999991+99/foo.txt",
  55                 "size": 3,
  56                 "basename": "foo.txt"
  57             },
  58             "bar": {
  59                 "class": "File",
  60                 "location": "keep:99999999999999999999999999999992+99/bar.txt",
  61                 "basename": "baz.txt",
  62                 "size": 4
  63             }
  64         })
  65
  66         final.copy.assert_has_calls([mock.call('bar.txt', 'baz.txt', overwrite=False, source_collection=readermock)])
  67         final.copy.assert_has_calls([mock.call('foo.txt', 'foo.txt', overwrite=False, source_collection=readermock)])
  68         final.save_new.assert_has_calls([mock.call(ensure_unique_name=True, name='Test output', owner_uuid='zzzzz-j7d0g-zzzzzzzzzzzzzzz', properties={}, storage_classes=['foo'])])
  69         self.assertEqual("""{
  70     "bar": {
  71         "basename": "baz.txt",
  72         "class": "File",
  73         "location": "baz.txt",
  74         "size": 4
  75     },
  76     "foo": {
  77         "basename": "foo.txt",
  78         "class": "File",
  79         "location": "foo.txt",
  80         "size": 3
  81     }
  82 }""", cwlout.getvalue())
  83
  84         self.assertIs(final, runner.final_output_collection)
  85         self.assertIs(final_uuid, runner.final_output_collection.manifest_locator())
  86         self.api.links().create.assert_has_calls([mock.call(body={"head_uuid": final_uuid, "link_class": "tag", "name": "tag0"}), mock.call().execute(num_retries=num_retries)])
  87         self.api.links().create.assert_has_calls([mock.call(body={"head_uuid": final_uuid, "link_class": "tag", "name": "tag1"}), mock.call().execute(num_retries=num_retries)])
  88         self.api.links().create.assert_has_calls([mock.call(body={"head_uuid": final_uuid, "link_class": "tag", "name": "tag2"}), mock.call().execute(num_retries=num_retries)])
  89
  90     @mock.patch("arvados.collection.Collection")
  91     @mock.patch("arvados.collection.CollectionReader")
  92     def test_make_output_for_multiple_file_targets(self, reader, col):
  93         keep_client = mock.MagicMock()
  94         runner = arvados_cwl.executor.ArvCwlExecutor(self.api, keep_client=keep_client)
  95         runner.project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
  96
  97         final = mock.MagicMock()
  98         col.return_value = final
  99         readermock = mock.MagicMock()
 100         reader.return_value = readermock
 101
 102         # This output describes a single file listed in 2 different directories
 103         _, runner.final_output_collection = runner.make_output_collection("Test output", ["foo"], "", {}, { 'out': [
 104         {
 105             'basename': 'testdir1',
 106             'listing': [
 107                 {
 108                     'basename': 'test.txt',
 109                     'nameroot': 'test',
 110                     'nameext': '.txt',
 111                     'location': 'keep:99999999999999999999999999999991+99/test.txt',
 112                     'class': 'File',
 113                     'size': 16
 114                 }
 115             ],
 116             'location': '_:99999999999999999999999999999992+99',
 117             'class': 'Directory'
 118         },
 119         {
 120             'basename': 'testdir2',
 121             'listing': [
 122                 {
 123                     'basename': 'test.txt',
 124                     'nameroot': 'test',
 125                     'nameext': '.txt',
 126                     'location': 'keep:99999999999999999999999999999991+99/test.txt',
 127                     'class':
 128                     'File',
 129                     'size': 16
 130                 }
 131             ],
 132             'location': '_:99999999999999999999999999999993+99',
 133             'class': 'Directory'
 134         }]})
 135
 136         # Check that copy is called on the collection for both locations
 137         final.copy.assert_any_call("test.txt", "testdir1/test.txt", source_collection=mock.ANY, overwrite=mock.ANY)
 138         final.copy.assert_any_call("test.txt", "testdir2/test.txt", source_collection=mock.ANY, overwrite=mock.ANY)
 139
 140     @mock.patch("arvados.collection.Collection")
 141     @mock.patch("arvados.collection.CollectionReader")
 142     def test_make_output_for_literal_name_conflicts(self, reader, col):
 143         keep_client = mock.MagicMock()
 144         runner = arvados_cwl.executor.ArvCwlExecutor(self.api, keep_client=keep_client)
 145         runner.project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 146
 147         final = mock.MagicMock()
 148         col.return_value = final
 149         readermock = mock.MagicMock()
 150         reader.return_value = readermock
 151
 152         # This output describes two literals with the same basename
 153         _, runner.final_output_collection = runner.make_output_collection("Test output", ["foo"], "",  {}, [
 154         {
 155             'lit':
 156             {
 157                 'basename': 'a_file',
 158                 'nameext': '',
 159                 'nameroot': 'a_file',
 160                 'location': '_:f168fc0c-4291-40aa-a04e-366d57390560',
 161                 'class': 'File',
 162                 'contents': 'Hello file literal.'
 163             }
 164         },
 165         {
 166             'lit':
 167             {
 168                 'basename': 'a_file',
 169                 'nameext': '',
 170                 'nameroot': 'a_file',
 171                 'location': '_:1728da8f-c64e-4a3e-b2e2-1ee356be7bc8',
 172                 'class': 'File',
 173                 'contents': 'Hello file literal.'
 174             }
 175         }])
 176
 177         # Check that the file name conflict is resolved and open is called for both
 178         final.open.assert_any_call("a_file", "wb")
 179         final.open.assert_any_call("a_file_2", "wb")