sdk/cwl/tests/test_make_output.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from future import standard_library
   6 standard_library.install_aliases()
   7
   8 import functools
   9 import json
  10 import logging
  11 import mock
  12 import os
  13 import io
  14 import unittest
  15
  16 import arvados
  17 import arvados_cwl
  18 import arvados_cwl.executor
  19 from .mock_discovery import get_rootDesc
  20
  21 class TestMakeOutput(unittest.TestCase):
  22     def setUp(self):
  23         self.api = mock.MagicMock()
  24         self.api._rootDesc = get_rootDesc()
  25
  26     @mock.patch("arvados.collection.Collection")
  27     @mock.patch("arvados.collection.CollectionReader")
  28     def test_make_output_collection(self, reader, col):
  29         keep_client = mock.MagicMock()
  30         runner = arvados_cwl.executor.ArvCwlExecutor(self.api, keep_client=keep_client)
  31         runner.project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
  32
  33         final = mock.MagicMock()
  34         col.return_value = final
  35         readermock = mock.MagicMock()
  36         reader.return_value = readermock
  37
  38         final_uuid = final.manifest_locator()
  39         num_retries = runner.num_retries
  40
  41         cwlout = io.StringIO()
  42         openmock = mock.MagicMock()
  43         final.open.return_value = openmock
  44         openmock.__enter__.return_value = cwlout
  45
  46         _, runner.final_output_collection = runner.make_output_collection("Test output", ["foo"], "tag0,tag1,tag2", {
  47             "foo": {
  48                 "class": "File",
  49                 "location": "keep:99999999999999999999999999999991+99/foo.txt",
  50                 "size": 3,
  51                 "basename": "foo.txt"
  52             },
  53             "bar": {
  54                 "class": "File",
  55                 "location": "keep:99999999999999999999999999999992+99/bar.txt",
  56                 "basename": "baz.txt",
  57                 "size": 4
  58             }
  59         })
  60
  61         final.copy.assert_has_calls([mock.call('bar.txt', 'baz.txt', overwrite=False, source_collection=readermock)])
  62         final.copy.assert_has_calls([mock.call('foo.txt', 'foo.txt', overwrite=False, source_collection=readermock)])
  63         final.save_new.assert_has_calls([mock.call(ensure_unique_name=True, name='Test output', owner_uuid='zzzzz-j7d0g-zzzzzzzzzzzzzzz', storage_classes=['foo'])])
  64         self.assertEqual("""{
  65     "bar": {
  66         "basename": "baz.txt",
  67         "class": "File",
  68         "location": "baz.txt",
  69         "size": 4
  70     },
  71     "foo": {
  72         "basename": "foo.txt",
  73         "class": "File",
  74         "location": "foo.txt",
  75         "size": 3
  76     }
  77 }""", cwlout.getvalue())
  78
  79         self.assertIs(final, runner.final_output_collection)
  80         self.assertIs(final_uuid, runner.final_output_collection.manifest_locator())
  81         self.api.links().create.assert_has_calls([mock.call(body={"head_uuid": final_uuid, "link_class": "tag", "name": "tag0"}), mock.call().execute(num_retries=num_retries)])
  82         self.api.links().create.assert_has_calls([mock.call(body={"head_uuid": final_uuid, "link_class": "tag", "name": "tag1"}), mock.call().execute(num_retries=num_retries)])
  83         self.api.links().create.assert_has_calls([mock.call(body={"head_uuid": final_uuid, "link_class": "tag", "name": "tag2"}), mock.call().execute(num_retries=num_retries)])
  84
  85     @mock.patch("arvados.collection.Collection")
  86     @mock.patch("arvados.collection.CollectionReader")
  87     def test_make_output_for_multiple_file_targets(self, reader, col):
  88         keep_client = mock.MagicMock()
  89         runner = arvados_cwl.executor.ArvCwlExecutor(self.api, keep_client=keep_client)
  90         runner.project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
  91
  92         final = mock.MagicMock()
  93         col.return_value = final
  94         readermock = mock.MagicMock()
  95         reader.return_value = readermock
  96
  97         # This output describes a single file listed in 2 different directories
  98         _, runner.final_output_collection = runner.make_output_collection("Test output", ["foo"], "", { 'out': [
  99         {
 100             'basename': 'testdir1',
 101             'listing': [
 102                 {
 103                     'basename': 'test.txt',
 104                     'nameroot': 'test',
 105                     'nameext': '.txt',
 106                     'location': 'keep:99999999999999999999999999999991+99/test.txt',
 107                     'class': 'File',
 108                     'size': 16
 109                 }
 110             ],
 111             'location': '_:99999999999999999999999999999992+99',
 112             'class': 'Directory'
 113         },
 114         {
 115             'basename': 'testdir2',
 116             'listing': [
 117                 {
 118                     'basename': 'test.txt',
 119                     'nameroot': 'test',
 120                     'nameext': '.txt',
 121                     'location': 'keep:99999999999999999999999999999991+99/test.txt',
 122                     'class':
 123                     'File',
 124                     'size': 16
 125                 }
 126             ],
 127             'location': '_:99999999999999999999999999999993+99',
 128             'class': 'Directory'
 129         }]})
 130
 131         # Check that copy is called on the collection for both locations
 132         final.copy.assert_any_call("test.txt", "testdir1/test.txt", source_collection=mock.ANY, overwrite=mock.ANY)
 133         final.copy.assert_any_call("test.txt", "testdir2/test.txt", source_collection=mock.ANY, overwrite=mock.ANY)
 134
 135     @mock.patch("arvados.collection.Collection")
 136     @mock.patch("arvados.collection.CollectionReader")
 137     def test_make_output_for_literal_name_conflicts(self, reader, col):
 138         keep_client = mock.MagicMock()
 139         runner = arvados_cwl.executor.ArvCwlExecutor(self.api, keep_client=keep_client)
 140         runner.project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 141
 142         final = mock.MagicMock()
 143         col.return_value = final
 144         readermock = mock.MagicMock()
 145         reader.return_value = readermock
 146
 147         # This output describes two literals with the same basename
 148         _, runner.final_output_collection = runner.make_output_collection("Test output", ["foo"], "",  [
 149         {
 150             'lit':
 151             {
 152                 'basename': 'a_file',
 153                 'nameext': '',
 154                 'nameroot': 'a_file',
 155                 'location': '_:f168fc0c-4291-40aa-a04e-366d57390560',
 156                 'class': 'File',
 157                 'contents': 'Hello file literal.'
 158             }
 159         },
 160         {
 161             'lit':
 162             {
 163                 'basename': 'a_file',
 164                 'nameext': '',
 165                 'nameroot': 'a_file',
 166                 'location': '_:1728da8f-c64e-4a3e-b2e2-1ee356be7bc8',
 167                 'class': 'File',
 168                 'contents': 'Hello file literal.'
 169             }
 170         }])
 171
 172         # Check that the file name conflict is resolved and open is called for both
 173         final.open.assert_any_call("a_file", "wb")
 174         final.open.assert_any_call("a_file_2", "wb")