1 # Copyright (C) The Arvados Authors. All rights reserved.
3 # SPDX-License-Identifier: Apache-2.0
12 from unittest import mock
16 import arvados.collection
18 import arvados_cwl.executor
20 from cwltool.pathmapper import MapperEnt
21 from .mock_discovery import get_rootDesc
23 from arvados_cwl.pathmapper import ArvPathMapper
25 def upload_mock(files, api, dry_run=False, num_retries=0, project=None, fnPattern="$(file %s/%s)", name=None, collection=None, packed=None):
26 pdh = "99999999999999999999999999999991+99"
28 c.keepref = "%s/%s" % (pdh, os.path.basename(c.fn))
29 c.fn = fnPattern % (pdh, os.path.basename(c.fn))
31 class TestPathmap(unittest.TestCase):
33 self.api = mock.MagicMock()
34 self.api._rootDesc = get_rootDesc()
37 root_logger = logging.getLogger('')
39 # Remove existing RuntimeStatusLoggingHandlers if they exist
40 handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
41 root_logger.handlers = handlers
43 def test_keepref(self):
44 """Test direct keep references."""
46 arvrunner = arvados_cwl.executor.ArvCwlExecutor(self.api)
48 p = ArvPathMapper(arvrunner, [{
50 "location": "keep:99999999999999999999999999999991+99/hw.py"
51 }], "", "/test/%s", "/test/%s/%s")
53 self.assertEqual({'keep:99999999999999999999999999999991+99/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File', staged=True)},
56 @mock.patch("arvados.commands.run.uploadfiles")
57 @mock.patch("arvados.commands.run.statfile")
58 def test_upload(self, statfile, upl):
59 """Test pathmapper uploading files."""
61 arvrunner = arvados_cwl.executor.ArvCwlExecutor(self.api)
63 def statfile_mock(prefix, fn, fnPattern="$(file %s/%s)", dirPattern="$(dir %s/%s/)", raiseOSError=False):
64 st = arvados.commands.run.UploadFile("", "tests/hw.py")
67 upl.side_effect = upload_mock
68 statfile.side_effect = statfile_mock
70 p = ArvPathMapper(arvrunner, [{
72 "location": "file:tests/hw.py"
73 }], "", "/test/%s", "/test/%s/%s")
75 self.assertEqual({'file:tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File', staged=True)},
78 @mock.patch("arvados.commands.run.uploadfiles")
79 @mock.patch("arvados.commands.run.statfile")
80 def test_statfile(self, statfile, upl):
81 """Test pathmapper handling ArvFile references."""
82 arvrunner = arvados_cwl.executor.ArvCwlExecutor(self.api)
84 # An ArvFile object returned from arvados.commands.run.statfile means the file is located on a
85 # keep mount, so we can construct a direct reference directly without upload.
86 def statfile_mock(prefix, fn, fnPattern="$(file %s/%s)", dirPattern="$(dir %s/%s/)", raiseOSError=False):
87 st = arvados.commands.run.ArvFile("", fnPattern % ("99999999999999999999999999999991+99", "hw.py"))
90 upl.side_effect = upload_mock
91 statfile.side_effect = statfile_mock
93 p = ArvPathMapper(arvrunner, [{
95 "location": "file:tests/hw.py"
96 }], "", "/test/%s", "/test/%s/%s")
98 self.assertEqual({'file:tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File', staged=True)},
101 @mock.patch("os.stat")
102 def test_missing_file(self, stat):
103 """Test pathmapper handling missing references."""
104 arvrunner = arvados_cwl.executor.ArvCwlExecutor(self.api)
106 stat.side_effect = OSError(2, "No such file or directory")
108 with self.assertRaises(OSError):
109 p = ArvPathMapper(arvrunner, [{
111 "location": "file:tests/hw.py"
112 }], "", "/test/%s", "/test/%s/%s")
114 def test_needs_new_collection(self):
115 arvrunner = arvados_cwl.executor.ArvCwlExecutor(self.api)
117 # Plain file. Don't need a new collection.
120 "location": "keep:99999999999999999999999999999991+99/hw.py",
123 p = ArvPathMapper(arvrunner, [], "", "%s", "%s/%s")
124 p._pathmap["keep:99999999999999999999999999999991+99/hw.py"] = True
125 self.assertFalse(p.needs_new_collection(a))
127 # A file that isn't in the pathmap (for some reason). Need a new collection.
128 p = ArvPathMapper(arvrunner, [], "", "%s", "%s/%s")
129 self.assertTrue(p.needs_new_collection(a))
131 # A file with a secondary file in the same collection. Don't need
135 "location": "keep:99999999999999999999999999999991+99/hw.py",
139 "location": "keep:99999999999999999999999999999991+99/hw.pyc",
143 p = ArvPathMapper(arvrunner, [], "", "%s", "%s/%s")
144 p._pathmap["keep:99999999999999999999999999999991+99/hw.py"] = True
145 p._pathmap["keep:99999999999999999999999999999991+99/hw.pyc"] = True
146 self.assertFalse(p.needs_new_collection(a))
148 # Secondary file is in a different collection from the
149 # a new collectionprimary. Need a new collection.
152 "location": "keep:99999999999999999999999999999991+99/hw.py",
156 "location": "keep:99999999999999999999999999999992+99/hw.pyc",
160 p = ArvPathMapper(arvrunner, [], "", "%s", "%s/%s")
161 p._pathmap["keep:99999999999999999999999999999991+99/hw.py"] = True
162 p._pathmap["keep:99999999999999999999999999999992+99/hw.pyc"] = True
163 self.assertTrue(p.needs_new_collection(a))
165 # Secondary file should be staged to a different name than
166 # path in location. Need a new collection.
169 "location": "keep:99999999999999999999999999999991+99/hw.py",
173 "location": "keep:99999999999999999999999999999991+99/hw.pyc",
174 "basename": "hw.other"
177 p = ArvPathMapper(arvrunner, [], "", "%s", "%s/%s")
178 p._pathmap["keep:99999999999999999999999999999991+99/hw.py"] = True
179 p._pathmap["keep:99999999999999999999999999999991+99/hw.pyc"] = True
180 self.assertTrue(p.needs_new_collection(a))
182 # Secondary file is a directory. Do not need a new collection.
185 "location": "keep:99999999999999999999999999999991+99/hw.py",
188 "class": "Directory",
189 "location": "keep:99999999999999999999999999999991+99/hw",
193 "location": "keep:99999999999999999999999999999991+99/hw/h2",
198 p = ArvPathMapper(arvrunner, [], "", "%s", "%s/%s")
199 p._pathmap["keep:99999999999999999999999999999991+99/hw.py"] = True
200 p._pathmap["keep:99999999999999999999999999999991+99/hw"] = True
201 p._pathmap["keep:99999999999999999999999999999991+99/hw/h2"] = True
202 self.assertFalse(p.needs_new_collection(a))
204 # Secondary file is a renamed directory. Need a new collection.
207 "location": "keep:99999999999999999999999999999991+99/hw.py",
210 "class": "Directory",
211 "location": "keep:99999999999999999999999999999991+99/hw",
215 "location": "keep:99999999999999999999999999999991+99/hw/h2",
220 p = ArvPathMapper(arvrunner, [], "", "%s", "%s/%s")
221 p._pathmap["keep:99999999999999999999999999999991+99/hw.py"] = True
222 p._pathmap["keep:99999999999999999999999999999991+99/hw"] = True
223 p._pathmap["keep:99999999999999999999999999999991+99/hw/h2"] = True
224 self.assertTrue(p.needs_new_collection(a))
226 # Secondary file is a file literal. Need a new collection.
229 "location": "keep:99999999999999999999999999999991+99/hw.py",
234 "basename": "hw.pyc",
238 p = ArvPathMapper(arvrunner, [], "", "%s", "%s/%s")
239 p._pathmap["keep:99999999999999999999999999999991+99/hw.py"] = True
240 p._pathmap["_:123"] = True
241 self.assertTrue(p.needs_new_collection(a))
243 def test_is_in_collection(self):
244 arvrunner = arvados_cwl.executor.ArvCwlExecutor(self.api)
245 self.maxDiff = 1000000
248 p = ArvPathMapper(arvrunner, [{
250 "location": "file://"+cwd+"/tests/fake-keep-mount/fake_collection_dir/subdir/banana.txt"
251 }], "", "/test/%s", "/test/%s/%s")
253 self.assertEqual({"file://"+cwd+"/tests/fake-keep-mount/fake_collection_dir/subdir/banana.txt": MapperEnt(resolved='keep:99999999999999999999999999999991+99/subdir/banana.txt', target='/test/99999999999999999999999999999991+99/subdir/banana.txt', type='File', staged=True)},