1 # Copyright (C) The Arvados Authors. All rights reserved.
3 # SPDX-License-Identifier: Apache-2.0
16 import arvados.collection
19 from cwltool.pathmapper import MapperEnt
20 from .mock_discovery import get_rootDesc
22 from arvados_cwl.pathmapper import ArvPathMapper
24 def upload_mock(files, api, dry_run=False, num_retries=0, project=None, fnPattern="$(file %s/%s)", name=None, collection=None, packed=None):
25 pdh = "99999999999999999999999999999991+99"
27 c.keepref = "%s/%s" % (pdh, os.path.basename(c.fn))
28 c.fn = fnPattern % (pdh, os.path.basename(c.fn))
30 class MockDateTime(datetime.datetime):
33 return datetime.datetime(2018, 1, 1, 0, 0, 0, 0)
35 datetime.datetime = MockDateTime
37 class TestPathmap(unittest.TestCase):
39 self.api = mock.MagicMock()
40 self.api._rootDesc = get_rootDesc()
42 def test_keepref(self):
43 """Test direct keep references."""
45 arvrunner = arvados_cwl.ArvCwlRunner(self.api)
47 p = ArvPathMapper(arvrunner, [{
49 "location": "keep:99999999999999999999999999999991+99/hw.py"
50 }], "", "/test/%s", "/test/%s/%s")
52 self.assertEqual({'keep:99999999999999999999999999999991+99/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File', staged=True)},
55 @mock.patch("arvados.commands.run.uploadfiles")
56 @mock.patch("arvados.commands.run.statfile")
57 def test_upload(self, statfile, upl):
58 """Test pathmapper uploading files."""
60 arvrunner = arvados_cwl.ArvCwlRunner(self.api)
62 def statfile_mock(prefix, fn, fnPattern="$(file %s/%s)", dirPattern="$(dir %s/%s/)", raiseOSError=False):
63 st = arvados.commands.run.UploadFile("", "tests/hw.py")
66 upl.side_effect = upload_mock
67 statfile.side_effect = statfile_mock
69 p = ArvPathMapper(arvrunner, [{
71 "location": "file:tests/hw.py"
72 }], "", "/test/%s", "/test/%s/%s")
74 self.assertEqual({'file:tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File', staged=True)},
77 @mock.patch("arvados.commands.run.uploadfiles")
78 @mock.patch("arvados.commands.run.statfile")
79 def test_statfile(self, statfile, upl):
80 """Test pathmapper handling ArvFile references."""
81 arvrunner = arvados_cwl.ArvCwlRunner(self.api)
83 # An ArvFile object returned from arvados.commands.run.statfile means the file is located on a
84 # keep mount, so we can construct a direct reference directly without upload.
85 def statfile_mock(prefix, fn, fnPattern="$(file %s/%s)", dirPattern="$(dir %s/%s/)", raiseOSError=False):
86 st = arvados.commands.run.ArvFile("", fnPattern % ("99999999999999999999999999999991+99", "hw.py"))
89 upl.side_effect = upload_mock
90 statfile.side_effect = statfile_mock
92 p = ArvPathMapper(arvrunner, [{
94 "location": "file:tests/hw.py"
95 }], "", "/test/%s", "/test/%s/%s")
97 self.assertEqual({'file:tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File', staged=True)},
100 @mock.patch("os.stat")
101 def test_missing_file(self, stat):
102 """Test pathmapper handling missing references."""
103 arvrunner = arvados_cwl.ArvCwlRunner(self.api)
105 stat.side_effect = OSError(2, "No such file or directory")
107 with self.assertRaises(OSError):
108 p = ArvPathMapper(arvrunner, [{
110 "location": "file:tests/hw.py"
111 }], "", "/test/%s", "/test/%s/%s")
113 def test_get_intermediate_collection_info(self):
114 self.api.containers().current().execute.return_value = {"uuid" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"}
115 arvrunner = arvados_cwl.ArvCwlRunner(self.api)
116 arvrunner.intermediate_output_ttl = 60
118 path_mapper = ArvPathMapper(arvrunner, [{
120 "location": "keep:99999999999999999999999999999991+99/hw.py"
121 }], "", "/test/%s", "/test/%s/%s")
123 info = path_mapper._get_intermediate_collection_info()
125 self.assertEqual(info["name"], "Intermediate collection")
126 self.assertEqual(info["trash_at"], datetime.datetime(2018, 1, 1, 0, 1))
127 self.assertEqual(info["properties"], {"type" : "Intermediate", "container" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"})