X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/ef6f7202858cba65e06cc1a32d52ee2305687bc8..11377b3b6eb9544b9d9bcabab8de495f6e6fcaa4:/sdk/cwl/tests/test_pathmapper.py diff --git a/sdk/cwl/tests/test_pathmapper.py b/sdk/cwl/tests/test_pathmapper.py index 7e13066d85..9649b83872 100644 --- a/sdk/cwl/tests/test_pathmapper.py +++ b/sdk/cwl/tests/test_pathmapper.py @@ -1,3 +1,7 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + import functools import mock import sys @@ -12,70 +16,82 @@ import arvados.collection import arvados_cwl from cwltool.pathmapper import MapperEnt +from .mock_discovery import get_rootDesc from arvados_cwl.pathmapper import ArvPathMapper -def upload_mock(files, api, dry_run=False, num_retries=0, project=None, fnPattern="$(file %s/%s)", name=None): +def upload_mock(files, api, dry_run=False, num_retries=0, project=None, fnPattern="$(file %s/%s)", name=None, collection=None): pdh = "99999999999999999999999999999991+99" for c in files: + c.keepref = "%s/%s" % (pdh, os.path.basename(c.fn)) c.fn = fnPattern % (pdh, os.path.basename(c.fn)) class TestPathmap(unittest.TestCase): + def setUp(self): + self.api = mock.MagicMock() + self.api._rootDesc = get_rootDesc() + def test_keepref(self): """Test direct keep references.""" - arvrunner = arvados_cwl.ArvCwlRunner(mock.MagicMock()) + arvrunner = arvados_cwl.ArvCwlRunner(self.api) p = ArvPathMapper(arvrunner, [{ "class": "File", "location": "keep:99999999999999999999999999999991+99/hw.py" }], "", "/test/%s", "/test/%s/%s") - self.assertEqual({'keep:99999999999999999999999999999991+99/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File')}, + self.assertEqual({'keep:99999999999999999999999999999991+99/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File', staged=True)}, p._pathmap) @mock.patch("arvados.commands.run.uploadfiles") - def test_upload(self, upl): + @mock.patch("arvados.commands.run.statfile") + def test_upload(self, statfile, upl): """Test pathmapper uploading files.""" - arvrunner = arvados_cwl.ArvCwlRunner(mock.MagicMock()) + arvrunner = arvados_cwl.ArvCwlRunner(self.api) + + def statfile_mock(prefix, fn, fnPattern="$(file %s/%s)", dirPattern="$(dir %s/%s/)", raiseOSError=False): + st = arvados.commands.run.UploadFile("", "tests/hw.py") + return st upl.side_effect = upload_mock + statfile.side_effect = statfile_mock p = ArvPathMapper(arvrunner, [{ "class": "File", - "location": "tests/hw.py" + "location": "file:tests/hw.py" }], "", "/test/%s", "/test/%s/%s") - self.assertEqual({'tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File')}, + self.assertEqual({'file:tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File', staged=True)}, p._pathmap) @mock.patch("arvados.commands.run.uploadfiles") def test_prev_uploaded(self, upl): """Test pathmapper handling previously uploaded files.""" - arvrunner = arvados_cwl.ArvCwlRunner(mock.MagicMock()) - arvrunner.add_uploaded('tests/hw.py', MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='', type='File')) + arvrunner = arvados_cwl.ArvCwlRunner(self.api) + arvrunner.add_uploaded('file:tests/hw.py', MapperEnt(resolved='keep:99999999999999999999999999999992+99/hw.py', target='', type='File', staged=True)) upl.side_effect = upload_mock p = ArvPathMapper(arvrunner, [{ "class": "File", - "location": "tests/hw.py" + "location": "file:tests/hw.py" }], "", "/test/%s", "/test/%s/%s") - self.assertEqual({'tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File')}, + self.assertEqual({'file:tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999992+99/hw.py', target='/test/99999999999999999999999999999992+99/hw.py', type='File', staged=True)}, p._pathmap) @mock.patch("arvados.commands.run.uploadfiles") @mock.patch("arvados.commands.run.statfile") def test_statfile(self, statfile, upl): """Test pathmapper handling ArvFile references.""" - arvrunner = arvados_cwl.ArvCwlRunner(mock.MagicMock()) + arvrunner = arvados_cwl.ArvCwlRunner(self.api) # An ArvFile object returned from arvados.commands.run.statfile means the file is located on a # keep mount, so we can construct a direct reference directly without upload. - def statfile_mock(prefix, fn, fnPattern="$(file %s/%s)", dirPattern="$(dir %s/%s/)"): + def statfile_mock(prefix, fn, fnPattern="$(file %s/%s)", dirPattern="$(dir %s/%s/)", raiseOSError=False): st = arvados.commands.run.ArvFile("", fnPattern % ("99999999999999999999999999999991+99", "hw.py")) return st @@ -84,8 +100,21 @@ class TestPathmap(unittest.TestCase): p = ArvPathMapper(arvrunner, [{ "class": "File", - "location": "tests/hw.py" + "location": "file:tests/hw.py" }], "", "/test/%s", "/test/%s/%s") - self.assertEqual({'tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File')}, + self.assertEqual({'file:tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File', staged=True)}, p._pathmap) + + @mock.patch("os.stat") + def test_missing_file(self, stat): + """Test pathmapper handling missing references.""" + arvrunner = arvados_cwl.ArvCwlRunner(self.api) + + stat.side_effect = OSError(2, "No such file or directory") + + with self.assertRaises(OSError): + p = ArvPathMapper(arvrunner, [{ + "class": "File", + "location": "file:tests/hw.py" + }], "", "/test/%s", "/test/%s/%s")