10221: Add path mapper tests direct keep references, uploaded files, and keep mounted...

diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index 5051d977b49298266d3554b1a12e1218f31ede46..288397a5012ed39f339d276e27e8a708324064f8 100644 (file)
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -41,7 +41,7 @@ class ArvPathMapper(PathMapper):
                 if isinstance(st, arvados.commands.run.UploadFile):
                     uploadfiles.add((src, ab, st))
                 elif isinstance(st, arvados.commands.run.ArvFile):
-                    self._pathmap[src] = MapperEnt("keep:"+ab, self.collection_pattern % ab, "File")
+                    self._pathmap[src] = MapperEnt("keep:"+st.keepref, st.fn, "File")
                 elif src.startswith("_:"):
                     if "contents" in srcobj:
                         pass
@@ -96,7 +96,7 @@ class ArvPathMapper(PathMapper):
                                              project=self.arvrunner.project_uuid)
 
         for src, ab, st in uploadfiles:
-            self._pathmap[src] = MapperEnt("keep:" + st.keepref, self.collection_pattern % st.keepref, "File")
+            self._pathmap[src] = MapperEnt("keep:" + st.keepref, st.fn, "File")
             self.arvrunner.add_uploaded(src, self._pathmap[src])
 
         for srcobj in referenced_files:

diff --git a/sdk/cwl/tests/hw.py b/sdk/cwl/tests/hw.py
new file mode 100644 (file)
index 0000000..62c813a
--- /dev/null
+++ b/sdk/cwl/tests/hw.py
@@ -0,0 +1 @@
+print "Hello world"

diff --git a/sdk/cwl/tests/test_pathmapper.py b/sdk/cwl/tests/test_pathmapper.py
new file mode 100644 (file)
index 0000000..065904d
--- /dev/null
+++ b/sdk/cwl/tests/test_pathmapper.py
@@ -0,0 +1,94 @@
+import functools
+import mock
+import sys
+import unittest
+import json
+import logging
+import os
+
+import arvados
+import arvados.keep
+import arvados.collection
+import arvados_cwl
+
+from cwltool.pathmapper import MapperEnt
+
+from arvados_cwl.pathmapper import ArvPathMapper
+
+def upload_mock(files, api, dry_run=False, num_retries=0, project=None, fnPattern="$(file %s/%s)", name=None):
+    pdh = "99999999999999999999999999999991+99"
+    for c in files:
+        c.fn = os.path.basename(c.fn)
+        c.keepref = "%s/%s" % (pdh, c.fn)
+        c.fn = fnPattern % (pdh, c.fn)
+
+class TestPathmap(unittest.TestCase):
+    def test_keepref(self):
+        """Test direct keep references."""
+
+        arvrunner = arvados_cwl.ArvCwlRunner(mock.MagicMock())
+
+        p = ArvPathMapper(arvrunner, [{
+            "class": "File",
+            "location": "keep:99999999999999999999999999999991+99/hw.py"
+        }], "", "/test/%s", "/test/%s/%s")
+
+        self.assertEqual({'keep:99999999999999999999999999999991+99/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File')},
+                         p._pathmap)
+
+    @mock.patch("arvados.commands.run.uploadfiles")
+    def test_upload(self, upl):
+        """Test pathmapper uploading files."""
+
+        arvrunner = arvados_cwl.ArvCwlRunner(mock.MagicMock())
+
+        upl.side_effect = upload_mock
+
+        p = ArvPathMapper(arvrunner, [{
+            "class": "File",
+            "location": "tests/hw.py"
+        }], "", "/test/%s", "/test/%s/%s")
+
+        self.assertEqual({'tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File')},
+                         p._pathmap)
+
+    @mock.patch("arvados.commands.run.uploadfiles")
+    def test_prev_uploaded(self, upl):
+        """Test pathmapper handling previously uploaded files."""
+
+        arvrunner = arvados_cwl.ArvCwlRunner(mock.MagicMock())
+        arvrunner.add_uploaded('tests/hw.py', MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='', type='File'))
+
+        upl.side_effect = upload_mock
+
+        p = ArvPathMapper(arvrunner, [{
+            "class": "File",
+            "location": "tests/hw.py"
+        }], "", "/test/%s", "/test/%s/%s")
+
+        self.assertEqual({'tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File')},
+                         p._pathmap)
+
+    @mock.patch("arvados.commands.run.uploadfiles")
+    @mock.patch("arvados.commands.run.statfile")
+    def test_statfile(self, statfile, upl):
+        """Test pathmapper handling ArvFile references."""
+        arvrunner = arvados_cwl.ArvCwlRunner(mock.MagicMock())
+
+        # An ArvFile object returned from arvados.commands.run.statfile means the file is located on a
+        # keep mount, so we can construct a direct reference directly without upload.
+        def statfile_mock(prefix, fn, fnPattern="$(file %s/%s)", dirPattern="$(dir %s/%s/)"):
+            st = arvados.commands.run.ArvFile("", fnPattern % ("99999999999999999999999999999991+99", "hw.py"))
+            st.keepref = "99999999999999999999999999999991+99/hw.py"
+            return st
+
+        upl.side_effect = upload_mock
+        statfile.side_effect = statfile_mock
+
+        p = ArvPathMapper(arvrunner, [{
+            "class": "File",
+            "location": "tests/hw.py"
+        }], "", "/test/%s", "/test/%s/%s")
+
+        self.assertEqual({'tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File')},
+                         p._pathmap)

diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 6674efb8c4f75c2391160a702588ea2354e377a3..007f809d4140a89136b9419e7e6d9fd2d801477f 100644 (file)
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -60,7 +60,13 @@ def stubs(func):
             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz5",
             "portable_data_hash": "99999999999999999999999999999995+99",
             "manifest_text": ""
-        }        )
+        },
+        {
+            "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz6",
+            "portable_data_hash": "99999999999999999999999999999996+99",
+            "manifest_text": ""
+        }
+        )
         stubs.api.collections().get().execute.return_value = {
             "portable_data_hash": "99999999999999999999999999999993+99", "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"}