11462: Log ArgumentError in make_output_collection. Add test for CollectionCache.

diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 1d2fa291ec1eee8953337f4ca1b97343ed7bcae2..ed784f56c55cfa1d0be2d0e8e78287a8e5c3115b 100644 (file)
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -247,10 +247,13 @@ class ArvCwlRunner(object):
                 raise Exception("Output source is not in keep or a literal")
             sp = k.split("/")
             srccollection = sp[0][5:]
-            reader = self.collection_cache.get(srccollection)
             try:
+                reader = self.collection_cache.get(srccollection)
                 srcpath = "/".join(sp[1:]) if len(sp) > 1 else "."
                 final.copy(srcpath, v.target, source_collection=reader, overwrite=False)
+            except arvados.errors.ArgumentError as e:
+                logger.error("Creating CollectionReader for '%s' '%s': %s", k, v, e)
+                raise
             except IOError as e:
                 logger.warn("While preparing output collection: %s", e)
 

diff --git a/sdk/cwl/arvados_cwl/fsaccess.py b/sdk/cwl/arvados_cwl/fsaccess.py
index 34d9cea3a4ad34dedcdd68dfc4e69811bb4d6886..534a675525ae60d90c08559e337dddc5b7d78934 100644 (file)
--- a/sdk/cwl/arvados_cwl/fsaccess.py
+++ b/sdk/cwl/arvados_cwl/fsaccess.py
@@ -4,6 +4,7 @@ import errno
 import urlparse
 import re
 import logging
+import threading
 
 import ruamel.yaml as yaml
 
@@ -25,13 +26,15 @@ class CollectionCache(object):
         self.api_client = api_client
         self.keep_client = keep_client
         self.collections = {}
+        self.lock = threading.Lock()
 
     def get(self, pdh):
-        if pdh not in self.collections:
-            logger.debug("Creating collection reader for %s", pdh)
-            self.collections[pdh] = arvados.collection.CollectionReader(pdh, api_client=self.api_client,
-                                                                        keep_client=self.keep_client)
-        return self.collections[pdh]
+        with self.lock:
+            if pdh not in self.collections:
+                logger.debug("Creating collection reader for %s", pdh)
+                self.collections[pdh] = arvados.collection.CollectionReader(pdh, api_client=self.api_client,
+                                                                            keep_client=self.keep_client)
+            return self.collections[pdh]
 
 
 class CollectionFsAccess(cwltool.stdfsaccess.StdFsAccess):

diff --git a/sdk/cwl/tests/test_fsaccess.py b/sdk/cwl/tests/test_fsaccess.py
new file mode 100644 (file)
index 0000000..4395661
--- /dev/null
+++ b/sdk/cwl/tests/test_fsaccess.py
@@ -0,0 +1,28 @@
+import functools
+import mock
+import sys
+import unittest
+import json
+import logging
+import os
+
+import arvados
+import arvados.keep
+import arvados.collection
+import arvados_cwl
+
+from cwltool.pathmapper import MapperEnt
+from .mock_discovery import get_rootDesc
+
+from arvados_cwl.fsaccess import CollectionCache
+
+class TestFsAccess(unittest.TestCase):
+    @mock.patch("arvados.collection.CollectionReader")
+    def test_collection_cache(self, cr):
+        cache = CollectionCache(mock.MagicMock(), mock.MagicMock(), 4)
+        c1 = cache.get("99999999999999999999999999999991+99")
+        c2 = cache.get("99999999999999999999999999999991+99")
+        self.assertIs(c1, c2)
+        self.assertEqual(1, cr.call_count)
+        c3 = cache.get("99999999999999999999999999999992+99")
+        self.assertEqual(2, cr.call_count)