17503: Merge branch 'master' into 17503-fix-deduplication-report-paper-cuts

[arvados.git] / services / fuse / arvados_fuse / fusedir.py

diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py
index 9853f6ac20d591f4e71da9f7ef32823e1a793607..e8da789fa5fbcff8682cbc0fca251bdf9a7c23a8 100644 (file)
--- a/services/fuse/arvados_fuse/fusedir.py
+++ b/services/fuse/arvados_fuse/fusedir.py
@@ -7,17 +7,17 @@ from __future__ import division
 from future.utils import viewitems
 from future.utils import itervalues
 from builtins import dict
-import logging
-import re
-import time
-import llfuse
-import arvados
 import apiclient
+import arvados
+import errno
 import functools
+import llfuse
+import logging
+import re
+import sys
 import threading
-from apiclient import errors as apiclient_errors
-import errno
 import time
+from apiclient import errors as apiclient_errors
 
 from .fusefile import StringFile, ObjectFile, FuncToJSONFile, FuseArvadosFile
 from .fresh import FreshBase, convertTime, use_counter, check_update
@@ -55,10 +55,24 @@ class Directory(FreshBase):
         self._entries = {}
         self._mtime = time.time()
 
+    def forward_slash_subst(self):
+        if not hasattr(self, '_fsns'):
+            self._fsns = None
+            config = self.apiconfig()
+            try:
+                self._fsns = config["Collections"]["ForwardSlashNameSubstitution"]
+            except KeyError:
+                # old API server with no FSNS config
+                self._fsns = '_'
+            else:
+                if self._fsns == '' or self._fsns == '/':
+                    self._fsns = None
+        return self._fsns
+
     def unsanitize_filename(self, incoming):
         """Replace ForwardSlashNameSubstitution value with /"""
-        fsns = self.apiconfig()["Collections"]["ForwardSlashNameSubstitution"]
-        if isinstance(fsns, str) and fsns != '' and fsns != '/':
+        fsns = self.forward_slash_subst()
+        if isinstance(fsns, str):
             return incoming.replace(fsns, '/')
         else:
             return incoming
@@ -76,8 +90,8 @@ class Directory(FreshBase):
         elif dirty == '..':
             return '__'
         else:
-            fsns = self.apiconfig()["Collections"]["ForwardSlashNameSubstitution"]
-            if isinstance(fsns, str) and fsns != '':
+            fsns = self.forward_slash_subst()
+            if isinstance(fsns, str):
                 dirty = dirty.replace('/', fsns)
             return _disallowed_filename_characters.sub('_', dirty)
 
@@ -669,13 +683,12 @@ and the directory will appear if it exists.
 
             if group_uuid_pattern.match(k):
                 project = self.api.groups().list(
-                    filters=[['group_class', '=', 'project'], ["uuid", "=", k]]).execute(num_retries=self.num_retries)
+                    filters=[['group_class', 'in', ['project','filter']], ["uuid", "=", k]]).execute(num_retries=self.num_retries)
                 if project[u'items_available'] == 0:
                     return False
                 e = self.inodes.add_entry(ProjectDirectory(
                     self.inode, self.inodes, self.api, self.num_retries, project[u'items'][0]))
             else:
-                import sys
                 e = self.inodes.add_entry(CollectionDirectory(
                         self.inode, self.inodes, self.api, self.num_retries, k))
 
@@ -798,7 +811,7 @@ class ProjectDirectory(Directory):
     """A special directory that contains the contents of a project."""
 
     def __init__(self, parent_inode, inodes, api, num_retries, project_object,
-                 poll=False, poll_time=60):
+                 poll=True, poll_time=3):
         super(ProjectDirectory, self).__init__(parent_inode, inodes, api.config)
         self.api = api
         self.num_retries = num_retries
@@ -881,14 +894,16 @@ class ProjectDirectory(Directory):
                 elif user_uuid_pattern.match(self.project_uuid):
                     self.project_object = self.api.users().get(
                         uuid=self.project_uuid).execute(num_retries=self.num_retries)
-
-                contents = arvados.util.list_all(self.api.groups().list,
+                # do this in 2 steps until #17424 is fixed
+                contents = arvados.util.list_all(self.api.groups().contents,
                                                  self.num_retries,
-                                                 filters=[["owner_uuid", "=", self.project_uuid],
-                                                          ["group_class", "=", "project"]])
-                contents.extend(arvados.util.list_all(self.api.collections().list,
+                                                 uuid=self.project_uuid,
+                                                 filters=[["uuid", "is_a", "arvados#group"],
+                                                          ["groups.group_class", "in", ["project","filter"]]])
+                contents.extend(arvados.util.list_all(self.api.groups().contents,
                                                       self.num_retries,
-                                                      filters=[["owner_uuid", "=", self.project_uuid]]))
+                                                      uuid=self.project_uuid,
+                                                      filters=[["uuid", "is_a", "arvados#collection"]]))
 
             # end with llfuse.lock_released, re-acquire lock
 
@@ -919,7 +934,7 @@ class ProjectDirectory(Directory):
             else:
                 namefilter = ["name", "in", [k, k2]]
             contents = self.api.groups().list(filters=[["owner_uuid", "=", self.project_uuid],
-                                                       ["group_class", "=", "project"],
+                                                       ["group_class", "in", ["project","filter"]],
                                                        namefilter],
                                               limit=2).execute(num_retries=self.num_retries)["items"]
             if not contents:
@@ -927,7 +942,7 @@ class ProjectDirectory(Directory):
                                                                 namefilter],
                                                        limit=2).execute(num_retries=self.num_retries)["items"]
         if contents:
-            if len(contents) > 1 and contents[1].name == k:
+            if len(contents) > 1 and contents[1]['name'] == k:
                 # If "foo/bar" and "foo[SUBST]bar" both exist, use
                 # "foo[SUBST]bar".
                 contents = [contents[1]]
@@ -1088,7 +1103,7 @@ class SharedDirectory(Directory):
                 if 'httpMethod' in methods.get('shared', {}):
                     page = []
                     while True:
-                        resp = self.api.groups().shared(filters=[['group_class', '=', 'project']]+page,
+                        resp = self.api.groups().shared(filters=[['group_class', 'in', ['project','filter']]]+page,
                                                         order="uuid",
                                                         limit=10000,
                                                         count="none",
@@ -1105,7 +1120,7 @@ class SharedDirectory(Directory):
                 else:
                     all_projects = arvados.util.list_all(
                         self.api.groups().list, self.num_retries,
-                        filters=[['group_class','=','project']],
+                        filters=[['group_class','in',['project','filter']]],
                         select=["uuid", "owner_uuid"])
                     for ob in all_projects:
                         objects[ob['uuid']] = ob