Merge branch 'master' into 14965-arv-mount-py-three

[arvados.git] / services / fuse / arvados_fuse / fusedir.py

diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py
index 769771e7beb52cafd5fa11033921973de501e54c..9d53df950b8cd4886398be7134d4207dcb0beef5 100644 (file)
--- a/services/fuse/arvados_fuse/fusedir.py
+++ b/services/fuse/arvados_fuse/fusedir.py
@@ -2,6 +2,10 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
+from __future__ import absolute_import
+from __future__ import division
+from future.utils import viewitems
+from builtins import dict
 import logging
 import re
 import time
@@ -14,8 +18,8 @@ from apiclient import errors as apiclient_errors
 import errno
 import time
 
-from fusefile import StringFile, ObjectFile, FuncToJSONFile, FuseArvadosFile
-from fresh import FreshBase, convertTime, use_counter, check_update
+from .fusefile import StringFile, ObjectFile, FuncToJSONFile, FuseArvadosFile
+from .fresh import FreshBase, convertTime, use_counter, check_update
 
 import arvados.collection
 from arvados.util import portable_data_hash_pattern, uuid_pattern, collection_uuid_pattern, group_uuid_pattern, user_uuid_pattern, link_uuid_pattern
@@ -163,7 +167,7 @@ class Directory(FreshBase):
     def in_use(self):
         if super(Directory, self).in_use():
             return True
-        for v in self._entries.itervalues():
+        for v in self._entries.values():
             if v.in_use():
                 return True
         return False
@@ -171,7 +175,7 @@ class Directory(FreshBase):
     def has_ref(self, only_children):
         if super(Directory, self).has_ref(only_children):
             return True
-        for v in self._entries.itervalues():
+        for v in self._entries.values():
             if v.has_ref(False):
                 return True
         return False
@@ -193,7 +197,7 @@ class Directory(FreshBase):
         # Find self on the parent in order to invalidate this path.
         # Calling the public items() method might trigger a refresh,
         # which we definitely don't want, so read the internal dict directly.
-        for k,v in parent._entries.items():
+        for k,v in viewitems(parent._entries):
             if v is self:
                 self.inodes.invalidate_entry(parent, k)
                 break
@@ -282,7 +286,7 @@ class CollectionDirectoryBase(Directory):
     def populate(self, mtime):
         self._mtime = mtime
         self.collection.subscribe(self.on_event)
-        for entry, item in self.collection.items():
+        for entry, item in viewitems(self.collection):
             self.new_entry(entry, item, self.mtime())
 
     def writable(self):
@@ -359,7 +363,7 @@ class CollectionDirectory(CollectionDirectoryBase):
         self.collection_record = None
         self._poll = True
         try:
-            self._poll_time = (api._rootDesc.get('blobSignatureTtl', 60*60*2)/2)
+            self._poll_time = (api._rootDesc.get('blobSignatureTtl', 60*60*2) // 2)
         except:
             _logger.debug("Error getting blobSignatureTtl from discovery document: %s", sys.exc_info()[0])
             self._poll_time = 60*60
@@ -925,7 +929,7 @@ class ProjectDirectory(Directory):
         with llfuse.lock_released:
             if not self._current_user:
                 self._current_user = self.api.users().current().execute(num_retries=self.num_retries)
-            return self._current_user["uuid"] in self.project_object["writable_by"]
+            return self._current_user["uuid"] in self.project_object.get("writable_by", [])
 
     def persisted(self):
         return True
@@ -1049,35 +1053,55 @@ class SharedDirectory(Directory):
                 if not self.stale():
                     return
 
-                all_projects = arvados.util.list_all(
-                    self.api.groups().list, self.num_retries,
-                    filters=[['group_class','=','project']],
-                    select=["uuid", "owner_uuid"])
-                objects = {}
-                for ob in all_projects:
-                    objects[ob['uuid']] = ob
-
+                contents = {}
                 roots = []
                 root_owners = set()
-                current_uuid = self.current_user['uuid']
-                for ob in all_projects:
-                    if ob['owner_uuid'] != current_uuid and ob['owner_uuid'] not in objects:
-                        roots.append(ob['uuid'])
-                        root_owners.add(ob['owner_uuid'])
-
-                lusers = arvados.util.list_all(
-                    self.api.users().list, self.num_retries,
-                    filters=[['uuid','in', list(root_owners)]])
-                lgroups = arvados.util.list_all(
-                    self.api.groups().list, self.num_retries,
-                    filters=[['uuid','in', list(root_owners)+roots]])
-
-                for l in lusers:
-                    objects[l["uuid"]] = l
-                for l in lgroups:
-                    objects[l["uuid"]] = l
+                objects = {}
+
+                methods = self.api._rootDesc.get('resources')["groups"]['methods']
+                if 'httpMethod' in methods.get('shared', {}):
+                    page = []
+                    while True:
+                        resp = self.api.groups().shared(filters=[['group_class', '=', 'project']]+page,
+                                                        order="uuid",
+                                                        limit=10000,
+                                                        count="none",
+                                                        include="owner_uuid").execute()
+                        if not resp["items"]:
+                            break
+                        page = [["uuid", ">", resp["items"][len(resp["items"])-1]["uuid"]]]
+                        for r in resp["items"]:
+                            objects[r["uuid"]] = r
+                            roots.append(r["uuid"])
+                        for r in resp["included"]:
+                            objects[r["uuid"]] = r
+                            root_owners.add(r["uuid"])
+                else:
+                    all_projects = arvados.util.list_all(
+                        self.api.groups().list, self.num_retries,
+                        filters=[['group_class','=','project']],
+                        select=["uuid", "owner_uuid"])
+                    for ob in all_projects:
+                        objects[ob['uuid']] = ob
+
+                    current_uuid = self.current_user['uuid']
+                    for ob in all_projects:
+                        if ob['owner_uuid'] != current_uuid and ob['owner_uuid'] not in objects:
+                            roots.append(ob['uuid'])
+                            root_owners.add(ob['owner_uuid'])
+
+                    lusers = arvados.util.list_all(
+                        self.api.users().list, self.num_retries,
+                        filters=[['uuid','in', list(root_owners)]])
+                    lgroups = arvados.util.list_all(
+                        self.api.groups().list, self.num_retries,
+                        filters=[['uuid','in', list(root_owners)+roots]])
+
+                    for l in lusers:
+                        objects[l["uuid"]] = l
+                    for l in lgroups:
+                        objects[l["uuid"]] = l
 
-                contents = {}
                 for r in root_owners:
                     if r in objects:
                         obr = objects[r]
@@ -1096,7 +1120,7 @@ class SharedDirectory(Directory):
 
             # end with llfuse.lock_released, re-acquire lock
 
-            self.merge(contents.items(),
+            self.merge(viewitems(contents),
                        lambda i: i[0],
                        lambda a, i: a.uuid() == i[1]['uuid'],
                        lambda i: ProjectDirectory(self.inode, self.inodes, self.api, self.num_retries, i[1], poll=self._poll, poll_time=self._poll_time))