Merge branch '17572-arvmount-storage-classes'
[arvados.git] / services / fuse / arvados_fuse / fusedir.py
index e8da789fa5fbcff8682cbc0fca251bdf9a7c23a8..78cbd0d8cfd06f1c638549151c56e74a32025237 100644 (file)
@@ -487,6 +487,8 @@ class CollectionDirectory(CollectionDirectoryBase):
                             new_collection_record["portable_data_hash"] = new_collection_record["uuid"]
                         if 'manifest_text' not in new_collection_record:
                             new_collection_record['manifest_text'] = coll_reader.manifest_text()
+                        if 'storage_classes_desired' not in new_collection_record:
+                            new_collection_record['storage_classes_desired'] = coll_reader.storage_classes_desired()
 
                         if self.collection_record is None or self.collection_record["portable_data_hash"] != new_collection_record.get("portable_data_hash"):
                             self.new_collection(new_collection_record, coll_reader)
@@ -571,11 +573,12 @@ class TmpCollectionDirectory(CollectionDirectoryBase):
         def save_new(self):
             pass
 
-    def __init__(self, parent_inode, inodes, api_client, num_retries):
+    def __init__(self, parent_inode, inodes, api_client, num_retries, storage_classes=None):
         collection = self.UnsaveableCollection(
             api_client=api_client,
             keep_client=api_client.keep,
-            num_retries=num_retries)
+            num_retries=num_retries,
+            storage_classes_desired=storage_classes)
         super(TmpCollectionDirectory, self).__init__(
             parent_inode, inodes, api_client.config, collection)
         self.collection_record_file = None
@@ -595,6 +598,7 @@ class TmpCollectionDirectory(CollectionDirectoryBase):
                 "uuid": None,
                 "manifest_text": self.collection.manifest_text(),
                 "portable_data_hash": self.collection.portable_data_hash(),
+                "storage_classes_desired": self.collection.storage_classes_desired(),
             }
 
     def __contains__(self, k):
@@ -653,11 +657,12 @@ and the directory will appear if it exists.
 
 """.lstrip()
 
-    def __init__(self, parent_inode, inodes, api, num_retries, pdh_only=False):
+    def __init__(self, parent_inode, inodes, api, num_retries, pdh_only=False, storage_classes=None):
         super(MagicDirectory, self).__init__(parent_inode, inodes, api.config)
         self.api = api
         self.num_retries = num_retries
         self.pdh_only = pdh_only
+        self.storage_classes = storage_classes
 
     def __setattr__(self, name, value):
         super(MagicDirectory, self).__setattr__(name, value)
@@ -687,7 +692,8 @@ and the directory will appear if it exists.
                 if project[u'items_available'] == 0:
                     return False
                 e = self.inodes.add_entry(ProjectDirectory(
-                    self.inode, self.inodes, self.api, self.num_retries, project[u'items'][0]))
+                    self.inode, self.inodes, self.api, self.num_retries,
+                    project[u'items'][0], storage_classes=self.storage_classes))
             else:
                 e = self.inodes.add_entry(CollectionDirectory(
                         self.inode, self.inodes, self.api, self.num_retries, k))
@@ -811,7 +817,7 @@ class ProjectDirectory(Directory):
     """A special directory that contains the contents of a project."""
 
     def __init__(self, parent_inode, inodes, api, num_retries, project_object,
-                 poll=True, poll_time=3):
+                 poll=True, poll_time=3, storage_classes=None):
         super(ProjectDirectory, self).__init__(parent_inode, inodes, api.config)
         self.api = api
         self.num_retries = num_retries
@@ -823,6 +829,7 @@ class ProjectDirectory(Directory):
         self._updating_lock = threading.Lock()
         self._current_user = None
         self._full_listing = False
+        self.storage_classes = storage_classes
 
     def want_event_subscribe(self):
         return True
@@ -831,7 +838,7 @@ class ProjectDirectory(Directory):
         if collection_uuid_pattern.match(i['uuid']):
             return CollectionDirectory(self.inode, self.inodes, self.api, self.num_retries, i)
         elif group_uuid_pattern.match(i['uuid']):
-            return ProjectDirectory(self.inode, self.inodes, self.api, self.num_retries, i, self._poll, self._poll_time)
+            return ProjectDirectory(self.inode, self.inodes, self.api, self.num_retries, i, self._poll, self._poll_time, self.storage_classes)
         elif link_uuid_pattern.match(i['uuid']):
             if i['head_kind'] == 'arvados#collection' or portable_data_hash_pattern.match(i['head_uuid']):
                 return CollectionDirectory(self.inode, self.inodes, self.api, self.num_retries, i['head_uuid'])
@@ -895,15 +902,17 @@ class ProjectDirectory(Directory):
                     self.project_object = self.api.users().get(
                         uuid=self.project_uuid).execute(num_retries=self.num_retries)
                 # do this in 2 steps until #17424 is fixed
-                contents = arvados.util.list_all(self.api.groups().contents,
-                                                 self.num_retries,
-                                                 uuid=self.project_uuid,
-                                                 filters=[["uuid", "is_a", "arvados#group"],
-                                                          ["groups.group_class", "in", ["project","filter"]]])
-                contents.extend(arvados.util.list_all(self.api.groups().contents,
-                                                      self.num_retries,
-                                                      uuid=self.project_uuid,
-                                                      filters=[["uuid", "is_a", "arvados#collection"]]))
+                contents = list(arvados.util.keyset_list_all(self.api.groups().contents,
+                                                        order_key="uuid",
+                                                        num_retries=self.num_retries,
+                                                        uuid=self.project_uuid,
+                                                        filters=[["uuid", "is_a", "arvados#group"],
+                                                                 ["groups.group_class", "in", ["project","filter"]]]))
+                contents.extend(arvados.util.keyset_list_all(self.api.groups().contents,
+                                                             order_key="uuid",
+                                                             num_retries=self.num_retries,
+                                                             uuid=self.project_uuid,
+                                                             filters=[["uuid", "is_a", "arvados#collection"]]))
 
             # end with llfuse.lock_released, re-acquire lock
 
@@ -980,9 +989,16 @@ class ProjectDirectory(Directory):
     def mkdir(self, name):
         try:
             with llfuse.lock_released:
-                self.api.collections().create(body={"owner_uuid": self.project_uuid,
-                                                    "name": name,
-                                                    "manifest_text": ""}).execute(num_retries=self.num_retries)
+                c = {
+                    "owner_uuid": self.project_uuid,
+                    "name": name,
+                    "manifest_text": "" }
+                if self.storage_classes is not None:
+                    c["storage_classes_desired"] = self.storage_classes
+                try:
+                    self.api.collections().create(body=c).execute(num_retries=self.num_retries)
+                except Exception as e:
+                    raise
             self.invalidate()
         except apiclient_errors.Error as error:
             _logger.error(error)
@@ -1077,7 +1093,7 @@ class SharedDirectory(Directory):
     """A special directory that represents users or groups who have shared projects with me."""
 
     def __init__(self, parent_inode, inodes, api, num_retries, exclude,
-                 poll=False, poll_time=60):
+                 poll=False, poll_time=60, storage_classes=None):
         super(SharedDirectory, self).__init__(parent_inode, inodes, api.config)
         self.api = api
         self.num_retries = num_retries
@@ -1085,6 +1101,7 @@ class SharedDirectory(Directory):
         self._poll = True
         self._poll_time = poll_time
         self._updating_lock = threading.Lock()
+        self.storage_classes = storage_classes
 
     @use_counter
     def update(self):
@@ -1118,10 +1135,12 @@ class SharedDirectory(Directory):
                             objects[r["uuid"]] = r
                             root_owners.add(r["uuid"])
                 else:
-                    all_projects = arvados.util.list_all(
-                        self.api.groups().list, self.num_retries,
+                    all_projects = list(arvados.util.keyset_list_all(
+                        self.api.groups().list,
+                        order_key="uuid",
+                        num_retries=self.num_retries,
                         filters=[['group_class','in',['project','filter']]],
-                        select=["uuid", "owner_uuid"])
+                        select=["uuid", "owner_uuid"]))
                     for ob in all_projects:
                         objects[ob['uuid']] = ob
 
@@ -1131,11 +1150,15 @@ class SharedDirectory(Directory):
                             roots.append(ob['uuid'])
                             root_owners.add(ob['owner_uuid'])
 
-                    lusers = arvados.util.list_all(
-                        self.api.users().list, self.num_retries,
+                    lusers = arvados.util.keyset_list_all(
+                        self.api.users().list,
+                        order_key="uuid",
+                        num_retries=self.num_retries,
                         filters=[['uuid','in', list(root_owners)]])
-                    lgroups = arvados.util.list_all(
-                        self.api.groups().list, self.num_retries,
+                    lgroups = arvados.util.keyset_list_all(
+                        self.api.groups().list,
+                        order_key="uuid",
+                        num_retries=self.num_retries,
                         filters=[['uuid','in', list(root_owners)+roots]])
 
                     for l in lusers:
@@ -1148,8 +1171,6 @@ class SharedDirectory(Directory):
                         obr = objects[r]
                         if obr.get("name"):
                             contents[obr["name"]] = obr
-                        #elif obr.get("username"):
-                        #    contents[obr["username"]] = obr
                         elif "first_name" in obr:
                             contents[u"{} {}".format(obr["first_name"], obr["last_name"])] = obr
 
@@ -1164,7 +1185,7 @@ class SharedDirectory(Directory):
             self.merge(viewitems(contents),
                        lambda i: i[0],
                        lambda a, i: a.uuid() == i[1]['uuid'],
-                       lambda i: ProjectDirectory(self.inode, self.inodes, self.api, self.num_retries, i[1], poll=self._poll, poll_time=self._poll_time))
+                       lambda i: ProjectDirectory(self.inode, self.inodes, self.api, self.num_retries, i[1], poll=self._poll, poll_time=self._poll_time, storage_classes=self.storage_classes))
         except Exception:
             _logger.exception("arv-mount shared dir error")
         finally: