20613: Update unconfigured logger check to accommodate NullHandler

[arvados.git] / sdk / python / arvados / collection.py

diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index d03265ca44b1b6e886e57d455dadcb7a613d0b6e..23b4393a948164fd474c45384ee121e1c5daf362 100644 (file)
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -827,7 +827,7 @@ class RichCollectionBase(CollectionBase):
             self.set_committed(False)
             self.notify(DEL, self, pathcomponents[0], deleteditem)
         else:
             self.set_committed(False)
             self.notify(DEL, self, pathcomponents[0], deleteditem)
         else:

-            item.remove(pathcomponents[1])
+            item.remove(pathcomponents[1], recursive=recursive)

 
     def _clonefrom(self, source):
         for k,v in listitems(source):
 
     def _clonefrom(self, source):
         for k,v in listitems(source):


@@ -1256,13 +1256,14 @@ class Collection(RichCollectionBase):
     def __init__(self, manifest_locator_or_text=None,
                  api_client=None,
                  keep_client=None,
     def __init__(self, manifest_locator_or_text=None,
                  api_client=None,
                  keep_client=None,

-                 num_retries=None,
+                 num_retries=10,

                  parent=None,
                  apiconfig=None,
                  block_manager=None,
                  replication_desired=None,
                  storage_classes_desired=None,
                  parent=None,
                  apiconfig=None,
                  block_manager=None,
                  replication_desired=None,
                  storage_classes_desired=None,

-                 put_threads=None):
+                 put_threads=None,
+                 get_threads=None):

         """Collection constructor.
 
         :manifest_locator_or_text:
         """Collection constructor.
 
         :manifest_locator_or_text:


@@ -1307,17 +1308,23 @@ class Collection(RichCollectionBase):
         super(Collection, self).__init__(parent)
         self._api_client = api_client
         self._keep_client = keep_client
         super(Collection, self).__init__(parent)
         self._api_client = api_client
         self._keep_client = keep_client

+
+        # Use the keep client from ThreadSafeApiCache
+        if self._keep_client is None and isinstance(self._api_client, ThreadSafeApiCache):
+            self._keep_client = self._api_client.keep
+

         self._block_manager = block_manager
         self.replication_desired = replication_desired
         self._storage_classes_desired = storage_classes_desired
         self.put_threads = put_threads
         self._block_manager = block_manager
         self.replication_desired = replication_desired
         self._storage_classes_desired = storage_classes_desired
         self.put_threads = put_threads

+        self.get_threads = get_threads

 
         if apiconfig:
             self._config = apiconfig
         else:
             self._config = config.settings()
 
 
         if apiconfig:
             self._config = apiconfig
         else:
             self._config = config.settings()
 

-        self.num_retries = num_retries if num_retries is not None else 0
+        self.num_retries = num_retries

         self._manifest_locator = None
         self._manifest_text = None
         self._portable_data_hash = None
         self._manifest_locator = None
         self._manifest_text = None
         self._portable_data_hash = None


@@ -1395,7 +1402,7 @@ class Collection(RichCollectionBase):
                 # our tokens.
                 return
             else:
                 # our tokens.
                 return
             else:

-                self._past_versions.add((response.get("modified_at"), response.get("portable_data_hash")))
+                self._remember_api_response(response)

             other = CollectionReader(response["manifest_text"])
         baseline = CollectionReader(self._manifest_text)
         self.apply(baseline.diff(other))
             other = CollectionReader(response["manifest_text"])
         baseline = CollectionReader(self._manifest_text)
         self.apply(baseline.diff(other))


@@ -1404,7 +1411,7 @@ class Collection(RichCollectionBase):
     @synchronized
     def _my_api(self):
         if self._api_client is None:
     @synchronized
     def _my_api(self):
         if self._api_client is None:

-            self._api_client = ThreadSafeApiCache(self._config)
+            self._api_client = ThreadSafeApiCache(self._config, version='v1')

             if self._keep_client is None:
                 self._keep_client = self._api_client.keep
         return self._api_client
             if self._keep_client is None:
                 self._keep_client = self._api_client.keep
         return self._api_client


@@ -1424,7 +1431,12 @@ class Collection(RichCollectionBase):
             copies = (self.replication_desired or
                       self._my_api()._rootDesc.get('defaultCollectionReplication',
                                                    2))
             copies = (self.replication_desired or
                       self._my_api()._rootDesc.get('defaultCollectionReplication',
                                                    2))

-            self._block_manager = _BlockManager(self._my_keep(), copies=copies, put_threads=self.put_threads, num_retries=self.num_retries, storage_classes_func=self.storage_classes_desired)
+            self._block_manager = _BlockManager(self._my_keep(),
+                                                copies=copies,
+                                                put_threads=self.put_threads,
+                                                num_retries=self.num_retries,
+                                                storage_classes_func=self.storage_classes_desired,
+                                                get_threads=self.get_threads,)

         return self._block_manager
 
     def _remember_api_response(self, response):
         return self._block_manager
 
     def _remember_api_response(self, response):


@@ -1546,7 +1558,8 @@ class Collection(RichCollectionBase):
              storage_classes=None,
              trash_at=None,
              merge=True,
              storage_classes=None,
              trash_at=None,
              merge=True,

-             num_retries=None):
+             num_retries=None,
+             preserve_version=False):

         """Save collection to an existing collection record.
 
         Commit pending buffer blocks to Keep, merge with remote record (if
         """Save collection to an existing collection record.
 
         Commit pending buffer blocks to Keep, merge with remote record (if


@@ -1576,6 +1589,13 @@ class Collection(RichCollectionBase):
         :num_retries:
           Retry count on API calls (if None,  use the collection default)
 
         :num_retries:
           Retry count on API calls (if None,  use the collection default)
 

+        :preserve_version:
+          If True, indicate that the collection content being saved right now
+          should be preserved in a version snapshot if the collection record is
+          updated in the future. Requires that the API server has
+          Collections.CollectionVersioning enabled, if not, setting this will
+          raise an exception.
+

         """
         if properties and type(properties) is not dict:
             raise errors.ArgumentError("properties must be dictionary type.")
         """
         if properties and type(properties) is not dict:
             raise errors.ArgumentError("properties must be dictionary type.")


@@ -1588,6 +1608,9 @@ class Collection(RichCollectionBase):
         if trash_at and type(trash_at) is not datetime.datetime:
             raise errors.ArgumentError("trash_at must be datetime type.")
 
         if trash_at and type(trash_at) is not datetime.datetime:
             raise errors.ArgumentError("trash_at must be datetime type.")
 

+        if preserve_version and not self._my_api().config()['Collections'].get('CollectionVersioning', False):
+            raise errors.ArgumentError("preserve_version is not supported when CollectionVersioning is not enabled.")
+

         body={}
         if properties:
             body["properties"] = properties
         body={}
         if properties:
             body["properties"] = properties


@@ -1596,6 +1619,8 @@ class Collection(RichCollectionBase):
         if trash_at:
             t = trash_at.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
             body["trash_at"] = t
         if trash_at:
             t = trash_at.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
             body["trash_at"] = t

+        if preserve_version:
+            body["preserve_version"] = preserve_version

 
         if not self.committed():
             if self._has_remote_blocks:
 
         if not self.committed():
             if self._has_remote_blocks:


@@ -1641,7 +1666,8 @@ class Collection(RichCollectionBase):
                  storage_classes=None,
                  trash_at=None,
                  ensure_unique_name=False,
                  storage_classes=None,
                  trash_at=None,
                  ensure_unique_name=False,

-                 num_retries=None):
+                 num_retries=None,
+                 preserve_version=False):

         """Save collection to a new collection record.
 
         Commit pending buffer blocks to Keep and, when create_collection_record
         """Save collection to a new collection record.
 
         Commit pending buffer blocks to Keep and, when create_collection_record


@@ -1680,6 +1706,13 @@ class Collection(RichCollectionBase):
         :num_retries:
           Retry count on API calls (if None,  use the collection default)
 
         :num_retries:
           Retry count on API calls (if None,  use the collection default)
 

+        :preserve_version:
+          If True, indicate that the collection content being saved right now
+          should be preserved in a version snapshot if the collection record is
+          updated in the future. Requires that the API server has
+          Collections.CollectionVersioning enabled, if not, setting this will
+          raise an exception.
+

         """
         if properties and type(properties) is not dict:
             raise errors.ArgumentError("properties must be dictionary type.")
         """
         if properties and type(properties) is not dict:
             raise errors.ArgumentError("properties must be dictionary type.")


@@ -1690,6 +1723,9 @@ class Collection(RichCollectionBase):
         if trash_at and type(trash_at) is not datetime.datetime:
             raise errors.ArgumentError("trash_at must be datetime type.")
 
         if trash_at and type(trash_at) is not datetime.datetime:
             raise errors.ArgumentError("trash_at must be datetime type.")
 

+        if preserve_version and not self._my_api().config()['Collections'].get('CollectionVersioning', False):
+            raise errors.ArgumentError("preserve_version is not supported when CollectionVersioning is not enabled.")
+

         if self._has_remote_blocks:
             # Copy any remote blocks to the local cluster.
             self._copy_remote_blocks(remote_blocks={})
         if self._has_remote_blocks:
             # Copy any remote blocks to the local cluster.
             self._copy_remote_blocks(remote_blocks={})


@@ -1718,6 +1754,8 @@ class Collection(RichCollectionBase):
             if trash_at:
                 t = trash_at.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
                 body["trash_at"] = t
             if trash_at:
                 t = trash_at.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
                 body["trash_at"] = t

+            if preserve_version:
+                body["preserve_version"] = preserve_version

 
             self._remember_api_response(self._my_api().collections().create(ensure_unique_name=ensure_unique_name, body=body).execute(num_retries=num_retries))
             text = self._api_response["manifest_text"]
 
             self._remember_api_response(self._my_api().collections().create(ensure_unique_name=ensure_unique_name, body=body).execute(num_retries=num_retries))
             text = self._api_response["manifest_text"]