From: Lucas Di Pentima Date: Tue, 16 Nov 2021 18:48:47 +0000 (-0300) Subject: Merge branch '17635-pysdk-collection-preserve-version' into main. Closes #17635 X-Git-Tag: 2.4.0~168 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/3849ee94bbe65ef79df8f50c87b5445a5b1d4877?hp=a02b821d78d93bd814d8bf2b8b532b8940e93ecf Merge branch '17635-pysdk-collection-preserve-version' into main. Closes #17635 Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima --- diff --git a/doc/user/topics/collection-versioning.html.textile.liquid b/doc/user/topics/collection-versioning.html.textile.liquid index 9a32de0d0b..d6a3bb4c10 100644 --- a/doc/user/topics/collection-versioning.html.textile.liquid +++ b/doc/user/topics/collection-versioning.html.textile.liquid @@ -18,7 +18,7 @@ A version will be saved when one of the following conditions is true: One is by "configuring (system-wide) the collection's idle time":{{site.baseurl}}/admin/collection-versioning.html. This idle time is checked against the @modified_at@ attribute so that the version is saved when one or more of the previously enumerated attributes get updated and the @modified_at@ is at least at the configured idle time in the past. This way, a frequently updated collection won't create lots of version records that may not be useful. -The other way to trigger a version save, is by setting @preserve_version@ to @true@ on the current version collection record: this ensures that the current state will be preserved as a version the next time it gets updated. +The other way to trigger a version save, is by setting @preserve_version@ to @true@ on the current version collection record: this ensures that the current state will be preserved as a version the next time it gets updated. This includes either creating a new collection or updating a preexisting one. In the case of using @preserve_version = true@ on a collection's create call, the new record state will be preserved as a snapshot on the next update. h3. Collection's past versions behavior & limitations diff --git a/lib/config/export.go b/lib/config/export.go index d224924a24..4c4e341f5a 100644 --- a/lib/config/export.go +++ b/lib/config/export.go @@ -96,7 +96,7 @@ var whitelist = map[string]bool{ "Collections.BlobTrashCheckInterval": false, "Collections.BlobTrashConcurrency": false, "Collections.BlobTrashLifetime": false, - "Collections.CollectionVersioning": false, + "Collections.CollectionVersioning": true, "Collections.DefaultReplication": true, "Collections.DefaultTrashLifetime": true, "Collections.ForwardSlashNameSubstitution": true, diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py index d03265ca44..55be40fa04 100644 --- a/sdk/python/arvados/collection.py +++ b/sdk/python/arvados/collection.py @@ -1546,7 +1546,8 @@ class Collection(RichCollectionBase): storage_classes=None, trash_at=None, merge=True, - num_retries=None): + num_retries=None, + preserve_version=False): """Save collection to an existing collection record. Commit pending buffer blocks to Keep, merge with remote record (if @@ -1576,6 +1577,13 @@ class Collection(RichCollectionBase): :num_retries: Retry count on API calls (if None, use the collection default) + :preserve_version: + If True, indicate that the collection content being saved right now + should be preserved in a version snapshot if the collection record is + updated in the future. Requires that the API server has + Collections.CollectionVersioning enabled, if not, setting this will + raise an exception. + """ if properties and type(properties) is not dict: raise errors.ArgumentError("properties must be dictionary type.") @@ -1588,6 +1596,9 @@ class Collection(RichCollectionBase): if trash_at and type(trash_at) is not datetime.datetime: raise errors.ArgumentError("trash_at must be datetime type.") + if preserve_version and not self._my_api().config()['Collections'].get('CollectionVersioning', False): + raise errors.ArgumentError("preserve_version is not supported when CollectionVersioning is not enabled.") + body={} if properties: body["properties"] = properties @@ -1596,6 +1607,8 @@ class Collection(RichCollectionBase): if trash_at: t = trash_at.strftime("%Y-%m-%dT%H:%M:%S.%fZ") body["trash_at"] = t + if preserve_version: + body["preserve_version"] = preserve_version if not self.committed(): if self._has_remote_blocks: @@ -1641,7 +1654,8 @@ class Collection(RichCollectionBase): storage_classes=None, trash_at=None, ensure_unique_name=False, - num_retries=None): + num_retries=None, + preserve_version=False): """Save collection to a new collection record. Commit pending buffer blocks to Keep and, when create_collection_record @@ -1680,6 +1694,13 @@ class Collection(RichCollectionBase): :num_retries: Retry count on API calls (if None, use the collection default) + :preserve_version: + If True, indicate that the collection content being saved right now + should be preserved in a version snapshot if the collection record is + updated in the future. Requires that the API server has + Collections.CollectionVersioning enabled, if not, setting this will + raise an exception. + """ if properties and type(properties) is not dict: raise errors.ArgumentError("properties must be dictionary type.") @@ -1690,6 +1711,9 @@ class Collection(RichCollectionBase): if trash_at and type(trash_at) is not datetime.datetime: raise errors.ArgumentError("trash_at must be datetime type.") + if preserve_version and not self._my_api().config()['Collections'].get('CollectionVersioning', False): + raise errors.ArgumentError("preserve_version is not supported when CollectionVersioning is not enabled.") + if self._has_remote_blocks: # Copy any remote blocks to the local cluster. self._copy_remote_blocks(remote_blocks={}) @@ -1718,6 +1742,8 @@ class Collection(RichCollectionBase): if trash_at: t = trash_at.strftime("%Y-%m-%dT%H:%M:%S.%fZ") body["trash_at"] = t + if preserve_version: + body["preserve_version"] = preserve_version self._remember_api_response(self._my_api().collections().create(ensure_unique_name=ensure_unique_name, body=body).execute(num_retries=num_retries)) text = self._api_response["manifest_text"] diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py index 6d2643a967..f917832509 100644 --- a/sdk/python/tests/run_test_server.py +++ b/sdk/python/tests/run_test_server.py @@ -791,6 +791,7 @@ def setup_config(): "UserProfileNotificationAddress": "arvados@example.com", }, "Collections": { + "CollectionVersioning": True, "BlobSigningKey": "zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc", "TrustAllContent": False, "ForwardSlashNameSubstitution": "/", diff --git a/sdk/python/tests/test_collections.py b/sdk/python/tests/test_collections.py index f821ff952f..a43e0d40df 100644 --- a/sdk/python/tests/test_collections.py +++ b/sdk/python/tests/test_collections.py @@ -1360,6 +1360,25 @@ class NewCollectionTestCaseWithServersAndTokens(run_test_server.TestCaseWithServ class NewCollectionTestCaseWithServers(run_test_server.TestCaseWithServers): + def test_preserve_version_on_save(self): + c = Collection() + c.save_new(preserve_version=True) + coll_record = arvados.api().collections().get(uuid=c.manifest_locator()).execute() + self.assertEqual(coll_record['version'], 1) + self.assertEqual(coll_record['preserve_version'], True) + with c.open("foo.txt", "wb") as foo: + foo.write(b"foo") + c.save(preserve_version=True) + coll_record = arvados.api().collections().get(uuid=c.manifest_locator()).execute() + self.assertEqual(coll_record['version'], 2) + self.assertEqual(coll_record['preserve_version'], True) + with c.open("bar.txt", "wb") as foo: + foo.write(b"bar") + c.save(preserve_version=False) + coll_record = arvados.api().collections().get(uuid=c.manifest_locator()).execute() + self.assertEqual(coll_record['version'], 3) + self.assertEqual(coll_record['preserve_version'], False) + def test_get_manifest_text_only_committed(self): c = Collection() with c.open("count.txt", "wb") as f: