From f7a870dde952902343656e367b26ad550fe3d63c Mon Sep 17 00:00:00 2001 From: Tom Clegg Date: Mon, 27 Nov 2023 11:48:10 -0500 Subject: [PATCH] Merge branch '21126-trash-when-ro' closes #21126 Arvados-DCO-1.1-Signed-off-by: Tom Clegg --- doc/Gemfile.lock | 2 +- doc/_config.yml | 1 + doc/admin/keep-balance.html.textile.liquid | 6 +-- .../keep-faster-gc-s3.html.textile.liquid | 41 +++++++++++++++++++ .../install-keep-balance.html.textile.liquid | 2 +- 5 files changed, 46 insertions(+), 6 deletions(-) create mode 100644 doc/admin/keep-faster-gc-s3.html.textile.liquid diff --git a/doc/Gemfile.lock b/doc/Gemfile.lock index 420e13146f..a6c85319c8 100644 --- a/doc/Gemfile.lock +++ b/doc/Gemfile.lock @@ -29,4 +29,4 @@ DEPENDENCIES zenweb BUNDLED WITH - 2.1.4 + 2.2.19 diff --git a/doc/_config.yml b/doc/_config.yml index cbb082aefb..04ff5caa9b 100644 --- a/doc/_config.yml +++ b/doc/_config.yml @@ -195,6 +195,7 @@ navbar: - admin/storage-classes.html.textile.liquid - admin/keep-recovering-data.html.textile.liquid - admin/keep-measuring-deduplication.html.textile.liquid + - admin/keep-faster-gc-s3.html.textile.liquid - Cloud: - admin/spot-instances.html.textile.liquid - admin/cloudtest.html.textile.liquid diff --git a/doc/admin/keep-balance.html.textile.liquid b/doc/admin/keep-balance.html.textile.liquid index 2785930de8..4d18307cf7 100644 --- a/doc/admin/keep-balance.html.textile.liquid +++ b/doc/admin/keep-balance.html.textile.liquid @@ -30,14 +30,12 @@ The @Collections.BalancePeriod@ value in @/etc/arvados/config.yml@ determines th Keep-balance can also be run with the @-once@ flag to do a single scan/balance operation and then exit. The exit code will be zero if the operation was successful. -h3. Committing - -Keep-balance computes and reports changes but does not implement them by sending pull and trash lists to the Keep services unless the @-commit-pull@ and @-commit-trash@ flags are used. - h3. Additional configuration For configuring resource usage tuning and lost block reporting, please see the @Collections.BlobMissingReport@, @Collections.BalanceCollectionBatch@, @Collections.BalanceCollectionBuffers@ option in the "default config.yml file":{{site.baseurl}}/admin/config.html. +The @Collections.BalancePullLimit@ and @Collections.BalanceTrashLimit@ configuration entries determine the maximum number of pull and trash operations keep-balance will attempt to apply on each keepstore server. If both values are zero, keep-balance will operate in "dry run" mode, where all changes are computed but none are committed. + h3. Limitations Keep-balance does not attempt to discover whether committed pull and trash requests ever get carried out -- only that they are accepted by the Keep services. If some services are full, new copies of under-replicated blocks might never get made, only repeatedly requested. diff --git a/doc/admin/keep-faster-gc-s3.html.textile.liquid b/doc/admin/keep-faster-gc-s3.html.textile.liquid new file mode 100644 index 0000000000..2230ac2491 --- /dev/null +++ b/doc/admin/keep-faster-gc-s3.html.textile.liquid @@ -0,0 +1,41 @@ +--- +layout: default +navsection: admin +title: "Faster garbage collection in S3" +... + +{% comment %} +Copyright (C) The Arvados Authors. All rights reserved. + +SPDX-License-Identifier: CC-BY-SA-3.0 +{% endcomment %} + +When there is a large number of unneeded blocks stored in an S3 bucket, particularly when using @PrefixLength: 0@, the speed of garbage collection can be severely limited by AWS API rate limits and Arvados's multi-step trash/delete process. + +The multi-step trash/delete process can be short-circuited by setting @BlobTrashLifetime@ to zero and enabling @UnsafeDelete@ on S3-backed volumes. However, on an actively used cluster such a configuration *can result in data loss* in the rare case where a given block is trashed and then rewritten soon afterward, and S3 processes the write and delete requests in the opposite order. + +The following steps can be used to temporarily disable writes on an S3 bucket to enable faster garbage collection without data loss or service interruption. Note that garbage collection on other S3 volumes will be temporarily disabled during this procedure. +# Create a new S3 bucket and configure it as an additional volume (this step may be skipped if the configuration already has enough writable volumes that clients will still be able to write blocks while the target volume is read-only). We recommend using @PrefixLength: 3@ for the new volume because this results in a much higher rate limit for I/O and garbage collection operations compared to the default @PrefixLength: 0@. If the target volume configuration specifies @StorageClasses@, use the same values for the new volume. +# Shut down the @keep-balance@ service. +# Update your configuration as follows: 
+  Collections:
+    BlobTrashLifetime: 0
+    BalancePullLimit: 0
+  [...]
+  Volumes:
+    target-volume-uuid:
+      ReadOnly: true
+      AllowTrashWhileReadOnly: true
+      DriverParameters:
+        UnsafeDelete: true
+
 Note that @BlobTrashLifetime: 0@ instructs keepstore to delete unneeded blocks outright (bypassing the recoverable trash phase); however, in this mode it will normally not trash any blocks at all on an S3 volume due to the safety issue mentioned above, unless the volume is configured with @UnsafeDelete: true@. +# Restart all @keepstore@ services with the updated configuration. +# Start the @keep-balance@ service. +# Objects will be deleted immediately instead of being first copied to trash on the S3 volume, which should significantly speed up cleanup of trashed objects. Monitor progress by watching @keep-balance@ logs and metrics. When garbage collection is complete, keep-balance logs will show an empty changeset: 
zzzzz-bi6l4-0123456789abcdef (keep0.zzzzz.arvadosapi.com:25107, disk): ChangeSet{Pulls:0, Trashes:0}
 +# Remove the @UnsafeDelete@ configuration entry on the target volume. +# Remove the @BlobTrashLifetime@ configuration entry (or restore it to its previous value). +# If the target volume has @PrefixLength: 0@ and the new volume has @PrefixLength: 3@, skip the next two steps: new data will be stored on the new volume, some existing data will be moved automatically to other volumes, and some will be left on the target volume as long as it's needed. +# If you want to resume writing new data to the target volume, revert to @ReadOnly: false@ and @AllowTrashWhileReadOnly: false@ on the target volume. +# If you want to stop writing new data to the newly created volume, set @ReadOnly: true@ and @AllowTrashWhileReadOnly: true@ on the new volume. +# Remove the @BalancePullLimit@ configuration entry (or restore its previous value), and restart @keep-balance@. +# Restart all @keepstore@ services with the updated configuration. diff --git a/doc/install/install-keep-balance.html.textile.liquid b/doc/install/install-keep-balance.html.textile.liquid index bb4ae7b3d8..05d27b7cb4 100644 --- a/doc/install/install-keep-balance.html.textile.liquid +++ b/doc/install/install-keep-balance.html.textile.liquid @@ -24,7 +24,7 @@ Keep-balance can be installed anywhere with network access to Keep services, arv {% include 'notebox_begin' %} -If you are installing keep-balance on an existing system with valuable data, you can run keep-balance in "dry run" mode first and review its logs as a precaution. To do this, edit your keep-balance startup script to use the flags @-commit-pulls=false -commit-trash=false -commit-confirmed-fields=false@. +If you are installing keep-balance on an existing system with valuable data, you can run keep-balance in "dry run" mode first and review its logs as a precaution. To do this, set the @Collections.BalancePullLimit@ and @Collections.BalanceTrashLimit@ configuration entries to zero. {% include 'notebox_end' %} -- 2.30.2