From 0e0f69c6164339f4d0babb1d9d5a68fae24c01d5 Mon Sep 17 00:00:00 2001 From: Tom Clegg Date: Tue, 17 Sep 2019 13:25:45 -0400 Subject: [PATCH] 13647: Update config keywords in docs, comments, and error messages. Arvados-DCO-1.1-Signed-off-by: Tom Clegg --- doc/install/install-keepstore.html.textile.liquid | 6 +++--- services/keepstore/azure_blob_volume.go | 4 ++-- services/keepstore/command.go | 4 ++-- services/keepstore/handler_test.go | 10 +++++----- services/keepstore/s3_volume.go | 8 ++++---- services/keepstore/s3_volume_test.go | 4 ++-- services/keepstore/unix_volume.go | 4 ++-- services/keepstore/volume.go | 11 +++++------ services/keepstore/volume_generic_test.go | 9 +++++---- 9 files changed, 30 insertions(+), 30 deletions(-) diff --git a/doc/install/install-keepstore.html.textile.liquid b/doc/install/install-keepstore.html.textile.liquid index 157ac589bd..a16de51627 100644 --- a/doc/install/install-keepstore.html.textile.liquid +++ b/doc/install/install-keepstore.html.textile.liquid @@ -83,11 +83,11 @@ h3. Notes on storage management On its own, a keepstore server never deletes data. The "keep-balance":install-keep-balance.html service determines which blocks are candidates for deletion and instructs the keepstore to move those blocks to the trash. -When a block is newly written, it is protected from deletion for the duration in @BlobSignatureTTL@. During this time, it cannot be trashed. +When a block is newly written, it is protected from deletion for the duration in @BlobSigningTTL@. During this time, it cannot be trashed or deleted. -If keep-balance instructs keepstore to trash a block which is older than @BlobSignatureTTL@, and @EnableDelete@ is true, the block will be moved to "trash". A block which is in the trash is no longer accessible by read requests, but has not yet been permanently deleted. Blocks which are in the trash may be recovered using the "untrash" API endpoint. Blocks are permanently deleted after they have been in the trash for the duration in @TrashLifetime@. +If keep-balance instructs keepstore to trash a block which is older than @BlobSigningTTL@, and @BlobTrashLifetime@ is non-zero, the block will be moved to "trash". A block which is in the trash is no longer accessible by read requests, but has not yet been permanently deleted. Blocks which are in the trash may be recovered using the "untrash" API endpoint. Blocks are permanently deleted after they have been in the trash for the duration in @BlobTrashLifetime@. -Keep-balance is also responsible for balancing the distribution of blocks across keepstore servers by asking servers to pull blocks from other servers (as determined by their "storage class":{{site.baseurl}}/admin/storage-classes.html and "rendezvous hashing order":{{site.baseurl}}/api/storage.html). Pulling a block makes a copy. If a block is overreplicated (i.e. there are excess copies) after pulling, it will be subsequently trashed on the original server. +Keep-balance is also responsible for balancing the distribution of blocks across keepstore servers by asking servers to pull blocks from other servers (as determined by their "storage class":{{site.baseurl}}/admin/storage-classes.html and "rendezvous hashing order":{{site.baseurl}}/api/storage.html). Pulling a block makes a copy. If a block is overreplicated (i.e. there are excess copies) after pulling, it will be subsequently trashed and deleted on the original server, subject to @BlobTrash@ and @BlobTrashLifetime@ settings. h3. Configure storage volumes diff --git a/services/keepstore/azure_blob_volume.go b/services/keepstore/azure_blob_volume.go index b52b706b26..6806c8c6ce 100644 --- a/services/keepstore/azure_blob_volume.go +++ b/services/keepstore/azure_blob_volume.go @@ -497,7 +497,7 @@ func (v *AzureBlobVolume) Trash(loc string) error { return nil } - // If TrashLifetime == 0, just delete it + // If BlobTrashLifetime == 0, just delete it if v.cluster.Collections.BlobTrashLifetime == 0 { return v.container.DeleteBlob(loc, &storage.DeleteBlobOptions{ IfMatch: props.Etag, @@ -567,7 +567,7 @@ func (v *AzureBlobVolume) isKeepBlock(s string) bool { return keepBlockRegexp.MatchString(s) } -// EmptyTrash looks for trashed blocks that exceeded TrashLifetime +// EmptyTrash looks for trashed blocks that exceeded BlobTrashLifetime // and deletes them from the volume. func (v *AzureBlobVolume) EmptyTrash() { if v.cluster.Collections.BlobDeleteConcurrency < 1 { diff --git a/services/keepstore/command.go b/services/keepstore/command.go index 51031d3607..c589e639f5 100644 --- a/services/keepstore/command.go +++ b/services/keepstore/command.go @@ -149,13 +149,13 @@ func (h *handler) setup(ctx context.Context, cluster *arvados.Cluster, token str h.Cluster = cluster h.Logger = ctxlog.FromContext(ctx) if h.Cluster.API.MaxKeepBlobBuffers <= 0 { - return fmt.Errorf("MaxBuffers must be greater than zero") + return fmt.Errorf("API.MaxKeepBlobBuffers must be greater than zero") } bufs = newBufferPool(h.Logger, h.Cluster.API.MaxKeepBlobBuffers, BlockSize) if h.Cluster.API.MaxConcurrentRequests < 1 { h.Cluster.API.MaxConcurrentRequests = h.Cluster.API.MaxKeepBlobBuffers * 2 - h.Logger.Warnf("MaxRequests <1 or not specified; defaulting to MaxKeepBlobBuffers * 2 == %d", h.Cluster.API.MaxConcurrentRequests) + h.Logger.Warnf("API.MaxConcurrentRequests <1 or not specified; defaulting to MaxKeepBlobBuffers * 2 == %d", h.Cluster.API.MaxConcurrentRequests) } if h.Cluster.Collections.BlobSigningKey != "" { diff --git a/services/keepstore/handler_test.go b/services/keepstore/handler_test.go index 299460297f..9d69b9fa47 100644 --- a/services/keepstore/handler_test.go +++ b/services/keepstore/handler_test.go @@ -327,7 +327,7 @@ func (s *HandlerSuite) TestPutAndDeleteSkipReadonlyVolumes(c *check.C) { // - authenticated /index/prefix request | superuser // // The only /index requests that should succeed are those issued by the -// superuser. They should pass regardless of the value of RequireSignatures. +// superuser. They should pass regardless of the value of BlobSigning. // func (s *HandlerSuite) TestIndexHandler(c *check.C) { c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil) @@ -393,7 +393,7 @@ func (s *HandlerSuite) TestIndexHandler(c *check.C) { // => UnauthorizedError response := IssueRequest(s.handler, unauthenticatedReq) ExpectStatusCode(c, - "RequireSignatures on, unauthenticated request", + "permissions on, unauthenticated request", UnauthorizedError.HTTPCode, response) @@ -520,7 +520,7 @@ func (s *HandlerSuite) TestDeleteHandler(c *check.C) { vols := s.handler.volmgr.AllWritable() vols[0].Put(context.Background(), TestHash, TestBlock) - // Explicitly set the BlobSignatureTTL to 0 for these + // Explicitly set the BlobSigningTTL to 0 for these // tests, to ensure the MockVolume deletes the blocks // even though they have just been created. s.cluster.Collections.BlobSigningTTL = arvados.Duration(0) @@ -611,7 +611,7 @@ func (s *HandlerSuite) TestDeleteHandler(c *check.C) { c.Error("superuserExistingBlockReq: block not deleted") } - // A DELETE request on a block newer than BlobSignatureTTL + // A DELETE request on a block newer than BlobSigningTTL // should return success but leave the block on the volume. vols[0].Put(context.Background(), TestHash, TestBlock) s.cluster.Collections.BlobSigningTTL = arvados.Duration(time.Hour) @@ -936,7 +936,7 @@ func (s *HandlerSuite) TestPutNeedsOnlyOneBuffer(c *check.C) { select { case <-ok: case <-time.After(time.Second): - c.Fatal("PUT deadlocks with MaxBuffers==1") + c.Fatal("PUT deadlocks with MaxKeepBlobBuffers==1") } } diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go index 22a38e2085..9fc87045e0 100644 --- a/services/keepstore/s3_volume.go +++ b/services/keepstore/s3_volume.go @@ -113,7 +113,7 @@ const ( var ( // ErrS3TrashDisabled is returned by Trash if that operation // is impossible with the current config. - ErrS3TrashDisabled = fmt.Errorf("trash function is disabled because -trash-lifetime=0 and -s3-unsafe-delete=false") + ErrS3TrashDisabled = fmt.Errorf("trash function is disabled because Collections.BlobTrashLifetime=0 and DriverParameters.UnsafeDelete=false") s3ACL = s3.Private @@ -663,7 +663,7 @@ func (v *S3Volume) translateError(err error) error { return err } -// EmptyTrash looks for trashed blocks that exceeded TrashLifetime +// EmptyTrash looks for trashed blocks that exceeded BlobTrashLifetime // and deletes them from the volume. func (v *S3Volume) EmptyTrash() { if v.cluster.Collections.BlobDeleteConcurrency < 1 { @@ -712,8 +712,8 @@ func (v *S3Volume) EmptyTrash() { // the raceWindow that starts if we // delete trash/X now. // - // Note this means (TrashCheckInterval - // < BlobSignatureTTL - raceWindow) is + // Note this means (TrashSweepInterval + // < BlobSigningTTL - raceWindow) is // necessary to avoid starvation. log.Printf("notice: %s: EmptyTrash: detected old race for %q, calling fixRace + Touch", v, loc) v.fixRace(loc) diff --git a/services/keepstore/s3_volume_test.go b/services/keepstore/s3_volume_test.go index b8c4458a5b..5c639d6295 100644 --- a/services/keepstore/s3_volume_test.go +++ b/services/keepstore/s3_volume_test.go @@ -316,12 +316,12 @@ func (s *StubbedS3Suite) TestBackendStates(c *check.C) { false, false, false, true, false, false, }, { - "Erroneously trashed during a race, detected before TrashLifetime", + "Erroneously trashed during a race, detected before BlobTrashLifetime", none, t0.Add(-30 * time.Minute), t0.Add(-29 * time.Minute), true, false, true, true, true, false, }, { - "Erroneously trashed during a race, rescue during EmptyTrash despite reaching TrashLifetime", + "Erroneously trashed during a race, rescue during EmptyTrash despite reaching BlobTrashLifetime", none, t0.Add(-90 * time.Minute), t0.Add(-89 * time.Minute), true, false, true, true, true, false, }, diff --git a/services/keepstore/unix_volume.go b/services/keepstore/unix_volume.go index 918555c2b1..6504f9c16b 100644 --- a/services/keepstore/unix_volume.go +++ b/services/keepstore/unix_volume.go @@ -418,9 +418,9 @@ func (v *UnixVolume) IndexTo(prefix string, w io.Writer) error { } // Trash trashes the block data from the unix storage -// If TrashLifetime == 0, the block is deleted +// If BlobTrashLifetime == 0, the block is deleted // Else, the block is renamed as path/{loc}.trash.{deadline}, -// where deadline = now + TrashLifetime +// where deadline = now + BlobTrashLifetime func (v *UnixVolume) Trash(loc string) error { // Touch() must be called before calling Write() on a block. Touch() // also uses lockfile(). This avoids a race condition between Write() diff --git a/services/keepstore/volume.go b/services/keepstore/volume.go index 8614355022..1dea6194d5 100644 --- a/services/keepstore/volume.go +++ b/services/keepstore/volume.go @@ -170,12 +170,12 @@ type Volume interface { // Trash moves the block data from the underlying storage // device to trash area. The block then stays in trash for - // -trash-lifetime interval before it is actually deleted. + // BlobTrashLifetime before it is actually deleted. // // loc is as described in Get. // // If the timestamp for the given locator is newer than - // BlobSignatureTTL, Trash must not trash the data. + // BlobSigningTTL, Trash must not trash the data. // // If a Trash operation overlaps with any Touch or Put // operations on the same locator, the implementation must @@ -196,8 +196,7 @@ type Volume interface { // reliably or fail outright. // // Corollary: A successful Touch or Put guarantees a block - // will not be trashed for at least BlobSignatureTTL - // seconds. + // will not be trashed for at least BlobSigningTTL seconds. Trash(loc string) error // Untrash moves block from trash back into store @@ -216,8 +215,8 @@ type Volume interface { // secrets. String() string - // EmptyTrash looks for trashed blocks that exceeded TrashLifetime - // and deletes them from the volume. + // EmptyTrash looks for trashed blocks that exceeded + // BlobTrashLifetime and deletes them from the volume. EmptyTrash() // Return a globally unique ID of the underlying storage diff --git a/services/keepstore/volume_generic_test.go b/services/keepstore/volume_generic_test.go index 683521c01a..cbb0eb3a3a 100644 --- a/services/keepstore/volume_generic_test.go +++ b/services/keepstore/volume_generic_test.go @@ -487,8 +487,8 @@ func (s *genericVolumeSuite) testDeleteNewBlock(t TB, factory TestableVolumeFact } // Calling Delete() for a block with a timestamp older than -// BlobSignatureTTL seconds in the past should delete the data. -// Test is intended for only writable volumes +// BlobSigningTTL seconds in the past should delete the data. Test is +// intended for only writable volumes func (s *genericVolumeSuite) testDeleteOldBlock(t TB, factory TestableVolumeFactory) { s.setup(t) s.cluster.Collections.BlobSigningTTL.Set("5m") @@ -834,7 +834,7 @@ func (s *genericVolumeSuite) testPutFullBlock(t TB, factory TestableVolumeFactor } } -// With TrashLifetime != 0, perform: +// With BlobTrashLifetime != 0, perform: // Trash an old block - which either raises ErrNotImplemented or succeeds // Untrash - which either raises ErrNotImplemented or succeeds // Get - which must succeed @@ -940,7 +940,8 @@ func (s *genericVolumeSuite) testTrashEmptyTrashUntrash(t TB, factory TestableVo err = v.Trash(TestHash) if err == MethodDisabledError || err == ErrNotImplemented { // Skip the trash tests for read-only volumes, and - // volume types that don't support TrashLifetime>0. + // volume types that don't support + // BlobTrashLifetime>0. return } -- 2.39.5