On its own, a keepstore server never deletes data. The "keep-balance":install-keep-balance.html service determines which blocks are candidates for deletion and instructs the keepstore to move those blocks to the trash.
-When a block is newly written, it is protected from deletion for the duration in @BlobSignatureTTL@. During this time, it cannot be trashed.
+When a block is newly written, it is protected from deletion for the duration in @BlobSigningTTL@. During this time, it cannot be trashed or deleted.
-If keep-balance instructs keepstore to trash a block which is older than @BlobSignatureTTL@, and @EnableDelete@ is true, the block will be moved to "trash". A block which is in the trash is no longer accessible by read requests, but has not yet been permanently deleted. Blocks which are in the trash may be recovered using the "untrash" API endpoint. Blocks are permanently deleted after they have been in the trash for the duration in @TrashLifetime@.
+If keep-balance instructs keepstore to trash a block which is older than @BlobSigningTTL@, and @BlobTrashLifetime@ is non-zero, the block will be moved to "trash". A block which is in the trash is no longer accessible by read requests, but has not yet been permanently deleted. Blocks which are in the trash may be recovered using the "untrash" API endpoint. Blocks are permanently deleted after they have been in the trash for the duration in @BlobTrashLifetime@.
-Keep-balance is also responsible for balancing the distribution of blocks across keepstore servers by asking servers to pull blocks from other servers (as determined by their "storage class":{{site.baseurl}}/admin/storage-classes.html and "rendezvous hashing order":{{site.baseurl}}/api/storage.html). Pulling a block makes a copy. If a block is overreplicated (i.e. there are excess copies) after pulling, it will be subsequently trashed on the original server.
+Keep-balance is also responsible for balancing the distribution of blocks across keepstore servers by asking servers to pull blocks from other servers (as determined by their "storage class":{{site.baseurl}}/admin/storage-classes.html and "rendezvous hashing order":{{site.baseurl}}/api/storage.html). Pulling a block makes a copy. If a block is overreplicated (i.e. there are excess copies) after pulling, it will be subsequently trashed and deleted on the original server, subject to @BlobTrash@ and @BlobTrashLifetime@ settings.
h3. Configure storage volumes
return nil
}
- // If TrashLifetime == 0, just delete it
+ // If BlobTrashLifetime == 0, just delete it
if v.cluster.Collections.BlobTrashLifetime == 0 {
return v.container.DeleteBlob(loc, &storage.DeleteBlobOptions{
IfMatch: props.Etag,
return keepBlockRegexp.MatchString(s)
}
-// EmptyTrash looks for trashed blocks that exceeded TrashLifetime
+// EmptyTrash looks for trashed blocks that exceeded BlobTrashLifetime
// and deletes them from the volume.
func (v *AzureBlobVolume) EmptyTrash() {
if v.cluster.Collections.BlobDeleteConcurrency < 1 {
h.Cluster = cluster
h.Logger = ctxlog.FromContext(ctx)
if h.Cluster.API.MaxKeepBlobBuffers <= 0 {
- return fmt.Errorf("MaxBuffers must be greater than zero")
+ return fmt.Errorf("API.MaxKeepBlobBuffers must be greater than zero")
}
bufs = newBufferPool(h.Logger, h.Cluster.API.MaxKeepBlobBuffers, BlockSize)
if h.Cluster.API.MaxConcurrentRequests < 1 {
h.Cluster.API.MaxConcurrentRequests = h.Cluster.API.MaxKeepBlobBuffers * 2
- h.Logger.Warnf("MaxRequests <1 or not specified; defaulting to MaxKeepBlobBuffers * 2 == %d", h.Cluster.API.MaxConcurrentRequests)
+ h.Logger.Warnf("API.MaxConcurrentRequests <1 or not specified; defaulting to MaxKeepBlobBuffers * 2 == %d", h.Cluster.API.MaxConcurrentRequests)
}
if h.Cluster.Collections.BlobSigningKey != "" {
// - authenticated /index/prefix request | superuser
//
// The only /index requests that should succeed are those issued by the
-// superuser. They should pass regardless of the value of RequireSignatures.
+// superuser. They should pass regardless of the value of BlobSigning.
//
func (s *HandlerSuite) TestIndexHandler(c *check.C) {
c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
// => UnauthorizedError
response := IssueRequest(s.handler, unauthenticatedReq)
ExpectStatusCode(c,
- "RequireSignatures on, unauthenticated request",
+ "permissions on, unauthenticated request",
UnauthorizedError.HTTPCode,
response)
vols := s.handler.volmgr.AllWritable()
vols[0].Put(context.Background(), TestHash, TestBlock)
- // Explicitly set the BlobSignatureTTL to 0 for these
+ // Explicitly set the BlobSigningTTL to 0 for these
// tests, to ensure the MockVolume deletes the blocks
// even though they have just been created.
s.cluster.Collections.BlobSigningTTL = arvados.Duration(0)
c.Error("superuserExistingBlockReq: block not deleted")
}
- // A DELETE request on a block newer than BlobSignatureTTL
+ // A DELETE request on a block newer than BlobSigningTTL
// should return success but leave the block on the volume.
vols[0].Put(context.Background(), TestHash, TestBlock)
s.cluster.Collections.BlobSigningTTL = arvados.Duration(time.Hour)
select {
case <-ok:
case <-time.After(time.Second):
- c.Fatal("PUT deadlocks with MaxBuffers==1")
+ c.Fatal("PUT deadlocks with MaxKeepBlobBuffers==1")
}
}
var (
// ErrS3TrashDisabled is returned by Trash if that operation
// is impossible with the current config.
- ErrS3TrashDisabled = fmt.Errorf("trash function is disabled because -trash-lifetime=0 and -s3-unsafe-delete=false")
+ ErrS3TrashDisabled = fmt.Errorf("trash function is disabled because Collections.BlobTrashLifetime=0 and DriverParameters.UnsafeDelete=false")
s3ACL = s3.Private
return err
}
-// EmptyTrash looks for trashed blocks that exceeded TrashLifetime
+// EmptyTrash looks for trashed blocks that exceeded BlobTrashLifetime
// and deletes them from the volume.
func (v *S3Volume) EmptyTrash() {
if v.cluster.Collections.BlobDeleteConcurrency < 1 {
// the raceWindow that starts if we
// delete trash/X now.
//
- // Note this means (TrashCheckInterval
- // < BlobSignatureTTL - raceWindow) is
+ // Note this means (TrashSweepInterval
+ // < BlobSigningTTL - raceWindow) is
// necessary to avoid starvation.
log.Printf("notice: %s: EmptyTrash: detected old race for %q, calling fixRace + Touch", v, loc)
v.fixRace(loc)
false, false, false, true, false, false,
},
{
- "Erroneously trashed during a race, detected before TrashLifetime",
+ "Erroneously trashed during a race, detected before BlobTrashLifetime",
none, t0.Add(-30 * time.Minute), t0.Add(-29 * time.Minute),
true, false, true, true, true, false,
},
{
- "Erroneously trashed during a race, rescue during EmptyTrash despite reaching TrashLifetime",
+ "Erroneously trashed during a race, rescue during EmptyTrash despite reaching BlobTrashLifetime",
none, t0.Add(-90 * time.Minute), t0.Add(-89 * time.Minute),
true, false, true, true, true, false,
},
}
// Trash trashes the block data from the unix storage
-// If TrashLifetime == 0, the block is deleted
+// If BlobTrashLifetime == 0, the block is deleted
// Else, the block is renamed as path/{loc}.trash.{deadline},
-// where deadline = now + TrashLifetime
+// where deadline = now + BlobTrashLifetime
func (v *UnixVolume) Trash(loc string) error {
// Touch() must be called before calling Write() on a block. Touch()
// also uses lockfile(). This avoids a race condition between Write()
// Trash moves the block data from the underlying storage
// device to trash area. The block then stays in trash for
- // -trash-lifetime interval before it is actually deleted.
+ // BlobTrashLifetime before it is actually deleted.
//
// loc is as described in Get.
//
// If the timestamp for the given locator is newer than
- // BlobSignatureTTL, Trash must not trash the data.
+ // BlobSigningTTL, Trash must not trash the data.
//
// If a Trash operation overlaps with any Touch or Put
// operations on the same locator, the implementation must
// reliably or fail outright.
//
// Corollary: A successful Touch or Put guarantees a block
- // will not be trashed for at least BlobSignatureTTL
- // seconds.
+ // will not be trashed for at least BlobSigningTTL seconds.
Trash(loc string) error
// Untrash moves block from trash back into store
// secrets.
String() string
- // EmptyTrash looks for trashed blocks that exceeded TrashLifetime
- // and deletes them from the volume.
+ // EmptyTrash looks for trashed blocks that exceeded
+ // BlobTrashLifetime and deletes them from the volume.
EmptyTrash()
// Return a globally unique ID of the underlying storage
}
// Calling Delete() for a block with a timestamp older than
-// BlobSignatureTTL seconds in the past should delete the data.
-// Test is intended for only writable volumes
+// BlobSigningTTL seconds in the past should delete the data. Test is
+// intended for only writable volumes
func (s *genericVolumeSuite) testDeleteOldBlock(t TB, factory TestableVolumeFactory) {
s.setup(t)
s.cluster.Collections.BlobSigningTTL.Set("5m")
}
}
-// With TrashLifetime != 0, perform:
+// With BlobTrashLifetime != 0, perform:
// Trash an old block - which either raises ErrNotImplemented or succeeds
// Untrash - which either raises ErrNotImplemented or succeeds
// Get - which must succeed
err = v.Trash(TestHash)
if err == MethodDisabledError || err == ErrNotImplemented {
// Skip the trash tests for read-only volumes, and
- // volume types that don't support TrashLifetime>0.
+ // volume types that don't support
+ // BlobTrashLifetime>0.
return
}
projects / arvados.git / commitdiff