// Turn on permission settings so we can generate signed locators.
enforcePermissions = true
PermissionSecret = []byte(knownKey)
- blob_signature_ttl = 300 * time.Second
+ blobSignatureTTL = 300 * time.Second
var (
unsignedLocator = "/" + TestHash
- validTimestamp = time.Now().Add(blob_signature_ttl)
+ validTimestamp = time.Now().Add(blobSignatureTTL)
expiredTimestamp = time.Now().Add(-time.Hour)
signedLocator = "/" + SignLocator(TestHash, knownToken, validTimestamp)
expiredLocator = "/" + SignLocator(TestHash, knownToken, expiredTimestamp)
// With a server key.
PermissionSecret = []byte(knownKey)
- blob_signature_ttl = 300 * time.Second
+ blobSignatureTTL = 300 * time.Second
// When a permission key is available, the locator returned
// from an authenticated PUT request will be signed.
func TestPutAndDeleteSkipReadonlyVolumes(t *testing.T) {
defer teardown()
- data_manager_token = "fake-data-manager-token"
+ dataManagerToken = "fake-data-manager-token"
vols := []*MockVolume{CreateMockVolume(), CreateMockVolume()}
vols[0].Readonly = true
KeepVM = MakeRRVolumeManager([]Volume{vols[0], vols[1]})
requestBody: TestBlock,
})
defer func(orig bool) {
- never_delete = orig
- }(never_delete)
- never_delete = false
+ neverDelete = orig
+ }(neverDelete)
+ neverDelete = false
IssueRequest(
&RequestTester{
method: "DELETE",
uri: "/" + TestHash,
requestBody: TestBlock,
- apiToken: data_manager_token,
+ apiToken: dataManagerToken,
})
type expect struct {
volnum int
vols[0].Put(TestHash+".meta", []byte("metadata"))
vols[1].Put(TestHash2+".meta", []byte("metadata"))
- data_manager_token = "DATA MANAGER TOKEN"
+ dataManagerToken = "DATA MANAGER TOKEN"
unauthenticatedReq := &RequestTester{
method: "GET",
superuserReq := &RequestTester{
method: "GET",
uri: "/index",
- apiToken: data_manager_token,
+ apiToken: dataManagerToken,
}
unauthPrefixReq := &RequestTester{
method: "GET",
superuserPrefixReq := &RequestTester{
method: "GET",
uri: "/index/" + TestHash[0:3],
- apiToken: data_manager_token,
+ apiToken: dataManagerToken,
}
// -------------------------------------------------------------
vols := KeepVM.AllWritable()
vols[0].Put(TestHash, TestBlock)
- // Explicitly set the blob_signature_ttl to 0 for these
+ // Explicitly set the blobSignatureTTL to 0 for these
// tests, to ensure the MockVolume deletes the blocks
// even though they have just been created.
- blob_signature_ttl = time.Duration(0)
+ blobSignatureTTL = time.Duration(0)
var userToken = "NOT DATA MANAGER TOKEN"
- data_manager_token = "DATA MANAGER TOKEN"
+ dataManagerToken = "DATA MANAGER TOKEN"
- never_delete = false
+ neverDelete = false
unauthReq := &RequestTester{
method: "DELETE",
superuserExistingBlockReq := &RequestTester{
method: "DELETE",
uri: "/" + TestHash,
- apiToken: data_manager_token,
+ apiToken: dataManagerToken,
}
superuserNonexistentBlockReq := &RequestTester{
method: "DELETE",
uri: "/" + TestHash2,
- apiToken: data_manager_token,
+ apiToken: dataManagerToken,
}
// Unauthenticated request returns PermissionError.
http.StatusNotFound,
response)
- // Authenticated admin request for existing block while never_delete is set.
- never_delete = true
+ // Authenticated admin request for existing block while neverDelete is set.
+ neverDelete = true
response = IssueRequest(superuserExistingBlockReq)
ExpectStatusCode(t,
"authenticated request, existing block, method disabled",
MethodDisabledError.HTTPCode,
response)
- never_delete = false
+ neverDelete = false
// Authenticated admin request for existing block.
response = IssueRequest(superuserExistingBlockReq)
t.Error("superuserExistingBlockReq: block not deleted")
}
- // A DELETE request on a block newer than blob_signature_ttl
+ // A DELETE request on a block newer than blobSignatureTTL
// should return success but leave the block on the volume.
vols[0].Put(TestHash, TestBlock)
- blob_signature_ttl = time.Hour
+ blobSignatureTTL = time.Hour
response = IssueRequest(superuserExistingBlockReq)
ExpectStatusCode(t,
defer teardown()
var userToken = "USER TOKEN"
- data_manager_token = "DATA MANAGER TOKEN"
+ dataManagerToken = "DATA MANAGER TOKEN"
pullq = NewWorkQueue()
},
{
"Valid pull request from the data manager",
- RequestTester{"/pull", data_manager_token, "PUT", goodJSON},
+ RequestTester{"/pull", dataManagerToken, "PUT", goodJSON},
http.StatusOK,
"Received 3 pull requests\n",
},
{
"Invalid pull request from the data manager",
- RequestTester{"/pull", data_manager_token, "PUT", badJSON},
+ RequestTester{"/pull", dataManagerToken, "PUT", badJSON},
http.StatusBadRequest,
"",
},
defer teardown()
var userToken = "USER TOKEN"
- data_manager_token = "DATA MANAGER TOKEN"
+ dataManagerToken = "DATA MANAGER TOKEN"
trashq = NewWorkQueue()
},
{
"Valid trash list from the data manager",
- RequestTester{"/trash", data_manager_token, "PUT", goodJSON},
+ RequestTester{"/trash", dataManagerToken, "PUT", goodJSON},
http.StatusOK,
"Received 3 trash requests\n",
},
{
"Invalid trash list from the data manager",
- RequestTester{"/trash", data_manager_token, "PUT", badJSON},
+ RequestTester{"/trash", dataManagerToken, "PUT", badJSON},
http.StatusBadRequest,
"",
},
returnHash := fmt.Sprintf("%s+%d", hash, req.ContentLength)
apiToken := GetApiToken(req)
if PermissionSecret != nil && apiToken != "" {
- expiry := time.Now().Add(blob_signature_ttl)
+ expiry := time.Now().Add(blobSignatureTTL)
returnHash = SignLocator(returnHash, apiToken, expiry)
}
resp.Write([]byte(returnHash + "\n"))
return
}
- if never_delete {
+ if neverDelete {
http.Error(resp, MethodDisabledError.Error(), MethodDisabledError.HTTPCode)
return
}
// IsDataManagerToken returns true if apiToken represents the data
// manager's token.
func IsDataManagerToken(apiToken string) bool {
- return data_manager_token != "" && apiToken == data_manager_token
+ return dataManagerToken != "" && apiToken == dataManagerToken
}
// Initialized by the -enforce-permissions flag.
var enforcePermissions bool
-// blob_signature_ttl is the time duration for which new permission
+// blobSignatureTTL is the time duration for which new permission
// signatures (returned by PUT requests) will be valid.
// Initialized by the -permission-ttl flag.
-var blob_signature_ttl time.Duration
+var blobSignatureTTL time.Duration
-// data_manager_token represents the API token used by the
+// dataManagerToken represents the API token used by the
// Data Manager, and is required on certain privileged operations.
// Initialized by the -data-manager-token-file flag.
-var data_manager_token string
+var dataManagerToken string
-// never_delete can be used to prevent the DELETE handler from
+// neverDelete can be used to prevent the DELETE handler from
// actually deleting anything.
-var never_delete = true
+var neverDelete = true
var maxBuffers = 128
var bufs *bufferPool
defer log.Println("keepstore exiting, pid", os.Getpid())
var (
- data_manager_token_file string
- listen string
- blob_signing_key_file string
- permission_ttl_sec int
- volumes volumeSet
- pidfile string
+ dataManagerTokenFile string
+ listen string
+ blobSigningKeyFile string
+ permissionTTLSec int
+ volumes volumeSet
+ pidfile string
)
flag.StringVar(
- &data_manager_token_file,
+ &dataManagerTokenFile,
"data-manager-token-file",
"",
"File with the API token used by the Data Manager. All DELETE "+
DefaultAddr,
"Listening address, in the form \"host:port\". e.g., 10.0.1.24:8000. Omit the host part to listen on all interfaces.")
flag.BoolVar(
- &never_delete,
+ &neverDelete,
"never-delete",
true,
"If set, nothing will be deleted. HTTP 405 will be returned "+
"for valid DELETE requests.")
flag.StringVar(
- &blob_signing_key_file,
+ &blobSigningKeyFile,
"permission-key-file",
"",
"Synonym for -blob-signing-key-file.")
flag.StringVar(
- &blob_signing_key_file,
+ &blobSigningKeyFile,
"blob-signing-key-file",
"",
"File containing the secret key for generating and verifying "+
"blob permission signatures.")
flag.IntVar(
- &permission_ttl_sec,
+ &permissionTTLSec,
"permission-ttl",
0,
"Synonym for -blob-signature-ttl.")
flag.IntVar(
- &permission_ttl_sec,
+ &permissionTTLSec,
"blob-signature-ttl",
int(time.Duration(2*7*24*time.Hour).Seconds()),
"Lifetime of blob permission signatures. "+
flag.Parse()
- if never_delete != true {
- log.Fatal("never_delete must be true, see #6221")
+ if neverDelete != true {
+ log.Fatal("neverDelete must be true, see #6221")
}
if maxBuffers < 0 {
// Initialize data manager token and permission key.
// If these tokens are specified but cannot be read,
// raise a fatal error.
- if data_manager_token_file != "" {
- if buf, err := ioutil.ReadFile(data_manager_token_file); err == nil {
- data_manager_token = strings.TrimSpace(string(buf))
+ if dataManagerTokenFile != "" {
+ if buf, err := ioutil.ReadFile(dataManagerTokenFile); err == nil {
+ dataManagerToken = strings.TrimSpace(string(buf))
} else {
log.Fatalf("reading data manager token: %s\n", err)
}
}
- if blob_signing_key_file != "" {
- if buf, err := ioutil.ReadFile(blob_signing_key_file); err == nil {
+ if blobSigningKeyFile != "" {
+ if buf, err := ioutil.ReadFile(blobSigningKeyFile); err == nil {
PermissionSecret = bytes.TrimSpace(buf)
} else {
log.Fatalf("reading permission key: %s\n", err)
}
}
- blob_signature_ttl = time.Duration(permission_ttl_sec) * time.Second
+ blobSignatureTTL = time.Duration(permissionTTLSec) * time.Second
if PermissionSecret == nil {
if enforcePermissions {
// teardown cleans up after each test.
func teardown() {
- data_manager_token = ""
+ dataManagerToken = ""
enforcePermissions = false
PermissionSecret = nil
KeepVM = nil
func (s *PullWorkerTestSuite) TestPullWorkerPullList_with_two_locators(c *C) {
defer teardown()
- data_manager_token = "DATA MANAGER TOKEN"
+ dataManagerToken = "DATA MANAGER TOKEN"
testData := PullWorkerTestData{
name: "TestPullWorkerPullList_with_two_locators",
- req: RequestTester{"/pull", data_manager_token, "PUT", firstPullList},
+ req: RequestTester{"/pull", dataManagerToken, "PUT", firstPullList},
responseCode: http.StatusOK,
responseBody: "Received 2 pull requests\n",
readContent: "hello",
func (s *PullWorkerTestSuite) TestPullWorkerPullList_with_one_locator(c *C) {
defer teardown()
- data_manager_token = "DATA MANAGER TOKEN"
+ dataManagerToken = "DATA MANAGER TOKEN"
testData := PullWorkerTestData{
name: "TestPullWorkerPullList_with_one_locator",
- req: RequestTester{"/pull", data_manager_token, "PUT", secondPullList},
+ req: RequestTester{"/pull", dataManagerToken, "PUT", secondPullList},
responseCode: http.StatusOK,
responseBody: "Received 1 pull requests\n",
readContent: "hola",
func (s *PullWorkerTestSuite) TestPullWorker_error_on_get_one_locator(c *C) {
defer teardown()
- data_manager_token = "DATA MANAGER TOKEN"
+ dataManagerToken = "DATA MANAGER TOKEN"
testData := PullWorkerTestData{
name: "TestPullWorker_error_on_get_one_locator",
- req: RequestTester{"/pull", data_manager_token, "PUT", secondPullList},
+ req: RequestTester{"/pull", dataManagerToken, "PUT", secondPullList},
responseCode: http.StatusOK,
responseBody: "Received 1 pull requests\n",
readContent: "unused",
func (s *PullWorkerTestSuite) TestPullWorker_error_on_get_two_locators(c *C) {
defer teardown()
- data_manager_token = "DATA MANAGER TOKEN"
+ dataManagerToken = "DATA MANAGER TOKEN"
testData := PullWorkerTestData{
name: "TestPullWorker_error_on_get_two_locators",
- req: RequestTester{"/pull", data_manager_token, "PUT", firstPullList},
+ req: RequestTester{"/pull", dataManagerToken, "PUT", firstPullList},
responseCode: http.StatusOK,
responseBody: "Received 2 pull requests\n",
readContent: "unused",
func (s *PullWorkerTestSuite) TestPullWorker_error_on_put_one_locator(c *C) {
defer teardown()
- data_manager_token = "DATA MANAGER TOKEN"
+ dataManagerToken = "DATA MANAGER TOKEN"
testData := PullWorkerTestData{
name: "TestPullWorker_error_on_put_one_locator",
- req: RequestTester{"/pull", data_manager_token, "PUT", secondPullList},
+ req: RequestTester{"/pull", dataManagerToken, "PUT", secondPullList},
responseCode: http.StatusOK,
responseBody: "Received 1 pull requests\n",
readContent: "hello hello",
func (s *PullWorkerTestSuite) TestPullWorker_error_on_put_two_locators(c *C) {
defer teardown()
- data_manager_token = "DATA MANAGER TOKEN"
+ dataManagerToken = "DATA MANAGER TOKEN"
testData := PullWorkerTestData{
name: "TestPullWorker_error_on_put_two_locators",
- req: RequestTester{"/pull", data_manager_token, "PUT", firstPullList},
+ req: RequestTester{"/pull", dataManagerToken, "PUT", firstPullList},
responseCode: http.StatusOK,
responseBody: "Received 2 pull requests\n",
readContent: "hello again",
pullq.ReplaceQueue(makeTestWorkList(firstInput))
testPullLists["Added_before_actual_test_item"] = string(1)
- data_manager_token = "DATA MANAGER TOKEN"
+ dataManagerToken = "DATA MANAGER TOKEN"
testData := PullWorkerTestData{
name: "TestPullWorkerPullList_with_two_items_latest_replacing_old",
- req: RequestTester{"/pull", data_manager_token, "PUT", secondPullList},
+ req: RequestTester{"/pull", dataManagerToken, "PUT", secondPullList},
responseCode: http.StatusOK,
responseBody: "Received 1 pull requests\n",
readContent: "hola de nuevo",
}
// In this case, the item will not be placed on pullq
-func (s *PullWorkerTestSuite) TestPullWorker_invalid_data_manager_token(c *C) {
+func (s *PullWorkerTestSuite) TestPullWorker_invalid_dataManagerToken(c *C) {
defer teardown()
- data_manager_token = "DATA MANAGER TOKEN"
+ dataManagerToken = "DATA MANAGER TOKEN"
testData := PullWorkerTestData{
name: "TestPullWorkerPullList_with_two_locators",
- req: RequestTester{"/pull", "invalid_data_manager_token", "PUT", firstPullList},
+ req: RequestTester{"/pull", "invalid_dataManagerToken", "PUT", firstPullList},
responseCode: http.StatusUnauthorized,
responseBody: "Unauthorized\n",
readContent: "hello",
// TrashItem deletes the indicated block from every writable volume.
func TrashItem(trashRequest TrashRequest) {
reqMtime := time.Unix(trashRequest.BlockMtime, 0)
- if time.Since(reqMtime) < blob_signature_ttl {
- log.Printf("WARNING: data manager asked to delete a %v old block %v (BlockMtime %d = %v), but my blob_signature_ttl is %v! Skipping.",
+ if time.Since(reqMtime) < blobSignatureTTL {
+ log.Printf("WARNING: data manager asked to delete a %v old block %v (BlockMtime %d = %v), but my blobSignatureTTL is %v! Skipping.",
time.Since(reqMtime),
trashRequest.Locator,
trashRequest.BlockMtime,
reqMtime,
- blob_signature_ttl)
+ blobSignatureTTL)
return
}
continue
}
- if never_delete {
- err = errors.New("did not delete block because never_delete is true")
+ if neverDelete {
+ err = errors.New("did not delete block because neverDelete is true")
} else {
err = volume.Delete(trashRequest.Locator)
}
Expect no errors.
*/
func TestTrashWorkerIntegration_GetNonExistingLocator(t *testing.T) {
- never_delete = false
+ neverDelete = false
testData := TrashWorkerTestData{
Locator1: "5d41402abc4b2a76b9719d911017c592",
Block1: []byte("hello"),
Expect the second locator in volume 2 to be unaffected.
*/
func TestTrashWorkerIntegration_LocatorInVolume1(t *testing.T) {
- never_delete = false
+ neverDelete = false
testData := TrashWorkerTestData{
Locator1: TestHash,
Block1: TestBlock,
Expect the first locator in volume 1 to be unaffected.
*/
func TestTrashWorkerIntegration_LocatorInVolume2(t *testing.T) {
- never_delete = false
+ neverDelete = false
testData := TrashWorkerTestData{
Locator1: TestHash,
Block1: TestBlock,
Expect locator to be deleted from both volumes.
*/
func TestTrashWorkerIntegration_LocatorInBothVolumes(t *testing.T) {
- never_delete = false
+ neverDelete = false
testData := TrashWorkerTestData{
Locator1: TestHash,
Block1: TestBlock,
Delete the second and expect the first to be still around.
*/
func TestTrashWorkerIntegration_MtimeMatchesForLocator1ButNotForLocator2(t *testing.T) {
- never_delete = false
+ neverDelete = false
testData := TrashWorkerTestData{
Locator1: TestHash,
Block1: TestBlock,
Expect the other unaffected.
*/
func TestTrashWorkerIntegration_TwoDifferentLocatorsInVolume1(t *testing.T) {
- never_delete = false
+ neverDelete = false
testData := TrashWorkerTestData{
Locator1: TestHash,
Block1: TestBlock,
will not be deleted becuase its Mtime is within the trash life time.
*/
func TestTrashWorkerIntegration_SameLocatorInTwoVolumesWithDefaultTrashLifeTime(t *testing.T) {
- never_delete = false
+ neverDelete = false
testData := TrashWorkerTestData{
Locator1: TestHash,
Block1: TestBlock,
performTrashWorkerTest(testData, t)
}
-/* Delete a block with matching mtime for locator in both volumes, but never_delete is true,
+/* Delete a block with matching mtime for locator in both volumes, but neverDelete is true,
so block won't be deleted.
*/
func TestTrashWorkerIntegration_NeverDelete(t *testing.T) {
- never_delete = true
+ neverDelete = true
testData := TrashWorkerTestData{
Locator1: TestHash,
Block1: TestBlock,
}
}
- oldBlockTime := time.Now().Add(-blob_signature_ttl - time.Minute)
+ oldBlockTime := time.Now().Add(-blobSignatureTTL - time.Minute)
// Create TrashRequest for the test
trashRequest := TrashRequest{
// loc is as described in Get.
//
// If the timestamp for the given locator is newer than
- // blob_signature_ttl, Delete must not delete the data.
+ // blobSignatureTTL, Delete must not delete the data.
//
// If a Delete operation overlaps with any Touch or Put
// operations on the same locator, the implementation must
// reliably or fail outright.
//
// Corollary: A successful Touch or Put guarantees a block
- // will not be deleted for at least blob_signature_ttl
+ // will not be deleted for at least blobSignatureTTL
// seconds.
Delete(loc string) error
}
// Calling Delete() for a block with a timestamp older than
-// blob_signature_ttl seconds in the past should delete the data.
+// blobSignatureTTL seconds in the past should delete the data.
// Test is intended for only writable volumes
func testDeleteOldBlock(t *testing.T, factory TestableVolumeFactory) {
v := factory(t)
}
v.Put(TestHash, TestBlock)
- v.TouchWithDate(TestHash, time.Now().Add(-2*blob_signature_ttl*time.Second))
+ v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL*time.Second))
if err := v.Delete(TestHash); err != nil {
t.Error(err)
return MethodDisabledError
}
if _, ok := v.Store[loc]; ok {
- if time.Since(v.Timestamps[loc]) < blob_signature_ttl {
+ if time.Since(v.Timestamps[loc]) < blobSignatureTTL {
return nil
}
delete(v.Store, loc)
}
defer unlockfile(f)
- // If the block has been PUT in the last blob_signature_ttl
+ // If the block has been PUT in the last blobSignatureTTL
// seconds, return success without removing the block. This
// protects data from garbage collection until it is no longer
// possible for clients to retrieve the unreferenced blocks
if fi, err := os.Stat(p); err != nil {
return err
} else {
- if time.Since(fi.ModTime()) < blob_signature_ttl {
+ if time.Since(fi.ModTime()) < blobSignatureTTL {
return nil
}
}