package main
import (
+ "bytes"
"fmt"
+ "io"
"io/ioutil"
"log"
"os"
type ServerRequiredSuite struct{}
type DoMainTestSuite struct{}
+var kc *keepclient.KeepClient
+var keepServicesJSON, blobSigningKey string
+var TestHash = "aaaa09c290d0fb1ca068ffaddf22cbd0"
+var TestHash2 = "aaaac516f788aec4f30932ffb6395c39"
+var allLocators []string
+
func (s *ServerRequiredSuite) SetUpSuite(c *C) {
arvadostest.StartAPI()
}
arvadostest.ResetEnv()
}
+var logBuffer bytes.Buffer
+
func (s *ServerRequiredSuite) SetUpTest(c *C) {
blobSigningKey = ""
keepServicesJSON = ""
- tempfile, err := ioutil.TempFile(os.TempDir(), "temp-log-file")
- c.Check(err, IsNil)
- log.SetOutput(tempfile)
- tempLogFileName = tempfile.Name()
+ logOutput := io.MultiWriter(&logBuffer)
+ log.SetOutput(logOutput)
}
func (s *ServerRequiredSuite) TearDownTest(c *C) {
arvadostest.StopKeep(2)
- os.Remove(tempLogFileName)
+ log.SetOutput(os.Stdout)
+ log.Printf("%v", logBuffer.String())
}
-var tempLogFileName = ""
-var initialArgs []string
-var kc *keepclient.KeepClient
-var keepServicesJSON, blobSigningKey string
-
func (s *DoMainTestSuite) SetUpSuite(c *C) {
- initialArgs = os.Args
}
+var testArgs = []string{}
+
func (s *DoMainTestSuite) SetUpTest(c *C) {
blobSigningKey = ""
keepServicesJSON = ""
- args := []string{"keep-block-check"}
- os.Args = args
-
- tempfile, err := ioutil.TempFile(os.TempDir(), "temp-log-file")
- c.Check(err, IsNil)
- log.SetOutput(tempfile)
- tempLogFileName = tempfile.Name()
+ logOutput := io.MultiWriter(&logBuffer)
+ log.SetOutput(logOutput)
}
func (s *DoMainTestSuite) TearDownTest(c *C) {
- os.Remove(tempLogFileName)
- os.Args = initialArgs
+ log.SetOutput(os.Stdout)
+ log.Printf("%v", logBuffer.String())
+ testArgs = []string{}
}
-var testKeepServicesJSON = "{ \"kind\":\"arvados#keepServiceList\", \"etag\":\"\", \"self_link\":\"\", \"offset\":null, \"limit\":null, \"items\":[ { \"href\":\"/keep_services/zzzzz-bi6l4-123456789012340\", \"kind\":\"arvados#keepService\", \"etag\":\"641234567890enhj7hzx432e5\", \"uuid\":\"zzzzz-bi6l4-123456789012340\", \"owner_uuid\":\"zzzzz-tpzed-123456789012345\", \"service_host\":\"keep0.zzzzz.arvadosapi.com\", \"service_port\":25107, \"service_ssl_flag\":false, \"service_type\":\"disk\", \"read_only\":false }, { \"href\":\"/keep_services/zzzzz-bi6l4-123456789012341\", \"kind\":\"arvados#keepService\", \"etag\":\"641234567890enhj7hzx432e5\", \"uuid\":\"zzzzz-bi6l4-123456789012341\", \"owner_uuid\":\"zzzzz-tpzed-123456789012345\", \"service_host\":\"keep0.zzzzz.arvadosapi.com\", \"service_port\":25108, \"service_ssl_flag\":false, \"service_type\":\"disk\", \"read_only\":false } ], \"items_available\":2 }"
-
-var TestHash = "aaaa09c290d0fb1ca068ffaddf22cbd0"
-var TestHash2 = "aaaac516f788aec4f30932ffb6395c39"
-
func setupKeepBlockCheck(c *C, enforcePermissions bool) {
var config apiConfig
config.APIHost = os.Getenv("ARVADOS_API_HOST")
config.APIToken = arvadostest.DataManagerToken
config.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))
if enforcePermissions {
- blobSigningKey = "zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc"
+ blobSigningKey = arvadostest.BlobSigningKey
}
// Start Keep servers
}
// Setup test data
-var allLocators []string
-
func setupTestData(c *C) {
allLocators = []string{}
fileContent += "\n"
fileContent += "ARVADOS_API_HOST_INSECURE=" + os.Getenv("ARVADOS_API_HOST_INSECURE") + "\n"
fileContent += " ARVADOS_EXTERNAL_CLIENT = false \n"
+ fileContent += " NotANameValuePairAndShouldGetIgnored \n"
fileContent += "ARVADOS_BLOB_SIGNING_KEY=abcdefg\n"
_, err = file.Write([]byte(fileContent))
}
func checkErrorLog(c *C, blocks []string, prefix, suffix string) {
- buf, _ := ioutil.ReadFile(tempLogFileName)
- if len(blocks) == 0 {
- expected := prefix + `.*` + suffix
- match, _ := regexp.MatchString(expected, string(buf))
- c.Assert(match, Equals, false)
- return
- }
for _, hash := range blocks {
expected := prefix + `.*` + hash + `.*` + suffix
- match, _ := regexp.MatchString(expected, string(buf))
+ match, _ := regexp.MatchString(expected, logBuffer.String())
c.Assert(match, Equals, true)
}
}
+func checkNoErrorsLogged(c *C, prefix, suffix string) {
+ expected := prefix + `.*` + suffix
+ match, _ := regexp.MatchString(expected, logBuffer.String())
+ c.Assert(match, Equals, false)
+}
+
func (s *ServerRequiredSuite) TestBlockCheck(c *C) {
setupKeepBlockCheck(c, false)
setupTestData(c)
- err := performKeepBlockCheck(kc, blobSigningKey, "", allLocators)
+ err := performKeepBlockCheck(kc, blobSigningKey, allLocators, true)
c.Check(err, IsNil)
- checkErrorLog(c, []string{}, "head", "Block not found") // no errors
+ checkNoErrorsLogged(c, "Error verifying block", "Block not found")
}
func (s *ServerRequiredSuite) TestBlockCheckWithBlobSigning(c *C) {
setupKeepBlockCheck(c, true)
setupTestData(c)
- err := performKeepBlockCheck(kc, blobSigningKey, "", allLocators)
+ err := performKeepBlockCheck(kc, blobSigningKey, allLocators, true)
c.Check(err, IsNil)
- checkErrorLog(c, []string{}, "head", "Block not found") // no errors
+ checkNoErrorsLogged(c, "Error verifying block", "Block not found")
}
func (s *ServerRequiredSuite) TestBlockCheck_NoSuchBlock(c *C) {
setupTestData(c)
allLocators = append(allLocators, TestHash)
allLocators = append(allLocators, TestHash2)
- err := performKeepBlockCheck(kc, blobSigningKey, "", allLocators)
+ err := performKeepBlockCheck(kc, blobSigningKey, allLocators, true)
c.Check(err, NotNil)
- c.Assert(err.Error(), Equals, "Head information not found for 2 out of 7 blocks with matching prefix.")
- checkErrorLog(c, []string{TestHash, TestHash2}, "head", "Block not found")
+ c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 7 blocks with matching prefix.")
+ checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "Block not found")
}
func (s *ServerRequiredSuite) TestBlockCheck_NoSuchBlock_WithMatchingPrefix(c *C) {
setupTestData(c)
allLocators = append(allLocators, TestHash)
allLocators = append(allLocators, TestHash2)
- err := performKeepBlockCheck(kc, blobSigningKey, "aaa", allLocators)
+ locatorFile := setupBlockHashFile(c, "block-hash", allLocators)
+ defer os.Remove(locatorFile)
+ locators, err := getBlockLocators(locatorFile, "aaa")
+ c.Check(err, IsNil)
+ err = performKeepBlockCheck(kc, blobSigningKey, locators, true)
c.Check(err, NotNil)
- // Of the 7 blocks given, only two match the prefix and hence only those are checked
- c.Assert(err.Error(), Equals, "Head information not found for 2 out of 2 blocks with matching prefix.")
- checkErrorLog(c, []string{TestHash, TestHash2}, "head", "Block not found")
+ // Of the 7 blocks in allLocators, only two match the prefix and hence only those are checked
+ c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix.")
+ checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "Block not found")
}
func (s *ServerRequiredSuite) TestBlockCheck_NoSuchBlock_WithPrefixMismatch(c *C) {
setupTestData(c)
allLocators = append(allLocators, TestHash)
allLocators = append(allLocators, TestHash2)
- err := performKeepBlockCheck(kc, blobSigningKey, "999", allLocators)
+ locatorFile := setupBlockHashFile(c, "block-hash", allLocators)
+ defer os.Remove(locatorFile)
+ locators, err := getBlockLocators(locatorFile, "999")
c.Check(err, IsNil)
- checkErrorLog(c, []string{}, "head", "Block not found") // no errors
+ err = performKeepBlockCheck(kc, blobSigningKey, locators, true)
+ c.Check(err, IsNil) // there were no matching locators and hence no errors
+}
+
+func (s *ServerRequiredSuite) TestBlockCheck_BadSignature(c *C) {
+ setupKeepBlockCheck(c, true)
+ setupTestData(c)
+ err := performKeepBlockCheck(kc, "badblobsigningkey", []string{TestHash, TestHash2}, false)
+ c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix.")
+ checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "HTTP 403")
+ // verbose logging not requested
+ c.Assert(strings.Contains(logBuffer.String(), "Checking block 1 of 2"), Equals, false)
}
+var testKeepServicesJSON = `{
+ "kind":"arvados#keepServiceList",
+ "etag":"",
+ "self_link":"",
+ "offset":null, "limit":null,
+ "items":[
+ {"href":"/keep_services/zzzzz-bi6l4-123456789012340",
+ "kind":"arvados#keepService",
+ "uuid":"zzzzz-bi6l4-123456789012340",
+ "service_host":"keep0.zzzzz.arvadosapi.com",
+ "service_port":25107,
+ "service_ssl_flag":false,
+ "service_type":"disk",
+ "read_only":false },
+ {"href":"/keep_services/zzzzz-bi6l4-123456789012341",
+ "kind":"arvados#keepService",
+ "uuid":"zzzzz-bi6l4-123456789012341",
+ "service_host":"keep0.zzzzz.arvadosapi.com",
+ "service_port":25108,
+ "service_ssl_flag":false,
+ "service_type":"disk",
+ "read_only":false }
+ ],
+ "items_available":2 }`
+
// Setup block-check using keepServicesJSON with fake keepservers.
// Expect error during performKeepBlockCheck due to unreachable keepservers.
func (s *ServerRequiredSuite) TestErrorDuringKeepBlockCheck_FakeKeepservers(c *C) {
keepServicesJSON = testKeepServicesJSON
setupKeepBlockCheck(c, false)
- err := performKeepBlockCheck(kc, blobSigningKey, "", []string{TestHash, TestHash2})
- c.Assert(err.Error(), Equals, "Head information not found for 2 out of 2 blocks with matching prefix.")
- checkErrorLog(c, []string{TestHash, TestHash2}, "head", "no such host")
-}
-
-func (s *ServerRequiredSuite) TestBlockCheck_BadSignature(c *C) {
- setupKeepBlockCheck(c, true)
- setupTestData(c)
- err := performKeepBlockCheck(kc, "badblobsigningkey", "", []string{TestHash, TestHash2})
- c.Assert(err.Error(), Equals, "Head information not found for 2 out of 2 blocks with matching prefix.")
- checkErrorLog(c, []string{TestHash, TestHash2}, "head", "HTTP 403")
+ err := performKeepBlockCheck(kc, blobSigningKey, []string{TestHash, TestHash2}, true)
+ c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix.")
+ checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "")
}
// Test keep-block-check initialization with keepServicesJSON
func (s *DoMainTestSuite) Test_doMain_WithNoConfig(c *C) {
args := []string{"-prefix", "a"}
- os.Args = append(os.Args, args...)
- err := doMain()
+ testArgs = append(testArgs, args...)
+ err := doMain(testArgs)
c.Check(err, NotNil)
c.Assert(strings.Contains(err.Error(), "config file not specified"), Equals, true)
}
func (s *DoMainTestSuite) Test_doMain_WithNoSuchConfigFile(c *C) {
args := []string{"-config", "no-such-file"}
- os.Args = append(os.Args, args...)
- err := doMain()
+ testArgs = append(testArgs, args...)
+ err := doMain(testArgs)
c.Check(err, NotNil)
c.Assert(strings.Contains(err.Error(), "no such file or directory"), Equals, true)
}
defer os.Remove(config)
args := []string{"-config", config}
- os.Args = append(os.Args, args...)
+ testArgs = append(testArgs, args...)
// Start keepservers.
arvadostest.StartKeep(2, false)
defer arvadostest.StopKeep(2)
- err := doMain()
+ err := doMain(testArgs)
c.Assert(strings.Contains(err.Error(), "block-hash-file not specified"), Equals, true)
}
defer os.Remove(config)
args := []string{"-config", config, "-block-hash-file", "no-such-file"}
- os.Args = append(os.Args, args...)
+ testArgs = append(testArgs, args...)
// Start keepservers.
arvadostest.StartKeep(2, false)
defer arvadostest.StopKeep(2)
- err := doMain()
+ err := doMain(testArgs)
c.Assert(strings.Contains(err.Error(), "no such file or directory"), Equals, true)
}
locatorFile := setupBlockHashFile(c, "block-hash", []string{TestHash, TestHash2})
defer os.Remove(locatorFile)
- args := []string{"-config", config, "-block-hash-file", locatorFile}
- os.Args = append(os.Args, args...)
+ args := []string{"-config", config, "-block-hash-file", locatorFile, "-v"}
+ testArgs = append(testArgs, args...)
- err := doMain()
+ err := doMain(testArgs)
c.Check(err, NotNil)
- c.Assert(err.Error(), Equals, "Head information not found for 2 out of 2 blocks with matching prefix.")
- checkErrorLog(c, []string{TestHash, TestHash2}, "head", "Block not found")
+ c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix.")
+ checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "Block not found")
+ c.Assert(strings.Contains(logBuffer.String(), "Checking block 1 of 2"), Equals, true)
}
projects / arvados.git / blobdiff