12 "git.curoverse.com/arvados.git/sdk/go/arvadostest"
13 "git.curoverse.com/arvados.git/sdk/go/keepclient"
18 // Gocheck boilerplate
19 func Test(t *testing.T) {
23 // Gocheck boilerplate
24 var _ = Suite(&ServerRequiredSuite{})
25 var _ = Suite(&ServerNotRequiredSuite{})
26 var _ = Suite(&DoMainTestSuite{})
28 // Tests that require the Keep server running
29 type ServerRequiredSuite struct{}
30 type ServerNotRequiredSuite struct{}
31 type DoMainTestSuite struct{}
33 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
35 arvadostest.StartAPI()
38 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
40 arvadostest.ResetEnv()
43 var initialArgs []string
45 func (s *DoMainTestSuite) SetUpSuite(c *C) {
49 var kcSrc, kcDst *keepclient.KeepClient
50 var srcKeepServicesJSON, dstKeepServicesJSON, blobSigningKey string
52 func (s *ServerRequiredSuite) SetUpTest(c *C) {
53 // reset all variables between tests
55 srcKeepServicesJSON = ""
56 dstKeepServicesJSON = ""
57 kcSrc = &keepclient.KeepClient{}
58 kcDst = &keepclient.KeepClient{}
61 func (s *ServerRequiredSuite) TearDownTest(c *C) {
62 arvadostest.StopKeep(3)
65 func (s *DoMainTestSuite) SetUpTest(c *C) {
66 args := []string{"keep-rsync"}
70 func (s *DoMainTestSuite) TearDownTest(c *C) {
74 var testKeepServicesJSON = "{ \"kind\":\"arvados#keepServiceList\", \"etag\":\"\", \"self_link\":\"\", \"offset\":null, \"limit\":null, \"items\":[ { \"href\":\"/keep_services/zzzzz-bi6l4-123456789012340\", \"kind\":\"arvados#keepService\", \"etag\":\"641234567890enhj7hzx432e5\", \"uuid\":\"zzzzz-bi6l4-123456789012340\", \"owner_uuid\":\"zzzzz-tpzed-123456789012345\", \"service_host\":\"keep0.zzzzz.arvadosapi.com\", \"service_port\":25107, \"service_ssl_flag\":false, \"service_type\":\"disk\", \"read_only\":false }, { \"href\":\"/keep_services/zzzzz-bi6l4-123456789012341\", \"kind\":\"arvados#keepService\", \"etag\":\"641234567890enhj7hzx432e5\", \"uuid\":\"zzzzz-bi6l4-123456789012341\", \"owner_uuid\":\"zzzzz-tpzed-123456789012345\", \"service_host\":\"keep0.zzzzz.arvadosapi.com\", \"service_port\":25108, \"service_ssl_flag\":false, \"service_type\":\"disk\", \"read_only\":false } ], \"items_available\":2 }"
76 // Testing keep-rsync needs two sets of keep services: src and dst.
77 // The test setup hence creates 3 servers instead of the default 2,
78 // and uses the first 2 as src and the 3rd as dst keep servers.
79 func setupRsync(c *C, enforcePermissions bool, replications int) {
81 var srcConfig apiConfig
82 srcConfig.APIHost = os.Getenv("ARVADOS_API_HOST")
83 srcConfig.APIToken = os.Getenv("ARVADOS_API_TOKEN")
84 srcConfig.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))
87 var dstConfig apiConfig
88 dstConfig.APIHost = os.Getenv("ARVADOS_API_HOST")
89 dstConfig.APIToken = os.Getenv("ARVADOS_API_TOKEN")
90 dstConfig.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))
92 if enforcePermissions {
93 blobSigningKey = "zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc"
97 arvadostest.StartKeep(3, enforcePermissions)
101 kcSrc, err = setupKeepClient(srcConfig, srcKeepServicesJSON, false, 0)
104 kcDst, err = setupKeepClient(dstConfig, dstKeepServicesJSON, true, replications)
107 for uuid := range kcSrc.LocalRoots() {
108 if strings.HasSuffix(uuid, "02") {
109 delete(kcSrc.LocalRoots(), uuid)
112 for uuid := range kcSrc.GatewayRoots() {
113 if strings.HasSuffix(uuid, "02") {
114 delete(kcSrc.GatewayRoots(), uuid)
117 for uuid := range kcSrc.WritableLocalRoots() {
118 if strings.HasSuffix(uuid, "02") {
119 delete(kcSrc.WritableLocalRoots(), uuid)
123 for uuid := range kcDst.LocalRoots() {
124 if strings.HasSuffix(uuid, "00") || strings.HasSuffix(uuid, "01") {
125 delete(kcDst.LocalRoots(), uuid)
128 for uuid := range kcDst.GatewayRoots() {
129 if strings.HasSuffix(uuid, "00") || strings.HasSuffix(uuid, "01") {
130 delete(kcDst.GatewayRoots(), uuid)
133 for uuid := range kcDst.WritableLocalRoots() {
134 if strings.HasSuffix(uuid, "00") || strings.HasSuffix(uuid, "01") {
135 delete(kcDst.WritableLocalRoots(), uuid)
139 if replications == 0 {
140 // Must have got default replications value of 2 from dst discovery document
141 c.Assert(kcDst.Want_replicas, Equals, 2)
143 // Since replications value is provided, it is used
144 c.Assert(kcDst.Want_replicas, Equals, replications)
148 func (s *ServerRequiredSuite) TestRsyncPutInOne_GetFromOtherShouldFail(c *C) {
149 setupRsync(c, false, 1)
151 // Put a block in src and verify that it is not found in dst
152 testNoCrosstalk(c, "test-data-1", kcSrc, kcDst)
154 // Put a block in dst and verify that it is not found in src
155 testNoCrosstalk(c, "test-data-2", kcDst, kcSrc)
158 func (s *ServerRequiredSuite) TestRsyncWithBlobSigning_PutInOne_GetFromOtherShouldFail(c *C) {
159 setupRsync(c, true, 1)
161 // Put a block in src and verify that it is not found in dst
162 testNoCrosstalk(c, "test-data-1", kcSrc, kcDst)
164 // Put a block in dst and verify that it is not found in src
165 testNoCrosstalk(c, "test-data-2", kcDst, kcSrc)
168 // Do a Put in the first and Get from the second,
169 // which should raise block not found error.
170 func testNoCrosstalk(c *C, testData string, kc1, kc2 *keepclient.KeepClient) {
171 // Put a block using kc1
172 locator, _, err := kc1.PutB([]byte(testData))
173 c.Assert(err, Equals, nil)
175 locator = strings.Split(locator, "+")[0]
176 _, _, _, err = kc2.Get(keepclient.SignLocator(locator, kc2.Arvados.ApiToken, time.Now().AddDate(0, 0, 1), []byte(blobSigningKey)))
177 c.Assert(err, NotNil)
178 c.Check(err.Error(), Equals, "Block not found")
181 // Test keep-rsync initialization, with srcKeepServicesJSON
182 func (s *ServerRequiredSuite) TestRsyncInitializeWithKeepServicesJSON(c *C) {
183 srcKeepServicesJSON = testKeepServicesJSON
185 setupRsync(c, false, 1)
187 localRoots := kcSrc.LocalRoots()
188 c.Check(localRoots, NotNil)
191 for k := range localRoots {
192 if k == "zzzzz-bi6l4-123456789012340" {
196 c.Check(foundIt, Equals, true)
199 for k := range localRoots {
200 if k == "zzzzz-bi6l4-123456789012341" {
204 c.Check(foundIt, Equals, true)
207 // Test keep-rsync initialization with default replications count
208 func (s *ServerRequiredSuite) TestInitializeRsyncDefaultReplicationsCount(c *C) {
209 setupRsync(c, false, 0)
212 // Test keep-rsync initialization with replications count argument
213 func (s *ServerRequiredSuite) TestInitializeRsyncReplicationsCount(c *C) {
214 setupRsync(c, false, 3)
217 // Put some blocks in Src and some more in Dst
218 // And copy missing blocks from Src to Dst
219 func (s *ServerRequiredSuite) TestKeepRsync(c *C) {
220 testKeepRsync(c, false, "")
223 // Put some blocks in Src and some more in Dst with blob signing enabled.
224 // And copy missing blocks from Src to Dst
225 func (s *ServerRequiredSuite) TestKeepRsync_WithBlobSigning(c *C) {
226 testKeepRsync(c, true, "")
229 // Put some blocks in Src and some more in Dst
230 // Use prefix while doing rsync
231 // And copy missing blocks from Src to Dst
232 func (s *ServerRequiredSuite) TestKeepRsync_WithPrefix(c *C) {
233 data := []byte("test-data-4")
234 hash := fmt.Sprintf("%x", md5.Sum(data))
236 testKeepRsync(c, false, hash[0:3])
237 c.Check(len(dstIndex) > len(dstLocators), Equals, true)
240 // Put some blocks in Src and some more in Dst
241 // Use prefix not in src while doing rsync
242 // And copy missing blocks from Src to Dst
243 func (s *ServerRequiredSuite) TestKeepRsync_WithNoSuchPrefixInSrc(c *C) {
244 testKeepRsync(c, false, "999")
245 c.Check(len(dstIndex), Equals, len(dstLocators))
248 // Put 5 blocks in src. Put 2 of those blocks in dst
249 // Hence there are 3 additional blocks in src
250 // Also, put 2 extra blocks in dst; they are hence only in dst
251 // Run rsync and verify that those 7 blocks are now available in dst
252 func testKeepRsync(c *C, enforcePermissions bool, prefix string) {
253 setupRsync(c, enforcePermissions, 1)
256 setupTestData(c, prefix)
258 err := performKeepRsync(kcSrc, kcDst, blobSigningKey, prefix)
261 // Now GetIndex from dst and verify that all 5 from src and the 2 extra blocks are found
262 dstIndex, err = getUniqueLocators(kcDst, "")
265 for _, locator := range srcLocatorsMatchingPrefix {
266 _, ok := dstIndex[locator]
267 c.Assert(ok, Equals, true)
270 for _, locator := range extraDstLocators {
271 _, ok := dstIndex[locator]
272 c.Assert(ok, Equals, true)
276 // all blocks from src and the two extra blocks
277 c.Assert(len(dstIndex), Equals, len(srcLocators)+len(extraDstLocators))
279 // 1 matching prefix and copied over, 2 that were initially copied into dst along with src, and the 2 extra blocks
280 c.Assert(len(dstIndex), Equals, len(srcLocatorsMatchingPrefix)+len(extraDstLocators)+2)
284 // Setup test data in src and dst.
285 var srcLocators, srcLocatorsMatchingPrefix, dstLocators, extraDstLocators []string
286 var dstIndex map[string]bool
288 func setupTestData(c *C, indexPrefix string) {
289 srcLocators = []string{}
290 srcLocatorsMatchingPrefix = []string{}
291 dstLocators = []string{}
292 extraDstLocators = []string{}
293 dstIndex = make(map[string]bool)
295 // Put a few blocks in src using kcSrc
296 for i := 0; i < 5; i++ {
297 hash, _, err := kcSrc.PutB([]byte(fmt.Sprintf("test-data-%d", i)))
300 srcLocators = append(srcLocators, strings.Split(hash, "+A")[0])
301 if strings.HasPrefix(hash, indexPrefix) {
302 srcLocatorsMatchingPrefix = append(srcLocatorsMatchingPrefix, strings.Split(hash, "+A")[0])
306 // Put first two of those src blocks in dst using kcDst
307 for i := 0; i < 2; i++ {
308 hash, _, err := kcDst.PutB([]byte(fmt.Sprintf("test-data-%d", i)))
310 dstLocators = append(dstLocators, strings.Split(hash, "+A")[0])
313 // Put two more blocks in dst; they are not in src at all
314 for i := 0; i < 2; i++ {
315 hash, _, err := kcDst.PutB([]byte(fmt.Sprintf("other-data-%d", i)))
317 dstLocators = append(dstLocators, strings.Split(hash, "+A")[0])
318 extraDstLocators = append(extraDstLocators, strings.Split(hash, "+A")[0])
322 // Setup rsync using srcKeepServicesJSON with fake keepservers.
323 // Expect error during performKeepRsync due to unreachable src keepservers.
324 func (s *ServerRequiredSuite) TestErrorDuringRsync_FakeSrcKeepservers(c *C) {
325 srcKeepServicesJSON = testKeepServicesJSON
327 setupRsync(c, false, 1)
329 err := performKeepRsync(kcSrc, kcDst, "", "")
330 c.Check(strings.HasSuffix(err.Error(), "no such host"), Equals, true)
333 // Setup rsync using dstKeepServicesJSON with fake keepservers.
334 // Expect error during performKeepRsync due to unreachable dst keepservers.
335 func (s *ServerRequiredSuite) TestErrorDuringRsync_FakeDstKeepservers(c *C) {
336 dstKeepServicesJSON = testKeepServicesJSON
338 setupRsync(c, false, 1)
340 err := performKeepRsync(kcSrc, kcDst, "", "")
341 c.Check(strings.HasSuffix(err.Error(), "no such host"), Equals, true)
344 // Test rsync with signature error during Get from src.
345 func (s *ServerRequiredSuite) TestErrorDuringRsync_ErrorGettingBlockFromSrc(c *C) {
346 setupRsync(c, true, 1)
348 // put some blocks in src and dst
351 // Change blob signing key to a fake key, so that Get from src fails
352 blobSigningKey = "thisisfakeblobsigningkey"
354 err := performKeepRsync(kcSrc, kcDst, blobSigningKey, "")
355 c.Check(strings.Contains(err.Error(), "HTTP 403 \"Forbidden\""), Equals, true)
358 // Test rsync with error during Put to src.
359 func (s *ServerRequiredSuite) TestErrorDuringRsync_ErrorPuttingBlockInDst(c *C) {
360 setupRsync(c, false, 1)
362 // put some blocks in src and dst
365 // Increase Want_replicas on dst to result in insufficient replicas error during Put
366 kcDst.Want_replicas = 2
368 err := performKeepRsync(kcSrc, kcDst, blobSigningKey, "")
369 c.Check(strings.HasSuffix(err.Error(), "Could not write sufficient replicas"), Equals, true)
372 // Test loadConfig func
373 func (s *ServerNotRequiredSuite) TestLoadConfig(c *C) {
374 // Setup a src config file
375 srcFile := setupConfigFile(c, "src-config")
376 defer os.Remove(srcFile.Name())
377 srcConfigFile := srcFile.Name()
379 // Setup a dst config file
380 dstFile := setupConfigFile(c, "dst-config")
381 defer os.Remove(dstFile.Name())
382 dstConfigFile := dstFile.Name()
384 // load configuration from those files
385 srcConfig, srcBlobSigningKey, err := loadConfig(srcConfigFile)
388 c.Assert(srcConfig.APIHost, Equals, os.Getenv("ARVADOS_API_HOST"))
389 c.Assert(srcConfig.APIToken, Equals, os.Getenv("ARVADOS_API_TOKEN"))
390 c.Assert(srcConfig.APIHostInsecure, Equals, matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE")))
391 c.Assert(srcConfig.ExternalClient, Equals, false)
393 dstConfig, _, err := loadConfig(dstConfigFile)
396 c.Assert(dstConfig.APIHost, Equals, os.Getenv("ARVADOS_API_HOST"))
397 c.Assert(dstConfig.APIToken, Equals, os.Getenv("ARVADOS_API_TOKEN"))
398 c.Assert(dstConfig.APIHostInsecure, Equals, matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE")))
399 c.Assert(dstConfig.ExternalClient, Equals, false)
401 c.Assert(srcBlobSigningKey, Equals, "abcdefg")
404 // Test loadConfig func without setting up the config files
405 func (s *ServerNotRequiredSuite) TestLoadConfig_MissingSrcConfig(c *C) {
406 _, _, err := loadConfig("")
407 c.Assert(err.Error(), Equals, "config file not specified")
410 // Test loadConfig func - error reading config
411 func (s *ServerNotRequiredSuite) TestLoadConfig_ErrorLoadingSrcConfig(c *C) {
412 _, _, err := loadConfig("no-such-config-file")
413 c.Assert(strings.HasSuffix(err.Error(), "no such file or directory"), Equals, true)
416 func setupConfigFile(c *C, name string) *os.File {
417 // Setup a config file
418 file, err := ioutil.TempFile(os.TempDir(), name)
421 fileContent := "ARVADOS_API_HOST=" + os.Getenv("ARVADOS_API_HOST") + "\n"
422 fileContent += "ARVADOS_API_TOKEN=" + os.Getenv("ARVADOS_API_TOKEN") + "\n"
423 fileContent += "ARVADOS_API_HOST_INSECURE=" + os.Getenv("ARVADOS_API_HOST_INSECURE") + "\n"
424 fileContent += "ARVADOS_EXTERNAL_CLIENT=false\n"
425 fileContent += "ARVADOS_BLOB_SIGNING_KEY=abcdefg"
427 _, err = file.Write([]byte(fileContent))
433 func (s *DoMainTestSuite) Test_doMain_NoSrcConfig(c *C) {
436 c.Assert(err.Error(), Equals, "Error loading src configuration from file: config file not specified")
439 func (s *DoMainTestSuite) Test_doMain_SrcButNoDstConfig(c *C) {
440 srcConfig := setupConfigFile(c, "src")
441 args := []string{"-replications", "3", "-src", srcConfig.Name()}
442 os.Args = append(os.Args, args...)
445 c.Assert(err.Error(), Equals, "Error loading dst configuration from file: config file not specified")
448 func (s *DoMainTestSuite) Test_doMain_BadSrcConfig(c *C) {
449 args := []string{"-src", "abcd"}
450 os.Args = append(os.Args, args...)
453 c.Assert(strings.HasPrefix(err.Error(), "Error loading src configuration from file: Error reading config file"), Equals, true)
456 func (s *DoMainTestSuite) Test_doMain_WithReplicationsButNoSrcConfig(c *C) {
457 args := []string{"-replications", "3"}
458 os.Args = append(os.Args, args...)
461 c.Assert(err.Error(), Equals, "Error loading src configuration from file: config file not specified")
464 func (s *DoMainTestSuite) Test_doMainWithSrcAndDstConfig(c *C) {
465 srcConfig := setupConfigFile(c, "src")
466 dstConfig := setupConfigFile(c, "dst")
467 args := []string{"-src", srcConfig.Name(), "-dst", dstConfig.Name()}
468 os.Args = append(os.Args, args...)
470 // Start keepservers. Since we are not doing any tweaking as in setupRsync func,
471 // kcSrc and kcDst will be the same and no actual copying to dst will happen, but that's ok.
472 arvadostest.StartKeep(2, false)