12 "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
13 "git.curoverse.com/arvados.git/sdk/go/arvadostest"
14 "git.curoverse.com/arvados.git/sdk/go/keepclient"
19 // Gocheck boilerplate
20 func Test(t *testing.T) {
24 // Gocheck boilerplate
25 var _ = Suite(&ServerRequiredSuite{})
27 // Tests that require the Keep server running
28 type ServerRequiredSuite struct{}
30 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
33 func (s *ServerRequiredSuite) SetUpTest(c *C) {
34 arvadostest.ResetEnv()
36 // reset all variables between tests
38 srcKeepServicesJSON = ""
39 dstKeepServicesJSON = ""
40 kcSrc = &keepclient.KeepClient{}
41 kcDst = &keepclient.KeepClient{}
44 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
45 arvadostest.StopKeep()
49 var kcSrc *keepclient.KeepClient
50 var kcDst *keepclient.KeepClient
51 var srcKeepServicesJSON string
52 var dstKeepServicesJSON string
54 var testKeepServicesJSON = "{ \"kind\":\"arvados#keepServiceList\", \"etag\":\"\", \"self_link\":\"\", \"offset\":null, \"limit\":null, \"items\":[ { \"href\":\"/keep_services/zzzzz-bi6l4-123456789012340\", \"kind\":\"arvados#keepService\", \"etag\":\"641234567890enhj7hzx432e5\", \"uuid\":\"zzzzz-bi6l4-123456789012340\", \"owner_uuid\":\"zzzzz-tpzed-123456789012345\", \"service_host\":\"keep0.zzzzz.arvadosapi.com\", \"service_port\":25107, \"service_ssl_flag\":false, \"service_type\":\"disk\", \"read_only\":false }, { \"href\":\"/keep_services/zzzzz-bi6l4-123456789012341\", \"kind\":\"arvados#keepService\", \"etag\":\"641234567890enhj7hzx432e5\", \"uuid\":\"zzzzz-bi6l4-123456789012341\", \"owner_uuid\":\"zzzzz-tpzed-123456789012345\", \"service_host\":\"keep0.zzzzz.arvadosapi.com\", \"service_port\":25108, \"service_ssl_flag\":false, \"service_type\":\"disk\", \"read_only\":false } ], \"items_available\":2 }"
56 // Testing keep-rsync needs two sets of keep services: src and dst.
57 // The test setup hence tweaks keep-rsync initialization to achieve this.
58 // First invoke setupKeepClients and then invoke StartKeepWithParams
59 // to create the keep servers to be used as destination.
60 func setupRsync(c *C, enforcePermissions, setupDstServers bool, replications int) {
62 var srcConfig arvadosclient.APIConfig
63 srcConfig.APIHost = os.Getenv("ARVADOS_API_HOST")
64 srcConfig.APIToken = os.Getenv("ARVADOS_API_TOKEN")
65 srcConfig.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))
68 var dstConfig arvadosclient.APIConfig
69 dstConfig.APIHost = os.Getenv("ARVADOS_API_HOST")
70 dstConfig.APIToken = os.Getenv("ARVADOS_API_TOKEN")
71 dstConfig.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))
73 if enforcePermissions {
74 blobSigningKey = "zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc"
77 // Start API and Keep servers
78 arvadostest.StartAPI()
79 arvadostest.StartKeepWithParams(false, enforcePermissions)
83 kcSrc, kcDst, err = setupKeepClients(srcConfig, dstConfig, srcKeepServicesJSON, dstKeepServicesJSON, replications)
86 // Create an additional keep server to be used as destination and reload kcDst
88 arvadostest.StartKeepWithParams(true, enforcePermissions)
89 kcDst, err = keepclient.MakeKeepClient(kcDst.Arvados)
91 kcDst.Want_replicas = replications
95 // Test keep-rsync initialization, with src and dst keep servers.
96 // Do a Put and Get in src, both of which should succeed.
97 // Do a Put and Get in dst, both of which should succeed.
98 // Do a Get in dst for the src hash, which should raise block not found error.
99 // Do a Get in src for the dst hash, which should raise block not found error.
100 func (s *ServerRequiredSuite) TestRsyncPutInOne_GetFromOtherShouldFail(c *C) {
101 setupRsync(c, false, true, 1)
103 // Put a block in src using kcSrc and Get it
104 srcData := []byte("test-data1")
105 locatorInSrc := fmt.Sprintf("%x", md5.Sum(srcData))
107 hash, rep, err := kcSrc.PutB(srcData)
108 c.Check(hash, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, locatorInSrc))
109 c.Check(rep, Equals, 2)
110 c.Check(err, Equals, nil)
112 reader, blocklen, _, err := kcSrc.Get(locatorInSrc)
114 c.Check(blocklen, Equals, int64(10))
115 all, err := ioutil.ReadAll(reader)
116 c.Check(all, DeepEquals, srcData)
118 // Put a different block in src using kcSrc and Get it
119 dstData := []byte("test-data2")
120 locatorInDst := fmt.Sprintf("%x", md5.Sum(dstData))
122 hash, rep, err = kcDst.PutB(dstData)
123 c.Check(hash, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, locatorInDst))
124 c.Check(rep, Equals, 1)
125 c.Check(err, Equals, nil)
127 reader, blocklen, _, err = kcDst.Get(locatorInDst)
129 c.Check(blocklen, Equals, int64(10))
130 all, err = ioutil.ReadAll(reader)
131 c.Check(all, DeepEquals, dstData)
133 // Get srcLocator using kcDst should fail with Not Found error
134 _, _, _, err = kcDst.Get(locatorInSrc)
135 c.Assert(err.Error(), Equals, "Block not found")
137 // Get dstLocator using kcSrc should fail with Not Found error
138 _, _, _, err = kcSrc.Get(locatorInDst)
139 c.Assert(err.Error(), Equals, "Block not found")
142 // Test keep-rsync initialization, with srcKeepServicesJSON
143 func (s *ServerRequiredSuite) TestRsyncInitializeWithKeepServicesJSON(c *C) {
144 srcKeepServicesJSON = testKeepServicesJSON
146 setupRsync(c, false, true, 1)
148 localRoots := kcSrc.LocalRoots()
149 c.Check(localRoots, NotNil)
152 for k := range localRoots {
153 if k == "zzzzz-bi6l4-123456789012340" {
157 c.Check(foundIt, Equals, true)
160 for k := range localRoots {
161 if k == "zzzzz-bi6l4-123456789012341" {
165 c.Check(foundIt, Equals, true)
168 // Test keep-rsync initialization, with src and dst keep servers with blobSigningKey.
169 // Do a Put and Get in src, both of which should succeed.
170 // Do a Put and Get in dst, both of which should succeed.
171 // Do a Get in dst for the src hash, which should raise block not found error.
172 // Do a Get in src for the dst hash, which should raise block not found error.
173 func (s *ServerRequiredSuite) TestRsyncWithBlobSigning_PutInOne_GetFromOtherShouldFail(c *C) {
174 setupRsync(c, true, true, 1)
176 // Put a block in src using kcSrc and Get it
177 srcData := []byte("test-data1")
178 locatorInSrc := fmt.Sprintf("%x", md5.Sum(srcData))
180 hash, rep, err := kcSrc.PutB(srcData)
181 c.Check(hash, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, locatorInSrc))
182 c.Check(rep, Equals, 2)
183 c.Check(err, Equals, nil)
185 tomorrow := time.Now().AddDate(0, 0, 1)
186 signedLocator := keepclient.SignLocator(locatorInSrc, kcSrc.Arvados.ApiToken, tomorrow, []byte(blobSigningKey))
188 reader, blocklen, _, err := kcSrc.Get(signedLocator)
190 c.Check(blocklen, Equals, int64(10))
191 all, err := ioutil.ReadAll(reader)
192 c.Check(all, DeepEquals, srcData)
194 // Put a different block in src using kcSrc and Get it
195 dstData := []byte("test-data2")
196 locatorInDst := fmt.Sprintf("%x", md5.Sum(dstData))
198 hash, rep, err = kcDst.PutB(dstData)
199 c.Check(hash, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, locatorInDst))
200 c.Check(rep, Equals, 1)
201 c.Check(err, Equals, nil)
203 signedLocator = keepclient.SignLocator(locatorInDst, kcDst.Arvados.ApiToken, tomorrow, []byte(blobSigningKey))
205 reader, blocklen, _, err = kcDst.Get(signedLocator)
207 c.Check(blocklen, Equals, int64(10))
208 all, err = ioutil.ReadAll(reader)
209 c.Check(all, DeepEquals, dstData)
211 // Get srcLocator using kcDst should fail with Not Found error
212 signedLocator = keepclient.SignLocator(locatorInSrc, kcDst.Arvados.ApiToken, tomorrow, []byte(blobSigningKey))
213 _, _, _, err = kcDst.Get(locatorInSrc)
214 c.Assert(err.Error(), Equals, "Block not found")
216 // Get dstLocator using kcSrc should fail with Not Found error
217 signedLocator = keepclient.SignLocator(locatorInDst, kcSrc.Arvados.ApiToken, tomorrow, []byte(blobSigningKey))
218 _, _, _, err = kcSrc.Get(locatorInDst)
219 c.Assert(err.Error(), Equals, "Block not found")
222 // Test keep-rsync initialization with default replications count
223 func (s *ServerRequiredSuite) TestInitializeRsyncDefaultReplicationsCount(c *C) {
224 setupRsync(c, false, false, 0)
226 // Must have got default replications value as 2 from dst discovery document
227 c.Assert(kcDst.Want_replicas, Equals, 2)
230 // Test keep-rsync initialization with replications count argument
231 func (s *ServerRequiredSuite) TestInitializeRsyncReplicationsCount(c *C) {
232 setupRsync(c, false, false, 3)
234 // Since replications value is provided, default is not used
235 c.Assert(kcDst.Want_replicas, Equals, 3)
238 // Put some blocks in Src and some more in Dst
239 // And copy missing blocks from Src to Dst
240 func (s *ServerRequiredSuite) TestKeepRsync(c *C) {
241 testKeepRsync(c, false, "")
244 // Put some blocks in Src and some more in Dst with blob signing enabled.
245 // And copy missing blocks from Src to Dst
246 func (s *ServerRequiredSuite) TestKeepRsync_WithBlobSigning(c *C) {
247 testKeepRsync(c, true, "")
250 // Put some blocks in Src and some more in Dst
251 // Use prefix while doing rsync
252 // And copy missing blocks from Src to Dst
253 func (s *ServerRequiredSuite) TestKeepRsync_WithPrefix(c *C) {
254 data := []byte("test-data-4")
255 hash := fmt.Sprintf("%x", md5.Sum(data))
257 testKeepRsync(c, false, hash[0:3])
260 // Put some blocks in Src and some more in Dst
261 // Use prefix not in src while doing rsync
262 // And copy missing blocks from Src to Dst
263 func (s *ServerRequiredSuite) TestKeepRsync_WithNoSuchPrefixInSrc(c *C) {
264 testKeepRsync(c, false, "999")
267 // Put 5 blocks in src. Put 2 of those blocks in dst
268 // Hence there are 3 additional blocks in src
269 // Also, put 2 extra blocks in dst; they are hence only in dst
270 // Run rsync and verify that those 7 blocks are now available in dst
271 func testKeepRsync(c *C, enforcePermissions bool, prefix string) {
272 setupRsync(c, enforcePermissions, true, 1)
275 setupTestData(c, enforcePermissions, prefix)
277 err := performKeepRsync(kcSrc, kcDst, prefix)
280 // Now GetIndex from dst and verify that all 5 from src and the 2 extra blocks are found
281 dstIndex, err := getUniqueLocators(kcDst, "")
285 for _, locator := range srcLocators {
286 _, ok := dstIndex[locator]
287 c.Assert(ok, Equals, true)
290 for _, locator := range srcLocatorsMatchingPrefix {
291 _, ok := dstIndex[locator]
292 c.Assert(ok, Equals, true)
296 for _, locator := range extraDstLocators {
297 _, ok := dstIndex[locator]
298 c.Assert(ok, Equals, true)
302 // all blocks from src and the two extra blocks
303 c.Assert(len(dstIndex), Equals, len(srcLocators)+len(extraDstLocators))
305 // one matching prefix, 2 that were initially copied into dst along with src, and the extra blocks
306 c.Assert(len(dstIndex), Equals, len(srcLocatorsMatchingPrefix)+len(extraDstLocators)+2)
310 // Setup test data in src and dst.
311 var srcLocators, srcLocatorsMatchingPrefix, dstLocators, extraDstLocators []string
312 func setupTestData(c *C, enforcePermissions bool, indexPrefix string) {
313 srcLocators = []string{}
314 srcLocatorsMatchingPrefix = []string{}
315 dstLocators = []string{}
316 extraDstLocators = []string{}
318 // Put a few blocks in src using kcSrc
319 for i := 0; i < 5; i++ {
320 hash, _, err := kcSrc.PutB([]byte(fmt.Sprintf("test-data-%d", i)))
323 srcLocators = append(srcLocators, strings.Split(hash, "+A")[0])
324 if strings.HasPrefix(hash, indexPrefix) {
325 srcLocatorsMatchingPrefix = append(srcLocatorsMatchingPrefix, strings.Split(hash, "+A")[0])
329 // Put first two of those src blocks in dst using kcDst
330 for i := 0; i < 2; i++ {
331 hash, _, err := kcDst.PutB([]byte(fmt.Sprintf("test-data-%d", i)))
333 dstLocators = append(dstLocators, strings.Split(hash, "+A")[0])
336 // Put two more blocks in dst; they are not in src at all
337 for i := 0; i < 2; i++ {
338 hash, _, err := kcDst.PutB([]byte(fmt.Sprintf("other-data-%d", i)))
340 dstLocators = append(dstLocators, strings.Split(hash, "+A")[0])
341 extraDstLocators = append(extraDstLocators, strings.Split(hash, "+A")[0])
345 // Setup rsync using srcKeepServicesJSON with fake keepservers.
346 // Expect error during performKeepRsync due to unreachable src keepservers.
347 func (s *ServerRequiredSuite) TestErrorDuringRsync_FakeSrcKeepservers(c *C) {
348 srcKeepServicesJSON = testKeepServicesJSON
350 setupRsync(c, false, false, 1)
352 err := performKeepRsync(kcSrc, kcDst, "")
353 c.Check(strings.HasSuffix(err.Error(), "no such host"), Equals, true)
356 // Setup rsync using dstKeepServicesJSON with fake keepservers.
357 // Expect error during performKeepRsync due to unreachable dst keepservers.
358 func (s *ServerRequiredSuite) TestErrorDuringRsync_FakeDstKeepservers(c *C) {
359 dstKeepServicesJSON = testKeepServicesJSON
361 setupRsync(c, false, false, 1)
363 err := performKeepRsync(kcSrc, kcDst, "")
364 c.Check(strings.HasSuffix(err.Error(), "no such host"), Equals, true)
367 // Test rsync with signature error during Get from src.
368 func (s *ServerRequiredSuite) TestErrorDuringRsync_ErrorGettingBlockFromSrc(c *C) {
369 setupRsync(c, true, true, 1)
371 // put some blocks in src and dst
372 setupTestData(c, true, "")
374 // Change blob signing key to a fake key, so that Get from src fails
375 blobSigningKey = "123456789012345678901234yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc"
377 err := performKeepRsync(kcSrc, kcDst, "")
378 c.Check(strings.HasSuffix(err.Error(), "Block not found"), Equals, true)
381 // Test rsync with error during Put to src.
382 func (s *ServerRequiredSuite) TestErrorDuringRsync_ErrorPuttingBlockInDst(c *C) {
383 setupRsync(c, false, true, 1)
385 // put some blocks in src and dst
386 setupTestData(c, true, "")
388 // Increase Want_replicas on dst to result in insufficient replicas error during Put
389 kcDst.Want_replicas = 2
391 err := performKeepRsync(kcSrc, kcDst, "")
392 c.Check(strings.HasSuffix(err.Error(), "Could not write sufficient replicas"), Equals, true)
395 // Test loadConfig func
396 func (s *ServerRequiredSuite) TestLoadConfig(c *C) {
397 // Setup a src config file
398 srcFile := setupConfigFile(c, "src-config")
399 defer os.Remove(srcFile.Name())
400 srcConfigFile := srcFile.Name()
402 // Setup a dst config file
403 dstFile := setupConfigFile(c, "dst-config")
404 defer os.Remove(dstFile.Name())
405 dstConfigFile := dstFile.Name()
407 // load configuration from those files
408 srcConfig, dstConfig, err := loadConfig(srcConfigFile, dstConfigFile)
411 c.Assert(srcConfig.APIHost, Equals, "testhost")
412 c.Assert(srcConfig.APIToken, Equals, "testtoken")
413 c.Assert(srcConfig.APIHostInsecure, Equals, true)
414 c.Assert(srcConfig.ExternalClient, Equals, false)
416 c.Assert(dstConfig.APIHost, Equals, "testhost")
417 c.Assert(dstConfig.APIToken, Equals, "testtoken")
418 c.Assert(dstConfig.APIHostInsecure, Equals, true)
419 c.Assert(dstConfig.ExternalClient, Equals, false)
421 c.Assert(blobSigningKey, Equals, "abcdefg")
424 // Test loadConfig func without setting up the config files
425 func (s *ServerRequiredSuite) TestLoadConfig_MissingSrcConfig(c *C) {
426 _, _, err := loadConfig("", "")
427 c.Assert(err.Error(), Equals, "-src-config-file must be specified")
430 // Test loadConfig func - error reading src config
431 func (s *ServerRequiredSuite) TestLoadConfig_ErrorLoadingSrcConfig(c *C) {
432 _, _, err := loadConfig("no-such-config-file", "")
433 c.Assert(strings.HasSuffix(err.Error(), "no such file or directory"), Equals, true)
436 // Test loadConfig func with no dst config file specified
437 func (s *ServerRequiredSuite) TestLoadConfig_MissingDstConfig(c *C) {
438 // Setup a src config file
439 srcFile := setupConfigFile(c, "src-config")
440 defer os.Remove(srcFile.Name())
441 srcConfigFile := srcFile.Name()
443 // load configuration
444 _, _, err := loadConfig(srcConfigFile, "")
445 c.Assert(err.Error(), Equals, "-dst-config-file must be specified")
448 // Test loadConfig func
449 func (s *ServerRequiredSuite) TestLoadConfig_ErrorLoadingDstConfig(c *C) {
450 // Setup a src config file
451 srcFile := setupConfigFile(c, "src-config")
452 defer os.Remove(srcFile.Name())
453 srcConfigFile := srcFile.Name()
455 // load configuration
456 _, _, err := loadConfig(srcConfigFile, "no-such-config-file")
457 c.Assert(strings.HasSuffix(err.Error(), "no such file or directory"), Equals, true)
460 func setupConfigFile(c *C, name string) *os.File {
461 // Setup a config file
462 file, err := ioutil.TempFile(os.TempDir(), name)
465 fileContent := "ARVADOS_API_HOST=testhost\n"
466 fileContent += "ARVADOS_API_TOKEN=testtoken\n"
467 fileContent += "ARVADOS_API_HOST_INSECURE=true\n"
468 fileContent += "ARVADOS_BLOB_SIGNING_KEY=abcdefg"
470 _, err = file.Write([]byte(fileContent))