12 "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
13 "git.curoverse.com/arvados.git/sdk/go/arvadostest"
14 "git.curoverse.com/arvados.git/sdk/go/keepclient"
19 var kcSrc, kcDst *keepclient.KeepClient
20 var srcKeepServicesJSON, dstKeepServicesJSON, blobSigningKey string
21 var blobSignatureTTL = time.Duration(2*7*24) * time.Hour
25 srcKeepServicesJSON = ""
26 dstKeepServicesJSON = ""
31 // Gocheck boilerplate
32 func Test(t *testing.T) {
36 var _ = Suite(&ServerRequiredSuite{})
37 var _ = Suite(&ServerNotRequiredSuite{})
38 var _ = Suite(&DoMainTestSuite{})
40 type ServerRequiredSuite struct{}
42 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
43 arvadostest.StartAPI()
46 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
48 arvadostest.ResetEnv()
51 func (s *ServerRequiredSuite) SetUpTest(c *C) {
55 func (s *ServerRequiredSuite) TearDownTest(c *C) {
56 arvadostest.StopKeep(3)
59 func (s *ServerNotRequiredSuite) SetUpTest(c *C) {
63 type ServerNotRequiredSuite struct{}
65 type DoMainTestSuite struct {
69 func (s *DoMainTestSuite) SetUpTest(c *C) {
70 s.initialArgs = os.Args
71 os.Args = []string{"keep-rsync"}
75 func (s *DoMainTestSuite) TearDownTest(c *C) {
76 os.Args = s.initialArgs
79 var testKeepServicesJSON = "{ \"kind\":\"arvados#keepServiceList\", \"etag\":\"\", \"self_link\":\"\", \"offset\":null, \"limit\":null, \"items\":[ { \"href\":\"/keep_services/zzzzz-bi6l4-123456789012340\", \"kind\":\"arvados#keepService\", \"etag\":\"641234567890enhj7hzx432e5\", \"uuid\":\"zzzzz-bi6l4-123456789012340\", \"owner_uuid\":\"zzzzz-tpzed-123456789012345\", \"service_host\":\"keep0.zzzzz.arvadosapi.com\", \"service_port\":25107, \"service_ssl_flag\":false, \"service_type\":\"disk\", \"read_only\":false }, { \"href\":\"/keep_services/zzzzz-bi6l4-123456789012341\", \"kind\":\"arvados#keepService\", \"etag\":\"641234567890enhj7hzx432e5\", \"uuid\":\"zzzzz-bi6l4-123456789012341\", \"owner_uuid\":\"zzzzz-tpzed-123456789012345\", \"service_host\":\"keep0.zzzzz.arvadosapi.com\", \"service_port\":25108, \"service_ssl_flag\":false, \"service_type\":\"disk\", \"read_only\":false } ], \"items_available\":2 }"
81 // Testing keep-rsync needs two sets of keep services: src and dst.
82 // The test setup hence creates 3 servers instead of the default 2,
83 // and uses the first 2 as src and the 3rd as dst keep servers.
84 func setupRsync(c *C, enforcePermissions bool, replications int) {
86 var srcConfig apiConfig
87 srcConfig.APIHost = os.Getenv("ARVADOS_API_HOST")
88 srcConfig.APIToken = arvadostest.DataManagerToken
89 srcConfig.APIHostInsecure = arvadosclient.StringBool(os.Getenv("ARVADOS_API_HOST_INSECURE"))
92 var dstConfig apiConfig
93 dstConfig.APIHost = os.Getenv("ARVADOS_API_HOST")
94 dstConfig.APIToken = arvadostest.DataManagerToken
95 dstConfig.APIHostInsecure = arvadosclient.StringBool(os.Getenv("ARVADOS_API_HOST_INSECURE"))
97 if enforcePermissions {
98 blobSigningKey = arvadostest.BlobSigningKey
101 // Start Keep servers
102 arvadostest.StartKeep(3, enforcePermissions)
103 keepclient.RefreshServiceDiscovery()
107 kcSrc, _, err = setupKeepClient(srcConfig, srcKeepServicesJSON, false, 0, blobSignatureTTL)
110 kcDst, _, err = setupKeepClient(dstConfig, dstKeepServicesJSON, true, replications, 0)
113 srcRoots := map[string]string{}
114 dstRoots := map[string]string{}
115 for uuid, root := range kcSrc.LocalRoots() {
116 if strings.HasSuffix(uuid, "02") {
117 dstRoots[uuid] = root
119 srcRoots[uuid] = root
122 if srcKeepServicesJSON == "" {
123 kcSrc.SetServiceRoots(srcRoots, srcRoots, srcRoots)
125 if dstKeepServicesJSON == "" {
126 kcDst.SetServiceRoots(dstRoots, dstRoots, dstRoots)
129 if replications == 0 {
130 // Must have got default replications value of 2 from dst discovery document
131 c.Assert(kcDst.Want_replicas, Equals, 2)
133 // Since replications value is provided, it is used
134 c.Assert(kcDst.Want_replicas, Equals, replications)
138 func (s *ServerRequiredSuite) TestRsyncPutInOne_GetFromOtherShouldFail(c *C) {
139 setupRsync(c, false, 1)
141 // Put a block in src and verify that it is not found in dst
142 testNoCrosstalk(c, "test-data-1", kcSrc, kcDst)
144 // Put a block in dst and verify that it is not found in src
145 testNoCrosstalk(c, "test-data-2", kcDst, kcSrc)
148 func (s *ServerRequiredSuite) TestRsyncWithBlobSigning_PutInOne_GetFromOtherShouldFail(c *C) {
149 setupRsync(c, true, 1)
151 // Put a block in src and verify that it is not found in dst
152 testNoCrosstalk(c, "test-data-1", kcSrc, kcDst)
154 // Put a block in dst and verify that it is not found in src
155 testNoCrosstalk(c, "test-data-2", kcDst, kcSrc)
158 // Do a Put in the first and Get from the second,
159 // which should raise block not found error.
160 func testNoCrosstalk(c *C, testData string, kc1, kc2 *keepclient.KeepClient) {
161 // Put a block using kc1
162 locator, _, err := kc1.PutB([]byte(testData))
163 c.Assert(err, Equals, nil)
165 locator = strings.Split(locator, "+")[0]
166 _, _, _, err = kc2.Get(keepclient.SignLocator(locator, kc2.Arvados.ApiToken, time.Now().AddDate(0, 0, 1), blobSignatureTTL, []byte(blobSigningKey)))
167 c.Assert(err, NotNil)
168 c.Check(err.Error(), Equals, "Block not found")
171 // Test keep-rsync initialization, with srcKeepServicesJSON
172 func (s *ServerRequiredSuite) TestRsyncInitializeWithKeepServicesJSON(c *C) {
173 srcKeepServicesJSON = testKeepServicesJSON
175 setupRsync(c, false, 1)
177 localRoots := kcSrc.LocalRoots()
178 c.Check(localRoots, NotNil)
179 c.Check(localRoots["zzzzz-bi6l4-123456789012340"], Not(Equals), "")
180 c.Check(localRoots["zzzzz-bi6l4-123456789012341"], Not(Equals), "")
183 // Test keep-rsync initialization with default replications count
184 func (s *ServerRequiredSuite) TestInitializeRsyncDefaultReplicationsCount(c *C) {
185 setupRsync(c, false, 0)
188 // Test keep-rsync initialization with replications count argument
189 func (s *ServerRequiredSuite) TestInitializeRsyncReplicationsCount(c *C) {
190 setupRsync(c, false, 3)
193 // Put some blocks in Src and some more in Dst
194 // And copy missing blocks from Src to Dst
195 func (s *ServerRequiredSuite) TestKeepRsync(c *C) {
196 testKeepRsync(c, false, "")
199 // Put some blocks in Src and some more in Dst with blob signing enabled.
200 // And copy missing blocks from Src to Dst
201 func (s *ServerRequiredSuite) TestKeepRsync_WithBlobSigning(c *C) {
202 testKeepRsync(c, true, "")
205 // Put some blocks in Src and some more in Dst
206 // Use prefix while doing rsync
207 // And copy missing blocks from Src to Dst
208 func (s *ServerRequiredSuite) TestKeepRsync_WithPrefix(c *C) {
209 data := []byte("test-data-4")
210 hash := fmt.Sprintf("%x", md5.Sum(data))
212 testKeepRsync(c, false, hash[0:3])
213 c.Check(len(dstIndex) > len(dstLocators), Equals, true)
216 // Put some blocks in Src and some more in Dst
217 // Use prefix not in src while doing rsync
218 // And copy missing blocks from Src to Dst
219 func (s *ServerRequiredSuite) TestKeepRsync_WithNoSuchPrefixInSrc(c *C) {
220 testKeepRsync(c, false, "999")
221 c.Check(len(dstIndex), Equals, len(dstLocators))
224 // Put 5 blocks in src. Put 2 of those blocks in dst
225 // Hence there are 3 additional blocks in src
226 // Also, put 2 extra blocks in dst; they are hence only in dst
227 // Run rsync and verify that those 7 blocks are now available in dst
228 func testKeepRsync(c *C, enforcePermissions bool, prefix string) {
229 setupRsync(c, enforcePermissions, 1)
232 setupTestData(c, prefix)
234 err := performKeepRsync(kcSrc, kcDst, blobSignatureTTL, blobSigningKey, prefix)
237 // Now GetIndex from dst and verify that all 5 from src and the 2 extra blocks are found
238 dstIndex, err = getUniqueLocators(kcDst, "")
241 for _, locator := range srcLocatorsMatchingPrefix {
242 _, ok := dstIndex[locator]
243 c.Assert(ok, Equals, true)
246 for _, locator := range extraDstLocators {
247 _, ok := dstIndex[locator]
248 c.Assert(ok, Equals, true)
252 // all blocks from src and the two extra blocks
253 c.Assert(len(dstIndex), Equals, len(srcLocators)+len(extraDstLocators))
255 // 1 matching prefix and copied over, 2 that were initially copied into dst along with src, and the 2 extra blocks
256 c.Assert(len(dstIndex), Equals, len(srcLocatorsMatchingPrefix)+len(extraDstLocators)+2)
260 // Setup test data in src and dst.
261 var srcLocators, srcLocatorsMatchingPrefix, dstLocators, extraDstLocators []string
262 var dstIndex map[string]bool
264 func setupTestData(c *C, indexPrefix string) {
265 srcLocators = []string{}
266 srcLocatorsMatchingPrefix = []string{}
267 dstLocators = []string{}
268 extraDstLocators = []string{}
269 dstIndex = make(map[string]bool)
271 // Put a few blocks in src using kcSrc
272 for i := 0; i < 5; i++ {
273 hash, _, err := kcSrc.PutB([]byte(fmt.Sprintf("test-data-%d", i)))
276 srcLocators = append(srcLocators, strings.Split(hash, "+A")[0])
277 if strings.HasPrefix(hash, indexPrefix) {
278 srcLocatorsMatchingPrefix = append(srcLocatorsMatchingPrefix, strings.Split(hash, "+A")[0])
282 // Put first two of those src blocks in dst using kcDst
283 for i := 0; i < 2; i++ {
284 hash, _, err := kcDst.PutB([]byte(fmt.Sprintf("test-data-%d", i)))
286 dstLocators = append(dstLocators, strings.Split(hash, "+A")[0])
289 // Put two more blocks in dst; they are not in src at all
290 for i := 0; i < 2; i++ {
291 hash, _, err := kcDst.PutB([]byte(fmt.Sprintf("other-data-%d", i)))
293 dstLocators = append(dstLocators, strings.Split(hash, "+A")[0])
294 extraDstLocators = append(extraDstLocators, strings.Split(hash, "+A")[0])
298 // Setup rsync using srcKeepServicesJSON with fake keepservers.
299 // Expect error during performKeepRsync due to unreachable src keepservers.
300 func (s *ServerRequiredSuite) TestErrorDuringRsync_FakeSrcKeepservers(c *C) {
301 srcKeepServicesJSON = testKeepServicesJSON
303 setupRsync(c, false, 1)
305 err := performKeepRsync(kcSrc, kcDst, blobSignatureTTL, "", "")
306 c.Assert(err, NotNil)
307 c.Check(err.Error(), Matches, ".*no such host.*")
310 // Setup rsync using dstKeepServicesJSON with fake keepservers.
311 // Expect error during performKeepRsync due to unreachable dst keepservers.
312 func (s *ServerRequiredSuite) TestErrorDuringRsync_FakeDstKeepservers(c *C) {
313 dstKeepServicesJSON = testKeepServicesJSON
315 setupRsync(c, false, 1)
317 err := performKeepRsync(kcSrc, kcDst, blobSignatureTTL, "", "")
318 c.Assert(err, NotNil)
319 c.Check(err.Error(), Matches, ".*no such host.*")
322 // Test rsync with signature error during Get from src.
323 func (s *ServerRequiredSuite) TestErrorDuringRsync_ErrorGettingBlockFromSrc(c *C) {
324 setupRsync(c, true, 1)
326 // put some blocks in src and dst
329 // Change blob signing key to a fake key, so that Get from src fails
330 blobSigningKey = "thisisfakeblobsigningkey"
332 err := performKeepRsync(kcSrc, kcDst, blobSignatureTTL, blobSigningKey, "")
333 c.Assert(err, NotNil)
334 c.Check(err.Error(), Matches, ".*HTTP 403 \"Forbidden\".*")
337 // Test rsync with error during Put to src.
338 func (s *ServerRequiredSuite) TestErrorDuringRsync_ErrorPuttingBlockInDst(c *C) {
339 setupRsync(c, false, 1)
341 // put some blocks in src and dst
344 // Increase Want_replicas on dst to result in insufficient replicas error during Put
345 kcDst.Want_replicas = 2
347 err := performKeepRsync(kcSrc, kcDst, blobSignatureTTL, blobSigningKey, "")
348 c.Assert(err, NotNil)
349 c.Check(err.Error(), Matches, ".*Could not write sufficient replicas.*")
352 // Test loadConfig func
353 func (s *ServerNotRequiredSuite) TestLoadConfig(c *C) {
354 // Setup a src config file
355 srcFile := setupConfigFile(c, "src-config")
356 defer os.Remove(srcFile.Name())
357 srcConfigFile := srcFile.Name()
359 // Setup a dst config file
360 dstFile := setupConfigFile(c, "dst-config")
361 defer os.Remove(dstFile.Name())
362 dstConfigFile := dstFile.Name()
364 // load configuration from those files
365 srcConfig, srcBlobSigningKey, err := loadConfig(srcConfigFile)
368 c.Assert(srcConfig.APIHost, Equals, os.Getenv("ARVADOS_API_HOST"))
369 c.Assert(srcConfig.APIToken, Equals, arvadostest.DataManagerToken)
370 c.Assert(srcConfig.APIHostInsecure, Equals, arvadosclient.StringBool(os.Getenv("ARVADOS_API_HOST_INSECURE")))
371 c.Assert(srcConfig.ExternalClient, Equals, false)
373 dstConfig, _, err := loadConfig(dstConfigFile)
376 c.Assert(dstConfig.APIHost, Equals, os.Getenv("ARVADOS_API_HOST"))
377 c.Assert(dstConfig.APIToken, Equals, arvadostest.DataManagerToken)
378 c.Assert(dstConfig.APIHostInsecure, Equals, arvadosclient.StringBool(os.Getenv("ARVADOS_API_HOST_INSECURE")))
379 c.Assert(dstConfig.ExternalClient, Equals, false)
381 c.Assert(srcBlobSigningKey, Equals, "abcdefg")
384 // Test loadConfig func without setting up the config files
385 func (s *ServerNotRequiredSuite) TestLoadConfig_MissingSrcConfig(c *C) {
386 _, _, err := loadConfig("")
387 c.Assert(err.Error(), Equals, "config file not specified")
390 // Test loadConfig func - error reading config
391 func (s *ServerNotRequiredSuite) TestLoadConfig_ErrorLoadingSrcConfig(c *C) {
392 _, _, err := loadConfig("no-such-config-file")
393 c.Assert(err, NotNil)
394 c.Check(err.Error(), Matches, ".*no such file or directory.*")
397 func (s *ServerNotRequiredSuite) TestSetupKeepClient_NoBlobSignatureTTL(c *C) {
398 var srcConfig apiConfig
399 srcConfig.APIHost = os.Getenv("ARVADOS_API_HOST")
400 srcConfig.APIToken = arvadostest.DataManagerToken
401 srcConfig.APIHostInsecure = arvadosclient.StringBool(os.Getenv("ARVADOS_API_HOST_INSECURE"))
403 _, ttl, err := setupKeepClient(srcConfig, srcKeepServicesJSON, false, 0, 0)
405 c.Assert(ttl, Equals, blobSignatureTTL)
408 func setupConfigFile(c *C, name string) *os.File {
409 // Setup a config file
410 file, err := ioutil.TempFile(os.TempDir(), name)
413 fileContent := "ARVADOS_API_HOST=" + os.Getenv("ARVADOS_API_HOST") + "\n"
414 fileContent += "ARVADOS_API_TOKEN=" + arvadostest.DataManagerToken + "\n"
415 fileContent += "ARVADOS_API_HOST_INSECURE=" + os.Getenv("ARVADOS_API_HOST_INSECURE") + "\n"
416 fileContent += "ARVADOS_EXTERNAL_CLIENT=false\n"
417 fileContent += "ARVADOS_BLOB_SIGNING_KEY=abcdefg"
419 _, err = file.Write([]byte(fileContent))
425 func (s *DoMainTestSuite) Test_doMain_NoSrcConfig(c *C) {
427 c.Assert(err, NotNil)
428 c.Assert(err.Error(), Equals, "Error loading src configuration from file: config file not specified")
431 func (s *DoMainTestSuite) Test_doMain_SrcButNoDstConfig(c *C) {
432 srcConfig := setupConfigFile(c, "src")
433 args := []string{"-replications", "3", "-src", srcConfig.Name()}
434 os.Args = append(os.Args, args...)
436 c.Assert(err, NotNil)
437 c.Assert(err.Error(), Equals, "Error loading dst configuration from file: config file not specified")
440 func (s *DoMainTestSuite) Test_doMain_BadSrcConfig(c *C) {
441 args := []string{"-src", "abcd"}
442 os.Args = append(os.Args, args...)
444 c.Assert(err, NotNil)
445 c.Assert(err.Error(), Matches, "Error loading src configuration from file: Error reading config file.*")
448 func (s *DoMainTestSuite) Test_doMain_WithReplicationsButNoSrcConfig(c *C) {
449 args := []string{"-replications", "3"}
450 os.Args = append(os.Args, args...)
453 c.Assert(err.Error(), Equals, "Error loading src configuration from file: config file not specified")
456 func (s *DoMainTestSuite) Test_doMainWithSrcAndDstConfig(c *C) {
457 srcConfig := setupConfigFile(c, "src")
458 dstConfig := setupConfigFile(c, "dst")
459 args := []string{"-src", srcConfig.Name(), "-dst", dstConfig.Name()}
460 os.Args = append(os.Args, args...)
462 // Start keepservers. Since we are not doing any tweaking as
463 // in setupRsync func, kcSrc and kcDst will be the same and no
464 // actual copying to dst will happen, but that's ok.
465 arvadostest.StartKeep(2, false)
466 defer arvadostest.StopKeep(2)
467 keepclient.RefreshServiceDiscovery()