Merge branch 'master' into 7748-datamanager-dry-run

Conflicts:
	services/datamanager/datamanager_test.go

diff --cc services/datamanager/datamanager_test.go
index f432c0717021dfc5145cc16af47385a57c93a1e7,75950d1314d71078acab0f027d3dcc4eb2f509c2..67b0e79f14fea4eb75a64d12f8907b9eabf4bf14
--- 1/services/datamanager/datamanager_test.go
--- 2/services/datamanager/datamanager_test.go
+++ b/services/datamanager/datamanager_test.go
@@@ -625,14 -624,17 +624,22 @@@ func createMultiStreamBlockCollection(
  // Also, create stray block and backdate it.
  // After datamanager run: expect blocks from the collection, but not the stray block.
  func TestManifestWithMultipleStreamsAndBlocks(t *testing.T) {
-       testManifestWithMultipleStreamsAndBlocks(t, false)
 -      testManifestWithMultipleStreamsAndBlocks(t, 100, 10, "")
++      testManifestWithMultipleStreamsAndBlocks(t, 100, 10, "", false)
  }
  
 -      testManifestWithMultipleStreamsAndBlocks(t, 2, 2, "testblobstore")
+ // Same test as TestManifestWithMultipleStreamsAndBlocks with an additional
+ // keepstore of a service type other than "disk". Only the "disk" type services
+ // will be indexed by datamanager and hence should work the same way.
+ func TestManifestWithMultipleStreamsAndBlocks_WithOneUnsupportedKeepServer(t *testing.T) {
 -func testManifestWithMultipleStreamsAndBlocks(t *testing.T, numStreams, numBlocks int, createExtraKeepServerWithType string) {
++      testManifestWithMultipleStreamsAndBlocks(t, 2, 2, "testblobstore", false)
+ }
+ 
-       testManifestWithMultipleStreamsAndBlocks(t, true)
++// Test datamanager with dry-run. Expect no block to be deleted.
 +func TestManifestWithMultipleStreamsAndBlocks_DryRun(t *testing.T) {
- func testManifestWithMultipleStreamsAndBlocks(t *testing.T, isDryRun bool) {
++      testManifestWithMultipleStreamsAndBlocks(t, 2, 2, "", true)
 +}
 +
++func testManifestWithMultipleStreamsAndBlocks(t *testing.T, numStreams, numBlocks int, createExtraKeepServerWithType string, isDryRun bool) {
        defer TearDownDataManagerTest(t)
        SetupDataManagerTest(t)
  
@@@ -658,15 -660,41 +665,47 @@@
        // also backdate the stray old block
        backdateBlocks(t, []string{strayOldBlock})
  
+       // If requested, create an extra keepserver with the given type
+       // This should be ignored during indexing and hence not change the datamanager outcome
+       var extraKeepServerUUID string
+       if createExtraKeepServerWithType != "" {
+               extraKeepServerUUID = addExtraKeepServer(t, createExtraKeepServerWithType)
+               defer deleteExtraKeepServer(extraKeepServerUUID)
+       }
+ 
        // run datamanager
 +      dryRun = isDryRun
        dataManagerSingleRun(t)
  
 -      // verify that strayOldBlock is not to be found, but the collections blocks are still there
 -      verifyBlocks(t, []string{strayOldBlock}, oldBlocks, 2)
 +      if dryRun {
 +              // verify that all blocks, including strayOldBlock, are still to be found
 +              verifyBlocks(t, nil, expected, 2)
 +      } else {
 +              // verify that strayOldBlock is not to be found, but the collections blocks are still there
 +              verifyBlocks(t, []string{strayOldBlock}, oldBlocks, 2)
 +      }
  }
+ 
+ // Add one more keepstore with the given service type
+ func addExtraKeepServer(t *testing.T, serviceType string) string {
+       defer switchToken(arvadostest.AdminToken)()
+ 
+       extraKeepService := make(arvadosclient.Dict)
+       err := arv.Create("keep_services",
+               arvadosclient.Dict{"keep_service": arvadosclient.Dict{
+                       "service_host":     "localhost",
+                       "service_port":     "21321",
+                       "service_ssl_flag": false,
+                       "service_type":     serviceType}},
+               &extraKeepService)
+       if err != nil {
+               t.Fatal(err)
+       }
+ 
+       return extraKeepService["uuid"].(string)
+ }
+ 
+ func deleteExtraKeepServer(uuid string) {
+       defer switchToken(arvadostest.AdminToken)()
+       arv.Delete("keep_services", uuid, nil, nil)
+ }

diff --cc services/datamanager/keep/keep.go
Simple merge

diff --cc services/datamanager/keep/keep_test.go
Simple merge