lib/config/deprecated_keepstore_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package config
   6
   7 import (
   8         "bytes"
   9         "encoding/json"
  10         "fmt"
  11         "io"
  12         "io/ioutil"
  13         "os"
  14         "sort"
  15         "strconv"
  16         "strings"
  17         "text/template"
  18         "time"
  19
  20         "git.curoverse.com/arvados.git/sdk/go/arvados"
  21         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  22         check "gopkg.in/check.v1"
  23 )
  24
  25 type KeepstoreMigrationSuite struct {
  26         hostname string // blank = use test system's hostname
  27         ksByPort map[int]arvados.KeepService
  28 }
  29
  30 var _ = check.Suite(&KeepstoreMigrationSuite{})
  31
  32 func (s *KeepstoreMigrationSuite) SetUpSuite(c *check.C) {
  33         // We don't need the keepstore servers, but we do need
  34         // keep_services listings that point to localhost, rather than
  35         // the apiserver fixtures that point to fictional hosts
  36         // keep*.zzzzz.arvadosapi.com.
  37
  38         client := arvados.NewClientFromEnv()
  39
  40         // Delete existing non-proxy listings.
  41         var svcList arvados.KeepServiceList
  42         err := client.RequestAndDecode(&svcList, "GET", "arvados/v1/keep_services", nil, nil)
  43         c.Assert(err, check.IsNil)
  44         for _, ks := range svcList.Items {
  45                 if ks.ServiceType != "proxy" {
  46                         err = client.RequestAndDecode(new(struct{}), "DELETE", "arvados/v1/keep_services/"+ks.UUID, nil, nil)
  47                         c.Assert(err, check.IsNil)
  48                 }
  49         }
  50         // Add new fake listings.
  51         s.ksByPort = map[int]arvados.KeepService{}
  52         for _, port := range []int{25107, 25108} {
  53                 var ks arvados.KeepService
  54                 err = client.RequestAndDecode(&ks, "POST", "arvados/v1/keep_services", nil, map[string]interface{}{
  55                         "keep_service": map[string]interface{}{
  56                                 "service_type": "disk",
  57                                 "service_host": "localhost",
  58                                 "service_port": port,
  59                         },
  60                 })
  61                 c.Assert(err, check.IsNil)
  62                 s.ksByPort[port] = ks
  63         }
  64 }
  65
  66 func (s *KeepstoreMigrationSuite) checkEquivalentWithKeepstoreConfig(c *check.C, keepstoreconfig, clusterconfig, expectedconfig string) {
  67         keepstorefile, err := ioutil.TempFile("", "")
  68         c.Assert(err, check.IsNil)
  69         defer os.Remove(keepstorefile.Name())
  70         _, err = io.WriteString(keepstorefile, keepstoreconfig)
  71         c.Assert(err, check.IsNil)
  72         err = keepstorefile.Close()
  73         c.Assert(err, check.IsNil)
  74
  75         gotldr := testLoader(c, clusterconfig, nil)
  76         gotldr.KeepstorePath = keepstorefile.Name()
  77         expectedldr := testLoader(c, expectedconfig, nil)
  78         checkEquivalentLoaders(c, gotldr, expectedldr)
  79 }
  80
  81 func (s *KeepstoreMigrationSuite) TestDeprecatedKeepstoreConfig(c *check.C) {
  82         keyfile, err := ioutil.TempFile("", "")
  83         c.Assert(err, check.IsNil)
  84         defer os.Remove(keyfile.Name())
  85         io.WriteString(keyfile, "blobsigningkey\n")
  86
  87         hostname, err := os.Hostname()
  88         c.Assert(err, check.IsNil)
  89
  90         s.checkEquivalentWithKeepstoreConfig(c, `
  91 Listen: ":25107"
  92 Debug: true
  93 LogFormat: text
  94 MaxBuffers: 1234
  95 MaxRequests: 2345
  96 BlobSignatureTTL: 123m
  97 BlobSigningKeyFile: `+keyfile.Name()+`
  98 Volumes:
  99 - Type: Directory
 100   Root: /tmp
 101 `, `
 102 Clusters:
 103   z1111:
 104     SystemRootToken: `+arvadostest.AdminToken+`
 105     TLS: {Insecure: true}
 106     Services:
 107       Controller:
 108         ExternalURL: "https://`+os.Getenv("ARVADOS_API_HOST")+`"
 109 `, `
 110 Clusters:
 111   z1111:
 112     SystemRootToken: `+arvadostest.AdminToken+`
 113     TLS: {Insecure: true}
 114     Services:
 115       Keepstore:
 116         InternalURLs:
 117           "http://`+hostname+`:25107": {Rendezvous: `+s.ksByPort[25107].UUID[12:]+`}
 118       Controller:
 119         ExternalURL: "https://`+os.Getenv("ARVADOS_API_HOST")+`"
 120     SystemLogs:
 121       Format: text
 122       LogLevel: debug
 123     API:
 124       MaxKeepBlobBuffers: 1234
 125       MaxConcurrentRequests: 2345
 126     Collections:
 127       BlobSigningTTL: 123m
 128       BlobSigningKey: blobsigningkey
 129     Volumes:
 130       z1111-nyw5e-`+s.ksByPort[25107].UUID[12:]+`:
 131         AccessViaHosts:
 132           "http://`+hostname+`:25107":
 133             ReadOnly: false
 134         Driver: Directory
 135         DriverParameters:
 136           Root: /tmp
 137           Serialize: false
 138         ReadOnly: false
 139         Replication: 1
 140         StorageClasses: {}
 141 `)
 142 }
 143
 144 func (s *KeepstoreMigrationSuite) TestDiscoverLocalVolumes(c *check.C) {
 145         tmpd, err := ioutil.TempDir("", "")
 146         c.Assert(err, check.IsNil)
 147         defer os.RemoveAll(tmpd)
 148         err = os.Mkdir(tmpd+"/keep", 0777)
 149         c.Assert(err, check.IsNil)
 150
 151         tmpf, err := ioutil.TempFile("", "")
 152         c.Assert(err, check.IsNil)
 153         defer os.Remove(tmpf.Name())
 154
 155         // read/write
 156         _, err = fmt.Fprintf(tmpf, "/dev/xvdb %s ext4 rw,noexec 0 0\n", tmpd)
 157         c.Assert(err, check.IsNil)
 158
 159         s.testDeprecatedVolume(c, "DiscoverVolumesFromMountsFile: "+tmpf.Name(), arvados.Volume{
 160                 Driver:      "Directory",
 161                 ReadOnly:    false,
 162                 Replication: 1,
 163         }, &arvados.DirectoryVolumeDriverParameters{
 164                 Root:      tmpd + "/keep",
 165                 Serialize: false,
 166         }, &arvados.DirectoryVolumeDriverParameters{})
 167
 168         // read-only
 169         tmpf.Seek(0, os.SEEK_SET)
 170         tmpf.Truncate(0)
 171         _, err = fmt.Fprintf(tmpf, "/dev/xvdb %s ext4 ro,noexec 0 0\n", tmpd)
 172         c.Assert(err, check.IsNil)
 173
 174         s.testDeprecatedVolume(c, "DiscoverVolumesFromMountsFile: "+tmpf.Name(), arvados.Volume{
 175                 Driver:      "Directory",
 176                 ReadOnly:    true,
 177                 Replication: 1,
 178         }, &arvados.DirectoryVolumeDriverParameters{
 179                 Root:      tmpd + "/keep",
 180                 Serialize: false,
 181         }, &arvados.DirectoryVolumeDriverParameters{})
 182 }
 183
 184 func (s *KeepstoreMigrationSuite) TestDeprecatedVolumes(c *check.C) {
 185         accesskeyfile, err := ioutil.TempFile("", "")
 186         c.Assert(err, check.IsNil)
 187         defer os.Remove(accesskeyfile.Name())
 188         io.WriteString(accesskeyfile, "accesskeydata\n")
 189
 190         secretkeyfile, err := ioutil.TempFile("", "")
 191         c.Assert(err, check.IsNil)
 192         defer os.Remove(secretkeyfile.Name())
 193         io.WriteString(secretkeyfile, "secretkeydata\n")
 194
 195         // s3, empty/default
 196         s.testDeprecatedVolume(c, `
 197 Volumes:
 198 - Type: S3
 199 `, arvados.Volume{
 200                 Driver:      "S3",
 201                 Replication: 1,
 202         }, &arvados.S3VolumeDriverParameters{}, &arvados.S3VolumeDriverParameters{})
 203
 204         // s3, fully configured
 205         s.testDeprecatedVolume(c, `
 206 Volumes:
 207 - Type: S3
 208   AccessKeyFile: `+accesskeyfile.Name()+`
 209   SecretKeyFile: `+secretkeyfile.Name()+`
 210   Endpoint: https://storage.googleapis.com
 211   Region: us-east-1z
 212   Bucket: testbucket
 213   LocationConstraint: true
 214   IndexPageSize: 1234
 215   S3Replication: 4
 216   ConnectTimeout: 3m
 217   ReadTimeout: 4m
 218   RaceWindow: 5m
 219   UnsafeDelete: true
 220 `, arvados.Volume{
 221                 Driver:      "S3",
 222                 Replication: 4,
 223         }, &arvados.S3VolumeDriverParameters{
 224                 AccessKey:          "accesskeydata",
 225                 SecretKey:          "secretkeydata",
 226                 Endpoint:           "https://storage.googleapis.com",
 227                 Region:             "us-east-1z",
 228                 Bucket:             "testbucket",
 229                 LocationConstraint: true,
 230                 IndexPageSize:      1234,
 231                 ConnectTimeout:     arvados.Duration(time.Minute * 3),
 232                 ReadTimeout:        arvados.Duration(time.Minute * 4),
 233                 RaceWindow:         arvados.Duration(time.Minute * 5),
 234                 UnsafeDelete:       true,
 235         }, &arvados.S3VolumeDriverParameters{})
 236
 237         // azure, empty/default
 238         s.testDeprecatedVolume(c, `
 239 Volumes:
 240 - Type: Azure
 241 `, arvados.Volume{
 242                 Driver:      "Azure",
 243                 Replication: 1,
 244         }, &arvados.AzureVolumeDriverParameters{}, &arvados.AzureVolumeDriverParameters{})
 245
 246         // azure, fully configured
 247         s.testDeprecatedVolume(c, `
 248 Volumes:
 249 - Type: Azure
 250   ReadOnly: true
 251   StorageAccountName: storageacctname
 252   StorageAccountKeyFile: `+secretkeyfile.Name()+`
 253   StorageBaseURL: https://example.example
 254   ContainerName: testctr
 255   LocationConstraint: true
 256   AzureReplication: 4
 257   RequestTimeout: 3m
 258   ListBlobsRetryDelay: 4m
 259   ListBlobsMaxAttempts: 5
 260 `, arvados.Volume{
 261                 Driver:      "Azure",
 262                 ReadOnly:    true,
 263                 Replication: 4,
 264         }, &arvados.AzureVolumeDriverParameters{
 265                 StorageAccountName:   "storageacctname",
 266                 StorageAccountKey:    "secretkeydata",
 267                 StorageBaseURL:       "https://example.example",
 268                 ContainerName:        "testctr",
 269                 RequestTimeout:       arvados.Duration(time.Minute * 3),
 270                 ListBlobsRetryDelay:  arvados.Duration(time.Minute * 4),
 271                 ListBlobsMaxAttempts: 5,
 272         }, &arvados.AzureVolumeDriverParameters{})
 273
 274         // directory, empty/default
 275         s.testDeprecatedVolume(c, `
 276 Volumes:
 277 - Type: Directory
 278   Root: /tmp/xyzzy
 279 `, arvados.Volume{
 280                 Driver:      "Directory",
 281                 Replication: 1,
 282         }, &arvados.DirectoryVolumeDriverParameters{
 283                 Root: "/tmp/xyzzy",
 284         }, &arvados.DirectoryVolumeDriverParameters{})
 285
 286         // directory, fully configured
 287         s.testDeprecatedVolume(c, `
 288 Volumes:
 289 - Type: Directory
 290   ReadOnly: true
 291   Root: /tmp/xyzzy
 292   DirectoryReplication: 4
 293   Serialize: true
 294 `, arvados.Volume{
 295                 Driver:      "Directory",
 296                 ReadOnly:    true,
 297                 Replication: 4,
 298         }, &arvados.DirectoryVolumeDriverParameters{
 299                 Root:      "/tmp/xyzzy",
 300                 Serialize: true,
 301         }, &arvados.DirectoryVolumeDriverParameters{})
 302 }
 303
 304 func (s *KeepstoreMigrationSuite) testDeprecatedVolume(c *check.C, oldconfigdata string, expectvol arvados.Volume, expectparams interface{}, paramsdst interface{}) {
 305         hostname := s.hostname
 306         if hostname == "" {
 307                 h, err := os.Hostname()
 308                 c.Assert(err, check.IsNil)
 309                 hostname = h
 310         }
 311
 312         oldconfig, err := ioutil.TempFile("", "")
 313         c.Assert(err, check.IsNil)
 314         defer os.Remove(oldconfig.Name())
 315         io.WriteString(oldconfig, "Listen: :12345\n"+oldconfigdata)
 316         if !strings.Contains(oldconfigdata, "DiscoverVolumesFromMountsFile") {
 317                 // Prevent tests from looking at the real /proc/mounts on the test host.
 318                 io.WriteString(oldconfig, "\nDiscoverVolumesFromMountsFile: /dev/null\n")
 319         }
 320
 321         ldr := testLoader(c, "Clusters: {z1111: {}}", nil)
 322         ldr.KeepstorePath = oldconfig.Name()
 323         cfg, err := ldr.Load()
 324         c.Assert(err, check.IsNil)
 325         cc := cfg.Clusters["z1111"]
 326         c.Check(cc.Volumes, check.HasLen, 1)
 327         for uuid, v := range cc.Volumes {
 328                 c.Check(uuid, check.HasLen, 27)
 329                 c.Check(v.Driver, check.Equals, expectvol.Driver)
 330                 c.Check(v.Replication, check.Equals, expectvol.Replication)
 331
 332                 avh, ok := v.AccessViaHosts[arvados.URL{Scheme: "http", Host: hostname + ":12345"}]
 333                 c.Check(ok, check.Equals, true)
 334                 c.Check(avh.ReadOnly, check.Equals, expectvol.ReadOnly)
 335
 336                 err := json.Unmarshal(v.DriverParameters, paramsdst)
 337                 c.Check(err, check.IsNil)
 338                 c.Check(paramsdst, check.DeepEquals, expectparams)
 339         }
 340 }
 341
 342 // How we handle a volume from a legacy keepstore config file depends
 343 // on whether it's writable, whether a volume using the same cloud
 344 // backend already exists in the cluster config, and (if so) whether
 345 // it already has an AccessViaHosts entry for this host.
 346 //
 347 // In all cases, we should end up with an AccessViaHosts entry for
 348 // this host, to indicate that the current host's volumes have been
 349 // migrated.
 350
 351 // Same backend already referenced in cluster config, this host
 352 // already listed in AccessViaHosts --> no change, except possibly
 353 // updating the ReadOnly flag on the AccessViaHosts entry.
 354 func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_AlreadyMigrated(c *check.C) {
 355         before, after := s.loadWithKeepstoreConfig(c, `
 356 Listen: :12345
 357 Volumes:
 358 - Type: S3
 359   Endpoint: https://storage.googleapis.com
 360   Region: us-east-1z
 361   Bucket: alreadymigrated
 362   S3Replication: 3
 363 `)
 364         checkEqualYAML(c, after, before)
 365 }
 366
 367 // Writable volume, same cloud backend already referenced in cluster
 368 // config --> change UUID to match this keepstore's UUID.
 369 func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_UpdateUUID(c *check.C) {
 370         port, expectUUID := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
 371
 372         before, after := s.loadWithKeepstoreConfig(c, `
 373 Listen: :`+strconv.Itoa(port)+`
 374 Volumes:
 375 - Type: S3
 376   Endpoint: https://storage.googleapis.com
 377   Region: us-east-1z
 378   Bucket: readonlyonother
 379   S3Replication: 3
 380 `)
 381         c.Check(after, check.HasLen, len(before))
 382         newuuids := s.findAddedVolumes(c, before, after, 1)
 383         newvol := after[newuuids[0]]
 384
 385         var params arvados.S3VolumeDriverParameters
 386         json.Unmarshal(newvol.DriverParameters, &params)
 387         c.Check(params.Bucket, check.Equals, "readonlyonother")
 388         c.Check(newuuids[0], check.Equals, expectUUID)
 389 }
 390
 391 // Writable volume, same cloud backend not yet referenced --> add a
 392 // new volume, with UUID to match this keepstore's UUID.
 393 func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_AddCloudVolume(c *check.C) {
 394         port, expectUUID := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
 395
 396         before, after := s.loadWithKeepstoreConfig(c, `
 397 Listen: :`+strconv.Itoa(port)+`
 398 Volumes:
 399 - Type: S3
 400   Endpoint: https://storage.googleapis.com
 401   Region: us-east-1z
 402   Bucket: bucket-to-migrate
 403   S3Replication: 3
 404 `)
 405         newuuids := s.findAddedVolumes(c, before, after, 1)
 406         newvol := after[newuuids[0]]
 407
 408         var params arvados.S3VolumeDriverParameters
 409         json.Unmarshal(newvol.DriverParameters, &params)
 410         c.Check(params.Bucket, check.Equals, "bucket-to-migrate")
 411         c.Check(newvol.Replication, check.Equals, 3)
 412
 413         c.Check(newuuids[0], check.Equals, expectUUID)
 414 }
 415
 416 // Writable volume, same filesystem backend already referenced in
 417 // cluster config, but this host isn't in AccessViaHosts --> add a new
 418 // volume, with UUID to match this keepstore's UUID (filesystem-backed
 419 // volumes are assumed to be different on different hosts, even if
 420 // paths are the same).
 421 func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_AddLocalVolume(c *check.C) {
 422         before, after := s.loadWithKeepstoreConfig(c, `
 423 Listen: :12345
 424 Volumes:
 425 - Type: Directory
 426   Root: /data/sdd
 427   DirectoryReplication: 2
 428 `)
 429         newuuids := s.findAddedVolumes(c, before, after, 1)
 430         newvol := after[newuuids[0]]
 431
 432         var params arvados.DirectoryVolumeDriverParameters
 433         json.Unmarshal(newvol.DriverParameters, &params)
 434         c.Check(params.Root, check.Equals, "/data/sdd")
 435         c.Check(newvol.Replication, check.Equals, 2)
 436 }
 437
 438 // Writable volume, same filesystem backend already referenced in
 439 // cluster config, and this host is already listed in AccessViaHosts
 440 // --> already migrated, don't change anything.
 441 func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_LocalVolumeAlreadyMigrated(c *check.C) {
 442         before, after := s.loadWithKeepstoreConfig(c, `
 443 Listen: :12345
 444 Volumes:
 445 - Type: Directory
 446   Root: /data/sde
 447   DirectoryReplication: 2
 448 `)
 449         checkEqualYAML(c, after, before)
 450 }
 451
 452 // Multiple writable cloud-backed volumes --> one of them will get a
 453 // UUID matching this keepstore's UUID.
 454 func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_AddMultipleCloudVolumes(c *check.C) {
 455         port, expectUUID := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
 456
 457         before, after := s.loadWithKeepstoreConfig(c, `
 458 Listen: :`+strconv.Itoa(port)+`
 459 Volumes:
 460 - Type: S3
 461   Endpoint: https://storage.googleapis.com
 462   Region: us-east-1z
 463   Bucket: first-bucket-to-migrate
 464   S3Replication: 3
 465 - Type: S3
 466   Endpoint: https://storage.googleapis.com
 467   Region: us-east-1z
 468   Bucket: second-bucket-to-migrate
 469   S3Replication: 3
 470 `)
 471         newuuids := s.findAddedVolumes(c, before, after, 2)
 472         // Sort by bucket name (so "first" comes before "second")
 473         params := map[string]arvados.S3VolumeDriverParameters{}
 474         for _, uuid := range newuuids {
 475                 var p arvados.S3VolumeDriverParameters
 476                 json.Unmarshal(after[uuid].DriverParameters, &p)
 477                 params[uuid] = p
 478         }
 479         sort.Slice(newuuids, func(i, j int) bool { return params[newuuids[i]].Bucket < params[newuuids[j]].Bucket })
 480         newvol0, newvol1 := after[newuuids[0]], after[newuuids[1]]
 481         params0, params1 := params[newuuids[0]], params[newuuids[1]]
 482
 483         c.Check(params0.Bucket, check.Equals, "first-bucket-to-migrate")
 484         c.Check(newvol0.Replication, check.Equals, 3)
 485
 486         c.Check(params1.Bucket, check.Equals, "second-bucket-to-migrate")
 487         c.Check(newvol1.Replication, check.Equals, 3)
 488
 489         // Don't care which one gets the special UUID
 490         if newuuids[0] != expectUUID {
 491                 c.Check(newuuids[1], check.Equals, expectUUID)
 492         }
 493 }
 494
 495 // Non-writable volume, same cloud backend already referenced in
 496 // cluster config --> add this host to AccessViaHosts with
 497 // ReadOnly==true
 498 func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_UpdateWithReadOnly(c *check.C) {
 499         port, _ := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
 500         before, after := s.loadWithKeepstoreConfig(c, `
 501 Listen: :`+strconv.Itoa(port)+`
 502 Volumes:
 503 - Type: S3
 504   Endpoint: https://storage.googleapis.com
 505   Region: us-east-1z
 506   Bucket: readonlyonother
 507   S3Replication: 3
 508   ReadOnly: true
 509 `)
 510         hostname, err := os.Hostname()
 511         c.Assert(err, check.IsNil)
 512         url := arvados.URL{
 513                 Scheme: "http",
 514                 Host:   fmt.Sprintf("%s:%d", hostname, port),
 515         }
 516         _, ok := before["zzzzz-nyw5e-readonlyonother"].AccessViaHosts[url]
 517         c.Check(ok, check.Equals, false)
 518         _, ok = after["zzzzz-nyw5e-readonlyonother"].AccessViaHosts[url]
 519         c.Check(ok, check.Equals, true)
 520 }
 521
 522 // Writable volume, same cloud backend already writable by another
 523 // keepstore server --> add this host to AccessViaHosts with
 524 // ReadOnly==true
 525 func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_UpdateAlreadyWritable(c *check.C) {
 526         port, _ := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
 527         before, after := s.loadWithKeepstoreConfig(c, `
 528 Listen: :`+strconv.Itoa(port)+`
 529 Volumes:
 530 - Type: S3
 531   Endpoint: https://storage.googleapis.com
 532   Region: us-east-1z
 533   Bucket: writableonother
 534   S3Replication: 3
 535   ReadOnly: false
 536 `)
 537         hostname, err := os.Hostname()
 538         c.Assert(err, check.IsNil)
 539         url := arvados.URL{
 540                 Scheme: "http",
 541                 Host:   fmt.Sprintf("%s:%d", hostname, port),
 542         }
 543         _, ok := before["zzzzz-nyw5e-writableonother"].AccessViaHosts[url]
 544         c.Check(ok, check.Equals, false)
 545         _, ok = after["zzzzz-nyw5e-writableonother"].AccessViaHosts[url]
 546         c.Check(ok, check.Equals, true)
 547 }
 548
 549 // Non-writable volume, same cloud backend not already referenced in
 550 // cluster config --> assign a new random volume UUID.
 551 func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_AddReadOnly(c *check.C) {
 552         port, _ := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
 553         before, after := s.loadWithKeepstoreConfig(c, `
 554 Listen: :`+strconv.Itoa(port)+`
 555 Volumes:
 556 - Type: S3
 557   Endpoint: https://storage.googleapis.com
 558   Region: us-east-1z
 559   Bucket: differentbucket
 560   S3Replication: 3
 561 `)
 562         newuuids := s.findAddedVolumes(c, before, after, 1)
 563         newvol := after[newuuids[0]]
 564
 565         var params arvados.S3VolumeDriverParameters
 566         json.Unmarshal(newvol.DriverParameters, &params)
 567         c.Check(params.Bucket, check.Equals, "differentbucket")
 568
 569         hostname, err := os.Hostname()
 570         c.Assert(err, check.IsNil)
 571         _, ok := newvol.AccessViaHosts[arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%d", hostname, port)}]
 572         c.Check(ok, check.Equals, true)
 573 }
 574
 575 // Ensure logs mention unmigrated servers.
 576 func (s *KeepstoreMigrationSuite) TestPendingKeepstoreMigrations(c *check.C) {
 577         client := arvados.NewClientFromEnv()
 578         for _, host := range []string{"keep0", "keep1"} {
 579                 err := client.RequestAndDecode(new(struct{}), "POST", "arvados/v1/keep_services", nil, map[string]interface{}{
 580                         "keep_service": map[string]interface{}{
 581                                 "service_type": "disk",
 582                                 "service_host": host + ".zzzzz.example.com",
 583                                 "service_port": 25107,
 584                         },
 585                 })
 586                 c.Assert(err, check.IsNil)
 587         }
 588
 589         port, _ := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
 590         logs := s.logsWithKeepstoreConfig(c, `
 591 Listen: :`+strconv.Itoa(port)+`
 592 Volumes:
 593 - Type: S3
 594   Endpoint: https://storage.googleapis.com
 595   Bucket: foo
 596 `)
 597         c.Check(logs, check.Matches, `(?ms).*you should remove the legacy keepstore config file.*`)
 598         c.Check(logs, check.Matches, `(?ms).*you should migrate the legacy keepstore configuration file on host keep1.zzzzz.example.com.*`)
 599         c.Check(logs, check.Not(check.Matches), `(?ms).*should migrate.*keep0.zzzzz.example.com.*`)
 600         c.Check(logs, check.Matches, `(?ms).*keepstore configured at http://keep2.zzzzz.example.com:25107 does not have access to any volumes.*`)
 601         c.Check(logs, check.Matches, `(?ms).*Volumes.zzzzz-nyw5e-possconfigerror.AccessViaHosts refers to nonexistent keepstore server http://keep00.zzzzz.example.com:25107.*`)
 602 }
 603
 604 const clusterConfigForKeepstoreMigrationTest = `
 605 Clusters:
 606   zzzzz:
 607     SystemRootToken: ` + arvadostest.AdminToken + `
 608     Services:
 609       Keepstore:
 610         InternalURLs:
 611           "http://{{.hostname}}:12345": {}
 612           "http://keep0.zzzzz.example.com:25107": {}
 613           "http://keep2.zzzzz.example.com:25107": {}
 614       Controller:
 615         ExternalURL: "https://{{.controller}}"
 616     TLS:
 617       Insecure: true
 618     Volumes:
 619
 620       zzzzz-nyw5e-alreadymigrated:
 621         AccessViaHosts:
 622           "http://{{.hostname}}:12345": {}
 623         Driver: S3
 624         DriverParameters:
 625           Endpoint: https://storage.googleapis.com
 626           Region: us-east-1z
 627           Bucket: alreadymigrated
 628         Replication: 3
 629
 630       zzzzz-nyw5e-readonlyonother:
 631         AccessViaHosts:
 632           "http://keep0.zzzzz.example.com:25107": {ReadOnly: true}
 633         Driver: S3
 634         DriverParameters:
 635           Endpoint: https://storage.googleapis.com
 636           Region: us-east-1z
 637           Bucket: readonlyonother
 638         Replication: 3
 639
 640       zzzzz-nyw5e-writableonother:
 641         AccessViaHosts:
 642           "http://keep0.zzzzz.example.com:25107": {}
 643         Driver: S3
 644         DriverParameters:
 645           Endpoint: https://storage.googleapis.com
 646           Region: us-east-1z
 647           Bucket: writableonother
 648         Replication: 3
 649
 650       zzzzz-nyw5e-localfilesystem:
 651         AccessViaHosts:
 652           "http://keep0.zzzzz.example.com:25107": {}
 653         Driver: Directory
 654         DriverParameters:
 655           Root: /data/sdd
 656         Replication: 1
 657
 658       zzzzz-nyw5e-localismigrated:
 659         AccessViaHosts:
 660           "http://{{.hostname}}:12345": {}
 661         Driver: Directory
 662         DriverParameters:
 663           Root: /data/sde
 664         Replication: 1
 665
 666       zzzzz-nyw5e-possconfigerror:
 667         AccessViaHosts:
 668           "http://keep00.zzzzz.example.com:25107": {}
 669         Driver: Directory
 670         DriverParameters:
 671           Root: /data/sdf
 672         Replication: 1
 673 `
 674
 675 // Determine the effect of combining the given legacy keepstore config
 676 // YAML (just the "Volumes" entries of an old keepstore config file)
 677 // with the example clusterConfigForKeepstoreMigrationTest config.
 678 //
 679 // Return two Volumes configs -- one without loading keepstoreYAML
 680 // ("before") and one with ("after") -- for the caller to compare.
 681 func (s *KeepstoreMigrationSuite) loadWithKeepstoreConfig(c *check.C, keepstoreYAML string) (before, after map[string]arvados.Volume) {
 682         ldr := testLoader(c, s.clusterConfigYAML(c), nil)
 683         cBefore, err := ldr.Load()
 684         c.Assert(err, check.IsNil)
 685
 686         keepstoreconfig, err := ioutil.TempFile("", "")
 687         c.Assert(err, check.IsNil)
 688         defer os.Remove(keepstoreconfig.Name())
 689         io.WriteString(keepstoreconfig, keepstoreYAML)
 690
 691         ldr = testLoader(c, s.clusterConfigYAML(c), nil)
 692         ldr.KeepstorePath = keepstoreconfig.Name()
 693         cAfter, err := ldr.Load()
 694         c.Assert(err, check.IsNil)
 695
 696         return cBefore.Clusters["zzzzz"].Volumes, cAfter.Clusters["zzzzz"].Volumes
 697 }
 698
 699 // Return the log messages emitted when loading keepstoreYAML along
 700 // with clusterConfigForKeepstoreMigrationTest.
 701 func (s *KeepstoreMigrationSuite) logsWithKeepstoreConfig(c *check.C, keepstoreYAML string) string {
 702         var logs bytes.Buffer
 703
 704         keepstoreconfig, err := ioutil.TempFile("", "")
 705         c.Assert(err, check.IsNil)
 706         defer os.Remove(keepstoreconfig.Name())
 707         io.WriteString(keepstoreconfig, keepstoreYAML)
 708
 709         ldr := testLoader(c, s.clusterConfigYAML(c), &logs)
 710         ldr.KeepstorePath = keepstoreconfig.Name()
 711         _, err = ldr.Load()
 712         c.Assert(err, check.IsNil)
 713
 714         return logs.String()
 715 }
 716
 717 func (s *KeepstoreMigrationSuite) clusterConfigYAML(c *check.C) string {
 718         hostname, err := os.Hostname()
 719         c.Assert(err, check.IsNil)
 720
 721         tmpl := template.Must(template.New("config").Parse(clusterConfigForKeepstoreMigrationTest))
 722
 723         var clusterconfigdata bytes.Buffer
 724         err = tmpl.Execute(&clusterconfigdata, map[string]interface{}{
 725                 "hostname":   hostname,
 726                 "controller": os.Getenv("ARVADOS_API_HOST"),
 727         })
 728         c.Assert(err, check.IsNil)
 729
 730         return clusterconfigdata.String()
 731 }
 732
 733 // Return the uuids of volumes that appear in "after" but not
 734 // "before".
 735 //
 736 // Assert the returned slice has at least minAdded entries.
 737 func (s *KeepstoreMigrationSuite) findAddedVolumes(c *check.C, before, after map[string]arvados.Volume, minAdded int) (uuids []string) {
 738         for uuid := range after {
 739                 if _, ok := before[uuid]; !ok {
 740                         uuids = append(uuids, uuid)
 741                 }
 742         }
 743         if len(uuids) < minAdded {
 744                 c.Assert(uuids, check.HasLen, minAdded)
 745         }
 746         return
 747 }
 748
 749 func (s *KeepstoreMigrationSuite) getTestKeepstorePortAndMatchingVolumeUUID(c *check.C) (int, string) {
 750         for port, ks := range s.ksByPort {
 751                 c.Assert(ks.UUID, check.HasLen, 27)
 752                 return port, "zzzzz-nyw5e-" + ks.UUID[12:]
 753         }
 754         c.Fatal("s.ksByPort is empty")
 755         return 0, ""
 756 }
 757
 758 func (s *KeepstoreMigrationSuite) TestKeepServiceIsMe(c *check.C) {
 759         for i, trial := range []struct {
 760                 match       bool
 761                 hostname    string
 762                 listen      string
 763                 serviceHost string
 764                 servicePort int
 765         }{
 766                 {true, "keep0", "keep0", "keep0", 80},
 767                 {true, "keep0", "[::1]:http", "keep0", 80},
 768                 {true, "keep0", "[::]:http", "keep0", 80},
 769                 {true, "keep0", "keep0:25107", "keep0", 25107},
 770                 {true, "keep0", ":25107", "keep0", 25107},
 771                 {true, "keep0.domain", ":25107", "keep0.domain.example", 25107},
 772                 {true, "keep0.domain.example", ":25107", "keep0.domain.example", 25107},
 773                 {true, "keep0", ":25107", "keep0.domain.example", 25107},
 774                 {true, "keep0", ":25107", "Keep0.domain.example", 25107},
 775                 {true, "keep0", ":http", "keep0.domain.example", 80},
 776                 {true, "keep0", ":25107", "localhost", 25107},
 777                 {true, "keep0", ":25107", "::1", 25107},
 778                 {false, "keep0", ":25107", "keep0", 1111},              // different port
 779                 {false, "keep0", ":25107", "localhost", 1111},          // different port
 780                 {false, "keep0", ":http", "keep0.domain.example", 443}, // different port
 781                 {false, "keep0", ":bogussss", "keep0", 25107},          // unresolvable port
 782                 {false, "keep0", ":25107", "keep1", 25107},             // different hostname
 783                 {false, "keep1", ":25107", "keep10", 25107},            // different hostname (prefix, but not on a "." boundary)
 784         } {
 785                 c.Check(keepServiceIsMe(arvados.KeepService{ServiceHost: trial.serviceHost, ServicePort: trial.servicePort}, trial.hostname, trial.listen), check.Equals, trial.match, check.Commentf("trial #%d: %#v", i, trial))
 786         }
 787 }