lib/config/deprecated_keepstore_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package config
   6
   7 import (
   8         "bytes"
   9         "encoding/json"
  10         "fmt"
  11         "io"
  12         "io/ioutil"
  13         "os"
  14         "sort"
  15         "strconv"
  16         "strings"
  17         "text/template"
  18         "time"
  19
  20         "git.arvados.org/arvados.git/sdk/go/arvados"
  21         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  22         check "gopkg.in/check.v1"
  23 )
  24
  25 type KeepstoreMigrationSuite struct {
  26         hostname string // blank = use test system's hostname
  27         ksByPort map[int]arvados.KeepService
  28 }
  29
  30 var _ = check.Suite(&KeepstoreMigrationSuite{})
  31
  32 func (s *KeepstoreMigrationSuite) SetUpSuite(c *check.C) {
  33         os.Setenv("ARVADOS_API_HOST", os.Getenv("ARVADOS_TEST_API_HOST"))
  34         os.Setenv("ARVADOS_API_HOST_INSECURE", "1")
  35         os.Setenv("ARVADOS_API_TOKEN", arvadostest.AdminToken)
  36
  37         // We don't need the keepstore servers, but we do need
  38         // keep_services listings that point to localhost, rather than
  39         // the apiserver fixtures that point to fictional hosts
  40         // keep*.zzzzz.arvadosapi.com.
  41
  42         client := arvados.NewClientFromEnv()
  43
  44         // Delete existing non-proxy listings.
  45         var svcList arvados.KeepServiceList
  46         err := client.RequestAndDecode(&svcList, "GET", "arvados/v1/keep_services", nil, nil)
  47         c.Assert(err, check.IsNil)
  48         for _, ks := range svcList.Items {
  49                 if ks.ServiceType != "proxy" {
  50                         err = client.RequestAndDecode(new(struct{}), "DELETE", "arvados/v1/keep_services/"+ks.UUID, nil, nil)
  51                         c.Assert(err, check.IsNil)
  52                 }
  53         }
  54         // Add new fake listings.
  55         s.ksByPort = map[int]arvados.KeepService{}
  56         for _, port := range []int{25107, 25108} {
  57                 var ks arvados.KeepService
  58                 err = client.RequestAndDecode(&ks, "POST", "arvados/v1/keep_services", nil, map[string]interface{}{
  59                         "keep_service": map[string]interface{}{
  60                                 "service_type": "disk",
  61                                 "service_host": "localhost",
  62                                 "service_port": port,
  63                         },
  64                 })
  65                 c.Assert(err, check.IsNil)
  66                 s.ksByPort[port] = ks
  67         }
  68 }
  69
  70 func (s *KeepstoreMigrationSuite) checkEquivalentWithKeepstoreConfig(c *check.C, keepstoreconfig, clusterconfig, expectedconfig string) {
  71         keepstorefile, err := ioutil.TempFile("", "")
  72         c.Assert(err, check.IsNil)
  73         defer os.Remove(keepstorefile.Name())
  74         _, err = io.WriteString(keepstorefile, keepstoreconfig)
  75         c.Assert(err, check.IsNil)
  76         err = keepstorefile.Close()
  77         c.Assert(err, check.IsNil)
  78
  79         gotldr := testLoader(c, clusterconfig, nil)
  80         gotldr.KeepstorePath = keepstorefile.Name()
  81         expectedldr := testLoader(c, expectedconfig, nil)
  82         checkEquivalentLoaders(c, gotldr, expectedldr)
  83 }
  84
  85 func (s *KeepstoreMigrationSuite) TestDeprecatedKeepstoreConfig(c *check.C) {
  86         keyfile, err := ioutil.TempFile("", "")
  87         c.Assert(err, check.IsNil)
  88         defer os.Remove(keyfile.Name())
  89         io.WriteString(keyfile, "blobsigningkey\n")
  90
  91         hostname, err := os.Hostname()
  92         c.Assert(err, check.IsNil)
  93
  94         s.checkEquivalentWithKeepstoreConfig(c, `
  95 Listen: ":25107"
  96 Debug: true
  97 LogFormat: text
  98 MaxBuffers: 1234
  99 MaxRequests: 2345
 100 BlobSignatureTTL: 123m
 101 BlobSigningKeyFile: `+keyfile.Name()+`
 102 Volumes:
 103 - Type: Directory
 104   Root: /tmp
 105 `, `
 106 Clusters:
 107   z1111:
 108     SystemRootToken: `+arvadostest.AdminToken+`
 109     TLS: {Insecure: true}
 110     Services:
 111       Controller:
 112         ExternalURL: "https://`+os.Getenv("ARVADOS_API_HOST")+`/"
 113 `, `
 114 Clusters:
 115   z1111:
 116     SystemRootToken: `+arvadostest.AdminToken+`
 117     TLS: {Insecure: true}
 118     Services:
 119       Keepstore:
 120         InternalURLs:
 121           "http://`+hostname+`:25107": {Rendezvous: `+s.ksByPort[25107].UUID[12:]+`}
 122       Controller:
 123         ExternalURL: "https://`+os.Getenv("ARVADOS_API_HOST")+`/"
 124     SystemLogs:
 125       Format: text
 126       LogLevel: debug
 127     API:
 128       MaxKeepBlobBuffers: 1234
 129       MaxConcurrentRequests: 2345
 130     Collections:
 131       BlobSigningTTL: 123m
 132       BlobSigningKey: blobsigningkey
 133     Volumes:
 134       z1111-nyw5e-`+s.ksByPort[25107].UUID[12:]+`:
 135         AccessViaHosts:
 136           "http://`+hostname+`:25107":
 137             ReadOnly: false
 138         Driver: Directory
 139         DriverParameters:
 140           Root: /tmp
 141           Serialize: false
 142         ReadOnly: false
 143         Replication: 1
 144         StorageClasses: {}
 145 `)
 146 }
 147
 148 func (s *KeepstoreMigrationSuite) TestDiscoverLocalVolumes(c *check.C) {
 149         tmpd, err := ioutil.TempDir("", "")
 150         c.Assert(err, check.IsNil)
 151         defer os.RemoveAll(tmpd)
 152         err = os.Mkdir(tmpd+"/keep", 0777)
 153         c.Assert(err, check.IsNil)
 154
 155         tmpf, err := ioutil.TempFile("", "")
 156         c.Assert(err, check.IsNil)
 157         defer os.Remove(tmpf.Name())
 158
 159         // read/write
 160         _, err = fmt.Fprintf(tmpf, "/dev/xvdb %s ext4 rw,noexec 0 0\n", tmpd)
 161         c.Assert(err, check.IsNil)
 162
 163         s.testDeprecatedVolume(c, "DiscoverVolumesFromMountsFile: "+tmpf.Name(), arvados.Volume{
 164                 Driver:      "Directory",
 165                 ReadOnly:    false,
 166                 Replication: 1,
 167         }, &arvados.DirectoryVolumeDriverParameters{
 168                 Root:      tmpd + "/keep",
 169                 Serialize: false,
 170         }, &arvados.DirectoryVolumeDriverParameters{})
 171
 172         // read-only
 173         tmpf.Seek(0, os.SEEK_SET)
 174         tmpf.Truncate(0)
 175         _, err = fmt.Fprintf(tmpf, "/dev/xvdb %s ext4 ro,noexec 0 0\n", tmpd)
 176         c.Assert(err, check.IsNil)
 177
 178         s.testDeprecatedVolume(c, "DiscoverVolumesFromMountsFile: "+tmpf.Name(), arvados.Volume{
 179                 Driver:      "Directory",
 180                 ReadOnly:    true,
 181                 Replication: 1,
 182         }, &arvados.DirectoryVolumeDriverParameters{
 183                 Root:      tmpd + "/keep",
 184                 Serialize: false,
 185         }, &arvados.DirectoryVolumeDriverParameters{})
 186 }
 187
 188 func (s *KeepstoreMigrationSuite) TestDeprecatedVolumes(c *check.C) {
 189         accesskeyfile, err := ioutil.TempFile("", "")
 190         c.Assert(err, check.IsNil)
 191         defer os.Remove(accesskeyfile.Name())
 192         io.WriteString(accesskeyfile, "accesskeydata\n")
 193
 194         secretkeyfile, err := ioutil.TempFile("", "")
 195         c.Assert(err, check.IsNil)
 196         defer os.Remove(secretkeyfile.Name())
 197         io.WriteString(secretkeyfile, "secretkeydata\n")
 198
 199         // s3, empty/default
 200         s.testDeprecatedVolume(c, `
 201 Volumes:
 202 - Type: S3
 203 `, arvados.Volume{
 204                 Driver:      "S3",
 205                 Replication: 1,
 206         }, &arvados.S3VolumeDriverParameters{}, &arvados.S3VolumeDriverParameters{})
 207
 208         // s3, fully configured
 209         s.testDeprecatedVolume(c, `
 210 Volumes:
 211 - Type: S3
 212   AccessKeyFile: `+accesskeyfile.Name()+`
 213   SecretKeyFile: `+secretkeyfile.Name()+`
 214   Endpoint: https://storage.googleapis.com
 215   Region: us-east-1z
 216   Bucket: testbucket
 217   LocationConstraint: true
 218   IndexPageSize: 1234
 219   S3Replication: 4
 220   ConnectTimeout: 3m
 221   ReadTimeout: 4m
 222   RaceWindow: 5m
 223   UnsafeDelete: true
 224 `, arvados.Volume{
 225                 Driver:      "S3",
 226                 Replication: 4,
 227         }, &arvados.S3VolumeDriverParameters{
 228                 AccessKeyID:        "accesskeydata",
 229                 SecretAccessKey:    "secretkeydata",
 230                 Endpoint:           "https://storage.googleapis.com",
 231                 Region:             "us-east-1z",
 232                 Bucket:             "testbucket",
 233                 LocationConstraint: true,
 234                 IndexPageSize:      1234,
 235                 ConnectTimeout:     arvados.Duration(time.Minute * 3),
 236                 ReadTimeout:        arvados.Duration(time.Minute * 4),
 237                 RaceWindow:         arvados.Duration(time.Minute * 5),
 238                 UnsafeDelete:       true,
 239         }, &arvados.S3VolumeDriverParameters{})
 240
 241         // azure, empty/default
 242         s.testDeprecatedVolume(c, `
 243 Volumes:
 244 - Type: Azure
 245 `, arvados.Volume{
 246                 Driver:      "Azure",
 247                 Replication: 1,
 248         }, &arvados.AzureVolumeDriverParameters{}, &arvados.AzureVolumeDriverParameters{})
 249
 250         // azure, fully configured
 251         s.testDeprecatedVolume(c, `
 252 Volumes:
 253 - Type: Azure
 254   ReadOnly: true
 255   StorageAccountName: storageacctname
 256   StorageAccountKeyFile: `+secretkeyfile.Name()+`
 257   StorageBaseURL: https://example.example/
 258   ContainerName: testctr
 259   LocationConstraint: true
 260   AzureReplication: 4
 261   RequestTimeout: 3m
 262   ListBlobsRetryDelay: 4m
 263   ListBlobsMaxAttempts: 5
 264 `, arvados.Volume{
 265                 Driver:      "Azure",
 266                 ReadOnly:    true,
 267                 Replication: 4,
 268         }, &arvados.AzureVolumeDriverParameters{
 269                 StorageAccountName:   "storageacctname",
 270                 StorageAccountKey:    "secretkeydata",
 271                 StorageBaseURL:       "https://example.example/",
 272                 ContainerName:        "testctr",
 273                 RequestTimeout:       arvados.Duration(time.Minute * 3),
 274                 ListBlobsRetryDelay:  arvados.Duration(time.Minute * 4),
 275                 ListBlobsMaxAttempts: 5,
 276         }, &arvados.AzureVolumeDriverParameters{})
 277
 278         // directory, empty/default
 279         s.testDeprecatedVolume(c, `
 280 Volumes:
 281 - Type: Directory
 282   Root: /tmp/xyzzy
 283 `, arvados.Volume{
 284                 Driver:      "Directory",
 285                 Replication: 1,
 286         }, &arvados.DirectoryVolumeDriverParameters{
 287                 Root: "/tmp/xyzzy",
 288         }, &arvados.DirectoryVolumeDriverParameters{})
 289
 290         // directory, fully configured
 291         s.testDeprecatedVolume(c, `
 292 Volumes:
 293 - Type: Directory
 294   ReadOnly: true
 295   Root: /tmp/xyzzy
 296   DirectoryReplication: 4
 297   Serialize: true
 298 `, arvados.Volume{
 299                 Driver:      "Directory",
 300                 ReadOnly:    true,
 301                 Replication: 4,
 302         }, &arvados.DirectoryVolumeDriverParameters{
 303                 Root:      "/tmp/xyzzy",
 304                 Serialize: true,
 305         }, &arvados.DirectoryVolumeDriverParameters{})
 306 }
 307
 308 func (s *KeepstoreMigrationSuite) testDeprecatedVolume(c *check.C, oldconfigdata string, expectvol arvados.Volume, expectparams interface{}, paramsdst interface{}) {
 309         hostname := s.hostname
 310         if hostname == "" {
 311                 h, err := os.Hostname()
 312                 c.Assert(err, check.IsNil)
 313                 hostname = h
 314         }
 315
 316         oldconfig, err := ioutil.TempFile("", "")
 317         c.Assert(err, check.IsNil)
 318         defer os.Remove(oldconfig.Name())
 319         io.WriteString(oldconfig, "Listen: :12345\n"+oldconfigdata)
 320         if !strings.Contains(oldconfigdata, "DiscoverVolumesFromMountsFile") {
 321                 // Prevent tests from looking at the real /proc/mounts on the test host.
 322                 io.WriteString(oldconfig, "\nDiscoverVolumesFromMountsFile: /dev/null\n")
 323         }
 324
 325         ldr := testLoader(c, "Clusters: {z1111: {}}", nil)
 326         ldr.KeepstorePath = oldconfig.Name()
 327         cfg, err := ldr.Load()
 328         c.Assert(err, check.IsNil)
 329         cc := cfg.Clusters["z1111"]
 330         c.Check(cc.Volumes, check.HasLen, 1)
 331         for uuid, v := range cc.Volumes {
 332                 c.Check(uuid, check.HasLen, 27)
 333                 c.Check(v.Driver, check.Equals, expectvol.Driver)
 334                 c.Check(v.Replication, check.Equals, expectvol.Replication)
 335
 336                 avh, ok := v.AccessViaHosts[arvados.URL{Scheme: "http", Host: hostname + ":12345", Path: "/"}]
 337                 c.Check(ok, check.Equals, true)
 338                 c.Check(avh.ReadOnly, check.Equals, expectvol.ReadOnly)
 339
 340                 err := json.Unmarshal(v.DriverParameters, paramsdst)
 341                 c.Check(err, check.IsNil)
 342                 c.Check(paramsdst, check.DeepEquals, expectparams)
 343         }
 344 }
 345
 346 // How we handle a volume from a legacy keepstore config file depends
 347 // on whether it's writable, whether a volume using the same cloud
 348 // backend already exists in the cluster config, and (if so) whether
 349 // it already has an AccessViaHosts entry for this host.
 350 //
 351 // In all cases, we should end up with an AccessViaHosts entry for
 352 // this host, to indicate that the current host's volumes have been
 353 // migrated.
 354
 355 // Same backend already referenced in cluster config, this host
 356 // already listed in AccessViaHosts --> no change, except possibly
 357 // updating the ReadOnly flag on the AccessViaHosts entry.
 358 func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_AlreadyMigrated(c *check.C) {
 359         before, after := s.loadWithKeepstoreConfig(c, `
 360 Listen: :12345
 361 Volumes:
 362 - Type: S3
 363   Endpoint: https://storage.googleapis.com
 364   Region: us-east-1z
 365   Bucket: alreadymigrated
 366   S3Replication: 3
 367 `)
 368         checkEqualYAML(c, after, before)
 369 }
 370
 371 // Writable volume, same cloud backend already referenced in cluster
 372 // config --> change UUID to match this keepstore's UUID.
 373 func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_UpdateUUID(c *check.C) {
 374         port, expectUUID := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
 375
 376         before, after := s.loadWithKeepstoreConfig(c, `
 377 Listen: :`+strconv.Itoa(port)+`
 378 Volumes:
 379 - Type: S3
 380   Endpoint: https://storage.googleapis.com
 381   Region: us-east-1z
 382   Bucket: readonlyonother
 383   S3Replication: 3
 384 `)
 385         c.Check(after, check.HasLen, len(before))
 386         newuuids := s.findAddedVolumes(c, before, after, 1)
 387         newvol := after[newuuids[0]]
 388
 389         var params arvados.S3VolumeDriverParameters
 390         json.Unmarshal(newvol.DriverParameters, &params)
 391         c.Check(params.Bucket, check.Equals, "readonlyonother")
 392         c.Check(newuuids[0], check.Equals, expectUUID)
 393 }
 394
 395 // Writable volume, same cloud backend not yet referenced --> add a
 396 // new volume, with UUID to match this keepstore's UUID.
 397 func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_AddCloudVolume(c *check.C) {
 398         port, expectUUID := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
 399
 400         before, after := s.loadWithKeepstoreConfig(c, `
 401 Listen: :`+strconv.Itoa(port)+`
 402 Volumes:
 403 - Type: S3
 404   Endpoint: https://storage.googleapis.com
 405   Region: us-east-1z
 406   Bucket: bucket-to-migrate
 407   S3Replication: 3
 408 `)
 409         newuuids := s.findAddedVolumes(c, before, after, 1)
 410         newvol := after[newuuids[0]]
 411
 412         var params arvados.S3VolumeDriverParameters
 413         json.Unmarshal(newvol.DriverParameters, &params)
 414         c.Check(params.Bucket, check.Equals, "bucket-to-migrate")
 415         c.Check(newvol.Replication, check.Equals, 3)
 416
 417         c.Check(newuuids[0], check.Equals, expectUUID)
 418 }
 419
 420 // Writable volume, same filesystem backend already referenced in
 421 // cluster config, but this host isn't in AccessViaHosts --> add a new
 422 // volume, with UUID to match this keepstore's UUID (filesystem-backed
 423 // volumes are assumed to be different on different hosts, even if
 424 // paths are the same).
 425 func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_AddLocalVolume(c *check.C) {
 426         before, after := s.loadWithKeepstoreConfig(c, `
 427 Listen: :12345
 428 Volumes:
 429 - Type: Directory
 430   Root: /data/sdd
 431   DirectoryReplication: 2
 432 `)
 433         newuuids := s.findAddedVolumes(c, before, after, 1)
 434         newvol := after[newuuids[0]]
 435
 436         var params arvados.DirectoryVolumeDriverParameters
 437         json.Unmarshal(newvol.DriverParameters, &params)
 438         c.Check(params.Root, check.Equals, "/data/sdd")
 439         c.Check(newvol.Replication, check.Equals, 2)
 440 }
 441
 442 // Writable volume, same filesystem backend already referenced in
 443 // cluster config, and this host is already listed in AccessViaHosts
 444 // --> already migrated, don't change anything.
 445 func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_LocalVolumeAlreadyMigrated(c *check.C) {
 446         before, after := s.loadWithKeepstoreConfig(c, `
 447 Listen: :12345
 448 Volumes:
 449 - Type: Directory
 450   Root: /data/sde
 451   DirectoryReplication: 2
 452 `)
 453         checkEqualYAML(c, after, before)
 454 }
 455
 456 // Multiple writable cloud-backed volumes --> one of them will get a
 457 // UUID matching this keepstore's UUID.
 458 func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_AddMultipleCloudVolumes(c *check.C) {
 459         port, expectUUID := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
 460
 461         before, after := s.loadWithKeepstoreConfig(c, `
 462 Listen: :`+strconv.Itoa(port)+`
 463 Volumes:
 464 - Type: S3
 465   Endpoint: https://storage.googleapis.com
 466   Region: us-east-1z
 467   Bucket: first-bucket-to-migrate
 468   S3Replication: 3
 469 - Type: S3
 470   Endpoint: https://storage.googleapis.com
 471   Region: us-east-1z
 472   Bucket: second-bucket-to-migrate
 473   S3Replication: 3
 474 `)
 475         newuuids := s.findAddedVolumes(c, before, after, 2)
 476         // Sort by bucket name (so "first" comes before "second")
 477         params := map[string]arvados.S3VolumeDriverParameters{}
 478         for _, uuid := range newuuids {
 479                 var p arvados.S3VolumeDriverParameters
 480                 json.Unmarshal(after[uuid].DriverParameters, &p)
 481                 params[uuid] = p
 482         }
 483         sort.Slice(newuuids, func(i, j int) bool { return params[newuuids[i]].Bucket < params[newuuids[j]].Bucket })
 484         newvol0, newvol1 := after[newuuids[0]], after[newuuids[1]]
 485         params0, params1 := params[newuuids[0]], params[newuuids[1]]
 486
 487         c.Check(params0.Bucket, check.Equals, "first-bucket-to-migrate")
 488         c.Check(newvol0.Replication, check.Equals, 3)
 489
 490         c.Check(params1.Bucket, check.Equals, "second-bucket-to-migrate")
 491         c.Check(newvol1.Replication, check.Equals, 3)
 492
 493         // Don't care which one gets the special UUID
 494         if newuuids[0] != expectUUID {
 495                 c.Check(newuuids[1], check.Equals, expectUUID)
 496         }
 497 }
 498
 499 // Non-writable volume, same cloud backend already referenced in
 500 // cluster config --> add this host to AccessViaHosts with
 501 // ReadOnly==true
 502 func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_UpdateWithReadOnly(c *check.C) {
 503         port, _ := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
 504         before, after := s.loadWithKeepstoreConfig(c, `
 505 Listen: :`+strconv.Itoa(port)+`
 506 Volumes:
 507 - Type: S3
 508   Endpoint: https://storage.googleapis.com
 509   Region: us-east-1z
 510   Bucket: readonlyonother
 511   S3Replication: 3
 512   ReadOnly: true
 513 `)
 514         hostname, err := os.Hostname()
 515         c.Assert(err, check.IsNil)
 516         url := arvados.URL{
 517                 Scheme: "http",
 518                 Host:   fmt.Sprintf("%s:%d", hostname, port),
 519                 Path:   "/",
 520         }
 521         _, ok := before["zzzzz-nyw5e-readonlyonother"].AccessViaHosts[url]
 522         c.Check(ok, check.Equals, false)
 523         _, ok = after["zzzzz-nyw5e-readonlyonother"].AccessViaHosts[url]
 524         c.Check(ok, check.Equals, true)
 525 }
 526
 527 // Writable volume, same cloud backend already writable by another
 528 // keepstore server --> add this host to AccessViaHosts with
 529 // ReadOnly==true
 530 func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_UpdateAlreadyWritable(c *check.C) {
 531         port, _ := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
 532         before, after := s.loadWithKeepstoreConfig(c, `
 533 Listen: :`+strconv.Itoa(port)+`
 534 Volumes:
 535 - Type: S3
 536   Endpoint: https://storage.googleapis.com
 537   Region: us-east-1z
 538   Bucket: writableonother
 539   S3Replication: 3
 540   ReadOnly: false
 541 `)
 542         hostname, err := os.Hostname()
 543         c.Assert(err, check.IsNil)
 544         url := arvados.URL{
 545                 Scheme: "http",
 546                 Host:   fmt.Sprintf("%s:%d", hostname, port),
 547                 Path:   "/",
 548         }
 549         _, ok := before["zzzzz-nyw5e-writableonother"].AccessViaHosts[url]
 550         c.Check(ok, check.Equals, false)
 551         _, ok = after["zzzzz-nyw5e-writableonother"].AccessViaHosts[url]
 552         c.Check(ok, check.Equals, true)
 553 }
 554
 555 // Non-writable volume, same cloud backend not already referenced in
 556 // cluster config --> assign a new random volume UUID.
 557 func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_AddReadOnly(c *check.C) {
 558         port, _ := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
 559         before, after := s.loadWithKeepstoreConfig(c, `
 560 Listen: :`+strconv.Itoa(port)+`
 561 Volumes:
 562 - Type: S3
 563   Endpoint: https://storage.googleapis.com
 564   Region: us-east-1z
 565   Bucket: differentbucket
 566   S3Replication: 3
 567 `)
 568         newuuids := s.findAddedVolumes(c, before, after, 1)
 569         newvol := after[newuuids[0]]
 570
 571         var params arvados.S3VolumeDriverParameters
 572         json.Unmarshal(newvol.DriverParameters, &params)
 573         c.Check(params.Bucket, check.Equals, "differentbucket")
 574
 575         hostname, err := os.Hostname()
 576         c.Assert(err, check.IsNil)
 577         _, ok := newvol.AccessViaHosts[arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%d", hostname, port), Path: "/"}]
 578         c.Check(ok, check.Equals, true)
 579 }
 580
 581 // Ensure logs mention unmigrated servers.
 582 func (s *KeepstoreMigrationSuite) TestPendingKeepstoreMigrations(c *check.C) {
 583         client := arvados.NewClientFromEnv()
 584         for _, host := range []string{"keep0", "keep1"} {
 585                 err := client.RequestAndDecode(new(struct{}), "POST", "arvados/v1/keep_services", nil, map[string]interface{}{
 586                         "keep_service": map[string]interface{}{
 587                                 "service_type": "disk",
 588                                 "service_host": host + ".zzzzz.example.com",
 589                                 "service_port": 25107,
 590                         },
 591                 })
 592                 c.Assert(err, check.IsNil)
 593         }
 594
 595         port, _ := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
 596         logs := s.logsWithKeepstoreConfig(c, `
 597 Listen: :`+strconv.Itoa(port)+`
 598 Volumes:
 599 - Type: S3
 600   Endpoint: https://storage.googleapis.com
 601   Bucket: foo
 602 `)
 603         c.Check(logs, check.Matches, `(?ms).*you should remove the legacy keepstore config file.*`)
 604         c.Check(logs, check.Matches, `(?ms).*you should migrate the legacy keepstore configuration file on host keep1.zzzzz.example.com.*`)
 605         c.Check(logs, check.Not(check.Matches), `(?ms).*should migrate.*keep0.zzzzz.example.com.*`)
 606         c.Check(logs, check.Matches, `(?ms).*keepstore configured at http://keep2.zzzzz.example.com:25107/ does not have access to any volumes.*`)
 607         c.Check(logs, check.Matches, `(?ms).*Volumes.zzzzz-nyw5e-possconfigerror.AccessViaHosts refers to nonexistent keepstore server http://keep00.zzzzz.example.com:25107.*`)
 608 }
 609
 610 const clusterConfigForKeepstoreMigrationTest = `
 611 Clusters:
 612   zzzzz:
 613     SystemRootToken: ` + arvadostest.AdminToken + `
 614     Services:
 615       Keepstore:
 616         InternalURLs:
 617           "http://{{.hostname}}:12345": {}
 618           "http://keep0.zzzzz.example.com:25107": {}
 619           "http://keep2.zzzzz.example.com:25107": {}
 620       Controller:
 621         ExternalURL: "https://{{.controller}}"
 622     TLS:
 623       Insecure: true
 624     Volumes:
 625
 626       zzzzz-nyw5e-alreadymigrated:
 627         AccessViaHosts:
 628           "http://{{.hostname}}:12345": {}
 629         Driver: S3
 630         DriverParameters:
 631           Endpoint: https://storage.googleapis.com
 632           Region: us-east-1z
 633           Bucket: alreadymigrated
 634         Replication: 3
 635
 636       zzzzz-nyw5e-readonlyonother:
 637         AccessViaHosts:
 638           "http://keep0.zzzzz.example.com:25107": {ReadOnly: true}
 639         Driver: S3
 640         DriverParameters:
 641           Endpoint: https://storage.googleapis.com
 642           Region: us-east-1z
 643           Bucket: readonlyonother
 644         Replication: 3
 645
 646       zzzzz-nyw5e-writableonother:
 647         AccessViaHosts:
 648           "http://keep0.zzzzz.example.com:25107": {}
 649         Driver: S3
 650         DriverParameters:
 651           Endpoint: https://storage.googleapis.com
 652           Region: us-east-1z
 653           Bucket: writableonother
 654         Replication: 3
 655
 656       zzzzz-nyw5e-localfilesystem:
 657         AccessViaHosts:
 658           "http://keep0.zzzzz.example.com:25107": {}
 659         Driver: Directory
 660         DriverParameters:
 661           Root: /data/sdd
 662         Replication: 1
 663
 664       zzzzz-nyw5e-localismigrated:
 665         AccessViaHosts:
 666           "http://{{.hostname}}:12345": {}
 667         Driver: Directory
 668         DriverParameters:
 669           Root: /data/sde
 670         Replication: 1
 671
 672       zzzzz-nyw5e-possconfigerror:
 673         AccessViaHosts:
 674           "http://keep00.zzzzz.example.com:25107": {}
 675         Driver: Directory
 676         DriverParameters:
 677           Root: /data/sdf
 678         Replication: 1
 679 `
 680
 681 // Determine the effect of combining the given legacy keepstore config
 682 // YAML (just the "Volumes" entries of an old keepstore config file)
 683 // with the example clusterConfigForKeepstoreMigrationTest config.
 684 //
 685 // Return two Volumes configs -- one without loading keepstoreYAML
 686 // ("before") and one with ("after") -- for the caller to compare.
 687 func (s *KeepstoreMigrationSuite) loadWithKeepstoreConfig(c *check.C, keepstoreYAML string) (before, after map[string]arvados.Volume) {
 688         ldr := testLoader(c, s.clusterConfigYAML(c), nil)
 689         cBefore, err := ldr.Load()
 690         c.Assert(err, check.IsNil)
 691
 692         keepstoreconfig, err := ioutil.TempFile("", "")
 693         c.Assert(err, check.IsNil)
 694         defer os.Remove(keepstoreconfig.Name())
 695         io.WriteString(keepstoreconfig, keepstoreYAML)
 696
 697         ldr = testLoader(c, s.clusterConfigYAML(c), nil)
 698         ldr.KeepstorePath = keepstoreconfig.Name()
 699         cAfter, err := ldr.Load()
 700         c.Assert(err, check.IsNil)
 701
 702         return cBefore.Clusters["zzzzz"].Volumes, cAfter.Clusters["zzzzz"].Volumes
 703 }
 704
 705 // Return the log messages emitted when loading keepstoreYAML along
 706 // with clusterConfigForKeepstoreMigrationTest.
 707 func (s *KeepstoreMigrationSuite) logsWithKeepstoreConfig(c *check.C, keepstoreYAML string) string {
 708         var logs bytes.Buffer
 709
 710         keepstoreconfig, err := ioutil.TempFile("", "")
 711         c.Assert(err, check.IsNil)
 712         defer os.Remove(keepstoreconfig.Name())
 713         io.WriteString(keepstoreconfig, keepstoreYAML)
 714
 715         ldr := testLoader(c, s.clusterConfigYAML(c), &logs)
 716         ldr.KeepstorePath = keepstoreconfig.Name()
 717         _, err = ldr.Load()
 718         c.Assert(err, check.IsNil)
 719
 720         return logs.String()
 721 }
 722
 723 func (s *KeepstoreMigrationSuite) clusterConfigYAML(c *check.C) string {
 724         hostname, err := os.Hostname()
 725         c.Assert(err, check.IsNil)
 726
 727         tmpl := template.Must(template.New("config").Parse(clusterConfigForKeepstoreMigrationTest))
 728
 729         var clusterconfigdata bytes.Buffer
 730         err = tmpl.Execute(&clusterconfigdata, map[string]interface{}{
 731                 "hostname":   hostname,
 732                 "controller": os.Getenv("ARVADOS_API_HOST"),
 733         })
 734         c.Assert(err, check.IsNil)
 735
 736         return clusterconfigdata.String()
 737 }
 738
 739 // Return the uuids of volumes that appear in "after" but not
 740 // "before".
 741 //
 742 // Assert the returned slice has at least minAdded entries.
 743 func (s *KeepstoreMigrationSuite) findAddedVolumes(c *check.C, before, after map[string]arvados.Volume, minAdded int) (uuids []string) {
 744         for uuid := range after {
 745                 if _, ok := before[uuid]; !ok {
 746                         uuids = append(uuids, uuid)
 747                 }
 748         }
 749         if len(uuids) < minAdded {
 750                 c.Assert(uuids, check.HasLen, minAdded)
 751         }
 752         return
 753 }
 754
 755 func (s *KeepstoreMigrationSuite) getTestKeepstorePortAndMatchingVolumeUUID(c *check.C) (int, string) {
 756         for port, ks := range s.ksByPort {
 757                 c.Assert(ks.UUID, check.HasLen, 27)
 758                 return port, "zzzzz-nyw5e-" + ks.UUID[12:]
 759         }
 760         c.Fatal("s.ksByPort is empty")
 761         return 0, ""
 762 }
 763
 764 func (s *KeepstoreMigrationSuite) TestKeepServiceIsMe(c *check.C) {
 765         for i, trial := range []struct {
 766                 match       bool
 767                 hostname    string
 768                 listen      string
 769                 serviceHost string
 770                 servicePort int
 771         }{
 772                 {true, "keep0", "keep0", "keep0", 80},
 773                 {true, "keep0", "[::1]:http", "keep0", 80},
 774                 {true, "keep0", "[::]:http", "keep0", 80},
 775                 {true, "keep0", "keep0:25107", "keep0", 25107},
 776                 {true, "keep0", ":25107", "keep0", 25107},
 777                 {true, "keep0.domain", ":25107", "keep0.domain.example", 25107},
 778                 {true, "keep0.domain.example", ":25107", "keep0.domain.example", 25107},
 779                 {true, "keep0", ":25107", "keep0.domain.example", 25107},
 780                 {true, "keep0", ":25107", "Keep0.domain.example", 25107},
 781                 {true, "keep0", ":http", "keep0.domain.example", 80},
 782                 {true, "keep0", ":25107", "localhost", 25107},
 783                 {true, "keep0", ":25107", "::1", 25107},
 784                 {false, "keep0", ":25107", "keep0", 1111},              // different port
 785                 {false, "keep0", ":25107", "localhost", 1111},          // different port
 786                 {false, "keep0", ":http", "keep0.domain.example", 443}, // different port
 787                 {false, "keep0", ":bogussss", "keep0", 25107},          // unresolvable port
 788                 {false, "keep0", ":25107", "keep1", 25107},             // different hostname
 789                 {false, "keep1", ":25107", "keep10", 25107},            // different hostname (prefix, but not on a "." boundary)
 790         } {
 791                 c.Check(keepServiceIsMe(arvados.KeepService{ServiceHost: trial.serviceHost, ServicePort: trial.servicePort}, trial.hostname, trial.listen), check.Equals, trial.match, check.Commentf("trial #%d: %#v", i, trial))
 792         }
 793 }