lib/config/deprecated_keepstore_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package config
   6
   7 import (
   8         "bytes"
   9         "encoding/json"
  10         "fmt"
  11         "io"
  12         "io/ioutil"
  13         "os"
  14         "sort"
  15         "strconv"
  16         "strings"
  17         "text/template"
  18         "time"
  19
  20         "git.arvados.org/arvados.git/sdk/go/arvados"
  21         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  22         check "gopkg.in/check.v1"
  23 )
  24
  25 type KeepstoreMigrationSuite struct {
  26         hostname string // blank = use test system's hostname
  27         ksByPort map[int]arvados.KeepService
  28 }
  29
  30 var _ = check.Suite(&KeepstoreMigrationSuite{})
  31
  32 func (s *KeepstoreMigrationSuite) SetUpSuite(c *check.C) {
  33         os.Setenv("ARVADOS_API_HOST", os.Getenv("ARVADOS_TEST_API_HOST"))
  34         os.Setenv("ARVADOS_API_HOST_INSECURE", "1")
  35         os.Setenv("ARVADOS_API_TOKEN", arvadostest.AdminToken)
  36
  37         // We don't need the keepstore servers, but we do need
  38         // keep_services listings that point to localhost, rather than
  39         // the apiserver fixtures that point to fictional hosts
  40         // keep*.zzzzz.arvadosapi.com.
  41
  42         client := arvados.NewClientFromEnv()
  43
  44         // Delete existing non-proxy listings.
  45         var svcList arvados.KeepServiceList
  46         err := client.RequestAndDecode(&svcList, "GET", "arvados/v1/keep_services", nil, nil)
  47         c.Assert(err, check.IsNil)
  48         for _, ks := range svcList.Items {
  49                 if ks.ServiceType != "proxy" {
  50                         err = client.RequestAndDecode(new(struct{}), "DELETE", "arvados/v1/keep_services/"+ks.UUID, nil, nil)
  51                         c.Assert(err, check.IsNil)
  52                 }
  53         }
  54         // Add new fake listings.
  55         s.ksByPort = map[int]arvados.KeepService{}
  56         for _, port := range []int{25107, 25108} {
  57                 var ks arvados.KeepService
  58                 err = client.RequestAndDecode(&ks, "POST", "arvados/v1/keep_services", nil, map[string]interface{}{
  59                         "keep_service": map[string]interface{}{
  60                                 "service_type": "disk",
  61                                 "service_host": "localhost",
  62                                 "service_port": port,
  63                         },
  64                 })
  65                 c.Assert(err, check.IsNil)
  66                 s.ksByPort[port] = ks
  67         }
  68 }
  69
  70 func (s *KeepstoreMigrationSuite) checkEquivalentWithKeepstoreConfig(c *check.C, keepstoreconfig, clusterconfig, expectedconfig string) {
  71         keepstorefile, err := ioutil.TempFile("", "")
  72         c.Assert(err, check.IsNil)
  73         defer os.Remove(keepstorefile.Name())
  74         _, err = io.WriteString(keepstorefile, keepstoreconfig)
  75         c.Assert(err, check.IsNil)
  76         err = keepstorefile.Close()
  77         c.Assert(err, check.IsNil)
  78
  79         gotldr := testLoader(c, clusterconfig, nil)
  80         gotldr.KeepstorePath = keepstorefile.Name()
  81         expectedldr := testLoader(c, expectedconfig, nil)
  82         checkEquivalentLoaders(c, gotldr, expectedldr)
  83 }
  84
  85 func (s *KeepstoreMigrationSuite) TestDeprecatedKeepstoreConfig(c *check.C) {
  86         keyfile, err := ioutil.TempFile("", "")
  87         c.Assert(err, check.IsNil)
  88         defer os.Remove(keyfile.Name())
  89         io.WriteString(keyfile, "blobsigningkey\n")
  90
  91         hostname, err := os.Hostname()
  92         c.Assert(err, check.IsNil)
  93
  94         s.checkEquivalentWithKeepstoreConfig(c, `
  95 Listen: ":25107"
  96 Debug: true
  97 LogFormat: text
  98 MaxBuffers: 1234
  99 MaxRequests: 2345
 100 BlobSignatureTTL: 123m
 101 BlobSigningKeyFile: `+keyfile.Name()+`
 102 Volumes:
 103 - Type: Directory
 104   Root: /tmp
 105 `, `
 106 Clusters:
 107   z1111:
 108     SystemRootToken: `+arvadostest.AdminToken+`
 109     TLS: {Insecure: true}
 110     Services:
 111       Controller:
 112         ExternalURL: "https://`+os.Getenv("ARVADOS_API_HOST")+`"
 113 `, `
 114 Clusters:
 115   z1111:
 116     SystemRootToken: `+arvadostest.AdminToken+`
 117     TLS: {Insecure: true}
 118     Services:
 119       Keepstore:
 120         InternalURLs:
 121           "http://`+hostname+`:25107": {Rendezvous: `+s.ksByPort[25107].UUID[12:]+`}
 122       Controller:
 123         ExternalURL: "https://`+os.Getenv("ARVADOS_API_HOST")+`"
 124     SystemLogs:
 125       Format: text
 126       LogLevel: debug
 127     API:
 128       MaxKeepBlobBuffers: 1234
 129       MaxConcurrentRequests: 2345
 130     Collections:
 131       BlobSigningTTL: 123m
 132       BlobSigningKey: blobsigningkey
 133     Volumes:
 134       z1111-nyw5e-`+s.ksByPort[25107].UUID[12:]+`:
 135         AccessViaHosts:
 136           "http://`+hostname+`:25107":
 137             ReadOnly: false
 138         Driver: Directory
 139         DriverParameters:
 140           Root: /tmp
 141           Serialize: false
 142         ReadOnly: false
 143         Replication: 1
 144         StorageClasses: {}
 145 `)
 146 }
 147
 148 func (s *KeepstoreMigrationSuite) TestDiscoverLocalVolumes(c *check.C) {
 149         tmpd, err := ioutil.TempDir("", "")
 150         c.Assert(err, check.IsNil)
 151         defer os.RemoveAll(tmpd)
 152         err = os.Mkdir(tmpd+"/keep", 0777)
 153         c.Assert(err, check.IsNil)
 154
 155         tmpf, err := ioutil.TempFile("", "")
 156         c.Assert(err, check.IsNil)
 157         defer os.Remove(tmpf.Name())
 158
 159         // read/write
 160         _, err = fmt.Fprintf(tmpf, "/dev/xvdb %s ext4 rw,noexec 0 0\n", tmpd)
 161         c.Assert(err, check.IsNil)
 162
 163         s.testDeprecatedVolume(c, "DiscoverVolumesFromMountsFile: "+tmpf.Name(), arvados.Volume{
 164                 Driver:      "Directory",
 165                 ReadOnly:    false,
 166                 Replication: 1,
 167         }, &arvados.DirectoryVolumeDriverParameters{
 168                 Root:      tmpd + "/keep",
 169                 Serialize: false,
 170         }, &arvados.DirectoryVolumeDriverParameters{})
 171
 172         // read-only
 173         tmpf.Seek(0, os.SEEK_SET)
 174         tmpf.Truncate(0)
 175         _, err = fmt.Fprintf(tmpf, "/dev/xvdb %s ext4 ro,noexec 0 0\n", tmpd)
 176         c.Assert(err, check.IsNil)
 177
 178         s.testDeprecatedVolume(c, "DiscoverVolumesFromMountsFile: "+tmpf.Name(), arvados.Volume{
 179                 Driver:      "Directory",
 180                 ReadOnly:    true,
 181                 Replication: 1,
 182         }, &arvados.DirectoryVolumeDriverParameters{
 183                 Root:      tmpd + "/keep",
 184                 Serialize: false,
 185         }, &arvados.DirectoryVolumeDriverParameters{})
 186 }
 187
 188 func (s *KeepstoreMigrationSuite) TestDeprecatedVolumes(c *check.C) {
 189         accesskeyfile, err := ioutil.TempFile("", "")
 190         c.Assert(err, check.IsNil)
 191         defer os.Remove(accesskeyfile.Name())
 192         io.WriteString(accesskeyfile, "accesskeydata\n")
 193
 194         secretkeyfile, err := ioutil.TempFile("", "")
 195         c.Assert(err, check.IsNil)
 196         defer os.Remove(secretkeyfile.Name())
 197         io.WriteString(secretkeyfile, "secretkeydata\n")
 198
 199         // s3, empty/default
 200         s.testDeprecatedVolume(c, `
 201 Volumes:
 202 - Type: S3
 203 `, arvados.Volume{
 204                 Driver:      "S3",
 205                 Replication: 1,
 206         }, &arvados.S3VolumeDriverParameters{}, &arvados.S3VolumeDriverParameters{})
 207
 208         // s3, fully configured
 209         s.testDeprecatedVolume(c, `
 210 Volumes:
 211 - Type: S3
 212   AccessKeyFile: `+accesskeyfile.Name()+`
 213   SecretKeyFile: `+secretkeyfile.Name()+`
 214   Endpoint: https://storage.googleapis.com
 215   Region: us-east-1z
 216   Bucket: testbucket
 217   LocationConstraint: true
 218   IndexPageSize: 1234
 219   S3Replication: 4
 220   ConnectTimeout: 3m
 221   ReadTimeout: 4m
 222   RaceWindow: 5m
 223   UnsafeDelete: true
 224 `, arvados.Volume{
 225                 Driver:      "S3",
 226                 Replication: 4,
 227         }, &arvados.S3VolumeDriverParameters{
 228                 AccessKey:          "accesskeydata",
 229                 SecretKey:          "secretkeydata",
 230                 Endpoint:           "https://storage.googleapis.com",
 231                 Region:             "us-east-1z",
 232                 Bucket:             "testbucket",
 233                 LocationConstraint: true,
 234                 IndexPageSize:      1234,
 235                 ConnectTimeout:     arvados.Duration(time.Minute * 3),
 236                 ReadTimeout:        arvados.Duration(time.Minute * 4),
 237                 RaceWindow:         arvados.Duration(time.Minute * 5),
 238                 UnsafeDelete:       true,
 239         }, &arvados.S3VolumeDriverParameters{})
 240
 241         // azure, empty/default
 242         s.testDeprecatedVolume(c, `
 243 Volumes:
 244 - Type: Azure
 245 `, arvados.Volume{
 246                 Driver:      "Azure",
 247                 Replication: 1,
 248         }, &arvados.AzureVolumeDriverParameters{}, &arvados.AzureVolumeDriverParameters{})
 249
 250         // azure, fully configured
 251         s.testDeprecatedVolume(c, `
 252 Volumes:
 253 - Type: Azure
 254   ReadOnly: true
 255   StorageAccountName: storageacctname
 256   StorageAccountKeyFile: `+secretkeyfile.Name()+`
 257   StorageBaseURL: https://example.example
 258   ContainerName: testctr
 259   LocationConstraint: true
 260   AzureReplication: 4
 261   RequestTimeout: 3m
 262   ListBlobsRetryDelay: 4m
 263   ListBlobsMaxAttempts: 5
 264 `, arvados.Volume{
 265                 Driver:      "Azure",
 266                 ReadOnly:    true,
 267                 Replication: 4,
 268         }, &arvados.AzureVolumeDriverParameters{
 269                 StorageAccountName:   "storageacctname",
 270                 StorageAccountKey:    "secretkeydata",
 271                 StorageBaseURL:       "https://example.example",
 272                 ContainerName:        "testctr",
 273                 RequestTimeout:       arvados.Duration(time.Minute * 3),
 274                 ListBlobsRetryDelay:  arvados.Duration(time.Minute * 4),
 275                 ListBlobsMaxAttempts: 5,
 276         }, &arvados.AzureVolumeDriverParameters{})
 277
 278         // directory, empty/default
 279         s.testDeprecatedVolume(c, `
 280 Volumes:
 281 - Type: Directory
 282   Root: /tmp/xyzzy
 283 `, arvados.Volume{
 284                 Driver:      "Directory",
 285                 Replication: 1,
 286         }, &arvados.DirectoryVolumeDriverParameters{
 287                 Root: "/tmp/xyzzy",
 288         }, &arvados.DirectoryVolumeDriverParameters{})
 289
 290         // directory, fully configured
 291         s.testDeprecatedVolume(c, `
 292 Volumes:
 293 - Type: Directory
 294   ReadOnly: true
 295   Root: /tmp/xyzzy
 296   DirectoryReplication: 4
 297   Serialize: true
 298 `, arvados.Volume{
 299                 Driver:      "Directory",
 300                 ReadOnly:    true,
 301                 Replication: 4,
 302         }, &arvados.DirectoryVolumeDriverParameters{
 303                 Root:      "/tmp/xyzzy",
 304                 Serialize: true,
 305         }, &arvados.DirectoryVolumeDriverParameters{})
 306 }
 307
 308 func (s *KeepstoreMigrationSuite) testDeprecatedVolume(c *check.C, oldconfigdata string, expectvol arvados.Volume, expectparams interface{}, paramsdst interface{}) {
 309         hostname := s.hostname
 310         if hostname == "" {
 311                 h, err := os.Hostname()
 312                 c.Assert(err, check.IsNil)
 313                 hostname = h
 314         }
 315
 316         oldconfig, err := ioutil.TempFile("", "")
 317         c.Assert(err, check.IsNil)
 318         defer os.Remove(oldconfig.Name())
 319         io.WriteString(oldconfig, "Listen: :12345\n"+oldconfigdata)
 320         if !strings.Contains(oldconfigdata, "DiscoverVolumesFromMountsFile") {
 321                 // Prevent tests from looking at the real /proc/mounts on the test host.
 322                 io.WriteString(oldconfig, "\nDiscoverVolumesFromMountsFile: /dev/null\n")
 323         }
 324
 325         ldr := testLoader(c, "Clusters: {z1111: {}}", nil)
 326         ldr.KeepstorePath = oldconfig.Name()
 327         cfg, err := ldr.Load()
 328         c.Assert(err, check.IsNil)
 329         cc := cfg.Clusters["z1111"]
 330         c.Check(cc.Volumes, check.HasLen, 1)
 331         for uuid, v := range cc.Volumes {
 332                 c.Check(uuid, check.HasLen, 27)
 333                 c.Check(v.Driver, check.Equals, expectvol.Driver)
 334                 c.Check(v.Replication, check.Equals, expectvol.Replication)
 335
 336                 avh, ok := v.AccessViaHosts[arvados.URL{Scheme: "http", Host: hostname + ":12345"}]
 337                 c.Check(ok, check.Equals, true)
 338                 c.Check(avh.ReadOnly, check.Equals, expectvol.ReadOnly)
 339
 340                 err := json.Unmarshal(v.DriverParameters, paramsdst)
 341                 c.Check(err, check.IsNil)
 342                 c.Check(paramsdst, check.DeepEquals, expectparams)
 343         }
 344 }
 345
 346 // How we handle a volume from a legacy keepstore config file depends
 347 // on whether it's writable, whether a volume using the same cloud
 348 // backend already exists in the cluster config, and (if so) whether
 349 // it already has an AccessViaHosts entry for this host.
 350 //
 351 // In all cases, we should end up with an AccessViaHosts entry for
 352 // this host, to indicate that the current host's volumes have been
 353 // migrated.
 354
 355 // Same backend already referenced in cluster config, this host
 356 // already listed in AccessViaHosts --> no change, except possibly
 357 // updating the ReadOnly flag on the AccessViaHosts entry.
 358 func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_AlreadyMigrated(c *check.C) {
 359         before, after := s.loadWithKeepstoreConfig(c, `
 360 Listen: :12345
 361 Volumes:
 362 - Type: S3
 363   Endpoint: https://storage.googleapis.com
 364   Region: us-east-1z
 365   Bucket: alreadymigrated
 366   S3Replication: 3
 367 `)
 368         checkEqualYAML(c, after, before)
 369 }
 370
 371 // Writable volume, same cloud backend already referenced in cluster
 372 // config --> change UUID to match this keepstore's UUID.
 373 func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_UpdateUUID(c *check.C) {
 374         port, expectUUID := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
 375
 376         before, after := s.loadWithKeepstoreConfig(c, `
 377 Listen: :`+strconv.Itoa(port)+`
 378 Volumes:
 379 - Type: S3
 380   Endpoint: https://storage.googleapis.com
 381   Region: us-east-1z
 382   Bucket: readonlyonother
 383   S3Replication: 3
 384 `)
 385         c.Check(after, check.HasLen, len(before))
 386         newuuids := s.findAddedVolumes(c, before, after, 1)
 387         newvol := after[newuuids[0]]
 388
 389         var params arvados.S3VolumeDriverParameters
 390         json.Unmarshal(newvol.DriverParameters, &params)
 391         c.Check(params.Bucket, check.Equals, "readonlyonother")
 392         c.Check(newuuids[0], check.Equals, expectUUID)
 393 }
 394
 395 // Writable volume, same cloud backend not yet referenced --> add a
 396 // new volume, with UUID to match this keepstore's UUID.
 397 func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_AddCloudVolume(c *check.C) {
 398         port, expectUUID := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
 399
 400         before, after := s.loadWithKeepstoreConfig(c, `
 401 Listen: :`+strconv.Itoa(port)+`
 402 Volumes:
 403 - Type: S3
 404   Endpoint: https://storage.googleapis.com
 405   Region: us-east-1z
 406   Bucket: bucket-to-migrate
 407   S3Replication: 3
 408 `)
 409         newuuids := s.findAddedVolumes(c, before, after, 1)
 410         newvol := after[newuuids[0]]
 411
 412         var params arvados.S3VolumeDriverParameters
 413         json.Unmarshal(newvol.DriverParameters, &params)
 414         c.Check(params.Bucket, check.Equals, "bucket-to-migrate")
 415         c.Check(newvol.Replication, check.Equals, 3)
 416
 417         c.Check(newuuids[0], check.Equals, expectUUID)
 418 }
 419
 420 // Writable volume, same filesystem backend already referenced in
 421 // cluster config, but this host isn't in AccessViaHosts --> add a new
 422 // volume, with UUID to match this keepstore's UUID (filesystem-backed
 423 // volumes are assumed to be different on different hosts, even if
 424 // paths are the same).
 425 func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_AddLocalVolume(c *check.C) {
 426         before, after := s.loadWithKeepstoreConfig(c, `
 427 Listen: :12345
 428 Volumes:
 429 - Type: Directory
 430   Root: /data/sdd
 431   DirectoryReplication: 2
 432 `)
 433         newuuids := s.findAddedVolumes(c, before, after, 1)
 434         newvol := after[newuuids[0]]
 435
 436         var params arvados.DirectoryVolumeDriverParameters
 437         json.Unmarshal(newvol.DriverParameters, &params)
 438         c.Check(params.Root, check.Equals, "/data/sdd")
 439         c.Check(newvol.Replication, check.Equals, 2)
 440 }
 441
 442 // Writable volume, same filesystem backend already referenced in
 443 // cluster config, and this host is already listed in AccessViaHosts
 444 // --> already migrated, don't change anything.
 445 func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_LocalVolumeAlreadyMigrated(c *check.C) {
 446         before, after := s.loadWithKeepstoreConfig(c, `
 447 Listen: :12345
 448 Volumes:
 449 - Type: Directory
 450   Root: /data/sde
 451   DirectoryReplication: 2
 452 `)
 453         checkEqualYAML(c, after, before)
 454 }
 455
 456 // Multiple writable cloud-backed volumes --> one of them will get a
 457 // UUID matching this keepstore's UUID.
 458 func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_AddMultipleCloudVolumes(c *check.C) {
 459         port, expectUUID := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
 460
 461         before, after := s.loadWithKeepstoreConfig(c, `
 462 Listen: :`+strconv.Itoa(port)+`
 463 Volumes:
 464 - Type: S3
 465   Endpoint: https://storage.googleapis.com
 466   Region: us-east-1z
 467   Bucket: first-bucket-to-migrate
 468   S3Replication: 3
 469 - Type: S3
 470   Endpoint: https://storage.googleapis.com
 471   Region: us-east-1z
 472   Bucket: second-bucket-to-migrate
 473   S3Replication: 3
 474 `)
 475         newuuids := s.findAddedVolumes(c, before, after, 2)
 476         // Sort by bucket name (so "first" comes before "second")
 477         params := map[string]arvados.S3VolumeDriverParameters{}
 478         for _, uuid := range newuuids {
 479                 var p arvados.S3VolumeDriverParameters
 480                 json.Unmarshal(after[uuid].DriverParameters, &p)
 481                 params[uuid] = p
 482         }
 483         sort.Slice(newuuids, func(i, j int) bool { return params[newuuids[i]].Bucket < params[newuuids[j]].Bucket })
 484         newvol0, newvol1 := after[newuuids[0]], after[newuuids[1]]
 485         params0, params1 := params[newuuids[0]], params[newuuids[1]]
 486
 487         c.Check(params0.Bucket, check.Equals, "first-bucket-to-migrate")
 488         c.Check(newvol0.Replication, check.Equals, 3)
 489
 490         c.Check(params1.Bucket, check.Equals, "second-bucket-to-migrate")
 491         c.Check(newvol1.Replication, check.Equals, 3)
 492
 493         // Don't care which one gets the special UUID
 494         if newuuids[0] != expectUUID {
 495                 c.Check(newuuids[1], check.Equals, expectUUID)
 496         }
 497 }
 498
 499 // Non-writable volume, same cloud backend already referenced in
 500 // cluster config --> add this host to AccessViaHosts with
 501 // ReadOnly==true
 502 func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_UpdateWithReadOnly(c *check.C) {
 503         port, _ := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
 504         before, after := s.loadWithKeepstoreConfig(c, `
 505 Listen: :`+strconv.Itoa(port)+`
 506 Volumes:
 507 - Type: S3
 508   Endpoint: https://storage.googleapis.com
 509   Region: us-east-1z
 510   Bucket: readonlyonother
 511   S3Replication: 3
 512   ReadOnly: true
 513 `)
 514         hostname, err := os.Hostname()
 515         c.Assert(err, check.IsNil)
 516         url := arvados.URL{
 517                 Scheme: "http",
 518                 Host:   fmt.Sprintf("%s:%d", hostname, port),
 519         }
 520         _, ok := before["zzzzz-nyw5e-readonlyonother"].AccessViaHosts[url]
 521         c.Check(ok, check.Equals, false)
 522         _, ok = after["zzzzz-nyw5e-readonlyonother"].AccessViaHosts[url]
 523         c.Check(ok, check.Equals, true)
 524 }
 525
 526 // Writable volume, same cloud backend already writable by another
 527 // keepstore server --> add this host to AccessViaHosts with
 528 // ReadOnly==true
 529 func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_UpdateAlreadyWritable(c *check.C) {
 530         port, _ := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
 531         before, after := s.loadWithKeepstoreConfig(c, `
 532 Listen: :`+strconv.Itoa(port)+`
 533 Volumes:
 534 - Type: S3
 535   Endpoint: https://storage.googleapis.com
 536   Region: us-east-1z
 537   Bucket: writableonother
 538   S3Replication: 3
 539   ReadOnly: false
 540 `)
 541         hostname, err := os.Hostname()
 542         c.Assert(err, check.IsNil)
 543         url := arvados.URL{
 544                 Scheme: "http",
 545                 Host:   fmt.Sprintf("%s:%d", hostname, port),
 546         }
 547         _, ok := before["zzzzz-nyw5e-writableonother"].AccessViaHosts[url]
 548         c.Check(ok, check.Equals, false)
 549         _, ok = after["zzzzz-nyw5e-writableonother"].AccessViaHosts[url]
 550         c.Check(ok, check.Equals, true)
 551 }
 552
 553 // Non-writable volume, same cloud backend not already referenced in
 554 // cluster config --> assign a new random volume UUID.
 555 func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_AddReadOnly(c *check.C) {
 556         port, _ := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
 557         before, after := s.loadWithKeepstoreConfig(c, `
 558 Listen: :`+strconv.Itoa(port)+`
 559 Volumes:
 560 - Type: S3
 561   Endpoint: https://storage.googleapis.com
 562   Region: us-east-1z
 563   Bucket: differentbucket
 564   S3Replication: 3
 565 `)
 566         newuuids := s.findAddedVolumes(c, before, after, 1)
 567         newvol := after[newuuids[0]]
 568
 569         var params arvados.S3VolumeDriverParameters
 570         json.Unmarshal(newvol.DriverParameters, &params)
 571         c.Check(params.Bucket, check.Equals, "differentbucket")
 572
 573         hostname, err := os.Hostname()
 574         c.Assert(err, check.IsNil)
 575         _, ok := newvol.AccessViaHosts[arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%d", hostname, port)}]
 576         c.Check(ok, check.Equals, true)
 577 }
 578
 579 // Ensure logs mention unmigrated servers.
 580 func (s *KeepstoreMigrationSuite) TestPendingKeepstoreMigrations(c *check.C) {
 581         client := arvados.NewClientFromEnv()
 582         for _, host := range []string{"keep0", "keep1"} {
 583                 err := client.RequestAndDecode(new(struct{}), "POST", "arvados/v1/keep_services", nil, map[string]interface{}{
 584                         "keep_service": map[string]interface{}{
 585                                 "service_type": "disk",
 586                                 "service_host": host + ".zzzzz.example.com",
 587                                 "service_port": 25107,
 588                         },
 589                 })
 590                 c.Assert(err, check.IsNil)
 591         }
 592
 593         port, _ := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
 594         logs := s.logsWithKeepstoreConfig(c, `
 595 Listen: :`+strconv.Itoa(port)+`
 596 Volumes:
 597 - Type: S3
 598   Endpoint: https://storage.googleapis.com
 599   Bucket: foo
 600 `)
 601         c.Check(logs, check.Matches, `(?ms).*you should remove the legacy keepstore config file.*`)
 602         c.Check(logs, check.Matches, `(?ms).*you should migrate the legacy keepstore configuration file on host keep1.zzzzz.example.com.*`)
 603         c.Check(logs, check.Not(check.Matches), `(?ms).*should migrate.*keep0.zzzzz.example.com.*`)
 604         c.Check(logs, check.Matches, `(?ms).*keepstore configured at http://keep2.zzzzz.example.com:25107 does not have access to any volumes.*`)
 605         c.Check(logs, check.Matches, `(?ms).*Volumes.zzzzz-nyw5e-possconfigerror.AccessViaHosts refers to nonexistent keepstore server http://keep00.zzzzz.example.com:25107.*`)
 606 }
 607
 608 const clusterConfigForKeepstoreMigrationTest = `
 609 Clusters:
 610   zzzzz:
 611     SystemRootToken: ` + arvadostest.AdminToken + `
 612     Services:
 613       Keepstore:
 614         InternalURLs:
 615           "http://{{.hostname}}:12345": {}
 616           "http://keep0.zzzzz.example.com:25107": {}
 617           "http://keep2.zzzzz.example.com:25107": {}
 618       Controller:
 619         ExternalURL: "https://{{.controller}}"
 620     TLS:
 621       Insecure: true
 622     Volumes:
 623
 624       zzzzz-nyw5e-alreadymigrated:
 625         AccessViaHosts:
 626           "http://{{.hostname}}:12345": {}
 627         Driver: S3
 628         DriverParameters:
 629           Endpoint: https://storage.googleapis.com
 630           Region: us-east-1z
 631           Bucket: alreadymigrated
 632         Replication: 3
 633
 634       zzzzz-nyw5e-readonlyonother:
 635         AccessViaHosts:
 636           "http://keep0.zzzzz.example.com:25107": {ReadOnly: true}
 637         Driver: S3
 638         DriverParameters:
 639           Endpoint: https://storage.googleapis.com
 640           Region: us-east-1z
 641           Bucket: readonlyonother
 642         Replication: 3
 643
 644       zzzzz-nyw5e-writableonother:
 645         AccessViaHosts:
 646           "http://keep0.zzzzz.example.com:25107": {}
 647         Driver: S3
 648         DriverParameters:
 649           Endpoint: https://storage.googleapis.com
 650           Region: us-east-1z
 651           Bucket: writableonother
 652         Replication: 3
 653
 654       zzzzz-nyw5e-localfilesystem:
 655         AccessViaHosts:
 656           "http://keep0.zzzzz.example.com:25107": {}
 657         Driver: Directory
 658         DriverParameters:
 659           Root: /data/sdd
 660         Replication: 1
 661
 662       zzzzz-nyw5e-localismigrated:
 663         AccessViaHosts:
 664           "http://{{.hostname}}:12345": {}
 665         Driver: Directory
 666         DriverParameters:
 667           Root: /data/sde
 668         Replication: 1
 669
 670       zzzzz-nyw5e-possconfigerror:
 671         AccessViaHosts:
 672           "http://keep00.zzzzz.example.com:25107": {}
 673         Driver: Directory
 674         DriverParameters:
 675           Root: /data/sdf
 676         Replication: 1
 677 `
 678
 679 // Determine the effect of combining the given legacy keepstore config
 680 // YAML (just the "Volumes" entries of an old keepstore config file)
 681 // with the example clusterConfigForKeepstoreMigrationTest config.
 682 //
 683 // Return two Volumes configs -- one without loading keepstoreYAML
 684 // ("before") and one with ("after") -- for the caller to compare.
 685 func (s *KeepstoreMigrationSuite) loadWithKeepstoreConfig(c *check.C, keepstoreYAML string) (before, after map[string]arvados.Volume) {
 686         ldr := testLoader(c, s.clusterConfigYAML(c), nil)
 687         cBefore, err := ldr.Load()
 688         c.Assert(err, check.IsNil)
 689
 690         keepstoreconfig, err := ioutil.TempFile("", "")
 691         c.Assert(err, check.IsNil)
 692         defer os.Remove(keepstoreconfig.Name())
 693         io.WriteString(keepstoreconfig, keepstoreYAML)
 694
 695         ldr = testLoader(c, s.clusterConfigYAML(c), nil)
 696         ldr.KeepstorePath = keepstoreconfig.Name()
 697         cAfter, err := ldr.Load()
 698         c.Assert(err, check.IsNil)
 699
 700         return cBefore.Clusters["zzzzz"].Volumes, cAfter.Clusters["zzzzz"].Volumes
 701 }
 702
 703 // Return the log messages emitted when loading keepstoreYAML along
 704 // with clusterConfigForKeepstoreMigrationTest.
 705 func (s *KeepstoreMigrationSuite) logsWithKeepstoreConfig(c *check.C, keepstoreYAML string) string {
 706         var logs bytes.Buffer
 707
 708         keepstoreconfig, err := ioutil.TempFile("", "")
 709         c.Assert(err, check.IsNil)
 710         defer os.Remove(keepstoreconfig.Name())
 711         io.WriteString(keepstoreconfig, keepstoreYAML)
 712
 713         ldr := testLoader(c, s.clusterConfigYAML(c), &logs)
 714         ldr.KeepstorePath = keepstoreconfig.Name()
 715         _, err = ldr.Load()
 716         c.Assert(err, check.IsNil)
 717
 718         return logs.String()
 719 }
 720
 721 func (s *KeepstoreMigrationSuite) clusterConfigYAML(c *check.C) string {
 722         hostname, err := os.Hostname()
 723         c.Assert(err, check.IsNil)
 724
 725         tmpl := template.Must(template.New("config").Parse(clusterConfigForKeepstoreMigrationTest))
 726
 727         var clusterconfigdata bytes.Buffer
 728         err = tmpl.Execute(&clusterconfigdata, map[string]interface{}{
 729                 "hostname":   hostname,
 730                 "controller": os.Getenv("ARVADOS_API_HOST"),
 731         })
 732         c.Assert(err, check.IsNil)
 733
 734         return clusterconfigdata.String()
 735 }
 736
 737 // Return the uuids of volumes that appear in "after" but not
 738 // "before".
 739 //
 740 // Assert the returned slice has at least minAdded entries.
 741 func (s *KeepstoreMigrationSuite) findAddedVolumes(c *check.C, before, after map[string]arvados.Volume, minAdded int) (uuids []string) {
 742         for uuid := range after {
 743                 if _, ok := before[uuid]; !ok {
 744                         uuids = append(uuids, uuid)
 745                 }
 746         }
 747         if len(uuids) < minAdded {
 748                 c.Assert(uuids, check.HasLen, minAdded)
 749         }
 750         return
 751 }
 752
 753 func (s *KeepstoreMigrationSuite) getTestKeepstorePortAndMatchingVolumeUUID(c *check.C) (int, string) {
 754         for port, ks := range s.ksByPort {
 755                 c.Assert(ks.UUID, check.HasLen, 27)
 756                 return port, "zzzzz-nyw5e-" + ks.UUID[12:]
 757         }
 758         c.Fatal("s.ksByPort is empty")
 759         return 0, ""
 760 }
 761
 762 func (s *KeepstoreMigrationSuite) TestKeepServiceIsMe(c *check.C) {
 763         for i, trial := range []struct {
 764                 match       bool
 765                 hostname    string
 766                 listen      string
 767                 serviceHost string
 768                 servicePort int
 769         }{
 770                 {true, "keep0", "keep0", "keep0", 80},
 771                 {true, "keep0", "[::1]:http", "keep0", 80},
 772                 {true, "keep0", "[::]:http", "keep0", 80},
 773                 {true, "keep0", "keep0:25107", "keep0", 25107},
 774                 {true, "keep0", ":25107", "keep0", 25107},
 775                 {true, "keep0.domain", ":25107", "keep0.domain.example", 25107},
 776                 {true, "keep0.domain.example", ":25107", "keep0.domain.example", 25107},
 777                 {true, "keep0", ":25107", "keep0.domain.example", 25107},
 778                 {true, "keep0", ":25107", "Keep0.domain.example", 25107},
 779                 {true, "keep0", ":http", "keep0.domain.example", 80},
 780                 {true, "keep0", ":25107", "localhost", 25107},
 781                 {true, "keep0", ":25107", "::1", 25107},
 782                 {false, "keep0", ":25107", "keep0", 1111},              // different port
 783                 {false, "keep0", ":25107", "localhost", 1111},          // different port
 784                 {false, "keep0", ":http", "keep0.domain.example", 443}, // different port
 785                 {false, "keep0", ":bogussss", "keep0", 25107},          // unresolvable port
 786                 {false, "keep0", ":25107", "keep1", 25107},             // different hostname
 787                 {false, "keep1", ":25107", "keep10", 25107},            // different hostname (prefix, but not on a "." boundary)
 788         } {
 789                 c.Check(keepServiceIsMe(arvados.KeepService{ServiceHost: trial.serviceHost, ServicePort: trial.servicePort}, trial.hostname, trial.listen), check.Equals, trial.match, check.Commentf("trial #%d: %#v", i, trial))
 790         }
 791 }