lib/config/deprecated_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package config
   6
   7 import (
   8         "flag"
   9         "fmt"
  10         "io/ioutil"
  11         "os"
  12         "time"
  13
  14         "git.arvados.org/arvados.git/sdk/go/arvados"
  15         check "gopkg.in/check.v1"
  16 )
  17
  18 // Configured at: sdk/python/tests/run_test_server.py
  19 const TestServerManagementToken = "e687950a23c3a9bceec28c6223a06c79"
  20
  21 func testLoadLegacyConfig(content []byte, mungeFlag string, c *check.C) (*arvados.Cluster, error) {
  22         tmpfile, err := ioutil.TempFile("", "example")
  23         if err != nil {
  24                 return nil, err
  25         }
  26         defer os.Remove(tmpfile.Name())
  27
  28         if _, err := tmpfile.Write(content); err != nil {
  29                 return nil, err
  30         }
  31         if err := tmpfile.Close(); err != nil {
  32                 return nil, err
  33         }
  34         flags := flag.NewFlagSet("test", flag.ExitOnError)
  35         ldr := testLoader(c, "Clusters: {zzzzz: {}}", nil)
  36         ldr.SetupFlags(flags)
  37         args := ldr.MungeLegacyConfigArgs(ldr.Logger, []string{"-config", tmpfile.Name()}, mungeFlag)
  38         err = flags.Parse(args)
  39         c.Assert(err, check.IsNil)
  40         c.Assert(flags.NArg(), check.Equals, 0)
  41         cfg, err := ldr.Load()
  42         if err != nil {
  43                 return nil, err
  44         }
  45         cluster, err := cfg.GetCluster("")
  46         if err != nil {
  47                 return nil, err
  48         }
  49         return cluster, nil
  50 }
  51
  52 func (s *LoadSuite) TestOldEmailConfiguration(c *check.C) {
  53         logs := checkEquivalent(c, `
  54 Clusters:
  55  z1111:
  56   Mail:
  57     SendUserSetupNotificationEmail: false
  58     SupportEmailAddress: "support@example.invalid"
  59 `, `
  60 Clusters:
  61  z1111:
  62   Users:
  63     SendUserSetupNotificationEmail: false
  64     SupportEmailAddress: "support@example.invalid"
  65 `)
  66         c.Check(logs, check.Matches, `(?ms).*deprecated or unknown config entry: .*Mail\.SendUserSetupNotificationEmail.*`)
  67         c.Check(logs, check.Matches, `(?ms).*deprecated or unknown config entry: .*Mail\.SupportEmailAddress.*`)
  68         c.Check(logs, check.Matches, `(?ms).*using your old config key Mail\.SendUserSetupNotificationEmail -- but you should rename it to Users\.SendUserSetupNotificationEmail.*`)
  69         c.Check(logs, check.Matches, `(?ms).*using your old config key Mail\.SupportEmailAddress -- but you should rename it to Users\.SupportEmailAddress.*`)
  70 }
  71
  72 func (s *LoadSuite) TestLegacyVolumeDriverParameters(c *check.C) {
  73         logs := checkEquivalent(c, `
  74 Clusters:
  75  z1111:
  76   Volumes:
  77    z1111-nyw5e-aaaaaaaaaaaaaaa:
  78     Driver: S3
  79     DriverParameters:
  80      AccessKey: exampleaccesskey
  81      SecretKey: examplesecretkey
  82      Region: foobar
  83      ReadTimeout: 1200s
  84 `, `
  85 Clusters:
  86  z1111:
  87   Volumes:
  88    z1111-nyw5e-aaaaaaaaaaaaaaa:
  89     Driver: S3
  90     DriverParameters:
  91      AccessKeyID: exampleaccesskey
  92      SecretAccessKey: examplesecretkey
  93      Region: foobar
  94      ReadTimeout: 1200s
  95 `)
  96         c.Check(logs, check.Matches, `(?ms).*deprecated or unknown config entry: .*AccessKey.*`)
  97         c.Check(logs, check.Matches, `(?ms).*deprecated or unknown config entry: .*SecretKey.*`)
  98         c.Check(logs, check.Matches, `(?ms).*using your old config keys z1111\.Volumes\.z1111-nyw5e-aaaaaaaaaaaaaaa\.DriverParameters\.AccessKey/SecretKey -- but you should rename them to AccessKeyID/SecretAccessKey.*`)
  99
 100         _, err := testLoader(c, `
 101 Clusters:
 102  z1111:
 103   Volumes:
 104    z1111-nyw5e-aaaaaaaaaaaaaaa:
 105     Driver: S3
 106     DriverParameters:
 107      AccessKey: exampleaccesskey
 108      SecretKey: examplesecretkey
 109      AccessKeyID: exampleaccesskey
 110 `, nil).Load()
 111         c.Check(err, check.ErrorMatches, `(?ms).*cannot use .*SecretKey.*and.*SecretAccessKey.*in z1111.Volumes.z1111-nyw5e-aaaaaaaaaaaaaaa.DriverParameters.*`)
 112 }
 113
 114 func (s *LoadSuite) TestDeprecatedNodeProfilesToServices(c *check.C) {
 115         hostname, err := os.Hostname()
 116         c.Assert(err, check.IsNil)
 117         checkEquivalent(c, `
 118 Clusters:
 119  z1111:
 120   NodeProfiles:
 121    "*":
 122     arvados-controller:
 123      listen: ":9004"
 124    `+hostname+`:
 125     arvados-api-server:
 126      listen: ":8000"
 127    dispatch-host:
 128     arvados-dispatch-cloud:
 129      listen: ":9006"
 130 `, `
 131 Clusters:
 132  z1111:
 133   Services:
 134    RailsAPI:
 135     InternalURLs:
 136      "http://localhost:8000": {}
 137    Controller:
 138     InternalURLs:
 139      "http://localhost:9004": {}
 140    DispatchCloud:
 141     InternalURLs:
 142      "http://dispatch-host:9006": {}
 143   NodeProfiles:
 144    "*":
 145     arvados-controller:
 146      listen: ":9004"
 147    `+hostname+`:
 148     arvados-api-server:
 149      listen: ":8000"
 150    dispatch-host:
 151     arvados-dispatch-cloud:
 152      listen: ":9006"
 153 `)
 154 }
 155
 156 func (s *LoadSuite) TestDeprecatedLoginBackend(c *check.C) {
 157         checkEquivalent(c, `
 158 Clusters:
 159  z1111:
 160   Login:
 161    GoogleClientID: aaaa
 162    GoogleClientSecret: bbbb
 163    GoogleAlternateEmailAddresses: true
 164 `, `
 165 Clusters:
 166  z1111:
 167   Login:
 168    Google:
 169     Enable: true
 170     ClientID: aaaa
 171     ClientSecret: bbbb
 172     AlternateEmailAddresses: true
 173 `)
 174         checkEquivalent(c, `
 175 Clusters:
 176  z1111:
 177   Login:
 178    ProviderAppID: aaaa
 179    ProviderAppSecret: bbbb
 180 `, `
 181 Clusters:
 182  z1111:
 183   Login:
 184    SSO:
 185     Enable: true
 186     ProviderAppID: aaaa
 187     ProviderAppSecret: bbbb
 188 `)
 189 }
 190
 191 func (s *LoadSuite) TestLegacyKeepWebConfig(c *check.C) {
 192         content := []byte(`
 193 {
 194         "Client": {
 195                 "Scheme": "",
 196                 "APIHost": "example.com",
 197                 "AuthToken": "abcdefg",
 198         },
 199         "Listen": ":80",
 200         "AnonymousTokens": [
 201                 "anonusertoken"
 202         ],
 203         "AttachmentOnlyHost": "download.example.com",
 204         "TrustAllContent": true,
 205         "Cache": {
 206                 "TTL": "1m",
 207                 "UUIDTTL": "1s",
 208                 "MaxCollectionEntries": 42,
 209                 "MaxCollectionBytes": 1234567890,
 210                 "MaxUUIDEntries": 100
 211         },
 212         "ManagementToken": "xyzzy"
 213 }
 214 `)
 215         cluster, err := testLoadLegacyConfig(content, "-legacy-keepweb-config", c)
 216         c.Assert(err, check.IsNil)
 217
 218         c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
 219         c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
 220
 221         c.Check(cluster.Collections.WebDAVCache.TTL, check.Equals, arvados.Duration(60*time.Second))
 222         c.Check(cluster.Collections.WebDAVCache.MaxCollectionBytes, check.Equals, arvados.ByteSize(1234567890))
 223
 224         c.Check(cluster.Services.WebDAVDownload.ExternalURL, check.Equals, arvados.URL{Host: "download.example.com", Path: "/"})
 225         c.Check(cluster.Services.WebDAVDownload.InternalURLs[arvados.URL{Host: ":80"}], check.NotNil)
 226         c.Check(cluster.Services.WebDAV.InternalURLs[arvados.URL{Host: ":80"}], check.NotNil)
 227
 228         c.Check(cluster.Collections.TrustAllContent, check.Equals, true)
 229         c.Check(cluster.Users.AnonymousUserToken, check.Equals, "anonusertoken")
 230         c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
 231 }
 232
 233 // Tests fix for https://dev.arvados.org/issues/15642
 234 func (s *LoadSuite) TestLegacyKeepWebConfigDoesntDisableMissingItems(c *check.C) {
 235         content := []byte(`
 236 {
 237         "Client": {
 238                 "Scheme": "",
 239                 "APIHost": "example.com",
 240                 "AuthToken": "abcdefg",
 241         }
 242 }
 243 `)
 244         cluster, err := testLoadLegacyConfig(content, "-legacy-keepweb-config", c)
 245         c.Assert(err, check.IsNil)
 246         // The resulting ManagementToken should be the one set up on the test server.
 247         c.Check(cluster.ManagementToken, check.Equals, TestServerManagementToken)
 248 }
 249
 250 func (s *LoadSuite) TestLegacyKeepproxyConfig(c *check.C) {
 251         f := "-legacy-keepproxy-config"
 252         content := []byte(fmtKeepproxyConfig("", true))
 253         cluster, err := testLoadLegacyConfig(content, f, c)
 254
 255         c.Assert(err, check.IsNil)
 256         c.Assert(cluster, check.NotNil)
 257         c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
 258         c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
 259         c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
 260         c.Check(cluster.Services.Keepproxy.InternalURLs[arvados.URL{Host: ":80"}], check.Equals, arvados.ServiceInstance{})
 261         c.Check(cluster.Collections.DefaultReplication, check.Equals, 0)
 262         c.Check(cluster.API.KeepServiceRequestTimeout.String(), check.Equals, "15s")
 263         c.Check(cluster.SystemLogs.LogLevel, check.Equals, "debug")
 264
 265         content = []byte(fmtKeepproxyConfig("", false))
 266         cluster, err = testLoadLegacyConfig(content, f, c)
 267         c.Check(err, check.IsNil)
 268         c.Check(cluster.SystemLogs.LogLevel, check.Equals, "info")
 269
 270         content = []byte(fmtKeepproxyConfig(`"DisableGet": true,`, true))
 271         _, err = testLoadLegacyConfig(content, f, c)
 272         c.Check(err, check.NotNil)
 273
 274         content = []byte(fmtKeepproxyConfig(`"DisablePut": true,`, true))
 275         _, err = testLoadLegacyConfig(content, f, c)
 276         c.Check(err, check.NotNil)
 277
 278         content = []byte(fmtKeepproxyConfig(`"PIDFile": "test",`, true))
 279         _, err = testLoadLegacyConfig(content, f, c)
 280         c.Check(err, check.NotNil)
 281
 282         content = []byte(fmtKeepproxyConfig(`"DisableGet": false, "DisablePut": false, "PIDFile": "",`, true))
 283         _, err = testLoadLegacyConfig(content, f, c)
 284         c.Check(err, check.IsNil)
 285 }
 286
 287 func fmtKeepproxyConfig(param string, debugLog bool) string {
 288         return fmt.Sprintf(`
 289 {
 290         "Client": {
 291                 "Scheme": "",
 292                 "APIHost": "example.com",
 293                 "AuthToken": "abcdefg",
 294                 "Insecure": false
 295         },
 296         "Listen": ":80",
 297         "DefaultReplicas": 0,
 298         "Timeout": "15s",
 299         "Debug": %t,
 300         %s
 301         "ManagementToken": "xyzzy"
 302 }
 303 `, debugLog, param)
 304 }
 305
 306 func (s *LoadSuite) TestLegacyKeepBalanceConfig(c *check.C) {
 307         f := "-legacy-keepbalance-config"
 308         content := []byte(fmtKeepBalanceConfig(""))
 309         cluster, err := testLoadLegacyConfig(content, f, c)
 310
 311         c.Assert(err, check.IsNil)
 312         c.Assert(cluster, check.NotNil)
 313         c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
 314         c.Check(cluster.Services.Keepbalance.InternalURLs[arvados.URL{Host: ":80"}], check.Equals, arvados.ServiceInstance{})
 315         c.Check(cluster.Collections.BalanceCollectionBuffers, check.Equals, 1000)
 316         c.Check(cluster.Collections.BalanceCollectionBatch, check.Equals, 100000)
 317         c.Check(cluster.Collections.BalancePeriod.String(), check.Equals, "10m")
 318         c.Check(cluster.Collections.BlobMissingReport, check.Equals, "testfile")
 319         c.Check(cluster.API.KeepServiceRequestTimeout.String(), check.Equals, "30m")
 320
 321         content = []byte(fmtKeepBalanceConfig(`"KeepServiceTypes":["disk"],`))
 322         _, err = testLoadLegacyConfig(content, f, c)
 323         c.Check(err, check.IsNil)
 324
 325         content = []byte(fmtKeepBalanceConfig(`"KeepServiceTypes":[],`))
 326         _, err = testLoadLegacyConfig(content, f, c)
 327         c.Check(err, check.IsNil)
 328
 329         content = []byte(fmtKeepBalanceConfig(`"KeepServiceTypes":["proxy"],`))
 330         _, err = testLoadLegacyConfig(content, f, c)
 331         c.Check(err, check.NotNil)
 332
 333         content = []byte(fmtKeepBalanceConfig(`"KeepServiceTypes":["disk", "proxy"],`))
 334         _, err = testLoadLegacyConfig(content, f, c)
 335         c.Check(err, check.NotNil)
 336
 337         content = []byte(fmtKeepBalanceConfig(`"KeepServiceList":{},`))
 338         _, err = testLoadLegacyConfig(content, f, c)
 339         c.Check(err, check.NotNil)
 340 }
 341
 342 func fmtKeepBalanceConfig(param string) string {
 343         return fmt.Sprintf(`
 344 {
 345         "Client": {
 346                 "Scheme": "",
 347                 "APIHost": "example.com",
 348                 "AuthToken": "abcdefg",
 349                 "Insecure": false
 350         },
 351         "Listen": ":80",
 352         %s
 353         "RunPeriod": "10m",
 354         "CollectionBatchSize": 100000,
 355         "CollectionBuffers": 1000,
 356         "RequestTimeout": "30m",
 357         "ManagementToken": "xyzzy",
 358         "LostBlocksFile": "testfile"
 359 }
 360 `, param)
 361 }