lib/config/deprecated_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package config
   6
   7 import (
   8         "flag"
   9         "fmt"
  10         "io/ioutil"
  11         "os"
  12         "time"
  13
  14         "git.arvados.org/arvados.git/sdk/go/arvados"
  15         check "gopkg.in/check.v1"
  16 )
  17
  18 // Configured at: sdk/python/tests/run_test_server.py
  19 const TestServerManagementToken = "e687950a23c3a9bceec28c6223a06c79"
  20
  21 func testLoadLegacyConfig(content []byte, mungeFlag string, c *check.C) (*arvados.Cluster, error) {
  22         tmpfile, err := ioutil.TempFile("", "example")
  23         if err != nil {
  24                 return nil, err
  25         }
  26         defer os.Remove(tmpfile.Name())
  27
  28         if _, err := tmpfile.Write(content); err != nil {
  29                 return nil, err
  30         }
  31         if err := tmpfile.Close(); err != nil {
  32                 return nil, err
  33         }
  34         flags := flag.NewFlagSet("test", flag.ExitOnError)
  35         ldr := testLoader(c, "Clusters: {zzzzz: {}}", nil)
  36         ldr.SetupFlags(flags)
  37         args := ldr.MungeLegacyConfigArgs(ldr.Logger, []string{"-config", tmpfile.Name()}, mungeFlag)
  38         flags.Parse(args)
  39         cfg, err := ldr.Load()
  40         if err != nil {
  41                 return nil, err
  42         }
  43         cluster, err := cfg.GetCluster("")
  44         if err != nil {
  45                 return nil, err
  46         }
  47         return cluster, nil
  48 }
  49
  50 func (s *LoadSuite) TestLegacyVolumeDriverParameters(c *check.C) {
  51         logs := checkEquivalent(c, `
  52 Clusters:
  53  z1111:
  54   Volumes:
  55    z1111-nyw5e-aaaaaaaaaaaaaaa:
  56     Driver: S3
  57     DriverParameters:
  58      AccessKey: exampleaccesskey
  59      SecretKey: examplesecretkey
  60      Region: foobar
  61      ReadTimeout: 1200s
  62 `, `
  63 Clusters:
  64  z1111:
  65   Volumes:
  66    z1111-nyw5e-aaaaaaaaaaaaaaa:
  67     Driver: S3
  68     DriverParameters:
  69      AccessKeyID: exampleaccesskey
  70      SecretAccessKey: examplesecretkey
  71      Region: foobar
  72      ReadTimeout: 1200s
  73 `)
  74         c.Check(logs, check.Matches, `(?ms).*deprecated or unknown config entry: .*AccessKey.*`)
  75         c.Check(logs, check.Matches, `(?ms).*deprecated or unknown config entry: .*SecretKey.*`)
  76         c.Check(logs, check.Matches, `(?ms).*using your old config keys z1111\.Volumes\.z1111-nyw5e-aaaaaaaaaaaaaaa\.DriverParameters\.AccessKey/SecretKey -- but you should rename them to AccessKeyID/SecretAccessKey.*`)
  77
  78         _, err := testLoader(c, `
  79 Clusters:
  80  z1111:
  81   Volumes:
  82    z1111-nyw5e-aaaaaaaaaaaaaaa:
  83     Driver: S3
  84     DriverParameters:
  85      AccessKey: exampleaccesskey
  86      SecretKey: examplesecretkey
  87      AccessKeyID: exampleaccesskey
  88 `, nil).Load()
  89         c.Check(err, check.ErrorMatches, `(?ms).*cannot use .*SecretKey.*and.*SecretAccessKey.*in z1111.Volumes.z1111-nyw5e-aaaaaaaaaaaaaaa.DriverParameters.*`)
  90 }
  91
  92 func (s *LoadSuite) TestDeprecatedNodeProfilesToServices(c *check.C) {
  93         hostname, err := os.Hostname()
  94         c.Assert(err, check.IsNil)
  95         checkEquivalent(c, `
  96 Clusters:
  97  z1111:
  98   NodeProfiles:
  99    "*":
 100     arvados-controller:
 101      listen: ":9004"
 102    `+hostname+`:
 103     arvados-api-server:
 104      listen: ":8000"
 105    dispatch-host:
 106     arvados-dispatch-cloud:
 107      listen: ":9006"
 108 `, `
 109 Clusters:
 110  z1111:
 111   Services:
 112    RailsAPI:
 113     InternalURLs:
 114      "http://localhost:8000": {}
 115    Controller:
 116     InternalURLs:
 117      "http://localhost:9004": {}
 118    DispatchCloud:
 119     InternalURLs:
 120      "http://dispatch-host:9006": {}
 121   NodeProfiles:
 122    "*":
 123     arvados-controller:
 124      listen: ":9004"
 125    `+hostname+`:
 126     arvados-api-server:
 127      listen: ":8000"
 128    dispatch-host:
 129     arvados-dispatch-cloud:
 130      listen: ":9006"
 131 `)
 132 }
 133
 134 func (s *LoadSuite) TestDeprecatedLoginBackend(c *check.C) {
 135         checkEquivalent(c, `
 136 Clusters:
 137  z1111:
 138   Login:
 139    GoogleClientID: aaaa
 140    GoogleClientSecret: bbbb
 141    GoogleAlternateEmailAddresses: true
 142 `, `
 143 Clusters:
 144  z1111:
 145   Login:
 146    Google:
 147     Enable: true
 148     ClientID: aaaa
 149     ClientSecret: bbbb
 150     AlternateEmailAddresses: true
 151 `)
 152         checkEquivalent(c, `
 153 Clusters:
 154  z1111:
 155   Login:
 156    ProviderAppID: aaaa
 157    ProviderAppSecret: bbbb
 158 `, `
 159 Clusters:
 160  z1111:
 161   Login:
 162    SSO:
 163     Enable: true
 164     ProviderAppID: aaaa
 165     ProviderAppSecret: bbbb
 166 `)
 167 }
 168
 169 func (s *LoadSuite) TestLegacyKeepWebConfig(c *check.C) {
 170         content := []byte(`
 171 {
 172         "Client": {
 173                 "Scheme": "",
 174                 "APIHost": "example.com",
 175                 "AuthToken": "abcdefg",
 176         },
 177         "Listen": ":80",
 178         "AnonymousTokens": [
 179                 "anonusertoken"
 180         ],
 181         "AttachmentOnlyHost": "download.example.com",
 182         "TrustAllContent": true,
 183         "Cache": {
 184                 "TTL": "1m",
 185                 "UUIDTTL": "1s",
 186                 "MaxCollectionEntries": 42,
 187                 "MaxCollectionBytes": 1234567890,
 188                 "MaxUUIDEntries": 100
 189         },
 190         "ManagementToken": "xyzzy"
 191 }
 192 `)
 193         cluster, err := testLoadLegacyConfig(content, "-legacy-keepweb-config", c)
 194         c.Assert(err, check.IsNil)
 195
 196         c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
 197         c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
 198
 199         c.Check(cluster.Collections.WebDAVCache.TTL, check.Equals, arvados.Duration(60*time.Second))
 200         c.Check(cluster.Collections.WebDAVCache.UUIDTTL, check.Equals, arvados.Duration(time.Second))
 201         c.Check(cluster.Collections.WebDAVCache.MaxCollectionEntries, check.Equals, 42)
 202         c.Check(cluster.Collections.WebDAVCache.MaxCollectionBytes, check.Equals, int64(1234567890))
 203         c.Check(cluster.Collections.WebDAVCache.MaxUUIDEntries, check.Equals, 100)
 204
 205         c.Check(cluster.Services.WebDAVDownload.ExternalURL, check.Equals, arvados.URL{Host: "download.example.com", Path: "/"})
 206         c.Check(cluster.Services.WebDAVDownload.InternalURLs[arvados.URL{Host: ":80"}], check.NotNil)
 207         c.Check(cluster.Services.WebDAV.InternalURLs[arvados.URL{Host: ":80"}], check.NotNil)
 208
 209         c.Check(cluster.Collections.TrustAllContent, check.Equals, true)
 210         c.Check(cluster.Users.AnonymousUserToken, check.Equals, "anonusertoken")
 211         c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
 212 }
 213
 214 // Tests fix for https://dev.arvados.org/issues/15642
 215 func (s *LoadSuite) TestLegacyKeepWebConfigDoesntDisableMissingItems(c *check.C) {
 216         content := []byte(`
 217 {
 218         "Client": {
 219                 "Scheme": "",
 220                 "APIHost": "example.com",
 221                 "AuthToken": "abcdefg",
 222         }
 223 }
 224 `)
 225         cluster, err := testLoadLegacyConfig(content, "-legacy-keepweb-config", c)
 226         c.Assert(err, check.IsNil)
 227         // The resulting ManagementToken should be the one set up on the test server.
 228         c.Check(cluster.ManagementToken, check.Equals, TestServerManagementToken)
 229 }
 230
 231 func (s *LoadSuite) TestLegacyKeepproxyConfig(c *check.C) {
 232         f := "-legacy-keepproxy-config"
 233         content := []byte(fmtKeepproxyConfig("", true))
 234         cluster, err := testLoadLegacyConfig(content, f, c)
 235
 236         c.Assert(err, check.IsNil)
 237         c.Assert(cluster, check.NotNil)
 238         c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
 239         c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
 240         c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
 241         c.Check(cluster.Services.Keepproxy.InternalURLs[arvados.URL{Host: ":80"}], check.Equals, arvados.ServiceInstance{})
 242         c.Check(cluster.Collections.DefaultReplication, check.Equals, 0)
 243         c.Check(cluster.API.KeepServiceRequestTimeout.String(), check.Equals, "15s")
 244         c.Check(cluster.SystemLogs.LogLevel, check.Equals, "debug")
 245
 246         content = []byte(fmtKeepproxyConfig("", false))
 247         cluster, err = testLoadLegacyConfig(content, f, c)
 248         c.Check(err, check.IsNil)
 249         c.Check(cluster.SystemLogs.LogLevel, check.Equals, "info")
 250
 251         content = []byte(fmtKeepproxyConfig(`"DisableGet": true,`, true))
 252         _, err = testLoadLegacyConfig(content, f, c)
 253         c.Check(err, check.NotNil)
 254
 255         content = []byte(fmtKeepproxyConfig(`"DisablePut": true,`, true))
 256         _, err = testLoadLegacyConfig(content, f, c)
 257         c.Check(err, check.NotNil)
 258
 259         content = []byte(fmtKeepproxyConfig(`"PIDFile": "test",`, true))
 260         _, err = testLoadLegacyConfig(content, f, c)
 261         c.Check(err, check.NotNil)
 262
 263         content = []byte(fmtKeepproxyConfig(`"DisableGet": false, "DisablePut": false, "PIDFile": "",`, true))
 264         _, err = testLoadLegacyConfig(content, f, c)
 265         c.Check(err, check.IsNil)
 266 }
 267
 268 func fmtKeepproxyConfig(param string, debugLog bool) string {
 269         return fmt.Sprintf(`
 270 {
 271         "Client": {
 272                 "Scheme": "",
 273                 "APIHost": "example.com",
 274                 "AuthToken": "abcdefg",
 275                 "Insecure": false
 276         },
 277         "Listen": ":80",
 278         "DefaultReplicas": 0,
 279         "Timeout": "15s",
 280         "Debug": %t,
 281         %s
 282         "ManagementToken": "xyzzy"
 283 }
 284 `, debugLog, param)
 285 }
 286
 287 func (s *LoadSuite) TestLegacyArvGitHttpdConfig(c *check.C) {
 288         content := []byte(`
 289 {
 290         "Client": {
 291                 "Scheme": "",
 292                 "APIHost": "example.com",
 293                 "AuthToken": "abcdefg",
 294         },
 295         "Listen": ":9000",
 296         "GitCommand": "/test/git",
 297         "GitoliteHome": "/test/gitolite",
 298         "RepoRoot": "/test/reporoot",
 299         "ManagementToken": "xyzzy"
 300 }
 301 `)
 302         f := "-legacy-git-httpd-config"
 303         cluster, err := testLoadLegacyConfig(content, f, c)
 304
 305         c.Assert(err, check.IsNil)
 306         c.Assert(cluster, check.NotNil)
 307         c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
 308         c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
 309         c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
 310         c.Check(cluster.Git.GitCommand, check.Equals, "/test/git")
 311         c.Check(cluster.Git.GitoliteHome, check.Equals, "/test/gitolite")
 312         c.Check(cluster.Git.Repositories, check.Equals, "/test/reporoot")
 313         c.Check(cluster.Services.Keepproxy.InternalURLs[arvados.URL{Host: ":9000"}], check.Equals, arvados.ServiceInstance{})
 314 }
 315
 316 // Tests fix for https://dev.arvados.org/issues/15642
 317 func (s *LoadSuite) TestLegacyArvGitHttpdConfigDoesntDisableMissingItems(c *check.C) {
 318         content := []byte(`
 319 {
 320         "Client": {
 321                 "Scheme": "",
 322                 "APIHost": "example.com",
 323                 "AuthToken": "abcdefg",
 324         }
 325 }
 326 `)
 327         cluster, err := testLoadLegacyConfig(content, "-legacy-git-httpd-config", c)
 328         c.Assert(err, check.IsNil)
 329         // The resulting ManagementToken should be the one set up on the test server.
 330         c.Check(cluster.ManagementToken, check.Equals, TestServerManagementToken)
 331 }
 332
 333 func (s *LoadSuite) TestLegacyKeepBalanceConfig(c *check.C) {
 334         f := "-legacy-keepbalance-config"
 335         content := []byte(fmtKeepBalanceConfig(""))
 336         cluster, err := testLoadLegacyConfig(content, f, c)
 337
 338         c.Assert(err, check.IsNil)
 339         c.Assert(cluster, check.NotNil)
 340         c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
 341         c.Check(cluster.Services.Keepbalance.InternalURLs[arvados.URL{Host: ":80"}], check.Equals, arvados.ServiceInstance{})
 342         c.Check(cluster.Collections.BalanceCollectionBuffers, check.Equals, 1000)
 343         c.Check(cluster.Collections.BalanceCollectionBatch, check.Equals, 100000)
 344         c.Check(cluster.Collections.BalancePeriod.String(), check.Equals, "10m")
 345         c.Check(cluster.Collections.BlobMissingReport, check.Equals, "testfile")
 346         c.Check(cluster.API.KeepServiceRequestTimeout.String(), check.Equals, "30m")
 347
 348         content = []byte(fmtKeepBalanceConfig(`"KeepServiceTypes":["disk"],`))
 349         _, err = testLoadLegacyConfig(content, f, c)
 350         c.Check(err, check.IsNil)
 351
 352         content = []byte(fmtKeepBalanceConfig(`"KeepServiceTypes":[],`))
 353         _, err = testLoadLegacyConfig(content, f, c)
 354         c.Check(err, check.IsNil)
 355
 356         content = []byte(fmtKeepBalanceConfig(`"KeepServiceTypes":["proxy"],`))
 357         _, err = testLoadLegacyConfig(content, f, c)
 358         c.Check(err, check.NotNil)
 359
 360         content = []byte(fmtKeepBalanceConfig(`"KeepServiceTypes":["disk", "proxy"],`))
 361         _, err = testLoadLegacyConfig(content, f, c)
 362         c.Check(err, check.NotNil)
 363
 364         content = []byte(fmtKeepBalanceConfig(`"KeepServiceList":{},`))
 365         _, err = testLoadLegacyConfig(content, f, c)
 366         c.Check(err, check.NotNil)
 367 }
 368
 369 func fmtKeepBalanceConfig(param string) string {
 370         return fmt.Sprintf(`
 371 {
 372         "Client": {
 373                 "Scheme": "",
 374                 "APIHost": "example.com",
 375                 "AuthToken": "abcdefg",
 376                 "Insecure": false
 377         },
 378         "Listen": ":80",
 379         %s
 380         "RunPeriod": "10m",
 381         "CollectionBatchSize": 100000,
 382         "CollectionBuffers": 1000,
 383         "RequestTimeout": "30m",
 384         "ManagementToken": "xyzzy",
 385         "LostBlocksFile": "testfile"
 386 }
 387 `, param)
 388 }