lib/config/deprecated_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package config
   6
   7 import (
   8         "flag"
   9         "fmt"
  10         "io/ioutil"
  11         "os"
  12         "time"
  13
  14         "git.arvados.org/arvados.git/sdk/go/arvados"
  15         check "gopkg.in/check.v1"
  16 )
  17
  18 // Configured at: sdk/python/tests/run_test_server.py
  19 const TestServerManagementToken = "e687950a23c3a9bceec28c6223a06c79"
  20
  21 func testLoadLegacyConfig(content []byte, mungeFlag string, c *check.C) (*arvados.Cluster, error) {
  22         tmpfile, err := ioutil.TempFile("", "example")
  23         if err != nil {
  24                 return nil, err
  25         }
  26         defer os.Remove(tmpfile.Name())
  27
  28         if _, err := tmpfile.Write(content); err != nil {
  29                 return nil, err
  30         }
  31         if err := tmpfile.Close(); err != nil {
  32                 return nil, err
  33         }
  34         flags := flag.NewFlagSet("test", flag.ExitOnError)
  35         ldr := testLoader(c, "Clusters: {zzzzz: {}}", nil)
  36         ldr.SetupFlags(flags)
  37         args := ldr.MungeLegacyConfigArgs(ldr.Logger, []string{"-config", tmpfile.Name()}, mungeFlag)
  38         flags.Parse(args)
  39         cfg, err := ldr.Load()
  40         if err != nil {
  41                 return nil, err
  42         }
  43         cluster, err := cfg.GetCluster("")
  44         if err != nil {
  45                 return nil, err
  46         }
  47         return cluster, nil
  48 }
  49
  50 func (s *LoadSuite) TestLegacyVolumeDriverParameters(c *check.C) {
  51         logs := checkEquivalent(c, `
  52 Clusters:
  53  z1111:
  54   Volumes:
  55    z1111-nyw5e-aaaaaaaaaaaaaaa:
  56     Driver: S3
  57     DriverParameters:
  58      AccessKey: exampleaccesskey
  59      SecretKey: examplesecretkey
  60      Region: foobar
  61      ReadTimeout: 1200s
  62 `, `
  63 Clusters:
  64  z1111:
  65   Volumes:
  66    z1111-nyw5e-aaaaaaaaaaaaaaa:
  67     Driver: S3
  68     DriverParameters:
  69      AccessKeyID: exampleaccesskey
  70      SecretAccessKey: examplesecretkey
  71      Region: foobar
  72      ReadTimeout: 1200s
  73 `)
  74         c.Check(logs, check.Matches, `(?ms).*deprecated or unknown config entry: .*AccessKey.*`)
  75         c.Check(logs, check.Matches, `(?ms).*deprecated or unknown config entry: .*SecretKey.*`)
  76         c.Check(logs, check.Matches, `(?ms).*using your old config keys z1111\.Volumes\.z1111-nyw5e-aaaaaaaaaaaaaaa\.DriverParameters\.AccessKey/SecretKey -- but you should rename them to AccessKeyID/SecretAccessKey.*`)
  77
  78         _, err := testLoader(c, `
  79 Clusters:
  80  z1111:
  81   Volumes:
  82    z1111-nyw5e-aaaaaaaaaaaaaaa:
  83     Driver: S3
  84     DriverParameters:
  85      AccessKey: exampleaccesskey
  86      SecretKey: examplesecretkey
  87      AccessKeyID: exampleaccesskey
  88 `, nil).Load()
  89         c.Check(err, check.ErrorMatches, `(?ms).*cannot use .*SecretKey.*and.*SecretAccessKey.*in z1111.Volumes.z1111-nyw5e-aaaaaaaaaaaaaaa.DriverParameters.*`)
  90 }
  91
  92 func (s *LoadSuite) TestDeprecatedNodeProfilesToServices(c *check.C) {
  93         hostname, err := os.Hostname()
  94         c.Assert(err, check.IsNil)
  95         checkEquivalent(c, `
  96 Clusters:
  97  z1111:
  98   NodeProfiles:
  99    "*":
 100     arvados-controller:
 101      listen: ":9004"
 102    `+hostname+`:
 103     arvados-api-server:
 104      listen: ":8000"
 105    dispatch-host:
 106     arvados-dispatch-cloud:
 107      listen: ":9006"
 108 `, `
 109 Clusters:
 110  z1111:
 111   Services:
 112    RailsAPI:
 113     InternalURLs:
 114      "http://localhost:8000": {}
 115    Controller:
 116     InternalURLs:
 117      "http://localhost:9004": {}
 118    DispatchCloud:
 119     InternalURLs:
 120      "http://dispatch-host:9006": {}
 121   NodeProfiles:
 122    "*":
 123     arvados-controller:
 124      listen: ":9004"
 125    `+hostname+`:
 126     arvados-api-server:
 127      listen: ":8000"
 128    dispatch-host:
 129     arvados-dispatch-cloud:
 130      listen: ":9006"
 131 `)
 132 }
 133
 134 func (s *LoadSuite) TestDeprecatedLoginBackend(c *check.C) {
 135         checkEquivalent(c, `
 136 Clusters:
 137  z1111:
 138   Login:
 139    GoogleClientID: aaaa
 140    GoogleClientSecret: bbbb
 141    GoogleAlternateEmailAddresses: true
 142 `, `
 143 Clusters:
 144  z1111:
 145   Login:
 146    Google:
 147     Enable: true
 148     ClientID: aaaa
 149     ClientSecret: bbbb
 150     AlternateEmailAddresses: true
 151 `)
 152         checkEquivalent(c, `
 153 Clusters:
 154  z1111:
 155   Login:
 156    ProviderAppID: aaaa
 157    ProviderAppSecret: bbbb
 158 `, `
 159 Clusters:
 160  z1111:
 161   Login:
 162    SSO:
 163     Enable: true
 164     ProviderAppID: aaaa
 165     ProviderAppSecret: bbbb
 166 `)
 167 }
 168
 169 func (s *LoadSuite) TestLegacyKeepWebConfig(c *check.C) {
 170         content := []byte(`
 171 {
 172         "Client": {
 173                 "Scheme": "",
 174                 "APIHost": "example.com",
 175                 "AuthToken": "abcdefg",
 176         },
 177         "Listen": ":80",
 178         "AnonymousTokens": [
 179                 "anonusertoken"
 180         ],
 181         "AttachmentOnlyHost": "download.example.com",
 182         "TrustAllContent": true,
 183         "Cache": {
 184                 "TTL": "1m",
 185                 "UUIDTTL": "1s",
 186                 "MaxCollectionEntries": 42,
 187                 "MaxCollectionBytes": 1234567890,
 188                 "MaxPermissionEntries": 100,
 189                 "MaxUUIDEntries": 100
 190         },
 191         "ManagementToken": "xyzzy"
 192 }
 193 `)
 194         cluster, err := testLoadLegacyConfig(content, "-legacy-keepweb-config", c)
 195         c.Assert(err, check.IsNil)
 196
 197         c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
 198         c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
 199
 200         c.Check(cluster.Collections.WebDAVCache.TTL, check.Equals, arvados.Duration(60*time.Second))
 201         c.Check(cluster.Collections.WebDAVCache.UUIDTTL, check.Equals, arvados.Duration(time.Second))
 202         c.Check(cluster.Collections.WebDAVCache.MaxCollectionEntries, check.Equals, 42)
 203         c.Check(cluster.Collections.WebDAVCache.MaxCollectionBytes, check.Equals, int64(1234567890))
 204         c.Check(cluster.Collections.WebDAVCache.MaxPermissionEntries, check.Equals, 100)
 205         c.Check(cluster.Collections.WebDAVCache.MaxUUIDEntries, check.Equals, 100)
 206
 207         c.Check(cluster.Services.WebDAVDownload.ExternalURL, check.Equals, arvados.URL{Host: "download.example.com", Path: "/"})
 208         c.Check(cluster.Services.WebDAVDownload.InternalURLs[arvados.URL{Host: ":80"}], check.NotNil)
 209         c.Check(cluster.Services.WebDAV.InternalURLs[arvados.URL{Host: ":80"}], check.NotNil)
 210
 211         c.Check(cluster.Collections.TrustAllContent, check.Equals, true)
 212         c.Check(cluster.Users.AnonymousUserToken, check.Equals, "anonusertoken")
 213         c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
 214 }
 215
 216 // Tests fix for https://dev.arvados.org/issues/15642
 217 func (s *LoadSuite) TestLegacyKeepWebConfigDoesntDisableMissingItems(c *check.C) {
 218         content := []byte(`
 219 {
 220         "Client": {
 221                 "Scheme": "",
 222                 "APIHost": "example.com",
 223                 "AuthToken": "abcdefg",
 224         }
 225 }
 226 `)
 227         cluster, err := testLoadLegacyConfig(content, "-legacy-keepweb-config", c)
 228         c.Assert(err, check.IsNil)
 229         // The resulting ManagementToken should be the one set up on the test server.
 230         c.Check(cluster.ManagementToken, check.Equals, TestServerManagementToken)
 231 }
 232
 233 func (s *LoadSuite) TestLegacyKeepproxyConfig(c *check.C) {
 234         f := "-legacy-keepproxy-config"
 235         content := []byte(fmtKeepproxyConfig("", true))
 236         cluster, err := testLoadLegacyConfig(content, f, c)
 237
 238         c.Assert(err, check.IsNil)
 239         c.Assert(cluster, check.NotNil)
 240         c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
 241         c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
 242         c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
 243         c.Check(cluster.Services.Keepproxy.InternalURLs[arvados.URL{Host: ":80"}], check.Equals, arvados.ServiceInstance{})
 244         c.Check(cluster.Collections.DefaultReplication, check.Equals, 0)
 245         c.Check(cluster.API.KeepServiceRequestTimeout.String(), check.Equals, "15s")
 246         c.Check(cluster.SystemLogs.LogLevel, check.Equals, "debug")
 247
 248         content = []byte(fmtKeepproxyConfig("", false))
 249         cluster, err = testLoadLegacyConfig(content, f, c)
 250         c.Check(err, check.IsNil)
 251         c.Check(cluster.SystemLogs.LogLevel, check.Equals, "info")
 252
 253         content = []byte(fmtKeepproxyConfig(`"DisableGet": true,`, true))
 254         _, err = testLoadLegacyConfig(content, f, c)
 255         c.Check(err, check.NotNil)
 256
 257         content = []byte(fmtKeepproxyConfig(`"DisablePut": true,`, true))
 258         _, err = testLoadLegacyConfig(content, f, c)
 259         c.Check(err, check.NotNil)
 260
 261         content = []byte(fmtKeepproxyConfig(`"PIDFile": "test",`, true))
 262         _, err = testLoadLegacyConfig(content, f, c)
 263         c.Check(err, check.NotNil)
 264
 265         content = []byte(fmtKeepproxyConfig(`"DisableGet": false, "DisablePut": false, "PIDFile": "",`, true))
 266         _, err = testLoadLegacyConfig(content, f, c)
 267         c.Check(err, check.IsNil)
 268 }
 269
 270 func fmtKeepproxyConfig(param string, debugLog bool) string {
 271         return fmt.Sprintf(`
 272 {
 273         "Client": {
 274                 "Scheme": "",
 275                 "APIHost": "example.com",
 276                 "AuthToken": "abcdefg",
 277                 "Insecure": false
 278         },
 279         "Listen": ":80",
 280         "DefaultReplicas": 0,
 281         "Timeout": "15s",
 282         "Debug": %t,
 283         %s
 284         "ManagementToken": "xyzzy"
 285 }
 286 `, debugLog, param)
 287 }
 288
 289 func (s *LoadSuite) TestLegacyArvGitHttpdConfig(c *check.C) {
 290         content := []byte(`
 291 {
 292         "Client": {
 293                 "Scheme": "",
 294                 "APIHost": "example.com",
 295                 "AuthToken": "abcdefg",
 296         },
 297         "Listen": ":9000",
 298         "GitCommand": "/test/git",
 299         "GitoliteHome": "/test/gitolite",
 300         "RepoRoot": "/test/reporoot",
 301         "ManagementToken": "xyzzy"
 302 }
 303 `)
 304         f := "-legacy-git-httpd-config"
 305         cluster, err := testLoadLegacyConfig(content, f, c)
 306
 307         c.Assert(err, check.IsNil)
 308         c.Assert(cluster, check.NotNil)
 309         c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
 310         c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
 311         c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
 312         c.Check(cluster.Git.GitCommand, check.Equals, "/test/git")
 313         c.Check(cluster.Git.GitoliteHome, check.Equals, "/test/gitolite")
 314         c.Check(cluster.Git.Repositories, check.Equals, "/test/reporoot")
 315         c.Check(cluster.Services.Keepproxy.InternalURLs[arvados.URL{Host: ":9000"}], check.Equals, arvados.ServiceInstance{})
 316 }
 317
 318 // Tests fix for https://dev.arvados.org/issues/15642
 319 func (s *LoadSuite) TestLegacyArvGitHttpdConfigDoesntDisableMissingItems(c *check.C) {
 320         content := []byte(`
 321 {
 322         "Client": {
 323                 "Scheme": "",
 324                 "APIHost": "example.com",
 325                 "AuthToken": "abcdefg",
 326         }
 327 }
 328 `)
 329         cluster, err := testLoadLegacyConfig(content, "-legacy-git-httpd-config", c)
 330         c.Assert(err, check.IsNil)
 331         // The resulting ManagementToken should be the one set up on the test server.
 332         c.Check(cluster.ManagementToken, check.Equals, TestServerManagementToken)
 333 }
 334
 335 func (s *LoadSuite) TestLegacyKeepBalanceConfig(c *check.C) {
 336         f := "-legacy-keepbalance-config"
 337         content := []byte(fmtKeepBalanceConfig(""))
 338         cluster, err := testLoadLegacyConfig(content, f, c)
 339
 340         c.Assert(err, check.IsNil)
 341         c.Assert(cluster, check.NotNil)
 342         c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
 343         c.Check(cluster.Services.Keepbalance.InternalURLs[arvados.URL{Host: ":80"}], check.Equals, arvados.ServiceInstance{})
 344         c.Check(cluster.Collections.BalanceCollectionBuffers, check.Equals, 1000)
 345         c.Check(cluster.Collections.BalanceCollectionBatch, check.Equals, 100000)
 346         c.Check(cluster.Collections.BalancePeriod.String(), check.Equals, "10m")
 347         c.Check(cluster.Collections.BlobMissingReport, check.Equals, "testfile")
 348         c.Check(cluster.API.KeepServiceRequestTimeout.String(), check.Equals, "30m")
 349
 350         content = []byte(fmtKeepBalanceConfig(`"KeepServiceTypes":["disk"],`))
 351         _, err = testLoadLegacyConfig(content, f, c)
 352         c.Check(err, check.IsNil)
 353
 354         content = []byte(fmtKeepBalanceConfig(`"KeepServiceTypes":[],`))
 355         _, err = testLoadLegacyConfig(content, f, c)
 356         c.Check(err, check.IsNil)
 357
 358         content = []byte(fmtKeepBalanceConfig(`"KeepServiceTypes":["proxy"],`))
 359         _, err = testLoadLegacyConfig(content, f, c)
 360         c.Check(err, check.NotNil)
 361
 362         content = []byte(fmtKeepBalanceConfig(`"KeepServiceTypes":["disk", "proxy"],`))
 363         _, err = testLoadLegacyConfig(content, f, c)
 364         c.Check(err, check.NotNil)
 365
 366         content = []byte(fmtKeepBalanceConfig(`"KeepServiceList":{},`))
 367         _, err = testLoadLegacyConfig(content, f, c)
 368         c.Check(err, check.NotNil)
 369 }
 370
 371 func fmtKeepBalanceConfig(param string) string {
 372         return fmt.Sprintf(`
 373 {
 374         "Client": {
 375                 "Scheme": "",
 376                 "APIHost": "example.com",
 377                 "AuthToken": "abcdefg",
 378                 "Insecure": false
 379         },
 380         "Listen": ":80",
 381         %s
 382         "RunPeriod": "10m",
 383         "CollectionBatchSize": 100000,
 384         "CollectionBuffers": 1000,
 385         "RequestTimeout": "30m",
 386         "ManagementToken": "xyzzy",
 387         "LostBlocksFile": "testfile"
 388 }
 389 `, param)
 390 }