lib/config/deprecated_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package config
   6
   7 import (
   8         "flag"
   9         "fmt"
  10         "io/ioutil"
  11         "os"
  12         "time"
  13
  14         "git.arvados.org/arvados.git/sdk/go/arvados"
  15         check "gopkg.in/check.v1"
  16 )
  17
  18 // Configured at: sdk/python/tests/run_test_server.py
  19 const TestServerManagementToken = "e687950a23c3a9bceec28c6223a06c79"
  20
  21 func testLoadLegacyConfig(content []byte, mungeFlag string, c *check.C) (*arvados.Cluster, error) {
  22         tmpfile, err := ioutil.TempFile("", "example")
  23         if err != nil {
  24                 return nil, err
  25         }
  26         defer os.Remove(tmpfile.Name())
  27
  28         if _, err := tmpfile.Write(content); err != nil {
  29                 return nil, err
  30         }
  31         if err := tmpfile.Close(); err != nil {
  32                 return nil, err
  33         }
  34         flags := flag.NewFlagSet("test", flag.ExitOnError)
  35         ldr := testLoader(c, "Clusters: {zzzzz: {}}", nil)
  36         ldr.SetupFlags(flags)
  37         args := ldr.MungeLegacyConfigArgs(ldr.Logger, []string{"-config", tmpfile.Name()}, mungeFlag)
  38         err = flags.Parse(args)
  39         c.Assert(err, check.IsNil)
  40         c.Assert(flags.NArg(), check.Equals, 0)
  41         cfg, err := ldr.Load()
  42         if err != nil {
  43                 return nil, err
  44         }
  45         cluster, err := cfg.GetCluster("")
  46         if err != nil {
  47                 return nil, err
  48         }
  49         return cluster, nil
  50 }
  51
  52 func (s *LoadSuite) TestLegacyVolumeDriverParameters(c *check.C) {
  53         logs := checkEquivalent(c, `
  54 Clusters:
  55  z1111:
  56   Volumes:
  57    z1111-nyw5e-aaaaaaaaaaaaaaa:
  58     Driver: S3
  59     DriverParameters:
  60      AccessKey: exampleaccesskey
  61      SecretKey: examplesecretkey
  62      Region: foobar
  63      ReadTimeout: 1200s
  64 `, `
  65 Clusters:
  66  z1111:
  67   Volumes:
  68    z1111-nyw5e-aaaaaaaaaaaaaaa:
  69     Driver: S3
  70     DriverParameters:
  71      AccessKeyID: exampleaccesskey
  72      SecretAccessKey: examplesecretkey
  73      Region: foobar
  74      ReadTimeout: 1200s
  75 `)
  76         c.Check(logs, check.Matches, `(?ms).*deprecated or unknown config entry: .*AccessKey.*`)
  77         c.Check(logs, check.Matches, `(?ms).*deprecated or unknown config entry: .*SecretKey.*`)
  78         c.Check(logs, check.Matches, `(?ms).*using your old config keys z1111\.Volumes\.z1111-nyw5e-aaaaaaaaaaaaaaa\.DriverParameters\.AccessKey/SecretKey -- but you should rename them to AccessKeyID/SecretAccessKey.*`)
  79
  80         _, err := testLoader(c, `
  81 Clusters:
  82  z1111:
  83   Volumes:
  84    z1111-nyw5e-aaaaaaaaaaaaaaa:
  85     Driver: S3
  86     DriverParameters:
  87      AccessKey: exampleaccesskey
  88      SecretKey: examplesecretkey
  89      AccessKeyID: exampleaccesskey
  90 `, nil).Load()
  91         c.Check(err, check.ErrorMatches, `(?ms).*cannot use .*SecretKey.*and.*SecretAccessKey.*in z1111.Volumes.z1111-nyw5e-aaaaaaaaaaaaaaa.DriverParameters.*`)
  92 }
  93
  94 func (s *LoadSuite) TestDeprecatedNodeProfilesToServices(c *check.C) {
  95         hostname, err := os.Hostname()
  96         c.Assert(err, check.IsNil)
  97         checkEquivalent(c, `
  98 Clusters:
  99  z1111:
 100   NodeProfiles:
 101    "*":
 102     arvados-controller:
 103      listen: ":9004"
 104    `+hostname+`:
 105     arvados-api-server:
 106      listen: ":8000"
 107    dispatch-host:
 108     arvados-dispatch-cloud:
 109      listen: ":9006"
 110 `, `
 111 Clusters:
 112  z1111:
 113   Services:
 114    RailsAPI:
 115     InternalURLs:
 116      "http://localhost:8000": {}
 117    Controller:
 118     InternalURLs:
 119      "http://localhost:9004": {}
 120    DispatchCloud:
 121     InternalURLs:
 122      "http://dispatch-host:9006": {}
 123   NodeProfiles:
 124    "*":
 125     arvados-controller:
 126      listen: ":9004"
 127    `+hostname+`:
 128     arvados-api-server:
 129      listen: ":8000"
 130    dispatch-host:
 131     arvados-dispatch-cloud:
 132      listen: ":9006"
 133 `)
 134 }
 135
 136 func (s *LoadSuite) TestDeprecatedLoginBackend(c *check.C) {
 137         checkEquivalent(c, `
 138 Clusters:
 139  z1111:
 140   Login:
 141    GoogleClientID: aaaa
 142    GoogleClientSecret: bbbb
 143    GoogleAlternateEmailAddresses: true
 144 `, `
 145 Clusters:
 146  z1111:
 147   Login:
 148    Google:
 149     Enable: true
 150     ClientID: aaaa
 151     ClientSecret: bbbb
 152     AlternateEmailAddresses: true
 153 `)
 154         checkEquivalent(c, `
 155 Clusters:
 156  z1111:
 157   Login:
 158    ProviderAppID: aaaa
 159    ProviderAppSecret: bbbb
 160 `, `
 161 Clusters:
 162  z1111:
 163   Login:
 164    SSO:
 165     Enable: true
 166     ProviderAppID: aaaa
 167     ProviderAppSecret: bbbb
 168 `)
 169 }
 170
 171 func (s *LoadSuite) TestLegacyKeepWebConfig(c *check.C) {
 172         content := []byte(`
 173 {
 174         "Client": {
 175                 "Scheme": "",
 176                 "APIHost": "example.com",
 177                 "AuthToken": "abcdefg",
 178         },
 179         "Listen": ":80",
 180         "AnonymousTokens": [
 181                 "anonusertoken"
 182         ],
 183         "AttachmentOnlyHost": "download.example.com",
 184         "TrustAllContent": true,
 185         "Cache": {
 186                 "TTL": "1m",
 187                 "UUIDTTL": "1s",
 188                 "MaxCollectionEntries": 42,
 189                 "MaxCollectionBytes": 1234567890,
 190                 "MaxUUIDEntries": 100
 191         },
 192         "ManagementToken": "xyzzy"
 193 }
 194 `)
 195         cluster, err := testLoadLegacyConfig(content, "-legacy-keepweb-config", c)
 196         c.Assert(err, check.IsNil)
 197
 198         c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
 199         c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
 200
 201         c.Check(cluster.Collections.WebDAVCache.TTL, check.Equals, arvados.Duration(60*time.Second))
 202         c.Check(cluster.Collections.WebDAVCache.MaxCollectionBytes, check.Equals, arvados.ByteSize(1234567890))
 203
 204         c.Check(cluster.Services.WebDAVDownload.ExternalURL, check.Equals, arvados.URL{Host: "download.example.com", Path: "/"})
 205         c.Check(cluster.Services.WebDAVDownload.InternalURLs[arvados.URL{Host: ":80"}], check.NotNil)
 206         c.Check(cluster.Services.WebDAV.InternalURLs[arvados.URL{Host: ":80"}], check.NotNil)
 207
 208         c.Check(cluster.Collections.TrustAllContent, check.Equals, true)
 209         c.Check(cluster.Users.AnonymousUserToken, check.Equals, "anonusertoken")
 210         c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
 211 }
 212
 213 // Tests fix for https://dev.arvados.org/issues/15642
 214 func (s *LoadSuite) TestLegacyKeepWebConfigDoesntDisableMissingItems(c *check.C) {
 215         content := []byte(`
 216 {
 217         "Client": {
 218                 "Scheme": "",
 219                 "APIHost": "example.com",
 220                 "AuthToken": "abcdefg",
 221         }
 222 }
 223 `)
 224         cluster, err := testLoadLegacyConfig(content, "-legacy-keepweb-config", c)
 225         c.Assert(err, check.IsNil)
 226         // The resulting ManagementToken should be the one set up on the test server.
 227         c.Check(cluster.ManagementToken, check.Equals, TestServerManagementToken)
 228 }
 229
 230 func (s *LoadSuite) TestLegacyKeepproxyConfig(c *check.C) {
 231         f := "-legacy-keepproxy-config"
 232         content := []byte(fmtKeepproxyConfig("", true))
 233         cluster, err := testLoadLegacyConfig(content, f, c)
 234
 235         c.Assert(err, check.IsNil)
 236         c.Assert(cluster, check.NotNil)
 237         c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
 238         c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
 239         c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
 240         c.Check(cluster.Services.Keepproxy.InternalURLs[arvados.URL{Host: ":80"}], check.Equals, arvados.ServiceInstance{})
 241         c.Check(cluster.Collections.DefaultReplication, check.Equals, 0)
 242         c.Check(cluster.API.KeepServiceRequestTimeout.String(), check.Equals, "15s")
 243         c.Check(cluster.SystemLogs.LogLevel, check.Equals, "debug")
 244
 245         content = []byte(fmtKeepproxyConfig("", false))
 246         cluster, err = testLoadLegacyConfig(content, f, c)
 247         c.Check(err, check.IsNil)
 248         c.Check(cluster.SystemLogs.LogLevel, check.Equals, "info")
 249
 250         content = []byte(fmtKeepproxyConfig(`"DisableGet": true,`, true))
 251         _, err = testLoadLegacyConfig(content, f, c)
 252         c.Check(err, check.NotNil)
 253
 254         content = []byte(fmtKeepproxyConfig(`"DisablePut": true,`, true))
 255         _, err = testLoadLegacyConfig(content, f, c)
 256         c.Check(err, check.NotNil)
 257
 258         content = []byte(fmtKeepproxyConfig(`"PIDFile": "test",`, true))
 259         _, err = testLoadLegacyConfig(content, f, c)
 260         c.Check(err, check.NotNil)
 261
 262         content = []byte(fmtKeepproxyConfig(`"DisableGet": false, "DisablePut": false, "PIDFile": "",`, true))
 263         _, err = testLoadLegacyConfig(content, f, c)
 264         c.Check(err, check.IsNil)
 265 }
 266
 267 func fmtKeepproxyConfig(param string, debugLog bool) string {
 268         return fmt.Sprintf(`
 269 {
 270         "Client": {
 271                 "Scheme": "",
 272                 "APIHost": "example.com",
 273                 "AuthToken": "abcdefg",
 274                 "Insecure": false
 275         },
 276         "Listen": ":80",
 277         "DefaultReplicas": 0,
 278         "Timeout": "15s",
 279         "Debug": %t,
 280         %s
 281         "ManagementToken": "xyzzy"
 282 }
 283 `, debugLog, param)
 284 }
 285
 286 func (s *LoadSuite) TestLegacyArvGitHttpdConfig(c *check.C) {
 287         content := []byte(`
 288 {
 289         "Client": {
 290                 "Scheme": "",
 291                 "APIHost": "example.com",
 292                 "AuthToken": "abcdefg",
 293         },
 294         "Listen": ":9000",
 295         "GitCommand": "/test/git",
 296         "GitoliteHome": "/test/gitolite",
 297         "RepoRoot": "/test/reporoot",
 298         "ManagementToken": "xyzzy"
 299 }
 300 `)
 301         f := "-legacy-git-httpd-config"
 302         cluster, err := testLoadLegacyConfig(content, f, c)
 303
 304         c.Assert(err, check.IsNil)
 305         c.Assert(cluster, check.NotNil)
 306         c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
 307         c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
 308         c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
 309         c.Check(cluster.Git.GitCommand, check.Equals, "/test/git")
 310         c.Check(cluster.Git.GitoliteHome, check.Equals, "/test/gitolite")
 311         c.Check(cluster.Git.Repositories, check.Equals, "/test/reporoot")
 312         c.Check(cluster.Services.Keepproxy.InternalURLs[arvados.URL{Host: ":9000"}], check.Equals, arvados.ServiceInstance{})
 313 }
 314
 315 // Tests fix for https://dev.arvados.org/issues/15642
 316 func (s *LoadSuite) TestLegacyArvGitHttpdConfigDoesntDisableMissingItems(c *check.C) {
 317         content := []byte(`
 318 {
 319         "Client": {
 320                 "Scheme": "",
 321                 "APIHost": "example.com",
 322                 "AuthToken": "abcdefg",
 323         }
 324 }
 325 `)
 326         cluster, err := testLoadLegacyConfig(content, "-legacy-git-httpd-config", c)
 327         c.Assert(err, check.IsNil)
 328         // The resulting ManagementToken should be the one set up on the test server.
 329         c.Check(cluster.ManagementToken, check.Equals, TestServerManagementToken)
 330 }
 331
 332 func (s *LoadSuite) TestLegacyKeepBalanceConfig(c *check.C) {
 333         f := "-legacy-keepbalance-config"
 334         content := []byte(fmtKeepBalanceConfig(""))
 335         cluster, err := testLoadLegacyConfig(content, f, c)
 336
 337         c.Assert(err, check.IsNil)
 338         c.Assert(cluster, check.NotNil)
 339         c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
 340         c.Check(cluster.Services.Keepbalance.InternalURLs[arvados.URL{Host: ":80"}], check.Equals, arvados.ServiceInstance{})
 341         c.Check(cluster.Collections.BalanceCollectionBuffers, check.Equals, 1000)
 342         c.Check(cluster.Collections.BalanceCollectionBatch, check.Equals, 100000)
 343         c.Check(cluster.Collections.BalancePeriod.String(), check.Equals, "10m")
 344         c.Check(cluster.Collections.BlobMissingReport, check.Equals, "testfile")
 345         c.Check(cluster.API.KeepServiceRequestTimeout.String(), check.Equals, "30m")
 346
 347         content = []byte(fmtKeepBalanceConfig(`"KeepServiceTypes":["disk"],`))
 348         _, err = testLoadLegacyConfig(content, f, c)
 349         c.Check(err, check.IsNil)
 350
 351         content = []byte(fmtKeepBalanceConfig(`"KeepServiceTypes":[],`))
 352         _, err = testLoadLegacyConfig(content, f, c)
 353         c.Check(err, check.IsNil)
 354
 355         content = []byte(fmtKeepBalanceConfig(`"KeepServiceTypes":["proxy"],`))
 356         _, err = testLoadLegacyConfig(content, f, c)
 357         c.Check(err, check.NotNil)
 358
 359         content = []byte(fmtKeepBalanceConfig(`"KeepServiceTypes":["disk", "proxy"],`))
 360         _, err = testLoadLegacyConfig(content, f, c)
 361         c.Check(err, check.NotNil)
 362
 363         content = []byte(fmtKeepBalanceConfig(`"KeepServiceList":{},`))
 364         _, err = testLoadLegacyConfig(content, f, c)
 365         c.Check(err, check.NotNil)
 366 }
 367
 368 func fmtKeepBalanceConfig(param string) string {
 369         return fmt.Sprintf(`
 370 {
 371         "Client": {
 372                 "Scheme": "",
 373                 "APIHost": "example.com",
 374                 "AuthToken": "abcdefg",
 375                 "Insecure": false
 376         },
 377         "Listen": ":80",
 378         %s
 379         "RunPeriod": "10m",
 380         "CollectionBatchSize": 100000,
 381         "CollectionBuffers": 1000,
 382         "RequestTimeout": "30m",
 383         "ManagementToken": "xyzzy",
 384         "LostBlocksFile": "testfile"
 385 }
 386 `, param)
 387 }