lib/config/deprecated_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package config
   6
   7 import (
   8         "flag"
   9         "fmt"
  10         "io/ioutil"
  11         "os"
  12         "time"
  13
  14         "git.arvados.org/arvados.git/sdk/go/arvados"
  15         check "gopkg.in/check.v1"
  16 )
  17
  18 // Configured at: sdk/python/tests/run_test_server.py
  19 const TestServerManagementToken = "e687950a23c3a9bceec28c6223a06c79"
  20
  21 func testLoadLegacyConfig(content []byte, mungeFlag string, c *check.C) (*arvados.Cluster, error) {
  22         tmpfile, err := ioutil.TempFile("", "example")
  23         if err != nil {
  24                 return nil, err
  25         }
  26         defer os.Remove(tmpfile.Name())
  27
  28         if _, err := tmpfile.Write(content); err != nil {
  29                 return nil, err
  30         }
  31         if err := tmpfile.Close(); err != nil {
  32                 return nil, err
  33         }
  34         flags := flag.NewFlagSet("test", flag.ExitOnError)
  35         ldr := testLoader(c, "Clusters: {zzzzz: {}}", nil)
  36         ldr.SetupFlags(flags)
  37         args := ldr.MungeLegacyConfigArgs(ldr.Logger, []string{"-config", tmpfile.Name()}, mungeFlag)
  38         flags.Parse(args)
  39         cfg, err := ldr.Load()
  40         if err != nil {
  41                 return nil, err
  42         }
  43         cluster, err := cfg.GetCluster("")
  44         if err != nil {
  45                 return nil, err
  46         }
  47         return cluster, nil
  48 }
  49
  50 func (s *LoadSuite) TestLegacyVolumeDriverParameters(c *check.C) {
  51         logs := checkEquivalent(c, `
  52 Clusters:
  53  z1111:
  54   Volumes:
  55    z1111-nyw5e-aaaaaaaaaaaaaaa:
  56     Driver: S3
  57     DriverParameters:
  58      AccessKey: exampleaccesskey
  59      SecretKey: examplesecretkey
  60      Region: foobar
  61      ReadTimeout: 1200s
  62 `, `
  63 Clusters:
  64  z1111:
  65   Volumes:
  66    z1111-nyw5e-aaaaaaaaaaaaaaa:
  67     Driver: S3
  68     DriverParameters:
  69      AccessKeyID: exampleaccesskey
  70      SecretAccessKey: examplesecretkey
  71      Region: foobar
  72      ReadTimeout: 1200s
  73 `)
  74         c.Check(logs, check.Matches, `(?ms).*deprecated or unknown config entry: .*AccessKey.*`)
  75         c.Check(logs, check.Matches, `(?ms).*deprecated or unknown config entry: .*SecretKey.*`)
  76         c.Check(logs, check.Matches, `(?ms).*using your old config keys z1111\.Volumes\.z1111-nyw5e-aaaaaaaaaaaaaaa\.DriverParameters\.AccessKey/SecretKey -- but you should rename them to AccessKeyID/SecretAccessKey.*`)
  77 }
  78
  79 func (s *LoadSuite) TestDeprecatedNodeProfilesToServices(c *check.C) {
  80         hostname, err := os.Hostname()
  81         c.Assert(err, check.IsNil)
  82         checkEquivalent(c, `
  83 Clusters:
  84  z1111:
  85   NodeProfiles:
  86    "*":
  87     arvados-controller:
  88      listen: ":9004"
  89    `+hostname+`:
  90     arvados-api-server:
  91      listen: ":8000"
  92    dispatch-host:
  93     arvados-dispatch-cloud:
  94      listen: ":9006"
  95 `, `
  96 Clusters:
  97  z1111:
  98   Services:
  99    RailsAPI:
 100     InternalURLs:
 101      "http://localhost:8000": {}
 102    Controller:
 103     InternalURLs:
 104      "http://localhost:9004": {}
 105    DispatchCloud:
 106     InternalURLs:
 107      "http://dispatch-host:9006": {}
 108   NodeProfiles:
 109    "*":
 110     arvados-controller:
 111      listen: ":9004"
 112    `+hostname+`:
 113     arvados-api-server:
 114      listen: ":8000"
 115    dispatch-host:
 116     arvados-dispatch-cloud:
 117      listen: ":9006"
 118 `)
 119 }
 120
 121 func (s *LoadSuite) TestDeprecatedLoginBackend(c *check.C) {
 122         checkEquivalent(c, `
 123 Clusters:
 124  z1111:
 125   Login:
 126    GoogleClientID: aaaa
 127    GoogleClientSecret: bbbb
 128    GoogleAlternateEmailAddresses: true
 129 `, `
 130 Clusters:
 131  z1111:
 132   Login:
 133    Google:
 134     Enable: true
 135     ClientID: aaaa
 136     ClientSecret: bbbb
 137     AlternateEmailAddresses: true
 138 `)
 139         checkEquivalent(c, `
 140 Clusters:
 141  z1111:
 142   Login:
 143    ProviderAppID: aaaa
 144    ProviderAppSecret: bbbb
 145 `, `
 146 Clusters:
 147  z1111:
 148   Login:
 149    SSO:
 150     Enable: true
 151     ProviderAppID: aaaa
 152     ProviderAppSecret: bbbb
 153 `)
 154 }
 155
 156 func (s *LoadSuite) TestLegacyKeepWebConfig(c *check.C) {
 157         content := []byte(`
 158 {
 159         "Client": {
 160                 "Scheme": "",
 161                 "APIHost": "example.com",
 162                 "AuthToken": "abcdefg",
 163         },
 164         "Listen": ":80",
 165         "AnonymousTokens": [
 166                 "anonusertoken"
 167         ],
 168         "AttachmentOnlyHost": "download.example.com",
 169         "TrustAllContent": true,
 170         "Cache": {
 171                 "TTL": "1m",
 172                 "UUIDTTL": "1s",
 173                 "MaxCollectionEntries": 42,
 174                 "MaxCollectionBytes": 1234567890,
 175                 "MaxPermissionEntries": 100,
 176                 "MaxUUIDEntries": 100
 177         },
 178         "ManagementToken": "xyzzy"
 179 }
 180 `)
 181         cluster, err := testLoadLegacyConfig(content, "-legacy-keepweb-config", c)
 182         c.Assert(err, check.IsNil)
 183
 184         c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
 185         c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
 186
 187         c.Check(cluster.Collections.WebDAVCache.TTL, check.Equals, arvados.Duration(60*time.Second))
 188         c.Check(cluster.Collections.WebDAVCache.UUIDTTL, check.Equals, arvados.Duration(time.Second))
 189         c.Check(cluster.Collections.WebDAVCache.MaxCollectionEntries, check.Equals, 42)
 190         c.Check(cluster.Collections.WebDAVCache.MaxCollectionBytes, check.Equals, int64(1234567890))
 191         c.Check(cluster.Collections.WebDAVCache.MaxPermissionEntries, check.Equals, 100)
 192         c.Check(cluster.Collections.WebDAVCache.MaxUUIDEntries, check.Equals, 100)
 193
 194         c.Check(cluster.Services.WebDAVDownload.ExternalURL, check.Equals, arvados.URL{Host: "download.example.com", Path: "/"})
 195         c.Check(cluster.Services.WebDAVDownload.InternalURLs[arvados.URL{Host: ":80"}], check.NotNil)
 196         c.Check(cluster.Services.WebDAV.InternalURLs[arvados.URL{Host: ":80"}], check.NotNil)
 197
 198         c.Check(cluster.Collections.TrustAllContent, check.Equals, true)
 199         c.Check(cluster.Users.AnonymousUserToken, check.Equals, "anonusertoken")
 200         c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
 201 }
 202
 203 // Tests fix for https://dev.arvados.org/issues/15642
 204 func (s *LoadSuite) TestLegacyKeepWebConfigDoesntDisableMissingItems(c *check.C) {
 205         content := []byte(`
 206 {
 207         "Client": {
 208                 "Scheme": "",
 209                 "APIHost": "example.com",
 210                 "AuthToken": "abcdefg",
 211         }
 212 }
 213 `)
 214         cluster, err := testLoadLegacyConfig(content, "-legacy-keepweb-config", c)
 215         c.Assert(err, check.IsNil)
 216         // The resulting ManagementToken should be the one set up on the test server.
 217         c.Check(cluster.ManagementToken, check.Equals, TestServerManagementToken)
 218 }
 219
 220 func (s *LoadSuite) TestLegacyKeepproxyConfig(c *check.C) {
 221         f := "-legacy-keepproxy-config"
 222         content := []byte(fmtKeepproxyConfig("", true))
 223         cluster, err := testLoadLegacyConfig(content, f, c)
 224
 225         c.Assert(err, check.IsNil)
 226         c.Assert(cluster, check.NotNil)
 227         c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
 228         c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
 229         c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
 230         c.Check(cluster.Services.Keepproxy.InternalURLs[arvados.URL{Host: ":80"}], check.Equals, arvados.ServiceInstance{})
 231         c.Check(cluster.Collections.DefaultReplication, check.Equals, 0)
 232         c.Check(cluster.API.KeepServiceRequestTimeout.String(), check.Equals, "15s")
 233         c.Check(cluster.SystemLogs.LogLevel, check.Equals, "debug")
 234
 235         content = []byte(fmtKeepproxyConfig("", false))
 236         cluster, err = testLoadLegacyConfig(content, f, c)
 237         c.Check(err, check.IsNil)
 238         c.Check(cluster.SystemLogs.LogLevel, check.Equals, "info")
 239
 240         content = []byte(fmtKeepproxyConfig(`"DisableGet": true,`, true))
 241         _, err = testLoadLegacyConfig(content, f, c)
 242         c.Check(err, check.NotNil)
 243
 244         content = []byte(fmtKeepproxyConfig(`"DisablePut": true,`, true))
 245         _, err = testLoadLegacyConfig(content, f, c)
 246         c.Check(err, check.NotNil)
 247
 248         content = []byte(fmtKeepproxyConfig(`"PIDFile": "test",`, true))
 249         _, err = testLoadLegacyConfig(content, f, c)
 250         c.Check(err, check.NotNil)
 251
 252         content = []byte(fmtKeepproxyConfig(`"DisableGet": false, "DisablePut": false, "PIDFile": "",`, true))
 253         _, err = testLoadLegacyConfig(content, f, c)
 254         c.Check(err, check.IsNil)
 255 }
 256
 257 func fmtKeepproxyConfig(param string, debugLog bool) string {
 258         return fmt.Sprintf(`
 259 {
 260         "Client": {
 261                 "Scheme": "",
 262                 "APIHost": "example.com",
 263                 "AuthToken": "abcdefg",
 264                 "Insecure": false
 265         },
 266         "Listen": ":80",
 267         "DefaultReplicas": 0,
 268         "Timeout": "15s",
 269         "Debug": %t,
 270         %s
 271         "ManagementToken": "xyzzy"
 272 }
 273 `, debugLog, param)
 274 }
 275
 276 func (s *LoadSuite) TestLegacyArvGitHttpdConfig(c *check.C) {
 277         content := []byte(`
 278 {
 279         "Client": {
 280                 "Scheme": "",
 281                 "APIHost": "example.com",
 282                 "AuthToken": "abcdefg",
 283         },
 284         "Listen": ":9000",
 285         "GitCommand": "/test/git",
 286         "GitoliteHome": "/test/gitolite",
 287         "RepoRoot": "/test/reporoot",
 288         "ManagementToken": "xyzzy"
 289 }
 290 `)
 291         f := "-legacy-git-httpd-config"
 292         cluster, err := testLoadLegacyConfig(content, f, c)
 293
 294         c.Assert(err, check.IsNil)
 295         c.Assert(cluster, check.NotNil)
 296         c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
 297         c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
 298         c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
 299         c.Check(cluster.Git.GitCommand, check.Equals, "/test/git")
 300         c.Check(cluster.Git.GitoliteHome, check.Equals, "/test/gitolite")
 301         c.Check(cluster.Git.Repositories, check.Equals, "/test/reporoot")
 302         c.Check(cluster.Services.Keepproxy.InternalURLs[arvados.URL{Host: ":9000"}], check.Equals, arvados.ServiceInstance{})
 303 }
 304
 305 // Tests fix for https://dev.arvados.org/issues/15642
 306 func (s *LoadSuite) TestLegacyArvGitHttpdConfigDoesntDisableMissingItems(c *check.C) {
 307         content := []byte(`
 308 {
 309         "Client": {
 310                 "Scheme": "",
 311                 "APIHost": "example.com",
 312                 "AuthToken": "abcdefg",
 313         }
 314 }
 315 `)
 316         cluster, err := testLoadLegacyConfig(content, "-legacy-git-httpd-config", c)
 317         c.Assert(err, check.IsNil)
 318         // The resulting ManagementToken should be the one set up on the test server.
 319         c.Check(cluster.ManagementToken, check.Equals, TestServerManagementToken)
 320 }
 321
 322 func (s *LoadSuite) TestLegacyKeepBalanceConfig(c *check.C) {
 323         f := "-legacy-keepbalance-config"
 324         content := []byte(fmtKeepBalanceConfig(""))
 325         cluster, err := testLoadLegacyConfig(content, f, c)
 326
 327         c.Assert(err, check.IsNil)
 328         c.Assert(cluster, check.NotNil)
 329         c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
 330         c.Check(cluster.Services.Keepbalance.InternalURLs[arvados.URL{Host: ":80"}], check.Equals, arvados.ServiceInstance{})
 331         c.Check(cluster.Collections.BalanceCollectionBuffers, check.Equals, 1000)
 332         c.Check(cluster.Collections.BalanceCollectionBatch, check.Equals, 100000)
 333         c.Check(cluster.Collections.BalancePeriod.String(), check.Equals, "10m")
 334         c.Check(cluster.Collections.BlobMissingReport, check.Equals, "testfile")
 335         c.Check(cluster.API.KeepServiceRequestTimeout.String(), check.Equals, "30m")
 336
 337         content = []byte(fmtKeepBalanceConfig(`"KeepServiceTypes":["disk"],`))
 338         _, err = testLoadLegacyConfig(content, f, c)
 339         c.Check(err, check.IsNil)
 340
 341         content = []byte(fmtKeepBalanceConfig(`"KeepServiceTypes":[],`))
 342         _, err = testLoadLegacyConfig(content, f, c)
 343         c.Check(err, check.IsNil)
 344
 345         content = []byte(fmtKeepBalanceConfig(`"KeepServiceTypes":["proxy"],`))
 346         _, err = testLoadLegacyConfig(content, f, c)
 347         c.Check(err, check.NotNil)
 348
 349         content = []byte(fmtKeepBalanceConfig(`"KeepServiceTypes":["disk", "proxy"],`))
 350         _, err = testLoadLegacyConfig(content, f, c)
 351         c.Check(err, check.NotNil)
 352
 353         content = []byte(fmtKeepBalanceConfig(`"KeepServiceList":{},`))
 354         _, err = testLoadLegacyConfig(content, f, c)
 355         c.Check(err, check.NotNil)
 356 }
 357
 358 func fmtKeepBalanceConfig(param string) string {
 359         return fmt.Sprintf(`
 360 {
 361         "Client": {
 362                 "Scheme": "",
 363                 "APIHost": "example.com",
 364                 "AuthToken": "abcdefg",
 365                 "Insecure": false
 366         },
 367         "Listen": ":80",
 368         %s
 369         "RunPeriod": "10m",
 370         "CollectionBatchSize": 100000,
 371         "CollectionBuffers": 1000,
 372         "RequestTimeout": "30m",
 373         "ManagementToken": "xyzzy",
 374         "LostBlocksFile": "testfile"
 375 }
 376 `, param)
 377 }