lib/config/deprecated_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package config
   6
   7 import (
   8         "flag"
   9         "fmt"
  10         "io/ioutil"
  11         "os"
  12         "time"
  13
  14         "git.arvados.org/arvados.git/sdk/go/arvados"
  15         check "gopkg.in/check.v1"
  16 )
  17
  18 // Configured at: sdk/python/tests/run_test_server.py
  19 const TestServerManagementToken = "e687950a23c3a9bceec28c6223a06c79"
  20
  21 func testLoadLegacyConfig(content []byte, mungeFlag string, c *check.C) (*arvados.Cluster, error) {
  22         tmpfile, err := ioutil.TempFile("", "example")
  23         if err != nil {
  24                 return nil, err
  25         }
  26         defer os.Remove(tmpfile.Name())
  27
  28         if _, err := tmpfile.Write(content); err != nil {
  29                 return nil, err
  30         }
  31         if err := tmpfile.Close(); err != nil {
  32                 return nil, err
  33         }
  34         flags := flag.NewFlagSet("test", flag.ExitOnError)
  35         ldr := testLoader(c, "Clusters: {zzzzz: {}}", nil)
  36         ldr.SetupFlags(flags)
  37         args := ldr.MungeLegacyConfigArgs(ldr.Logger, []string{"-config", tmpfile.Name()}, mungeFlag)
  38         err = flags.Parse(args)
  39         c.Assert(err, check.IsNil)
  40         c.Assert(flags.NArg(), check.Equals, 0)
  41         cfg, err := ldr.Load()
  42         if err != nil {
  43                 return nil, err
  44         }
  45         cluster, err := cfg.GetCluster("")
  46         if err != nil {
  47                 return nil, err
  48         }
  49         return cluster, nil
  50 }
  51
  52 func (s *LoadSuite) TestLegacyVolumeDriverParameters(c *check.C) {
  53         logs := checkEquivalent(c, `
  54 Clusters:
  55  z1111:
  56   Volumes:
  57    z1111-nyw5e-aaaaaaaaaaaaaaa:
  58     Driver: S3
  59     DriverParameters:
  60      AccessKey: exampleaccesskey
  61      SecretKey: examplesecretkey
  62      Region: foobar
  63      ReadTimeout: 1200s
  64 `, `
  65 Clusters:
  66  z1111:
  67   Volumes:
  68    z1111-nyw5e-aaaaaaaaaaaaaaa:
  69     Driver: S3
  70     DriverParameters:
  71      AccessKeyID: exampleaccesskey
  72      SecretAccessKey: examplesecretkey
  73      Region: foobar
  74      ReadTimeout: 1200s
  75 `)
  76         c.Check(logs, check.Matches, `(?ms).*deprecated or unknown config entry: .*AccessKey.*`)
  77         c.Check(logs, check.Matches, `(?ms).*deprecated or unknown config entry: .*SecretKey.*`)
  78         c.Check(logs, check.Matches, `(?ms).*using your old config keys z1111\.Volumes\.z1111-nyw5e-aaaaaaaaaaaaaaa\.DriverParameters\.AccessKey/SecretKey -- but you should rename them to AccessKeyID/SecretAccessKey.*`)
  79
  80         _, err := testLoader(c, `
  81 Clusters:
  82  z1111:
  83   Volumes:
  84    z1111-nyw5e-aaaaaaaaaaaaaaa:
  85     Driver: S3
  86     DriverParameters:
  87      AccessKey: exampleaccesskey
  88      SecretKey: examplesecretkey
  89      AccessKeyID: exampleaccesskey
  90 `, nil).Load()
  91         c.Check(err, check.ErrorMatches, `(?ms).*cannot use .*SecretKey.*and.*SecretAccessKey.*in z1111.Volumes.z1111-nyw5e-aaaaaaaaaaaaaaa.DriverParameters.*`)
  92 }
  93
  94 func (s *LoadSuite) TestDeprecatedNodeProfilesToServices(c *check.C) {
  95         hostname, err := os.Hostname()
  96         c.Assert(err, check.IsNil)
  97         checkEquivalent(c, `
  98 Clusters:
  99  z1111:
 100   NodeProfiles:
 101    "*":
 102     arvados-controller:
 103      listen: ":9004"
 104    `+hostname+`:
 105     arvados-api-server:
 106      listen: ":8000"
 107    dispatch-host:
 108     arvados-dispatch-cloud:
 109      listen: ":9006"
 110 `, `
 111 Clusters:
 112  z1111:
 113   Services:
 114    RailsAPI:
 115     InternalURLs:
 116      "http://localhost:8000": {}
 117    Controller:
 118     InternalURLs:
 119      "http://localhost:9004": {}
 120    DispatchCloud:
 121     InternalURLs:
 122      "http://dispatch-host:9006": {}
 123   NodeProfiles:
 124    "*":
 125     arvados-controller:
 126      listen: ":9004"
 127    `+hostname+`:
 128     arvados-api-server:
 129      listen: ":8000"
 130    dispatch-host:
 131     arvados-dispatch-cloud:
 132      listen: ":9006"
 133 `)
 134 }
 135
 136 func (s *LoadSuite) TestDeprecatedLoginBackend(c *check.C) {
 137         checkEquivalent(c, `
 138 Clusters:
 139  z1111:
 140   Login:
 141    GoogleClientID: aaaa
 142    GoogleClientSecret: bbbb
 143    GoogleAlternateEmailAddresses: true
 144 `, `
 145 Clusters:
 146  z1111:
 147   Login:
 148    Google:
 149     Enable: true
 150     ClientID: aaaa
 151     ClientSecret: bbbb
 152     AlternateEmailAddresses: true
 153 `)
 154         checkEquivalent(c, `
 155 Clusters:
 156  z1111:
 157   Login:
 158    ProviderAppID: aaaa
 159    ProviderAppSecret: bbbb
 160 `, `
 161 Clusters:
 162  z1111:
 163   Login:
 164    SSO:
 165     Enable: true
 166     ProviderAppID: aaaa
 167     ProviderAppSecret: bbbb
 168 `)
 169 }
 170
 171 func (s *LoadSuite) TestLegacyKeepWebConfig(c *check.C) {
 172         content := []byte(`
 173 {
 174         "Client": {
 175                 "Scheme": "",
 176                 "APIHost": "example.com",
 177                 "AuthToken": "abcdefg",
 178         },
 179         "Listen": ":80",
 180         "AnonymousTokens": [
 181                 "anonusertoken"
 182         ],
 183         "AttachmentOnlyHost": "download.example.com",
 184         "TrustAllContent": true,
 185         "Cache": {
 186                 "TTL": "1m",
 187                 "UUIDTTL": "1s",
 188                 "MaxCollectionEntries": 42,
 189                 "MaxCollectionBytes": 1234567890,
 190                 "MaxUUIDEntries": 100
 191         },
 192         "ManagementToken": "xyzzy"
 193 }
 194 `)
 195         cluster, err := testLoadLegacyConfig(content, "-legacy-keepweb-config", c)
 196         c.Assert(err, check.IsNil)
 197
 198         c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
 199         c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
 200
 201         c.Check(cluster.Collections.WebDAVCache.TTL, check.Equals, arvados.Duration(60*time.Second))
 202         c.Check(cluster.Collections.WebDAVCache.UUIDTTL, check.Equals, arvados.Duration(time.Second))
 203         c.Check(cluster.Collections.WebDAVCache.MaxCollectionEntries, check.Equals, 42)
 204         c.Check(cluster.Collections.WebDAVCache.MaxCollectionBytes, check.Equals, int64(1234567890))
 205         c.Check(cluster.Collections.WebDAVCache.MaxUUIDEntries, check.Equals, 100)
 206
 207         c.Check(cluster.Services.WebDAVDownload.ExternalURL, check.Equals, arvados.URL{Host: "download.example.com", Path: "/"})
 208         c.Check(cluster.Services.WebDAVDownload.InternalURLs[arvados.URL{Host: ":80"}], check.NotNil)
 209         c.Check(cluster.Services.WebDAV.InternalURLs[arvados.URL{Host: ":80"}], check.NotNil)
 210
 211         c.Check(cluster.Collections.TrustAllContent, check.Equals, true)
 212         c.Check(cluster.Users.AnonymousUserToken, check.Equals, "anonusertoken")
 213         c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
 214 }
 215
 216 // Tests fix for https://dev.arvados.org/issues/15642
 217 func (s *LoadSuite) TestLegacyKeepWebConfigDoesntDisableMissingItems(c *check.C) {
 218         content := []byte(`
 219 {
 220         "Client": {
 221                 "Scheme": "",
 222                 "APIHost": "example.com",
 223                 "AuthToken": "abcdefg",
 224         }
 225 }
 226 `)
 227         cluster, err := testLoadLegacyConfig(content, "-legacy-keepweb-config", c)
 228         c.Assert(err, check.IsNil)
 229         // The resulting ManagementToken should be the one set up on the test server.
 230         c.Check(cluster.ManagementToken, check.Equals, TestServerManagementToken)
 231 }
 232
 233 func (s *LoadSuite) TestLegacyKeepproxyConfig(c *check.C) {
 234         f := "-legacy-keepproxy-config"
 235         content := []byte(fmtKeepproxyConfig("", true))
 236         cluster, err := testLoadLegacyConfig(content, f, c)
 237
 238         c.Assert(err, check.IsNil)
 239         c.Assert(cluster, check.NotNil)
 240         c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
 241         c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
 242         c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
 243         c.Check(cluster.Services.Keepproxy.InternalURLs[arvados.URL{Host: ":80"}], check.Equals, arvados.ServiceInstance{})
 244         c.Check(cluster.Collections.DefaultReplication, check.Equals, 0)
 245         c.Check(cluster.API.KeepServiceRequestTimeout.String(), check.Equals, "15s")
 246         c.Check(cluster.SystemLogs.LogLevel, check.Equals, "debug")
 247
 248         content = []byte(fmtKeepproxyConfig("", false))
 249         cluster, err = testLoadLegacyConfig(content, f, c)
 250         c.Check(err, check.IsNil)
 251         c.Check(cluster.SystemLogs.LogLevel, check.Equals, "info")
 252
 253         content = []byte(fmtKeepproxyConfig(`"DisableGet": true,`, true))
 254         _, err = testLoadLegacyConfig(content, f, c)
 255         c.Check(err, check.NotNil)
 256
 257         content = []byte(fmtKeepproxyConfig(`"DisablePut": true,`, true))
 258         _, err = testLoadLegacyConfig(content, f, c)
 259         c.Check(err, check.NotNil)
 260
 261         content = []byte(fmtKeepproxyConfig(`"PIDFile": "test",`, true))
 262         _, err = testLoadLegacyConfig(content, f, c)
 263         c.Check(err, check.NotNil)
 264
 265         content = []byte(fmtKeepproxyConfig(`"DisableGet": false, "DisablePut": false, "PIDFile": "",`, true))
 266         _, err = testLoadLegacyConfig(content, f, c)
 267         c.Check(err, check.IsNil)
 268 }
 269
 270 func fmtKeepproxyConfig(param string, debugLog bool) string {
 271         return fmt.Sprintf(`
 272 {
 273         "Client": {
 274                 "Scheme": "",
 275                 "APIHost": "example.com",
 276                 "AuthToken": "abcdefg",
 277                 "Insecure": false
 278         },
 279         "Listen": ":80",
 280         "DefaultReplicas": 0,
 281         "Timeout": "15s",
 282         "Debug": %t,
 283         %s
 284         "ManagementToken": "xyzzy"
 285 }
 286 `, debugLog, param)
 287 }
 288
 289 func (s *LoadSuite) TestLegacyArvGitHttpdConfig(c *check.C) {
 290         content := []byte(`
 291 {
 292         "Client": {
 293                 "Scheme": "",
 294                 "APIHost": "example.com",
 295                 "AuthToken": "abcdefg",
 296         },
 297         "Listen": ":9000",
 298         "GitCommand": "/test/git",
 299         "GitoliteHome": "/test/gitolite",
 300         "RepoRoot": "/test/reporoot",
 301         "ManagementToken": "xyzzy"
 302 }
 303 `)
 304         f := "-legacy-git-httpd-config"
 305         cluster, err := testLoadLegacyConfig(content, f, c)
 306
 307         c.Assert(err, check.IsNil)
 308         c.Assert(cluster, check.NotNil)
 309         c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
 310         c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
 311         c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
 312         c.Check(cluster.Git.GitCommand, check.Equals, "/test/git")
 313         c.Check(cluster.Git.GitoliteHome, check.Equals, "/test/gitolite")
 314         c.Check(cluster.Git.Repositories, check.Equals, "/test/reporoot")
 315         c.Check(cluster.Services.Keepproxy.InternalURLs[arvados.URL{Host: ":9000"}], check.Equals, arvados.ServiceInstance{})
 316 }
 317
 318 // Tests fix for https://dev.arvados.org/issues/15642
 319 func (s *LoadSuite) TestLegacyArvGitHttpdConfigDoesntDisableMissingItems(c *check.C) {
 320         content := []byte(`
 321 {
 322         "Client": {
 323                 "Scheme": "",
 324                 "APIHost": "example.com",
 325                 "AuthToken": "abcdefg",
 326         }
 327 }
 328 `)
 329         cluster, err := testLoadLegacyConfig(content, "-legacy-git-httpd-config", c)
 330         c.Assert(err, check.IsNil)
 331         // The resulting ManagementToken should be the one set up on the test server.
 332         c.Check(cluster.ManagementToken, check.Equals, TestServerManagementToken)
 333 }
 334
 335 func (s *LoadSuite) TestLegacyKeepBalanceConfig(c *check.C) {
 336         f := "-legacy-keepbalance-config"
 337         content := []byte(fmtKeepBalanceConfig(""))
 338         cluster, err := testLoadLegacyConfig(content, f, c)
 339
 340         c.Assert(err, check.IsNil)
 341         c.Assert(cluster, check.NotNil)
 342         c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
 343         c.Check(cluster.Services.Keepbalance.InternalURLs[arvados.URL{Host: ":80"}], check.Equals, arvados.ServiceInstance{})
 344         c.Check(cluster.Collections.BalanceCollectionBuffers, check.Equals, 1000)
 345         c.Check(cluster.Collections.BalanceCollectionBatch, check.Equals, 100000)
 346         c.Check(cluster.Collections.BalancePeriod.String(), check.Equals, "10m")
 347         c.Check(cluster.Collections.BlobMissingReport, check.Equals, "testfile")
 348         c.Check(cluster.API.KeepServiceRequestTimeout.String(), check.Equals, "30m")
 349
 350         content = []byte(fmtKeepBalanceConfig(`"KeepServiceTypes":["disk"],`))
 351         _, err = testLoadLegacyConfig(content, f, c)
 352         c.Check(err, check.IsNil)
 353
 354         content = []byte(fmtKeepBalanceConfig(`"KeepServiceTypes":[],`))
 355         _, err = testLoadLegacyConfig(content, f, c)
 356         c.Check(err, check.IsNil)
 357
 358         content = []byte(fmtKeepBalanceConfig(`"KeepServiceTypes":["proxy"],`))
 359         _, err = testLoadLegacyConfig(content, f, c)
 360         c.Check(err, check.NotNil)
 361
 362         content = []byte(fmtKeepBalanceConfig(`"KeepServiceTypes":["disk", "proxy"],`))
 363         _, err = testLoadLegacyConfig(content, f, c)
 364         c.Check(err, check.NotNil)
 365
 366         content = []byte(fmtKeepBalanceConfig(`"KeepServiceList":{},`))
 367         _, err = testLoadLegacyConfig(content, f, c)
 368         c.Check(err, check.NotNil)
 369 }
 370
 371 func fmtKeepBalanceConfig(param string) string {
 372         return fmt.Sprintf(`
 373 {
 374         "Client": {
 375                 "Scheme": "",
 376                 "APIHost": "example.com",
 377                 "AuthToken": "abcdefg",
 378                 "Insecure": false
 379         },
 380         "Listen": ":80",
 381         %s
 382         "RunPeriod": "10m",
 383         "CollectionBatchSize": 100000,
 384         "CollectionBuffers": 1000,
 385         "RequestTimeout": "30m",
 386         "ManagementToken": "xyzzy",
 387         "LostBlocksFile": "testfile"
 388 }
 389 `, param)
 390 }