lib/config/deprecated_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package config
   6
   7 import (
   8         "flag"
   9         "fmt"
  10         "io/ioutil"
  11         "os"
  12         "time"
  13
  14         "git.arvados.org/arvados.git/sdk/go/arvados"
  15         check "gopkg.in/check.v1"
  16 )
  17
  18 // Configured at: sdk/python/tests/run_test_server.py
  19 const TestServerManagementToken = "e687950a23c3a9bceec28c6223a06c79"
  20
  21 func testLoadLegacyConfig(content []byte, mungeFlag string, c *check.C) (*arvados.Cluster, error) {
  22         tmpfile, err := ioutil.TempFile("", "example")
  23         if err != nil {
  24                 return nil, err
  25         }
  26         defer os.Remove(tmpfile.Name())
  27
  28         if _, err := tmpfile.Write(content); err != nil {
  29                 return nil, err
  30         }
  31         if err := tmpfile.Close(); err != nil {
  32                 return nil, err
  33         }
  34         flags := flag.NewFlagSet("test", flag.ExitOnError)
  35         ldr := testLoader(c, "Clusters: {zzzzz: {}}", nil)
  36         ldr.SetupFlags(flags)
  37         args := ldr.MungeLegacyConfigArgs(ldr.Logger, []string{"-config", tmpfile.Name()}, mungeFlag)
  38         flags.Parse(args)
  39         cfg, err := ldr.Load()
  40         if err != nil {
  41                 return nil, err
  42         }
  43         cluster, err := cfg.GetCluster("")
  44         if err != nil {
  45                 return nil, err
  46         }
  47         return cluster, nil
  48 }
  49
  50 func (s *LoadSuite) TestLegacyVolumeDriverParameters(c *check.C) {
  51         logs := checkEquivalent(c, `
  52 Clusters:
  53  z1111:
  54   Volumes:
  55    z1111-nyw5e-aaaaaaaaaaaaaaa:
  56     Driver: s3
  57     DriverParameters:
  58      AccessKey: exampleaccesskey
  59      SecretKey: examplesecretkey
  60      Region: foobar
  61      ReadTimeout: 1200s
  62 `, `
  63 Clusters:
  64  z1111:
  65   Volumes:
  66    z1111-nyw5e-aaaaaaaaaaaaaaa:
  67     Driver: s3
  68     DriverParameters:
  69      AccessKeyID: exampleaccesskey
  70      SecretAccessKey: examplesecretkey
  71      Region: foobar
  72      ReadTimeout: 1200s
  73 `)
  74         c.Check(logs, check.Matches, `(?ms).*deprecated or unknown config entry: .*AccessKey.*`)
  75         c.Check(logs, check.Matches, `(?ms).*deprecated or unknown config entry: .*SecretKey.*`)
  76 }
  77
  78 func (s *LoadSuite) TestDeprecatedNodeProfilesToServices(c *check.C) {
  79         hostname, err := os.Hostname()
  80         c.Assert(err, check.IsNil)
  81         checkEquivalent(c, `
  82 Clusters:
  83  z1111:
  84   NodeProfiles:
  85    "*":
  86     arvados-controller:
  87      listen: ":9004"
  88    `+hostname+`:
  89     arvados-api-server:
  90      listen: ":8000"
  91    dispatch-host:
  92     arvados-dispatch-cloud:
  93      listen: ":9006"
  94 `, `
  95 Clusters:
  96  z1111:
  97   Services:
  98    RailsAPI:
  99     InternalURLs:
 100      "http://localhost:8000": {}
 101    Controller:
 102     InternalURLs:
 103      "http://localhost:9004": {}
 104    DispatchCloud:
 105     InternalURLs:
 106      "http://dispatch-host:9006": {}
 107   NodeProfiles:
 108    "*":
 109     arvados-controller:
 110      listen: ":9004"
 111    `+hostname+`:
 112     arvados-api-server:
 113      listen: ":8000"
 114    dispatch-host:
 115     arvados-dispatch-cloud:
 116      listen: ":9006"
 117 `)
 118 }
 119
 120 func (s *LoadSuite) TestDeprecatedLoginBackend(c *check.C) {
 121         checkEquivalent(c, `
 122 Clusters:
 123  z1111:
 124   Login:
 125    GoogleClientID: aaaa
 126    GoogleClientSecret: bbbb
 127    GoogleAlternateEmailAddresses: true
 128 `, `
 129 Clusters:
 130  z1111:
 131   Login:
 132    Google:
 133     Enable: true
 134     ClientID: aaaa
 135     ClientSecret: bbbb
 136     AlternateEmailAddresses: true
 137 `)
 138         checkEquivalent(c, `
 139 Clusters:
 140  z1111:
 141   Login:
 142    ProviderAppID: aaaa
 143    ProviderAppSecret: bbbb
 144 `, `
 145 Clusters:
 146  z1111:
 147   Login:
 148    SSO:
 149     Enable: true
 150     ProviderAppID: aaaa
 151     ProviderAppSecret: bbbb
 152 `)
 153 }
 154
 155 func (s *LoadSuite) TestLegacyKeepWebConfig(c *check.C) {
 156         content := []byte(`
 157 {
 158         "Client": {
 159                 "Scheme": "",
 160                 "APIHost": "example.com",
 161                 "AuthToken": "abcdefg",
 162         },
 163         "Listen": ":80",
 164         "AnonymousTokens": [
 165                 "anonusertoken"
 166         ],
 167         "AttachmentOnlyHost": "download.example.com",
 168         "TrustAllContent": true,
 169         "Cache": {
 170                 "TTL": "1m",
 171                 "UUIDTTL": "1s",
 172                 "MaxCollectionEntries": 42,
 173                 "MaxCollectionBytes": 1234567890,
 174                 "MaxPermissionEntries": 100,
 175                 "MaxUUIDEntries": 100
 176         },
 177         "ManagementToken": "xyzzy"
 178 }
 179 `)
 180         cluster, err := testLoadLegacyConfig(content, "-legacy-keepweb-config", c)
 181         c.Assert(err, check.IsNil)
 182
 183         c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
 184         c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
 185
 186         c.Check(cluster.Collections.WebDAVCache.TTL, check.Equals, arvados.Duration(60*time.Second))
 187         c.Check(cluster.Collections.WebDAVCache.UUIDTTL, check.Equals, arvados.Duration(time.Second))
 188         c.Check(cluster.Collections.WebDAVCache.MaxCollectionEntries, check.Equals, 42)
 189         c.Check(cluster.Collections.WebDAVCache.MaxCollectionBytes, check.Equals, int64(1234567890))
 190         c.Check(cluster.Collections.WebDAVCache.MaxPermissionEntries, check.Equals, 100)
 191         c.Check(cluster.Collections.WebDAVCache.MaxUUIDEntries, check.Equals, 100)
 192
 193         c.Check(cluster.Services.WebDAVDownload.ExternalURL, check.Equals, arvados.URL{Host: "download.example.com", Path: "/"})
 194         c.Check(cluster.Services.WebDAVDownload.InternalURLs[arvados.URL{Host: ":80"}], check.NotNil)
 195         c.Check(cluster.Services.WebDAV.InternalURLs[arvados.URL{Host: ":80"}], check.NotNil)
 196
 197         c.Check(cluster.Collections.TrustAllContent, check.Equals, true)
 198         c.Check(cluster.Users.AnonymousUserToken, check.Equals, "anonusertoken")
 199         c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
 200 }
 201
 202 // Tests fix for https://dev.arvados.org/issues/15642
 203 func (s *LoadSuite) TestLegacyKeepWebConfigDoesntDisableMissingItems(c *check.C) {
 204         content := []byte(`
 205 {
 206         "Client": {
 207                 "Scheme": "",
 208                 "APIHost": "example.com",
 209                 "AuthToken": "abcdefg",
 210         }
 211 }
 212 `)
 213         cluster, err := testLoadLegacyConfig(content, "-legacy-keepweb-config", c)
 214         c.Assert(err, check.IsNil)
 215         // The resulting ManagementToken should be the one set up on the test server.
 216         c.Check(cluster.ManagementToken, check.Equals, TestServerManagementToken)
 217 }
 218
 219 func (s *LoadSuite) TestLegacyKeepproxyConfig(c *check.C) {
 220         f := "-legacy-keepproxy-config"
 221         content := []byte(fmtKeepproxyConfig("", true))
 222         cluster, err := testLoadLegacyConfig(content, f, c)
 223
 224         c.Assert(err, check.IsNil)
 225         c.Assert(cluster, check.NotNil)
 226         c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
 227         c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
 228         c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
 229         c.Check(cluster.Services.Keepproxy.InternalURLs[arvados.URL{Host: ":80"}], check.Equals, arvados.ServiceInstance{})
 230         c.Check(cluster.Collections.DefaultReplication, check.Equals, 0)
 231         c.Check(cluster.API.KeepServiceRequestTimeout.String(), check.Equals, "15s")
 232         c.Check(cluster.SystemLogs.LogLevel, check.Equals, "debug")
 233
 234         content = []byte(fmtKeepproxyConfig("", false))
 235         cluster, err = testLoadLegacyConfig(content, f, c)
 236         c.Check(err, check.IsNil)
 237         c.Check(cluster.SystemLogs.LogLevel, check.Equals, "info")
 238
 239         content = []byte(fmtKeepproxyConfig(`"DisableGet": true,`, true))
 240         _, err = testLoadLegacyConfig(content, f, c)
 241         c.Check(err, check.NotNil)
 242
 243         content = []byte(fmtKeepproxyConfig(`"DisablePut": true,`, true))
 244         _, err = testLoadLegacyConfig(content, f, c)
 245         c.Check(err, check.NotNil)
 246
 247         content = []byte(fmtKeepproxyConfig(`"PIDFile": "test",`, true))
 248         _, err = testLoadLegacyConfig(content, f, c)
 249         c.Check(err, check.NotNil)
 250
 251         content = []byte(fmtKeepproxyConfig(`"DisableGet": false, "DisablePut": false, "PIDFile": "",`, true))
 252         _, err = testLoadLegacyConfig(content, f, c)
 253         c.Check(err, check.IsNil)
 254 }
 255
 256 func fmtKeepproxyConfig(param string, debugLog bool) string {
 257         return fmt.Sprintf(`
 258 {
 259         "Client": {
 260                 "Scheme": "",
 261                 "APIHost": "example.com",
 262                 "AuthToken": "abcdefg",
 263                 "Insecure": false
 264         },
 265         "Listen": ":80",
 266         "DefaultReplicas": 0,
 267         "Timeout": "15s",
 268         "Debug": %t,
 269         %s
 270         "ManagementToken": "xyzzy"
 271 }
 272 `, debugLog, param)
 273 }
 274
 275 func (s *LoadSuite) TestLegacyArvGitHttpdConfig(c *check.C) {
 276         content := []byte(`
 277 {
 278         "Client": {
 279                 "Scheme": "",
 280                 "APIHost": "example.com",
 281                 "AuthToken": "abcdefg",
 282         },
 283         "Listen": ":9000",
 284         "GitCommand": "/test/git",
 285         "GitoliteHome": "/test/gitolite",
 286         "RepoRoot": "/test/reporoot",
 287         "ManagementToken": "xyzzy"
 288 }
 289 `)
 290         f := "-legacy-git-httpd-config"
 291         cluster, err := testLoadLegacyConfig(content, f, c)
 292
 293         c.Assert(err, check.IsNil)
 294         c.Assert(cluster, check.NotNil)
 295         c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
 296         c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
 297         c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
 298         c.Check(cluster.Git.GitCommand, check.Equals, "/test/git")
 299         c.Check(cluster.Git.GitoliteHome, check.Equals, "/test/gitolite")
 300         c.Check(cluster.Git.Repositories, check.Equals, "/test/reporoot")
 301         c.Check(cluster.Services.Keepproxy.InternalURLs[arvados.URL{Host: ":9000"}], check.Equals, arvados.ServiceInstance{})
 302 }
 303
 304 // Tests fix for https://dev.arvados.org/issues/15642
 305 func (s *LoadSuite) TestLegacyArvGitHttpdConfigDoesntDisableMissingItems(c *check.C) {
 306         content := []byte(`
 307 {
 308         "Client": {
 309                 "Scheme": "",
 310                 "APIHost": "example.com",
 311                 "AuthToken": "abcdefg",
 312         }
 313 }
 314 `)
 315         cluster, err := testLoadLegacyConfig(content, "-legacy-git-httpd-config", c)
 316         c.Assert(err, check.IsNil)
 317         // The resulting ManagementToken should be the one set up on the test server.
 318         c.Check(cluster.ManagementToken, check.Equals, TestServerManagementToken)
 319 }
 320
 321 func (s *LoadSuite) TestLegacyKeepBalanceConfig(c *check.C) {
 322         f := "-legacy-keepbalance-config"
 323         content := []byte(fmtKeepBalanceConfig(""))
 324         cluster, err := testLoadLegacyConfig(content, f, c)
 325
 326         c.Assert(err, check.IsNil)
 327         c.Assert(cluster, check.NotNil)
 328         c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
 329         c.Check(cluster.Services.Keepbalance.InternalURLs[arvados.URL{Host: ":80"}], check.Equals, arvados.ServiceInstance{})
 330         c.Check(cluster.Collections.BalanceCollectionBuffers, check.Equals, 1000)
 331         c.Check(cluster.Collections.BalanceCollectionBatch, check.Equals, 100000)
 332         c.Check(cluster.Collections.BalancePeriod.String(), check.Equals, "10m")
 333         c.Check(cluster.Collections.BlobMissingReport, check.Equals, "testfile")
 334         c.Check(cluster.API.KeepServiceRequestTimeout.String(), check.Equals, "30m")
 335
 336         content = []byte(fmtKeepBalanceConfig(`"KeepServiceTypes":["disk"],`))
 337         _, err = testLoadLegacyConfig(content, f, c)
 338         c.Check(err, check.IsNil)
 339
 340         content = []byte(fmtKeepBalanceConfig(`"KeepServiceTypes":[],`))
 341         _, err = testLoadLegacyConfig(content, f, c)
 342         c.Check(err, check.IsNil)
 343
 344         content = []byte(fmtKeepBalanceConfig(`"KeepServiceTypes":["proxy"],`))
 345         _, err = testLoadLegacyConfig(content, f, c)
 346         c.Check(err, check.NotNil)
 347
 348         content = []byte(fmtKeepBalanceConfig(`"KeepServiceTypes":["disk", "proxy"],`))
 349         _, err = testLoadLegacyConfig(content, f, c)
 350         c.Check(err, check.NotNil)
 351
 352         content = []byte(fmtKeepBalanceConfig(`"KeepServiceList":{},`))
 353         _, err = testLoadLegacyConfig(content, f, c)
 354         c.Check(err, check.NotNil)
 355 }
 356
 357 func fmtKeepBalanceConfig(param string) string {
 358         return fmt.Sprintf(`
 359 {
 360         "Client": {
 361                 "Scheme": "",
 362                 "APIHost": "example.com",
 363                 "AuthToken": "abcdefg",
 364                 "Insecure": false
 365         },
 366         "Listen": ":80",
 367         %s
 368         "RunPeriod": "10m",
 369         "CollectionBatchSize": 100000,
 370         "CollectionBuffers": 1000,
 371         "RequestTimeout": "30m",
 372         "ManagementToken": "xyzzy",
 373         "LostBlocksFile": "testfile"
 374 }
 375 `, param)
 376 }