{"errors":["Forbidden"],"error_token":"1533044555+684b532c"}
</code></pre>
</notextile>
+
+h3(#confirm-config). Confirm the public configuration is OK
+
+Confirm the publicly accessible configuration endpoint does not reveal any sensitive information (e.g., a secret that was mistakenly entered under the wrong configuration key). Use the jq program, if you have installed it, to make the JSON document easier to read.
+
+<notextile>
+<pre><code>~$ <span class="userinput">curl http://0.0.0.0:<b>9004</b>/arvados/v1/config | jq .</span>
+{
+ "API": {
+ "MaxItemsPerResponse": 1000,
+ "MaxRequestAmplification": 4,
+ "RequestTimeout": "5m"
+ },
+ ...
+</code></pre>
+</notextile>
"os"
"os/exec"
- "git.curoverse.com/arvados.git/lib/cmd"
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/ctxlog"
"github.com/ghodss/yaml"
)
-var DumpCommand cmd.Handler = dumpCommand{}
+var DumpCommand dumpCommand
type dumpCommand struct{}
return 0
}
-var CheckCommand cmd.Handler = checkCommand{}
+var CheckCommand checkCommand
type checkCommand struct{}
import (
"bytes"
+ "git.curoverse.com/arvados.git/lib/cmd"
check "gopkg.in/check.v1"
)
var _ = check.Suite(&CommandSuite{})
+var (
+ // Commands must satisfy cmd.Handler interface
+ _ cmd.Handler = dumpCommand{}
+ _ cmd.Handler = checkCommand{}
+)
+
type CommandSuite struct{}
func (s *CommandSuite) TestBadArg(c *check.C) {
`
code := CheckCommand.RunCommand("arvados config-check", []string{"-config", "-"}, bytes.NewBufferString(in), &stdout, &stderr)
c.Check(code, check.Equals, 1)
- c.Check(stdout.String(), check.Matches, `(?ms).*API:\n\- +.*MaxItemsPerResponse: 1000\n\+ +MaxItemsPerResponse: 1234\n.*`)
+ c.Check(stdout.String(), check.Matches, `(?ms).*\n\- +.*MaxItemsPerResponse: 1000\n\+ +MaxItemsPerResponse: 1234\n.*`)
}
func (s *CommandSuite) TestCheckUnknownKey(c *check.C) {
InternalURLs: {}
ExternalURL: ""
GitSSH:
+ InternalURLs: {}
ExternalURL: ""
DispatchCloud:
InternalURLs: {}
ExternalURL: "-"
SSO:
+ InternalURLs: {}
ExternalURL: ""
Keepproxy:
InternalURLs: {}
InternalURLs: {}
ExternalURL: "-"
Composer:
+ InternalURLs: {}
ExternalURL: ""
WebShell:
+ InternalURLs: {}
ExternalURL: ""
Workbench1:
InternalURLs: {}
ExternalURL: ""
Workbench2:
+ InternalURLs: {}
ExternalURL: ""
Nodemanager:
InternalURLs: {}
# Interval (seconds) between asynchronous permission view updates. Any
# permission-updating API called with the 'async' parameter schedules a an
# update on the permission view in the future, if not already scheduled.
- AsyncPermissionsUpdateInterval: 20
+ AsyncPermissionsUpdateInterval: 20s
# Maximum number of concurrent outgoing requests to make while
# serving a single incoming multi-cluster (federated) request.
# Interval (seconds) between trash sweeps. During a trash sweep,
# collections are marked as trash if their trash_at time has
# arrived, and deleted if their delete_at time has arrived.
- TrashSweepInterval: 60
+ TrashSweepInterval: 60s
# If true, enable collection versioning.
# When a collection's preserve_version field is true or the current version
# the current collection.
CollectionVersioning: false
- # 0 = auto-create a new version on every update.
- # -1 = never auto-create new versions.
- # > 0 = auto-create a new version when older than the specified number of seconds.
- PreserveVersionIfIdle: -1
+ # 0s = auto-create a new version on every update.
+ # -1s = never auto-create new versions.
+ # > 0s = auto-create a new version when older than the specified number of seconds.
+ PreserveVersionIfIdle: -1s
Login:
# These settings are provided by your OAuth2 provider (e.g.,
# scheduling parameter parameter set.
UsePreemptibleInstances: false
- # Include details about job reuse decisions in the server log. This
- # causes additional database queries to run, so it should not be
- # enabled unless you expect to examine the resulting logs for
- # troubleshooting purposes.
- LogReuseDecisions: false
-
# PEM encoded SSH key (RSA, DSA, or ECDSA) used by the
# (experimental) cloud dispatcher for executing containers on
# worker VMs. Begins with "-----BEGIN RSA PRIVATE KEY-----\n"
LogBytesPerEvent: 4096
LogSecondsBetweenEvents: 1
- # The sample period for throttling logs, in seconds.
- LogThrottlePeriod: 60
+ # The sample period for throttling logs.
+ LogThrottlePeriod: 60s
# Maximum number of bytes that job can log over crunch_log_throttle_period
# before being silenced until the end of the period.
# silenced by throttling are not counted against this total.
LimitLogBytesPerJob: 67108864
- LogPartialLineThrottlePeriod: 5
+ LogPartialLineThrottlePeriod: 5s
- # Container logs are written to Keep and saved in a collection,
- # which is updated periodically while the container runs. This
- # value sets the interval (given in seconds) between collection
- # updates.
- LogUpdatePeriod: 1800
+ # Container logs are written to Keep and saved in a
+ # collection, which is updated periodically while the
+ # container runs. This value sets the interval between
+ # collection updates.
+ LogUpdatePeriod: 30m
# The log collection is also updated when the specified amount of
# log data (given in bytes) is produced in less than one update
# period.
- LogUpdateSize: 33554432
+ LogUpdateSize: 32MiB
SLURM:
Managed:
TimeoutShutdown: 10s
# Worker VM image ID.
- ImageID: ami-01234567890abcdef
+ ImageID: ""
# Tags to add on all resources (VMs, NICs, disks) created by
# the container dispatcher. (Arvados's own tags --
Insecure: false
ActivateUsers: false
SAMPLE:
+ # API endpoint host or host:port; default is {id}.arvadosapi.com
Host: sample.arvadosapi.com
+
+ # Perform a proxy request when a local client requests an
+ # object belonging to this remote.
Proxy: false
+
+ # Default "https". Can be set to "http" for testing.
Scheme: https
+
+ # Disable TLS verify. Can be set to true for testing.
Insecure: false
+
+ # When users present tokens issued by this remote cluster, and
+ # their accounts are active on the remote cluster, activate
+ # them on this cluster too.
ActivateUsers: false
+
+ Workbench:
+ # Workbench1 configs
+ Theme: default
+ ActivationContactLink: mailto:info@arvados.org
+ ArvadosDocsite: https://doc.arvados.org
+ ArvadosPublicDataDocURL: https://playground.arvados.org/projects/public
+ ShowUserAgreementInline: false
+ SecretToken: ""
+ SecretKeyBase: ""
+ RepositoryCache: /var/www/arvados-workbench/current/tmp/git
+ UserProfileFormFields:
+ SAMPLE:
+ Type: text
+ FormFieldTitle: ""
+ FormFieldDescription: ""
+ Required: true
+ UserProfileFormMessage: 'Welcome to Arvados. All <span style="color:red">required fields</span> must be completed before you can proceed.'
+ ApplicationMimetypesWithViewIcon:
+ cwl: {}
+ fasta: {}
+ go: {}
+ javascript: {}
+ json: {}
+ pdf: {}
+ python: {}
+ x-python: {}
+ r: {}
+ rtf: {}
+ sam: {}
+ x-sh: {}
+ vnd.realvnc.bed: {}
+ xml: {}
+ xsl: {}
+ LogViewerMaxBytes: 1M
+ EnablePublicProjectsPage: true
+ EnableGettingStartedPopup: false
+ APIResponseCompression: true
+ APIClientConnectTimeout: 2m
+ APIClientReceiveTimeout: 5m
+ RunningJobLogRecordsToFetch: 2000
+ ShowRecentCollectionsOnDashboard: true
+ ShowUserNotifications: true
+ MultiSiteSearch: false
+ Repositories: true
+ SiteName: Arvados Workbench
+
+ # Workbench2 configs
+ VocabularyURL: ""
+ FileViewersConfigURL: ""
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package config
+
+import (
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "strings"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+)
+
+// ExportJSON writes a JSON object with the safe (non-secret) portions
+// of the cluster config to w.
+func ExportJSON(w io.Writer, cluster *arvados.Cluster) error {
+ buf, err := json.Marshal(cluster)
+ if err != nil {
+ return err
+ }
+ var m map[string]interface{}
+ err = json.Unmarshal(buf, &m)
+ if err != nil {
+ return err
+ }
+ err = redactUnsafe(m, "", "")
+ if err != nil {
+ return err
+ }
+ return json.NewEncoder(w).Encode(m)
+}
+
+// whitelist classifies configs as safe/unsafe to reveal to
+// unauthenticated clients.
+//
+// Every config entry must either be listed explicitly here along with
+// all of its parent keys (e.g., "API" + "API.RequestTimeout"), or
+// have an ancestor listed as false (e.g.,
+// "PostgreSQL.Connection.password" has an ancestor
+// "PostgreSQL.Connection" with a false value). Otherwise, it is a bug
+// which should be caught by tests.
+//
+// Example: API.RequestTimeout is safe because whitelist["API"] == and
+// whitelist["API.RequestTimeout"] == true.
+//
+// Example: PostgreSQL.Connection.password is not safe because
+// whitelist["PostgreSQL.Connection"] == false.
+//
+// Example: PostgreSQL.BadKey would cause an error because
+// whitelist["PostgreSQL"] isn't false, and neither
+// whitelist["PostgreSQL.BadKey"] nor whitelist["PostgreSQL.*"]
+// exists.
+var whitelist = map[string]bool{
+ // | sort -t'"' -k2,2
+ "API": true,
+ "API.AsyncPermissionsUpdateInterval": false,
+ "API.DisabledAPIs": false,
+ "API.MaxIndexDatabaseRead": false,
+ "API.MaxItemsPerResponse": true,
+ "API.MaxRequestAmplification": false,
+ "API.MaxRequestSize": true,
+ "API.RailsSessionSecretToken": false,
+ "API.RequestTimeout": true,
+ "AuditLogs": false,
+ "AuditLogs.MaxAge": false,
+ "AuditLogs.MaxDeleteBatch": false,
+ "AuditLogs.UnloggedAttributes": false,
+ "Collections": true,
+ "Collections.BlobSigning": true,
+ "Collections.BlobSigningKey": false,
+ "Collections.BlobSigningTTL": true,
+ "Collections.CollectionVersioning": false,
+ "Collections.DefaultReplication": true,
+ "Collections.DefaultTrashLifetime": true,
+ "Collections.PreserveVersionIfIdle": true,
+ "Collections.TrashSweepInterval": false,
+ "Containers": true,
+ "Containers.CloudVMs": false,
+ "Containers.DefaultKeepCacheRAM": true,
+ "Containers.DispatchPrivateKey": false,
+ "Containers.JobsAPI": true,
+ "Containers.JobsAPI.CrunchJobUser": false,
+ "Containers.JobsAPI.CrunchJobWrapper": false,
+ "Containers.JobsAPI.CrunchRefreshTrigger": false,
+ "Containers.JobsAPI.DefaultDockerImage": false,
+ "Containers.JobsAPI.Enable": true,
+ "Containers.JobsAPI.GitInternalDir": false,
+ "Containers.JobsAPI.ReuseJobIfOutputsDiffer": false,
+ "Containers.Logging": false,
+ "Containers.LogReuseDecisions": false,
+ "Containers.MaxComputeVMs": false,
+ "Containers.MaxDispatchAttempts": false,
+ "Containers.MaxRetryAttempts": true,
+ "Containers.SLURM": false,
+ "Containers.StaleLockTimeout": false,
+ "Containers.SupportedDockerImageFormats": true,
+ "Containers.UsePreemptibleInstances": true,
+ "Git": false,
+ "InstanceTypes": true,
+ "InstanceTypes.*": true,
+ "InstanceTypes.*.*": true,
+ "Login": false,
+ "Mail": false,
+ "ManagementToken": false,
+ "PostgreSQL": false,
+ "RemoteClusters": true,
+ "RemoteClusters.*": true,
+ "RemoteClusters.*.ActivateUsers": true,
+ "RemoteClusters.*.Host": true,
+ "RemoteClusters.*.Insecure": true,
+ "RemoteClusters.*.Proxy": true,
+ "RemoteClusters.*.Scheme": true,
+ "Services": true,
+ "Services.*": true,
+ "Services.*.ExternalURL": true,
+ "Services.*.InternalURLs": false,
+ "SystemLogs": false,
+ "SystemRootToken": false,
+ "TLS": false,
+ "Users": false,
+ "Workbench": false,
+}
+
+func redactUnsafe(m map[string]interface{}, mPrefix, lookupPrefix string) error {
+ var errs []string
+ for k, v := range m {
+ lookupKey := k
+ safe, ok := whitelist[lookupPrefix+k]
+ if !ok {
+ lookupKey = "*"
+ safe, ok = whitelist[lookupPrefix+"*"]
+ }
+ if !ok {
+ errs = append(errs, fmt.Sprintf("config bug: key %q not in whitelist map", lookupPrefix+k))
+ continue
+ }
+ if !safe {
+ delete(m, k)
+ continue
+ }
+ if v, ok := v.(map[string]interface{}); ok {
+ err := redactUnsafe(v, mPrefix+k+".", lookupPrefix+lookupKey+".")
+ if err != nil {
+ errs = append(errs, err.Error())
+ }
+ }
+ }
+ if len(errs) > 0 {
+ return errors.New(strings.Join(errs, "\n"))
+ }
+ return nil
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package config
+
+import (
+ "bytes"
+ "regexp"
+ "strings"
+
+ "git.curoverse.com/arvados.git/sdk/go/ctxlog"
+ check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&ExportSuite{})
+
+type ExportSuite struct{}
+
+func (s *ExportSuite) TestExport(c *check.C) {
+ confdata := bytes.Replace(DefaultYAML, []byte("SAMPLE"), []byte("testkey"), -1)
+ cfg, err := Load(bytes.NewBuffer(confdata), ctxlog.TestLogger(c))
+ c.Assert(err, check.IsNil)
+ cluster := cfg.Clusters["xxxxx"]
+ cluster.ManagementToken = "abcdefg"
+
+ var exported bytes.Buffer
+ err = ExportJSON(&exported, &cluster)
+ c.Check(err, check.IsNil)
+ if err != nil {
+ c.Logf("If all the new keys are safe, add these to whitelist in export.go:")
+ for _, k := range regexp.MustCompile(`"[^"]*"`).FindAllString(err.Error(), -1) {
+ c.Logf("\t%q: true,", strings.Replace(k, `"`, "", -1))
+ }
+ }
+ c.Check(exported.String(), check.Not(check.Matches), `(?ms).*abcdefg.*`)
+}
InternalURLs: {}
ExternalURL: ""
GitSSH:
+ InternalURLs: {}
ExternalURL: ""
DispatchCloud:
InternalURLs: {}
ExternalURL: "-"
SSO:
+ InternalURLs: {}
ExternalURL: ""
Keepproxy:
InternalURLs: {}
InternalURLs: {}
ExternalURL: "-"
Composer:
+ InternalURLs: {}
ExternalURL: ""
WebShell:
+ InternalURLs: {}
ExternalURL: ""
Workbench1:
InternalURLs: {}
ExternalURL: ""
Workbench2:
+ InternalURLs: {}
ExternalURL: ""
Nodemanager:
InternalURLs: {}
# Interval (seconds) between asynchronous permission view updates. Any
# permission-updating API called with the 'async' parameter schedules a an
# update on the permission view in the future, if not already scheduled.
- AsyncPermissionsUpdateInterval: 20
+ AsyncPermissionsUpdateInterval: 20s
# Maximum number of concurrent outgoing requests to make while
# serving a single incoming multi-cluster (federated) request.
# Interval (seconds) between trash sweeps. During a trash sweep,
# collections are marked as trash if their trash_at time has
# arrived, and deleted if their delete_at time has arrived.
- TrashSweepInterval: 60
+ TrashSweepInterval: 60s
# If true, enable collection versioning.
# When a collection's preserve_version field is true or the current version
# the current collection.
CollectionVersioning: false
- # 0 = auto-create a new version on every update.
- # -1 = never auto-create new versions.
- # > 0 = auto-create a new version when older than the specified number of seconds.
- PreserveVersionIfIdle: -1
+ # 0s = auto-create a new version on every update.
+ # -1s = never auto-create new versions.
+ # > 0s = auto-create a new version when older than the specified number of seconds.
+ PreserveVersionIfIdle: -1s
Login:
# These settings are provided by your OAuth2 provider (e.g.,
# scheduling parameter parameter set.
UsePreemptibleInstances: false
- # Include details about job reuse decisions in the server log. This
- # causes additional database queries to run, so it should not be
- # enabled unless you expect to examine the resulting logs for
- # troubleshooting purposes.
- LogReuseDecisions: false
-
# PEM encoded SSH key (RSA, DSA, or ECDSA) used by the
# (experimental) cloud dispatcher for executing containers on
# worker VMs. Begins with "-----BEGIN RSA PRIVATE KEY-----\n"
LogBytesPerEvent: 4096
LogSecondsBetweenEvents: 1
- # The sample period for throttling logs, in seconds.
- LogThrottlePeriod: 60
+ # The sample period for throttling logs.
+ LogThrottlePeriod: 60s
# Maximum number of bytes that job can log over crunch_log_throttle_period
# before being silenced until the end of the period.
# silenced by throttling are not counted against this total.
LimitLogBytesPerJob: 67108864
- LogPartialLineThrottlePeriod: 5
+ LogPartialLineThrottlePeriod: 5s
- # Container logs are written to Keep and saved in a collection,
- # which is updated periodically while the container runs. This
- # value sets the interval (given in seconds) between collection
- # updates.
- LogUpdatePeriod: 1800
+ # Container logs are written to Keep and saved in a
+ # collection, which is updated periodically while the
+ # container runs. This value sets the interval between
+ # collection updates.
+ LogUpdatePeriod: 30m
# The log collection is also updated when the specified amount of
# log data (given in bytes) is produced in less than one update
# period.
- LogUpdateSize: 33554432
+ LogUpdateSize: 32MiB
SLURM:
Managed:
TimeoutShutdown: 10s
# Worker VM image ID.
- ImageID: ami-01234567890abcdef
+ ImageID: ""
# Tags to add on all resources (VMs, NICs, disks) created by
# the container dispatcher. (Arvados's own tags --
Insecure: false
ActivateUsers: false
SAMPLE:
+ # API endpoint host or host:port; default is {id}.arvadosapi.com
Host: sample.arvadosapi.com
+
+ # Perform a proxy request when a local client requests an
+ # object belonging to this remote.
Proxy: false
+
+ # Default "https". Can be set to "http" for testing.
Scheme: https
+
+ # Disable TLS verify. Can be set to true for testing.
Insecure: false
+
+ # When users present tokens issued by this remote cluster, and
+ # their accounts are active on the remote cluster, activate
+ # them on this cluster too.
ActivateUsers: false
+
+ Workbench:
+ # Workbench1 configs
+ Theme: default
+ ActivationContactLink: mailto:info@arvados.org
+ ArvadosDocsite: https://doc.arvados.org
+ ArvadosPublicDataDocURL: https://playground.arvados.org/projects/public
+ ShowUserAgreementInline: false
+ SecretToken: ""
+ SecretKeyBase: ""
+ RepositoryCache: /var/www/arvados-workbench/current/tmp/git
+ UserProfileFormFields:
+ SAMPLE:
+ Type: text
+ FormFieldTitle: ""
+ FormFieldDescription: ""
+ Required: true
+ UserProfileFormMessage: 'Welcome to Arvados. All <span style="color:red">required fields</span> must be completed before you can proceed.'
+ ApplicationMimetypesWithViewIcon:
+ cwl: {}
+ fasta: {}
+ go: {}
+ javascript: {}
+ json: {}
+ pdf: {}
+ python: {}
+ x-python: {}
+ r: {}
+ rtf: {}
+ sam: {}
+ x-sh: {}
+ vnd.realvnc.bed: {}
+ xml: {}
+ xsl: {}
+ LogViewerMaxBytes: 1M
+ EnablePublicProjectsPage: true
+ EnableGettingStartedPopup: false
+ APIResponseCompression: true
+ APIClientConnectTimeout: 2m
+ APIClientReceiveTimeout: 5m
+ RunningJobLogRecordsToFetch: 2000
+ ShowRecentCollectionsOnDashboard: true
+ ShowUserNotifications: true
+ MultiSiteSearch: false
+ Repositories: true
+ SiteName: Arvados Workbench
+
+ # Workbench2 configs
+ VocabularyURL: ""
+ FileViewersConfigURL: ""
`)
c.Check(logs, check.HasLen, 2)
}
+func (s *LoadSuite) TestNoUnrecognizedKeysInDefaultConfig(c *check.C) {
+ var logbuf bytes.Buffer
+ logger := logrus.New()
+ logger.Out = &logbuf
+ var supplied map[string]interface{}
+ yaml.Unmarshal(DefaultYAML, &supplied)
+ cfg, err := Load(bytes.NewBuffer(DefaultYAML), logger)
+ c.Assert(err, check.IsNil)
+ var loaded map[string]interface{}
+ buf, err := yaml.Marshal(cfg)
+ c.Assert(err, check.IsNil)
+ err = yaml.Unmarshal(buf, &loaded)
+ c.Assert(err, check.IsNil)
+
+ logExtraKeys(logger, loaded, supplied, "")
+ c.Check(logbuf.String(), check.Equals, "")
+}
+
func (s *LoadSuite) TestNoWarningsForDumpedConfig(c *check.C) {
var logbuf bytes.Buffer
logger := logrus.New()
cluster := &arvados.Cluster{
ClusterID: "zhome",
PostgreSQL: integrationTestCluster().PostgreSQL,
- TLS: arvados.TLS{Insecure: true},
- API: arvados.API{
- MaxItemsPerResponse: 1000,
- MaxRequestAmplification: 4,
- },
}
+ cluster.TLS.Insecure = true
+ cluster.API.MaxItemsPerResponse = 1000
+ cluster.API.MaxRequestAmplification = 4
arvadostest.SetServiceURL(&cluster.Services.RailsAPI, "http://localhost:1/")
arvadostest.SetServiceURL(&cluster.Services.Controller, "http://localhost:/")
s.testHandler = &Handler{Cluster: cluster}
package controller
import (
+ "bytes"
"context"
"database/sql"
"errors"
"fmt"
+ "io"
"net/http"
"net/url"
"strings"
"sync"
"time"
+ "git.curoverse.com/arvados.git/lib/config"
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/health"
"git.curoverse.com/arvados.git/sdk/go/httpserver"
Prefix: "/_health/",
Routes: health.Routes{"ping": func() error { _, err := h.db(&http.Request{}); return err }},
})
+
+ mux.Handle("/arvados/v1/config", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ var buf bytes.Buffer
+ err := config.ExportJSON(&buf, h.Cluster)
+ if err != nil {
+ httpserver.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ w.Header().Set("Content-Type", "application/json")
+ io.Copy(w, &buf)
+ }))
+
hs := http.NotFoundHandler()
hs = prepend(hs, h.proxyRailsAPI)
hs = h.setupProxyRemoteCluster(hs)
s.cluster = &arvados.Cluster{
ClusterID: "zzzzz",
PostgreSQL: integrationTestCluster().PostgreSQL,
- TLS: arvados.TLS{Insecure: true},
}
+ s.cluster.TLS.Insecure = true
arvadostest.SetServiceURL(&s.cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
arvadostest.SetServiceURL(&s.cluster.Services.Controller, "http://localhost:/")
s.handler = newHandler(s.ctx, s.cluster, "")
s.cancel()
}
+func (s *HandlerSuite) TestConfigExport(c *check.C) {
+ s.cluster.ManagementToken = "secret"
+ s.cluster.SystemRootToken = "secret"
+ s.cluster.Collections.BlobSigning = true
+ s.cluster.Collections.BlobSigningTTL = arvados.Duration(23 * time.Second)
+ req := httptest.NewRequest("GET", "/arvados/v1/config", nil)
+ resp := httptest.NewRecorder()
+ s.handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusOK)
+ var cluster arvados.Cluster
+ c.Log(resp.Body.String())
+ err := json.Unmarshal(resp.Body.Bytes(), &cluster)
+ c.Check(err, check.IsNil)
+ c.Check(cluster.ManagementToken, check.Equals, "")
+ c.Check(cluster.SystemRootToken, check.Equals, "")
+ c.Check(cluster.Collections.BlobSigning, check.DeepEquals, true)
+ c.Check(cluster.Collections.BlobSigningTTL, check.Equals, arvados.Duration(23*time.Second))
+}
+
func (s *HandlerSuite) TestProxyDiscoveryDoc(c *check.C) {
req := httptest.NewRequest("GET", "/discovery/v1/apis/arvados/v1/rest", nil)
resp := httptest.NewRecorder()
handler := &Handler{Cluster: &arvados.Cluster{
ClusterID: "zzzzz",
PostgreSQL: integrationTestCluster().PostgreSQL,
- TLS: arvados.TLS{Insecure: true},
}}
+ handler.Cluster.TLS.Insecure = true
arvadostest.SetServiceURL(&handler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
arvadostest.SetServiceURL(&handler.Cluster.Services.Controller, "http://localhost:/")
}
}
-type API struct {
- MaxItemsPerResponse int
- MaxRequestAmplification int
- RequestTimeout Duration
-}
-
type Cluster struct {
ClusterID string `json:"-"`
ManagementToken string
Containers ContainersConfig
RemoteClusters map[string]RemoteCluster
PostgreSQL PostgreSQL
- API API
- SystemLogs SystemLogs
- TLS TLS
+
+ API struct {
+ AsyncPermissionsUpdateInterval Duration
+ DisabledAPIs []string
+ MaxIndexDatabaseRead int
+ MaxItemsPerResponse int
+ MaxRequestAmplification int
+ MaxRequestSize int
+ RailsSessionSecretToken string
+ RequestTimeout Duration
+ }
+ AuditLogs struct {
+ MaxAge Duration
+ MaxDeleteBatch int
+ UnloggedAttributes []string
+ }
+ Collections struct {
+ BlobSigning bool
+ BlobSigningKey string
+ DefaultReplication int
+ BlobSigningTTL Duration
+ DefaultTrashLifetime Duration
+ TrashSweepInterval Duration
+ CollectionVersioning bool
+ PreserveVersionIfIdle Duration
+ }
+ Git struct {
+ Repositories string
+ }
+ Login struct {
+ ProviderAppSecret string
+ ProviderAppID string
+ }
+ Mail struct {
+ MailchimpAPIKey string
+ MailchimpListID string
+ SendUserSetupNotificationEmail string
+ IssueReporterEmailFrom string
+ IssueReporterEmailTo string
+ SupportEmailAddress string
+ EmailFrom string
+ }
+ SystemLogs struct {
+ LogLevel string
+ Format string
+ MaxRequestLogParamsSize int
+ }
+ TLS struct {
+ Certificate string
+ Key string
+ Insecure bool
+ }
+ Users struct {
+ AdminNotifierEmailFrom string
+ AutoAdminFirstUser bool
+ AutoAdminUserWithEmail string
+ AutoSetupNewUsers bool
+ AutoSetupNewUsersWithRepository bool
+ AutoSetupNewUsersWithVmUUID string
+ AutoSetupUsernameBlacklist []string
+ EmailSubjectPrefix string
+ NewInactiveUserNotificationRecipients []string
+ NewUserNotificationRecipients []string
+ NewUsersAreActive bool
+ UserNotifierEmailFrom string
+ UserProfileNotificationAddress string
+ }
+ Workbench struct {
+ ActivationContactLink string
+ APIClientConnectTimeout Duration
+ APIClientReceiveTimeout Duration
+ APIResponseCompression bool
+ ApplicationMimetypesWithViewIcon map[string]struct{}
+ ArvadosDocsite string
+ ArvadosPublicDataDocURL string
+ EnableGettingStartedPopup bool
+ EnablePublicProjectsPage bool
+ FileViewersConfigURL string
+ LogViewerMaxBytes ByteSize
+ MultiSiteSearch bool
+ Repositories bool
+ RepositoryCache string
+ RunningJobLogRecordsToFetch int
+ SecretKeyBase string
+ SecretToken string
+ ShowRecentCollectionsOnDashboard bool
+ ShowUserAgreementInline bool
+ ShowUserNotifications bool
+ SiteName string
+ Theme string
+ UserProfileFormFields map[string]struct {
+ Type string
+ FormFieldTitle string
+ FormFieldDescription string
+ Required bool
+ }
+ UserProfileFormMessage string
+ VocabularyURL string
+ }
}
type Services struct {
- Controller Service
- DispatchCloud Service
- Health Service
- Keepbalance Service
- Keepproxy Service
- Keepstore Service
- Nodemanager Service
- RailsAPI Service
- WebDAV Service
- Websocket Service
- Workbench1 Service
- Workbench2 Service
+ Composer Service
+ Controller Service
+ DispatchCloud Service
+ GitHTTP Service
+ GitSSH Service
+ Health Service
+ Keepbalance Service
+ Keepproxy Service
+ Keepstore Service
+ Nodemanager Service
+ RailsAPI Service
+ SSO Service
+ WebDAVDownload Service
+ WebDAV Service
+ WebShell Service
+ Websocket Service
+ Workbench1 Service
+ Workbench2 Service
}
type Service struct {
- InternalURLs map[URL]ServiceInstance `json:",omitempty"`
+ InternalURLs map[URL]ServiceInstance
ExternalURL URL
}
type ServiceInstance struct{}
-type SystemLogs struct {
- LogLevel string
- Format string
- MaxRequestLogParamsSize int
-}
-
type PostgreSQL struct {
Connection PostgreSQLConnection
ConnectionPool int
type PostgreSQLConnection map[string]string
type RemoteCluster struct {
- // API endpoint host or host:port; default is {id}.arvadosapi.com
- Host string
- // Perform a proxy request when a local client requests an
- // object belonging to this remote.
- Proxy bool
- // Scheme, default "https". Can be set to "http" for testing.
- Scheme string
- // Disable TLS verify. Can be set to true for testing.
- Insecure bool
+ Host string
+ Proxy bool
+ Scheme string
+ Insecure bool
+ ActivateUsers bool
}
type InstanceType struct {
}
type ContainersConfig struct {
- CloudVMs CloudVMsConfig
- DispatchPrivateKey string
- StaleLockTimeout Duration
+ CloudVMs CloudVMsConfig
+ DefaultKeepCacheRAM ByteSize
+ DispatchPrivateKey string
+ LogReuseDecisions bool
+ MaxComputeVMs int
+ MaxDispatchAttempts int
+ MaxRetryAttempts int
+ StaleLockTimeout Duration
+ SupportedDockerImageFormats []string
+ UsePreemptibleInstances bool
+
+ JobsAPI struct {
+ Enable string
+ GitInternalDir string
+ DefaultDockerImage string
+ CrunchJobWrapper string
+ CrunchJobUser string
+ CrunchRefreshTrigger string
+ ReuseJobIfOutputsDiffer bool
+ }
+ Logging struct {
+ MaxAge Duration
+ LogBytesPerEvent int
+ LogSecondsBetweenEvents int
+ LogThrottlePeriod Duration
+ LogThrottleBytes int
+ LogThrottleLines int
+ LimitLogBytesPerJob int
+ LogPartialLineThrottlePeriod Duration
+ LogUpdatePeriod Duration
+ LogUpdateSize ByteSize
+ }
+ SLURM struct {
+ Managed struct {
+ DNSServerConfDir string
+ DNSServerConfTemplate string
+ DNSServerReloadCommand string
+ DNSServerUpdateCommand string
+ ComputeNodeDomain string
+ ComputeNodeNameservers []string
+ AssignNodeHostname string
+ }
+ }
}
type CloudVMsConfig struct {
ServiceNameKeepstore: svcs.Keepstore,
}
}
-
-type TLS struct {
- Certificate string
- Key string
- Insecure bool
-}
if data[0] == '"' {
return d.Set(string(data[1 : len(data)-1]))
}
- return fmt.Errorf("duration must be given as a string like \"600s\" or \"1h30m\"")
+ // Mimic error message returned by ParseDuration for a number
+ // without units.
+ return fmt.Errorf("missing unit in duration %s", data)
}
// MarshalJSON implements json.Marshaler.
c.Check(string(buf), check.Equals, `"`+trial.out+`"`)
}
}
+
+func (s *DurationSuite) TestUnmarshalJSON(c *check.C) {
+ var d struct {
+ D Duration
+ }
+ err := json.Unmarshal([]byte(`{"D":1.234}`), &d)
+ c.Check(err, check.ErrorMatches, `missing unit in duration 1.234`)
+ err = json.Unmarshal([]byte(`{"D":"1.234"}`), &d)
+ c.Check(err, check.ErrorMatches, `.*missing unit in duration 1.234`)
+ err = json.Unmarshal([]byte(`{"D":"1"}`), &d)
+ c.Check(err, check.ErrorMatches, `.*missing unit in duration 1`)
+ err = json.Unmarshal([]byte(`{"D":"foobar"}`), &d)
+ c.Check(err, check.ErrorMatches, `.*invalid duration foobar`)
+ err = json.Unmarshal([]byte(`{"D":"60s"}`), &d)
+ c.Check(err, check.IsNil)
+ c.Check(d.D.Duration(), check.Equals, time.Minute)
+}
if cfg[k].is_a? Integer
cfg[k] = cfg[k].seconds
elsif cfg[k].is_a? String
- cfg[k] = ConfigLoader.parse_duration cfg[k]
+ cfg[k] = ConfigLoader.parse_duration(cfg[k], cfgkey: cfgkey)
end
end
cfg[k] = URI(cfg[k])
end
+ if cfgtype == Integer && cfg[k].is_a?(String)
+ v = cfg[k].sub(/B\s*$/, '')
+ if mt = /(-?\d*\.?\d+)\s*([KMGTPE]i?)$/.match(v)
+ if mt[1].index('.')
+ v = mt[1].to_f
+ else
+ v = mt[1].to_i
+ end
+ cfg[k] = v * {
+ 'K' => 1000,
+ 'Ki' => 1 << 10,
+ 'M' => 1000000,
+ 'Mi' => 1 << 20,
+ "G" => 1000000000,
+ "Gi" => 1 << 30,
+ "T" => 1000000000000,
+ "Ti" => 1 << 40,
+ "P" => 1000000000000000,
+ "Pi" => 1 << 50,
+ "E" => 1000000000000000000,
+ "Ei" => 1 << 60,
+ }[mt[2]]
+ end
+ end
+
if !cfg[k].is_a? cfgtype
raise "#{cfgkey} expected #{cfgtype} but was #{cfg[k].class}"
end
end
end
- def self.parse_duration durstr
- duration_re = /(\d+(\.\d+)?)(s|m|h)/
+ def self.parse_duration durstr, cfgkey:
+ duration_re = /-?(\d+(\.\d+)?)(s|m|h)/
dursec = 0
while durstr != ""
mt = duration_re.match durstr
if !mt
- raise "#{cfgkey} not a valid duration: '#{cfg[k]}', accepted suffixes are s, m, h"
+ raise "#{cfgkey} not a valid duration: '#{durstr}', accepted suffixes are s, m, h"
end
multiplier = {s: 1, m: 60, h: 3600}
dursec += (Float(mt[1]) * multiplier[mt[3].to_sym])
projects / arvados.git / commitdiff