Services:
RailsAPI:
InternalURLs: {}
- GitHTTP:
- InternalURLs: {}
- ExternalURL: ""
- Keepstore:
- InternalURLs: {}
+ ExternalURL: "-"
Controller:
InternalURLs: {}
ExternalURL: ""
ExternalURL: ""
Keepbalance:
InternalURLs: {}
+ ExternalURL: "-"
GitHTTP:
InternalURLs: {}
ExternalURL: ""
ExternalURL: ""
DispatchCloud:
InternalURLs: {}
+ ExternalURL: "-"
SSO:
ExternalURL: ""
Keepproxy:
ExternalURL: ""
Keepstore:
InternalURLs: {}
+ ExternalURL: "-"
Composer:
ExternalURL: ""
WebShell:
ExternalURL: ""
Workbench2:
ExternalURL: ""
+ Nodemanager:
+ InternalURLs: {}
+ ExternalURL: "-"
+ Health:
+ InternalURLs: {}
+ ExternalURL: "-"
+
PostgreSQL:
# max concurrent connections per arvados server daemon
ConnectionPool: 32
# site secret. It should be at least 50 characters.
RailsSessionSecretToken: ""
+ # Maximum wall clock time to spend handling an incoming request.
+ RequestTimeout: 5m
+
Users:
# Config parameters to automatically setup new users. If enabled,
# this users will be able to self-activate. Enable this if you want
UnloggedAttributes: []
SystemLogs:
+
+ # Logging threshold: panic, fatal, error, warn, info, debug, or
+ # trace
+ LogLevel: info
+
+ # Logging format: json or text
+ Format: json
+
# Maximum characters of (JSON-encoded) query parameters to include
# in each request log entry. When params exceed this size, they will
# be JSON-encoded, truncated to this size, and logged as
Repositories: /var/lib/arvados/git/repositories
TLS:
+ Certificate: ""
+ Key: ""
Insecure: false
Containers:
# troubleshooting purposes.
LogReuseDecisions: false
+ # PEM encoded SSH key (RSA, DSA, or ECDSA) used by the
+ # (experimental) cloud dispatcher for executing containers on
+ # worker VMs. Begins with "-----BEGIN RSA PRIVATE KEY-----\n"
+ # and ends with "\n-----END RSA PRIVATE KEY-----\n".
+ DispatchPrivateKey: none
+
+ # Maximum time to wait for workers to come up before abandoning
+ # stale locks from a previous dispatch process.
+ StaleLockTimeout: 1m
+
Logging:
# When you run the db:delete_old_container_logs task, it will find
# containers that have been finished for at least this many seconds,
# original job reuse behavior, and is still the default).
ReuseJobIfOutputsDiffer: false
+ CloudVMs:
+ # Enable the cloud scheduler (experimental).
+ Enable: false
+
+ # Name/number of port where workers' SSH services listen.
+ SSHPort: "22"
+
+ # Interval between queue polls.
+ PollInterval: 10s
+
+ # Shell command to execute on each worker to determine whether
+ # the worker is booted and ready to run containers. It should
+ # exit zero if the worker is ready.
+ BootProbeCommand: "docker ps"
+
+ # Minimum interval between consecutive probes to a single
+ # worker.
+ ProbeInterval: 10s
+
+ # Maximum probes per second, across all workers in a pool.
+ MaxProbesPerSecond: 10
+
+ # Time before repeating SIGTERM when killing a container.
+ TimeoutSignal: 5s
+
+ # Time to give up on SIGTERM and write off the worker.
+ TimeoutTERM: 2m
+
+ # Maximum create/destroy-instance operations per second (0 =
+ # unlimited).
+ MaxCloudOpsPerSecond: 0
+
+ # Interval between cloud provider syncs/updates ("list all
+ # instances").
+ SyncInterval: 1m
+
+ # Time to leave an idle worker running (in case new containers
+ # appear in the queue that it can run) before shutting it
+ # down.
+ TimeoutIdle: 1m
+
+ # Time to wait for a new worker to boot (i.e., pass
+ # BootProbeCommand) before giving up and shutting it down.
+ TimeoutBooting: 10m
+
+ # Maximum time a worker can stay alive with no successful
+ # probes before being automatically shut down.
+ TimeoutProbe: 10m
+
+ # Time after shutting down a worker to retry the
+ # shutdown/destroy operation.
+ TimeoutShutdown: 10s
+
+ # Worker VM image ID.
+ ImageID: ami-01234567890abcdef
+
+ # Cloud driver: "azure" (Microsoft Azure) or "ec2" (Amazon AWS).
+ Driver: ec2
+
+ # Cloud-specific driver parameters.
+ DriverParameters:
+
+ # (ec2) Credentials.
+ AccessKeyID: ""
+ SecretAccessKey: ""
+
+ # (ec2) Instance configuration.
+ SecurityGroupIDs:
+ - ""
+ SubnetID: ""
+ Region: ""
+ EBSVolumeType: gp2
+ AdminUsername: debian
+
+ # (azure) Credentials.
+ SubscriptionID: ""
+ ClientID: ""
+ ClientSecret: ""
+ TenantID: ""
+
+ # (azure) Instance configuration.
+ CloudEnvironment: AzurePublicCloud
+ ResourceGroup: ""
+ Location: centralus
+ Network: ""
+ Subnet: ""
+ StorageAccount: ""
+ BlobContainer: ""
+ DeleteDanglingResourcesAfter: 20s
+ AdminUsername: arvados
+
+ InstanceTypes:
+
+ # Use the instance type name as the key (in place of "SAMPLE" in
+ # this sample entry).
+ SAMPLE:
+ # Cloud provider's instance type. Defaults to the configured type name.
+ ProviderType: ""
+ VCPUs: 1
+ RAM: 128MiB
+ IncludedScratch: 16GB
+ AddedScratch: 0
+ Price: 0.1
+ Preemptible: false
+
Mail:
MailchimpAPIKey: ""
MailchimpListID: ""
EmailFrom: ""
RemoteClusters:
"*":
+ Host: ""
Proxy: false
+ Scheme: https
+ Insecure: false
ActivateUsers: false
SAMPLE:
Host: sample.arvadosapi.com
Services:
RailsAPI:
InternalURLs: {}
- GitHTTP:
- InternalURLs: {}
- ExternalURL: ""
- Keepstore:
- InternalURLs: {}
+ ExternalURL: "-"
Controller:
InternalURLs: {}
ExternalURL: ""
ExternalURL: ""
Keepbalance:
InternalURLs: {}
+ ExternalURL: "-"
GitHTTP:
InternalURLs: {}
ExternalURL: ""
ExternalURL: ""
DispatchCloud:
InternalURLs: {}
+ ExternalURL: "-"
SSO:
ExternalURL: ""
Keepproxy:
ExternalURL: ""
Keepstore:
InternalURLs: {}
+ ExternalURL: "-"
Composer:
ExternalURL: ""
WebShell:
ExternalURL: ""
Workbench2:
ExternalURL: ""
+ Nodemanager:
+ InternalURLs: {}
+ ExternalURL: "-"
+ Health:
+ InternalURLs: {}
+ ExternalURL: "-"
+
PostgreSQL:
# max concurrent connections per arvados server daemon
ConnectionPool: 32
# site secret. It should be at least 50 characters.
RailsSessionSecretToken: ""
+ # Maximum wall clock time to spend handling an incoming request.
+ RequestTimeout: 5m
+
Users:
# Config parameters to automatically setup new users. If enabled,
# this users will be able to self-activate. Enable this if you want
UnloggedAttributes: []
SystemLogs:
+
+ # Logging threshold: panic, fatal, error, warn, info, debug, or
+ # trace
+ LogLevel: info
+
+ # Logging format: json or text
+ Format: json
+
# Maximum characters of (JSON-encoded) query parameters to include
# in each request log entry. When params exceed this size, they will
# be JSON-encoded, truncated to this size, and logged as
Repositories: /var/lib/arvados/git/repositories
TLS:
+ Certificate: ""
+ Key: ""
Insecure: false
Containers:
# troubleshooting purposes.
LogReuseDecisions: false
+ # PEM encoded SSH key (RSA, DSA, or ECDSA) used by the
+ # (experimental) cloud dispatcher for executing containers on
+ # worker VMs. Begins with "-----BEGIN RSA PRIVATE KEY-----\n"
+ # and ends with "\n-----END RSA PRIVATE KEY-----\n".
+ DispatchPrivateKey: none
+
+ # Maximum time to wait for workers to come up before abandoning
+ # stale locks from a previous dispatch process.
+ StaleLockTimeout: 1m
+
Logging:
# When you run the db:delete_old_container_logs task, it will find
# containers that have been finished for at least this many seconds,
# original job reuse behavior, and is still the default).
ReuseJobIfOutputsDiffer: false
+ CloudVMs:
+ # Enable the cloud scheduler (experimental).
+ Enable: false
+
+ # Name/number of port where workers' SSH services listen.
+ SSHPort: "22"
+
+ # Interval between queue polls.
+ PollInterval: 10s
+
+ # Shell command to execute on each worker to determine whether
+ # the worker is booted and ready to run containers. It should
+ # exit zero if the worker is ready.
+ BootProbeCommand: "docker ps"
+
+ # Minimum interval between consecutive probes to a single
+ # worker.
+ ProbeInterval: 10s
+
+ # Maximum probes per second, across all workers in a pool.
+ MaxProbesPerSecond: 10
+
+ # Time before repeating SIGTERM when killing a container.
+ TimeoutSignal: 5s
+
+ # Time to give up on SIGTERM and write off the worker.
+ TimeoutTERM: 2m
+
+ # Maximum create/destroy-instance operations per second (0 =
+ # unlimited).
+ MaxCloudOpsPerSecond: 0
+
+ # Interval between cloud provider syncs/updates ("list all
+ # instances").
+ SyncInterval: 1m
+
+ # Time to leave an idle worker running (in case new containers
+ # appear in the queue that it can run) before shutting it
+ # down.
+ TimeoutIdle: 1m
+
+ # Time to wait for a new worker to boot (i.e., pass
+ # BootProbeCommand) before giving up and shutting it down.
+ TimeoutBooting: 10m
+
+ # Maximum time a worker can stay alive with no successful
+ # probes before being automatically shut down.
+ TimeoutProbe: 10m
+
+ # Time after shutting down a worker to retry the
+ # shutdown/destroy operation.
+ TimeoutShutdown: 10s
+
+ # Worker VM image ID.
+ ImageID: ami-01234567890abcdef
+
+ # Cloud driver: "azure" (Microsoft Azure) or "ec2" (Amazon AWS).
+ Driver: ec2
+
+ # Cloud-specific driver parameters.
+ DriverParameters:
+
+ # (ec2) Credentials.
+ AccessKeyID: ""
+ SecretAccessKey: ""
+
+ # (ec2) Instance configuration.
+ SecurityGroupIDs:
+ - ""
+ SubnetID: ""
+ Region: ""
+ EBSVolumeType: gp2
+ AdminUsername: debian
+
+ # (azure) Credentials.
+ SubscriptionID: ""
+ ClientID: ""
+ ClientSecret: ""
+ TenantID: ""
+
+ # (azure) Instance configuration.
+ CloudEnvironment: AzurePublicCloud
+ ResourceGroup: ""
+ Location: centralus
+ Network: ""
+ Subnet: ""
+ StorageAccount: ""
+ BlobContainer: ""
+ DeleteDanglingResourcesAfter: 20s
+ AdminUsername: arvados
+
+ InstanceTypes:
+
+ # Use the instance type name as the key (in place of "SAMPLE" in
+ # this sample entry).
+ SAMPLE:
+ # Cloud provider's instance type. Defaults to the configured type name.
+ ProviderType: ""
+ VCPUs: 1
+ RAM: 128MiB
+ IncludedScratch: 16GB
+ AddedScratch: 0
+ Price: 0.1
+ Preemptible: false
+
Mail:
MailchimpAPIKey: ""
MailchimpListID: ""
EmailFrom: ""
RemoteClusters:
"*":
+ Host: ""
Proxy: false
+ Scheme: https
+ Insecure: false
ActivateUsers: false
SAMPLE:
Host: sample.arvadosapi.com
c.Check(logs, check.HasLen, 2)
}
+func (s *LoadSuite) TestNoWarningsForDumpedConfig(c *check.C) {
+ var logbuf bytes.Buffer
+ logger := logrus.New()
+ logger.Out = &logbuf
+ cfg, err := Load(bytes.NewBufferString(`{"Clusters":{"zzzzz":{}}}`), logger)
+ c.Assert(err, check.IsNil)
+ yaml, err := yaml.Marshal(cfg)
+ c.Assert(err, check.IsNil)
+ cfgDumped, err := Load(bytes.NewBuffer(yaml), logger)
+ c.Assert(err, check.IsNil)
+ c.Check(cfg, check.DeepEquals, cfgDumped)
+ c.Check(logbuf.String(), check.Equals, "")
+}
+
func (s *LoadSuite) TestPostgreSQLKeyConflict(c *check.C) {
_, err := Load(bytes.NewBufferString(`
Clusters:
req.URL.Path = strings.Replace(req.URL.Path, "//", "/", -1)
}
}
- if h.Cluster.HTTPRequestTimeout > 0 {
- ctx, cancel := context.WithDeadline(req.Context(), time.Now().Add(time.Duration(h.Cluster.HTTPRequestTimeout)))
+ if h.Cluster.API.RequestTimeout > 0 {
+ ctx, cancel := context.WithDeadline(req.Context(), time.Now().Add(time.Duration(h.Cluster.API.RequestTimeout)))
req = req.WithContext(ctx)
defer cancel()
}
}
func (s *HandlerSuite) TestRequestTimeout(c *check.C) {
- s.cluster.HTTPRequestTimeout = arvados.Duration(time.Nanosecond)
+ s.cluster.API.RequestTimeout = arvados.Duration(time.Nanosecond)
req := httptest.NewRequest("GET", "/discovery/v1/apis/arvados/v1/rest", nil)
resp := httptest.NewRecorder()
s.handler.ServeHTTP(resp, req)
if err != nil {
return 1
}
- log = ctxlog.New(stderr, cluster.Logging.Format, cluster.Logging.Level).WithFields(logrus.Fields{
+ log = ctxlog.New(stderr, cluster.SystemLogs.Format, cluster.SystemLogs.LogLevel).WithFields(logrus.Fields{
"PID": os.Getpid(),
})
ctx := ctxlog.Context(c.ctx, log)
type API struct {
MaxItemsPerResponse int
MaxRequestAmplification int
+ RequestTimeout Duration
}
type Cluster struct {
- ClusterID string `json:"-"`
- ManagementToken string
- SystemRootToken string
- Services Services
- NodeProfiles map[string]NodeProfile
- InstanceTypes InstanceTypeMap
- CloudVMs CloudVMs
- Dispatch Dispatch
- HTTPRequestTimeout Duration
- RemoteClusters map[string]RemoteCluster
- PostgreSQL PostgreSQL
- API API
- Logging Logging
- TLS TLS
+ ClusterID string `json:"-"`
+ ManagementToken string
+ SystemRootToken string
+ Services Services
+ NodeProfiles map[string]NodeProfile
+ InstanceTypes InstanceTypeMap
+ Containers ContainersConfig
+ RemoteClusters map[string]RemoteCluster
+ PostgreSQL PostgreSQL
+ API API
+ SystemLogs SystemLogs
+ TLS TLS
}
type Services struct {
}
type Service struct {
- InternalURLs map[URL]ServiceInstance
+ InternalURLs map[URL]ServiceInstance `json:",omitempty"`
ExternalURL URL
}
type ServiceInstance struct{}
-type Logging struct {
- Level string
- Format string
+type SystemLogs struct {
+ LogLevel string
+ Format string
+ MaxRequestLogParamsSize int
}
type PostgreSQL struct {
Preemptible bool
}
-type Dispatch struct {
- // PEM encoded SSH key (RSA, DSA, or ECDSA) able to log in to
- // cloud VMs.
- PrivateKey string
-
- // Max time for workers to come up before abandoning stale
- // locks from previous run
- StaleLockTimeout Duration
-
- // Interval between queue polls
- PollInterval Duration
-
- // Interval between probes to each worker
- ProbeInterval Duration
-
- // Maximum total worker probes per second
- MaxProbesPerSecond int
-
- // Time before repeating SIGTERM when killing a container
- TimeoutSignal Duration
-
- // Time to give up on SIGTERM and write off the worker
- TimeoutTERM Duration
+type ContainersConfig struct {
+ CloudVMs CloudVMsConfig
+ DispatchPrivateKey string
+ StaleLockTimeout Duration
}
-type CloudVMs struct {
- // Shell command that exits zero IFF the VM is fully booted
- // and ready to run containers, e.g., "mount | grep
- // /encrypted-tmp"
- BootProbeCommand string
-
- // Listening port (name or number) of SSH servers on worker
- // VMs
- SSHPort string
+type CloudVMsConfig struct {
+ Enable bool
- SyncInterval Duration
-
- // Maximum idle time before automatic shutdown
- TimeoutIdle Duration
-
- // Maximum booting time before automatic shutdown
- TimeoutBooting Duration
-
- // Maximum time with no successful probes before automatic shutdown
- TimeoutProbe Duration
-
- // Time after shutdown to retry shutdown
- TimeoutShutdown Duration
-
- // Maximum create/destroy-instance operations per second
+ BootProbeCommand string
+ ImageID string
MaxCloudOpsPerSecond int
-
- ImageID string
+ MaxProbesPerSecond int
+ PollInterval Duration
+ ProbeInterval Duration
+ SSHPort string
+ SyncInterval Duration
+ TimeoutBooting Duration
+ TimeoutIdle Duration
+ TimeoutProbe Duration
+ TimeoutShutdown Duration
+ TimeoutSignal Duration
+ TimeoutTERM Duration
Driver string
DriverParameters json.RawMessage
}
// MarshalJSON implements json.Marshaler.
-func (d *Duration) MarshalJSON() ([]byte, error) {
+func (d Duration) MarshalJSON() ([]byte, error) {
return json.Marshal(d.String())
}
Clusters:
zzzzz:
ManagementToken: e687950a23c3a9bceec28c6223a06c79
- HTTPRequestTimeout: 30s
+ API:
+ RequestTimeout: 30s
PostgreSQL:
ConnectionPool: 32
Connection:
projects / arvados.git / commitdiff