X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/cb13593decb097f501c0a1d64510a653b3233395..d4ed3e6460469f2766e1f1676c538d6c86e000b6:/sdk/go/arvados/config.go diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go index 6b3150c6f0..93f52f3c80 100644 --- a/sdk/go/arvados/config.go +++ b/sdk/go/arvados/config.go @@ -14,7 +14,13 @@ import ( "git.curoverse.com/arvados.git/sdk/go/config" ) -const DefaultConfigFile = "/etc/arvados/config.yml" +var DefaultConfigFile = func() string { + if path := os.Getenv("ARVADOS_CONFIG"); path != "" { + return path + } else { + return "/etc/arvados/config.yml" + } +}() type Config struct { Clusters map[string]Cluster @@ -51,41 +57,162 @@ func (sc *Config) GetCluster(clusterID string) (*Cluster, error) { } } -type API struct { - MaxItemsPerResponse int - MaxRequestAmplification int +type WebDAVCacheConfig struct { + TTL Duration + UUIDTTL Duration + MaxBlockEntries int + MaxCollectionEntries int + MaxCollectionBytes int64 + MaxPermissionEntries int + MaxUUIDEntries int } - type Cluster struct { - ClusterID string `json:"-"` - ManagementToken string - SystemRootToken string - Services Services - NodeProfiles map[string]NodeProfile - InstanceTypes InstanceTypeMap - CloudVMs CloudVMs - Dispatch Dispatch - HTTPRequestTimeout Duration - RemoteClusters map[string]RemoteCluster - PostgreSQL PostgreSQL - API API - Logging Logging - TLS TLS + ClusterID string `json:"-"` + ManagementToken string + SystemRootToken string + Services Services + InstanceTypes InstanceTypeMap + Containers ContainersConfig + RemoteClusters map[string]RemoteCluster + PostgreSQL PostgreSQL + + API struct { + AsyncPermissionsUpdateInterval Duration + DisabledAPIs StringSet + MaxIndexDatabaseRead int + MaxItemsPerResponse int + MaxRequestAmplification int + MaxRequestSize int + RailsSessionSecretToken string + RequestTimeout Duration + SendTimeout Duration + WebsocketClientEventQueue int + WebsocketServerEventQueue int + KeepServiceRequestTimeout Duration + } + AuditLogs struct { + MaxAge Duration + MaxDeleteBatch int + UnloggedAttributes StringSet + } + Collections struct { + BlobSigning bool + BlobSigningKey string + BlobSigningTTL Duration + CollectionVersioning bool + DefaultTrashLifetime Duration + DefaultReplication int + ManagedProperties map[string]struct { + Value interface{} + Function string + Protected bool + } + PreserveVersionIfIdle Duration + TrashSweepInterval Duration + TrustAllContent bool + + WebDAVCache WebDAVCacheConfig + } + Git struct { + Repositories string + } + Login struct { + ProviderAppSecret string + ProviderAppID string + } + Mail struct { + MailchimpAPIKey string + MailchimpListID string + SendUserSetupNotificationEmail bool + IssueReporterEmailFrom string + IssueReporterEmailTo string + SupportEmailAddress string + EmailFrom string + } + SystemLogs struct { + LogLevel string + Format string + MaxRequestLogParamsSize int + } + TLS struct { + Certificate string + Key string + Insecure bool + } + Users struct { + AnonymousUserToken string + AdminNotifierEmailFrom string + AutoAdminFirstUser bool + AutoAdminUserWithEmail string + AutoSetupNewUsers bool + AutoSetupNewUsersWithRepository bool + AutoSetupNewUsersWithVmUUID string + AutoSetupUsernameBlacklist StringSet + EmailSubjectPrefix string + NewInactiveUserNotificationRecipients StringSet + NewUserNotificationRecipients StringSet + NewUsersAreActive bool + UserNotifierEmailFrom string + UserProfileNotificationAddress string + } + Workbench struct { + ActivationContactLink string + APIClientConnectTimeout Duration + APIClientReceiveTimeout Duration + APIResponseCompression bool + ApplicationMimetypesWithViewIcon StringSet + ArvadosDocsite string + ArvadosPublicDataDocURL string + DefaultOpenIdPrefix string + EnableGettingStartedPopup bool + EnablePublicProjectsPage bool + FileViewersConfigURL string + LogViewerMaxBytes ByteSize + MultiSiteSearch string + ProfilingEnabled bool + Repositories bool + RepositoryCache string + RunningJobLogRecordsToFetch int + SecretKeyBase string + ShowRecentCollectionsOnDashboard bool + ShowUserAgreementInline bool + ShowUserNotifications bool + SiteName string + Theme string + UserProfileFormFields map[string]struct { + Type string + FormFieldTitle string + FormFieldDescription string + Required bool + Position int + Options map[string]struct{} + } + UserProfileFormMessage string + VocabularyURL string + } + + EnableBetaController14287 bool } type Services struct { - Controller Service - DispatchCloud Service - Health Service - Keepbalance Service - Keepproxy Service - Keepstore Service - Nodemanager Service - RailsAPI Service - WebDAV Service - Websocket Service - Workbench1 Service - Workbench2 Service + Composer Service + Controller Service + DispatchCloud Service + GitHTTP Service + GitSSH Service + Health Service + Keepbalance Service + Keepproxy Service + Keepstore Service + Nodemanager Service + RailsAPI Service + SSO Service + WebDAVDownload Service + WebDAV Service + WebShell Service + Websocket Service + Workbench1 Service + Workbench2 Service } type Service struct { @@ -112,11 +239,6 @@ func (su URL) MarshalText() ([]byte, error) { type ServiceInstance struct{} -type Logging struct { - Level string - Format string -} - type PostgreSQL struct { Connection PostgreSQLConnection ConnectionPool int @@ -125,15 +247,11 @@ type PostgreSQL struct { type PostgreSQLConnection map[string]string type RemoteCluster struct { - // API endpoint host or host:port; default is {id}.arvadosapi.com - Host string - // Perform a proxy request when a local client requests an - // object belonging to this remote. - Proxy bool - // Scheme, default "https". Can be set to "http" for testing. - Scheme string - // Disable TLS verify. Can be set to true for testing. - Insecure bool + Host string + Proxy bool + Scheme string + Insecure bool + ActivateUsers bool } type InstanceType struct { @@ -148,59 +266,73 @@ type InstanceType struct { Preemptible bool } -type Dispatch struct { - // PEM encoded SSH key (RSA, DSA, or ECDSA) able to log in to - // cloud VMs. - PrivateKey string - - // Max time for workers to come up before abandoning stale - // locks from previous run - StaleLockTimeout Duration - - // Interval between queue polls - PollInterval Duration - - // Interval between probes to each worker - ProbeInterval Duration - - // Maximum total worker probes per second - MaxProbesPerSecond int - - // Time before repeating SIGTERM when killing a container - TimeoutSignal Duration - - // Time to give up on SIGTERM and write off the worker - TimeoutTERM Duration +type ContainersConfig struct { + CloudVMs CloudVMsConfig + CrunchRunCommand string + CrunchRunArgumentsList []string + DefaultKeepCacheRAM ByteSize + DispatchPrivateKey string + LogReuseDecisions bool + MaxComputeVMs int + MaxDispatchAttempts int + MaxRetryAttempts int + MinRetryPeriod Duration + ReserveExtraRAM ByteSize + StaleLockTimeout Duration + SupportedDockerImageFormats StringSet + UsePreemptibleInstances bool + + JobsAPI struct { + Enable string + GitInternalDir string + } + Logging struct { + MaxAge Duration + LogBytesPerEvent int + LogSecondsBetweenEvents int + LogThrottlePeriod Duration + LogThrottleBytes int + LogThrottleLines int + LimitLogBytesPerJob int + LogPartialLineThrottlePeriod Duration + LogUpdatePeriod Duration + LogUpdateSize ByteSize + } + SLURM struct { + PrioritySpread int64 + SbatchArgumentsList []string + SbatchEnvironmentVariables map[string]string + Managed struct { + DNSServerConfDir string + DNSServerConfTemplate string + DNSServerReloadCommand string + DNSServerUpdateCommand string + ComputeNodeDomain string + ComputeNodeNameservers StringSet + AssignNodeHostname string + } + } } -type CloudVMs struct { - // Shell command that exits zero IFF the VM is fully booted - // and ready to run containers, e.g., "mount | grep - // /encrypted-tmp" - BootProbeCommand string +type CloudVMsConfig struct { + Enable bool - // Listening port (name or number) of SSH servers on worker - // VMs - SSHPort string - - SyncInterval Duration - - // Maximum idle time before automatic shutdown - TimeoutIdle Duration - - // Maximum booting time before automatic shutdown - TimeoutBooting Duration - - // Maximum time with no successful probes before automatic shutdown - TimeoutProbe Duration - - // Time after shutdown to retry shutdown - TimeoutShutdown Duration - - // Maximum create/destroy-instance operations per second + BootProbeCommand string + ImageID string MaxCloudOpsPerSecond int - - ImageID string + MaxProbesPerSecond int + PollInterval Duration + ProbeInterval Duration + SSHPort string + SyncInterval Duration + TimeoutBooting Duration + TimeoutIdle Duration + TimeoutProbe Duration + TimeoutShutdown Duration + TimeoutSignal Duration + TimeoutTERM Duration + ResourceTags map[string]string + TagKeyPrefix string Driver string DriverParameters json.RawMessage @@ -264,41 +396,38 @@ func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error { return nil } -// GetNodeProfile returns a NodeProfile for the given hostname. An -// error is returned if the appropriate configuration can't be -// determined (e.g., this does not appear to be a system node). If -// node is empty, use the OS-reported hostname. -func (cc *Cluster) GetNodeProfile(node string) (*NodeProfile, error) { - if node == "" { - hostname, err := os.Hostname() +type StringSet map[string]struct{} + +// UnmarshalJSON handles old config files that provide an array of +// instance types instead of a hash. +func (ss *StringSet) UnmarshalJSON(data []byte) error { + if len(data) > 0 && data[0] == '[' { + var arr []string + err := json.Unmarshal(data, &arr) if err != nil { - return nil, err + return err + } + if len(arr) == 0 { + *ss = nil + return nil + } + *ss = make(map[string]struct{}, len(arr)) + for _, t := range arr { + (*ss)[t] = struct{}{} } - node = hostname + return nil } - if cfg, ok := cc.NodeProfiles[node]; ok { - return &cfg, nil + var hash map[string]struct{} + err := json.Unmarshal(data, &hash) + if err != nil { + return err } - // If node is not listed, but "*" gives a default system node - // config, use the default config. - if cfg, ok := cc.NodeProfiles["*"]; ok { - return &cfg, nil + *ss = make(map[string]struct{}, len(hash)) + for t, _ := range hash { + (*ss)[t] = struct{}{} } - return nil, fmt.Errorf("config does not provision host %q as a system node", node) -} -type NodeProfile struct { - Controller SystemServiceInstance `json:"arvados-controller"` - Health SystemServiceInstance `json:"arvados-health"` - Keepbalance SystemServiceInstance `json:"keep-balance"` - Keepproxy SystemServiceInstance `json:"keepproxy"` - Keepstore SystemServiceInstance `json:"keepstore"` - Keepweb SystemServiceInstance `json:"keep-web"` - Nodemanager SystemServiceInstance `json:"arvados-node-manager"` - DispatchCloud SystemServiceInstance `json:"arvados-dispatch-cloud"` - RailsAPI SystemServiceInstance `json:"arvados-api-server"` - Websocket SystemServiceInstance `json:"arvados-ws"` - Workbench SystemServiceInstance `json:"arvados-workbench"` + return nil } type ServiceName string @@ -307,8 +436,10 @@ const ( ServiceNameRailsAPI ServiceName = "arvados-api-server" ServiceNameController ServiceName = "arvados-controller" ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud" + ServiceNameHealth ServiceName = "arvados-health" ServiceNameNodemanager ServiceName = "arvados-node-manager" - ServiceNameWorkbench ServiceName = "arvados-workbench" + ServiceNameWorkbench1 ServiceName = "arvados-workbench1" + ServiceNameWorkbench2 ServiceName = "arvados-workbench2" ServiceNameWebsocket ServiceName = "arvados-ws" ServiceNameKeepbalance ServiceName = "keep-balance" ServiceNameKeepweb ServiceName = "keep-web" @@ -316,31 +447,21 @@ const ( ServiceNameKeepstore ServiceName = "keepstore" ) -// ServicePorts returns the configured listening address (or "" if -// disabled) for each service on the node. -func (np *NodeProfile) ServicePorts() map[ServiceName]string { - return map[ServiceName]string{ - ServiceNameRailsAPI: np.RailsAPI.Listen, - ServiceNameController: np.Controller.Listen, - ServiceNameDispatchCloud: np.DispatchCloud.Listen, - ServiceNameNodemanager: np.Nodemanager.Listen, - ServiceNameWorkbench: np.Workbench.Listen, - ServiceNameWebsocket: np.Websocket.Listen, - ServiceNameKeepbalance: np.Keepbalance.Listen, - ServiceNameKeepweb: np.Keepweb.Listen, - ServiceNameKeepproxy: np.Keepproxy.Listen, - ServiceNameKeepstore: np.Keepstore.Listen, +// Map returns all services as a map, suitable for iterating over all +// services or looking up a service by name. +func (svcs Services) Map() map[ServiceName]Service { + return map[ServiceName]Service{ + ServiceNameRailsAPI: svcs.RailsAPI, + ServiceNameController: svcs.Controller, + ServiceNameDispatchCloud: svcs.DispatchCloud, + ServiceNameHealth: svcs.Health, + ServiceNameNodemanager: svcs.Nodemanager, + ServiceNameWorkbench1: svcs.Workbench1, + ServiceNameWorkbench2: svcs.Workbench2, + ServiceNameWebsocket: svcs.Websocket, + ServiceNameKeepbalance: svcs.Keepbalance, + ServiceNameKeepweb: svcs.WebDAV, + ServiceNameKeepproxy: svcs.Keepproxy, + ServiceNameKeepstore: svcs.Keepstore, } } - -type SystemServiceInstance struct { - Listen string - TLS bool - Insecure bool -} - -type TLS struct { - Certificate string - Key string - Insecure bool -}