X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/42a0609a6e287a82ed565413c7392d40141388ae..3b4bb3d393adc3bd3ddfb4442a65087275a5c5c3:/sdk/go/arvados/config.go diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go index b0c7069cd9..dbd9f71099 100644 --- a/sdk/go/arvados/config.go +++ b/sdk/go/arvados/config.go @@ -5,16 +5,26 @@ package arvados import ( + "encoding/json" + "errors" "fmt" + "net/url" "os" - "git.curoverse.com/arvados.git/sdk/go/config" + "git.arvados.org/arvados.git/sdk/go/config" ) -const DefaultConfigFile = "/etc/arvados/config.yml" +var DefaultConfigFile = func() string { + if path := os.Getenv("ARVADOS_CONFIG"); path != "" { + return path + } else { + return "/etc/arvados/config.yml" + } +}() type Config struct { - Clusters map[string]Cluster + Clusters map[string]Cluster + AutoReloadConfig bool } // GetConfig returns the current system config, loading it from @@ -48,73 +58,535 @@ func (sc *Config) GetCluster(clusterID string) (*Cluster, error) { } } +type WebDAVCacheConfig struct { + TTL Duration + UUIDTTL Duration + MaxBlockEntries int + MaxCollectionEntries int + MaxCollectionBytes int64 + MaxPermissionEntries int + MaxUUIDEntries int +} + type Cluster struct { ClusterID string `json:"-"` ManagementToken string - SystemNodes map[string]SystemNode - InstanceTypes []InstanceType + SystemRootToken string + Services Services + InstanceTypes InstanceTypeMap + Containers ContainersConfig + RemoteClusters map[string]RemoteCluster + PostgreSQL PostgreSQL + + API struct { + AsyncPermissionsUpdateInterval Duration + DisabledAPIs StringSet + MaxIndexDatabaseRead int + MaxItemsPerResponse int + MaxConcurrentRequests int + MaxKeepBlobBuffers int + MaxRequestAmplification int + MaxRequestSize int + RailsSessionSecretToken string + RequestTimeout Duration + SendTimeout Duration + WebsocketClientEventQueue int + WebsocketServerEventQueue int + KeepServiceRequestTimeout Duration + } + AuditLogs struct { + MaxAge Duration + MaxDeleteBatch int + UnloggedAttributes StringSet + } + Collections struct { + BlobSigning bool + BlobSigningKey string + BlobSigningTTL Duration + BlobTrash bool + BlobTrashLifetime Duration + BlobTrashCheckInterval Duration + BlobTrashConcurrency int + BlobDeleteConcurrency int + BlobReplicateConcurrency int + CollectionVersioning bool + DefaultTrashLifetime Duration + DefaultReplication int + ManagedProperties map[string]struct { + Value interface{} + Function string + Protected bool + } + PreserveVersionIfIdle Duration + TrashSweepInterval Duration + TrustAllContent bool + ForwardSlashNameSubstitution string + + BlobMissingReport string + BalancePeriod Duration + BalanceCollectionBatch int + BalanceCollectionBuffers int + + WebDAVCache WebDAVCacheConfig + } + Git struct { + GitCommand string + GitoliteHome string + Repositories string + } + Login struct { + LDAP struct { + Enable bool + URL URL + StartTLS bool + InsecureTLS bool + StripDomain string + AppendDomain string + SearchAttribute string + SearchBindUser string + SearchBindPassword string + SearchBase string + SearchFilters string + EmailAttribute string + UsernameAttribute string + } + Google struct { + Enable bool + ClientID string + ClientSecret string + AlternateEmailAddresses bool + } + OpenIDConnect struct { + Enable bool + Issuer string + ClientID string + ClientSecret string + } + PAM struct { + Enable bool + Service string + DefaultEmailDomain string + } + SSO struct { + Enable bool + ProviderAppID string + ProviderAppSecret string + } + LoginCluster string + RemoteTokenRefresh Duration + } + Mail struct { + MailchimpAPIKey string + MailchimpListID string + SendUserSetupNotificationEmail bool + IssueReporterEmailFrom string + IssueReporterEmailTo string + SupportEmailAddress string + EmailFrom string + } + SystemLogs struct { + LogLevel string + Format string + MaxRequestLogParamsSize int + } + TLS struct { + Certificate string + Key string + Insecure bool + } + Users struct { + AnonymousUserToken string + AdminNotifierEmailFrom string + AutoAdminFirstUser bool + AutoAdminUserWithEmail string + AutoSetupNewUsers bool + AutoSetupNewUsersWithRepository bool + AutoSetupNewUsersWithVmUUID string + AutoSetupUsernameBlacklist StringSet + EmailSubjectPrefix string + NewInactiveUserNotificationRecipients StringSet + NewUserNotificationRecipients StringSet + NewUsersAreActive bool + UserNotifierEmailFrom string + UserProfileNotificationAddress string + PreferDomainForUsername string + } + Volumes map[string]Volume + Workbench struct { + ActivationContactLink string + APIClientConnectTimeout Duration + APIClientReceiveTimeout Duration + APIResponseCompression bool + ApplicationMimetypesWithViewIcon StringSet + ArvadosDocsite string + ArvadosPublicDataDocURL string + DefaultOpenIdPrefix string + EnableGettingStartedPopup bool + EnablePublicProjectsPage bool + FileViewersConfigURL string + LogViewerMaxBytes ByteSize + MultiSiteSearch string + ProfilingEnabled bool + Repositories bool + RepositoryCache string + RunningJobLogRecordsToFetch int + SecretKeyBase string + ShowRecentCollectionsOnDashboard bool + ShowUserAgreementInline bool + ShowUserNotifications bool + SiteName string + Theme string + UserProfileFormFields map[string]struct { + Type string + FormFieldTitle string + FormFieldDescription string + Required bool + Position int + Options map[string]struct{} + } + UserProfileFormMessage string + VocabularyURL string + WelcomePageHTML string + InactivePageHTML string + SSHHelpPageHTML string + SSHHelpHostSuffix string + } + + ForceLegacyAPI14 bool } -type InstanceType struct { - Name string - ProviderType string - VCPUs int - RAM int64 - Scratch int64 - Price float64 - Preemptable bool -} - -// GetThisSystemNode returns a SystemNode for the node we're running -// on right now. -func (cc *Cluster) GetThisSystemNode() (*SystemNode, error) { - hostname, err := os.Hostname() - if err != nil { - return nil, err +type Volume struct { + AccessViaHosts map[URL]VolumeAccess + ReadOnly bool + Replication int + StorageClasses map[string]bool + Driver string + DriverParameters json.RawMessage +} + +type S3VolumeDriverParameters struct { + IAMRole string + AccessKey string + SecretKey string + Endpoint string + Region string + Bucket string + LocationConstraint bool + V2Signature bool + IndexPageSize int + ConnectTimeout Duration + ReadTimeout Duration + RaceWindow Duration + UnsafeDelete bool +} + +type AzureVolumeDriverParameters struct { + StorageAccountName string + StorageAccountKey string + StorageBaseURL string + ContainerName string + RequestTimeout Duration + ListBlobsRetryDelay Duration + ListBlobsMaxAttempts int +} + +type DirectoryVolumeDriverParameters struct { + Root string + Serialize bool +} + +type VolumeAccess struct { + ReadOnly bool +} + +type Services struct { + Composer Service + Controller Service + DispatchCloud Service + GitHTTP Service + GitSSH Service + Health Service + Keepbalance Service + Keepproxy Service + Keepstore Service + Nodemanager Service + RailsAPI Service + SSO Service + WebDAVDownload Service + WebDAV Service + WebShell Service + Websocket Service + Workbench1 Service + Workbench2 Service +} + +type Service struct { + InternalURLs map[URL]ServiceInstance + ExternalURL URL +} + +// URL is a url.URL that is also usable as a JSON key/value. +type URL url.URL + +// UnmarshalText implements encoding.TextUnmarshaler so URL can be +// used as a JSON key/value. +func (su *URL) UnmarshalText(text []byte) error { + u, err := url.Parse(string(text)) + if err == nil { + *su = URL(*u) + if su.Path == "" && su.Host != "" { + // http://example really means http://example/ + su.Path = "/" + } } - return cc.GetSystemNode(hostname) + return err +} + +func (su URL) MarshalText() ([]byte, error) { + return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil +} + +func (su URL) String() string { + return (*url.URL)(&su).String() } -// GetSystemNode returns a SystemNode for the given hostname. An error -// is returned if the appropriate configuration can't be determined -// (e.g., this does not appear to be a system node). -func (cc *Cluster) GetSystemNode(node string) (*SystemNode, error) { - if cfg, ok := cc.SystemNodes[node]; ok { - return &cfg, nil +type ServiceInstance struct { + Rendezvous string `json:",omitempty"` +} + +type PostgreSQL struct { + Connection PostgreSQLConnection + ConnectionPool int +} + +type PostgreSQLConnection map[string]string + +type RemoteCluster struct { + Host string + Proxy bool + Scheme string + Insecure bool + ActivateUsers bool +} + +type InstanceType struct { + Name string + ProviderType string + VCPUs int + RAM ByteSize + Scratch ByteSize + IncludedScratch ByteSize + AddedScratch ByteSize + Price float64 + Preemptible bool +} + +type ContainersConfig struct { + CloudVMs CloudVMsConfig + CrunchRunCommand string + CrunchRunArgumentsList []string + DefaultKeepCacheRAM ByteSize + DispatchPrivateKey string + LogReuseDecisions bool + MaxComputeVMs int + MaxDispatchAttempts int + MaxRetryAttempts int + MinRetryPeriod Duration + ReserveExtraRAM ByteSize + StaleLockTimeout Duration + SupportedDockerImageFormats StringSet + UsePreemptibleInstances bool + + JobsAPI struct { + Enable string + GitInternalDir string + } + Logging struct { + MaxAge Duration + LogBytesPerEvent int + LogSecondsBetweenEvents Duration + LogThrottlePeriod Duration + LogThrottleBytes int + LogThrottleLines int + LimitLogBytesPerJob int + LogPartialLineThrottlePeriod Duration + LogUpdatePeriod Duration + LogUpdateSize ByteSize } - // If node is not listed, but "*" gives a default system node - // config, use the default config. - if cfg, ok := cc.SystemNodes["*"]; ok { - return &cfg, nil + SLURM struct { + PrioritySpread int64 + SbatchArgumentsList []string + SbatchEnvironmentVariables map[string]string + Managed struct { + DNSServerConfDir string + DNSServerConfTemplate string + DNSServerReloadCommand string + DNSServerUpdateCommand string + ComputeNodeDomain string + ComputeNodeNameservers StringSet + AssignNodeHostname string + } } - return nil, fmt.Errorf("config does not provision host %q as a system node", node) } -type SystemNode struct { - Health SystemServiceInstance `json:"arvados-health"` - Keepproxy SystemServiceInstance `json:"keepproxy"` - Keepstore SystemServiceInstance `json:"keepstore"` - Keepweb SystemServiceInstance `json:"keep-web"` - Nodemanager SystemServiceInstance `json:"arvados-node-manager"` - RailsAPI SystemServiceInstance `json:"arvados-api-server"` - Websocket SystemServiceInstance `json:"arvados-ws"` - Workbench SystemServiceInstance `json:"arvados-workbench"` +type CloudVMsConfig struct { + Enable bool + + BootProbeCommand string + DeployRunnerBinary string + ImageID string + MaxCloudOpsPerSecond int + MaxProbesPerSecond int + PollInterval Duration + ProbeInterval Duration + SSHPort string + SyncInterval Duration + TimeoutBooting Duration + TimeoutIdle Duration + TimeoutProbe Duration + TimeoutShutdown Duration + TimeoutSignal Duration + TimeoutTERM Duration + ResourceTags map[string]string + TagKeyPrefix string + + Driver string + DriverParameters json.RawMessage } -// ServicePorts returns the configured listening address (or "" if -// disabled) for each service on the node. -func (sn *SystemNode) ServicePorts() map[string]string { - return map[string]string{ - "arvados-api-server": sn.RailsAPI.Listen, - "arvados-node-manager": sn.Nodemanager.Listen, - "arvados-workbench": sn.Workbench.Listen, - "arvados-ws": sn.Websocket.Listen, - "keep-web": sn.Keepweb.Listen, - "keepproxy": sn.Keepproxy.Listen, - "keepstore": sn.Keepstore.Listen, +type InstanceTypeMap map[string]InstanceType + +var errDuplicateInstanceTypeName = errors.New("duplicate instance type name") + +// UnmarshalJSON handles old config files that provide an array of +// instance types instead of a hash. +func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error { + fixup := func(t InstanceType) (InstanceType, error) { + if t.ProviderType == "" { + t.ProviderType = t.Name + } + if t.Scratch == 0 { + t.Scratch = t.IncludedScratch + t.AddedScratch + } else if t.AddedScratch == 0 { + t.AddedScratch = t.Scratch - t.IncludedScratch + } else if t.IncludedScratch == 0 { + t.IncludedScratch = t.Scratch - t.AddedScratch + } + + if t.Scratch != (t.IncludedScratch + t.AddedScratch) { + return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name) + } + return t, nil } + + if len(data) > 0 && data[0] == '[' { + var arr []InstanceType + err := json.Unmarshal(data, &arr) + if err != nil { + return err + } + if len(arr) == 0 { + *it = nil + return nil + } + *it = make(map[string]InstanceType, len(arr)) + for _, t := range arr { + if _, ok := (*it)[t.Name]; ok { + return errDuplicateInstanceTypeName + } + t, err := fixup(t) + if err != nil { + return err + } + (*it)[t.Name] = t + } + return nil + } + var hash map[string]InstanceType + err := json.Unmarshal(data, &hash) + if err != nil { + return err + } + // Fill in Name field (and ProviderType field, if not + // specified) using hash key. + *it = InstanceTypeMap(hash) + for name, t := range *it { + t.Name = name + t, err := fixup(t) + if err != nil { + return err + } + (*it)[name] = t + } + return nil } -type SystemServiceInstance struct { - Listen string +type StringSet map[string]struct{} + +// UnmarshalJSON handles old config files that provide an array of +// instance types instead of a hash. +func (ss *StringSet) UnmarshalJSON(data []byte) error { + if len(data) > 0 && data[0] == '[' { + var arr []string + err := json.Unmarshal(data, &arr) + if err != nil { + return err + } + if len(arr) == 0 { + *ss = nil + return nil + } + *ss = make(map[string]struct{}, len(arr)) + for _, t := range arr { + (*ss)[t] = struct{}{} + } + return nil + } + var hash map[string]struct{} + err := json.Unmarshal(data, &hash) + if err != nil { + return err + } + *ss = make(map[string]struct{}, len(hash)) + for t, _ := range hash { + (*ss)[t] = struct{}{} + } + + return nil +} + +type ServiceName string + +const ( + ServiceNameRailsAPI ServiceName = "arvados-api-server" + ServiceNameController ServiceName = "arvados-controller" + ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud" + ServiceNameHealth ServiceName = "arvados-health" + ServiceNameNodemanager ServiceName = "arvados-node-manager" + ServiceNameWorkbench1 ServiceName = "arvados-workbench1" + ServiceNameWorkbench2 ServiceName = "arvados-workbench2" + ServiceNameWebsocket ServiceName = "arvados-ws" + ServiceNameKeepbalance ServiceName = "keep-balance" + ServiceNameKeepweb ServiceName = "keep-web" + ServiceNameKeepproxy ServiceName = "keepproxy" + ServiceNameKeepstore ServiceName = "keepstore" +) + +// Map returns all services as a map, suitable for iterating over all +// services or looking up a service by name. +func (svcs Services) Map() map[ServiceName]Service { + return map[ServiceName]Service{ + ServiceNameRailsAPI: svcs.RailsAPI, + ServiceNameController: svcs.Controller, + ServiceNameDispatchCloud: svcs.DispatchCloud, + ServiceNameHealth: svcs.Health, + ServiceNameNodemanager: svcs.Nodemanager, + ServiceNameWorkbench1: svcs.Workbench1, + ServiceNameWorkbench2: svcs.Workbench2, + ServiceNameWebsocket: svcs.Websocket, + ServiceNameKeepbalance: svcs.Keepbalance, + ServiceNameKeepweb: svcs.WebDAV, + ServiceNameKeepproxy: svcs.Keepproxy, + ServiceNameKeepstore: svcs.Keepstore, + } }