X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/bdc8a7630030494c63fb0426be4c15a93a9a37cb..c9aaa4d329b08c8b1c8cfd566c5282ab57e97f05:/sdk/go/arvados/config.go diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go index 72128a9dcd..ad9a4da03c 100644 --- a/sdk/go/arvados/config.go +++ b/sdk/go/arvados/config.go @@ -11,19 +11,19 @@ import ( "net/url" "os" - "git.curoverse.com/arvados.git/sdk/go/config" + "git.arvados.org/arvados.git/sdk/go/config" ) var DefaultConfigFile = func() string { if path := os.Getenv("ARVADOS_CONFIG"); path != "" { return path - } else { - return "/etc/arvados/config.yml" } + return "/etc/arvados/config.yml" }() type Config struct { - Clusters map[string]Cluster + Clusters map[string]Cluster + AutoReloadConfig bool } // GetConfig returns the current system config, loading it from @@ -49,12 +49,12 @@ func (sc *Config) GetCluster(clusterID string) (*Cluster, error) { } } } - if cc, ok := sc.Clusters[clusterID]; !ok { + cc, ok := sc.Clusters[clusterID] + if !ok { return nil, fmt.Errorf("cluster %q is not configured", clusterID) - } else { - cc.ClusterID = clusterID - return &cc, nil } + cc.ClusterID = clusterID + return &cc, nil } type WebDAVCacheConfig struct { @@ -63,9 +63,20 @@ type WebDAVCacheConfig struct { MaxBlockEntries int MaxCollectionEntries int MaxCollectionBytes int64 - MaxPermissionEntries int MaxUUIDEntries int + MaxSessions int } + +type UploadDownloadPermission struct { + Upload bool + Download bool +} + +type UploadDownloadRolePermissions struct { + User UploadDownloadPermission + Admin UploadDownloadPermission +} + type Cluster struct { ClusterID string `json:"-"` ManagementToken string @@ -85,7 +96,7 @@ type Cluster struct { MaxKeepBlobBuffers int MaxRequestAmplification int MaxRequestSize int - RailsSessionSecretToken string + MaxTokenLifetime Duration RequestTimeout Duration SendTimeout Duration WebsocketClientEventQueue int @@ -115,16 +126,24 @@ type Cluster struct { Function string Protected bool } - PreserveVersionIfIdle Duration - TrashSweepInterval Duration - TrustAllContent bool + PreserveVersionIfIdle Duration + TrashSweepInterval Duration + TrustAllContent bool + ForwardSlashNameSubstitution string + S3FolderObjects bool BlobMissingReport string BalancePeriod Duration BalanceCollectionBatch int BalanceCollectionBuffers int + BalanceTimeout Duration + BalanceUpdateLimit int WebDAVCache WebDAVCacheConfig + + KeepproxyPermission UploadDownloadRolePermissions + WebDAVPermission UploadDownloadRolePermissions + WebDAVLogEvents bool } Git struct { GitCommand string @@ -132,13 +151,54 @@ type Cluster struct { Repositories string } Login struct { - GoogleClientID string - GoogleClientSecret string - GoogleAlternateEmailAddresses bool - ProviderAppID string - ProviderAppSecret string - LoginCluster string - RemoteTokenRefresh Duration + LDAP struct { + Enable bool + URL URL + StartTLS bool + InsecureTLS bool + StripDomain string + AppendDomain string + SearchAttribute string + SearchBindUser string + SearchBindPassword string + SearchBase string + SearchFilters string + EmailAttribute string + UsernameAttribute string + } + Google struct { + Enable bool + ClientID string + ClientSecret string + AlternateEmailAddresses bool + AuthenticationRequestParameters map[string]string + } + OpenIDConnect struct { + Enable bool + Issuer string + ClientID string + ClientSecret string + EmailClaim string + EmailVerifiedClaim string + UsernameClaim string + AcceptAccessToken bool + AcceptAccessTokenScope string + AuthenticationRequestParameters map[string]string + } + PAM struct { + Enable bool + Service string + DefaultEmailDomain string + } + Test struct { + Enable bool + Users map[string]TestUser + } + LoginCluster string + RemoteTokenRefresh Duration + TokenLifetime Duration + TrustedClients map[string]struct{} + IssueTrustedTokens bool } Mail struct { MailchimpAPIKey string @@ -173,11 +233,14 @@ type Cluster struct { NewUserNotificationRecipients StringSet NewUsersAreActive bool UserNotifierEmailFrom string + UserNotifierEmailBcc StringSet UserProfileNotificationAddress string PreferDomainForUsername string + UserSetupMailText string } - Volumes map[string]Volume - Workbench struct { + StorageClasses map[string]StorageClassConfig + Volumes map[string]Volume + Workbench struct { ActivationContactLink string APIClientConnectTimeout Duration APIClientReceiveTimeout Duration @@ -213,9 +276,15 @@ type Cluster struct { VocabularyURL string WelcomePageHTML string InactivePageHTML string + SSHHelpPageHTML string + SSHHelpHostSuffix string + IdleTimeout Duration } +} - EnableBetaController14287 bool +type StorageClassConfig struct { + Default bool + Priority int } type Volume struct { @@ -228,12 +297,15 @@ type Volume struct { } type S3VolumeDriverParameters struct { - AccessKey string - SecretKey string + IAMRole string + AccessKeyID string + SecretAccessKey string Endpoint string Region string Bucket string LocationConstraint bool + V2Signature bool + UseAWSS3v2Driver bool IndexPageSize int ConnectTimeout Duration ReadTimeout Duration @@ -264,15 +336,14 @@ type Services struct { Composer Service Controller Service DispatchCloud Service + DispatchLSF Service GitHTTP Service GitSSH Service Health Service Keepbalance Service Keepproxy Service Keepstore Service - Nodemanager Service RailsAPI Service - SSO Service WebDAVDownload Service WebDAV Service WebShell Service @@ -286,6 +357,11 @@ type Service struct { ExternalURL URL } +type TestUser struct { + Email string + Password string +} + // URL is a url.URL that is also usable as a JSON key/value. type URL url.URL @@ -295,6 +371,10 @@ func (su *URL) UnmarshalText(text []byte) error { u, err := url.Parse(string(text)) if err == nil { *su = URL(*u) + if su.Path == "" && su.Host != "" { + // http://example really means http://example/ + su.Path = "/" + } } return err } @@ -353,6 +433,7 @@ type ContainersConfig struct { StaleLockTimeout Duration SupportedDockerImageFormats StringSet UsePreemptibleInstances bool + RuntimeEngine string JobsAPI struct { Enable string @@ -361,7 +442,7 @@ type ContainersConfig struct { Logging struct { MaxAge Duration LogBytesPerEvent int - LogSecondsBetweenEvents int + LogSecondsBetweenEvents Duration LogThrottlePeriod Duration LogThrottleBytes int LogThrottleLines int @@ -370,6 +451,10 @@ type ContainersConfig struct { LogUpdatePeriod Duration LogUpdateSize ByteSize } + ShellAccess struct { + Admin bool + User bool + } SLURM struct { PrioritySpread int64 SbatchArgumentsList []string @@ -384,27 +469,34 @@ type ContainersConfig struct { AssignNodeHostname string } } + LSF struct { + BsubSudoUser string + BsubArgumentsList []string + } } type CloudVMsConfig struct { Enable bool - BootProbeCommand string - ImageID string - MaxCloudOpsPerSecond int - MaxProbesPerSecond int - PollInterval Duration - ProbeInterval Duration - SSHPort string - SyncInterval Duration - TimeoutBooting Duration - TimeoutIdle Duration - TimeoutProbe Duration - TimeoutShutdown Duration - TimeoutSignal Duration - TimeoutTERM Duration - ResourceTags map[string]string - TagKeyPrefix string + BootProbeCommand string + DeployRunnerBinary string + ImageID string + MaxCloudOpsPerSecond int + MaxProbesPerSecond int + MaxConcurrentInstanceCreateOps int + PollInterval Duration + ProbeInterval Duration + SSHPort string + SyncInterval Duration + TimeoutBooting Duration + TimeoutIdle Duration + TimeoutProbe Duration + TimeoutShutdown Duration + TimeoutSignal Duration + TimeoutStaleRunLock Duration + TimeoutTERM Duration + ResourceTags map[string]string + TagKeyPrefix string Driver string DriverParameters json.RawMessage @@ -417,6 +509,24 @@ var errDuplicateInstanceTypeName = errors.New("duplicate instance type name") // UnmarshalJSON handles old config files that provide an array of // instance types instead of a hash. func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error { + fixup := func(t InstanceType) (InstanceType, error) { + if t.ProviderType == "" { + t.ProviderType = t.Name + } + if t.Scratch == 0 { + t.Scratch = t.IncludedScratch + t.AddedScratch + } else if t.AddedScratch == 0 { + t.AddedScratch = t.Scratch - t.IncludedScratch + } else if t.IncludedScratch == 0 { + t.IncludedScratch = t.Scratch - t.AddedScratch + } + + if t.Scratch != (t.IncludedScratch + t.AddedScratch) { + return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name) + } + return t, nil + } + if len(data) > 0 && data[0] == '[' { var arr []InstanceType err := json.Unmarshal(data, &arr) @@ -432,19 +542,9 @@ func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error { if _, ok := (*it)[t.Name]; ok { return errDuplicateInstanceTypeName } - if t.ProviderType == "" { - t.ProviderType = t.Name - } - if t.Scratch == 0 { - t.Scratch = t.IncludedScratch + t.AddedScratch - } else if t.AddedScratch == 0 { - t.AddedScratch = t.Scratch - t.IncludedScratch - } else if t.IncludedScratch == 0 { - t.IncludedScratch = t.Scratch - t.AddedScratch - } - - if t.Scratch != (t.IncludedScratch + t.AddedScratch) { - return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name) + t, err := fixup(t) + if err != nil { + return err } (*it)[t.Name] = t } @@ -460,8 +560,9 @@ func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error { *it = InstanceTypeMap(hash) for name, t := range *it { t.Name = name - if t.ProviderType == "" { - t.ProviderType = name + t, err := fixup(t) + if err != nil { + return err } (*it)[name] = t } @@ -495,7 +596,7 @@ func (ss *StringSet) UnmarshalJSON(data []byte) error { return err } *ss = make(map[string]struct{}, len(hash)) - for t, _ := range hash { + for t := range hash { (*ss)[t] = struct{}{} } @@ -508,8 +609,8 @@ const ( ServiceNameRailsAPI ServiceName = "arvados-api-server" ServiceNameController ServiceName = "arvados-controller" ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud" + ServiceNameDispatchLSF ServiceName = "arvados-dispatch-lsf" ServiceNameHealth ServiceName = "arvados-health" - ServiceNameNodemanager ServiceName = "arvados-node-manager" ServiceNameWorkbench1 ServiceName = "arvados-workbench1" ServiceNameWorkbench2 ServiceName = "arvados-workbench2" ServiceNameWebsocket ServiceName = "arvados-ws" @@ -526,8 +627,8 @@ func (svcs Services) Map() map[ServiceName]Service { ServiceNameRailsAPI: svcs.RailsAPI, ServiceNameController: svcs.Controller, ServiceNameDispatchCloud: svcs.DispatchCloud, + ServiceNameDispatchLSF: svcs.DispatchLSF, ServiceNameHealth: svcs.Health, - ServiceNameNodemanager: svcs.Nodemanager, ServiceNameWorkbench1: svcs.Workbench1, ServiceNameWorkbench2: svcs.Workbench2, ServiceNameWebsocket: svcs.Websocket,