X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/ef23c3d124b24a461f6947868a28e67e7a0a1010..0945afa5523fb45f827750e4d1700df4ff222295:/sdk/go/arvados/config.go diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go index 93f52f3c80..8222035a33 100644 --- a/sdk/go/arvados/config.go +++ b/sdk/go/arvados/config.go @@ -11,19 +11,19 @@ import ( "net/url" "os" - "git.curoverse.com/arvados.git/sdk/go/config" + "git.arvados.org/arvados.git/sdk/go/config" ) var DefaultConfigFile = func() string { if path := os.Getenv("ARVADOS_CONFIG"); path != "" { return path - } else { - return "/etc/arvados/config.yml" } + return "/etc/arvados/config.yml" }() type Config struct { - Clusters map[string]Cluster + Clusters map[string]Cluster + AutoReloadConfig bool } // GetConfig returns the current system config, loading it from @@ -49,12 +49,12 @@ func (sc *Config) GetCluster(clusterID string) (*Cluster, error) { } } } - if cc, ok := sc.Clusters[clusterID]; !ok { + cc, ok := sc.Clusters[clusterID] + if !ok { return nil, fmt.Errorf("cluster %q is not configured", clusterID) - } else { - cc.ClusterID = clusterID - return &cc, nil } + cc.ClusterID = clusterID + return &cc, nil } type WebDAVCacheConfig struct { @@ -66,6 +66,7 @@ type WebDAVCacheConfig struct { MaxPermissionEntries int MaxUUIDEntries int } + type Cluster struct { ClusterID string `json:"-"` ManagementToken string @@ -81,9 +82,10 @@ type Cluster struct { DisabledAPIs StringSet MaxIndexDatabaseRead int MaxItemsPerResponse int + MaxConcurrentRequests int + MaxKeepBlobBuffers int MaxRequestAmplification int MaxRequestSize int - RailsSessionSecretToken string RequestTimeout Duration SendTimeout Duration WebsocketClientEventQueue int @@ -96,29 +98,91 @@ type Cluster struct { UnloggedAttributes StringSet } Collections struct { - BlobSigning bool - BlobSigningKey string - BlobSigningTTL Duration - CollectionVersioning bool - DefaultTrashLifetime Duration - DefaultReplication int - ManagedProperties map[string]struct { + BlobSigning bool + BlobSigningKey string + BlobSigningTTL Duration + BlobTrash bool + BlobTrashLifetime Duration + BlobTrashCheckInterval Duration + BlobTrashConcurrency int + BlobDeleteConcurrency int + BlobReplicateConcurrency int + CollectionVersioning bool + DefaultTrashLifetime Duration + DefaultReplication int + ManagedProperties map[string]struct { Value interface{} Function string Protected bool } - PreserveVersionIfIdle Duration - TrashSweepInterval Duration - TrustAllContent bool + PreserveVersionIfIdle Duration + TrashSweepInterval Duration + TrustAllContent bool + ForwardSlashNameSubstitution string + S3FolderObjects bool + + BlobMissingReport string + BalancePeriod Duration + BalanceCollectionBatch int + BalanceCollectionBuffers int + BalanceTimeout Duration WebDAVCache WebDAVCacheConfig } Git struct { + GitCommand string + GitoliteHome string Repositories string } Login struct { - ProviderAppSecret string - ProviderAppID string + LDAP struct { + Enable bool + URL URL + StartTLS bool + InsecureTLS bool + StripDomain string + AppendDomain string + SearchAttribute string + SearchBindUser string + SearchBindPassword string + SearchBase string + SearchFilters string + EmailAttribute string + UsernameAttribute string + } + Google struct { + Enable bool + ClientID string + ClientSecret string + AlternateEmailAddresses bool + } + OpenIDConnect struct { + Enable bool + Issuer string + ClientID string + ClientSecret string + EmailClaim string + EmailVerifiedClaim string + UsernameClaim string + } + PAM struct { + Enable bool + Service string + DefaultEmailDomain string + } + SSO struct { + Enable bool + ProviderAppID string + ProviderAppSecret string + } + Test struct { + Enable bool + Users map[string]TestUser + } + LoginCluster string + RemoteTokenRefresh Duration + TokenLifetime Duration + TrustedClients map[string]struct{} } Mail struct { MailchimpAPIKey string @@ -154,7 +218,10 @@ type Cluster struct { NewUsersAreActive bool UserNotifierEmailFrom string UserProfileNotificationAddress string + PreferDomainForUsername string + UserSetupMailText string } + Volumes map[string]Volume Workbench struct { ActivationContactLink string APIClientConnectTimeout Duration @@ -189,9 +256,59 @@ type Cluster struct { } UserProfileFormMessage string VocabularyURL string + WelcomePageHTML string + InactivePageHTML string + SSHHelpPageHTML string + SSHHelpHostSuffix string + IdleTimeout Duration } - EnableBetaController14287 bool + ForceLegacyAPI14 bool +} + +type Volume struct { + AccessViaHosts map[URL]VolumeAccess + ReadOnly bool + Replication int + StorageClasses map[string]bool + Driver string + DriverParameters json.RawMessage +} + +type S3VolumeDriverParameters struct { + IAMRole string + AccessKey string + SecretKey string + Endpoint string + Region string + Bucket string + LocationConstraint bool + V2Signature bool + UseAWSS3v2Driver bool + IndexPageSize int + ConnectTimeout Duration + ReadTimeout Duration + RaceWindow Duration + UnsafeDelete bool +} + +type AzureVolumeDriverParameters struct { + StorageAccountName string + StorageAccountKey string + StorageBaseURL string + ContainerName string + RequestTimeout Duration + ListBlobsRetryDelay Duration + ListBlobsMaxAttempts int +} + +type DirectoryVolumeDriverParameters struct { + Root string + Serialize bool +} + +type VolumeAccess struct { + ReadOnly bool } type Services struct { @@ -204,7 +321,6 @@ type Services struct { Keepbalance Service Keepproxy Service Keepstore Service - Nodemanager Service RailsAPI Service SSO Service WebDAVDownload Service @@ -220,6 +336,11 @@ type Service struct { ExternalURL URL } +type TestUser struct { + Email string + Password string +} + // URL is a url.URL that is also usable as a JSON key/value. type URL url.URL @@ -229,6 +350,10 @@ func (su *URL) UnmarshalText(text []byte) error { u, err := url.Parse(string(text)) if err == nil { *su = URL(*u) + if su.Path == "" && su.Host != "" { + // http://example really means http://example/ + su.Path = "/" + } } return err } @@ -237,7 +362,13 @@ func (su URL) MarshalText() ([]byte, error) { return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil } -type ServiceInstance struct{} +func (su URL) String() string { + return (*url.URL)(&su).String() +} + +type ServiceInstance struct { + Rendezvous string `json:",omitempty"` +} type PostgreSQL struct { Connection PostgreSQLConnection @@ -289,7 +420,7 @@ type ContainersConfig struct { Logging struct { MaxAge Duration LogBytesPerEvent int - LogSecondsBetweenEvents int + LogSecondsBetweenEvents Duration LogThrottlePeriod Duration LogThrottleBytes int LogThrottleLines int @@ -298,6 +429,10 @@ type ContainersConfig struct { LogUpdatePeriod Duration LogUpdateSize ByteSize } + ShellAccess struct { + Admin bool + User bool + } SLURM struct { PrioritySpread int64 SbatchArgumentsList []string @@ -317,22 +452,25 @@ type ContainersConfig struct { type CloudVMsConfig struct { Enable bool - BootProbeCommand string - ImageID string - MaxCloudOpsPerSecond int - MaxProbesPerSecond int - PollInterval Duration - ProbeInterval Duration - SSHPort string - SyncInterval Duration - TimeoutBooting Duration - TimeoutIdle Duration - TimeoutProbe Duration - TimeoutShutdown Duration - TimeoutSignal Duration - TimeoutTERM Duration - ResourceTags map[string]string - TagKeyPrefix string + BootProbeCommand string + DeployRunnerBinary string + ImageID string + MaxCloudOpsPerSecond int + MaxProbesPerSecond int + MaxConcurrentInstanceCreateOps int + PollInterval Duration + ProbeInterval Duration + SSHPort string + SyncInterval Duration + TimeoutBooting Duration + TimeoutIdle Duration + TimeoutProbe Duration + TimeoutShutdown Duration + TimeoutSignal Duration + TimeoutStaleRunLock Duration + TimeoutTERM Duration + ResourceTags map[string]string + TagKeyPrefix string Driver string DriverParameters json.RawMessage @@ -345,6 +483,24 @@ var errDuplicateInstanceTypeName = errors.New("duplicate instance type name") // UnmarshalJSON handles old config files that provide an array of // instance types instead of a hash. func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error { + fixup := func(t InstanceType) (InstanceType, error) { + if t.ProviderType == "" { + t.ProviderType = t.Name + } + if t.Scratch == 0 { + t.Scratch = t.IncludedScratch + t.AddedScratch + } else if t.AddedScratch == 0 { + t.AddedScratch = t.Scratch - t.IncludedScratch + } else if t.IncludedScratch == 0 { + t.IncludedScratch = t.Scratch - t.AddedScratch + } + + if t.Scratch != (t.IncludedScratch + t.AddedScratch) { + return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name) + } + return t, nil + } + if len(data) > 0 && data[0] == '[' { var arr []InstanceType err := json.Unmarshal(data, &arr) @@ -360,19 +516,9 @@ func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error { if _, ok := (*it)[t.Name]; ok { return errDuplicateInstanceTypeName } - if t.ProviderType == "" { - t.ProviderType = t.Name - } - if t.Scratch == 0 { - t.Scratch = t.IncludedScratch + t.AddedScratch - } else if t.AddedScratch == 0 { - t.AddedScratch = t.Scratch - t.IncludedScratch - } else if t.IncludedScratch == 0 { - t.IncludedScratch = t.Scratch - t.AddedScratch - } - - if t.Scratch != (t.IncludedScratch + t.AddedScratch) { - return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name) + t, err := fixup(t) + if err != nil { + return err } (*it)[t.Name] = t } @@ -388,8 +534,9 @@ func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error { *it = InstanceTypeMap(hash) for name, t := range *it { t.Name = name - if t.ProviderType == "" { - t.ProviderType = name + t, err := fixup(t) + if err != nil { + return err } (*it)[name] = t } @@ -423,7 +570,7 @@ func (ss *StringSet) UnmarshalJSON(data []byte) error { return err } *ss = make(map[string]struct{}, len(hash)) - for t, _ := range hash { + for t := range hash { (*ss)[t] = struct{}{} } @@ -437,7 +584,6 @@ const ( ServiceNameController ServiceName = "arvados-controller" ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud" ServiceNameHealth ServiceName = "arvados-health" - ServiceNameNodemanager ServiceName = "arvados-node-manager" ServiceNameWorkbench1 ServiceName = "arvados-workbench1" ServiceNameWorkbench2 ServiceName = "arvados-workbench2" ServiceNameWebsocket ServiceName = "arvados-ws" @@ -455,7 +601,6 @@ func (svcs Services) Map() map[ServiceName]Service { ServiceNameController: svcs.Controller, ServiceNameDispatchCloud: svcs.DispatchCloud, ServiceNameHealth: svcs.Health, - ServiceNameNodemanager: svcs.Nodemanager, ServiceNameWorkbench1: svcs.Workbench1, ServiceNameWorkbench2: svcs.Workbench2, ServiceNameWebsocket: svcs.Websocket,