X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/6b3a880d607ee3e3dd273f019981fd6cae62373c..2c61bf045e35d8971412880031412c4f1a9660f6:/sdk/go/arvados/config.go diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go index b84e1eefaa..ad663b23ef 100644 --- a/sdk/go/arvados/config.go +++ b/sdk/go/arvados/config.go @@ -10,6 +10,7 @@ import ( "fmt" "net/url" "os" + "time" "git.arvados.org/arvados.git/sdk/go/config" ) @@ -24,6 +25,8 @@ var DefaultConfigFile = func() string { type Config struct { Clusters map[string]Cluster AutoReloadConfig bool + SourceTimestamp time.Time + SourceSHA256 string } // GetConfig returns the current system config, loading it from @@ -77,6 +80,12 @@ type UploadDownloadRolePermissions struct { Admin UploadDownloadPermission } +type ManagedProperties map[string]struct { + Value interface{} + Function string + Protected bool +} + type Cluster struct { ClusterID string `json:"-"` ManagementToken string @@ -88,20 +97,24 @@ type Cluster struct { PostgreSQL PostgreSQL API struct { - AsyncPermissionsUpdateInterval Duration - DisabledAPIs StringSet - MaxIndexDatabaseRead int - MaxItemsPerResponse int - MaxConcurrentRequests int - MaxKeepBlobBuffers int - MaxRequestAmplification int - MaxRequestSize int - MaxTokenLifetime Duration - RequestTimeout Duration - SendTimeout Duration - WebsocketClientEventQueue int - WebsocketServerEventQueue int - KeepServiceRequestTimeout Duration + AsyncPermissionsUpdateInterval Duration + DisabledAPIs StringSet + MaxIndexDatabaseRead int + MaxItemsPerResponse int + MaxConcurrentRequests int + MaxKeepBlobBuffers int + MaxRequestAmplification int + MaxRequestSize int + MaxTokenLifetime Duration + RequestTimeout Duration + SendTimeout Duration + WebsocketClientEventQueue int + WebsocketServerEventQueue int + KeepServiceRequestTimeout Duration + VocabularyPath string + FreezeProjectRequiresDescription bool + FreezeProjectRequiresProperties StringSet + UnfreezeProjectRequiresAdmin bool } AuditLogs struct { MaxAge Duration @@ -109,23 +122,19 @@ type Cluster struct { UnloggedAttributes StringSet } Collections struct { - BlobSigning bool - BlobSigningKey string - BlobSigningTTL Duration - BlobTrash bool - BlobTrashLifetime Duration - BlobTrashCheckInterval Duration - BlobTrashConcurrency int - BlobDeleteConcurrency int - BlobReplicateConcurrency int - CollectionVersioning bool - DefaultTrashLifetime Duration - DefaultReplication int - ManagedProperties map[string]struct { - Value interface{} - Function string - Protected bool - } + BlobSigning bool + BlobSigningKey string + BlobSigningTTL Duration + BlobTrash bool + BlobTrashLifetime Duration + BlobTrashCheckInterval Duration + BlobTrashConcurrency int + BlobDeleteConcurrency int + BlobReplicateConcurrency int + CollectionVersioning bool + DefaultTrashLifetime Duration + DefaultReplication int + ManagedProperties ManagedProperties PreserveVersionIfIdle Duration TrashSweepInterval Duration TrustAllContent bool @@ -220,6 +229,7 @@ type Cluster struct { Insecure bool } Users struct { + ActivatedUsersAreVisibleToOthers bool AnonymousUserToken string AdminNotifierEmailFrom string AutoAdminFirstUser bool @@ -237,6 +247,7 @@ type Cluster struct { UserProfileNotificationAddress string PreferDomainForUsername string UserSetupMailText string + RoleGroupsVisibleToAll bool } StorageClasses map[string]StorageClassConfig Volumes map[string]Volume @@ -249,6 +260,7 @@ type Cluster struct { ArvadosDocsite string ArvadosPublicDataDocURL string DefaultOpenIdPrefix string + DisableSharingURLsUI bool EnableGettingStartedPopup bool EnablePublicProjectsPage bool FileViewersConfigURL string @@ -273,7 +285,6 @@ type Cluster struct { Options map[string]struct{} } UserProfileFormMessage string - VocabularyURL string WelcomePageHTML string InactivePageHTML string SSHHelpPageHTML string @@ -338,6 +349,7 @@ type Services struct { Controller Service DispatchCloud Service DispatchLSF Service + DispatchSLURM Service GitHTTP Service GitSSH Service Health Service @@ -354,8 +366,9 @@ type Services struct { } type Service struct { - InternalURLs map[URL]ServiceInstance - ExternalURL URL + ListenAddress string + InternalURLs map[URL]ServiceInstance + ExternalURL URL } type TestUser struct { @@ -407,35 +420,44 @@ type RemoteCluster struct { ActivateUsers bool } +type CUDAFeatures struct { + DriverVersion string + HardwareCapability string + DeviceCount int +} + type InstanceType struct { - Name string + Name string `json:"-"` ProviderType string VCPUs int RAM ByteSize - Scratch ByteSize + Scratch ByteSize `json:"-"` IncludedScratch ByteSize AddedScratch ByteSize Price float64 Preemptible bool + CUDA CUDAFeatures } type ContainersConfig struct { - CloudVMs CloudVMsConfig - CrunchRunCommand string - CrunchRunArgumentsList []string - DefaultKeepCacheRAM ByteSize - DispatchPrivateKey string - LogReuseDecisions bool - MaxComputeVMs int - MaxDispatchAttempts int - MaxRetryAttempts int - MinRetryPeriod Duration - ReserveExtraRAM ByteSize - StaleLockTimeout Duration - SupportedDockerImageFormats StringSet - UsePreemptibleInstances bool - RuntimeEngine string - LocalKeepBlobBuffersPerVCPU int + CloudVMs CloudVMsConfig + CrunchRunCommand string + CrunchRunArgumentsList []string + DefaultKeepCacheRAM ByteSize + DispatchPrivateKey string + LogReuseDecisions bool + MaxComputeVMs int + MaxDispatchAttempts int + MaxRetryAttempts int + MinRetryPeriod Duration + ReserveExtraRAM ByteSize + StaleLockTimeout Duration + SupportedDockerImageFormats StringSet + AlwaysUsePreemptibleInstances bool + PreemptiblePriceFactor float64 + RuntimeEngine string + LocalKeepBlobBuffersPerVCPU int + LocalKeepLogsToContainerLog string JobsAPI struct { Enable string @@ -474,6 +496,7 @@ type ContainersConfig struct { LSF struct { BsubSudoUser string BsubArgumentsList []string + BsubCUDAArguments []string } } @@ -508,49 +531,23 @@ type InstanceTypeMap map[string]InstanceType var errDuplicateInstanceTypeName = errors.New("duplicate instance type name") -// UnmarshalJSON handles old config files that provide an array of -// instance types instead of a hash. +// UnmarshalJSON does special handling of InstanceTypes: +// * populate computed fields (Name and Scratch) +// * error out if InstancesTypes are populated as an array, which was +// deprecated in Arvados 1.2.0 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error { fixup := func(t InstanceType) (InstanceType, error) { if t.ProviderType == "" { t.ProviderType = t.Name } - if t.Scratch == 0 { - t.Scratch = t.IncludedScratch + t.AddedScratch - } else if t.AddedScratch == 0 { - t.AddedScratch = t.Scratch - t.IncludedScratch - } else if t.IncludedScratch == 0 { - t.IncludedScratch = t.Scratch - t.AddedScratch - } - - if t.Scratch != (t.IncludedScratch + t.AddedScratch) { - return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name) - } + // If t.Scratch is set in the configuration file, it will be ignored and overwritten. + // It will also generate a "deprecated or unknown config entry" warning. + t.Scratch = t.IncludedScratch + t.AddedScratch return t, nil } if len(data) > 0 && data[0] == '[' { - var arr []InstanceType - err := json.Unmarshal(data, &arr) - if err != nil { - return err - } - if len(arr) == 0 { - *it = nil - return nil - } - *it = make(map[string]InstanceType, len(arr)) - for _, t := range arr { - if _, ok := (*it)[t.Name]; ok { - return errDuplicateInstanceTypeName - } - t, err := fixup(t) - if err != nil { - return err - } - (*it)[t.Name] = t - } - return nil + return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html") } var hash map[string]InstanceType err := json.Unmarshal(data, &hash) @@ -608,35 +605,39 @@ func (ss *StringSet) UnmarshalJSON(data []byte) error { type ServiceName string const ( - ServiceNameRailsAPI ServiceName = "arvados-api-server" ServiceNameController ServiceName = "arvados-controller" ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud" ServiceNameDispatchLSF ServiceName = "arvados-dispatch-lsf" + ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm" + ServiceNameGitHTTP ServiceName = "arvados-git-httpd" ServiceNameHealth ServiceName = "arvados-health" - ServiceNameWorkbench1 ServiceName = "arvados-workbench1" - ServiceNameWorkbench2 ServiceName = "arvados-workbench2" - ServiceNameWebsocket ServiceName = "arvados-ws" ServiceNameKeepbalance ServiceName = "keep-balance" - ServiceNameKeepweb ServiceName = "keep-web" ServiceNameKeepproxy ServiceName = "keepproxy" ServiceNameKeepstore ServiceName = "keepstore" + ServiceNameKeepweb ServiceName = "keep-web" + ServiceNameRailsAPI ServiceName = "arvados-api-server" + ServiceNameWebsocket ServiceName = "arvados-ws" + ServiceNameWorkbench1 ServiceName = "arvados-workbench1" + ServiceNameWorkbench2 ServiceName = "arvados-workbench2" ) // Map returns all services as a map, suitable for iterating over all // services or looking up a service by name. func (svcs Services) Map() map[ServiceName]Service { return map[ServiceName]Service{ - ServiceNameRailsAPI: svcs.RailsAPI, ServiceNameController: svcs.Controller, ServiceNameDispatchCloud: svcs.DispatchCloud, ServiceNameDispatchLSF: svcs.DispatchLSF, + ServiceNameDispatchSLURM: svcs.DispatchSLURM, + ServiceNameGitHTTP: svcs.GitHTTP, ServiceNameHealth: svcs.Health, - ServiceNameWorkbench1: svcs.Workbench1, - ServiceNameWorkbench2: svcs.Workbench2, - ServiceNameWebsocket: svcs.Websocket, ServiceNameKeepbalance: svcs.Keepbalance, - ServiceNameKeepweb: svcs.WebDAV, ServiceNameKeepproxy: svcs.Keepproxy, ServiceNameKeepstore: svcs.Keepstore, + ServiceNameKeepweb: svcs.WebDAV, + ServiceNameRailsAPI: svcs.RailsAPI, + ServiceNameWebsocket: svcs.Websocket, + ServiceNameWorkbench1: svcs.Workbench1, + ServiceNameWorkbench2: svcs.Workbench2, } }