package arvados
import (
+ "encoding/json"
+ "errors"
"fmt"
+ "net/url"
"os"
"git.curoverse.com/arvados.git/sdk/go/config"
)
-const DefaultConfigFile = "/etc/arvados/config.yml"
+var DefaultConfigFile = func() string {
+ if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
+ return path
+ } else {
+ return "/etc/arvados/config.yml"
+ }
+}()
type Config struct {
Clusters map[string]Cluster
}
}
+type WebDAVCacheConfig struct {
+ TTL Duration
+ UUIDTTL Duration
+ MaxBlockEntries int
+ MaxCollectionEntries int
+ MaxCollectionBytes int64
+ MaxPermissionEntries int
+ MaxUUIDEntries int
+}
type Cluster struct {
ClusterID string `json:"-"`
ManagementToken string
- SystemNodes map[string]SystemNode
- InstanceTypes []InstanceType
+ SystemRootToken string
+ Services Services
+ InstanceTypes InstanceTypeMap
+ Containers ContainersConfig
+ RemoteClusters map[string]RemoteCluster
+ PostgreSQL PostgreSQL
+
+ API struct {
+ AsyncPermissionsUpdateInterval Duration
+ DisabledAPIs StringSet
+ MaxIndexDatabaseRead int
+ MaxItemsPerResponse int
+ MaxRequestAmplification int
+ MaxRequestSize int
+ RailsSessionSecretToken string
+ RequestTimeout Duration
+ SendTimeout Duration
+ WebsocketClientEventQueue int
+ WebsocketServerEventQueue int
+ }
+ AuditLogs struct {
+ MaxAge Duration
+ MaxDeleteBatch int
+ UnloggedAttributes StringSet
+ }
+ Collections struct {
+ BlobSigning bool
+ BlobSigningKey string
+ BlobSigningTTL Duration
+ CollectionVersioning bool
+ DefaultTrashLifetime Duration
+ DefaultReplication int
+ ManagedProperties map[string]struct {
+ Value interface{}
+ Function string
+ Protected bool
+ }
+ PreserveVersionIfIdle Duration
+ TrashSweepInterval Duration
+ TrustAllContent bool
+
+ WebDAVCache WebDAVCacheConfig
+ }
+ Git struct {
+ Repositories string
+ }
+ Login struct {
+ ProviderAppSecret string
+ ProviderAppID string
+ }
+ Mail struct {
+ MailchimpAPIKey string
+ MailchimpListID string
+ SendUserSetupNotificationEmail bool
+ IssueReporterEmailFrom string
+ IssueReporterEmailTo string
+ SupportEmailAddress string
+ EmailFrom string
+ }
+ SystemLogs struct {
+ LogLevel string
+ Format string
+ MaxRequestLogParamsSize int
+ }
+ TLS struct {
+ Certificate string
+ Key string
+ Insecure bool
+ }
+ Users struct {
+ AnonymousUserToken string
+ AdminNotifierEmailFrom string
+ AutoAdminFirstUser bool
+ AutoAdminUserWithEmail string
+ AutoSetupNewUsers bool
+ AutoSetupNewUsersWithRepository bool
+ AutoSetupNewUsersWithVmUUID string
+ AutoSetupUsernameBlacklist StringSet
+ EmailSubjectPrefix string
+ NewInactiveUserNotificationRecipients StringSet
+ NewUserNotificationRecipients StringSet
+ NewUsersAreActive bool
+ UserNotifierEmailFrom string
+ UserProfileNotificationAddress string
+ }
+ Workbench struct {
+ ActivationContactLink string
+ APIClientConnectTimeout Duration
+ APIClientReceiveTimeout Duration
+ APIResponseCompression bool
+ ApplicationMimetypesWithViewIcon StringSet
+ ArvadosDocsite string
+ ArvadosPublicDataDocURL string
+ DefaultOpenIdPrefix string
+ EnableGettingStartedPopup bool
+ EnablePublicProjectsPage bool
+ FileViewersConfigURL string
+ LogViewerMaxBytes ByteSize
+ MultiSiteSearch string
+ ProfilingEnabled bool
+ Repositories bool
+ RepositoryCache string
+ RunningJobLogRecordsToFetch int
+ SecretKeyBase string
+ ShowRecentCollectionsOnDashboard bool
+ ShowUserAgreementInline bool
+ ShowUserNotifications bool
+ SiteName string
+ Theme string
+ UserProfileFormFields map[string]struct {
+ Type string
+ FormFieldTitle string
+ FormFieldDescription string
+ Required bool
+ Position int
+ Options map[string]struct{}
+ }
+ UserProfileFormMessage string
+ VocabularyURL string
+ }
+
+ EnableBetaController14287 bool
}
-type InstanceType struct {
- Name string
- ProviderType string
- VCPUs int
- RAM int64
- Scratch int64
- Price float64
- Preemptable bool
-}
-
-// GetThisSystemNode returns a SystemNode for the node we're running
-// on right now.
-func (cc *Cluster) GetThisSystemNode() (*SystemNode, error) {
- hostname, err := os.Hostname()
- if err != nil {
- return nil, err
+type Services struct {
+ Composer Service
+ Controller Service
+ DispatchCloud Service
+ GitHTTP Service
+ GitSSH Service
+ Health Service
+ Keepbalance Service
+ Keepproxy Service
+ Keepstore Service
+ Nodemanager Service
+ RailsAPI Service
+ SSO Service
+ WebDAVDownload Service
+ WebDAV Service
+ WebShell Service
+ Websocket Service
+ Workbench1 Service
+ Workbench2 Service
+}
+
+type Service struct {
+ InternalURLs map[URL]ServiceInstance
+ ExternalURL URL
+}
+
+// URL is a url.URL that is also usable as a JSON key/value.
+type URL url.URL
+
+// UnmarshalText implements encoding.TextUnmarshaler so URL can be
+// used as a JSON key/value.
+func (su *URL) UnmarshalText(text []byte) error {
+ u, err := url.Parse(string(text))
+ if err == nil {
+ *su = URL(*u)
}
- return cc.GetSystemNode(hostname)
+ return err
+}
+
+func (su URL) MarshalText() ([]byte, error) {
+ return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
+}
+
+type ServiceInstance struct{}
+
+type PostgreSQL struct {
+ Connection PostgreSQLConnection
+ ConnectionPool int
+}
+
+type PostgreSQLConnection map[string]string
+
+type RemoteCluster struct {
+ Host string
+ Proxy bool
+ Scheme string
+ Insecure bool
+ ActivateUsers bool
+}
+
+type InstanceType struct {
+ Name string
+ ProviderType string
+ VCPUs int
+ RAM ByteSize
+ Scratch ByteSize
+ IncludedScratch ByteSize
+ AddedScratch ByteSize
+ Price float64
+ Preemptible bool
}
-// GetSystemNode returns a SystemNode for the given hostname. An error
-// is returned if the appropriate configuration can't be determined
-// (e.g., this does not appear to be a system node).
-func (cc *Cluster) GetSystemNode(node string) (*SystemNode, error) {
- if cfg, ok := cc.SystemNodes[node]; ok {
- return &cfg, nil
+type ContainersConfig struct {
+ CloudVMs CloudVMsConfig
+ CrunchRunCommand string
+ CrunchRunArgumentsList []string
+ DefaultKeepCacheRAM ByteSize
+ DispatchPrivateKey string
+ LogReuseDecisions bool
+ MaxComputeVMs int
+ MaxDispatchAttempts int
+ MaxRetryAttempts int
+ MinRetryPeriod Duration
+ ReserveExtraRAM ByteSize
+ StaleLockTimeout Duration
+ SupportedDockerImageFormats StringSet
+ UsePreemptibleInstances bool
+
+ JobsAPI struct {
+ Enable string
+ GitInternalDir string
}
- // If node is not listed, but "*" gives a default system node
- // config, use the default config.
- if cfg, ok := cc.SystemNodes["*"]; ok {
- return &cfg, nil
+ Logging struct {
+ MaxAge Duration
+ LogBytesPerEvent int
+ LogSecondsBetweenEvents int
+ LogThrottlePeriod Duration
+ LogThrottleBytes int
+ LogThrottleLines int
+ LimitLogBytesPerJob int
+ LogPartialLineThrottlePeriod Duration
+ LogUpdatePeriod Duration
+ LogUpdateSize ByteSize
+ }
+ SLURM struct {
+ PrioritySpread int64
+ SbatchArgumentsList []string
+ SbatchEnvironmentVariables map[string]string
+ Managed struct {
+ DNSServerConfDir string
+ DNSServerConfTemplate string
+ DNSServerReloadCommand string
+ DNSServerUpdateCommand string
+ ComputeNodeDomain string
+ ComputeNodeNameservers StringSet
+ AssignNodeHostname string
+ }
}
- return nil, fmt.Errorf("config does not provision host %q as a system node", node)
}
-type SystemNode struct {
- Health SystemServiceInstance `json:"arvados-health"`
- Keepproxy SystemServiceInstance `json:"keepproxy"`
- Keepstore SystemServiceInstance `json:"keepstore"`
- Keepweb SystemServiceInstance `json:"keep-web"`
- Nodemanager SystemServiceInstance `json:"arvados-node-manager"`
- RailsAPI SystemServiceInstance `json:"arvados-api-server"`
- Websocket SystemServiceInstance `json:"arvados-ws"`
- Workbench SystemServiceInstance `json:"arvados-workbench"`
+type CloudVMsConfig struct {
+ Enable bool
+
+ BootProbeCommand string
+ ImageID string
+ MaxCloudOpsPerSecond int
+ MaxProbesPerSecond int
+ PollInterval Duration
+ ProbeInterval Duration
+ SSHPort string
+ SyncInterval Duration
+ TimeoutBooting Duration
+ TimeoutIdle Duration
+ TimeoutProbe Duration
+ TimeoutShutdown Duration
+ TimeoutSignal Duration
+ TimeoutTERM Duration
+ ResourceTags map[string]string
+ TagKeyPrefix string
+
+ Driver string
+ DriverParameters json.RawMessage
}
-// ServicePorts returns the configured listening address (or "" if
-// disabled) for each service on the node.
-func (sn *SystemNode) ServicePorts() map[string]string {
- return map[string]string{
- "arvados-api-server": sn.RailsAPI.Listen,
- "arvados-node-manager": sn.Nodemanager.Listen,
- "arvados-workbench": sn.Workbench.Listen,
- "arvados-ws": sn.Websocket.Listen,
- "keep-web": sn.Keepweb.Listen,
- "keepproxy": sn.Keepproxy.Listen,
- "keepstore": sn.Keepstore.Listen,
+type InstanceTypeMap map[string]InstanceType
+
+var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
+
+// UnmarshalJSON handles old config files that provide an array of
+// instance types instead of a hash.
+func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
+ if len(data) > 0 && data[0] == '[' {
+ var arr []InstanceType
+ err := json.Unmarshal(data, &arr)
+ if err != nil {
+ return err
+ }
+ if len(arr) == 0 {
+ *it = nil
+ return nil
+ }
+ *it = make(map[string]InstanceType, len(arr))
+ for _, t := range arr {
+ if _, ok := (*it)[t.Name]; ok {
+ return errDuplicateInstanceTypeName
+ }
+ if t.ProviderType == "" {
+ t.ProviderType = t.Name
+ }
+ if t.Scratch == 0 {
+ t.Scratch = t.IncludedScratch + t.AddedScratch
+ } else if t.AddedScratch == 0 {
+ t.AddedScratch = t.Scratch - t.IncludedScratch
+ } else if t.IncludedScratch == 0 {
+ t.IncludedScratch = t.Scratch - t.AddedScratch
+ }
+
+ if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
+ return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
+ }
+ (*it)[t.Name] = t
+ }
+ return nil
+ }
+ var hash map[string]InstanceType
+ err := json.Unmarshal(data, &hash)
+ if err != nil {
+ return err
}
+ // Fill in Name field (and ProviderType field, if not
+ // specified) using hash key.
+ *it = InstanceTypeMap(hash)
+ for name, t := range *it {
+ t.Name = name
+ if t.ProviderType == "" {
+ t.ProviderType = name
+ }
+ (*it)[name] = t
+ }
+ return nil
}
-type SystemServiceInstance struct {
- Listen string
+type StringSet map[string]struct{}
+
+// UnmarshalJSON handles old config files that provide an array of
+// instance types instead of a hash.
+func (ss *StringSet) UnmarshalJSON(data []byte) error {
+ if len(data) > 0 && data[0] == '[' {
+ var arr []string
+ err := json.Unmarshal(data, &arr)
+ if err != nil {
+ return err
+ }
+ if len(arr) == 0 {
+ *ss = nil
+ return nil
+ }
+ *ss = make(map[string]struct{}, len(arr))
+ for _, t := range arr {
+ (*ss)[t] = struct{}{}
+ }
+ return nil
+ }
+ var hash map[string]struct{}
+ err := json.Unmarshal(data, &hash)
+ if err != nil {
+ return err
+ }
+ *ss = make(map[string]struct{}, len(hash))
+ for t, _ := range hash {
+ (*ss)[t] = struct{}{}
+ }
+
+ return nil
+}
+
+type ServiceName string
+
+const (
+ ServiceNameRailsAPI ServiceName = "arvados-api-server"
+ ServiceNameController ServiceName = "arvados-controller"
+ ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
+ ServiceNameHealth ServiceName = "arvados-health"
+ ServiceNameNodemanager ServiceName = "arvados-node-manager"
+ ServiceNameWorkbench1 ServiceName = "arvados-workbench1"
+ ServiceNameWorkbench2 ServiceName = "arvados-workbench2"
+ ServiceNameWebsocket ServiceName = "arvados-ws"
+ ServiceNameKeepbalance ServiceName = "keep-balance"
+ ServiceNameKeepweb ServiceName = "keep-web"
+ ServiceNameKeepproxy ServiceName = "keepproxy"
+ ServiceNameKeepstore ServiceName = "keepstore"
+)
+
+// Map returns all services as a map, suitable for iterating over all
+// services or looking up a service by name.
+func (svcs Services) Map() map[ServiceName]Service {
+ return map[ServiceName]Service{
+ ServiceNameRailsAPI: svcs.RailsAPI,
+ ServiceNameController: svcs.Controller,
+ ServiceNameDispatchCloud: svcs.DispatchCloud,
+ ServiceNameHealth: svcs.Health,
+ ServiceNameNodemanager: svcs.Nodemanager,
+ ServiceNameWorkbench1: svcs.Workbench1,
+ ServiceNameWorkbench2: svcs.Workbench2,
+ ServiceNameWebsocket: svcs.Websocket,
+ ServiceNameKeepbalance: svcs.Keepbalance,
+ ServiceNameKeepweb: svcs.WebDAV,
+ ServiceNameKeepproxy: svcs.Keepproxy,
+ ServiceNameKeepstore: svcs.Keepstore,
+ }
}
projects / arvados.git / blobdiff