X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/a4fa040579473b598a532ee38173b1a28c6b1694..6e73eff3926a2e7345333edd02531e8e6fbe15ef:/sdk/go/arvados/config.go diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go index 636cf350c1..275eb9e190 100644 --- a/sdk/go/arvados/config.go +++ b/sdk/go/arvados/config.go @@ -8,7 +8,7 @@ import ( "encoding/json" "errors" "fmt" - "os" + "net/url" "git.curoverse.com/arvados.git/sdk/go/config" ) @@ -50,22 +50,69 @@ func (sc *Config) GetCluster(clusterID string) (*Cluster, error) { } } -type RequestLimits struct { - MaxItemsPerResponse int - MultiClusterRequestConcurrency int +type API struct { + MaxItemsPerResponse int + MaxRequestAmplification int + RequestTimeout Duration } type Cluster struct { - ClusterID string `json:"-"` - ManagementToken string - NodeProfiles map[string]NodeProfile - InstanceTypes InstanceTypeMap - CloudVMs CloudVMs - Dispatch Dispatch - HTTPRequestTimeout Duration - RemoteClusters map[string]RemoteCluster - PostgreSQL PostgreSQL - RequestLimits RequestLimits + ClusterID string `json:"-"` + ManagementToken string + SystemRootToken string + Services Services + InstanceTypes InstanceTypeMap + Containers ContainersConfig + RemoteClusters map[string]RemoteCluster + PostgreSQL PostgreSQL + API API + SystemLogs SystemLogs + TLS TLS +} + +type Services struct { + Controller Service + DispatchCloud Service + Health Service + Keepbalance Service + Keepproxy Service + Keepstore Service + Nodemanager Service + RailsAPI Service + WebDAV Service + Websocket Service + Workbench1 Service + Workbench2 Service +} + +type Service struct { + InternalURLs map[URL]ServiceInstance `json:",omitempty"` + ExternalURL URL +} + +// URL is a url.URL that is also usable as a JSON key/value. +type URL url.URL + +// UnmarshalText implements encoding.TextUnmarshaler so URL can be +// used as a JSON key/value. +func (su *URL) UnmarshalText(text []byte) error { + u, err := url.Parse(string(text)) + if err == nil { + *su = URL(*u) + } + return err +} + +func (su URL) MarshalText() ([]byte, error) { + return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil +} + +type ServiceInstance struct{} + +type SystemLogs struct { + LogLevel string + Format string + MaxRequestLogParamsSize int } type PostgreSQL struct { @@ -88,59 +135,40 @@ type RemoteCluster struct { } type InstanceType struct { - Name string - ProviderType string - VCPUs int - RAM ByteSize - Scratch ByteSize - Price float64 - Preemptible bool + Name string + ProviderType string + VCPUs int + RAM ByteSize + Scratch ByteSize + IncludedScratch ByteSize + AddedScratch ByteSize + Price float64 + Preemptible bool } -type Dispatch struct { - // PEM encoded SSH key (RSA, DSA, or ECDSA) able to log in to - // cloud VMs. - PrivateKey string - - // Max time for workers to come up before abandoning stale - // locks from previous run - StaleLockTimeout Duration - - // Interval between queue polls - PollInterval Duration - - // Interval between probes to each worker - ProbeInterval Duration - - // Maximum total worker probes per second - MaxProbesPerSecond int +type ContainersConfig struct { + CloudVMs CloudVMsConfig + DispatchPrivateKey string + StaleLockTimeout Duration } -type CloudVMs struct { - // Shell command that exits zero IFF the VM is fully booted - // and ready to run containers, e.g., "mount | grep - // /encrypted-tmp" - BootProbeCommand string - - // Listening port (name or number) of SSH servers on worker - // VMs - SSHPort string - - SyncInterval Duration - - // Maximum idle time before automatic shutdown - TimeoutIdle Duration - - // Maximum booting time before automatic shutdown - TimeoutBooting Duration - - // Maximum time with no successful probes before automatic shutdown - TimeoutProbe Duration - - // Time after shutdown to retry shutdown - TimeoutShutdown Duration - - ImageID string +type CloudVMsConfig struct { + Enable bool + + BootProbeCommand string + ImageID string + MaxCloudOpsPerSecond int + MaxProbesPerSecond int + PollInterval Duration + ProbeInterval Duration + SSHPort string + SyncInterval Duration + TimeoutBooting Duration + TimeoutIdle Duration + TimeoutProbe Duration + TimeoutShutdown Duration + TimeoutSignal Duration + TimeoutTERM Duration Driver string DriverParameters json.RawMessage @@ -168,6 +196,20 @@ func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error { if _, ok := (*it)[t.Name]; ok { return errDuplicateInstanceTypeName } + if t.ProviderType == "" { + t.ProviderType = t.Name + } + if t.Scratch == 0 { + t.Scratch = t.IncludedScratch + t.AddedScratch + } else if t.AddedScratch == 0 { + t.AddedScratch = t.Scratch - t.IncludedScratch + } else if t.IncludedScratch == 0 { + t.IncludedScratch = t.Scratch - t.AddedScratch + } + + if t.Scratch != (t.IncludedScratch + t.AddedScratch) { + return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name) + } (*it)[t.Name] = t } return nil @@ -177,60 +219,29 @@ func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error { if err != nil { return err } - // Fill in Name field using hash key. + // Fill in Name field (and ProviderType field, if not + // specified) using hash key. *it = InstanceTypeMap(hash) for name, t := range *it { t.Name = name + if t.ProviderType == "" { + t.ProviderType = name + } (*it)[name] = t } return nil } -// GetNodeProfile returns a NodeProfile for the given hostname. An -// error is returned if the appropriate configuration can't be -// determined (e.g., this does not appear to be a system node). If -// node is empty, use the OS-reported hostname. -func (cc *Cluster) GetNodeProfile(node string) (*NodeProfile, error) { - if node == "" { - hostname, err := os.Hostname() - if err != nil { - return nil, err - } - node = hostname - } - if cfg, ok := cc.NodeProfiles[node]; ok { - return &cfg, nil - } - // If node is not listed, but "*" gives a default system node - // config, use the default config. - if cfg, ok := cc.NodeProfiles["*"]; ok { - return &cfg, nil - } - return nil, fmt.Errorf("config does not provision host %q as a system node", node) -} - -type NodeProfile struct { - Controller SystemServiceInstance `json:"arvados-controller"` - Health SystemServiceInstance `json:"arvados-health"` - Keepbalance SystemServiceInstance `json:"keep-balance"` - Keepproxy SystemServiceInstance `json:"keepproxy"` - Keepstore SystemServiceInstance `json:"keepstore"` - Keepweb SystemServiceInstance `json:"keep-web"` - Nodemanager SystemServiceInstance `json:"arvados-node-manager"` - DispatchCloud SystemServiceInstance `json:"arvados-dispatch-cloud"` - RailsAPI SystemServiceInstance `json:"arvados-api-server"` - Websocket SystemServiceInstance `json:"arvados-ws"` - Workbench SystemServiceInstance `json:"arvados-workbench"` -} - type ServiceName string const ( ServiceNameRailsAPI ServiceName = "arvados-api-server" ServiceNameController ServiceName = "arvados-controller" ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud" + ServiceNameHealth ServiceName = "arvados-health" ServiceNameNodemanager ServiceName = "arvados-node-manager" - ServiceNameWorkbench ServiceName = "arvados-workbench" + ServiceNameWorkbench1 ServiceName = "arvados-workbench1" + ServiceNameWorkbench2 ServiceName = "arvados-workbench2" ServiceNameWebsocket ServiceName = "arvados-ws" ServiceNameKeepbalance ServiceName = "keep-balance" ServiceNameKeepweb ServiceName = "keep-web" @@ -238,39 +249,27 @@ const ( ServiceNameKeepstore ServiceName = "keepstore" ) -// ServicePorts returns the configured listening address (or "" if -// disabled) for each service on the node. -func (np *NodeProfile) ServicePorts() map[ServiceName]string { - return map[ServiceName]string{ - ServiceNameRailsAPI: np.RailsAPI.Listen, - ServiceNameController: np.Controller.Listen, - ServiceNameDispatchCloud: np.DispatchCloud.Listen, - ServiceNameNodemanager: np.Nodemanager.Listen, - ServiceNameWorkbench: np.Workbench.Listen, - ServiceNameWebsocket: np.Websocket.Listen, - ServiceNameKeepbalance: np.Keepbalance.Listen, - ServiceNameKeepweb: np.Keepweb.Listen, - ServiceNameKeepproxy: np.Keepproxy.Listen, - ServiceNameKeepstore: np.Keepstore.Listen, +// Map returns all services as a map, suitable for iterating over all +// services or looking up a service by name. +func (svcs Services) Map() map[ServiceName]Service { + return map[ServiceName]Service{ + ServiceNameRailsAPI: svcs.RailsAPI, + ServiceNameController: svcs.Controller, + ServiceNameDispatchCloud: svcs.DispatchCloud, + ServiceNameHealth: svcs.Health, + ServiceNameNodemanager: svcs.Nodemanager, + ServiceNameWorkbench1: svcs.Workbench1, + ServiceNameWorkbench2: svcs.Workbench2, + ServiceNameWebsocket: svcs.Websocket, + ServiceNameKeepbalance: svcs.Keepbalance, + ServiceNameKeepweb: svcs.WebDAV, + ServiceNameKeepproxy: svcs.Keepproxy, + ServiceNameKeepstore: svcs.Keepstore, } } -func (h RequestLimits) GetMultiClusterRequestConcurrency() int { - if h.MultiClusterRequestConcurrency == 0 { - return 4 - } - return h.MultiClusterRequestConcurrency -} - -func (h RequestLimits) GetMaxItemsPerResponse() int { - if h.MaxItemsPerResponse == 0 { - return 1000 - } - return h.MaxItemsPerResponse -} - -type SystemServiceInstance struct { - Listen string - TLS bool - Insecure bool +type TLS struct { + Certificate string + Key string + Insecure bool }