Merge branch '13306-arvados-cwl-runner-py3-support'

[arvados.git] / sdk / go / arvados / config.go

diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index 6edd18418bb8015087f8b486acf6ee21d2d26db4..1154f922ba45f0b76484e08ac182276c147641db 100644 (file)
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -50,14 +50,22 @@ func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
        }
 }
 
+type RequestLimits struct {
+       MaxItemsPerResponse            int
+       MultiClusterRequestConcurrency int
+}
+
 type Cluster struct {
        ClusterID          string `json:"-"`
        ManagementToken    string
        NodeProfiles       map[string]NodeProfile
        InstanceTypes      InstanceTypeMap
+       CloudVMs           CloudVMs
+       Dispatch           Dispatch
        HTTPRequestTimeout Duration
        RemoteClusters     map[string]RemoteCluster
        PostgreSQL         PostgreSQL
+       RequestLimits      RequestLimits
 }
 
 type PostgreSQL struct {
@@ -89,6 +97,55 @@ type InstanceType struct {
        Preemptible  bool
 }
 
+type Dispatch struct {
+       // PEM encoded SSH key (RSA, DSA, or ECDSA) able to log in to
+       // cloud VMs.
+       PrivateKey []byte
+
+       // Max time for workers to come up before abandoning stale
+       // locks from previous run
+       StaleLockTimeout Duration
+
+       // Interval between queue polls
+       PollInterval Duration
+
+       // Interval between probes to each worker
+       ProbeInterval Duration
+
+       // Maximum total worker probes per second
+       MaxProbesPerSecond int
+}
+
+type CloudVMs struct {
+       // Shell command that exits zero IFF the VM is fully booted
+       // and ready to run containers, e.g., "mount | grep
+       // /encrypted-tmp"
+       BootProbeCommand string
+
+       // Listening port (name or number) of SSH servers on worker
+       // VMs
+       SSHPort string
+
+       SyncInterval Duration
+
+       // Maximum idle time before automatic shutdown
+       TimeoutIdle Duration
+
+       // Maximum booting time before automatic shutdown
+       TimeoutBooting Duration
+
+       // Maximum time with no successful probes before automatic shutdown
+       TimeoutProbe Duration
+
+       // Time after shutdown to retry shutdown
+       TimeoutShutdown Duration
+
+       ImageID string
+
+       Driver           string
+       DriverParameters map[string]interface{}
+}
+
 type InstanceTypeMap map[string]InstanceType
 
 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
@@ -153,43 +210,63 @@ func (cc *Cluster) GetNodeProfile(node string) (*NodeProfile, error) {
 }
 
 type NodeProfile struct {
-       Controller  SystemServiceInstance `json:"arvados-controller"`
-       Health      SystemServiceInstance `json:"arvados-health"`
-       Keepproxy   SystemServiceInstance `json:"keepproxy"`
-       Keepstore   SystemServiceInstance `json:"keepstore"`
-       Keepweb     SystemServiceInstance `json:"keep-web"`
-       Nodemanager SystemServiceInstance `json:"arvados-node-manager"`
-       RailsAPI    SystemServiceInstance `json:"arvados-api-server"`
-       Websocket   SystemServiceInstance `json:"arvados-ws"`
-       Workbench   SystemServiceInstance `json:"arvados-workbench"`
+       Controller    SystemServiceInstance `json:"arvados-controller"`
+       Health        SystemServiceInstance `json:"arvados-health"`
+       Keepbalance   SystemServiceInstance `json:"keep-balance"`
+       Keepproxy     SystemServiceInstance `json:"keepproxy"`
+       Keepstore     SystemServiceInstance `json:"keepstore"`
+       Keepweb       SystemServiceInstance `json:"keep-web"`
+       Nodemanager   SystemServiceInstance `json:"arvados-node-manager"`
+       DispatchCloud SystemServiceInstance `json:"arvados-dispatch-cloud"`
+       RailsAPI      SystemServiceInstance `json:"arvados-api-server"`
+       Websocket     SystemServiceInstance `json:"arvados-ws"`
+       Workbench     SystemServiceInstance `json:"arvados-workbench"`
 }
 
 type ServiceName string
 
 const (
-       ServiceNameRailsAPI    ServiceName = "arvados-api-server"
-       ServiceNameController  ServiceName = "arvados-controller"
-       ServiceNameNodemanager ServiceName = "arvados-node-manager"
-       ServiceNameWorkbench   ServiceName = "arvados-workbench"
-       ServiceNameWebsocket   ServiceName = "arvados-ws"
-       ServiceNameKeepweb     ServiceName = "keep-web"
-       ServiceNameKeepproxy   ServiceName = "keepproxy"
-       ServiceNameKeepstore   ServiceName = "keepstore"
+       ServiceNameRailsAPI      ServiceName = "arvados-api-server"
+       ServiceNameController    ServiceName = "arvados-controller"
+       ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
+       ServiceNameNodemanager   ServiceName = "arvados-node-manager"
+       ServiceNameWorkbench     ServiceName = "arvados-workbench"
+       ServiceNameWebsocket     ServiceName = "arvados-ws"
+       ServiceNameKeepbalance   ServiceName = "keep-balance"
+       ServiceNameKeepweb       ServiceName = "keep-web"
+       ServiceNameKeepproxy     ServiceName = "keepproxy"
+       ServiceNameKeepstore     ServiceName = "keepstore"
 )
 
 // ServicePorts returns the configured listening address (or "" if
 // disabled) for each service on the node.
 func (np *NodeProfile) ServicePorts() map[ServiceName]string {
        return map[ServiceName]string{
-               ServiceNameRailsAPI:    np.RailsAPI.Listen,
-               ServiceNameController:  np.Controller.Listen,
-               ServiceNameNodemanager: np.Nodemanager.Listen,
-               ServiceNameWorkbench:   np.Workbench.Listen,
-               ServiceNameWebsocket:   np.Websocket.Listen,
-               ServiceNameKeepweb:     np.Keepweb.Listen,
-               ServiceNameKeepproxy:   np.Keepproxy.Listen,
-               ServiceNameKeepstore:   np.Keepstore.Listen,
+               ServiceNameRailsAPI:      np.RailsAPI.Listen,
+               ServiceNameController:    np.Controller.Listen,
+               ServiceNameDispatchCloud: np.DispatchCloud.Listen,
+               ServiceNameNodemanager:   np.Nodemanager.Listen,
+               ServiceNameWorkbench:     np.Workbench.Listen,
+               ServiceNameWebsocket:     np.Websocket.Listen,
+               ServiceNameKeepbalance:   np.Keepbalance.Listen,
+               ServiceNameKeepweb:       np.Keepweb.Listen,
+               ServiceNameKeepproxy:     np.Keepproxy.Listen,
+               ServiceNameKeepstore:     np.Keepstore.Listen,
+       }
+}
+
+func (h RequestLimits) GetMultiClusterRequestConcurrency() int {
+       if h.MultiClusterRequestConcurrency == 0 {
+               return 4
+       }
+       return h.MultiClusterRequestConcurrency
+}
+
+func (h RequestLimits) GetMaxItemsPerResponse() int {
+       if h.MaxItemsPerResponse == 0 {
+               return 1000
        }
+       return h.MaxItemsPerResponse
 }
 
 type SystemServiceInstance struct {