sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12
  13         "git.curoverse.com/arvados.git/sdk/go/config"
  14 )
  15
  16 const DefaultConfigFile = "/etc/arvados/config.yml"
  17
  18 type Config struct {
  19         Clusters map[string]Cluster
  20 }
  21
  22 // GetConfig returns the current system config, loading it from
  23 // configFile if needed.
  24 func GetConfig(configFile string) (*Config, error) {
  25         var cfg Config
  26         err := config.LoadFile(&cfg, configFile)
  27         return &cfg, err
  28 }
  29
  30 // GetCluster returns the cluster ID and config for the given
  31 // cluster, or the default/only configured cluster if clusterID is "".
  32 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  33         if clusterID == "" {
  34                 if len(sc.Clusters) == 0 {
  35                         return nil, fmt.Errorf("no clusters configured")
  36                 } else if len(sc.Clusters) > 1 {
  37                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  38                 } else {
  39                         for id, cc := range sc.Clusters {
  40                                 cc.ClusterID = id
  41                                 return &cc, nil
  42                         }
  43                 }
  44         }
  45         if cc, ok := sc.Clusters[clusterID]; !ok {
  46                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  47         } else {
  48                 cc.ClusterID = clusterID
  49                 return &cc, nil
  50         }
  51 }
  52
  53 type Cluster struct {
  54         ClusterID       string `json:"-"`
  55         ManagementToken string
  56         SystemRootToken string
  57         Services        Services
  58         InstanceTypes   InstanceTypeMap
  59         Containers      ContainersConfig
  60         RemoteClusters  map[string]RemoteCluster
  61         PostgreSQL      PostgreSQL
  62
  63         API struct {
  64                 AsyncPermissionsUpdateInterval Duration
  65                 DisabledAPIs                   []string
  66                 MaxIndexDatabaseRead           int
  67                 MaxItemsPerResponse            int
  68                 MaxRequestAmplification        int
  69                 MaxRequestSize                 int
  70                 RailsSessionSecretToken        string
  71                 RequestTimeout                 Duration
  72         }
  73         AuditLogs struct {
  74                 MaxAge             Duration
  75                 MaxDeleteBatch     int
  76                 UnloggedAttributes []string
  77         }
  78         Collections struct {
  79                 BlobSigning          bool
  80                 BlobSigningKey       string
  81                 BlobSigningTTL       Duration
  82                 CollectionVersioning bool
  83                 DefaultTrashLifetime Duration
  84                 DefaultReplication   int
  85                 ManagedProperties    map[string]struct {
  86                         Value     interface{}
  87                         Function  string
  88                         Protected bool
  89                 }
  90                 PreserveVersionIfIdle Duration
  91                 TrashSweepInterval    Duration
  92                 TrustAllContent       bool
  93         }
  94         Git struct {
  95                 Repositories string
  96         }
  97         Login struct {
  98                 ProviderAppSecret string
  99                 ProviderAppID     string
 100         }
 101         Mail struct {
 102                 MailchimpAPIKey                string
 103                 MailchimpListID                string
 104                 SendUserSetupNotificationEmail bool
 105                 IssueReporterEmailFrom         string
 106                 IssueReporterEmailTo           string
 107                 SupportEmailAddress            string
 108                 EmailFrom                      string
 109         }
 110         SystemLogs struct {
 111                 LogLevel                string
 112                 Format                  string
 113                 MaxRequestLogParamsSize int
 114         }
 115         TLS struct {
 116                 Certificate string
 117                 Key         string
 118                 Insecure    bool
 119         }
 120         Users struct {
 121                 AnonymousUserToken                    string
 122                 AdminNotifierEmailFrom                string
 123                 AutoAdminFirstUser                    bool
 124                 AutoAdminUserWithEmail                string
 125                 AutoSetupNewUsers                     bool
 126                 AutoSetupNewUsersWithRepository       bool
 127                 AutoSetupNewUsersWithVmUUID           string
 128                 AutoSetupUsernameBlacklist            []string
 129                 EmailSubjectPrefix                    string
 130                 NewInactiveUserNotificationRecipients []string
 131                 NewUserNotificationRecipients         []string
 132                 NewUsersAreActive                     bool
 133                 UserNotifierEmailFrom                 string
 134                 UserProfileNotificationAddress        string
 135         }
 136         Workbench struct {
 137                 ActivationContactLink            string
 138                 APIClientConnectTimeout          Duration
 139                 APIClientReceiveTimeout          Duration
 140                 APIResponseCompression           bool
 141                 ApplicationMimetypesWithViewIcon map[string]struct{}
 142                 ArvadosDocsite                   string
 143                 ArvadosPublicDataDocURL          string
 144                 DefaultOpenIdPrefix              string
 145                 EnableGettingStartedPopup        bool
 146                 EnablePublicProjectsPage         bool
 147                 FileViewersConfigURL             string
 148                 LogViewerMaxBytes                ByteSize
 149                 MultiSiteSearch                  string
 150                 ProfilingEnabled                 bool
 151                 Repositories                     bool
 152                 RepositoryCache                  string
 153                 RunningJobLogRecordsToFetch      int
 154                 SecretKeyBase                    string
 155                 ShowRecentCollectionsOnDashboard bool
 156                 ShowUserAgreementInline          bool
 157                 ShowUserNotifications            bool
 158                 SiteName                         string
 159                 Theme                            string
 160                 UserProfileFormFields            map[string]struct {
 161                         Type                 string
 162                         FormFieldTitle       string
 163                         FormFieldDescription string
 164                         Required             bool
 165                         Position             int
 166                         Options              map[string]struct{}
 167                 }
 168                 UserProfileFormMessage string
 169                 VocabularyURL          string
 170         }
 171
 172         EnableBetaController14287 bool
 173 }
 174
 175 type Services struct {
 176         Composer       Service
 177         Controller     Service
 178         DispatchCloud  Service
 179         GitHTTP        Service
 180         GitSSH         Service
 181         Health         Service
 182         Keepbalance    Service
 183         Keepproxy      Service
 184         Keepstore      Service
 185         Nodemanager    Service
 186         RailsAPI       Service
 187         SSO            Service
 188         WebDAVDownload Service
 189         WebDAV         Service
 190         WebShell       Service
 191         Websocket      Service
 192         Workbench1     Service
 193         Workbench2     Service
 194 }
 195
 196 type Service struct {
 197         InternalURLs map[URL]ServiceInstance
 198         ExternalURL  URL
 199 }
 200
 201 // URL is a url.URL that is also usable as a JSON key/value.
 202 type URL url.URL
 203
 204 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 205 // used as a JSON key/value.
 206 func (su *URL) UnmarshalText(text []byte) error {
 207         u, err := url.Parse(string(text))
 208         if err == nil {
 209                 *su = URL(*u)
 210         }
 211         return err
 212 }
 213
 214 func (su URL) MarshalText() ([]byte, error) {
 215         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 216 }
 217
 218 type ServiceInstance struct{}
 219
 220 type PostgreSQL struct {
 221         Connection     PostgreSQLConnection
 222         ConnectionPool int
 223 }
 224
 225 type PostgreSQLConnection map[string]string
 226
 227 type RemoteCluster struct {
 228         Host          string
 229         Proxy         bool
 230         Scheme        string
 231         Insecure      bool
 232         ActivateUsers bool
 233 }
 234
 235 type InstanceType struct {
 236         Name            string
 237         ProviderType    string
 238         VCPUs           int
 239         RAM             ByteSize
 240         Scratch         ByteSize
 241         IncludedScratch ByteSize
 242         AddedScratch    ByteSize
 243         Price           float64
 244         Preemptible     bool
 245 }
 246
 247 type ContainersConfig struct {
 248         CloudVMs                    CloudVMsConfig
 249         DefaultKeepCacheRAM         ByteSize
 250         DispatchPrivateKey          string
 251         LogReuseDecisions           bool
 252         MaxComputeVMs               int
 253         MaxDispatchAttempts         int
 254         MaxRetryAttempts            int
 255         StaleLockTimeout            Duration
 256         SupportedDockerImageFormats []string
 257         UsePreemptibleInstances     bool
 258
 259         JobsAPI struct {
 260                 Enable                  string
 261                 GitInternalDir          string
 262                 DefaultDockerImage      string
 263                 CrunchJobWrapper        string
 264                 CrunchJobUser           string
 265                 CrunchRefreshTrigger    string
 266                 ReuseJobIfOutputsDiffer bool
 267         }
 268         Logging struct {
 269                 MaxAge                       Duration
 270                 LogBytesPerEvent             int
 271                 LogSecondsBetweenEvents      int
 272                 LogThrottlePeriod            Duration
 273                 LogThrottleBytes             int
 274                 LogThrottleLines             int
 275                 LimitLogBytesPerJob          int
 276                 LogPartialLineThrottlePeriod Duration
 277                 LogUpdatePeriod              Duration
 278                 LogUpdateSize                ByteSize
 279         }
 280         SLURM struct {
 281                 Managed struct {
 282                         DNSServerConfDir       string
 283                         DNSServerConfTemplate  string
 284                         DNSServerReloadCommand string
 285                         DNSServerUpdateCommand string
 286                         ComputeNodeDomain      string
 287                         ComputeNodeNameservers []string
 288                         AssignNodeHostname     string
 289                 }
 290         }
 291 }
 292
 293 type CloudVMsConfig struct {
 294         Enable bool
 295
 296         BootProbeCommand     string
 297         ImageID              string
 298         MaxCloudOpsPerSecond int
 299         MaxProbesPerSecond   int
 300         PollInterval         Duration
 301         ProbeInterval        Duration
 302         SSHPort              string
 303         SyncInterval         Duration
 304         TimeoutBooting       Duration
 305         TimeoutIdle          Duration
 306         TimeoutProbe         Duration
 307         TimeoutShutdown      Duration
 308         TimeoutSignal        Duration
 309         TimeoutTERM          Duration
 310         ResourceTags         map[string]string
 311         TagKeyPrefix         string
 312
 313         Driver           string
 314         DriverParameters json.RawMessage
 315 }
 316
 317 type InstanceTypeMap map[string]InstanceType
 318
 319 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 320
 321 // UnmarshalJSON handles old config files that provide an array of
 322 // instance types instead of a hash.
 323 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 324         if len(data) > 0 && data[0] == '[' {
 325                 var arr []InstanceType
 326                 err := json.Unmarshal(data, &arr)
 327                 if err != nil {
 328                         return err
 329                 }
 330                 if len(arr) == 0 {
 331                         *it = nil
 332                         return nil
 333                 }
 334                 *it = make(map[string]InstanceType, len(arr))
 335                 for _, t := range arr {
 336                         if _, ok := (*it)[t.Name]; ok {
 337                                 return errDuplicateInstanceTypeName
 338                         }
 339                         if t.ProviderType == "" {
 340                                 t.ProviderType = t.Name
 341                         }
 342                         if t.Scratch == 0 {
 343                                 t.Scratch = t.IncludedScratch + t.AddedScratch
 344                         } else if t.AddedScratch == 0 {
 345                                 t.AddedScratch = t.Scratch - t.IncludedScratch
 346                         } else if t.IncludedScratch == 0 {
 347                                 t.IncludedScratch = t.Scratch - t.AddedScratch
 348                         }
 349
 350                         if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 351                                 return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 352                         }
 353                         (*it)[t.Name] = t
 354                 }
 355                 return nil
 356         }
 357         var hash map[string]InstanceType
 358         err := json.Unmarshal(data, &hash)
 359         if err != nil {
 360                 return err
 361         }
 362         // Fill in Name field (and ProviderType field, if not
 363         // specified) using hash key.
 364         *it = InstanceTypeMap(hash)
 365         for name, t := range *it {
 366                 t.Name = name
 367                 if t.ProviderType == "" {
 368                         t.ProviderType = name
 369                 }
 370                 (*it)[name] = t
 371         }
 372         return nil
 373 }
 374
 375 type ServiceName string
 376
 377 const (
 378         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 379         ServiceNameController    ServiceName = "arvados-controller"
 380         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 381         ServiceNameHealth        ServiceName = "arvados-health"
 382         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 383         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 384         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 385         ServiceNameWebsocket     ServiceName = "arvados-ws"
 386         ServiceNameKeepbalance   ServiceName = "keep-balance"
 387         ServiceNameKeepweb       ServiceName = "keep-web"
 388         ServiceNameKeepproxy     ServiceName = "keepproxy"
 389         ServiceNameKeepstore     ServiceName = "keepstore"
 390 )
 391
 392 // Map returns all services as a map, suitable for iterating over all
 393 // services or looking up a service by name.
 394 func (svcs Services) Map() map[ServiceName]Service {
 395         return map[ServiceName]Service{
 396                 ServiceNameRailsAPI:      svcs.RailsAPI,
 397                 ServiceNameController:    svcs.Controller,
 398                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 399                 ServiceNameHealth:        svcs.Health,
 400                 ServiceNameNodemanager:   svcs.Nodemanager,
 401                 ServiceNameWorkbench1:    svcs.Workbench1,
 402                 ServiceNameWorkbench2:    svcs.Workbench2,
 403                 ServiceNameWebsocket:     svcs.Websocket,
 404                 ServiceNameKeepbalance:   svcs.Keepbalance,
 405                 ServiceNameKeepweb:       svcs.WebDAV,
 406                 ServiceNameKeepproxy:     svcs.Keepproxy,
 407                 ServiceNameKeepstore:     svcs.Keepstore,
 408         }
 409 }