sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12
  13         "git.curoverse.com/arvados.git/sdk/go/config"
  14 )
  15
  16 const DefaultConfigFile = "/etc/arvados/config.yml"
  17
  18 type Config struct {
  19         Clusters map[string]Cluster
  20 }
  21
  22 // GetConfig returns the current system config, loading it from
  23 // configFile if needed.
  24 func GetConfig(configFile string) (*Config, error) {
  25         var cfg Config
  26         err := config.LoadFile(&cfg, configFile)
  27         return &cfg, err
  28 }
  29
  30 // GetCluster returns the cluster ID and config for the given
  31 // cluster, or the default/only configured cluster if clusterID is "".
  32 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  33         if clusterID == "" {
  34                 if len(sc.Clusters) == 0 {
  35                         return nil, fmt.Errorf("no clusters configured")
  36                 } else if len(sc.Clusters) > 1 {
  37                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  38                 } else {
  39                         for id, cc := range sc.Clusters {
  40                                 cc.ClusterID = id
  41                                 return &cc, nil
  42                         }
  43                 }
  44         }
  45         if cc, ok := sc.Clusters[clusterID]; !ok {
  46                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  47         } else {
  48                 cc.ClusterID = clusterID
  49                 return &cc, nil
  50         }
  51 }
  52
  53 type Cluster struct {
  54         ClusterID       string `json:"-"`
  55         ManagementToken string
  56         SystemRootToken string
  57         Services        Services
  58         InstanceTypes   InstanceTypeMap
  59         Containers      ContainersConfig
  60         RemoteClusters  map[string]RemoteCluster
  61         PostgreSQL      PostgreSQL
  62
  63         API struct {
  64                 AsyncPermissionsUpdateInterval Duration
  65                 DisabledAPIs                   []string
  66                 MaxIndexDatabaseRead           int
  67                 MaxItemsPerResponse            int
  68                 MaxRequestAmplification        int
  69                 MaxRequestSize                 int
  70                 RailsSessionSecretToken        string
  71                 RequestTimeout                 Duration
  72         }
  73         AuditLogs struct {
  74                 MaxAge             Duration
  75                 MaxDeleteBatch     int
  76                 UnloggedAttributes []string
  77         }
  78         Collections struct {
  79                 BlobSigning           bool
  80                 BlobSigningKey        string
  81                 BlobSigningTTL        Duration
  82                 CollectionVersioning  bool
  83                 DefaultTrashLifetime  Duration
  84                 DefaultReplication    int
  85                 ManagedProperties     map[string]interface{}
  86                 PreserveVersionIfIdle Duration
  87                 TrashSweepInterval    Duration
  88         }
  89         Git struct {
  90                 Repositories string
  91         }
  92         Login struct {
  93                 ProviderAppSecret string
  94                 ProviderAppID     string
  95         }
  96         Mail struct {
  97                 MailchimpAPIKey                string
  98                 MailchimpListID                string
  99                 SendUserSetupNotificationEmail string
 100                 IssueReporterEmailFrom         string
 101                 IssueReporterEmailTo           string
 102                 SupportEmailAddress            string
 103                 EmailFrom                      string
 104         }
 105         SystemLogs struct {
 106                 LogLevel                string
 107                 Format                  string
 108                 MaxRequestLogParamsSize int
 109         }
 110         TLS struct {
 111                 Certificate string
 112                 Key         string
 113                 Insecure    bool
 114         }
 115         Users struct {
 116                 AdminNotifierEmailFrom                string
 117                 AutoAdminFirstUser                    bool
 118                 AutoAdminUserWithEmail                string
 119                 AutoSetupNewUsers                     bool
 120                 AutoSetupNewUsersWithRepository       bool
 121                 AutoSetupNewUsersWithVmUUID           string
 122                 AutoSetupUsernameBlacklist            []string
 123                 EmailSubjectPrefix                    string
 124                 NewInactiveUserNotificationRecipients []string
 125                 NewUserNotificationRecipients         []string
 126                 NewUsersAreActive                     bool
 127                 UserNotifierEmailFrom                 string
 128                 UserProfileNotificationAddress        string
 129         }
 130         Workbench struct {
 131                 ActivationContactLink            string
 132                 APIClientConnectTimeout          Duration
 133                 APIClientReceiveTimeout          Duration
 134                 APIResponseCompression           bool
 135                 ApplicationMimetypesWithViewIcon map[string]struct{}
 136                 ArvadosDocsite                   string
 137                 ArvadosPublicDataDocURL          string
 138                 EnableGettingStartedPopup        bool
 139                 EnablePublicProjectsPage         bool
 140                 FileViewersConfigURL             string
 141                 LogViewerMaxBytes                ByteSize
 142                 MultiSiteSearch                  bool
 143                 Repositories                     bool
 144                 RepositoryCache                  string
 145                 RunningJobLogRecordsToFetch      int
 146                 SecretKeyBase                    string
 147                 SecretToken                      string
 148                 ShowRecentCollectionsOnDashboard bool
 149                 ShowUserAgreementInline          bool
 150                 ShowUserNotifications            bool
 151                 SiteName                         string
 152                 Theme                            string
 153                 UserProfileFormFields            map[string]struct {
 154                         Type                 string
 155                         FormFieldTitle       string
 156                         FormFieldDescription string
 157                         Required             bool
 158                 }
 159                 UserProfileFormMessage string
 160                 VocabularyURL          string
 161         }
 162 }
 163
 164 type Services struct {
 165         Composer       Service
 166         Controller     Service
 167         DispatchCloud  Service
 168         GitHTTP        Service
 169         GitSSH         Service
 170         Health         Service
 171         Keepbalance    Service
 172         Keepproxy      Service
 173         Keepstore      Service
 174         Nodemanager    Service
 175         RailsAPI       Service
 176         SSO            Service
 177         WebDAVDownload Service
 178         WebDAV         Service
 179         WebShell       Service
 180         Websocket      Service
 181         Workbench1     Service
 182         Workbench2     Service
 183 }
 184
 185 type Service struct {
 186         InternalURLs map[URL]ServiceInstance
 187         ExternalURL  URL
 188 }
 189
 190 // URL is a url.URL that is also usable as a JSON key/value.
 191 type URL url.URL
 192
 193 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 194 // used as a JSON key/value.
 195 func (su *URL) UnmarshalText(text []byte) error {
 196         u, err := url.Parse(string(text))
 197         if err == nil {
 198                 *su = URL(*u)
 199         }
 200         return err
 201 }
 202
 203 func (su URL) MarshalText() ([]byte, error) {
 204         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 205 }
 206
 207 type ServiceInstance struct{}
 208
 209 type PostgreSQL struct {
 210         Connection     PostgreSQLConnection
 211         ConnectionPool int
 212 }
 213
 214 type PostgreSQLConnection map[string]string
 215
 216 type RemoteCluster struct {
 217         Host          string
 218         Proxy         bool
 219         Scheme        string
 220         Insecure      bool
 221         ActivateUsers bool
 222 }
 223
 224 type InstanceType struct {
 225         Name            string
 226         ProviderType    string
 227         VCPUs           int
 228         RAM             ByteSize
 229         Scratch         ByteSize
 230         IncludedScratch ByteSize
 231         AddedScratch    ByteSize
 232         Price           float64
 233         Preemptible     bool
 234 }
 235
 236 type ContainersConfig struct {
 237         CloudVMs                    CloudVMsConfig
 238         DefaultKeepCacheRAM         ByteSize
 239         DispatchPrivateKey          string
 240         LogReuseDecisions           bool
 241         MaxComputeVMs               int
 242         MaxDispatchAttempts         int
 243         MaxRetryAttempts            int
 244         StaleLockTimeout            Duration
 245         SupportedDockerImageFormats []string
 246         UsePreemptibleInstances     bool
 247
 248         JobsAPI struct {
 249                 Enable                  string
 250                 GitInternalDir          string
 251                 DefaultDockerImage      string
 252                 CrunchJobWrapper        string
 253                 CrunchJobUser           string
 254                 CrunchRefreshTrigger    string
 255                 ReuseJobIfOutputsDiffer bool
 256         }
 257         Logging struct {
 258                 MaxAge                       Duration
 259                 LogBytesPerEvent             int
 260                 LogSecondsBetweenEvents      int
 261                 LogThrottlePeriod            Duration
 262                 LogThrottleBytes             int
 263                 LogThrottleLines             int
 264                 LimitLogBytesPerJob          int
 265                 LogPartialLineThrottlePeriod Duration
 266                 LogUpdatePeriod              Duration
 267                 LogUpdateSize                ByteSize
 268         }
 269         SLURM struct {
 270                 Managed struct {
 271                         DNSServerConfDir       string
 272                         DNSServerConfTemplate  string
 273                         DNSServerReloadCommand string
 274                         DNSServerUpdateCommand string
 275                         ComputeNodeDomain      string
 276                         ComputeNodeNameservers []string
 277                         AssignNodeHostname     string
 278                 }
 279         }
 280 }
 281
 282 type CloudVMsConfig struct {
 283         Enable bool
 284
 285         BootProbeCommand     string
 286         ImageID              string
 287         MaxCloudOpsPerSecond int
 288         MaxProbesPerSecond   int
 289         PollInterval         Duration
 290         ProbeInterval        Duration
 291         SSHPort              string
 292         SyncInterval         Duration
 293         TimeoutBooting       Duration
 294         TimeoutIdle          Duration
 295         TimeoutProbe         Duration
 296         TimeoutShutdown      Duration
 297         TimeoutSignal        Duration
 298         TimeoutTERM          Duration
 299         ResourceTags         map[string]string
 300         TagKeyPrefix         string
 301
 302         Driver           string
 303         DriverParameters json.RawMessage
 304 }
 305
 306 type InstanceTypeMap map[string]InstanceType
 307
 308 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 309
 310 // UnmarshalJSON handles old config files that provide an array of
 311 // instance types instead of a hash.
 312 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 313         if len(data) > 0 && data[0] == '[' {
 314                 var arr []InstanceType
 315                 err := json.Unmarshal(data, &arr)
 316                 if err != nil {
 317                         return err
 318                 }
 319                 if len(arr) == 0 {
 320                         *it = nil
 321                         return nil
 322                 }
 323                 *it = make(map[string]InstanceType, len(arr))
 324                 for _, t := range arr {
 325                         if _, ok := (*it)[t.Name]; ok {
 326                                 return errDuplicateInstanceTypeName
 327                         }
 328                         if t.ProviderType == "" {
 329                                 t.ProviderType = t.Name
 330                         }
 331                         if t.Scratch == 0 {
 332                                 t.Scratch = t.IncludedScratch + t.AddedScratch
 333                         } else if t.AddedScratch == 0 {
 334                                 t.AddedScratch = t.Scratch - t.IncludedScratch
 335                         } else if t.IncludedScratch == 0 {
 336                                 t.IncludedScratch = t.Scratch - t.AddedScratch
 337                         }
 338
 339                         if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 340                                 return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 341                         }
 342                         (*it)[t.Name] = t
 343                 }
 344                 return nil
 345         }
 346         var hash map[string]InstanceType
 347         err := json.Unmarshal(data, &hash)
 348         if err != nil {
 349                 return err
 350         }
 351         // Fill in Name field (and ProviderType field, if not
 352         // specified) using hash key.
 353         *it = InstanceTypeMap(hash)
 354         for name, t := range *it {
 355                 t.Name = name
 356                 if t.ProviderType == "" {
 357                         t.ProviderType = name
 358                 }
 359                 (*it)[name] = t
 360         }
 361         return nil
 362 }
 363
 364 type ServiceName string
 365
 366 const (
 367         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 368         ServiceNameController    ServiceName = "arvados-controller"
 369         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 370         ServiceNameHealth        ServiceName = "arvados-health"
 371         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 372         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 373         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 374         ServiceNameWebsocket     ServiceName = "arvados-ws"
 375         ServiceNameKeepbalance   ServiceName = "keep-balance"
 376         ServiceNameKeepweb       ServiceName = "keep-web"
 377         ServiceNameKeepproxy     ServiceName = "keepproxy"
 378         ServiceNameKeepstore     ServiceName = "keepstore"
 379 )
 380
 381 // Map returns all services as a map, suitable for iterating over all
 382 // services or looking up a service by name.
 383 func (svcs Services) Map() map[ServiceName]Service {
 384         return map[ServiceName]Service{
 385                 ServiceNameRailsAPI:      svcs.RailsAPI,
 386                 ServiceNameController:    svcs.Controller,
 387                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 388                 ServiceNameHealth:        svcs.Health,
 389                 ServiceNameNodemanager:   svcs.Nodemanager,
 390                 ServiceNameWorkbench1:    svcs.Workbench1,
 391                 ServiceNameWorkbench2:    svcs.Workbench2,
 392                 ServiceNameWebsocket:     svcs.Websocket,
 393                 ServiceNameKeepbalance:   svcs.Keepbalance,
 394                 ServiceNameKeepweb:       svcs.WebDAV,
 395                 ServiceNameKeepproxy:     svcs.Keepproxy,
 396                 ServiceNameKeepstore:     svcs.Keepstore,
 397         }
 398 }