sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12
  13         "git.curoverse.com/arvados.git/sdk/go/config"
  14 )
  15
  16 const DefaultConfigFile = "/etc/arvados/config.yml"
  17
  18 type Config struct {
  19         Clusters map[string]Cluster
  20 }
  21
  22 // GetConfig returns the current system config, loading it from
  23 // configFile if needed.
  24 func GetConfig(configFile string) (*Config, error) {
  25         var cfg Config
  26         err := config.LoadFile(&cfg, configFile)
  27         return &cfg, err
  28 }
  29
  30 // GetCluster returns the cluster ID and config for the given
  31 // cluster, or the default/only configured cluster if clusterID is "".
  32 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  33         if clusterID == "" {
  34                 if len(sc.Clusters) == 0 {
  35                         return nil, fmt.Errorf("no clusters configured")
  36                 } else if len(sc.Clusters) > 1 {
  37                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  38                 } else {
  39                         for id, cc := range sc.Clusters {
  40                                 cc.ClusterID = id
  41                                 return &cc, nil
  42                         }
  43                 }
  44         }
  45         if cc, ok := sc.Clusters[clusterID]; !ok {
  46                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  47         } else {
  48                 cc.ClusterID = clusterID
  49                 return &cc, nil
  50         }
  51 }
  52
  53 type Cluster struct {
  54         ClusterID       string `json:"-"`
  55         ManagementToken string
  56         SystemRootToken string
  57         Services        Services
  58         InstanceTypes   InstanceTypeMap
  59         Containers      ContainersConfig
  60         RemoteClusters  map[string]RemoteCluster
  61         PostgreSQL      PostgreSQL
  62
  63         API struct {
  64                 AsyncPermissionsUpdateInterval Duration
  65                 DisabledAPIs                   []string
  66                 MaxIndexDatabaseRead           int
  67                 MaxItemsPerResponse            int
  68                 MaxRequestAmplification        int
  69                 MaxRequestSize                 int
  70                 RailsSessionSecretToken        string
  71                 RequestTimeout                 Duration
  72         }
  73         AuditLogs struct {
  74                 MaxAge             Duration
  75                 MaxDeleteBatch     int
  76                 UnloggedAttributes []string
  77         }
  78         Collections struct {
  79                 BlobSigning           bool
  80                 BlobSigningKey        string
  81                 BlobSigningTTL        Duration
  82                 CollectionVersioning  bool
  83                 DefaultTrashLifetime  Duration
  84                 DefaultReplication    int
  85                 ManagedProperties     map[string]interface{}
  86                 PreserveVersionIfIdle Duration
  87                 TrashSweepInterval    Duration
  88         }
  89         Git struct {
  90                 Repositories string
  91         }
  92         Login struct {
  93                 ProviderAppSecret string
  94                 ProviderAppID     string
  95         }
  96         Mail struct {
  97                 MailchimpAPIKey                string
  98                 MailchimpListID                string
  99                 SendUserSetupNotificationEmail bool
 100                 IssueReporterEmailFrom         string
 101                 IssueReporterEmailTo           string
 102                 SupportEmailAddress            string
 103                 EmailFrom                      string
 104         }
 105         SystemLogs struct {
 106                 LogLevel                string
 107                 Format                  string
 108                 MaxRequestLogParamsSize int
 109         }
 110         TLS struct {
 111                 Certificate string
 112                 Key         string
 113                 Insecure    bool
 114         }
 115         Users struct {
 116                 AnonymousUserToken                    string
 117                 AdminNotifierEmailFrom                string
 118                 AutoAdminFirstUser                    bool
 119                 AutoAdminUserWithEmail                string
 120                 AutoSetupNewUsers                     bool
 121                 AutoSetupNewUsersWithRepository       bool
 122                 AutoSetupNewUsersWithVmUUID           string
 123                 AutoSetupUsernameBlacklist            []string
 124                 EmailSubjectPrefix                    string
 125                 NewInactiveUserNotificationRecipients []string
 126                 NewUserNotificationRecipients         []string
 127                 NewUsersAreActive                     bool
 128                 UserNotifierEmailFrom                 string
 129                 UserProfileNotificationAddress        string
 130         }
 131         Workbench struct {
 132                 ActivationContactLink            string
 133                 APIClientConnectTimeout          Duration
 134                 APIClientReceiveTimeout          Duration
 135                 APIResponseCompression           bool
 136                 ApplicationMimetypesWithViewIcon map[string]struct{}
 137                 ArvadosDocsite                   string
 138                 ArvadosPublicDataDocURL          string
 139                 EnableGettingStartedPopup        bool
 140                 EnablePublicProjectsPage         bool
 141                 FileViewersConfigURL             string
 142                 LogViewerMaxBytes                ByteSize
 143                 MultiSiteSearch                  string
 144                 ProfilingEnabled                 bool
 145                 Repositories                     bool
 146                 RepositoryCache                  string
 147                 RunningJobLogRecordsToFetch      int
 148                 SecretKeyBase                    string
 149                 ShowRecentCollectionsOnDashboard bool
 150                 ShowUserAgreementInline          bool
 151                 ShowUserNotifications            bool
 152                 SiteName                         string
 153                 Theme                            string
 154                 TrustAllContent                  bool
 155                 UserProfileFormFields            map[string]struct {
 156                         Type                 string
 157                         FormFieldTitle       string
 158                         FormFieldDescription string
 159                         Required             bool
 160                         Position             int
 161                         Options              map[string]struct{}
 162                 }
 163                 UserProfileFormMessage string
 164                 VocabularyURL          string
 165         }
 166
 167         EnableBetaController14287 bool
 168 }
 169
 170 type Services struct {
 171         Composer       Service
 172         Controller     Service
 173         DispatchCloud  Service
 174         GitHTTP        Service
 175         GitSSH         Service
 176         Health         Service
 177         Keepbalance    Service
 178         Keepproxy      Service
 179         Keepstore      Service
 180         Nodemanager    Service
 181         RailsAPI       Service
 182         SSO            Service
 183         WebDAVDownload Service
 184         WebDAV         Service
 185         WebShell       Service
 186         Websocket      Service
 187         Workbench1     Service
 188         Workbench2     Service
 189 }
 190
 191 type Service struct {
 192         InternalURLs map[URL]ServiceInstance
 193         ExternalURL  URL
 194 }
 195
 196 // URL is a url.URL that is also usable as a JSON key/value.
 197 type URL url.URL
 198
 199 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 200 // used as a JSON key/value.
 201 func (su *URL) UnmarshalText(text []byte) error {
 202         u, err := url.Parse(string(text))
 203         if err == nil {
 204                 *su = URL(*u)
 205         }
 206         return err
 207 }
 208
 209 func (su URL) MarshalText() ([]byte, error) {
 210         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 211 }
 212
 213 type ServiceInstance struct{}
 214
 215 type PostgreSQL struct {
 216         Connection     PostgreSQLConnection
 217         ConnectionPool int
 218 }
 219
 220 type PostgreSQLConnection map[string]string
 221
 222 type RemoteCluster struct {
 223         Host          string
 224         Proxy         bool
 225         Scheme        string
 226         Insecure      bool
 227         ActivateUsers bool
 228 }
 229
 230 type InstanceType struct {
 231         Name            string
 232         ProviderType    string
 233         VCPUs           int
 234         RAM             ByteSize
 235         Scratch         ByteSize
 236         IncludedScratch ByteSize
 237         AddedScratch    ByteSize
 238         Price           float64
 239         Preemptible     bool
 240 }
 241
 242 type ContainersConfig struct {
 243         CloudVMs                    CloudVMsConfig
 244         DefaultKeepCacheRAM         ByteSize
 245         DispatchPrivateKey          string
 246         LogReuseDecisions           bool
 247         MaxComputeVMs               int
 248         MaxDispatchAttempts         int
 249         MaxRetryAttempts            int
 250         StaleLockTimeout            Duration
 251         SupportedDockerImageFormats []string
 252         UsePreemptibleInstances     bool
 253
 254         JobsAPI struct {
 255                 Enable                  string
 256                 GitInternalDir          string
 257                 DefaultDockerImage      string
 258                 CrunchJobWrapper        string
 259                 CrunchJobUser           string
 260                 CrunchRefreshTrigger    string
 261                 ReuseJobIfOutputsDiffer bool
 262         }
 263         Logging struct {
 264                 MaxAge                       Duration
 265                 LogBytesPerEvent             int
 266                 LogSecondsBetweenEvents      int
 267                 LogThrottlePeriod            Duration
 268                 LogThrottleBytes             int
 269                 LogThrottleLines             int
 270                 LimitLogBytesPerJob          int
 271                 LogPartialLineThrottlePeriod Duration
 272                 LogUpdatePeriod              Duration
 273                 LogUpdateSize                ByteSize
 274         }
 275         SLURM struct {
 276                 Managed struct {
 277                         DNSServerConfDir       string
 278                         DNSServerConfTemplate  string
 279                         DNSServerReloadCommand string
 280                         DNSServerUpdateCommand string
 281                         ComputeNodeDomain      string
 282                         ComputeNodeNameservers []string
 283                         AssignNodeHostname     string
 284                 }
 285         }
 286 }
 287
 288 type CloudVMsConfig struct {
 289         Enable bool
 290
 291         BootProbeCommand     string
 292         ImageID              string
 293         MaxCloudOpsPerSecond int
 294         MaxProbesPerSecond   int
 295         PollInterval         Duration
 296         ProbeInterval        Duration
 297         SSHPort              string
 298         SyncInterval         Duration
 299         TimeoutBooting       Duration
 300         TimeoutIdle          Duration
 301         TimeoutProbe         Duration
 302         TimeoutShutdown      Duration
 303         TimeoutSignal        Duration
 304         TimeoutTERM          Duration
 305         ResourceTags         map[string]string
 306         TagKeyPrefix         string
 307
 308         Driver           string
 309         DriverParameters json.RawMessage
 310 }
 311
 312 type InstanceTypeMap map[string]InstanceType
 313
 314 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 315
 316 // UnmarshalJSON handles old config files that provide an array of
 317 // instance types instead of a hash.
 318 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 319         if len(data) > 0 && data[0] == '[' {
 320                 var arr []InstanceType
 321                 err := json.Unmarshal(data, &arr)
 322                 if err != nil {
 323                         return err
 324                 }
 325                 if len(arr) == 0 {
 326                         *it = nil
 327                         return nil
 328                 }
 329                 *it = make(map[string]InstanceType, len(arr))
 330                 for _, t := range arr {
 331                         if _, ok := (*it)[t.Name]; ok {
 332                                 return errDuplicateInstanceTypeName
 333                         }
 334                         if t.ProviderType == "" {
 335                                 t.ProviderType = t.Name
 336                         }
 337                         if t.Scratch == 0 {
 338                                 t.Scratch = t.IncludedScratch + t.AddedScratch
 339                         } else if t.AddedScratch == 0 {
 340                                 t.AddedScratch = t.Scratch - t.IncludedScratch
 341                         } else if t.IncludedScratch == 0 {
 342                                 t.IncludedScratch = t.Scratch - t.AddedScratch
 343                         }
 344
 345                         if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 346                                 return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 347                         }
 348                         (*it)[t.Name] = t
 349                 }
 350                 return nil
 351         }
 352         var hash map[string]InstanceType
 353         err := json.Unmarshal(data, &hash)
 354         if err != nil {
 355                 return err
 356         }
 357         // Fill in Name field (and ProviderType field, if not
 358         // specified) using hash key.
 359         *it = InstanceTypeMap(hash)
 360         for name, t := range *it {
 361                 t.Name = name
 362                 if t.ProviderType == "" {
 363                         t.ProviderType = name
 364                 }
 365                 (*it)[name] = t
 366         }
 367         return nil
 368 }
 369
 370 type ServiceName string
 371
 372 const (
 373         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 374         ServiceNameController    ServiceName = "arvados-controller"
 375         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 376         ServiceNameHealth        ServiceName = "arvados-health"
 377         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 378         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 379         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 380         ServiceNameWebsocket     ServiceName = "arvados-ws"
 381         ServiceNameKeepbalance   ServiceName = "keep-balance"
 382         ServiceNameKeepweb       ServiceName = "keep-web"
 383         ServiceNameKeepproxy     ServiceName = "keepproxy"
 384         ServiceNameKeepstore     ServiceName = "keepstore"
 385 )
 386
 387 // Map returns all services as a map, suitable for iterating over all
 388 // services or looking up a service by name.
 389 func (svcs Services) Map() map[ServiceName]Service {
 390         return map[ServiceName]Service{
 391                 ServiceNameRailsAPI:      svcs.RailsAPI,
 392                 ServiceNameController:    svcs.Controller,
 393                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 394                 ServiceNameHealth:        svcs.Health,
 395                 ServiceNameNodemanager:   svcs.Nodemanager,
 396                 ServiceNameWorkbench1:    svcs.Workbench1,
 397                 ServiceNameWorkbench2:    svcs.Workbench2,
 398                 ServiceNameWebsocket:     svcs.Websocket,
 399                 ServiceNameKeepbalance:   svcs.Keepbalance,
 400                 ServiceNameKeepweb:       svcs.WebDAV,
 401                 ServiceNameKeepproxy:     svcs.Keepproxy,
 402                 ServiceNameKeepstore:     svcs.Keepstore,
 403         }
 404 }