sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12
  13         "git.curoverse.com/arvados.git/sdk/go/config"
  14 )
  15
  16 const DefaultConfigFile = "/etc/arvados/config.yml"
  17
  18 type Config struct {
  19         Clusters map[string]Cluster
  20 }
  21
  22 // GetConfig returns the current system config, loading it from
  23 // configFile if needed.
  24 func GetConfig(configFile string) (*Config, error) {
  25         var cfg Config
  26         err := config.LoadFile(&cfg, configFile)
  27         return &cfg, err
  28 }
  29
  30 // GetCluster returns the cluster ID and config for the given
  31 // cluster, or the default/only configured cluster if clusterID is "".
  32 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  33         if clusterID == "" {
  34                 if len(sc.Clusters) == 0 {
  35                         return nil, fmt.Errorf("no clusters configured")
  36                 } else if len(sc.Clusters) > 1 {
  37                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  38                 } else {
  39                         for id, cc := range sc.Clusters {
  40                                 cc.ClusterID = id
  41                                 return &cc, nil
  42                         }
  43                 }
  44         }
  45         if cc, ok := sc.Clusters[clusterID]; !ok {
  46                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  47         } else {
  48                 cc.ClusterID = clusterID
  49                 return &cc, nil
  50         }
  51 }
  52
  53 type Cluster struct {
  54         ClusterID       string `json:"-"`
  55         ManagementToken string
  56         SystemRootToken string
  57         Services        Services
  58         InstanceTypes   InstanceTypeMap
  59         Containers      ContainersConfig
  60         RemoteClusters  map[string]RemoteCluster
  61         PostgreSQL      PostgreSQL
  62
  63         API struct {
  64                 AsyncPermissionsUpdateInterval Duration
  65                 DisabledAPIs                   []string
  66                 MaxIndexDatabaseRead           int
  67                 MaxItemsPerResponse            int
  68                 MaxRequestAmplification        int
  69                 MaxRequestSize                 int
  70                 RailsSessionSecretToken        string
  71                 RequestTimeout                 Duration
  72         }
  73         AuditLogs struct {
  74                 MaxAge             Duration
  75                 MaxDeleteBatch     int
  76                 UnloggedAttributes []string
  77         }
  78         Collections struct {
  79                 BlobSigning           bool
  80                 BlobSigningKey        string
  81                 BlobSigningTTL        Duration
  82                 CollectionVersioning  bool
  83                 DefaultTrashLifetime  Duration
  84                 DefaultReplication    int
  85                 ManagedProperties     map[string]interface{}
  86                 PreserveVersionIfIdle Duration
  87                 TrashSweepInterval    Duration
  88         }
  89         Git struct {
  90                 Repositories string
  91         }
  92         Login struct {
  93                 ProviderAppSecret string
  94                 ProviderAppID     string
  95         }
  96         Mail struct {
  97                 MailchimpAPIKey                string
  98                 MailchimpListID                string
  99                 SendUserSetupNotificationEmail bool
 100                 IssueReporterEmailFrom         string
 101                 IssueReporterEmailTo           string
 102                 SupportEmailAddress            string
 103                 EmailFrom                      string
 104         }
 105         SystemLogs struct {
 106                 LogLevel                string
 107                 Format                  string
 108                 MaxRequestLogParamsSize int
 109         }
 110         TLS struct {
 111                 Certificate string
 112                 Key         string
 113                 Insecure    bool
 114         }
 115         Users struct {
 116                 AnonymousUserToken                    string
 117                 AdminNotifierEmailFrom                string
 118                 AutoAdminFirstUser                    bool
 119                 AutoAdminUserWithEmail                string
 120                 AutoSetupNewUsers                     bool
 121                 AutoSetupNewUsersWithRepository       bool
 122                 AutoSetupNewUsersWithVmUUID           string
 123                 AutoSetupUsernameBlacklist            []string
 124                 EmailSubjectPrefix                    string
 125                 NewInactiveUserNotificationRecipients []string
 126                 NewUserNotificationRecipients         []string
 127                 NewUsersAreActive                     bool
 128                 UserNotifierEmailFrom                 string
 129                 UserProfileNotificationAddress        string
 130         }
 131         Workbench struct {
 132                 ActivationContactLink            string
 133                 APIClientConnectTimeout          Duration
 134                 APIClientReceiveTimeout          Duration
 135                 APIResponseCompression           bool
 136                 ApplicationMimetypesWithViewIcon map[string]struct{}
 137                 ArvadosDocsite                   string
 138                 ArvadosPublicDataDocURL          string
 139                 EnableGettingStartedPopup        bool
 140                 EnablePublicProjectsPage         bool
 141                 FileViewersConfigURL             string
 142                 LogViewerMaxBytes                ByteSize
 143                 MultiSiteSearch                  string
 144                 Repositories                     bool
 145                 RepositoryCache                  string
 146                 RunningJobLogRecordsToFetch      int
 147                 SecretKeyBase                    string
 148                 ShowRecentCollectionsOnDashboard bool
 149                 ShowUserAgreementInline          bool
 150                 ShowUserNotifications            bool
 151                 SiteName                         string
 152                 Theme                            string
 153                 TrustAllContent                  bool
 154                 UserProfileFormFields            map[string]struct {
 155                         Type                 string
 156                         FormFieldTitle       string
 157                         FormFieldDescription string
 158                         Required             bool
 159                         Position             int
 160                         Options              map[string]struct{}
 161                 }
 162                 UserProfileFormMessage string
 163                 VocabularyURL          string
 164         }
 165
 166         EnableBetaController14287 bool
 167 }
 168
 169 type Services struct {
 170         Composer       Service
 171         Controller     Service
 172         DispatchCloud  Service
 173         GitHTTP        Service
 174         GitSSH         Service
 175         Health         Service
 176         Keepbalance    Service
 177         Keepproxy      Service
 178         Keepstore      Service
 179         Nodemanager    Service
 180         RailsAPI       Service
 181         SSO            Service
 182         WebDAVDownload Service
 183         WebDAV         Service
 184         WebShell       Service
 185         Websocket      Service
 186         Workbench1     Service
 187         Workbench2     Service
 188 }
 189
 190 type Service struct {
 191         InternalURLs map[URL]ServiceInstance
 192         ExternalURL  URL
 193 }
 194
 195 // URL is a url.URL that is also usable as a JSON key/value.
 196 type URL url.URL
 197
 198 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 199 // used as a JSON key/value.
 200 func (su *URL) UnmarshalText(text []byte) error {
 201         u, err := url.Parse(string(text))
 202         if err == nil {
 203                 *su = URL(*u)
 204         }
 205         return err
 206 }
 207
 208 func (su URL) MarshalText() ([]byte, error) {
 209         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 210 }
 211
 212 type ServiceInstance struct{}
 213
 214 type PostgreSQL struct {
 215         Connection     PostgreSQLConnection
 216         ConnectionPool int
 217 }
 218
 219 type PostgreSQLConnection map[string]string
 220
 221 type RemoteCluster struct {
 222         Host          string
 223         Proxy         bool
 224         Scheme        string
 225         Insecure      bool
 226         ActivateUsers bool
 227 }
 228
 229 type InstanceType struct {
 230         Name            string
 231         ProviderType    string
 232         VCPUs           int
 233         RAM             ByteSize
 234         Scratch         ByteSize
 235         IncludedScratch ByteSize
 236         AddedScratch    ByteSize
 237         Price           float64
 238         Preemptible     bool
 239 }
 240
 241 type ContainersConfig struct {
 242         CloudVMs                    CloudVMsConfig
 243         DefaultKeepCacheRAM         ByteSize
 244         DispatchPrivateKey          string
 245         LogReuseDecisions           bool
 246         MaxComputeVMs               int
 247         MaxDispatchAttempts         int
 248         MaxRetryAttempts            int
 249         StaleLockTimeout            Duration
 250         SupportedDockerImageFormats []string
 251         UsePreemptibleInstances     bool
 252
 253         JobsAPI struct {
 254                 Enable                  string
 255                 GitInternalDir          string
 256                 DefaultDockerImage      string
 257                 CrunchJobWrapper        string
 258                 CrunchJobUser           string
 259                 CrunchRefreshTrigger    string
 260                 ReuseJobIfOutputsDiffer bool
 261         }
 262         Logging struct {
 263                 MaxAge                       Duration
 264                 LogBytesPerEvent             int
 265                 LogSecondsBetweenEvents      int
 266                 LogThrottlePeriod            Duration
 267                 LogThrottleBytes             int
 268                 LogThrottleLines             int
 269                 LimitLogBytesPerJob          int
 270                 LogPartialLineThrottlePeriod Duration
 271                 LogUpdatePeriod              Duration
 272                 LogUpdateSize                ByteSize
 273         }
 274         SLURM struct {
 275                 Managed struct {
 276                         DNSServerConfDir       string
 277                         DNSServerConfTemplate  string
 278                         DNSServerReloadCommand string
 279                         DNSServerUpdateCommand string
 280                         ComputeNodeDomain      string
 281                         ComputeNodeNameservers []string
 282                         AssignNodeHostname     string
 283                 }
 284         }
 285 }
 286
 287 type CloudVMsConfig struct {
 288         Enable bool
 289
 290         BootProbeCommand     string
 291         ImageID              string
 292         MaxCloudOpsPerSecond int
 293         MaxProbesPerSecond   int
 294         PollInterval         Duration
 295         ProbeInterval        Duration
 296         SSHPort              string
 297         SyncInterval         Duration
 298         TimeoutBooting       Duration
 299         TimeoutIdle          Duration
 300         TimeoutProbe         Duration
 301         TimeoutShutdown      Duration
 302         TimeoutSignal        Duration
 303         TimeoutTERM          Duration
 304         ResourceTags         map[string]string
 305         TagKeyPrefix         string
 306
 307         Driver           string
 308         DriverParameters json.RawMessage
 309 }
 310
 311 type InstanceTypeMap map[string]InstanceType
 312
 313 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 314
 315 // UnmarshalJSON handles old config files that provide an array of
 316 // instance types instead of a hash.
 317 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 318         if len(data) > 0 && data[0] == '[' {
 319                 var arr []InstanceType
 320                 err := json.Unmarshal(data, &arr)
 321                 if err != nil {
 322                         return err
 323                 }
 324                 if len(arr) == 0 {
 325                         *it = nil
 326                         return nil
 327                 }
 328                 *it = make(map[string]InstanceType, len(arr))
 329                 for _, t := range arr {
 330                         if _, ok := (*it)[t.Name]; ok {
 331                                 return errDuplicateInstanceTypeName
 332                         }
 333                         if t.ProviderType == "" {
 334                                 t.ProviderType = t.Name
 335                         }
 336                         if t.Scratch == 0 {
 337                                 t.Scratch = t.IncludedScratch + t.AddedScratch
 338                         } else if t.AddedScratch == 0 {
 339                                 t.AddedScratch = t.Scratch - t.IncludedScratch
 340                         } else if t.IncludedScratch == 0 {
 341                                 t.IncludedScratch = t.Scratch - t.AddedScratch
 342                         }
 343
 344                         if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 345                                 return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 346                         }
 347                         (*it)[t.Name] = t
 348                 }
 349                 return nil
 350         }
 351         var hash map[string]InstanceType
 352         err := json.Unmarshal(data, &hash)
 353         if err != nil {
 354                 return err
 355         }
 356         // Fill in Name field (and ProviderType field, if not
 357         // specified) using hash key.
 358         *it = InstanceTypeMap(hash)
 359         for name, t := range *it {
 360                 t.Name = name
 361                 if t.ProviderType == "" {
 362                         t.ProviderType = name
 363                 }
 364                 (*it)[name] = t
 365         }
 366         return nil
 367 }
 368
 369 type ServiceName string
 370
 371 const (
 372         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 373         ServiceNameController    ServiceName = "arvados-controller"
 374         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 375         ServiceNameHealth        ServiceName = "arvados-health"
 376         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 377         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 378         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 379         ServiceNameWebsocket     ServiceName = "arvados-ws"
 380         ServiceNameKeepbalance   ServiceName = "keep-balance"
 381         ServiceNameKeepweb       ServiceName = "keep-web"
 382         ServiceNameKeepproxy     ServiceName = "keepproxy"
 383         ServiceNameKeepstore     ServiceName = "keepstore"
 384 )
 385
 386 // Map returns all services as a map, suitable for iterating over all
 387 // services or looking up a service by name.
 388 func (svcs Services) Map() map[ServiceName]Service {
 389         return map[ServiceName]Service{
 390                 ServiceNameRailsAPI:      svcs.RailsAPI,
 391                 ServiceNameController:    svcs.Controller,
 392                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 393                 ServiceNameHealth:        svcs.Health,
 394                 ServiceNameNodemanager:   svcs.Nodemanager,
 395                 ServiceNameWorkbench1:    svcs.Workbench1,
 396                 ServiceNameWorkbench2:    svcs.Workbench2,
 397                 ServiceNameWebsocket:     svcs.Websocket,
 398                 ServiceNameKeepbalance:   svcs.Keepbalance,
 399                 ServiceNameKeepweb:       svcs.WebDAV,
 400                 ServiceNameKeepproxy:     svcs.Keepproxy,
 401                 ServiceNameKeepstore:     svcs.Keepstore,
 402         }
 403 }