sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12
  13         "git.curoverse.com/arvados.git/sdk/go/config"
  14 )
  15
  16 const DefaultConfigFile = "/etc/arvados/config.yml"
  17
  18 type Config struct {
  19         Clusters map[string]Cluster
  20 }
  21
  22 // GetConfig returns the current system config, loading it from
  23 // configFile if needed.
  24 func GetConfig(configFile string) (*Config, error) {
  25         var cfg Config
  26         err := config.LoadFile(&cfg, configFile)
  27         return &cfg, err
  28 }
  29
  30 // GetCluster returns the cluster ID and config for the given
  31 // cluster, or the default/only configured cluster if clusterID is "".
  32 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  33         if clusterID == "" {
  34                 if len(sc.Clusters) == 0 {
  35                         return nil, fmt.Errorf("no clusters configured")
  36                 } else if len(sc.Clusters) > 1 {
  37                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  38                 } else {
  39                         for id, cc := range sc.Clusters {
  40                                 cc.ClusterID = id
  41                                 return &cc, nil
  42                         }
  43                 }
  44         }
  45         if cc, ok := sc.Clusters[clusterID]; !ok {
  46                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  47         } else {
  48                 cc.ClusterID = clusterID
  49                 return &cc, nil
  50         }
  51 }
  52
  53 type Cluster struct {
  54         ClusterID       string `json:"-"`
  55         ManagementToken string
  56         SystemRootToken string
  57         Services        Services
  58         InstanceTypes   InstanceTypeMap
  59         Containers      ContainersConfig
  60         RemoteClusters  map[string]RemoteCluster
  61         PostgreSQL      PostgreSQL
  62
  63         API struct {
  64                 AsyncPermissionsUpdateInterval Duration
  65                 DisabledAPIs                   []string
  66                 MaxIndexDatabaseRead           int
  67                 MaxItemsPerResponse            int
  68                 MaxRequestAmplification        int
  69                 MaxRequestSize                 int
  70                 RailsSessionSecretToken        string
  71                 RequestTimeout                 Duration
  72         }
  73         AuditLogs struct {
  74                 MaxAge             Duration
  75                 MaxDeleteBatch     int
  76                 UnloggedAttributes []string
  77         }
  78         Collections struct {
  79                 BlobSigning           bool
  80                 BlobSigningKey        string
  81                 BlobSigningTTL        Duration
  82                 CollectionVersioning  bool
  83                 DefaultTrashLifetime  Duration
  84                 DefaultReplication    int
  85                 ManagedProperties     map[string]interface{}
  86                 PreserveVersionIfIdle Duration
  87                 TrashSweepInterval    Duration
  88         }
  89         Git struct {
  90                 Repositories string
  91         }
  92         Login struct {
  93                 ProviderAppSecret string
  94                 ProviderAppID     string
  95         }
  96         Mail struct {
  97                 MailchimpAPIKey                string
  98                 MailchimpListID                string
  99                 SendUserSetupNotificationEmail string
 100                 IssueReporterEmailFrom         string
 101                 IssueReporterEmailTo           string
 102                 SupportEmailAddress            string
 103                 EmailFrom                      string
 104         }
 105         SystemLogs struct {
 106                 LogLevel                string
 107                 Format                  string
 108                 MaxRequestLogParamsSize int
 109         }
 110         TLS struct {
 111                 Certificate string
 112                 Key         string
 113                 Insecure    bool
 114         }
 115         Users struct {
 116                 AdminNotifierEmailFrom                string
 117                 AutoAdminFirstUser                    bool
 118                 AutoAdminUserWithEmail                string
 119                 AutoSetupNewUsers                     bool
 120                 AutoSetupNewUsersWithRepository       bool
 121                 AutoSetupNewUsersWithVmUUID           string
 122                 AutoSetupUsernameBlacklist            []string
 123                 EmailSubjectPrefix                    string
 124                 NewInactiveUserNotificationRecipients []string
 125                 NewUserNotificationRecipients         []string
 126                 NewUsersAreActive                     bool
 127                 UserNotifierEmailFrom                 string
 128                 UserProfileNotificationAddress        string
 129         }
 130         Workbench struct {
 131                 ActivationContactLink            string
 132                 APIClientConnectTimeout          Duration
 133                 APIClientReceiveTimeout          Duration
 134                 APIResponseCompression           bool
 135                 ApplicationMimetypesWithViewIcon map[string]struct{}
 136                 ArvadosDocsite                   string
 137                 ArvadosPublicDataDocURL          string
 138                 EnableGettingStartedPopup        bool
 139                 EnablePublicProjectsPage         bool
 140                 FileViewersConfigURL             string
 141                 LogViewerMaxBytes                ByteSize
 142                 MultiSiteSearch                  bool
 143                 Repositories                     bool
 144                 RepositoryCache                  string
 145                 RunningJobLogRecordsToFetch      int
 146                 SecretKeyBase                    string
 147                 SecretToken                      string
 148                 ShowRecentCollectionsOnDashboard bool
 149                 ShowUserAgreementInline          bool
 150                 ShowUserNotifications            bool
 151                 SiteName                         string
 152                 Theme                            string
 153                 UserProfileFormFields            map[string]struct {
 154                         Type                 string
 155                         FormFieldTitle       string
 156                         FormFieldDescription string
 157                         Required             bool
 158                 }
 159                 UserProfileFormMessage string
 160                 VocabularyURL          string
 161         }
 162
 163         EnableBetaController14287 bool
 164 }
 165
 166 type Services struct {
 167         Composer       Service
 168         Controller     Service
 169         DispatchCloud  Service
 170         GitHTTP        Service
 171         GitSSH         Service
 172         Health         Service
 173         Keepbalance    Service
 174         Keepproxy      Service
 175         Keepstore      Service
 176         Nodemanager    Service
 177         RailsAPI       Service
 178         SSO            Service
 179         WebDAVDownload Service
 180         WebDAV         Service
 181         WebShell       Service
 182         Websocket      Service
 183         Workbench1     Service
 184         Workbench2     Service
 185 }
 186
 187 type Service struct {
 188         InternalURLs map[URL]ServiceInstance
 189         ExternalURL  URL
 190 }
 191
 192 // URL is a url.URL that is also usable as a JSON key/value.
 193 type URL url.URL
 194
 195 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 196 // used as a JSON key/value.
 197 func (su *URL) UnmarshalText(text []byte) error {
 198         u, err := url.Parse(string(text))
 199         if err == nil {
 200                 *su = URL(*u)
 201         }
 202         return err
 203 }
 204
 205 func (su URL) MarshalText() ([]byte, error) {
 206         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 207 }
 208
 209 type ServiceInstance struct{}
 210
 211 type PostgreSQL struct {
 212         Connection     PostgreSQLConnection
 213         ConnectionPool int
 214 }
 215
 216 type PostgreSQLConnection map[string]string
 217
 218 type RemoteCluster struct {
 219         Host          string
 220         Proxy         bool
 221         Scheme        string
 222         Insecure      bool
 223         ActivateUsers bool
 224 }
 225
 226 type InstanceType struct {
 227         Name            string
 228         ProviderType    string
 229         VCPUs           int
 230         RAM             ByteSize
 231         Scratch         ByteSize
 232         IncludedScratch ByteSize
 233         AddedScratch    ByteSize
 234         Price           float64
 235         Preemptible     bool
 236 }
 237
 238 type ContainersConfig struct {
 239         CloudVMs                    CloudVMsConfig
 240         DefaultKeepCacheRAM         ByteSize
 241         DispatchPrivateKey          string
 242         LogReuseDecisions           bool
 243         MaxComputeVMs               int
 244         MaxDispatchAttempts         int
 245         MaxRetryAttempts            int
 246         StaleLockTimeout            Duration
 247         SupportedDockerImageFormats []string
 248         UsePreemptibleInstances     bool
 249
 250         JobsAPI struct {
 251                 Enable                  string
 252                 GitInternalDir          string
 253                 DefaultDockerImage      string
 254                 CrunchJobWrapper        string
 255                 CrunchJobUser           string
 256                 CrunchRefreshTrigger    string
 257                 ReuseJobIfOutputsDiffer bool
 258         }
 259         Logging struct {
 260                 MaxAge                       Duration
 261                 LogBytesPerEvent             int
 262                 LogSecondsBetweenEvents      int
 263                 LogThrottlePeriod            Duration
 264                 LogThrottleBytes             int
 265                 LogThrottleLines             int
 266                 LimitLogBytesPerJob          int
 267                 LogPartialLineThrottlePeriod Duration
 268                 LogUpdatePeriod              Duration
 269                 LogUpdateSize                ByteSize
 270         }
 271         SLURM struct {
 272                 Managed struct {
 273                         DNSServerConfDir       string
 274                         DNSServerConfTemplate  string
 275                         DNSServerReloadCommand string
 276                         DNSServerUpdateCommand string
 277                         ComputeNodeDomain      string
 278                         ComputeNodeNameservers []string
 279                         AssignNodeHostname     string
 280                 }
 281         }
 282 }
 283
 284 type CloudVMsConfig struct {
 285         Enable bool
 286
 287         BootProbeCommand     string
 288         ImageID              string
 289         MaxCloudOpsPerSecond int
 290         MaxProbesPerSecond   int
 291         PollInterval         Duration
 292         ProbeInterval        Duration
 293         SSHPort              string
 294         SyncInterval         Duration
 295         TimeoutBooting       Duration
 296         TimeoutIdle          Duration
 297         TimeoutProbe         Duration
 298         TimeoutShutdown      Duration
 299         TimeoutSignal        Duration
 300         TimeoutTERM          Duration
 301         ResourceTags         map[string]string
 302         TagKeyPrefix         string
 303
 304         Driver           string
 305         DriverParameters json.RawMessage
 306 }
 307
 308 type InstanceTypeMap map[string]InstanceType
 309
 310 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 311
 312 // UnmarshalJSON handles old config files that provide an array of
 313 // instance types instead of a hash.
 314 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 315         if len(data) > 0 && data[0] == '[' {
 316                 var arr []InstanceType
 317                 err := json.Unmarshal(data, &arr)
 318                 if err != nil {
 319                         return err
 320                 }
 321                 if len(arr) == 0 {
 322                         *it = nil
 323                         return nil
 324                 }
 325                 *it = make(map[string]InstanceType, len(arr))
 326                 for _, t := range arr {
 327                         if _, ok := (*it)[t.Name]; ok {
 328                                 return errDuplicateInstanceTypeName
 329                         }
 330                         if t.ProviderType == "" {
 331                                 t.ProviderType = t.Name
 332                         }
 333                         if t.Scratch == 0 {
 334                                 t.Scratch = t.IncludedScratch + t.AddedScratch
 335                         } else if t.AddedScratch == 0 {
 336                                 t.AddedScratch = t.Scratch - t.IncludedScratch
 337                         } else if t.IncludedScratch == 0 {
 338                                 t.IncludedScratch = t.Scratch - t.AddedScratch
 339                         }
 340
 341                         if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 342                                 return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 343                         }
 344                         (*it)[t.Name] = t
 345                 }
 346                 return nil
 347         }
 348         var hash map[string]InstanceType
 349         err := json.Unmarshal(data, &hash)
 350         if err != nil {
 351                 return err
 352         }
 353         // Fill in Name field (and ProviderType field, if not
 354         // specified) using hash key.
 355         *it = InstanceTypeMap(hash)
 356         for name, t := range *it {
 357                 t.Name = name
 358                 if t.ProviderType == "" {
 359                         t.ProviderType = name
 360                 }
 361                 (*it)[name] = t
 362         }
 363         return nil
 364 }
 365
 366 type ServiceName string
 367
 368 const (
 369         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 370         ServiceNameController    ServiceName = "arvados-controller"
 371         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 372         ServiceNameHealth        ServiceName = "arvados-health"
 373         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 374         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 375         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 376         ServiceNameWebsocket     ServiceName = "arvados-ws"
 377         ServiceNameKeepbalance   ServiceName = "keep-balance"
 378         ServiceNameKeepweb       ServiceName = "keep-web"
 379         ServiceNameKeepproxy     ServiceName = "keepproxy"
 380         ServiceNameKeepstore     ServiceName = "keepstore"
 381 )
 382
 383 // Map returns all services as a map, suitable for iterating over all
 384 // services or looking up a service by name.
 385 func (svcs Services) Map() map[ServiceName]Service {
 386         return map[ServiceName]Service{
 387                 ServiceNameRailsAPI:      svcs.RailsAPI,
 388                 ServiceNameController:    svcs.Controller,
 389                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 390                 ServiceNameHealth:        svcs.Health,
 391                 ServiceNameNodemanager:   svcs.Nodemanager,
 392                 ServiceNameWorkbench1:    svcs.Workbench1,
 393                 ServiceNameWorkbench2:    svcs.Workbench2,
 394                 ServiceNameWebsocket:     svcs.Websocket,
 395                 ServiceNameKeepbalance:   svcs.Keepbalance,
 396                 ServiceNameKeepweb:       svcs.WebDAV,
 397                 ServiceNameKeepproxy:     svcs.Keepproxy,
 398                 ServiceNameKeepstore:     svcs.Keepstore,
 399         }
 400 }