sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12
  13         "git.curoverse.com/arvados.git/sdk/go/config"
  14 )
  15
  16 const DefaultConfigFile = "/etc/arvados/config.yml"
  17
  18 type Config struct {
  19         Clusters map[string]Cluster
  20 }
  21
  22 // GetConfig returns the current system config, loading it from
  23 // configFile if needed.
  24 func GetConfig(configFile string) (*Config, error) {
  25         var cfg Config
  26         err := config.LoadFile(&cfg, configFile)
  27         return &cfg, err
  28 }
  29
  30 // GetCluster returns the cluster ID and config for the given
  31 // cluster, or the default/only configured cluster if clusterID is "".
  32 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  33         if clusterID == "" {
  34                 if len(sc.Clusters) == 0 {
  35                         return nil, fmt.Errorf("no clusters configured")
  36                 } else if len(sc.Clusters) > 1 {
  37                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  38                 } else {
  39                         for id, cc := range sc.Clusters {
  40                                 cc.ClusterID = id
  41                                 return &cc, nil
  42                         }
  43                 }
  44         }
  45         if cc, ok := sc.Clusters[clusterID]; !ok {
  46                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  47         } else {
  48                 cc.ClusterID = clusterID
  49                 return &cc, nil
  50         }
  51 }
  52
  53 type Cluster struct {
  54         ClusterID       string `json:"-"`
  55         ManagementToken string
  56         SystemRootToken string
  57         Services        Services
  58         InstanceTypes   InstanceTypeMap
  59         Containers      ContainersConfig
  60         RemoteClusters  map[string]RemoteCluster
  61         PostgreSQL      PostgreSQL
  62
  63         API struct {
  64                 AsyncPermissionsUpdateInterval Duration
  65                 DisabledAPIs                   []string
  66                 MaxIndexDatabaseRead           int
  67                 MaxItemsPerResponse            int
  68                 MaxRequestAmplification        int
  69                 MaxRequestSize                 int
  70                 RailsSessionSecretToken        string
  71                 RequestTimeout                 Duration
  72         }
  73         AuditLogs struct {
  74                 MaxAge             Duration
  75                 MaxDeleteBatch     int
  76                 UnloggedAttributes []string
  77         }
  78         Collections struct {
  79                 BlobSigning           bool
  80                 BlobSigningKey        string
  81                 BlobSigningTTL        Duration
  82                 CollectionVersioning  bool
  83                 DefaultTrashLifetime  Duration
  84                 DefaultReplication    int
  85                 ManagedProperties     map[string]interface{}
  86                 PreserveVersionIfIdle Duration
  87                 TrashSweepInterval    Duration
  88                 TrustAllContent       bool
  89         }
  90         Git struct {
  91                 Repositories string
  92         }
  93         Login struct {
  94                 ProviderAppSecret string
  95                 ProviderAppID     string
  96         }
  97         Mail struct {
  98                 MailchimpAPIKey                string
  99                 MailchimpListID                string
 100                 SendUserSetupNotificationEmail bool
 101                 IssueReporterEmailFrom         string
 102                 IssueReporterEmailTo           string
 103                 SupportEmailAddress            string
 104                 EmailFrom                      string
 105         }
 106         SystemLogs struct {
 107                 LogLevel                string
 108                 Format                  string
 109                 MaxRequestLogParamsSize int
 110         }
 111         TLS struct {
 112                 Certificate string
 113                 Key         string
 114                 Insecure    bool
 115         }
 116         Users struct {
 117                 AnonymousUserToken                    string
 118                 AdminNotifierEmailFrom                string
 119                 AutoAdminFirstUser                    bool
 120                 AutoAdminUserWithEmail                string
 121                 AutoSetupNewUsers                     bool
 122                 AutoSetupNewUsersWithRepository       bool
 123                 AutoSetupNewUsersWithVmUUID           string
 124                 AutoSetupUsernameBlacklist            []string
 125                 EmailSubjectPrefix                    string
 126                 NewInactiveUserNotificationRecipients []string
 127                 NewUserNotificationRecipients         []string
 128                 NewUsersAreActive                     bool
 129                 UserNotifierEmailFrom                 string
 130                 UserProfileNotificationAddress        string
 131         }
 132         Workbench struct {
 133                 ActivationContactLink            string
 134                 APIClientConnectTimeout          Duration
 135                 APIClientReceiveTimeout          Duration
 136                 APIResponseCompression           bool
 137                 ApplicationMimetypesWithViewIcon map[string]struct{}
 138                 ArvadosDocsite                   string
 139                 ArvadosPublicDataDocURL          string
 140                 DefaultOpenIdPrefix              string
 141                 EnableGettingStartedPopup        bool
 142                 EnablePublicProjectsPage         bool
 143                 FileViewersConfigURL             string
 144                 LogViewerMaxBytes                ByteSize
 145                 MultiSiteSearch                  string
 146                 ProfilingEnabled                 bool
 147                 Repositories                     bool
 148                 RepositoryCache                  string
 149                 RunningJobLogRecordsToFetch      int
 150                 SecretKeyBase                    string
 151                 ShowRecentCollectionsOnDashboard bool
 152                 ShowUserAgreementInline          bool
 153                 ShowUserNotifications            bool
 154                 SiteName                         string
 155                 Theme                            string
 156                 UserProfileFormFields            map[string]struct {
 157                         Type                 string
 158                         FormFieldTitle       string
 159                         FormFieldDescription string
 160                         Required             bool
 161                         Position             int
 162                         Options              map[string]struct{}
 163                 }
 164                 UserProfileFormMessage string
 165                 VocabularyURL          string
 166         }
 167
 168         EnableBetaController14287 bool
 169 }
 170
 171 type Services struct {
 172         Composer       Service
 173         Controller     Service
 174         DispatchCloud  Service
 175         GitHTTP        Service
 176         GitSSH         Service
 177         Health         Service
 178         Keepbalance    Service
 179         Keepproxy      Service
 180         Keepstore      Service
 181         Nodemanager    Service
 182         RailsAPI       Service
 183         SSO            Service
 184         WebDAVDownload Service
 185         WebDAV         Service
 186         WebShell       Service
 187         Websocket      Service
 188         Workbench1     Service
 189         Workbench2     Service
 190 }
 191
 192 type Service struct {
 193         InternalURLs map[URL]ServiceInstance
 194         ExternalURL  URL
 195 }
 196
 197 // URL is a url.URL that is also usable as a JSON key/value.
 198 type URL url.URL
 199
 200 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 201 // used as a JSON key/value.
 202 func (su *URL) UnmarshalText(text []byte) error {
 203         u, err := url.Parse(string(text))
 204         if err == nil {
 205                 *su = URL(*u)
 206         }
 207         return err
 208 }
 209
 210 func (su URL) MarshalText() ([]byte, error) {
 211         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 212 }
 213
 214 type ServiceInstance struct{}
 215
 216 type PostgreSQL struct {
 217         Connection     PostgreSQLConnection
 218         ConnectionPool int
 219 }
 220
 221 type PostgreSQLConnection map[string]string
 222
 223 type RemoteCluster struct {
 224         Host          string
 225         Proxy         bool
 226         Scheme        string
 227         Insecure      bool
 228         ActivateUsers bool
 229 }
 230
 231 type InstanceType struct {
 232         Name            string
 233         ProviderType    string
 234         VCPUs           int
 235         RAM             ByteSize
 236         Scratch         ByteSize
 237         IncludedScratch ByteSize
 238         AddedScratch    ByteSize
 239         Price           float64
 240         Preemptible     bool
 241 }
 242
 243 type ContainersConfig struct {
 244         CloudVMs                    CloudVMsConfig
 245         DefaultKeepCacheRAM         ByteSize
 246         DispatchPrivateKey          string
 247         LogReuseDecisions           bool
 248         MaxComputeVMs               int
 249         MaxDispatchAttempts         int
 250         MaxRetryAttempts            int
 251         StaleLockTimeout            Duration
 252         SupportedDockerImageFormats []string
 253         UsePreemptibleInstances     bool
 254
 255         JobsAPI struct {
 256                 Enable                  string
 257                 GitInternalDir          string
 258                 DefaultDockerImage      string
 259                 CrunchJobWrapper        string
 260                 CrunchJobUser           string
 261                 CrunchRefreshTrigger    string
 262                 ReuseJobIfOutputsDiffer bool
 263         }
 264         Logging struct {
 265                 MaxAge                       Duration
 266                 LogBytesPerEvent             int
 267                 LogSecondsBetweenEvents      int
 268                 LogThrottlePeriod            Duration
 269                 LogThrottleBytes             int
 270                 LogThrottleLines             int
 271                 LimitLogBytesPerJob          int
 272                 LogPartialLineThrottlePeriod Duration
 273                 LogUpdatePeriod              Duration
 274                 LogUpdateSize                ByteSize
 275         }
 276         SLURM struct {
 277                 Managed struct {
 278                         DNSServerConfDir       string
 279                         DNSServerConfTemplate  string
 280                         DNSServerReloadCommand string
 281                         DNSServerUpdateCommand string
 282                         ComputeNodeDomain      string
 283                         ComputeNodeNameservers []string
 284                         AssignNodeHostname     string
 285                 }
 286         }
 287 }
 288
 289 type CloudVMsConfig struct {
 290         Enable bool
 291
 292         BootProbeCommand     string
 293         ImageID              string
 294         MaxCloudOpsPerSecond int
 295         MaxProbesPerSecond   int
 296         PollInterval         Duration
 297         ProbeInterval        Duration
 298         SSHPort              string
 299         SyncInterval         Duration
 300         TimeoutBooting       Duration
 301         TimeoutIdle          Duration
 302         TimeoutProbe         Duration
 303         TimeoutShutdown      Duration
 304         TimeoutSignal        Duration
 305         TimeoutTERM          Duration
 306         ResourceTags         map[string]string
 307         TagKeyPrefix         string
 308
 309         Driver           string
 310         DriverParameters json.RawMessage
 311 }
 312
 313 type InstanceTypeMap map[string]InstanceType
 314
 315 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 316
 317 // UnmarshalJSON handles old config files that provide an array of
 318 // instance types instead of a hash.
 319 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 320         if len(data) > 0 && data[0] == '[' {
 321                 var arr []InstanceType
 322                 err := json.Unmarshal(data, &arr)
 323                 if err != nil {
 324                         return err
 325                 }
 326                 if len(arr) == 0 {
 327                         *it = nil
 328                         return nil
 329                 }
 330                 *it = make(map[string]InstanceType, len(arr))
 331                 for _, t := range arr {
 332                         if _, ok := (*it)[t.Name]; ok {
 333                                 return errDuplicateInstanceTypeName
 334                         }
 335                         if t.ProviderType == "" {
 336                                 t.ProviderType = t.Name
 337                         }
 338                         if t.Scratch == 0 {
 339                                 t.Scratch = t.IncludedScratch + t.AddedScratch
 340                         } else if t.AddedScratch == 0 {
 341                                 t.AddedScratch = t.Scratch - t.IncludedScratch
 342                         } else if t.IncludedScratch == 0 {
 343                                 t.IncludedScratch = t.Scratch - t.AddedScratch
 344                         }
 345
 346                         if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 347                                 return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 348                         }
 349                         (*it)[t.Name] = t
 350                 }
 351                 return nil
 352         }
 353         var hash map[string]InstanceType
 354         err := json.Unmarshal(data, &hash)
 355         if err != nil {
 356                 return err
 357         }
 358         // Fill in Name field (and ProviderType field, if not
 359         // specified) using hash key.
 360         *it = InstanceTypeMap(hash)
 361         for name, t := range *it {
 362                 t.Name = name
 363                 if t.ProviderType == "" {
 364                         t.ProviderType = name
 365                 }
 366                 (*it)[name] = t
 367         }
 368         return nil
 369 }
 370
 371 type ServiceName string
 372
 373 const (
 374         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 375         ServiceNameController    ServiceName = "arvados-controller"
 376         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 377         ServiceNameHealth        ServiceName = "arvados-health"
 378         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 379         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 380         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 381         ServiceNameWebsocket     ServiceName = "arvados-ws"
 382         ServiceNameKeepbalance   ServiceName = "keep-balance"
 383         ServiceNameKeepweb       ServiceName = "keep-web"
 384         ServiceNameKeepproxy     ServiceName = "keepproxy"
 385         ServiceNameKeepstore     ServiceName = "keepstore"
 386 )
 387
 388 // Map returns all services as a map, suitable for iterating over all
 389 // services or looking up a service by name.
 390 func (svcs Services) Map() map[ServiceName]Service {
 391         return map[ServiceName]Service{
 392                 ServiceNameRailsAPI:      svcs.RailsAPI,
 393                 ServiceNameController:    svcs.Controller,
 394                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 395                 ServiceNameHealth:        svcs.Health,
 396                 ServiceNameNodemanager:   svcs.Nodemanager,
 397                 ServiceNameWorkbench1:    svcs.Workbench1,
 398                 ServiceNameWorkbench2:    svcs.Workbench2,
 399                 ServiceNameWebsocket:     svcs.Websocket,
 400                 ServiceNameKeepbalance:   svcs.Keepbalance,
 401                 ServiceNameKeepweb:       svcs.WebDAV,
 402                 ServiceNameKeepproxy:     svcs.Keepproxy,
 403                 ServiceNameKeepstore:     svcs.Keepstore,
 404         }
 405 }