sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.curoverse.com/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters map[string]Cluster
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         if cc, ok := sc.Clusters[clusterID]; !ok {
  53                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  54         } else {
  55                 cc.ClusterID = clusterID
  56                 return &cc, nil
  57         }
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxCollectionEntries int
  64         MaxCollectionBytes   int64
  65         MaxPermissionEntries int
  66         MaxUUIDEntries       int
  67 }
  68 type Cluster struct {
  69         ClusterID       string `json:"-"`
  70         ManagementToken string
  71         SystemRootToken string
  72         Services        Services
  73         InstanceTypes   InstanceTypeMap
  74         Containers      ContainersConfig
  75         RemoteClusters  map[string]RemoteCluster
  76         PostgreSQL      PostgreSQL
  77
  78         API struct {
  79                 AsyncPermissionsUpdateInterval Duration
  80                 DisabledAPIs                   StringSet
  81                 MaxIndexDatabaseRead           int
  82                 MaxItemsPerResponse            int
  83                 MaxRequestAmplification        int
  84                 MaxRequestSize                 int
  85                 RailsSessionSecretToken        string
  86                 RequestTimeout                 Duration
  87                 SendTimeout                    Duration
  88                 WebsocketClientEventQueue      int
  89                 WebsocketServerEventQueue      int
  90         }
  91         AuditLogs struct {
  92                 MaxAge             Duration
  93                 MaxDeleteBatch     int
  94                 UnloggedAttributes StringSet
  95         }
  96         Collections struct {
  97                 BlobSigning          bool
  98                 BlobSigningKey       string
  99                 BlobSigningTTL       Duration
 100                 CollectionVersioning bool
 101                 DefaultTrashLifetime Duration
 102                 DefaultReplication   int
 103                 ManagedProperties    map[string]struct {
 104                         Value     interface{}
 105                         Function  string
 106                         Protected bool
 107                 }
 108                 PreserveVersionIfIdle Duration
 109                 TrashSweepInterval    Duration
 110                 TrustAllContent       bool
 111
 112                 WebDAVCache WebDAVCacheConfig
 113         }
 114         Git struct {
 115                 Repositories string
 116         }
 117         Login struct {
 118                 ProviderAppSecret string
 119                 ProviderAppID     string
 120         }
 121         Mail struct {
 122                 MailchimpAPIKey                string
 123                 MailchimpListID                string
 124                 SendUserSetupNotificationEmail bool
 125                 IssueReporterEmailFrom         string
 126                 IssueReporterEmailTo           string
 127                 SupportEmailAddress            string
 128                 EmailFrom                      string
 129         }
 130         SystemLogs struct {
 131                 LogLevel                string
 132                 Format                  string
 133                 MaxRequestLogParamsSize int
 134         }
 135         TLS struct {
 136                 Certificate string
 137                 Key         string
 138                 Insecure    bool
 139         }
 140         Users struct {
 141                 AnonymousUserToken                    string
 142                 AdminNotifierEmailFrom                string
 143                 AutoAdminFirstUser                    bool
 144                 AutoAdminUserWithEmail                string
 145                 AnonymousUserToken                    string
 146                 AutoSetupNewUsers                     bool
 147                 AutoSetupNewUsersWithRepository       bool
 148                 AutoSetupNewUsersWithVmUUID           string
 149                 AutoSetupUsernameBlacklist            StringSet
 150                 EmailSubjectPrefix                    string
 151                 NewInactiveUserNotificationRecipients StringSet
 152                 NewUserNotificationRecipients         StringSet
 153                 NewUsersAreActive                     bool
 154                 UserNotifierEmailFrom                 string
 155                 UserProfileNotificationAddress        string
 156         }
 157         Workbench struct {
 158                 ActivationContactLink            string
 159                 APIClientConnectTimeout          Duration
 160                 APIClientReceiveTimeout          Duration
 161                 APIResponseCompression           bool
 162                 ApplicationMimetypesWithViewIcon StringSet
 163                 ArvadosDocsite                   string
 164                 ArvadosPublicDataDocURL          string
 165                 DefaultOpenIdPrefix              string
 166                 EnableGettingStartedPopup        bool
 167                 EnablePublicProjectsPage         bool
 168                 FileViewersConfigURL             string
 169                 LogViewerMaxBytes                ByteSize
 170                 MultiSiteSearch                  string
 171                 ProfilingEnabled                 bool
 172                 Repositories                     bool
 173                 RepositoryCache                  string
 174                 RunningJobLogRecordsToFetch      int
 175                 SecretKeyBase                    string
 176                 ShowRecentCollectionsOnDashboard bool
 177                 ShowUserAgreementInline          bool
 178                 ShowUserNotifications            bool
 179                 SiteName                         string
 180                 Theme                            string
 181                 UserProfileFormFields            map[string]struct {
 182                         Type                 string
 183                         FormFieldTitle       string
 184                         FormFieldDescription string
 185                         Required             bool
 186                         Position             int
 187                         Options              map[string]struct{}
 188                 }
 189                 UserProfileFormMessage string
 190                 VocabularyURL          string
 191         }
 192
 193         EnableBetaController14287 bool
 194 }
 195
 196 type Services struct {
 197         Composer       Service
 198         Controller     Service
 199         DispatchCloud  Service
 200         GitHTTP        Service
 201         GitSSH         Service
 202         Health         Service
 203         Keepbalance    Service
 204         Keepproxy      Service
 205         Keepstore      Service
 206         Nodemanager    Service
 207         RailsAPI       Service
 208         SSO            Service
 209         WebDAVDownload Service
 210         WebDAV         Service
 211         WebShell       Service
 212         Websocket      Service
 213         Workbench1     Service
 214         Workbench2     Service
 215 }
 216
 217 type Service struct {
 218         InternalURLs map[URL]ServiceInstance
 219         ExternalURL  URL
 220 }
 221
 222 // URL is a url.URL that is also usable as a JSON key/value.
 223 type URL url.URL
 224
 225 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 226 // used as a JSON key/value.
 227 func (su *URL) UnmarshalText(text []byte) error {
 228         u, err := url.Parse(string(text))
 229         if err == nil {
 230                 *su = URL(*u)
 231         }
 232         return err
 233 }
 234
 235 func (su URL) MarshalText() ([]byte, error) {
 236         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 237 }
 238
 239 type ServiceInstance struct{}
 240
 241 type PostgreSQL struct {
 242         Connection     PostgreSQLConnection
 243         ConnectionPool int
 244 }
 245
 246 type PostgreSQLConnection map[string]string
 247
 248 type RemoteCluster struct {
 249         Host          string
 250         Proxy         bool
 251         Scheme        string
 252         Insecure      bool
 253         ActivateUsers bool
 254 }
 255
 256 type InstanceType struct {
 257         Name            string
 258         ProviderType    string
 259         VCPUs           int
 260         RAM             ByteSize
 261         Scratch         ByteSize
 262         IncludedScratch ByteSize
 263         AddedScratch    ByteSize
 264         Price           float64
 265         Preemptible     bool
 266 }
 267
 268 type ContainersConfig struct {
 269         CloudVMs                    CloudVMsConfig
 270         CrunchRunCommand            string
 271         CrunchRunArgumentsList      []string
 272         DefaultKeepCacheRAM         ByteSize
 273         DispatchPrivateKey          string
 274         LogReuseDecisions           bool
 275         MaxComputeVMs               int
 276         MaxDispatchAttempts         int
 277         MaxRetryAttempts            int
 278         MinRetryPeriod              Duration
 279         ReserveExtraRAM             ByteSize
 280         StaleLockTimeout            Duration
 281         SupportedDockerImageFormats StringSet
 282         UsePreemptibleInstances     bool
 283
 284         JobsAPI struct {
 285                 Enable                  string
 286                 GitInternalDir          string
 287                 DefaultDockerImage      string
 288                 CrunchJobWrapper        string
 289                 CrunchJobUser           string
 290                 CrunchRefreshTrigger    string
 291                 ReuseJobIfOutputsDiffer bool
 292         }
 293         Logging struct {
 294                 MaxAge                       Duration
 295                 LogBytesPerEvent             int
 296                 LogSecondsBetweenEvents      int
 297                 LogThrottlePeriod            Duration
 298                 LogThrottleBytes             int
 299                 LogThrottleLines             int
 300                 LimitLogBytesPerJob          int
 301                 LogPartialLineThrottlePeriod Duration
 302                 LogUpdatePeriod              Duration
 303                 LogUpdateSize                ByteSize
 304         }
 305         SLURM struct {
 306                 PrioritySpread      int64
 307                 SbatchArgumentsList []string
 308                 Managed             struct {
 309                         DNSServerConfDir       string
 310                         DNSServerConfTemplate  string
 311                         DNSServerReloadCommand string
 312                         DNSServerUpdateCommand string
 313                         ComputeNodeDomain      string
 314                         ComputeNodeNameservers StringSet
 315                         AssignNodeHostname     string
 316                 }
 317         }
 318 }
 319
 320 type CloudVMsConfig struct {
 321         Enable bool
 322
 323         BootProbeCommand     string
 324         ImageID              string
 325         MaxCloudOpsPerSecond int
 326         MaxProbesPerSecond   int
 327         PollInterval         Duration
 328         ProbeInterval        Duration
 329         SSHPort              string
 330         SyncInterval         Duration
 331         TimeoutBooting       Duration
 332         TimeoutIdle          Duration
 333         TimeoutProbe         Duration
 334         TimeoutShutdown      Duration
 335         TimeoutSignal        Duration
 336         TimeoutTERM          Duration
 337         ResourceTags         map[string]string
 338         TagKeyPrefix         string
 339
 340         Driver           string
 341         DriverParameters json.RawMessage
 342 }
 343
 344 type InstanceTypeMap map[string]InstanceType
 345
 346 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 347
 348 // UnmarshalJSON handles old config files that provide an array of
 349 // instance types instead of a hash.
 350 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 351         if len(data) > 0 && data[0] == '[' {
 352                 var arr []InstanceType
 353                 err := json.Unmarshal(data, &arr)
 354                 if err != nil {
 355                         return err
 356                 }
 357                 if len(arr) == 0 {
 358                         *it = nil
 359                         return nil
 360                 }
 361                 *it = make(map[string]InstanceType, len(arr))
 362                 for _, t := range arr {
 363                         if _, ok := (*it)[t.Name]; ok {
 364                                 return errDuplicateInstanceTypeName
 365                         }
 366                         if t.ProviderType == "" {
 367                                 t.ProviderType = t.Name
 368                         }
 369                         if t.Scratch == 0 {
 370                                 t.Scratch = t.IncludedScratch + t.AddedScratch
 371                         } else if t.AddedScratch == 0 {
 372                                 t.AddedScratch = t.Scratch - t.IncludedScratch
 373                         } else if t.IncludedScratch == 0 {
 374                                 t.IncludedScratch = t.Scratch - t.AddedScratch
 375                         }
 376
 377                         if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 378                                 return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 379                         }
 380                         (*it)[t.Name] = t
 381                 }
 382                 return nil
 383         }
 384         var hash map[string]InstanceType
 385         err := json.Unmarshal(data, &hash)
 386         if err != nil {
 387                 return err
 388         }
 389         // Fill in Name field (and ProviderType field, if not
 390         // specified) using hash key.
 391         *it = InstanceTypeMap(hash)
 392         for name, t := range *it {
 393                 t.Name = name
 394                 if t.ProviderType == "" {
 395                         t.ProviderType = name
 396                 }
 397                 (*it)[name] = t
 398         }
 399         return nil
 400 }
 401
 402 type StringSet map[string]struct{}
 403
 404 // UnmarshalJSON handles old config files that provide an array of
 405 // instance types instead of a hash.
 406 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 407         if len(data) > 0 && data[0] == '[' {
 408                 var arr []string
 409                 err := json.Unmarshal(data, &arr)
 410                 if err != nil {
 411                         return err
 412                 }
 413                 if len(arr) == 0 {
 414                         *ss = nil
 415                         return nil
 416                 }
 417                 *ss = make(map[string]struct{}, len(arr))
 418                 for _, t := range arr {
 419                         (*ss)[t] = struct{}{}
 420                 }
 421                 return nil
 422         }
 423         var hash map[string]struct{}
 424         err := json.Unmarshal(data, &hash)
 425         if err != nil {
 426                 return err
 427         }
 428         *ss = make(map[string]struct{}, len(hash))
 429         for t, _ := range hash {
 430                 (*ss)[t] = struct{}{}
 431         }
 432
 433         return nil
 434 }
 435
 436 type ServiceName string
 437
 438 const (
 439         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 440         ServiceNameController    ServiceName = "arvados-controller"
 441         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 442         ServiceNameHealth        ServiceName = "arvados-health"
 443         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 444         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 445         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 446         ServiceNameWebsocket     ServiceName = "arvados-ws"
 447         ServiceNameKeepbalance   ServiceName = "keep-balance"
 448         ServiceNameKeepweb       ServiceName = "keep-web"
 449         ServiceNameKeepproxy     ServiceName = "keepproxy"
 450         ServiceNameKeepstore     ServiceName = "keepstore"
 451 )
 452
 453 // Map returns all services as a map, suitable for iterating over all
 454 // services or looking up a service by name.
 455 func (svcs Services) Map() map[ServiceName]Service {
 456         return map[ServiceName]Service{
 457                 ServiceNameRailsAPI:      svcs.RailsAPI,
 458                 ServiceNameController:    svcs.Controller,
 459                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 460                 ServiceNameHealth:        svcs.Health,
 461                 ServiceNameNodemanager:   svcs.Nodemanager,
 462                 ServiceNameWorkbench1:    svcs.Workbench1,
 463                 ServiceNameWorkbench2:    svcs.Workbench2,
 464                 ServiceNameWebsocket:     svcs.Websocket,
 465                 ServiceNameKeepbalance:   svcs.Keepbalance,
 466                 ServiceNameKeepweb:       svcs.WebDAV,
 467                 ServiceNameKeepproxy:     svcs.Keepproxy,
 468                 ServiceNameKeepstore:     svcs.Keepstore,
 469         }
 470 }