sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.curoverse.com/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters map[string]Cluster
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         if cc, ok := sc.Clusters[clusterID]; !ok {
  53                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  54         } else {
  55                 cc.ClusterID = clusterID
  56                 return &cc, nil
  57         }
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxPermissionEntries int
  67         MaxUUIDEntries       int
  68 }
  69 type Cluster struct {
  70         ClusterID       string `json:"-"`
  71         ManagementToken string
  72         SystemRootToken string
  73         Services        Services
  74         InstanceTypes   InstanceTypeMap
  75         Containers      ContainersConfig
  76         RemoteClusters  map[string]RemoteCluster
  77         PostgreSQL      PostgreSQL
  78
  79         API struct {
  80                 AsyncPermissionsUpdateInterval Duration
  81                 DisabledAPIs                   StringSet
  82                 MaxIndexDatabaseRead           int
  83                 MaxItemsPerResponse            int
  84                 MaxConcurrentRequests          int
  85                 MaxKeepBlobBuffers             int
  86                 MaxRequestAmplification        int
  87                 MaxRequestSize                 int
  88                 RailsSessionSecretToken        string
  89                 RequestTimeout                 Duration
  90                 SendTimeout                    Duration
  91                 WebsocketClientEventQueue      int
  92                 WebsocketServerEventQueue      int
  93                 KeepServiceRequestTimeout      Duration
  94         }
  95         AuditLogs struct {
  96                 MaxAge             Duration
  97                 MaxDeleteBatch     int
  98                 UnloggedAttributes StringSet
  99         }
 100         Collections struct {
 101                 BlobSigning              bool
 102                 BlobSigningKey           string
 103                 BlobSigningTTL           Duration
 104                 BlobTrash                bool
 105                 BlobTrashLifetime        Duration
 106                 BlobTrashCheckInterval   Duration
 107                 BlobTrashConcurrency     int
 108                 BlobDeleteConcurrency    int
 109                 BlobReplicateConcurrency int
 110                 CollectionVersioning     bool
 111                 DefaultTrashLifetime     Duration
 112                 DefaultReplication       int
 113                 ManagedProperties        map[string]struct {
 114                         Value     interface{}
 115                         Function  string
 116                         Protected bool
 117                 }
 118                 PreserveVersionIfIdle Duration
 119                 TrashSweepInterval    Duration
 120                 TrustAllContent       bool
 121
 122                 BlobMissingReport        string
 123                 BalancePeriod            Duration
 124                 BalanceCollectionBatch   int
 125                 BalanceCollectionBuffers int
 126
 127                 WebDAVCache WebDAVCacheConfig
 128         }
 129         Git struct {
 130                 GitCommand   string
 131                 GitoliteHome string
 132                 Repositories string
 133         }
 134         Login struct {
 135                 ProviderAppSecret  string
 136                 ProviderAppID      string
 137                 LoginCluster       string
 138                 RemoteTokenRefresh Duration
 139         }
 140         Mail struct {
 141                 MailchimpAPIKey                string
 142                 MailchimpListID                string
 143                 SendUserSetupNotificationEmail bool
 144                 IssueReporterEmailFrom         string
 145                 IssueReporterEmailTo           string
 146                 SupportEmailAddress            string
 147                 EmailFrom                      string
 148         }
 149         SystemLogs struct {
 150                 LogLevel                string
 151                 Format                  string
 152                 MaxRequestLogParamsSize int
 153         }
 154         TLS struct {
 155                 Certificate string
 156                 Key         string
 157                 Insecure    bool
 158         }
 159         Users struct {
 160                 AnonymousUserToken                    string
 161                 AdminNotifierEmailFrom                string
 162                 AutoAdminFirstUser                    bool
 163                 AutoAdminUserWithEmail                string
 164                 AutoSetupNewUsers                     bool
 165                 AutoSetupNewUsersWithRepository       bool
 166                 AutoSetupNewUsersWithVmUUID           string
 167                 AutoSetupUsernameBlacklist            StringSet
 168                 EmailSubjectPrefix                    string
 169                 NewInactiveUserNotificationRecipients StringSet
 170                 NewUserNotificationRecipients         StringSet
 171                 NewUsersAreActive                     bool
 172                 UserNotifierEmailFrom                 string
 173                 UserProfileNotificationAddress        string
 174         }
 175         Volumes   map[string]Volume
 176         Workbench struct {
 177                 ActivationContactLink            string
 178                 APIClientConnectTimeout          Duration
 179                 APIClientReceiveTimeout          Duration
 180                 APIResponseCompression           bool
 181                 ApplicationMimetypesWithViewIcon StringSet
 182                 ArvadosDocsite                   string
 183                 ArvadosPublicDataDocURL          string
 184                 DefaultOpenIdPrefix              string
 185                 EnableGettingStartedPopup        bool
 186                 EnablePublicProjectsPage         bool
 187                 FileViewersConfigURL             string
 188                 LogViewerMaxBytes                ByteSize
 189                 MultiSiteSearch                  string
 190                 ProfilingEnabled                 bool
 191                 Repositories                     bool
 192                 RepositoryCache                  string
 193                 RunningJobLogRecordsToFetch      int
 194                 SecretKeyBase                    string
 195                 ShowRecentCollectionsOnDashboard bool
 196                 ShowUserAgreementInline          bool
 197                 ShowUserNotifications            bool
 198                 SiteName                         string
 199                 Theme                            string
 200                 UserProfileFormFields            map[string]struct {
 201                         Type                 string
 202                         FormFieldTitle       string
 203                         FormFieldDescription string
 204                         Required             bool
 205                         Position             int
 206                         Options              map[string]struct{}
 207                 }
 208                 UserProfileFormMessage string
 209                 VocabularyURL          string
 210                 WelcomePageHTML        string
 211         }
 212
 213         EnableBetaController14287 bool
 214 }
 215
 216 type Volume struct {
 217         AccessViaHosts   map[URL]VolumeAccess
 218         ReadOnly         bool
 219         Replication      int
 220         StorageClasses   map[string]bool
 221         Driver           string
 222         DriverParameters json.RawMessage
 223 }
 224
 225 type S3VolumeDriverParameters struct {
 226         AccessKey          string
 227         SecretKey          string
 228         Endpoint           string
 229         Region             string
 230         Bucket             string
 231         LocationConstraint bool
 232         IndexPageSize      int
 233         ConnectTimeout     Duration
 234         ReadTimeout        Duration
 235         RaceWindow         Duration
 236         UnsafeDelete       bool
 237 }
 238
 239 type AzureVolumeDriverParameters struct {
 240         StorageAccountName   string
 241         StorageAccountKey    string
 242         StorageBaseURL       string
 243         ContainerName        string
 244         RequestTimeout       Duration
 245         ListBlobsRetryDelay  Duration
 246         ListBlobsMaxAttempts int
 247 }
 248
 249 type DirectoryVolumeDriverParameters struct {
 250         Root      string
 251         Serialize bool
 252 }
 253
 254 type VolumeAccess struct {
 255         ReadOnly bool
 256 }
 257
 258 type Services struct {
 259         Composer       Service
 260         Controller     Service
 261         DispatchCloud  Service
 262         GitHTTP        Service
 263         GitSSH         Service
 264         Health         Service
 265         Keepbalance    Service
 266         Keepproxy      Service
 267         Keepstore      Service
 268         Nodemanager    Service
 269         RailsAPI       Service
 270         SSO            Service
 271         WebDAVDownload Service
 272         WebDAV         Service
 273         WebShell       Service
 274         Websocket      Service
 275         Workbench1     Service
 276         Workbench2     Service
 277 }
 278
 279 type Service struct {
 280         InternalURLs map[URL]ServiceInstance
 281         ExternalURL  URL
 282 }
 283
 284 // URL is a url.URL that is also usable as a JSON key/value.
 285 type URL url.URL
 286
 287 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 288 // used as a JSON key/value.
 289 func (su *URL) UnmarshalText(text []byte) error {
 290         u, err := url.Parse(string(text))
 291         if err == nil {
 292                 *su = URL(*u)
 293         }
 294         return err
 295 }
 296
 297 func (su URL) MarshalText() ([]byte, error) {
 298         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 299 }
 300
 301 func (su URL) String() string {
 302         return (*url.URL)(&su).String()
 303 }
 304
 305 type ServiceInstance struct {
 306         Rendezvous string `json:",omitempty"`
 307 }
 308
 309 type PostgreSQL struct {
 310         Connection     PostgreSQLConnection
 311         ConnectionPool int
 312 }
 313
 314 type PostgreSQLConnection map[string]string
 315
 316 type RemoteCluster struct {
 317         Host          string
 318         Proxy         bool
 319         Scheme        string
 320         Insecure      bool
 321         ActivateUsers bool
 322 }
 323
 324 type InstanceType struct {
 325         Name            string
 326         ProviderType    string
 327         VCPUs           int
 328         RAM             ByteSize
 329         Scratch         ByteSize
 330         IncludedScratch ByteSize
 331         AddedScratch    ByteSize
 332         Price           float64
 333         Preemptible     bool
 334 }
 335
 336 type ContainersConfig struct {
 337         CloudVMs                    CloudVMsConfig
 338         CrunchRunCommand            string
 339         CrunchRunArgumentsList      []string
 340         DefaultKeepCacheRAM         ByteSize
 341         DispatchPrivateKey          string
 342         LogReuseDecisions           bool
 343         MaxComputeVMs               int
 344         MaxDispatchAttempts         int
 345         MaxRetryAttempts            int
 346         MinRetryPeriod              Duration
 347         ReserveExtraRAM             ByteSize
 348         StaleLockTimeout            Duration
 349         SupportedDockerImageFormats StringSet
 350         UsePreemptibleInstances     bool
 351
 352         JobsAPI struct {
 353                 Enable         string
 354                 GitInternalDir string
 355         }
 356         Logging struct {
 357                 MaxAge                       Duration
 358                 LogBytesPerEvent             int
 359                 LogSecondsBetweenEvents      int
 360                 LogThrottlePeriod            Duration
 361                 LogThrottleBytes             int
 362                 LogThrottleLines             int
 363                 LimitLogBytesPerJob          int
 364                 LogPartialLineThrottlePeriod Duration
 365                 LogUpdatePeriod              Duration
 366                 LogUpdateSize                ByteSize
 367         }
 368         SLURM struct {
 369                 PrioritySpread             int64
 370                 SbatchArgumentsList        []string
 371                 SbatchEnvironmentVariables map[string]string
 372                 Managed                    struct {
 373                         DNSServerConfDir       string
 374                         DNSServerConfTemplate  string
 375                         DNSServerReloadCommand string
 376                         DNSServerUpdateCommand string
 377                         ComputeNodeDomain      string
 378                         ComputeNodeNameservers StringSet
 379                         AssignNodeHostname     string
 380                 }
 381         }
 382 }
 383
 384 type CloudVMsConfig struct {
 385         Enable bool
 386
 387         BootProbeCommand     string
 388         ImageID              string
 389         MaxCloudOpsPerSecond int
 390         MaxProbesPerSecond   int
 391         PollInterval         Duration
 392         ProbeInterval        Duration
 393         SSHPort              string
 394         SyncInterval         Duration
 395         TimeoutBooting       Duration
 396         TimeoutIdle          Duration
 397         TimeoutProbe         Duration
 398         TimeoutShutdown      Duration
 399         TimeoutSignal        Duration
 400         TimeoutTERM          Duration
 401         ResourceTags         map[string]string
 402         TagKeyPrefix         string
 403
 404         Driver           string
 405         DriverParameters json.RawMessage
 406 }
 407
 408 type InstanceTypeMap map[string]InstanceType
 409
 410 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 411
 412 // UnmarshalJSON handles old config files that provide an array of
 413 // instance types instead of a hash.
 414 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 415         if len(data) > 0 && data[0] == '[' {
 416                 var arr []InstanceType
 417                 err := json.Unmarshal(data, &arr)
 418                 if err != nil {
 419                         return err
 420                 }
 421                 if len(arr) == 0 {
 422                         *it = nil
 423                         return nil
 424                 }
 425                 *it = make(map[string]InstanceType, len(arr))
 426                 for _, t := range arr {
 427                         if _, ok := (*it)[t.Name]; ok {
 428                                 return errDuplicateInstanceTypeName
 429                         }
 430                         if t.ProviderType == "" {
 431                                 t.ProviderType = t.Name
 432                         }
 433                         if t.Scratch == 0 {
 434                                 t.Scratch = t.IncludedScratch + t.AddedScratch
 435                         } else if t.AddedScratch == 0 {
 436                                 t.AddedScratch = t.Scratch - t.IncludedScratch
 437                         } else if t.IncludedScratch == 0 {
 438                                 t.IncludedScratch = t.Scratch - t.AddedScratch
 439                         }
 440
 441                         if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 442                                 return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 443                         }
 444                         (*it)[t.Name] = t
 445                 }
 446                 return nil
 447         }
 448         var hash map[string]InstanceType
 449         err := json.Unmarshal(data, &hash)
 450         if err != nil {
 451                 return err
 452         }
 453         // Fill in Name field (and ProviderType field, if not
 454         // specified) using hash key.
 455         *it = InstanceTypeMap(hash)
 456         for name, t := range *it {
 457                 t.Name = name
 458                 if t.ProviderType == "" {
 459                         t.ProviderType = name
 460                 }
 461                 (*it)[name] = t
 462         }
 463         return nil
 464 }
 465
 466 type StringSet map[string]struct{}
 467
 468 // UnmarshalJSON handles old config files that provide an array of
 469 // instance types instead of a hash.
 470 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 471         if len(data) > 0 && data[0] == '[' {
 472                 var arr []string
 473                 err := json.Unmarshal(data, &arr)
 474                 if err != nil {
 475                         return err
 476                 }
 477                 if len(arr) == 0 {
 478                         *ss = nil
 479                         return nil
 480                 }
 481                 *ss = make(map[string]struct{}, len(arr))
 482                 for _, t := range arr {
 483                         (*ss)[t] = struct{}{}
 484                 }
 485                 return nil
 486         }
 487         var hash map[string]struct{}
 488         err := json.Unmarshal(data, &hash)
 489         if err != nil {
 490                 return err
 491         }
 492         *ss = make(map[string]struct{}, len(hash))
 493         for t, _ := range hash {
 494                 (*ss)[t] = struct{}{}
 495         }
 496
 497         return nil
 498 }
 499
 500 type ServiceName string
 501
 502 const (
 503         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 504         ServiceNameController    ServiceName = "arvados-controller"
 505         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 506         ServiceNameHealth        ServiceName = "arvados-health"
 507         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 508         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 509         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 510         ServiceNameWebsocket     ServiceName = "arvados-ws"
 511         ServiceNameKeepbalance   ServiceName = "keep-balance"
 512         ServiceNameKeepweb       ServiceName = "keep-web"
 513         ServiceNameKeepproxy     ServiceName = "keepproxy"
 514         ServiceNameKeepstore     ServiceName = "keepstore"
 515 )
 516
 517 // Map returns all services as a map, suitable for iterating over all
 518 // services or looking up a service by name.
 519 func (svcs Services) Map() map[ServiceName]Service {
 520         return map[ServiceName]Service{
 521                 ServiceNameRailsAPI:      svcs.RailsAPI,
 522                 ServiceNameController:    svcs.Controller,
 523                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 524                 ServiceNameHealth:        svcs.Health,
 525                 ServiceNameNodemanager:   svcs.Nodemanager,
 526                 ServiceNameWorkbench1:    svcs.Workbench1,
 527                 ServiceNameWorkbench2:    svcs.Workbench2,
 528                 ServiceNameWebsocket:     svcs.Websocket,
 529                 ServiceNameKeepbalance:   svcs.Keepbalance,
 530                 ServiceNameKeepweb:       svcs.WebDAV,
 531                 ServiceNameKeepproxy:     svcs.Keepproxy,
 532                 ServiceNameKeepstore:     svcs.Keepstore,
 533         }
 534 }