sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.curoverse.com/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters map[string]Cluster
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         if cc, ok := sc.Clusters[clusterID]; !ok {
  53                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  54         } else {
  55                 cc.ClusterID = clusterID
  56                 return &cc, nil
  57         }
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxPermissionEntries int
  67         MaxUUIDEntries       int
  68 }
  69 type Cluster struct {
  70         ClusterID       string `json:"-"`
  71         ManagementToken string
  72         SystemRootToken string
  73         Services        Services
  74         InstanceTypes   InstanceTypeMap
  75         Containers      ContainersConfig
  76         RemoteClusters  map[string]RemoteCluster
  77         PostgreSQL      PostgreSQL
  78
  79         API struct {
  80                 AsyncPermissionsUpdateInterval Duration
  81                 DisabledAPIs                   StringSet
  82                 MaxIndexDatabaseRead           int
  83                 MaxItemsPerResponse            int
  84                 MaxConcurrentRequests          int
  85                 MaxKeepBlobBuffers             int
  86                 MaxRequestAmplification        int
  87                 MaxRequestSize                 int
  88                 RailsSessionSecretToken        string
  89                 RequestTimeout                 Duration
  90                 SendTimeout                    Duration
  91                 WebsocketClientEventQueue      int
  92                 WebsocketServerEventQueue      int
  93                 KeepServiceRequestTimeout      Duration
  94         }
  95         AuditLogs struct {
  96                 MaxAge             Duration
  97                 MaxDeleteBatch     int
  98                 UnloggedAttributes StringSet
  99         }
 100         Collections struct {
 101                 BlobSigning              bool
 102                 BlobSigningKey           string
 103                 BlobSigningTTL           Duration
 104                 BlobTrash                bool
 105                 BlobTrashLifetime        Duration
 106                 BlobTrashCheckInterval   Duration
 107                 BlobTrashConcurrency     int
 108                 BlobDeleteConcurrency    int
 109                 BlobReplicateConcurrency int
 110                 CollectionVersioning     bool
 111                 DefaultTrashLifetime     Duration
 112                 DefaultReplication       int
 113                 ManagedProperties        map[string]struct {
 114                         Value     interface{}
 115                         Function  string
 116                         Protected bool
 117                 }
 118                 PreserveVersionIfIdle Duration
 119                 TrashSweepInterval    Duration
 120                 TrustAllContent       bool
 121
 122                 WebDAVCache WebDAVCacheConfig
 123         }
 124         Git struct {
 125                 GitCommand   string
 126                 GitoliteHome string
 127                 Repositories string
 128         }
 129         Login struct {
 130                 ProviderAppSecret  string
 131                 ProviderAppID      string
 132                 LoginCluster       string
 133                 RemoteTokenRefresh Duration
 134         }
 135         Mail struct {
 136                 MailchimpAPIKey                string
 137                 MailchimpListID                string
 138                 SendUserSetupNotificationEmail bool
 139                 IssueReporterEmailFrom         string
 140                 IssueReporterEmailTo           string
 141                 SupportEmailAddress            string
 142                 EmailFrom                      string
 143         }
 144         SystemLogs struct {
 145                 LogLevel                string
 146                 Format                  string
 147                 MaxRequestLogParamsSize int
 148         }
 149         TLS struct {
 150                 Certificate string
 151                 Key         string
 152                 Insecure    bool
 153         }
 154         Users struct {
 155                 AnonymousUserToken                    string
 156                 AdminNotifierEmailFrom                string
 157                 AutoAdminFirstUser                    bool
 158                 AutoAdminUserWithEmail                string
 159                 AutoSetupNewUsers                     bool
 160                 AutoSetupNewUsersWithRepository       bool
 161                 AutoSetupNewUsersWithVmUUID           string
 162                 AutoSetupUsernameBlacklist            StringSet
 163                 EmailSubjectPrefix                    string
 164                 NewInactiveUserNotificationRecipients StringSet
 165                 NewUserNotificationRecipients         StringSet
 166                 NewUsersAreActive                     bool
 167                 UserNotifierEmailFrom                 string
 168                 UserProfileNotificationAddress        string
 169         }
 170         Volumes   map[string]Volume
 171         Workbench struct {
 172                 ActivationContactLink            string
 173                 APIClientConnectTimeout          Duration
 174                 APIClientReceiveTimeout          Duration
 175                 APIResponseCompression           bool
 176                 ApplicationMimetypesWithViewIcon StringSet
 177                 ArvadosDocsite                   string
 178                 ArvadosPublicDataDocURL          string
 179                 DefaultOpenIdPrefix              string
 180                 EnableGettingStartedPopup        bool
 181                 EnablePublicProjectsPage         bool
 182                 FileViewersConfigURL             string
 183                 LogViewerMaxBytes                ByteSize
 184                 MultiSiteSearch                  string
 185                 ProfilingEnabled                 bool
 186                 Repositories                     bool
 187                 RepositoryCache                  string
 188                 RunningJobLogRecordsToFetch      int
 189                 SecretKeyBase                    string
 190                 ShowRecentCollectionsOnDashboard bool
 191                 ShowUserAgreementInline          bool
 192                 ShowUserNotifications            bool
 193                 SiteName                         string
 194                 Theme                            string
 195                 UserProfileFormFields            map[string]struct {
 196                         Type                 string
 197                         FormFieldTitle       string
 198                         FormFieldDescription string
 199                         Required             bool
 200                         Position             int
 201                         Options              map[string]struct{}
 202                 }
 203                 UserProfileFormMessage string
 204                 VocabularyURL          string
 205         }
 206
 207         EnableBetaController14287 bool
 208 }
 209
 210 type Volume struct {
 211         AccessViaHosts   map[URL]VolumeAccess
 212         ReadOnly         bool
 213         Replication      int
 214         StorageClasses   map[string]bool
 215         Driver           string
 216         DriverParameters json.RawMessage
 217 }
 218
 219 type S3VolumeDriverParameters struct {
 220         AccessKey          string
 221         SecretKey          string
 222         Endpoint           string
 223         Region             string
 224         Bucket             string
 225         LocationConstraint bool
 226         IndexPageSize      int
 227         ConnectTimeout     Duration
 228         ReadTimeout        Duration
 229         RaceWindow         Duration
 230         UnsafeDelete       bool
 231 }
 232
 233 type AzureVolumeDriverParameters struct {
 234         StorageAccountName   string
 235         StorageAccountKey    string
 236         StorageBaseURL       string
 237         ContainerName        string
 238         RequestTimeout       Duration
 239         ListBlobsRetryDelay  Duration
 240         ListBlobsMaxAttempts int
 241 }
 242
 243 type DirectoryVolumeDriverParameters struct {
 244         Root      string
 245         Serialize bool
 246 }
 247
 248 type VolumeAccess struct {
 249         ReadOnly bool
 250 }
 251
 252 type Services struct {
 253         Composer       Service
 254         Controller     Service
 255         DispatchCloud  Service
 256         GitHTTP        Service
 257         GitSSH         Service
 258         Health         Service
 259         Keepbalance    Service
 260         Keepproxy      Service
 261         Keepstore      Service
 262         Nodemanager    Service
 263         RailsAPI       Service
 264         SSO            Service
 265         WebDAVDownload Service
 266         WebDAV         Service
 267         WebShell       Service
 268         Websocket      Service
 269         Workbench1     Service
 270         Workbench2     Service
 271 }
 272
 273 type Service struct {
 274         InternalURLs map[URL]ServiceInstance
 275         ExternalURL  URL
 276 }
 277
 278 // URL is a url.URL that is also usable as a JSON key/value.
 279 type URL url.URL
 280
 281 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 282 // used as a JSON key/value.
 283 func (su *URL) UnmarshalText(text []byte) error {
 284         u, err := url.Parse(string(text))
 285         if err == nil {
 286                 *su = URL(*u)
 287         }
 288         return err
 289 }
 290
 291 func (su URL) MarshalText() ([]byte, error) {
 292         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 293 }
 294
 295 func (su URL) String() string {
 296         return (*url.URL)(&su).String()
 297 }
 298
 299 type ServiceInstance struct {
 300         Rendezvous string `json:",omitempty"`
 301 }
 302
 303 type PostgreSQL struct {
 304         Connection     PostgreSQLConnection
 305         ConnectionPool int
 306 }
 307
 308 type PostgreSQLConnection map[string]string
 309
 310 type RemoteCluster struct {
 311         Host          string
 312         Proxy         bool
 313         Scheme        string
 314         Insecure      bool
 315         ActivateUsers bool
 316 }
 317
 318 type InstanceType struct {
 319         Name            string
 320         ProviderType    string
 321         VCPUs           int
 322         RAM             ByteSize
 323         Scratch         ByteSize
 324         IncludedScratch ByteSize
 325         AddedScratch    ByteSize
 326         Price           float64
 327         Preemptible     bool
 328 }
 329
 330 type ContainersConfig struct {
 331         CloudVMs                    CloudVMsConfig
 332         CrunchRunCommand            string
 333         CrunchRunArgumentsList      []string
 334         DefaultKeepCacheRAM         ByteSize
 335         DispatchPrivateKey          string
 336         LogReuseDecisions           bool
 337         MaxComputeVMs               int
 338         MaxDispatchAttempts         int
 339         MaxRetryAttempts            int
 340         MinRetryPeriod              Duration
 341         ReserveExtraRAM             ByteSize
 342         StaleLockTimeout            Duration
 343         SupportedDockerImageFormats StringSet
 344         UsePreemptibleInstances     bool
 345
 346         JobsAPI struct {
 347                 Enable         string
 348                 GitInternalDir string
 349         }
 350         Logging struct {
 351                 MaxAge                       Duration
 352                 LogBytesPerEvent             int
 353                 LogSecondsBetweenEvents      int
 354                 LogThrottlePeriod            Duration
 355                 LogThrottleBytes             int
 356                 LogThrottleLines             int
 357                 LimitLogBytesPerJob          int
 358                 LogPartialLineThrottlePeriod Duration
 359                 LogUpdatePeriod              Duration
 360                 LogUpdateSize                ByteSize
 361         }
 362         SLURM struct {
 363                 PrioritySpread             int64
 364                 SbatchArgumentsList        []string
 365                 SbatchEnvironmentVariables map[string]string
 366                 Managed                    struct {
 367                         DNSServerConfDir       string
 368                         DNSServerConfTemplate  string
 369                         DNSServerReloadCommand string
 370                         DNSServerUpdateCommand string
 371                         ComputeNodeDomain      string
 372                         ComputeNodeNameservers StringSet
 373                         AssignNodeHostname     string
 374                 }
 375         }
 376 }
 377
 378 type CloudVMsConfig struct {
 379         Enable bool
 380
 381         BootProbeCommand     string
 382         ImageID              string
 383         MaxCloudOpsPerSecond int
 384         MaxProbesPerSecond   int
 385         PollInterval         Duration
 386         ProbeInterval        Duration
 387         SSHPort              string
 388         SyncInterval         Duration
 389         TimeoutBooting       Duration
 390         TimeoutIdle          Duration
 391         TimeoutProbe         Duration
 392         TimeoutShutdown      Duration
 393         TimeoutSignal        Duration
 394         TimeoutTERM          Duration
 395         ResourceTags         map[string]string
 396         TagKeyPrefix         string
 397
 398         Driver           string
 399         DriverParameters json.RawMessage
 400 }
 401
 402 type InstanceTypeMap map[string]InstanceType
 403
 404 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 405
 406 // UnmarshalJSON handles old config files that provide an array of
 407 // instance types instead of a hash.
 408 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 409         if len(data) > 0 && data[0] == '[' {
 410                 var arr []InstanceType
 411                 err := json.Unmarshal(data, &arr)
 412                 if err != nil {
 413                         return err
 414                 }
 415                 if len(arr) == 0 {
 416                         *it = nil
 417                         return nil
 418                 }
 419                 *it = make(map[string]InstanceType, len(arr))
 420                 for _, t := range arr {
 421                         if _, ok := (*it)[t.Name]; ok {
 422                                 return errDuplicateInstanceTypeName
 423                         }
 424                         if t.ProviderType == "" {
 425                                 t.ProviderType = t.Name
 426                         }
 427                         if t.Scratch == 0 {
 428                                 t.Scratch = t.IncludedScratch + t.AddedScratch
 429                         } else if t.AddedScratch == 0 {
 430                                 t.AddedScratch = t.Scratch - t.IncludedScratch
 431                         } else if t.IncludedScratch == 0 {
 432                                 t.IncludedScratch = t.Scratch - t.AddedScratch
 433                         }
 434
 435                         if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 436                                 return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 437                         }
 438                         (*it)[t.Name] = t
 439                 }
 440                 return nil
 441         }
 442         var hash map[string]InstanceType
 443         err := json.Unmarshal(data, &hash)
 444         if err != nil {
 445                 return err
 446         }
 447         // Fill in Name field (and ProviderType field, if not
 448         // specified) using hash key.
 449         *it = InstanceTypeMap(hash)
 450         for name, t := range *it {
 451                 t.Name = name
 452                 if t.ProviderType == "" {
 453                         t.ProviderType = name
 454                 }
 455                 (*it)[name] = t
 456         }
 457         return nil
 458 }
 459
 460 type StringSet map[string]struct{}
 461
 462 // UnmarshalJSON handles old config files that provide an array of
 463 // instance types instead of a hash.
 464 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 465         if len(data) > 0 && data[0] == '[' {
 466                 var arr []string
 467                 err := json.Unmarshal(data, &arr)
 468                 if err != nil {
 469                         return err
 470                 }
 471                 if len(arr) == 0 {
 472                         *ss = nil
 473                         return nil
 474                 }
 475                 *ss = make(map[string]struct{}, len(arr))
 476                 for _, t := range arr {
 477                         (*ss)[t] = struct{}{}
 478                 }
 479                 return nil
 480         }
 481         var hash map[string]struct{}
 482         err := json.Unmarshal(data, &hash)
 483         if err != nil {
 484                 return err
 485         }
 486         *ss = make(map[string]struct{}, len(hash))
 487         for t, _ := range hash {
 488                 (*ss)[t] = struct{}{}
 489         }
 490
 491         return nil
 492 }
 493
 494 type ServiceName string
 495
 496 const (
 497         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 498         ServiceNameController    ServiceName = "arvados-controller"
 499         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 500         ServiceNameHealth        ServiceName = "arvados-health"
 501         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 502         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 503         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 504         ServiceNameWebsocket     ServiceName = "arvados-ws"
 505         ServiceNameKeepbalance   ServiceName = "keep-balance"
 506         ServiceNameKeepweb       ServiceName = "keep-web"
 507         ServiceNameKeepproxy     ServiceName = "keepproxy"
 508         ServiceNameKeepstore     ServiceName = "keepstore"
 509 )
 510
 511 // Map returns all services as a map, suitable for iterating over all
 512 // services or looking up a service by name.
 513 func (svcs Services) Map() map[ServiceName]Service {
 514         return map[ServiceName]Service{
 515                 ServiceNameRailsAPI:      svcs.RailsAPI,
 516                 ServiceNameController:    svcs.Controller,
 517                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 518                 ServiceNameHealth:        svcs.Health,
 519                 ServiceNameNodemanager:   svcs.Nodemanager,
 520                 ServiceNameWorkbench1:    svcs.Workbench1,
 521                 ServiceNameWorkbench2:    svcs.Workbench2,
 522                 ServiceNameWebsocket:     svcs.Websocket,
 523                 ServiceNameKeepbalance:   svcs.Keepbalance,
 524                 ServiceNameKeepweb:       svcs.WebDAV,
 525                 ServiceNameKeepproxy:     svcs.Keepproxy,
 526                 ServiceNameKeepstore:     svcs.Keepstore,
 527         }
 528 }