sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters map[string]Cluster
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         if cc, ok := sc.Clusters[clusterID]; !ok {
  53                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  54         } else {
  55                 cc.ClusterID = clusterID
  56                 return &cc, nil
  57         }
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxPermissionEntries int
  67         MaxUUIDEntries       int
  68 }
  69 type Cluster struct {
  70         ClusterID       string `json:"-"`
  71         ManagementToken string
  72         SystemRootToken string
  73         Services        Services
  74         InstanceTypes   InstanceTypeMap
  75         Containers      ContainersConfig
  76         RemoteClusters  map[string]RemoteCluster
  77         PostgreSQL      PostgreSQL
  78
  79         API struct {
  80                 AsyncPermissionsUpdateInterval Duration
  81                 DisabledAPIs                   StringSet
  82                 MaxIndexDatabaseRead           int
  83                 MaxItemsPerResponse            int
  84                 MaxConcurrentRequests          int
  85                 MaxKeepBlobBuffers             int
  86                 MaxRequestAmplification        int
  87                 MaxRequestSize                 int
  88                 RailsSessionSecretToken        string
  89                 RequestTimeout                 Duration
  90                 SendTimeout                    Duration
  91                 WebsocketClientEventQueue      int
  92                 WebsocketServerEventQueue      int
  93                 KeepServiceRequestTimeout      Duration
  94         }
  95         AuditLogs struct {
  96                 MaxAge             Duration
  97                 MaxDeleteBatch     int
  98                 UnloggedAttributes StringSet
  99         }
 100         Collections struct {
 101                 BlobSigning              bool
 102                 BlobSigningKey           string
 103                 BlobSigningTTL           Duration
 104                 BlobTrash                bool
 105                 BlobTrashLifetime        Duration
 106                 BlobTrashCheckInterval   Duration
 107                 BlobTrashConcurrency     int
 108                 BlobDeleteConcurrency    int
 109                 BlobReplicateConcurrency int
 110                 CollectionVersioning     bool
 111                 DefaultTrashLifetime     Duration
 112                 DefaultReplication       int
 113                 ManagedProperties        map[string]struct {
 114                         Value     interface{}
 115                         Function  string
 116                         Protected bool
 117                 }
 118                 PreserveVersionIfIdle        Duration
 119                 TrashSweepInterval           Duration
 120                 TrustAllContent              bool
 121                 ForwardSlashNameSubstitution string
 122
 123                 BlobMissingReport        string
 124                 BalancePeriod            Duration
 125                 BalanceCollectionBatch   int
 126                 BalanceCollectionBuffers int
 127
 128                 WebDAVCache WebDAVCacheConfig
 129         }
 130         Git struct {
 131                 GitCommand   string
 132                 GitoliteHome string
 133                 Repositories string
 134         }
 135         Login struct {
 136                 GoogleClientID                string
 137                 GoogleClientSecret            string
 138                 GoogleAlternateEmailAddresses bool
 139                 ProviderAppID                 string
 140                 ProviderAppSecret             string
 141                 LoginCluster                  string
 142                 RemoteTokenRefresh            Duration
 143         }
 144         Mail struct {
 145                 MailchimpAPIKey                string
 146                 MailchimpListID                string
 147                 SendUserSetupNotificationEmail bool
 148                 IssueReporterEmailFrom         string
 149                 IssueReporterEmailTo           string
 150                 SupportEmailAddress            string
 151                 EmailFrom                      string
 152         }
 153         SystemLogs struct {
 154                 LogLevel                string
 155                 Format                  string
 156                 MaxRequestLogParamsSize int
 157         }
 158         TLS struct {
 159                 Certificate string
 160                 Key         string
 161                 Insecure    bool
 162         }
 163         Users struct {
 164                 AnonymousUserToken                    string
 165                 AdminNotifierEmailFrom                string
 166                 AutoAdminFirstUser                    bool
 167                 AutoAdminUserWithEmail                string
 168                 AutoSetupNewUsers                     bool
 169                 AutoSetupNewUsersWithRepository       bool
 170                 AutoSetupNewUsersWithVmUUID           string
 171                 AutoSetupUsernameBlacklist            StringSet
 172                 EmailSubjectPrefix                    string
 173                 NewInactiveUserNotificationRecipients StringSet
 174                 NewUserNotificationRecipients         StringSet
 175                 NewUsersAreActive                     bool
 176                 UserNotifierEmailFrom                 string
 177                 UserProfileNotificationAddress        string
 178                 PreferDomainForUsername               string
 179         }
 180         Volumes   map[string]Volume
 181         Workbench struct {
 182                 ActivationContactLink            string
 183                 APIClientConnectTimeout          Duration
 184                 APIClientReceiveTimeout          Duration
 185                 APIResponseCompression           bool
 186                 ApplicationMimetypesWithViewIcon StringSet
 187                 ArvadosDocsite                   string
 188                 ArvadosPublicDataDocURL          string
 189                 DefaultOpenIdPrefix              string
 190                 EnableGettingStartedPopup        bool
 191                 EnablePublicProjectsPage         bool
 192                 FileViewersConfigURL             string
 193                 LogViewerMaxBytes                ByteSize
 194                 MultiSiteSearch                  string
 195                 ProfilingEnabled                 bool
 196                 Repositories                     bool
 197                 RepositoryCache                  string
 198                 RunningJobLogRecordsToFetch      int
 199                 SecretKeyBase                    string
 200                 ShowRecentCollectionsOnDashboard bool
 201                 ShowUserAgreementInline          bool
 202                 ShowUserNotifications            bool
 203                 SiteName                         string
 204                 Theme                            string
 205                 UserProfileFormFields            map[string]struct {
 206                         Type                 string
 207                         FormFieldTitle       string
 208                         FormFieldDescription string
 209                         Required             bool
 210                         Position             int
 211                         Options              map[string]struct{}
 212                 }
 213                 UserProfileFormMessage string
 214                 VocabularyURL          string
 215                 WelcomePageHTML        string
 216                 InactivePageHTML       string
 217         }
 218
 219         ForceLegacyAPI14 bool
 220 }
 221
 222 type Volume struct {
 223         AccessViaHosts   map[URL]VolumeAccess
 224         ReadOnly         bool
 225         Replication      int
 226         StorageClasses   map[string]bool
 227         Driver           string
 228         DriverParameters json.RawMessage
 229 }
 230
 231 type S3VolumeDriverParameters struct {
 232         AccessKey          string
 233         SecretKey          string
 234         Endpoint           string
 235         Region             string
 236         Bucket             string
 237         LocationConstraint bool
 238         IndexPageSize      int
 239         ConnectTimeout     Duration
 240         ReadTimeout        Duration
 241         RaceWindow         Duration
 242         UnsafeDelete       bool
 243 }
 244
 245 type AzureVolumeDriverParameters struct {
 246         StorageAccountName   string
 247         StorageAccountKey    string
 248         StorageBaseURL       string
 249         ContainerName        string
 250         RequestTimeout       Duration
 251         ListBlobsRetryDelay  Duration
 252         ListBlobsMaxAttempts int
 253 }
 254
 255 type DirectoryVolumeDriverParameters struct {
 256         Root      string
 257         Serialize bool
 258 }
 259
 260 type VolumeAccess struct {
 261         ReadOnly bool
 262 }
 263
 264 type Services struct {
 265         Composer       Service
 266         Controller     Service
 267         DispatchCloud  Service
 268         GitHTTP        Service
 269         GitSSH         Service
 270         Health         Service
 271         Keepbalance    Service
 272         Keepproxy      Service
 273         Keepstore      Service
 274         Nodemanager    Service
 275         RailsAPI       Service
 276         SSO            Service
 277         WebDAVDownload Service
 278         WebDAV         Service
 279         WebShell       Service
 280         Websocket      Service
 281         Workbench1     Service
 282         Workbench2     Service
 283 }
 284
 285 type Service struct {
 286         InternalURLs map[URL]ServiceInstance
 287         ExternalURL  URL
 288 }
 289
 290 // URL is a url.URL that is also usable as a JSON key/value.
 291 type URL url.URL
 292
 293 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 294 // used as a JSON key/value.
 295 func (su *URL) UnmarshalText(text []byte) error {
 296         u, err := url.Parse(string(text))
 297         if err == nil {
 298                 *su = URL(*u)
 299         }
 300         return err
 301 }
 302
 303 func (su URL) MarshalText() ([]byte, error) {
 304         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 305 }
 306
 307 func (su URL) String() string {
 308         return (*url.URL)(&su).String()
 309 }
 310
 311 type ServiceInstance struct {
 312         Rendezvous string `json:",omitempty"`
 313 }
 314
 315 type PostgreSQL struct {
 316         Connection     PostgreSQLConnection
 317         ConnectionPool int
 318 }
 319
 320 type PostgreSQLConnection map[string]string
 321
 322 type RemoteCluster struct {
 323         Host          string
 324         Proxy         bool
 325         Scheme        string
 326         Insecure      bool
 327         ActivateUsers bool
 328 }
 329
 330 type InstanceType struct {
 331         Name            string
 332         ProviderType    string
 333         VCPUs           int
 334         RAM             ByteSize
 335         Scratch         ByteSize
 336         IncludedScratch ByteSize
 337         AddedScratch    ByteSize
 338         Price           float64
 339         Preemptible     bool
 340 }
 341
 342 type ContainersConfig struct {
 343         CloudVMs                    CloudVMsConfig
 344         CrunchRunCommand            string
 345         CrunchRunArgumentsList      []string
 346         DefaultKeepCacheRAM         ByteSize
 347         DispatchPrivateKey          string
 348         LogReuseDecisions           bool
 349         MaxComputeVMs               int
 350         MaxDispatchAttempts         int
 351         MaxRetryAttempts            int
 352         MinRetryPeriod              Duration
 353         ReserveExtraRAM             ByteSize
 354         StaleLockTimeout            Duration
 355         SupportedDockerImageFormats StringSet
 356         UsePreemptibleInstances     bool
 357
 358         JobsAPI struct {
 359                 Enable         string
 360                 GitInternalDir string
 361         }
 362         Logging struct {
 363                 MaxAge                       Duration
 364                 LogBytesPerEvent             int
 365                 LogSecondsBetweenEvents      Duration
 366                 LogThrottlePeriod            Duration
 367                 LogThrottleBytes             int
 368                 LogThrottleLines             int
 369                 LimitLogBytesPerJob          int
 370                 LogPartialLineThrottlePeriod Duration
 371                 LogUpdatePeriod              Duration
 372                 LogUpdateSize                ByteSize
 373         }
 374         SLURM struct {
 375                 PrioritySpread             int64
 376                 SbatchArgumentsList        []string
 377                 SbatchEnvironmentVariables map[string]string
 378                 Managed                    struct {
 379                         DNSServerConfDir       string
 380                         DNSServerConfTemplate  string
 381                         DNSServerReloadCommand string
 382                         DNSServerUpdateCommand string
 383                         ComputeNodeDomain      string
 384                         ComputeNodeNameservers StringSet
 385                         AssignNodeHostname     string
 386                 }
 387         }
 388 }
 389
 390 type CloudVMsConfig struct {
 391         Enable bool
 392
 393         BootProbeCommand     string
 394         DeployRunnerBinary   string
 395         ImageID              string
 396         MaxCloudOpsPerSecond int
 397         MaxProbesPerSecond   int
 398         PollInterval         Duration
 399         ProbeInterval        Duration
 400         SSHPort              string
 401         SyncInterval         Duration
 402         TimeoutBooting       Duration
 403         TimeoutIdle          Duration
 404         TimeoutProbe         Duration
 405         TimeoutShutdown      Duration
 406         TimeoutSignal        Duration
 407         TimeoutTERM          Duration
 408         ResourceTags         map[string]string
 409         TagKeyPrefix         string
 410
 411         Driver           string
 412         DriverParameters json.RawMessage
 413 }
 414
 415 type InstanceTypeMap map[string]InstanceType
 416
 417 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 418
 419 // UnmarshalJSON handles old config files that provide an array of
 420 // instance types instead of a hash.
 421 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 422         if len(data) > 0 && data[0] == '[' {
 423                 var arr []InstanceType
 424                 err := json.Unmarshal(data, &arr)
 425                 if err != nil {
 426                         return err
 427                 }
 428                 if len(arr) == 0 {
 429                         *it = nil
 430                         return nil
 431                 }
 432                 *it = make(map[string]InstanceType, len(arr))
 433                 for _, t := range arr {
 434                         if _, ok := (*it)[t.Name]; ok {
 435                                 return errDuplicateInstanceTypeName
 436                         }
 437                         if t.ProviderType == "" {
 438                                 t.ProviderType = t.Name
 439                         }
 440                         if t.Scratch == 0 {
 441                                 t.Scratch = t.IncludedScratch + t.AddedScratch
 442                         } else if t.AddedScratch == 0 {
 443                                 t.AddedScratch = t.Scratch - t.IncludedScratch
 444                         } else if t.IncludedScratch == 0 {
 445                                 t.IncludedScratch = t.Scratch - t.AddedScratch
 446                         }
 447
 448                         if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 449                                 return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 450                         }
 451                         (*it)[t.Name] = t
 452                 }
 453                 return nil
 454         }
 455         var hash map[string]InstanceType
 456         err := json.Unmarshal(data, &hash)
 457         if err != nil {
 458                 return err
 459         }
 460         // Fill in Name field (and ProviderType field, if not
 461         // specified) using hash key.
 462         *it = InstanceTypeMap(hash)
 463         for name, t := range *it {
 464                 t.Name = name
 465                 if t.ProviderType == "" {
 466                         t.ProviderType = name
 467                 }
 468                 (*it)[name] = t
 469         }
 470         return nil
 471 }
 472
 473 type StringSet map[string]struct{}
 474
 475 // UnmarshalJSON handles old config files that provide an array of
 476 // instance types instead of a hash.
 477 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 478         if len(data) > 0 && data[0] == '[' {
 479                 var arr []string
 480                 err := json.Unmarshal(data, &arr)
 481                 if err != nil {
 482                         return err
 483                 }
 484                 if len(arr) == 0 {
 485                         *ss = nil
 486                         return nil
 487                 }
 488                 *ss = make(map[string]struct{}, len(arr))
 489                 for _, t := range arr {
 490                         (*ss)[t] = struct{}{}
 491                 }
 492                 return nil
 493         }
 494         var hash map[string]struct{}
 495         err := json.Unmarshal(data, &hash)
 496         if err != nil {
 497                 return err
 498         }
 499         *ss = make(map[string]struct{}, len(hash))
 500         for t, _ := range hash {
 501                 (*ss)[t] = struct{}{}
 502         }
 503
 504         return nil
 505 }
 506
 507 type ServiceName string
 508
 509 const (
 510         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 511         ServiceNameController    ServiceName = "arvados-controller"
 512         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 513         ServiceNameHealth        ServiceName = "arvados-health"
 514         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 515         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 516         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 517         ServiceNameWebsocket     ServiceName = "arvados-ws"
 518         ServiceNameKeepbalance   ServiceName = "keep-balance"
 519         ServiceNameKeepweb       ServiceName = "keep-web"
 520         ServiceNameKeepproxy     ServiceName = "keepproxy"
 521         ServiceNameKeepstore     ServiceName = "keepstore"
 522 )
 523
 524 // Map returns all services as a map, suitable for iterating over all
 525 // services or looking up a service by name.
 526 func (svcs Services) Map() map[ServiceName]Service {
 527         return map[ServiceName]Service{
 528                 ServiceNameRailsAPI:      svcs.RailsAPI,
 529                 ServiceNameController:    svcs.Controller,
 530                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 531                 ServiceNameHealth:        svcs.Health,
 532                 ServiceNameNodemanager:   svcs.Nodemanager,
 533                 ServiceNameWorkbench1:    svcs.Workbench1,
 534                 ServiceNameWorkbench2:    svcs.Workbench2,
 535                 ServiceNameWebsocket:     svcs.Websocket,
 536                 ServiceNameKeepbalance:   svcs.Keepbalance,
 537                 ServiceNameKeepweb:       svcs.WebDAV,
 538                 ServiceNameKeepproxy:     svcs.Keepproxy,
 539                 ServiceNameKeepstore:     svcs.Keepstore,
 540         }
 541 }