sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters map[string]Cluster
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         if cc, ok := sc.Clusters[clusterID]; !ok {
  53                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  54         } else {
  55                 cc.ClusterID = clusterID
  56                 return &cc, nil
  57         }
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxPermissionEntries int
  67         MaxUUIDEntries       int
  68 }
  69 type Cluster struct {
  70         ClusterID       string `json:"-"`
  71         ManagementToken string
  72         SystemRootToken string
  73         Services        Services
  74         InstanceTypes   InstanceTypeMap
  75         Containers      ContainersConfig
  76         RemoteClusters  map[string]RemoteCluster
  77         PostgreSQL      PostgreSQL
  78
  79         API struct {
  80                 AsyncPermissionsUpdateInterval Duration
  81                 DisabledAPIs                   StringSet
  82                 MaxIndexDatabaseRead           int
  83                 MaxItemsPerResponse            int
  84                 MaxConcurrentRequests          int
  85                 MaxKeepBlobBuffers             int
  86                 MaxRequestAmplification        int
  87                 MaxRequestSize                 int
  88                 RailsSessionSecretToken        string
  89                 RequestTimeout                 Duration
  90                 SendTimeout                    Duration
  91                 WebsocketClientEventQueue      int
  92                 WebsocketServerEventQueue      int
  93                 KeepServiceRequestTimeout      Duration
  94         }
  95         AuditLogs struct {
  96                 MaxAge             Duration
  97                 MaxDeleteBatch     int
  98                 UnloggedAttributes StringSet
  99         }
 100         Collections struct {
 101                 BlobSigning              bool
 102                 BlobSigningKey           string
 103                 BlobSigningTTL           Duration
 104                 BlobTrash                bool
 105                 BlobTrashLifetime        Duration
 106                 BlobTrashCheckInterval   Duration
 107                 BlobTrashConcurrency     int
 108                 BlobDeleteConcurrency    int
 109                 BlobReplicateConcurrency int
 110                 CollectionVersioning     bool
 111                 DefaultTrashLifetime     Duration
 112                 DefaultReplication       int
 113                 ManagedProperties        map[string]struct {
 114                         Value     interface{}
 115                         Function  string
 116                         Protected bool
 117                 }
 118                 PreserveVersionIfIdle        Duration
 119                 TrashSweepInterval           Duration
 120                 TrustAllContent              bool
 121                 ForwardSlashNameSubstitution string
 122
 123                 BlobMissingReport        string
 124                 BalancePeriod            Duration
 125                 BalanceCollectionBatch   int
 126                 BalanceCollectionBuffers int
 127
 128                 WebDAVCache WebDAVCacheConfig
 129         }
 130         Git struct {
 131                 GitCommand   string
 132                 GitoliteHome string
 133                 Repositories string
 134         }
 135         Login struct {
 136                 GoogleClientID                string
 137                 GoogleClientSecret            string
 138                 GoogleAlternateEmailAddresses bool
 139                 ProviderAppID                 string
 140                 ProviderAppSecret             string
 141                 LoginCluster                  string
 142                 RemoteTokenRefresh            Duration
 143         }
 144         Mail struct {
 145                 MailchimpAPIKey                string
 146                 MailchimpListID                string
 147                 SendUserSetupNotificationEmail bool
 148                 IssueReporterEmailFrom         string
 149                 IssueReporterEmailTo           string
 150                 SupportEmailAddress            string
 151                 EmailFrom                      string
 152         }
 153         SystemLogs struct {
 154                 LogLevel                string
 155                 Format                  string
 156                 MaxRequestLogParamsSize int
 157         }
 158         TLS struct {
 159                 Certificate string
 160                 Key         string
 161                 Insecure    bool
 162         }
 163         Users struct {
 164                 AnonymousUserToken                    string
 165                 AdminNotifierEmailFrom                string
 166                 AutoAdminFirstUser                    bool
 167                 AutoAdminUserWithEmail                string
 168                 AutoSetupNewUsers                     bool
 169                 AutoSetupNewUsersWithRepository       bool
 170                 AutoSetupNewUsersWithVmUUID           string
 171                 AutoSetupUsernameBlacklist            StringSet
 172                 EmailSubjectPrefix                    string
 173                 NewInactiveUserNotificationRecipients StringSet
 174                 NewUserNotificationRecipients         StringSet
 175                 NewUsersAreActive                     bool
 176                 UserNotifierEmailFrom                 string
 177                 UserProfileNotificationAddress        string
 178                 PreferDomainForUsername               string
 179         }
 180         Volumes   map[string]Volume
 181         Workbench struct {
 182                 ActivationContactLink            string
 183                 APIClientConnectTimeout          Duration
 184                 APIClientReceiveTimeout          Duration
 185                 APIResponseCompression           bool
 186                 ApplicationMimetypesWithViewIcon StringSet
 187                 ArvadosDocsite                   string
 188                 ArvadosPublicDataDocURL          string
 189                 DefaultOpenIdPrefix              string
 190                 EnableGettingStartedPopup        bool
 191                 EnablePublicProjectsPage         bool
 192                 FileViewersConfigURL             string
 193                 LogViewerMaxBytes                ByteSize
 194                 MultiSiteSearch                  string
 195                 ProfilingEnabled                 bool
 196                 Repositories                     bool
 197                 RepositoryCache                  string
 198                 RunningJobLogRecordsToFetch      int
 199                 SecretKeyBase                    string
 200                 ShowRecentCollectionsOnDashboard bool
 201                 ShowUserAgreementInline          bool
 202                 ShowUserNotifications            bool
 203                 SiteName                         string
 204                 Theme                            string
 205                 UserProfileFormFields            map[string]struct {
 206                         Type                 string
 207                         FormFieldTitle       string
 208                         FormFieldDescription string
 209                         Required             bool
 210                         Position             int
 211                         Options              map[string]struct{}
 212                 }
 213                 UserProfileFormMessage string
 214                 VocabularyURL          string
 215                 WelcomePageHTML        string
 216                 InactivePageHTML       string
 217                 SSHHelpPageHTML        string
 218         }
 219
 220         ForceLegacyAPI14 bool
 221 }
 222
 223 type Volume struct {
 224         AccessViaHosts   map[URL]VolumeAccess
 225         ReadOnly         bool
 226         Replication      int
 227         StorageClasses   map[string]bool
 228         Driver           string
 229         DriverParameters json.RawMessage
 230 }
 231
 232 type S3VolumeDriverParameters struct {
 233         AccessKey          string
 234         SecretKey          string
 235         Endpoint           string
 236         Region             string
 237         Bucket             string
 238         LocationConstraint bool
 239         IndexPageSize      int
 240         ConnectTimeout     Duration
 241         ReadTimeout        Duration
 242         RaceWindow         Duration
 243         UnsafeDelete       bool
 244 }
 245
 246 type AzureVolumeDriverParameters struct {
 247         StorageAccountName   string
 248         StorageAccountKey    string
 249         StorageBaseURL       string
 250         ContainerName        string
 251         RequestTimeout       Duration
 252         ListBlobsRetryDelay  Duration
 253         ListBlobsMaxAttempts int
 254 }
 255
 256 type DirectoryVolumeDriverParameters struct {
 257         Root      string
 258         Serialize bool
 259 }
 260
 261 type VolumeAccess struct {
 262         ReadOnly bool
 263 }
 264
 265 type Services struct {
 266         Composer       Service
 267         Controller     Service
 268         DispatchCloud  Service
 269         GitHTTP        Service
 270         GitSSH         Service
 271         Health         Service
 272         Keepbalance    Service
 273         Keepproxy      Service
 274         Keepstore      Service
 275         Nodemanager    Service
 276         RailsAPI       Service
 277         SSO            Service
 278         WebDAVDownload Service
 279         WebDAV         Service
 280         WebShell       Service
 281         Websocket      Service
 282         Workbench1     Service
 283         Workbench2     Service
 284 }
 285
 286 type Service struct {
 287         InternalURLs map[URL]ServiceInstance
 288         ExternalURL  URL
 289 }
 290
 291 // URL is a url.URL that is also usable as a JSON key/value.
 292 type URL url.URL
 293
 294 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 295 // used as a JSON key/value.
 296 func (su *URL) UnmarshalText(text []byte) error {
 297         u, err := url.Parse(string(text))
 298         if err == nil {
 299                 *su = URL(*u)
 300         }
 301         return err
 302 }
 303
 304 func (su URL) MarshalText() ([]byte, error) {
 305         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 306 }
 307
 308 func (su URL) String() string {
 309         return (*url.URL)(&su).String()
 310 }
 311
 312 type ServiceInstance struct {
 313         Rendezvous string `json:",omitempty"`
 314 }
 315
 316 type PostgreSQL struct {
 317         Connection     PostgreSQLConnection
 318         ConnectionPool int
 319 }
 320
 321 type PostgreSQLConnection map[string]string
 322
 323 type RemoteCluster struct {
 324         Host          string
 325         Proxy         bool
 326         Scheme        string
 327         Insecure      bool
 328         ActivateUsers bool
 329 }
 330
 331 type InstanceType struct {
 332         Name            string
 333         ProviderType    string
 334         VCPUs           int
 335         RAM             ByteSize
 336         Scratch         ByteSize
 337         IncludedScratch ByteSize
 338         AddedScratch    ByteSize
 339         Price           float64
 340         Preemptible     bool
 341 }
 342
 343 type ContainersConfig struct {
 344         CloudVMs                    CloudVMsConfig
 345         CrunchRunCommand            string
 346         CrunchRunArgumentsList      []string
 347         DefaultKeepCacheRAM         ByteSize
 348         DispatchPrivateKey          string
 349         LogReuseDecisions           bool
 350         MaxComputeVMs               int
 351         MaxDispatchAttempts         int
 352         MaxRetryAttempts            int
 353         MinRetryPeriod              Duration
 354         ReserveExtraRAM             ByteSize
 355         StaleLockTimeout            Duration
 356         SupportedDockerImageFormats StringSet
 357         UsePreemptibleInstances     bool
 358
 359         JobsAPI struct {
 360                 Enable         string
 361                 GitInternalDir string
 362         }
 363         Logging struct {
 364                 MaxAge                       Duration
 365                 LogBytesPerEvent             int
 366                 LogSecondsBetweenEvents      Duration
 367                 LogThrottlePeriod            Duration
 368                 LogThrottleBytes             int
 369                 LogThrottleLines             int
 370                 LimitLogBytesPerJob          int
 371                 LogPartialLineThrottlePeriod Duration
 372                 LogUpdatePeriod              Duration
 373                 LogUpdateSize                ByteSize
 374         }
 375         SLURM struct {
 376                 PrioritySpread             int64
 377                 SbatchArgumentsList        []string
 378                 SbatchEnvironmentVariables map[string]string
 379                 Managed                    struct {
 380                         DNSServerConfDir       string
 381                         DNSServerConfTemplate  string
 382                         DNSServerReloadCommand string
 383                         DNSServerUpdateCommand string
 384                         ComputeNodeDomain      string
 385                         ComputeNodeNameservers StringSet
 386                         AssignNodeHostname     string
 387                 }
 388         }
 389 }
 390
 391 type CloudVMsConfig struct {
 392         Enable bool
 393
 394         BootProbeCommand     string
 395         DeployRunnerBinary   string
 396         ImageID              string
 397         MaxCloudOpsPerSecond int
 398         MaxProbesPerSecond   int
 399         PollInterval         Duration
 400         ProbeInterval        Duration
 401         SSHPort              string
 402         SyncInterval         Duration
 403         TimeoutBooting       Duration
 404         TimeoutIdle          Duration
 405         TimeoutProbe         Duration
 406         TimeoutShutdown      Duration
 407         TimeoutSignal        Duration
 408         TimeoutTERM          Duration
 409         ResourceTags         map[string]string
 410         TagKeyPrefix         string
 411
 412         Driver           string
 413         DriverParameters json.RawMessage
 414 }
 415
 416 type InstanceTypeMap map[string]InstanceType
 417
 418 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 419
 420 // UnmarshalJSON handles old config files that provide an array of
 421 // instance types instead of a hash.
 422 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 423         if len(data) > 0 && data[0] == '[' {
 424                 var arr []InstanceType
 425                 err := json.Unmarshal(data, &arr)
 426                 if err != nil {
 427                         return err
 428                 }
 429                 if len(arr) == 0 {
 430                         *it = nil
 431                         return nil
 432                 }
 433                 *it = make(map[string]InstanceType, len(arr))
 434                 for _, t := range arr {
 435                         if _, ok := (*it)[t.Name]; ok {
 436                                 return errDuplicateInstanceTypeName
 437                         }
 438                         if t.ProviderType == "" {
 439                                 t.ProviderType = t.Name
 440                         }
 441                         if t.Scratch == 0 {
 442                                 t.Scratch = t.IncludedScratch + t.AddedScratch
 443                         } else if t.AddedScratch == 0 {
 444                                 t.AddedScratch = t.Scratch - t.IncludedScratch
 445                         } else if t.IncludedScratch == 0 {
 446                                 t.IncludedScratch = t.Scratch - t.AddedScratch
 447                         }
 448
 449                         if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 450                                 return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 451                         }
 452                         (*it)[t.Name] = t
 453                 }
 454                 return nil
 455         }
 456         var hash map[string]InstanceType
 457         err := json.Unmarshal(data, &hash)
 458         if err != nil {
 459                 return err
 460         }
 461         // Fill in Name field (and ProviderType field, if not
 462         // specified) using hash key.
 463         *it = InstanceTypeMap(hash)
 464         for name, t := range *it {
 465                 t.Name = name
 466                 if t.ProviderType == "" {
 467                         t.ProviderType = name
 468                 }
 469                 (*it)[name] = t
 470         }
 471         return nil
 472 }
 473
 474 type StringSet map[string]struct{}
 475
 476 // UnmarshalJSON handles old config files that provide an array of
 477 // instance types instead of a hash.
 478 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 479         if len(data) > 0 && data[0] == '[' {
 480                 var arr []string
 481                 err := json.Unmarshal(data, &arr)
 482                 if err != nil {
 483                         return err
 484                 }
 485                 if len(arr) == 0 {
 486                         *ss = nil
 487                         return nil
 488                 }
 489                 *ss = make(map[string]struct{}, len(arr))
 490                 for _, t := range arr {
 491                         (*ss)[t] = struct{}{}
 492                 }
 493                 return nil
 494         }
 495         var hash map[string]struct{}
 496         err := json.Unmarshal(data, &hash)
 497         if err != nil {
 498                 return err
 499         }
 500         *ss = make(map[string]struct{}, len(hash))
 501         for t, _ := range hash {
 502                 (*ss)[t] = struct{}{}
 503         }
 504
 505         return nil
 506 }
 507
 508 type ServiceName string
 509
 510 const (
 511         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 512         ServiceNameController    ServiceName = "arvados-controller"
 513         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 514         ServiceNameHealth        ServiceName = "arvados-health"
 515         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 516         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 517         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 518         ServiceNameWebsocket     ServiceName = "arvados-ws"
 519         ServiceNameKeepbalance   ServiceName = "keep-balance"
 520         ServiceNameKeepweb       ServiceName = "keep-web"
 521         ServiceNameKeepproxy     ServiceName = "keepproxy"
 522         ServiceNameKeepstore     ServiceName = "keepstore"
 523 )
 524
 525 // Map returns all services as a map, suitable for iterating over all
 526 // services or looking up a service by name.
 527 func (svcs Services) Map() map[ServiceName]Service {
 528         return map[ServiceName]Service{
 529                 ServiceNameRailsAPI:      svcs.RailsAPI,
 530                 ServiceNameController:    svcs.Controller,
 531                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 532                 ServiceNameHealth:        svcs.Health,
 533                 ServiceNameNodemanager:   svcs.Nodemanager,
 534                 ServiceNameWorkbench1:    svcs.Workbench1,
 535                 ServiceNameWorkbench2:    svcs.Workbench2,
 536                 ServiceNameWebsocket:     svcs.Websocket,
 537                 ServiceNameKeepbalance:   svcs.Keepbalance,
 538                 ServiceNameKeepweb:       svcs.WebDAV,
 539                 ServiceNameKeepproxy:     svcs.Keepproxy,
 540                 ServiceNameKeepstore:     svcs.Keepstore,
 541         }
 542 }