sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.curoverse.com/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters map[string]Cluster
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         if cc, ok := sc.Clusters[clusterID]; !ok {
  53                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  54         } else {
  55                 cc.ClusterID = clusterID
  56                 return &cc, nil
  57         }
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxPermissionEntries int
  67         MaxUUIDEntries       int
  68 }
  69 type Cluster struct {
  70         ClusterID       string `json:"-"`
  71         ManagementToken string
  72         SystemRootToken string
  73         Services        Services
  74         InstanceTypes   InstanceTypeMap
  75         Containers      ContainersConfig
  76         RemoteClusters  map[string]RemoteCluster
  77         PostgreSQL      PostgreSQL
  78
  79         API struct {
  80                 AsyncPermissionsUpdateInterval Duration
  81                 DisabledAPIs                   StringSet
  82                 MaxIndexDatabaseRead           int
  83                 MaxItemsPerResponse            int
  84                 MaxConcurrentRequests          int
  85                 MaxKeepBlobBuffers             int
  86                 MaxRequestAmplification        int
  87                 MaxRequestSize                 int
  88                 RailsSessionSecretToken        string
  89                 RequestTimeout                 Duration
  90                 SendTimeout                    Duration
  91                 WebsocketClientEventQueue      int
  92                 WebsocketServerEventQueue      int
  93                 KeepServiceRequestTimeout      Duration
  94         }
  95         AuditLogs struct {
  96                 MaxAge             Duration
  97                 MaxDeleteBatch     int
  98                 UnloggedAttributes StringSet
  99         }
 100         Collections struct {
 101                 BlobSigning              bool
 102                 BlobSigningKey           string
 103                 BlobSigningTTL           Duration
 104                 BlobTrash                bool
 105                 BlobTrashLifetime        Duration
 106                 BlobTrashCheckInterval   Duration
 107                 BlobTrashConcurrency     int
 108                 BlobDeleteConcurrency    int
 109                 BlobReplicateConcurrency int
 110                 CollectionVersioning     bool
 111                 DefaultTrashLifetime     Duration
 112                 DefaultReplication       int
 113                 ManagedProperties        map[string]struct {
 114                         Value     interface{}
 115                         Function  string
 116                         Protected bool
 117                 }
 118                 PreserveVersionIfIdle Duration
 119                 TrashSweepInterval    Duration
 120                 TrustAllContent       bool
 121
 122                 BlobMissingReport        string
 123                 BalancePeriod            Duration
 124                 BalanceCollectionBatch   int
 125                 BalanceCollectionBuffers int
 126
 127                 WebDAVCache WebDAVCacheConfig
 128         }
 129         Git struct {
 130                 GitCommand   string
 131                 GitoliteHome string
 132                 Repositories string
 133         }
 134         Login struct {
 135                 GoogleClientID     string
 136                 GoogleClientSecret string
 137                 ProviderAppID      string
 138                 ProviderAppSecret  string
 139                 LoginCluster       string
 140                 RemoteTokenRefresh Duration
 141         }
 142         Mail struct {
 143                 MailchimpAPIKey                string
 144                 MailchimpListID                string
 145                 SendUserSetupNotificationEmail bool
 146                 IssueReporterEmailFrom         string
 147                 IssueReporterEmailTo           string
 148                 SupportEmailAddress            string
 149                 EmailFrom                      string
 150         }
 151         SystemLogs struct {
 152                 LogLevel                string
 153                 Format                  string
 154                 MaxRequestLogParamsSize int
 155         }
 156         TLS struct {
 157                 Certificate string
 158                 Key         string
 159                 Insecure    bool
 160         }
 161         Users struct {
 162                 AnonymousUserToken                    string
 163                 AdminNotifierEmailFrom                string
 164                 AutoAdminFirstUser                    bool
 165                 AutoAdminUserWithEmail                string
 166                 AutoSetupNewUsers                     bool
 167                 AutoSetupNewUsersWithRepository       bool
 168                 AutoSetupNewUsersWithVmUUID           string
 169                 AutoSetupUsernameBlacklist            StringSet
 170                 EmailSubjectPrefix                    string
 171                 NewInactiveUserNotificationRecipients StringSet
 172                 NewUserNotificationRecipients         StringSet
 173                 NewUsersAreActive                     bool
 174                 UserNotifierEmailFrom                 string
 175                 UserProfileNotificationAddress        string
 176         }
 177         Volumes   map[string]Volume
 178         Workbench struct {
 179                 ActivationContactLink            string
 180                 APIClientConnectTimeout          Duration
 181                 APIClientReceiveTimeout          Duration
 182                 APIResponseCompression           bool
 183                 ApplicationMimetypesWithViewIcon StringSet
 184                 ArvadosDocsite                   string
 185                 ArvadosPublicDataDocURL          string
 186                 DefaultOpenIdPrefix              string
 187                 EnableGettingStartedPopup        bool
 188                 EnablePublicProjectsPage         bool
 189                 FileViewersConfigURL             string
 190                 LogViewerMaxBytes                ByteSize
 191                 MultiSiteSearch                  string
 192                 ProfilingEnabled                 bool
 193                 Repositories                     bool
 194                 RepositoryCache                  string
 195                 RunningJobLogRecordsToFetch      int
 196                 SecretKeyBase                    string
 197                 ShowRecentCollectionsOnDashboard bool
 198                 ShowUserAgreementInline          bool
 199                 ShowUserNotifications            bool
 200                 SiteName                         string
 201                 Theme                            string
 202                 UserProfileFormFields            map[string]struct {
 203                         Type                 string
 204                         FormFieldTitle       string
 205                         FormFieldDescription string
 206                         Required             bool
 207                         Position             int
 208                         Options              map[string]struct{}
 209                 }
 210                 UserProfileFormMessage string
 211                 VocabularyURL          string
 212                 WelcomePageHTML        string
 213                 InactivePageHTML       string
 214         }
 215
 216         EnableBetaController14287 bool
 217 }
 218
 219 type Volume struct {
 220         AccessViaHosts   map[URL]VolumeAccess
 221         ReadOnly         bool
 222         Replication      int
 223         StorageClasses   map[string]bool
 224         Driver           string
 225         DriverParameters json.RawMessage
 226 }
 227
 228 type S3VolumeDriverParameters struct {
 229         AccessKey          string
 230         SecretKey          string
 231         Endpoint           string
 232         Region             string
 233         Bucket             string
 234         LocationConstraint bool
 235         IndexPageSize      int
 236         ConnectTimeout     Duration
 237         ReadTimeout        Duration
 238         RaceWindow         Duration
 239         UnsafeDelete       bool
 240 }
 241
 242 type AzureVolumeDriverParameters struct {
 243         StorageAccountName   string
 244         StorageAccountKey    string
 245         StorageBaseURL       string
 246         ContainerName        string
 247         RequestTimeout       Duration
 248         ListBlobsRetryDelay  Duration
 249         ListBlobsMaxAttempts int
 250 }
 251
 252 type DirectoryVolumeDriverParameters struct {
 253         Root      string
 254         Serialize bool
 255 }
 256
 257 type VolumeAccess struct {
 258         ReadOnly bool
 259 }
 260
 261 type Services struct {
 262         Composer       Service
 263         Controller     Service
 264         DispatchCloud  Service
 265         GitHTTP        Service
 266         GitSSH         Service
 267         Health         Service
 268         Keepbalance    Service
 269         Keepproxy      Service
 270         Keepstore      Service
 271         Nodemanager    Service
 272         RailsAPI       Service
 273         SSO            Service
 274         WebDAVDownload Service
 275         WebDAV         Service
 276         WebShell       Service
 277         Websocket      Service
 278         Workbench1     Service
 279         Workbench2     Service
 280 }
 281
 282 type Service struct {
 283         InternalURLs map[URL]ServiceInstance
 284         ExternalURL  URL
 285 }
 286
 287 // URL is a url.URL that is also usable as a JSON key/value.
 288 type URL url.URL
 289
 290 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 291 // used as a JSON key/value.
 292 func (su *URL) UnmarshalText(text []byte) error {
 293         u, err := url.Parse(string(text))
 294         if err == nil {
 295                 *su = URL(*u)
 296         }
 297         return err
 298 }
 299
 300 func (su URL) MarshalText() ([]byte, error) {
 301         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 302 }
 303
 304 func (su URL) String() string {
 305         return (*url.URL)(&su).String()
 306 }
 307
 308 type ServiceInstance struct {
 309         Rendezvous string `json:",omitempty"`
 310 }
 311
 312 type PostgreSQL struct {
 313         Connection     PostgreSQLConnection
 314         ConnectionPool int
 315 }
 316
 317 type PostgreSQLConnection map[string]string
 318
 319 type RemoteCluster struct {
 320         Host          string
 321         Proxy         bool
 322         Scheme        string
 323         Insecure      bool
 324         ActivateUsers bool
 325 }
 326
 327 type InstanceType struct {
 328         Name            string
 329         ProviderType    string
 330         VCPUs           int
 331         RAM             ByteSize
 332         Scratch         ByteSize
 333         IncludedScratch ByteSize
 334         AddedScratch    ByteSize
 335         Price           float64
 336         Preemptible     bool
 337 }
 338
 339 type ContainersConfig struct {
 340         CloudVMs                    CloudVMsConfig
 341         CrunchRunCommand            string
 342         CrunchRunArgumentsList      []string
 343         DefaultKeepCacheRAM         ByteSize
 344         DispatchPrivateKey          string
 345         LogReuseDecisions           bool
 346         MaxComputeVMs               int
 347         MaxDispatchAttempts         int
 348         MaxRetryAttempts            int
 349         MinRetryPeriod              Duration
 350         ReserveExtraRAM             ByteSize
 351         StaleLockTimeout            Duration
 352         SupportedDockerImageFormats StringSet
 353         UsePreemptibleInstances     bool
 354
 355         JobsAPI struct {
 356                 Enable         string
 357                 GitInternalDir string
 358         }
 359         Logging struct {
 360                 MaxAge                       Duration
 361                 LogBytesPerEvent             int
 362                 LogSecondsBetweenEvents      int
 363                 LogThrottlePeriod            Duration
 364                 LogThrottleBytes             int
 365                 LogThrottleLines             int
 366                 LimitLogBytesPerJob          int
 367                 LogPartialLineThrottlePeriod Duration
 368                 LogUpdatePeriod              Duration
 369                 LogUpdateSize                ByteSize
 370         }
 371         SLURM struct {
 372                 PrioritySpread             int64
 373                 SbatchArgumentsList        []string
 374                 SbatchEnvironmentVariables map[string]string
 375                 Managed                    struct {
 376                         DNSServerConfDir       string
 377                         DNSServerConfTemplate  string
 378                         DNSServerReloadCommand string
 379                         DNSServerUpdateCommand string
 380                         ComputeNodeDomain      string
 381                         ComputeNodeNameservers StringSet
 382                         AssignNodeHostname     string
 383                 }
 384         }
 385 }
 386
 387 type CloudVMsConfig struct {
 388         Enable bool
 389
 390         BootProbeCommand     string
 391         ImageID              string
 392         MaxCloudOpsPerSecond int
 393         MaxProbesPerSecond   int
 394         PollInterval         Duration
 395         ProbeInterval        Duration
 396         SSHPort              string
 397         SyncInterval         Duration
 398         TimeoutBooting       Duration
 399         TimeoutIdle          Duration
 400         TimeoutProbe         Duration
 401         TimeoutShutdown      Duration
 402         TimeoutSignal        Duration
 403         TimeoutTERM          Duration
 404         ResourceTags         map[string]string
 405         TagKeyPrefix         string
 406
 407         Driver           string
 408         DriverParameters json.RawMessage
 409 }
 410
 411 type InstanceTypeMap map[string]InstanceType
 412
 413 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 414
 415 // UnmarshalJSON handles old config files that provide an array of
 416 // instance types instead of a hash.
 417 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 418         if len(data) > 0 && data[0] == '[' {
 419                 var arr []InstanceType
 420                 err := json.Unmarshal(data, &arr)
 421                 if err != nil {
 422                         return err
 423                 }
 424                 if len(arr) == 0 {
 425                         *it = nil
 426                         return nil
 427                 }
 428                 *it = make(map[string]InstanceType, len(arr))
 429                 for _, t := range arr {
 430                         if _, ok := (*it)[t.Name]; ok {
 431                                 return errDuplicateInstanceTypeName
 432                         }
 433                         if t.ProviderType == "" {
 434                                 t.ProviderType = t.Name
 435                         }
 436                         if t.Scratch == 0 {
 437                                 t.Scratch = t.IncludedScratch + t.AddedScratch
 438                         } else if t.AddedScratch == 0 {
 439                                 t.AddedScratch = t.Scratch - t.IncludedScratch
 440                         } else if t.IncludedScratch == 0 {
 441                                 t.IncludedScratch = t.Scratch - t.AddedScratch
 442                         }
 443
 444                         if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 445                                 return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 446                         }
 447                         (*it)[t.Name] = t
 448                 }
 449                 return nil
 450         }
 451         var hash map[string]InstanceType
 452         err := json.Unmarshal(data, &hash)
 453         if err != nil {
 454                 return err
 455         }
 456         // Fill in Name field (and ProviderType field, if not
 457         // specified) using hash key.
 458         *it = InstanceTypeMap(hash)
 459         for name, t := range *it {
 460                 t.Name = name
 461                 if t.ProviderType == "" {
 462                         t.ProviderType = name
 463                 }
 464                 (*it)[name] = t
 465         }
 466         return nil
 467 }
 468
 469 type StringSet map[string]struct{}
 470
 471 // UnmarshalJSON handles old config files that provide an array of
 472 // instance types instead of a hash.
 473 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 474         if len(data) > 0 && data[0] == '[' {
 475                 var arr []string
 476                 err := json.Unmarshal(data, &arr)
 477                 if err != nil {
 478                         return err
 479                 }
 480                 if len(arr) == 0 {
 481                         *ss = nil
 482                         return nil
 483                 }
 484                 *ss = make(map[string]struct{}, len(arr))
 485                 for _, t := range arr {
 486                         (*ss)[t] = struct{}{}
 487                 }
 488                 return nil
 489         }
 490         var hash map[string]struct{}
 491         err := json.Unmarshal(data, &hash)
 492         if err != nil {
 493                 return err
 494         }
 495         *ss = make(map[string]struct{}, len(hash))
 496         for t, _ := range hash {
 497                 (*ss)[t] = struct{}{}
 498         }
 499
 500         return nil
 501 }
 502
 503 type ServiceName string
 504
 505 const (
 506         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 507         ServiceNameController    ServiceName = "arvados-controller"
 508         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 509         ServiceNameHealth        ServiceName = "arvados-health"
 510         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 511         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 512         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 513         ServiceNameWebsocket     ServiceName = "arvados-ws"
 514         ServiceNameKeepbalance   ServiceName = "keep-balance"
 515         ServiceNameKeepweb       ServiceName = "keep-web"
 516         ServiceNameKeepproxy     ServiceName = "keepproxy"
 517         ServiceNameKeepstore     ServiceName = "keepstore"
 518 )
 519
 520 // Map returns all services as a map, suitable for iterating over all
 521 // services or looking up a service by name.
 522 func (svcs Services) Map() map[ServiceName]Service {
 523         return map[ServiceName]Service{
 524                 ServiceNameRailsAPI:      svcs.RailsAPI,
 525                 ServiceNameController:    svcs.Controller,
 526                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 527                 ServiceNameHealth:        svcs.Health,
 528                 ServiceNameNodemanager:   svcs.Nodemanager,
 529                 ServiceNameWorkbench1:    svcs.Workbench1,
 530                 ServiceNameWorkbench2:    svcs.Workbench2,
 531                 ServiceNameWebsocket:     svcs.Websocket,
 532                 ServiceNameKeepbalance:   svcs.Keepbalance,
 533                 ServiceNameKeepweb:       svcs.WebDAV,
 534                 ServiceNameKeepproxy:     svcs.Keepproxy,
 535                 ServiceNameKeepstore:     svcs.Keepstore,
 536         }
 537 }