sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters         map[string]Cluster
  27         AutoReloadConfig bool
  28 }
  29
  30 // GetConfig returns the current system config, loading it from
  31 // configFile if needed.
  32 func GetConfig(configFile string) (*Config, error) {
  33         var cfg Config
  34         err := config.LoadFile(&cfg, configFile)
  35         return &cfg, err
  36 }
  37
  38 // GetCluster returns the cluster ID and config for the given
  39 // cluster, or the default/only configured cluster if clusterID is "".
  40 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  41         if clusterID == "" {
  42                 if len(sc.Clusters) == 0 {
  43                         return nil, fmt.Errorf("no clusters configured")
  44                 } else if len(sc.Clusters) > 1 {
  45                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  46                 } else {
  47                         for id, cc := range sc.Clusters {
  48                                 cc.ClusterID = id
  49                                 return &cc, nil
  50                         }
  51                 }
  52         }
  53         if cc, ok := sc.Clusters[clusterID]; !ok {
  54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  55         } else {
  56                 cc.ClusterID = clusterID
  57                 return &cc, nil
  58         }
  59 }
  60
  61 type WebDAVCacheConfig struct {
  62         TTL                  Duration
  63         UUIDTTL              Duration
  64         MaxBlockEntries      int
  65         MaxCollectionEntries int
  66         MaxCollectionBytes   int64
  67         MaxPermissionEntries int
  68         MaxUUIDEntries       int
  69 }
  70
  71 type Cluster struct {
  72         ClusterID       string `json:"-"`
  73         ManagementToken string
  74         SystemRootToken string
  75         Services        Services
  76         InstanceTypes   InstanceTypeMap
  77         Containers      ContainersConfig
  78         RemoteClusters  map[string]RemoteCluster
  79         PostgreSQL      PostgreSQL
  80
  81         API struct {
  82                 AsyncPermissionsUpdateInterval Duration
  83                 DisabledAPIs                   StringSet
  84                 MaxIndexDatabaseRead           int
  85                 MaxItemsPerResponse            int
  86                 MaxConcurrentRequests          int
  87                 MaxKeepBlobBuffers             int
  88                 MaxRequestAmplification        int
  89                 MaxRequestSize                 int
  90                 RailsSessionSecretToken        string
  91                 RequestTimeout                 Duration
  92                 SendTimeout                    Duration
  93                 WebsocketClientEventQueue      int
  94                 WebsocketServerEventQueue      int
  95                 KeepServiceRequestTimeout      Duration
  96         }
  97         AuditLogs struct {
  98                 MaxAge             Duration
  99                 MaxDeleteBatch     int
 100                 UnloggedAttributes StringSet
 101         }
 102         Collections struct {
 103                 BlobSigning              bool
 104                 BlobSigningKey           string
 105                 BlobSigningTTL           Duration
 106                 BlobTrash                bool
 107                 BlobTrashLifetime        Duration
 108                 BlobTrashCheckInterval   Duration
 109                 BlobTrashConcurrency     int
 110                 BlobDeleteConcurrency    int
 111                 BlobReplicateConcurrency int
 112                 CollectionVersioning     bool
 113                 DefaultTrashLifetime     Duration
 114                 DefaultReplication       int
 115                 ManagedProperties        map[string]struct {
 116                         Value     interface{}
 117                         Function  string
 118                         Protected bool
 119                 }
 120                 PreserveVersionIfIdle        Duration
 121                 TrashSweepInterval           Duration
 122                 TrustAllContent              bool
 123                 ForwardSlashNameSubstitution string
 124
 125                 BlobMissingReport        string
 126                 BalancePeriod            Duration
 127                 BalanceCollectionBatch   int
 128                 BalanceCollectionBuffers int
 129
 130                 WebDAVCache WebDAVCacheConfig
 131         }
 132         Git struct {
 133                 GitCommand   string
 134                 GitoliteHome string
 135                 Repositories string
 136         }
 137         Login struct {
 138                 GoogleClientID                string
 139                 GoogleClientSecret            string
 140                 GoogleAlternateEmailAddresses bool
 141                 PAM                           bool
 142                 PAMService                    string
 143                 PAMDefaultEmailDomain         string
 144                 ProviderAppID                 string
 145                 ProviderAppSecret             string
 146                 LoginCluster                  string
 147                 RemoteTokenRefresh            Duration
 148         }
 149         Mail struct {
 150                 MailchimpAPIKey                string
 151                 MailchimpListID                string
 152                 SendUserSetupNotificationEmail bool
 153                 IssueReporterEmailFrom         string
 154                 IssueReporterEmailTo           string
 155                 SupportEmailAddress            string
 156                 EmailFrom                      string
 157         }
 158         SystemLogs struct {
 159                 LogLevel                string
 160                 Format                  string
 161                 MaxRequestLogParamsSize int
 162         }
 163         TLS struct {
 164                 Certificate string
 165                 Key         string
 166                 Insecure    bool
 167         }
 168         Users struct {
 169                 AnonymousUserToken                    string
 170                 AdminNotifierEmailFrom                string
 171                 AutoAdminFirstUser                    bool
 172                 AutoAdminUserWithEmail                string
 173                 AutoSetupNewUsers                     bool
 174                 AutoSetupNewUsersWithRepository       bool
 175                 AutoSetupNewUsersWithVmUUID           string
 176                 AutoSetupUsernameBlacklist            StringSet
 177                 EmailSubjectPrefix                    string
 178                 NewInactiveUserNotificationRecipients StringSet
 179                 NewUserNotificationRecipients         StringSet
 180                 NewUsersAreActive                     bool
 181                 UserNotifierEmailFrom                 string
 182                 UserProfileNotificationAddress        string
 183                 PreferDomainForUsername               string
 184         }
 185         Volumes   map[string]Volume
 186         Workbench struct {
 187                 ActivationContactLink            string
 188                 APIClientConnectTimeout          Duration
 189                 APIClientReceiveTimeout          Duration
 190                 APIResponseCompression           bool
 191                 ApplicationMimetypesWithViewIcon StringSet
 192                 ArvadosDocsite                   string
 193                 ArvadosPublicDataDocURL          string
 194                 DefaultOpenIdPrefix              string
 195                 EnableGettingStartedPopup        bool
 196                 EnablePublicProjectsPage         bool
 197                 FileViewersConfigURL             string
 198                 LogViewerMaxBytes                ByteSize
 199                 MultiSiteSearch                  string
 200                 ProfilingEnabled                 bool
 201                 Repositories                     bool
 202                 RepositoryCache                  string
 203                 RunningJobLogRecordsToFetch      int
 204                 SecretKeyBase                    string
 205                 ShowRecentCollectionsOnDashboard bool
 206                 ShowUserAgreementInline          bool
 207                 ShowUserNotifications            bool
 208                 SiteName                         string
 209                 Theme                            string
 210                 UserProfileFormFields            map[string]struct {
 211                         Type                 string
 212                         FormFieldTitle       string
 213                         FormFieldDescription string
 214                         Required             bool
 215                         Position             int
 216                         Options              map[string]struct{}
 217                 }
 218                 UserProfileFormMessage string
 219                 VocabularyURL          string
 220                 WelcomePageHTML        string
 221                 InactivePageHTML       string
 222                 SSHHelpPageHTML        string
 223                 SSHHelpHostSuffix      string
 224         }
 225
 226         ForceLegacyAPI14 bool
 227 }
 228
 229 type Volume struct {
 230         AccessViaHosts   map[URL]VolumeAccess
 231         ReadOnly         bool
 232         Replication      int
 233         StorageClasses   map[string]bool
 234         Driver           string
 235         DriverParameters json.RawMessage
 236 }
 237
 238 type S3VolumeDriverParameters struct {
 239         AccessKey          string
 240         SecretKey          string
 241         Endpoint           string
 242         Region             string
 243         Bucket             string
 244         LocationConstraint bool
 245         IndexPageSize      int
 246         ConnectTimeout     Duration
 247         ReadTimeout        Duration
 248         RaceWindow         Duration
 249         UnsafeDelete       bool
 250 }
 251
 252 type AzureVolumeDriverParameters struct {
 253         StorageAccountName   string
 254         StorageAccountKey    string
 255         StorageBaseURL       string
 256         ContainerName        string
 257         RequestTimeout       Duration
 258         ListBlobsRetryDelay  Duration
 259         ListBlobsMaxAttempts int
 260 }
 261
 262 type DirectoryVolumeDriverParameters struct {
 263         Root      string
 264         Serialize bool
 265 }
 266
 267 type VolumeAccess struct {
 268         ReadOnly bool
 269 }
 270
 271 type Services struct {
 272         Composer       Service
 273         Controller     Service
 274         DispatchCloud  Service
 275         GitHTTP        Service
 276         GitSSH         Service
 277         Health         Service
 278         Keepbalance    Service
 279         Keepproxy      Service
 280         Keepstore      Service
 281         Nodemanager    Service
 282         RailsAPI       Service
 283         SSO            Service
 284         WebDAVDownload Service
 285         WebDAV         Service
 286         WebShell       Service
 287         Websocket      Service
 288         Workbench1     Service
 289         Workbench2     Service
 290 }
 291
 292 type Service struct {
 293         InternalURLs map[URL]ServiceInstance
 294         ExternalURL  URL
 295 }
 296
 297 // URL is a url.URL that is also usable as a JSON key/value.
 298 type URL url.URL
 299
 300 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 301 // used as a JSON key/value.
 302 func (su *URL) UnmarshalText(text []byte) error {
 303         u, err := url.Parse(string(text))
 304         if err == nil {
 305                 *su = URL(*u)
 306         }
 307         return err
 308 }
 309
 310 func (su URL) MarshalText() ([]byte, error) {
 311         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 312 }
 313
 314 func (su URL) String() string {
 315         return (*url.URL)(&su).String()
 316 }
 317
 318 type ServiceInstance struct {
 319         Rendezvous string `json:",omitempty"`
 320 }
 321
 322 type PostgreSQL struct {
 323         Connection     PostgreSQLConnection
 324         ConnectionPool int
 325 }
 326
 327 type PostgreSQLConnection map[string]string
 328
 329 type RemoteCluster struct {
 330         Host          string
 331         Proxy         bool
 332         Scheme        string
 333         Insecure      bool
 334         ActivateUsers bool
 335 }
 336
 337 type InstanceType struct {
 338         Name            string
 339         ProviderType    string
 340         VCPUs           int
 341         RAM             ByteSize
 342         Scratch         ByteSize
 343         IncludedScratch ByteSize
 344         AddedScratch    ByteSize
 345         Price           float64
 346         Preemptible     bool
 347 }
 348
 349 type ContainersConfig struct {
 350         CloudVMs                    CloudVMsConfig
 351         CrunchRunCommand            string
 352         CrunchRunArgumentsList      []string
 353         DefaultKeepCacheRAM         ByteSize
 354         DispatchPrivateKey          string
 355         LogReuseDecisions           bool
 356         MaxComputeVMs               int
 357         MaxDispatchAttempts         int
 358         MaxRetryAttempts            int
 359         MinRetryPeriod              Duration
 360         ReserveExtraRAM             ByteSize
 361         StaleLockTimeout            Duration
 362         SupportedDockerImageFormats StringSet
 363         UsePreemptibleInstances     bool
 364
 365         JobsAPI struct {
 366                 Enable         string
 367                 GitInternalDir string
 368         }
 369         Logging struct {
 370                 MaxAge                       Duration
 371                 LogBytesPerEvent             int
 372                 LogSecondsBetweenEvents      Duration
 373                 LogThrottlePeriod            Duration
 374                 LogThrottleBytes             int
 375                 LogThrottleLines             int
 376                 LimitLogBytesPerJob          int
 377                 LogPartialLineThrottlePeriod Duration
 378                 LogUpdatePeriod              Duration
 379                 LogUpdateSize                ByteSize
 380         }
 381         SLURM struct {
 382                 PrioritySpread             int64
 383                 SbatchArgumentsList        []string
 384                 SbatchEnvironmentVariables map[string]string
 385                 Managed                    struct {
 386                         DNSServerConfDir       string
 387                         DNSServerConfTemplate  string
 388                         DNSServerReloadCommand string
 389                         DNSServerUpdateCommand string
 390                         ComputeNodeDomain      string
 391                         ComputeNodeNameservers StringSet
 392                         AssignNodeHostname     string
 393                 }
 394         }
 395 }
 396
 397 type CloudVMsConfig struct {
 398         Enable bool
 399
 400         BootProbeCommand     string
 401         DeployRunnerBinary   string
 402         ImageID              string
 403         MaxCloudOpsPerSecond int
 404         MaxProbesPerSecond   int
 405         PollInterval         Duration
 406         ProbeInterval        Duration
 407         SSHPort              string
 408         SyncInterval         Duration
 409         TimeoutBooting       Duration
 410         TimeoutIdle          Duration
 411         TimeoutProbe         Duration
 412         TimeoutShutdown      Duration
 413         TimeoutSignal        Duration
 414         TimeoutTERM          Duration
 415         ResourceTags         map[string]string
 416         TagKeyPrefix         string
 417
 418         Driver           string
 419         DriverParameters json.RawMessage
 420 }
 421
 422 type InstanceTypeMap map[string]InstanceType
 423
 424 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 425
 426 // UnmarshalJSON handles old config files that provide an array of
 427 // instance types instead of a hash.
 428 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 429         fixup := func(t InstanceType) (InstanceType, error) {
 430                 if t.ProviderType == "" {
 431                         t.ProviderType = t.Name
 432                 }
 433                 if t.Scratch == 0 {
 434                         t.Scratch = t.IncludedScratch + t.AddedScratch
 435                 } else if t.AddedScratch == 0 {
 436                         t.AddedScratch = t.Scratch - t.IncludedScratch
 437                 } else if t.IncludedScratch == 0 {
 438                         t.IncludedScratch = t.Scratch - t.AddedScratch
 439                 }
 440
 441                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 442                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 443                 }
 444                 return t, nil
 445         }
 446
 447         if len(data) > 0 && data[0] == '[' {
 448                 var arr []InstanceType
 449                 err := json.Unmarshal(data, &arr)
 450                 if err != nil {
 451                         return err
 452                 }
 453                 if len(arr) == 0 {
 454                         *it = nil
 455                         return nil
 456                 }
 457                 *it = make(map[string]InstanceType, len(arr))
 458                 for _, t := range arr {
 459                         if _, ok := (*it)[t.Name]; ok {
 460                                 return errDuplicateInstanceTypeName
 461                         }
 462                         t, err := fixup(t)
 463                         if err != nil {
 464                                 return err
 465                         }
 466                         (*it)[t.Name] = t
 467                 }
 468                 return nil
 469         }
 470         var hash map[string]InstanceType
 471         err := json.Unmarshal(data, &hash)
 472         if err != nil {
 473                 return err
 474         }
 475         // Fill in Name field (and ProviderType field, if not
 476         // specified) using hash key.
 477         *it = InstanceTypeMap(hash)
 478         for name, t := range *it {
 479                 t.Name = name
 480                 t, err := fixup(t)
 481                 if err != nil {
 482                         return err
 483                 }
 484                 (*it)[name] = t
 485         }
 486         return nil
 487 }
 488
 489 type StringSet map[string]struct{}
 490
 491 // UnmarshalJSON handles old config files that provide an array of
 492 // instance types instead of a hash.
 493 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 494         if len(data) > 0 && data[0] == '[' {
 495                 var arr []string
 496                 err := json.Unmarshal(data, &arr)
 497                 if err != nil {
 498                         return err
 499                 }
 500                 if len(arr) == 0 {
 501                         *ss = nil
 502                         return nil
 503                 }
 504                 *ss = make(map[string]struct{}, len(arr))
 505                 for _, t := range arr {
 506                         (*ss)[t] = struct{}{}
 507                 }
 508                 return nil
 509         }
 510         var hash map[string]struct{}
 511         err := json.Unmarshal(data, &hash)
 512         if err != nil {
 513                 return err
 514         }
 515         *ss = make(map[string]struct{}, len(hash))
 516         for t, _ := range hash {
 517                 (*ss)[t] = struct{}{}
 518         }
 519
 520         return nil
 521 }
 522
 523 type ServiceName string
 524
 525 const (
 526         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 527         ServiceNameController    ServiceName = "arvados-controller"
 528         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 529         ServiceNameHealth        ServiceName = "arvados-health"
 530         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 531         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 532         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 533         ServiceNameWebsocket     ServiceName = "arvados-ws"
 534         ServiceNameKeepbalance   ServiceName = "keep-balance"
 535         ServiceNameKeepweb       ServiceName = "keep-web"
 536         ServiceNameKeepproxy     ServiceName = "keepproxy"
 537         ServiceNameKeepstore     ServiceName = "keepstore"
 538 )
 539
 540 // Map returns all services as a map, suitable for iterating over all
 541 // services or looking up a service by name.
 542 func (svcs Services) Map() map[ServiceName]Service {
 543         return map[ServiceName]Service{
 544                 ServiceNameRailsAPI:      svcs.RailsAPI,
 545                 ServiceNameController:    svcs.Controller,
 546                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 547                 ServiceNameHealth:        svcs.Health,
 548                 ServiceNameNodemanager:   svcs.Nodemanager,
 549                 ServiceNameWorkbench1:    svcs.Workbench1,
 550                 ServiceNameWorkbench2:    svcs.Workbench2,
 551                 ServiceNameWebsocket:     svcs.Websocket,
 552                 ServiceNameKeepbalance:   svcs.Keepbalance,
 553                 ServiceNameKeepweb:       svcs.WebDAV,
 554                 ServiceNameKeepproxy:     svcs.Keepproxy,
 555                 ServiceNameKeepstore:     svcs.Keepstore,
 556         }
 557 }