sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13         "time"
  14
  15         "git.arvados.org/arvados.git/sdk/go/config"
  16 )
  17
  18 var DefaultConfigFile = func() string {
  19         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  20                 return path
  21         }
  22         return "/etc/arvados/config.yml"
  23 }()
  24
  25 type Config struct {
  26         Clusters         map[string]Cluster
  27         AutoReloadConfig bool
  28         SourceTimestamp  time.Time
  29         SourceSHA256     string
  30 }
  31
  32 // GetConfig returns the current system config, loading it from
  33 // configFile if needed.
  34 func GetConfig(configFile string) (*Config, error) {
  35         var cfg Config
  36         err := config.LoadFile(&cfg, configFile)
  37         return &cfg, err
  38 }
  39
  40 // GetCluster returns the cluster ID and config for the given
  41 // cluster, or the default/only configured cluster if clusterID is "".
  42 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  43         if clusterID == "" {
  44                 if len(sc.Clusters) == 0 {
  45                         return nil, fmt.Errorf("no clusters configured")
  46                 } else if len(sc.Clusters) > 1 {
  47                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  48                 } else {
  49                         for id, cc := range sc.Clusters {
  50                                 cc.ClusterID = id
  51                                 return &cc, nil
  52                         }
  53                 }
  54         }
  55         cc, ok := sc.Clusters[clusterID]
  56         if !ok {
  57                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  58         }
  59         cc.ClusterID = clusterID
  60         return &cc, nil
  61 }
  62
  63 type WebDAVCacheConfig struct {
  64         TTL                  Duration
  65         UUIDTTL              Duration
  66         MaxBlockEntries      int
  67         MaxCollectionEntries int
  68         MaxCollectionBytes   int64
  69         MaxUUIDEntries       int
  70         MaxSessions          int
  71 }
  72
  73 type UploadDownloadPermission struct {
  74         Upload   bool
  75         Download bool
  76 }
  77
  78 type UploadDownloadRolePermissions struct {
  79         User  UploadDownloadPermission
  80         Admin UploadDownloadPermission
  81 }
  82
  83 type ManagedProperties map[string]struct {
  84         Value     interface{}
  85         Function  string
  86         Protected bool
  87 }
  88
  89 type Cluster struct {
  90         ClusterID       string `json:"-"`
  91         ManagementToken string
  92         SystemRootToken string
  93         Services        Services
  94         InstanceTypes   InstanceTypeMap
  95         Containers      ContainersConfig
  96         RemoteClusters  map[string]RemoteCluster
  97         PostgreSQL      PostgreSQL
  98
  99         API struct {
 100                 AsyncPermissionsUpdateInterval   Duration
 101                 DisabledAPIs                     StringSet
 102                 MaxIndexDatabaseRead             int
 103                 MaxItemsPerResponse              int
 104                 MaxConcurrentRequests            int
 105                 MaxKeepBlobBuffers               int
 106                 MaxRequestAmplification          int
 107                 MaxRequestSize                   int
 108                 MaxTokenLifetime                 Duration
 109                 RequestTimeout                   Duration
 110                 SendTimeout                      Duration
 111                 WebsocketClientEventQueue        int
 112                 WebsocketServerEventQueue        int
 113                 KeepServiceRequestTimeout        Duration
 114                 VocabularyPath                   string
 115                 FreezeProjectRequiresDescription bool
 116                 FreezeProjectRequiresProperties  StringSet
 117                 UnfreezeProjectRequiresAdmin     bool
 118         }
 119         AuditLogs struct {
 120                 MaxAge             Duration
 121                 MaxDeleteBatch     int
 122                 UnloggedAttributes StringSet
 123         }
 124         Collections struct {
 125                 BlobSigning                  bool
 126                 BlobSigningKey               string
 127                 BlobSigningTTL               Duration
 128                 BlobTrash                    bool
 129                 BlobTrashLifetime            Duration
 130                 BlobTrashCheckInterval       Duration
 131                 BlobTrashConcurrency         int
 132                 BlobDeleteConcurrency        int
 133                 BlobReplicateConcurrency     int
 134                 CollectionVersioning         bool
 135                 DefaultTrashLifetime         Duration
 136                 DefaultReplication           int
 137                 ManagedProperties            ManagedProperties
 138                 PreserveVersionIfIdle        Duration
 139                 TrashSweepInterval           Duration
 140                 TrustAllContent              bool
 141                 ForwardSlashNameSubstitution string
 142                 S3FolderObjects              bool
 143
 144                 BlobMissingReport        string
 145                 BalancePeriod            Duration
 146                 BalanceCollectionBatch   int
 147                 BalanceCollectionBuffers int
 148                 BalanceTimeout           Duration
 149                 BalanceUpdateLimit       int
 150
 151                 WebDAVCache WebDAVCacheConfig
 152
 153                 KeepproxyPermission UploadDownloadRolePermissions
 154                 WebDAVPermission    UploadDownloadRolePermissions
 155                 WebDAVLogEvents     bool
 156         }
 157         Git struct {
 158                 GitCommand   string
 159                 GitoliteHome string
 160                 Repositories string
 161         }
 162         Login struct {
 163                 LDAP struct {
 164                         Enable             bool
 165                         URL                URL
 166                         StartTLS           bool
 167                         InsecureTLS        bool
 168                         StripDomain        string
 169                         AppendDomain       string
 170                         SearchAttribute    string
 171                         SearchBindUser     string
 172                         SearchBindPassword string
 173                         SearchBase         string
 174                         SearchFilters      string
 175                         EmailAttribute     string
 176                         UsernameAttribute  string
 177                 }
 178                 Google struct {
 179                         Enable                          bool
 180                         ClientID                        string
 181                         ClientSecret                    string
 182                         AlternateEmailAddresses         bool
 183                         AuthenticationRequestParameters map[string]string
 184                 }
 185                 OpenIDConnect struct {
 186                         Enable                          bool
 187                         Issuer                          string
 188                         ClientID                        string
 189                         ClientSecret                    string
 190                         EmailClaim                      string
 191                         EmailVerifiedClaim              string
 192                         UsernameClaim                   string
 193                         AcceptAccessToken               bool
 194                         AcceptAccessTokenScope          string
 195                         AuthenticationRequestParameters map[string]string
 196                 }
 197                 PAM struct {
 198                         Enable             bool
 199                         Service            string
 200                         DefaultEmailDomain string
 201                 }
 202                 Test struct {
 203                         Enable bool
 204                         Users  map[string]TestUser
 205                 }
 206                 LoginCluster       string
 207                 RemoteTokenRefresh Duration
 208                 TokenLifetime      Duration
 209                 TrustedClients     map[string]struct{}
 210                 IssueTrustedTokens bool
 211         }
 212         Mail struct {
 213                 MailchimpAPIKey                string
 214                 MailchimpListID                string
 215                 SendUserSetupNotificationEmail bool
 216                 IssueReporterEmailFrom         string
 217                 IssueReporterEmailTo           string
 218                 SupportEmailAddress            string
 219                 EmailFrom                      string
 220         }
 221         SystemLogs struct {
 222                 LogLevel                string
 223                 Format                  string
 224                 MaxRequestLogParamsSize int
 225         }
 226         TLS struct {
 227                 Certificate string
 228                 Key         string
 229                 Insecure    bool
 230         }
 231         Users struct {
 232                 ActivatedUsersAreVisibleToOthers      bool
 233                 AnonymousUserToken                    string
 234                 AdminNotifierEmailFrom                string
 235                 AutoAdminFirstUser                    bool
 236                 AutoAdminUserWithEmail                string
 237                 AutoSetupNewUsers                     bool
 238                 AutoSetupNewUsersWithRepository       bool
 239                 AutoSetupNewUsersWithVmUUID           string
 240                 AutoSetupUsernameBlacklist            StringSet
 241                 EmailSubjectPrefix                    string
 242                 NewInactiveUserNotificationRecipients StringSet
 243                 NewUserNotificationRecipients         StringSet
 244                 NewUsersAreActive                     bool
 245                 UserNotifierEmailFrom                 string
 246                 UserNotifierEmailBcc                  StringSet
 247                 UserProfileNotificationAddress        string
 248                 PreferDomainForUsername               string
 249                 UserSetupMailText                     string
 250                 RoleGroupsVisibleToAll                bool
 251         }
 252         StorageClasses map[string]StorageClassConfig
 253         Volumes        map[string]Volume
 254         Workbench      struct {
 255                 ActivationContactLink            string
 256                 APIClientConnectTimeout          Duration
 257                 APIClientReceiveTimeout          Duration
 258                 APIResponseCompression           bool
 259                 ApplicationMimetypesWithViewIcon StringSet
 260                 ArvadosDocsite                   string
 261                 ArvadosPublicDataDocURL          string
 262                 DefaultOpenIdPrefix              string
 263                 EnableGettingStartedPopup        bool
 264                 EnablePublicProjectsPage         bool
 265                 FileViewersConfigURL             string
 266                 LogViewerMaxBytes                ByteSize
 267                 MultiSiteSearch                  string
 268                 ProfilingEnabled                 bool
 269                 Repositories                     bool
 270                 RepositoryCache                  string
 271                 RunningJobLogRecordsToFetch      int
 272                 SecretKeyBase                    string
 273                 ShowRecentCollectionsOnDashboard bool
 274                 ShowUserAgreementInline          bool
 275                 ShowUserNotifications            bool
 276                 SiteName                         string
 277                 Theme                            string
 278                 UserProfileFormFields            map[string]struct {
 279                         Type                 string
 280                         FormFieldTitle       string
 281                         FormFieldDescription string
 282                         Required             bool
 283                         Position             int
 284                         Options              map[string]struct{}
 285                 }
 286                 UserProfileFormMessage string
 287                 WelcomePageHTML        string
 288                 InactivePageHTML       string
 289                 SSHHelpPageHTML        string
 290                 SSHHelpHostSuffix      string
 291                 IdleTimeout            Duration
 292         }
 293 }
 294
 295 type StorageClassConfig struct {
 296         Default  bool
 297         Priority int
 298 }
 299
 300 type Volume struct {
 301         AccessViaHosts   map[URL]VolumeAccess
 302         ReadOnly         bool
 303         Replication      int
 304         StorageClasses   map[string]bool
 305         Driver           string
 306         DriverParameters json.RawMessage
 307 }
 308
 309 type S3VolumeDriverParameters struct {
 310         IAMRole            string
 311         AccessKeyID        string
 312         SecretAccessKey    string
 313         Endpoint           string
 314         Region             string
 315         Bucket             string
 316         LocationConstraint bool
 317         V2Signature        bool
 318         UseAWSS3v2Driver   bool
 319         IndexPageSize      int
 320         ConnectTimeout     Duration
 321         ReadTimeout        Duration
 322         RaceWindow         Duration
 323         UnsafeDelete       bool
 324         PrefixLength       int
 325 }
 326
 327 type AzureVolumeDriverParameters struct {
 328         StorageAccountName   string
 329         StorageAccountKey    string
 330         StorageBaseURL       string
 331         ContainerName        string
 332         RequestTimeout       Duration
 333         ListBlobsRetryDelay  Duration
 334         ListBlobsMaxAttempts int
 335 }
 336
 337 type DirectoryVolumeDriverParameters struct {
 338         Root      string
 339         Serialize bool
 340 }
 341
 342 type VolumeAccess struct {
 343         ReadOnly bool
 344 }
 345
 346 type Services struct {
 347         Composer       Service
 348         Controller     Service
 349         DispatchCloud  Service
 350         DispatchLSF    Service
 351         DispatchSLURM  Service
 352         GitHTTP        Service
 353         GitSSH         Service
 354         Health         Service
 355         Keepbalance    Service
 356         Keepproxy      Service
 357         Keepstore      Service
 358         RailsAPI       Service
 359         WebDAVDownload Service
 360         WebDAV         Service
 361         WebShell       Service
 362         Websocket      Service
 363         Workbench1     Service
 364         Workbench2     Service
 365 }
 366
 367 type Service struct {
 368         InternalURLs map[URL]ServiceInstance
 369         ExternalURL  URL
 370 }
 371
 372 type TestUser struct {
 373         Email    string
 374         Password string
 375 }
 376
 377 // URL is a url.URL that is also usable as a JSON key/value.
 378 type URL url.URL
 379
 380 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 381 // used as a JSON key/value.
 382 func (su *URL) UnmarshalText(text []byte) error {
 383         u, err := url.Parse(string(text))
 384         if err == nil {
 385                 *su = URL(*u)
 386                 if su.Path == "" && su.Host != "" {
 387                         // http://example really means http://example/
 388                         su.Path = "/"
 389                 }
 390         }
 391         return err
 392 }
 393
 394 func (su URL) MarshalText() ([]byte, error) {
 395         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 396 }
 397
 398 func (su URL) String() string {
 399         return (*url.URL)(&su).String()
 400 }
 401
 402 type ServiceInstance struct {
 403         Rendezvous string `json:",omitempty"`
 404 }
 405
 406 type PostgreSQL struct {
 407         Connection     PostgreSQLConnection
 408         ConnectionPool int
 409 }
 410
 411 type PostgreSQLConnection map[string]string
 412
 413 type RemoteCluster struct {
 414         Host          string
 415         Proxy         bool
 416         Scheme        string
 417         Insecure      bool
 418         ActivateUsers bool
 419 }
 420
 421 type CUDAFeatures struct {
 422         DriverVersion      string
 423         HardwareCapability string
 424         DeviceCount        int
 425 }
 426
 427 type InstanceType struct {
 428         Name            string `json:"-"`
 429         ProviderType    string
 430         VCPUs           int
 431         RAM             ByteSize
 432         Scratch         ByteSize `json:"-"`
 433         IncludedScratch ByteSize
 434         AddedScratch    ByteSize
 435         Price           float64
 436         Preemptible     bool
 437         CUDA            CUDAFeatures
 438 }
 439
 440 type ContainersConfig struct {
 441         CloudVMs                      CloudVMsConfig
 442         CrunchRunCommand              string
 443         CrunchRunArgumentsList        []string
 444         DefaultKeepCacheRAM           ByteSize
 445         DispatchPrivateKey            string
 446         LogReuseDecisions             bool
 447         MaxComputeVMs                 int
 448         MaxDispatchAttempts           int
 449         MaxRetryAttempts              int
 450         MinRetryPeriod                Duration
 451         ReserveExtraRAM               ByteSize
 452         StaleLockTimeout              Duration
 453         SupportedDockerImageFormats   StringSet
 454         AlwaysUsePreemptibleInstances bool
 455         PreemptiblePriceFactor        float64
 456         RuntimeEngine                 string
 457         LocalKeepBlobBuffersPerVCPU   int
 458         LocalKeepLogsToContainerLog   string
 459
 460         JobsAPI struct {
 461                 Enable         string
 462                 GitInternalDir string
 463         }
 464         Logging struct {
 465                 MaxAge                       Duration
 466                 LogBytesPerEvent             int
 467                 LogSecondsBetweenEvents      Duration
 468                 LogThrottlePeriod            Duration
 469                 LogThrottleBytes             int
 470                 LogThrottleLines             int
 471                 LimitLogBytesPerJob          int
 472                 LogPartialLineThrottlePeriod Duration
 473                 LogUpdatePeriod              Duration
 474                 LogUpdateSize                ByteSize
 475         }
 476         ShellAccess struct {
 477                 Admin bool
 478                 User  bool
 479         }
 480         SLURM struct {
 481                 PrioritySpread             int64
 482                 SbatchArgumentsList        []string
 483                 SbatchEnvironmentVariables map[string]string
 484                 Managed                    struct {
 485                         DNSServerConfDir       string
 486                         DNSServerConfTemplate  string
 487                         DNSServerReloadCommand string
 488                         DNSServerUpdateCommand string
 489                         ComputeNodeDomain      string
 490                         ComputeNodeNameservers StringSet
 491                         AssignNodeHostname     string
 492                 }
 493         }
 494         LSF struct {
 495                 BsubSudoUser      string
 496                 BsubArgumentsList []string
 497                 BsubCUDAArguments []string
 498         }
 499 }
 500
 501 type CloudVMsConfig struct {
 502         Enable bool
 503
 504         BootProbeCommand               string
 505         DeployRunnerBinary             string
 506         ImageID                        string
 507         MaxCloudOpsPerSecond           int
 508         MaxProbesPerSecond             int
 509         MaxConcurrentInstanceCreateOps int
 510         PollInterval                   Duration
 511         ProbeInterval                  Duration
 512         SSHPort                        string
 513         SyncInterval                   Duration
 514         TimeoutBooting                 Duration
 515         TimeoutIdle                    Duration
 516         TimeoutProbe                   Duration
 517         TimeoutShutdown                Duration
 518         TimeoutSignal                  Duration
 519         TimeoutStaleRunLock            Duration
 520         TimeoutTERM                    Duration
 521         ResourceTags                   map[string]string
 522         TagKeyPrefix                   string
 523
 524         Driver           string
 525         DriverParameters json.RawMessage
 526 }
 527
 528 type InstanceTypeMap map[string]InstanceType
 529
 530 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 531
 532 // UnmarshalJSON does special handling of InstanceTypes:
 533 // * populate computed fields (Name and Scratch)
 534 // * error out if InstancesTypes are populated as an array, which was
 535 //   deprecated in Arvados 1.2.0
 536 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 537         fixup := func(t InstanceType) (InstanceType, error) {
 538                 if t.ProviderType == "" {
 539                         t.ProviderType = t.Name
 540                 }
 541                 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
 542                 // It will also generate a "deprecated or unknown config entry" warning.
 543                 t.Scratch = t.IncludedScratch + t.AddedScratch
 544                 return t, nil
 545         }
 546
 547         if len(data) > 0 && data[0] == '[' {
 548                 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
 549         }
 550         var hash map[string]InstanceType
 551         err := json.Unmarshal(data, &hash)
 552         if err != nil {
 553                 return err
 554         }
 555         // Fill in Name field (and ProviderType field, if not
 556         // specified) using hash key.
 557         *it = InstanceTypeMap(hash)
 558         for name, t := range *it {
 559                 t.Name = name
 560                 t, err := fixup(t)
 561                 if err != nil {
 562                         return err
 563                 }
 564                 (*it)[name] = t
 565         }
 566         return nil
 567 }
 568
 569 type StringSet map[string]struct{}
 570
 571 // UnmarshalJSON handles old config files that provide an array of
 572 // instance types instead of a hash.
 573 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 574         if len(data) > 0 && data[0] == '[' {
 575                 var arr []string
 576                 err := json.Unmarshal(data, &arr)
 577                 if err != nil {
 578                         return err
 579                 }
 580                 if len(arr) == 0 {
 581                         *ss = nil
 582                         return nil
 583                 }
 584                 *ss = make(map[string]struct{}, len(arr))
 585                 for _, t := range arr {
 586                         (*ss)[t] = struct{}{}
 587                 }
 588                 return nil
 589         }
 590         var hash map[string]struct{}
 591         err := json.Unmarshal(data, &hash)
 592         if err != nil {
 593                 return err
 594         }
 595         *ss = make(map[string]struct{}, len(hash))
 596         for t := range hash {
 597                 (*ss)[t] = struct{}{}
 598         }
 599
 600         return nil
 601 }
 602
 603 type ServiceName string
 604
 605 const (
 606         ServiceNameController    ServiceName = "arvados-controller"
 607         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 608         ServiceNameDispatchLSF   ServiceName = "arvados-dispatch-lsf"
 609         ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
 610         ServiceNameGitHTTP       ServiceName = "arvados-git-httpd"
 611         ServiceNameHealth        ServiceName = "arvados-health"
 612         ServiceNameKeepbalance   ServiceName = "keep-balance"
 613         ServiceNameKeepproxy     ServiceName = "keepproxy"
 614         ServiceNameKeepstore     ServiceName = "keepstore"
 615         ServiceNameKeepweb       ServiceName = "keep-web"
 616         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 617         ServiceNameWebsocket     ServiceName = "arvados-ws"
 618         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 619         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 620 )
 621
 622 // Map returns all services as a map, suitable for iterating over all
 623 // services or looking up a service by name.
 624 func (svcs Services) Map() map[ServiceName]Service {
 625         return map[ServiceName]Service{
 626                 ServiceNameController:    svcs.Controller,
 627                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 628                 ServiceNameDispatchLSF:   svcs.DispatchLSF,
 629                 ServiceNameDispatchSLURM: svcs.DispatchSLURM,
 630                 ServiceNameGitHTTP:       svcs.GitHTTP,
 631                 ServiceNameHealth:        svcs.Health,
 632                 ServiceNameKeepbalance:   svcs.Keepbalance,
 633                 ServiceNameKeepproxy:     svcs.Keepproxy,
 634                 ServiceNameKeepstore:     svcs.Keepstore,
 635                 ServiceNameKeepweb:       svcs.WebDAV,
 636                 ServiceNameRailsAPI:      svcs.RailsAPI,
 637                 ServiceNameWebsocket:     svcs.Websocket,
 638                 ServiceNameWorkbench1:    svcs.Workbench1,
 639                 ServiceNameWorkbench2:    svcs.Workbench2,
 640         }
 641 }