sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.curoverse.com/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters map[string]Cluster
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         if cc, ok := sc.Clusters[clusterID]; !ok {
  53                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  54         } else {
  55                 cc.ClusterID = clusterID
  56                 return &cc, nil
  57         }
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxPermissionEntries int
  67         MaxUUIDEntries       int
  68 }
  69 type Cluster struct {
  70         ClusterID       string `json:"-"`
  71         ManagementToken string
  72         SystemRootToken string
  73         Services        Services
  74         InstanceTypes   InstanceTypeMap
  75         Containers      ContainersConfig
  76         RemoteClusters  map[string]RemoteCluster
  77         PostgreSQL      PostgreSQL
  78
  79         API struct {
  80                 AsyncPermissionsUpdateInterval Duration
  81                 DisabledAPIs                   StringSet
  82                 MaxIndexDatabaseRead           int
  83                 MaxItemsPerResponse            int
  84                 MaxConcurrentRequests          int
  85                 MaxKeepBlockBuffers            int
  86                 MaxRequestAmplification        int
  87                 MaxRequestSize                 int
  88                 RailsSessionSecretToken        string
  89                 RequestTimeout                 Duration
  90                 SendTimeout                    Duration
  91                 WebsocketClientEventQueue      int
  92                 WebsocketServerEventQueue      int
  93                 KeepServiceRequestTimeout      Duration
  94         }
  95         AuditLogs struct {
  96                 MaxAge             Duration
  97                 MaxDeleteBatch     int
  98                 UnloggedAttributes StringSet
  99         }
 100         Collections struct {
 101                 BlobSigning              bool
 102                 BlobSigningKey           string
 103                 BlobSigningTTL           Duration
 104                 BlobTrash                bool
 105                 BlobTrashLifetime        Duration
 106                 BlobTrashCheckInterval   Duration
 107                 BlobTrashConcurrency     int
 108                 BlobDeleteConcurrency    int
 109                 BlobReplicateConcurrency int
 110                 CollectionVersioning     bool
 111                 DefaultTrashLifetime     Duration
 112                 DefaultReplication       int
 113                 ManagedProperties        map[string]struct {
 114                         Value     interface{}
 115                         Function  string
 116                         Protected bool
 117                 }
 118                 PreserveVersionIfIdle Duration
 119                 TrashSweepInterval    Duration
 120                 TrustAllContent       bool
 121
 122                 WebDAVCache WebDAVCacheConfig
 123         }
 124         Git struct {
 125                 GitCommand   string
 126                 GitoliteHome string
 127                 Repositories string
 128         }
 129         Login struct {
 130                 ProviderAppSecret string
 131                 ProviderAppID     string
 132         }
 133         Mail struct {
 134                 MailchimpAPIKey                string
 135                 MailchimpListID                string
 136                 SendUserSetupNotificationEmail bool
 137                 IssueReporterEmailFrom         string
 138                 IssueReporterEmailTo           string
 139                 SupportEmailAddress            string
 140                 EmailFrom                      string
 141         }
 142         SystemLogs struct {
 143                 LogLevel                string
 144                 Format                  string
 145                 MaxRequestLogParamsSize int
 146         }
 147         TLS struct {
 148                 Certificate string
 149                 Key         string
 150                 Insecure    bool
 151         }
 152         Users struct {
 153                 AnonymousUserToken                    string
 154                 AdminNotifierEmailFrom                string
 155                 AutoAdminFirstUser                    bool
 156                 AutoAdminUserWithEmail                string
 157                 AutoSetupNewUsers                     bool
 158                 AutoSetupNewUsersWithRepository       bool
 159                 AutoSetupNewUsersWithVmUUID           string
 160                 AutoSetupUsernameBlacklist            StringSet
 161                 EmailSubjectPrefix                    string
 162                 NewInactiveUserNotificationRecipients StringSet
 163                 NewUserNotificationRecipients         StringSet
 164                 NewUsersAreActive                     bool
 165                 UserNotifierEmailFrom                 string
 166                 UserProfileNotificationAddress        string
 167         }
 168         Volumes   map[string]Volume
 169         Workbench struct {
 170                 ActivationContactLink            string
 171                 APIClientConnectTimeout          Duration
 172                 APIClientReceiveTimeout          Duration
 173                 APIResponseCompression           bool
 174                 ApplicationMimetypesWithViewIcon StringSet
 175                 ArvadosDocsite                   string
 176                 ArvadosPublicDataDocURL          string
 177                 DefaultOpenIdPrefix              string
 178                 EnableGettingStartedPopup        bool
 179                 EnablePublicProjectsPage         bool
 180                 FileViewersConfigURL             string
 181                 LogViewerMaxBytes                ByteSize
 182                 MultiSiteSearch                  string
 183                 ProfilingEnabled                 bool
 184                 Repositories                     bool
 185                 RepositoryCache                  string
 186                 RunningJobLogRecordsToFetch      int
 187                 SecretKeyBase                    string
 188                 ShowRecentCollectionsOnDashboard bool
 189                 ShowUserAgreementInline          bool
 190                 ShowUserNotifications            bool
 191                 SiteName                         string
 192                 Theme                            string
 193                 UserProfileFormFields            map[string]struct {
 194                         Type                 string
 195                         FormFieldTitle       string
 196                         FormFieldDescription string
 197                         Required             bool
 198                         Position             int
 199                         Options              map[string]struct{}
 200                 }
 201                 UserProfileFormMessage string
 202                 VocabularyURL          string
 203         }
 204
 205         EnableBetaController14287 bool
 206 }
 207
 208 type Volume struct {
 209         AccessViaHosts   map[URL]VolumeAccess
 210         ReadOnly         bool
 211         Replication      int
 212         StorageClasses   map[string]bool
 213         Driver           string
 214         DriverParameters json.RawMessage
 215 }
 216
 217 type S3VolumeDriverParameters struct {
 218         AccessKey          string
 219         SecretKey          string
 220         Endpoint           string
 221         Region             string
 222         Bucket             string
 223         LocationConstraint bool
 224         IndexPageSize      int
 225         ConnectTimeout     Duration
 226         ReadTimeout        Duration
 227         RaceWindow         Duration
 228         UnsafeDelete       bool
 229 }
 230
 231 type AzureVolumeDriverParameters struct {
 232         StorageAccountName   string
 233         StorageAccountKey    string
 234         StorageBaseURL       string
 235         ContainerName        string
 236         RequestTimeout       Duration
 237         ListBlobsRetryDelay  Duration
 238         ListBlobsMaxAttempts int
 239 }
 240
 241 type DirectoryVolumeDriverParameters struct {
 242         Root      string
 243         Serialize bool
 244 }
 245
 246 type VolumeAccess struct {
 247         ReadOnly bool
 248 }
 249
 250 type Services struct {
 251         Composer       Service
 252         Controller     Service
 253         DispatchCloud  Service
 254         GitHTTP        Service
 255         GitSSH         Service
 256         Health         Service
 257         Keepbalance    Service
 258         Keepproxy      Service
 259         Keepstore      Service
 260         Nodemanager    Service
 261         RailsAPI       Service
 262         SSO            Service
 263         WebDAVDownload Service
 264         WebDAV         Service
 265         WebShell       Service
 266         Websocket      Service
 267         Workbench1     Service
 268         Workbench2     Service
 269 }
 270
 271 type Service struct {
 272         InternalURLs map[URL]ServiceInstance
 273         ExternalURL  URL
 274 }
 275
 276 // URL is a url.URL that is also usable as a JSON key/value.
 277 type URL url.URL
 278
 279 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 280 // used as a JSON key/value.
 281 func (su *URL) UnmarshalText(text []byte) error {
 282         u, err := url.Parse(string(text))
 283         if err == nil {
 284                 *su = URL(*u)
 285         }
 286         return err
 287 }
 288
 289 func (su URL) MarshalText() ([]byte, error) {
 290         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 291 }
 292
 293 func (su URL) String() string {
 294         return (*url.URL)(&su).String()
 295 }
 296
 297 type ServiceInstance struct{}
 298
 299 type PostgreSQL struct {
 300         Connection     PostgreSQLConnection
 301         ConnectionPool int
 302 }
 303
 304 type PostgreSQLConnection map[string]string
 305
 306 type RemoteCluster struct {
 307         Host          string
 308         Proxy         bool
 309         Scheme        string
 310         Insecure      bool
 311         ActivateUsers bool
 312 }
 313
 314 type InstanceType struct {
 315         Name            string
 316         ProviderType    string
 317         VCPUs           int
 318         RAM             ByteSize
 319         Scratch         ByteSize
 320         IncludedScratch ByteSize
 321         AddedScratch    ByteSize
 322         Price           float64
 323         Preemptible     bool
 324 }
 325
 326 type ContainersConfig struct {
 327         CloudVMs                    CloudVMsConfig
 328         CrunchRunCommand            string
 329         CrunchRunArgumentsList      []string
 330         DefaultKeepCacheRAM         ByteSize
 331         DispatchPrivateKey          string
 332         LogReuseDecisions           bool
 333         MaxComputeVMs               int
 334         MaxDispatchAttempts         int
 335         MaxRetryAttempts            int
 336         MinRetryPeriod              Duration
 337         ReserveExtraRAM             ByteSize
 338         StaleLockTimeout            Duration
 339         SupportedDockerImageFormats StringSet
 340         UsePreemptibleInstances     bool
 341
 342         JobsAPI struct {
 343                 Enable         string
 344                 GitInternalDir string
 345         }
 346         Logging struct {
 347                 MaxAge                       Duration
 348                 LogBytesPerEvent             int
 349                 LogSecondsBetweenEvents      int
 350                 LogThrottlePeriod            Duration
 351                 LogThrottleBytes             int
 352                 LogThrottleLines             int
 353                 LimitLogBytesPerJob          int
 354                 LogPartialLineThrottlePeriod Duration
 355                 LogUpdatePeriod              Duration
 356                 LogUpdateSize                ByteSize
 357         }
 358         SLURM struct {
 359                 PrioritySpread             int64
 360                 SbatchArgumentsList        []string
 361                 SbatchEnvironmentVariables map[string]string
 362                 Managed                    struct {
 363                         DNSServerConfDir       string
 364                         DNSServerConfTemplate  string
 365                         DNSServerReloadCommand string
 366                         DNSServerUpdateCommand string
 367                         ComputeNodeDomain      string
 368                         ComputeNodeNameservers StringSet
 369                         AssignNodeHostname     string
 370                 }
 371         }
 372 }
 373
 374 type CloudVMsConfig struct {
 375         Enable bool
 376
 377         BootProbeCommand     string
 378         ImageID              string
 379         MaxCloudOpsPerSecond int
 380         MaxProbesPerSecond   int
 381         PollInterval         Duration
 382         ProbeInterval        Duration
 383         SSHPort              string
 384         SyncInterval         Duration
 385         TimeoutBooting       Duration
 386         TimeoutIdle          Duration
 387         TimeoutProbe         Duration
 388         TimeoutShutdown      Duration
 389         TimeoutSignal        Duration
 390         TimeoutTERM          Duration
 391         ResourceTags         map[string]string
 392         TagKeyPrefix         string
 393
 394         Driver           string
 395         DriverParameters json.RawMessage
 396 }
 397
 398 type InstanceTypeMap map[string]InstanceType
 399
 400 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 401
 402 // UnmarshalJSON handles old config files that provide an array of
 403 // instance types instead of a hash.
 404 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 405         if len(data) > 0 && data[0] == '[' {
 406                 var arr []InstanceType
 407                 err := json.Unmarshal(data, &arr)
 408                 if err != nil {
 409                         return err
 410                 }
 411                 if len(arr) == 0 {
 412                         *it = nil
 413                         return nil
 414                 }
 415                 *it = make(map[string]InstanceType, len(arr))
 416                 for _, t := range arr {
 417                         if _, ok := (*it)[t.Name]; ok {
 418                                 return errDuplicateInstanceTypeName
 419                         }
 420                         if t.ProviderType == "" {
 421                                 t.ProviderType = t.Name
 422                         }
 423                         if t.Scratch == 0 {
 424                                 t.Scratch = t.IncludedScratch + t.AddedScratch
 425                         } else if t.AddedScratch == 0 {
 426                                 t.AddedScratch = t.Scratch - t.IncludedScratch
 427                         } else if t.IncludedScratch == 0 {
 428                                 t.IncludedScratch = t.Scratch - t.AddedScratch
 429                         }
 430
 431                         if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 432                                 return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 433                         }
 434                         (*it)[t.Name] = t
 435                 }
 436                 return nil
 437         }
 438         var hash map[string]InstanceType
 439         err := json.Unmarshal(data, &hash)
 440         if err != nil {
 441                 return err
 442         }
 443         // Fill in Name field (and ProviderType field, if not
 444         // specified) using hash key.
 445         *it = InstanceTypeMap(hash)
 446         for name, t := range *it {
 447                 t.Name = name
 448                 if t.ProviderType == "" {
 449                         t.ProviderType = name
 450                 }
 451                 (*it)[name] = t
 452         }
 453         return nil
 454 }
 455
 456 type StringSet map[string]struct{}
 457
 458 // UnmarshalJSON handles old config files that provide an array of
 459 // instance types instead of a hash.
 460 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 461         if len(data) > 0 && data[0] == '[' {
 462                 var arr []string
 463                 err := json.Unmarshal(data, &arr)
 464                 if err != nil {
 465                         return err
 466                 }
 467                 if len(arr) == 0 {
 468                         *ss = nil
 469                         return nil
 470                 }
 471                 *ss = make(map[string]struct{}, len(arr))
 472                 for _, t := range arr {
 473                         (*ss)[t] = struct{}{}
 474                 }
 475                 return nil
 476         }
 477         var hash map[string]struct{}
 478         err := json.Unmarshal(data, &hash)
 479         if err != nil {
 480                 return err
 481         }
 482         *ss = make(map[string]struct{}, len(hash))
 483         for t, _ := range hash {
 484                 (*ss)[t] = struct{}{}
 485         }
 486
 487         return nil
 488 }
 489
 490 type ServiceName string
 491
 492 const (
 493         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 494         ServiceNameController    ServiceName = "arvados-controller"
 495         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 496         ServiceNameHealth        ServiceName = "arvados-health"
 497         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 498         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 499         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 500         ServiceNameWebsocket     ServiceName = "arvados-ws"
 501         ServiceNameKeepbalance   ServiceName = "keep-balance"
 502         ServiceNameKeepweb       ServiceName = "keep-web"
 503         ServiceNameKeepproxy     ServiceName = "keepproxy"
 504         ServiceNameKeepstore     ServiceName = "keepstore"
 505 )
 506
 507 // Map returns all services as a map, suitable for iterating over all
 508 // services or looking up a service by name.
 509 func (svcs Services) Map() map[ServiceName]Service {
 510         return map[ServiceName]Service{
 511                 ServiceNameRailsAPI:      svcs.RailsAPI,
 512                 ServiceNameController:    svcs.Controller,
 513                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 514                 ServiceNameHealth:        svcs.Health,
 515                 ServiceNameNodemanager:   svcs.Nodemanager,
 516                 ServiceNameWorkbench1:    svcs.Workbench1,
 517                 ServiceNameWorkbench2:    svcs.Workbench2,
 518                 ServiceNameWebsocket:     svcs.Websocket,
 519                 ServiceNameKeepbalance:   svcs.Keepbalance,
 520                 ServiceNameKeepweb:       svcs.WebDAV,
 521                 ServiceNameKeepproxy:     svcs.Keepproxy,
 522                 ServiceNameKeepstore:     svcs.Keepstore,
 523         }
 524 }