sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.curoverse.com/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters map[string]Cluster
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         if cc, ok := sc.Clusters[clusterID]; !ok {
  53                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  54         } else {
  55                 cc.ClusterID = clusterID
  56                 return &cc, nil
  57         }
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxCollectionEntries int
  64         MaxCollectionBytes   int64
  65         MaxPermissionEntries int
  66         MaxUUIDEntries       int
  67 }
  68 type Cluster struct {
  69         ClusterID       string `json:"-"`
  70         ManagementToken string
  71         SystemRootToken string
  72         Services        Services
  73         InstanceTypes   InstanceTypeMap
  74         Containers      ContainersConfig
  75         RemoteClusters  map[string]RemoteCluster
  76         PostgreSQL      PostgreSQL
  77
  78         API struct {
  79                 AsyncPermissionsUpdateInterval Duration
  80                 DisabledAPIs                   StringSet
  81                 MaxIndexDatabaseRead           int
  82                 MaxItemsPerResponse            int
  83                 MaxRequestAmplification        int
  84                 MaxRequestSize                 int
  85                 RailsSessionSecretToken        string
  86                 RequestTimeout                 Duration
  87                 SendTimeout                    Duration
  88                 WebsocketClientEventQueue      int
  89                 WebsocketServerEventQueue      int
  90         }
  91         AuditLogs struct {
  92                 MaxAge             Duration
  93                 MaxDeleteBatch     int
  94                 UnloggedAttributes StringSet
  95         }
  96         Collections struct {
  97                 BlobSigning          bool
  98                 BlobSigningKey       string
  99                 BlobSigningTTL       Duration
 100                 CollectionVersioning bool
 101                 DefaultTrashLifetime Duration
 102                 DefaultReplication   int
 103                 ManagedProperties    map[string]struct {
 104                         Value     interface{}
 105                         Function  string
 106                         Protected bool
 107                 }
 108                 PreserveVersionIfIdle Duration
 109                 TrashSweepInterval    Duration
 110                 TrustAllContent       bool
 111
 112                 WebDAVCache WebDAVCacheConfig
 113         }
 114         Git struct {
 115                 Repositories string
 116         }
 117         Login struct {
 118                 ProviderAppSecret string
 119                 ProviderAppID     string
 120         }
 121         Mail struct {
 122                 MailchimpAPIKey                string
 123                 MailchimpListID                string
 124                 SendUserSetupNotificationEmail bool
 125                 IssueReporterEmailFrom         string
 126                 IssueReporterEmailTo           string
 127                 SupportEmailAddress            string
 128                 EmailFrom                      string
 129         }
 130         SystemLogs struct {
 131                 LogLevel                string
 132                 Format                  string
 133                 MaxRequestLogParamsSize int
 134         }
 135         TLS struct {
 136                 Certificate string
 137                 Key         string
 138                 Insecure    bool
 139         }
 140         Users struct {
 141                 AnonymousUserToken                    string
 142                 AdminNotifierEmailFrom                string
 143                 AutoAdminFirstUser                    bool
 144                 AutoAdminUserWithEmail                string
 145                 AutoSetupNewUsers                     bool
 146                 AutoSetupNewUsersWithRepository       bool
 147                 AutoSetupNewUsersWithVmUUID           string
 148                 AutoSetupUsernameBlacklist            StringSet
 149                 EmailSubjectPrefix                    string
 150                 NewInactiveUserNotificationRecipients StringSet
 151                 NewUserNotificationRecipients         StringSet
 152                 NewUsersAreActive                     bool
 153                 UserNotifierEmailFrom                 string
 154                 UserProfileNotificationAddress        string
 155         }
 156         Workbench struct {
 157                 ActivationContactLink            string
 158                 APIClientConnectTimeout          Duration
 159                 APIClientReceiveTimeout          Duration
 160                 APIResponseCompression           bool
 161                 ApplicationMimetypesWithViewIcon StringSet
 162                 ArvadosDocsite                   string
 163                 ArvadosPublicDataDocURL          string
 164                 DefaultOpenIdPrefix              string
 165                 EnableGettingStartedPopup        bool
 166                 EnablePublicProjectsPage         bool
 167                 FileViewersConfigURL             string
 168                 LogViewerMaxBytes                ByteSize
 169                 MultiSiteSearch                  string
 170                 ProfilingEnabled                 bool
 171                 Repositories                     bool
 172                 RepositoryCache                  string
 173                 RunningJobLogRecordsToFetch      int
 174                 SecretKeyBase                    string
 175                 ShowRecentCollectionsOnDashboard bool
 176                 ShowUserAgreementInline          bool
 177                 ShowUserNotifications            bool
 178                 SiteName                         string
 179                 Theme                            string
 180                 UserProfileFormFields            map[string]struct {
 181                         Type                 string
 182                         FormFieldTitle       string
 183                         FormFieldDescription string
 184                         Required             bool
 185                         Position             int
 186                         Options              map[string]struct{}
 187                 }
 188                 UserProfileFormMessage string
 189                 VocabularyURL          string
 190         }
 191
 192         EnableBetaController14287 bool
 193 }
 194
 195 type Services struct {
 196         Composer       Service
 197         Controller     Service
 198         DispatchCloud  Service
 199         GitHTTP        Service
 200         GitSSH         Service
 201         Health         Service
 202         Keepbalance    Service
 203         Keepproxy      Service
 204         Keepstore      Service
 205         Nodemanager    Service
 206         RailsAPI       Service
 207         SSO            Service
 208         WebDAVDownload Service
 209         WebDAV         Service
 210         WebShell       Service
 211         Websocket      Service
 212         Workbench1     Service
 213         Workbench2     Service
 214 }
 215
 216 type Service struct {
 217         InternalURLs map[URL]ServiceInstance
 218         ExternalURL  URL
 219 }
 220
 221 // URL is a url.URL that is also usable as a JSON key/value.
 222 type URL url.URL
 223
 224 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 225 // used as a JSON key/value.
 226 func (su *URL) UnmarshalText(text []byte) error {
 227         u, err := url.Parse(string(text))
 228         if err == nil {
 229                 *su = URL(*u)
 230         }
 231         return err
 232 }
 233
 234 func (su URL) MarshalText() ([]byte, error) {
 235         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 236 }
 237
 238 type ServiceInstance struct{}
 239
 240 type PostgreSQL struct {
 241         Connection     PostgreSQLConnection
 242         ConnectionPool int
 243 }
 244
 245 type PostgreSQLConnection map[string]string
 246
 247 type RemoteCluster struct {
 248         Host          string
 249         Proxy         bool
 250         Scheme        string
 251         Insecure      bool
 252         ActivateUsers bool
 253 }
 254
 255 type InstanceType struct {
 256         Name            string
 257         ProviderType    string
 258         VCPUs           int
 259         RAM             ByteSize
 260         Scratch         ByteSize
 261         IncludedScratch ByteSize
 262         AddedScratch    ByteSize
 263         Price           float64
 264         Preemptible     bool
 265 }
 266
 267 type ContainersConfig struct {
 268         CloudVMs                    CloudVMsConfig
 269         CrunchRunCommand            string
 270         CrunchRunArgumentsList      []string
 271         DefaultKeepCacheRAM         ByteSize
 272         DispatchPrivateKey          string
 273         LogReuseDecisions           bool
 274         MaxComputeVMs               int
 275         MaxDispatchAttempts         int
 276         MaxRetryAttempts            int
 277         MinRetryPeriod              Duration
 278         ReserveExtraRAM             ByteSize
 279         StaleLockTimeout            Duration
 280         SupportedDockerImageFormats StringSet
 281         UsePreemptibleInstances     bool
 282
 283         JobsAPI struct {
 284                 Enable                  string
 285                 GitInternalDir          string
 286                 DefaultDockerImage      string
 287                 CrunchJobWrapper        string
 288                 CrunchJobUser           string
 289                 CrunchRefreshTrigger    string
 290                 ReuseJobIfOutputsDiffer bool
 291         }
 292         Logging struct {
 293                 MaxAge                       Duration
 294                 LogBytesPerEvent             int
 295                 LogSecondsBetweenEvents      int
 296                 LogThrottlePeriod            Duration
 297                 LogThrottleBytes             int
 298                 LogThrottleLines             int
 299                 LimitLogBytesPerJob          int
 300                 LogPartialLineThrottlePeriod Duration
 301                 LogUpdatePeriod              Duration
 302                 LogUpdateSize                ByteSize
 303         }
 304         SLURM struct {
 305                 PrioritySpread             int64
 306                 SbatchArgumentsList        []string
 307                 SbatchEnvironmentVariables map[string]string
 308                 Managed                    struct {
 309                         DNSServerConfDir       string
 310                         DNSServerConfTemplate  string
 311                         DNSServerReloadCommand string
 312                         DNSServerUpdateCommand string
 313                         ComputeNodeDomain      string
 314                         ComputeNodeNameservers StringSet
 315                         AssignNodeHostname     string
 316                 }
 317         }
 318 }
 319
 320 type CloudVMsConfig struct {
 321         Enable bool
 322
 323         BootProbeCommand     string
 324         ImageID              string
 325         MaxCloudOpsPerSecond int
 326         MaxProbesPerSecond   int
 327         PollInterval         Duration
 328         ProbeInterval        Duration
 329         SSHPort              string
 330         SyncInterval         Duration
 331         TimeoutBooting       Duration
 332         TimeoutIdle          Duration
 333         TimeoutProbe         Duration
 334         TimeoutShutdown      Duration
 335         TimeoutSignal        Duration
 336         TimeoutTERM          Duration
 337         ResourceTags         map[string]string
 338         TagKeyPrefix         string
 339
 340         Driver           string
 341         DriverParameters json.RawMessage
 342 }
 343
 344 type InstanceTypeMap map[string]InstanceType
 345
 346 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 347
 348 // UnmarshalJSON handles old config files that provide an array of
 349 // instance types instead of a hash.
 350 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 351         if len(data) > 0 && data[0] == '[' {
 352                 var arr []InstanceType
 353                 err := json.Unmarshal(data, &arr)
 354                 if err != nil {
 355                         return err
 356                 }
 357                 if len(arr) == 0 {
 358                         *it = nil
 359                         return nil
 360                 }
 361                 *it = make(map[string]InstanceType, len(arr))
 362                 for _, t := range arr {
 363                         if _, ok := (*it)[t.Name]; ok {
 364                                 return errDuplicateInstanceTypeName
 365                         }
 366                         if t.ProviderType == "" {
 367                                 t.ProviderType = t.Name
 368                         }
 369                         if t.Scratch == 0 {
 370                                 t.Scratch = t.IncludedScratch + t.AddedScratch
 371                         } else if t.AddedScratch == 0 {
 372                                 t.AddedScratch = t.Scratch - t.IncludedScratch
 373                         } else if t.IncludedScratch == 0 {
 374                                 t.IncludedScratch = t.Scratch - t.AddedScratch
 375                         }
 376
 377                         if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 378                                 return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 379                         }
 380                         (*it)[t.Name] = t
 381                 }
 382                 return nil
 383         }
 384         var hash map[string]InstanceType
 385         err := json.Unmarshal(data, &hash)
 386         if err != nil {
 387                 return err
 388         }
 389         // Fill in Name field (and ProviderType field, if not
 390         // specified) using hash key.
 391         *it = InstanceTypeMap(hash)
 392         for name, t := range *it {
 393                 t.Name = name
 394                 if t.ProviderType == "" {
 395                         t.ProviderType = name
 396                 }
 397                 (*it)[name] = t
 398         }
 399         return nil
 400 }
 401
 402 type StringSet map[string]struct{}
 403
 404 // UnmarshalJSON handles old config files that provide an array of
 405 // instance types instead of a hash.
 406 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 407         if len(data) > 0 && data[0] == '[' {
 408                 var arr []string
 409                 err := json.Unmarshal(data, &arr)
 410                 if err != nil {
 411                         return err
 412                 }
 413                 if len(arr) == 0 {
 414                         *ss = nil
 415                         return nil
 416                 }
 417                 *ss = make(map[string]struct{}, len(arr))
 418                 for _, t := range arr {
 419                         (*ss)[t] = struct{}{}
 420                 }
 421                 return nil
 422         }
 423         var hash map[string]struct{}
 424         err := json.Unmarshal(data, &hash)
 425         if err != nil {
 426                 return err
 427         }
 428         *ss = make(map[string]struct{}, len(hash))
 429         for t, _ := range hash {
 430                 (*ss)[t] = struct{}{}
 431         }
 432
 433         return nil
 434 }
 435
 436 type ServiceName string
 437
 438 const (
 439         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 440         ServiceNameController    ServiceName = "arvados-controller"
 441         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 442         ServiceNameHealth        ServiceName = "arvados-health"
 443         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 444         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 445         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 446         ServiceNameWebsocket     ServiceName = "arvados-ws"
 447         ServiceNameKeepbalance   ServiceName = "keep-balance"
 448         ServiceNameKeepweb       ServiceName = "keep-web"
 449         ServiceNameKeepproxy     ServiceName = "keepproxy"
 450         ServiceNameKeepstore     ServiceName = "keepstore"
 451 )
 452
 453 // Map returns all services as a map, suitable for iterating over all
 454 // services or looking up a service by name.
 455 func (svcs Services) Map() map[ServiceName]Service {
 456         return map[ServiceName]Service{
 457                 ServiceNameRailsAPI:      svcs.RailsAPI,
 458                 ServiceNameController:    svcs.Controller,
 459                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 460                 ServiceNameHealth:        svcs.Health,
 461                 ServiceNameNodemanager:   svcs.Nodemanager,
 462                 ServiceNameWorkbench1:    svcs.Workbench1,
 463                 ServiceNameWorkbench2:    svcs.Workbench2,
 464                 ServiceNameWebsocket:     svcs.Websocket,
 465                 ServiceNameKeepbalance:   svcs.Keepbalance,
 466                 ServiceNameKeepweb:       svcs.WebDAV,
 467                 ServiceNameKeepproxy:     svcs.Keepproxy,
 468                 ServiceNameKeepstore:     svcs.Keepstore,
 469         }
 470 }