sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "crypto/tls"
   9         "encoding/json"
  10         "errors"
  11         "fmt"
  12         "net/url"
  13         "os"
  14         "time"
  15
  16         "git.arvados.org/arvados.git/sdk/go/config"
  17 )
  18
  19 var DefaultConfigFile = func() string {
  20         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  21                 return path
  22         }
  23         return "/etc/arvados/config.yml"
  24 }()
  25
  26 type Config struct {
  27         Clusters         map[string]Cluster
  28         AutoReloadConfig bool
  29         SourceTimestamp  time.Time
  30         SourceSHA256     string
  31 }
  32
  33 // GetConfig returns the current system config, loading it from
  34 // configFile if needed.
  35 func GetConfig(configFile string) (*Config, error) {
  36         var cfg Config
  37         err := config.LoadFile(&cfg, configFile)
  38         return &cfg, err
  39 }
  40
  41 // GetCluster returns the cluster ID and config for the given
  42 // cluster, or the default/only configured cluster if clusterID is "".
  43 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  44         if clusterID == "" {
  45                 if len(sc.Clusters) == 0 {
  46                         return nil, fmt.Errorf("no clusters configured")
  47                 } else if len(sc.Clusters) > 1 {
  48                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  49                 } else {
  50                         for id, cc := range sc.Clusters {
  51                                 cc.ClusterID = id
  52                                 return &cc, nil
  53                         }
  54                 }
  55         }
  56         cc, ok := sc.Clusters[clusterID]
  57         if !ok {
  58                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  59         }
  60         cc.ClusterID = clusterID
  61         return &cc, nil
  62 }
  63
  64 type WebDAVCacheConfig struct {
  65         TTL                Duration
  66         MaxBlockEntries    int
  67         MaxCollectionBytes int64
  68         MaxSessions        int
  69 }
  70
  71 type UploadDownloadPermission struct {
  72         Upload   bool
  73         Download bool
  74 }
  75
  76 type UploadDownloadRolePermissions struct {
  77         User  UploadDownloadPermission
  78         Admin UploadDownloadPermission
  79 }
  80
  81 type ManagedProperties map[string]struct {
  82         Value     interface{}
  83         Function  string
  84         Protected bool
  85 }
  86
  87 type Cluster struct {
  88         ClusterID       string `json:"-"`
  89         ManagementToken string
  90         SystemRootToken string
  91         Services        Services
  92         InstanceTypes   InstanceTypeMap
  93         Containers      ContainersConfig
  94         RemoteClusters  map[string]RemoteCluster
  95         PostgreSQL      PostgreSQL
  96
  97         API struct {
  98                 AsyncPermissionsUpdateInterval   Duration
  99                 DisabledAPIs                     StringSet
 100                 MaxIndexDatabaseRead             int
 101                 MaxItemsPerResponse              int
 102                 MaxConcurrentRequests            int
 103                 MaxQueuedRequests                int
 104                 MaxQueueTimeForLockRequests      Duration
 105                 LogCreateRequestFraction         float64
 106                 MaxKeepBlobBuffers               int
 107                 MaxRequestAmplification          int
 108                 MaxRequestSize                   int
 109                 MaxTokenLifetime                 Duration
 110                 RequestTimeout                   Duration
 111                 SendTimeout                      Duration
 112                 WebsocketClientEventQueue        int
 113                 WebsocketServerEventQueue        int
 114                 KeepServiceRequestTimeout        Duration
 115                 VocabularyPath                   string
 116                 FreezeProjectRequiresDescription bool
 117                 FreezeProjectRequiresProperties  StringSet
 118                 UnfreezeProjectRequiresAdmin     bool
 119                 LockBeforeUpdate                 bool
 120         }
 121         AuditLogs struct {
 122                 MaxAge             Duration
 123                 MaxDeleteBatch     int
 124                 UnloggedAttributes StringSet
 125         }
 126         Collections struct {
 127                 BlobSigning                  bool
 128                 BlobSigningKey               string
 129                 BlobSigningTTL               Duration
 130                 BlobTrash                    bool
 131                 BlobTrashLifetime            Duration
 132                 BlobTrashCheckInterval       Duration
 133                 BlobTrashConcurrency         int
 134                 BlobDeleteConcurrency        int
 135                 BlobReplicateConcurrency     int
 136                 CollectionVersioning         bool
 137                 DefaultTrashLifetime         Duration
 138                 DefaultReplication           int
 139                 ManagedProperties            ManagedProperties
 140                 PreserveVersionIfIdle        Duration
 141                 TrashSweepInterval           Duration
 142                 TrustAllContent              bool
 143                 ForwardSlashNameSubstitution string
 144                 S3FolderObjects              bool
 145
 146                 BlobMissingReport        string
 147                 BalancePeriod            Duration
 148                 BalanceCollectionBatch   int
 149                 BalanceCollectionBuffers int
 150                 BalanceTimeout           Duration
 151                 BalanceUpdateLimit       int
 152
 153                 WebDAVCache WebDAVCacheConfig
 154
 155                 KeepproxyPermission UploadDownloadRolePermissions
 156                 WebDAVPermission    UploadDownloadRolePermissions
 157                 WebDAVLogEvents     bool
 158         }
 159         Git struct {
 160                 GitCommand   string
 161                 GitoliteHome string
 162                 Repositories string
 163         }
 164         Login struct {
 165                 LDAP struct {
 166                         Enable             bool
 167                         URL                URL
 168                         StartTLS           bool
 169                         InsecureTLS        bool
 170                         MinTLSVersion      TLSVersion
 171                         StripDomain        string
 172                         AppendDomain       string
 173                         SearchAttribute    string
 174                         SearchBindUser     string
 175                         SearchBindPassword string
 176                         SearchBase         string
 177                         SearchFilters      string
 178                         EmailAttribute     string
 179                         UsernameAttribute  string
 180                 }
 181                 Google struct {
 182                         Enable                          bool
 183                         ClientID                        string
 184                         ClientSecret                    string
 185                         AlternateEmailAddresses         bool
 186                         AuthenticationRequestParameters map[string]string
 187                 }
 188                 OpenIDConnect struct {
 189                         Enable                          bool
 190                         Issuer                          string
 191                         ClientID                        string
 192                         ClientSecret                    string
 193                         EmailClaim                      string
 194                         EmailVerifiedClaim              string
 195                         UsernameClaim                   string
 196                         AcceptAccessToken               bool
 197                         AcceptAccessTokenScope          string
 198                         AuthenticationRequestParameters map[string]string
 199                 }
 200                 PAM struct {
 201                         Enable             bool
 202                         Service            string
 203                         DefaultEmailDomain string
 204                 }
 205                 Test struct {
 206                         Enable bool
 207                         Users  map[string]TestUser
 208                 }
 209                 LoginCluster         string
 210                 RemoteTokenRefresh   Duration
 211                 TokenLifetime        Duration
 212                 TrustedClients       map[URL]struct{}
 213                 TrustPrivateNetworks bool
 214                 IssueTrustedTokens   bool
 215         }
 216         Mail struct {
 217                 MailchimpAPIKey                string
 218                 MailchimpListID                string
 219                 SendUserSetupNotificationEmail bool
 220                 IssueReporterEmailFrom         string
 221                 IssueReporterEmailTo           string
 222                 SupportEmailAddress            string
 223                 EmailFrom                      string
 224         }
 225         SystemLogs struct {
 226                 LogLevel                  string
 227                 Format                    string
 228                 MaxRequestLogParamsSize   int
 229                 RequestQueueDumpDirectory string
 230         }
 231         TLS struct {
 232                 Certificate string
 233                 Key         string
 234                 Insecure    bool
 235                 ACME        struct {
 236                         Server string
 237                 }
 238         }
 239         Users struct {
 240                 ActivatedUsersAreVisibleToOthers      bool
 241                 AnonymousUserToken                    string
 242                 AdminNotifierEmailFrom                string
 243                 AutoAdminFirstUser                    bool
 244                 AutoAdminUserWithEmail                string
 245                 AutoSetupNewUsers                     bool
 246                 AutoSetupNewUsersWithRepository       bool
 247                 AutoSetupNewUsersWithVmUUID           string
 248                 AutoSetupUsernameBlacklist            StringSet
 249                 EmailSubjectPrefix                    string
 250                 NewInactiveUserNotificationRecipients StringSet
 251                 NewUserNotificationRecipients         StringSet
 252                 NewUsersAreActive                     bool
 253                 UserNotifierEmailFrom                 string
 254                 UserNotifierEmailBcc                  StringSet
 255                 UserProfileNotificationAddress        string
 256                 PreferDomainForUsername               string
 257                 UserSetupMailText                     string
 258                 RoleGroupsVisibleToAll                bool
 259                 CanCreateRoleGroups                   bool
 260                 ActivityLoggingPeriod                 Duration
 261                 SyncIgnoredGroups                     []string
 262                 SyncRequiredGroups                    []string
 263                 SyncUserAccounts                      bool
 264                 SyncUserAPITokens                     bool
 265                 SyncUserGroups                        bool
 266                 SyncUserSSHKeys                       bool
 267         }
 268         StorageClasses map[string]StorageClassConfig
 269         Volumes        map[string]Volume
 270         Workbench      struct {
 271                 ActivationContactLink            string
 272                 APIClientConnectTimeout          Duration
 273                 APIClientReceiveTimeout          Duration
 274                 APIResponseCompression           bool
 275                 ApplicationMimetypesWithViewIcon StringSet
 276                 ArvadosDocsite                   string
 277                 ArvadosPublicDataDocURL          string
 278                 DefaultOpenIdPrefix              string
 279                 DisableSharingURLsUI             bool
 280                 EnableGettingStartedPopup        bool
 281                 EnablePublicProjectsPage         bool
 282                 FileViewersConfigURL             string
 283                 LogViewerMaxBytes                ByteSize
 284                 MultiSiteSearch                  string
 285                 ProfilingEnabled                 bool
 286                 Repositories                     bool
 287                 RepositoryCache                  string
 288                 RunningJobLogRecordsToFetch      int
 289                 SecretKeyBase                    string
 290                 ShowRecentCollectionsOnDashboard bool
 291                 ShowUserAgreementInline          bool
 292                 ShowUserNotifications            bool
 293                 SiteName                         string
 294                 Theme                            string
 295                 UserProfileFormFields            map[string]struct {
 296                         Type                 string
 297                         FormFieldTitle       string
 298                         FormFieldDescription string
 299                         Required             bool
 300                         Position             int
 301                         Options              map[string]struct{}
 302                 }
 303                 UserProfileFormMessage string
 304                 WelcomePageHTML        string
 305                 InactivePageHTML       string
 306                 SSHHelpPageHTML        string
 307                 SSHHelpHostSuffix      string
 308                 IdleTimeout            Duration
 309                 BannerUUID             string
 310         }
 311 }
 312
 313 type StorageClassConfig struct {
 314         Default  bool
 315         Priority int
 316 }
 317
 318 type Volume struct {
 319         AccessViaHosts   map[URL]VolumeAccess
 320         ReadOnly         bool
 321         Replication      int
 322         StorageClasses   map[string]bool
 323         Driver           string
 324         DriverParameters json.RawMessage
 325 }
 326
 327 type S3VolumeDriverParameters struct {
 328         IAMRole            string
 329         AccessKeyID        string
 330         SecretAccessKey    string
 331         Endpoint           string
 332         Region             string
 333         Bucket             string
 334         LocationConstraint bool
 335         V2Signature        bool
 336         IndexPageSize      int
 337         ConnectTimeout     Duration
 338         ReadTimeout        Duration
 339         RaceWindow         Duration
 340         UnsafeDelete       bool
 341         PrefixLength       int
 342 }
 343
 344 type AzureVolumeDriverParameters struct {
 345         StorageAccountName   string
 346         StorageAccountKey    string
 347         StorageBaseURL       string
 348         ContainerName        string
 349         RequestTimeout       Duration
 350         ListBlobsRetryDelay  Duration
 351         ListBlobsMaxAttempts int
 352 }
 353
 354 type DirectoryVolumeDriverParameters struct {
 355         Root      string
 356         Serialize bool
 357 }
 358
 359 type VolumeAccess struct {
 360         ReadOnly bool
 361 }
 362
 363 type Services struct {
 364         Composer       Service
 365         Controller     Service
 366         DispatchCloud  Service
 367         DispatchLSF    Service
 368         DispatchSLURM  Service
 369         GitHTTP        Service
 370         GitSSH         Service
 371         Health         Service
 372         Keepbalance    Service
 373         Keepproxy      Service
 374         Keepstore      Service
 375         RailsAPI       Service
 376         WebDAVDownload Service
 377         WebDAV         Service
 378         WebShell       Service
 379         Websocket      Service
 380         Workbench1     Service
 381         Workbench2     Service
 382 }
 383
 384 type Service struct {
 385         InternalURLs map[URL]ServiceInstance
 386         ExternalURL  URL
 387 }
 388
 389 type TestUser struct {
 390         Email    string
 391         Password string
 392 }
 393
 394 // URL is a url.URL that is also usable as a JSON key/value.
 395 type URL url.URL
 396
 397 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 398 // used as a JSON key/value.
 399 func (su *URL) UnmarshalText(text []byte) error {
 400         u, err := url.Parse(string(text))
 401         if err == nil {
 402                 *su = URL(*u)
 403                 if su.Path == "" && su.Host != "" {
 404                         // http://example really means http://example/
 405                         su.Path = "/"
 406                 }
 407         }
 408         return err
 409 }
 410
 411 func (su URL) MarshalText() ([]byte, error) {
 412         return []byte(su.String()), nil
 413 }
 414
 415 func (su URL) String() string {
 416         return (*url.URL)(&su).String()
 417 }
 418
 419 type TLSVersion uint16
 420
 421 func (v TLSVersion) MarshalText() ([]byte, error) {
 422         switch v {
 423         case 0:
 424                 return []byte{}, nil
 425         case tls.VersionTLS10:
 426                 return []byte("1.0"), nil
 427         case tls.VersionTLS11:
 428                 return []byte("1.1"), nil
 429         case tls.VersionTLS12:
 430                 return []byte("1.2"), nil
 431         case tls.VersionTLS13:
 432                 return []byte("1.3"), nil
 433         default:
 434                 return nil, fmt.Errorf("unsupported TLSVersion %x", v)
 435         }
 436 }
 437
 438 func (v *TLSVersion) UnmarshalJSON(text []byte) error {
 439         if len(text) > 0 && text[0] == '"' {
 440                 var s string
 441                 err := json.Unmarshal(text, &s)
 442                 if err != nil {
 443                         return err
 444                 }
 445                 text = []byte(s)
 446         }
 447         switch string(text) {
 448         case "":
 449                 *v = 0
 450         case "1.0":
 451                 *v = tls.VersionTLS10
 452         case "1.1":
 453                 *v = tls.VersionTLS11
 454         case "1.2":
 455                 *v = tls.VersionTLS12
 456         case "1.3":
 457                 *v = tls.VersionTLS13
 458         default:
 459                 return fmt.Errorf("unsupported TLSVersion %q", text)
 460         }
 461         return nil
 462 }
 463
 464 type ServiceInstance struct {
 465         ListenURL  URL
 466         Rendezvous string `json:",omitempty"`
 467 }
 468
 469 type PostgreSQL struct {
 470         Connection     PostgreSQLConnection
 471         ConnectionPool int
 472 }
 473
 474 type PostgreSQLConnection map[string]string
 475
 476 type RemoteCluster struct {
 477         Host          string
 478         Proxy         bool
 479         Scheme        string
 480         Insecure      bool
 481         ActivateUsers bool
 482 }
 483
 484 type CUDAFeatures struct {
 485         DriverVersion      string
 486         HardwareCapability string
 487         DeviceCount        int
 488 }
 489
 490 type InstanceType struct {
 491         Name            string `json:"-"`
 492         ProviderType    string
 493         VCPUs           int
 494         RAM             ByteSize
 495         Scratch         ByteSize `json:"-"`
 496         IncludedScratch ByteSize
 497         AddedScratch    ByteSize
 498         Price           float64
 499         Preemptible     bool
 500         CUDA            CUDAFeatures
 501 }
 502
 503 type ContainersConfig struct {
 504         CloudVMs                      CloudVMsConfig
 505         CrunchRunCommand              string
 506         CrunchRunArgumentsList        []string
 507         DefaultKeepCacheRAM           ByteSize
 508         DispatchPrivateKey            string
 509         LogReuseDecisions             bool
 510         MaxDispatchAttempts           int
 511         MaxRetryAttempts              int
 512         MinRetryPeriod                Duration
 513         ReserveExtraRAM               ByteSize
 514         StaleLockTimeout              Duration
 515         SupportedDockerImageFormats   StringSet
 516         AlwaysUsePreemptibleInstances bool
 517         PreemptiblePriceFactor        float64
 518         RuntimeEngine                 string
 519         LocalKeepBlobBuffersPerVCPU   int
 520         LocalKeepLogsToContainerLog   string
 521
 522         JobsAPI struct {
 523                 Enable         string
 524                 GitInternalDir string
 525         }
 526         Logging struct {
 527                 MaxAge                       Duration
 528                 SweepInterval                Duration
 529                 LogBytesPerEvent             int
 530                 LogSecondsBetweenEvents      Duration
 531                 LogThrottlePeriod            Duration
 532                 LogThrottleBytes             int
 533                 LogThrottleLines             int
 534                 LimitLogBytesPerJob          int
 535                 LogPartialLineThrottlePeriod Duration
 536                 LogUpdatePeriod              Duration
 537                 LogUpdateSize                ByteSize
 538         }
 539         ShellAccess struct {
 540                 Admin bool
 541                 User  bool
 542         }
 543         SLURM struct {
 544                 PrioritySpread             int64
 545                 SbatchArgumentsList        []string
 546                 SbatchEnvironmentVariables map[string]string
 547                 Managed                    struct {
 548                         DNSServerConfDir       string
 549                         DNSServerConfTemplate  string
 550                         DNSServerReloadCommand string
 551                         DNSServerUpdateCommand string
 552                         ComputeNodeDomain      string
 553                         ComputeNodeNameservers StringSet
 554                         AssignNodeHostname     string
 555                 }
 556         }
 557         LSF struct {
 558                 BsubSudoUser      string
 559                 BsubArgumentsList []string
 560                 BsubCUDAArguments []string
 561         }
 562 }
 563
 564 type CloudVMsConfig struct {
 565         Enable bool
 566
 567         BootProbeCommand               string
 568         InstanceInitCommand            string
 569         DeployRunnerBinary             string
 570         DeployPublicKey                bool
 571         ImageID                        string
 572         MaxCloudOpsPerSecond           int
 573         MaxProbesPerSecond             int
 574         MaxConcurrentInstanceCreateOps int
 575         MaxInstances                   int
 576         InitialQuotaEstimate           int
 577         SupervisorFraction             float64
 578         PollInterval                   Duration
 579         ProbeInterval                  Duration
 580         SSHPort                        string
 581         SyncInterval                   Duration
 582         TimeoutBooting                 Duration
 583         TimeoutIdle                    Duration
 584         TimeoutProbe                   Duration
 585         TimeoutShutdown                Duration
 586         TimeoutSignal                  Duration
 587         TimeoutStaleRunLock            Duration
 588         TimeoutTERM                    Duration
 589         ResourceTags                   map[string]string
 590         TagKeyPrefix                   string
 591
 592         Driver           string
 593         DriverParameters json.RawMessage
 594 }
 595
 596 type InstanceTypeMap map[string]InstanceType
 597
 598 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 599
 600 // UnmarshalJSON does special handling of InstanceTypes:
 601 //
 602 // - populate computed fields (Name and Scratch)
 603 //
 604 // - error out if InstancesTypes are populated as an array, which was
 605 // deprecated in Arvados 1.2.0
 606 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 607         fixup := func(t InstanceType) (InstanceType, error) {
 608                 if t.ProviderType == "" {
 609                         t.ProviderType = t.Name
 610                 }
 611                 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
 612                 // It will also generate a "deprecated or unknown config entry" warning.
 613                 t.Scratch = t.IncludedScratch + t.AddedScratch
 614                 return t, nil
 615         }
 616
 617         if len(data) > 0 && data[0] == '[' {
 618                 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
 619         }
 620         var hash map[string]InstanceType
 621         err := json.Unmarshal(data, &hash)
 622         if err != nil {
 623                 return err
 624         }
 625         // Fill in Name field (and ProviderType field, if not
 626         // specified) using hash key.
 627         *it = InstanceTypeMap(hash)
 628         for name, t := range *it {
 629                 t.Name = name
 630                 t, err := fixup(t)
 631                 if err != nil {
 632                         return err
 633                 }
 634                 (*it)[name] = t
 635         }
 636         return nil
 637 }
 638
 639 type StringSet map[string]struct{}
 640
 641 // UnmarshalJSON handles old config files that provide an array of
 642 // instance types instead of a hash.
 643 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 644         if len(data) > 0 && data[0] == '[' {
 645                 var arr []string
 646                 err := json.Unmarshal(data, &arr)
 647                 if err != nil {
 648                         return err
 649                 }
 650                 if len(arr) == 0 {
 651                         *ss = nil
 652                         return nil
 653                 }
 654                 *ss = make(map[string]struct{}, len(arr))
 655                 for _, t := range arr {
 656                         (*ss)[t] = struct{}{}
 657                 }
 658                 return nil
 659         }
 660         var hash map[string]struct{}
 661         err := json.Unmarshal(data, &hash)
 662         if err != nil {
 663                 return err
 664         }
 665         *ss = make(map[string]struct{}, len(hash))
 666         for t := range hash {
 667                 (*ss)[t] = struct{}{}
 668         }
 669
 670         return nil
 671 }
 672
 673 type ServiceName string
 674
 675 const (
 676         ServiceNameController    ServiceName = "arvados-controller"
 677         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 678         ServiceNameDispatchLSF   ServiceName = "arvados-dispatch-lsf"
 679         ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
 680         ServiceNameGitHTTP       ServiceName = "arvados-git-httpd"
 681         ServiceNameHealth        ServiceName = "arvados-health"
 682         ServiceNameKeepbalance   ServiceName = "keep-balance"
 683         ServiceNameKeepproxy     ServiceName = "keepproxy"
 684         ServiceNameKeepstore     ServiceName = "keepstore"
 685         ServiceNameKeepweb       ServiceName = "keep-web"
 686         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 687         ServiceNameWebsocket     ServiceName = "arvados-ws"
 688         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 689         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 690 )
 691
 692 // Map returns all services as a map, suitable for iterating over all
 693 // services or looking up a service by name.
 694 func (svcs Services) Map() map[ServiceName]Service {
 695         return map[ServiceName]Service{
 696                 ServiceNameController:    svcs.Controller,
 697                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 698                 ServiceNameDispatchLSF:   svcs.DispatchLSF,
 699                 ServiceNameDispatchSLURM: svcs.DispatchSLURM,
 700                 ServiceNameGitHTTP:       svcs.GitHTTP,
 701                 ServiceNameHealth:        svcs.Health,
 702                 ServiceNameKeepbalance:   svcs.Keepbalance,
 703                 ServiceNameKeepproxy:     svcs.Keepproxy,
 704                 ServiceNameKeepstore:     svcs.Keepstore,
 705                 ServiceNameKeepweb:       svcs.WebDAV,
 706                 ServiceNameRailsAPI:      svcs.RailsAPI,
 707                 ServiceNameWebsocket:     svcs.Websocket,
 708                 ServiceNameWorkbench1:    svcs.Workbench1,
 709                 ServiceNameWorkbench2:    svcs.Workbench2,
 710         }
 711 }