17574: Merge branch 'main'

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@curii.com>

diff --combined lib/config/config.default.yml
index c5bc1c8e87f6e8c6598eadc93f2f3766d222e703,3ce160776ec7bbd4a39530567930ded21c4babbe..66f508b5adf2c3c10f2bca8ce2d72f0e44a0f327
--- 1/lib/config/config.default.yml
--- 2/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@@ -52,6 -52,9 +52,9 @@@ Clusters
        DispatchCloud:
          InternalURLs: {SAMPLE: {}}
          ExternalURL: "-"
+       DispatchLSF:
+         InternalURLs: {SAMPLE: {}}
+         ExternalURL: "-"
        Keepproxy:
          InternalURLs: {SAMPLE: {}}
          ExternalURL: ""
@@@ -456,13 -459,6 +459,13 @@@
        # long-running balancing operation.
        BalanceTimeout: 6h
  
 +      # Maximum number of replication_confirmed /
 +      # storage_classes_confirmed updates to write to the database
 +      # after a rebalancing run. When many updates are needed, this
 +      # spreads them over a few runs rather than applying them all at
 +      # once.
 +      BalanceUpdateLimit: 100000
 +
        # Default lifetime for ephemeral collections: 2 weeks. This must not
        # be less than BlobSigningTTL.
        DefaultTrashLifetime: 336h
@@@ -524,10 -520,10 +527,10 @@@
        # WebDAV would have to expose XSS vulnerabilities in order to
        # handle the redirect (see discussion on Services.WebDAV).
        #
-       # This setting has no effect in the recommended configuration,
-       # where the WebDAV is configured to have a separate domain for
-       # every collection; in this case XSS protection is provided by
-       # browsers' same-origin policy.
+       # This setting has no effect in the recommended configuration, where the
+       # WebDAV service is configured to have a separate domain for every
+       # collection and XSS protection is provided by browsers' same-origin
+       # policy.
        #
        # The default setting (false) is appropriate for a multi-user site.
        TrustAllContent: false
@@@ -1019,6 -1015,19 +1022,19 @@@
            # (See http://ruby-doc.org/core-2.2.2/Kernel.html#method-i-format for more.)
            AssignNodeHostname: "compute%<slot_number>d"
  
+       LSF:
+         # Additional arguments to bsub when submitting Arvados
+         # containers as LSF jobs.
+         BsubArgumentsList: []
+ 
+         # Use sudo to switch to this user account when submitting LSF
+         # jobs.
+         #
+         # This account must exist on the hosts where LSF jobs run
+         # ("execution hosts"), as well as on the host where the
+         # Arvados LSF dispatcher runs ("submission host").
+         BsubSudoUser: "crunch"
+ 
        JobsAPI:
          # Enable the legacy 'jobs' API (crunch v1).  This value must be a string.
          #

diff --combined lib/config/export.go
index 2faacc85953ffb724015b427f29796cb4634002e,da5495352a53e7450ada8dabdbb578f9ec13647c..bbc5ea6c55b885244fc0c33e51a50f36c0f64ca1
--- 1/lib/config/export.go
--- 2/lib/config/export.go
+++ b/lib/config/export.go
@@@ -84,7 -84,6 +84,7 @@@ var whitelist = map[string]bool
        "Collections.BalanceCollectionBuffers":                false,
        "Collections.BalancePeriod":                           false,
        "Collections.BalanceTimeout":                          false,
 +      "Collections.BalanceUpdateLimit":                      false,
        "Collections.BlobDeleteConcurrency":                   false,
        "Collections.BlobMissingReport":                       false,
        "Collections.BlobReplicateConcurrency":                false,
@@@ -121,6 -120,7 +121,7 @@@
        "Containers.JobsAPI.GitInternalDir":                   false,
        "Containers.Logging":                                  false,
        "Containers.LogReuseDecisions":                        false,
+       "Containers.LSF":                                      false,
        "Containers.MaxComputeVMs":                            false,
        "Containers.MaxDispatchAttempts":                      false,
        "Containers.MaxRetryAttempts":                         true,

diff --combined lib/config/generated_config.go
index fb9f888ebb87361b071136ab7226665f0a1e51c5,4a726d1ec7e5c5ce615fdfa23ffb64db43826917..ee230841354522ad155e25ce794dbca6f549b58f
--- 1/lib/config/generated_config.go
--- 2/lib/config/generated_config.go
+++ b/lib/config/generated_config.go
@@@ -58,6 -58,9 +58,9 @@@ Clusters
        DispatchCloud:
          InternalURLs: {SAMPLE: {}}
          ExternalURL: "-"
+       DispatchLSF:
+         InternalURLs: {SAMPLE: {}}
+         ExternalURL: "-"
        Keepproxy:
          InternalURLs: {SAMPLE: {}}
          ExternalURL: ""
@@@ -462,13 -465,6 +465,13 @@@
        # long-running balancing operation.
        BalanceTimeout: 6h
  
 +      # Maximum number of replication_confirmed /
 +      # storage_classes_confirmed updates to write to the database
 +      # after a rebalancing run. When many updates are needed, this
 +      # spreads them over a few runs rather than applying them all at
 +      # once.
 +      BalanceUpdateLimit: 100000
 +
        # Default lifetime for ephemeral collections: 2 weeks. This must not
        # be less than BlobSigningTTL.
        DefaultTrashLifetime: 336h
@@@ -530,10 -526,10 +533,10 @@@
        # WebDAV would have to expose XSS vulnerabilities in order to
        # handle the redirect (see discussion on Services.WebDAV).
        #
-       # This setting has no effect in the recommended configuration,
-       # where the WebDAV is configured to have a separate domain for
-       # every collection; in this case XSS protection is provided by
-       # browsers' same-origin policy.
+       # This setting has no effect in the recommended configuration, where the
+       # WebDAV service is configured to have a separate domain for every
+       # collection and XSS protection is provided by browsers' same-origin
+       # policy.
        #
        # The default setting (false) is appropriate for a multi-user site.
        TrustAllContent: false
@@@ -1025,6 -1021,19 +1028,19 @@@
            # (See http://ruby-doc.org/core-2.2.2/Kernel.html#method-i-format for more.)
            AssignNodeHostname: "compute%<slot_number>d"
  
+       LSF:
+         # Additional arguments to bsub when submitting Arvados
+         # containers as LSF jobs.
+         BsubArgumentsList: []
+ 
+         # Use sudo to switch to this user account when submitting LSF
+         # jobs.
+         #
+         # This account must exist on the hosts where LSF jobs run
+         # ("execution hosts"), as well as on the host where the
+         # Arvados LSF dispatcher runs ("submission host").
+         BsubSudoUser: "crunch"
+ 
        JobsAPI:
          # Enable the legacy 'jobs' API (crunch v1).  This value must be a string.
          #

diff --combined sdk/go/arvados/config.go
index dcffcd25e73051985c1242441cce0372ed0eb7dd,844991f41e03b219c5b1d1c0c537d5125d9b2413..9e7eb521eec079a145c94a840a14b35e502b6f18
--- 1/sdk/go/arvados/config.go
--- 2/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@@ -138,7 -138,6 +138,7 @@@ type Cluster struct 
                BalanceCollectionBatch   int
                BalanceCollectionBuffers int
                BalanceTimeout           Duration
 +              BalanceUpdateLimit       int
  
                WebDAVCache WebDAVCacheConfig
  
@@@ -330,6 -329,7 +330,7 @@@ type Services struct 
        Composer       Service
        Controller     Service
        DispatchCloud  Service
+       DispatchLSF    Service
        GitHTTP        Service
        GitSSH         Service
        Health         Service
@@@ -462,6 -462,10 +463,10 @@@ type ContainersConfig struct 
                        AssignNodeHostname     string
                }
        }
+       LSF struct {
+               BsubSudoUser      string
+               BsubArgumentsList []string
+       }
  }
  
  type CloudVMsConfig struct {
@@@ -598,6 -602,7 +603,7 @@@ const 
        ServiceNameRailsAPI      ServiceName = "arvados-api-server"
        ServiceNameController    ServiceName = "arvados-controller"
        ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
+       ServiceNameDispatchLSF   ServiceName = "arvados-dispatch-lsf"
        ServiceNameHealth        ServiceName = "arvados-health"
        ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
        ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
@@@ -615,6 -620,7 +621,7 @@@ func (svcs Services) Map() map[ServiceN
                ServiceNameRailsAPI:      svcs.RailsAPI,
                ServiceNameController:    svcs.Controller,
                ServiceNameDispatchCloud: svcs.DispatchCloud,
+               ServiceNameDispatchLSF:   svcs.DispatchLSF,
                ServiceNameHealth:        svcs.Health,
                ServiceNameWorkbench1:    svcs.Workbench1,
                ServiceNameWorkbench2:    svcs.Workbench2,

diff --combined sdk/go/arvadostest/fixtures.go
index d770ca76d1876ed56274a89435c4322db536fbfa,3de4225d568a95324e5f81ecc34e8b2486433f8e..9281f51d0cf0ee2b46ca97c2e59fde9f68051d4d
--- 1/sdk/go/arvadostest/fixtures.go
--- 2/sdk/go/arvadostest/fixtures.go
+++ b/sdk/go/arvadostest/fixtures.go
@@@ -31,10 -31,7 +31,10 @@@ const 
        UserAgreementPDH        = "b519d9cb706a29fc7ea24dbea2f05851+93"
        HelloWorldPdh           = "55713e6a34081eb03609e7ad5fcad129+62"
  
 -      MultilevelCollection1 = "zzzzz-4zz18-pyw8yp9g3pr7irn"
 +      MultilevelCollection1                        = "zzzzz-4zz18-pyw8yp9g3pr7irn"
 +      StorageClassesDesiredDefaultConfirmedDefault = "zzzzz-4zz18-3t236wr12769tga"
 +      StorageClassesDesiredArchiveConfirmedDefault = "zzzzz-4zz18-3t236wr12769qqa"
 +      EmptyCollectionUUID                          = "zzzzz-4zz18-gs9ooj1h9sd5mde"
  
        AProjectUUID    = "zzzzz-j7d0g-v955i6s2oi1cbso"
        ASubprojectUUID = "zzzzz-j7d0g-axqo7eu9pwvna1x"
@@@ -48,6 -45,8 +48,8 @@@
        QueuedContainerRequestUUID = "zzzzz-xvhdp-cr4queuedcontnr"
        QueuedContainerUUID        = "zzzzz-dz642-queuedcontainer"
  
+       LockedContainerUUID = "zzzzz-dz642-lockedcontainer"
+ 
        RunningContainerUUID = "zzzzz-dz642-runningcontainr"
  
        CompletedContainerUUID         = "zzzzz-dz642-compltcontainer"