14714: Adds keep-balance to cluster config loading

[arvados.git] / lib / config / generated_config.go

diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go
index 117f189d9a533a023c77182535f5f8784461657d..3806bbd8ad6680743fb4eda0b87dbf5ec86ace58 100644 (file)
--- a/lib/config/generated_config.go
+++ b/lib/config/generated_config.go
@@ -31,7 +31,13 @@ Clusters:
       # listening, and reachable from other hosts in the cluster.
       SAMPLE:
         InternalURLs:
-          "http://example.host:12345": {}
+          "http://host1.example:12345": {}
+          "http://host2.example:12345":
+            # Rendezvous is normally empty/omitted. When changing the
+            # URL of a Keepstore service, Rendezvous should be set to
+            # the old URL (with trailing slash omitted) to preserve
+            # rendezvous ordering.
+            Rendezvous: ""
           SAMPLE:
             Rendezvous: ""
         ExternalURL: "-"
@@ -319,15 +325,9 @@ Clusters:
       MaxRequestLogParamsSize: 2000
 
     Collections:
-      # Allow clients to create collections by providing a manifest with
-      # unsigned data blob locators. IMPORTANT: This effectively disables
-      # access controls for data stored in Keep: a client who knows a hash
-      # can write a manifest that references the hash, pass it to
-      # collections.create (which will create a permission link), use
-      # collections.get to obtain a signature for that data locator, and
-      # use that signed locator to retrieve the data from Keep. Therefore,
-      # do not turn this on if your users expect to keep data private from
-      # one another!
+
+      # Enable access controls for data stored in Keep. This should
+      # always be set to true on a production cluster.
       BlobSigning: true
 
       # BlobSigningKey is a string of alphanumeric characters used to
@@ -382,16 +382,42 @@ Clusters:
       # still has permission) the client can retrieve the collection again
       # to get fresh signatures.
       #
-      # This must be exactly equal to the -blob-signature-ttl flag used by
-      # keepstore servers.  Otherwise, reading data blocks and saving
-      # collections will fail with HTTP 403 permission errors.
-      #
       # Modifying BlobSigningTTL invalidates existing signatures; see
       # BlobSigningKey note above.
       #
       # The default is 2 weeks.
       BlobSigningTTL: 336h
 
+      # When running keep-balance, this is the destination filename for the
+      # list of lost block hashes if there are any, one per line. Updated atomically during
+      # each successful run.
+      BlobMissingReport: ""
+
+      # keep-balance operates periodically, i.e.: do a
+      # scan/balance operation, sleep, repeat.
+      #
+      # BalancePeriod determines the interval between start times of
+      # successive scan/balance operations. If a scan/balance operation
+      # takes longer than RunPeriod, the next one will follow it
+      # immediately.
+      #
+      # If SIGUSR1 is received during an idle period between operations,
+      # the next operation will start immediately.
+      BalancePeriod: 10m
+
+      # Limits the number of collections retrieved by keep-balance per
+      # API transaction. If this is zero, page size is
+      # determined by the API server's own page size limits (see
+      # API.MaxItemsPerResponse and API.MaxIndexDatabaseRead).
+      BalanceCollectionBatch: 100000
+
+      # The size of keep-balance's internal queue of
+      # collections. Higher values use more memory and improve throughput
+      # by allowing keep-balance to fetch the next page of collections
+      # while the current page is still being processed. If this is zero
+      # or omitted, pages are processed serially.
+      BalanceCollectionBuffers: 1000
+
       # Default lifetime for ephemeral collections: 2 weeks. This must not
       # be less than BlobSigningTTL.
       DefaultTrashLifetime: 336h
@@ -797,9 +823,24 @@ Clusters:
 
     Volumes:
       SAMPLE:
+        # AccessViaHosts specifies which keepstore processes can read
+        # and write data on the volume.
+        #
+        # For a local filesystem, AccessViaHosts has one entry,
+        # indicating which server the filesystem is located on.
+        #
+        # For a network-attached backend accessible by all keepstore
+        # servers, like a cloud storage bucket or an NFS mount,
+        # AccessViaHosts can be empty/omitted.
+        #
+        # Further info/examples:
+        # https://doc.arvados.org/install/configure-fs-storage.html
+        # https://doc.arvados.org/install/configure-s3-object-storage.html
+        # https://doc.arvados.org/install/configure-azure-blob-storage.html
         AccessViaHosts:
           SAMPLE:
             ReadOnly: false
+          "http://host1.example:25107": {}
         ReadOnly: false
         Replication: 1
         StorageClasses:
@@ -808,7 +849,8 @@ Clusters:
         Driver: s3
         DriverParameters:
 
-          # for s3 driver
+          # for s3 driver -- see
+          # https://doc.arvados.org/install/configure-s3-object-storage.html
           AccessKey: aaaaa
           SecretKey: aaaaa
           Endpoint: ""
@@ -821,7 +863,8 @@ Clusters:
           RaceWindow: 24h
           UnsafeDelete: false
 
-          # for azure driver
+          # for azure driver -- see
+          # https://doc.arvados.org/install/configure-azure-blob-storage.html
           StorageAccountName: aaaaa
           StorageAccountKey: aaaaa
           StorageBaseURL: core.windows.net
@@ -833,7 +876,8 @@ Clusters:
           WriteRaceInterval: 15s
           WriteRacePollTime: 1s
 
-          # for local directory driver
+          # for local directory driver -- see
+          # https://doc.arvados.org/install/configure-fs-storage.html
           Root: /var/lib/arvados/keep-data
           Serialize: false