X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/20a69e126c68236a3c73daab101e351346f8bfaf..fd38b59aac9b4453cf04fb7d6e1b8ad51549d6c3:/lib/config/generated_config.go diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go index 89cfb80097..d21bb2d284 100644 --- a/lib/config/generated_config.go +++ b/lib/config/generated_config.go @@ -31,7 +31,13 @@ Clusters: # listening, and reachable from other hosts in the cluster. SAMPLE: InternalURLs: - "http://example.host:12345": {} + "http://host1.example:12345": {} + "http://host2.example:12345": + # Rendezvous is normally empty/omitted. When changing the + # URL of a Keepstore service, Rendezvous should be set to + # the old URL (with trailing slash omitted) to preserve + # rendezvous ordering. + Rendezvous: "" SAMPLE: Rendezvous: "" ExternalURL: "-" @@ -190,7 +196,7 @@ Clusters: # Maximum number of 64MiB memory buffers per keepstore server # process, or 0 for no limit. - MaxKeepBlockBuffers: 128 + MaxKeepBlobBuffers: 128 # API methods to disable. Disabled methods are not listed in the # discovery document, and respond 404 to all requests. @@ -319,15 +325,9 @@ Clusters: MaxRequestLogParamsSize: 2000 Collections: - # Allow clients to create collections by providing a manifest with - # unsigned data blob locators. IMPORTANT: This effectively disables - # access controls for data stored in Keep: a client who knows a hash - # can write a manifest that references the hash, pass it to - # collections.create (which will create a permission link), use - # collections.get to obtain a signature for that data locator, and - # use that signed locator to retrieve the data from Keep. Therefore, - # do not turn this on if your users expect to keep data private from - # one another! + + # Enable access controls for data stored in Keep. This should + # always be set to true on a production cluster. BlobSigning: true # BlobSigningKey is a string of alphanumeric characters used to @@ -374,17 +374,26 @@ Clusters: # collection's replication_desired attribute is nil. DefaultReplication: 2 - # Lifetime (in seconds) of blob permission signatures generated by - # the API server. This determines how long a client can take (after - # retrieving a collection record) to retrieve the collection data - # from Keep. If the client needs more time than that (assuming the - # collection still has the same content and the relevant user/token - # still has permission) the client can retrieve the collection again - # to get fresh signatures. + # BlobSigningTTL determines the minimum lifetime of transient + # data, i.e., blocks that are not referenced by + # collections. Unreferenced blocks exist for two reasons: + # + # 1) A data block must be written to a disk/cloud backend device + # before a collection can be created/updated with a reference to + # it. + # + # 2) Deleting or updating a collection can remove the last + # remaining reference to a data block. # - # This must be exactly equal to the -blob-signature-ttl flag used by - # keepstore servers. Otherwise, reading data blocks and saving - # collections will fail with HTTP 403 permission errors. + # If BlobSigningTTL is too short, long-running + # processes/containers will fail when they take too long (a) + # between writing blocks and writing collections that reference + # them, or (b) between reading collections and reading the + # referenced blocks. + # + # If BlobSigningTTL is too long, data will still be stored long + # after the referring collections are deleted, and you will + # needlessly fill up disks or waste money on cloud storage. # # Modifying BlobSigningTTL invalidates existing signatures; see # BlobSigningKey note above. @@ -392,6 +401,36 @@ Clusters: # The default is 2 weeks. BlobSigningTTL: 336h + # When running keep-balance, this is the destination filename for + # the list of lost block hashes if there are any, one per line. + # Updated automically during each successful run. + BlobMissingReport: "" + + # keep-balance operates periodically, i.e.: do a + # scan/balance operation, sleep, repeat. + # + # BalancePeriod determines the interval between start times of + # successive scan/balance operations. If a scan/balance operation + # takes longer than RunPeriod, the next one will follow it + # immediately. + # + # If SIGUSR1 is received during an idle period between operations, + # the next operation will start immediately. + BalancePeriod: 10m + + # Limits the number of collections retrieved by keep-balance per + # API transaction. If this is zero, page size is + # determined by the API server's own page size limits (see + # API.MaxItemsPerResponse and API.MaxIndexDatabaseRead). + BalanceCollectionBatch: 0 + + # The size of keep-balance's internal queue of + # collections. Higher values use more memory and improve throughput + # by allowing keep-balance to fetch the next page of collections + # while the current page is still being processed. If this is zero + # or omitted, pages are processed serially. + BalanceCollectionBuffers: 1000 + # Default lifetime for ephemeral collections: 2 weeks. This must not # be less than BlobSigningTTL. DefaultTrashLifetime: 336h @@ -458,11 +497,20 @@ Clusters: MaxUUIDEntries: 1000 Login: - # These settings are provided by your OAuth2 provider (e.g., - # sso-provider). + # These settings are provided by your OAuth2 provider (eg + # Google) used to perform upstream authentication. ProviderAppSecret: "" ProviderAppID: "" + # The cluster ID to delegate the user database. When set, + # logins on this cluster will be redirected to the login cluster + # (login cluster must appear in RemoteHosts with Proxy: true) + LoginCluster: "" + + # How long a cached token belonging to a remote cluster will + # remain valid before it needs to be revalidated. + RemoteTokenRefresh: 5m + Git: # Path to git or gitolite-shell executable. Each authenticated # request will execute this program with the single argument "http-backend" @@ -788,9 +836,24 @@ Clusters: Volumes: SAMPLE: + # AccessViaHosts specifies which keepstore processes can read + # and write data on the volume. + # + # For a local filesystem, AccessViaHosts has one entry, + # indicating which server the filesystem is located on. + # + # For a network-attached backend accessible by all keepstore + # servers, like a cloud storage bucket or an NFS mount, + # AccessViaHosts can be empty/omitted. + # + # Further info/examples: + # https://doc.arvados.org/install/configure-fs-storage.html + # https://doc.arvados.org/install/configure-s3-object-storage.html + # https://doc.arvados.org/install/configure-azure-blob-storage.html AccessViaHosts: SAMPLE: ReadOnly: false + "http://host1.example:25107": {} ReadOnly: false Replication: 1 StorageClasses: @@ -799,7 +862,9 @@ Clusters: Driver: s3 DriverParameters: - # for s3 driver + # for s3 driver -- see + # https://doc.arvados.org/install/configure-s3-object-storage.html + IAMRole: aaaaa AccessKey: aaaaa SecretKey: aaaaa Endpoint: "" @@ -812,7 +877,8 @@ Clusters: RaceWindow: 24h UnsafeDelete: false - # for azure driver + # for azure driver -- see + # https://doc.arvados.org/install/configure-azure-blob-storage.html StorageAccountName: aaaaa StorageAccountKey: aaaaa StorageBaseURL: core.windows.net @@ -824,7 +890,8 @@ Clusters: WriteRaceInterval: 15s WriteRacePollTime: 1s - # for local directory driver + # for local directory driver -- see + # https://doc.arvados.org/install/configure-fs-storage.html Root: /var/lib/arvados/keep-data Serialize: false