X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/080c940d7a8134a6e277a53b7e45eb27e2b2c87f..6f5e2a33099a0120f4acb2f24430fb74dcfc689a:/doc/install/configure-s3-object-storage.html.textile.liquid?ds=sidebyside
diff --git a/doc/install/configure-s3-object-storage.html.textile.liquid b/doc/install/configure-s3-object-storage.html.textile.liquid
index 88172fa9f7..b4e0c1a312 100644
--- a/doc/install/configure-s3-object-storage.html.textile.liquid
+++ b/doc/install/configure-s3-object-storage.html.textile.liquid
@@ -9,104 +9,142 @@ Copyright (C) The Arvados Authors. All rights reserved.
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-Keepstore can store data in object storage compatible with the S3 API, such as Amazon S3, Google Cloud Storage, or Ceph RADOS.
-
-h2. Configure keepstore
-
-Copy the "access key" and "secret key" to files where they will be accessible to keepstore at startup time.
+Keepstore can store data in object storage compatible with the S3 API, such as Amazon S3, Google Cloud Storage, Ceph RADOS, NetApp StorageGRID, and others.
+
+Volumes are configured in the @Volumes@ section of the cluster configuration file.
+
+# "Configuration example":#example
+# "IAM Policy":#IAM
+
+h2(#example). Configuration example
+
+{% include 'assign_volume_uuid' %}
+
+ Volumes:
+ ClusterID-nyw5e-000000000000000:
+ AccessViaHosts:
+ # This section determines which keepstore servers access the
+ # volume. In this example, keep0 has read/write access, and
+ # keep1 has read-only access.
+ #
+ # If the AccessViaHosts section is empty or omitted, all
+ # keepstore servers will have read/write access to the
+ # volume.
+ "http://keep0.ClusterID.example.com:25107": {}
+ "http://keep1.ClusterID.example.com:25107": {ReadOnly: true}
+
+ Driver: S3
+ DriverParameters:
+ # Bucket name.
+ Bucket: example-bucket-name
+
+ # IAM role name to use when retrieving credentials from
+ # instance metadata. It can be omitted, in which case the
+ # role name itself will be retrieved from instance metadata
+ # -- but setting it explicitly may protect you from using
+ # the wrong credentials in the event of an
+ # installation/configuration error.
+ IAMRole: ""
+
+ # If you are not using an IAM role for authentication,
+ # specify access credentials here instead.
+ AccessKeyID: ""
+ SecretAccessKey: ""
+
+ # Storage provider region. If Endpoint is specified, the
+ # region determines the request signing method, and defaults
+ # to "us-east-1".
+ Region: us-east-1
+
+ # Storage provider endpoint. For Amazon S3, use "" or
+ # omit. For Google Cloud Storage, use
+ # "https://storage.googleapis.com".
+ Endpoint: ""
+
+ # Change to true if the region requires a LocationConstraint
+ # declaration.
+ LocationConstraint: false
+
+ # Use V2 signatures instead of the default V4. Amazon S3
+ # supports V4 signatures in all regions, but this option
+ # might be needed for other S3-compatible services.
+ V2Signature: false
+
+ # Use the AWS S3 v2 Go driver instead of the goamz driver.
+ UseAWSS3v2Driver: false
+
+ # By default keepstore stores data using the MD5 checksum
+ # (32 hexadecimal characters) as the object name, e.g.,
+ # "0123456abc...". Setting PrefixLength to 3 changes this
+ # naming scheme to "012/0123456abc...". This can improve
+ # performance, depending on the S3 service being used. For
+ # example, PrefixLength 3 is recommended to avoid AWS
+ # limitations on the number of read/write operations per
+ # second per prefix (see
+ # https://aws.amazon.com/premiumsupport/knowledge-center/s3-request-limit-avoid-throttling/).
+ #
+ # Note that changing PrefixLength on an existing volume is
+ # not currently supported. Once you have started using a
+ # bucket as an Arvados volume, you should not change its
+ # configured PrefixLength, or configure another volume using
+ # the same bucket and a different PrefixLength.
+ PrefixLength: 0
+
+ # Requested page size for "list bucket contents" requests.
+ IndexPageSize: 1000
+
+ # Maximum time to wait while making the initial connection
+ # to the backend before failing the request.
+ ConnectTimeout: 1m
+
+ # Maximum time to wait for a complete response from the
+ # backend before failing the request.
+ ReadTimeout: 2m
+
+ # Maximum eventual consistency latency
+ RaceWindow: 24h
+
+ # How much replication is provided by the underlying bucket.
+ # This is used to inform replication decisions at the Keep
+ # layer.
+ Replication: 2
+
+ # If true, do not accept write or trash operations, even if
+ # AccessViaHosts.*.ReadOnly is false.
+ #
+ # If false or omitted, enable write access (subject to
+ # AccessViaHosts.*.ReadOnly, where applicable).
+ ReadOnly: false
+
+ # Storage classes to associate with this volume. See "Storage
+ # classes" in the "Admin" section of doc.arvados.org.
+ StorageClasses: null
+
-~$ sudo sh -c 'cat >/etc/arvados/keepstore/aws_s3_access_key.txt <<EOF'
-zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz==
-EOF
-~$ sudo chmod 0400 /etc/arvados/keepstore/aws_s3_access_key.txt
-
-Volumes: -- # The volume type, this indicates object storage compatible with the S3 API - Type: S3 - - # Storage provider. If blank, uses Amazon S3 by default. - # See below for example alternate configuration for Google cloud - # storage. - Endpoint: "" - - # The bucket to use for the backing store. - Bucket: example-bucket-name - - # The region where the bucket is located. - Region: us-east-1 - - # The credentials to use to access the bucket. - AccessKeyFile: /etc/arvados/keepstore/aws_s3_access_key.txt - SecretKeyFile: /etc/arvados/keepstore/aws_s3_secret_key.txt - - # Maximum time to wait making the initial connection to the backend before - # failing the request. - ConnectTimeout: 1m0s - - # Page size for s3 "list bucket contents" requests - IndexPageSize: 1000 - - # True if the region requires a LocationConstraint declaration - LocationConstraint: false - - # Maximum eventual consistency latency - RaceWindow: 24h0m0s - - # If true, do not accept write or trash operations, only reads. - ReadOnly: false - - # Maximum time to wait for a complete response from the backend before - # failing the request. - ReadTimeout: 2m0s - - # How much replication is performed by the underlying bucket. - # This is used to inform replication decisions at the Keep layer. - S3Replication: 2 - - # Storage classes to associate with this volume. See - # "Storage classes" in the "Admin" section of doc.arvados.org. - StorageClasses: null - - # Enable deletion (garbage collection) even when TrashLifetime is - # zero. WARNING: eventual consistency may result in race conditions - # that can cause data loss. Do not enable this unless you know what - # you are doing. - UnsafeDelete: false +{ + "Id": "arvados-keepstore policy", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:*" + ], + "Resource": "arn:aws:s3:::xarv1-nyw5e-000000000000000-volume" + "Resource": "arn:aws:s3:::xarv1-nyw5e-000000000000000-volume/*" + } + ] +}- -Start (or restart) keepstore, and check its log file to confirm it is using the new configuration. - -h3. Example config for Google cloud storage - -See previous section for documentation of configuration fields. - -
-Volumes: -- # Example configuration using alternate storage provider - # Configuration for Google cloud storage - Endpoint: https://storage.googleapis.com - Region: "" - - AccessKeyFile: /etc/arvados/keepstore/gce_s3_access_key.txt - SecretKeyFile: /etc/arvados/keepstore/gce_s3_secret_key.txt - Bucket: example-bucket-name - ConnectTimeout: 1m0s - IndexPageSize: 1000 - LocationConstraint: false - RaceWindow: 24h0m0s - ReadOnly: false - ReadTimeout: 2m0s - S3Replication: 2 - StorageClasses: null - UnsafeDelete: false -- -Start (or restart) keepstore, and check its log file to confirm it is using the new configuration. +