X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/0bb53c1cbcbcb8b3be50e6ecf3fdf0bb7cbd96b5..83f05664d99a7d80b2d2ae9c0517004cbfb5d00d:/doc/install/configure-s3-object-storage.html.textile.liquid diff --git a/doc/install/configure-s3-object-storage.html.textile.liquid b/doc/install/configure-s3-object-storage.html.textile.liquid index 6e725b3449..e9866d5103 100644 --- a/doc/install/configure-s3-object-storage.html.textile.liquid +++ b/doc/install/configure-s3-object-storage.html.textile.liquid @@ -15,10 +15,8 @@ Volumes are configured in the @Volumes@ section of the cluster configuration fil {% include 'assign_volume_uuid' %} -
Clusters:
-  uuid_prefix:
-    Volumes:
-      uuid_prefix-nyw5e-000000000000000:
+
    Volumes:
+      ClusterID-nyw5e-000000000000000:
         AccessViaHosts:
           # This section determines which keepstore servers access the
           # volume. In this example, keep0 has read/write access, and
@@ -27,30 +25,64 @@ Volumes are configured in the @Volumes@ section of the cluster configuration fil
           # If the AccessViaHosts section is empty or omitted, all
           # keepstore servers will have read/write access to the
           # volume.
-          "http://keep0.uuid_prefix.example.com:25107/": {}
-          "http://keep1.uuid_prefix.example.com:25107/": {ReadOnly: true}
+          "http://keep0.ClusterID.example.com:25107": {}
+          "http://keep1.ClusterID.example.com:25107": {ReadOnly: true}
 
-        Driver: S3
+        Driver: S3
         DriverParameters:
-          # The credentials to use to access the bucket.
-          AccessKey: aaaaa
-          SecretKey: aaaaa
+          # Bucket name.
+          Bucket: example-bucket-name
+
+          # IAM role name to use when retrieving credentials from
+          # instance metadata. It can be omitted, in which case the
+          # role name itself will be retrieved from instance metadata
+          # -- but setting it explicitly may protect you from using
+          # the wrong credentials in the event of an
+          # installation/configuration error.
+          IAMRole: ""
+
+          # If you are not using an IAM role for authentication,
+          # specify access credentials here instead.
+          AccessKeyID: ""
+          SecretAccessKey: ""
+
+          # Storage provider region. For Google Cloud Storage, use ""
+          # or omit.
+          Region: us-east-1
 
           # Storage provider endpoint. For Amazon S3, use "" or
           # omit. For Google Cloud Storage, use
           # "https://storage.googleapis.com".
           Endpoint: ""
 
-          # Storage provider region. For Google Cloud Storage, use ""
-          # or omit.
-          Region: us-east-1a
-
           # Change to true if the region requires a LocationConstraint
           # declaration.
           LocationConstraint: false
 
-          # Bucket name.
-          Bucket: example-bucket-name
+          # Use V2 signatures instead of the default V4. Amazon S3
+          # supports V4 signatures in all regions, but this option
+          # might be needed for other S3-compatible services.
+          V2Signature: false
+
+          # Use the AWS S3 v2 Go driver instead of the goamz driver.
+          UseAWSS3v2Driver: false
+
+          # By default keepstore stores data using the MD5 checksum
+          # (32 hexadecimal characters) as the object name, e.g.,
+          # "0123456abc...". Setting PrefixLength to 3 changes this
+          # naming scheme to "012/0123456abc...". This can improve
+          # performance, depending on the S3 service being used. For
+          # example, PrefixLength 3 is recommended to avoid AWS
+          # limitations on the number of read/write operations per
+          # second per prefix (see
+          # https://aws.amazon.com/premiumsupport/knowledge-center/s3-request-limit-avoid-throttling/).
+          #
+          # Note that changing PrefixLength on an existing volume is
+          # not currently supported. Once you have started using a
+          # bucket as an Arvados volume, you should not change its
+          # configured PrefixLength, or configure another volume using
+          # the same bucket and a different PrefixLength.
+          PrefixLength: 0
 
           # Requested page size for "list bucket contents" requests.
           IndexPageSize: 1000
@@ -66,13 +98,6 @@ Volumes are configured in the @Volumes@ section of the cluster configuration fil
           # Maximum eventual consistency latency
           RaceWindow: 24h
 
-          # Enable deletion (garbage collection) even when the
-          # configured BlobTrashLifetime is zero.  WARNING: eventual
-          # consistency may result in race conditions that can cause
-          # data loss.  Do not enable this unless you understand and
-          # accept the risk.
-          UnsafeDelete: false
-
         # How much replication is provided by the underlying bucket.
         # This is used to inform replication decisions at the Keep
         # layer.
@@ -89,3 +114,9 @@ Volumes are configured in the @Volumes@ section of the cluster configuration fil
         # classes" in the "Admin" section of doc.arvados.org.
         StorageClasses: null
 

+
+Two S3 drivers are available. Historically, Arvados has used the @goamz@ driver to talk to S3-compatible services. More recently, support for the @aws-sdk-go-v2@ driver was added. This driver can be activated by setting the @UseAWSS3v2Driver@ flag to @true@.
+
+The @aws-sdk-go-v2@ does not support the old S3 v2 signing algorithm. This will not affect interacting with AWS S3, but it might be an issue when Keep is backed by a very old version of a third party S3-compatible service.
+
+The @aws-sdk-go-v2@ driver can improve read performance by 50-100% over the @goamz@ driver, but it has not had as much production use. See the "wiki":https://dev.arvados.org/projects/arvados/wiki/Keep_real_world_performance_numbers for details.