From 1a37db7b0a70b5672cd0f7c86b19b85ba824378f Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Thu, 14 Jun 2018 12:22:09 -0400 Subject: [PATCH] 13579: Update from comments Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- ...ure-azure-blob-storage.html.textile.liquid | 4 +- .../configure-fs-storage.html.textile.liquid | 56 +++++++++++++ ...gure-s3-object-storage.html.textile.liquid | 23 ++++-- .../install-keepstore.html.textile.liquid | 78 ++++--------------- 4 files changed, 92 insertions(+), 69 deletions(-) create mode 100644 doc/install/configure-fs-storage.html.textile.liquid diff --git a/doc/install/configure-azure-blob-storage.html.textile.liquid b/doc/install/configure-azure-blob-storage.html.textile.liquid index e343737264..d2c4e709df 100644 --- a/doc/install/configure-azure-blob-storage.html.textile.liquid +++ b/doc/install/configure-azure-blob-storage.html.textile.liquid @@ -9,7 +9,7 @@ Copyright (C) The Arvados Authors. All rights reserved. SPDX-License-Identifier: CC-BY-SA-3.0 {% endcomment %} -As an alternative to local and network-attached POSIX filesystems, Keepstore can store data in an Azure Storage container. +Keepstore can store data in one or more Azure Storage containers. h2. Create a storage container @@ -64,7 +64,7 @@ Volumes: ReadOnly: false # Amount of time to wait for a response before failing the request - RequestTimeout: 10m0s + RequestTimeout: 2m0s # The storage account name, used for authentication StorageAccountName: exampleStorageAccountName diff --git a/doc/install/configure-fs-storage.html.textile.liquid b/doc/install/configure-fs-storage.html.textile.liquid new file mode 100644 index 0000000000..730eed6653 --- /dev/null +++ b/doc/install/configure-fs-storage.html.textile.liquid @@ -0,0 +1,56 @@ +--- +layout: default +navsection: installguide +title: Filesystem storage +... +{% comment %} +Copyright (C) The Arvados Authors. All rights reserved. + +SPDX-License-Identifier: CC-BY-SA-3.0 +{% endcomment %} + +Keepstore can store data in local and network-attached POSIX filesystems. + +h2. Setting up filesystem mounts + +Volumes are configured in the @Volumes@ section of the configuration file. You may provide multiple volumes for a single keepstore process to manage multiple disks. Keepstore distributes blocks among volumes in round-robin fashion. + +
+Volumes:
+- # The volume type, indicates this is a filesystem directory.
+  Type: Directory
+
+  # The directory that will be used as the backing store.
+  Root: /mnt/local-disk
+
+  # How much replication is performed by the underlying filesystem.
+  # (for example, a network filesystem may provide its own replication).
+  # This is used to inform replication decisions at the Keep layer.
+  DirectoryReplication: 1
+
+  # If true, do not accept write or trash operations, only reads.
+  ReadOnly: false
+
+  # When true, read and write operations (for whole 64MiB blocks) on
+  # an individual volume will queued and issued serially.  When
+  # false, read and write operations will be issued concurrently.
+  #
+  # May improve throughput if you experience contention when there are
+  # multiple requests to the same volume.
+  #
+  # When using SSDs, RAID, or a parallel network filesystem, you probably
+  # don't want this.
+  Serialize: false
+
+  # Storage classes to associate with this volume.  See "Configuring
+  # storage classes" in the "Admin" section of doc.arvados.org.
+  StorageClasses: null
+
+  # Example of a second volume section
+- DirectoryReplication: 2
+  ReadOnly: false
+  Root: /mnt/network-disk
+  Serialize: false
+  StorageClasses: null
+  Type: Directory
+
diff --git a/doc/install/configure-s3-object-storage.html.textile.liquid b/doc/install/configure-s3-object-storage.html.textile.liquid index 33195f3eac..b4d9e929cb 100644 --- a/doc/install/configure-s3-object-storage.html.textile.liquid +++ b/doc/install/configure-s3-object-storage.html.textile.liquid @@ -9,7 +9,7 @@ Copyright (C) The Arvados Authors. All rights reserved. SPDX-License-Identifier: CC-BY-SA-3.0 {% endcomment %} -As an alternative to local and network-attached POSIX filesystems, Keepstore can store data in object storage compatible with the S3 API, such as Amazon S3, Google Cloud Storage, or Ceph RADOS. +Keepstore can store data in object storage compatible with the S3 API, such as Amazon S3, Google Cloud Storage, or Ceph RADOS. h2. Configure keepstore @@ -23,14 +23,18 @@ EOF -Next, edit the @Volumes@ section of the @keepstore.yml@ config file: +Next, edit the @Volumes@ section of the @keepstore.yml@ config file. + +h3. Example config for Amazon S3 
 Volumes:
 - # The volume type, this indicates object storage compatible with the S3 API
   Type: S3
 
-  # Storage provider (blank uses Amazon S3 by default)
+  # Storage provider.  If blank, uses Amazon S3 by default.
+  # See below for example alternate configuration for Google cloud
+  # storage.
   Endpoint: ""
 
   # The bucket to use for the backing store.
@@ -61,7 +65,7 @@ Volumes:
 
   # Maximum time to wait for a complete response from the backend before
   # failing the request.
-  ReadTimeout: 5m0s
+  ReadTimeout: 2m0s
 
   # How much replication is performed by the underlying bucket.
   # This is used to inform replication decisions at the Keep layer.
@@ -76,7 +80,16 @@ Volumes:
   # that can cause data loss.  Do not enable this unless you know what
   # you are doing.
   UnsafeDelete: false
+
+ +Start (or restart) keepstore, and check its log file to confirm it is using the new configuration. +h3. Example config for Google cloud storage + +See previous section for documentation of configuration fields. + + +Volumes: - # Example configuration using alternate storage provider # Configuration for Google cloud storage Endpoint: https://storage.googleapis.com @@ -90,7 +103,7 @@ Volumes: LocationConstraint: false RaceWindow: 24h0m0s ReadOnly: false - ReadTimeout: 5m0s + ReadTimeout: 2m0s S3Replication: 2 StorageClasses: null UnsafeDelete: false diff --git a/doc/install/install-keepstore.html.textile.liquid b/doc/install/install-keepstore.html.textile.liquid index 61553f73ea..402d15cb53 100644 --- a/doc/install/install-keepstore.html.textile.liquid +++ b/doc/install/install-keepstore.html.textile.liquid @@ -116,8 +116,9 @@ MaxRequests: 0 PIDFile: "" # Maximum number of concurrent pull operations. Default is 1, i.e., -# pull lists are processed serially. -PullWorkers: 0 +# pull lists are processed serially. A pull operation copies a block +# from another keepstore server. +PullWorkers: 1 # Honor read requests only if a valid signature is provided. This # should be true, except for development use and when migrating from @@ -150,79 +151,32 @@ TrashCheckInterval: 24h0m0s # recovered using an /untrash request. TrashLifetime: 336h0m0s -# Maximum number of concurrent trash operations. Default is 1, i.e., -# trash lists are processed serially. +# Maximum number of concurrent trash operations (moving a block to the +# trash, or permanently deleting it) . Default is 1, i.e., trash lists +# are processed serially. If individual trash operations have high +# latency (eg some cloud platforms) you should increase this. TrashWorkers: 1 h3. Notes on storage management -On its own, a keepstore server never deletes data. The "keep-balance":install-keep-balance.html service service determines which blocks are candidates for deletion and instructs the keepstore to move those blocks to the trash. +On its own, a keepstore server never deletes data. The "keep-balance":install-keep-balance.html service determines which blocks are candidates for deletion and instructs the keepstore to move those blocks to the trash. When a block is newly written, it is protected from deletion for the duration in @BlobSignatureTTL@. During this time, it cannot be trashed. -If keep-balance instructs keepstore to trash a block which is older than @BlobSignatureTTL@, and @EnableDelete@ is true, the block will be moved to "trash". A block which is in the trash has been moved out of the way and is no longer accessible by read requests, but has not yet been permanently deleted. Blocks which are in the trash may be recovered using the "untrash" API endpoint. Blocks are permanently deleted after they have been in the trash for the duration in @TrashLifetime@. +If keep-balance instructs keepstore to trash a block which is older than @BlobSignatureTTL@, and @EnableDelete@ is true, the block will be moved to "trash". A block which is in the trash is no longer accessible by read requests, but has not yet been permanently deleted. Blocks which are in the trash may be recovered using the "untrash" API endpoint. Blocks are permanently deleted after they have been in the trash for the duration in @TrashLifetime@. -h3. Configure storage volumes - -Available storage volume types include cloud object storage and POSIX filesystems. - -If you are using S3-compatible object storage (including Amazon S3, Google Cloud Storage, and Ceph RADOS), follow the setup instructions "S3 Object Storage":configure-s3-object-storage.html page instead and then "Run keepstore as a supervised service.":#keepstoreservice - -If you are using Azure Blob Storage, follow the setup instructions "Azure Blob Storage":configure-azure-blob-storage.html and then proceed to "Run keepstore as a supervised service.":#keepstoreservice +Keep-balance is also responsible for balancing the distribution of blocks across keepstore servers by asking servers to pull blocks from other servers (as determined by their storage class and "rendezvous hashing order":{{site.baseurl}}/api/storage.html). Pulling a block makes a copy. If a block is overreplicated (i.e. there are excess copies) after pulling, it will be subsequently trashed on the original server. -To use a POSIX filesystem, including both local filesystems (ext4, xfs) and network file system such as GPFS or Lustre, continue reading this section. - -h4. Setting up filesystem mounts +h3. Configure storage volumes -Volumes are configured in the @Volumes@ section of the configuration -file. You may provide multiple volumes for a single keepstore process -to manage multiple disks. Keepstore distributes blocks among volumes -in round-robin fashion. +Available storage volume types include POSIX filesystems and cloud object storage. -
-Volumes:
-- # The volume type, indicates this is a filesystem directory.
-  Type: Directory
-
-  # The actual directory that will be used as the backing store.
-  Root: /mnt/local-disk
-
-  # How much replication is performed by the underlying filesystem.
-  # (for example, a network filesystem may provide its own replication).
-  # This is used to inform replication decisions at the Keep layer.
-  DirectoryReplication: 1
-
-  # If true, do not accept write or trash operations, only reads.
-  ReadOnly: false
-
-  # When true, read and write operations (for whole 64MiB blocks) on
-  # an individual volume will queued and issued sequentially.  When
-  # false, read and write operations will be issued concurrently as
-  # they come in.
-  #
-  # When using spinning disks where storage partitions map 1:1 to
-  # physical disks that are dedicated to Keepstore, enabling this may
-  # reduce contention and improve throughput by minimizing seeks.
-  #
-  # When using SSDs, RAID, or a parallel network filesystem, you probably
-  # don't want this.
-  Serialize: true
-
-  # Storage classes to associate with this volume.  See "Configuring
-  # storage classes" in the "Admin" section of doc.arvados.org.
-  StorageClasses: null
-
-  # Example of a second volume section
-- DirectoryReplication: 2
-  ReadOnly: false
-  Root: /mnt/network-disk
-  Serialize: false
-  StorageClasses: null
-  Type: Directory
-
+* To use a POSIX filesystem, including both local filesystems (ext4, xfs) and network file system such as GPFS or Lustre, follow the setup instructions on "Filesystem storage":configure-fs-storage.html +* If you are using S3-compatible object storage (including Amazon S3, Google Cloud Storage, and Ceph RADOS), follow the setup instructions on "S3 Object Storage":configure-s3-object-storage.html +* If you are using Azure Blob Storage, follow the setup instructions on "Azure Blob Storage":configure-azure-blob-storage.html -h3(#keepstoreservice). Run keepstore as a supervised service +h3. Run keepstore as a supervised service Install runit to supervise the keepstore daemon. {% include 'install_runit' %} -- 2.30.2