image builder.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward@curii.com>
</code></pre>
</notextile>
+Example policy for the IAM role used by the cloud dispatcher:
+
+<notextile>
+<pre>
+{
+ "Version": "2012-10-17",
+ "Id": "arvados-dispatch-cloud policy",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": [
+ "iam:PassRole",
+ "ec2:DescribeKeyPairs",
+ "ec2:ImportKeyPair",
+ "ec2:RunInstances",
+ "ec2:DescribeInstances",
+ "ec2:CreateTags",
+ "ec2:TerminateInstances"
+ ],
+ "Resource": "*"
+ }
+ ]
+}
+</pre>
+</notextile>
+
h4. Minimal configuration example for Azure
Using managed disks:
Region: ""
EBSVolumeType: gp2
AdminUsername: debian
+ # (ec2) name of the IamInstanceProfile for instances started by
+ # the cloud dispatcher. Leave blank when not needed.
IamInstanceProfile: ""
# (azure) Credentials.
"aws_profile": "",
"aws_secret_key": "",
"aws_source_ami": "ami-031283ff8a43b021c",
+ "aws_ebs_autoscale": "",
"build_environment": "aws",
"public_key_file": "",
"mksquashfs_mem": "",
"type": "file",
"source": "scripts/usr-local-bin-ensure-encrypted-partitions.sh",
"destination": "/tmp/usr-local-bin-ensure-encrypted-partitions.sh"
+ },{
+ "type": "file",
+ "source": "scripts/usr-local-bin-ensure-encrypted-partitions-aws-ebs-autoscale.sh",
+ "destination": "/tmp/usr-local-bin-ensure-encrypted-partitions-aws-ebs-autoscale.sh"
+ },{
+ "type": "file",
+ "source": "scripts/create-ebs-volume-nvme.patch",
+ "destination": "/tmp/create-ebs-volume-nvme.patch"
},{
"type": "file",
"source": "{{user `public_key_file`}}",
"type": "shell",
"execute_command": "sudo -S env {{ .Vars }} /bin/bash '{{ .Path }}'",
"script": "scripts/base.sh",
- "environment_vars": ["RESOLVER={{user `resolver`}}","REPOSUFFIX={{user `reposuffix`}}","MKSQUASHFS_MEM={{user `mksquashfs_mem`}}","NVIDIA_GPU_SUPPORT={{user `nvidia_gpu_support`}}","CLOUD=aws"]
+ "environment_vars": ["RESOLVER={{user `resolver`}}","REPOSUFFIX={{user `reposuffix`}}","MKSQUASHFS_MEM={{user `mksquashfs_mem`}}","NVIDIA_GPU_SUPPORT={{user `nvidia_gpu_support`}}","CLOUD=aws","AWS_EBS_AUTOSCALE={{user `aws_ebs_autoscale`}}"]
}]
}
VPC id for AWS, otherwise packer will pick the default one
--aws-subnet-id
Subnet id for AWS otherwise packer will pick the default one for the VPC
+ --aws-ebs-autoscale (default: false)
+ Install the AWS EBS autoscaler daemon.
--gcp-project-id (default: false, required if building for GCP)
GCP project id
--gcp-account-file (default: false, required if building for GCP)
--debug (default: false)
Output debug information
+For more information, see the Arvados documentation at https://doc.arvados.org/install/crunch2-cloud/install-compute-node.html
+
EOF
JSON_FILE=
AWS_SOURCE_AMI=
AWS_VPC_ID=
AWS_SUBNET_ID=
+AWS_EBS_AUTOSCALE=
GCP_PROJECT_ID=
GCP_ACCOUNT_FILE=
GCP_ZONE=
NVIDIA_GPU_SUPPORT=
PARSEDOPTS=$(getopt --name "$0" --longoptions \
- help,json-file:,arvados-cluster-id:,aws-source-ami:,aws-profile:,aws-secrets-file:,aws-region:,aws-vpc-id:,aws-subnet-id:,gcp-project-id:,gcp-account-file:,gcp-zone:,azure-secrets-file:,azure-resource-group:,azure-location:,azure-sku:,azure-cloud-environment:,ssh_user:,resolver:,reposuffix:,public-key-file:,mksquashfs-mem:,nvidia-gpu-support,debug \
+ help,json-file:,arvados-cluster-id:,aws-source-ami:,aws-profile:,aws-secrets-file:,aws-region:,aws-vpc-id:,aws-subnet-id:,aws-ebs-autoscale,gcp-project-id:,gcp-account-file:,gcp-zone:,azure-secrets-file:,azure-resource-group:,azure-location:,azure-sku:,azure-cloud-environment:,ssh_user:,resolver:,reposuffix:,public-key-file:,mksquashfs-mem:,nvidia-gpu-support,debug \
-- "" "$@")
if [ $? -ne 0 ]; then
exit 1
--aws-subnet-id)
AWS_SUBNET_ID="$2"; shift
;;
+ --aws-ebs-autoscale)
+ AWS_EBS_AUTOSCALE=1
+ ;;
--gcp-project-id)
GCP_PROJECT_ID="$2"; shift
;;
if [[ "$AWS_DEFAULT_REGION" != "" ]]; then
EXTRA2+=" -var aws_default_region=$AWS_DEFAULT_REGION"
fi
+if [[ "$AWS_EBS_AUTOSCALE" != "" ]]; then
+ EXTRA2+=" -var aws_ebs_autoscale=$AWS_EBS_AUTOSCALE"
+fi
if [[ "$GCP_PROJECT_ID" != "" ]]; then
EXTRA2+=" -var project_id=$GCP_PROJECT_ID"
fi
if [ "x$RESOLVER" != "x" ]; then
$SUDO sed -i "s/#prepend domain-name-servers 127.0.0.1;/prepend domain-name-servers ${RESOLVER};/" /etc/dhcp/dhclient.conf
fi
-# Set up the cloud-init script that will ensure encrypted disks
-$SUDO mv /tmp/usr-local-bin-ensure-encrypted-partitions.sh /usr/local/bin/ensure-encrypted-partitions.sh
+
+if [ "$AWS_EBS_AUTOSCALE" != "1" ]; then
+ # Set up the cloud-init script that will ensure encrypted disks
+ $SUDO mv /tmp/usr-local-bin-ensure-encrypted-partitions.sh /usr/local/bin/ensure-encrypted-partitions.sh
+else
+ wait_for_apt_locks && $SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes install jq unzip
+
+ curl -s "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "/tmp/awscliv2.zip"
+ unzip -q /tmp/awscliv2.zip -d /tmp && $SUDO /tmp/aws/install
+ # Pinned to v2.4.5 because we apply a patch below
+ #export EBS_AUTOSCALE_VERSION=$(curl --silent "https://api.github.com/repos/awslabs/amazon-ebs-autoscale/releases/latest" | jq -r .tag_name)
+ export EBS_AUTOSCALE_VERSION="v2.4.5"
+ cd /opt && $SUDO git clone https://github.com/awslabs/amazon-ebs-autoscale.git
+ cd /opt/amazon-ebs-autoscale && $SUDO git checkout $EBS_AUTOSCALE_VERSION
+ cd bin
+ $SUDO patch -p1 < /tmp/create-ebs-volume-nvme.patch
+
+ # This script really requires bash and the shebang line is wrong
+ $SUDO sed -i 's|^#!/bin/sh|#!/bin/bash|' /opt/amazon-ebs-autoscale/bin/ebs-autoscale
+
+ # Set up the cloud-init script that makes use of the AWS EBS autoscaler
+ $SUDO mv /tmp/usr-local-bin-ensure-encrypted-partitions-aws-ebs-autoscale.sh /usr/local/bin/ensure-encrypted-partitions.sh
+fi
+
$SUDO chmod 755 /usr/local/bin/ensure-encrypted-partitions.sh
$SUDO chown root:root /usr/local/bin/ensure-encrypted-partitions.sh
$SUDO mv /tmp/etc-cloud-cloud.cfg.d-07_compute_arvados_dispatch_cloud.cfg /etc/cloud/cloud.cfg.d/07_compute_arvados_dispatch_cloud.cfg
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+Make the create-ebs-volume script work with nvme devices.
+
+--- a/create-ebs-volume 2022-02-18 15:24:19.866607848 -0500
++++ b/create-ebs-volume 2022-02-18 16:23:17.931870970 -0500
+@@ -149,9 +152,20 @@
+ for letter in ${alphabet[@]}; do
+ # use /dev/xvdb* device names to avoid contention for /dev/sd* and /dev/xvda names
+ # only supported by HVM instances
+- if [ ! -b "/dev/xvdb${letter}" ]; then
++ if [[ $created_volumes =~ .*/dev/xvdb${letter}.* ]]; then
++ continue
++ fi
+ echo "/dev/xvdb${letter}"
+ break
++ done
++}
++
++numbers=( {1..255} )
++function get_next_logical_nvme_device() {
++ for num in ${numbers[@]}; do
++ if [ ! -b "/dev/nvme${num}n1" ]; then
++ echo "/dev/nvme${num}"
++ break
+ fi
+ done
+ }
+@@ -243,10 +257,12 @@
+
+ # check if there are available device names
+ local device=$(get_next_logical_device)
++ local nvme_device=$(get_next_logical_nvme_device)
+ if [ -z "$device" ]; then
+ error "no device names available for volume"
+ fi
+ logthis "next available device: $device"
++ logthis "next available nvme device: $nvme_device"
+
+ # create the volume
+ local tmpfile=$(mktemp /tmp/ebs-autoscale.create-volume.XXXXXXXXXX)
+@@ -323,8 +339,8 @@
+
+ logthis "waiting for volume $volume_id on filesystem"
+ while true; do
+- if [ -e "$device" ]; then
+- logthis "volume $volume_id on filesystem as $device"
++ if [ -e "$nvme_device" ]; then
++ logthis "volume $volume_id on filesystem as $nvme_device (aws device $device)"
+ break
+ fi
+ sleep 1
+@@ -338,7 +354,7 @@
+ > /dev/null
+ logthis "volume $volume_id DeleteOnTermination ENABLED"
+
+- echo $device
++ echo "$nvme_device"n1
+ }
+
+ create_and_attach_volume
--- /dev/null
+#!/bin/bash
+
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+set -e
+set -x
+
+MOUNTPATH=/tmp
+
+findmntq() {
+ findmnt "$@" >/dev/null
+}
+
+ensure_umount() {
+ if findmntq "$1"; then
+ umount "$1"
+ fi
+}
+
+# First make sure docker is not using /tmp, then unmount everything under it.
+if [ -d /etc/sv/docker.io ]
+then
+ sv stop docker.io || service stop docker.io || true
+else
+ service docker stop || true
+fi
+
+ensure_umount "$MOUNTPATH/docker/aufs"
+
+/bin/bash /opt/amazon-ebs-autoscale/install.sh -f lvm.ext4 -m $MOUNTPATH 2>&1 > /var/log/ebs-autoscale-install.log
+
+# Make sure docker uses the big partition
+cat <<EOF > /etc/docker/daemon.json
+{
+ "data-root": "$MOUNTPATH/docker-data"
+}
+EOF
+
+# restart docker
+if [ -d /etc/sv/docker.io ]
+then
+ ## runit
+ sv up docker.io
+else
+ service docker start
+fi
+
+end=$((SECONDS+60))
+
+while [ $SECONDS -lt $end ]; do
+ if /usr/bin/docker ps -q >/dev/null; then
+ exit 0
+ fi
+ sleep 1
+done
+
+# Docker didn't start within a minute, abort
+exit 1
projects / arvados.git / commitdiff