tools/compute-images/build.sh

   1 #!/bin/bash
   2
   3 # Copyright (C) The Arvados Authors. All rights reserved.
   4 #
   5 # SPDX-License-Identifier: Apache-2.0
   6
   7 JSON_FILE=$1
   8 ARVADOS_CLUSTER=$2
   9 PROJECT_ID=$3
  10 ACCOUNT_FILE=$4
  11
  12 read -rd "\000" helpmessage <<EOF
  13 $(basename $0): Build cloud images for arvados-dispatch-cloud
  14
  15 Syntax:
  16         $(basename $0) [options]
  17
  18 Options:
  19
  20   --json-file (required)
  21       Path to the packer json file
  22   --arvados-cluster-id (required)
  23       The ID of the Arvados cluster, e.g. zzzzz
  24   --aws-profile (default: false)
  25       AWS profile to use (valid profile from ~/.aws/config
  26   --aws-secrets-file (default: false, required if building for AWS)
  27       AWS secrets file which will be sourced from this script
  28   --aws-source-ami (default: false, required if building for AWS)
  29       The AMI to use as base for building the images
  30   --aws-region (default: us-east-1)
  31       The AWS region to use for building the images
  32   --aws-vpc-id (optional)
  33       VPC id for AWS, otherwise packer will pick the default one
  34   --aws-subnet-id
  35       Subnet id for AWS otherwise packer will pick the default one for the VPC
  36   --aws-ebs-autoscale (default: false)
  37       Install the AWS EBS autoscaler daemon.
  38   --gcp-project-id (default: false, required if building for GCP)
  39       GCP project id
  40   --gcp-account-file (default: false, required if building for GCP)
  41       GCP account file
  42   --gcp-zone (default: us-central1-f)
  43       GCP zone
  44   --azure-secrets-file (default: false, required if building for Azure)
  45       Azure secrets file which will be sourced from this script
  46   --azure-resource-group (default: false, required if building for Azure)
  47       Azure resource group
  48   --azure-location (default: false, required if building for Azure)
  49       Azure location, e.g. centralus, eastus, westeurope
  50   --azure-sku (default: unset, required if building for Azure, e.g. 16.04-LTS)
  51       Azure SKU image to use
  52   --ssh_user  (default: packer)
  53       The user packer will use to log into the image
  54   --resolver (default: host's network provided)
  55       The dns resolver for the machine
  56   --reposuffix (default: unset)
  57       Set this to "-dev" to track the unstable/dev Arvados repositories
  58   --public-key-file (required)
  59       Path to the public key file that a-d-c will use to log into the compute node
  60   --mksquashfs-mem (default: 256M)
  61       Only relevant when using Singularity. This is the amount of memory mksquashfs is allowed to use.
  62   --nvidia-gpu-support (default: false)
  63       Install all the necessary tooling for Nvidia GPU support
  64   --debug (default: false)
  65       Output debug information
  66
  67 For more information, see the Arvados documentation at https://doc.arvados.org/install/crunch2-cloud/install-compute-node.html
  68
  69 EOF
  70
  71 JSON_FILE=
  72 ARVADOS_CLUSTER_ID=
  73 AWS_PROFILE=
  74 AWS_SECRETS_FILE=
  75 AWS_SOURCE_AMI=
  76 AWS_VPC_ID=
  77 AWS_SUBNET_ID=
  78 AWS_EBS_AUTOSCALE=
  79 GCP_PROJECT_ID=
  80 GCP_ACCOUNT_FILE=
  81 GCP_ZONE=
  82 AZURE_SECRETS_FILE=
  83 AZURE_RESOURCE_GROUP=
  84 AZURE_LOCATION=
  85 AZURE_CLOUD_ENVIRONMENT=
  86 DEBUG=
  87 SSH_USER=
  88 AWS_DEFAULT_REGION=us-east-1
  89 PUBLIC_KEY_FILE=
  90 MKSQUASHFS_MEM=256M
  91 NVIDIA_GPU_SUPPORT=
  92
  93 PARSEDOPTS=$(getopt --name "$0" --longoptions \
  94     help,json-file:,arvados-cluster-id:,aws-source-ami:,aws-profile:,aws-secrets-file:,aws-region:,aws-vpc-id:,aws-subnet-id:,aws-ebs-autoscale,gcp-project-id:,gcp-account-file:,gcp-zone:,azure-secrets-file:,azure-resource-group:,azure-location:,azure-sku:,azure-cloud-environment:,ssh_user:,resolver:,reposuffix:,public-key-file:,mksquashfs-mem:,nvidia-gpu-support,debug \
  95     -- "" "$@")
  96 if [ $? -ne 0 ]; then
  97     exit 1
  98 fi
  99
 100 eval set -- "$PARSEDOPTS"
 101 while [ $# -gt 0 ]; do
 102     case "$1" in
 103         --help)
 104             echo >&2 "$helpmessage"
 105             echo >&2
 106             exit 1
 107             ;;
 108         --json-file)
 109             JSON_FILE="$2"; shift
 110             ;;
 111         --arvados-cluster-id)
 112             ARVADOS_CLUSTER_ID="$2"; shift
 113             ;;
 114         --aws-source-ami)
 115             AWS_SOURCE_AMI="$2"; shift
 116             ;;
 117         --aws-profile)
 118             AWS_PROFILE="$2"; shift
 119             ;;
 120         --aws-secrets-file)
 121             AWS_SECRETS_FILE="$2"; shift
 122             ;;
 123         --aws-region)
 124             AWS_DEFAULT_REGION="$2"; shift
 125             ;;
 126         --aws-vpc-id)
 127             AWS_VPC_ID="$2"; shift
 128             ;;
 129         --aws-subnet-id)
 130             AWS_SUBNET_ID="$2"; shift
 131             ;;
 132         --aws-ebs-autoscale)
 133             AWS_EBS_AUTOSCALE=1
 134             ;;
 135         --gcp-project-id)
 136             GCP_PROJECT_ID="$2"; shift
 137             ;;
 138         --gcp-account-file)
 139             GCP_ACCOUNT_FILE="$2"; shift
 140             ;;
 141         --gcp-zone)
 142             GCP_ZONE="$2"; shift
 143             ;;
 144         --azure-secrets-file)
 145             AZURE_SECRETS_FILE="$2"; shift
 146             ;;
 147         --azure-resource-group)
 148             AZURE_RESOURCE_GROUP="$2"; shift
 149             ;;
 150         --azure-location)
 151             AZURE_LOCATION="$2"; shift
 152             ;;
 153         --azure-sku)
 154             AZURE_SKU="$2"; shift
 155             ;;
 156         --azure-cloud-environment)
 157             AZURE_CLOUD_ENVIRONMENT="$2"; shift
 158             ;;
 159         --ssh_user)
 160             SSH_USER="$2"; shift
 161             ;;
 162         --resolver)
 163             RESOLVER="$2"; shift
 164             ;;
 165         --reposuffix)
 166             REPOSUFFIX="$2"; shift
 167             ;;
 168         --public-key-file)
 169             PUBLIC_KEY_FILE="$2"; shift
 170             ;;
 171         --mksquashfs-mem)
 172             MKSQUASHFS_MEM="$2"; shift
 173             ;;
 174         --nvidia-gpu-support)
 175             NVIDIA_GPU_SUPPORT=1
 176             ;;
 177         --debug)
 178             # If you want to debug a build issue, add the -debug flag to the build
 179             # command in question.
 180             # This will allow you to ssh in, if you use the .pem file that packer
 181             # generates in this directory as the ssh key. The base image uses the admin
 182             # user and ssh port 22.
 183             EXTRA=" -debug"
 184             ;;
 185         --)
 186             if [ $# -gt 1 ]; then
 187                 echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
 188                 exit 1
 189             fi
 190             ;;
 191     esac
 192     shift
 193 done
 194
 195
 196 if [[ -z "$JSON_FILE" ]] || [[ ! -f "$JSON_FILE" ]]; then
 197   echo >&2 "$helpmessage"
 198   echo >&2
 199   echo >&2 "ERROR: packer json file not found"
 200   echo >&2
 201   exit 1
 202 fi
 203
 204 if [[ -z "$ARVADOS_CLUSTER_ID" ]]; then
 205   echo >&2 "$helpmessage"
 206   echo >&2
 207   echo >&2 "ERROR: arvados cluster id not specified"
 208   echo >&2
 209   exit 1
 210 fi
 211
 212 if [[ -z "$PUBLIC_KEY_FILE" ]] || [[ ! -f "$PUBLIC_KEY_FILE" ]]; then
 213   echo >&2 "$helpmessage"
 214   echo >&2
 215   echo >&2 "ERROR: public key file file not found"
 216   echo >&2
 217   exit 1
 218 fi
 219
 220 if [[ ! -z "$AWS_SECRETS_FILE" ]]; then
 221   source $AWS_SECRETS_FILE
 222 fi
 223
 224 if [[ ! -z "$AZURE_SECRETS_FILE" ]]; then
 225   source $AZURE_SECRETS_FILE
 226 fi
 227
 228
 229 EXTRA2=""
 230
 231 if [[ -n "$AWS_SOURCE_AMI" ]]; then
 232   EXTRA2+=" -var aws_source_ami=$AWS_SOURCE_AMI"
 233 fi
 234 if [[ -n "$AWS_PROFILE" ]]; then
 235   EXTRA2+=" -var aws_profile=$AWS_PROFILE"
 236 fi
 237 if [[ -n "$AWS_VPC_ID" ]]; then
 238   EXTRA2+=" -var vpc_id=$AWS_VPC_ID -var associate_public_ip_address=true "
 239 fi
 240 if [[ -n "$AWS_SUBNET_ID" ]]; then
 241   EXTRA2+=" -var subnet_id=$AWS_SUBNET_ID -var associate_public_ip_address=true "
 242 fi
 243 if [[ -n "$AWS_DEFAULT_REGION" ]]; then
 244   EXTRA2+=" -var aws_default_region=$AWS_DEFAULT_REGION"
 245 fi
 246 if [[ -n "$AWS_EBS_AUTOSCALE" ]]; then
 247   EXTRA2+=" -var aws_ebs_autoscale=$AWS_EBS_AUTOSCALE"
 248 fi
 249 if [[ -n "$GCP_PROJECT_ID" ]]; then
 250   EXTRA2+=" -var project_id=$GCP_PROJECT_ID"
 251 fi
 252 if [[ -n "$GCP_ACCOUNT_FILE" ]]; then
 253   EXTRA2+=" -var account_file=$GCP_ACCOUNT_FILE"
 254 fi
 255 if [[ -n "$GCP_ZONE" ]]; then
 256   EXTRA2+=" -var zone=$GCP_ZONE"
 257 fi
 258 if [[ -n "$AZURE_RESOURCE_GROUP" ]]; then
 259   EXTRA2+=" -var resource_group=$AZURE_RESOURCE_GROUP"
 260 fi
 261 if [[ -n "$AZURE_LOCATION" ]]; then
 262   EXTRA2+=" -var location=$AZURE_LOCATION"
 263 fi
 264 if [[ -n "$AZURE_SKU" ]]; then
 265   EXTRA2+=" -var image_sku=$AZURE_SKU"
 266 fi
 267 if [[ -n "$AZURE_CLOUD_ENVIRONMENT" ]]; then
 268   EXTRA2+=" -var cloud_environment_name=$AZURE_CLOUD_ENVIRONMENT"
 269 fi
 270 if [[ -n "$SSH_USER" ]]; then
 271   EXTRA2+=" -var ssh_user=$SSH_USER"
 272 fi
 273 if [[ -n "$RESOLVER" ]]; then
 274   EXTRA2+=" -var resolver=$RESOLVER"
 275 fi
 276 if [[ -n "$REPOSUFFIX" ]]; then
 277   EXTRA2+=" -var reposuffix=$REPOSUFFIX"
 278 fi
 279 if [[ -n "$PUBLIC_KEY_FILE" ]]; then
 280   EXTRA2+=" -var public_key_file=$PUBLIC_KEY_FILE"
 281 fi
 282 if [[ -n "$MKSQUASHFS_MEM" ]]; then
 283   EXTRA2+=" -var mksquashfs_mem=$MKSQUASHFS_MEM"
 284 fi
 285 if [[ -n "$NVIDIA_GPU_SUPPORT" ]]; then
 286   EXTRA2+=" -var nvidia_gpu_support=$NVIDIA_GPU_SUPPORT"
 287 fi
 288
 289 GOVERSION=$(grep 'const goversion =' ../../lib/install/deps.go |awk -F'"' '{print $2}')
 290 EXTRA2+=" -var goversion=$GOVERSION"
 291
 292 echo
 293 packer version
 294 echo
 295 echo packer build$EXTRA -var "arvados_cluster=$ARVADOS_CLUSTER_ID"$EXTRA2 $JSON_FILE
 296 packer build$EXTRA -var "arvados_cluster=$ARVADOS_CLUSTER_ID"$EXTRA2 $JSON_FILE