tools/compute-images/build.sh

   1 #!/bin/bash
   2
   3 # Copyright (C) The Arvados Authors. All rights reserved.
   4 #
   5 # SPDX-License-Identifier: Apache-2.0
   6
   7 JSON_FILE=$1
   8 ARVADOS_CLUSTER=$2
   9 PROJECT_ID=$3
  10 ACCOUNT_FILE=$4
  11
  12 read -rd "\000" helpmessage <<EOF
  13 $(basename $0): Build cloud images for arvados-dispatch-cloud
  14
  15 Syntax:
  16         $(basename $0) [options]
  17
  18 Options:
  19
  20   --json-file (required)
  21       Path to the packer json file
  22   --arvados-cluster-id (required)
  23       The ID of the Arvados cluster, e.g. zzzzz
  24   --aws-profile (default: false)
  25       AWS profile to use (valid profile from ~/.aws/config
  26   --aws-secrets-file (default: false, required if building for AWS)
  27       AWS secrets file which will be sourced from this script
  28   --aws-source-ami (default: false, required if building for AWS)
  29       The AMI to use as base for building the images
  30   --aws-region (default: us-east-1)
  31       The AWS region to use for building the images
  32   --aws-vpc-id (optional)
  33       VPC id for AWS, otherwise packer will pick the default one
  34   --aws-subnet-id
  35       Subnet id for AWS otherwise packer will pick the default one for the VPC
  36   --aws-ebs-autoscale (default: false)
  37       Install the AWS EBS autoscaler daemon.
  38   --aws-associate-public-ip (default: true if building for AWS)
  39       Associate a public IP address with the node used for building the compute image.
  40       Required when the machine running packer can not reach the node used for building
  41       the compute image via its private IP.
  42       Note: if the subnet has "Auto-assign public IPv4 address" enabled, disabling this
  43       flag will have no effect.
  44   --aws-ena-support (default: true if building for AWS)
  45       Enable enhanced networking
  46   --gcp-project-id (default: false, required if building for GCP)
  47       GCP project id
  48   --gcp-account-file (default: false, required if building for GCP)
  49       GCP account file
  50   --gcp-zone (default: us-central1-f)
  51       GCP zone
  52   --azure-secrets-file (default: false, required if building for Azure)
  53       Azure secrets file which will be sourced from this script
  54   --azure-resource-group (default: false, required if building for Azure)
  55       Azure resource group
  56   --azure-location (default: false, required if building for Azure)
  57       Azure location, e.g. centralus, eastus, westeurope
  58   --azure-sku (default: unset, required if building for Azure, e.g. 16.04-LTS)
  59       Azure SKU image to use
  60   --ssh_user  (default: packer)
  61       The user packer will use to log into the image
  62   --resolver (default: host's network provided)
  63       The dns resolver for the machine
  64   --reposuffix (default: unset)
  65       Set this to "-dev" to track the unstable/dev Arvados repositories
  66   --public-key-file (required)
  67       Path to the public key file that a-d-c will use to log into the compute node
  68   --mksquashfs-mem (default: 256M)
  69       Only relevant when using Singularity. This is the amount of memory mksquashfs is allowed to use.
  70   --nvidia-gpu-support (default: false)
  71       Install all the necessary tooling for Nvidia GPU support
  72   --debug (default: false)
  73       Output debug information
  74
  75 For more information, see the Arvados documentation at https://doc.arvados.org/install/crunch2-cloud/install-compute-node.html
  76
  77 EOF
  78
  79 JSON_FILE=
  80 ARVADOS_CLUSTER_ID=
  81 AWS_PROFILE=
  82 AWS_SECRETS_FILE=
  83 AWS_SOURCE_AMI=
  84 AWS_VPC_ID=
  85 AWS_SUBNET_ID=
  86 AWS_EBS_AUTOSCALE=
  87 AWS_ASSOCIATE_PUBLIC_IP=true
  88 AWS_ENA_SUPPORT=true
  89 GCP_PROJECT_ID=
  90 GCP_ACCOUNT_FILE=
  91 GCP_ZONE=
  92 AZURE_SECRETS_FILE=
  93 AZURE_RESOURCE_GROUP=
  94 AZURE_LOCATION=
  95 AZURE_CLOUD_ENVIRONMENT=
  96 DEBUG=
  97 SSH_USER=
  98 AWS_DEFAULT_REGION=us-east-1
  99 PUBLIC_KEY_FILE=
 100 MKSQUASHFS_MEM=256M
 101 NVIDIA_GPU_SUPPORT=
 102
 103 PARSEDOPTS=$(getopt --name "$0" --longoptions \
 104     help,json-file:,arvados-cluster-id:,aws-source-ami:,aws-profile:,aws-secrets-file:,aws-region:,aws-vpc-id:,aws-subnet-id:,aws-ebs-autoscale,aws-associate-public-ip:,aws-ena-support:,gcp-project-id:,gcp-account-file:,gcp-zone:,azure-secrets-file:,azure-resource-group:,azure-location:,azure-sku:,azure-cloud-environment:,ssh_user:,resolver:,reposuffix:,public-key-file:,mksquashfs-mem:,nvidia-gpu-support,debug \
 105     -- "" "$@")
 106 if [ $? -ne 0 ]; then
 107     exit 1
 108 fi
 109
 110 eval set -- "$PARSEDOPTS"
 111 while [ $# -gt 0 ]; do
 112     case "$1" in
 113         --help)
 114             echo >&2 "$helpmessage"
 115             echo >&2
 116             exit 1
 117             ;;
 118         --json-file)
 119             JSON_FILE="$2"; shift
 120             ;;
 121         --arvados-cluster-id)
 122             ARVADOS_CLUSTER_ID="$2"; shift
 123             ;;
 124         --aws-source-ami)
 125             AWS_SOURCE_AMI="$2"; shift
 126             ;;
 127         --aws-profile)
 128             AWS_PROFILE="$2"; shift
 129             ;;
 130         --aws-secrets-file)
 131             AWS_SECRETS_FILE="$2"; shift
 132             ;;
 133         --aws-region)
 134             AWS_DEFAULT_REGION="$2"; shift
 135             ;;
 136         --aws-vpc-id)
 137             AWS_VPC_ID="$2"; shift
 138             ;;
 139         --aws-subnet-id)
 140             AWS_SUBNET_ID="$2"; shift
 141             ;;
 142         --aws-ebs-autoscale)
 143             AWS_EBS_AUTOSCALE=1
 144             ;;
 145         --aws-associate-public-ip)
 146             AWS_ASSOCIATE_PUBLIC_IP="$2"; shift
 147             ;;
 148         --aws-ena-support)
 149             AWS_ENA_SUPPORT="$2"; shift
 150             ;;
 151         --gcp-project-id)
 152             GCP_PROJECT_ID="$2"; shift
 153             ;;
 154         --gcp-account-file)
 155             GCP_ACCOUNT_FILE="$2"; shift
 156             ;;
 157         --gcp-zone)
 158             GCP_ZONE="$2"; shift
 159             ;;
 160         --azure-secrets-file)
 161             AZURE_SECRETS_FILE="$2"; shift
 162             ;;
 163         --azure-resource-group)
 164             AZURE_RESOURCE_GROUP="$2"; shift
 165             ;;
 166         --azure-location)
 167             AZURE_LOCATION="$2"; shift
 168             ;;
 169         --azure-sku)
 170             AZURE_SKU="$2"; shift
 171             ;;
 172         --azure-cloud-environment)
 173             AZURE_CLOUD_ENVIRONMENT="$2"; shift
 174             ;;
 175         --ssh_user)
 176             SSH_USER="$2"; shift
 177             ;;
 178         --resolver)
 179             RESOLVER="$2"; shift
 180             ;;
 181         --reposuffix)
 182             REPOSUFFIX="$2"; shift
 183             ;;
 184         --public-key-file)
 185             PUBLIC_KEY_FILE="$2"; shift
 186             ;;
 187         --mksquashfs-mem)
 188             MKSQUASHFS_MEM="$2"; shift
 189             ;;
 190         --nvidia-gpu-support)
 191             NVIDIA_GPU_SUPPORT=1
 192             ;;
 193         --debug)
 194             # If you want to debug a build issue, add the -debug flag to the build
 195             # command in question.
 196             # This will allow you to ssh in, if you use the .pem file that packer
 197             # generates in this directory as the ssh key. The base image uses the admin
 198             # user and ssh port 22.
 199             EXTRA=" -debug"
 200             ;;
 201         --)
 202             if [ $# -gt 1 ]; then
 203                 echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
 204                 exit 1
 205             fi
 206             ;;
 207     esac
 208     shift
 209 done
 210
 211
 212 if [[ -z "$JSON_FILE" ]] || [[ ! -f "$JSON_FILE" ]]; then
 213   echo >&2 "$helpmessage"
 214   echo >&2
 215   echo >&2 "ERROR: packer json file not found"
 216   echo >&2
 217   exit 1
 218 fi
 219
 220 if [[ -z "$ARVADOS_CLUSTER_ID" ]]; then
 221   echo >&2 "$helpmessage"
 222   echo >&2
 223   echo >&2 "ERROR: arvados cluster id not specified"
 224   echo >&2
 225   exit 1
 226 fi
 227
 228 if [[ -z "$PUBLIC_KEY_FILE" ]] || [[ ! -f "$PUBLIC_KEY_FILE" ]]; then
 229   echo >&2 "$helpmessage"
 230   echo >&2
 231   echo >&2 "ERROR: public key file file not found"
 232   echo >&2
 233   exit 1
 234 fi
 235
 236 if [[ ! -z "$AWS_SECRETS_FILE" ]]; then
 237   source $AWS_SECRETS_FILE
 238 fi
 239
 240 if [[ ! -z "$AZURE_SECRETS_FILE" ]]; then
 241   source $AZURE_SECRETS_FILE
 242 fi
 243
 244
 245 AWS=0
 246 EXTRA2=""
 247
 248 if [[ -n "$AWS_SOURCE_AMI" ]]; then
 249   EXTRA2+=" -var aws_source_ami=$AWS_SOURCE_AMI"
 250   AWS=1
 251 fi
 252 if [[ -n "$AWS_PROFILE" ]]; then
 253   EXTRA2+=" -var aws_profile=$AWS_PROFILE"
 254   AWS=1
 255 fi
 256 if [[ -n "$AWS_VPC_ID" ]]; then
 257   EXTRA2+=" -var vpc_id=$AWS_VPC_ID"
 258   AWS=1
 259 fi
 260 if [[ -n "$AWS_SUBNET_ID" ]]; then
 261   EXTRA2+=" -var subnet_id=$AWS_SUBNET_ID"
 262   AWS=1
 263 fi
 264 if [[ -n "$AWS_DEFAULT_REGION" ]]; then
 265   EXTRA2+=" -var aws_default_region=$AWS_DEFAULT_REGION"
 266   AWS=1
 267 fi
 268 if [[ -n "$AWS_EBS_AUTOSCALE" ]]; then
 269   EXTRA2+=" -var aws_ebs_autoscale=$AWS_EBS_AUTOSCALE"
 270   AWS=1
 271 fi
 272 if [[ $AWS -eq 1 ]]; then
 273   EXTRA2+=" -var aws_associate_public_ip_address=$AWS_ASSOCIATE_PUBLIC_IP"
 274   EXTRA2+=" -var aws_ena_support=$AWS_ENA_SUPPORT"
 275 fi
 276 if [[ -n "$GCP_PROJECT_ID" ]]; then
 277   EXTRA2+=" -var project_id=$GCP_PROJECT_ID"
 278 fi
 279 if [[ -n "$GCP_ACCOUNT_FILE" ]]; then
 280   EXTRA2+=" -var account_file=$GCP_ACCOUNT_FILE"
 281 fi
 282 if [[ -n "$GCP_ZONE" ]]; then
 283   EXTRA2+=" -var zone=$GCP_ZONE"
 284 fi
 285 if [[ -n "$AZURE_RESOURCE_GROUP" ]]; then
 286   EXTRA2+=" -var resource_group=$AZURE_RESOURCE_GROUP"
 287 fi
 288 if [[ -n "$AZURE_LOCATION" ]]; then
 289   EXTRA2+=" -var location=$AZURE_LOCATION"
 290 fi
 291 if [[ -n "$AZURE_SKU" ]]; then
 292   EXTRA2+=" -var image_sku=$AZURE_SKU"
 293 fi
 294 if [[ -n "$AZURE_CLOUD_ENVIRONMENT" ]]; then
 295   EXTRA2+=" -var cloud_environment_name=$AZURE_CLOUD_ENVIRONMENT"
 296 fi
 297 if [[ -n "$SSH_USER" ]]; then
 298   EXTRA2+=" -var ssh_user=$SSH_USER"
 299 fi
 300 if [[ -n "$RESOLVER" ]]; then
 301   EXTRA2+=" -var resolver=$RESOLVER"
 302 fi
 303 if [[ -n "$REPOSUFFIX" ]]; then
 304   EXTRA2+=" -var reposuffix=$REPOSUFFIX"
 305 fi
 306 if [[ -n "$PUBLIC_KEY_FILE" ]]; then
 307   EXTRA2+=" -var public_key_file=$PUBLIC_KEY_FILE"
 308 fi
 309 if [[ -n "$MKSQUASHFS_MEM" ]]; then
 310   EXTRA2+=" -var mksquashfs_mem=$MKSQUASHFS_MEM"
 311 fi
 312 if [[ -n "$NVIDIA_GPU_SUPPORT" ]]; then
 313   EXTRA2+=" -var nvidia_gpu_support=$NVIDIA_GPU_SUPPORT"
 314 fi
 315
 316 GOVERSION=$(grep 'const goversion =' ../../lib/install/deps.go |awk -F'"' '{print $2}')
 317 EXTRA2+=" -var goversion=$GOVERSION"
 318
 319 echo
 320 packer version
 321 echo
 322 echo packer build$EXTRA -var "arvados_cluster=$ARVADOS_CLUSTER_ID"$EXTRA2 $JSON_FILE
 323 packer build$EXTRA -var "arvados_cluster=$ARVADOS_CLUSTER_ID"$EXTRA2 $JSON_FILE