tools/compute-images/build.sh

   1 #!/bin/bash
   2
   3 # Copyright (C) The Arvados Authors. All rights reserved.
   4 #
   5 # SPDX-License-Identifier: Apache-2.0
   6
   7 JSON_FILE=$1
   8 ARVADOS_CLUSTER=$2
   9 PROJECT_ID=$3
  10 ACCOUNT_FILE=$4
  11
  12 read -rd "\000" helpmessage <<EOF
  13 $(basename $0): Build cloud images for arvados-dispatch-cloud
  14
  15 Syntax:
  16         $(basename $0) [options]
  17
  18 Options:
  19
  20   --json-file <path>
  21       Path to the packer json file (required)
  22   --arvados-cluster-id <xxxxx>
  23       The ID of the Arvados cluster, e.g. zzzzz(required)
  24   --aws-profile <profile>
  25       AWS profile to use (valid profile from ~/.aws/config (optional)
  26   --aws-secrets-file <path>
  27       AWS secrets file which will be sourced from this script (optional)
  28       When building for AWS, either an AWS profile or an AWS secrets file
  29       must be provided.
  30   --aws-source-ami <ami-xxxxxxxxxxxxxxxxx>
  31       The AMI to use as base for building the images (required if building for AWS)
  32   --aws-region <region> (default: us-east-1)
  33       The AWS region to use for building the images
  34   --aws-vpc-id <vpc-id>
  35       VPC id for AWS, if not specified packer will derive from the subnet id or pick the default one.
  36   --aws-subnet-id <subnet-xxxxxxxxxxxxxxxxx>
  37       Subnet id for AWS, if not specified packer will pick the default one for the VPC.
  38   --aws-ebs-autoscale
  39       Install the AWS EBS autoscaler daemon (default: do not install the AWS EBS autoscaler).
  40   --aws-associate-public-ip <true|false>
  41       Associate a public IP address with the node used for building the compute image.
  42       Required when the machine running packer can not reach the node used for building
  43       the compute image via its private IP. (default: true if building for AWS)
  44       Note: if the subnet has "Auto-assign public IPv4 address" enabled, disabling this
  45       flag will have no effect.
  46   --aws-ena-support <true|false>
  47       Enable enhanced networking (default: true if building for AWS)
  48   --gcp-project-id <project-id>
  49       GCP project id (required if building for GCP)
  50   --gcp-account-file <path>
  51       GCP account file (required if building for GCP)
  52   --gcp-zone <zone> (default: us-central1-f)
  53       GCP zone
  54   --azure-secrets-file <patch>
  55       Azure secrets file which will be sourced from this script (required if building for Azure)
  56   --azure-resource-group <resouce-group>
  57       Azure resource group (required if building for Azure)
  58   --azure-location <location>
  59       Azure location, e.g. centralus, eastus, westeurope (required if building for Azure)
  60   --azure-sku <sku> (required if building for Azure, e.g. 16.04-LTS)
  61       Azure SKU image to use
  62   --ssh_user <user> (default: packer)
  63       The user packer will use to log into the image
  64   --workdir <path> (default: /tmp)
  65       The directory where data files are staged and setup scripts are run
  66   --resolver <resolver_IP>
  67       The dns resolver for the machine (default: host's network provided)
  68   --reposuffix <suffix>
  69       Set this to "-dev" to track the unstable/dev Arvados repositories
  70   --pin-packages, --no-pin-packages
  71       These flags determine whether or not to configure apt pins for Arvados
  72       and third-party packages it depends on. By default packages are pinned
  73       unless you set \`--reposuffix -dev\`.
  74   --public-key-file <path>
  75       Path to the public key file that a-d-c will use to log into the compute node (required)
  76   --mksquashfs-mem (default: 256M)
  77       Only relevant when using Singularity. This is the amount of memory mksquashfs is allowed to use.
  78   --nvidia-gpu-support
  79       Install all the necessary tooling for Nvidia GPU support (default: do not install Nvidia GPU support)
  80   --debug
  81       Output debug information (default: no debug output is printed)
  82
  83 For more information, see the Arvados documentation at https://doc.arvados.org/install/crunch2-cloud/install-compute-node.html
  84
  85 EOF
  86
  87 set -e -o pipefail
  88
  89 JSON_FILE=
  90 ARVADOS_CLUSTER_ID=
  91 AWS_PROFILE=
  92 AWS_SECRETS_FILE=
  93 AWS_SOURCE_AMI=
  94 AWS_VPC_ID=
  95 AWS_SUBNET_ID=
  96 AWS_EBS_AUTOSCALE=
  97 AWS_ASSOCIATE_PUBLIC_IP=true
  98 AWS_ENA_SUPPORT=true
  99 GCP_PROJECT_ID=
 100 GCP_ACCOUNT_FILE=
 101 GCP_ZONE=
 102 AZURE_SECRETS_FILE=
 103 AZURE_RESOURCE_GROUP=
 104 AZURE_LOCATION=
 105 AZURE_CLOUD_ENVIRONMENT=
 106 DEBUG=
 107 SSH_USER=
 108 WORKDIR=
 109 AWS_DEFAULT_REGION=us-east-1
 110 PIN_PACKAGES=
 111 PUBLIC_KEY_FILE=
 112 MKSQUASHFS_MEM=256M
 113 NVIDIA_GPU_SUPPORT=
 114
 115 PARSEDOPTS=$(getopt --name "$0" --longoptions \
 116     help,json-file:,arvados-cluster-id:,aws-source-ami:,aws-profile:,aws-secrets-file:,aws-region:,aws-vpc-id:,aws-subnet-id:,aws-ebs-autoscale,aws-associate-public-ip:,aws-ena-support:,gcp-project-id:,gcp-account-file:,gcp-zone:,azure-secrets-file:,azure-resource-group:,azure-location:,azure-sku:,azure-cloud-environment:,ssh_user:,workdir:,resolver:,reposuffix:,pin-packages,no-pin-packages,public-key-file:,mksquashfs-mem:,nvidia-gpu-support,debug \
 117     -- "" "$@")
 118 if [ $? -ne 0 ]; then
 119     exit 1
 120 fi
 121
 122 eval set -- "$PARSEDOPTS"
 123 while [ $# -gt 0 ]; do
 124     case "$1" in
 125         --help)
 126             echo >&2 "$helpmessage"
 127             echo >&2
 128             exit 1
 129             ;;
 130         --json-file)
 131             JSON_FILE="$2"; shift
 132             ;;
 133         --arvados-cluster-id)
 134             ARVADOS_CLUSTER_ID="$2"; shift
 135             ;;
 136         --aws-source-ami)
 137             AWS_SOURCE_AMI="$2"; shift
 138             ;;
 139         --aws-profile)
 140             AWS_PROFILE="$2"; shift
 141             ;;
 142         --aws-secrets-file)
 143             AWS_SECRETS_FILE="$2"; shift
 144             ;;
 145         --aws-region)
 146             AWS_DEFAULT_REGION="$2"; shift
 147             ;;
 148         --aws-vpc-id)
 149             AWS_VPC_ID="$2"; shift
 150             ;;
 151         --aws-subnet-id)
 152             AWS_SUBNET_ID="$2"; shift
 153             ;;
 154         --aws-ebs-autoscale)
 155             AWS_EBS_AUTOSCALE=1
 156             ;;
 157         --aws-associate-public-ip)
 158             AWS_ASSOCIATE_PUBLIC_IP="$2"; shift
 159             ;;
 160         --aws-ena-support)
 161             AWS_ENA_SUPPORT="$2"; shift
 162             ;;
 163         --gcp-project-id)
 164             GCP_PROJECT_ID="$2"; shift
 165             ;;
 166         --gcp-account-file)
 167             GCP_ACCOUNT_FILE="$2"; shift
 168             ;;
 169         --gcp-zone)
 170             GCP_ZONE="$2"; shift
 171             ;;
 172         --azure-secrets-file)
 173             AZURE_SECRETS_FILE="$2"; shift
 174             ;;
 175         --azure-resource-group)
 176             AZURE_RESOURCE_GROUP="$2"; shift
 177             ;;
 178         --azure-location)
 179             AZURE_LOCATION="$2"; shift
 180             ;;
 181         --azure-sku)
 182             AZURE_SKU="$2"; shift
 183             ;;
 184         --azure-cloud-environment)
 185             AZURE_CLOUD_ENVIRONMENT="$2"; shift
 186             ;;
 187         --ssh_user)
 188             SSH_USER="$2"; shift
 189             ;;
 190         --workdir)
 191             WORKDIR="$2"; shift
 192             ;;
 193         --resolver)
 194             RESOLVER="$2"; shift
 195             ;;
 196         --reposuffix)
 197             REPOSUFFIX="$2"; shift
 198             ;;
 199         --pin-packages)
 200             PIN_PACKAGES=true
 201             ;;
 202         --no-pin-packages)
 203             PIN_PACKAGES=false
 204             ;;
 205         --public-key-file)
 206             PUBLIC_KEY_FILE="$2"; shift
 207             ;;
 208         --mksquashfs-mem)
 209             MKSQUASHFS_MEM="$2"; shift
 210             ;;
 211         --nvidia-gpu-support)
 212             NVIDIA_GPU_SUPPORT=1
 213             ;;
 214         --debug)
 215             # If you want to debug a build issue, add the -debug flag to the build
 216             # command in question.
 217             # This will allow you to ssh in, if you use the .pem file that packer
 218             # generates in this directory as the ssh key. The base image uses the admin
 219             # user and ssh port 22.
 220             EXTRA=" -debug"
 221             ;;
 222         --)
 223             if [ $# -gt 1 ]; then
 224                 echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
 225                 exit 1
 226             fi
 227             ;;
 228     esac
 229     shift
 230 done
 231
 232
 233 if [[ -z "$JSON_FILE" ]] || [[ ! -f "$JSON_FILE" ]]; then
 234   echo >&2 "$helpmessage"
 235   echo >&2
 236   echo >&2 "ERROR: packer json file not found"
 237   echo >&2
 238   exit 1
 239 fi
 240
 241 if [[ -z "$ARVADOS_CLUSTER_ID" ]]; then
 242   echo >&2 "$helpmessage"
 243   echo >&2
 244   echo >&2 "ERROR: arvados cluster id not specified"
 245   echo >&2
 246   exit 1
 247 fi
 248
 249 if [[ -z "$PUBLIC_KEY_FILE" ]] || [[ ! -f "$PUBLIC_KEY_FILE" ]]; then
 250   echo >&2 "$helpmessage"
 251   echo >&2
 252   echo >&2 "ERROR: public key file file not found"
 253   echo >&2
 254   exit 1
 255 fi
 256
 257 if [[ ! -z "$AWS_SECRETS_FILE" ]]; then
 258   source $AWS_SECRETS_FILE
 259 fi
 260
 261 if [[ ! -z "$AZURE_SECRETS_FILE" ]]; then
 262   source $AZURE_SECRETS_FILE
 263 fi
 264
 265
 266 AWS=0
 267 EXTRA2=""
 268
 269 if [[ -n "$AWS_SOURCE_AMI" ]]; then
 270   EXTRA2+=" -var aws_source_ami=$AWS_SOURCE_AMI"
 271   AWS=1
 272 fi
 273 if [[ -n "$AWS_PROFILE" ]]; then
 274   EXTRA2+=" -var aws_profile=$AWS_PROFILE"
 275   AWS=1
 276 fi
 277 if [[ -n "$AWS_VPC_ID" ]]; then
 278   EXTRA2+=" -var vpc_id=$AWS_VPC_ID"
 279   AWS=1
 280 fi
 281 if [[ -n "$AWS_SUBNET_ID" ]]; then
 282   EXTRA2+=" -var subnet_id=$AWS_SUBNET_ID"
 283   AWS=1
 284 fi
 285 if [[ -n "$AWS_DEFAULT_REGION" ]]; then
 286   EXTRA2+=" -var aws_default_region=$AWS_DEFAULT_REGION"
 287   AWS=1
 288 fi
 289 if [[ -n "$AWS_EBS_AUTOSCALE" ]]; then
 290   EXTRA2+=" -var aws_ebs_autoscale=$AWS_EBS_AUTOSCALE"
 291   AWS=1
 292 fi
 293 if [[ $AWS -eq 1 ]]; then
 294   EXTRA2+=" -var aws_associate_public_ip_address=$AWS_ASSOCIATE_PUBLIC_IP"
 295   EXTRA2+=" -var aws_ena_support=$AWS_ENA_SUPPORT"
 296 fi
 297 if [[ -n "$GCP_PROJECT_ID" ]]; then
 298   EXTRA2+=" -var project_id=$GCP_PROJECT_ID"
 299 fi
 300 if [[ -n "$GCP_ACCOUNT_FILE" ]]; then
 301   EXTRA2+=" -var account_file=$GCP_ACCOUNT_FILE"
 302 fi
 303 if [[ -n "$GCP_ZONE" ]]; then
 304   EXTRA2+=" -var zone=$GCP_ZONE"
 305 fi
 306 if [[ -n "$AZURE_RESOURCE_GROUP" ]]; then
 307   EXTRA2+=" -var resource_group=$AZURE_RESOURCE_GROUP"
 308 fi
 309 if [[ -n "$AZURE_LOCATION" ]]; then
 310   EXTRA2+=" -var location=$AZURE_LOCATION"
 311 fi
 312 if [[ -n "$AZURE_SKU" ]]; then
 313   EXTRA2+=" -var image_sku=$AZURE_SKU"
 314 fi
 315 if [[ -n "$AZURE_CLOUD_ENVIRONMENT" ]]; then
 316   EXTRA2+=" -var cloud_environment_name=$AZURE_CLOUD_ENVIRONMENT"
 317 fi
 318 if [[ -n "$SSH_USER" ]]; then
 319   EXTRA2+=" -var ssh_user=$SSH_USER"
 320 fi
 321 if [[ -n "$WORKDIR" ]]; then
 322   EXTRA2+=" -var workdir=$WORKDIR"
 323 fi
 324 if [[ -n "$RESOLVER" ]]; then
 325   EXTRA2+=" -var resolver=$RESOLVER"
 326 fi
 327 if [[ -n "$REPOSUFFIX" ]]; then
 328   EXTRA2+=" -var reposuffix=$REPOSUFFIX"
 329 fi
 330 if [[ -z "$PIN_PACKAGES" ]]; then
 331     case "$REPOSUFFIX" in
 332         -dev) PIN_PACKAGES=false ;;
 333         *) PIN_PACKAGES=true ;;
 334     esac
 335 fi
 336 EXTRA2+=" -var pin_packages=$PIN_PACKAGES"
 337 if [[ -n "$PUBLIC_KEY_FILE" ]]; then
 338   EXTRA2+=" -var public_key_file=$PUBLIC_KEY_FILE"
 339 fi
 340 if [[ -n "$MKSQUASHFS_MEM" ]]; then
 341   EXTRA2+=" -var mksquashfs_mem=$MKSQUASHFS_MEM"
 342 fi
 343 if [[ -n "$NVIDIA_GPU_SUPPORT" ]]; then
 344   EXTRA2+=" -var nvidia_gpu_support=$NVIDIA_GPU_SUPPORT"
 345 fi
 346
 347 GOVERSION=$(grep 'const goversion =' ../../lib/install/deps.go |awk -F'"' '{print $2}')
 348 EXTRA2+=" -var goversion=$GOVERSION"
 349
 350 logfile=packer-$(date -Iseconds).log
 351
 352 echo
 353 packer version
 354 echo
 355 echo packer build$EXTRA -var "arvados_cluster=$ARVADOS_CLUSTER_ID"$EXTRA2 $JSON_FILE | tee -a $logfile
 356 packer build$EXTRA -var "arvados_cluster=$ARVADOS_CLUSTER_ID"$EXTRA2 $JSON_FILE 2>&1 | tee -a $logfile