tools/compute-images/build.sh

   1 #!/bin/bash
   2
   3 # Copyright (C) The Arvados Authors. All rights reserved.
   4 #
   5 # SPDX-License-Identifier: Apache-2.0
   6
   7 JSON_FILE=$1
   8 ARVADOS_CLUSTER=$2
   9 PROJECT_ID=$3
  10 ACCOUNT_FILE=$4
  11
  12 read -rd "\000" helpmessage <<EOF
  13 $(basename $0): Build cloud images for arvados-dispatch-cloud
  14
  15 Syntax:
  16         $(basename $0) [options]
  17
  18 Options:
  19
  20   --json-file <path>
  21       Path to the packer json file (required)
  22   --arvados-cluster-id <xxxxx>
  23       The ID of the Arvados cluster, e.g. zzzzz(required)
  24   --aws-profile <profile>
  25       AWS profile to use (valid profile from ~/.aws/config (optional)
  26   --aws-secrets-file <path>
  27       AWS secrets file which will be sourced from this script (optional)
  28       When building for AWS, either an AWS profile or an AWS secrets file
  29       must be provided.
  30   --aws-source-ami <ami-xxxxxxxxxxxxxxxxx>
  31       The AMI to use as base for building the images (required if building for AWS)
  32   --aws-region <region> (default: us-east-1)
  33       The AWS region to use for building the images
  34   --aws-vpc-id <vpc-id>
  35       VPC id for AWS, if not specified packer will derive from the subnet id or pick the default one.
  36   --aws-subnet-id <subnet-xxxxxxxxxxxxxxxxx>
  37       Subnet id for AWS, if not specified packer will pick the default one for the VPC.
  38   --aws-ebs-autoscale
  39       Install the AWS EBS autoscaler daemon (default: do not install the AWS EBS autoscaler).
  40   --aws-associate-public-ip <true|false>
  41       Associate a public IP address with the node used for building the compute image.
  42       Required when the machine running packer can not reach the node used for building
  43       the compute image via its private IP. (default: true if building for AWS)
  44       Note: if the subnet has "Auto-assign public IPv4 address" enabled, disabling this
  45       flag will have no effect.
  46   --aws-ena-support <true|false>
  47       Enable enhanced networking (default: true if building for AWS)
  48   --gcp-project-id <project-id>
  49       GCP project id (required if building for GCP)
  50   --gcp-account-file <path>
  51       GCP account file (required if building for GCP)
  52   --gcp-zone <zone> (default: us-central1-f)
  53       GCP zone
  54   --azure-secrets-file <patch>
  55       Azure secrets file which will be sourced from this script (required if building for Azure)
  56   --azure-resource-group <resouce-group>
  57       Azure resource group (required if building for Azure)
  58   --azure-location <location>
  59       Azure location, e.g. centralus, eastus, westeurope (required if building for Azure)
  60   --azure-sku <sku> (required if building for Azure, e.g. 16.04-LTS)
  61       Azure SKU image to use
  62   --ssh_user <user> (default: packer)
  63       The user packer will use to log into the image
  64   --resolver <resolver_IP>
  65       The dns resolver for the machine (default: host's network provided)
  66   --reposuffix <suffix>
  67       Set this to "-dev" to track the unstable/dev Arvados repositories
  68   --public-key-file <path>
  69       Path to the public key file that a-d-c will use to log into the compute node (required)
  70   --mksquashfs-mem (default: 256M)
  71       Only relevant when using Singularity. This is the amount of memory mksquashfs is allowed to use.
  72   --nvidia-gpu-support
  73       Install all the necessary tooling for Nvidia GPU support (default: do not install Nvidia GPU support)
  74   --debug
  75       Output debug information (default: no debug output is printed)
  76
  77 For more information, see the Arvados documentation at https://doc.arvados.org/install/crunch2-cloud/install-compute-node.html
  78
  79 EOF
  80
  81 JSON_FILE=
  82 ARVADOS_CLUSTER_ID=
  83 AWS_PROFILE=
  84 AWS_SECRETS_FILE=
  85 AWS_SOURCE_AMI=
  86 AWS_VPC_ID=
  87 AWS_SUBNET_ID=
  88 AWS_EBS_AUTOSCALE=
  89 AWS_ASSOCIATE_PUBLIC_IP=true
  90 AWS_ENA_SUPPORT=true
  91 GCP_PROJECT_ID=
  92 GCP_ACCOUNT_FILE=
  93 GCP_ZONE=
  94 AZURE_SECRETS_FILE=
  95 AZURE_RESOURCE_GROUP=
  96 AZURE_LOCATION=
  97 AZURE_CLOUD_ENVIRONMENT=
  98 DEBUG=
  99 SSH_USER=
 100 AWS_DEFAULT_REGION=us-east-1
 101 PUBLIC_KEY_FILE=
 102 MKSQUASHFS_MEM=256M
 103 NVIDIA_GPU_SUPPORT=
 104
 105 PARSEDOPTS=$(getopt --name "$0" --longoptions \
 106     help,json-file:,arvados-cluster-id:,aws-source-ami:,aws-profile:,aws-secrets-file:,aws-region:,aws-vpc-id:,aws-subnet-id:,aws-ebs-autoscale,aws-associate-public-ip:,aws-ena-support:,gcp-project-id:,gcp-account-file:,gcp-zone:,azure-secrets-file:,azure-resource-group:,azure-location:,azure-sku:,azure-cloud-environment:,ssh_user:,resolver:,reposuffix:,public-key-file:,mksquashfs-mem:,nvidia-gpu-support,debug \
 107     -- "" "$@")
 108 if [ $? -ne 0 ]; then
 109     exit 1
 110 fi
 111
 112 eval set -- "$PARSEDOPTS"
 113 while [ $# -gt 0 ]; do
 114     case "$1" in
 115         --help)
 116             echo >&2 "$helpmessage"
 117             echo >&2
 118             exit 1
 119             ;;
 120         --json-file)
 121             JSON_FILE="$2"; shift
 122             ;;
 123         --arvados-cluster-id)
 124             ARVADOS_CLUSTER_ID="$2"; shift
 125             ;;
 126         --aws-source-ami)
 127             AWS_SOURCE_AMI="$2"; shift
 128             ;;
 129         --aws-profile)
 130             AWS_PROFILE="$2"; shift
 131             ;;
 132         --aws-secrets-file)
 133             AWS_SECRETS_FILE="$2"; shift
 134             ;;
 135         --aws-region)
 136             AWS_DEFAULT_REGION="$2"; shift
 137             ;;
 138         --aws-vpc-id)
 139             AWS_VPC_ID="$2"; shift
 140             ;;
 141         --aws-subnet-id)
 142             AWS_SUBNET_ID="$2"; shift
 143             ;;
 144         --aws-ebs-autoscale)
 145             AWS_EBS_AUTOSCALE=1
 146             ;;
 147         --aws-associate-public-ip)
 148             AWS_ASSOCIATE_PUBLIC_IP="$2"; shift
 149             ;;
 150         --aws-ena-support)
 151             AWS_ENA_SUPPORT="$2"; shift
 152             ;;
 153         --gcp-project-id)
 154             GCP_PROJECT_ID="$2"; shift
 155             ;;
 156         --gcp-account-file)
 157             GCP_ACCOUNT_FILE="$2"; shift
 158             ;;
 159         --gcp-zone)
 160             GCP_ZONE="$2"; shift
 161             ;;
 162         --azure-secrets-file)
 163             AZURE_SECRETS_FILE="$2"; shift
 164             ;;
 165         --azure-resource-group)
 166             AZURE_RESOURCE_GROUP="$2"; shift
 167             ;;
 168         --azure-location)
 169             AZURE_LOCATION="$2"; shift
 170             ;;
 171         --azure-sku)
 172             AZURE_SKU="$2"; shift
 173             ;;
 174         --azure-cloud-environment)
 175             AZURE_CLOUD_ENVIRONMENT="$2"; shift
 176             ;;
 177         --ssh_user)
 178             SSH_USER="$2"; shift
 179             ;;
 180         --resolver)
 181             RESOLVER="$2"; shift
 182             ;;
 183         --reposuffix)
 184             REPOSUFFIX="$2"; shift
 185             ;;
 186         --public-key-file)
 187             PUBLIC_KEY_FILE="$2"; shift
 188             ;;
 189         --mksquashfs-mem)
 190             MKSQUASHFS_MEM="$2"; shift
 191             ;;
 192         --nvidia-gpu-support)
 193             NVIDIA_GPU_SUPPORT=1
 194             ;;
 195         --debug)
 196             # If you want to debug a build issue, add the -debug flag to the build
 197             # command in question.
 198             # This will allow you to ssh in, if you use the .pem file that packer
 199             # generates in this directory as the ssh key. The base image uses the admin
 200             # user and ssh port 22.
 201             EXTRA=" -debug"
 202             ;;
 203         --)
 204             if [ $# -gt 1 ]; then
 205                 echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
 206                 exit 1
 207             fi
 208             ;;
 209     esac
 210     shift
 211 done
 212
 213
 214 if [[ -z "$JSON_FILE" ]] || [[ ! -f "$JSON_FILE" ]]; then
 215   echo >&2 "$helpmessage"
 216   echo >&2
 217   echo >&2 "ERROR: packer json file not found"
 218   echo >&2
 219   exit 1
 220 fi
 221
 222 if [[ -z "$ARVADOS_CLUSTER_ID" ]]; then
 223   echo >&2 "$helpmessage"
 224   echo >&2
 225   echo >&2 "ERROR: arvados cluster id not specified"
 226   echo >&2
 227   exit 1
 228 fi
 229
 230 if [[ -z "$PUBLIC_KEY_FILE" ]] || [[ ! -f "$PUBLIC_KEY_FILE" ]]; then
 231   echo >&2 "$helpmessage"
 232   echo >&2
 233   echo >&2 "ERROR: public key file file not found"
 234   echo >&2
 235   exit 1
 236 fi
 237
 238 if [[ ! -z "$AWS_SECRETS_FILE" ]]; then
 239   source $AWS_SECRETS_FILE
 240 fi
 241
 242 if [[ ! -z "$AZURE_SECRETS_FILE" ]]; then
 243   source $AZURE_SECRETS_FILE
 244 fi
 245
 246
 247 AWS=0
 248 EXTRA2=""
 249
 250 if [[ -n "$AWS_SOURCE_AMI" ]]; then
 251   EXTRA2+=" -var aws_source_ami=$AWS_SOURCE_AMI"
 252   AWS=1
 253 fi
 254 if [[ -n "$AWS_PROFILE" ]]; then
 255   EXTRA2+=" -var aws_profile=$AWS_PROFILE"
 256   AWS=1
 257 fi
 258 if [[ -n "$AWS_VPC_ID" ]]; then
 259   EXTRA2+=" -var vpc_id=$AWS_VPC_ID"
 260   AWS=1
 261 fi
 262 if [[ -n "$AWS_SUBNET_ID" ]]; then
 263   EXTRA2+=" -var subnet_id=$AWS_SUBNET_ID"
 264   AWS=1
 265 fi
 266 if [[ -n "$AWS_DEFAULT_REGION" ]]; then
 267   EXTRA2+=" -var aws_default_region=$AWS_DEFAULT_REGION"
 268   AWS=1
 269 fi
 270 if [[ -n "$AWS_EBS_AUTOSCALE" ]]; then
 271   EXTRA2+=" -var aws_ebs_autoscale=$AWS_EBS_AUTOSCALE"
 272   AWS=1
 273 fi
 274 if [[ $AWS -eq 1 ]]; then
 275   EXTRA2+=" -var aws_associate_public_ip_address=$AWS_ASSOCIATE_PUBLIC_IP"
 276   EXTRA2+=" -var aws_ena_support=$AWS_ENA_SUPPORT"
 277 fi
 278 if [[ -n "$GCP_PROJECT_ID" ]]; then
 279   EXTRA2+=" -var project_id=$GCP_PROJECT_ID"
 280 fi
 281 if [[ -n "$GCP_ACCOUNT_FILE" ]]; then
 282   EXTRA2+=" -var account_file=$GCP_ACCOUNT_FILE"
 283 fi
 284 if [[ -n "$GCP_ZONE" ]]; then
 285   EXTRA2+=" -var zone=$GCP_ZONE"
 286 fi
 287 if [[ -n "$AZURE_RESOURCE_GROUP" ]]; then
 288   EXTRA2+=" -var resource_group=$AZURE_RESOURCE_GROUP"
 289 fi
 290 if [[ -n "$AZURE_LOCATION" ]]; then
 291   EXTRA2+=" -var location=$AZURE_LOCATION"
 292 fi
 293 if [[ -n "$AZURE_SKU" ]]; then
 294   EXTRA2+=" -var image_sku=$AZURE_SKU"
 295 fi
 296 if [[ -n "$AZURE_CLOUD_ENVIRONMENT" ]]; then
 297   EXTRA2+=" -var cloud_environment_name=$AZURE_CLOUD_ENVIRONMENT"
 298 fi
 299 if [[ -n "$SSH_USER" ]]; then
 300   EXTRA2+=" -var ssh_user=$SSH_USER"
 301 fi
 302 if [[ -n "$RESOLVER" ]]; then
 303   EXTRA2+=" -var resolver=$RESOLVER"
 304 fi
 305 if [[ -n "$REPOSUFFIX" ]]; then
 306   EXTRA2+=" -var reposuffix=$REPOSUFFIX"
 307 fi
 308 if [[ -n "$PUBLIC_KEY_FILE" ]]; then
 309   EXTRA2+=" -var public_key_file=$PUBLIC_KEY_FILE"
 310 fi
 311 if [[ -n "$MKSQUASHFS_MEM" ]]; then
 312   EXTRA2+=" -var mksquashfs_mem=$MKSQUASHFS_MEM"
 313 fi
 314 if [[ -n "$NVIDIA_GPU_SUPPORT" ]]; then
 315   EXTRA2+=" -var nvidia_gpu_support=$NVIDIA_GPU_SUPPORT"
 316 fi
 317
 318 GOVERSION=$(grep 'const goversion =' ../../lib/install/deps.go |awk -F'"' '{print $2}')
 319 EXTRA2+=" -var goversion=$GOVERSION"
 320
 321 logfile=packer-$(date -Iseconds).log
 322
 323 echo
 324 packer version
 325 echo
 326 echo packer build$EXTRA -var "arvados_cluster=$ARVADOS_CLUSTER_ID"$EXTRA2 $JSON_FILE | tee -a $logfile
 327 packer build$EXTRA -var "arvados_cluster=$ARVADOS_CLUSTER_ID"$EXTRA2 $JSON_FILE 2>&1 | tee -a $logfile