tools/compute-images/build.sh

   1 #!/bin/bash
   2
   3 # Copyright (C) The Arvados Authors. All rights reserved.
   4 #
   5 # SPDX-License-Identifier: Apache-2.0
   6
   7 JSON_FILE=$1
   8 ARVADOS_CLUSTER=$2
   9 PROJECT_ID=$3
  10 ACCOUNT_FILE=$4
  11
  12 read -rd "\000" helpmessage <<EOF
  13 $(basename $0): Build cloud images for arvados-dispatch-cloud
  14
  15 Syntax:
  16         $(basename $0) [options]
  17
  18 Options:
  19
  20   --json-file <path>
  21       Path to the packer json file (required)
  22   --arvados-cluster-id <xxxxx>
  23       The ID of the Arvados cluster, e.g. zzzzz(required)
  24   --aws-profile <profile>
  25       AWS profile to use (valid profile from ~/.aws/config (optional)
  26   --aws-secrets-file <path>
  27       AWS secrets file which will be sourced from this script (optional)
  28       When building for AWS, either an AWS profile or an AWS secrets file
  29       must be provided.
  30   --aws-source-ami <ami-xxxxxxxxxxxxxxxxx>
  31       The AMI to use as base for building the images (required if building for AWS)
  32   --aws-region <region> (default: us-east-1)
  33       The AWS region to use for building the images
  34   --aws-vpc-id <vpc-id>
  35       VPC id for AWS, if not specified packer will derive from the subnet id or pick the default one.
  36   --aws-subnet-id <subnet-xxxxxxxxxxxxxxxxx>
  37       Subnet id for AWS, if not specified packer will pick the default one for the VPC.
  38   --aws-ebs-autoscale
  39       Install the AWS EBS autoscaler daemon (default: do not install the AWS EBS autoscaler).
  40   --aws-associate-public-ip <true|false>
  41       Associate a public IP address with the node used for building the compute image.
  42       Required when the machine running packer can not reach the node used for building
  43       the compute image via its private IP. (default: true if building for AWS)
  44       Note: if the subnet has "Auto-assign public IPv4 address" enabled, disabling this
  45       flag will have no effect.
  46   --aws-ena-support <true|false>
  47       Enable enhanced networking (default: true if building for AWS)
  48   --gcp-project-id <project-id>
  49       GCP project id (required if building for GCP)
  50   --gcp-account-file <path>
  51       GCP account file (required if building for GCP)
  52   --gcp-zone <zone> (default: us-central1-f)
  53       GCP zone
  54   --azure-secrets-file <patch>
  55       Azure secrets file which will be sourced from this script (required if building for Azure)
  56   --azure-resource-group <resouce-group>
  57       Azure resource group (required if building for Azure)
  58   --azure-location <location>
  59       Azure location, e.g. centralus, eastus, westeurope (required if building for Azure)
  60   --azure-sku <sku> (required if building for Azure, e.g. 16.04-LTS)
  61       Azure SKU image to use
  62   --ssh_user <user> (default: packer)
  63       The user packer will use to log into the image
  64   --workdir <path> (default: /tmp)
  65       The directory where data files are staged and setup scripts are run
  66   --resolver <resolver_IP>
  67       The dns resolver for the machine (default: host's network provided)
  68   --reposuffix <suffix>
  69       Set this to "-dev" to track the unstable/dev Arvados repositories
  70   --public-key-file <path>
  71       Path to the public key file that a-d-c will use to log into the compute node (required)
  72   --mksquashfs-mem (default: 256M)
  73       Only relevant when using Singularity. This is the amount of memory mksquashfs is allowed to use.
  74   --nvidia-gpu-support
  75       Install all the necessary tooling for Nvidia GPU support (default: do not install Nvidia GPU support)
  76   --debug
  77       Output debug information (default: no debug output is printed)
  78
  79 For more information, see the Arvados documentation at https://doc.arvados.org/install/crunch2-cloud/install-compute-node.html
  80
  81 EOF
  82
  83 JSON_FILE=
  84 ARVADOS_CLUSTER_ID=
  85 AWS_PROFILE=
  86 AWS_SECRETS_FILE=
  87 AWS_SOURCE_AMI=
  88 AWS_VPC_ID=
  89 AWS_SUBNET_ID=
  90 AWS_EBS_AUTOSCALE=
  91 AWS_ASSOCIATE_PUBLIC_IP=true
  92 AWS_ENA_SUPPORT=true
  93 GCP_PROJECT_ID=
  94 GCP_ACCOUNT_FILE=
  95 GCP_ZONE=
  96 AZURE_SECRETS_FILE=
  97 AZURE_RESOURCE_GROUP=
  98 AZURE_LOCATION=
  99 AZURE_CLOUD_ENVIRONMENT=
 100 DEBUG=
 101 SSH_USER=
 102 WORKDIR=
 103 AWS_DEFAULT_REGION=us-east-1
 104 PUBLIC_KEY_FILE=
 105 MKSQUASHFS_MEM=256M
 106 NVIDIA_GPU_SUPPORT=
 107
 108 PARSEDOPTS=$(getopt --name "$0" --longoptions \
 109     help,json-file:,arvados-cluster-id:,aws-source-ami:,aws-profile:,aws-secrets-file:,aws-region:,aws-vpc-id:,aws-subnet-id:,aws-ebs-autoscale,aws-associate-public-ip:,aws-ena-support:,gcp-project-id:,gcp-account-file:,gcp-zone:,azure-secrets-file:,azure-resource-group:,azure-location:,azure-sku:,azure-cloud-environment:,ssh_user:,workdir:,resolver:,reposuffix:,public-key-file:,mksquashfs-mem:,nvidia-gpu-support,debug \
 110     -- "" "$@")
 111 if [ $? -ne 0 ]; then
 112     exit 1
 113 fi
 114
 115 eval set -- "$PARSEDOPTS"
 116 while [ $# -gt 0 ]; do
 117     case "$1" in
 118         --help)
 119             echo >&2 "$helpmessage"
 120             echo >&2
 121             exit 1
 122             ;;
 123         --json-file)
 124             JSON_FILE="$2"; shift
 125             ;;
 126         --arvados-cluster-id)
 127             ARVADOS_CLUSTER_ID="$2"; shift
 128             ;;
 129         --aws-source-ami)
 130             AWS_SOURCE_AMI="$2"; shift
 131             ;;
 132         --aws-profile)
 133             AWS_PROFILE="$2"; shift
 134             ;;
 135         --aws-secrets-file)
 136             AWS_SECRETS_FILE="$2"; shift
 137             ;;
 138         --aws-region)
 139             AWS_DEFAULT_REGION="$2"; shift
 140             ;;
 141         --aws-vpc-id)
 142             AWS_VPC_ID="$2"; shift
 143             ;;
 144         --aws-subnet-id)
 145             AWS_SUBNET_ID="$2"; shift
 146             ;;
 147         --aws-ebs-autoscale)
 148             AWS_EBS_AUTOSCALE=1
 149             ;;
 150         --aws-associate-public-ip)
 151             AWS_ASSOCIATE_PUBLIC_IP="$2"; shift
 152             ;;
 153         --aws-ena-support)
 154             AWS_ENA_SUPPORT="$2"; shift
 155             ;;
 156         --gcp-project-id)
 157             GCP_PROJECT_ID="$2"; shift
 158             ;;
 159         --gcp-account-file)
 160             GCP_ACCOUNT_FILE="$2"; shift
 161             ;;
 162         --gcp-zone)
 163             GCP_ZONE="$2"; shift
 164             ;;
 165         --azure-secrets-file)
 166             AZURE_SECRETS_FILE="$2"; shift
 167             ;;
 168         --azure-resource-group)
 169             AZURE_RESOURCE_GROUP="$2"; shift
 170             ;;
 171         --azure-location)
 172             AZURE_LOCATION="$2"; shift
 173             ;;
 174         --azure-sku)
 175             AZURE_SKU="$2"; shift
 176             ;;
 177         --azure-cloud-environment)
 178             AZURE_CLOUD_ENVIRONMENT="$2"; shift
 179             ;;
 180         --ssh_user)
 181             SSH_USER="$2"; shift
 182             ;;
 183         --workdir)
 184             WORKDIR="$2"; shift
 185             ;;
 186         --resolver)
 187             RESOLVER="$2"; shift
 188             ;;
 189         --reposuffix)
 190             REPOSUFFIX="$2"; shift
 191             ;;
 192         --public-key-file)
 193             PUBLIC_KEY_FILE="$2"; shift
 194             ;;
 195         --mksquashfs-mem)
 196             MKSQUASHFS_MEM="$2"; shift
 197             ;;
 198         --nvidia-gpu-support)
 199             NVIDIA_GPU_SUPPORT=1
 200             ;;
 201         --debug)
 202             # If you want to debug a build issue, add the -debug flag to the build
 203             # command in question.
 204             # This will allow you to ssh in, if you use the .pem file that packer
 205             # generates in this directory as the ssh key. The base image uses the admin
 206             # user and ssh port 22.
 207             EXTRA=" -debug"
 208             ;;
 209         --)
 210             if [ $# -gt 1 ]; then
 211                 echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
 212                 exit 1
 213             fi
 214             ;;
 215     esac
 216     shift
 217 done
 218
 219
 220 if [[ -z "$JSON_FILE" ]] || [[ ! -f "$JSON_FILE" ]]; then
 221   echo >&2 "$helpmessage"
 222   echo >&2
 223   echo >&2 "ERROR: packer json file not found"
 224   echo >&2
 225   exit 1
 226 fi
 227
 228 if [[ -z "$ARVADOS_CLUSTER_ID" ]]; then
 229   echo >&2 "$helpmessage"
 230   echo >&2
 231   echo >&2 "ERROR: arvados cluster id not specified"
 232   echo >&2
 233   exit 1
 234 fi
 235
 236 if [[ -z "$PUBLIC_KEY_FILE" ]] || [[ ! -f "$PUBLIC_KEY_FILE" ]]; then
 237   echo >&2 "$helpmessage"
 238   echo >&2
 239   echo >&2 "ERROR: public key file file not found"
 240   echo >&2
 241   exit 1
 242 fi
 243
 244 if [[ ! -z "$AWS_SECRETS_FILE" ]]; then
 245   source $AWS_SECRETS_FILE
 246 fi
 247
 248 if [[ ! -z "$AZURE_SECRETS_FILE" ]]; then
 249   source $AZURE_SECRETS_FILE
 250 fi
 251
 252
 253 AWS=0
 254 EXTRA2=""
 255
 256 if [[ -n "$AWS_SOURCE_AMI" ]]; then
 257   EXTRA2+=" -var aws_source_ami=$AWS_SOURCE_AMI"
 258   AWS=1
 259 fi
 260 if [[ -n "$AWS_PROFILE" ]]; then
 261   EXTRA2+=" -var aws_profile=$AWS_PROFILE"
 262   AWS=1
 263 fi
 264 if [[ -n "$AWS_VPC_ID" ]]; then
 265   EXTRA2+=" -var vpc_id=$AWS_VPC_ID"
 266   AWS=1
 267 fi
 268 if [[ -n "$AWS_SUBNET_ID" ]]; then
 269   EXTRA2+=" -var subnet_id=$AWS_SUBNET_ID"
 270   AWS=1
 271 fi
 272 if [[ -n "$AWS_DEFAULT_REGION" ]]; then
 273   EXTRA2+=" -var aws_default_region=$AWS_DEFAULT_REGION"
 274   AWS=1
 275 fi
 276 if [[ -n "$AWS_EBS_AUTOSCALE" ]]; then
 277   EXTRA2+=" -var aws_ebs_autoscale=$AWS_EBS_AUTOSCALE"
 278   AWS=1
 279 fi
 280 if [[ $AWS -eq 1 ]]; then
 281   EXTRA2+=" -var aws_associate_public_ip_address=$AWS_ASSOCIATE_PUBLIC_IP"
 282   EXTRA2+=" -var aws_ena_support=$AWS_ENA_SUPPORT"
 283 fi
 284 if [[ -n "$GCP_PROJECT_ID" ]]; then
 285   EXTRA2+=" -var project_id=$GCP_PROJECT_ID"
 286 fi
 287 if [[ -n "$GCP_ACCOUNT_FILE" ]]; then
 288   EXTRA2+=" -var account_file=$GCP_ACCOUNT_FILE"
 289 fi
 290 if [[ -n "$GCP_ZONE" ]]; then
 291   EXTRA2+=" -var zone=$GCP_ZONE"
 292 fi
 293 if [[ -n "$AZURE_RESOURCE_GROUP" ]]; then
 294   EXTRA2+=" -var resource_group=$AZURE_RESOURCE_GROUP"
 295 fi
 296 if [[ -n "$AZURE_LOCATION" ]]; then
 297   EXTRA2+=" -var location=$AZURE_LOCATION"
 298 fi
 299 if [[ -n "$AZURE_SKU" ]]; then
 300   EXTRA2+=" -var image_sku=$AZURE_SKU"
 301 fi
 302 if [[ -n "$AZURE_CLOUD_ENVIRONMENT" ]]; then
 303   EXTRA2+=" -var cloud_environment_name=$AZURE_CLOUD_ENVIRONMENT"
 304 fi
 305 if [[ -n "$SSH_USER" ]]; then
 306   EXTRA2+=" -var ssh_user=$SSH_USER"
 307 fi
 308 if [[ -n "$WORKDIR" ]]; then
 309   EXTRA2+=" -var workdir=$WORKDIR"
 310 fi
 311 if [[ -n "$RESOLVER" ]]; then
 312   EXTRA2+=" -var resolver=$RESOLVER"
 313 fi
 314 if [[ -n "$REPOSUFFIX" ]]; then
 315   EXTRA2+=" -var reposuffix=$REPOSUFFIX"
 316 fi
 317 if [[ -n "$PUBLIC_KEY_FILE" ]]; then
 318   EXTRA2+=" -var public_key_file=$PUBLIC_KEY_FILE"
 319 fi
 320 if [[ -n "$MKSQUASHFS_MEM" ]]; then
 321   EXTRA2+=" -var mksquashfs_mem=$MKSQUASHFS_MEM"
 322 fi
 323 if [[ -n "$NVIDIA_GPU_SUPPORT" ]]; then
 324   EXTRA2+=" -var nvidia_gpu_support=$NVIDIA_GPU_SUPPORT"
 325 fi
 326
 327 GOVERSION=$(grep 'const goversion =' ../../lib/install/deps.go |awk -F'"' '{print $2}')
 328 EXTRA2+=" -var goversion=$GOVERSION"
 329
 330 logfile=packer-$(date -Iseconds).log
 331
 332 echo
 333 packer version
 334 echo
 335 echo packer build$EXTRA -var "arvados_cluster=$ARVADOS_CLUSTER_ID"$EXTRA2 $JSON_FILE | tee -a $logfile
 336 packer build$EXTRA -var "arvados_cluster=$ARVADOS_CLUSTER_ID"$EXTRA2 $JSON_FILE 2>&1 | tee -a $logfile