tools/compute-images/build.sh

   1 #!/bin/bash
   2
   3 # Copyright (C) The Arvados Authors. All rights reserved.
   4 #
   5 # SPDX-License-Identifier: Apache-2.0
   6
   7 JSON_FILE=$1
   8 ARVADOS_CLUSTER=$2
   9 PROJECT_ID=$3
  10 ACCOUNT_FILE=$4
  11
  12 read -rd "\000" helpmessage <<EOF
  13 $(basename $0): Build cloud images for arvados-dispatch-cloud
  14
  15 Syntax:
  16         $(basename $0) [options]
  17
  18 Options:
  19
  20   --json-file (required)
  21       Path to the packer json file
  22   --arvados-cluster-id (required)
  23       The ID of the Arvados cluster, e.g. zzzzz
  24   --aws-profile (default: false)
  25       AWS profile to use (valid profile from ~/.aws/config
  26   --aws-secrets-file (default: false, required if building for AWS)
  27       AWS secrets file which will be sourced from this script
  28   --aws-source-ami (default: false, required if building for AWS)
  29       The AMI to use as base for building the images
  30   --aws-region (default: us-east-1)
  31       The AWS region to use for building the images
  32   --aws-vpc-id (optional)
  33       VPC id for AWS, otherwise packer will pick the default one
  34   --aws-subnet-id
  35       Subnet id for AWS otherwise packer will pick the default one for the VPC
  36   --gcp-project-id (default: false, required if building for GCP)
  37       GCP project id
  38   --gcp-account-file (default: false, required if building for GCP)
  39       GCP account file
  40   --gcp-zone (default: us-central1-f)
  41       GCP zone
  42   --azure-secrets-file (default: false, required if building for Azure)
  43       Azure secrets file which will be sourced from this script
  44   --azure-resource-group (default: false, required if building for Azure)
  45       Azure resource group
  46   --azure-location (default: false, required if building for Azure)
  47       Azure location, e.g. centralus, eastus, westeurope
  48   --azure-sku (default: unset, required if building for Azure, e.g. 16.04-LTS)
  49       Azure SKU image to use
  50   --ssh_user  (default: packer)
  51       The user packer will use to log into the image
  52   --resolver (default: host's network provided)
  53       The dns resolver for the machine
  54   --reposuffix (default: unset)
  55       Set this to "-dev" to track the unstable/dev Arvados repositories
  56   --public-key-file (required)
  57       Path to the public key file that a-d-c will use to log into the compute node
  58   --mksquashfs-mem (default: 256M)
  59       Only relevant when using Singularity. This is the amount of memory mksquashfs is allowed to use.
  60   --nvidia-gpu-support (default: false)
  61       Install all the necessary tooling for Nvidia GPU support
  62   --debug (default: false)
  63       Output debug information
  64
  65 EOF
  66
  67 JSON_FILE=
  68 ARVADOS_CLUSTER_ID=
  69 AWS_PROFILE=
  70 AWS_SECRETS_FILE=
  71 AWS_SOURCE_AMI=
  72 AWS_VPC_ID=
  73 AWS_SUBNET_ID=
  74 GCP_PROJECT_ID=
  75 GCP_ACCOUNT_FILE=
  76 GCP_ZONE=
  77 AZURE_SECRETS_FILE=
  78 AZURE_RESOURCE_GROUP=
  79 AZURE_LOCATION=
  80 AZURE_CLOUD_ENVIRONMENT=
  81 DEBUG=
  82 SSH_USER=
  83 AWS_DEFAULT_REGION=us-east-1
  84 PUBLIC_KEY_FILE=
  85 MKSQUASHFS_MEM=256M
  86 NVIDIA_GPU_SUPPORT=
  87
  88 PARSEDOPTS=$(getopt --name "$0" --longoptions \
  89     help,json-file:,arvados-cluster-id:,aws-source-ami:,aws-profile:,aws-secrets-file:,aws-region:,aws-vpc-id:,aws-subnet-id:,gcp-project-id:,gcp-account-file:,gcp-zone:,azure-secrets-file:,azure-resource-group:,azure-location:,azure-sku:,azure-cloud-environment:,ssh_user:,resolver:,reposuffix:,public-key-file:,mksquashfs-mem:,nvidia-gpu-support,debug \
  90     -- "" "$@")
  91 if [ $? -ne 0 ]; then
  92     exit 1
  93 fi
  94
  95 eval set -- "$PARSEDOPTS"
  96 while [ $# -gt 0 ]; do
  97     case "$1" in
  98         --help)
  99             echo >&2 "$helpmessage"
 100             echo >&2
 101             exit 1
 102             ;;
 103         --json-file)
 104             JSON_FILE="$2"; shift
 105             ;;
 106         --arvados-cluster-id)
 107             ARVADOS_CLUSTER_ID="$2"; shift
 108             ;;
 109         --aws-source-ami)
 110             AWS_SOURCE_AMI="$2"; shift
 111             ;;
 112         --aws-profile)
 113             AWS_PROFILE="$2"; shift
 114             ;;
 115         --aws-secrets-file)
 116             AWS_SECRETS_FILE="$2"; shift
 117             ;;
 118         --aws-region)
 119             AWS_DEFAULT_REGION="$2"; shift
 120             ;;
 121         --aws-vpc-id)
 122             AWS_VPC_ID="$2"; shift
 123             ;;
 124         --aws-subnet-id)
 125             AWS_SUBNET_ID="$2"; shift
 126             ;;
 127         --gcp-project-id)
 128             GCP_PROJECT_ID="$2"; shift
 129             ;;
 130         --gcp-account-file)
 131             GCP_ACCOUNT_FILE="$2"; shift
 132             ;;
 133         --gcp-zone)
 134             GCP_ZONE="$2"; shift
 135             ;;
 136         --azure-secrets-file)
 137             AZURE_SECRETS_FILE="$2"; shift
 138             ;;
 139         --azure-resource-group)
 140             AZURE_RESOURCE_GROUP="$2"; shift
 141             ;;
 142         --azure-location)
 143             AZURE_LOCATION="$2"; shift
 144             ;;
 145         --azure-sku)
 146             AZURE_SKU="$2"; shift
 147             ;;
 148         --azure-cloud-environment)
 149             AZURE_CLOUD_ENVIRONMENT="$2"; shift
 150             ;;
 151         --ssh_user)
 152             SSH_USER="$2"; shift
 153             ;;
 154         --resolver)
 155             RESOLVER="$2"; shift
 156             ;;
 157         --reposuffix)
 158             REPOSUFFIX="$2"; shift
 159             ;;
 160         --public-key-file)
 161             PUBLIC_KEY_FILE="$2"; shift
 162             ;;
 163         --mksquashfs-mem)
 164             MKSQUASHFS_MEM="$2"; shift
 165             ;;
 166         --nvidia-gpu-support)
 167             NVIDIA_GPU_SUPPORT=1
 168             ;;
 169         --debug)
 170             # If you want to debug a build issue, add the -debug flag to the build
 171             # command in question.
 172             # This will allow you to ssh in, if you use the .pem file that packer
 173             # generates in this directory as the ssh key. The base image uses the admin
 174             # user and ssh port 22.
 175             EXTRA=" -debug"
 176             ;;
 177         --)
 178             if [ $# -gt 1 ]; then
 179                 echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
 180                 exit 1
 181             fi
 182             ;;
 183     esac
 184     shift
 185 done
 186
 187
 188 if [[ "$JSON_FILE" == "" ]] || [[ ! -f "$JSON_FILE" ]]; then
 189   echo >&2 "$helpmessage"
 190   echo >&2
 191   echo >&2 "ERROR: packer json file not found"
 192   echo >&2
 193   exit 1
 194 fi
 195
 196 if [[ -z "$ARVADOS_CLUSTER_ID" ]]; then
 197   echo >&2 "$helpmessage"
 198   echo >&2
 199   echo >&2 "ERROR: arvados cluster id not specified"
 200   echo >&2
 201   exit 1
 202 fi
 203
 204 if [[ "$PUBLIC_KEY_FILE" == "" ]] || [[ ! -f "$PUBLIC_KEY_FILE" ]]; then
 205   echo >&2 "$helpmessage"
 206   echo >&2
 207   echo >&2 "ERROR: public key file file not found"
 208   echo >&2
 209   exit 1
 210 fi
 211
 212 if [[ ! -z "$AWS_SECRETS_FILE" ]]; then
 213   source $AWS_SECRETS_FILE
 214 fi
 215
 216 if [[ ! -z "$AZURE_SECRETS_FILE" ]]; then
 217   source $AZURE_SECRETS_FILE
 218 fi
 219
 220
 221 EXTRA2=""
 222
 223 if [[ "$AWS_SOURCE_AMI" != "" ]]; then
 224   EXTRA2+=" -var aws_source_ami=$AWS_SOURCE_AMI"
 225 fi
 226 if [[ "$AWS_PROFILE" != "" ]]; then
 227   EXTRA2+=" -var aws_profile=$AWS_PROFILE"
 228 fi
 229 if [[ "$AWS_VPC_ID" != "" ]]; then
 230   EXTRA2+=" -var vpc_id=$AWS_VPC_ID -var associate_public_ip_address=true "
 231 fi
 232 if [[ "$AWS_SUBNET_ID" != "" ]]; then
 233   EXTRA2+=" -var subnet_id=$AWS_SUBNET_ID -var associate_public_ip_address=true "
 234 fi
 235 if [[ "$AWS_DEFAULT_REGION" != "" ]]; then
 236   EXTRA2+=" -var aws_default_region=$AWS_DEFAULT_REGION"
 237 fi
 238 if [[ "$GCP_PROJECT_ID" != "" ]]; then
 239   EXTRA2+=" -var project_id=$GCP_PROJECT_ID"
 240 fi
 241 if [[ "$GCP_ACCOUNT_FILE" != "" ]]; then
 242   EXTRA2+=" -var account_file=$GCP_ACCOUNT_FILE"
 243 fi
 244 if [[ "$GCP_ZONE" != "" ]]; then
 245   EXTRA2+=" -var zone=$GCP_ZONE"
 246 fi
 247 if [[ "$AZURE_RESOURCE_GROUP" != "" ]]; then
 248   EXTRA2+=" -var resource_group=$AZURE_RESOURCE_GROUP"
 249 fi
 250 if [[ "$AZURE_LOCATION" != "" ]]; then
 251   EXTRA2+=" -var location=$AZURE_LOCATION"
 252 fi
 253 if [[ "$AZURE_SKU" != "" ]]; then
 254   EXTRA2+=" -var image_sku=$AZURE_SKU"
 255 fi
 256 if [[ "$AZURE_CLOUD_ENVIRONMENT" != "" ]]; then
 257   EXTRA2+=" -var cloud_environment_name=$AZURE_CLOUD_ENVIRONMENT"
 258 fi
 259 if [[ "$SSH_USER" != "" ]]; then
 260   EXTRA2+=" -var ssh_user=$SSH_USER"
 261 fi
 262 if [[ "$RESOLVER" != "" ]]; then
 263   EXTRA2+=" -var resolver=$RESOLVER"
 264 fi
 265 if [[ "$REPOSUFFIX" != "" ]]; then
 266   EXTRA2+=" -var reposuffix=$REPOSUFFIX"
 267 fi
 268 if [[ "$PUBLIC_KEY_FILE" != "" ]]; then
 269   EXTRA2+=" -var public_key_file=$PUBLIC_KEY_FILE"
 270 fi
 271 if [[ "$MKSQUASHFS_MEM" != "" ]]; then
 272   EXTRA2+=" -var mksquashfs_mem=$MKSQUASHFS_MEM"
 273 fi
 274 if [[ "$NVIDIA_GPU_SUPPORT" != "" ]]; then
 275   EXTRA2+=" -var nvidia_gpu_support=$NVIDIA_GPU_SUPPORT"
 276 fi
 277
 278
 279
 280 echo
 281 packer version
 282 echo
 283 echo packer build$EXTRA -var "arvados_cluster=$ARVADOS_CLUSTER_ID"$EXTRA2 $JSON_FILE
 284 packer build$EXTRA -var "arvados_cluster=$ARVADOS_CLUSTER_ID"$EXTRA2 $JSON_FILE