tools/compute-images/build.sh

   1 #!/bin/bash
   2
   3 # Copyright (C) The Arvados Authors. All rights reserved.
   4 #
   5 # SPDX-License-Identifier: Apache-2.0
   6
   7 JSON_FILE=$1
   8 ARVADOS_CLUSTER=$2
   9 PROJECT_ID=$3
  10 ACCOUNT_FILE=$4
  11
  12 read -rd "\000" helpmessage <<EOF
  13 $(basename $0): Build cloud images for arvados-dispatch-cloud
  14
  15 Syntax:
  16         $(basename $0) [options]
  17
  18 Options:
  19
  20   --json-file <path>
  21       Path to the packer json file (required)
  22   --arvados-cluster-id <xxxxx>
  23       The ID of the Arvados cluster, e.g. zzzzz(required)
  24   --aws-profile <profile>
  25       AWS profile to use (valid profile from ~/.aws/config (optional)
  26   --aws-secrets-file <path>
  27       AWS secrets file which will be sourced from this script (optional)
  28       When building for AWS, either an AWS profile or an AWS secrets file
  29       must be provided.
  30   --aws-source-ami <ami-xxxxxxxxxxxxxxxxx>
  31       The AMI to use as base for building the images (required if building for AWS)
  32   --aws-region <region> (default: us-east-1)
  33       The AWS region to use for building the images
  34   --aws-vpc-id <vpc-id>
  35       VPC id for AWS, if not specified packer will derive from the subnet id or pick the default one.
  36   --aws-subnet-id <subnet-xxxxxxxxxxxxxxxxx>
  37       Subnet id for AWS, if not specified packer will pick the default one for the VPC.
  38   --aws-ebs-autoscale
  39       Install the AWS EBS autoscaler daemon (default: do not install the AWS EBS autoscaler).
  40   --aws-associate-public-ip <true|false>
  41       Associate a public IP address with the node used for building the compute image.
  42       Required when the machine running packer can not reach the node used for building
  43       the compute image via its private IP. (default: true if building for AWS)
  44       Note: if the subnet has "Auto-assign public IPv4 address" enabled, disabling this
  45       flag will have no effect.
  46   --aws-ena-support <true|false>
  47       Enable enhanced networking (default: true if building for AWS)
  48   --gcp-project-id <project-id>
  49       GCP project id (required if building for GCP)
  50   --gcp-account-file <path>
  51       GCP account file (required if building for GCP)
  52   --gcp-zone <zone> (default: us-central1-f)
  53       GCP zone
  54   --azure-secrets-file <patch>
  55       Azure secrets file which will be sourced from this script (required if building for Azure)
  56   --azure-resource-group <resouce-group>
  57       Azure resource group (required if building for Azure)
  58   --azure-location <location>
  59       Azure location, e.g. centralus, eastus, westeurope (required if building for Azure)
  60   --azure-sku <sku> (required if building for Azure, e.g. 16.04-LTS)
  61       Azure SKU image to use
  62   --ssh_user <user> (default: packer)
  63       The user packer will use to log into the image
  64   --workdir <path> (default: /tmp)
  65       The directory where data files are staged and setup scripts are run
  66   --resolver <resolver_IP>
  67       The dns resolver for the machine (default: host's network provided)
  68   --reposuffix <suffix>
  69       Set this to "-dev" to track the unstable/dev Arvados repositories
  70   --pin-packages, --no-pin-packages
  71       These flags determine whether or not to configure apt pins for Arvados
  72       and third-party packages it depends on. By default packages are pinned
  73       unless you set \`--reposuffix -dev\`.
  74   --public-key-file <path>
  75       Path to the public key file that a-d-c will use to log into the compute node (required)
  76   --mksquashfs-mem (default: 256M)
  77       Only relevant when using Singularity. This is the amount of memory mksquashfs is allowed to use.
  78   --nvidia-gpu-support
  79       Install all the necessary tooling for Nvidia GPU support (default: do not install Nvidia GPU support)
  80   --debug
  81       Output debug information (default: no debug output is printed)
  82
  83 For more information, see the Arvados documentation at https://doc.arvados.org/install/crunch2-cloud/install-compute-node.html
  84
  85 EOF
  86
  87 JSON_FILE=
  88 ARVADOS_CLUSTER_ID=
  89 AWS_PROFILE=
  90 AWS_SECRETS_FILE=
  91 AWS_SOURCE_AMI=
  92 AWS_VPC_ID=
  93 AWS_SUBNET_ID=
  94 AWS_EBS_AUTOSCALE=
  95 AWS_ASSOCIATE_PUBLIC_IP=true
  96 AWS_ENA_SUPPORT=true
  97 GCP_PROJECT_ID=
  98 GCP_ACCOUNT_FILE=
  99 GCP_ZONE=
 100 AZURE_SECRETS_FILE=
 101 AZURE_RESOURCE_GROUP=
 102 AZURE_LOCATION=
 103 AZURE_CLOUD_ENVIRONMENT=
 104 DEBUG=
 105 SSH_USER=
 106 WORKDIR=
 107 AWS_DEFAULT_REGION=us-east-1
 108 PIN_PACKAGES=
 109 PUBLIC_KEY_FILE=
 110 MKSQUASHFS_MEM=256M
 111 NVIDIA_GPU_SUPPORT=
 112
 113 PARSEDOPTS=$(getopt --name "$0" --longoptions \
 114     help,json-file:,arvados-cluster-id:,aws-source-ami:,aws-profile:,aws-secrets-file:,aws-region:,aws-vpc-id:,aws-subnet-id:,aws-ebs-autoscale,aws-associate-public-ip:,aws-ena-support:,gcp-project-id:,gcp-account-file:,gcp-zone:,azure-secrets-file:,azure-resource-group:,azure-location:,azure-sku:,azure-cloud-environment:,ssh_user:,workdir:,resolver:,reposuffix:,pin-packages,no-pin-packages,public-key-file:,mksquashfs-mem:,nvidia-gpu-support,debug \
 115     -- "" "$@")
 116 if [ $? -ne 0 ]; then
 117     exit 1
 118 fi
 119
 120 eval set -- "$PARSEDOPTS"
 121 while [ $# -gt 0 ]; do
 122     case "$1" in
 123         --help)
 124             echo >&2 "$helpmessage"
 125             echo >&2
 126             exit 1
 127             ;;
 128         --json-file)
 129             JSON_FILE="$2"; shift
 130             ;;
 131         --arvados-cluster-id)
 132             ARVADOS_CLUSTER_ID="$2"; shift
 133             ;;
 134         --aws-source-ami)
 135             AWS_SOURCE_AMI="$2"; shift
 136             ;;
 137         --aws-profile)
 138             AWS_PROFILE="$2"; shift
 139             ;;
 140         --aws-secrets-file)
 141             AWS_SECRETS_FILE="$2"; shift
 142             ;;
 143         --aws-region)
 144             AWS_DEFAULT_REGION="$2"; shift
 145             ;;
 146         --aws-vpc-id)
 147             AWS_VPC_ID="$2"; shift
 148             ;;
 149         --aws-subnet-id)
 150             AWS_SUBNET_ID="$2"; shift
 151             ;;
 152         --aws-ebs-autoscale)
 153             AWS_EBS_AUTOSCALE=1
 154             ;;
 155         --aws-associate-public-ip)
 156             AWS_ASSOCIATE_PUBLIC_IP="$2"; shift
 157             ;;
 158         --aws-ena-support)
 159             AWS_ENA_SUPPORT="$2"; shift
 160             ;;
 161         --gcp-project-id)
 162             GCP_PROJECT_ID="$2"; shift
 163             ;;
 164         --gcp-account-file)
 165             GCP_ACCOUNT_FILE="$2"; shift
 166             ;;
 167         --gcp-zone)
 168             GCP_ZONE="$2"; shift
 169             ;;
 170         --azure-secrets-file)
 171             AZURE_SECRETS_FILE="$2"; shift
 172             ;;
 173         --azure-resource-group)
 174             AZURE_RESOURCE_GROUP="$2"; shift
 175             ;;
 176         --azure-location)
 177             AZURE_LOCATION="$2"; shift
 178             ;;
 179         --azure-sku)
 180             AZURE_SKU="$2"; shift
 181             ;;
 182         --azure-cloud-environment)
 183             AZURE_CLOUD_ENVIRONMENT="$2"; shift
 184             ;;
 185         --ssh_user)
 186             SSH_USER="$2"; shift
 187             ;;
 188         --workdir)
 189             WORKDIR="$2"; shift
 190             ;;
 191         --resolver)
 192             RESOLVER="$2"; shift
 193             ;;
 194         --reposuffix)
 195             REPOSUFFIX="$2"; shift
 196             ;;
 197         --pin-packages)
 198             PIN_PACKAGES=true
 199             ;;
 200         --no-pin-packages)
 201             PIN_PACKAGES=false
 202             ;;
 203         --public-key-file)
 204             PUBLIC_KEY_FILE="$2"; shift
 205             ;;
 206         --mksquashfs-mem)
 207             MKSQUASHFS_MEM="$2"; shift
 208             ;;
 209         --nvidia-gpu-support)
 210             NVIDIA_GPU_SUPPORT=1
 211             ;;
 212         --debug)
 213             # If you want to debug a build issue, add the -debug flag to the build
 214             # command in question.
 215             # This will allow you to ssh in, if you use the .pem file that packer
 216             # generates in this directory as the ssh key. The base image uses the admin
 217             # user and ssh port 22.
 218             EXTRA=" -debug"
 219             ;;
 220         --)
 221             if [ $# -gt 1 ]; then
 222                 echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
 223                 exit 1
 224             fi
 225             ;;
 226     esac
 227     shift
 228 done
 229
 230
 231 if [[ -z "$JSON_FILE" ]] || [[ ! -f "$JSON_FILE" ]]; then
 232   echo >&2 "$helpmessage"
 233   echo >&2
 234   echo >&2 "ERROR: packer json file not found"
 235   echo >&2
 236   exit 1
 237 fi
 238
 239 if [[ -z "$ARVADOS_CLUSTER_ID" ]]; then
 240   echo >&2 "$helpmessage"
 241   echo >&2
 242   echo >&2 "ERROR: arvados cluster id not specified"
 243   echo >&2
 244   exit 1
 245 fi
 246
 247 if [[ -z "$PUBLIC_KEY_FILE" ]] || [[ ! -f "$PUBLIC_KEY_FILE" ]]; then
 248   echo >&2 "$helpmessage"
 249   echo >&2
 250   echo >&2 "ERROR: public key file file not found"
 251   echo >&2
 252   exit 1
 253 fi
 254
 255 if [[ ! -z "$AWS_SECRETS_FILE" ]]; then
 256   source $AWS_SECRETS_FILE
 257 fi
 258
 259 if [[ ! -z "$AZURE_SECRETS_FILE" ]]; then
 260   source $AZURE_SECRETS_FILE
 261 fi
 262
 263
 264 AWS=0
 265 EXTRA2=""
 266
 267 if [[ -n "$AWS_SOURCE_AMI" ]]; then
 268   EXTRA2+=" -var aws_source_ami=$AWS_SOURCE_AMI"
 269   AWS=1
 270 fi
 271 if [[ -n "$AWS_PROFILE" ]]; then
 272   EXTRA2+=" -var aws_profile=$AWS_PROFILE"
 273   AWS=1
 274 fi
 275 if [[ -n "$AWS_VPC_ID" ]]; then
 276   EXTRA2+=" -var vpc_id=$AWS_VPC_ID"
 277   AWS=1
 278 fi
 279 if [[ -n "$AWS_SUBNET_ID" ]]; then
 280   EXTRA2+=" -var subnet_id=$AWS_SUBNET_ID"
 281   AWS=1
 282 fi
 283 if [[ -n "$AWS_DEFAULT_REGION" ]]; then
 284   EXTRA2+=" -var aws_default_region=$AWS_DEFAULT_REGION"
 285   AWS=1
 286 fi
 287 if [[ -n "$AWS_EBS_AUTOSCALE" ]]; then
 288   EXTRA2+=" -var aws_ebs_autoscale=$AWS_EBS_AUTOSCALE"
 289   AWS=1
 290 fi
 291 if [[ $AWS -eq 1 ]]; then
 292   EXTRA2+=" -var aws_associate_public_ip_address=$AWS_ASSOCIATE_PUBLIC_IP"
 293   EXTRA2+=" -var aws_ena_support=$AWS_ENA_SUPPORT"
 294 fi
 295 if [[ -n "$GCP_PROJECT_ID" ]]; then
 296   EXTRA2+=" -var project_id=$GCP_PROJECT_ID"
 297 fi
 298 if [[ -n "$GCP_ACCOUNT_FILE" ]]; then
 299   EXTRA2+=" -var account_file=$GCP_ACCOUNT_FILE"
 300 fi
 301 if [[ -n "$GCP_ZONE" ]]; then
 302   EXTRA2+=" -var zone=$GCP_ZONE"
 303 fi
 304 if [[ -n "$AZURE_RESOURCE_GROUP" ]]; then
 305   EXTRA2+=" -var resource_group=$AZURE_RESOURCE_GROUP"
 306 fi
 307 if [[ -n "$AZURE_LOCATION" ]]; then
 308   EXTRA2+=" -var location=$AZURE_LOCATION"
 309 fi
 310 if [[ -n "$AZURE_SKU" ]]; then
 311   EXTRA2+=" -var image_sku=$AZURE_SKU"
 312 fi
 313 if [[ -n "$AZURE_CLOUD_ENVIRONMENT" ]]; then
 314   EXTRA2+=" -var cloud_environment_name=$AZURE_CLOUD_ENVIRONMENT"
 315 fi
 316 if [[ -n "$SSH_USER" ]]; then
 317   EXTRA2+=" -var ssh_user=$SSH_USER"
 318 fi
 319 if [[ -n "$WORKDIR" ]]; then
 320   EXTRA2+=" -var workdir=$WORKDIR"
 321 fi
 322 if [[ -n "$RESOLVER" ]]; then
 323   EXTRA2+=" -var resolver=$RESOLVER"
 324 fi
 325 if [[ -n "$REPOSUFFIX" ]]; then
 326   EXTRA2+=" -var reposuffix=$REPOSUFFIX"
 327 fi
 328 if [[ -z "$PIN_PACKAGES" ]]; then
 329     case "$REPOSUFFIX" in
 330         -dev) PIN_PACKAGES=false ;;
 331         *) PIN_PACKAGES=true ;;
 332     esac
 333 fi
 334 EXTRA2+=" -var pin_packages=$PIN_PACKAGES"
 335 if [[ -n "$PUBLIC_KEY_FILE" ]]; then
 336   EXTRA2+=" -var public_key_file=$PUBLIC_KEY_FILE"
 337 fi
 338 if [[ -n "$MKSQUASHFS_MEM" ]]; then
 339   EXTRA2+=" -var mksquashfs_mem=$MKSQUASHFS_MEM"
 340 fi
 341 if [[ -n "$NVIDIA_GPU_SUPPORT" ]]; then
 342   EXTRA2+=" -var nvidia_gpu_support=$NVIDIA_GPU_SUPPORT"
 343 fi
 344
 345 GOVERSION=$(grep 'const goversion =' ../../lib/install/deps.go |awk -F'"' '{print $2}')
 346 EXTRA2+=" -var goversion=$GOVERSION"
 347
 348 logfile=packer-$(date -Iseconds).log
 349
 350 echo
 351 packer version
 352 echo
 353 echo packer build$EXTRA -var "arvados_cluster=$ARVADOS_CLUSTER_ID"$EXTRA2 $JSON_FILE | tee -a $logfile
 354 packer build$EXTRA -var "arvados_cluster=$ARVADOS_CLUSTER_ID"$EXTRA2 $JSON_FILE 2>&1 | tee -a $logfile