Make compute node image script compatible with Ubuntu.

[arvados.git] / tools / compute-images / build.sh

diff --git a/tools/compute-images/build.sh b/tools/compute-images/build.sh
index a714bafc1d89666cf80cdea9982a5b3119983f59..7b45a43c27ea8b515296381c8cccbcb7d8c2df00 100755 (executable)
--- a/tools/compute-images/build.sh
+++ b/tools/compute-images/build.sh
@@ -17,48 +17,64 @@ Syntax:
 
 Options:
 
 
 Options:
 

-  --json-file (required)
-      Path to the packer json file
-  --arvados-cluster-id (required)
-      The ID of the Arvados cluster, e.g. zzzzz
-  --aws-profile (default: false)
-      AWS profile to use (valid profile from ~/.aws/config
-  --aws-secrets-file (default: false, required if building for AWS)
-      AWS secrets file which will be sourced from this script
-  --aws-source-ami (default: false, required if building for AWS)
-      The AMI to use as base for building the images
-  --aws-region (default: us-east-1)
+  --json-file <path>
+      Path to the packer json file (required)
+  --arvados-cluster-id <xxxxx>
+      The ID of the Arvados cluster, e.g. zzzzz(required)
+  --aws-profile <profile>
+      AWS profile to use (valid profile from ~/.aws/config (optional)
+  --aws-secrets-file <path>
+      AWS secrets file which will be sourced from this script (optional)
+      When building for AWS, either an AWS profile or an AWS secrets file
+      must be provided.
+  --aws-source-ami <ami-xxxxxxxxxxxxxxxxx>
+      The AMI to use as base for building the images (required if building for AWS)
+  --aws-region <region> (default: us-east-1)

       The AWS region to use for building the images
       The AWS region to use for building the images

-  --aws-vpc-id (optional)
-      VPC id for AWS, otherwise packer will pick the default one
-  --aws-subnet-id
-      Subnet id for AWS otherwise packer will pick the default one for the VPC
-  --gcp-project-id (default: false, required if building for GCP)
-      GCP project id
-  --gcp-account-file (default: false, required if building for GCP)
-      GCP account file
-  --gcp-zone (default: us-central1-f)
+  --aws-vpc-id <vpc-id>
+      VPC id for AWS, if not specified packer will derive from the subnet id or pick the default one.
+  --aws-subnet-id <subnet-xxxxxxxxxxxxxxxxx>
+      Subnet id for AWS, if not specified packer will pick the default one for the VPC.
+  --aws-ebs-autoscale
+      Install the AWS EBS autoscaler daemon (default: do not install the AWS EBS autoscaler).
+  --aws-associate-public-ip <true|false>
+      Associate a public IP address with the node used for building the compute image.
+      Required when the machine running packer can not reach the node used for building
+      the compute image via its private IP. (default: true if building for AWS)
+      Note: if the subnet has "Auto-assign public IPv4 address" enabled, disabling this
+      flag will have no effect.
+  --aws-ena-support <true|false>
+      Enable enhanced networking (default: true if building for AWS)
+  --gcp-project-id <project-id>
+      GCP project id (required if building for GCP)
+  --gcp-account-file <path>
+      GCP account file (required if building for GCP)
+  --gcp-zone <zone> (default: us-central1-f)

       GCP zone
       GCP zone

-  --azure-secrets-file (default: false, required if building for Azure)
-      Azure secrets file which will be sourced from this script
-  --azure-resource-group (default: false, required if building for Azure)
-      Azure resource group
-  --azure-location (default: false, required if building for Azure)
-      Azure location, e.g. centralus, eastus, westeurope
-  --azure-sku (default: unset, required if building for Azure, e.g. 16.04-LTS)
+  --azure-secrets-file <patch>
+      Azure secrets file which will be sourced from this script (required if building for Azure)
+  --azure-resource-group <resouce-group>
+      Azure resource group (required if building for Azure)
+  --azure-location <location>
+      Azure location, e.g. centralus, eastus, westeurope (required if building for Azure)
+  --azure-sku <sku> (required if building for Azure, e.g. 16.04-LTS)

       Azure SKU image to use
       Azure SKU image to use

-  --ssh_user  (default: packer)
+  --ssh_user <user> (default: packer)

       The user packer will use to log into the image
       The user packer will use to log into the image

-  --resolver (default: host's network provided)
-      The dns resolver for the machine
-  --reposuffix (default: unset)
+  --resolver <resolver_IP>
+      The dns resolver for the machine (default: host's network provided)
+  --reposuffix <suffix>

       Set this to "-dev" to track the unstable/dev Arvados repositories
       Set this to "-dev" to track the unstable/dev Arvados repositories

-  --public-key-file (required)
-      Path to the public key file that a-d-c will use to log into the compute node
-  --mksquashfs-mem (default: 512M)
+  --public-key-file <path>
+      Path to the public key file that a-d-c will use to log into the compute node (required)
+  --mksquashfs-mem (default: 256M)

       Only relevant when using Singularity. This is the amount of memory mksquashfs is allowed to use.
       Only relevant when using Singularity. This is the amount of memory mksquashfs is allowed to use.

+  --nvidia-gpu-support
+      Install all the necessary tooling for Nvidia GPU support (default: do not install Nvidia GPU support)

   --debug
   --debug

-      Output debug information (default: false)
+      Output debug information (default: no debug output is printed)
+
+For more information, see the Arvados documentation at https://doc.arvados.org/install/crunch2-cloud/install-compute-node.html

 
 EOF
 
 
 EOF
 


@@ -69,6 +85,9 @@ AWS_SECRETS_FILE=
 AWS_SOURCE_AMI=
 AWS_VPC_ID=
 AWS_SUBNET_ID=
 AWS_SOURCE_AMI=
 AWS_VPC_ID=
 AWS_SUBNET_ID=

+AWS_EBS_AUTOSCALE=
+AWS_ASSOCIATE_PUBLIC_IP=true
+AWS_ENA_SUPPORT=true

 GCP_PROJECT_ID=
 GCP_ACCOUNT_FILE=
 GCP_ZONE=
 GCP_PROJECT_ID=
 GCP_ACCOUNT_FILE=
 GCP_ZONE=


@@ -80,10 +99,11 @@ DEBUG=
 SSH_USER=
 AWS_DEFAULT_REGION=us-east-1
 PUBLIC_KEY_FILE=
 SSH_USER=
 AWS_DEFAULT_REGION=us-east-1
 PUBLIC_KEY_FILE=

-MKSQUASHFS_MEM=512M
+MKSQUASHFS_MEM=256M
+NVIDIA_GPU_SUPPORT=

 
 PARSEDOPTS=$(getopt --name "$0" --longoptions \
 
 PARSEDOPTS=$(getopt --name "$0" --longoptions \

-    help,json-file:,arvados-cluster-id:,aws-source-ami:,aws-profile:,aws-secrets-file:,aws-region:,aws-vpc-id:,aws-subnet-id:,gcp-project-id:,gcp-account-file:,gcp-zone:,azure-secrets-file:,azure-resource-group:,azure-location:,azure-sku:,azure-cloud-environment:,ssh_user:,resolver:,reposuffix:,public-key-file:,mksquashfs-mem:,debug \
+    help,json-file:,arvados-cluster-id:,aws-source-ami:,aws-profile:,aws-secrets-file:,aws-region:,aws-vpc-id:,aws-subnet-id:,aws-ebs-autoscale,aws-associate-public-ip:,aws-ena-support:,gcp-project-id:,gcp-account-file:,gcp-zone:,azure-secrets-file:,azure-resource-group:,azure-location:,azure-sku:,azure-cloud-environment:,ssh_user:,resolver:,reposuffix:,public-key-file:,mksquashfs-mem:,nvidia-gpu-support,debug \

     -- "" "$@")
 if [ $? -ne 0 ]; then
     exit 1
     -- "" "$@")
 if [ $? -ne 0 ]; then
     exit 1


@@ -121,6 +141,15 @@ while [ $# -gt 0 ]; do
         --aws-subnet-id)
             AWS_SUBNET_ID="$2"; shift
             ;;
         --aws-subnet-id)
             AWS_SUBNET_ID="$2"; shift
             ;;

+        --aws-ebs-autoscale)
+            AWS_EBS_AUTOSCALE=1
+            ;;
+        --aws-associate-public-ip)
+            AWS_ASSOCIATE_PUBLIC_IP="$2"; shift
+            ;;
+        --aws-ena-support)
+            AWS_ENA_SUPPORT="$2"; shift
+            ;;

         --gcp-project-id)
             GCP_PROJECT_ID="$2"; shift
             ;;
         --gcp-project-id)
             GCP_PROJECT_ID="$2"; shift
             ;;


@@ -160,6 +189,9 @@ while [ $# -gt 0 ]; do
         --mksquashfs-mem)
             MKSQUASHFS_MEM="$2"; shift
             ;;
         --mksquashfs-mem)
             MKSQUASHFS_MEM="$2"; shift
             ;;

+        --nvidia-gpu-support)
+            NVIDIA_GPU_SUPPORT=1
+            ;;

         --debug)
             # If you want to debug a build issue, add the -debug flag to the build
             # command in question.
         --debug)
             # If you want to debug a build issue, add the -debug flag to the build
             # command in question.


@@ -179,7 +211,7 @@ while [ $# -gt 0 ]; do
 done
 
 
 done
 
 

-if [[ "$JSON_FILE" == "" ]] || [[ ! -f "$JSON_FILE" ]]; then
+if [[ -z "$JSON_FILE" ]] || [[ ! -f "$JSON_FILE" ]]; then

   echo >&2 "$helpmessage"
   echo >&2
   echo >&2 "ERROR: packer json file not found"
   echo >&2 "$helpmessage"
   echo >&2
   echo >&2 "ERROR: packer json file not found"


@@ -195,7 +227,7 @@ if [[ -z "$ARVADOS_CLUSTER_ID" ]]; then
   exit 1
 fi
 
   exit 1
 fi
 

-if [[ "$PUBLIC_KEY_FILE" == "" ]] || [[ ! -f "$PUBLIC_KEY_FILE" ]]; then
+if [[ -z "$PUBLIC_KEY_FILE" ]] || [[ ! -f "$PUBLIC_KEY_FILE" ]]; then

   echo >&2 "$helpmessage"
   echo >&2
   echo >&2 "ERROR: public key file file not found"
   echo >&2 "$helpmessage"
   echo >&2
   echo >&2 "ERROR: public key file file not found"


@@ -212,60 +244,76 @@ if [[ ! -z "$AZURE_SECRETS_FILE" ]]; then
 fi
 
 
 fi
 
 

+AWS=0

 EXTRA2=""
 
 EXTRA2=""
 

-if [[ "$AWS_SOURCE_AMI" != "" ]]; then
+if [[ -n "$AWS_SOURCE_AMI" ]]; then

   EXTRA2+=" -var aws_source_ami=$AWS_SOURCE_AMI"
   EXTRA2+=" -var aws_source_ami=$AWS_SOURCE_AMI"

+  AWS=1

 fi
 fi

-if [[ "$AWS_PROFILE" != "" ]]; then
+if [[ -n "$AWS_PROFILE" ]]; then

   EXTRA2+=" -var aws_profile=$AWS_PROFILE"
   EXTRA2+=" -var aws_profile=$AWS_PROFILE"

+  AWS=1

 fi
 fi

-if [[ "$AWS_VPC_ID" != "" ]]; then
-  EXTRA2+=" -var vpc_id=$AWS_VPC_ID -var associate_public_ip_address=true "
+if [[ -n "$AWS_VPC_ID" ]]; then
+  EXTRA2+=" -var vpc_id=$AWS_VPC_ID"
+  AWS=1

 fi
 fi

-if [[ "$AWS_SUBNET_ID" != "" ]]; then
-  EXTRA2+=" -var subnet_id=$AWS_SUBNET_ID -var associate_public_ip_address=true "
+if [[ -n "$AWS_SUBNET_ID" ]]; then
+  EXTRA2+=" -var subnet_id=$AWS_SUBNET_ID"
+  AWS=1

 fi
 fi

-if [[ "$AWS_DEFAULT_REGION" != "" ]]; then
+if [[ -n "$AWS_DEFAULT_REGION" ]]; then

   EXTRA2+=" -var aws_default_region=$AWS_DEFAULT_REGION"
   EXTRA2+=" -var aws_default_region=$AWS_DEFAULT_REGION"

+  AWS=1

 fi
 fi

-if [[ "$GCP_PROJECT_ID" != "" ]]; then
+if [[ -n "$AWS_EBS_AUTOSCALE" ]]; then
+  EXTRA2+=" -var aws_ebs_autoscale=$AWS_EBS_AUTOSCALE"
+  AWS=1
+fi
+if [[ $AWS -eq 1 ]]; then
+  EXTRA2+=" -var aws_associate_public_ip_address=$AWS_ASSOCIATE_PUBLIC_IP"
+  EXTRA2+=" -var aws_ena_support=$AWS_ENA_SUPPORT"
+fi
+if [[ -n "$GCP_PROJECT_ID" ]]; then

   EXTRA2+=" -var project_id=$GCP_PROJECT_ID"
 fi
   EXTRA2+=" -var project_id=$GCP_PROJECT_ID"
 fi

-if [[ "$GCP_ACCOUNT_FILE" != "" ]]; then
+if [[ -n "$GCP_ACCOUNT_FILE" ]]; then

   EXTRA2+=" -var account_file=$GCP_ACCOUNT_FILE"
 fi
   EXTRA2+=" -var account_file=$GCP_ACCOUNT_FILE"
 fi

-if [[ "$GCP_ZONE" != "" ]]; then
+if [[ -n "$GCP_ZONE" ]]; then

   EXTRA2+=" -var zone=$GCP_ZONE"
 fi
   EXTRA2+=" -var zone=$GCP_ZONE"
 fi

-if [[ "$AZURE_RESOURCE_GROUP" != "" ]]; then
+if [[ -n "$AZURE_RESOURCE_GROUP" ]]; then

   EXTRA2+=" -var resource_group=$AZURE_RESOURCE_GROUP"
 fi
   EXTRA2+=" -var resource_group=$AZURE_RESOURCE_GROUP"
 fi

-if [[ "$AZURE_LOCATION" != "" ]]; then
+if [[ -n "$AZURE_LOCATION" ]]; then

   EXTRA2+=" -var location=$AZURE_LOCATION"
 fi
   EXTRA2+=" -var location=$AZURE_LOCATION"
 fi

-if [[ "$AZURE_SKU" != "" ]]; then
+if [[ -n "$AZURE_SKU" ]]; then

   EXTRA2+=" -var image_sku=$AZURE_SKU"
 fi
   EXTRA2+=" -var image_sku=$AZURE_SKU"
 fi

-if [[ "$AZURE_CLOUD_ENVIRONMENT" != "" ]]; then
+if [[ -n "$AZURE_CLOUD_ENVIRONMENT" ]]; then

   EXTRA2+=" -var cloud_environment_name=$AZURE_CLOUD_ENVIRONMENT"
 fi
   EXTRA2+=" -var cloud_environment_name=$AZURE_CLOUD_ENVIRONMENT"
 fi

-if [[ "$SSH_USER" != "" ]]; then
+if [[ -n "$SSH_USER" ]]; then

   EXTRA2+=" -var ssh_user=$SSH_USER"
 fi
   EXTRA2+=" -var ssh_user=$SSH_USER"
 fi

-if [[ "$RESOLVER" != "" ]]; then
+if [[ -n "$RESOLVER" ]]; then

   EXTRA2+=" -var resolver=$RESOLVER"
 fi
   EXTRA2+=" -var resolver=$RESOLVER"
 fi

-if [[ "$REPOSUFFIX" != "" ]]; then
+if [[ -n "$REPOSUFFIX" ]]; then

   EXTRA2+=" -var reposuffix=$REPOSUFFIX"
 fi
   EXTRA2+=" -var reposuffix=$REPOSUFFIX"
 fi

-if [[ "$PUBLIC_KEY_FILE" != "" ]]; then
+if [[ -n "$PUBLIC_KEY_FILE" ]]; then

   EXTRA2+=" -var public_key_file=$PUBLIC_KEY_FILE"
 fi
   EXTRA2+=" -var public_key_file=$PUBLIC_KEY_FILE"
 fi

-if [[ "$MKSQUASHFS_MEM" != "" ]]; then
+if [[ -n "$MKSQUASHFS_MEM" ]]; then

   EXTRA2+=" -var mksquashfs_mem=$MKSQUASHFS_MEM"
 fi
   EXTRA2+=" -var mksquashfs_mem=$MKSQUASHFS_MEM"
 fi

-
+if [[ -n "$NVIDIA_GPU_SUPPORT" ]]; then
+  EXTRA2+=" -var nvidia_gpu_support=$NVIDIA_GPU_SUPPORT"
+fi

 
 echo
 packer version
 
 echo
 packer version