18289: add support to set `mksquashfs mem` for Singularity in our

       compute node image builder for cloud setups.

Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward@curii.com>

diff --git a/tools/compute-images/arvados-images-aws.json b/tools/compute-images/arvados-images-aws.json
index 4d757abfd2d9f5530d2995c4241e8c702e6f095f..b1b4c909d2491bdfb77f188267c68a9bd186b332 100644 (file)
--- a/tools/compute-images/arvados-images-aws.json
+++ b/tools/compute-images/arvados-images-aws.json
@@ -8,6 +8,7 @@
     "aws_source_ami": "ami-04d70e069399af2e9",
     "build_environment": "aws",
     "public_key_file": "",
+    "mksquashfs_mem": "",
     "reposuffix": "",
     "resolver": "",
     "ssh_user": "admin",
@@ -76,6 +77,6 @@
     "type": "shell",
     "execute_command": "sudo -S env {{ .Vars }} /bin/bash '{{ .Path }}'",
     "script": "scripts/base.sh",
-    "environment_vars": ["RESOLVER={{user `resolver`}}","REPOSUFFIX={{user `reposuffix`}}"]
+    "environment_vars": ["RESOLVER={{user `resolver`}}","REPOSUFFIX={{user `reposuffix`}}","MKSQUASHFS_MEM={{user `mksquashfs_mem`}}"]
   }]
 }

diff --git a/tools/compute-images/arvados-images-azure.json b/tools/compute-images/arvados-images-azure.json
index ec1d9b6a6379a0fddba94ffed7184f8e4f2d07ea..20f776d041f328594956053a53b0a7f5c2c771bd 100644 (file)
--- a/tools/compute-images/arvados-images-azure.json
+++ b/tools/compute-images/arvados-images-azure.json
@@ -10,6 +10,7 @@
     "location": "centralus",
     "project_id": "",
     "public_key_file": "",
+    "mksquashfs_mem": "",
     "reposuffix": "",
     "resolver": "",
     "resource_group": null,
@@ -65,6 +66,6 @@
     "type": "shell",
     "execute_command": "sudo -S env {{ .Vars }} /bin/bash '{{ .Path }}'",
     "script": "scripts/base.sh",
-    "environment_vars": ["RESOLVER={{user `resolver`}}","REPOSUFFIX={{user `reposuffix`}}"]
+    "environment_vars": ["RESOLVER={{user `resolver`}}","REPOSUFFIX={{user `reposuffix`}}","MKSQUASHFS_MEM={{user `mksquashfs_mem`}}"]
   }]
 }

diff --git a/tools/compute-images/build.sh b/tools/compute-images/build.sh
index a2dd2ed288884b27f382ea808ae10b76efb85662..a714bafc1d89666cf80cdea9982a5b3119983f59 100755 (executable)
--- a/tools/compute-images/build.sh
+++ b/tools/compute-images/build.sh
@@ -55,6 +55,8 @@ Options:
       Set this to "-dev" to track the unstable/dev Arvados repositories
   --public-key-file (required)
       Path to the public key file that a-d-c will use to log into the compute node
+  --mksquashfs-mem (default: 512M)
+      Only relevant when using Singularity. This is the amount of memory mksquashfs is allowed to use.
   --debug
       Output debug information (default: false)
 
@@ -78,9 +80,10 @@ DEBUG=
 SSH_USER=
 AWS_DEFAULT_REGION=us-east-1
 PUBLIC_KEY_FILE=
+MKSQUASHFS_MEM=512M
 
 PARSEDOPTS=$(getopt --name "$0" --longoptions \
-    help,json-file:,arvados-cluster-id:,aws-source-ami:,aws-profile:,aws-secrets-file:,aws-region:,aws-vpc-id:,aws-subnet-id:,gcp-project-id:,gcp-account-file:,gcp-zone:,azure-secrets-file:,azure-resource-group:,azure-location:,azure-sku:,azure-cloud-environment:,ssh_user:,resolver:,reposuffix:,public-key-file:,debug \
+    help,json-file:,arvados-cluster-id:,aws-source-ami:,aws-profile:,aws-secrets-file:,aws-region:,aws-vpc-id:,aws-subnet-id:,gcp-project-id:,gcp-account-file:,gcp-zone:,azure-secrets-file:,azure-resource-group:,azure-location:,azure-sku:,azure-cloud-environment:,ssh_user:,resolver:,reposuffix:,public-key-file:,mksquashfs-mem:,debug \
     -- "" "$@")
 if [ $? -ne 0 ]; then
     exit 1
@@ -154,6 +157,9 @@ while [ $# -gt 0 ]; do
         --public-key-file)
             PUBLIC_KEY_FILE="$2"; shift
             ;;
+        --mksquashfs-mem)
+            MKSQUASHFS_MEM="$2"; shift
+            ;;
         --debug)
             # If you want to debug a build issue, add the -debug flag to the build
             # command in question.
@@ -256,6 +262,10 @@ fi
 if [[ "$PUBLIC_KEY_FILE" != "" ]]; then
   EXTRA2+=" -var public_key_file=$PUBLIC_KEY_FILE"
 fi
+if [[ "$MKSQUASHFS_MEM" != "" ]]; then
+  EXTRA2+=" -var mksquashfs_mem=$MKSQUASHFS_MEM"
+fi
+
 
 echo
 packer version

diff --git a/tools/compute-images/scripts/base.sh b/tools/compute-images/scripts/base.sh
index 022f4a7e5a5745f6de131a6c475f8f139de45af8..0ab51223b94aae29bc349e764179ce69ba1bcc7c 100644 (file)
--- a/tools/compute-images/scripts/base.sh
+++ b/tools/compute-images/scripts/base.sh
@@ -89,6 +89,11 @@ make -C ./builddir
 make -C ./builddir install
 ln -sf /var/lib/arvados/bin/* /usr/local/bin/
 
+# set `mksquashfs mem` in the singularity config file if it is configured
+if [ "$MKSQUASHFS_MEM" != "" ]; then
+  echo "mksquashfs mem = ${MKSQUASHFS_MEM}" >> /var/lib/arvados/etc/singularity/singularity.conf
+fi
+
 # Print singularity version installed
 singularity --version