Merge branch '17212-fix-port-conflicts'

fixes #17212

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@curii.com>

diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index 771dc2ee799584e4853c5e59c1769bd46bf44509..4c817852e05fbd8c67ad2b15b157e6103aae633f 100644 (file)
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -197,7 +197,7 @@ Clusters:
       # * 1.1) fits comfortably in memory. On a host dedicated to running
       # Keepstore, divide total memory by 88MiB to suggest a suitable value.
       # For example, if grep MemTotal /proc/meminfo reports MemTotal: 7125440
-      # kB, compute 7125440 / (88 * 1024)=79 and configure MaxBuffers: 79
+      # kB, compute 7125440 / (88 * 1024)=79 and set MaxKeepBlobBuffers: 79
       MaxKeepBlobBuffers: 128
 
       # API methods to disable. Disabled methods are not listed in the

diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go
index a202a540476ed54a36097244005c24822503ccf2..9d1ec706d5a9b6440fa2a1a76e9a4b532e0e2127 100644 (file)
--- a/lib/config/generated_config.go
+++ b/lib/config/generated_config.go
@@ -203,7 +203,7 @@ Clusters:
       # * 1.1) fits comfortably in memory. On a host dedicated to running
       # Keepstore, divide total memory by 88MiB to suggest a suitable value.
       # For example, if grep MemTotal /proc/meminfo reports MemTotal: 7125440
-      # kB, compute 7125440 / (88 * 1024)=79 and configure MaxBuffers: 79
+      # kB, compute 7125440 / (88 * 1024)=79 and set MaxKeepBlobBuffers: 79
       MaxKeepBlobBuffers: 128
 
       # API methods to disable. Disabled methods are not listed in the

diff --git a/lib/dispatchcloud/node_size.go b/lib/dispatchcloud/node_size.go
index fd04860861a14f9ff6b526ac6a562de02f0c74e4..7e8ce0bf4206834c80a176a7411ea614c2e1e3f7 100644 (file)
--- a/lib/dispatchcloud/node_size.go
+++ b/lib/dispatchcloud/node_size.go
@@ -96,6 +96,7 @@ func ChooseInstanceType(cc *arvados.Cluster, ctr *arvados.Container) (best arvad
        needVCPUs := ctr.RuntimeConstraints.VCPUs
 
        needRAM := ctr.RuntimeConstraints.RAM + ctr.RuntimeConstraints.KeepCacheRAM
+       needRAM += int64(cc.Containers.ReserveExtraRAM)
        needRAM = (needRAM * 100) / int64(100-discountConfiguredRAMPercent)
 
        ok := false

diff --git a/lib/dispatchcloud/node_size_test.go b/lib/dispatchcloud/node_size_test.go
index ea98efe1d2b175c70eb7394d106af45e491e41e6..abd292cbaf1278c7ca147e7da7352d74d0ad39d0 100644 (file)
--- a/lib/dispatchcloud/node_size_test.go
+++ b/lib/dispatchcloud/node_size_test.go
@@ -73,8 +73,14 @@ func (*NodeSizeSuite) TestChoose(c *check.C) {
                        "best":   {Price: 3.3, RAM: 4000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "best"},
                        "costly": {Price: 4.4, RAM: 4000000000, VCPUs: 8, Scratch: 2 * GiB, Name: "costly"},
                },
+               {
+                       "small":  {Price: 1.1, RAM: 1000000000, VCPUs: 2, Scratch: GiB, Name: "small"},
+                       "nearly": {Price: 2.2, RAM: 1200000000, VCPUs: 4, Scratch: 2 * GiB, Name: "nearly"},
+                       "best":   {Price: 3.3, RAM: 4000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "best"},
+                       "costly": {Price: 4.4, RAM: 4000000000, VCPUs: 8, Scratch: 2 * GiB, Name: "costly"},
+               },
        } {
-               best, err := ChooseInstanceType(&arvados.Cluster{InstanceTypes: menu}, &arvados.Container{
+               best, err := ChooseInstanceType(&arvados.Cluster{InstanceTypes: menu, Containers: arvados.ContainersConfig{ReserveExtraRAM: 268435456}}, &arvados.Container{
                        Mounts: map[string]arvados.Mount{
                                "/tmp": {Kind: "tmp", Capacity: 2 * int64(GiB)},
                        },

diff --git a/tools/compute-images/arvados-images-aws.json b/tools/compute-images/arvados-images-aws.json
index b83207bd28286ccceac19190c985124c9b635029..6a1c45da2676d903f03155c92f1ccfd13bb09f41 100644 (file)
--- a/tools/compute-images/arvados-images-aws.json
+++ b/tools/compute-images/arvados-images-aws.json
@@ -6,7 +6,6 @@
     "build_environment": "aws",
     "arvados_cluster": "",
     "aws_source_ami": "ami-04d70e069399af2e9",
-    "fqdn": "",
     "ssh_user": "admin",
     "vpc_id": "",
     "subnet_id": "",
@@ -75,6 +74,6 @@
     "type": "shell",
     "execute_command": "sudo -S env {{ .Vars }} /bin/bash '{{ .Path }}'",
     "script": "scripts/base.sh",
-    "environment_vars": ["ROLE=compute","RESOLVER={{user `resolver`}}","REPOSUFFIX={{user `reposuffix`}}"]
+    "environment_vars": ["RESOLVER={{user `resolver`}}","REPOSUFFIX={{user `reposuffix`}}"]
   }]
 }

diff --git a/tools/compute-images/arvados-images-azure.json b/tools/compute-images/arvados-images-azure.json
index c8db9499cda716240ed8ba421d9f72d496a0bf27..a0278d515af8c5d9515dc2f5833f6310ae9e8149 100644 (file)
--- a/tools/compute-images/arvados-images-azure.json
+++ b/tools/compute-images/arvados-images-azure.json
@@ -14,7 +14,6 @@
     "arvados_cluster": "",
     "project_id": "",
     "account_file": "",
-    "fqdn": "",
     "resolver": "",
     "reposuffix": "",
     "public_key_file": ""
@@ -66,6 +65,6 @@
     "type": "shell",
     "execute_command": "sudo -S env {{ .Vars }} /bin/bash '{{ .Path }}'",
     "script": "scripts/base.sh",
-    "environment_vars": ["ROLE=compute","RESOLVER={{user `resolver`}}","REPOSUFFIX={{user `reposuffix`}}"]
+    "environment_vars": ["RESOLVER={{user `resolver`}}","REPOSUFFIX={{user `reposuffix`}}"]
   }]
 }

diff --git a/tools/compute-images/build.sh b/tools/compute-images/build.sh
index 030eb410b8d52fcf7c1e72e2a8be79c0af90bf7d..fb02ce944210c852b5e9d6cc3c3919d2abc7645d 100755 (executable)
--- a/tools/compute-images/build.sh
+++ b/tools/compute-images/build.sh
@@ -49,8 +49,6 @@ Options:
       Azure SKU image to use
   --ssh_user  (default: packer)
       The user packer will use to log into the image
-  --domain  (default: arvadosapi.com)
-      The domain part of the FQDN for the cluster
   --resolver (default: 8.8.8.8)
       The dns resolver for the machine
   --reposuffix (default: unset)
@@ -78,12 +76,11 @@ AZURE_LOCATION=
 AZURE_CLOUD_ENVIRONMENT=
 DEBUG=
 SSH_USER=
-DOMAIN="arvadosapi.com"
 AWS_DEFAULT_REGION=us-east-1
 PUBLIC_KEY_FILE=
 
 PARSEDOPTS=$(getopt --name "$0" --longoptions \
-    help,json-file:,arvados-cluster-id:,aws-source-ami:,aws-profile:,aws-secrets-file:,aws-region:,aws-vpc-id:,aws-subnet-id:,gcp-project-id:,gcp-account-file:,gcp-zone:,azure-secrets-file:,azure-resource-group:,azure-location:,azure-sku:,azure-cloud-environment:,ssh_user:,domain:,resolver:,reposuffix:,public-key-file:,debug \
+    help,json-file:,arvados-cluster-id:,aws-source-ami:,aws-profile:,aws-secrets-file:,aws-region:,aws-vpc-id:,aws-subnet-id:,gcp-project-id:,gcp-account-file:,gcp-zone:,azure-secrets-file:,azure-resource-group:,azure-location:,azure-sku:,azure-cloud-environment:,ssh_user:,resolver:,reposuffix:,public-key-file:,debug \
     -- "" "$@")
 if [ $? -ne 0 ]; then
     exit 1
@@ -148,9 +145,6 @@ while [ $# -gt 0 ]; do
         --ssh_user)
             SSH_USER="$2"; shift
             ;;
-        --domain)
-            DOMAIN="$2"; shift
-            ;;
         --resolver)
             RESOLVER="$2"; shift
             ;;
@@ -211,7 +205,6 @@ if [[ ! -z "$AZURE_SECRETS_FILE" ]]; then
   source $AZURE_SECRETS_FILE
 fi
 
-FQDN=" -var fqdn=compute.$ARVADOS_CLUSTER_ID.$DOMAIN ";
 
 EXTRA2=""
 
@@ -264,5 +257,5 @@ if [[ "$PUBLIC_KEY_FILE" != "" ]]; then
   EXTRA2+=" -var public_key_file=$PUBLIC_KEY_FILE"
 fi
 
-echo packer build$EXTRA$FQDN -var "role=$role" -var "arvados_cluster=$ARVADOS_CLUSTER_ID"$EXTRA2 $JSON_FILE
-packer build$EXTRA$FQDN -var "role=$role" -var "arvados_cluster=$ARVADOS_CLUSTER_ID"$EXTRA2 $JSON_FILE
+echo packer build$EXTRA -var "arvados_cluster=$ARVADOS_CLUSTER_ID"$EXTRA2 $JSON_FILE
+packer build$EXTRA -var "arvados_cluster=$ARVADOS_CLUSTER_ID"$EXTRA2 $JSON_FILE