5642: crunch-job uses higher memory limit when running few tasks.

This gives a task all of the node's RAM when it's the only task
running this round, which is more in line with user expectations.

diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job
index 912159a36f1e4659fcd1d655186563dcb8e2c0fa..a6ccebcd91dd4d5f46221a575d1a1d4e0cb4da6e 100755 (executable)
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@ -659,6 +659,10 @@ update_progress_stats();
 
 
 THISROUND:
+my $tasks_this_level = 0;
+foreach my $id (@jobstep_todo) {
+  $tasks_this_level++ if ($jobstep[$id]->{level} == $level);
+}
 for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
 {
   my $id = $jobstep_todo[$todo_ptr];
@@ -716,6 +720,11 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
 
     $ENV{"GZIP"} = "-n";
 
+    my $max_node_concurrent_tasks = $ENV{CRUNCH_NODE_SLOTS};
+    if ($tasks_this_level < $max_node_concurrent_tasks) {
+      $max_node_concurrent_tasks = $tasks_this_level;
+    }
+
     my @srunargs = (
       "srun",
       "--nodelist=".$childnode->{name},
@@ -730,7 +739,7 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
         # $command.  No tool is expected to read these values directly.
         .q{&& MEM=$(awk '($1 == "MemTotal:"){print $2}' </proc/meminfo) }
         .q{&& SWAP=$(awk '($1 == "SwapTotal:"){print $2}' </proc/meminfo) }
-        ."&& MEMLIMIT=\$(( (\$MEM * 95) / ($ENV{CRUNCH_NODE_SLOTS} * 100) )) "
+        ."&& MEMLIMIT=\$(( (\$MEM * 95) / ($max_node_concurrent_tasks * 100) )) "
         ."&& let SWAPLIMIT=\$MEMLIMIT+\$SWAP ";
     $command .= "&& exec arv-mount --by-id --allow-other $ENV{TASK_KEEPMOUNT} --exec ";
     if ($docker_hash)