X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/eda00143401effca0bbc0b2f6e4fde8ee2ede8a0..4139c5dfde60cdd20d39f385ca36107b0c44906f:/services/api/test/unit/crunch_dispatch_test.rb

diff --git a/services/api/test/unit/crunch_dispatch_test.rb b/services/api/test/unit/crunch_dispatch_test.rb
index a4503a845e..3a8f90a66b 100644
--- a/services/api/test/unit/crunch_dispatch_test.rb
+++ b/services/api/test/unit/crunch_dispatch_test.rb
@@ -1,7 +1,14 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
 require 'test_helper'
 require 'crunch_dispatch'
+require 'helpers/git_test_helper'
 
 class CrunchDispatchTest < ActiveSupport::TestCase
+  include GitTestHelper
+
   test 'choose cheaper nodes first' do
     act_as_system_user do
       # Replace test fixtures with a set suitable for testing dispatch
@@ -56,7 +63,8 @@ class CrunchDispatchTest < ActiveSupport::TestCase
      [2, 16384, ['compute2', 'compute1']],
      [2, 8000, ['compute4', 'compute3']],
     ].each do |min_nodes, min_ram, expect_nodes|
-      job = Job.new(runtime_constraints: {
+      job = Job.new(uuid: 'zzzzz-8i9sb-382lhiizavzhqlp',
+                    runtime_constraints: {
                       'min_nodes' => min_nodes,
                       'min_ram_mb_per_node' => min_ram,
                     })
@@ -71,17 +79,6 @@ class CrunchDispatchTest < ActiveSupport::TestCase
     begin
       pid = Process.fork do
         begin
-          # Abandon database connections inherited from parent
-          # process.  Credit to
-          # https://github.com/kstephens/rails_is_forked
-          ActiveRecord::Base.connection_handler.connection_pools.each_value do |pool|
-            pool.instance_eval do
-              @reserved_connections = {}
-              @connections = []
-            end
-          end
-          ActiveRecord::Base.establish_connection
-
           dispatch = CrunchDispatch.new
           dispatch.stubs(:did_recently).returns true
           dispatch.run []
@@ -100,6 +97,30 @@ class CrunchDispatchTest < ActiveSupport::TestCase
     end
   end
 
+  test 'override --cgroup-root with CRUNCH_CGROUP_ROOT' do
+    ENV['CRUNCH_CGROUP_ROOT'] = '/path/to/cgroup'
+    Rails.configuration.Containers.JobsAPI.CrunchJobWrapper = "none"
+    act_as_system_user do
+      j = Job.create(repository: 'active/foo',
+                     script: 'hash',
+                     script_version: '4fe459abe02d9b365932b8f5dc419439ab4e2577',
+                     script_parameters: {})
+      ok = false
+      Open3.expects(:popen3).at_least_once.with do |*args|
+        if args.index(j.uuid)
+          ok = ((i = args.index '--cgroup-root') and
+                (args[i+1] == '/path/to/cgroup'))
+        end
+        true
+      end.raises(StandardError.new('all is well'))
+      dispatch = CrunchDispatch.new
+      dispatch.parse_argv ['--jobs']
+      dispatch.refresh_todo
+      dispatch.start_jobs
+      assert ok
+    end
+  end
+
   def assert_with_timeout timeout, message
     t = 0
     while (t += 0.1) < timeout
@@ -119,7 +140,7 @@ class CrunchDispatchTest < ActiveSupport::TestCase
 
   test 'rate limit of partial line segments' do
     act_as_system_user do
-      Rails.configuration.crunch_log_partial_line_throttle_period = 1
+      Rails.configuration.Containers.Logging.LogPartialLineThrottlePeriod = 1
 
       job = {}
       job[:bytes_logged] = 0
@@ -152,12 +173,6 @@ class CrunchDispatchTest < ActiveSupport::TestCase
       assert_equal false, limit
       assert_equal 2, job[:log_throttle_lines_so_far]
 
-      # crunchstat partial line segment is also skipped
-      line = "#{now} localhost 100 0 stderr crunchstat [...] second partial line segment within the interval [...]"
-      limit = dispatch.rate_limit(job, line)
-      assert_equal false, limit
-      assert_equal 2, job[:log_throttle_lines_so_far]
-
       # next partial line after interval is counted towards skipped lines
       sleep(1)
       line = "#{now} localhost 100 0 stderr [...] third partial line segment after the interval [...]"
@@ -180,4 +195,24 @@ class CrunchDispatchTest < ActiveSupport::TestCase
       assert_equal 5, job[:log_throttle_lines_so_far]
     end
   end
+
+  test 'scancel orphaned job nodes' do
+    Rails.configuration.Containers.JobsAPI.CrunchJobWrapper = "slurm_immediate"
+    act_as_system_user do
+      dispatch = CrunchDispatch.new
+
+      squeue_resp = IO.popen("echo zzzzz-8i9sb-pshmckwoma9plh7\necho thisisnotvalidjobuuid\necho zzzzz-8i9sb-4cf0abc123e809j\necho zzzzz-dz642-o04e3r651turtdr\n")
+      scancel_resp = IO.popen("true")
+
+      IO.expects(:popen).
+        with(['squeue', '-a', '-h', '-o', '%j']).
+        returns(squeue_resp)
+
+      IO.expects(:popen).
+        with(dispatch.sudo_preface + ['scancel', '-n', 'zzzzz-8i9sb-4cf0abc123e809j']).
+        returns(scancel_resp)
+
+      dispatch.check_orphaned_slurm_jobs
+    end
+  end
 end