2 require 'crunch_dispatch'
4 class CrunchDispatchTest < ActiveSupport::TestCase
5 test 'choose cheaper nodes first' do
7 # Replace test fixtures with a set suitable for testing dispatch
10 # Idle nodes with different prices
11 [['compute1', 3.20, 32],
12 ['compute2', 1.60, 16],
13 ['compute3', 0.80, 8]].each do |hostname, price, cores|
14 Node.create!(hostname: hostname,
16 'slurm_state' => 'idle',
22 'total_cpu_cores' => cores,
23 'total_ram_mb' => cores*1024,
24 'total_scratch_mb' => cores*10000,
28 # Node with no price information
29 Node.create!(hostname: 'compute4',
31 'slurm_state' => 'idle',
34 'total_cpu_cores' => 8,
35 'total_ram_mb' => 8192,
36 'total_scratch_mb' => 80000,
40 Node.create!(hostname: 'compute5',
42 'slurm_state' => 'alloc',
48 'total_cpu_cores' => 32,
49 'total_ram_mb' => 32768,
50 'total_scratch_mb' => 320000,
54 dispatch = CrunchDispatch.new
55 [[1, 16384, ['compute2']],
56 [2, 16384, ['compute2', 'compute1']],
57 [2, 8000, ['compute4', 'compute3']],
58 ].each do |min_nodes, min_ram, expect_nodes|
59 job = Job.new(runtime_constraints: {
60 'min_nodes' => min_nodes,
61 'min_ram_mb_per_node' => min_ram,
63 nodes = dispatch.nodes_available_for_job_now job
64 assert_equal expect_nodes, nodes
68 test 'respond to TERM' do
69 lockfile = Rails.root.join 'tmp', 'dispatch.lock'
70 ENV['CRUNCH_DISPATCH_LOCKFILE'] = lockfile.to_s
74 # Abandon database connections inherited from parent
76 # https://github.com/kstephens/rails_is_forked
77 ActiveRecord::Base.connection_handler.connection_pools.each_value do |pool|
79 @reserved_connections = {}
83 ActiveRecord::Base.establish_connection
85 dispatch = CrunchDispatch.new
86 dispatch.stubs(:did_recently).returns true
92 assert_with_timeout 5, "Dispatch did not lock #{lockfile}" do
96 Process.kill("TERM", pid)
98 assert_with_timeout 20, "Dispatch did not unlock #{lockfile}" do
103 def assert_with_timeout timeout, message
105 while (t += 0.1) < timeout
111 assert false, message + " (waited #{timeout} seconds)"
114 def can_lock lockfile
115 lockfile.open(File::RDWR|File::CREAT, 0644) do |f|
116 return f.flock(File::LOCK_EX|File::LOCK_NB)
120 test 'rate limit of partial line segments' do
121 act_as_system_user do
122 Rails.configuration.crunch_log_partial_line_throttle_period = 1
125 job[:bytes_logged] = 0
126 job[:log_throttle_bytes_so_far] = 0
127 job[:log_throttle_lines_so_far] = 0
128 job[:log_throttle_bytes_skipped] = 0
129 job[:log_throttle_is_open] = true
130 job[:log_throttle_partial_line_last_at] = Time.new(0)
131 job[:log_throttle_first_partial_line] = true
133 dispatch = CrunchDispatch.new
135 line = "first log line"
136 limit = dispatch.rate_limit(job, line)
137 assert_equal true, limit
138 assert_equal "first log line", line
139 assert_equal 1, job[:log_throttle_lines_so_far]
141 # first partial line segment is skipped and counted towards skipped lines
142 now = Time.now.strftime('%Y-%m-%d-%H:%M:%S')
143 line = "#{now} localhost 100 0 stderr [...] this is first partial line segment [...]"
144 limit = dispatch.rate_limit(job, line)
145 assert_equal true, limit
146 assert_includes line, "Rate-limiting partial segments of long lines", line
147 assert_equal 2, job[:log_throttle_lines_so_far]
149 # next partial line segment within throttle interval is skipped but not counted towards skipped lines
150 line = "#{now} localhost 100 0 stderr [...] second partial line segment within the interval [...]"
151 limit = dispatch.rate_limit(job, line)
152 assert_equal false, limit
153 assert_equal 2, job[:log_throttle_lines_so_far]
155 # crunchstat partial line segment is also skipped
156 line = "#{now} localhost 100 0 stderr crunchstat [...] second partial line segment within the interval [...]"
157 limit = dispatch.rate_limit(job, line)
158 assert_equal false, limit
159 assert_equal 2, job[:log_throttle_lines_so_far]
161 # next partial line after interval is counted towards skipped lines
163 line = "#{now} localhost 100 0 stderr [...] third partial line segment after the interval [...]"
164 limit = dispatch.rate_limit(job, line)
165 assert_equal false, limit
166 assert_equal 3, job[:log_throttle_lines_so_far]
168 # this is not a valid line segment
169 line = "#{now} localhost 100 0 stderr [...] does not end with [...] and is not a partial segment"
170 limit = dispatch.rate_limit(job, line)
171 assert_equal true, limit
172 assert_equal "#{now} localhost 100 0 stderr [...] does not end with [...] and is not a partial segment", line
173 assert_equal 4, job[:log_throttle_lines_so_far]
175 # this also is not a valid line segment
176 line = "#{now} localhost 100 0 stderr does not start correctly but ends with [...]"
177 limit = dispatch.rate_limit(job, line)
178 assert_equal true, limit
179 assert_equal "#{now} localhost 100 0 stderr does not start correctly but ends with [...]", line
180 assert_equal 5, job[:log_throttle_lines_so_far]