services/api/test/unit/fail_jobs_test.rb

   1 require 'test_helper'
   2 require 'crunch_dispatch'
   3
   4 class FailJobsTest < ActiveSupport::TestCase
   5   include DbCurrentTime
   6
   7   BOOT_TIME = 1448378837
   8
   9   setup do
  10     @job = {}
  11     act_as_user users(:admin) do
  12       @job[:before_reboot] = Job.create!(state: 'Running',
  13                                          running: true,
  14                                          started_at: Time.at(BOOT_TIME - 300))
  15       @job[:after_reboot] = Job.create!(state: 'Running',
  16                                         running: true,
  17                                         started_at: Time.at(BOOT_TIME + 300))
  18       @job[:complete] = Job.create!(state: 'Running',
  19                                     running: true,
  20                                     started_at: Time.at(BOOT_TIME - 300))
  21       @job[:complete].update_attributes(state: 'Complete')
  22       @job[:complete].update_attributes(finished_at: Time.at(BOOT_TIME + 100))
  23       @job[:queued] = jobs(:queued)
  24
  25       @job.values.each do |job|
  26         # backdate timestamps
  27         Job.where(uuid: job.uuid).
  28           update_all(created_at: Time.at(BOOT_TIME - 330),
  29                      modified_at: (job.finished_at ||
  30                                    job.started_at ||
  31                                    Time.at(BOOT_TIME - 300)))
  32       end
  33     end
  34     @dispatch = CrunchDispatch.new
  35     @test_start_time = db_current_time
  36   end
  37
  38   test 'cancel slurm jobs' do
  39     Rails.configuration.crunch_job_wrapper = :slurm_immediate
  40     Rails.configuration.crunch_job_user = 'foobar'
  41     fake_squeue = IO.popen("echo #{@job[:before_reboot].uuid}")
  42     fake_scancel = IO.popen("true")
  43     IO.expects(:popen).
  44       with(['squeue', '-a', '-h', '-o', '%j']).
  45       returns(fake_squeue)
  46     IO.expects(:popen).
  47       with(includes('sudo', '-u', 'foobar', 'scancel', '-n', @job[:before_reboot].uuid)).
  48       returns(fake_scancel)
  49     @dispatch.fail_jobs(before: Time.at(BOOT_TIME).to_s)
  50     assert_end_states
  51   end
  52
  53   test 'use reboot time' do
  54     Rails.configuration.crunch_job_wrapper = nil
  55     @dispatch.expects(:open).once.with('/proc/stat').
  56       returns open(Rails.root.join('test/fixtures/files/proc_stat'))
  57     @dispatch.fail_jobs(before: 'reboot')
  58     assert_end_states
  59   end
  60
  61   test 'command line help' do
  62     cmd = Rails.root.join('script/fail-jobs.rb').to_s
  63     assert_match(/Options:.*--before=/m, File.popen([cmd, '--help']).read)
  64   end
  65
  66   protected
  67
  68   def assert_end_states
  69     @job.values.map(&:reload)
  70     assert_equal 'Failed', @job[:before_reboot].state
  71     assert_equal false, @job[:before_reboot].running
  72     assert_equal false, @job[:before_reboot].success
  73     assert_operator @job[:before_reboot].finished_at, :>=, @test_start_time
  74     assert_operator @job[:before_reboot].finished_at, :<=, db_current_time
  75     assert_equal 'Running', @job[:after_reboot].state
  76     assert_equal 'Complete', @job[:complete].state
  77     assert_equal 'Queued', @job[:queued].state
  78   end
  79 end