services/api/test/unit/fail_jobs_test.rb

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 require 'test_helper'
   6 require 'crunch_dispatch'
   7
   8 class FailJobsTest < ActiveSupport::TestCase
   9   include DbCurrentTime
  10
  11   BOOT_TIME = 1448378837
  12
  13   setup do
  14     @job = {}
  15     act_as_user users(:admin) do
  16       @job[:before_reboot] = Job.create!(state: 'Running',
  17                                          running: true,
  18                                          started_at: Time.at(BOOT_TIME - 300))
  19       @job[:after_reboot] = Job.create!(state: 'Running',
  20                                         running: true,
  21                                         started_at: Time.at(BOOT_TIME + 300))
  22       @job[:complete] = Job.create!(state: 'Running',
  23                                     running: true,
  24                                     started_at: Time.at(BOOT_TIME - 300))
  25       @job[:complete].update_attributes(state: 'Complete')
  26       @job[:complete].update_attributes(finished_at: Time.at(BOOT_TIME + 100))
  27       @job[:queued] = jobs(:queued)
  28
  29       @job.values.each do |job|
  30         # backdate timestamps
  31         Job.where(uuid: job.uuid).
  32           update_all(created_at: Time.at(BOOT_TIME - 330),
  33                      modified_at: (job.finished_at ||
  34                                    job.started_at ||
  35                                    Time.at(BOOT_TIME - 300)))
  36       end
  37     end
  38     @dispatch = CrunchDispatch.new
  39     @test_start_time = db_current_time
  40   end
  41
  42   test 'cancel slurm jobs' do
  43     Rails.configuration.crunch_job_wrapper = :slurm_immediate
  44     Rails.configuration.crunch_job_user = 'foobar'
  45     fake_squeue = IO.popen("echo #{@job[:before_reboot].uuid}")
  46     fake_scancel = IO.popen("true")
  47     IO.expects(:popen).
  48       with(['squeue', '-a', '-h', '-o', '%j']).
  49       returns(fake_squeue)
  50     IO.expects(:popen).
  51       with(includes('sudo', '-u', 'foobar', 'scancel', '-n', @job[:before_reboot].uuid)).
  52       returns(fake_scancel)
  53     @dispatch.fail_jobs(before: Time.at(BOOT_TIME).to_s)
  54     assert_end_states
  55   end
  56
  57   test 'use reboot time' do
  58     Rails.configuration.crunch_job_wrapper = nil
  59     @dispatch.expects(:open).once.with('/proc/stat').
  60       returns open(Rails.root.join('test/fixtures/files/proc_stat'))
  61     @dispatch.fail_jobs(before: 'reboot')
  62     assert_end_states
  63   end
  64
  65   test 'command line help' do
  66     cmd = Rails.root.join('script/fail-jobs.rb').to_s
  67     assert_match(/Options:.*--before=/m, File.popen([cmd, '--help']).read)
  68   end
  69
  70   protected
  71
  72   def assert_end_states
  73     @job.values.map(&:reload)
  74     assert_equal 'Failed', @job[:before_reboot].state
  75     assert_equal false, @job[:before_reboot].running
  76     assert_equal false, @job[:before_reboot].success
  77     assert_operator @job[:before_reboot].finished_at, :>=, @test_start_time
  78     assert_operator @job[:before_reboot].finished_at, :<=, db_current_time
  79     assert_equal 'Running', @job[:after_reboot].state
  80     assert_equal 'Complete', @job[:complete].state
  81     assert_equal 'Queued', @job[:queued].state
  82   end
  83 end