services/api/test/unit/job_test.rb

   1 require 'test_helper'
   2 require 'helpers/git_test_helper'
   3 require 'helpers/docker_migration_helper'
   4
   5 class JobTest < ActiveSupport::TestCase
   6   include DockerMigrationHelper
   7   include GitTestHelper
   8
   9   BAD_COLLECTION = "#{'f' * 32}+0"
  10
  11   setup do
  12     set_user_from_auth :active
  13   end
  14
  15   def job_attrs merge_me={}
  16     # Default (valid) set of attributes, with given overrides
  17     {
  18       script: "hash",
  19       script_version: "master",
  20       repository: "active/foo",
  21     }.merge(merge_me)
  22   end
  23
  24   test "Job without Docker image doesn't get locator" do
  25     job = Job.new job_attrs
  26     assert job.valid?, job.errors.full_messages.to_s
  27     assert_nil job.docker_image_locator
  28   end
  29
  30   { 'name' => [:links, :docker_image_collection_tag, :name],
  31     'hash' => [:links, :docker_image_collection_hash, :name],
  32     'locator' => [:collections, :docker_image, :portable_data_hash],
  33   }.each_pair do |spec_type, (fixture_type, fixture_name, fixture_attr)|
  34     test "Job initialized with Docker image #{spec_type} gets locator" do
  35       image_spec = send(fixture_type, fixture_name).send(fixture_attr)
  36       job = Job.new job_attrs(runtime_constraints:
  37                               {'docker_image' => image_spec})
  38       assert job.valid?, job.errors.full_messages.to_s
  39       assert_equal(collections(:docker_image).portable_data_hash, job.docker_image_locator)
  40     end
  41
  42     test "Job modified with Docker image #{spec_type} gets locator" do
  43       job = Job.new job_attrs
  44       assert job.valid?, job.errors.full_messages.to_s
  45       assert_nil job.docker_image_locator
  46       image_spec = send(fixture_type, fixture_name).send(fixture_attr)
  47       job.runtime_constraints['docker_image'] = image_spec
  48       assert job.valid?, job.errors.full_messages.to_s
  49       assert_equal(collections(:docker_image).portable_data_hash, job.docker_image_locator)
  50     end
  51   end
  52
  53   test "removing a Docker runtime constraint removes the locator" do
  54     image_locator = collections(:docker_image).portable_data_hash
  55     job = Job.new job_attrs(runtime_constraints:
  56                             {'docker_image' => image_locator})
  57     assert job.valid?, job.errors.full_messages.to_s
  58     assert_equal(image_locator, job.docker_image_locator)
  59     job.runtime_constraints = {}
  60     assert job.valid?, job.errors.full_messages.to_s + "after clearing runtime constraints"
  61     assert_nil job.docker_image_locator
  62   end
  63
  64   test "locate a Docker image with a repository + tag" do
  65     image_repo, image_tag =
  66       links(:docker_image_collection_tag2).name.split(':', 2)
  67     job = Job.new job_attrs(runtime_constraints:
  68                             {'docker_image' => image_repo,
  69                               'docker_image_tag' => image_tag})
  70     assert job.valid?, job.errors.full_messages.to_s
  71     assert_equal(collections(:docker_image).portable_data_hash, job.docker_image_locator)
  72   end
  73
  74   test "can't locate a Docker image with a nonexistent tag" do
  75     image_repo = links(:docker_image_collection_tag).name
  76     image_tag = '__nonexistent tag__'
  77     job = Job.new job_attrs(runtime_constraints:
  78                             {'docker_image' => image_repo,
  79                               'docker_image_tag' => image_tag})
  80     assert(job.invalid?, "Job with bad Docker tag valid")
  81   end
  82
  83   [
  84     false,
  85     true
  86   ].each do |use_config|
  87     test "Job with no Docker image uses default docker image when configuration is set #{use_config}" do
  88       default_docker_image = collections(:docker_image)[:portable_data_hash]
  89       Rails.configuration.default_docker_image_for_jobs = default_docker_image if use_config
  90
  91       job = Job.new job_attrs
  92       assert job.valid?, job.errors.full_messages.to_s
  93
  94       if use_config
  95         refute_nil job.docker_image_locator
  96         assert_equal default_docker_image, job.docker_image_locator
  97       else
  98         assert_nil job.docker_image_locator
  99       end
 100     end
 101   end
 102
 103   test "create a job with a disambiguated script_version branch name" do
 104     job = Job.
 105       new(script: "testscript",
 106           script_version: "heads/7387838c69a21827834586cc42b467ff6c63293b",
 107           repository: "active/shabranchnames",
 108           script_parameters: {})
 109     assert(job.save)
 110     assert_equal("abec49829bf1758413509b7ffcab32a771b71e81", job.script_version)
 111   end
 112
 113   test "locate a Docker image with a partial hash" do
 114     image_hash = links(:docker_image_collection_hash).name[0..24]
 115     job = Job.new job_attrs(runtime_constraints:
 116                             {'docker_image' => image_hash})
 117     assert job.valid?, job.errors.full_messages.to_s + " with partial hash #{image_hash}"
 118     assert_equal(collections(:docker_image).portable_data_hash, job.docker_image_locator)
 119   end
 120
 121   { 'name' => 'arvados_test_nonexistent',
 122     'hash' => 'f' * 64,
 123     'locator' => BAD_COLLECTION,
 124   }.each_pair do |spec_type, image_spec|
 125     test "Job validation fails with nonexistent Docker image #{spec_type}" do
 126       job = Job.new job_attrs(runtime_constraints:
 127                               {'docker_image' => image_spec})
 128       assert(job.invalid?, "nonexistent Docker image #{spec_type} was valid")
 129     end
 130   end
 131
 132   test "Job validation fails with non-Docker Collection constraint" do
 133     job = Job.new job_attrs(runtime_constraints:
 134                             {'docker_image' => collections(:foo_file).uuid})
 135     assert(job.invalid?, "non-Docker Collection constraint was valid")
 136   end
 137
 138   test "can create Job with Docker image Collection without Docker links" do
 139     image_uuid = collections(:unlinked_docker_image).portable_data_hash
 140     job = Job.new job_attrs(runtime_constraints: {"docker_image" => image_uuid})
 141     assert(job.valid?, "Job created with unlinked Docker image was invalid")
 142     assert_equal(image_uuid, job.docker_image_locator)
 143   end
 144
 145   def check_attrs_unset(job, attrs)
 146     assert_empty(attrs.each_key.map { |key| job.send(key) }.compact,
 147                  "job has values for #{attrs.keys}")
 148   end
 149
 150   def check_creation_prohibited(attrs)
 151     begin
 152       job = Job.new(job_attrs(attrs))
 153     rescue ActiveModel::MassAssignmentSecurity::Error
 154       # Test passes - expected attribute protection
 155     else
 156       check_attrs_unset(job, attrs)
 157     end
 158   end
 159
 160   def check_modification_prohibited(attrs)
 161     job = Job.new(job_attrs)
 162     attrs.each_pair do |key, value|
 163       assert_raises(NoMethodError) { job.send("{key}=".to_sym, value) }
 164     end
 165     check_attrs_unset(job, attrs)
 166   end
 167
 168   test "can't create Job with Docker image locator" do
 169     check_creation_prohibited(docker_image_locator: BAD_COLLECTION)
 170   end
 171
 172   test "can't assign Docker image locator to Job" do
 173     check_modification_prohibited(docker_image_locator: BAD_COLLECTION)
 174   end
 175
 176   [
 177    {script_parameters: ""},
 178    {script_parameters: []},
 179    {script_parameters: {["foo"] => ["bar"]}},
 180    {runtime_constraints: ""},
 181    {runtime_constraints: []},
 182    {tasks_summary: ""},
 183    {tasks_summary: []},
 184   ].each do |invalid_attrs|
 185     test "validation failures set error messages: #{invalid_attrs.to_json}" do
 186       # Ensure valid_attrs doesn't produce errors -- otherwise we will
 187       # not know whether errors reported below are actually caused by
 188       # invalid_attrs.
 189       Job.new(job_attrs).save!
 190
 191       err = assert_raises(ArgumentError) do
 192         Job.new(job_attrs(invalid_attrs)).save!
 193       end
 194       assert_match /parameters|constraints|summary/, err.message
 195     end
 196   end
 197
 198   test "invalid script_version" do
 199     invalid = {
 200       script_version: "no/branch/could/ever/possibly/have/this/name",
 201     }
 202     err = assert_raises(ActiveRecord::RecordInvalid) do
 203       Job.new(job_attrs(invalid)).save!
 204     end
 205     assert_match /Script version .* does not resolve to a commit/, err.message
 206   end
 207
 208   [
 209     # Each test case is of the following format
 210     # Array of parameters where each parameter is of the format:
 211     #  attr name to be changed, attr value, and array of expectations (where each expectation is an array)
 212     [['running', false, [['state', 'Queued']]]],
 213     [['state', 'Running', [['started_at', 'not_nil']]]],
 214     [['is_locked_by_uuid', 'use_current_user_uuid', [['state', 'Queued']]], ['state', 'Running', [['running', true], ['started_at', 'not_nil'], ['success', 'nil']]]],
 215     [['running', false, [['state', 'Queued']]], ['state', 'Complete', [['success', true]]]],
 216     [['running', true, [['state', 'Running']]], ['cancelled_at', Time.now, [['state', 'Cancelled']]]],
 217     [['running', true, [['state', 'Running']]], ['state', 'Cancelled', [['cancelled_at', 'not_nil']]]],
 218     [['running', true, [['state', 'Running']]], ['success', true, [['state', 'Complete']]]],
 219     [['running', true, [['state', 'Running']]], ['success', false, [['state', 'Failed']]]],
 220     [['running', true, [['state', 'Running']]], ['state', 'Complete', [['success', true],['finished_at', 'not_nil']]]],
 221     [['running', true, [['state', 'Running']]], ['state', 'Failed', [['success', false],['finished_at', 'not_nil']]]],
 222     [['cancelled_at', Time.now, [['state', 'Cancelled']]], ['success', false, [['state', 'Cancelled'],['finished_at', 'nil'], ['cancelled_at', 'not_nil']]]],
 223     [['cancelled_at', Time.now, [['state', 'Cancelled'],['running', false]]], ['success', true, [['state', 'Cancelled'],['running', false],['finished_at', 'nil'],['cancelled_at', 'not_nil']]]],
 224     # potential migration cases
 225     [['state', nil, [['state', 'Queued']]]],
 226     [['state', nil, [['state', 'Queued']]], ['cancelled_at', Time.now, [['state', 'Cancelled']]]],
 227     [['running', true, [['state', 'Running']]], ['state', nil, [['state', 'Running']]]],
 228   ].each do |parameters|
 229     test "verify job status #{parameters}" do
 230       job = Job.create! job_attrs
 231       assert_equal 'Queued', job.state, "job.state"
 232
 233       parameters.each do |parameter|
 234         expectations = parameter[2]
 235         if 'use_current_user_uuid' == parameter[1]
 236           parameter[1] = Thread.current[:user].uuid
 237         end
 238
 239         if expectations.instance_of? Array
 240           job[parameter[0]] = parameter[1]
 241           assert_equal true, job.save, job.errors.full_messages.to_s
 242           expectations.each do |expectation|
 243             if expectation[1] == 'not_nil'
 244               assert_not_nil job[expectation[0]], expectation[0]
 245             elsif expectation[1] == 'nil'
 246               assert_nil job[expectation[0]], expectation[0]
 247             else
 248               assert_equal expectation[1], job[expectation[0]], expectation[0]
 249             end
 250           end
 251         else
 252           raise 'I do not know how to handle this expectation'
 253         end
 254       end
 255     end
 256   end
 257
 258   test "Test job state changes" do
 259     all = ["Queued", "Running", "Complete", "Failed", "Cancelled"]
 260     valid = {"Queued" => all, "Running" => ["Complete", "Failed", "Cancelled"]}
 261     all.each do |start|
 262       all.each do |finish|
 263         if start != finish
 264           job = Job.create! job_attrs(state: start)
 265           assert_equal start, job.state
 266           job.state = finish
 267           job.save
 268           job.reload
 269           if valid[start] and valid[start].include? finish
 270             assert_equal finish, job.state
 271           else
 272             assert_equal start, job.state
 273           end
 274         end
 275       end
 276     end
 277   end
 278
 279   test "Test job locking" do
 280     set_user_from_auth :active_trustedclient
 281     job = Job.create! job_attrs
 282
 283     assert_equal "Queued", job.state
 284
 285     # Should be able to lock successfully
 286     job.lock current_user.uuid
 287     assert_equal "Running", job.state
 288
 289     assert_raises ArvadosModel::AlreadyLockedError do
 290       # Can't lock it again
 291       job.lock current_user.uuid
 292     end
 293     job.reload
 294     assert_equal "Running", job.state
 295
 296     set_user_from_auth :project_viewer
 297     assert_raises ArvadosModel::AlreadyLockedError do
 298       # Can't lock it as a different user either
 299       job.lock current_user.uuid
 300     end
 301     job.reload
 302     assert_equal "Running", job.state
 303
 304     assert_raises ArvadosModel::PermissionDeniedError do
 305       # Can't update fields as a different user
 306       job.update_attributes(state: "Failed")
 307     end
 308     job.reload
 309     assert_equal "Running", job.state
 310
 311
 312     set_user_from_auth :active_trustedclient
 313
 314     # Can update fields as the locked_by user
 315     job.update_attributes(state: "Failed")
 316     assert_equal "Failed", job.state
 317   end
 318
 319   test "admin user can cancel a running job despite lock" do
 320     set_user_from_auth :active_trustedclient
 321     job = Job.create! job_attrs
 322     job.lock current_user.uuid
 323     assert_equal Job::Running, job.state
 324
 325     set_user_from_auth :spectator
 326     assert_raises do
 327       job.update_attributes!(state: Job::Cancelled)
 328     end
 329
 330     set_user_from_auth :admin
 331     job.reload
 332     assert_equal Job::Running, job.state
 333     job.update_attributes!(state: Job::Cancelled)
 334     assert_equal Job::Cancelled, job.state
 335   end
 336
 337   test "verify job queue position" do
 338     job1 = Job.create! job_attrs
 339     assert_equal 'Queued', job1.state, "Incorrect job state for newly created job1"
 340
 341     job2 = Job.create! job_attrs
 342     assert_equal 'Queued', job2.state, "Incorrect job state for newly created job2"
 343
 344     assert_not_nil job1.queue_position, "Expected non-nil queue position for job1"
 345     assert_not_nil job2.queue_position, "Expected non-nil queue position for job2"
 346   end
 347
 348   SDK_MASTER = "ca68b24e51992e790f29df5cc4bc54ce1da4a1c2"
 349   SDK_TAGGED = "00634b2b8a492d6f121e3cf1d6587b821136a9a7"
 350
 351   def sdk_constraint(version)
 352     {runtime_constraints: {
 353         "arvados_sdk_version" => version,
 354         "docker_image" => links(:docker_image_collection_tag).name,
 355       }}
 356   end
 357
 358   def check_job_sdk_version(expected)
 359     job = yield
 360     if expected.nil?
 361       refute(job.valid?, "job valid with bad Arvados SDK version")
 362     else
 363       assert(job.valid?, "job not valid with good Arvados SDK version")
 364       assert_equal(expected, job.arvados_sdk_version)
 365     end
 366   end
 367
 368   { "master" => SDK_MASTER,
 369     "commit2" => SDK_TAGGED,
 370     SDK_TAGGED[0, 8] => SDK_TAGGED,
 371     "__nonexistent__" => nil,
 372   }.each_pair do |search, commit_hash|
 373     test "creating job with SDK version '#{search}'" do
 374       check_job_sdk_version(commit_hash) do
 375         Job.new(job_attrs(sdk_constraint(search)))
 376       end
 377     end
 378
 379     test "updating job from no SDK to version '#{search}'" do
 380       job = Job.create!(job_attrs)
 381       assert_nil job.arvados_sdk_version
 382       check_job_sdk_version(commit_hash) do
 383         job.runtime_constraints = sdk_constraint(search)[:runtime_constraints]
 384         job
 385       end
 386     end
 387
 388     test "updating job from SDK version 'master' to '#{search}'" do
 389       job = Job.create!(job_attrs(sdk_constraint("master")))
 390       assert_equal(SDK_MASTER, job.arvados_sdk_version)
 391       check_job_sdk_version(commit_hash) do
 392         job.runtime_constraints = sdk_constraint(search)[:runtime_constraints]
 393         job
 394       end
 395     end
 396   end
 397
 398   test "clear the SDK version" do
 399     job = Job.create!(job_attrs(sdk_constraint("master")))
 400     assert_equal(SDK_MASTER, job.arvados_sdk_version)
 401     job.runtime_constraints = {}
 402     assert(job.valid?, "job invalid after clearing SDK version")
 403     assert_nil(job.arvados_sdk_version)
 404   end
 405
 406   test "job with SDK constraint, without Docker image is invalid" do
 407     sdk_attrs = sdk_constraint("master")
 408     sdk_attrs[:runtime_constraints].delete("docker_image")
 409     job = Job.create(job_attrs(sdk_attrs))
 410     refute(job.valid?, "Job valid with SDK version, without Docker image")
 411     sdk_errors = job.errors.messages[:arvados_sdk_version] || []
 412     refute_empty(sdk_errors.grep(/\bDocker\b/),
 413                  "no Job SDK errors mention that Docker is required")
 414   end
 415
 416   test "invalid to clear Docker image constraint when SDK constraint exists" do
 417     job = Job.create!(job_attrs(sdk_constraint("master")))
 418     job.runtime_constraints.delete("docker_image")
 419     refute(job.valid?,
 420            "Job with SDK constraint valid after clearing Docker image")
 421   end
 422
 423   test "use migrated docker image if requesting old-format image by tag" do
 424     Rails.configuration.docker_image_formats = ['v2']
 425     add_docker19_migration_link
 426     job = Job.create!(
 427       job_attrs(
 428         script: 'foo',
 429         runtime_constraints: {
 430           'docker_image' => links(:docker_image_collection_tag).name}))
 431     assert(job.valid?)
 432     assert_equal(job.docker_image_locator, collections(:docker_image_1_12).portable_data_hash)
 433   end
 434
 435   test "use migrated docker image if requesting old-format image by pdh" do
 436     Rails.configuration.docker_image_formats = ['v2']
 437     add_docker19_migration_link
 438     job = Job.create!(
 439       job_attrs(
 440         script: 'foo',
 441         runtime_constraints: {
 442           'docker_image' => collections(:docker_image).portable_data_hash}))
 443     assert(job.valid?)
 444     assert_equal(job.docker_image_locator, collections(:docker_image_1_12).portable_data_hash)
 445   end
 446
 447   [[:docker_image, :docker_image, :docker_image_1_12],
 448    [:docker_image_1_12, :docker_image, :docker_image_1_12],
 449    [:docker_image, :docker_image_1_12, :docker_image_1_12],
 450    [:docker_image_1_12, :docker_image_1_12, :docker_image_1_12],
 451   ].each do |existing_image, request_image, expect_image|
 452     test "if a #{existing_image} job exists, #{request_image} yields #{expect_image} after migration" do
 453       Rails.configuration.docker_image_formats = ['v1']
 454
 455       if existing_image == :docker_image
 456         oldjob = Job.create!(
 457           job_attrs(
 458             script: 'foobar1',
 459             runtime_constraints: {
 460               'docker_image' => collections(existing_image).portable_data_hash}))
 461         oldjob.reload
 462         assert_equal(oldjob.docker_image_locator,
 463                      collections(existing_image).portable_data_hash)
 464       elsif existing_image == :docker_image_1_12
 465         assert_raises(ActiveRecord::RecordInvalid,
 466                       "Should not resolve v2 image when only v1 is supported") do
 467         oldjob = Job.create!(
 468           job_attrs(
 469             script: 'foobar1',
 470             runtime_constraints: {
 471               'docker_image' => collections(existing_image).portable_data_hash}))
 472         end
 473       end
 474
 475       Rails.configuration.docker_image_formats = ['v2']
 476       add_docker19_migration_link
 477
 478       # Check that both v1 and v2 images get resolved to v2.
 479       newjob = Job.create!(
 480         job_attrs(
 481           script: 'foobar1',
 482           runtime_constraints: {
 483             'docker_image' => collections(request_image).portable_data_hash}))
 484       newjob.reload
 485       assert_equal(newjob.docker_image_locator,
 486                    collections(expect_image).portable_data_hash)
 487     end
 488   end
 489
 490   test "can't create job with SDK version assigned directly" do
 491     check_creation_prohibited(arvados_sdk_version: SDK_MASTER)
 492   end
 493
 494   test "can't modify job to assign SDK version directly" do
 495     check_modification_prohibited(arvados_sdk_version: SDK_MASTER)
 496   end
 497
 498   test "job validation fails when collection uuid found in script_parameters" do
 499     bad_params = {
 500       script_parameters: {
 501         'input' => {
 502           'param1' => 'the collection uuid zzzzz-4zz18-012345678901234'
 503         }
 504       }
 505     }
 506     assert_raises(ActiveRecord::RecordInvalid,
 507                   "created job with a collection uuid in script_parameters") do
 508       Job.create!(job_attrs(bad_params))
 509     end
 510   end
 511
 512   test "job validation succeeds when no collection uuid in script_parameters" do
 513     good_params = {
 514       script_parameters: {
 515         'arg1' => 'foo',
 516         'arg2' => [ 'bar', 'baz' ],
 517         'arg3' => {
 518           'a' => 1,
 519           'b' => [2, 3, 4],
 520         }
 521       }
 522     }
 523     job = Job.create!(job_attrs(good_params))
 524     assert job.valid?
 525   end
 526
 527   test 'update job uuid tag in internal.git when version changes' do
 528     authorize_with :active
 529     j = jobs :queued
 530     j.update_attributes repository: 'active/foo', script_version: 'b1'
 531     assert_equal('1de84a854e2b440dc53bf42f8548afa4c17da332',
 532                  internal_tag(j.uuid))
 533     j.update_attributes repository: 'active/foo', script_version: 'master'
 534     assert_equal('077ba2ad3ea24a929091a9e6ce545c93199b8e57',
 535                  internal_tag(j.uuid))
 536   end
 537
 538   test 'script_parameters_digest is independent of key order' do
 539     j1 = Job.new(job_attrs(script_parameters: {'a' => 'a', 'ddee' => {'d' => 'd', 'e' => 'e'}}))
 540     j2 = Job.new(job_attrs(script_parameters: {'ddee' => {'e' => 'e', 'd' => 'd'}, 'a' => 'a'}))
 541     assert j1.valid?
 542     assert j2.valid?
 543     assert_equal(j1.script_parameters_digest, j2.script_parameters_digest)
 544   end
 545
 546   test 'job fixtures have correct script_parameters_digest' do
 547     Job.all.each do |j|
 548       d = j.script_parameters_digest
 549       assert_equal(j.update_script_parameters_digest, d,
 550                    "wrong script_parameters_digest for #{j.uuid}")
 551     end
 552   end
 553
 554   test 'deep_sort_hash on array of hashes' do
 555     a = {'z' => [[{'a' => 'a', 'b' => 'b'}]]}
 556     b = {'z' => [[{'b' => 'b', 'a' => 'a'}]]}
 557     assert_equal Job.deep_sort_hash(a).to_json, Job.deep_sort_hash(b).to_json
 558   end
 559
 560   test 'find_reusable without logging' do
 561     Rails.logger.expects(:info).never
 562     try_find_reusable
 563   end
 564
 565   test 'find_reusable with logging' do
 566     Rails.configuration.log_reuse_decisions = true
 567     Rails.logger.expects(:info).at_least(3)
 568     try_find_reusable
 569   end
 570
 571   def try_find_reusable
 572     foobar = jobs(:foobar)
 573     example_attrs = {
 574       script_version: foobar.script_version,
 575       script: foobar.script,
 576       script_parameters: foobar.script_parameters,
 577       repository: foobar.repository,
 578     }
 579
 580     # Two matching jobs exist with identical outputs. The older one
 581     # should be reused.
 582     j = Job.find_reusable(example_attrs, {}, [], [users(:active)])
 583     assert j
 584     assert_equal foobar.uuid, j.uuid
 585
 586     # Two matching jobs exist with different outputs. Neither should
 587     # be reused.
 588     Job.where(uuid: jobs(:job_with_latest_version).uuid).
 589       update_all(output: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa+1')
 590     assert_nil Job.find_reusable(example_attrs, {}, [], [users(:active)])
 591
 592     # ...unless config says to reuse the earlier job in such cases.
 593     Rails.configuration.reuse_job_if_outputs_differ = true
 594     j = Job.find_reusable(example_attrs, {}, [], [users(:active)])
 595     assert_equal foobar.uuid, j.uuid
 596   end
 597
 598   [
 599     true,
 600     false,
 601   ].each do |cascade|
 602     test "cancel job with cascade #{cascade}" do
 603       job = Job.find_by_uuid jobs(:running_job_with_components_at_level_1).uuid
 604       job.cancel cascade: cascade
 605       assert_equal Job::Cancelled, job.state
 606
 607       descendents = ['zzzzz-8i9sb-jobcomponentsl2',
 608                      'zzzzz-d1hrv-picomponentsl02',
 609                      'zzzzz-8i9sb-job1atlevel3noc',
 610                      'zzzzz-8i9sb-job2atlevel3noc']
 611
 612       jobs = Job.where(uuid: descendents)
 613       jobs.each do |j|
 614         assert_equal ('Cancelled' == j.state), cascade
 615       end
 616
 617       pipelines = PipelineInstance.where(uuid: descendents)
 618       pipelines.each do |pi|
 619         assert_equal ('Paused' == pi.state), cascade
 620       end
 621     end
 622   end
 623
 624   test 'cancelling a completed job raises error' do
 625     job = Job.find_by_uuid jobs(:job_with_latest_version).uuid
 626     assert job
 627     assert_equal 'Complete', job.state
 628
 629     assert_raises(ArvadosModel::InvalidStateTransitionError) do
 630       job.cancel
 631     end
 632   end
 633
 634   test 'cancelling a job with circular relationship with another does not result in an infinite loop' do
 635     job = Job.find_by_uuid jobs(:running_job_2_with_circular_component_relationship).uuid
 636
 637     job.cancel cascade: true
 638
 639     assert_equal Job::Cancelled, job.state
 640
 641     child = Job.find_by_uuid job.components.collect{|_, uuid| uuid}[0]
 642     assert_equal Job::Cancelled, child.state
 643   end
 644 end