services/api/test/unit/job_test.rb

   1 require 'test_helper'
   2 require 'helpers/git_test_helper'
   3 require 'helpers/docker_migration_helper'
   4
   5 class JobTest < ActiveSupport::TestCase
   6   include DockerMigrationHelper
   7   include GitTestHelper
   8
   9   BAD_COLLECTION = "#{'f' * 32}+0"
  10
  11   setup do
  12     set_user_from_auth :active
  13   end
  14
  15   def job_attrs merge_me={}
  16     # Default (valid) set of attributes, with given overrides
  17     {
  18       script: "hash",
  19       script_version: "master",
  20       repository: "active/foo",
  21     }.merge(merge_me)
  22   end
  23
  24   test "Job without Docker image doesn't get locator" do
  25     job = Job.new job_attrs
  26     assert job.valid?, job.errors.full_messages.to_s
  27     assert_nil job.docker_image_locator
  28   end
  29
  30   { 'name' => [:links, :docker_image_collection_tag, :name],
  31     'hash' => [:links, :docker_image_collection_hash, :name],
  32     'locator' => [:collections, :docker_image, :portable_data_hash],
  33   }.each_pair do |spec_type, (fixture_type, fixture_name, fixture_attr)|
  34     test "Job initialized with Docker image #{spec_type} gets locator" do
  35       image_spec = send(fixture_type, fixture_name).send(fixture_attr)
  36       job = Job.new job_attrs(runtime_constraints:
  37                               {'docker_image' => image_spec})
  38       assert job.valid?, job.errors.full_messages.to_s
  39       assert_equal(collections(:docker_image).portable_data_hash, job.docker_image_locator)
  40     end
  41
  42     test "Job modified with Docker image #{spec_type} gets locator" do
  43       job = Job.new job_attrs
  44       assert job.valid?, job.errors.full_messages.to_s
  45       assert_nil job.docker_image_locator
  46       image_spec = send(fixture_type, fixture_name).send(fixture_attr)
  47       job.runtime_constraints['docker_image'] = image_spec
  48       assert job.valid?, job.errors.full_messages.to_s
  49       assert_equal(collections(:docker_image).portable_data_hash, job.docker_image_locator)
  50     end
  51   end
  52
  53   test "removing a Docker runtime constraint removes the locator" do
  54     image_locator = collections(:docker_image).portable_data_hash
  55     job = Job.new job_attrs(runtime_constraints:
  56                             {'docker_image' => image_locator})
  57     assert job.valid?, job.errors.full_messages.to_s
  58     assert_equal(image_locator, job.docker_image_locator)
  59     job.runtime_constraints = {}
  60     assert job.valid?, job.errors.full_messages.to_s + "after clearing runtime constraints"
  61     assert_nil job.docker_image_locator
  62   end
  63
  64   test "locate a Docker image with a repository + tag" do
  65     image_repo, image_tag =
  66       links(:docker_image_collection_tag2).name.split(':', 2)
  67     job = Job.new job_attrs(runtime_constraints:
  68                             {'docker_image' => image_repo,
  69                               'docker_image_tag' => image_tag})
  70     assert job.valid?, job.errors.full_messages.to_s
  71     assert_equal(collections(:docker_image).portable_data_hash, job.docker_image_locator)
  72   end
  73
  74   test "can't locate a Docker image with a nonexistent tag" do
  75     image_repo = links(:docker_image_collection_tag).name
  76     image_tag = '__nonexistent tag__'
  77     job = Job.new job_attrs(runtime_constraints:
  78                             {'docker_image' => image_repo,
  79                               'docker_image_tag' => image_tag})
  80     assert(job.invalid?, "Job with bad Docker tag valid")
  81   end
  82
  83   [
  84     false,
  85     true
  86   ].each do |use_config|
  87     test "Job with no Docker image uses default docker image when configuration is set #{use_config}" do
  88       default_docker_image = collections(:docker_image)[:portable_data_hash]
  89       Rails.configuration.default_docker_image_for_jobs = default_docker_image if use_config
  90
  91       job = Job.new job_attrs
  92       assert job.valid?, job.errors.full_messages.to_s
  93
  94       if use_config
  95         refute_nil job.docker_image_locator
  96         assert_equal default_docker_image, job.docker_image_locator
  97       else
  98         assert_nil job.docker_image_locator
  99       end
 100     end
 101   end
 102
 103   test "create a job with a disambiguated script_version branch name" do
 104     job = Job.
 105       new(script: "testscript",
 106           script_version: "heads/7387838c69a21827834586cc42b467ff6c63293b",
 107           repository: "active/shabranchnames",
 108           script_parameters: {})
 109     assert(job.save)
 110     assert_equal("abec49829bf1758413509b7ffcab32a771b71e81", job.script_version)
 111   end
 112
 113   test "locate a Docker image with a partial hash" do
 114     image_hash = links(:docker_image_collection_hash).name[0..24]
 115     job = Job.new job_attrs(runtime_constraints:
 116                             {'docker_image' => image_hash})
 117     assert job.valid?, job.errors.full_messages.to_s + " with partial hash #{image_hash}"
 118     assert_equal(collections(:docker_image).portable_data_hash, job.docker_image_locator)
 119   end
 120
 121   { 'name' => 'arvados_test_nonexistent',
 122     'hash' => 'f' * 64,
 123     'locator' => BAD_COLLECTION,
 124   }.each_pair do |spec_type, image_spec|
 125     test "Job validation fails with nonexistent Docker image #{spec_type}" do
 126       job = Job.new job_attrs(runtime_constraints:
 127                               {'docker_image' => image_spec})
 128       assert(job.invalid?, "nonexistent Docker image #{spec_type} was valid")
 129     end
 130   end
 131
 132   test "Job validation fails with non-Docker Collection constraint" do
 133     job = Job.new job_attrs(runtime_constraints:
 134                             {'docker_image' => collections(:foo_file).uuid})
 135     assert(job.invalid?, "non-Docker Collection constraint was valid")
 136   end
 137
 138   test "can create Job with Docker image Collection without Docker links" do
 139     image_uuid = collections(:unlinked_docker_image).portable_data_hash
 140     job = Job.new job_attrs(runtime_constraints: {"docker_image" => image_uuid})
 141     assert(job.valid?, "Job created with unlinked Docker image was invalid")
 142     assert_equal(image_uuid, job.docker_image_locator)
 143   end
 144
 145   def check_attrs_unset(job, attrs)
 146     assert_empty(attrs.each_key.map { |key| job.send(key) }.compact,
 147                  "job has values for #{attrs.keys}")
 148   end
 149
 150   def check_creation_prohibited(attrs)
 151     begin
 152       job = Job.new(job_attrs(attrs))
 153     rescue ActiveModel::MassAssignmentSecurity::Error
 154       # Test passes - expected attribute protection
 155     else
 156       check_attrs_unset(job, attrs)
 157     end
 158   end
 159
 160   def check_modification_prohibited(attrs)
 161     job = Job.new(job_attrs)
 162     attrs.each_pair do |key, value|
 163       assert_raises(NoMethodError) { job.send("{key}=".to_sym, value) }
 164     end
 165     check_attrs_unset(job, attrs)
 166   end
 167
 168   test "can't create Job with Docker image locator" do
 169     check_creation_prohibited(docker_image_locator: BAD_COLLECTION)
 170   end
 171
 172   test "can't assign Docker image locator to Job" do
 173     check_modification_prohibited(docker_image_locator: BAD_COLLECTION)
 174   end
 175
 176   [
 177    {script_parameters: ""},
 178    {script_parameters: []},
 179    {script_parameters: {symbols: :are_not_allowed_here}},
 180    {runtime_constraints: ""},
 181    {runtime_constraints: []},
 182    {tasks_summary: ""},
 183    {tasks_summary: []},
 184    {script_version: "no/branch/could/ever/possibly/have/this/name"},
 185   ].each do |invalid_attrs|
 186     test "validation failures set error messages: #{invalid_attrs.to_json}" do
 187       # Ensure valid_attrs doesn't produce errors -- otherwise we will
 188       # not know whether errors reported below are actually caused by
 189       # invalid_attrs.
 190       Job.create! job_attrs
 191
 192       job = Job.create job_attrs(invalid_attrs)
 193       assert_raises(ActiveRecord::RecordInvalid, ArgumentError,
 194                     "save! did not raise the expected exception") do
 195         job.save!
 196       end
 197       assert_not_empty job.errors, "validation failure did not provide errors"
 198     end
 199   end
 200
 201   [
 202     # Each test case is of the following format
 203     # Array of parameters where each parameter is of the format:
 204     #  attr name to be changed, attr value, and array of expectations (where each expectation is an array)
 205     [['running', false, [['state', 'Queued']]]],
 206     [['state', 'Running', [['started_at', 'not_nil']]]],
 207     [['is_locked_by_uuid', 'use_current_user_uuid', [['state', 'Queued']]], ['state', 'Running', [['running', true], ['started_at', 'not_nil'], ['success', 'nil']]]],
 208     [['running', false, [['state', 'Queued']]], ['state', 'Complete', [['success', true]]]],
 209     [['running', true, [['state', 'Running']]], ['cancelled_at', Time.now, [['state', 'Cancelled']]]],
 210     [['running', true, [['state', 'Running']]], ['state', 'Cancelled', [['cancelled_at', 'not_nil']]]],
 211     [['running', true, [['state', 'Running']]], ['success', true, [['state', 'Complete']]]],
 212     [['running', true, [['state', 'Running']]], ['success', false, [['state', 'Failed']]]],
 213     [['running', true, [['state', 'Running']]], ['state', 'Complete', [['success', true],['finished_at', 'not_nil']]]],
 214     [['running', true, [['state', 'Running']]], ['state', 'Failed', [['success', false],['finished_at', 'not_nil']]]],
 215     [['cancelled_at', Time.now, [['state', 'Cancelled']]], ['success', false, [['state', 'Cancelled'],['finished_at', 'nil'], ['cancelled_at', 'not_nil']]]],
 216     [['cancelled_at', Time.now, [['state', 'Cancelled'],['running', false]]], ['success', true, [['state', 'Cancelled'],['running', false],['finished_at', 'nil'],['cancelled_at', 'not_nil']]]],
 217     # potential migration cases
 218     [['state', nil, [['state', 'Queued']]]],
 219     [['state', nil, [['state', 'Queued']]], ['cancelled_at', Time.now, [['state', 'Cancelled']]]],
 220     [['running', true, [['state', 'Running']]], ['state', nil, [['state', 'Running']]]],
 221   ].each do |parameters|
 222     test "verify job status #{parameters}" do
 223       job = Job.create! job_attrs
 224       assert_equal 'Queued', job.state, "job.state"
 225
 226       parameters.each do |parameter|
 227         expectations = parameter[2]
 228         if 'use_current_user_uuid' == parameter[1]
 229           parameter[1] = Thread.current[:user].uuid
 230         end
 231
 232         if expectations.instance_of? Array
 233           job[parameter[0]] = parameter[1]
 234           assert_equal true, job.save, job.errors.full_messages.to_s
 235           expectations.each do |expectation|
 236             if expectation[1] == 'not_nil'
 237               assert_not_nil job[expectation[0]], expectation[0]
 238             elsif expectation[1] == 'nil'
 239               assert_nil job[expectation[0]], expectation[0]
 240             else
 241               assert_equal expectation[1], job[expectation[0]], expectation[0]
 242             end
 243           end
 244         else
 245           raise 'I do not know how to handle this expectation'
 246         end
 247       end
 248     end
 249   end
 250
 251   test "Test job state changes" do
 252     all = ["Queued", "Running", "Complete", "Failed", "Cancelled"]
 253     valid = {"Queued" => all, "Running" => ["Complete", "Failed", "Cancelled"]}
 254     all.each do |start|
 255       all.each do |finish|
 256         if start != finish
 257           job = Job.create! job_attrs(state: start)
 258           assert_equal start, job.state
 259           job.state = finish
 260           job.save
 261           job.reload
 262           if valid[start] and valid[start].include? finish
 263             assert_equal finish, job.state
 264           else
 265             assert_equal start, job.state
 266           end
 267         end
 268       end
 269     end
 270   end
 271
 272   test "Test job locking" do
 273     set_user_from_auth :active_trustedclient
 274     job = Job.create! job_attrs
 275
 276     assert_equal "Queued", job.state
 277
 278     # Should be able to lock successfully
 279     job.lock current_user.uuid
 280     assert_equal "Running", job.state
 281
 282     assert_raises ArvadosModel::AlreadyLockedError do
 283       # Can't lock it again
 284       job.lock current_user.uuid
 285     end
 286     job.reload
 287     assert_equal "Running", job.state
 288
 289     set_user_from_auth :project_viewer
 290     assert_raises ArvadosModel::AlreadyLockedError do
 291       # Can't lock it as a different user either
 292       job.lock current_user.uuid
 293     end
 294     job.reload
 295     assert_equal "Running", job.state
 296
 297     assert_raises ArvadosModel::PermissionDeniedError do
 298       # Can't update fields as a different user
 299       job.update_attributes(state: "Failed")
 300     end
 301     job.reload
 302     assert_equal "Running", job.state
 303
 304
 305     set_user_from_auth :active_trustedclient
 306
 307     # Can update fields as the locked_by user
 308     job.update_attributes(state: "Failed")
 309     assert_equal "Failed", job.state
 310   end
 311
 312   test "admin user can cancel a running job despite lock" do
 313     set_user_from_auth :active_trustedclient
 314     job = Job.create! job_attrs
 315     job.lock current_user.uuid
 316     assert_equal Job::Running, job.state
 317
 318     set_user_from_auth :spectator
 319     assert_raises do
 320       job.update_attributes!(state: Job::Cancelled)
 321     end
 322
 323     set_user_from_auth :admin
 324     job.reload
 325     assert_equal Job::Running, job.state
 326     job.update_attributes!(state: Job::Cancelled)
 327     assert_equal Job::Cancelled, job.state
 328   end
 329
 330   test "verify job queue position" do
 331     job1 = Job.create! job_attrs
 332     assert_equal 'Queued', job1.state, "Incorrect job state for newly created job1"
 333
 334     job2 = Job.create! job_attrs
 335     assert_equal 'Queued', job2.state, "Incorrect job state for newly created job2"
 336
 337     assert_not_nil job1.queue_position, "Expected non-nil queue position for job1"
 338     assert_not_nil job2.queue_position, "Expected non-nil queue position for job2"
 339   end
 340
 341   SDK_MASTER = "ca68b24e51992e790f29df5cc4bc54ce1da4a1c2"
 342   SDK_TAGGED = "00634b2b8a492d6f121e3cf1d6587b821136a9a7"
 343
 344   def sdk_constraint(version)
 345     {runtime_constraints: {
 346         "arvados_sdk_version" => version,
 347         "docker_image" => links(:docker_image_collection_tag).name,
 348       }}
 349   end
 350
 351   def check_job_sdk_version(expected)
 352     job = yield
 353     if expected.nil?
 354       refute(job.valid?, "job valid with bad Arvados SDK version")
 355     else
 356       assert(job.valid?, "job not valid with good Arvados SDK version")
 357       assert_equal(expected, job.arvados_sdk_version)
 358     end
 359   end
 360
 361   { "master" => SDK_MASTER,
 362     "commit2" => SDK_TAGGED,
 363     SDK_TAGGED[0, 8] => SDK_TAGGED,
 364     "__nonexistent__" => nil,
 365   }.each_pair do |search, commit_hash|
 366     test "creating job with SDK version '#{search}'" do
 367       check_job_sdk_version(commit_hash) do
 368         Job.new(job_attrs(sdk_constraint(search)))
 369       end
 370     end
 371
 372     test "updating job from no SDK to version '#{search}'" do
 373       job = Job.create!(job_attrs)
 374       assert_nil job.arvados_sdk_version
 375       check_job_sdk_version(commit_hash) do
 376         job.runtime_constraints = sdk_constraint(search)[:runtime_constraints]
 377         job
 378       end
 379     end
 380
 381     test "updating job from SDK version 'master' to '#{search}'" do
 382       job = Job.create!(job_attrs(sdk_constraint("master")))
 383       assert_equal(SDK_MASTER, job.arvados_sdk_version)
 384       check_job_sdk_version(commit_hash) do
 385         job.runtime_constraints = sdk_constraint(search)[:runtime_constraints]
 386         job
 387       end
 388     end
 389   end
 390
 391   test "clear the SDK version" do
 392     job = Job.create!(job_attrs(sdk_constraint("master")))
 393     assert_equal(SDK_MASTER, job.arvados_sdk_version)
 394     job.runtime_constraints = {}
 395     assert(job.valid?, "job invalid after clearing SDK version")
 396     assert_nil(job.arvados_sdk_version)
 397   end
 398
 399   test "job with SDK constraint, without Docker image is invalid" do
 400     sdk_attrs = sdk_constraint("master")
 401     sdk_attrs[:runtime_constraints].delete("docker_image")
 402     job = Job.create(job_attrs(sdk_attrs))
 403     refute(job.valid?, "Job valid with SDK version, without Docker image")
 404     sdk_errors = job.errors.messages[:arvados_sdk_version] || []
 405     refute_empty(sdk_errors.grep(/\bDocker\b/),
 406                  "no Job SDK errors mention that Docker is required")
 407   end
 408
 409   test "invalid to clear Docker image constraint when SDK constraint exists" do
 410     job = Job.create!(job_attrs(sdk_constraint("master")))
 411     job.runtime_constraints.delete("docker_image")
 412     refute(job.valid?,
 413            "Job with SDK constraint valid after clearing Docker image")
 414   end
 415
 416   test "use migrated docker image if requesting old-format image by tag" do
 417     Rails.configuration.docker_image_formats = ['v2']
 418     add_docker19_migration_link
 419     job = Job.create!(
 420       job_attrs(
 421         script: 'foo',
 422         runtime_constraints: {
 423           'docker_image' => links(:docker_image_collection_tag).name}))
 424     assert(job.valid?)
 425     assert_equal(job.docker_image_locator, collections(:docker_image_1_12).portable_data_hash)
 426   end
 427
 428   test "use migrated docker image if requesting old-format image by pdh" do
 429     Rails.configuration.docker_image_formats = ['v2']
 430     add_docker19_migration_link
 431     job = Job.create!(
 432       job_attrs(
 433         script: 'foo',
 434         runtime_constraints: {
 435           'docker_image' => collections(:docker_image).portable_data_hash}))
 436     assert(job.valid?)
 437     assert_equal(job.docker_image_locator, collections(:docker_image_1_12).portable_data_hash)
 438   end
 439
 440   [[:docker_image, :docker_image, :docker_image_1_12],
 441    [:docker_image_1_12, :docker_image, :docker_image_1_12],
 442    [:docker_image, :docker_image_1_12, :docker_image_1_12],
 443    [:docker_image_1_12, :docker_image_1_12, :docker_image_1_12],
 444   ].each do |existing_image, request_image, expect_image|
 445     test "if a #{existing_image} job exists, #{request_image} yields #{expect_image} after migration" do
 446       Rails.configuration.docker_image_formats = ['v1']
 447
 448       if existing_image == :docker_image
 449         oldjob = Job.create!(
 450           job_attrs(
 451             script: 'foobar1',
 452             runtime_constraints: {
 453               'docker_image' => collections(existing_image).portable_data_hash}))
 454         oldjob.reload
 455         assert_equal(oldjob.docker_image_locator,
 456                      collections(existing_image).portable_data_hash)
 457       elsif existing_image == :docker_image_1_12
 458         assert_raises(ActiveRecord::RecordInvalid,
 459                       "Should not resolve v2 image when only v1 is supported") do
 460         oldjob = Job.create!(
 461           job_attrs(
 462             script: 'foobar1',
 463             runtime_constraints: {
 464               'docker_image' => collections(existing_image).portable_data_hash}))
 465         end
 466       end
 467
 468       Rails.configuration.docker_image_formats = ['v2']
 469       add_docker19_migration_link
 470
 471       # Check that both v1 and v2 images get resolved to v2.
 472       newjob = Job.create!(
 473         job_attrs(
 474           script: 'foobar1',
 475           runtime_constraints: {
 476             'docker_image' => collections(request_image).portable_data_hash}))
 477       newjob.reload
 478       assert_equal(newjob.docker_image_locator,
 479                    collections(expect_image).portable_data_hash)
 480     end
 481   end
 482
 483   test "can't create job with SDK version assigned directly" do
 484     check_creation_prohibited(arvados_sdk_version: SDK_MASTER)
 485   end
 486
 487   test "can't modify job to assign SDK version directly" do
 488     check_modification_prohibited(arvados_sdk_version: SDK_MASTER)
 489   end
 490
 491   test "job validation fails when collection uuid found in script_parameters" do
 492     bad_params = {
 493       script_parameters: {
 494         'input' => {
 495           'param1' => 'the collection uuid zzzzz-4zz18-012345678901234'
 496         }
 497       }
 498     }
 499     assert_raises(ActiveRecord::RecordInvalid,
 500                   "created job with a collection uuid in script_parameters") do
 501       Job.create!(job_attrs(bad_params))
 502     end
 503   end
 504
 505   test "job validation succeeds when no collection uuid in script_parameters" do
 506     good_params = {
 507       script_parameters: {
 508         'arg1' => 'foo',
 509         'arg2' => [ 'bar', 'baz' ],
 510         'arg3' => {
 511           'a' => 1,
 512           'b' => [2, 3, 4],
 513         }
 514       }
 515     }
 516     job = Job.create!(job_attrs(good_params))
 517     assert job.valid?
 518   end
 519
 520   test 'update job uuid tag in internal.git when version changes' do
 521     authorize_with :active
 522     j = jobs :queued
 523     j.update_attributes repository: 'active/foo', script_version: 'b1'
 524     assert_equal('1de84a854e2b440dc53bf42f8548afa4c17da332',
 525                  internal_tag(j.uuid))
 526     j.update_attributes repository: 'active/foo', script_version: 'master'
 527     assert_equal('077ba2ad3ea24a929091a9e6ce545c93199b8e57',
 528                  internal_tag(j.uuid))
 529   end
 530
 531   test 'script_parameters_digest is independent of key order' do
 532     j1 = Job.new(job_attrs(script_parameters: {'a' => 'a', 'ddee' => {'d' => 'd', 'e' => 'e'}}))
 533     j2 = Job.new(job_attrs(script_parameters: {'ddee' => {'e' => 'e', 'd' => 'd'}, 'a' => 'a'}))
 534     assert j1.valid?
 535     assert j2.valid?
 536     assert_equal(j1.script_parameters_digest, j2.script_parameters_digest)
 537   end
 538
 539   test 'job fixtures have correct script_parameters_digest' do
 540     Job.all.each do |j|
 541       d = j.script_parameters_digest
 542       assert_equal(j.update_script_parameters_digest, d,
 543                    "wrong script_parameters_digest for #{j.uuid}")
 544     end
 545   end
 546
 547   test 'deep_sort_hash on array of hashes' do
 548     a = {'z' => [[{'a' => 'a', 'b' => 'b'}]]}
 549     b = {'z' => [[{'b' => 'b', 'a' => 'a'}]]}
 550     assert_equal Job.deep_sort_hash(a).to_json, Job.deep_sort_hash(b).to_json
 551   end
 552
 553   test 'find_reusable' do
 554     foobar = jobs(:foobar)
 555     example_attrs = {
 556       script_version: foobar.script_version,
 557       script: foobar.script,
 558       script_parameters: foobar.script_parameters,
 559       repository: foobar.repository,
 560     }
 561
 562     # Two matching jobs exist with identical outputs. The older one
 563     # should be reused.
 564     j = Job.find_reusable(example_attrs, {}, [], [users(:active)])
 565     assert j
 566     assert_equal foobar.uuid, j.uuid
 567
 568     # Two matching jobs exist with different outputs. Neither should
 569     # be reused.
 570     Job.where(uuid: jobs(:job_with_latest_version).uuid).
 571       update_all(output: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa+1')
 572     assert_nil Job.find_reusable(example_attrs, {}, [], [users(:active)])
 573   end
 574
 575   [
 576     true,
 577     false,
 578   ].each do |cascade|
 579     test "cancel job with cascade #{cascade}" do
 580       job = Job.find_by_uuid jobs(:running_job_with_components_at_level_1).uuid
 581       job.cancel cascade: cascade
 582       assert_equal Job::Cancelled, job.state
 583
 584       descendents = ['zzzzz-8i9sb-jobcomponentsl2',
 585                      'zzzzz-d1hrv-picomponentsl02',
 586                      'zzzzz-8i9sb-job1atlevel3noc',
 587                      'zzzzz-8i9sb-job2atlevel3noc']
 588
 589       jobs = Job.where(uuid: descendents)
 590       jobs.each do |j|
 591         assert_equal ('Cancelled' == j.state), cascade
 592       end
 593
 594       pipelines = PipelineInstance.where(uuid: descendents)
 595       pipelines.each do |pi|
 596         assert_equal ('Paused' == pi.state), cascade
 597       end
 598     end
 599   end
 600
 601   test 'cancelling a completed job raises error' do
 602     job = Job.find_by_uuid jobs(:job_with_latest_version).uuid
 603     assert job
 604     assert_equal 'Complete', job.state
 605
 606     assert_raises(ArvadosModel::InvalidStateTransitionError) do
 607       job.cancel
 608     end
 609   end
 610
 611   test 'cancelling a job with circular relationship with another does not result in an infinite loop' do
 612     job = Job.find_by_uuid jobs(:running_job_2_with_circular_component_relationship).uuid
 613
 614     job.cancel cascade: true
 615
 616     assert_equal Job::Cancelled, job.state
 617
 618     child = Job.find_by_uuid job.components.collect{|_, uuid| uuid}[0]
 619     assert_equal Job::Cancelled, child.state
 620   end
 621 end