services/api/test/unit/job_test.rb

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 require 'test_helper'
   6 require 'helpers/git_test_helper'
   7 require 'helpers/docker_migration_helper'
   8
   9 class JobTest < ActiveSupport::TestCase
  10   include DockerMigrationHelper
  11   include GitTestHelper
  12
  13   BAD_COLLECTION = "#{'f' * 32}+0"
  14
  15   setup do
  16     set_user_from_auth :active
  17   end
  18
  19   def job_attrs merge_me={}
  20     # Default (valid) set of attributes, with given overrides
  21     {
  22       script: "hash",
  23       script_version: "master",
  24       repository: "active/foo",
  25     }.merge(merge_me)
  26   end
  27
  28   test "Job without Docker image doesn't get locator" do
  29     job = Job.new job_attrs
  30     assert job.valid?, job.errors.full_messages.to_s
  31     assert_nil job.docker_image_locator
  32   end
  33
  34   { 'name' => [:links, :docker_image_collection_tag, :name],
  35     'hash' => [:links, :docker_image_collection_hash, :name],
  36     'locator' => [:collections, :docker_image, :portable_data_hash],
  37   }.each_pair do |spec_type, (fixture_type, fixture_name, fixture_attr)|
  38     test "Job initialized with Docker image #{spec_type} gets locator" do
  39       image_spec = send(fixture_type, fixture_name).send(fixture_attr)
  40       job = Job.new job_attrs(runtime_constraints:
  41                               {'docker_image' => image_spec})
  42       assert job.valid?, job.errors.full_messages.to_s
  43       assert_equal(collections(:docker_image).portable_data_hash, job.docker_image_locator)
  44     end
  45
  46     test "Job modified with Docker image #{spec_type} gets locator" do
  47       job = Job.new job_attrs
  48       assert job.valid?, job.errors.full_messages.to_s
  49       assert_nil job.docker_image_locator
  50       image_spec = send(fixture_type, fixture_name).send(fixture_attr)
  51       job.runtime_constraints['docker_image'] = image_spec
  52       assert job.valid?, job.errors.full_messages.to_s
  53       assert_equal(collections(:docker_image).portable_data_hash, job.docker_image_locator)
  54     end
  55   end
  56
  57   test "removing a Docker runtime constraint removes the locator" do
  58     image_locator = collections(:docker_image).portable_data_hash
  59     job = Job.new job_attrs(runtime_constraints:
  60                             {'docker_image' => image_locator})
  61     assert job.valid?, job.errors.full_messages.to_s
  62     assert_equal(image_locator, job.docker_image_locator)
  63     job.runtime_constraints = {}
  64     assert job.valid?, job.errors.full_messages.to_s + "after clearing runtime constraints"
  65     assert_nil job.docker_image_locator
  66   end
  67
  68   test "locate a Docker image with a repository + tag" do
  69     image_repo, image_tag =
  70       links(:docker_image_collection_tag2).name.split(':', 2)
  71     job = Job.new job_attrs(runtime_constraints:
  72                             {'docker_image' => image_repo,
  73                               'docker_image_tag' => image_tag})
  74     assert job.valid?, job.errors.full_messages.to_s
  75     assert_equal(collections(:docker_image).portable_data_hash, job.docker_image_locator)
  76   end
  77
  78   test "can't locate a Docker image with a nonexistent tag" do
  79     image_repo = links(:docker_image_collection_tag).name
  80     image_tag = '__nonexistent tag__'
  81     job = Job.new job_attrs(runtime_constraints:
  82                             {'docker_image' => image_repo,
  83                               'docker_image_tag' => image_tag})
  84     assert(job.invalid?, "Job with bad Docker tag valid")
  85   end
  86
  87   [
  88     false,
  89     true
  90   ].each do |use_config|
  91     test "Job with no Docker image uses default docker image when configuration is set #{use_config}" do
  92       default_docker_image = collections(:docker_image)[:portable_data_hash]
  93       Rails.configuration.default_docker_image_for_jobs = default_docker_image if use_config
  94
  95       job = Job.new job_attrs
  96       assert job.valid?, job.errors.full_messages.to_s
  97
  98       if use_config
  99         refute_nil job.docker_image_locator
 100         assert_equal default_docker_image, job.docker_image_locator
 101       else
 102         assert_nil job.docker_image_locator
 103       end
 104     end
 105   end
 106
 107   test "create a job with a disambiguated script_version branch name" do
 108     job = Job.
 109       new(script: "testscript",
 110           script_version: "heads/7387838c69a21827834586cc42b467ff6c63293b",
 111           repository: "active/shabranchnames",
 112           script_parameters: {})
 113     assert(job.save)
 114     assert_equal("abec49829bf1758413509b7ffcab32a771b71e81", job.script_version)
 115   end
 116
 117   test "locate a Docker image with a partial hash" do
 118     image_hash = links(:docker_image_collection_hash).name[0..24]
 119     job = Job.new job_attrs(runtime_constraints:
 120                             {'docker_image' => image_hash})
 121     assert job.valid?, job.errors.full_messages.to_s + " with partial hash #{image_hash}"
 122     assert_equal(collections(:docker_image).portable_data_hash, job.docker_image_locator)
 123   end
 124
 125   { 'name' => 'arvados_test_nonexistent',
 126     'hash' => 'f' * 64,
 127     'locator' => BAD_COLLECTION,
 128   }.each_pair do |spec_type, image_spec|
 129     test "Job validation fails with nonexistent Docker image #{spec_type}" do
 130       job = Job.new job_attrs(runtime_constraints:
 131                               {'docker_image' => image_spec})
 132       assert(job.invalid?, "nonexistent Docker image #{spec_type} was valid")
 133     end
 134   end
 135
 136   test "Job validation fails with non-Docker Collection constraint" do
 137     job = Job.new job_attrs(runtime_constraints:
 138                             {'docker_image' => collections(:foo_file).uuid})
 139     assert(job.invalid?, "non-Docker Collection constraint was valid")
 140   end
 141
 142   test "can create Job with Docker image Collection without Docker links" do
 143     image_uuid = collections(:unlinked_docker_image).portable_data_hash
 144     job = Job.new job_attrs(runtime_constraints: {"docker_image" => image_uuid})
 145     assert(job.valid?, "Job created with unlinked Docker image was invalid")
 146     assert_equal(image_uuid, job.docker_image_locator)
 147   end
 148
 149   def check_attrs_unset(job, attrs)
 150     assert_empty(attrs.each_key.map { |key| job.send(key) }.compact,
 151                  "job has values for #{attrs.keys}")
 152   end
 153
 154   def check_creation_prohibited(attrs)
 155     begin
 156       job = Job.new(job_attrs(attrs))
 157     rescue ActiveModel::MassAssignmentSecurity::Error
 158       # Test passes - expected attribute protection
 159     else
 160       check_attrs_unset(job, attrs)
 161     end
 162   end
 163
 164   def check_modification_prohibited(attrs)
 165     job = Job.new(job_attrs)
 166     attrs.each_pair do |key, value|
 167       assert_raises(NoMethodError) { job.send("{key}=".to_sym, value) }
 168     end
 169     check_attrs_unset(job, attrs)
 170   end
 171
 172   test "can't create Job with Docker image locator" do
 173     check_creation_prohibited(docker_image_locator: BAD_COLLECTION)
 174   end
 175
 176   test "can't assign Docker image locator to Job" do
 177     check_modification_prohibited(docker_image_locator: BAD_COLLECTION)
 178   end
 179
 180   [
 181    {script_parameters: ""},
 182    {script_parameters: []},
 183    {script_parameters: {["foo"] => ["bar"]}},
 184    {runtime_constraints: ""},
 185    {runtime_constraints: []},
 186    {tasks_summary: ""},
 187    {tasks_summary: []},
 188   ].each do |invalid_attrs|
 189     test "validation failures set error messages: #{invalid_attrs.to_json}" do
 190       # Ensure valid_attrs doesn't produce errors -- otherwise we will
 191       # not know whether errors reported below are actually caused by
 192       # invalid_attrs.
 193       Job.new(job_attrs).save!
 194
 195       err = assert_raises(ArgumentError) do
 196         Job.new(job_attrs(invalid_attrs)).save!
 197       end
 198       assert_match /parameters|constraints|summary/, err.message
 199     end
 200   end
 201
 202   test "invalid script_version" do
 203     invalid = {
 204       script_version: "no/branch/could/ever/possibly/have/this/name",
 205     }
 206     err = assert_raises(ActiveRecord::RecordInvalid) do
 207       Job.new(job_attrs(invalid)).save!
 208     end
 209     assert_match /Script version .* does not resolve to a commit/, err.message
 210   end
 211
 212   [
 213     # Each test case is of the following format
 214     # Array of parameters where each parameter is of the format:
 215     #  attr name to be changed, attr value, and array of expectations (where each expectation is an array)
 216     [['running', false, [['state', 'Queued']]]],
 217     [['state', 'Running', [['started_at', 'not_nil']]]],
 218     [['is_locked_by_uuid', 'use_current_user_uuid', [['state', 'Queued']]], ['state', 'Running', [['running', true], ['started_at', 'not_nil'], ['success', 'nil']]]],
 219     [['running', false, [['state', 'Queued']]], ['state', 'Complete', [['success', true]]]],
 220     [['running', true, [['state', 'Running']]], ['cancelled_at', Time.now, [['state', 'Cancelled']]]],
 221     [['running', true, [['state', 'Running']]], ['state', 'Cancelled', [['cancelled_at', 'not_nil']]]],
 222     [['running', true, [['state', 'Running']]], ['success', true, [['state', 'Complete']]]],
 223     [['running', true, [['state', 'Running']]], ['success', false, [['state', 'Failed']]]],
 224     [['running', true, [['state', 'Running']]], ['state', 'Complete', [['success', true],['finished_at', 'not_nil']]]],
 225     [['running', true, [['state', 'Running']]], ['state', 'Failed', [['success', false],['finished_at', 'not_nil']]]],
 226     [['cancelled_at', Time.now, [['state', 'Cancelled']]], ['success', false, [['state', 'Cancelled'],['finished_at', 'nil'], ['cancelled_at', 'not_nil']]]],
 227     [['cancelled_at', Time.now, [['state', 'Cancelled'],['running', false]]], ['success', true, [['state', 'Cancelled'],['running', false],['finished_at', 'nil'],['cancelled_at', 'not_nil']]]],
 228     # potential migration cases
 229     [['state', nil, [['state', 'Queued']]]],
 230     [['state', nil, [['state', 'Queued']]], ['cancelled_at', Time.now, [['state', 'Cancelled']]]],
 231     [['running', true, [['state', 'Running']]], ['state', nil, [['state', 'Running']]]],
 232   ].each do |parameters|
 233     test "verify job status #{parameters}" do
 234       job = Job.create! job_attrs
 235       assert_equal 'Queued', job.state, "job.state"
 236
 237       parameters.each do |parameter|
 238         expectations = parameter[2]
 239         if 'use_current_user_uuid' == parameter[1]
 240           parameter[1] = Thread.current[:user].uuid
 241         end
 242
 243         if expectations.instance_of? Array
 244           job[parameter[0]] = parameter[1]
 245           assert_equal true, job.save, job.errors.full_messages.to_s
 246           expectations.each do |expectation|
 247             if expectation[1] == 'not_nil'
 248               assert_not_nil job[expectation[0]], expectation[0]
 249             elsif expectation[1] == 'nil'
 250               assert_nil job[expectation[0]], expectation[0]
 251             else
 252               assert_equal expectation[1], job[expectation[0]], expectation[0]
 253             end
 254           end
 255         else
 256           raise 'I do not know how to handle this expectation'
 257         end
 258       end
 259     end
 260   end
 261
 262   test "Test job state changes" do
 263     all = ["Queued", "Running", "Complete", "Failed", "Cancelled"]
 264     valid = {"Queued" => all, "Running" => ["Complete", "Failed", "Cancelled"]}
 265     all.each do |start|
 266       all.each do |finish|
 267         if start != finish
 268           job = Job.create! job_attrs(state: start)
 269           assert_equal start, job.state
 270           job.state = finish
 271           job.save
 272           job.reload
 273           if valid[start] and valid[start].include? finish
 274             assert_equal finish, job.state
 275           else
 276             assert_equal start, job.state
 277           end
 278         end
 279       end
 280     end
 281   end
 282
 283   test "Test job locking" do
 284     set_user_from_auth :active_trustedclient
 285     job = Job.create! job_attrs
 286
 287     assert_equal "Queued", job.state
 288
 289     # Should be able to lock successfully
 290     job.lock current_user.uuid
 291     assert_equal "Running", job.state
 292
 293     assert_raises ArvadosModel::AlreadyLockedError do
 294       # Can't lock it again
 295       job.lock current_user.uuid
 296     end
 297     job.reload
 298     assert_equal "Running", job.state
 299
 300     set_user_from_auth :project_viewer
 301     assert_raises ArvadosModel::AlreadyLockedError do
 302       # Can't lock it as a different user either
 303       job.lock current_user.uuid
 304     end
 305     job.reload
 306     assert_equal "Running", job.state
 307
 308     assert_raises ArvadosModel::PermissionDeniedError do
 309       # Can't update fields as a different user
 310       job.update_attributes(state: "Failed")
 311     end
 312     job.reload
 313     assert_equal "Running", job.state
 314
 315
 316     set_user_from_auth :active_trustedclient
 317
 318     # Can update fields as the locked_by user
 319     job.update_attributes(state: "Failed")
 320     assert_equal "Failed", job.state
 321   end
 322
 323   test "admin user can cancel a running job despite lock" do
 324     set_user_from_auth :active_trustedclient
 325     job = Job.create! job_attrs
 326     job.lock current_user.uuid
 327     assert_equal Job::Running, job.state
 328
 329     set_user_from_auth :spectator
 330     assert_raises do
 331       job.update_attributes!(state: Job::Cancelled)
 332     end
 333
 334     set_user_from_auth :admin
 335     job.reload
 336     assert_equal Job::Running, job.state
 337     job.update_attributes!(state: Job::Cancelled)
 338     assert_equal Job::Cancelled, job.state
 339   end
 340
 341   test "verify job queue position" do
 342     job1 = Job.create! job_attrs
 343     assert_equal 'Queued', job1.state, "Incorrect job state for newly created job1"
 344
 345     job2 = Job.create! job_attrs
 346     assert_equal 'Queued', job2.state, "Incorrect job state for newly created job2"
 347
 348     assert_not_nil job1.queue_position, "Expected non-nil queue position for job1"
 349     assert_not_nil job2.queue_position, "Expected non-nil queue position for job2"
 350   end
 351
 352   SDK_MASTER = "ca68b24e51992e790f29df5cc4bc54ce1da4a1c2"
 353   SDK_TAGGED = "00634b2b8a492d6f121e3cf1d6587b821136a9a7"
 354
 355   def sdk_constraint(version)
 356     {runtime_constraints: {
 357         "arvados_sdk_version" => version,
 358         "docker_image" => links(:docker_image_collection_tag).name,
 359       }}
 360   end
 361
 362   def check_job_sdk_version(expected)
 363     job = yield
 364     if expected.nil?
 365       refute(job.valid?, "job valid with bad Arvados SDK version")
 366     else
 367       assert(job.valid?, "job not valid with good Arvados SDK version")
 368       assert_equal(expected, job.arvados_sdk_version)
 369     end
 370   end
 371
 372   { "master" => SDK_MASTER,
 373     "commit2" => SDK_TAGGED,
 374     SDK_TAGGED[0, 8] => SDK_TAGGED,
 375     "__nonexistent__" => nil,
 376   }.each_pair do |search, commit_hash|
 377     test "creating job with SDK version '#{search}'" do
 378       check_job_sdk_version(commit_hash) do
 379         Job.new(job_attrs(sdk_constraint(search)))
 380       end
 381     end
 382
 383     test "updating job from no SDK to version '#{search}'" do
 384       job = Job.create!(job_attrs)
 385       assert_nil job.arvados_sdk_version
 386       check_job_sdk_version(commit_hash) do
 387         job.runtime_constraints = sdk_constraint(search)[:runtime_constraints]
 388         job
 389       end
 390     end
 391
 392     test "updating job from SDK version 'master' to '#{search}'" do
 393       job = Job.create!(job_attrs(sdk_constraint("master")))
 394       assert_equal(SDK_MASTER, job.arvados_sdk_version)
 395       check_job_sdk_version(commit_hash) do
 396         job.runtime_constraints = sdk_constraint(search)[:runtime_constraints]
 397         job
 398       end
 399     end
 400   end
 401
 402   test "clear the SDK version" do
 403     job = Job.create!(job_attrs(sdk_constraint("master")))
 404     assert_equal(SDK_MASTER, job.arvados_sdk_version)
 405     job.runtime_constraints = {}
 406     assert(job.valid?, "job invalid after clearing SDK version")
 407     assert_nil(job.arvados_sdk_version)
 408   end
 409
 410   test "job with SDK constraint, without Docker image is invalid" do
 411     sdk_attrs = sdk_constraint("master")
 412     sdk_attrs[:runtime_constraints].delete("docker_image")
 413     job = Job.create(job_attrs(sdk_attrs))
 414     refute(job.valid?, "Job valid with SDK version, without Docker image")
 415     sdk_errors = job.errors.messages[:arvados_sdk_version] || []
 416     refute_empty(sdk_errors.grep(/\bDocker\b/),
 417                  "no Job SDK errors mention that Docker is required")
 418   end
 419
 420   test "invalid to clear Docker image constraint when SDK constraint exists" do
 421     job = Job.create!(job_attrs(sdk_constraint("master")))
 422     job.runtime_constraints.delete("docker_image")
 423     refute(job.valid?,
 424            "Job with SDK constraint valid after clearing Docker image")
 425   end
 426
 427   test "use migrated docker image if requesting old-format image by tag" do
 428     Rails.configuration.docker_image_formats = ['v2']
 429     add_docker19_migration_link
 430     job = Job.create!(
 431       job_attrs(
 432         script: 'foo',
 433         runtime_constraints: {
 434           'docker_image' => links(:docker_image_collection_tag).name}))
 435     assert(job.valid?)
 436     assert_equal(job.docker_image_locator, collections(:docker_image_1_12).portable_data_hash)
 437   end
 438
 439   test "use migrated docker image if requesting old-format image by pdh" do
 440     Rails.configuration.docker_image_formats = ['v2']
 441     add_docker19_migration_link
 442     job = Job.create!(
 443       job_attrs(
 444         script: 'foo',
 445         runtime_constraints: {
 446           'docker_image' => collections(:docker_image).portable_data_hash}))
 447     assert(job.valid?)
 448     assert_equal(job.docker_image_locator, collections(:docker_image_1_12).portable_data_hash)
 449   end
 450
 451   [[:docker_image, :docker_image, :docker_image_1_12],
 452    [:docker_image_1_12, :docker_image, :docker_image_1_12],
 453    [:docker_image, :docker_image_1_12, :docker_image_1_12],
 454    [:docker_image_1_12, :docker_image_1_12, :docker_image_1_12],
 455   ].each do |existing_image, request_image, expect_image|
 456     test "if a #{existing_image} job exists, #{request_image} yields #{expect_image} after migration" do
 457       Rails.configuration.docker_image_formats = ['v1']
 458
 459       if existing_image == :docker_image
 460         oldjob = Job.create!(
 461           job_attrs(
 462             script: 'foobar1',
 463             runtime_constraints: {
 464               'docker_image' => collections(existing_image).portable_data_hash}))
 465         oldjob.reload
 466         assert_equal(oldjob.docker_image_locator,
 467                      collections(existing_image).portable_data_hash)
 468       elsif existing_image == :docker_image_1_12
 469         assert_raises(ActiveRecord::RecordInvalid,
 470                       "Should not resolve v2 image when only v1 is supported") do
 471         oldjob = Job.create!(
 472           job_attrs(
 473             script: 'foobar1',
 474             runtime_constraints: {
 475               'docker_image' => collections(existing_image).portable_data_hash}))
 476         end
 477       end
 478
 479       Rails.configuration.docker_image_formats = ['v2']
 480       add_docker19_migration_link
 481
 482       # Check that both v1 and v2 images get resolved to v2.
 483       newjob = Job.create!(
 484         job_attrs(
 485           script: 'foobar1',
 486           runtime_constraints: {
 487             'docker_image' => collections(request_image).portable_data_hash}))
 488       newjob.reload
 489       assert_equal(newjob.docker_image_locator,
 490                    collections(expect_image).portable_data_hash)
 491     end
 492   end
 493
 494   test "can't create job with SDK version assigned directly" do
 495     check_creation_prohibited(arvados_sdk_version: SDK_MASTER)
 496   end
 497
 498   test "can't modify job to assign SDK version directly" do
 499     check_modification_prohibited(arvados_sdk_version: SDK_MASTER)
 500   end
 501
 502   test "job validation fails when collection uuid found in script_parameters" do
 503     bad_params = {
 504       script_parameters: {
 505         'input' => {
 506           'param1' => 'the collection uuid zzzzz-4zz18-012345678901234'
 507         }
 508       }
 509     }
 510     assert_raises(ActiveRecord::RecordInvalid,
 511                   "created job with a collection uuid in script_parameters") do
 512       Job.create!(job_attrs(bad_params))
 513     end
 514   end
 515
 516   test "job validation succeeds when no collection uuid in script_parameters" do
 517     good_params = {
 518       script_parameters: {
 519         'arg1' => 'foo',
 520         'arg2' => [ 'bar', 'baz' ],
 521         'arg3' => {
 522           'a' => 1,
 523           'b' => [2, 3, 4],
 524         }
 525       }
 526     }
 527     job = Job.create!(job_attrs(good_params))
 528     assert job.valid?
 529   end
 530
 531   test 'update job uuid tag in internal.git when version changes' do
 532     authorize_with :active
 533     j = jobs :queued
 534     j.update_attributes repository: 'active/foo', script_version: 'b1'
 535     assert_equal('1de84a854e2b440dc53bf42f8548afa4c17da332',
 536                  internal_tag(j.uuid))
 537     j.update_attributes repository: 'active/foo', script_version: 'master'
 538     assert_equal('077ba2ad3ea24a929091a9e6ce545c93199b8e57',
 539                  internal_tag(j.uuid))
 540   end
 541
 542   test 'script_parameters_digest is independent of key order' do
 543     j1 = Job.new(job_attrs(script_parameters: {'a' => 'a', 'ddee' => {'d' => 'd', 'e' => 'e'}}))
 544     j2 = Job.new(job_attrs(script_parameters: {'ddee' => {'e' => 'e', 'd' => 'd'}, 'a' => 'a'}))
 545     assert j1.valid?
 546     assert j2.valid?
 547     assert_equal(j1.script_parameters_digest, j2.script_parameters_digest)
 548   end
 549
 550   test 'job fixtures have correct script_parameters_digest' do
 551     Job.all.each do |j|
 552       d = j.script_parameters_digest
 553       assert_equal(j.update_script_parameters_digest, d,
 554                    "wrong script_parameters_digest for #{j.uuid}")
 555     end
 556   end
 557
 558   test 'deep_sort_hash on array of hashes' do
 559     a = {'z' => [[{'a' => 'a', 'b' => 'b'}]]}
 560     b = {'z' => [[{'b' => 'b', 'a' => 'a'}]]}
 561     assert_equal Job.deep_sort_hash(a).to_json, Job.deep_sort_hash(b).to_json
 562   end
 563
 564   test 'find_reusable without logging' do
 565     Rails.logger.expects(:info).never
 566     try_find_reusable
 567   end
 568
 569   test 'find_reusable with logging' do
 570     Rails.configuration.log_reuse_decisions = true
 571     Rails.logger.expects(:info).at_least(3)
 572     try_find_reusable
 573   end
 574
 575   def try_find_reusable
 576     foobar = jobs(:foobar)
 577     example_attrs = {
 578       script_version: foobar.script_version,
 579       script: foobar.script,
 580       script_parameters: foobar.script_parameters,
 581       repository: foobar.repository,
 582     }
 583
 584     # Two matching jobs exist with identical outputs. The older one
 585     # should be reused.
 586     j = Job.find_reusable(example_attrs, {}, [], [users(:active)])
 587     assert j
 588     assert_equal foobar.uuid, j.uuid
 589
 590     # Two matching jobs exist with different outputs. Neither should
 591     # be reused.
 592     Job.where(uuid: jobs(:job_with_latest_version).uuid).
 593       update_all(output: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa+1')
 594     assert_nil Job.find_reusable(example_attrs, {}, [], [users(:active)])
 595
 596     # ...unless config says to reuse the earlier job in such cases.
 597     Rails.configuration.reuse_job_if_outputs_differ = true
 598     j = Job.find_reusable(example_attrs, {}, [], [users(:active)])
 599     assert_equal foobar.uuid, j.uuid
 600   end
 601
 602   [
 603     true,
 604     false,
 605   ].each do |cascade|
 606     test "cancel job with cascade #{cascade}" do
 607       job = Job.find_by_uuid jobs(:running_job_with_components_at_level_1).uuid
 608       job.cancel cascade: cascade
 609       assert_equal Job::Cancelled, job.state
 610
 611       descendents = ['zzzzz-8i9sb-jobcomponentsl2',
 612                      'zzzzz-d1hrv-picomponentsl02',
 613                      'zzzzz-8i9sb-job1atlevel3noc',
 614                      'zzzzz-8i9sb-job2atlevel3noc']
 615
 616       jobs = Job.where(uuid: descendents)
 617       jobs.each do |j|
 618         assert_equal ('Cancelled' == j.state), cascade
 619       end
 620
 621       pipelines = PipelineInstance.where(uuid: descendents)
 622       pipelines.each do |pi|
 623         assert_equal ('Paused' == pi.state), cascade
 624       end
 625     end
 626   end
 627
 628   test 'cancelling a completed job raises error' do
 629     job = Job.find_by_uuid jobs(:job_with_latest_version).uuid
 630     assert job
 631     assert_equal 'Complete', job.state
 632
 633     assert_raises(ArvadosModel::InvalidStateTransitionError) do
 634       job.cancel
 635     end
 636   end
 637
 638   test 'cancelling a job with circular relationship with another does not result in an infinite loop' do
 639     job = Job.find_by_uuid jobs(:running_job_2_with_circular_component_relationship).uuid
 640
 641     job.cancel cascade: true
 642
 643     assert_equal Job::Cancelled, job.state
 644
 645     child = Job.find_by_uuid job.components.collect{|_, uuid| uuid}[0]
 646     assert_equal Job::Cancelled, child.state
 647   end
 648 end