X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/927524f1be454de021180b74999d682780b8cb6b..HEAD:/services/api/test/unit/container_test.rb diff --git a/services/api/test/unit/container_test.rb b/services/api/test/unit/container_test.rb index 375ab5a7bb..88496b793f 100644 --- a/services/api/test/unit/container_test.rb +++ b/services/api/test/unit/container_test.rb @@ -14,7 +14,7 @@ class ContainerTest < ActiveSupport::TestCase container_image: 'fa3c1a9cb6783f85f2ecda037e07b8c3+167', output_path: '/tmp', priority: 1, - runtime_constraints: {"vcpus" => 1, "ram" => 1}, + runtime_constraints: {"vcpus" => 1, "ram" => 1, "cuda" => {"device_count":0, "driver_version": "", "hardware_capability": ""}}, } REUSABLE_COMMON_ATTRS = { @@ -22,11 +22,13 @@ class ContainerTest < ActiveSupport::TestCase cwd: "test", command: ["echo", "hello"], output_path: "test", + output_glob: [], runtime_constraints: { "API" => false, + "keep_cache_disk" => 0, "keep_cache_ram" => 0, "ram" => 12000000000, - "vcpus" => 4, + "vcpus" => 4 }, mounts: { "test" => {"kind" => "json"}, @@ -36,7 +38,29 @@ class ContainerTest < ActiveSupport::TestCase }, secret_mounts: {}, runtime_user_uuid: "zzzzz-tpzed-xurymjxw79nv3jz", - runtime_auth_scopes: ["all"] + runtime_auth_scopes: ["all"], + scheduling_parameters: {}, + } + + REUSABLE_ATTRS_SLIM = { + command: ["echo", "slim"], + container_image: "9ae44d5792468c58bcf85ce7353c7027+124", + cwd: "test", + environment: {}, + mounts: {}, + output_path: "test", + output_glob: [], + runtime_auth_scopes: ["all"], + runtime_constraints: { + "API" => false, + "keep_cache_disk" => 0, + "keep_cache_ram" => 0, + "ram" => 8 << 30, + "vcpus" => 4 + }, + runtime_user_uuid: "zzzzz-tpzed-xurymjxw79nv3jz", + secret_mounts: {}, + scheduling_parameters: {}, } def request_only attrs @@ -54,7 +78,7 @@ class ContainerTest < ActiveSupport::TestCase def check_illegal_updates c, bad_updates bad_updates.each do |u| - refute c.update_attributes(u), u.inspect + refute c.update(u), u.inspect refute c.valid?, u.inspect c.reload end @@ -151,15 +175,15 @@ class ContainerTest < ActiveSupport::TestCase assert_equal Container::Queued, c.state set_user_from_auth :dispatch1 - c.update_attributes! state: Container::Locked - c.update_attributes! state: Container::Running + c.update! state: Container::Locked + c.update! state: Container::Running [ 'error', 'errorDetail', 'warning', 'warningDetail', 'activity' ].each do |k| # String type is allowed string_val = 'A string is accepted' - c.update_attributes! runtime_status: {k => string_val} + c.update! runtime_status: {k => string_val} assert_equal string_val, c.runtime_status[k] # Other types aren't allowed @@ -167,7 +191,7 @@ class ContainerTest < ActiveSupport::TestCase 42, false, [], {}, nil ].each do |unallowed_val| assert_raises ActiveRecord::RecordInvalid do - c.update_attributes! runtime_status: {k => unallowed_val} + c.update! runtime_status: {k => unallowed_val} end end end @@ -187,41 +211,41 @@ class ContainerTest < ActiveSupport::TestCase assert_equal Container::Queued, c1.state assert_raises ArvadosModel::PermissionDeniedError do - c1.update_attributes! runtime_status: {'error' => 'Oops!'} + c1.update! runtime_status: {'error' => 'Oops!'} end set_user_from_auth :dispatch1 # Allow updates when state = Locked - c1.update_attributes! state: Container::Locked - c1.update_attributes! runtime_status: {'error' => 'Oops!'} + c1.update! state: Container::Locked + c1.update! runtime_status: {'error' => 'Oops!'} assert c1.runtime_status.key? 'error' # Reset when transitioning from Locked to Queued - c1.update_attributes! state: Container::Queued + c1.update! state: Container::Queued assert_equal c1.runtime_status, {} # Allow updates when state = Running - c1.update_attributes! state: Container::Locked - c1.update_attributes! state: Container::Running - c1.update_attributes! runtime_status: {'error' => 'Oops!'} + c1.update! state: Container::Locked + c1.update! state: Container::Running + c1.update! runtime_status: {'error' => 'Oops!'} assert c1.runtime_status.key? 'error' # Don't allow updates on other states - c1.update_attributes! state: Container::Complete + c1.update! state: Container::Complete assert_raises ActiveRecord::RecordInvalid do - c1.update_attributes! runtime_status: {'error' => 'Some other error'} + c1.update! runtime_status: {'error' => 'Some other error'} end set_user_from_auth :active c2, _ = minimal_new(attrs) assert_equal c2.runtime_status, {} set_user_from_auth :dispatch1 - c2.update_attributes! state: Container::Locked - c2.update_attributes! state: Container::Running - c2.update_attributes! state: Container::Cancelled + c2.update! state: Container::Locked + c2.update! state: Container::Running + c2.update! state: Container::Cancelled assert_raises ActiveRecord::RecordInvalid do - c2.update_attributes! runtime_status: {'error' => 'Oops!'} + c2.update! runtime_status: {'error' => 'Oops!'} end end @@ -229,7 +253,7 @@ class ContainerTest < ActiveSupport::TestCase set_user_from_auth :active env = {"C" => "3", "B" => "2", "A" => "1"} m = {"F" => {"kind" => "3"}, "E" => {"kind" => "2"}, "D" => {"kind" => "1"}} - rc = {"vcpus" => 1, "ram" => 1, "keep_cache_ram" => 1, "API" => true} + rc = {"vcpus" => 1, "ram" => 1, "keep_cache_ram" => 1, "keep_cache_disk" => 0, "API" => true, "cuda" => {"device_count":0, "driver_version": "", "hardware_capability": ""}} c, _ = minimal_new(environment: env, mounts: m, runtime_constraints: rc) c.reload assert_equal Container.deep_sort_hash(env).to_json, c.environment.to_json @@ -272,13 +296,13 @@ class ContainerTest < ActiveSupport::TestCase assert_not_equal c_older.uuid, c_recent.uuid set_user_from_auth :dispatch1 - c_older.update_attributes!({state: Container::Locked}) - c_older.update_attributes!({state: Container::Running}) - c_older.update_attributes!(completed_attrs) + c_older.update!({state: Container::Locked}) + c_older.update!({state: Container::Running}) + c_older.update!(completed_attrs) - c_recent.update_attributes!({state: Container::Locked}) - c_recent.update_attributes!({state: Container::Running}) - c_recent.update_attributes!(completed_attrs) + c_recent.update!({state: Container::Locked}) + c_recent.update!({state: Container::Running}) + c_recent.update!(completed_attrs) reused = Container.find_reusable(common_attrs) assert_not_nil reused @@ -311,15 +335,15 @@ class ContainerTest < ActiveSupport::TestCase set_user_from_auth :dispatch1 out1 = '1f4b0bc7583c2a7f9102c395f4ffc5e3+45' - log1 = collections(:real_log_collection).portable_data_hash - c_output1.update_attributes!({state: Container::Locked}) - c_output1.update_attributes!({state: Container::Running}) - c_output1.update_attributes!(completed_attrs.merge({log: log1, output: out1})) + log1 = collections(:log_collection).portable_data_hash + c_output1.update!({state: Container::Locked}) + c_output1.update!({state: Container::Running}) + c_output1.update!(completed_attrs.merge({log: log1, output: out1})) out2 = 'fa7aeb5140e2848d39b416daeef4ffc5+45' - c_output2.update_attributes!({state: Container::Locked}) - c_output2.update_attributes!({state: Container::Running}) - c_output2.update_attributes!(completed_attrs.merge({log: log1, output: out2})) + c_output2.update!({state: Container::Locked}) + c_output2.update!({state: Container::Running}) + c_output2.update!(completed_attrs.merge({log: log1, output: out2})) set_user_from_auth :active reused = Container.resolve(ContainerRequest.new(request_only(common_attrs))) @@ -335,14 +359,14 @@ class ContainerTest < ActiveSupport::TestCase # Confirm the 3 container UUIDs are different. assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length set_user_from_auth :dispatch1 - c_slower.update_attributes!({state: Container::Locked}) - c_slower.update_attributes!({state: Container::Running, + c_slower.update!({state: Container::Locked}) + c_slower.update!({state: Container::Running, progress: 0.1}) - c_faster_started_first.update_attributes!({state: Container::Locked}) - c_faster_started_first.update_attributes!({state: Container::Running, + c_faster_started_first.update!({state: Container::Locked}) + c_faster_started_first.update!({state: Container::Running, progress: 0.15}) - c_faster_started_second.update_attributes!({state: Container::Locked}) - c_faster_started_second.update_attributes!({state: Container::Running, + c_faster_started_second.update!({state: Container::Locked}) + c_faster_started_second.update!({state: Container::Running, progress: 0.15}) reused = Container.find_reusable(common_attrs) assert_not_nil reused @@ -359,14 +383,14 @@ class ContainerTest < ActiveSupport::TestCase # Confirm the 3 container UUIDs are different. assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length set_user_from_auth :dispatch1 - c_slower.update_attributes!({state: Container::Locked}) - c_slower.update_attributes!({state: Container::Running, + c_slower.update!({state: Container::Locked}) + c_slower.update!({state: Container::Running, progress: 0.1}) - c_faster_started_first.update_attributes!({state: Container::Locked}) - c_faster_started_first.update_attributes!({state: Container::Running, + c_faster_started_first.update!({state: Container::Locked}) + c_faster_started_first.update!({state: Container::Running, progress: 0.15}) - c_faster_started_second.update_attributes!({state: Container::Locked}) - c_faster_started_second.update_attributes!({state: Container::Running, + c_faster_started_second.update!({state: Container::Locked}) + c_faster_started_second.update!({state: Container::Running, progress: 0.2}) reused = Container.find_reusable(common_attrs) assert_not_nil reused @@ -383,16 +407,16 @@ class ContainerTest < ActiveSupport::TestCase # Confirm the 3 container UUIDs are different. assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length set_user_from_auth :dispatch1 - c_slower.update_attributes!({state: Container::Locked}) - c_slower.update_attributes!({state: Container::Running, + c_slower.update!({state: Container::Locked}) + c_slower.update!({state: Container::Running, progress: 0.1}) - c_faster_started_first.update_attributes!({state: Container::Locked}) - c_faster_started_first.update_attributes!({state: Container::Running, + c_faster_started_first.update!({state: Container::Locked}) + c_faster_started_first.update!({state: Container::Running, runtime_status: {'warning' => 'This is not an error'}, progress: 0.15}) - c_faster_started_second.update_attributes!({state: Container::Locked}) + c_faster_started_second.update!({state: Container::Locked}) assert_equal 0, Container.where("runtime_status->'error' is not null").count - c_faster_started_second.update_attributes!({state: Container::Running, + c_faster_started_second.update!({state: Container::Running, runtime_status: {'error' => 'Something bad happened'}, progress: 0.2}) assert_equal 1, Container.where("runtime_status->'error' is not null").count @@ -411,11 +435,11 @@ class ContainerTest < ActiveSupport::TestCase # Confirm the 3 container UUIDs are different. assert_equal 3, [c_low_priority.uuid, c_high_priority_older.uuid, c_high_priority_newer.uuid].uniq.length set_user_from_auth :dispatch1 - c_low_priority.update_attributes!({state: Container::Locked, + c_low_priority.update!({state: Container::Locked, priority: 1}) - c_high_priority_older.update_attributes!({state: Container::Locked, + c_high_priority_older.update!({state: Container::Locked, priority: 2}) - c_high_priority_newer.update_attributes!({state: Container::Locked, + c_high_priority_newer.update!({state: Container::Locked, priority: 2}) reused = Container.find_reusable(common_attrs) assert_not_nil reused @@ -429,14 +453,14 @@ class ContainerTest < ActiveSupport::TestCase c_running, _ = minimal_new(common_attrs.merge({use_existing: false})) assert_not_equal c_failed.uuid, c_running.uuid set_user_from_auth :dispatch1 - c_failed.update_attributes!({state: Container::Locked}) - c_failed.update_attributes!({state: Container::Running}) - c_failed.update_attributes!({state: Container::Complete, + c_failed.update!({state: Container::Locked}) + c_failed.update!({state: Container::Running}) + c_failed.update!({state: Container::Complete, exit_code: 42, log: 'ea10d51bcf88862dbcc36eb292017dfd+45', output: 'ea10d51bcf88862dbcc36eb292017dfd+45'}) - c_running.update_attributes!({state: Container::Locked}) - c_running.update_attributes!({state: Container::Running, + c_running.update!({state: Container::Locked}) + c_running.update!({state: Container::Running, progress: 0.15}) reused = Container.find_reusable(common_attrs) assert_not_nil reused @@ -450,14 +474,14 @@ class ContainerTest < ActiveSupport::TestCase c_running, _ = minimal_new(common_attrs.merge({use_existing: false})) assert_not_equal c_completed.uuid, c_running.uuid set_user_from_auth :dispatch1 - c_completed.update_attributes!({state: Container::Locked}) - c_completed.update_attributes!({state: Container::Running}) - c_completed.update_attributes!({state: Container::Complete, + c_completed.update!({state: Container::Locked}) + c_completed.update!({state: Container::Running}) + c_completed.update!({state: Container::Complete, exit_code: 0, log: 'ea10d51bcf88862dbcc36eb292017dfd+45', output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'}) - c_running.update_attributes!({state: Container::Locked}) - c_running.update_attributes!({state: Container::Running, + c_running.update!({state: Container::Locked}) + c_running.update!({state: Container::Running, progress: 0.15}) reused = Container.find_reusable(common_attrs) assert_not_nil reused @@ -471,9 +495,9 @@ class ContainerTest < ActiveSupport::TestCase c_running, _ = minimal_new(common_attrs.merge({use_existing: false})) assert_not_equal c_running.uuid, c_locked.uuid set_user_from_auth :dispatch1 - c_locked.update_attributes!({state: Container::Locked}) - c_running.update_attributes!({state: Container::Locked}) - c_running.update_attributes!({state: Container::Running, + c_locked.update!({state: Container::Locked}) + c_running.update!({state: Container::Locked}) + c_running.update!({state: Container::Running, progress: 0.15}) reused = Container.find_reusable(common_attrs) assert_not_nil reused @@ -487,7 +511,7 @@ class ContainerTest < ActiveSupport::TestCase c_queued, _ = minimal_new(common_attrs.merge({use_existing: false})) assert_not_equal c_queued.uuid, c_locked.uuid set_user_from_auth :dispatch1 - c_locked.update_attributes!({state: Container::Locked}) + c_locked.update!({state: Container::Locked}) reused = Container.find_reusable(common_attrs) assert_not_nil reused assert_equal reused.uuid, c_locked.uuid @@ -498,14 +522,42 @@ class ContainerTest < ActiveSupport::TestCase attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed"}}) c, _ = minimal_new(attrs) set_user_from_auth :dispatch1 - c.update_attributes!({state: Container::Locked}) - c.update_attributes!({state: Container::Running}) - c.update_attributes!({state: Container::Complete, + c.update!({state: Container::Locked}) + c.update!({state: Container::Running}) + c.update!({state: Container::Complete, exit_code: 33}) reused = Container.find_reusable(attrs) assert_nil reused end + [[false, false, true], + [false, true, true], + [true, false, false], + [true, true, true] + ].each do |c1_preemptible, c2_preemptible, should_reuse| + [[Container::Queued, 1], + [Container::Locked, 1], + [Container::Running, 0], # not cancelled yet, but obviously will be soon + ].each do |c1_state, c1_priority| + test "find_reusable for #{c2_preemptible ? '' : 'non-'}preemptible req should #{should_reuse ? '' : 'not'} reuse a #{c1_state} #{c1_preemptible ? '' : 'non-'}preemptible container with priority #{c1_priority}" do + configure_preemptible_instance_type + set_user_from_auth :active + c1_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"test" => name, "state" => c1_state}, scheduling_parameters: {"preemptible" => c1_preemptible}}) + c1, _ = minimal_new(c1_attrs) + set_user_from_auth :dispatch1 + c1.update!({state: Container::Locked}) if c1_state != Container::Queued + c1.update!({state: Container::Running, priority: c1_priority}) if c1_state == Container::Running + c2_attrs = c1_attrs.merge({scheduling_parameters: {"preemptible" => c2_preemptible}}) + reused = Container.find_reusable(c2_attrs) + if should_reuse && c1_priority > 0 + assert_not_nil reused + else + assert_nil reused + end + end + end + end + test "find_reusable with logging disabled" do set_user_from_auth :active Rails.logger.expects(:info).never @@ -590,6 +642,33 @@ class ContainerTest < ActiveSupport::TestCase assert_equal c1.uuid, reused.uuid end + test "find_reusable method with cuda" do + set_user_from_auth :active + # No cuda + no_cuda_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}, + runtime_constraints: {"vcpus" => 1, "ram" => 1, "keep_cache_disk"=>0, "keep_cache_ram"=>268435456, "API" => false, + "cuda" => {"device_count":0, "driver_version": "", "hardware_capability": ""}},}) + c1, _ = minimal_new(no_cuda_attrs) + assert_equal Container::Queued, c1.state + + # has cuda + cuda_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}, + runtime_constraints: {"vcpus" => 1, "ram" => 1, "keep_cache_disk"=>0, "keep_cache_ram"=>268435456, "API" => false, + "cuda" => {"device_count":1, "driver_version": "11.0", "hardware_capability": "9.0"}},}) + c2, _ = minimal_new(cuda_attrs) + assert_equal Container::Queued, c2.state + + # should find the no cuda one + reused = Container.find_reusable(no_cuda_attrs) + assert_not_nil reused + assert_equal reused.uuid, c1.uuid + + # should find the cuda one + reused = Container.find_reusable(cuda_attrs) + assert_not_nil reused + assert_equal reused.uuid, c2.uuid + end + test "Container running" do set_user_from_auth :active c, _ = minimal_new priority: 1 @@ -599,7 +678,7 @@ class ContainerTest < ActiveSupport::TestCase {state: Container::Complete}] c.lock - c.update_attributes! state: Container::Running + c.update! state: Container::Running check_illegal_modify c check_bogus_states c @@ -607,7 +686,7 @@ class ContainerTest < ActiveSupport::TestCase check_illegal_updates c, [{state: Container::Queued}] c.reload - c.update_attributes! priority: 3 + c.update! priority: 3 end test "Lock and unlock" do @@ -622,11 +701,11 @@ class ContainerTest < ActiveSupport::TestCase c.lock end c.reload - assert cr.update_attributes priority: 1 + assert cr.update priority: 1 - refute c.update_attributes(state: Container::Running), "not locked" + refute c.update(state: Container::Running), "not locked" c.reload - refute c.update_attributes(state: Container::Complete), "not locked" + refute c.update(state: Container::Complete), "not locked" c.reload assert c.lock, show_errors(c) @@ -640,13 +719,13 @@ class ContainerTest < ActiveSupport::TestCase refute c.locked_by_uuid refute c.auth_uuid - refute c.update_attributes(state: Container::Running), "not locked" + refute c.update(state: Container::Running), "not locked" c.reload refute c.locked_by_uuid refute c.auth_uuid assert c.lock, show_errors(c) - assert c.update_attributes(state: Container::Running), show_errors(c) + assert c.update(state: Container::Running), show_errors(c) assert c.locked_by_uuid assert c.auth_uuid @@ -663,7 +742,7 @@ class ContainerTest < ActiveSupport::TestCase end c.reload - assert c.update_attributes(state: Container::Complete), show_errors(c) + assert c.update(state: Container::Complete), show_errors(c) refute c.locked_by_uuid refute c.auth_uuid @@ -723,7 +802,7 @@ class ContainerTest < ActiveSupport::TestCase set_user_from_auth :active c, cr = minimal_new({container_count_max: 1}) set_user_from_auth :dispatch1 - assert c.update_attributes(state: Container::Cancelled), show_errors(c) + assert c.update(state: Container::Cancelled), show_errors(c) check_no_change_from_cancelled c cr.reload assert_equal ContainerRequest::Final, cr.state @@ -746,7 +825,7 @@ class ContainerTest < ActiveSupport::TestCase c, _ = minimal_new set_user_from_auth :dispatch1 assert c.lock, show_errors(c) - assert c.update_attributes(state: Container::Cancelled), show_errors(c) + assert c.update(state: Container::Cancelled), show_errors(c) check_no_change_from_cancelled c end @@ -766,9 +845,9 @@ class ContainerTest < ActiveSupport::TestCase c, _ = minimal_new set_user_from_auth :dispatch1 assert c.lock, show_errors(c) - assert c.update_attributes( + assert c.update( state: Container::Cancelled, - log: collections(:real_log_collection).portable_data_hash, + log: collections(:log_collection).portable_data_hash, ), show_errors(c) check_no_change_from_cancelled c end @@ -778,8 +857,8 @@ class ContainerTest < ActiveSupport::TestCase c, _ = minimal_new set_user_from_auth :dispatch1 c.lock - c.update_attributes! state: Container::Running - c.update_attributes! state: Container::Cancelled + c.update! state: Container::Running + c.update! state: Container::Cancelled check_no_change_from_cancelled c end @@ -829,36 +908,34 @@ class ContainerTest < ActiveSupport::TestCase set_user_from_auth :dispatch1 c.lock if start_state != Container::Locked - c.update_attributes! state: Container::Running + c.update! state: Container::Running if start_state != Container::Running - c.update_attributes! state: start_state + c.update! state: start_state end end end assert_equal c.state, start_state set_user_from_auth :active assert_raises(ArvadosModel::PermissionDeniedError) do - c.update_attributes! updates + c.update! updates end end end - test "Container only set exit code on complete" do + test "can only change exit code while running and at completion" do set_user_from_auth :active c, _ = minimal_new set_user_from_auth :dispatch1 c.lock - c.update_attributes! state: Container::Running - - check_illegal_updates c, [{exit_code: 1}, - {exit_code: 1, state: Container::Cancelled}] - - assert c.update_attributes(exit_code: 1, state: Container::Complete) + check_illegal_updates c, [{exit_code: 1}] + c.update! state: Container::Running + assert c.update(exit_code: 1) + assert c.update(exit_code: 1, state: Container::Complete) end test "locked_by_uuid can update log when locked/running, and output when running" do set_user_from_auth :active - logcoll = collections(:real_log_collection) + logcoll = collections(:container_log_collection) c, cr1 = minimal_new cr2 = ContainerRequest.new(DEFAULT_ATTRS) cr2.state = ContainerRequest::Committed @@ -872,8 +949,8 @@ class ContainerTest < ActiveSupport::TestCase set_user_from_auth :dispatch1 c.lock assert_equal c.locked_by_uuid, Thread.current[:api_client_authorization].uuid - c.update_attributes!(log: logpdh_time1) - c.update_attributes!(state: Container::Running) + c.update!(log: logpdh_time1) + c.update!(state: Container::Running) cr1.reload cr2.reload cr1log_uuid = cr1.log_uuid @@ -884,29 +961,29 @@ class ContainerTest < ActiveSupport::TestCase assert_not_equal logcoll.uuid, cr2log_uuid assert_not_equal cr1log_uuid, cr2log_uuid - logcoll.update_attributes!(manifest_text: logcoll.manifest_text + ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n") + logcoll.update!(manifest_text: logcoll.manifest_text + ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n") logpdh_time2 = logcoll.portable_data_hash - assert c.update_attributes(output: collections(:collection_owned_by_active).portable_data_hash) - assert c.update_attributes(log: logpdh_time2) - assert c.update_attributes(state: Container::Complete, log: logcoll.portable_data_hash) + assert c.update(output: collections(:collection_owned_by_active).portable_data_hash) + assert c.update(log: logpdh_time2) + assert c.update(state: Container::Complete, log: logcoll.portable_data_hash) c.reload assert_equal collections(:collection_owned_by_active).portable_data_hash, c.output assert_equal logpdh_time2, c.log - refute c.update_attributes(output: nil) - refute c.update_attributes(log: nil) + refute c.update(output: nil) + refute c.update(log: nil) cr1.reload cr2.reload assert_equal cr1log_uuid, cr1.log_uuid assert_equal cr2log_uuid, cr2.log_uuid assert_equal 1, Collection.where(uuid: [cr1log_uuid, cr2log_uuid]).to_a.collect(&:portable_data_hash).uniq.length - assert_equal ". acbd18db4cc2f85cedef654fccc4a4d8+3 cdd549ae79fe6640fa3d5c6261d8303c+195 0:3:foo.txt 3:195:zzzzz-8i9sb-0vsrcqi7whchuil.log.txt -./log\\040for\\040container\\040#{cr1.container_uuid} acbd18db4cc2f85cedef654fccc4a4d8+3 cdd549ae79fe6640fa3d5c6261d8303c+195 0:3:foo.txt 3:195:zzzzz-8i9sb-0vsrcqi7whchuil.log.txt + assert_equal ". 8c12f5f5297b7337598170c6f531fcee+7882 acbd18db4cc2f85cedef654fccc4a4d8+3 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 7882:3:foo.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt +./log\\040for\\040container\\040#{cr1.container_uuid} 8c12f5f5297b7337598170c6f531fcee+7882 acbd18db4cc2f85cedef654fccc4a4d8+3 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 7882:3:foo.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt ", Collection.find_by_uuid(cr1log_uuid).manifest_text end ["auth_uuid", "runtime_token"].each do |tok| - test "#{tok} can set output, progress, runtime_status, state on running container -- but not log" do + test "#{tok} can set output, progress, runtime_status, state, exit_code on running container -- but not log" do if tok == "runtime_token" set_user_from_auth :spectator c, _ = minimal_new(container_image: "9ae44d5792468c58bcf85ce7353c7027+124", @@ -917,7 +994,7 @@ class ContainerTest < ActiveSupport::TestCase end set_user_from_auth :dispatch1 c.lock - c.update_attributes! state: Container::Running + c.update! state: Container::Running if tok == "runtime_token" auth = ApiClientAuthorization.validate(token: c.runtime_token) @@ -933,12 +1010,14 @@ class ContainerTest < ActiveSupport::TestCase Thread.current[:user] = auth.user end - assert c.update_attributes(output: collections(:collection_owned_by_active).portable_data_hash) - assert c.update_attributes(runtime_status: {'warning' => 'something happened'}) - assert c.update_attributes(progress: 0.5) - refute c.update_attributes(log: collections(:real_log_collection).portable_data_hash) + assert c.update(gateway_address: "127.0.0.1:9") + assert c.update(output: collections(:collection_owned_by_active).portable_data_hash) + assert c.update(runtime_status: {'warning' => 'something happened'}) + assert c.update(progress: 0.5) + assert c.update(exit_code: 0) + refute c.update(log: collections(:log_collection).portable_data_hash) c.reload - assert c.update_attributes(state: Container::Complete, exit_code: 0) + assert c.update(state: Container::Complete, exit_code: 0) end end @@ -947,13 +1026,13 @@ class ContainerTest < ActiveSupport::TestCase c, _ = minimal_new set_user_from_auth :dispatch1 c.lock - c.update_attributes! state: Container::Running + c.update! state: Container::Running Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid) Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id) assert_raises ActiveRecord::RecordInvalid do - c.update_attributes! output: collections(:collection_not_readable_by_active).portable_data_hash + c.update! output: collections(:collection_not_readable_by_active).portable_data_hash end end @@ -962,11 +1041,11 @@ class ContainerTest < ActiveSupport::TestCase c, _ = minimal_new set_user_from_auth :dispatch1 c.lock - c.update_attributes! state: Container::Running + c.update! state: Container::Running set_user_from_auth :running_to_be_deleted_container_auth assert_raises(ArvadosModel::PermissionDeniedError) do - c.update_attributes(output: collections(:foo_file).portable_data_hash) + c.update(output: collections(:foo_file).portable_data_hash) end end @@ -975,13 +1054,13 @@ class ContainerTest < ActiveSupport::TestCase c, _ = minimal_new set_user_from_auth :dispatch1 c.lock - c.update_attributes! state: Container::Running + c.update! state: Container::Running output = Collection.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jk') assert output.is_trashed - assert c.update_attributes output: output.portable_data_hash - assert c.update_attributes! state: Container::Complete + assert c.update output: output.portable_data_hash + assert c.update! state: Container::Complete end test "not allowed to set trashed output that is not readable by current user" do @@ -989,7 +1068,7 @@ class ContainerTest < ActiveSupport::TestCase c, _ = minimal_new set_user_from_auth :dispatch1 c.lock - c.update_attributes! state: Container::Running + c.update! state: Container::Running output = Collection.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jr') @@ -997,7 +1076,7 @@ class ContainerTest < ActiveSupport::TestCase Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id) assert_raises ActiveRecord::RecordInvalid do - c.update_attributes! output: output.portable_data_hash + c.update! output: output.portable_data_hash end end @@ -1020,12 +1099,12 @@ class ContainerTest < ActiveSupport::TestCase container_count_max: 1, runtime_token: api_client_authorizations(:active).token) set_user_from_auth :dispatch1 c.lock - c.update_attributes!(state: Container::Running) + c.update!(state: Container::Running) c.reload assert c.secret_mounts.has_key?('/secret') assert_equal api_client_authorizations(:active).token, c.runtime_token - c.update_attributes!(final_attrs) + c.update!(final_attrs) c.reload assert_equal({}, c.secret_mounts) assert_nil c.runtime_token @@ -1035,4 +1114,266 @@ class ContainerTest < ActiveSupport::TestCase assert_no_secrets_logged end end + + def configure_preemptible_instance_type + Rails.configuration.InstanceTypes = ConfigLoader.to_OrderedOptions({ + "a1.small.pre" => { + "Preemptible" => true, + "Price" => 0.1, + "ProviderType" => "a1.small", + "VCPUs" => 1, + "RAM" => 1000000000, + }, + }) + end + + def vary_parameters(**kwargs) + # kwargs is a hash that maps parameters to an array of values. + # This function enumerates every possible hash where each key has one of + # the values from its array. + # The output keys are strings since that's what container hash attributes + # want. + # A nil value yields a hash without that key. + [[:_, nil]].product( + *kwargs.map { |(key, values)| [key.to_s].product(values) }, + ).map { |param_pairs| Hash[param_pairs].compact } + end + + def retry_with_scheduling_parameters(param_hashes) + set_user_from_auth :admin + containers = {} + requests = [] + param_hashes.each do |scheduling_parameters| + container, request = minimal_new(scheduling_parameters: scheduling_parameters) + containers[container.uuid] = container + requests << request + end + refute(containers.empty?, "buggy test: no scheduling parameters enumerated") + assert_equal(1, containers.length) + _, container1 = containers.shift + container1.lock + container1.update!(state: Container::Cancelled) + container1.reload + request1 = requests.shift + request1.reload + assert_not_equal(container1.uuid, request1.container_uuid) + requests.each do |request| + request.reload + assert_equal(request1.container_uuid, request.container_uuid) + end + container2 = Container.find_by_uuid(request1.container_uuid) + assert_not_nil(container2) + return container2 + end + + preemptible_values = [true, false, nil] + preemptible_values.permutation(1).chain( + preemptible_values.product(preemptible_values), + preemptible_values.product(preemptible_values, preemptible_values), + ).each do |preemptible_a| + # If the first req has preemptible=true but a subsequent req + # doesn't, we want to avoid reusing the first container, so this + # test isn't appropriate. + next if preemptible_a[0] && + ((preemptible_a.length > 1 && !preemptible_a[1]) || + (preemptible_a.length > 2 && !preemptible_a[2])) + test "retry requests scheduled with preemptible=#{preemptible_a}" do + configure_preemptible_instance_type + param_hashes = vary_parameters(preemptible: preemptible_a) + container = retry_with_scheduling_parameters(param_hashes) + assert_equal(preemptible_a.all?, + container.scheduling_parameters["preemptible"] || false) + end + end + + partition_values = [nil, [], ["alpha"], ["alpha", "bravo"], ["bravo", "charlie"]] + partition_values.permutation(1).chain( + partition_values.permutation(2), + ).each do |partitions_a| + test "retry requests scheduled with partitions=#{partitions_a}" do + param_hashes = vary_parameters(partitions: partitions_a) + container = retry_with_scheduling_parameters(param_hashes) + expected = if partitions_a.any? { |value| value.nil? or value.empty? } + [] + else + partitions_a.flatten.uniq + end + actual = container.scheduling_parameters["partitions"] || [] + assert_equal(expected.sort, actual.sort) + end + end + + runtime_values = [nil, 0, 1, 2, 3] + runtime_values.permutation(1).chain( + runtime_values.permutation(2), + runtime_values.permutation(3), + ).each do |max_run_time_a| + test "retry requests scheduled with max_run_time=#{max_run_time_a}" do + param_hashes = vary_parameters(max_run_time: max_run_time_a) + container = retry_with_scheduling_parameters(param_hashes) + expected = if max_run_time_a.any? { |value| value.nil? or value == 0 } + 0 + else + max_run_time_a.max + end + actual = container.scheduling_parameters["max_run_time"] || 0 + assert_equal(expected, actual) + end + end + + test "retry requests with multi-varied scheduling parameters" do + configure_preemptible_instance_type + param_hashes = [{ + "partitions": ["alpha", "bravo"], + "preemptible": false, + "max_run_time": 10, + }, { + "partitions": ["alpha", "charlie"], + "max_run_time": 20, + }, { + "partitions": ["bravo", "charlie"], + "preemptible": true, + "max_run_time": 30, + }] + container = retry_with_scheduling_parameters(param_hashes) + actual = container.scheduling_parameters + assert_equal(["alpha", "bravo", "charlie"], actual["partitions"]&.sort) + assert_equal(false, actual["preemptible"] || false) + assert_equal(30, actual["max_run_time"]) + end + + test "retry requests with unset scheduling parameters" do + configure_preemptible_instance_type + param_hashes = vary_parameters( + preemptible: [nil, true], + partitions: [nil, ["alpha"]], + max_run_time: [nil, 5], + ) + container = retry_with_scheduling_parameters(param_hashes) + actual = container.scheduling_parameters + assert_equal([], actual["partitions"] || []) + assert_equal(false, actual["preemptible"] || false) + assert_equal(0, actual["max_run_time"] || 0) + end + + test "retry requests with default scheduling parameters" do + configure_preemptible_instance_type + param_hashes = vary_parameters( + preemptible: [false, true], + partitions: [[], ["bravo"]], + max_run_time: [0, 1], + ) + container = retry_with_scheduling_parameters(param_hashes) + actual = container.scheduling_parameters + assert_equal([], actual["partitions"] || []) + assert_equal(false, actual["preemptible"] || false) + assert_equal(0, actual["max_run_time"] || 0) + end + + def run_container(request_params, final_attrs) + final_attrs[:state] ||= Container::Complete + if final_attrs[:state] == Container::Complete + final_attrs[:exit_code] ||= 0 + final_attrs[:log] ||= collections(:log_collection).portable_data_hash + final_attrs[:output] ||= collections(:multilevel_collection_1).portable_data_hash + end + container, request = minimal_new(request_params) + container.lock + container.update!(state: Container::Running) + container.update!(final_attrs) + return container, request + end + + def check_reuse_with_variations(default_keep_cache_ram, vary_attr, start_value, variations) + container_params = REUSABLE_ATTRS_SLIM.merge(vary_attr => start_value) + orig_default = Rails.configuration.Containers.DefaultKeepCacheRAM + begin + Rails.configuration.Containers.DefaultKeepCacheRAM = default_keep_cache_ram + set_user_from_auth :admin + expected, _ = run_container(container_params, {}) + variations.each do |variation| + full_variation = REUSABLE_ATTRS_SLIM[vary_attr].merge(variation) + parameters = REUSABLE_ATTRS_SLIM.merge(vary_attr => full_variation) + actual = Container.find_reusable(parameters) + assert_equal(expected.uuid, actual&.uuid, + "request with #{vary_attr}=#{variation} did not reuse container") + end + ensure + Rails.configuration.Containers.DefaultKeepCacheRAM = orig_default + end + end + + # Test that we can reuse a container with a known keep_cache_ram constraint, + # no matter what keep_cache_* constraints the new request uses. + [0, 2 << 30, 4 << 30].product( + [0, 1], + [true, false], + ).each do |(default_keep_cache_ram, multiplier, keep_disk_constraint)| + test "reuse request with DefaultKeepCacheRAM=#{default_keep_cache_ram}, keep_cache_ram*=#{multiplier}, keep_cache_disk=#{keep_disk_constraint}" do + runtime_constraints = REUSABLE_ATTRS_SLIM[:runtime_constraints].merge( + "keep_cache_ram" => default_keep_cache_ram * multiplier, + ) + if not keep_disk_constraint + # Simulate a container that predates keep_cache_disk by deleting + # the constraint entirely. + runtime_constraints.delete("keep_cache_disk") + end + # Important values are: + # * 0 + # * 2GiB, the minimum default keep_cache_disk + # * 8GiB, the default keep_cache_disk based on container ram + # * 32GiB, the maximum default keep_cache_disk + # Check these values and values in between. + vary_values = [0, 1, 2, 6, 8, 10, 32, 33].map { |v| v << 30 }.to_a + variations = vary_parameters(keep_cache_ram: vary_values) + .chain(vary_parameters(keep_cache_disk: vary_values)) + check_reuse_with_variations( + default_keep_cache_ram, + :runtime_constraints, + runtime_constraints, + variations, + ) + end + end + + # Test that we can reuse a container with a known keep_cache_disk constraint, + # no matter what keep_cache_* constraints the new request uses. + # keep_cache_disk values are the important values discussed in the test above. + [0, 2 << 30, 4 << 30] + .product([0, 2 << 30, 8 << 30, 32 << 30]) + .each do |(default_keep_cache_ram, keep_cache_disk)| + test "reuse request with DefaultKeepCacheRAM=#{default_keep_cache_ram} and keep_cache_disk=#{keep_cache_disk}" do + runtime_constraints = REUSABLE_ATTRS_SLIM[:runtime_constraints].merge( + "keep_cache_disk" => keep_cache_disk, + ) + vary_values = [0, 1, 2, 6, 8, 10, 32, 33].map { |v| v << 30 }.to_a + variations = vary_parameters(keep_cache_ram: vary_values) + .chain(vary_parameters(keep_cache_disk: vary_values)) + check_reuse_with_variations( + default_keep_cache_ram, + :runtime_constraints, + runtime_constraints, + variations, + ) + end + end + + # Test that a container request can reuse a container with an exactly + # matching keep_cache_* constraint, no matter what the defaults. + [0, 2 << 30, 4 << 30].product( + ["keep_cache_disk", "keep_cache_ram"], + [135790, 13 << 30, 135 << 30], + ).each do |(default_keep_cache_ram, constraint_key, constraint_value)| + test "reuse request with #{constraint_key}=#{constraint_value} and DefaultKeepCacheRAM=#{default_keep_cache_ram}" do + runtime_constraints = REUSABLE_ATTRS_SLIM[:runtime_constraints].merge( + constraint_key => constraint_value, + ) + check_reuse_with_variations( + default_keep_cache_ram, + :runtime_constraints, + runtime_constraints, + [runtime_constraints], + ) + end + end end