X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/c031d4145d8ab1a11463acf5b20ef4df1afe00a4..40dc04d6fa45b6b727e94986491c06132db4a582:/services/api/test/unit/container_test.rb diff --git a/services/api/test/unit/container_test.rb b/services/api/test/unit/container_test.rb index 2a9ff5bf4c..bcf99da2e3 100644 --- a/services/api/test/unit/container_test.rb +++ b/services/api/test/unit/container_test.rb @@ -14,7 +14,7 @@ class ContainerTest < ActiveSupport::TestCase container_image: 'fa3c1a9cb6783f85f2ecda037e07b8c3+167', output_path: '/tmp', priority: 1, - runtime_constraints: {"vcpus" => 1, "ram" => 1}, + runtime_constraints: {"vcpus" => 1, "ram" => 1, "cuda" => {"device_count":0, "driver_version": "", "hardware_capability": ""}}, } REUSABLE_COMMON_ATTRS = { @@ -23,8 +23,10 @@ class ContainerTest < ActiveSupport::TestCase command: ["echo", "hello"], output_path: "test", runtime_constraints: { + "API" => false, + "keep_cache_ram" => 0, "ram" => 12000000000, - "vcpus" => 4, + "vcpus" => 4 }, mounts: { "test" => {"kind" => "json"}, @@ -184,7 +186,7 @@ class ContainerTest < ActiveSupport::TestCase assert_equal c1.runtime_status, {} assert_equal Container::Queued, c1.state - assert_raises ActiveRecord::RecordInvalid do + assert_raises ArvadosModel::PermissionDeniedError do c1.update_attributes! runtime_status: {'error' => 'Oops!'} end @@ -227,11 +229,12 @@ class ContainerTest < ActiveSupport::TestCase set_user_from_auth :active env = {"C" => "3", "B" => "2", "A" => "1"} m = {"F" => {"kind" => "3"}, "E" => {"kind" => "2"}, "D" => {"kind" => "1"}} - rc = {"vcpus" => 1, "ram" => 1, "keep_cache_ram" => 1} + rc = {"vcpus" => 1, "ram" => 1, "keep_cache_ram" => 1, "API" => true, "cuda" => {"device_count":0, "driver_version": "", "hardware_capability": ""}} c, _ = minimal_new(environment: env, mounts: m, runtime_constraints: rc) - assert_equal c.environment.to_json, Container.deep_sort_hash(env).to_json - assert_equal c.mounts.to_json, Container.deep_sort_hash(m).to_json - assert_equal c.runtime_constraints.to_json, Container.deep_sort_hash(rc).to_json + c.reload + assert_equal Container.deep_sort_hash(env).to_json, c.environment.to_json + assert_equal Container.deep_sort_hash(m).to_json, c.mounts.to_json + assert_equal Container.deep_sort_hash(rc).to_json, c.runtime_constraints.to_json end test 'deep_sort_hash on array of hashes' do @@ -241,7 +244,7 @@ class ContainerTest < ActiveSupport::TestCase end test "find_reusable method should select higher priority queued container" do - Rails.configuration.log_reuse_decisions = true + Rails.configuration.Containers.LogReuseDecisions = true set_user_from_auth :active common_attrs = REUSABLE_COMMON_ATTRS.merge({environment:{"var" => "queued"}}) c_low_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:1})) @@ -388,9 +391,11 @@ class ContainerTest < ActiveSupport::TestCase runtime_status: {'warning' => 'This is not an error'}, progress: 0.15}) c_faster_started_second.update_attributes!({state: Container::Locked}) + assert_equal 0, Container.where("runtime_status->'error' is not null").count c_faster_started_second.update_attributes!({state: Container::Running, runtime_status: {'error' => 'Something bad happened'}, progress: 0.2}) + assert_equal 1, Container.where("runtime_status->'error' is not null").count reused = Container.find_reusable(common_attrs) assert_not_nil reused # Selected the non-failing container even if it's the one with less progress done @@ -509,7 +514,7 @@ class ContainerTest < ActiveSupport::TestCase test "find_reusable with logging enabled" do set_user_from_auth :active - Rails.configuration.log_reuse_decisions = true + Rails.configuration.Containers.LogReuseDecisions = true Rails.logger.expects(:info).at_least(3) Container.find_reusable(REUSABLE_COMMON_ATTRS) end @@ -559,6 +564,7 @@ class ContainerTest < ActiveSupport::TestCase assert_equal Container::Queued, c1.state reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token))) # See #14584 + assert_not_nil reused assert_equal c1.uuid, reused.uuid end @@ -569,6 +575,7 @@ class ContainerTest < ActiveSupport::TestCase assert_equal Container::Queued, c1.state reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token))) # See #14584 + assert_not_nil reused assert_equal c1.uuid, reused.uuid end @@ -579,9 +586,37 @@ class ContainerTest < ActiveSupport::TestCase assert_equal Container::Queued, c1.state reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token))) # See #14584 + assert_not_nil reused assert_equal c1.uuid, reused.uuid end + test "find_reusable method with cuda" do + set_user_from_auth :active + # No cuda + no_cuda_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}, + runtime_constraints: {"vcpus" => 1, "ram" => 1, "keep_cache_ram"=>268435456, "API" => false, + "cuda" => {"device_count":0, "driver_version": "", "hardware_capability": ""}},}) + c1, _ = minimal_new(no_cuda_attrs) + assert_equal Container::Queued, c1.state + + # has cuda + cuda_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}, + runtime_constraints: {"vcpus" => 1, "ram" => 1, "keep_cache_ram"=>268435456, "API" => false, + "cuda" => {"device_count":1, "driver_version": "11.0", "hardware_capability": "9.0"}},}) + c2, _ = minimal_new(cuda_attrs) + assert_equal Container::Queued, c2.state + + # should find the no cuda one + reused = Container.find_reusable(no_cuda_attrs) + assert_not_nil reused + assert_equal reused.uuid, c1.uuid + + # should find the cuda one + reused = Container.find_reusable(cuda_attrs) + assert_not_nil reused + assert_equal reused.uuid, c2.uuid + end + test "Container running" do set_user_from_auth :active c, _ = minimal_new priority: 1 @@ -661,6 +696,54 @@ class ContainerTest < ActiveSupport::TestCase auth_exp = ApiClientAuthorization.find_by_uuid(auth_uuid_was).expires_at assert_operator auth_exp, :<, db_current_time + + assert_nil ApiClientAuthorization.validate(token: ApiClientAuthorization.find_by_uuid(auth_uuid_was).token) + end + + test "Exceed maximum lock-unlock cycles" do + Rails.configuration.Containers.MaxDispatchAttempts = 3 + + set_user_from_auth :active + c, cr = minimal_new + + set_user_from_auth :dispatch1 + assert_equal Container::Queued, c.state + assert_equal 0, c.lock_count + + c.lock + c.reload + assert_equal 1, c.lock_count + assert_equal Container::Locked, c.state + + c.unlock + c.reload + assert_equal 1, c.lock_count + assert_equal Container::Queued, c.state + + c.lock + c.reload + assert_equal 2, c.lock_count + assert_equal Container::Locked, c.state + + c.unlock + c.reload + assert_equal 2, c.lock_count + assert_equal Container::Queued, c.state + + c.lock + c.reload + assert_equal 3, c.lock_count + assert_equal Container::Locked, c.state + + c.unlock + c.reload + assert_equal 3, c.lock_count + assert_equal Container::Cancelled, c.state + + assert_raise(ArvadosModel::LockFailedError) do + # Cancelled to Locked is not allowed + c.lock + end end test "Container queued cancel" do @@ -677,6 +760,14 @@ class ContainerTest < ActiveSupport::TestCase assert_equal 1, Container.readable_by(users(:active)).where(state: "Queued").count end + test "Containers with no matching request are readable by admin" do + uuids = Container.includes('container_requests').where(container_requests: {uuid: nil}).collect(&:uuid) + assert_not_empty uuids + assert_empty Container.readable_by(users(:active)).where(uuid: uuids) + assert_not_empty Container.readable_by(users(:admin)).where(uuid: uuids) + assert_equal uuids.count, Container.readable_by(users(:admin)).where(uuid: uuids).count + end + test "Container locked cancel" do set_user_from_auth :active c, _ = minimal_new @@ -686,6 +777,17 @@ class ContainerTest < ActiveSupport::TestCase check_no_change_from_cancelled c end + test "Container locked with non-expiring token" do + Rails.configuration.API.TokenMaxLifetime = 1.hour + set_user_from_auth :active + c, _ = minimal_new + set_user_from_auth :dispatch1 + assert c.lock, show_errors(c) + refute c.auth.nil? + assert c.auth.expires_at.nil? + assert c.auth.user_id == User.find_by_uuid(users(:active).uuid).id + end + test "Container locked cancel with log" do set_user_from_auth :active c, _ = minimal_new @@ -721,16 +823,61 @@ class ContainerTest < ActiveSupport::TestCase end end - test "Container only set exit code on complete" do + [ + [Container::Queued, {state: Container::Locked}], + [Container::Queued, {state: Container::Running}], + [Container::Queued, {state: Container::Complete}], + [Container::Queued, {state: Container::Cancelled}], + [Container::Queued, {priority: 123456789}], + [Container::Queued, {runtime_status: {'error' => 'oops'}}], + [Container::Queued, {cwd: '/'}], + [Container::Locked, {state: Container::Running}], + [Container::Locked, {state: Container::Queued}], + [Container::Locked, {priority: 123456789}], + [Container::Locked, {runtime_status: {'error' => 'oops'}}], + [Container::Locked, {cwd: '/'}], + [Container::Running, {state: Container::Complete}], + [Container::Running, {state: Container::Cancelled}], + [Container::Running, {priority: 123456789}], + [Container::Running, {runtime_status: {'error' => 'oops'}}], + [Container::Running, {cwd: '/'}], + [Container::Running, {gateway_address: "172.16.0.1:12345"}], + [Container::Running, {interactive_session_started: true}], + [Container::Complete, {state: Container::Cancelled}], + [Container::Complete, {priority: 123456789}], + [Container::Complete, {runtime_status: {'error' => 'oops'}}], + [Container::Complete, {cwd: '/'}], + [Container::Cancelled, {cwd: '/'}], + ].each do |start_state, updates| + test "Container update #{updates.inspect} when #{start_state} forbidden for non-admin" do + set_user_from_auth :active + c, _ = minimal_new + if start_state != Container::Queued + set_user_from_auth :dispatch1 + c.lock + if start_state != Container::Locked + c.update_attributes! state: Container::Running + if start_state != Container::Running + c.update_attributes! state: start_state + end + end + end + assert_equal c.state, start_state + set_user_from_auth :active + assert_raises(ArvadosModel::PermissionDeniedError) do + c.update_attributes! updates + end + end + end + + test "can only change exit code while running and at completion" do set_user_from_auth :active c, _ = minimal_new set_user_from_auth :dispatch1 c.lock + check_illegal_updates c, [{exit_code: 1}] c.update_attributes! state: Container::Running - - check_illegal_updates c, [{exit_code: 1}, - {exit_code: 1, state: Container::Cancelled}] - + assert c.update_attributes(exit_code: 1) assert c.update_attributes(exit_code: 1, state: Container::Complete) end @@ -777,11 +924,14 @@ class ContainerTest < ActiveSupport::TestCase cr2.reload assert_equal cr1log_uuid, cr1.log_uuid assert_equal cr2log_uuid, cr2.log_uuid - assert_equal [logpdh_time2], Collection.where(uuid: [cr1log_uuid, cr2log_uuid]).to_a.collect(&:portable_data_hash).uniq + assert_equal 1, Collection.where(uuid: [cr1log_uuid, cr2log_uuid]).to_a.collect(&:portable_data_hash).uniq.length + assert_equal ". acbd18db4cc2f85cedef654fccc4a4d8+3 cdd549ae79fe6640fa3d5c6261d8303c+195 0:3:foo.txt 3:195:zzzzz-8i9sb-0vsrcqi7whchuil.log.txt +./log\\040for\\040container\\040#{cr1.container_uuid} acbd18db4cc2f85cedef654fccc4a4d8+3 cdd549ae79fe6640fa3d5c6261d8303c+195 0:3:foo.txt 3:195:zzzzz-8i9sb-0vsrcqi7whchuil.log.txt +", Collection.find_by_uuid(cr1log_uuid).manifest_text end ["auth_uuid", "runtime_token"].each do |tok| - test "#{tok} can set output, progress, runtime_status, state on running container -- but not log" do + test "#{tok} can set output, progress, runtime_status, state, exit_code on running container -- but not log" do if tok == "runtime_token" set_user_from_auth :spectator c, _ = minimal_new(container_image: "9ae44d5792468c58bcf85ce7353c7027+124", @@ -811,6 +961,7 @@ class ContainerTest < ActiveSupport::TestCase assert c.update_attributes(output: collections(:collection_owned_by_active).portable_data_hash) assert c.update_attributes(runtime_status: {'warning' => 'something happened'}) assert c.update_attributes(progress: 0.5) + assert c.update_attributes(exit_code: 0) refute c.update_attributes(log: collections(:real_log_collection).portable_data_hash) c.reload assert c.update_attributes(state: Container::Complete, exit_code: 0) @@ -840,7 +991,9 @@ class ContainerTest < ActiveSupport::TestCase c.update_attributes! state: Container::Running set_user_from_auth :running_to_be_deleted_container_auth - refute c.update_attributes(output: collections(:foo_file).portable_data_hash) + assert_raises(ArvadosModel::PermissionDeniedError) do + c.update_attributes(output: collections(:foo_file).portable_data_hash) + end end test "can set trashed output on running container" do @@ -874,6 +1027,15 @@ class ContainerTest < ActiveSupport::TestCase end end + test "user cannot delete" do + set_user_from_auth :active + c, _ = minimal_new + assert_raises ArvadosModel::PermissionDeniedError do + c.destroy + end + assert Container.find_by_uuid(c.uuid) + end + [ {state: Container::Complete, exit_code: 0, output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'}, {state: Container::Cancelled},