X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/6ebfa334f7965f2c4dbe1f52676993dcbd745f65..0f644e242ef37c911ad3dc25aca8135c339de349:/services/api/test/unit/container_request_test.rb diff --git a/services/api/test/unit/container_request_test.rb b/services/api/test/unit/container_request_test.rb index ecfbe465e7..280e93962c 100644 --- a/services/api/test/unit/container_request_test.rb +++ b/services/api/test/unit/container_request_test.rb @@ -1,6 +1,14 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + require 'test_helper' +require 'helpers/docker_migration_helper' class ContainerRequestTest < ActiveSupport::TestCase + include DockerMigrationHelper + include DbCurrentTime + def create_minimal_req! attrs={} defaults = { command: ["echo", "foo"], @@ -120,6 +128,8 @@ class ContainerRequestTest < ActiveSupport::TestCase cr.reload + assert_equal({"vcpus" => 2, "ram" => 30}, cr.runtime_constraints) + assert_not_nil cr.container_uuid c = Container.find_by_uuid cr.container_uuid assert_not_nil c @@ -129,7 +139,7 @@ class ContainerRequestTest < ActiveSupport::TestCase assert_equal({}, c.environment) assert_equal({"/out" => {"kind"=>"tmp", "capacity"=>1000000}}, c.mounts) assert_equal "/out", c.output_path - assert_equal({"vcpus" => 2, "ram" => 30}, c.runtime_constraints) + assert_equal({"keep_cache_ram"=>268435456, "vcpus" => 2, "ram" => 30}, c.runtime_constraints) assert_equal 1, c.priority assert_raises(ActiveRecord::RecordInvalid) do @@ -230,10 +240,12 @@ class ContainerRequestTest < ActiveSupport::TestCase cr.reload assert_equal "Committed", cr.state + output_pdh = '1f4b0bc7583c2a7f9102c395f4ffc5e3+45' + log_pdh = 'fa7aeb5140e2848d39b416daeef4ffc5+45' act_as_system_user do c.update_attributes!(state: Container::Complete, - output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45', - log: 'fa7aeb5140e2848d39b416daeef4ffc5+45') + output: output_pdh, + log: log_pdh) end cr.reload @@ -244,6 +256,12 @@ class ContainerRequestTest < ActiveSupport::TestCase owner_uuid: project.uuid).count, "Container #{out_type} should be copied to #{project.uuid}") end + assert_not_nil cr.output_uuid + assert_not_nil cr.log_uuid + output = Collection.find_by_uuid cr.output_uuid + assert_equal output_pdh, output.portable_data_hash + log = Collection.find_by_uuid cr.log_uuid + assert_equal log_pdh, log.portable_data_hash end test "Container makes container request, then is cancelled" do @@ -276,7 +294,7 @@ class ContainerRequestTest < ActiveSupport::TestCase end [ - ['active', 'zzzzz-dz642-runningcontainr'], + ['running_container_auth', 'zzzzz-dz642-runningcontainr'], ['active_no_prefs', nil], ].each do |token, expected| test "create as #{token} and expect requesting_container_uuid to be #{expected}" do @@ -299,8 +317,7 @@ class ContainerRequestTest < ActiveSupport::TestCase lambda { |resolved| resolved["ram"] == 1234234234 }], ].each do |rc, okfunc| test "resolve runtime constraint range #{rc} to values" do - cr = ContainerRequest.new(runtime_constraints: rc) - resolved = cr.send :runtime_constraints_for_container + resolved = Container.resolve_runtime_constraints(rc) assert(okfunc.call(resolved), "container runtime_constraints was #{resolved.inspect}") end @@ -332,10 +349,9 @@ class ContainerRequestTest < ActiveSupport::TestCase ].each do |mounts, okfunc| test "resolve mounts #{mounts.inspect} to values" do set_user_from_auth :active - cr = ContainerRequest.new(mounts: mounts) - resolved = cr.send :mounts_for_container + resolved = Container.resolve_mounts(mounts) assert(okfunc.call(resolved), - "mounts_for_container returned #{resolved.inspect}") + "Container.resolve_mounts returned #{resolved.inspect}") end end @@ -348,9 +364,8 @@ class ContainerRequestTest < ActiveSupport::TestCase "path" => "/foo", }, } - cr = ContainerRequest.new(mounts: m) assert_raises(ArvadosModel::UnresolvableContainerError) do - cr.send :mounts_for_container + Container.resolve_mounts(m) end end @@ -364,9 +379,8 @@ class ContainerRequestTest < ActiveSupport::TestCase "path" => "/foo", }, } - cr = ContainerRequest.new(mounts: m) assert_raises(ArgumentError) do - cr.send :mounts_for_container + Container.resolve_mounts(m) end end @@ -374,20 +388,19 @@ class ContainerRequestTest < ActiveSupport::TestCase 'arvados/apitestfixture', 'd8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678', ].each do |tag| - test "container_image_for_container(#{tag.inspect})" do + test "Container.resolve_container_image(#{tag.inspect})" do set_user_from_auth :active - cr = ContainerRequest.new(container_image: tag) - resolved = cr.send :container_image_for_container + resolved = Container.resolve_container_image(tag) assert_equal resolved, collections(:docker_image).portable_data_hash end end - test "container_image_for_container(pdh)" do + test "Container.resolve_container_image(pdh)" do set_user_from_auth :active - [:docker_image, :docker_image_1_12].each do |coll| + [[:docker_image, 'v1'], [:docker_image_1_12, 'v2']].each do |coll, ver| + Rails.configuration.docker_image_formats = [ver] pdh = collections(coll).portable_data_hash - cr = ContainerRequest.new(container_image: pdh) - resolved = cr.send :container_image_for_container + resolved = Container.resolve_container_image(pdh) assert_equal resolved, pdh end end @@ -398,13 +411,79 @@ class ContainerRequestTest < ActiveSupport::TestCase ].each do |img| test "container_image_for_container(#{img.inspect}) => 422" do set_user_from_auth :active - cr = ContainerRequest.new(container_image: img) assert_raises(ArvadosModel::UnresolvableContainerError) do - cr.send :container_image_for_container + Container.resolve_container_image(img) end end end + test "migrated docker image" do + Rails.configuration.docker_image_formats = ['v2'] + add_docker19_migration_link + + # Test that it returns only v2 images even though request is for v1 image. + + set_user_from_auth :active + cr = create_minimal_req!(command: ["true", "1"], + container_image: collections(:docker_image).portable_data_hash) + assert_equal(Container.resolve_container_image(cr.container_image), + collections(:docker_image_1_12).portable_data_hash) + + cr = create_minimal_req!(command: ["true", "2"], + container_image: links(:docker_image_collection_tag).name) + assert_equal(Container.resolve_container_image(cr.container_image), + collections(:docker_image_1_12).portable_data_hash) + end + + test "use unmigrated docker image" do + Rails.configuration.docker_image_formats = ['v1'] + add_docker19_migration_link + + # Test that it returns only supported v1 images even though there is a + # migration link. + + set_user_from_auth :active + cr = create_minimal_req!(command: ["true", "1"], + container_image: collections(:docker_image).portable_data_hash) + assert_equal(Container.resolve_container_image(cr.container_image), + collections(:docker_image).portable_data_hash) + + cr = create_minimal_req!(command: ["true", "2"], + container_image: links(:docker_image_collection_tag).name) + assert_equal(Container.resolve_container_image(cr.container_image), + collections(:docker_image).portable_data_hash) + end + + test "incompatible docker image v1" do + Rails.configuration.docker_image_formats = ['v1'] + add_docker19_migration_link + + # Don't return unsupported v2 image even if we ask for it directly. + set_user_from_auth :active + cr = create_minimal_req!(command: ["true", "1"], + container_image: collections(:docker_image_1_12).portable_data_hash) + assert_raises(ArvadosModel::UnresolvableContainerError) do + Container.resolve_container_image(cr.container_image) + end + end + + test "incompatible docker image v2" do + Rails.configuration.docker_image_formats = ['v2'] + # No migration link, don't return unsupported v1 image, + + set_user_from_auth :active + cr = create_minimal_req!(command: ["true", "1"], + container_image: collections(:docker_image).portable_data_hash) + assert_raises(ArvadosModel::UnresolvableContainerError) do + Container.resolve_container_image(cr.container_image) + end + cr = create_minimal_req!(command: ["true", "2"], + container_image: links(:docker_image_collection_tag).name) + assert_raises(ArvadosModel::UnresolvableContainerError) do + Container.resolve_container_image(cr.container_image) + end + end + test "requestor can retrieve container owned by dispatch" do assert_not_empty Container.readable_by(users(:admin)).where(uuid: containers(:running).uuid) assert_not_empty Container.readable_by(users(:active)).where(uuid: containers(:running).uuid) @@ -462,6 +541,7 @@ class ContainerRequestTest < ActiveSupport::TestCase test "Retry on container cancelled" do set_user_from_auth :active cr = create_minimal_req!(priority: 1, state: "Committed", container_count_max: 2) + cr2 = create_minimal_req!(priority: 1, state: "Committed", container_count_max: 2, command: ["echo", "baz"]) prev_container_uuid = cr.container_uuid c = act_as_system_user do @@ -472,8 +552,10 @@ class ContainerRequestTest < ActiveSupport::TestCase end cr.reload + cr2.reload assert_equal "Committed", cr.state assert_equal prev_container_uuid, cr.container_uuid + assert_not_equal cr2.container_uuid, cr.container_uuid prev_container_uuid = cr.container_uuid act_as_system_user do @@ -481,8 +563,10 @@ class ContainerRequestTest < ActiveSupport::TestCase end cr.reload + cr2.reload assert_equal "Committed", cr.state assert_not_equal prev_container_uuid, cr.container_uuid + assert_not_equal cr2.container_uuid, cr.container_uuid prev_container_uuid = cr.container_uuid c = act_as_system_user do @@ -492,15 +576,73 @@ class ContainerRequestTest < ActiveSupport::TestCase end cr.reload + cr2.reload assert_equal "Final", cr.state assert_equal prev_container_uuid, cr.container_uuid + assert_not_equal cr2.container_uuid, cr.container_uuid end - test "Finalize committed request when reusing a finished container" do + test "Output collection name setting using output_name with name collision resolution" do set_user_from_auth :active - cr = create_minimal_req!(priority: 1, state: ContainerRequest::Committed) + output_name = 'unimaginative name' + Collection.create!(name: output_name) + + cr = create_minimal_req!(priority: 1, + state: ContainerRequest::Committed, + output_name: output_name) + run_container(cr) cr.reload - assert_equal ContainerRequest::Committed, cr.state + assert_equal ContainerRequest::Final, cr.state + output_coll = Collection.find_by_uuid(cr.output_uuid) + # Make sure the resulting output collection name include the original name + # plus the date + assert_not_equal output_name, output_coll.name, + "more than one collection with the same owner and name" + assert output_coll.name.include?(output_name), + "New name should include original name" + assert_match /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z/, output_coll.name, + "New name should include ISO8601 date" + end + + [[0, :check_output_ttl_0], + [1, :check_output_ttl_1s], + [365*86400, :check_output_ttl_1y], + ].each do |ttl, checker| + test "output_ttl=#{ttl}" do + act_as_user users(:active) do + cr = create_minimal_req!(priority: 1, + state: ContainerRequest::Committed, + output_name: 'foo', + output_ttl: ttl) + run_container(cr) + cr.reload + output = Collection.find_by_uuid(cr.output_uuid) + send(checker, db_current_time, output.trash_at, output.delete_at) + end + end + end + + def check_output_ttl_0(now, trash, delete) + assert_nil(trash) + assert_nil(delete) + end + + def check_output_ttl_1s(now, trash, delete) + assert_not_nil(trash) + assert_not_nil(delete) + assert_in_delta(trash, now + 1.second, 10) + assert_in_delta(delete, now + Rails.configuration.blob_signature_ttl.second, 10) + end + + def check_output_ttl_1y(now, trash, delete) + year = (86400*365).second + assert_not_nil(trash) + assert_not_nil(delete) + assert_in_delta(trash, now + year, 10) + assert_in_delta(delete, now + year, 10) + end + + def run_container(cr) act_as_system_user do c = Container.find_by_uuid(cr.container_uuid) c.update_attributes!(state: Container::Locked) @@ -509,7 +651,16 @@ class ContainerRequestTest < ActiveSupport::TestCase exit_code: 0, output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45', log: 'fa7aeb5140e2848d39b416daeef4ffc5+45') + c end + end + + test "Finalize committed request when reusing a finished container" do + set_user_from_auth :active + cr = create_minimal_req!(priority: 1, state: ContainerRequest::Committed) + cr.reload + assert_equal ContainerRequest::Committed, cr.state + run_container(cr) cr.reload assert_equal ContainerRequest::Final, cr.state @@ -523,4 +674,80 @@ class ContainerRequestTest < ActiveSupport::TestCase assert_equal cr.container_uuid, cr3.container_uuid assert_equal ContainerRequest::Final, cr3.state end + + [ + [{"partitions" => ["fastcpu","vfastcpu", 100]}, ContainerRequest::Committed, ActiveRecord::RecordInvalid], + [{"partitions" => ["fastcpu","vfastcpu", 100]}, ContainerRequest::Uncommitted], + [{"partitions" => "fastcpu"}, ContainerRequest::Committed, ActiveRecord::RecordInvalid], + [{"partitions" => "fastcpu"}, ContainerRequest::Uncommitted], + [{"partitions" => ["fastcpu","vfastcpu"]}, ContainerRequest::Committed], + ].each do |sp, state, expected| + test "create container request with scheduling_parameters #{sp} in state #{state} and verify #{expected}" do + common_attrs = {cwd: "test", + priority: 1, + command: ["echo", "hello"], + output_path: "test", + scheduling_parameters: sp, + mounts: {"test" => {"kind" => "json"}}} + set_user_from_auth :active + + if expected == ActiveRecord::RecordInvalid + assert_raises(ActiveRecord::RecordInvalid) do + create_minimal_req!(common_attrs.merge({state: state})) + end + else + cr = create_minimal_req!(common_attrs.merge({state: state})) + assert_equal sp, cr.scheduling_parameters + + if state == ContainerRequest::Committed + c = Container.find_by_uuid(cr.container_uuid) + assert_equal sp, c.scheduling_parameters + end + end + end + end + + [['Committed', true, {name: "foobar", priority: 123}], + ['Committed', false, {container_count: 2}], + ['Committed', false, {container_count: 0}], + ['Committed', false, {container_count: nil}], + ['Final', false, {state: ContainerRequest::Committed, name: "foobar"}], + ['Final', false, {name: "foobar", priority: 123}], + ['Final', false, {name: "foobar", output_uuid: "zzzzz-4zz18-znfnqtbbv4spc3w"}], + ['Final', false, {name: "foobar", log_uuid: "zzzzz-4zz18-znfnqtbbv4spc3w"}], + ['Final', false, {log_uuid: "zzzzz-4zz18-znfnqtbbv4spc3w"}], + ['Final', false, {priority: 123}], + ['Final', false, {mounts: {}}], + ['Final', false, {container_count: 2}], + ['Final', true, {name: "foobar"}], + ['Final', true, {name: "foobar", description: "baz"}], + ].each do |state, permitted, updates| + test "state=#{state} can#{'not' if !permitted} update #{updates.inspect}" do + act_as_user users(:active) do + cr = create_minimal_req!(priority: 1, + state: "Committed", + container_count_max: 1) + case state + when 'Committed' + # already done + when 'Final' + act_as_system_user do + Container.find_by_uuid(cr.container_uuid). + update_attributes!(state: Container::Cancelled) + end + cr.reload + else + raise 'broken test case' + end + assert_equal state, cr.state + if permitted + assert cr.update_attributes!(updates) + else + assert_raises(ActiveRecord::RecordInvalid) do + cr.update_attributes!(updates) + end + end + end + end + end end