# SPDX-License-Identifier: AGPL-3.0
require 'test_helper'
+require 'helpers/container_test_helper'
require 'helpers/docker_migration_helper'
class ContainerRequestTest < ActiveSupport::TestCase
include DockerMigrationHelper
include DbCurrentTime
+ include ContainerTestHelper
+
+ def with_container_auth(ctr)
+ auth_was = Thread.current[:api_client_authorization]
+ Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(ctr.auth_uuid)
+ begin
+ yield
+ ensure
+ Thread.current[:api_client_authorization] = auth_was
+ end
+ end
+
+ def lock_and_run(ctr)
+ act_as_system_user do
+ ctr.update_attributes!(state: Container::Locked)
+ ctr.update_attributes!(state: Container::Running)
+ end
+ end
def create_minimal_req! attrs={}
defaults = {
cr = create_minimal_req!
assert_nil cr.container_uuid
- assert_nil cr.priority
+ assert_equal 0, cr.priority
check_bogus_states cr
cr.container_image = "img3"
cr.cwd = "/tmp3"
cr.environment = {"BUP" => "BOP"}
- cr.mounts = {"BAR" => "BAZ"}
+ cr.mounts = {"BAR" => {"kind" => "BAZ"}}
cr.output_path = "/tmp4"
cr.priority = 2
cr.runtime_constraints = {"vcpus" => 4}
end
[
- {"vcpus" => 1},
- {"vcpus" => 1, "ram" => nil},
- {"vcpus" => 0, "ram" => 123},
- {"vcpus" => "1", "ram" => "123"}
- ].each do |invalid_constraints|
- test "Create with #{invalid_constraints}" do
+ {"runtime_constraints" => {"vcpus" => 1}},
+ {"runtime_constraints" => {"vcpus" => 1, "ram" => nil}},
+ {"runtime_constraints" => {"vcpus" => 0, "ram" => 123}},
+ {"runtime_constraints" => {"vcpus" => "1", "ram" => "123"}},
+ {"mounts" => {"FOO" => "BAR"}},
+ {"mounts" => {"FOO" => {}}},
+ {"mounts" => {"FOO" => {"kind" => "tmp", "capacity" => 42.222}}},
+ {"command" => ["echo", 55]},
+ {"environment" => {"FOO" => 55}}
+ ].each do |value|
+ test "Create with invalid #{value}" do
set_user_from_auth :active
assert_raises(ActiveRecord::RecordInvalid) do
- cr = create_minimal_req!(state: "Committed",
- priority: 1,
- runtime_constraints: invalid_constraints)
+ cr = create_minimal_req!({state: "Committed",
+ priority: 1}.merge(value))
cr.save!
end
end
- test "Update with #{invalid_constraints}" do
+ test "Update with invalid #{value}" do
set_user_from_auth :active
cr = create_minimal_req!(state: "Uncommitted", priority: 1)
cr.save!
assert_raises(ActiveRecord::RecordInvalid) do
cr = ContainerRequest.find_by_uuid cr.uuid
- cr.update_attributes!(state: "Committed",
- runtime_constraints: invalid_constraints)
+ cr.update_attributes!({state: "Committed",
+ priority: 1}.merge(value))
end
end
end
test "Container request priority must be non-nil" do
set_user_from_auth :active
- cr = create_minimal_req!(priority: nil)
+ cr = create_minimal_req!
+ cr.priority = nil
cr.state = "Committed"
assert_raises(ActiveRecord::RecordInvalid) do
cr.save!
assert_equal({"/out" => {"kind"=>"tmp", "capacity"=>1000000}}, c.mounts)
assert_equal "/out", c.output_path
assert_equal({"keep_cache_ram"=>268435456, "vcpus" => 2, "ram" => 30}, c.runtime_constraints)
- assert_equal 1, c.priority
+ assert_operator 0, :<, c.priority
assert_raises(ActiveRecord::RecordInvalid) do
cr.priority = nil
assert_equal 0, c.priority
end
-
- test "Container request max priority" do
- set_user_from_auth :active
- cr = create_minimal_req!(priority: 5, state: "Committed")
-
- c = Container.find_by_uuid cr.container_uuid
- assert_equal 5, c.priority
-
- cr2 = create_minimal_req!
- cr2.priority = 10
- cr2.state = "Committed"
- cr2.container_uuid = cr.container_uuid
- act_as_system_user do
- cr2.save!
- end
-
- # cr and cr2 have priority 5 and 10, and are being satisfied by
- # the same container c, so c's priority should be
- # max(priority)=10.
- c.reload
- assert_equal 10, c.priority
-
- cr2.update_attributes!(priority: 0)
-
- c.reload
- assert_equal 5, c.priority
-
- cr.update_attributes!(priority: 0)
-
- c.reload
- assert_equal 0, c.priority
- end
-
-
test "Independent container requests" do
set_user_from_auth :active
cr1 = create_minimal_req!(command: ["foo", "1"], priority: 5, state: "Committed")
cr2 = create_minimal_req!(command: ["foo", "2"], priority: 10, state: "Committed")
c1 = Container.find_by_uuid cr1.container_uuid
- assert_equal 5, c1.priority
+ assert_operator 0, :<, c1.priority
c2 = Container.find_by_uuid cr2.container_uuid
- assert_equal 10, c2.priority
+ assert_operator c1.priority, :<, c2.priority
+ c2priority_was = c2.priority
cr1.update_attributes!(priority: 0)
assert_equal 0, c1.priority
c2.reload
- assert_equal 10, c2.priority
+ assert_equal c2priority_was, c2.priority
end
test "Request is finalized when its container is cancelled" do
set_user_from_auth :active
cr = create_minimal_req!(priority: 1, state: "Committed", container_count_max: 1)
+ assert_equal users(:active).uuid, cr.modified_by_user_uuid
act_as_system_user do
Container.find_by_uuid(cr.container_uuid).
cr.reload
assert_equal "Final", cr.state
+ assert_equal users(:active).uuid, cr.modified_by_user_uuid
end
test "Request is finalized when its container is completed" do
cr = create_minimal_req!(owner_uuid: project.uuid,
priority: 1,
state: "Committed")
+ assert_equal users(:active).uuid, cr.modified_by_user_uuid
c = act_as_system_user do
c = Container.find_by_uuid(cr.container_uuid)
cr.reload
assert_equal "Final", cr.state
+ assert_equal users(:active).uuid, cr.modified_by_user_uuid
['output', 'log'].each do |out_type|
pdh = Container.find_by_uuid(cr.container_uuid).send(out_type)
assert_equal(1, Collection.where(portable_data_hash: pdh,
cr = create_minimal_req!(priority: 5, state: "Committed", container_count_max: 1)
c = Container.find_by_uuid cr.container_uuid
- assert_equal 5, c.priority
+ assert_operator 0, :<, c.priority
+ lock_and_run(c)
- cr2 = create_minimal_req!
- cr2.update_attributes!(priority: 10, state: "Committed", requesting_container_uuid: c.uuid, command: ["echo", "foo2"], container_count_max: 1)
- cr2.reload
+ cr2 = with_container_auth(c) do
+ create_minimal_req!(priority: 10, state: "Committed", container_count_max: 1, command: ["echo", "foo2"])
+ end
+ assert_not_nil cr2.requesting_container_uuid
+ assert_equal users(:active).uuid, cr2.modified_by_user_uuid
c2 = Container.find_by_uuid cr2.container_uuid
- assert_equal 10, c2.priority
+ assert_operator 0, :<, c2.priority
act_as_system_user do
c.state = "Cancelled"
cr2.reload
assert_equal 0, cr2.priority
+ assert_equal users(:active).uuid, cr2.modified_by_user_uuid
c2.reload
assert_equal 0, c2.priority
end
+ test "child container priority follows same ordering as corresponding top-level ancestors" do
+ findctr = lambda { |cr| Container.find_by_uuid(cr.container_uuid) }
+
+ set_user_from_auth :active
+
+ toplevel_crs = [
+ create_minimal_req!(priority: 5, state: "Committed", environment: {"workflow" => "0"}),
+ create_minimal_req!(priority: 5, state: "Committed", environment: {"workflow" => "1"}),
+ create_minimal_req!(priority: 5, state: "Committed", environment: {"workflow" => "2"}),
+ ]
+ parents = toplevel_crs.map(&findctr)
+
+ children = parents.map do |parent|
+ lock_and_run(parent)
+ with_container_auth(parent) do
+ create_minimal_req!(state: "Committed",
+ priority: 1,
+ environment: {"child" => parent.environment["workflow"]})
+ end
+ end.map(&findctr)
+
+ grandchildren = children.reverse.map do |child|
+ lock_and_run(child)
+ with_container_auth(child) do
+ create_minimal_req!(state: "Committed",
+ priority: 1,
+ environment: {"grandchild" => child.environment["child"]})
+ end
+ end.reverse.map(&findctr)
+
+ shared_grandchildren = children.map do |child|
+ with_container_auth(child) do
+ create_minimal_req!(state: "Committed",
+ priority: 1,
+ environment: {"grandchild" => "shared"})
+ end
+ end.map(&findctr)
+
+ assert_equal shared_grandchildren[0].uuid, shared_grandchildren[1].uuid
+ assert_equal shared_grandchildren[0].uuid, shared_grandchildren[2].uuid
+ shared_grandchild = shared_grandchildren[0]
+
+ set_user_from_auth :active
+
+ # parents should be prioritized by submit time.
+ assert_operator parents[0].priority, :>, parents[1].priority
+ assert_operator parents[1].priority, :>, parents[2].priority
+
+ # children should be prioritized in same order as their respective
+ # parents.
+ assert_operator children[0].priority, :>, children[1].priority
+ assert_operator children[1].priority, :>, children[2].priority
+
+ # grandchildren should also be prioritized in the same order,
+ # despite having been submitted in the opposite order.
+ assert_operator grandchildren[0].priority, :>, grandchildren[1].priority
+ assert_operator grandchildren[1].priority, :>, grandchildren[2].priority
+
+ # shared grandchild container should be prioritized above
+ # everything that isn't needed by parents[0], but not above
+ # earlier-submitted descendants of parents[0]
+ assert_operator shared_grandchild.priority, :>, grandchildren[1].priority
+ assert_operator shared_grandchild.priority, :>, children[1].priority
+ assert_operator shared_grandchild.priority, :>, parents[1].priority
+ assert_operator shared_grandchild.priority, :<=, grandchildren[0].priority
+ assert_operator shared_grandchild.priority, :<=, children[0].priority
+ assert_operator shared_grandchild.priority, :<=, parents[0].priority
+
+ # increasing priority of the most recent toplevel container should
+ # reprioritize all of its descendants (including the shared
+ # grandchild) above everything else.
+ toplevel_crs[2].update_attributes!(priority: 72)
+ (parents + children + grandchildren + [shared_grandchild]).map(&:reload)
+ assert_operator shared_grandchild.priority, :>, grandchildren[0].priority
+ assert_operator shared_grandchild.priority, :>, children[0].priority
+ assert_operator shared_grandchild.priority, :>, parents[0].priority
+ assert_operator shared_grandchild.priority, :>, grandchildren[1].priority
+ assert_operator shared_grandchild.priority, :>, children[1].priority
+ assert_operator shared_grandchild.priority, :>, parents[1].priority
+ # ...but the shared container should not have higher priority than
+ # the earlier-submitted descendants of the high-priority workflow.
+ assert_operator shared_grandchild.priority, :<=, grandchildren[2].priority
+ assert_operator shared_grandchild.priority, :<=, children[2].priority
+ assert_operator shared_grandchild.priority, :<=, parents[2].priority
+ end
+
[
- ['running_container_auth', 'zzzzz-dz642-runningcontainr'],
- ['active_no_prefs', nil],
- ].each do |token, expected|
+ ['running_container_auth', 'zzzzz-dz642-runningcontainr', 501],
+ ['active_no_prefs', nil, 0],
+ ].each do |token, expected, expected_priority|
test "create as #{token} and expect requesting_container_uuid to be #{expected}" do
set_user_from_auth token
cr = ContainerRequest.create(container_image: "img", output_path: "/tmp", command: ["echo", "foo"])
assert_not_nil cr.uuid, 'uuid should be set for newly created container_request'
assert_equal expected, cr.requesting_container_uuid
+ assert_equal expected_priority, cr.priority
end
end
test "requesting_container_uuid at create is not allowed" do
set_user_from_auth :active
- assert_raises(ActiveRecord::RecordNotSaved) do
+ assert_raises(ActiveRecord::RecordInvalid) do
create_minimal_req!(state: "Uncommitted", priority: 1, requesting_container_uuid: 'youcantdothat')
end
end
assert_equal ContainerRequest::Final, cr3.state
end
+ [
+ [false, ActiveRecord::RecordInvalid],
+ [true, nil],
+ ].each do |preemptible_conf, expected|
+ test "having Rails.configuration.preemptible_instances=#{preemptible_conf}, create preemptible container request and verify #{expected}" do
+ sp = {"preemptible" => true}
+ common_attrs = {cwd: "test",
+ priority: 1,
+ command: ["echo", "hello"],
+ output_path: "test",
+ scheduling_parameters: sp,
+ mounts: {"test" => {"kind" => "json"}}}
+ Rails.configuration.preemptible_instances = preemptible_conf
+ set_user_from_auth :active
+
+ cr = create_minimal_req!(common_attrs)
+ cr.state = ContainerRequest::Committed
+
+ if !expected.nil?
+ assert_raises(expected) do
+ cr.save!
+ end
+ else
+ cr.save!
+ assert_equal sp, cr.scheduling_parameters
+ end
+ end
+ end
+
+ [
+ 'zzzzz-dz642-runningcontainr',
+ nil,
+ ].each do |requesting_c|
+ test "having preemptible instances active on the API server, a committed #{requesting_c.nil? ? 'non-':''}child CR should not ask for preemptible instance if parameter already set to false" do
+ common_attrs = {cwd: "test",
+ priority: 1,
+ command: ["echo", "hello"],
+ output_path: "test",
+ scheduling_parameters: {"preemptible" => false},
+ mounts: {"test" => {"kind" => "json"}}}
+
+ Rails.configuration.preemptible_instances = true
+ set_user_from_auth :active
+
+ if requesting_c
+ cr = with_container_auth(Container.find_by_uuid requesting_c) do
+ create_minimal_req!(common_attrs)
+ end
+ assert_not_nil cr.requesting_container_uuid
+ else
+ cr = create_minimal_req!(common_attrs)
+ end
+
+ cr.state = ContainerRequest::Committed
+ cr.save!
+
+ assert_equal false, cr.scheduling_parameters['preemptible']
+ end
+ end
+
+ [
+ [true, 'zzzzz-dz642-runningcontainr', true],
+ [true, nil, nil],
+ [false, 'zzzzz-dz642-runningcontainr', nil],
+ [false, nil, nil],
+ ].each do |preemptible_conf, requesting_c, schedule_preemptible|
+ test "having Rails.configuration.preemptible_instances=#{preemptible_conf}, #{requesting_c.nil? ? 'non-':''}child CR should #{schedule_preemptible ? '':'not'} ask for preemptible instance by default" do
+ common_attrs = {cwd: "test",
+ priority: 1,
+ command: ["echo", "hello"],
+ output_path: "test",
+ mounts: {"test" => {"kind" => "json"}}}
+
+ Rails.configuration.preemptible_instances = preemptible_conf
+ set_user_from_auth :active
+
+ if requesting_c
+ cr = with_container_auth(Container.find_by_uuid requesting_c) do
+ create_minimal_req!(common_attrs)
+ end
+ assert_not_nil cr.requesting_container_uuid
+ else
+ cr = create_minimal_req!(common_attrs)
+ end
+
+ cr.state = ContainerRequest::Committed
+ cr.save!
+
+ assert_equal schedule_preemptible, cr.scheduling_parameters['preemptible']
+ end
+ end
+
[
[{"partitions" => ["fastcpu","vfastcpu", 100]}, ContainerRequest::Committed, ActiveRecord::RecordInvalid],
[{"partitions" => ["fastcpu","vfastcpu", 100]}, ContainerRequest::Uncommitted],
[{"partitions" => "fastcpu"}, ContainerRequest::Committed, ActiveRecord::RecordInvalid],
[{"partitions" => "fastcpu"}, ContainerRequest::Uncommitted],
[{"partitions" => ["fastcpu","vfastcpu"]}, ContainerRequest::Committed],
+ [{"max_run_time" => "one day"}, ContainerRequest::Committed, ActiveRecord::RecordInvalid],
+ [{"max_run_time" => "one day"}, ContainerRequest::Uncommitted],
+ [{"max_run_time" => -1}, ContainerRequest::Committed, ActiveRecord::RecordInvalid],
+ [{"max_run_time" => -1}, ContainerRequest::Uncommitted],
+ [{"max_run_time" => 86400}, ContainerRequest::Committed],
].each do |sp, state, expected|
test "create container request with scheduling_parameters #{sp} in state #{state} and verify #{expected}" do
common_attrs = {cwd: "test",
end
end
+ test "Having preemptible_instances=true create a committed child container request and verify the scheduling parameter of its container" do
+ common_attrs = {cwd: "test",
+ priority: 1,
+ command: ["echo", "hello"],
+ output_path: "test",
+ state: ContainerRequest::Committed,
+ mounts: {"test" => {"kind" => "json"}}}
+ set_user_from_auth :active
+ Rails.configuration.preemptible_instances = true
+
+ cr = with_container_auth(Container.find_by_uuid 'zzzzz-dz642-runningcontainr') do
+ create_minimal_req!(common_attrs)
+ end
+ assert_equal 'zzzzz-dz642-runningcontainr', cr.requesting_container_uuid
+ assert_equal true, cr.scheduling_parameters["preemptible"]
+
+ c = Container.find_by_uuid(cr.container_uuid)
+ assert_equal true, c.scheduling_parameters["preemptible"]
+ end
+
[['Committed', true, {name: "foobar", priority: 123}],
['Committed', false, {container_count: 2}],
['Committed', false, {container_count: 0}],
c = Container.find_by_uuid(cr.container_uuid)
assert_equal 1, c.priority
- # destroy the cr
- assert_nothing_raised {cr.destroy}
+ cr.destroy
# the cr's container now has priority of 0
c = Container.find_by_uuid(cr.container_uuid)
assert_nothing_raised {cr.destroy}
end
end
+
+ test "Container request valid priority" do
+ set_user_from_auth :active
+ cr = create_minimal_req!
+
+ assert_raises(ActiveRecord::RecordInvalid) do
+ cr.priority = -1
+ cr.save!
+ end
+
+ cr.priority = 0
+ cr.save!
+
+ cr.priority = 1
+ cr.save!
+
+ cr.priority = 500
+ cr.save!
+
+ cr.priority = 999
+ cr.save!
+
+ cr.priority = 1000
+ cr.save!
+
+ assert_raises(ActiveRecord::RecordInvalid) do
+ cr.priority = 1001
+ cr.save!
+ end
+ end
+
+ # Note: some of these tests might look redundant because they test
+ # that out-of-order spellings of hashes are still considered equal
+ # regardless of whether the existing (container) or new (container
+ # request) hash needs to be re-ordered.
+ secrets = {"/foo" => {"kind" => "text", "content" => "xyzzy"}}
+ same_secrets = {"/foo" => {"content" => "xyzzy", "kind" => "text"}}
+ different_secrets = {"/foo" => {"kind" => "text", "content" => "something completely different"}}
+ [
+ [true, nil, nil],
+ [true, nil, {}],
+ [true, {}, nil],
+ [true, {}, {}],
+ [true, secrets, same_secrets],
+ [true, same_secrets, secrets],
+ [false, nil, secrets],
+ [false, {}, secrets],
+ [false, secrets, {}],
+ [false, secrets, nil],
+ [false, secrets, different_secrets],
+ ].each do |expect_reuse, sm1, sm2|
+ test "container reuse secret_mounts #{sm1.inspect}, #{sm2.inspect}" do
+ set_user_from_auth :active
+ cr1 = create_minimal_req!(state: "Committed", priority: 1, secret_mounts: sm1)
+ cr2 = create_minimal_req!(state: "Committed", priority: 1, secret_mounts: sm2)
+ assert_not_nil cr1.container_uuid
+ assert_not_nil cr2.container_uuid
+ if expect_reuse
+ assert_equal cr1.container_uuid, cr2.container_uuid
+ else
+ assert_not_equal cr1.container_uuid, cr2.container_uuid
+ end
+ end
+ end
+
+ test "scrub secret_mounts but reuse container for request with identical secret_mounts" do
+ set_user_from_auth :active
+ sm = {'/secret/foo' => {'kind' => 'text', 'content' => secret_string}}
+ cr1 = create_minimal_req!(state: "Committed", priority: 1, secret_mounts: sm.dup)
+ run_container(cr1)
+ cr1.reload
+
+ # secret_mounts scrubbed from db
+ c = Container.where(uuid: cr1.container_uuid).first
+ assert_equal({}, c.secret_mounts)
+ assert_equal({}, cr1.secret_mounts)
+
+ # can reuse container if secret_mounts match
+ cr2 = create_minimal_req!(state: "Committed", priority: 1, secret_mounts: sm.dup)
+ assert_equal cr1.container_uuid, cr2.container_uuid
+
+ # don't reuse container if secret_mounts don't match
+ cr3 = create_minimal_req!(state: "Committed", priority: 1, secret_mounts: {})
+ assert_not_equal cr1.container_uuid, cr3.container_uuid
+
+ assert_no_secrets_logged
+ end
+
+ test "conflicting key in mounts and secret_mounts" do
+ sm = {'/secret/foo' => {'kind' => 'text', 'content' => secret_string}}
+ set_user_from_auth :active
+ cr = create_minimal_req!
+ assert_equal false, cr.update_attributes(state: "Committed",
+ priority: 1,
+ mounts: cr.mounts.merge(sm),
+ secret_mounts: sm)
+ assert_equal [:secret_mounts], cr.errors.messages.keys
+ end
end
projects / arvados.git / blobdiff