X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/1b4fa5760aab91a8422dc5d84c73bd627ff1dc51..24bcfa0b87b87e4510fffe8a961a5d4a9fd34948:/services/api/test/unit/container_test.rb

diff --git a/services/api/test/unit/container_test.rb b/services/api/test/unit/container_test.rb
index a25f2af467..8894ed9d4c 100644
--- a/services/api/test/unit/container_test.rb
+++ b/services/api/test/unit/container_test.rb
@@ -1,110 +1,82 @@
 require 'test_helper'
 
 class ContainerTest < ActiveSupport::TestCase
-  def minimal_new
-    c = Container.new
-    c.command = ["echo", "foo"]
-    c.container_image = "img"
-    c.output_path = "/tmp"
-    c
-  end
-
-  def check_illegal_modify c
-    c.reload
-    c.command = ["echo", "bar"]
-    assert_raises(ActiveRecord::RecordInvalid) do
-      c.save!
-    end
-
-    c.reload
-    c.container_image = "img2"
-    assert_raises(ActiveRecord::RecordInvalid) do
-      c.save!
-    end
-
-    c.reload
-    c.cwd = "/tmp2"
-    assert_raises(ActiveRecord::RecordInvalid) do
-      c.save!
-    end
-
-    c.reload
-    c.environment = {"FOO" => "BAR"}
-    assert_raises(ActiveRecord::RecordInvalid) do
-      c.save!
+  include DbCurrentTime
+
+  DEFAULT_ATTRS = {
+    command: ['echo', 'foo'],
+    container_image: 'img',
+    output_path: '/tmp',
+    priority: 1,
+    runtime_constraints: {"vcpus" => 1, "ram" => 1},
+  }
+
+  REUSABLE_COMMON_ATTRS = {container_image: "test",
+                           cwd: "test",
+                           command: ["echo", "hello"],
+                           output_path: "test",
+                           runtime_constraints: {"vcpus" => 4,
+                                                 "ram" => 12000000000},
+                           mounts: {"test" => {"kind" => "json"}},
+                           environment: {"var" => 'val'}}
+
+  def minimal_new attrs={}
+    cr = ContainerRequest.new DEFAULT_ATTRS.merge(attrs)
+    act_as_user users(:active) do
+      cr.save!
     end
-
-    c.reload
-    c.mounts = {"FOO" => "BAR"}
-    assert_raises(ActiveRecord::RecordInvalid) do
+    c = Container.new DEFAULT_ATTRS.merge(attrs)
+    act_as_system_user do
       c.save!
+      assert cr.update_attributes(container_uuid: c.uuid,
+                                  state: ContainerRequest::Committed,
+                                  ), show_errors(cr)
     end
+    return c, cr
+  end
 
-    c.reload
-    c.output_path = "/tmp3"
-    assert_raises(ActiveRecord::RecordInvalid) do
-      c.save!
+  def check_illegal_updates c, bad_updates
+    bad_updates.each do |u|
+      refute c.update_attributes(u), u.inspect
+      refute c.valid?, u.inspect
+      c.reload
     end
+  end
 
-    c.reload
-    c.runtime_constraints = {"FOO" => "BAR"}
-    assert_raises(ActiveRecord::RecordInvalid) do
-      c.save!
-    end
+  def check_illegal_modify c
+    check_illegal_updates c, [{command: ["echo", "bar"]},
+                              {container_image: "img2"},
+                              {cwd: "/tmp2"},
+                              {environment: {"FOO" => "BAR"}},
+                              {mounts: {"FOO" => "BAR"}},
+                              {output_path: "/tmp3"},
+                              {locked_by_uuid: "zzzzz-gj3su-027z32aux8dg2s1"},
+                              {auth_uuid: "zzzzz-gj3su-017z32aux8dg2s1"},
+                              {runtime_constraints: {"FOO" => "BAR"}}]
   end
 
   def check_bogus_states c
-    c.reload
-    c.state = nil
-    assert_raises(ActiveRecord::RecordInvalid) do
-      c.save!
-    end
-
-    c.reload
-    c.state = "Flubber"
-    assert_raises(ActiveRecord::RecordInvalid) do
-      c.save!
-    end
+    check_illegal_updates c, [{state: nil},
+                              {state: "Flubber"}]
   end
 
-  def check_no_change_from_complete c
+  def check_no_change_from_cancelled c
     check_illegal_modify c
     check_bogus_states c
-
-    c.reload
-    c.priority = 3
-    assert_raises(ActiveRecord::RecordInvalid) do
-      c.save!
-    end
-
-    c.reload
-    c.state = "Queued"
-    assert_raises(ActiveRecord::RecordInvalid) do
-      c.save!
-    end
-
-    c.reload
-    c.state = "Running"
-    assert_raises(ActiveRecord::RecordInvalid) do
-      c.save!
-    end
-
-    c.reload
-    c.state = "Complete"
-    assert_raises(ActiveRecord::RecordInvalid) do
-      c.save!
-    end
+    check_illegal_updates c, [{ priority: 3 },
+                              { state: Container::Queued },
+                              { state: Container::Locked },
+                              { state: Container::Running },
+                              { state: Container::Complete }]
   end
 
   test "Container create" do
     act_as_system_user do
-      c = minimal_new
-      c.environment = {}
-      c.mounts = {"BAR" => "FOO"}
-      c.output_path = "/tmp"
-      c.priority = 1
-      c.runtime_constraints = {}
-      c.save!
+      c, _ = minimal_new(environment: {},
+                      mounts: {"BAR" => "FOO"},
+                      output_path: "/tmp",
+                      priority: 1,
+                      runtime_constraints: {"vcpus" => 1, "ram" => 1})
 
       check_illegal_modify c
       check_bogus_states c
@@ -115,69 +87,325 @@ class ContainerTest < ActiveSupport::TestCase
     end
   end
 
-  test "Container running" do
-    act_as_system_user do
-      c = minimal_new
-      c.save!
+  test "Container serialized hash attributes sorted before save" do
+    env = {"C" => 3, "B" => 2, "A" => 1}
+    m = {"F" => 3, "E" => 2, "D" => 1}
+    rc = {"vcpus" => 1, "ram" => 1}
+    c, _ = minimal_new(environment: env, mounts: m, runtime_constraints: rc)
+    assert_equal c.environment.to_json, Container.deep_sort_hash(env).to_json
+    assert_equal c.mounts.to_json, Container.deep_sort_hash(m).to_json
+    assert_equal c.runtime_constraints.to_json, Container.deep_sort_hash(rc).to_json
+  end
 
-      c.reload
-      c.state = "Complete"
-      assert_raises(ActiveRecord::RecordInvalid) do
-        c.save!
-      end
+  test 'deep_sort_hash on array of hashes' do
+    a = {'z' => [[{'a' => 'a', 'b' => 'b'}]]}
+    b = {'z' => [[{'b' => 'b', 'a' => 'a'}]]}
+    assert_equal Container.deep_sort_hash(a).to_json, Container.deep_sort_hash(b).to_json
+  end
 
-      c.reload
-      c.state = "Running"
-      c.save!
+  test "find_reusable method should select higher priority queued container" do
+    set_user_from_auth :active
+    common_attrs = REUSABLE_COMMON_ATTRS.merge({environment:{"var" => "queued"}})
+    c_low_priority, _ = minimal_new(common_attrs.merge({priority:1}))
+    c_high_priority, _ = minimal_new(common_attrs.merge({priority:2}))
+    assert_equal Container::Queued, c_low_priority.state
+    assert_equal Container::Queued, c_high_priority.state
+    reused = Container.find_reusable(common_attrs)
+    assert_not_nil reused
+    assert_equal reused.uuid, c_high_priority.uuid
+  end
 
-      check_illegal_modify c
-      check_bogus_states c
+  test "find_reusable method should select latest completed container" do
+    set_user_from_auth :active
+    common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "complete"}})
+    completed_attrs = {
+      state: Container::Complete,
+      exit_code: 0,
+      log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
+      output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
+    }
+
+    c_older, _ = minimal_new(common_attrs)
+    c_recent, _ = minimal_new(common_attrs)
+
+    set_user_from_auth :dispatch1
+    c_older.update_attributes!({state: Container::Locked})
+    c_older.update_attributes!({state: Container::Running})
+    c_older.update_attributes!(completed_attrs)
+
+    c_recent.update_attributes!({state: Container::Locked})
+    c_recent.update_attributes!({state: Container::Running})
+    c_recent.update_attributes!(completed_attrs)
+
+    reused = Container.find_reusable(common_attrs)
+    assert_not_nil reused
+    assert_equal reused.uuid, c_older.uuid
+  end
 
-      c.reload
-      c.state = "Queued"
-      assert_raises(ActiveRecord::RecordInvalid) do
-        c.save!
-      end
+  test "find_reusable method should not select completed container when inconsistent outputs exist" do
+    set_user_from_auth :active
+    common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "complete"}})
+    completed_attrs = {
+      state: Container::Complete,
+      exit_code: 0,
+      log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
+    }
+
+    c_output1, _ = minimal_new(common_attrs)
+    c_output2, _ = minimal_new(common_attrs)
+
+    set_user_from_auth :dispatch1
+    c_output1.update_attributes!({state: Container::Locked})
+    c_output1.update_attributes!({state: Container::Running})
+    c_output1.update_attributes!(completed_attrs.merge({output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'}))
+
+    c_output2.update_attributes!({state: Container::Locked})
+    c_output2.update_attributes!({state: Container::Running})
+    c_output2.update_attributes!(completed_attrs.merge({output: 'fa7aeb5140e2848d39b416daeef4ffc5+45'}))
+
+    reused = Container.find_reusable(common_attrs)
+    assert_nil reused
+  end
 
-      c.reload
-      c.priority = 3
-      c.save!
-    end
+  test "find_reusable method should select running container by start date" do
+    set_user_from_auth :active
+    common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running"}})
+    c_slower, _ = minimal_new(common_attrs)
+    c_faster_started_first, _ = minimal_new(common_attrs)
+    c_faster_started_second, _ = minimal_new(common_attrs)
+    set_user_from_auth :dispatch1
+    c_slower.update_attributes!({state: Container::Locked})
+    c_slower.update_attributes!({state: Container::Running,
+                                 progress: 0.1})
+    c_faster_started_first.update_attributes!({state: Container::Locked})
+    c_faster_started_first.update_attributes!({state: Container::Running,
+                                               progress: 0.15})
+    c_faster_started_second.update_attributes!({state: Container::Locked})
+    c_faster_started_second.update_attributes!({state: Container::Running,
+                                                progress: 0.15})
+    reused = Container.find_reusable(common_attrs)
+    assert_not_nil reused
+    # Selected container is the one that started first
+    assert_equal reused.uuid, c_faster_started_first.uuid
   end
 
-  test "Container queued cancel" do
-    act_as_system_user do
-      c = minimal_new
-      c.save!
+  test "find_reusable method should select running container by progress" do
+    set_user_from_auth :active
+    common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running2"}})
+    c_slower, _ = minimal_new(common_attrs)
+    c_faster_started_first, _ = minimal_new(common_attrs)
+    c_faster_started_second, _ = minimal_new(common_attrs)
+    set_user_from_auth :dispatch1
+    c_slower.update_attributes!({state: Container::Locked})
+    c_slower.update_attributes!({state: Container::Running,
+                                 progress: 0.1})
+    c_faster_started_first.update_attributes!({state: Container::Locked})
+    c_faster_started_first.update_attributes!({state: Container::Running,
+                                               progress: 0.15})
+    c_faster_started_second.update_attributes!({state: Container::Locked})
+    c_faster_started_second.update_attributes!({state: Container::Running,
+                                                progress: 0.2})
+    reused = Container.find_reusable(common_attrs)
+    assert_not_nil reused
+    # Selected container is the one with most progress done
+    assert_equal reused.uuid, c_faster_started_second.uuid
+  end
 
-      c.reload
-      c.state = "Cancelled"
-      c.save!
+  test "find_reusable method should select locked container most likely to start sooner" do
+    set_user_from_auth :active
+    common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "locked"}})
+    c_low_priority, _ = minimal_new(common_attrs)
+    c_high_priority_older, _ = minimal_new(common_attrs)
+    c_high_priority_newer, _ = minimal_new(common_attrs)
+    set_user_from_auth :dispatch1
+    c_low_priority.update_attributes!({state: Container::Locked,
+                                       priority: 1})
+    c_high_priority_older.update_attributes!({state: Container::Locked,
+                                              priority: 2})
+    c_high_priority_newer.update_attributes!({state: Container::Locked,
+                                              priority: 2})
+    reused = Container.find_reusable(common_attrs)
+    assert_not_nil reused
+    assert_equal reused.uuid, c_high_priority_older.uuid
+  end
 
-      check_no_change_from_complete c
-    end
+  test "find_reusable method should select running over failed container" do
+    set_user_from_auth :active
+    common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed_vs_running"}})
+    c_failed, _ = minimal_new(common_attrs)
+    c_running, _ = minimal_new(common_attrs)
+    set_user_from_auth :dispatch1
+    c_failed.update_attributes!({state: Container::Locked})
+    c_failed.update_attributes!({state: Container::Running})
+    c_failed.update_attributes!({state: Container::Complete,
+                                 exit_code: 42,
+                                 log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
+                                 output: 'ea10d51bcf88862dbcc36eb292017dfd+45'})
+    c_running.update_attributes!({state: Container::Locked})
+    c_running.update_attributes!({state: Container::Running,
+                                  progress: 0.15})
+    reused = Container.find_reusable(common_attrs)
+    assert_not_nil reused
+    assert_equal reused.uuid, c_running.uuid
   end
 
-  test "Container running cancel" do
-    act_as_system_user do
-      c = minimal_new
-      c.save!
+  test "find_reusable method should select complete over running container" do
+    set_user_from_auth :active
+    common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "completed_vs_running"}})
+    c_completed, _ = minimal_new(common_attrs)
+    c_running, _ = minimal_new(common_attrs)
+    set_user_from_auth :dispatch1
+    c_completed.update_attributes!({state: Container::Locked})
+    c_completed.update_attributes!({state: Container::Running})
+    c_completed.update_attributes!({state: Container::Complete,
+                                    exit_code: 0,
+                                    log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
+                                    output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'})
+    c_running.update_attributes!({state: Container::Locked})
+    c_running.update_attributes!({state: Container::Running,
+                                  progress: 0.15})
+    reused = Container.find_reusable(common_attrs)
+    assert_not_nil reused
+    assert_equal c_completed.uuid, reused.uuid
+  end
 
-      c.reload
-      c.state = "Running"
-      c.save!
+  test "find_reusable method should select running over locked container" do
+    set_user_from_auth :active
+    common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}})
+    c_locked, _ = minimal_new(common_attrs)
+    c_running, _ = minimal_new(common_attrs)
+    set_user_from_auth :dispatch1
+    c_locked.update_attributes!({state: Container::Locked})
+    c_running.update_attributes!({state: Container::Locked})
+    c_running.update_attributes!({state: Container::Running,
+                                  progress: 0.15})
+    reused = Container.find_reusable(common_attrs)
+    assert_not_nil reused
+    assert_equal reused.uuid, c_running.uuid
+  end
 
-      c.reload
-      c.state = "Cancelled"
-      c.save!
+  test "find_reusable method should select locked over queued container" do
+    set_user_from_auth :active
+    common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}})
+    c_locked, _ = minimal_new(common_attrs)
+    c_queued, _ = minimal_new(common_attrs)
+    set_user_from_auth :dispatch1
+    c_locked.update_attributes!({state: Container::Locked})
+    reused = Container.find_reusable(common_attrs)
+    assert_not_nil reused
+    assert_equal reused.uuid, c_locked.uuid
+  end
 
-      check_no_change_from_complete c
-    end
+  test "find_reusable method should not select failed container" do
+    set_user_from_auth :active
+    attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed"}})
+    c, _ = minimal_new(attrs)
+    set_user_from_auth :dispatch1
+    c.update_attributes!({state: Container::Locked})
+    c.update_attributes!({state: Container::Running})
+    c.update_attributes!({state: Container::Complete,
+                          exit_code: 33})
+    reused = Container.find_reusable(attrs)
+    assert_nil reused
+  end
+
+  test "Container running" do
+    c, _ = minimal_new priority: 1
+
+    set_user_from_auth :dispatch1
+    check_illegal_updates c, [{state: Container::Running},
+                              {state: Container::Complete}]
+
+    c.lock
+    c.update_attributes! state: Container::Running
+
+    check_illegal_modify c
+    check_bogus_states c
+
+    check_illegal_updates c, [{state: Container::Queued}]
+    c.reload
+
+    c.update_attributes! priority: 3
+  end
+
+  test "Lock and unlock" do
+    c, cr = minimal_new priority: 0
+
+    set_user_from_auth :dispatch1
+    assert_equal Container::Queued, c.state
+
+    assert_raise(ActiveRecord::RecordInvalid) {c.lock} # "no priority"
+    c.reload
+    assert cr.update_attributes priority: 1
+
+    refute c.update_attributes(state: Container::Running), "not locked"
+    c.reload
+    refute c.update_attributes(state: Container::Complete), "not locked"
+    c.reload
+
+    assert c.lock, show_errors(c)
+    assert c.locked_by_uuid
+    assert c.auth_uuid
+
+    assert_raise(ArvadosModel::AlreadyLockedError) {c.lock}
+    c.reload
+
+    assert c.unlock, show_errors(c)
+    refute c.locked_by_uuid
+    refute c.auth_uuid
+
+    refute c.update_attributes(state: Container::Running), "not locked"
+    c.reload
+    refute c.locked_by_uuid
+    refute c.auth_uuid
+
+    assert c.lock, show_errors(c)
+    assert c.update_attributes(state: Container::Running), show_errors(c)
+    assert c.locked_by_uuid
+    assert c.auth_uuid
+
+    auth_uuid_was = c.auth_uuid
+
+    assert_raise(ActiveRecord::RecordInvalid) {c.lock} # Running to Locked is not allowed
+    c.reload
+    assert_raise(ActiveRecord::RecordInvalid) {c.unlock} # Running to Queued is not allowed
+    c.reload
+
+    assert c.update_attributes(state: Container::Complete), show_errors(c)
+    refute c.locked_by_uuid
+    refute c.auth_uuid
+
+    auth_exp = ApiClientAuthorization.find_by_uuid(auth_uuid_was).expires_at
+    assert_operator auth_exp, :<, db_current_time
+  end
+
+  test "Container queued cancel" do
+    c, _ = minimal_new
+    set_user_from_auth :dispatch1
+    assert c.update_attributes(state: Container::Cancelled), show_errors(c)
+    check_no_change_from_cancelled c
+  end
+
+  test "Container locked cancel" do
+    c, _ = minimal_new
+    set_user_from_auth :dispatch1
+    assert c.lock, show_errors(c)
+    assert c.update_attributes(state: Container::Cancelled), show_errors(c)
+    check_no_change_from_cancelled c
+  end
+
+  test "Container running cancel" do
+    c, _ = minimal_new
+    set_user_from_auth :dispatch1
+    c.lock
+    c.update_attributes! state: Container::Running
+    c.update_attributes! state: Container::Cancelled
+    check_no_change_from_cancelled c
   end
 
   test "Container create forbidden for non-admin" do
     set_user_from_auth :active_trustedclient
-    c = minimal_new
+    c = Container.new DEFAULT_ATTRS
     c.environment = {}
     c.mounts = {"BAR" => "FOO"}
     c.output_path = "/tmp"
@@ -189,31 +417,14 @@ class ContainerTest < ActiveSupport::TestCase
   end
 
   test "Container only set exit code on complete" do
-    act_as_system_user do
-      c = minimal_new
-      c.save!
+    c, _ = minimal_new
+    set_user_from_auth :dispatch1
+    c.lock
+    c.update_attributes! state: Container::Running
 
-      c.reload
-      c.state = "Running"
-      c.save!
-
-      c.reload
-      c.exit_code = 1
-      assert_raises(ActiveRecord::RecordInvalid) do
-        c.save!
-      end
-
-      c.reload
-      c.exit_code = 1
-      c.state = "Cancelled"
-      assert_raises(ActiveRecord::RecordInvalid) do
-        c.save!
-      end
+    check_illegal_updates c, [{exit_code: 1},
+                              {exit_code: 1, state: Container::Cancelled}]
 
-      c.reload
-      c.exit_code = 1
-      c.state = "Complete"
-      c.save!
-    end
+    assert c.update_attributes(exit_code: 1, state: Container::Complete)
   end
 end