18948: Update test.

[arvados.git] / services / api / test / unit / container_test.rb

diff --git a/services/api/test/unit/container_test.rb b/services/api/test/unit/container_test.rb
index 491022ad8d5a9cd6e47e1cf7727a5cba92d54ce4..bcf99da2e3442ba088d9effbe80a633e6467f857 100644 (file)
--- a/services/api/test/unit/container_test.rb
+++ b/services/api/test/unit/container_test.rb
@@ -14,7 +14,7 @@ class ContainerTest < ActiveSupport::TestCase
     container_image: 'fa3c1a9cb6783f85f2ecda037e07b8c3+167',
     output_path: '/tmp',
     priority: 1,
     container_image: 'fa3c1a9cb6783f85f2ecda037e07b8c3+167',
     output_path: '/tmp',
     priority: 1,

-    runtime_constraints: {"vcpus" => 1, "ram" => 1},
+    runtime_constraints: {"vcpus" => 1, "ram" => 1, "cuda" => {"device_count":0, "driver_version": "", "hardware_capability": ""}},

   }
 
   REUSABLE_COMMON_ATTRS = {
   }
 
   REUSABLE_COMMON_ATTRS = {


@@ -23,8 +23,10 @@ class ContainerTest < ActiveSupport::TestCase
     command: ["echo", "hello"],
     output_path: "test",
     runtime_constraints: {
     command: ["echo", "hello"],
     output_path: "test",
     runtime_constraints: {

+      "API" => false,
+      "keep_cache_ram" => 0,

       "ram" => 12000000000,
       "ram" => 12000000000,

-      "vcpus" => 4,
+      "vcpus" => 4

     },
     mounts: {
       "test" => {"kind" => "json"},
     },
     mounts: {
       "test" => {"kind" => "json"},


@@ -184,7 +186,7 @@ class ContainerTest < ActiveSupport::TestCase
     assert_equal c1.runtime_status, {}
 
     assert_equal Container::Queued, c1.state
     assert_equal c1.runtime_status, {}
 
     assert_equal Container::Queued, c1.state

-    assert_raises ActiveRecord::RecordInvalid do
+    assert_raises ArvadosModel::PermissionDeniedError do

       c1.update_attributes! runtime_status: {'error' => 'Oops!'}
     end
 
       c1.update_attributes! runtime_status: {'error' => 'Oops!'}
     end
 


@@ -227,11 +229,12 @@ class ContainerTest < ActiveSupport::TestCase
     set_user_from_auth :active
     env = {"C" => "3", "B" => "2", "A" => "1"}
     m = {"F" => {"kind" => "3"}, "E" => {"kind" => "2"}, "D" => {"kind" => "1"}}
     set_user_from_auth :active
     env = {"C" => "3", "B" => "2", "A" => "1"}
     m = {"F" => {"kind" => "3"}, "E" => {"kind" => "2"}, "D" => {"kind" => "1"}}

-    rc = {"vcpus" => 1, "ram" => 1, "keep_cache_ram" => 1}
+    rc = {"vcpus" => 1, "ram" => 1, "keep_cache_ram" => 1, "API" => true, "cuda" => {"device_count":0, "driver_version": "", "hardware_capability": ""}}

     c, _ = minimal_new(environment: env, mounts: m, runtime_constraints: rc)
     c, _ = minimal_new(environment: env, mounts: m, runtime_constraints: rc)

-    assert_equal c.environment.to_json, Container.deep_sort_hash(env).to_json
-    assert_equal c.mounts.to_json, Container.deep_sort_hash(m).to_json
-    assert_equal c.runtime_constraints.to_json, Container.deep_sort_hash(rc).to_json
+    c.reload
+    assert_equal Container.deep_sort_hash(env).to_json, c.environment.to_json
+    assert_equal Container.deep_sort_hash(m).to_json, c.mounts.to_json
+    assert_equal Container.deep_sort_hash(rc).to_json, c.runtime_constraints.to_json

   end
 
   test 'deep_sort_hash on array of hashes' do
   end
 
   test 'deep_sort_hash on array of hashes' do


@@ -241,7 +244,7 @@ class ContainerTest < ActiveSupport::TestCase
   end
 
   test "find_reusable method should select higher priority queued container" do
   end
 
   test "find_reusable method should select higher priority queued container" do

-        Rails.configuration.log_reuse_decisions = true
+        Rails.configuration.Containers.LogReuseDecisions = true

     set_user_from_auth :active
     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment:{"var" => "queued"}})
     c_low_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:1}))
     set_user_from_auth :active
     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment:{"var" => "queued"}})
     c_low_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:1}))


@@ -388,9 +391,11 @@ class ContainerTest < ActiveSupport::TestCase
                                                runtime_status: {'warning' => 'This is not an error'},
                                                progress: 0.15})
     c_faster_started_second.update_attributes!({state: Container::Locked})
                                                runtime_status: {'warning' => 'This is not an error'},
                                                progress: 0.15})
     c_faster_started_second.update_attributes!({state: Container::Locked})

+    assert_equal 0, Container.where("runtime_status->'error' is not null").count

     c_faster_started_second.update_attributes!({state: Container::Running,
                                                 runtime_status: {'error' => 'Something bad happened'},
                                                 progress: 0.2})
     c_faster_started_second.update_attributes!({state: Container::Running,
                                                 runtime_status: {'error' => 'Something bad happened'},
                                                 progress: 0.2})

+    assert_equal 1, Container.where("runtime_status->'error' is not null").count

     reused = Container.find_reusable(common_attrs)
     assert_not_nil reused
     # Selected the non-failing container even if it's the one with less progress done
     reused = Container.find_reusable(common_attrs)
     assert_not_nil reused
     # Selected the non-failing container even if it's the one with less progress done


@@ -509,7 +514,7 @@ class ContainerTest < ActiveSupport::TestCase
 
   test "find_reusable with logging enabled" do
     set_user_from_auth :active
 
   test "find_reusable with logging enabled" do
     set_user_from_auth :active

-    Rails.configuration.log_reuse_decisions = true
+    Rails.configuration.Containers.LogReuseDecisions = true

     Rails.logger.expects(:info).at_least(3)
     Container.find_reusable(REUSABLE_COMMON_ATTRS)
   end
     Rails.logger.expects(:info).at_least(3)
     Container.find_reusable(REUSABLE_COMMON_ATTRS)
   end


@@ -558,7 +563,9 @@ class ContainerTest < ActiveSupport::TestCase
     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:active).token}))
     assert_equal Container::Queued, c1.state
     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:active).token}))
     assert_equal Container::Queued, c1.state
     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))

-    assert_nil reused
+    # See #14584
+    assert_not_nil reused
+    assert_equal c1.uuid, reused.uuid

   end
 
   test "find_reusable method with nil runtime_token, then runtime_token with different user" do
   end
 
   test "find_reusable method with nil runtime_token, then runtime_token with different user" do


@@ -567,7 +574,9 @@ class ContainerTest < ActiveSupport::TestCase
     c1, _ = minimal_new(common_attrs.merge({runtime_token: nil}))
     assert_equal Container::Queued, c1.state
     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
     c1, _ = minimal_new(common_attrs.merge({runtime_token: nil}))
     assert_equal Container::Queued, c1.state
     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))

-    assert_nil reused
+    # See #14584
+    assert_not_nil reused
+    assert_equal c1.uuid, reused.uuid

   end
 
   test "find_reusable method with different runtime_token, different scope, same user" do
   end
 
   test "find_reusable method with different runtime_token, different scope, same user" do


@@ -576,7 +585,36 @@ class ContainerTest < ActiveSupport::TestCase
     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:runtime_token_limited_scope).token}))
     assert_equal Container::Queued, c1.state
     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:runtime_token_limited_scope).token}))
     assert_equal Container::Queued, c1.state
     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))

-    assert_nil reused
+    # See #14584
+    assert_not_nil reused
+    assert_equal c1.uuid, reused.uuid
+  end
+
+  test "find_reusable method with cuda" do
+    set_user_from_auth :active
+    # No cuda
+    no_cuda_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"},
+                                                runtime_constraints: {"vcpus" => 1, "ram" => 1, "keep_cache_ram"=>268435456, "API" => false,
+                                                                      "cuda" => {"device_count":0, "driver_version": "", "hardware_capability": ""}},})
+    c1, _ = minimal_new(no_cuda_attrs)
+    assert_equal Container::Queued, c1.state
+
+    # has cuda
+    cuda_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"},
+                                                runtime_constraints: {"vcpus" => 1, "ram" => 1, "keep_cache_ram"=>268435456, "API" => false,
+                                                                      "cuda" => {"device_count":1, "driver_version": "11.0", "hardware_capability": "9.0"}},})
+    c2, _ = minimal_new(cuda_attrs)
+    assert_equal Container::Queued, c2.state
+
+    # should find the no cuda one
+    reused = Container.find_reusable(no_cuda_attrs)
+    assert_not_nil reused
+    assert_equal reused.uuid, c1.uuid
+
+    # should find the cuda one
+    reused = Container.find_reusable(cuda_attrs)
+    assert_not_nil reused
+    assert_equal reused.uuid, c2.uuid

   end
 
   test "Container running" do
   end
 
   test "Container running" do


@@ -658,6 +696,54 @@ class ContainerTest < ActiveSupport::TestCase
 
     auth_exp = ApiClientAuthorization.find_by_uuid(auth_uuid_was).expires_at
     assert_operator auth_exp, :<, db_current_time
 
     auth_exp = ApiClientAuthorization.find_by_uuid(auth_uuid_was).expires_at
     assert_operator auth_exp, :<, db_current_time

+
+    assert_nil ApiClientAuthorization.validate(token: ApiClientAuthorization.find_by_uuid(auth_uuid_was).token)
+  end
+
+  test "Exceed maximum lock-unlock cycles" do
+    Rails.configuration.Containers.MaxDispatchAttempts = 3
+
+    set_user_from_auth :active
+    c, cr = minimal_new
+
+    set_user_from_auth :dispatch1
+    assert_equal Container::Queued, c.state
+    assert_equal 0, c.lock_count
+
+    c.lock
+    c.reload
+    assert_equal 1, c.lock_count
+    assert_equal Container::Locked, c.state
+
+    c.unlock
+    c.reload
+    assert_equal 1, c.lock_count
+    assert_equal Container::Queued, c.state
+
+    c.lock
+    c.reload
+    assert_equal 2, c.lock_count
+    assert_equal Container::Locked, c.state
+
+    c.unlock
+    c.reload
+    assert_equal 2, c.lock_count
+    assert_equal Container::Queued, c.state
+
+    c.lock
+    c.reload
+    assert_equal 3, c.lock_count
+    assert_equal Container::Locked, c.state
+
+    c.unlock
+    c.reload
+    assert_equal 3, c.lock_count
+    assert_equal Container::Cancelled, c.state
+
+    assert_raise(ArvadosModel::LockFailedError) do
+      # Cancelled to Locked is not allowed
+      c.lock
+    end

   end
 
   test "Container queued cancel" do
   end
 
   test "Container queued cancel" do


@@ -674,6 +760,14 @@ class ContainerTest < ActiveSupport::TestCase
     assert_equal 1, Container.readable_by(users(:active)).where(state: "Queued").count
   end
 
     assert_equal 1, Container.readable_by(users(:active)).where(state: "Queued").count
   end
 

+  test "Containers with no matching request are readable by admin" do
+    uuids = Container.includes('container_requests').where(container_requests: {uuid: nil}).collect(&:uuid)
+    assert_not_empty uuids
+    assert_empty Container.readable_by(users(:active)).where(uuid: uuids)
+    assert_not_empty Container.readable_by(users(:admin)).where(uuid: uuids)
+    assert_equal uuids.count, Container.readable_by(users(:admin)).where(uuid: uuids).count
+  end
+

   test "Container locked cancel" do
     set_user_from_auth :active
     c, _ = minimal_new
   test "Container locked cancel" do
     set_user_from_auth :active
     c, _ = minimal_new


@@ -683,6 +777,17 @@ class ContainerTest < ActiveSupport::TestCase
     check_no_change_from_cancelled c
   end
 
     check_no_change_from_cancelled c
   end
 

+  test "Container locked with non-expiring token" do
+    Rails.configuration.API.TokenMaxLifetime = 1.hour
+    set_user_from_auth :active
+    c, _ = minimal_new
+    set_user_from_auth :dispatch1
+    assert c.lock, show_errors(c)
+    refute c.auth.nil?
+    assert c.auth.expires_at.nil?
+    assert c.auth.user_id == User.find_by_uuid(users(:active).uuid).id
+  end
+

   test "Container locked cancel with log" do
     set_user_from_auth :active
     c, _ = minimal_new
   test "Container locked cancel with log" do
     set_user_from_auth :active
     c, _ = minimal_new


@@ -718,16 +823,61 @@ class ContainerTest < ActiveSupport::TestCase
     end
   end
 
     end
   end
 

-  test "Container only set exit code on complete" do
+  [
+    [Container::Queued, {state: Container::Locked}],
+    [Container::Queued, {state: Container::Running}],
+    [Container::Queued, {state: Container::Complete}],
+    [Container::Queued, {state: Container::Cancelled}],
+    [Container::Queued, {priority: 123456789}],
+    [Container::Queued, {runtime_status: {'error' => 'oops'}}],
+    [Container::Queued, {cwd: '/'}],
+    [Container::Locked, {state: Container::Running}],
+    [Container::Locked, {state: Container::Queued}],
+    [Container::Locked, {priority: 123456789}],
+    [Container::Locked, {runtime_status: {'error' => 'oops'}}],
+    [Container::Locked, {cwd: '/'}],
+    [Container::Running, {state: Container::Complete}],
+    [Container::Running, {state: Container::Cancelled}],
+    [Container::Running, {priority: 123456789}],
+    [Container::Running, {runtime_status: {'error' => 'oops'}}],
+    [Container::Running, {cwd: '/'}],
+    [Container::Running, {gateway_address: "172.16.0.1:12345"}],
+    [Container::Running, {interactive_session_started: true}],
+    [Container::Complete, {state: Container::Cancelled}],
+    [Container::Complete, {priority: 123456789}],
+    [Container::Complete, {runtime_status: {'error' => 'oops'}}],
+    [Container::Complete, {cwd: '/'}],
+    [Container::Cancelled, {cwd: '/'}],
+  ].each do |start_state, updates|
+    test "Container update #{updates.inspect} when #{start_state} forbidden for non-admin" do
+      set_user_from_auth :active
+      c, _ = minimal_new
+      if start_state != Container::Queued
+        set_user_from_auth :dispatch1
+        c.lock
+        if start_state != Container::Locked
+          c.update_attributes! state: Container::Running
+          if start_state != Container::Running
+            c.update_attributes! state: start_state
+          end
+        end
+      end
+      assert_equal c.state, start_state
+      set_user_from_auth :active
+      assert_raises(ArvadosModel::PermissionDeniedError) do
+        c.update_attributes! updates
+      end
+    end
+  end
+
+  test "can only change exit code while running and at completion" do

     set_user_from_auth :active
     c, _ = minimal_new
     set_user_from_auth :dispatch1
     c.lock
     set_user_from_auth :active
     c, _ = minimal_new
     set_user_from_auth :dispatch1
     c.lock

+    check_illegal_updates c, [{exit_code: 1}]

     c.update_attributes! state: Container::Running
     c.update_attributes! state: Container::Running

-
-    check_illegal_updates c, [{exit_code: 1},
-                              {exit_code: 1, state: Container::Cancelled}]
-
+    assert c.update_attributes(exit_code: 1)

     assert c.update_attributes(exit_code: 1, state: Container::Complete)
   end
 
     assert c.update_attributes(exit_code: 1, state: Container::Complete)
   end
 


@@ -774,28 +924,48 @@ class ContainerTest < ActiveSupport::TestCase
     cr2.reload
     assert_equal cr1log_uuid, cr1.log_uuid
     assert_equal cr2log_uuid, cr2.log_uuid
     cr2.reload
     assert_equal cr1log_uuid, cr1.log_uuid
     assert_equal cr2log_uuid, cr2.log_uuid

-    assert_equal [logpdh_time2], Collection.where(uuid: [cr1log_uuid, cr2log_uuid]).to_a.collect(&:portable_data_hash).uniq
+    assert_equal 1, Collection.where(uuid: [cr1log_uuid, cr2log_uuid]).to_a.collect(&:portable_data_hash).uniq.length
+    assert_equal ". acbd18db4cc2f85cedef654fccc4a4d8+3 cdd549ae79fe6640fa3d5c6261d8303c+195 0:3:foo.txt 3:195:zzzzz-8i9sb-0vsrcqi7whchuil.log.txt
+./log\\040for\\040container\\040#{cr1.container_uuid} acbd18db4cc2f85cedef654fccc4a4d8+3 cdd549ae79fe6640fa3d5c6261d8303c+195 0:3:foo.txt 3:195:zzzzz-8i9sb-0vsrcqi7whchuil.log.txt
+", Collection.find_by_uuid(cr1log_uuid).manifest_text

   end
 
   end
 

-  test "auth_uuid can set output, progress, runtime_status, state on running container -- but not log" do
-    set_user_from_auth :active
-    c, _ = minimal_new
-    set_user_from_auth :dispatch1
-    c.lock
-    c.update_attributes! state: Container::Running
-
-    auth = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
-    Thread.current[:api_client_authorization] = auth
-    Thread.current[:api_client] = auth.api_client
-    Thread.current[:token] = auth.token
-    Thread.current[:user] = auth.user
+  ["auth_uuid", "runtime_token"].each do |tok|
+    test "#{tok} can set output, progress, runtime_status, state, exit_code on running container -- but not log" do
+      if tok == "runtime_token"
+        set_user_from_auth :spectator
+        c, _ = minimal_new(container_image: "9ae44d5792468c58bcf85ce7353c7027+124",
+                           runtime_token: api_client_authorizations(:active).token)
+      else
+        set_user_from_auth :active
+        c, _ = minimal_new
+      end
+      set_user_from_auth :dispatch1
+      c.lock
+      c.update_attributes! state: Container::Running
+
+      if tok == "runtime_token"
+        auth = ApiClientAuthorization.validate(token: c.runtime_token)
+        Thread.current[:api_client_authorization] = auth
+        Thread.current[:api_client] = auth.api_client
+        Thread.current[:token] = auth.token
+        Thread.current[:user] = auth.user
+      else
+        auth = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
+        Thread.current[:api_client_authorization] = auth
+        Thread.current[:api_client] = auth.api_client
+        Thread.current[:token] = auth.token
+        Thread.current[:user] = auth.user
+      end

 
 

-    assert c.update_attributes(output: collections(:collection_owned_by_active).portable_data_hash)
-    assert c.update_attributes(runtime_status: {'warning' => 'something happened'})
-    assert c.update_attributes(progress: 0.5)
-    refute c.update_attributes(log: collections(:real_log_collection).portable_data_hash)
-    c.reload
-    assert c.update_attributes(state: Container::Complete, exit_code: 0)
+      assert c.update_attributes(output: collections(:collection_owned_by_active).portable_data_hash)
+      assert c.update_attributes(runtime_status: {'warning' => 'something happened'})
+      assert c.update_attributes(progress: 0.5)
+      assert c.update_attributes(exit_code: 0)
+      refute c.update_attributes(log: collections(:real_log_collection).portable_data_hash)
+      c.reload
+      assert c.update_attributes(state: Container::Complete, exit_code: 0)
+    end

   end
 
   test "not allowed to set output that is not readable by current user" do
   end
 
   test "not allowed to set output that is not readable by current user" do


@@ -821,7 +991,9 @@ class ContainerTest < ActiveSupport::TestCase
     c.update_attributes! state: Container::Running
 
     set_user_from_auth :running_to_be_deleted_container_auth
     c.update_attributes! state: Container::Running
 
     set_user_from_auth :running_to_be_deleted_container_auth

-    refute c.update_attributes(output: collections(:foo_file).portable_data_hash)
+    assert_raises(ArvadosModel::PermissionDeniedError) do
+      c.update_attributes(output: collections(:foo_file).portable_data_hash)
+    end

   end
 
   test "can set trashed output on running container" do
   end
 
   test "can set trashed output on running container" do


@@ -855,6 +1027,15 @@ class ContainerTest < ActiveSupport::TestCase
     end
   end
 
     end
   end
 

+  test "user cannot delete" do
+    set_user_from_auth :active
+    c, _ = minimal_new
+    assert_raises ArvadosModel::PermissionDeniedError do
+      c.destroy
+    end
+    assert Container.find_by_uuid(c.uuid)
+  end
+

   [
     {state: Container::Complete, exit_code: 0, output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'},
     {state: Container::Cancelled},
   [
     {state: Container::Complete, exit_code: 0, output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'},
     {state: Container::Cancelled},