Merge branch '14291-cdc-aws' refs #14291

[arvados.git] / services / api / test / unit / container_test.rb

diff --git a/services/api/test/unit/container_test.rb b/services/api/test/unit/container_test.rb
index eb4f35fea3d293c922d07fc6d16981f81b5092ee..1a53df7dab4a2b76e260b00d213f659a35db1b29 100644 (file)
--- a/services/api/test/unit/container_test.rb
+++ b/services/api/test/unit/container_test.rb
@@ -3,9 +3,11 @@
 # SPDX-License-Identifier: AGPL-3.0
 
 require 'test_helper'
 # SPDX-License-Identifier: AGPL-3.0
 
 require 'test_helper'

+require 'helpers/container_test_helper'

 
 class ContainerTest < ActiveSupport::TestCase
   include DbCurrentTime
 
 class ContainerTest < ActiveSupport::TestCase
   include DbCurrentTime

+  include ContainerTestHelper

 
   DEFAULT_ATTRS = {
     command: ['echo', 'foo'],
 
   DEFAULT_ATTRS = {
     command: ['echo', 'foo'],


@@ -30,14 +32,19 @@ class ContainerTest < ActiveSupport::TestCase
     environment: {
       "var" => "val",
     },
     environment: {
       "var" => "val",
     },

+    secret_mounts: {},
+    runtime_user_uuid: "zzzzz-tpzed-xurymjxw79nv3jz",
+    runtime_auth_scopes: ["all"]

   }
 
   }
 

+  def request_only attrs
+    attrs.reject {|k| [:runtime_user_uuid, :runtime_auth_scopes].include? k}
+  end
+

   def minimal_new attrs={}
   def minimal_new attrs={}

-    cr = ContainerRequest.new DEFAULT_ATTRS.merge(attrs)
+    cr = ContainerRequest.new request_only(DEFAULT_ATTRS.merge(attrs))

     cr.state = ContainerRequest::Committed
     cr.state = ContainerRequest::Committed

-    act_as_user users(:active) do
-      cr.save!
-    end
+    cr.save!

     c = Container.find_by_uuid cr.container_uuid
     assert_not_nil c
     return c, cr
     c = Container.find_by_uuid cr.container_uuid
     assert_not_nil c
     return c, cr


@@ -81,7 +88,7 @@ class ContainerTest < ActiveSupport::TestCase
   test "Container create" do
     act_as_system_user do
       c, _ = minimal_new(environment: {},
   test "Container create" do
     act_as_system_user do
       c, _ = minimal_new(environment: {},

-                      mounts: {"BAR" => "FOO"},
+                      mounts: {"BAR" => {"kind" => "FOO"}},

                       output_path: "/tmp",
                       priority: 1,
                       runtime_constraints: {"vcpus" => 1, "ram" => 1})
                       output_path: "/tmp",
                       priority: 1,
                       runtime_constraints: {"vcpus" => 1, "ram" => 1})


@@ -98,7 +105,7 @@ class ContainerTest < ActiveSupport::TestCase
   test "Container valid priority" do
     act_as_system_user do
       c, _ = minimal_new(environment: {},
   test "Container valid priority" do
     act_as_system_user do
       c, _ = minimal_new(environment: {},

-                      mounts: {"BAR" => "FOO"},
+                      mounts: {"BAR" => {"kind" => "FOO"}},

                       output_path: "/tmp",
                       priority: 1,
                       runtime_constraints: {"vcpus" => 1, "ram" => 1})
                       output_path: "/tmp",
                       priority: 1,
                       runtime_constraints: {"vcpus" => 1, "ram" => 1})


@@ -123,17 +130,103 @@ class ContainerTest < ActiveSupport::TestCase
       c.priority = 1000
       c.save!
 
       c.priority = 1000
       c.save!
 

-      assert_raises(ActiveRecord::RecordInvalid) do
-        c.priority = 1001
-        c.save!
+      c.priority = 1000 << 50
+      c.save!
+    end
+  end
+
+  test "Container runtime_status data types" do
+    set_user_from_auth :active
+    attrs = {
+      environment: {},
+      mounts: {"BAR" => {"kind" => "FOO"}},
+      output_path: "/tmp",
+      priority: 1,
+      runtime_constraints: {"vcpus" => 1, "ram" => 1}
+    }
+    c, _ = minimal_new(attrs)
+    assert_equal c.runtime_status, {}
+    assert_equal Container::Queued, c.state
+
+    set_user_from_auth :dispatch1
+    c.update_attributes! state: Container::Locked
+    c.update_attributes! state: Container::Running
+
+    [
+      'error', 'errorDetail', 'warning', 'warningDetail', 'activity'
+    ].each do |k|
+      # String type is allowed
+      string_val = 'A string is accepted'
+      c.update_attributes! runtime_status: {k => string_val}
+      assert_equal string_val, c.runtime_status[k]
+
+      # Other types aren't allowed
+      [
+        42, false, [], {}, nil
+      ].each do |unallowed_val|
+        assert_raises ActiveRecord::RecordInvalid do
+          c.update_attributes! runtime_status: {k => unallowed_val}
+        end

       end
     end
   end
 
       end
     end
   end
 

+  test "Container runtime_status updates" do
+    set_user_from_auth :active
+    attrs = {
+      environment: {},
+      mounts: {"BAR" => {"kind" => "FOO"}},
+      output_path: "/tmp",
+      priority: 1,
+      runtime_constraints: {"vcpus" => 1, "ram" => 1}
+    }
+    c1, _ = minimal_new(attrs)
+    assert_equal c1.runtime_status, {}
+
+    assert_equal Container::Queued, c1.state
+    assert_raises ActiveRecord::RecordInvalid do
+      c1.update_attributes! runtime_status: {'error' => 'Oops!'}
+    end
+
+    set_user_from_auth :dispatch1
+
+    # Allow updates when state = Locked
+    c1.update_attributes! state: Container::Locked
+    c1.update_attributes! runtime_status: {'error' => 'Oops!'}
+    assert c1.runtime_status.key? 'error'
+
+    # Reset when transitioning from Locked to Queued
+    c1.update_attributes! state: Container::Queued
+    assert_equal c1.runtime_status, {}
+
+    # Allow updates when state = Running
+    c1.update_attributes! state: Container::Locked
+    c1.update_attributes! state: Container::Running
+    c1.update_attributes! runtime_status: {'error' => 'Oops!'}
+    assert c1.runtime_status.key? 'error'
+
+    # Don't allow updates on other states
+    c1.update_attributes! state: Container::Complete
+    assert_raises ActiveRecord::RecordInvalid do
+      c1.update_attributes! runtime_status: {'error' => 'Some other error'}
+    end
+
+    set_user_from_auth :active
+    c2, _ = minimal_new(attrs)
+    assert_equal c2.runtime_status, {}
+    set_user_from_auth :dispatch1
+    c2.update_attributes! state: Container::Locked
+    c2.update_attributes! state: Container::Running
+    c2.update_attributes! state: Container::Cancelled
+    assert_raises ActiveRecord::RecordInvalid do
+      c2.update_attributes! runtime_status: {'error' => 'Oops!'}
+    end
+  end

 
   test "Container serialized hash attributes sorted before save" do
 
   test "Container serialized hash attributes sorted before save" do

-    env = {"C" => 3, "B" => 2, "A" => 1}
-    m = {"F" => {"kind" => 3}, "E" => {"kind" => 2}, "D" => {"kind" => 1}}
+    set_user_from_auth :active
+    env = {"C" => "3", "B" => "2", "A" => "1"}
+    m = {"F" => {"kind" => "3"}, "E" => {"kind" => "2"}, "D" => {"kind" => "1"}}

     rc = {"vcpus" => 1, "ram" => 1, "keep_cache_ram" => 1}
     c, _ = minimal_new(environment: env, mounts: m, runtime_constraints: rc)
     assert_equal c.environment.to_json, Container.deep_sort_hash(env).to_json
     rc = {"vcpus" => 1, "ram" => 1, "keep_cache_ram" => 1}
     c, _ = minimal_new(environment: env, mounts: m, runtime_constraints: rc)
     assert_equal c.environment.to_json, Container.deep_sort_hash(env).to_json


@@ -148,6 +241,7 @@ class ContainerTest < ActiveSupport::TestCase
   end
 
   test "find_reusable method should select higher priority queued container" do
   end
 
   test "find_reusable method should select higher priority queued container" do

+        Rails.configuration.log_reuse_decisions = true

     set_user_from_auth :active
     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment:{"var" => "queued"}})
     c_low_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:1}))
     set_user_from_auth :active
     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment:{"var" => "queued"}})
     c_low_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:1}))


@@ -197,13 +291,13 @@ class ContainerTest < ActiveSupport::TestCase
       log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
     }
 
       log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
     }
 

-    cr = ContainerRequest.new common_attrs
+    cr = ContainerRequest.new request_only(common_attrs)

     cr.use_existing = false
     cr.state = ContainerRequest::Committed
     cr.save!
     c_output1 = Container.where(uuid: cr.container_uuid).first
 
     cr.use_existing = false
     cr.state = ContainerRequest::Committed
     cr.save!
     c_output1 = Container.where(uuid: cr.container_uuid).first
 

-    cr = ContainerRequest.new common_attrs
+    cr = ContainerRequest.new request_only(common_attrs)

     cr.use_existing = false
     cr.state = ContainerRequest::Committed
     cr.save!
     cr.use_existing = false
     cr.state = ContainerRequest::Committed
     cr.save!


@@ -224,7 +318,8 @@ class ContainerTest < ActiveSupport::TestCase
     c_output2.update_attributes!({state: Container::Running})
     c_output2.update_attributes!(completed_attrs.merge({log: log1, output: out2}))
 
     c_output2.update_attributes!({state: Container::Running})
     c_output2.update_attributes!(completed_attrs.merge({log: log1, output: out2}))
 

-    reused = Container.resolve(ContainerRequest.new(common_attrs))
+    set_user_from_auth :active
+    reused = Container.resolve(ContainerRequest.new(request_only(common_attrs)))

     assert_equal c_output1.uuid, reused.uuid
   end
 
     assert_equal c_output1.uuid, reused.uuid
   end
 


@@ -276,6 +371,32 @@ class ContainerTest < ActiveSupport::TestCase
     assert_equal reused.uuid, c_faster_started_second.uuid
   end
 
     assert_equal reused.uuid, c_faster_started_second.uuid
   end
 

+  test "find_reusable method should select non-failing running container" do
+    set_user_from_auth :active
+    common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running2"}})
+    c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
+    c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
+    c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
+    # Confirm the 3 container UUIDs are different.
+    assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
+    set_user_from_auth :dispatch1
+    c_slower.update_attributes!({state: Container::Locked})
+    c_slower.update_attributes!({state: Container::Running,
+                                 progress: 0.1})
+    c_faster_started_first.update_attributes!({state: Container::Locked})
+    c_faster_started_first.update_attributes!({state: Container::Running,
+                                               runtime_status: {'warning' => 'This is not an error'},
+                                               progress: 0.15})
+    c_faster_started_second.update_attributes!({state: Container::Locked})
+    c_faster_started_second.update_attributes!({state: Container::Running,
+                                                runtime_status: {'error' => 'Something bad happened'},
+                                                progress: 0.2})
+    reused = Container.find_reusable(common_attrs)
+    assert_not_nil reused
+    # Selected the non-failing container even if it's the one with less progress done
+    assert_equal reused.uuid, c_faster_started_first.uuid
+  end
+

   test "find_reusable method should select locked container most likely to start sooner" do
     set_user_from_auth :active
     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "locked"}})
   test "find_reusable method should select locked container most likely to start sooner" do
     set_user_from_auth :active
     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "locked"}})


@@ -393,7 +514,76 @@ class ContainerTest < ActiveSupport::TestCase
     Container.find_reusable(REUSABLE_COMMON_ATTRS)
   end
 
     Container.find_reusable(REUSABLE_COMMON_ATTRS)
   end
 

+  def runtime_token_attr tok
+    auth = api_client_authorizations(tok)
+    {runtime_user_uuid: User.find_by_id(auth.user_id).uuid,
+     runtime_auth_scopes: auth.scopes,
+     runtime_token: auth.token}
+  end
+
+  test "find_reusable method with same runtime_token" do
+    set_user_from_auth :active
+    common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
+    c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:container_runtime_token).token}))
+    assert_equal Container::Queued, c1.state
+    reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
+    assert_not_nil reused
+    assert_equal reused.uuid, c1.uuid
+  end
+
+  test "find_reusable method with different runtime_token, same user" do
+    set_user_from_auth :active
+    common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
+    c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:crt_user).token}))
+    assert_equal Container::Queued, c1.state
+    reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
+    assert_not_nil reused
+    assert_equal reused.uuid, c1.uuid
+  end
+
+  test "find_reusable method with nil runtime_token, then runtime_token with same user" do
+    set_user_from_auth :crt_user
+    common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
+    c1, _ = minimal_new(common_attrs)
+    assert_equal Container::Queued, c1.state
+    assert_equal users(:container_runtime_token_user).uuid, c1.runtime_user_uuid
+    reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
+    assert_not_nil reused
+    assert_equal reused.uuid, c1.uuid
+  end
+
+  test "find_reusable method with different runtime_token, different user" do
+    set_user_from_auth :crt_user
+    common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
+    c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:active).token}))
+    assert_equal Container::Queued, c1.state
+    reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
+    # See #14584
+    assert_equal c1.uuid, reused.uuid
+  end
+
+  test "find_reusable method with nil runtime_token, then runtime_token with different user" do
+    set_user_from_auth :active
+    common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
+    c1, _ = minimal_new(common_attrs.merge({runtime_token: nil}))
+    assert_equal Container::Queued, c1.state
+    reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
+    # See #14584
+    assert_equal c1.uuid, reused.uuid
+  end
+
+  test "find_reusable method with different runtime_token, different scope, same user" do
+    set_user_from_auth :active
+    common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
+    c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:runtime_token_limited_scope).token}))
+    assert_equal Container::Queued, c1.state
+    reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
+    # See #14584
+    assert_equal c1.uuid, reused.uuid
+  end
+

   test "Container running" do
   test "Container running" do

+    set_user_from_auth :active

     c, _ = minimal_new priority: 1
 
     set_user_from_auth :dispatch1
     c, _ = minimal_new priority: 1
 
     set_user_from_auth :dispatch1


@@ -413,6 +603,7 @@ class ContainerTest < ActiveSupport::TestCase
   end
 
   test "Lock and unlock" do
   end
 
   test "Lock and unlock" do

+    set_user_from_auth :active

     c, cr = minimal_new priority: 0
 
     set_user_from_auth :dispatch1
     c, cr = minimal_new priority: 0
 
     set_user_from_auth :dispatch1


@@ -472,18 +663,76 @@ class ContainerTest < ActiveSupport::TestCase
     assert_operator auth_exp, :<, db_current_time
   end
 
     assert_operator auth_exp, :<, db_current_time
   end
 

+  test "Exceed maximum lock-unlock cycles" do
+    Rails.configuration.max_container_dispatch_attempts = 3
+
+    set_user_from_auth :active
+    c, cr = minimal_new
+
+    set_user_from_auth :dispatch1
+    assert_equal Container::Queued, c.state
+    assert_equal 0, c.lock_count
+
+    c.lock
+    c.reload
+    assert_equal 1, c.lock_count
+    assert_equal Container::Locked, c.state
+
+    c.unlock
+    c.reload
+    assert_equal 1, c.lock_count
+    assert_equal Container::Queued, c.state
+
+    c.lock
+    c.reload
+    assert_equal 2, c.lock_count
+    assert_equal Container::Locked, c.state
+
+    c.unlock
+    c.reload
+    assert_equal 2, c.lock_count
+    assert_equal Container::Queued, c.state
+
+    c.lock
+    c.reload
+    assert_equal 3, c.lock_count
+    assert_equal Container::Locked, c.state
+
+    c.unlock
+    c.reload
+    assert_equal 3, c.lock_count
+    assert_equal Container::Cancelled, c.state
+
+    assert_raise(ArvadosModel::LockFailedError) do
+      # Cancelled to Locked is not allowed
+      c.lock
+    end
+  end
+

   test "Container queued cancel" do
   test "Container queued cancel" do

-    c, _ = minimal_new
+    set_user_from_auth :active
+    c, cr = minimal_new({container_count_max: 1})

     set_user_from_auth :dispatch1
     assert c.update_attributes(state: Container::Cancelled), show_errors(c)
     check_no_change_from_cancelled c
     set_user_from_auth :dispatch1
     assert c.update_attributes(state: Container::Cancelled), show_errors(c)
     check_no_change_from_cancelled c

+    cr.reload
+    assert_equal ContainerRequest::Final, cr.state

   end
 
   test "Container queued count" do
     assert_equal 1, Container.readable_by(users(:active)).where(state: "Queued").count
   end
 
   end
 
   test "Container queued count" do
     assert_equal 1, Container.readable_by(users(:active)).where(state: "Queued").count
   end
 

+  test "Containers with no matching request are readable by admin" do
+    uuids = Container.includes('container_requests').where(container_requests: {uuid: nil}).collect(&:uuid)
+    assert_not_empty uuids
+    assert_empty Container.readable_by(users(:active)).where(uuid: uuids)
+    assert_not_empty Container.readable_by(users(:admin)).where(uuid: uuids)
+    assert_equal uuids.count, Container.readable_by(users(:admin)).where(uuid: uuids).count
+  end
+

   test "Container locked cancel" do
   test "Container locked cancel" do

+    set_user_from_auth :active

     c, _ = minimal_new
     set_user_from_auth :dispatch1
     assert c.lock, show_errors(c)
     c, _ = minimal_new
     set_user_from_auth :dispatch1
     assert c.lock, show_errors(c)


@@ -492,6 +741,7 @@ class ContainerTest < ActiveSupport::TestCase
   end
 
   test "Container locked cancel with log" do
   end
 
   test "Container locked cancel with log" do

+    set_user_from_auth :active

     c, _ = minimal_new
     set_user_from_auth :dispatch1
     assert c.lock, show_errors(c)
     c, _ = minimal_new
     set_user_from_auth :dispatch1
     assert c.lock, show_errors(c)


@@ -503,6 +753,7 @@ class ContainerTest < ActiveSupport::TestCase
   end
 
   test "Container running cancel" do
   end
 
   test "Container running cancel" do

+    set_user_from_auth :active

     c, _ = minimal_new
     set_user_from_auth :dispatch1
     c.lock
     c, _ = minimal_new
     set_user_from_auth :dispatch1
     c.lock


@@ -525,6 +776,7 @@ class ContainerTest < ActiveSupport::TestCase
   end
 
   test "Container only set exit code on complete" do
   end
 
   test "Container only set exit code on complete" do

+    set_user_from_auth :active

     c, _ = minimal_new
     set_user_from_auth :dispatch1
     c.lock
     c, _ = minimal_new
     set_user_from_auth :dispatch1
     c.lock


@@ -536,35 +788,94 @@ class ContainerTest < ActiveSupport::TestCase
     assert c.update_attributes(exit_code: 1, state: Container::Complete)
   end
 
     assert c.update_attributes(exit_code: 1, state: Container::Complete)
   end
 

-  test "locked_by_uuid can set output on running container" do
-    c, _ = minimal_new
-    set_user_from_auth :dispatch1
-    c.lock
-    c.update_attributes! state: Container::Running
-
-    assert_equal c.locked_by_uuid, Thread.current[:api_client_authorization].uuid
+  test "locked_by_uuid can update log when locked/running, and output when running" do
+    set_user_from_auth :active
+    logcoll = collections(:real_log_collection)
+    c, cr1 = minimal_new
+    cr2 = ContainerRequest.new(DEFAULT_ATTRS)
+    cr2.state = ContainerRequest::Committed
+    act_as_user users(:active) do
+      cr2.save!
+    end
+    assert_equal cr1.container_uuid, cr2.container_uuid

 
 

-    assert c.update_attributes output: collections(:collection_owned_by_active).portable_data_hash
-    assert c.update_attributes! state: Container::Complete
-  end
+    logpdh_time1 = logcoll.portable_data_hash

 
 

-  test "auth_uuid can set output on running container, but not change container state" do
-    c, _ = minimal_new

     set_user_from_auth :dispatch1
     c.lock
     set_user_from_auth :dispatch1
     c.lock

-    c.update_attributes! state: Container::Running
-
-    Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
-    Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
-    assert c.update_attributes output: collections(:collection_owned_by_active).portable_data_hash
+    assert_equal c.locked_by_uuid, Thread.current[:api_client_authorization].uuid
+    c.update_attributes!(log: logpdh_time1)
+    c.update_attributes!(state: Container::Running)
+    cr1.reload
+    cr2.reload
+    cr1log_uuid = cr1.log_uuid
+    cr2log_uuid = cr2.log_uuid
+    assert_not_nil cr1log_uuid
+    assert_not_nil cr2log_uuid
+    assert_not_equal logcoll.uuid, cr1log_uuid
+    assert_not_equal logcoll.uuid, cr2log_uuid
+    assert_not_equal cr1log_uuid, cr2log_uuid
+
+    logcoll.update_attributes!(manifest_text: logcoll.manifest_text + ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n")
+    logpdh_time2 = logcoll.portable_data_hash
+
+    assert c.update_attributes(output: collections(:collection_owned_by_active).portable_data_hash)
+    assert c.update_attributes(log: logpdh_time2)
+    assert c.update_attributes(state: Container::Complete, log: logcoll.portable_data_hash)
+    c.reload
+    assert_equal collections(:collection_owned_by_active).portable_data_hash, c.output
+    assert_equal logpdh_time2, c.log
+    refute c.update_attributes(output: nil)
+    refute c.update_attributes(log: nil)
+    cr1.reload
+    cr2.reload
+    assert_equal cr1log_uuid, cr1.log_uuid
+    assert_equal cr2log_uuid, cr2.log_uuid
+    assert_equal 1, Collection.where(uuid: [cr1log_uuid, cr2log_uuid]).to_a.collect(&:portable_data_hash).uniq.length
+    assert_equal ". acbd18db4cc2f85cedef654fccc4a4d8+3 cdd549ae79fe6640fa3d5c6261d8303c+195 0:3:foo.txt 3:195:zzzzz-8i9sb-0vsrcqi7whchuil.log.txt
+./log\\040for\\040container\\040#{cr1.container_uuid} acbd18db4cc2f85cedef654fccc4a4d8+3 cdd549ae79fe6640fa3d5c6261d8303c+195 0:3:foo.txt 3:195:zzzzz-8i9sb-0vsrcqi7whchuil.log.txt
+", Collection.find_by_uuid(cr1log_uuid).manifest_text
+  end
+
+  ["auth_uuid", "runtime_token"].each do |tok|
+    test "#{tok} can set output, progress, runtime_status, state on running container -- but not log" do
+      if tok == "runtime_token"
+        set_user_from_auth :spectator
+        c, _ = minimal_new(container_image: "9ae44d5792468c58bcf85ce7353c7027+124",
+                           runtime_token: api_client_authorizations(:active).token)
+      else
+        set_user_from_auth :active
+        c, _ = minimal_new
+      end
+      set_user_from_auth :dispatch1
+      c.lock
+      c.update_attributes! state: Container::Running
+
+      if tok == "runtime_token"
+        auth = ApiClientAuthorization.validate(token: c.runtime_token)
+        Thread.current[:api_client_authorization] = auth
+        Thread.current[:api_client] = auth.api_client
+        Thread.current[:token] = auth.token
+        Thread.current[:user] = auth.user
+      else
+        auth = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
+        Thread.current[:api_client_authorization] = auth
+        Thread.current[:api_client] = auth.api_client
+        Thread.current[:token] = auth.token
+        Thread.current[:user] = auth.user
+      end

 
 

-    assert_raises ArvadosModel::PermissionDeniedError do
-      # auth_uuid cannot set container state
-      c.update_attributes state: Container::Complete
+      assert c.update_attributes(output: collections(:collection_owned_by_active).portable_data_hash)
+      assert c.update_attributes(runtime_status: {'warning' => 'something happened'})
+      assert c.update_attributes(progress: 0.5)
+      refute c.update_attributes(log: collections(:real_log_collection).portable_data_hash)
+      c.reload
+      assert c.update_attributes(state: Container::Complete, exit_code: 0)

     end
   end
 
   test "not allowed to set output that is not readable by current user" do
     end
   end
 
   test "not allowed to set output that is not readable by current user" do

+    set_user_from_auth :active

     c, _ = minimal_new
     set_user_from_auth :dispatch1
     c.lock
     c, _ = minimal_new
     set_user_from_auth :dispatch1
     c.lock


@@ -579,24 +890,24 @@ class ContainerTest < ActiveSupport::TestCase
   end
 
   test "other token cannot set output on running container" do
   end
 
   test "other token cannot set output on running container" do

+    set_user_from_auth :active

     c, _ = minimal_new
     set_user_from_auth :dispatch1
     c.lock
     c.update_attributes! state: Container::Running
 
     c, _ = minimal_new
     set_user_from_auth :dispatch1
     c.lock
     c.update_attributes! state: Container::Running
 

-    set_user_from_auth :not_running_container_auth
-    assert_raises ArvadosModel::PermissionDeniedError do
-      c.update_attributes! output: collections(:foo_file).portable_data_hash
-    end
+    set_user_from_auth :running_to_be_deleted_container_auth
+    refute c.update_attributes(output: collections(:foo_file).portable_data_hash)

   end
 
   test "can set trashed output on running container" do
   end
 
   test "can set trashed output on running container" do

+    set_user_from_auth :active

     c, _ = minimal_new
     set_user_from_auth :dispatch1
     c.lock
     c.update_attributes! state: Container::Running
 
     c, _ = minimal_new
     set_user_from_auth :dispatch1
     c.lock
     c.update_attributes! state: Container::Running
 

-    output = Collection.unscoped.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jk')
+    output = Collection.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jk')

 
     assert output.is_trashed
     assert c.update_attributes output: output.portable_data_hash
 
     assert output.is_trashed
     assert c.update_attributes output: output.portable_data_hash


@@ -604,12 +915,13 @@ class ContainerTest < ActiveSupport::TestCase
   end
 
   test "not allowed to set trashed output that is not readable by current user" do
   end
 
   test "not allowed to set trashed output that is not readable by current user" do

+    set_user_from_auth :active

     c, _ = minimal_new
     set_user_from_auth :dispatch1
     c.lock
     c.update_attributes! state: Container::Running
 
     c, _ = minimal_new
     set_user_from_auth :dispatch1
     c.lock
     c.update_attributes! state: Container::Running
 

-    output = Collection.unscoped.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jr')
+    output = Collection.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jr')

 
     Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
     Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
 
     Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
     Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)


@@ -619,4 +931,29 @@ class ContainerTest < ActiveSupport::TestCase
     end
   end
 
     end
   end
 

+  [
+    {state: Container::Complete, exit_code: 0, output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'},
+    {state: Container::Cancelled},
+  ].each do |final_attrs|
+    test "secret_mounts and runtime_token are null after container is #{final_attrs[:state]}" do
+      set_user_from_auth :active
+      c, cr = minimal_new(secret_mounts: {'/secret' => {'kind' => 'text', 'content' => 'foo'}},
+                          container_count_max: 1, runtime_token: api_client_authorizations(:active).token)
+      set_user_from_auth :dispatch1
+      c.lock
+      c.update_attributes!(state: Container::Running)
+      c.reload
+      assert c.secret_mounts.has_key?('/secret')
+      assert_equal api_client_authorizations(:active).token, c.runtime_token
+
+      c.update_attributes!(final_attrs)
+      c.reload
+      assert_equal({}, c.secret_mounts)
+      assert_nil c.runtime_token
+      cr.reload
+      assert_equal({}, cr.secret_mounts)
+      assert_nil cr.runtime_token
+      assert_no_secrets_logged
+    end
+  end

 end
 end