From: Peter Amstutz <pamstutz@veritasgenetics.com>
Date: Thu, 11 Oct 2018 22:27:51 +0000 (-0400)
Subject: 14260: Added runtime_token to container record
X-Git-Tag: 1.3.0~70^2~15
X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/3453440f95dc76db1bd39fe8c85d9a898d474c6b

14260: Added runtime_token to container record

* runtime_token, runtime_user_uuid and runtime_auth_scopes are now
  part of container initialization and reuse decisions

* Determine runtime_user_uuid and runtime_auth_scopes as part of
  Container.resolve

* Use runtime_user_uuid to create container token (when runtime_token
  is not set)

* act_as runtime_user_uuid when resolving container request fields

* tokens used for runtime_token will be left untouched (remove expire_destroy)

* added/updated/fixed tests

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz@veritasgenetics.com>
---

diff --git a/services/api/app/models/api_client_authorization.rb b/services/api/app/models/api_client_authorization.rb
index 0a2793aded..12ef8eb3eb 100644
--- a/services/api/app/models/api_client_authorization.rb
+++ b/services/api/app/models/api_client_authorization.rb
@@ -92,27 +92,6 @@ class ApiClientAuthorization < ArvadosModel
        uuid_prefix+".arvadosapi.com")
   end
 
-  # Delete token, if remote, attempt to delete on the host as well.
-  def expire_destroy
-      uuid_prefix = self.uuid[0..4]
-      if uuid_prefix != Rails.configuration.uuid_prefix
-        # remote token
-        host = remote_host(uuid_prefix: uuid_prefix)
-        begin
-          clnt = HTTPClient.new
-          if Rails.configuration.sso_insecure
-            clnt.ssl_config.verify_mode = OpenSSL::SSL::VERIFY_NONE
-          end
-          result = SafeJSON.load(
-            clnt.delete('https://' + host + '/arvados/v1/users/current',
-                        {'Authorization' => 'Bearer ' + token}))
-        rescue => e
-          Rails.logger.warn "deleting remote token #{self.uuid} failed: #{e}"
-        end
-      end
-      destroy
-  end
-
   def self.validate(token:, remote: nil)
     return nil if !token
     remote ||= Rails.configuration.uuid_prefix
diff --git a/services/api/app/models/container.rb b/services/api/app/models/container.rb
index ba5c1c28ca..86201955aa 100644
--- a/services/api/app/models/container.rb
+++ b/services/api/app/models/container.rb
@@ -37,7 +37,7 @@ class Container < ArvadosModel
   after_validation :assign_auth
   before_save :sort_serialized_attrs
   before_save :update_secret_mounts_md5
-  before_save :scrub_secret_mounts
+  before_save :scrub_secrets
   before_save :clear_runtime_status_when_queued
   after_save :update_cr_logs
   after_save :handle_completed
@@ -91,15 +91,15 @@ class Container < ArvadosModel
   end
 
   def self.full_text_searchable_columns
-    super - ["secret_mounts", "secret_mounts_md5"]
+    super - ["secret_mounts", "secret_mounts_md5", "runtime_token"]
   end
 
   def self.searchable_columns *args
-    super - ["secret_mounts_md5"]
+    super - ["secret_mounts_md5", "runtime_token"]
   end
 
   def logged_attributes
-    super.except('secret_mounts')
+    super.except('secret_mounts', 'runtime_token')
   end
 
   def state_transitions
@@ -146,17 +146,37 @@ class Container < ArvadosModel
   # Create a new container (or find an existing one) to satisfy the
   # given container request.
   def self.resolve(req)
-    c_attrs = {
-      command: req.command,
-      cwd: req.cwd,
-      environment: req.environment,
-      output_path: req.output_path,
-      container_image: resolve_container_image(req.container_image),
-      mounts: resolve_mounts(req.mounts),
-      runtime_constraints: resolve_runtime_constraints(req.runtime_constraints),
-      scheduling_parameters: req.scheduling_parameters,
-      secret_mounts: req.secret_mounts,
-    }
+    if req.runtime_token.nil?
+      runtime_user = if req.modified_by_user_uuid.nil?
+                       current_user
+                     else
+                       User.find_by_uuid(req.modified_by_user_uuid)
+                     end
+      runtime_auth_scopes = ["all"]
+    else
+      auth = ApiClientAuthorization.validate(token: req.runtime_token)
+      if auth.nil?
+        raise ArgumentError.new "Invalid runtime token"
+      end
+      runtime_user = User.find_by_id(auth.user_id)
+      runtime_auth_scopes = auth.scopes
+    end
+    c_attrs = act_as_user runtime_user do
+      {
+        command: req.command,
+        cwd: req.cwd,
+        environment: req.environment,
+        output_path: req.output_path,
+        container_image: resolve_container_image(req.container_image),
+        mounts: resolve_mounts(req.mounts),
+        runtime_constraints: resolve_runtime_constraints(req.runtime_constraints),
+        scheduling_parameters: req.scheduling_parameters,
+        secret_mounts: req.secret_mounts,
+        runtime_token: req.runtime_token,
+        runtime_user_uuid: runtime_user.uuid,
+        runtime_auth_scopes: runtime_auth_scopes
+      }
+    end
     act_as_system_user do
       if req.use_existing && (reusable = find_reusable(c_attrs))
         reusable
@@ -259,6 +279,14 @@ class Container < ArvadosModel
     candidates = candidates.where_serialized(:runtime_constraints, resolve_runtime_constraints(attrs[:runtime_constraints]), md5: true)
     log_reuse_info(candidates) { "after filtering on runtime_constraints #{attrs[:runtime_constraints].inspect}" }
 
+    candidates = candidates.where('runtime_user_uuid = ? or (runtime_user_uuid is NULL and runtime_auth_scopes is NULL)',
+                                  attrs[:runtime_user_uuid])
+    log_reuse_info(candidates) { "after filtering on runtime_user_uuid #{attrs[:runtime_user_uuid].inspect}" }
+
+    candidates = candidates.where('runtime_auth_scopes = ? or (runtime_user_uuid is NULL and runtime_auth_scopes is NULL)',
+                                  SafeJSON.dump(attrs[:runtime_auth_scopes]))
+    log_reuse_info(candidates) { "after filtering on runtime_auth_scopes #{attrs[:runtime_auth_scopes].inspect}" }
+
     log_reuse_info { "checking for state=Complete with readable output and log..." }
 
     select_readable_pdh = Collection.
@@ -415,7 +443,8 @@ class Container < ArvadosModel
       permitted.push(:owner_uuid, :command, :container_image, :cwd,
                      :environment, :mounts, :output_path, :priority,
                      :runtime_constraints, :scheduling_parameters,
-                     :secret_mounts)
+                     :secret_mounts, :runtime_token,
+                     :runtime_user_uuid, :runtime_auth_scopes)
     end
 
     case self.state
@@ -522,28 +551,30 @@ class Container < ArvadosModel
       # already have one
       return
     end
-    cr = ContainerRequest.
-      where('container_uuid=? and priority>0', self.uuid).
-      order('priority desc').
-      first
-    if !cr
-      return errors.add :auth_uuid, "cannot be assigned because priority <= 0"
-    end
-    if cr.runtime_token.nil?
+    if self.runtime_token.nil?
+      if self.runtime_user_uuid.nil?
+        cr = ContainerRequest.
+               where('container_uuid=? and priority>0', self.uuid).
+               order('priority desc').
+               first
+        if !cr
+          return errors.add :auth_uuid, "cannot be assigned because priority <= 0"
+        end
+        self.runtime_user_uuid = cr.modified_by_user_uuid
+        self.runtime_auth_scopes = ["all"]
+      end
+
       # generate a new token
       self.auth = ApiClientAuthorization.
-                    create!(user_id: User.find_by_uuid(cr.modified_by_user_uuid).id,
-                            api_client_id: 0)
-      self.runtime_user_uuid = cr.modified_by_user_uuid
-      self.runtime_auth_scopes = self.auth.scopes
+                    create!(user_id: User.find_by_uuid(self.runtime_user_uuid).id,
+                            api_client_id: 0,
+                            scopes: self.runtime_auth_scopes)
     else
-      # using cr.runtime_token
-      self.auth = ApiClientAuthorization.validate(token: cr.runtime_token)
+      # using runtime_token
+      self.auth = ApiClientAuthorization.validate(token: self.runtime_token)
       if self.auth.nil?
         raise ArgumentError.new "Invalid runtime token"
       end
-      self.runtime_user_uuid = User.find_by_id(self.auth.user_id).uuid
-      self.runtime_auth_scopes = self.auth.scopes
     end
   end
 
@@ -569,12 +600,13 @@ class Container < ArvadosModel
     end
   end
 
-  def scrub_secret_mounts
+  def scrub_secrets
     # this runs after update_secret_mounts_md5, so the
     # secret_mounts_md5 will still reflect the secrets that are being
     # scrubbed here.
     if self.state_changed? && self.final?
       self.secret_mounts = {}
+      self.runtime_token = nil
     end
   end
 
diff --git a/services/api/app/models/container_request.rb b/services/api/app/models/container_request.rb
index b75775c878..cd68517096 100644
--- a/services/api/app/models/container_request.rb
+++ b/services/api/app/models/container_request.rb
@@ -106,7 +106,7 @@ class ContainerRequest < ArvadosModel
   end
 
   def skip_uuid_read_permission_check
-    # XXX temporary until permissions are sorted out.
+  # XXX temporary until permissions are sorted out.
     %w(modified_by_client_uuid container_uuid requesting_container_uuid)
   end
 
@@ -359,14 +359,7 @@ class ContainerRequest < ArvadosModel
   def scrub_secrets
     if self.state == Final
       self.secret_mounts = {}
-      if !self.runtime_token.nil?
-        _, uuid, secret = self.runtime_token.split('/')
-        tok = ApiClientAuthorization.find_by_uuid(uuid)
-        if !tok.nil?
-          tok.expire_destroy
-        end
-        self.runtime_token = nil
-      end
+      self.runtime_token = nil
     end
   end
 
diff --git a/services/api/db/migrate/20181011184200_add_runtime_token_to_container.rb b/services/api/db/migrate/20181011184200_add_runtime_token_to_container.rb
new file mode 100644
index 0000000000..09201f5146
--- /dev/null
+++ b/services/api/db/migrate/20181011184200_add_runtime_token_to_container.rb
@@ -0,0 +1,5 @@
+class AddRuntimeTokenToContainer < ActiveRecord::Migration
+  def change
+    add_column :containers, :runtime_token, :text, :null => true
+  end
+end
diff --git a/services/api/db/structure.sql b/services/api/db/structure.sql
index d1eb8d8d06..636306f976 100644
--- a/services/api/db/structure.sql
+++ b/services/api/db/structure.sql
@@ -358,7 +358,8 @@ CREATE TABLE public.containers (
     secret_mounts_md5 character varying DEFAULT '99914b932bd37a50b983c5e7c90ae93b'::character varying,
     runtime_status jsonb DEFAULT '{}'::jsonb,
     runtime_user_uuid text,
-    runtime_auth_scopes jsonb
+    runtime_auth_scopes jsonb,
+    runtime_token text
 );
 
 
@@ -3176,3 +3177,5 @@ INSERT INTO schema_migrations (version) VALUES ('20180917205609');
 
 INSERT INTO schema_migrations (version) VALUES ('20181005192222');
 
+INSERT INTO schema_migrations (version) VALUES ('20181011184200');
+
diff --git a/services/api/lib/sweep_trashed_objects.rb b/services/api/lib/sweep_trashed_objects.rb
index 59008c0fc3..162bebf513 100644
--- a/services/api/lib/sweep_trashed_objects.rb
+++ b/services/api/lib/sweep_trashed_objects.rb
@@ -48,6 +48,11 @@ module SweepTrashedObjects
         where({group_class: 'project'}).
         where('is_trashed = false and trash_at < statement_timestamp()').
         update_all('is_trashed = true')
+
+      # Sweep expired tokens
+      ApiClientAuthorization.
+        where("expires_at <= statement_timestamp()").
+        destroy_all
     end
   end
 
diff --git a/services/api/test/fixtures/api_client_authorizations.yml b/services/api/test/fixtures/api_client_authorizations.yml
index 9074c5ffc1..d8ef63120b 100644
--- a/services/api/test/fixtures/api_client_authorizations.yml
+++ b/services/api/test/fixtures/api_client_authorizations.yml
@@ -345,6 +345,21 @@ foo_collection_sharing_token:
 container_runtime_token:
   uuid: zzzzz-gj3su-2nj68s291f50gd9
   api_client: untrusted
-  user: spectator
+  user: container_runtime_token_user
   api_token: 2d19ue6ofx26o3mm7fs9u6t7hov9um0v92dzwk1o2xed3abprw
   expires_at: 2038-01-01 00:00:00
+
+crt_user:
+  uuid: zzzzz-gj3su-3r47qqy5ja5d54v
+  api_client: untrusted
+  user: container_runtime_token_user
+  api_token: 13z1tz9deoryml3twep0vsahi4862097pe5lsmesugnkgpgpwk
+  expires_at: 2038-01-01 00:00:00
+
+runtime_token_limited_scope:
+  uuid: zzzzz-gj3su-2fljvypjrr4yr9m
+  api_client: untrusted
+  user: container_runtime_token_user
+  api_token: 1fwc3be1m13qkypix2gd01i4bq5ju483zjfc0cf4babjseirbm
+  expires_at: 2038-01-01 00:00:00
+  scopes: ["GET /"]
diff --git a/services/api/test/fixtures/containers.yml b/services/api/test/fixtures/containers.yml
index ce61c01eed..eefb0297c6 100644
--- a/services/api/test/fixtures/containers.yml
+++ b/services/api/test/fixtures/containers.yml
@@ -271,6 +271,9 @@ runtime_token:
   cwd: test
   output_path: test
   command: ["echo", "hello"]
+  runtime_token: v2/zzzzz-gj3su-2nj68s291f50gd9/2d19ue6ofx26o3mm7fs9u6t7hov9um0v92dzwk1o2xed3abprw
+  runtime_user_uuid: zzzzz-tpzed-l3skomkti0c4vg4
+  runtime_auth_scopes: ["all"]
   runtime_constraints:
     ram: 12000000000
     vcpus: 4
diff --git a/services/api/test/fixtures/links.yml b/services/api/test/fixtures/links.yml
index 8a33f696a9..2b247a960d 100644
--- a/services/api/test/fixtures/links.yml
+++ b/services/api/test/fixtures/links.yml
@@ -597,6 +597,20 @@ active_user_permission_to_unlinked_docker_image_collection:
   head_uuid: zzzzz-4zz18-d0d8z5wofvfgwad
   properties: {}
 
+crt_user_permission_to_unlinked_docker_image_collection:
+  uuid: zzzzz-o0j2j-20zvdi9b4odcfz3
+  owner_uuid: zzzzz-tpzed-000000000000000
+  created_at: 2014-01-24 20:42:26 -0800
+  modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+  modified_by_user_uuid: zzzzz-tpzed-000000000000000
+  modified_at: 2014-01-24 20:42:26 -0800
+  updated_at: 2014-01-24 20:42:26 -0800
+  tail_uuid: zzzzz-tpzed-l3skomkti0c4vg4
+  link_class: permission
+  name: can_read
+  head_uuid: zzzzz-4zz18-d0d8z5wofvfgwad
+  properties: {}
+
 docker_image_collection_hash:
   uuid: zzzzz-o0j2j-dockercollhasha
   owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
diff --git a/services/api/test/fixtures/users.yml b/services/api/test/fixtures/users.yml
index 8d25869219..7d6b1fc3ae 100644
--- a/services/api/test/fixtures/users.yml
+++ b/services/api/test/fixtures/users.yml
@@ -165,6 +165,22 @@ spectator:
       role: Computational biologist
     getting_started_shown: 2015-03-26 12:34:56.789000000 Z
 
+container_runtime_token_user:
+  owner_uuid: zzzzz-tpzed-000000000000000
+  uuid: zzzzz-tpzed-l3skomkti0c4vg4
+  email: spectator@arvados.local
+  first_name: Spect
+  last_name: Ator
+  identity_url: https://spectator.openid.local
+  is_active: true
+  is_admin: false
+  username: containerruntimetokenuser
+  prefs:
+    profile:
+      organization: example.com
+      role: Computational biologist
+    getting_started_shown: 2015-03-26 12:34:56.789000000 Z
+
 inactive_uninvited:
   owner_uuid: zzzzz-tpzed-000000000000000
   uuid: zzzzz-tpzed-rf2ec3ryh4vb5ma
diff --git a/services/api/test/unit/container_request_test.rb b/services/api/test/unit/container_request_test.rb
index 00c341f74f..14fa5796d0 100644
--- a/services/api/test/unit/container_request_test.rb
+++ b/services/api/test/unit/container_request_test.rb
@@ -1076,8 +1076,8 @@ class ContainerRequestTest < ActiveSupport::TestCase
   end
 
   test "using runtime_token" do
-    set_user_from_auth :active
-    spec = api_client_authorizations(:spectator)
+    set_user_from_auth :spectator
+    spec = api_client_authorizations(:active)
     cr = create_minimal_req!(state: "Committed", runtime_token: spec.token, priority: 1)
     cr.save!
     c = Container.find_by_uuid cr.container_uuid
@@ -1096,13 +1096,13 @@ class ContainerRequestTest < ActiveSupport::TestCase
     cr.reload
     c.reload
     assert_nil cr.runtime_token
-    assert_nil ApiClientAuthorization.find_by_uuid(spec.uuid)
+    assert_nil c.runtime_token
   end
 
   test "invalid runtime_token" do
     set_user_from_auth :active
     spec = api_client_authorizations(:spectator)
-    assert_raises(ActiveRecord::RecordInvalid) do
+    assert_raises(ArgumentError) do
       cr = create_minimal_req!(state: "Committed", runtime_token: "#{spec.token}xx")
       cr.save!
     end
diff --git a/services/api/test/unit/container_test.rb b/services/api/test/unit/container_test.rb
index 11ae0bfe3b..39fde79a33 100644
--- a/services/api/test/unit/container_test.rb
+++ b/services/api/test/unit/container_test.rb
@@ -33,14 +33,18 @@ class ContainerTest < ActiveSupport::TestCase
       "var" => "val",
     },
     secret_mounts: {},
+    runtime_user_uuid: "zzzzz-tpzed-xurymjxw79nv3jz",
+    runtime_auth_scopes: ["all"]
   }
 
+  def request_only attrs
+    attrs.reject {|k| [:runtime_user_uuid, :runtime_auth_scopes].include? k}
+  end
+
   def minimal_new attrs={}
-    cr = ContainerRequest.new DEFAULT_ATTRS.merge(attrs)
+    cr = ContainerRequest.new request_only(DEFAULT_ATTRS.merge(attrs))
     cr.state = ContainerRequest::Committed
-    act_as_user users(:active) do
-      cr.save!
-    end
+    cr.save!
     c = Container.find_by_uuid cr.container_uuid
     assert_not_nil c
     return c, cr
@@ -220,6 +224,7 @@ class ContainerTest < ActiveSupport::TestCase
   end
 
   test "Container serialized hash attributes sorted before save" do
+    set_user_from_auth :active
     env = {"C" => "3", "B" => "2", "A" => "1"}
     m = {"F" => {"kind" => "3"}, "E" => {"kind" => "2"}, "D" => {"kind" => "1"}}
     rc = {"vcpus" => 1, "ram" => 1, "keep_cache_ram" => 1}
@@ -236,6 +241,7 @@ class ContainerTest < ActiveSupport::TestCase
   end
 
   test "find_reusable method should select higher priority queued container" do
+        Rails.configuration.log_reuse_decisions = true
     set_user_from_auth :active
     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment:{"var" => "queued"}})
     c_low_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:1}))
@@ -285,13 +291,13 @@ class ContainerTest < ActiveSupport::TestCase
       log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
     }
 
-    cr = ContainerRequest.new common_attrs
+    cr = ContainerRequest.new request_only(common_attrs)
     cr.use_existing = false
     cr.state = ContainerRequest::Committed
     cr.save!
     c_output1 = Container.where(uuid: cr.container_uuid).first
 
-    cr = ContainerRequest.new common_attrs
+    cr = ContainerRequest.new request_only(common_attrs)
     cr.use_existing = false
     cr.state = ContainerRequest::Committed
     cr.save!
@@ -312,7 +318,8 @@ class ContainerTest < ActiveSupport::TestCase
     c_output2.update_attributes!({state: Container::Running})
     c_output2.update_attributes!(completed_attrs.merge({log: log1, output: out2}))
 
-    reused = Container.resolve(ContainerRequest.new(common_attrs))
+    set_user_from_auth :active
+    reused = Container.resolve(ContainerRequest.new(request_only(common_attrs)))
     assert_equal c_output1.uuid, reused.uuid
   end
 
@@ -507,7 +514,73 @@ class ContainerTest < ActiveSupport::TestCase
     Container.find_reusable(REUSABLE_COMMON_ATTRS)
   end
 
+  def runtime_token_attr tok
+    auth = api_client_authorizations(tok)
+    {runtime_user_uuid: User.find_by_id(auth.user_id).uuid,
+     runtime_auth_scopes: auth.scopes,
+     runtime_token: auth.token}
+  end
+
+  test "find_reusable method with same runtime_token" do
+    set_user_from_auth :active
+    common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
+    c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:container_runtime_token).token}))
+    assert_equal Container::Queued, c1.state
+    reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
+    assert_not_nil reused
+    assert_equal reused.uuid, c1.uuid
+  end
+
+  test "find_reusable method with different runtime_token, same user" do
+    set_user_from_auth :active
+    common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
+    c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:crt_user).token}))
+    assert_equal Container::Queued, c1.state
+    reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
+    assert_not_nil reused
+    assert_equal reused.uuid, c1.uuid
+  end
+
+  test "find_reusable method with nil runtime_token, then runtime_token with same user" do
+    set_user_from_auth :crt_user
+    common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
+    c1, _ = minimal_new(common_attrs)
+    assert_equal Container::Queued, c1.state
+    assert_equal users(:container_runtime_token_user).uuid, c1.runtime_user_uuid
+    reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
+    assert_not_nil reused
+    assert_equal reused.uuid, c1.uuid
+  end
+
+  test "find_reusable method with different runtime_token, different user" do
+    set_user_from_auth :crt_user
+    common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
+    c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:active).token}))
+    assert_equal Container::Queued, c1.state
+    reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
+    assert_nil reused
+  end
+
+  test "find_reusable method with nil runtime_token, then runtime_token with different user" do
+    set_user_from_auth :active
+    common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
+    c1, _ = minimal_new(common_attrs.merge({runtime_token: nil}))
+    assert_equal Container::Queued, c1.state
+    reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
+    assert_nil reused
+  end
+
+  test "find_reusable method with different runtime_token, different scope, same user" do
+    set_user_from_auth :active
+    common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
+    c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:runtime_token_limited_scope).token}))
+    assert_equal Container::Queued, c1.state
+    reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
+    assert_nil reused
+  end
+
   test "Container running" do
+    set_user_from_auth :active
     c, _ = minimal_new priority: 1
 
     set_user_from_auth :dispatch1
@@ -527,6 +600,7 @@ class ContainerTest < ActiveSupport::TestCase
   end
 
   test "Lock and unlock" do
+    set_user_from_auth :active
     c, cr = minimal_new priority: 0
 
     set_user_from_auth :dispatch1
@@ -587,6 +661,7 @@ class ContainerTest < ActiveSupport::TestCase
   end
 
   test "Container queued cancel" do
+    set_user_from_auth :active
     c, cr = minimal_new({container_count_max: 1})
     set_user_from_auth :dispatch1
     assert c.update_attributes(state: Container::Cancelled), show_errors(c)
@@ -596,10 +671,11 @@ class ContainerTest < ActiveSupport::TestCase
   end
 
   test "Container queued count" do
-    assert_equal 1, Container.readable_by(users(:active)).where(state: "Queued").count
+    assert_equal 2, Container.readable_by(users(:active)).where(state: "Queued").count
   end
 
   test "Container locked cancel" do
+    set_user_from_auth :active
     c, _ = minimal_new
     set_user_from_auth :dispatch1
     assert c.lock, show_errors(c)
@@ -608,6 +684,7 @@ class ContainerTest < ActiveSupport::TestCase
   end
 
   test "Container locked cancel with log" do
+    set_user_from_auth :active
     c, _ = minimal_new
     set_user_from_auth :dispatch1
     assert c.lock, show_errors(c)
@@ -619,6 +696,7 @@ class ContainerTest < ActiveSupport::TestCase
   end
 
   test "Container running cancel" do
+    set_user_from_auth :active
     c, _ = minimal_new
     set_user_from_auth :dispatch1
     c.lock
@@ -641,6 +719,7 @@ class ContainerTest < ActiveSupport::TestCase
   end
 
   test "Container only set exit code on complete" do
+    set_user_from_auth :active
     c, _ = minimal_new
     set_user_from_auth :dispatch1
     c.lock
@@ -653,6 +732,7 @@ class ContainerTest < ActiveSupport::TestCase
   end
 
   test "locked_by_uuid can update log when locked/running, and output when running" do
+    set_user_from_auth :active
     logcoll = collections(:real_log_collection)
     c, cr1 = minimal_new
     cr2 = ContainerRequest.new(DEFAULT_ATTRS)
@@ -698,6 +778,7 @@ class ContainerTest < ActiveSupport::TestCase
   end
 
   test "auth_uuid can set output, progress, runtime_status, state on running container -- but not log" do
+    set_user_from_auth :active
     c, _ = minimal_new
     set_user_from_auth :dispatch1
     c.lock
@@ -718,6 +799,7 @@ class ContainerTest < ActiveSupport::TestCase
   end
 
   test "not allowed to set output that is not readable by current user" do
+    set_user_from_auth :active
     c, _ = minimal_new
     set_user_from_auth :dispatch1
     c.lock
@@ -732,6 +814,7 @@ class ContainerTest < ActiveSupport::TestCase
   end
 
   test "other token cannot set output on running container" do
+    set_user_from_auth :active
     c, _ = minimal_new
     set_user_from_auth :dispatch1
     c.lock
@@ -742,6 +825,7 @@ class ContainerTest < ActiveSupport::TestCase
   end
 
   test "can set trashed output on running container" do
+    set_user_from_auth :active
     c, _ = minimal_new
     set_user_from_auth :dispatch1
     c.lock
@@ -755,6 +839,7 @@ class ContainerTest < ActiveSupport::TestCase
   end
 
   test "not allowed to set trashed output that is not readable by current user" do
+    set_user_from_auth :active
     c, _ = minimal_new
     set_user_from_auth :dispatch1
     c.lock
@@ -774,20 +859,24 @@ class ContainerTest < ActiveSupport::TestCase
     {state: Container::Complete, exit_code: 0, output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'},
     {state: Container::Cancelled},
   ].each do |final_attrs|
-    test "secret_mounts is null after container is #{final_attrs[:state]}" do
+    test "secret_mounts and runtime_token are null after container is #{final_attrs[:state]}" do
+      set_user_from_auth :active
       c, cr = minimal_new(secret_mounts: {'/secret' => {'kind' => 'text', 'content' => 'foo'}},
-                          container_count_max: 1)
+                          container_count_max: 1, runtime_token: api_client_authorizations(:active).token)
       set_user_from_auth :dispatch1
       c.lock
       c.update_attributes!(state: Container::Running)
       c.reload
       assert c.secret_mounts.has_key?('/secret')
+      assert_equal api_client_authorizations(:active).token, c.runtime_token
 
       c.update_attributes!(final_attrs)
       c.reload
       assert_equal({}, c.secret_mounts)
+      assert_nil c.runtime_token
       cr.reload
       assert_equal({}, cr.secret_mounts)
+      assert_nil cr.runtime_token
       assert_no_secrets_logged
     end
   end