services/api/test/unit/container_test.rb

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 require 'test_helper'
   6 require 'helpers/container_test_helper'
   7
   8 class ContainerTest < ActiveSupport::TestCase
   9   include DbCurrentTime
  10   include ContainerTestHelper
  11
  12   DEFAULT_ATTRS = {
  13     command: ['echo', 'foo'],
  14     container_image: 'fa3c1a9cb6783f85f2ecda037e07b8c3+167',
  15     output_path: '/tmp',
  16     priority: 1,
  17     runtime_constraints: {"vcpus" => 1, "ram" => 1},
  18   }
  19
  20   REUSABLE_COMMON_ATTRS = {
  21     container_image: "9ae44d5792468c58bcf85ce7353c7027+124",
  22     cwd: "test",
  23     command: ["echo", "hello"],
  24     output_path: "test",
  25     runtime_constraints: {
  26       "ram" => 12000000000,
  27       "vcpus" => 4,
  28     },
  29     mounts: {
  30       "test" => {"kind" => "json"},
  31     },
  32     environment: {
  33       "var" => "val",
  34     },
  35   }
  36
  37   def minimal_new attrs={}
  38     cr = ContainerRequest.new DEFAULT_ATTRS.merge(attrs)
  39     cr.state = ContainerRequest::Committed
  40     act_as_user users(:active) do
  41       cr.save!
  42     end
  43     c = Container.find_by_uuid cr.container_uuid
  44     assert_not_nil c
  45     return c, cr
  46   end
  47
  48   def check_illegal_updates c, bad_updates
  49     bad_updates.each do |u|
  50       refute c.update_attributes(u), u.inspect
  51       refute c.valid?, u.inspect
  52       c.reload
  53     end
  54   end
  55
  56   def check_illegal_modify c
  57     check_illegal_updates c, [{command: ["echo", "bar"]},
  58                               {container_image: "arvados/apitestfixture:june10"},
  59                               {cwd: "/tmp2"},
  60                               {environment: {"FOO" => "BAR"}},
  61                               {mounts: {"FOO" => "BAR"}},
  62                               {output_path: "/tmp3"},
  63                               {locked_by_uuid: "zzzzz-gj3su-027z32aux8dg2s1"},
  64                               {auth_uuid: "zzzzz-gj3su-017z32aux8dg2s1"},
  65                               {runtime_constraints: {"FOO" => "BAR"}}]
  66   end
  67
  68   def check_bogus_states c
  69     check_illegal_updates c, [{state: nil},
  70                               {state: "Flubber"}]
  71   end
  72
  73   def check_no_change_from_cancelled c
  74     check_illegal_modify c
  75     check_bogus_states c
  76     check_illegal_updates c, [{ priority: 3 },
  77                               { state: Container::Queued },
  78                               { state: Container::Locked },
  79                               { state: Container::Running },
  80                               { state: Container::Complete }]
  81   end
  82
  83   test "Container create" do
  84     act_as_system_user do
  85       c, _ = minimal_new(environment: {},
  86                       mounts: {"BAR" => "FOO"},
  87                       output_path: "/tmp",
  88                       priority: 1,
  89                       runtime_constraints: {"vcpus" => 1, "ram" => 1})
  90
  91       check_illegal_modify c
  92       check_bogus_states c
  93
  94       c.reload
  95       c.priority = 2
  96       c.save!
  97     end
  98   end
  99
 100   test "Container valid priority" do
 101     act_as_system_user do
 102       c, _ = minimal_new(environment: {},
 103                       mounts: {"BAR" => "FOO"},
 104                       output_path: "/tmp",
 105                       priority: 1,
 106                       runtime_constraints: {"vcpus" => 1, "ram" => 1})
 107
 108       assert_raises(ActiveRecord::RecordInvalid) do
 109         c.priority = -1
 110         c.save!
 111       end
 112
 113       c.priority = 0
 114       c.save!
 115
 116       c.priority = 1
 117       c.save!
 118
 119       c.priority = 500
 120       c.save!
 121
 122       c.priority = 999
 123       c.save!
 124
 125       c.priority = 1000
 126       c.save!
 127
 128       assert_raises(ActiveRecord::RecordInvalid) do
 129         c.priority = 1001
 130         c.save!
 131       end
 132     end
 133   end
 134
 135
 136   test "Container serialized hash attributes sorted before save" do
 137     env = {"C" => 3, "B" => 2, "A" => 1}
 138     m = {"F" => {"kind" => 3}, "E" => {"kind" => 2}, "D" => {"kind" => 1}}
 139     rc = {"vcpus" => 1, "ram" => 1, "keep_cache_ram" => 1}
 140     c, _ = minimal_new(environment: env, mounts: m, runtime_constraints: rc)
 141     assert_equal c.environment.to_json, Container.deep_sort_hash(env).to_json
 142     assert_equal c.mounts.to_json, Container.deep_sort_hash(m).to_json
 143     assert_equal c.runtime_constraints.to_json, Container.deep_sort_hash(rc).to_json
 144   end
 145
 146   test 'deep_sort_hash on array of hashes' do
 147     a = {'z' => [[{'a' => 'a', 'b' => 'b'}]]}
 148     b = {'z' => [[{'b' => 'b', 'a' => 'a'}]]}
 149     assert_equal Container.deep_sort_hash(a).to_json, Container.deep_sort_hash(b).to_json
 150   end
 151
 152   test "find_reusable method should select higher priority queued container" do
 153     set_user_from_auth :active
 154     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment:{"var" => "queued"}})
 155     c_low_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:1}))
 156     c_high_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:2}))
 157     assert_not_equal c_low_priority.uuid, c_high_priority.uuid
 158     assert_equal Container::Queued, c_low_priority.state
 159     assert_equal Container::Queued, c_high_priority.state
 160     reused = Container.find_reusable(common_attrs)
 161     assert_not_nil reused
 162     assert_equal reused.uuid, c_high_priority.uuid
 163   end
 164
 165   test "find_reusable method should select latest completed container" do
 166     set_user_from_auth :active
 167     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "complete"}})
 168     completed_attrs = {
 169       state: Container::Complete,
 170       exit_code: 0,
 171       log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 172       output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
 173     }
 174
 175     c_older, _ = minimal_new(common_attrs.merge({use_existing: false}))
 176     c_recent, _ = minimal_new(common_attrs.merge({use_existing: false}))
 177     assert_not_equal c_older.uuid, c_recent.uuid
 178
 179     set_user_from_auth :dispatch1
 180     c_older.update_attributes!({state: Container::Locked})
 181     c_older.update_attributes!({state: Container::Running})
 182     c_older.update_attributes!(completed_attrs)
 183
 184     c_recent.update_attributes!({state: Container::Locked})
 185     c_recent.update_attributes!({state: Container::Running})
 186     c_recent.update_attributes!(completed_attrs)
 187
 188     reused = Container.find_reusable(common_attrs)
 189     assert_not_nil reused
 190     assert_equal reused.uuid, c_older.uuid
 191   end
 192
 193   test "find_reusable method should select oldest completed container when inconsistent outputs exist" do
 194     set_user_from_auth :active
 195     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "complete"}, priority: 1})
 196     completed_attrs = {
 197       state: Container::Complete,
 198       exit_code: 0,
 199       log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 200     }
 201
 202     cr = ContainerRequest.new common_attrs
 203     cr.use_existing = false
 204     cr.state = ContainerRequest::Committed
 205     cr.save!
 206     c_output1 = Container.where(uuid: cr.container_uuid).first
 207
 208     cr = ContainerRequest.new common_attrs
 209     cr.use_existing = false
 210     cr.state = ContainerRequest::Committed
 211     cr.save!
 212     c_output2 = Container.where(uuid: cr.container_uuid).first
 213
 214     assert_not_equal c_output1.uuid, c_output2.uuid
 215
 216     set_user_from_auth :dispatch1
 217
 218     out1 = '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
 219     log1 = collections(:real_log_collection).portable_data_hash
 220     c_output1.update_attributes!({state: Container::Locked})
 221     c_output1.update_attributes!({state: Container::Running})
 222     c_output1.update_attributes!(completed_attrs.merge({log: log1, output: out1}))
 223
 224     out2 = 'fa7aeb5140e2848d39b416daeef4ffc5+45'
 225     c_output2.update_attributes!({state: Container::Locked})
 226     c_output2.update_attributes!({state: Container::Running})
 227     c_output2.update_attributes!(completed_attrs.merge({log: log1, output: out2}))
 228
 229     reused = Container.resolve(ContainerRequest.new(common_attrs))
 230     assert_equal c_output1.uuid, reused.uuid
 231   end
 232
 233   test "find_reusable method should select running container by start date" do
 234     set_user_from_auth :active
 235     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running"}})
 236     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
 237     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
 238     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
 239     # Confirm the 3 container UUIDs are different.
 240     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
 241     set_user_from_auth :dispatch1
 242     c_slower.update_attributes!({state: Container::Locked})
 243     c_slower.update_attributes!({state: Container::Running,
 244                                  progress: 0.1})
 245     c_faster_started_first.update_attributes!({state: Container::Locked})
 246     c_faster_started_first.update_attributes!({state: Container::Running,
 247                                                progress: 0.15})
 248     c_faster_started_second.update_attributes!({state: Container::Locked})
 249     c_faster_started_second.update_attributes!({state: Container::Running,
 250                                                 progress: 0.15})
 251     reused = Container.find_reusable(common_attrs)
 252     assert_not_nil reused
 253     # Selected container is the one that started first
 254     assert_equal reused.uuid, c_faster_started_first.uuid
 255   end
 256
 257   test "find_reusable method should select running container by progress" do
 258     set_user_from_auth :active
 259     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running2"}})
 260     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
 261     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
 262     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
 263     # Confirm the 3 container UUIDs are different.
 264     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
 265     set_user_from_auth :dispatch1
 266     c_slower.update_attributes!({state: Container::Locked})
 267     c_slower.update_attributes!({state: Container::Running,
 268                                  progress: 0.1})
 269     c_faster_started_first.update_attributes!({state: Container::Locked})
 270     c_faster_started_first.update_attributes!({state: Container::Running,
 271                                                progress: 0.15})
 272     c_faster_started_second.update_attributes!({state: Container::Locked})
 273     c_faster_started_second.update_attributes!({state: Container::Running,
 274                                                 progress: 0.2})
 275     reused = Container.find_reusable(common_attrs)
 276     assert_not_nil reused
 277     # Selected container is the one with most progress done
 278     assert_equal reused.uuid, c_faster_started_second.uuid
 279   end
 280
 281   test "find_reusable method should select locked container most likely to start sooner" do
 282     set_user_from_auth :active
 283     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "locked"}})
 284     c_low_priority, _ = minimal_new(common_attrs.merge({use_existing: false}))
 285     c_high_priority_older, _ = minimal_new(common_attrs.merge({use_existing: false}))
 286     c_high_priority_newer, _ = minimal_new(common_attrs.merge({use_existing: false}))
 287     # Confirm the 3 container UUIDs are different.
 288     assert_equal 3, [c_low_priority.uuid, c_high_priority_older.uuid, c_high_priority_newer.uuid].uniq.length
 289     set_user_from_auth :dispatch1
 290     c_low_priority.update_attributes!({state: Container::Locked,
 291                                        priority: 1})
 292     c_high_priority_older.update_attributes!({state: Container::Locked,
 293                                               priority: 2})
 294     c_high_priority_newer.update_attributes!({state: Container::Locked,
 295                                               priority: 2})
 296     reused = Container.find_reusable(common_attrs)
 297     assert_not_nil reused
 298     assert_equal reused.uuid, c_high_priority_older.uuid
 299   end
 300
 301   test "find_reusable method should select running over failed container" do
 302     set_user_from_auth :active
 303     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed_vs_running"}})
 304     c_failed, _ = minimal_new(common_attrs.merge({use_existing: false}))
 305     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
 306     assert_not_equal c_failed.uuid, c_running.uuid
 307     set_user_from_auth :dispatch1
 308     c_failed.update_attributes!({state: Container::Locked})
 309     c_failed.update_attributes!({state: Container::Running})
 310     c_failed.update_attributes!({state: Container::Complete,
 311                                  exit_code: 42,
 312                                  log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 313                                  output: 'ea10d51bcf88862dbcc36eb292017dfd+45'})
 314     c_running.update_attributes!({state: Container::Locked})
 315     c_running.update_attributes!({state: Container::Running,
 316                                   progress: 0.15})
 317     reused = Container.find_reusable(common_attrs)
 318     assert_not_nil reused
 319     assert_equal reused.uuid, c_running.uuid
 320   end
 321
 322   test "find_reusable method should select complete over running container" do
 323     set_user_from_auth :active
 324     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "completed_vs_running"}})
 325     c_completed, _ = minimal_new(common_attrs.merge({use_existing: false}))
 326     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
 327     assert_not_equal c_completed.uuid, c_running.uuid
 328     set_user_from_auth :dispatch1
 329     c_completed.update_attributes!({state: Container::Locked})
 330     c_completed.update_attributes!({state: Container::Running})
 331     c_completed.update_attributes!({state: Container::Complete,
 332                                     exit_code: 0,
 333                                     log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 334                                     output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'})
 335     c_running.update_attributes!({state: Container::Locked})
 336     c_running.update_attributes!({state: Container::Running,
 337                                   progress: 0.15})
 338     reused = Container.find_reusable(common_attrs)
 339     assert_not_nil reused
 340     assert_equal c_completed.uuid, reused.uuid
 341   end
 342
 343   test "find_reusable method should select running over locked container" do
 344     set_user_from_auth :active
 345     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}})
 346     c_locked, _ = minimal_new(common_attrs.merge({use_existing: false}))
 347     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
 348     assert_not_equal c_running.uuid, c_locked.uuid
 349     set_user_from_auth :dispatch1
 350     c_locked.update_attributes!({state: Container::Locked})
 351     c_running.update_attributes!({state: Container::Locked})
 352     c_running.update_attributes!({state: Container::Running,
 353                                   progress: 0.15})
 354     reused = Container.find_reusable(common_attrs)
 355     assert_not_nil reused
 356     assert_equal reused.uuid, c_running.uuid
 357   end
 358
 359   test "find_reusable method should select locked over queued container" do
 360     set_user_from_auth :active
 361     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}})
 362     c_locked, _ = minimal_new(common_attrs.merge({use_existing: false}))
 363     c_queued, _ = minimal_new(common_attrs.merge({use_existing: false}))
 364     assert_not_equal c_queued.uuid, c_locked.uuid
 365     set_user_from_auth :dispatch1
 366     c_locked.update_attributes!({state: Container::Locked})
 367     reused = Container.find_reusable(common_attrs)
 368     assert_not_nil reused
 369     assert_equal reused.uuid, c_locked.uuid
 370   end
 371
 372   test "find_reusable method should not select failed container" do
 373     set_user_from_auth :active
 374     attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed"}})
 375     c, _ = minimal_new(attrs)
 376     set_user_from_auth :dispatch1
 377     c.update_attributes!({state: Container::Locked})
 378     c.update_attributes!({state: Container::Running})
 379     c.update_attributes!({state: Container::Complete,
 380                           exit_code: 33})
 381     reused = Container.find_reusable(attrs)
 382     assert_nil reused
 383   end
 384
 385   test "find_reusable with logging disabled" do
 386     set_user_from_auth :active
 387     Rails.logger.expects(:info).never
 388     Container.find_reusable(REUSABLE_COMMON_ATTRS)
 389   end
 390
 391   test "find_reusable with logging enabled" do
 392     set_user_from_auth :active
 393     Rails.configuration.log_reuse_decisions = true
 394     Rails.logger.expects(:info).at_least(3)
 395     Container.find_reusable(REUSABLE_COMMON_ATTRS)
 396   end
 397
 398   test "Container running" do
 399     c, _ = minimal_new priority: 1
 400
 401     set_user_from_auth :dispatch1
 402     check_illegal_updates c, [{state: Container::Running},
 403                               {state: Container::Complete}]
 404
 405     c.lock
 406     c.update_attributes! state: Container::Running
 407
 408     check_illegal_modify c
 409     check_bogus_states c
 410
 411     check_illegal_updates c, [{state: Container::Queued}]
 412     c.reload
 413
 414     c.update_attributes! priority: 3
 415   end
 416
 417   test "Lock and unlock" do
 418     c, cr = minimal_new priority: 0
 419
 420     set_user_from_auth :dispatch1
 421     assert_equal Container::Queued, c.state
 422
 423     assert_raise(ArvadosModel::LockFailedError) do
 424       # "no priority"
 425       c.lock
 426     end
 427     c.reload
 428     assert cr.update_attributes priority: 1
 429
 430     refute c.update_attributes(state: Container::Running), "not locked"
 431     c.reload
 432     refute c.update_attributes(state: Container::Complete), "not locked"
 433     c.reload
 434
 435     assert c.lock, show_errors(c)
 436     assert c.locked_by_uuid
 437     assert c.auth_uuid
 438
 439     assert_raise(ArvadosModel::LockFailedError) {c.lock}
 440     c.reload
 441
 442     assert c.unlock, show_errors(c)
 443     refute c.locked_by_uuid
 444     refute c.auth_uuid
 445
 446     refute c.update_attributes(state: Container::Running), "not locked"
 447     c.reload
 448     refute c.locked_by_uuid
 449     refute c.auth_uuid
 450
 451     assert c.lock, show_errors(c)
 452     assert c.update_attributes(state: Container::Running), show_errors(c)
 453     assert c.locked_by_uuid
 454     assert c.auth_uuid
 455
 456     auth_uuid_was = c.auth_uuid
 457
 458     assert_raise(ArvadosModel::LockFailedError) do
 459       # Running to Locked is not allowed
 460       c.lock
 461     end
 462     c.reload
 463     assert_raise(ArvadosModel::InvalidStateTransitionError) do
 464       # Running to Queued is not allowed
 465       c.unlock
 466     end
 467     c.reload
 468
 469     assert c.update_attributes(state: Container::Complete), show_errors(c)
 470     refute c.locked_by_uuid
 471     refute c.auth_uuid
 472
 473     auth_exp = ApiClientAuthorization.find_by_uuid(auth_uuid_was).expires_at
 474     assert_operator auth_exp, :<, db_current_time
 475   end
 476
 477   test "Container queued cancel" do
 478     c, cr = minimal_new({container_count_max: 1})
 479     set_user_from_auth :dispatch1
 480     assert c.update_attributes(state: Container::Cancelled), show_errors(c)
 481     check_no_change_from_cancelled c
 482     cr.reload
 483     assert_equal ContainerRequest::Final, cr.state
 484   end
 485
 486   test "Container queued count" do
 487     assert_equal 1, Container.readable_by(users(:active)).where(state: "Queued").count
 488   end
 489
 490   test "Container locked cancel" do
 491     c, _ = minimal_new
 492     set_user_from_auth :dispatch1
 493     assert c.lock, show_errors(c)
 494     assert c.update_attributes(state: Container::Cancelled), show_errors(c)
 495     check_no_change_from_cancelled c
 496   end
 497
 498   test "Container locked cancel with log" do
 499     c, _ = minimal_new
 500     set_user_from_auth :dispatch1
 501     assert c.lock, show_errors(c)
 502     assert c.update_attributes(
 503              state: Container::Cancelled,
 504              log: collections(:real_log_collection).portable_data_hash,
 505            ), show_errors(c)
 506     check_no_change_from_cancelled c
 507   end
 508
 509   test "Container running cancel" do
 510     c, _ = minimal_new
 511     set_user_from_auth :dispatch1
 512     c.lock
 513     c.update_attributes! state: Container::Running
 514     c.update_attributes! state: Container::Cancelled
 515     check_no_change_from_cancelled c
 516   end
 517
 518   test "Container create forbidden for non-admin" do
 519     set_user_from_auth :active_trustedclient
 520     c = Container.new DEFAULT_ATTRS
 521     c.environment = {}
 522     c.mounts = {"BAR" => "FOO"}
 523     c.output_path = "/tmp"
 524     c.priority = 1
 525     c.runtime_constraints = {}
 526     assert_raises(ArvadosModel::PermissionDeniedError) do
 527       c.save!
 528     end
 529   end
 530
 531   test "Container only set exit code on complete" do
 532     c, _ = minimal_new
 533     set_user_from_auth :dispatch1
 534     c.lock
 535     c.update_attributes! state: Container::Running
 536
 537     check_illegal_updates c, [{exit_code: 1},
 538                               {exit_code: 1, state: Container::Cancelled}]
 539
 540     assert c.update_attributes(exit_code: 1, state: Container::Complete)
 541   end
 542
 543   test "locked_by_uuid can set output on running container" do
 544     c, _ = minimal_new
 545     set_user_from_auth :dispatch1
 546     c.lock
 547     c.update_attributes! state: Container::Running
 548
 549     assert_equal c.locked_by_uuid, Thread.current[:api_client_authorization].uuid
 550
 551     assert c.update_attributes output: collections(:collection_owned_by_active).portable_data_hash
 552     assert c.update_attributes! state: Container::Complete
 553   end
 554
 555   test "auth_uuid can set output on running container, but not change container state" do
 556     c, _ = minimal_new
 557     set_user_from_auth :dispatch1
 558     c.lock
 559     c.update_attributes! state: Container::Running
 560
 561     Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
 562     Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
 563     assert c.update_attributes output: collections(:collection_owned_by_active).portable_data_hash
 564
 565     assert_raises ArvadosModel::PermissionDeniedError do
 566       # auth_uuid cannot set container state
 567       c.update_attributes state: Container::Complete
 568     end
 569   end
 570
 571   test "not allowed to set output that is not readable by current user" do
 572     c, _ = minimal_new
 573     set_user_from_auth :dispatch1
 574     c.lock
 575     c.update_attributes! state: Container::Running
 576
 577     Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
 578     Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
 579
 580     assert_raises ActiveRecord::RecordInvalid do
 581       c.update_attributes! output: collections(:collection_not_readable_by_active).portable_data_hash
 582     end
 583   end
 584
 585   test "other token cannot set output on running container" do
 586     c, _ = minimal_new
 587     set_user_from_auth :dispatch1
 588     c.lock
 589     c.update_attributes! state: Container::Running
 590
 591     set_user_from_auth :running_to_be_deleted_container_auth
 592     assert_raises ArvadosModel::PermissionDeniedError do
 593       c.update_attributes! output: collections(:foo_file).portable_data_hash
 594     end
 595   end
 596
 597   test "can set trashed output on running container" do
 598     c, _ = minimal_new
 599     set_user_from_auth :dispatch1
 600     c.lock
 601     c.update_attributes! state: Container::Running
 602
 603     output = Collection.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jk')
 604
 605     assert output.is_trashed
 606     assert c.update_attributes output: output.portable_data_hash
 607     assert c.update_attributes! state: Container::Complete
 608   end
 609
 610   test "not allowed to set trashed output that is not readable by current user" do
 611     c, _ = minimal_new
 612     set_user_from_auth :dispatch1
 613     c.lock
 614     c.update_attributes! state: Container::Running
 615
 616     output = Collection.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jr')
 617
 618     Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
 619     Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
 620
 621     assert_raises ActiveRecord::RecordInvalid do
 622       c.update_attributes! output: output.portable_data_hash
 623     end
 624   end
 625
 626   [
 627     Container::Complete,
 628     Container::Cancelled,
 629   ].each do |final_state|
 630     test "secret_mounts is null after container is #{final_state}" do
 631       assert_no_secrets_logged
 632     end
 633   end
 634 end