services/api/test/unit/container_test.rb

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 require 'test_helper'
   6 require 'helpers/container_test_helper'
   7
   8 class ContainerTest < ActiveSupport::TestCase
   9   include DbCurrentTime
  10   include ContainerTestHelper
  11
  12   DEFAULT_ATTRS = {
  13     command: ['echo', 'foo'],
  14     container_image: 'fa3c1a9cb6783f85f2ecda037e07b8c3+167',
  15     output_path: '/tmp',
  16     priority: 1,
  17     runtime_constraints: {"vcpus" => 1, "ram" => 1},
  18   }
  19
  20   REUSABLE_COMMON_ATTRS = {
  21     container_image: "9ae44d5792468c58bcf85ce7353c7027+124",
  22     cwd: "test",
  23     command: ["echo", "hello"],
  24     output_path: "test",
  25     runtime_constraints: {
  26       "ram" => 12000000000,
  27       "vcpus" => 4,
  28     },
  29     mounts: {
  30       "test" => {"kind" => "json"},
  31     },
  32     environment: {
  33       "var" => "val",
  34     },
  35     secret_mounts: {},
  36   }
  37
  38   def minimal_new attrs={}
  39     cr = ContainerRequest.new DEFAULT_ATTRS.merge(attrs)
  40     cr.state = ContainerRequest::Committed
  41     act_as_user users(:active) do
  42       cr.save!
  43     end
  44     c = Container.find_by_uuid cr.container_uuid
  45     assert_not_nil c
  46     return c, cr
  47   end
  48
  49   def check_illegal_updates c, bad_updates
  50     bad_updates.each do |u|
  51       refute c.update_attributes(u), u.inspect
  52       refute c.valid?, u.inspect
  53       c.reload
  54     end
  55   end
  56
  57   def check_illegal_modify c
  58     check_illegal_updates c, [{command: ["echo", "bar"]},
  59                               {container_image: "arvados/apitestfixture:june10"},
  60                               {cwd: "/tmp2"},
  61                               {environment: {"FOO" => "BAR"}},
  62                               {mounts: {"FOO" => "BAR"}},
  63                               {output_path: "/tmp3"},
  64                               {locked_by_uuid: "zzzzz-gj3su-027z32aux8dg2s1"},
  65                               {auth_uuid: "zzzzz-gj3su-017z32aux8dg2s1"},
  66                               {runtime_constraints: {"FOO" => "BAR"}}]
  67   end
  68
  69   def check_bogus_states c
  70     check_illegal_updates c, [{state: nil},
  71                               {state: "Flubber"}]
  72   end
  73
  74   def check_no_change_from_cancelled c
  75     check_illegal_modify c
  76     check_bogus_states c
  77     check_illegal_updates c, [{ priority: 3 },
  78                               { state: Container::Queued },
  79                               { state: Container::Locked },
  80                               { state: Container::Running },
  81                               { state: Container::Complete }]
  82   end
  83
  84   test "Container create" do
  85     act_as_system_user do
  86       c, _ = minimal_new(environment: {},
  87                       mounts: {"BAR" => {"kind" => "FOO"}},
  88                       output_path: "/tmp",
  89                       priority: 1,
  90                       runtime_constraints: {"vcpus" => 1, "ram" => 1})
  91
  92       check_illegal_modify c
  93       check_bogus_states c
  94
  95       c.reload
  96       c.priority = 2
  97       c.save!
  98     end
  99   end
 100
 101   test "Container valid priority" do
 102     act_as_system_user do
 103       c, _ = minimal_new(environment: {},
 104                       mounts: {"BAR" => {"kind" => "FOO"}},
 105                       output_path: "/tmp",
 106                       priority: 1,
 107                       runtime_constraints: {"vcpus" => 1, "ram" => 1})
 108
 109       assert_raises(ActiveRecord::RecordInvalid) do
 110         c.priority = -1
 111         c.save!
 112       end
 113
 114       c.priority = 0
 115       c.save!
 116
 117       c.priority = 1
 118       c.save!
 119
 120       c.priority = 500
 121       c.save!
 122
 123       c.priority = 999
 124       c.save!
 125
 126       c.priority = 1000
 127       c.save!
 128
 129       c.priority = 1000 << 50
 130       c.save!
 131     end
 132   end
 133
 134   test "Container runtime_status data types" do
 135     set_user_from_auth :active
 136     attrs = {
 137       environment: {},
 138       mounts: {"BAR" => {"kind" => "FOO"}},
 139       output_path: "/tmp",
 140       priority: 1,
 141       runtime_constraints: {"vcpus" => 1, "ram" => 1}
 142     }
 143     c, _ = minimal_new(attrs)
 144     assert_equal c.runtime_status, {}
 145     assert_equal Container::Queued, c.state
 146
 147     set_user_from_auth :dispatch1
 148     c.update_attributes! state: Container::Locked
 149     c.update_attributes! state: Container::Running
 150
 151     [
 152       'error', 'errorDetail', 'warning', 'warningDetail', 'activity'
 153     ].each do |k|
 154       # String type is allowed
 155       string_val = 'A string is accepted'
 156       c.update_attributes! runtime_status: {k => string_val}
 157       assert_equal string_val, c.runtime_status[k]
 158
 159       # Other types aren't allowed
 160       [
 161         42, false, [], {}, nil
 162       ].each do |unallowed_val|
 163         assert_raises ActiveRecord::RecordInvalid do
 164           c.update_attributes! runtime_status: {k => unallowed_val}
 165         end
 166       end
 167     end
 168   end
 169
 170   test "Container runtime_status updates" do
 171     set_user_from_auth :active
 172     attrs = {
 173       environment: {},
 174       mounts: {"BAR" => {"kind" => "FOO"}},
 175       output_path: "/tmp",
 176       priority: 1,
 177       runtime_constraints: {"vcpus" => 1, "ram" => 1}
 178     }
 179     c1, _ = minimal_new(attrs)
 180     assert_equal c1.runtime_status, {}
 181
 182     assert_equal Container::Queued, c1.state
 183     assert_raises ActiveRecord::RecordInvalid do
 184       c1.update_attributes! runtime_status: {'error' => 'Oops!'}
 185     end
 186
 187     set_user_from_auth :dispatch1
 188
 189     # Allow updates when state = Locked
 190     c1.update_attributes! state: Container::Locked
 191     c1.update_attributes! runtime_status: {'error' => 'Oops!'}
 192     assert c1.runtime_status.key? 'error'
 193
 194     # Reset when transitioning from Locked to Queued
 195     c1.update_attributes! state: Container::Queued
 196     assert_equal c1.runtime_status, {}
 197
 198     # Allow updates when state = Running
 199     c1.update_attributes! state: Container::Locked
 200     c1.update_attributes! state: Container::Running
 201     c1.update_attributes! runtime_status: {'error' => 'Oops!'}
 202     assert c1.runtime_status.key? 'error'
 203
 204     # Don't allow updates on other states
 205     c1.update_attributes! state: Container::Complete
 206     assert_raises ActiveRecord::RecordInvalid do
 207       c1.update_attributes! runtime_status: {'error' => 'Some other error'}
 208     end
 209
 210     set_user_from_auth :active
 211     c2, _ = minimal_new(attrs)
 212     assert_equal c2.runtime_status, {}
 213     set_user_from_auth :dispatch1
 214     c2.update_attributes! state: Container::Locked
 215     c2.update_attributes! state: Container::Running
 216     c2.update_attributes! state: Container::Cancelled
 217     assert_raises ActiveRecord::RecordInvalid do
 218       c2.update_attributes! runtime_status: {'error' => 'Oops!'}
 219     end
 220   end
 221
 222   test "Container serialized hash attributes sorted before save" do
 223     env = {"C" => "3", "B" => "2", "A" => "1"}
 224     m = {"F" => {"kind" => "3"}, "E" => {"kind" => "2"}, "D" => {"kind" => "1"}}
 225     rc = {"vcpus" => 1, "ram" => 1, "keep_cache_ram" => 1}
 226     c, _ = minimal_new(environment: env, mounts: m, runtime_constraints: rc)
 227     assert_equal c.environment.to_json, Container.deep_sort_hash(env).to_json
 228     assert_equal c.mounts.to_json, Container.deep_sort_hash(m).to_json
 229     assert_equal c.runtime_constraints.to_json, Container.deep_sort_hash(rc).to_json
 230   end
 231
 232   test 'deep_sort_hash on array of hashes' do
 233     a = {'z' => [[{'a' => 'a', 'b' => 'b'}]]}
 234     b = {'z' => [[{'b' => 'b', 'a' => 'a'}]]}
 235     assert_equal Container.deep_sort_hash(a).to_json, Container.deep_sort_hash(b).to_json
 236   end
 237
 238   test "find_reusable method should select higher priority queued container" do
 239     set_user_from_auth :active
 240     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment:{"var" => "queued"}})
 241     c_low_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:1}))
 242     c_high_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:2}))
 243     assert_not_equal c_low_priority.uuid, c_high_priority.uuid
 244     assert_equal Container::Queued, c_low_priority.state
 245     assert_equal Container::Queued, c_high_priority.state
 246     reused = Container.find_reusable(common_attrs)
 247     assert_not_nil reused
 248     assert_equal reused.uuid, c_high_priority.uuid
 249   end
 250
 251   test "find_reusable method should select latest completed container" do
 252     set_user_from_auth :active
 253     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "complete"}})
 254     completed_attrs = {
 255       state: Container::Complete,
 256       exit_code: 0,
 257       log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 258       output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
 259     }
 260
 261     c_older, _ = minimal_new(common_attrs.merge({use_existing: false}))
 262     c_recent, _ = minimal_new(common_attrs.merge({use_existing: false}))
 263     assert_not_equal c_older.uuid, c_recent.uuid
 264
 265     set_user_from_auth :dispatch1
 266     c_older.update_attributes!({state: Container::Locked})
 267     c_older.update_attributes!({state: Container::Running})
 268     c_older.update_attributes!(completed_attrs)
 269
 270     c_recent.update_attributes!({state: Container::Locked})
 271     c_recent.update_attributes!({state: Container::Running})
 272     c_recent.update_attributes!(completed_attrs)
 273
 274     reused = Container.find_reusable(common_attrs)
 275     assert_not_nil reused
 276     assert_equal reused.uuid, c_older.uuid
 277   end
 278
 279   test "find_reusable method should select oldest completed container when inconsistent outputs exist" do
 280     set_user_from_auth :active
 281     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "complete"}, priority: 1})
 282     completed_attrs = {
 283       state: Container::Complete,
 284       exit_code: 0,
 285       log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 286     }
 287
 288     cr = ContainerRequest.new common_attrs
 289     cr.use_existing = false
 290     cr.state = ContainerRequest::Committed
 291     cr.save!
 292     c_output1 = Container.where(uuid: cr.container_uuid).first
 293
 294     cr = ContainerRequest.new common_attrs
 295     cr.use_existing = false
 296     cr.state = ContainerRequest::Committed
 297     cr.save!
 298     c_output2 = Container.where(uuid: cr.container_uuid).first
 299
 300     assert_not_equal c_output1.uuid, c_output2.uuid
 301
 302     set_user_from_auth :dispatch1
 303
 304     out1 = '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
 305     log1 = collections(:real_log_collection).portable_data_hash
 306     c_output1.update_attributes!({state: Container::Locked})
 307     c_output1.update_attributes!({state: Container::Running})
 308     c_output1.update_attributes!(completed_attrs.merge({log: log1, output: out1}))
 309
 310     out2 = 'fa7aeb5140e2848d39b416daeef4ffc5+45'
 311     c_output2.update_attributes!({state: Container::Locked})
 312     c_output2.update_attributes!({state: Container::Running})
 313     c_output2.update_attributes!(completed_attrs.merge({log: log1, output: out2}))
 314
 315     reused = Container.resolve(ContainerRequest.new(common_attrs))
 316     assert_equal c_output1.uuid, reused.uuid
 317   end
 318
 319   test "find_reusable method should select running container by start date" do
 320     set_user_from_auth :active
 321     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running"}})
 322     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
 323     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
 324     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
 325     # Confirm the 3 container UUIDs are different.
 326     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
 327     set_user_from_auth :dispatch1
 328     c_slower.update_attributes!({state: Container::Locked})
 329     c_slower.update_attributes!({state: Container::Running,
 330                                  progress: 0.1})
 331     c_faster_started_first.update_attributes!({state: Container::Locked})
 332     c_faster_started_first.update_attributes!({state: Container::Running,
 333                                                progress: 0.15})
 334     c_faster_started_second.update_attributes!({state: Container::Locked})
 335     c_faster_started_second.update_attributes!({state: Container::Running,
 336                                                 progress: 0.15})
 337     reused = Container.find_reusable(common_attrs)
 338     assert_not_nil reused
 339     # Selected container is the one that started first
 340     assert_equal reused.uuid, c_faster_started_first.uuid
 341   end
 342
 343   test "find_reusable method should select running container by progress" do
 344     set_user_from_auth :active
 345     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running2"}})
 346     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
 347     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
 348     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
 349     # Confirm the 3 container UUIDs are different.
 350     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
 351     set_user_from_auth :dispatch1
 352     c_slower.update_attributes!({state: Container::Locked})
 353     c_slower.update_attributes!({state: Container::Running,
 354                                  progress: 0.1})
 355     c_faster_started_first.update_attributes!({state: Container::Locked})
 356     c_faster_started_first.update_attributes!({state: Container::Running,
 357                                                progress: 0.15})
 358     c_faster_started_second.update_attributes!({state: Container::Locked})
 359     c_faster_started_second.update_attributes!({state: Container::Running,
 360                                                 progress: 0.2})
 361     reused = Container.find_reusable(common_attrs)
 362     assert_not_nil reused
 363     # Selected container is the one with most progress done
 364     assert_equal reused.uuid, c_faster_started_second.uuid
 365   end
 366
 367   test "find_reusable method should select non-failing running container" do
 368     set_user_from_auth :active
 369     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running2"}})
 370     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
 371     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
 372     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
 373     # Confirm the 3 container UUIDs are different.
 374     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
 375     set_user_from_auth :dispatch1
 376     c_slower.update_attributes!({state: Container::Locked})
 377     c_slower.update_attributes!({state: Container::Running,
 378                                  progress: 0.1})
 379     c_faster_started_first.update_attributes!({state: Container::Locked})
 380     c_faster_started_first.update_attributes!({state: Container::Running,
 381                                                runtime_status: {'warning' => 'This is not an error'},
 382                                                progress: 0.15})
 383     c_faster_started_second.update_attributes!({state: Container::Locked})
 384     c_faster_started_second.update_attributes!({state: Container::Running,
 385                                                 runtime_status: {'error' => 'Something bad happened'},
 386                                                 progress: 0.2})
 387     reused = Container.find_reusable(common_attrs)
 388     assert_not_nil reused
 389     # Selected the non-failing container even if it's the one with less progress done
 390     assert_equal reused.uuid, c_faster_started_first.uuid
 391   end
 392
 393   test "find_reusable method should select locked container most likely to start sooner" do
 394     set_user_from_auth :active
 395     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "locked"}})
 396     c_low_priority, _ = minimal_new(common_attrs.merge({use_existing: false}))
 397     c_high_priority_older, _ = minimal_new(common_attrs.merge({use_existing: false}))
 398     c_high_priority_newer, _ = minimal_new(common_attrs.merge({use_existing: false}))
 399     # Confirm the 3 container UUIDs are different.
 400     assert_equal 3, [c_low_priority.uuid, c_high_priority_older.uuid, c_high_priority_newer.uuid].uniq.length
 401     set_user_from_auth :dispatch1
 402     c_low_priority.update_attributes!({state: Container::Locked,
 403                                        priority: 1})
 404     c_high_priority_older.update_attributes!({state: Container::Locked,
 405                                               priority: 2})
 406     c_high_priority_newer.update_attributes!({state: Container::Locked,
 407                                               priority: 2})
 408     reused = Container.find_reusable(common_attrs)
 409     assert_not_nil reused
 410     assert_equal reused.uuid, c_high_priority_older.uuid
 411   end
 412
 413   test "find_reusable method should select running over failed container" do
 414     set_user_from_auth :active
 415     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed_vs_running"}})
 416     c_failed, _ = minimal_new(common_attrs.merge({use_existing: false}))
 417     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
 418     assert_not_equal c_failed.uuid, c_running.uuid
 419     set_user_from_auth :dispatch1
 420     c_failed.update_attributes!({state: Container::Locked})
 421     c_failed.update_attributes!({state: Container::Running})
 422     c_failed.update_attributes!({state: Container::Complete,
 423                                  exit_code: 42,
 424                                  log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 425                                  output: 'ea10d51bcf88862dbcc36eb292017dfd+45'})
 426     c_running.update_attributes!({state: Container::Locked})
 427     c_running.update_attributes!({state: Container::Running,
 428                                   progress: 0.15})
 429     reused = Container.find_reusable(common_attrs)
 430     assert_not_nil reused
 431     assert_equal reused.uuid, c_running.uuid
 432   end
 433
 434   test "find_reusable method should select complete over running container" do
 435     set_user_from_auth :active
 436     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "completed_vs_running"}})
 437     c_completed, _ = minimal_new(common_attrs.merge({use_existing: false}))
 438     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
 439     assert_not_equal c_completed.uuid, c_running.uuid
 440     set_user_from_auth :dispatch1
 441     c_completed.update_attributes!({state: Container::Locked})
 442     c_completed.update_attributes!({state: Container::Running})
 443     c_completed.update_attributes!({state: Container::Complete,
 444                                     exit_code: 0,
 445                                     log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 446                                     output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'})
 447     c_running.update_attributes!({state: Container::Locked})
 448     c_running.update_attributes!({state: Container::Running,
 449                                   progress: 0.15})
 450     reused = Container.find_reusable(common_attrs)
 451     assert_not_nil reused
 452     assert_equal c_completed.uuid, reused.uuid
 453   end
 454
 455   test "find_reusable method should select running over locked container" do
 456     set_user_from_auth :active
 457     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}})
 458     c_locked, _ = minimal_new(common_attrs.merge({use_existing: false}))
 459     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
 460     assert_not_equal c_running.uuid, c_locked.uuid
 461     set_user_from_auth :dispatch1
 462     c_locked.update_attributes!({state: Container::Locked})
 463     c_running.update_attributes!({state: Container::Locked})
 464     c_running.update_attributes!({state: Container::Running,
 465                                   progress: 0.15})
 466     reused = Container.find_reusable(common_attrs)
 467     assert_not_nil reused
 468     assert_equal reused.uuid, c_running.uuid
 469   end
 470
 471   test "find_reusable method should select locked over queued container" do
 472     set_user_from_auth :active
 473     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}})
 474     c_locked, _ = minimal_new(common_attrs.merge({use_existing: false}))
 475     c_queued, _ = minimal_new(common_attrs.merge({use_existing: false}))
 476     assert_not_equal c_queued.uuid, c_locked.uuid
 477     set_user_from_auth :dispatch1
 478     c_locked.update_attributes!({state: Container::Locked})
 479     reused = Container.find_reusable(common_attrs)
 480     assert_not_nil reused
 481     assert_equal reused.uuid, c_locked.uuid
 482   end
 483
 484   test "find_reusable method should not select failed container" do
 485     set_user_from_auth :active
 486     attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed"}})
 487     c, _ = minimal_new(attrs)
 488     set_user_from_auth :dispatch1
 489     c.update_attributes!({state: Container::Locked})
 490     c.update_attributes!({state: Container::Running})
 491     c.update_attributes!({state: Container::Complete,
 492                           exit_code: 33})
 493     reused = Container.find_reusable(attrs)
 494     assert_nil reused
 495   end
 496
 497   test "find_reusable with logging disabled" do
 498     set_user_from_auth :active
 499     Rails.logger.expects(:info).never
 500     Container.find_reusable(REUSABLE_COMMON_ATTRS)
 501   end
 502
 503   test "find_reusable with logging enabled" do
 504     set_user_from_auth :active
 505     Rails.configuration.log_reuse_decisions = true
 506     Rails.logger.expects(:info).at_least(3)
 507     Container.find_reusable(REUSABLE_COMMON_ATTRS)
 508   end
 509
 510   test "Container running" do
 511     c, _ = minimal_new priority: 1
 512
 513     set_user_from_auth :dispatch1
 514     check_illegal_updates c, [{state: Container::Running},
 515                               {state: Container::Complete}]
 516
 517     c.lock
 518     c.update_attributes! state: Container::Running
 519
 520     check_illegal_modify c
 521     check_bogus_states c
 522
 523     check_illegal_updates c, [{state: Container::Queued}]
 524     c.reload
 525
 526     c.update_attributes! priority: 3
 527   end
 528
 529   test "Lock and unlock" do
 530     c, cr = minimal_new priority: 0
 531
 532     set_user_from_auth :dispatch1
 533     assert_equal Container::Queued, c.state
 534
 535     assert_raise(ArvadosModel::LockFailedError) do
 536       # "no priority"
 537       c.lock
 538     end
 539     c.reload
 540     assert cr.update_attributes priority: 1
 541
 542     refute c.update_attributes(state: Container::Running), "not locked"
 543     c.reload
 544     refute c.update_attributes(state: Container::Complete), "not locked"
 545     c.reload
 546
 547     assert c.lock, show_errors(c)
 548     assert c.locked_by_uuid
 549     assert c.auth_uuid
 550
 551     assert_raise(ArvadosModel::LockFailedError) {c.lock}
 552     c.reload
 553
 554     assert c.unlock, show_errors(c)
 555     refute c.locked_by_uuid
 556     refute c.auth_uuid
 557
 558     refute c.update_attributes(state: Container::Running), "not locked"
 559     c.reload
 560     refute c.locked_by_uuid
 561     refute c.auth_uuid
 562
 563     assert c.lock, show_errors(c)
 564     assert c.update_attributes(state: Container::Running), show_errors(c)
 565     assert c.locked_by_uuid
 566     assert c.auth_uuid
 567
 568     auth_uuid_was = c.auth_uuid
 569
 570     assert_raise(ArvadosModel::LockFailedError) do
 571       # Running to Locked is not allowed
 572       c.lock
 573     end
 574     c.reload
 575     assert_raise(ArvadosModel::InvalidStateTransitionError) do
 576       # Running to Queued is not allowed
 577       c.unlock
 578     end
 579     c.reload
 580
 581     assert c.update_attributes(state: Container::Complete), show_errors(c)
 582     refute c.locked_by_uuid
 583     refute c.auth_uuid
 584
 585     auth_exp = ApiClientAuthorization.find_by_uuid(auth_uuid_was).expires_at
 586     assert_operator auth_exp, :<, db_current_time
 587   end
 588
 589   test "Container queued cancel" do
 590     c, cr = minimal_new({container_count_max: 1})
 591     set_user_from_auth :dispatch1
 592     assert c.update_attributes(state: Container::Cancelled), show_errors(c)
 593     check_no_change_from_cancelled c
 594     cr.reload
 595     assert_equal ContainerRequest::Final, cr.state
 596   end
 597
 598   test "Container queued count" do
 599     assert_equal 1, Container.readable_by(users(:active)).where(state: "Queued").count
 600   end
 601
 602   test "Container locked cancel" do
 603     c, _ = minimal_new
 604     set_user_from_auth :dispatch1
 605     assert c.lock, show_errors(c)
 606     assert c.update_attributes(state: Container::Cancelled), show_errors(c)
 607     check_no_change_from_cancelled c
 608   end
 609
 610   test "Container locked cancel with log" do
 611     c, _ = minimal_new
 612     set_user_from_auth :dispatch1
 613     assert c.lock, show_errors(c)
 614     assert c.update_attributes(
 615              state: Container::Cancelled,
 616              log: collections(:real_log_collection).portable_data_hash,
 617            ), show_errors(c)
 618     check_no_change_from_cancelled c
 619   end
 620
 621   test "Container running cancel" do
 622     c, _ = minimal_new
 623     set_user_from_auth :dispatch1
 624     c.lock
 625     c.update_attributes! state: Container::Running
 626     c.update_attributes! state: Container::Cancelled
 627     check_no_change_from_cancelled c
 628   end
 629
 630   test "Container create forbidden for non-admin" do
 631     set_user_from_auth :active_trustedclient
 632     c = Container.new DEFAULT_ATTRS
 633     c.environment = {}
 634     c.mounts = {"BAR" => "FOO"}
 635     c.output_path = "/tmp"
 636     c.priority = 1
 637     c.runtime_constraints = {}
 638     assert_raises(ArvadosModel::PermissionDeniedError) do
 639       c.save!
 640     end
 641   end
 642
 643   test "Container only set exit code on complete" do
 644     c, _ = minimal_new
 645     set_user_from_auth :dispatch1
 646     c.lock
 647     c.update_attributes! state: Container::Running
 648
 649     check_illegal_updates c, [{exit_code: 1},
 650                               {exit_code: 1, state: Container::Cancelled}]
 651
 652     assert c.update_attributes(exit_code: 1, state: Container::Complete)
 653   end
 654
 655   test "locked_by_uuid can update log when locked/running, and output when running" do
 656     logcoll = collections(:real_log_collection)
 657     c, cr1 = minimal_new
 658     cr2 = ContainerRequest.new(DEFAULT_ATTRS)
 659     cr2.state = ContainerRequest::Committed
 660     act_as_user users(:active) do
 661       cr2.save!
 662     end
 663     assert_equal cr1.container_uuid, cr2.container_uuid
 664
 665     logpdh_time1 = logcoll.portable_data_hash
 666
 667     set_user_from_auth :dispatch1
 668     c.lock
 669     assert_equal c.locked_by_uuid, Thread.current[:api_client_authorization].uuid
 670     c.update_attributes!(log: logpdh_time1)
 671     c.update_attributes!(state: Container::Running)
 672     cr1.reload
 673     cr2.reload
 674     cr1log_uuid = cr1.log_uuid
 675     cr2log_uuid = cr2.log_uuid
 676     assert_not_nil cr1log_uuid
 677     assert_not_nil cr2log_uuid
 678     assert_not_equal logcoll.uuid, cr1log_uuid
 679     assert_not_equal logcoll.uuid, cr2log_uuid
 680     assert_not_equal cr1log_uuid, cr2log_uuid
 681
 682     logcoll.update_attributes!(manifest_text: logcoll.manifest_text + ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n")
 683     logpdh_time2 = logcoll.portable_data_hash
 684
 685     assert c.update_attributes(output: collections(:collection_owned_by_active).portable_data_hash)
 686     assert c.update_attributes(log: logpdh_time2)
 687     assert c.update_attributes(state: Container::Complete, log: logcoll.portable_data_hash)
 688     c.reload
 689     assert_equal collections(:collection_owned_by_active).portable_data_hash, c.output
 690     assert_equal logpdh_time2, c.log
 691     refute c.update_attributes(output: nil)
 692     refute c.update_attributes(log: nil)
 693     cr1.reload
 694     cr2.reload
 695     assert_equal cr1log_uuid, cr1.log_uuid
 696     assert_equal cr2log_uuid, cr2.log_uuid
 697     assert_equal [logpdh_time2], Collection.where(uuid: [cr1log_uuid, cr2log_uuid]).to_a.collect(&:portable_data_hash).uniq
 698   end
 699
 700   test "auth_uuid can set output, progress, runtime_status, state on running container -- but not log" do
 701     c, _ = minimal_new
 702     set_user_from_auth :dispatch1
 703     c.lock
 704     c.update_attributes! state: Container::Running
 705
 706     auth = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
 707     Thread.current[:api_client_authorization] = auth
 708     Thread.current[:api_client] = auth.api_client
 709     Thread.current[:token] = auth.token
 710     Thread.current[:user] = auth.user
 711
 712     assert c.update_attributes(output: collections(:collection_owned_by_active).portable_data_hash)
 713     assert c.update_attributes(runtime_status: {'warning' => 'something happened'})
 714     assert c.update_attributes(progress: 0.5)
 715     refute c.update_attributes(log: collections(:real_log_collection).portable_data_hash)
 716     c.reload
 717     assert c.update_attributes(state: Container::Complete, exit_code: 0)
 718   end
 719
 720   test "not allowed to set output that is not readable by current user" do
 721     c, _ = minimal_new
 722     set_user_from_auth :dispatch1
 723     c.lock
 724     c.update_attributes! state: Container::Running
 725
 726     Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
 727     Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
 728
 729     assert_raises ActiveRecord::RecordInvalid do
 730       c.update_attributes! output: collections(:collection_not_readable_by_active).portable_data_hash
 731     end
 732   end
 733
 734   test "other token cannot set output on running container" do
 735     c, _ = minimal_new
 736     set_user_from_auth :dispatch1
 737     c.lock
 738     c.update_attributes! state: Container::Running
 739
 740     set_user_from_auth :running_to_be_deleted_container_auth
 741     refute c.update_attributes(output: collections(:foo_file).portable_data_hash)
 742   end
 743
 744   test "can set trashed output on running container" do
 745     c, _ = minimal_new
 746     set_user_from_auth :dispatch1
 747     c.lock
 748     c.update_attributes! state: Container::Running
 749
 750     output = Collection.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jk')
 751
 752     assert output.is_trashed
 753     assert c.update_attributes output: output.portable_data_hash
 754     assert c.update_attributes! state: Container::Complete
 755   end
 756
 757   test "not allowed to set trashed output that is not readable by current user" do
 758     c, _ = minimal_new
 759     set_user_from_auth :dispatch1
 760     c.lock
 761     c.update_attributes! state: Container::Running
 762
 763     output = Collection.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jr')
 764
 765     Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
 766     Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
 767
 768     assert_raises ActiveRecord::RecordInvalid do
 769       c.update_attributes! output: output.portable_data_hash
 770     end
 771   end
 772
 773   [
 774     {state: Container::Complete, exit_code: 0, output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'},
 775     {state: Container::Cancelled},
 776   ].each do |final_attrs|
 777     test "secret_mounts is null after container is #{final_attrs[:state]}" do
 778       c, cr = minimal_new(secret_mounts: {'/secret' => {'kind' => 'text', 'content' => 'foo'}},
 779                           container_count_max: 1)
 780       set_user_from_auth :dispatch1
 781       c.lock
 782       c.update_attributes!(state: Container::Running)
 783       c.reload
 784       assert c.secret_mounts.has_key?('/secret')
 785
 786       c.update_attributes!(final_attrs)
 787       c.reload
 788       assert_equal({}, c.secret_mounts)
 789       cr.reload
 790       assert_equal({}, cr.secret_mounts)
 791       assert_no_secrets_logged
 792     end
 793   end
 794 end