services/api/test/unit/container_test.rb

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 require 'test_helper'
   6 require 'helpers/container_test_helper'
   7
   8 class ContainerTest < ActiveSupport::TestCase
   9   include DbCurrentTime
  10   include ContainerTestHelper
  11
  12   DEFAULT_ATTRS = {
  13     command: ['echo', 'foo'],
  14     container_image: 'fa3c1a9cb6783f85f2ecda037e07b8c3+167',
  15     output_path: '/tmp',
  16     priority: 1,
  17     runtime_constraints: {"vcpus" => 1, "ram" => 1, "cuda" => {"device_count":0, "driver_version": "", "hardware_capability": ""}},
  18   }
  19
  20   REUSABLE_COMMON_ATTRS = {
  21     container_image: "9ae44d5792468c58bcf85ce7353c7027+124",
  22     cwd: "test",
  23     command: ["echo", "hello"],
  24     output_path: "test",
  25     runtime_constraints: {
  26       "API" => false,
  27       "keep_cache_ram" => 0,
  28       "ram" => 12000000000,
  29       "vcpus" => 4
  30     },
  31     mounts: {
  32       "test" => {"kind" => "json"},
  33     },
  34     environment: {
  35       "var" => "val",
  36     },
  37     secret_mounts: {},
  38     runtime_user_uuid: "zzzzz-tpzed-xurymjxw79nv3jz",
  39     runtime_auth_scopes: ["all"]
  40   }
  41
  42   def request_only attrs
  43     attrs.reject {|k| [:runtime_user_uuid, :runtime_auth_scopes].include? k}
  44   end
  45
  46   def minimal_new attrs={}
  47     cr = ContainerRequest.new request_only(DEFAULT_ATTRS.merge(attrs))
  48     cr.state = ContainerRequest::Committed
  49     cr.save!
  50     c = Container.find_by_uuid cr.container_uuid
  51     assert_not_nil c
  52     return c, cr
  53   end
  54
  55   def check_illegal_updates c, bad_updates
  56     bad_updates.each do |u|
  57       refute c.update_attributes(u), u.inspect
  58       refute c.valid?, u.inspect
  59       c.reload
  60     end
  61   end
  62
  63   def check_illegal_modify c
  64     check_illegal_updates c, [{command: ["echo", "bar"]},
  65                               {container_image: "arvados/apitestfixture:june10"},
  66                               {cwd: "/tmp2"},
  67                               {environment: {"FOO" => "BAR"}},
  68                               {mounts: {"FOO" => "BAR"}},
  69                               {output_path: "/tmp3"},
  70                               {locked_by_uuid: "zzzzz-gj3su-027z32aux8dg2s1"},
  71                               {auth_uuid: "zzzzz-gj3su-017z32aux8dg2s1"},
  72                               {runtime_constraints: {"FOO" => "BAR"}}]
  73   end
  74
  75   def check_bogus_states c
  76     check_illegal_updates c, [{state: nil},
  77                               {state: "Flubber"}]
  78   end
  79
  80   def check_no_change_from_cancelled c
  81     check_illegal_modify c
  82     check_bogus_states c
  83     check_illegal_updates c, [{ priority: 3 },
  84                               { state: Container::Queued },
  85                               { state: Container::Locked },
  86                               { state: Container::Running },
  87                               { state: Container::Complete }]
  88   end
  89
  90   test "Container create" do
  91     act_as_system_user do
  92       c, _ = minimal_new(environment: {},
  93                       mounts: {"BAR" => {"kind" => "FOO"}},
  94                       output_path: "/tmp",
  95                       priority: 1,
  96                       runtime_constraints: {"vcpus" => 1, "ram" => 1})
  97
  98       check_illegal_modify c
  99       check_bogus_states c
 100
 101       c.reload
 102       c.priority = 2
 103       c.save!
 104     end
 105   end
 106
 107   test "Container valid priority" do
 108     act_as_system_user do
 109       c, _ = minimal_new(environment: {},
 110                       mounts: {"BAR" => {"kind" => "FOO"}},
 111                       output_path: "/tmp",
 112                       priority: 1,
 113                       runtime_constraints: {"vcpus" => 1, "ram" => 1})
 114
 115       assert_raises(ActiveRecord::RecordInvalid) do
 116         c.priority = -1
 117         c.save!
 118       end
 119
 120       c.priority = 0
 121       c.save!
 122
 123       c.priority = 1
 124       c.save!
 125
 126       c.priority = 500
 127       c.save!
 128
 129       c.priority = 999
 130       c.save!
 131
 132       c.priority = 1000
 133       c.save!
 134
 135       c.priority = 1000 << 50
 136       c.save!
 137     end
 138   end
 139
 140   test "Container runtime_status data types" do
 141     set_user_from_auth :active
 142     attrs = {
 143       environment: {},
 144       mounts: {"BAR" => {"kind" => "FOO"}},
 145       output_path: "/tmp",
 146       priority: 1,
 147       runtime_constraints: {"vcpus" => 1, "ram" => 1}
 148     }
 149     c, _ = minimal_new(attrs)
 150     assert_equal c.runtime_status, {}
 151     assert_equal Container::Queued, c.state
 152
 153     set_user_from_auth :dispatch1
 154     c.update_attributes! state: Container::Locked
 155     c.update_attributes! state: Container::Running
 156
 157     [
 158       'error', 'errorDetail', 'warning', 'warningDetail', 'activity'
 159     ].each do |k|
 160       # String type is allowed
 161       string_val = 'A string is accepted'
 162       c.update_attributes! runtime_status: {k => string_val}
 163       assert_equal string_val, c.runtime_status[k]
 164
 165       # Other types aren't allowed
 166       [
 167         42, false, [], {}, nil
 168       ].each do |unallowed_val|
 169         assert_raises ActiveRecord::RecordInvalid do
 170           c.update_attributes! runtime_status: {k => unallowed_val}
 171         end
 172       end
 173     end
 174   end
 175
 176   test "Container runtime_status updates" do
 177     set_user_from_auth :active
 178     attrs = {
 179       environment: {},
 180       mounts: {"BAR" => {"kind" => "FOO"}},
 181       output_path: "/tmp",
 182       priority: 1,
 183       runtime_constraints: {"vcpus" => 1, "ram" => 1}
 184     }
 185     c1, _ = minimal_new(attrs)
 186     assert_equal c1.runtime_status, {}
 187
 188     assert_equal Container::Queued, c1.state
 189     assert_raises ArvadosModel::PermissionDeniedError do
 190       c1.update_attributes! runtime_status: {'error' => 'Oops!'}
 191     end
 192
 193     set_user_from_auth :dispatch1
 194
 195     # Allow updates when state = Locked
 196     c1.update_attributes! state: Container::Locked
 197     c1.update_attributes! runtime_status: {'error' => 'Oops!'}
 198     assert c1.runtime_status.key? 'error'
 199
 200     # Reset when transitioning from Locked to Queued
 201     c1.update_attributes! state: Container::Queued
 202     assert_equal c1.runtime_status, {}
 203
 204     # Allow updates when state = Running
 205     c1.update_attributes! state: Container::Locked
 206     c1.update_attributes! state: Container::Running
 207     c1.update_attributes! runtime_status: {'error' => 'Oops!'}
 208     assert c1.runtime_status.key? 'error'
 209
 210     # Don't allow updates on other states
 211     c1.update_attributes! state: Container::Complete
 212     assert_raises ActiveRecord::RecordInvalid do
 213       c1.update_attributes! runtime_status: {'error' => 'Some other error'}
 214     end
 215
 216     set_user_from_auth :active
 217     c2, _ = minimal_new(attrs)
 218     assert_equal c2.runtime_status, {}
 219     set_user_from_auth :dispatch1
 220     c2.update_attributes! state: Container::Locked
 221     c2.update_attributes! state: Container::Running
 222     c2.update_attributes! state: Container::Cancelled
 223     assert_raises ActiveRecord::RecordInvalid do
 224       c2.update_attributes! runtime_status: {'error' => 'Oops!'}
 225     end
 226   end
 227
 228   test "Container serialized hash attributes sorted before save" do
 229     set_user_from_auth :active
 230     env = {"C" => "3", "B" => "2", "A" => "1"}
 231     m = {"F" => {"kind" => "3"}, "E" => {"kind" => "2"}, "D" => {"kind" => "1"}}
 232     rc = {"vcpus" => 1, "ram" => 1, "keep_cache_ram" => 1, "API" => true, "cuda" => {"device_count":0, "driver_version": "", "hardware_capability": ""}}
 233     c, _ = minimal_new(environment: env, mounts: m, runtime_constraints: rc)
 234     c.reload
 235     assert_equal Container.deep_sort_hash(env).to_json, c.environment.to_json
 236     assert_equal Container.deep_sort_hash(m).to_json, c.mounts.to_json
 237     assert_equal Container.deep_sort_hash(rc).to_json, c.runtime_constraints.to_json
 238   end
 239
 240   test 'deep_sort_hash on array of hashes' do
 241     a = {'z' => [[{'a' => 'a', 'b' => 'b'}]]}
 242     b = {'z' => [[{'b' => 'b', 'a' => 'a'}]]}
 243     assert_equal Container.deep_sort_hash(a).to_json, Container.deep_sort_hash(b).to_json
 244   end
 245
 246   test "find_reusable method should select higher priority queued container" do
 247         Rails.configuration.Containers.LogReuseDecisions = true
 248     set_user_from_auth :active
 249     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment:{"var" => "queued"}})
 250     c_low_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:1}))
 251     c_high_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:2}))
 252     assert_not_equal c_low_priority.uuid, c_high_priority.uuid
 253     assert_equal Container::Queued, c_low_priority.state
 254     assert_equal Container::Queued, c_high_priority.state
 255     reused = Container.find_reusable(common_attrs)
 256     assert_not_nil reused
 257     assert_equal reused.uuid, c_high_priority.uuid
 258   end
 259
 260   test "find_reusable method should select latest completed container" do
 261     set_user_from_auth :active
 262     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "complete"}})
 263     completed_attrs = {
 264       state: Container::Complete,
 265       exit_code: 0,
 266       log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 267       output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
 268     }
 269
 270     c_older, _ = minimal_new(common_attrs.merge({use_existing: false}))
 271     c_recent, _ = minimal_new(common_attrs.merge({use_existing: false}))
 272     assert_not_equal c_older.uuid, c_recent.uuid
 273
 274     set_user_from_auth :dispatch1
 275     c_older.update_attributes!({state: Container::Locked})
 276     c_older.update_attributes!({state: Container::Running})
 277     c_older.update_attributes!(completed_attrs)
 278
 279     c_recent.update_attributes!({state: Container::Locked})
 280     c_recent.update_attributes!({state: Container::Running})
 281     c_recent.update_attributes!(completed_attrs)
 282
 283     reused = Container.find_reusable(common_attrs)
 284     assert_not_nil reused
 285     assert_equal reused.uuid, c_older.uuid
 286   end
 287
 288   test "find_reusable method should select oldest completed container when inconsistent outputs exist" do
 289     set_user_from_auth :active
 290     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "complete"}, priority: 1})
 291     completed_attrs = {
 292       state: Container::Complete,
 293       exit_code: 0,
 294       log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 295     }
 296
 297     cr = ContainerRequest.new request_only(common_attrs)
 298     cr.use_existing = false
 299     cr.state = ContainerRequest::Committed
 300     cr.save!
 301     c_output1 = Container.where(uuid: cr.container_uuid).first
 302
 303     cr = ContainerRequest.new request_only(common_attrs)
 304     cr.use_existing = false
 305     cr.state = ContainerRequest::Committed
 306     cr.save!
 307     c_output2 = Container.where(uuid: cr.container_uuid).first
 308
 309     assert_not_equal c_output1.uuid, c_output2.uuid
 310
 311     set_user_from_auth :dispatch1
 312
 313     out1 = '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
 314     log1 = collections(:real_log_collection).portable_data_hash
 315     c_output1.update_attributes!({state: Container::Locked})
 316     c_output1.update_attributes!({state: Container::Running})
 317     c_output1.update_attributes!(completed_attrs.merge({log: log1, output: out1}))
 318
 319     out2 = 'fa7aeb5140e2848d39b416daeef4ffc5+45'
 320     c_output2.update_attributes!({state: Container::Locked})
 321     c_output2.update_attributes!({state: Container::Running})
 322     c_output2.update_attributes!(completed_attrs.merge({log: log1, output: out2}))
 323
 324     set_user_from_auth :active
 325     reused = Container.resolve(ContainerRequest.new(request_only(common_attrs)))
 326     assert_equal c_output1.uuid, reused.uuid
 327   end
 328
 329   test "find_reusable method should select running container by start date" do
 330     set_user_from_auth :active
 331     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running"}})
 332     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
 333     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
 334     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
 335     # Confirm the 3 container UUIDs are different.
 336     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
 337     set_user_from_auth :dispatch1
 338     c_slower.update_attributes!({state: Container::Locked})
 339     c_slower.update_attributes!({state: Container::Running,
 340                                  progress: 0.1})
 341     c_faster_started_first.update_attributes!({state: Container::Locked})
 342     c_faster_started_first.update_attributes!({state: Container::Running,
 343                                                progress: 0.15})
 344     c_faster_started_second.update_attributes!({state: Container::Locked})
 345     c_faster_started_second.update_attributes!({state: Container::Running,
 346                                                 progress: 0.15})
 347     reused = Container.find_reusable(common_attrs)
 348     assert_not_nil reused
 349     # Selected container is the one that started first
 350     assert_equal reused.uuid, c_faster_started_first.uuid
 351   end
 352
 353   test "find_reusable method should select running container by progress" do
 354     set_user_from_auth :active
 355     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running2"}})
 356     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
 357     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
 358     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
 359     # Confirm the 3 container UUIDs are different.
 360     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
 361     set_user_from_auth :dispatch1
 362     c_slower.update_attributes!({state: Container::Locked})
 363     c_slower.update_attributes!({state: Container::Running,
 364                                  progress: 0.1})
 365     c_faster_started_first.update_attributes!({state: Container::Locked})
 366     c_faster_started_first.update_attributes!({state: Container::Running,
 367                                                progress: 0.15})
 368     c_faster_started_second.update_attributes!({state: Container::Locked})
 369     c_faster_started_second.update_attributes!({state: Container::Running,
 370                                                 progress: 0.2})
 371     reused = Container.find_reusable(common_attrs)
 372     assert_not_nil reused
 373     # Selected container is the one with most progress done
 374     assert_equal reused.uuid, c_faster_started_second.uuid
 375   end
 376
 377   test "find_reusable method should select non-failing running container" do
 378     set_user_from_auth :active
 379     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running2"}})
 380     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
 381     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
 382     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
 383     # Confirm the 3 container UUIDs are different.
 384     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
 385     set_user_from_auth :dispatch1
 386     c_slower.update_attributes!({state: Container::Locked})
 387     c_slower.update_attributes!({state: Container::Running,
 388                                  progress: 0.1})
 389     c_faster_started_first.update_attributes!({state: Container::Locked})
 390     c_faster_started_first.update_attributes!({state: Container::Running,
 391                                                runtime_status: {'warning' => 'This is not an error'},
 392                                                progress: 0.15})
 393     c_faster_started_second.update_attributes!({state: Container::Locked})
 394     assert_equal 0, Container.where("runtime_status->'error' is not null").count
 395     c_faster_started_second.update_attributes!({state: Container::Running,
 396                                                 runtime_status: {'error' => 'Something bad happened'},
 397                                                 progress: 0.2})
 398     assert_equal 1, Container.where("runtime_status->'error' is not null").count
 399     reused = Container.find_reusable(common_attrs)
 400     assert_not_nil reused
 401     # Selected the non-failing container even if it's the one with less progress done
 402     assert_equal reused.uuid, c_faster_started_first.uuid
 403   end
 404
 405   test "find_reusable method should select locked container most likely to start sooner" do
 406     set_user_from_auth :active
 407     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "locked"}})
 408     c_low_priority, _ = minimal_new(common_attrs.merge({use_existing: false}))
 409     c_high_priority_older, _ = minimal_new(common_attrs.merge({use_existing: false}))
 410     c_high_priority_newer, _ = minimal_new(common_attrs.merge({use_existing: false}))
 411     # Confirm the 3 container UUIDs are different.
 412     assert_equal 3, [c_low_priority.uuid, c_high_priority_older.uuid, c_high_priority_newer.uuid].uniq.length
 413     set_user_from_auth :dispatch1
 414     c_low_priority.update_attributes!({state: Container::Locked,
 415                                        priority: 1})
 416     c_high_priority_older.update_attributes!({state: Container::Locked,
 417                                               priority: 2})
 418     c_high_priority_newer.update_attributes!({state: Container::Locked,
 419                                               priority: 2})
 420     reused = Container.find_reusable(common_attrs)
 421     assert_not_nil reused
 422     assert_equal reused.uuid, c_high_priority_older.uuid
 423   end
 424
 425   test "find_reusable method should select running over failed container" do
 426     set_user_from_auth :active
 427     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed_vs_running"}})
 428     c_failed, _ = minimal_new(common_attrs.merge({use_existing: false}))
 429     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
 430     assert_not_equal c_failed.uuid, c_running.uuid
 431     set_user_from_auth :dispatch1
 432     c_failed.update_attributes!({state: Container::Locked})
 433     c_failed.update_attributes!({state: Container::Running})
 434     c_failed.update_attributes!({state: Container::Complete,
 435                                  exit_code: 42,
 436                                  log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 437                                  output: 'ea10d51bcf88862dbcc36eb292017dfd+45'})
 438     c_running.update_attributes!({state: Container::Locked})
 439     c_running.update_attributes!({state: Container::Running,
 440                                   progress: 0.15})
 441     reused = Container.find_reusable(common_attrs)
 442     assert_not_nil reused
 443     assert_equal reused.uuid, c_running.uuid
 444   end
 445
 446   test "find_reusable method should select complete over running container" do
 447     set_user_from_auth :active
 448     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "completed_vs_running"}})
 449     c_completed, _ = minimal_new(common_attrs.merge({use_existing: false}))
 450     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
 451     assert_not_equal c_completed.uuid, c_running.uuid
 452     set_user_from_auth :dispatch1
 453     c_completed.update_attributes!({state: Container::Locked})
 454     c_completed.update_attributes!({state: Container::Running})
 455     c_completed.update_attributes!({state: Container::Complete,
 456                                     exit_code: 0,
 457                                     log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 458                                     output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'})
 459     c_running.update_attributes!({state: Container::Locked})
 460     c_running.update_attributes!({state: Container::Running,
 461                                   progress: 0.15})
 462     reused = Container.find_reusable(common_attrs)
 463     assert_not_nil reused
 464     assert_equal c_completed.uuid, reused.uuid
 465   end
 466
 467   test "find_reusable method should select running over locked container" do
 468     set_user_from_auth :active
 469     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}})
 470     c_locked, _ = minimal_new(common_attrs.merge({use_existing: false}))
 471     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
 472     assert_not_equal c_running.uuid, c_locked.uuid
 473     set_user_from_auth :dispatch1
 474     c_locked.update_attributes!({state: Container::Locked})
 475     c_running.update_attributes!({state: Container::Locked})
 476     c_running.update_attributes!({state: Container::Running,
 477                                   progress: 0.15})
 478     reused = Container.find_reusable(common_attrs)
 479     assert_not_nil reused
 480     assert_equal reused.uuid, c_running.uuid
 481   end
 482
 483   test "find_reusable method should select locked over queued container" do
 484     set_user_from_auth :active
 485     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}})
 486     c_locked, _ = minimal_new(common_attrs.merge({use_existing: false}))
 487     c_queued, _ = minimal_new(common_attrs.merge({use_existing: false}))
 488     assert_not_equal c_queued.uuid, c_locked.uuid
 489     set_user_from_auth :dispatch1
 490     c_locked.update_attributes!({state: Container::Locked})
 491     reused = Container.find_reusable(common_attrs)
 492     assert_not_nil reused
 493     assert_equal reused.uuid, c_locked.uuid
 494   end
 495
 496   test "find_reusable method should not select failed container" do
 497     set_user_from_auth :active
 498     attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed"}})
 499     c, _ = minimal_new(attrs)
 500     set_user_from_auth :dispatch1
 501     c.update_attributes!({state: Container::Locked})
 502     c.update_attributes!({state: Container::Running})
 503     c.update_attributes!({state: Container::Complete,
 504                           exit_code: 33})
 505     reused = Container.find_reusable(attrs)
 506     assert_nil reused
 507   end
 508
 509   test "find_reusable with logging disabled" do
 510     set_user_from_auth :active
 511     Rails.logger.expects(:info).never
 512     Container.find_reusable(REUSABLE_COMMON_ATTRS)
 513   end
 514
 515   test "find_reusable with logging enabled" do
 516     set_user_from_auth :active
 517     Rails.configuration.Containers.LogReuseDecisions = true
 518     Rails.logger.expects(:info).at_least(3)
 519     Container.find_reusable(REUSABLE_COMMON_ATTRS)
 520   end
 521
 522   def runtime_token_attr tok
 523     auth = api_client_authorizations(tok)
 524     {runtime_user_uuid: User.find_by_id(auth.user_id).uuid,
 525      runtime_auth_scopes: auth.scopes,
 526      runtime_token: auth.token}
 527   end
 528
 529   test "find_reusable method with same runtime_token" do
 530     set_user_from_auth :active
 531     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 532     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:container_runtime_token).token}))
 533     assert_equal Container::Queued, c1.state
 534     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 535     assert_not_nil reused
 536     assert_equal reused.uuid, c1.uuid
 537   end
 538
 539   test "find_reusable method with different runtime_token, same user" do
 540     set_user_from_auth :active
 541     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 542     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:crt_user).token}))
 543     assert_equal Container::Queued, c1.state
 544     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 545     assert_not_nil reused
 546     assert_equal reused.uuid, c1.uuid
 547   end
 548
 549   test "find_reusable method with nil runtime_token, then runtime_token with same user" do
 550     set_user_from_auth :crt_user
 551     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 552     c1, _ = minimal_new(common_attrs)
 553     assert_equal Container::Queued, c1.state
 554     assert_equal users(:container_runtime_token_user).uuid, c1.runtime_user_uuid
 555     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 556     assert_not_nil reused
 557     assert_equal reused.uuid, c1.uuid
 558   end
 559
 560   test "find_reusable method with different runtime_token, different user" do
 561     set_user_from_auth :crt_user
 562     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 563     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:active).token}))
 564     assert_equal Container::Queued, c1.state
 565     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 566     # See #14584
 567     assert_not_nil reused
 568     assert_equal c1.uuid, reused.uuid
 569   end
 570
 571   test "find_reusable method with nil runtime_token, then runtime_token with different user" do
 572     set_user_from_auth :active
 573     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 574     c1, _ = minimal_new(common_attrs.merge({runtime_token: nil}))
 575     assert_equal Container::Queued, c1.state
 576     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 577     # See #14584
 578     assert_not_nil reused
 579     assert_equal c1.uuid, reused.uuid
 580   end
 581
 582   test "find_reusable method with different runtime_token, different scope, same user" do
 583     set_user_from_auth :active
 584     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 585     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:runtime_token_limited_scope).token}))
 586     assert_equal Container::Queued, c1.state
 587     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 588     # See #14584
 589     assert_not_nil reused
 590     assert_equal c1.uuid, reused.uuid
 591   end
 592
 593   test "find_reusable method with cuda" do
 594     set_user_from_auth :active
 595     # No cuda
 596     no_cuda_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"},
 597                                                 runtime_constraints: {"vcpus" => 1, "ram" => 1, "keep_cache_ram"=>268435456, "API" => false,
 598                                                                       "cuda" => {"device_count":0, "driver_version": "", "hardware_capability": ""}},})
 599     c1, _ = minimal_new(no_cuda_attrs)
 600     assert_equal Container::Queued, c1.state
 601
 602     # has cuda
 603     cuda_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"},
 604                                                 runtime_constraints: {"vcpus" => 1, "ram" => 1, "keep_cache_ram"=>268435456, "API" => false,
 605                                                                       "cuda" => {"device_count":1, "driver_version": "11.0", "hardware_capability": "9.0"}},})
 606     c2, _ = minimal_new(cuda_attrs)
 607     assert_equal Container::Queued, c2.state
 608
 609     # should find the no cuda one
 610     reused = Container.find_reusable(no_cuda_attrs)
 611     assert_not_nil reused
 612     assert_equal reused.uuid, c1.uuid
 613
 614     # should find the cuda one
 615     reused = Container.find_reusable(cuda_attrs)
 616     assert_not_nil reused
 617     assert_equal reused.uuid, c2.uuid
 618   end
 619
 620   test "Container running" do
 621     set_user_from_auth :active
 622     c, _ = minimal_new priority: 1
 623
 624     set_user_from_auth :dispatch1
 625     check_illegal_updates c, [{state: Container::Running},
 626                               {state: Container::Complete}]
 627
 628     c.lock
 629     c.update_attributes! state: Container::Running
 630
 631     check_illegal_modify c
 632     check_bogus_states c
 633
 634     check_illegal_updates c, [{state: Container::Queued}]
 635     c.reload
 636
 637     c.update_attributes! priority: 3
 638   end
 639
 640   test "Lock and unlock" do
 641     set_user_from_auth :active
 642     c, cr = minimal_new priority: 0
 643
 644     set_user_from_auth :dispatch1
 645     assert_equal Container::Queued, c.state
 646
 647     assert_raise(ArvadosModel::LockFailedError) do
 648       # "no priority"
 649       c.lock
 650     end
 651     c.reload
 652     assert cr.update_attributes priority: 1
 653
 654     refute c.update_attributes(state: Container::Running), "not locked"
 655     c.reload
 656     refute c.update_attributes(state: Container::Complete), "not locked"
 657     c.reload
 658
 659     assert c.lock, show_errors(c)
 660     assert c.locked_by_uuid
 661     assert c.auth_uuid
 662
 663     assert_raise(ArvadosModel::LockFailedError) {c.lock}
 664     c.reload
 665
 666     assert c.unlock, show_errors(c)
 667     refute c.locked_by_uuid
 668     refute c.auth_uuid
 669
 670     refute c.update_attributes(state: Container::Running), "not locked"
 671     c.reload
 672     refute c.locked_by_uuid
 673     refute c.auth_uuid
 674
 675     assert c.lock, show_errors(c)
 676     assert c.update_attributes(state: Container::Running), show_errors(c)
 677     assert c.locked_by_uuid
 678     assert c.auth_uuid
 679
 680     auth_uuid_was = c.auth_uuid
 681
 682     assert_raise(ArvadosModel::LockFailedError) do
 683       # Running to Locked is not allowed
 684       c.lock
 685     end
 686     c.reload
 687     assert_raise(ArvadosModel::InvalidStateTransitionError) do
 688       # Running to Queued is not allowed
 689       c.unlock
 690     end
 691     c.reload
 692
 693     assert c.update_attributes(state: Container::Complete), show_errors(c)
 694     refute c.locked_by_uuid
 695     refute c.auth_uuid
 696
 697     auth_exp = ApiClientAuthorization.find_by_uuid(auth_uuid_was).expires_at
 698     assert_operator auth_exp, :<, db_current_time
 699
 700     assert_nil ApiClientAuthorization.validate(token: ApiClientAuthorization.find_by_uuid(auth_uuid_was).token)
 701   end
 702
 703   test "Exceed maximum lock-unlock cycles" do
 704     Rails.configuration.Containers.MaxDispatchAttempts = 3
 705
 706     set_user_from_auth :active
 707     c, cr = minimal_new
 708
 709     set_user_from_auth :dispatch1
 710     assert_equal Container::Queued, c.state
 711     assert_equal 0, c.lock_count
 712
 713     c.lock
 714     c.reload
 715     assert_equal 1, c.lock_count
 716     assert_equal Container::Locked, c.state
 717
 718     c.unlock
 719     c.reload
 720     assert_equal 1, c.lock_count
 721     assert_equal Container::Queued, c.state
 722
 723     c.lock
 724     c.reload
 725     assert_equal 2, c.lock_count
 726     assert_equal Container::Locked, c.state
 727
 728     c.unlock
 729     c.reload
 730     assert_equal 2, c.lock_count
 731     assert_equal Container::Queued, c.state
 732
 733     c.lock
 734     c.reload
 735     assert_equal 3, c.lock_count
 736     assert_equal Container::Locked, c.state
 737
 738     c.unlock
 739     c.reload
 740     assert_equal 3, c.lock_count
 741     assert_equal Container::Cancelled, c.state
 742
 743     assert_raise(ArvadosModel::LockFailedError) do
 744       # Cancelled to Locked is not allowed
 745       c.lock
 746     end
 747   end
 748
 749   test "Container queued cancel" do
 750     set_user_from_auth :active
 751     c, cr = minimal_new({container_count_max: 1})
 752     set_user_from_auth :dispatch1
 753     assert c.update_attributes(state: Container::Cancelled), show_errors(c)
 754     check_no_change_from_cancelled c
 755     cr.reload
 756     assert_equal ContainerRequest::Final, cr.state
 757   end
 758
 759   test "Container queued count" do
 760     assert_equal 1, Container.readable_by(users(:active)).where(state: "Queued").count
 761   end
 762
 763   test "Containers with no matching request are readable by admin" do
 764     uuids = Container.includes('container_requests').where(container_requests: {uuid: nil}).collect(&:uuid)
 765     assert_not_empty uuids
 766     assert_empty Container.readable_by(users(:active)).where(uuid: uuids)
 767     assert_not_empty Container.readable_by(users(:admin)).where(uuid: uuids)
 768     assert_equal uuids.count, Container.readable_by(users(:admin)).where(uuid: uuids).count
 769   end
 770
 771   test "Container locked cancel" do
 772     set_user_from_auth :active
 773     c, _ = minimal_new
 774     set_user_from_auth :dispatch1
 775     assert c.lock, show_errors(c)
 776     assert c.update_attributes(state: Container::Cancelled), show_errors(c)
 777     check_no_change_from_cancelled c
 778   end
 779
 780   test "Container locked with non-expiring token" do
 781     Rails.configuration.API.TokenMaxLifetime = 1.hour
 782     set_user_from_auth :active
 783     c, _ = minimal_new
 784     set_user_from_auth :dispatch1
 785     assert c.lock, show_errors(c)
 786     refute c.auth.nil?
 787     assert c.auth.expires_at.nil?
 788     assert c.auth.user_id == User.find_by_uuid(users(:active).uuid).id
 789   end
 790
 791   test "Container locked cancel with log" do
 792     set_user_from_auth :active
 793     c, _ = minimal_new
 794     set_user_from_auth :dispatch1
 795     assert c.lock, show_errors(c)
 796     assert c.update_attributes(
 797              state: Container::Cancelled,
 798              log: collections(:real_log_collection).portable_data_hash,
 799            ), show_errors(c)
 800     check_no_change_from_cancelled c
 801   end
 802
 803   test "Container running cancel" do
 804     set_user_from_auth :active
 805     c, _ = minimal_new
 806     set_user_from_auth :dispatch1
 807     c.lock
 808     c.update_attributes! state: Container::Running
 809     c.update_attributes! state: Container::Cancelled
 810     check_no_change_from_cancelled c
 811   end
 812
 813   test "Container create forbidden for non-admin" do
 814     set_user_from_auth :active_trustedclient
 815     c = Container.new DEFAULT_ATTRS
 816     c.environment = {}
 817     c.mounts = {"BAR" => "FOO"}
 818     c.output_path = "/tmp"
 819     c.priority = 1
 820     c.runtime_constraints = {}
 821     assert_raises(ArvadosModel::PermissionDeniedError) do
 822       c.save!
 823     end
 824   end
 825
 826   [
 827     [Container::Queued, {state: Container::Locked}],
 828     [Container::Queued, {state: Container::Running}],
 829     [Container::Queued, {state: Container::Complete}],
 830     [Container::Queued, {state: Container::Cancelled}],
 831     [Container::Queued, {priority: 123456789}],
 832     [Container::Queued, {runtime_status: {'error' => 'oops'}}],
 833     [Container::Queued, {cwd: '/'}],
 834     [Container::Locked, {state: Container::Running}],
 835     [Container::Locked, {state: Container::Queued}],
 836     [Container::Locked, {priority: 123456789}],
 837     [Container::Locked, {runtime_status: {'error' => 'oops'}}],
 838     [Container::Locked, {cwd: '/'}],
 839     [Container::Running, {state: Container::Complete}],
 840     [Container::Running, {state: Container::Cancelled}],
 841     [Container::Running, {priority: 123456789}],
 842     [Container::Running, {runtime_status: {'error' => 'oops'}}],
 843     [Container::Running, {cwd: '/'}],
 844     [Container::Running, {gateway_address: "172.16.0.1:12345"}],
 845     [Container::Running, {interactive_session_started: true}],
 846     [Container::Complete, {state: Container::Cancelled}],
 847     [Container::Complete, {priority: 123456789}],
 848     [Container::Complete, {runtime_status: {'error' => 'oops'}}],
 849     [Container::Complete, {cwd: '/'}],
 850     [Container::Cancelled, {cwd: '/'}],
 851   ].each do |start_state, updates|
 852     test "Container update #{updates.inspect} when #{start_state} forbidden for non-admin" do
 853       set_user_from_auth :active
 854       c, _ = minimal_new
 855       if start_state != Container::Queued
 856         set_user_from_auth :dispatch1
 857         c.lock
 858         if start_state != Container::Locked
 859           c.update_attributes! state: Container::Running
 860           if start_state != Container::Running
 861             c.update_attributes! state: start_state
 862           end
 863         end
 864       end
 865       assert_equal c.state, start_state
 866       set_user_from_auth :active
 867       assert_raises(ArvadosModel::PermissionDeniedError) do
 868         c.update_attributes! updates
 869       end
 870     end
 871   end
 872
 873   test "can only change exit code while running and at completion" do
 874     set_user_from_auth :active
 875     c, _ = minimal_new
 876     set_user_from_auth :dispatch1
 877     c.lock
 878     check_illegal_updates c, [{exit_code: 1}]
 879     c.update_attributes! state: Container::Running
 880     assert c.update_attributes(exit_code: 1)
 881     assert c.update_attributes(exit_code: 1, state: Container::Complete)
 882   end
 883
 884   test "locked_by_uuid can update log when locked/running, and output when running" do
 885     set_user_from_auth :active
 886     logcoll = collections(:real_log_collection)
 887     c, cr1 = minimal_new
 888     cr2 = ContainerRequest.new(DEFAULT_ATTRS)
 889     cr2.state = ContainerRequest::Committed
 890     act_as_user users(:active) do
 891       cr2.save!
 892     end
 893     assert_equal cr1.container_uuid, cr2.container_uuid
 894
 895     logpdh_time1 = logcoll.portable_data_hash
 896
 897     set_user_from_auth :dispatch1
 898     c.lock
 899     assert_equal c.locked_by_uuid, Thread.current[:api_client_authorization].uuid
 900     c.update_attributes!(log: logpdh_time1)
 901     c.update_attributes!(state: Container::Running)
 902     cr1.reload
 903     cr2.reload
 904     cr1log_uuid = cr1.log_uuid
 905     cr2log_uuid = cr2.log_uuid
 906     assert_not_nil cr1log_uuid
 907     assert_not_nil cr2log_uuid
 908     assert_not_equal logcoll.uuid, cr1log_uuid
 909     assert_not_equal logcoll.uuid, cr2log_uuid
 910     assert_not_equal cr1log_uuid, cr2log_uuid
 911
 912     logcoll.update_attributes!(manifest_text: logcoll.manifest_text + ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n")
 913     logpdh_time2 = logcoll.portable_data_hash
 914
 915     assert c.update_attributes(output: collections(:collection_owned_by_active).portable_data_hash)
 916     assert c.update_attributes(log: logpdh_time2)
 917     assert c.update_attributes(state: Container::Complete, log: logcoll.portable_data_hash)
 918     c.reload
 919     assert_equal collections(:collection_owned_by_active).portable_data_hash, c.output
 920     assert_equal logpdh_time2, c.log
 921     refute c.update_attributes(output: nil)
 922     refute c.update_attributes(log: nil)
 923     cr1.reload
 924     cr2.reload
 925     assert_equal cr1log_uuid, cr1.log_uuid
 926     assert_equal cr2log_uuid, cr2.log_uuid
 927     assert_equal 1, Collection.where(uuid: [cr1log_uuid, cr2log_uuid]).to_a.collect(&:portable_data_hash).uniq.length
 928     assert_equal ". acbd18db4cc2f85cedef654fccc4a4d8+3 cdd549ae79fe6640fa3d5c6261d8303c+195 0:3:foo.txt 3:195:zzzzz-8i9sb-0vsrcqi7whchuil.log.txt
 929 ./log\\040for\\040container\\040#{cr1.container_uuid} acbd18db4cc2f85cedef654fccc4a4d8+3 cdd549ae79fe6640fa3d5c6261d8303c+195 0:3:foo.txt 3:195:zzzzz-8i9sb-0vsrcqi7whchuil.log.txt
 930 ", Collection.find_by_uuid(cr1log_uuid).manifest_text
 931   end
 932
 933   ["auth_uuid", "runtime_token"].each do |tok|
 934     test "#{tok} can set output, progress, runtime_status, state, exit_code on running container -- but not log" do
 935       if tok == "runtime_token"
 936         set_user_from_auth :spectator
 937         c, _ = minimal_new(container_image: "9ae44d5792468c58bcf85ce7353c7027+124",
 938                            runtime_token: api_client_authorizations(:active).token)
 939       else
 940         set_user_from_auth :active
 941         c, _ = minimal_new
 942       end
 943       set_user_from_auth :dispatch1
 944       c.lock
 945       c.update_attributes! state: Container::Running
 946
 947       if tok == "runtime_token"
 948         auth = ApiClientAuthorization.validate(token: c.runtime_token)
 949         Thread.current[:api_client_authorization] = auth
 950         Thread.current[:api_client] = auth.api_client
 951         Thread.current[:token] = auth.token
 952         Thread.current[:user] = auth.user
 953       else
 954         auth = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
 955         Thread.current[:api_client_authorization] = auth
 956         Thread.current[:api_client] = auth.api_client
 957         Thread.current[:token] = auth.token
 958         Thread.current[:user] = auth.user
 959       end
 960
 961       assert c.update_attributes(gateway_address: "127.0.0.1:9")
 962       assert c.update_attributes(output: collections(:collection_owned_by_active).portable_data_hash)
 963       assert c.update_attributes(runtime_status: {'warning' => 'something happened'})
 964       assert c.update_attributes(progress: 0.5)
 965       assert c.update_attributes(exit_code: 0)
 966       refute c.update_attributes(log: collections(:real_log_collection).portable_data_hash)
 967       c.reload
 968       assert c.update_attributes(state: Container::Complete, exit_code: 0)
 969     end
 970   end
 971
 972   test "not allowed to set output that is not readable by current user" do
 973     set_user_from_auth :active
 974     c, _ = minimal_new
 975     set_user_from_auth :dispatch1
 976     c.lock
 977     c.update_attributes! state: Container::Running
 978
 979     Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
 980     Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
 981
 982     assert_raises ActiveRecord::RecordInvalid do
 983       c.update_attributes! output: collections(:collection_not_readable_by_active).portable_data_hash
 984     end
 985   end
 986
 987   test "other token cannot set output on running container" do
 988     set_user_from_auth :active
 989     c, _ = minimal_new
 990     set_user_from_auth :dispatch1
 991     c.lock
 992     c.update_attributes! state: Container::Running
 993
 994     set_user_from_auth :running_to_be_deleted_container_auth
 995     assert_raises(ArvadosModel::PermissionDeniedError) do
 996       c.update_attributes(output: collections(:foo_file).portable_data_hash)
 997     end
 998   end
 999
1000   test "can set trashed output on running container" do
1001     set_user_from_auth :active
1002     c, _ = minimal_new
1003     set_user_from_auth :dispatch1
1004     c.lock
1005     c.update_attributes! state: Container::Running
1006
1007     output = Collection.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jk')
1008
1009     assert output.is_trashed
1010     assert c.update_attributes output: output.portable_data_hash
1011     assert c.update_attributes! state: Container::Complete
1012   end
1013
1014   test "not allowed to set trashed output that is not readable by current user" do
1015     set_user_from_auth :active
1016     c, _ = minimal_new
1017     set_user_from_auth :dispatch1
1018     c.lock
1019     c.update_attributes! state: Container::Running
1020
1021     output = Collection.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jr')
1022
1023     Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
1024     Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
1025
1026     assert_raises ActiveRecord::RecordInvalid do
1027       c.update_attributes! output: output.portable_data_hash
1028     end
1029   end
1030
1031   test "user cannot delete" do
1032     set_user_from_auth :active
1033     c, _ = minimal_new
1034     assert_raises ArvadosModel::PermissionDeniedError do
1035       c.destroy
1036     end
1037     assert Container.find_by_uuid(c.uuid)
1038   end
1039
1040   [
1041     {state: Container::Complete, exit_code: 0, output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'},
1042     {state: Container::Cancelled},
1043   ].each do |final_attrs|
1044     test "secret_mounts and runtime_token are null after container is #{final_attrs[:state]}" do
1045       set_user_from_auth :active
1046       c, cr = minimal_new(secret_mounts: {'/secret' => {'kind' => 'text', 'content' => 'foo'}},
1047                           container_count_max: 1, runtime_token: api_client_authorizations(:active).token)
1048       set_user_from_auth :dispatch1
1049       c.lock
1050       c.update_attributes!(state: Container::Running)
1051       c.reload
1052       assert c.secret_mounts.has_key?('/secret')
1053       assert_equal api_client_authorizations(:active).token, c.runtime_token
1054
1055       c.update_attributes!(final_attrs)
1056       c.reload
1057       assert_equal({}, c.secret_mounts)
1058       assert_nil c.runtime_token
1059       cr.reload
1060       assert_equal({}, cr.secret_mounts)
1061       assert_nil cr.runtime_token
1062       assert_no_secrets_logged
1063     end
1064   end
1065 end