services/api/test/unit/container_test.rb

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 require 'test_helper'
   6 require 'helpers/container_test_helper'
   7
   8 class ContainerTest < ActiveSupport::TestCase
   9   include DbCurrentTime
  10   include ContainerTestHelper
  11
  12   DEFAULT_ATTRS = {
  13     command: ['echo', 'foo'],
  14     container_image: 'fa3c1a9cb6783f85f2ecda037e07b8c3+167',
  15     output_path: '/tmp',
  16     priority: 1,
  17     runtime_constraints: {"vcpus" => 1, "ram" => 1},
  18   }
  19
  20   REUSABLE_COMMON_ATTRS = {
  21     container_image: "9ae44d5792468c58bcf85ce7353c7027+124",
  22     cwd: "test",
  23     command: ["echo", "hello"],
  24     output_path: "test",
  25     runtime_constraints: {
  26       "API" => false,
  27       "keep_cache_ram" => 0,
  28       "ram" => 12000000000,
  29       "vcpus" => 4,
  30     },
  31     mounts: {
  32       "test" => {"kind" => "json"},
  33     },
  34     environment: {
  35       "var" => "val",
  36     },
  37     secret_mounts: {},
  38     runtime_user_uuid: "zzzzz-tpzed-xurymjxw79nv3jz",
  39     runtime_auth_scopes: ["all"]
  40   }
  41
  42   def request_only attrs
  43     attrs.reject {|k| [:runtime_user_uuid, :runtime_auth_scopes].include? k}
  44   end
  45
  46   def minimal_new attrs={}
  47     cr = ContainerRequest.new request_only(DEFAULT_ATTRS.merge(attrs))
  48     cr.state = ContainerRequest::Committed
  49     cr.save!
  50     c = Container.find_by_uuid cr.container_uuid
  51     assert_not_nil c
  52     return c, cr
  53   end
  54
  55   def check_illegal_updates c, bad_updates
  56     bad_updates.each do |u|
  57       refute c.update_attributes(u), u.inspect
  58       refute c.valid?, u.inspect
  59       c.reload
  60     end
  61   end
  62
  63   def check_illegal_modify c
  64     check_illegal_updates c, [{command: ["echo", "bar"]},
  65                               {container_image: "arvados/apitestfixture:june10"},
  66                               {cwd: "/tmp2"},
  67                               {environment: {"FOO" => "BAR"}},
  68                               {mounts: {"FOO" => "BAR"}},
  69                               {output_path: "/tmp3"},
  70                               {locked_by_uuid: "zzzzz-gj3su-027z32aux8dg2s1"},
  71                               {auth_uuid: "zzzzz-gj3su-017z32aux8dg2s1"},
  72                               {runtime_constraints: {"FOO" => "BAR"}}]
  73   end
  74
  75   def check_bogus_states c
  76     check_illegal_updates c, [{state: nil},
  77                               {state: "Flubber"}]
  78   end
  79
  80   def check_no_change_from_cancelled c
  81     check_illegal_modify c
  82     check_bogus_states c
  83     check_illegal_updates c, [{ priority: 3 },
  84                               { state: Container::Queued },
  85                               { state: Container::Locked },
  86                               { state: Container::Running },
  87                               { state: Container::Complete }]
  88   end
  89
  90   test "Container create" do
  91     act_as_system_user do
  92       c, _ = minimal_new(environment: {},
  93                       mounts: {"BAR" => {"kind" => "FOO"}},
  94                       output_path: "/tmp",
  95                       priority: 1,
  96                       runtime_constraints: {"vcpus" => 1, "ram" => 1})
  97
  98       check_illegal_modify c
  99       check_bogus_states c
 100
 101       c.reload
 102       c.priority = 2
 103       c.save!
 104     end
 105   end
 106
 107   test "Container valid priority" do
 108     act_as_system_user do
 109       c, _ = minimal_new(environment: {},
 110                       mounts: {"BAR" => {"kind" => "FOO"}},
 111                       output_path: "/tmp",
 112                       priority: 1,
 113                       runtime_constraints: {"vcpus" => 1, "ram" => 1})
 114
 115       assert_raises(ActiveRecord::RecordInvalid) do
 116         c.priority = -1
 117         c.save!
 118       end
 119
 120       c.priority = 0
 121       c.save!
 122
 123       c.priority = 1
 124       c.save!
 125
 126       c.priority = 500
 127       c.save!
 128
 129       c.priority = 999
 130       c.save!
 131
 132       c.priority = 1000
 133       c.save!
 134
 135       c.priority = 1000 << 50
 136       c.save!
 137     end
 138   end
 139
 140   test "Container runtime_status data types" do
 141     set_user_from_auth :active
 142     attrs = {
 143       environment: {},
 144       mounts: {"BAR" => {"kind" => "FOO"}},
 145       output_path: "/tmp",
 146       priority: 1,
 147       runtime_constraints: {"vcpus" => 1, "ram" => 1}
 148     }
 149     c, _ = minimal_new(attrs)
 150     assert_equal c.runtime_status, {}
 151     assert_equal Container::Queued, c.state
 152
 153     set_user_from_auth :dispatch1
 154     c.update_attributes! state: Container::Locked
 155     c.update_attributes! state: Container::Running
 156
 157     [
 158       'error', 'errorDetail', 'warning', 'warningDetail', 'activity'
 159     ].each do |k|
 160       # String type is allowed
 161       string_val = 'A string is accepted'
 162       c.update_attributes! runtime_status: {k => string_val}
 163       assert_equal string_val, c.runtime_status[k]
 164
 165       # Other types aren't allowed
 166       [
 167         42, false, [], {}, nil
 168       ].each do |unallowed_val|
 169         assert_raises ActiveRecord::RecordInvalid do
 170           c.update_attributes! runtime_status: {k => unallowed_val}
 171         end
 172       end
 173     end
 174   end
 175
 176   test "Container runtime_status updates" do
 177     set_user_from_auth :active
 178     attrs = {
 179       environment: {},
 180       mounts: {"BAR" => {"kind" => "FOO"}},
 181       output_path: "/tmp",
 182       priority: 1,
 183       runtime_constraints: {"vcpus" => 1, "ram" => 1}
 184     }
 185     c1, _ = minimal_new(attrs)
 186     assert_equal c1.runtime_status, {}
 187
 188     assert_equal Container::Queued, c1.state
 189     assert_raises ArvadosModel::PermissionDeniedError do
 190       c1.update_attributes! runtime_status: {'error' => 'Oops!'}
 191     end
 192
 193     set_user_from_auth :dispatch1
 194
 195     # Allow updates when state = Locked
 196     c1.update_attributes! state: Container::Locked
 197     c1.update_attributes! runtime_status: {'error' => 'Oops!'}
 198     assert c1.runtime_status.key? 'error'
 199
 200     # Reset when transitioning from Locked to Queued
 201     c1.update_attributes! state: Container::Queued
 202     assert_equal c1.runtime_status, {}
 203
 204     # Allow updates when state = Running
 205     c1.update_attributes! state: Container::Locked
 206     c1.update_attributes! state: Container::Running
 207     c1.update_attributes! runtime_status: {'error' => 'Oops!'}
 208     assert c1.runtime_status.key? 'error'
 209
 210     # Don't allow updates on other states
 211     c1.update_attributes! state: Container::Complete
 212     assert_raises ActiveRecord::RecordInvalid do
 213       c1.update_attributes! runtime_status: {'error' => 'Some other error'}
 214     end
 215
 216     set_user_from_auth :active
 217     c2, _ = minimal_new(attrs)
 218     assert_equal c2.runtime_status, {}
 219     set_user_from_auth :dispatch1
 220     c2.update_attributes! state: Container::Locked
 221     c2.update_attributes! state: Container::Running
 222     c2.update_attributes! state: Container::Cancelled
 223     assert_raises ActiveRecord::RecordInvalid do
 224       c2.update_attributes! runtime_status: {'error' => 'Oops!'}
 225     end
 226   end
 227
 228   test "Container serialized hash attributes sorted before save" do
 229     set_user_from_auth :active
 230     env = {"C" => "3", "B" => "2", "A" => "1"}
 231     m = {"F" => {"kind" => "3"}, "E" => {"kind" => "2"}, "D" => {"kind" => "1"}}
 232     rc = {"vcpus" => 1, "ram" => 1, "keep_cache_ram" => 1, "API" => true}
 233     c, _ = minimal_new(environment: env, mounts: m, runtime_constraints: rc)
 234     c.reload
 235     assert_equal Container.deep_sort_hash(env).to_json, c.environment.to_json
 236     assert_equal Container.deep_sort_hash(m).to_json, c.mounts.to_json
 237     assert_equal Container.deep_sort_hash(rc).to_json, c.runtime_constraints.to_json
 238   end
 239
 240   test 'deep_sort_hash on array of hashes' do
 241     a = {'z' => [[{'a' => 'a', 'b' => 'b'}]]}
 242     b = {'z' => [[{'b' => 'b', 'a' => 'a'}]]}
 243     assert_equal Container.deep_sort_hash(a).to_json, Container.deep_sort_hash(b).to_json
 244   end
 245
 246   test "find_reusable method should select higher priority queued container" do
 247         Rails.configuration.Containers.LogReuseDecisions = true
 248     set_user_from_auth :active
 249     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment:{"var" => "queued"}})
 250     c_low_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:1}))
 251     c_high_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:2}))
 252     assert_not_equal c_low_priority.uuid, c_high_priority.uuid
 253     assert_equal Container::Queued, c_low_priority.state
 254     assert_equal Container::Queued, c_high_priority.state
 255     reused = Container.find_reusable(common_attrs)
 256     assert_not_nil reused
 257     assert_equal reused.uuid, c_high_priority.uuid
 258   end
 259
 260   test "find_reusable method should select latest completed container" do
 261     set_user_from_auth :active
 262     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "complete"}})
 263     completed_attrs = {
 264       state: Container::Complete,
 265       exit_code: 0,
 266       log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 267       output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
 268     }
 269
 270     c_older, _ = minimal_new(common_attrs.merge({use_existing: false}))
 271     c_recent, _ = minimal_new(common_attrs.merge({use_existing: false}))
 272     assert_not_equal c_older.uuid, c_recent.uuid
 273
 274     set_user_from_auth :dispatch1
 275     c_older.update_attributes!({state: Container::Locked})
 276     c_older.update_attributes!({state: Container::Running})
 277     c_older.update_attributes!(completed_attrs)
 278
 279     c_recent.update_attributes!({state: Container::Locked})
 280     c_recent.update_attributes!({state: Container::Running})
 281     c_recent.update_attributes!(completed_attrs)
 282
 283     reused = Container.find_reusable(common_attrs)
 284     assert_not_nil reused
 285     assert_equal reused.uuid, c_older.uuid
 286   end
 287
 288   test "find_reusable method should select oldest completed container when inconsistent outputs exist" do
 289     set_user_from_auth :active
 290     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "complete"}, priority: 1})
 291     completed_attrs = {
 292       state: Container::Complete,
 293       exit_code: 0,
 294       log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 295     }
 296
 297     cr = ContainerRequest.new request_only(common_attrs)
 298     cr.use_existing = false
 299     cr.state = ContainerRequest::Committed
 300     cr.save!
 301     c_output1 = Container.where(uuid: cr.container_uuid).first
 302
 303     cr = ContainerRequest.new request_only(common_attrs)
 304     cr.use_existing = false
 305     cr.state = ContainerRequest::Committed
 306     cr.save!
 307     c_output2 = Container.where(uuid: cr.container_uuid).first
 308
 309     assert_not_equal c_output1.uuid, c_output2.uuid
 310
 311     set_user_from_auth :dispatch1
 312
 313     out1 = '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
 314     log1 = collections(:real_log_collection).portable_data_hash
 315     c_output1.update_attributes!({state: Container::Locked})
 316     c_output1.update_attributes!({state: Container::Running})
 317     c_output1.update_attributes!(completed_attrs.merge({log: log1, output: out1}))
 318
 319     out2 = 'fa7aeb5140e2848d39b416daeef4ffc5+45'
 320     c_output2.update_attributes!({state: Container::Locked})
 321     c_output2.update_attributes!({state: Container::Running})
 322     c_output2.update_attributes!(completed_attrs.merge({log: log1, output: out2}))
 323
 324     set_user_from_auth :active
 325     reused = Container.resolve(ContainerRequest.new(request_only(common_attrs)))
 326     assert_equal c_output1.uuid, reused.uuid
 327   end
 328
 329   test "find_reusable method should select running container by start date" do
 330     set_user_from_auth :active
 331     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running"}})
 332     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
 333     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
 334     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
 335     # Confirm the 3 container UUIDs are different.
 336     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
 337     set_user_from_auth :dispatch1
 338     c_slower.update_attributes!({state: Container::Locked})
 339     c_slower.update_attributes!({state: Container::Running,
 340                                  progress: 0.1})
 341     c_faster_started_first.update_attributes!({state: Container::Locked})
 342     c_faster_started_first.update_attributes!({state: Container::Running,
 343                                                progress: 0.15})
 344     c_faster_started_second.update_attributes!({state: Container::Locked})
 345     c_faster_started_second.update_attributes!({state: Container::Running,
 346                                                 progress: 0.15})
 347     reused = Container.find_reusable(common_attrs)
 348     assert_not_nil reused
 349     # Selected container is the one that started first
 350     assert_equal reused.uuid, c_faster_started_first.uuid
 351   end
 352
 353   test "find_reusable method should select running container by progress" do
 354     set_user_from_auth :active
 355     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running2"}})
 356     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
 357     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
 358     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
 359     # Confirm the 3 container UUIDs are different.
 360     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
 361     set_user_from_auth :dispatch1
 362     c_slower.update_attributes!({state: Container::Locked})
 363     c_slower.update_attributes!({state: Container::Running,
 364                                  progress: 0.1})
 365     c_faster_started_first.update_attributes!({state: Container::Locked})
 366     c_faster_started_first.update_attributes!({state: Container::Running,
 367                                                progress: 0.15})
 368     c_faster_started_second.update_attributes!({state: Container::Locked})
 369     c_faster_started_second.update_attributes!({state: Container::Running,
 370                                                 progress: 0.2})
 371     reused = Container.find_reusable(common_attrs)
 372     assert_not_nil reused
 373     # Selected container is the one with most progress done
 374     assert_equal reused.uuid, c_faster_started_second.uuid
 375   end
 376
 377   test "find_reusable method should select non-failing running container" do
 378     set_user_from_auth :active
 379     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running2"}})
 380     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
 381     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
 382     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
 383     # Confirm the 3 container UUIDs are different.
 384     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
 385     set_user_from_auth :dispatch1
 386     c_slower.update_attributes!({state: Container::Locked})
 387     c_slower.update_attributes!({state: Container::Running,
 388                                  progress: 0.1})
 389     c_faster_started_first.update_attributes!({state: Container::Locked})
 390     c_faster_started_first.update_attributes!({state: Container::Running,
 391                                                runtime_status: {'warning' => 'This is not an error'},
 392                                                progress: 0.15})
 393     c_faster_started_second.update_attributes!({state: Container::Locked})
 394     assert_equal 0, Container.where("runtime_status->'error' is not null").count
 395     c_faster_started_second.update_attributes!({state: Container::Running,
 396                                                 runtime_status: {'error' => 'Something bad happened'},
 397                                                 progress: 0.2})
 398     assert_equal 1, Container.where("runtime_status->'error' is not null").count
 399     reused = Container.find_reusable(common_attrs)
 400     assert_not_nil reused
 401     # Selected the non-failing container even if it's the one with less progress done
 402     assert_equal reused.uuid, c_faster_started_first.uuid
 403   end
 404
 405   test "find_reusable method should select locked container most likely to start sooner" do
 406     set_user_from_auth :active
 407     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "locked"}})
 408     c_low_priority, _ = minimal_new(common_attrs.merge({use_existing: false}))
 409     c_high_priority_older, _ = minimal_new(common_attrs.merge({use_existing: false}))
 410     c_high_priority_newer, _ = minimal_new(common_attrs.merge({use_existing: false}))
 411     # Confirm the 3 container UUIDs are different.
 412     assert_equal 3, [c_low_priority.uuid, c_high_priority_older.uuid, c_high_priority_newer.uuid].uniq.length
 413     set_user_from_auth :dispatch1
 414     c_low_priority.update_attributes!({state: Container::Locked,
 415                                        priority: 1})
 416     c_high_priority_older.update_attributes!({state: Container::Locked,
 417                                               priority: 2})
 418     c_high_priority_newer.update_attributes!({state: Container::Locked,
 419                                               priority: 2})
 420     reused = Container.find_reusable(common_attrs)
 421     assert_not_nil reused
 422     assert_equal reused.uuid, c_high_priority_older.uuid
 423   end
 424
 425   test "find_reusable method should select running over failed container" do
 426     set_user_from_auth :active
 427     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed_vs_running"}})
 428     c_failed, _ = minimal_new(common_attrs.merge({use_existing: false}))
 429     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
 430     assert_not_equal c_failed.uuid, c_running.uuid
 431     set_user_from_auth :dispatch1
 432     c_failed.update_attributes!({state: Container::Locked})
 433     c_failed.update_attributes!({state: Container::Running})
 434     c_failed.update_attributes!({state: Container::Complete,
 435                                  exit_code: 42,
 436                                  log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 437                                  output: 'ea10d51bcf88862dbcc36eb292017dfd+45'})
 438     c_running.update_attributes!({state: Container::Locked})
 439     c_running.update_attributes!({state: Container::Running,
 440                                   progress: 0.15})
 441     reused = Container.find_reusable(common_attrs)
 442     assert_not_nil reused
 443     assert_equal reused.uuid, c_running.uuid
 444   end
 445
 446   test "find_reusable method should select complete over running container" do
 447     set_user_from_auth :active
 448     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "completed_vs_running"}})
 449     c_completed, _ = minimal_new(common_attrs.merge({use_existing: false}))
 450     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
 451     assert_not_equal c_completed.uuid, c_running.uuid
 452     set_user_from_auth :dispatch1
 453     c_completed.update_attributes!({state: Container::Locked})
 454     c_completed.update_attributes!({state: Container::Running})
 455     c_completed.update_attributes!({state: Container::Complete,
 456                                     exit_code: 0,
 457                                     log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 458                                     output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'})
 459     c_running.update_attributes!({state: Container::Locked})
 460     c_running.update_attributes!({state: Container::Running,
 461                                   progress: 0.15})
 462     reused = Container.find_reusable(common_attrs)
 463     assert_not_nil reused
 464     assert_equal c_completed.uuid, reused.uuid
 465   end
 466
 467   test "find_reusable method should select running over locked container" do
 468     set_user_from_auth :active
 469     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}})
 470     c_locked, _ = minimal_new(common_attrs.merge({use_existing: false}))
 471     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
 472     assert_not_equal c_running.uuid, c_locked.uuid
 473     set_user_from_auth :dispatch1
 474     c_locked.update_attributes!({state: Container::Locked})
 475     c_running.update_attributes!({state: Container::Locked})
 476     c_running.update_attributes!({state: Container::Running,
 477                                   progress: 0.15})
 478     reused = Container.find_reusable(common_attrs)
 479     assert_not_nil reused
 480     assert_equal reused.uuid, c_running.uuid
 481   end
 482
 483   test "find_reusable method should select locked over queued container" do
 484     set_user_from_auth :active
 485     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}})
 486     c_locked, _ = minimal_new(common_attrs.merge({use_existing: false}))
 487     c_queued, _ = minimal_new(common_attrs.merge({use_existing: false}))
 488     assert_not_equal c_queued.uuid, c_locked.uuid
 489     set_user_from_auth :dispatch1
 490     c_locked.update_attributes!({state: Container::Locked})
 491     reused = Container.find_reusable(common_attrs)
 492     assert_not_nil reused
 493     assert_equal reused.uuid, c_locked.uuid
 494   end
 495
 496   test "find_reusable method should not select failed container" do
 497     set_user_from_auth :active
 498     attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed"}})
 499     c, _ = minimal_new(attrs)
 500     set_user_from_auth :dispatch1
 501     c.update_attributes!({state: Container::Locked})
 502     c.update_attributes!({state: Container::Running})
 503     c.update_attributes!({state: Container::Complete,
 504                           exit_code: 33})
 505     reused = Container.find_reusable(attrs)
 506     assert_nil reused
 507   end
 508
 509   test "find_reusable with logging disabled" do
 510     set_user_from_auth :active
 511     Rails.logger.expects(:info).never
 512     Container.find_reusable(REUSABLE_COMMON_ATTRS)
 513   end
 514
 515   test "find_reusable with logging enabled" do
 516     set_user_from_auth :active
 517     Rails.configuration.Containers.LogReuseDecisions = true
 518     Rails.logger.expects(:info).at_least(3)
 519     Container.find_reusable(REUSABLE_COMMON_ATTRS)
 520   end
 521
 522   def runtime_token_attr tok
 523     auth = api_client_authorizations(tok)
 524     {runtime_user_uuid: User.find_by_id(auth.user_id).uuid,
 525      runtime_auth_scopes: auth.scopes,
 526      runtime_token: auth.token}
 527   end
 528
 529   test "find_reusable method with same runtime_token" do
 530     set_user_from_auth :active
 531     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 532     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:container_runtime_token).token}))
 533     assert_equal Container::Queued, c1.state
 534     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 535     assert_not_nil reused
 536     assert_equal reused.uuid, c1.uuid
 537   end
 538
 539   test "find_reusable method with different runtime_token, same user" do
 540     set_user_from_auth :active
 541     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 542     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:crt_user).token}))
 543     assert_equal Container::Queued, c1.state
 544     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 545     assert_not_nil reused
 546     assert_equal reused.uuid, c1.uuid
 547   end
 548
 549   test "find_reusable method with nil runtime_token, then runtime_token with same user" do
 550     set_user_from_auth :crt_user
 551     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 552     c1, _ = minimal_new(common_attrs)
 553     assert_equal Container::Queued, c1.state
 554     assert_equal users(:container_runtime_token_user).uuid, c1.runtime_user_uuid
 555     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 556     assert_not_nil reused
 557     assert_equal reused.uuid, c1.uuid
 558   end
 559
 560   test "find_reusable method with different runtime_token, different user" do
 561     set_user_from_auth :crt_user
 562     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 563     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:active).token}))
 564     assert_equal Container::Queued, c1.state
 565     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 566     # See #14584
 567     assert_not_nil reused
 568     assert_equal c1.uuid, reused.uuid
 569   end
 570
 571   test "find_reusable method with nil runtime_token, then runtime_token with different user" do
 572     set_user_from_auth :active
 573     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 574     c1, _ = minimal_new(common_attrs.merge({runtime_token: nil}))
 575     assert_equal Container::Queued, c1.state
 576     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 577     # See #14584
 578     assert_not_nil reused
 579     assert_equal c1.uuid, reused.uuid
 580   end
 581
 582   test "find_reusable method with different runtime_token, different scope, same user" do
 583     set_user_from_auth :active
 584     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 585     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:runtime_token_limited_scope).token}))
 586     assert_equal Container::Queued, c1.state
 587     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 588     # See #14584
 589     assert_not_nil reused
 590     assert_equal c1.uuid, reused.uuid
 591   end
 592
 593   test "Container running" do
 594     set_user_from_auth :active
 595     c, _ = minimal_new priority: 1
 596
 597     set_user_from_auth :dispatch1
 598     check_illegal_updates c, [{state: Container::Running},
 599                               {state: Container::Complete}]
 600
 601     c.lock
 602     c.update_attributes! state: Container::Running
 603
 604     check_illegal_modify c
 605     check_bogus_states c
 606
 607     check_illegal_updates c, [{state: Container::Queued}]
 608     c.reload
 609
 610     c.update_attributes! priority: 3
 611   end
 612
 613   test "Lock and unlock" do
 614     set_user_from_auth :active
 615     c, cr = minimal_new priority: 0
 616
 617     set_user_from_auth :dispatch1
 618     assert_equal Container::Queued, c.state
 619
 620     assert_raise(ArvadosModel::LockFailedError) do
 621       # "no priority"
 622       c.lock
 623     end
 624     c.reload
 625     assert cr.update_attributes priority: 1
 626
 627     refute c.update_attributes(state: Container::Running), "not locked"
 628     c.reload
 629     refute c.update_attributes(state: Container::Complete), "not locked"
 630     c.reload
 631
 632     assert c.lock, show_errors(c)
 633     assert c.locked_by_uuid
 634     assert c.auth_uuid
 635
 636     assert_raise(ArvadosModel::LockFailedError) {c.lock}
 637     c.reload
 638
 639     assert c.unlock, show_errors(c)
 640     refute c.locked_by_uuid
 641     refute c.auth_uuid
 642
 643     refute c.update_attributes(state: Container::Running), "not locked"
 644     c.reload
 645     refute c.locked_by_uuid
 646     refute c.auth_uuid
 647
 648     assert c.lock, show_errors(c)
 649     assert c.update_attributes(state: Container::Running), show_errors(c)
 650     assert c.locked_by_uuid
 651     assert c.auth_uuid
 652
 653     auth_uuid_was = c.auth_uuid
 654
 655     assert_raise(ArvadosModel::LockFailedError) do
 656       # Running to Locked is not allowed
 657       c.lock
 658     end
 659     c.reload
 660     assert_raise(ArvadosModel::InvalidStateTransitionError) do
 661       # Running to Queued is not allowed
 662       c.unlock
 663     end
 664     c.reload
 665
 666     assert c.update_attributes(state: Container::Complete), show_errors(c)
 667     refute c.locked_by_uuid
 668     refute c.auth_uuid
 669
 670     auth_exp = ApiClientAuthorization.find_by_uuid(auth_uuid_was).expires_at
 671     assert_operator auth_exp, :<, db_current_time
 672
 673     assert_nil ApiClientAuthorization.validate(token: ApiClientAuthorization.find_by_uuid(auth_uuid_was).token)
 674   end
 675
 676   test "Exceed maximum lock-unlock cycles" do
 677     Rails.configuration.Containers.MaxDispatchAttempts = 3
 678
 679     set_user_from_auth :active
 680     c, cr = minimal_new
 681
 682     set_user_from_auth :dispatch1
 683     assert_equal Container::Queued, c.state
 684     assert_equal 0, c.lock_count
 685
 686     c.lock
 687     c.reload
 688     assert_equal 1, c.lock_count
 689     assert_equal Container::Locked, c.state
 690
 691     c.unlock
 692     c.reload
 693     assert_equal 1, c.lock_count
 694     assert_equal Container::Queued, c.state
 695
 696     c.lock
 697     c.reload
 698     assert_equal 2, c.lock_count
 699     assert_equal Container::Locked, c.state
 700
 701     c.unlock
 702     c.reload
 703     assert_equal 2, c.lock_count
 704     assert_equal Container::Queued, c.state
 705
 706     c.lock
 707     c.reload
 708     assert_equal 3, c.lock_count
 709     assert_equal Container::Locked, c.state
 710
 711     c.unlock
 712     c.reload
 713     assert_equal 3, c.lock_count
 714     assert_equal Container::Cancelled, c.state
 715
 716     assert_raise(ArvadosModel::LockFailedError) do
 717       # Cancelled to Locked is not allowed
 718       c.lock
 719     end
 720   end
 721
 722   test "Container queued cancel" do
 723     set_user_from_auth :active
 724     c, cr = minimal_new({container_count_max: 1})
 725     set_user_from_auth :dispatch1
 726     assert c.update_attributes(state: Container::Cancelled), show_errors(c)
 727     check_no_change_from_cancelled c
 728     cr.reload
 729     assert_equal ContainerRequest::Final, cr.state
 730   end
 731
 732   test "Container queued count" do
 733     assert_equal 1, Container.readable_by(users(:active)).where(state: "Queued").count
 734   end
 735
 736   test "Containers with no matching request are readable by admin" do
 737     uuids = Container.includes('container_requests').where(container_requests: {uuid: nil}).collect(&:uuid)
 738     assert_not_empty uuids
 739     assert_empty Container.readable_by(users(:active)).where(uuid: uuids)
 740     assert_not_empty Container.readable_by(users(:admin)).where(uuid: uuids)
 741     assert_equal uuids.count, Container.readable_by(users(:admin)).where(uuid: uuids).count
 742   end
 743
 744   test "Container locked cancel" do
 745     set_user_from_auth :active
 746     c, _ = minimal_new
 747     set_user_from_auth :dispatch1
 748     assert c.lock, show_errors(c)
 749     assert c.update_attributes(state: Container::Cancelled), show_errors(c)
 750     check_no_change_from_cancelled c
 751   end
 752
 753   test "Container locked with non-expiring token" do
 754     Rails.configuration.API.TokenMaxLifetime = 1.hour
 755     set_user_from_auth :active
 756     c, _ = minimal_new
 757     set_user_from_auth :dispatch1
 758     assert c.lock, show_errors(c)
 759     refute c.auth.nil?
 760     assert c.auth.expires_at.nil?
 761     assert c.auth.user_id == User.find_by_uuid(users(:active).uuid).id
 762   end
 763
 764   test "Container locked cancel with log" do
 765     set_user_from_auth :active
 766     c, _ = minimal_new
 767     set_user_from_auth :dispatch1
 768     assert c.lock, show_errors(c)
 769     assert c.update_attributes(
 770              state: Container::Cancelled,
 771              log: collections(:real_log_collection).portable_data_hash,
 772            ), show_errors(c)
 773     check_no_change_from_cancelled c
 774   end
 775
 776   test "Container running cancel" do
 777     set_user_from_auth :active
 778     c, _ = minimal_new
 779     set_user_from_auth :dispatch1
 780     c.lock
 781     c.update_attributes! state: Container::Running
 782     c.update_attributes! state: Container::Cancelled
 783     check_no_change_from_cancelled c
 784   end
 785
 786   test "Container create forbidden for non-admin" do
 787     set_user_from_auth :active_trustedclient
 788     c = Container.new DEFAULT_ATTRS
 789     c.environment = {}
 790     c.mounts = {"BAR" => "FOO"}
 791     c.output_path = "/tmp"
 792     c.priority = 1
 793     c.runtime_constraints = {}
 794     assert_raises(ArvadosModel::PermissionDeniedError) do
 795       c.save!
 796     end
 797   end
 798
 799   [
 800     [Container::Queued, {state: Container::Locked}],
 801     [Container::Queued, {state: Container::Running}],
 802     [Container::Queued, {state: Container::Complete}],
 803     [Container::Queued, {state: Container::Cancelled}],
 804     [Container::Queued, {priority: 123456789}],
 805     [Container::Queued, {runtime_status: {'error' => 'oops'}}],
 806     [Container::Queued, {cwd: '/'}],
 807     [Container::Locked, {state: Container::Running}],
 808     [Container::Locked, {state: Container::Queued}],
 809     [Container::Locked, {priority: 123456789}],
 810     [Container::Locked, {runtime_status: {'error' => 'oops'}}],
 811     [Container::Locked, {cwd: '/'}],
 812     [Container::Running, {state: Container::Complete}],
 813     [Container::Running, {state: Container::Cancelled}],
 814     [Container::Running, {priority: 123456789}],
 815     [Container::Running, {runtime_status: {'error' => 'oops'}}],
 816     [Container::Running, {cwd: '/'}],
 817     [Container::Running, {gateway_address: "172.16.0.1:12345"}],
 818     [Container::Running, {interactive_session_started: true}],
 819     [Container::Complete, {state: Container::Cancelled}],
 820     [Container::Complete, {priority: 123456789}],
 821     [Container::Complete, {runtime_status: {'error' => 'oops'}}],
 822     [Container::Complete, {cwd: '/'}],
 823     [Container::Cancelled, {cwd: '/'}],
 824   ].each do |start_state, updates|
 825     test "Container update #{updates.inspect} when #{start_state} forbidden for non-admin" do
 826       set_user_from_auth :active
 827       c, _ = minimal_new
 828       if start_state != Container::Queued
 829         set_user_from_auth :dispatch1
 830         c.lock
 831         if start_state != Container::Locked
 832           c.update_attributes! state: Container::Running
 833           if start_state != Container::Running
 834             c.update_attributes! state: start_state
 835           end
 836         end
 837       end
 838       assert_equal c.state, start_state
 839       set_user_from_auth :active
 840       assert_raises(ArvadosModel::PermissionDeniedError) do
 841         c.update_attributes! updates
 842       end
 843     end
 844   end
 845
 846   test "Container only set exit code on complete" do
 847     set_user_from_auth :active
 848     c, _ = minimal_new
 849     set_user_from_auth :dispatch1
 850     c.lock
 851     c.update_attributes! state: Container::Running
 852
 853     check_illegal_updates c, [{exit_code: 1},
 854                               {exit_code: 1, state: Container::Cancelled}]
 855
 856     assert c.update_attributes(exit_code: 1, state: Container::Complete)
 857   end
 858
 859   test "locked_by_uuid can update log when locked/running, and output when running" do
 860     set_user_from_auth :active
 861     logcoll = collections(:real_log_collection)
 862     c, cr1 = minimal_new
 863     cr2 = ContainerRequest.new(DEFAULT_ATTRS)
 864     cr2.state = ContainerRequest::Committed
 865     act_as_user users(:active) do
 866       cr2.save!
 867     end
 868     assert_equal cr1.container_uuid, cr2.container_uuid
 869
 870     logpdh_time1 = logcoll.portable_data_hash
 871
 872     set_user_from_auth :dispatch1
 873     c.lock
 874     assert_equal c.locked_by_uuid, Thread.current[:api_client_authorization].uuid
 875     c.update_attributes!(log: logpdh_time1)
 876     c.update_attributes!(state: Container::Running)
 877     cr1.reload
 878     cr2.reload
 879     cr1log_uuid = cr1.log_uuid
 880     cr2log_uuid = cr2.log_uuid
 881     assert_not_nil cr1log_uuid
 882     assert_not_nil cr2log_uuid
 883     assert_not_equal logcoll.uuid, cr1log_uuid
 884     assert_not_equal logcoll.uuid, cr2log_uuid
 885     assert_not_equal cr1log_uuid, cr2log_uuid
 886
 887     logcoll.update_attributes!(manifest_text: logcoll.manifest_text + ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n")
 888     logpdh_time2 = logcoll.portable_data_hash
 889
 890     assert c.update_attributes(output: collections(:collection_owned_by_active).portable_data_hash)
 891     assert c.update_attributes(log: logpdh_time2)
 892     assert c.update_attributes(state: Container::Complete, log: logcoll.portable_data_hash)
 893     c.reload
 894     assert_equal collections(:collection_owned_by_active).portable_data_hash, c.output
 895     assert_equal logpdh_time2, c.log
 896     refute c.update_attributes(output: nil)
 897     refute c.update_attributes(log: nil)
 898     cr1.reload
 899     cr2.reload
 900     assert_equal cr1log_uuid, cr1.log_uuid
 901     assert_equal cr2log_uuid, cr2.log_uuid
 902     assert_equal 1, Collection.where(uuid: [cr1log_uuid, cr2log_uuid]).to_a.collect(&:portable_data_hash).uniq.length
 903     assert_equal ". acbd18db4cc2f85cedef654fccc4a4d8+3 cdd549ae79fe6640fa3d5c6261d8303c+195 0:3:foo.txt 3:195:zzzzz-8i9sb-0vsrcqi7whchuil.log.txt
 904 ./log\\040for\\040container\\040#{cr1.container_uuid} acbd18db4cc2f85cedef654fccc4a4d8+3 cdd549ae79fe6640fa3d5c6261d8303c+195 0:3:foo.txt 3:195:zzzzz-8i9sb-0vsrcqi7whchuil.log.txt
 905 ", Collection.find_by_uuid(cr1log_uuid).manifest_text
 906   end
 907
 908   ["auth_uuid", "runtime_token"].each do |tok|
 909     test "#{tok} can set output, progress, runtime_status, state on running container -- but not log" do
 910       if tok == "runtime_token"
 911         set_user_from_auth :spectator
 912         c, _ = minimal_new(container_image: "9ae44d5792468c58bcf85ce7353c7027+124",
 913                            runtime_token: api_client_authorizations(:active).token)
 914       else
 915         set_user_from_auth :active
 916         c, _ = minimal_new
 917       end
 918       set_user_from_auth :dispatch1
 919       c.lock
 920       c.update_attributes! state: Container::Running
 921
 922       if tok == "runtime_token"
 923         auth = ApiClientAuthorization.validate(token: c.runtime_token)
 924         Thread.current[:api_client_authorization] = auth
 925         Thread.current[:api_client] = auth.api_client
 926         Thread.current[:token] = auth.token
 927         Thread.current[:user] = auth.user
 928       else
 929         auth = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
 930         Thread.current[:api_client_authorization] = auth
 931         Thread.current[:api_client] = auth.api_client
 932         Thread.current[:token] = auth.token
 933         Thread.current[:user] = auth.user
 934       end
 935
 936       assert c.update_attributes(output: collections(:collection_owned_by_active).portable_data_hash)
 937       assert c.update_attributes(runtime_status: {'warning' => 'something happened'})
 938       assert c.update_attributes(progress: 0.5)
 939       refute c.update_attributes(log: collections(:real_log_collection).portable_data_hash)
 940       c.reload
 941       assert c.update_attributes(state: Container::Complete, exit_code: 0)
 942     end
 943   end
 944
 945   test "not allowed to set output that is not readable by current user" do
 946     set_user_from_auth :active
 947     c, _ = minimal_new
 948     set_user_from_auth :dispatch1
 949     c.lock
 950     c.update_attributes! state: Container::Running
 951
 952     Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
 953     Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
 954
 955     assert_raises ActiveRecord::RecordInvalid do
 956       c.update_attributes! output: collections(:collection_not_readable_by_active).portable_data_hash
 957     end
 958   end
 959
 960   test "other token cannot set output on running container" do
 961     set_user_from_auth :active
 962     c, _ = minimal_new
 963     set_user_from_auth :dispatch1
 964     c.lock
 965     c.update_attributes! state: Container::Running
 966
 967     set_user_from_auth :running_to_be_deleted_container_auth
 968     assert_raises(ArvadosModel::PermissionDeniedError) do
 969       c.update_attributes(output: collections(:foo_file).portable_data_hash)
 970     end
 971   end
 972
 973   test "can set trashed output on running container" do
 974     set_user_from_auth :active
 975     c, _ = minimal_new
 976     set_user_from_auth :dispatch1
 977     c.lock
 978     c.update_attributes! state: Container::Running
 979
 980     output = Collection.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jk')
 981
 982     assert output.is_trashed
 983     assert c.update_attributes output: output.portable_data_hash
 984     assert c.update_attributes! state: Container::Complete
 985   end
 986
 987   test "not allowed to set trashed output that is not readable by current user" do
 988     set_user_from_auth :active
 989     c, _ = minimal_new
 990     set_user_from_auth :dispatch1
 991     c.lock
 992     c.update_attributes! state: Container::Running
 993
 994     output = Collection.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jr')
 995
 996     Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
 997     Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
 998
 999     assert_raises ActiveRecord::RecordInvalid do
1000       c.update_attributes! output: output.portable_data_hash
1001     end
1002   end
1003
1004   test "user cannot delete" do
1005     set_user_from_auth :active
1006     c, _ = minimal_new
1007     assert_raises ArvadosModel::PermissionDeniedError do
1008       c.destroy
1009     end
1010     assert Container.find_by_uuid(c.uuid)
1011   end
1012
1013   [
1014     {state: Container::Complete, exit_code: 0, output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'},
1015     {state: Container::Cancelled},
1016   ].each do |final_attrs|
1017     test "secret_mounts and runtime_token are null after container is #{final_attrs[:state]}" do
1018       set_user_from_auth :active
1019       c, cr = minimal_new(secret_mounts: {'/secret' => {'kind' => 'text', 'content' => 'foo'}},
1020                           container_count_max: 1, runtime_token: api_client_authorizations(:active).token)
1021       set_user_from_auth :dispatch1
1022       c.lock
1023       c.update_attributes!(state: Container::Running)
1024       c.reload
1025       assert c.secret_mounts.has_key?('/secret')
1026       assert_equal api_client_authorizations(:active).token, c.runtime_token
1027
1028       c.update_attributes!(final_attrs)
1029       c.reload
1030       assert_equal({}, c.secret_mounts)
1031       assert_nil c.runtime_token
1032       cr.reload
1033       assert_equal({}, cr.secret_mounts)
1034       assert_nil cr.runtime_token
1035       assert_no_secrets_logged
1036     end
1037   end
1038 end