services/api/test/unit/container_test.rb

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 require 'test_helper'
   6 require 'helpers/container_test_helper'
   7
   8 class ContainerTest < ActiveSupport::TestCase
   9   include DbCurrentTime
  10   include ContainerTestHelper
  11
  12   DEFAULT_ATTRS = {
  13     command: ['echo', 'foo'],
  14     container_image: 'fa3c1a9cb6783f85f2ecda037e07b8c3+167',
  15     output_path: '/tmp',
  16     priority: 1,
  17     runtime_constraints: {"vcpus" => 1, "ram" => 1, "cuda" => {"device_count":0, "driver_version": "", "hardware_capability": ""}},
  18   }
  19
  20   REUSABLE_COMMON_ATTRS = {
  21     container_image: "9ae44d5792468c58bcf85ce7353c7027+124",
  22     cwd: "test",
  23     command: ["echo", "hello"],
  24     output_path: "test",
  25     output_glob: [],
  26     runtime_constraints: {
  27       "API" => false,
  28       "keep_cache_disk" => 0,
  29       "keep_cache_ram" => 0,
  30       "ram" => 12000000000,
  31       "vcpus" => 4
  32     },
  33     mounts: {
  34       "test" => {"kind" => "json"},
  35     },
  36     environment: {
  37       "var" => "val",
  38     },
  39     secret_mounts: {},
  40     runtime_user_uuid: "zzzzz-tpzed-xurymjxw79nv3jz",
  41     runtime_auth_scopes: ["all"],
  42     scheduling_parameters: {},
  43   }
  44
  45   REUSABLE_ATTRS_SLIM = {
  46     command: ["echo", "slim"],
  47     container_image: "9ae44d5792468c58bcf85ce7353c7027+124",
  48     cwd: "test",
  49     environment: {},
  50     mounts: {},
  51     output_path: "test",
  52     output_glob: [],
  53     runtime_auth_scopes: ["all"],
  54     runtime_constraints: {
  55       "API" => false,
  56       "keep_cache_disk" => 0,
  57       "keep_cache_ram" => 0,
  58       "ram" => 8 << 30,
  59       "vcpus" => 4
  60     },
  61     runtime_user_uuid: "zzzzz-tpzed-xurymjxw79nv3jz",
  62     secret_mounts: {},
  63     scheduling_parameters: {},
  64   }
  65
  66   def request_only attrs
  67     attrs.reject {|k| [:runtime_user_uuid, :runtime_auth_scopes].include? k}
  68   end
  69
  70   def minimal_new attrs={}
  71     cr = ContainerRequest.new request_only(DEFAULT_ATTRS.merge(attrs))
  72     cr.state = ContainerRequest::Committed
  73     cr.save!
  74     c = Container.find_by_uuid cr.container_uuid
  75     assert_not_nil c
  76     return c, cr
  77   end
  78
  79   def check_illegal_updates c, bad_updates
  80     bad_updates.each do |u|
  81       refute c.update(u), u.inspect
  82       refute c.valid?, u.inspect
  83       c.reload
  84     end
  85   end
  86
  87   def check_illegal_modify c
  88     check_illegal_updates c, [{command: ["echo", "bar"]},
  89                               {container_image: "arvados/apitestfixture:june10"},
  90                               {cwd: "/tmp2"},
  91                               {environment: {"FOO" => "BAR"}},
  92                               {mounts: {"FOO" => "BAR"}},
  93                               {output_path: "/tmp3"},
  94                               {locked_by_uuid: "zzzzz-gj3su-027z32aux8dg2s1"},
  95                               {auth_uuid: "zzzzz-gj3su-017z32aux8dg2s1"},
  96                               {runtime_constraints: {"FOO" => "BAR"}}]
  97   end
  98
  99   def check_bogus_states c
 100     check_illegal_updates c, [{state: nil},
 101                               {state: "Flubber"}]
 102   end
 103
 104   def check_no_change_from_cancelled c
 105     check_illegal_modify c
 106     check_bogus_states c
 107     check_illegal_updates c, [{ priority: 3 },
 108                               { state: Container::Queued },
 109                               { state: Container::Locked },
 110                               { state: Container::Running },
 111                               { state: Container::Complete }]
 112   end
 113
 114   test "Container create" do
 115     act_as_system_user do
 116       c, _ = minimal_new(environment: {},
 117                       mounts: {"BAR" => {"kind" => "FOO"}},
 118                       output_path: "/tmp",
 119                       priority: 1,
 120                       runtime_constraints: {"vcpus" => 1, "ram" => 1})
 121
 122       check_illegal_modify c
 123       check_bogus_states c
 124
 125       c.reload
 126       c.priority = 2
 127       c.save!
 128     end
 129   end
 130
 131   test "Container valid priority" do
 132     act_as_system_user do
 133       c, _ = minimal_new(environment: {},
 134                       mounts: {"BAR" => {"kind" => "FOO"}},
 135                       output_path: "/tmp",
 136                       priority: 1,
 137                       runtime_constraints: {"vcpus" => 1, "ram" => 1})
 138
 139       assert_raises(ActiveRecord::RecordInvalid) do
 140         c.priority = -1
 141         c.save!
 142       end
 143
 144       c.priority = 0
 145       c.save!
 146
 147       c.priority = 1
 148       c.save!
 149
 150       c.priority = 500
 151       c.save!
 152
 153       c.priority = 999
 154       c.save!
 155
 156       c.priority = 1000
 157       c.save!
 158
 159       c.priority = 1000 << 50
 160       c.save!
 161     end
 162   end
 163
 164   test "Container runtime_status data types" do
 165     set_user_from_auth :active
 166     attrs = {
 167       environment: {},
 168       mounts: {"BAR" => {"kind" => "FOO"}},
 169       output_path: "/tmp",
 170       priority: 1,
 171       runtime_constraints: {"vcpus" => 1, "ram" => 1}
 172     }
 173     c, _ = minimal_new(attrs)
 174     assert_equal c.runtime_status, {}
 175     assert_equal Container::Queued, c.state
 176
 177     set_user_from_auth :dispatch1
 178     c.update! state: Container::Locked
 179     c.update! state: Container::Running
 180
 181     [
 182       'error', 'errorDetail', 'warning', 'warningDetail', 'activity'
 183     ].each do |k|
 184       # String type is allowed
 185       string_val = 'A string is accepted'
 186       c.update! runtime_status: {k => string_val}
 187       assert_equal string_val, c.runtime_status[k]
 188
 189       # Other types aren't allowed
 190       [
 191         42, false, [], {}, nil
 192       ].each do |unallowed_val|
 193         assert_raises ActiveRecord::RecordInvalid do
 194           c.update! runtime_status: {k => unallowed_val}
 195         end
 196       end
 197     end
 198   end
 199
 200   test "Container runtime_status updates" do
 201     set_user_from_auth :active
 202     attrs = {
 203       environment: {},
 204       mounts: {"BAR" => {"kind" => "FOO"}},
 205       output_path: "/tmp",
 206       priority: 1,
 207       runtime_constraints: {"vcpus" => 1, "ram" => 1}
 208     }
 209     c1, _ = minimal_new(attrs)
 210     assert_equal c1.runtime_status, {}
 211
 212     assert_equal Container::Queued, c1.state
 213     assert_raises ArvadosModel::PermissionDeniedError do
 214       c1.update! runtime_status: {'error' => 'Oops!'}
 215     end
 216
 217     set_user_from_auth :dispatch1
 218
 219     # Allow updates when state = Locked
 220     c1.update! state: Container::Locked
 221     c1.update! runtime_status: {'error' => 'Oops!'}
 222     assert c1.runtime_status.key? 'error'
 223
 224     # Reset when transitioning from Locked to Queued
 225     c1.update! state: Container::Queued
 226     assert_equal c1.runtime_status, {}
 227
 228     # Allow updates when state = Running
 229     c1.update! state: Container::Locked
 230     c1.update! state: Container::Running
 231     c1.update! runtime_status: {'error' => 'Oops!'}
 232     assert c1.runtime_status.key? 'error'
 233
 234     # Don't allow updates on other states
 235     c1.update! state: Container::Complete
 236     assert_raises ActiveRecord::RecordInvalid do
 237       c1.update! runtime_status: {'error' => 'Some other error'}
 238     end
 239
 240     set_user_from_auth :active
 241     c2, _ = minimal_new(attrs)
 242     assert_equal c2.runtime_status, {}
 243     set_user_from_auth :dispatch1
 244     c2.update! state: Container::Locked
 245     c2.update! state: Container::Running
 246     c2.update! state: Container::Cancelled
 247     assert_raises ActiveRecord::RecordInvalid do
 248       c2.update! runtime_status: {'error' => 'Oops!'}
 249     end
 250   end
 251
 252   test "Container serialized hash attributes sorted before save" do
 253     set_user_from_auth :active
 254     env = {"C" => "3", "B" => "2", "A" => "1"}
 255     m = {"F" => {"kind" => "3"}, "E" => {"kind" => "2"}, "D" => {"kind" => "1"}}
 256     rc = {"vcpus" => 1, "ram" => 1, "keep_cache_ram" => 1, "keep_cache_disk" => 0, "API" => true, "cuda" => {"device_count":0, "driver_version": "", "hardware_capability": ""}}
 257     c, _ = minimal_new(environment: env, mounts: m, runtime_constraints: rc)
 258     c.reload
 259     assert_equal Container.deep_sort_hash(env).to_json, c.environment.to_json
 260     assert_equal Container.deep_sort_hash(m).to_json, c.mounts.to_json
 261     assert_equal Container.deep_sort_hash(rc).to_json, c.runtime_constraints.to_json
 262   end
 263
 264   test 'deep_sort_hash on array of hashes' do
 265     a = {'z' => [[{'a' => 'a', 'b' => 'b'}]]}
 266     b = {'z' => [[{'b' => 'b', 'a' => 'a'}]]}
 267     assert_equal Container.deep_sort_hash(a).to_json, Container.deep_sort_hash(b).to_json
 268   end
 269
 270   test "find_reusable method should select higher priority queued container" do
 271         Rails.configuration.Containers.LogReuseDecisions = true
 272     set_user_from_auth :active
 273     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment:{"var" => "queued"}})
 274     c_low_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:1}))
 275     c_high_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:2}))
 276     assert_not_equal c_low_priority.uuid, c_high_priority.uuid
 277     assert_equal Container::Queued, c_low_priority.state
 278     assert_equal Container::Queued, c_high_priority.state
 279     reused = Container.find_reusable(common_attrs)
 280     assert_not_nil reused
 281     assert_equal reused.uuid, c_high_priority.uuid
 282   end
 283
 284   test "find_reusable method should select latest completed container" do
 285     set_user_from_auth :active
 286     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "complete"}})
 287     completed_attrs = {
 288       state: Container::Complete,
 289       exit_code: 0,
 290       log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 291       output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
 292     }
 293
 294     c_older, _ = minimal_new(common_attrs.merge({use_existing: false}))
 295     c_recent, _ = minimal_new(common_attrs.merge({use_existing: false}))
 296     assert_not_equal c_older.uuid, c_recent.uuid
 297
 298     set_user_from_auth :dispatch1
 299     c_older.update!({state: Container::Locked})
 300     c_older.update!({state: Container::Running})
 301     c_older.update!(completed_attrs)
 302
 303     c_recent.update!({state: Container::Locked})
 304     c_recent.update!({state: Container::Running})
 305     c_recent.update!(completed_attrs)
 306
 307     reused = Container.find_reusable(common_attrs)
 308     assert_not_nil reused
 309     assert_equal reused.uuid, c_older.uuid
 310   end
 311
 312   test "find_reusable method should select oldest completed container when inconsistent outputs exist" do
 313     set_user_from_auth :active
 314     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "complete"}, priority: 1})
 315     completed_attrs = {
 316       state: Container::Complete,
 317       exit_code: 0,
 318       log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 319     }
 320
 321     cr = ContainerRequest.new request_only(common_attrs)
 322     cr.use_existing = false
 323     cr.state = ContainerRequest::Committed
 324     cr.save!
 325     c_output1 = Container.where(uuid: cr.container_uuid).first
 326
 327     cr = ContainerRequest.new request_only(common_attrs)
 328     cr.use_existing = false
 329     cr.state = ContainerRequest::Committed
 330     cr.save!
 331     c_output2 = Container.where(uuid: cr.container_uuid).first
 332
 333     assert_not_equal c_output1.uuid, c_output2.uuid
 334
 335     set_user_from_auth :dispatch1
 336
 337     out1 = '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
 338     log1 = collections(:log_collection).portable_data_hash
 339     c_output1.update!({state: Container::Locked})
 340     c_output1.update!({state: Container::Running})
 341     c_output1.update!(completed_attrs.merge({log: log1, output: out1}))
 342
 343     out2 = 'fa7aeb5140e2848d39b416daeef4ffc5+45'
 344     c_output2.update!({state: Container::Locked})
 345     c_output2.update!({state: Container::Running})
 346     c_output2.update!(completed_attrs.merge({log: log1, output: out2}))
 347
 348     set_user_from_auth :active
 349     reused = Container.resolve(ContainerRequest.new(request_only(common_attrs)))
 350     assert_equal c_output1.uuid, reused.uuid
 351   end
 352
 353   test "find_reusable method should select running container by start date" do
 354     set_user_from_auth :active
 355     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running"}})
 356     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
 357     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
 358     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
 359     # Confirm the 3 container UUIDs are different.
 360     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
 361     set_user_from_auth :dispatch1
 362     c_slower.update!({state: Container::Locked})
 363     c_slower.update!({state: Container::Running,
 364                                  progress: 0.1})
 365     c_faster_started_first.update!({state: Container::Locked})
 366     c_faster_started_first.update!({state: Container::Running,
 367                                                progress: 0.15})
 368     c_faster_started_second.update!({state: Container::Locked})
 369     c_faster_started_second.update!({state: Container::Running,
 370                                                 progress: 0.15})
 371     reused = Container.find_reusable(common_attrs)
 372     assert_not_nil reused
 373     # Selected container is the one that started first
 374     assert_equal reused.uuid, c_faster_started_first.uuid
 375   end
 376
 377   test "find_reusable method should select running container by progress" do
 378     set_user_from_auth :active
 379     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running2"}})
 380     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
 381     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
 382     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
 383     # Confirm the 3 container UUIDs are different.
 384     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
 385     set_user_from_auth :dispatch1
 386     c_slower.update!({state: Container::Locked})
 387     c_slower.update!({state: Container::Running,
 388                                  progress: 0.1})
 389     c_faster_started_first.update!({state: Container::Locked})
 390     c_faster_started_first.update!({state: Container::Running,
 391                                                progress: 0.15})
 392     c_faster_started_second.update!({state: Container::Locked})
 393     c_faster_started_second.update!({state: Container::Running,
 394                                                 progress: 0.2})
 395     reused = Container.find_reusable(common_attrs)
 396     assert_not_nil reused
 397     # Selected container is the one with most progress done
 398     assert_equal reused.uuid, c_faster_started_second.uuid
 399   end
 400
 401   test "find_reusable method should select non-failing running container" do
 402     set_user_from_auth :active
 403     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running2"}})
 404     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
 405     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
 406     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
 407     # Confirm the 3 container UUIDs are different.
 408     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
 409     set_user_from_auth :dispatch1
 410     c_slower.update!({state: Container::Locked})
 411     c_slower.update!({state: Container::Running,
 412                                  progress: 0.1})
 413     c_faster_started_first.update!({state: Container::Locked})
 414     c_faster_started_first.update!({state: Container::Running,
 415                                                runtime_status: {'warning' => 'This is not an error'},
 416                                                progress: 0.15})
 417     c_faster_started_second.update!({state: Container::Locked})
 418     assert_equal 0, Container.where("runtime_status->'error' is not null").count
 419     c_faster_started_second.update!({state: Container::Running,
 420                                                 runtime_status: {'error' => 'Something bad happened'},
 421                                                 progress: 0.2})
 422     assert_equal 1, Container.where("runtime_status->'error' is not null").count
 423     reused = Container.find_reusable(common_attrs)
 424     assert_not_nil reused
 425     # Selected the non-failing container even if it's the one with less progress done
 426     assert_equal reused.uuid, c_faster_started_first.uuid
 427   end
 428
 429   test "find_reusable method should select locked container most likely to start sooner" do
 430     set_user_from_auth :active
 431     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "locked"}})
 432     c_low_priority, _ = minimal_new(common_attrs.merge({use_existing: false}))
 433     c_high_priority_older, _ = minimal_new(common_attrs.merge({use_existing: false}))
 434     c_high_priority_newer, _ = minimal_new(common_attrs.merge({use_existing: false}))
 435     # Confirm the 3 container UUIDs are different.
 436     assert_equal 3, [c_low_priority.uuid, c_high_priority_older.uuid, c_high_priority_newer.uuid].uniq.length
 437     set_user_from_auth :dispatch1
 438     c_low_priority.update!({state: Container::Locked,
 439                                        priority: 1})
 440     c_high_priority_older.update!({state: Container::Locked,
 441                                               priority: 2})
 442     c_high_priority_newer.update!({state: Container::Locked,
 443                                               priority: 2})
 444     reused = Container.find_reusable(common_attrs)
 445     assert_not_nil reused
 446     assert_equal reused.uuid, c_high_priority_older.uuid
 447   end
 448
 449   test "find_reusable method should select running over failed container" do
 450     set_user_from_auth :active
 451     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed_vs_running"}})
 452     c_failed, _ = minimal_new(common_attrs.merge({use_existing: false}))
 453     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
 454     assert_not_equal c_failed.uuid, c_running.uuid
 455     set_user_from_auth :dispatch1
 456     c_failed.update!({state: Container::Locked})
 457     c_failed.update!({state: Container::Running})
 458     c_failed.update!({state: Container::Complete,
 459                                  exit_code: 42,
 460                                  log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 461                                  output: 'ea10d51bcf88862dbcc36eb292017dfd+45'})
 462     c_running.update!({state: Container::Locked})
 463     c_running.update!({state: Container::Running,
 464                                   progress: 0.15})
 465     reused = Container.find_reusable(common_attrs)
 466     assert_not_nil reused
 467     assert_equal reused.uuid, c_running.uuid
 468   end
 469
 470   test "find_reusable method should select complete over running container" do
 471     set_user_from_auth :active
 472     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "completed_vs_running"}})
 473     c_completed, _ = minimal_new(common_attrs.merge({use_existing: false}))
 474     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
 475     assert_not_equal c_completed.uuid, c_running.uuid
 476     set_user_from_auth :dispatch1
 477     c_completed.update!({state: Container::Locked})
 478     c_completed.update!({state: Container::Running})
 479     c_completed.update!({state: Container::Complete,
 480                                     exit_code: 0,
 481                                     log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 482                                     output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'})
 483     c_running.update!({state: Container::Locked})
 484     c_running.update!({state: Container::Running,
 485                                   progress: 0.15})
 486     reused = Container.find_reusable(common_attrs)
 487     assert_not_nil reused
 488     assert_equal c_completed.uuid, reused.uuid
 489   end
 490
 491   test "find_reusable method should select running over locked container" do
 492     set_user_from_auth :active
 493     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}})
 494     c_locked, _ = minimal_new(common_attrs.merge({use_existing: false}))
 495     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
 496     assert_not_equal c_running.uuid, c_locked.uuid
 497     set_user_from_auth :dispatch1
 498     c_locked.update!({state: Container::Locked})
 499     c_running.update!({state: Container::Locked})
 500     c_running.update!({state: Container::Running,
 501                                   progress: 0.15})
 502     reused = Container.find_reusable(common_attrs)
 503     assert_not_nil reused
 504     assert_equal reused.uuid, c_running.uuid
 505   end
 506
 507   test "find_reusable method should select locked over queued container" do
 508     set_user_from_auth :active
 509     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}})
 510     c_locked, _ = minimal_new(common_attrs.merge({use_existing: false}))
 511     c_queued, _ = minimal_new(common_attrs.merge({use_existing: false}))
 512     assert_not_equal c_queued.uuid, c_locked.uuid
 513     set_user_from_auth :dispatch1
 514     c_locked.update!({state: Container::Locked})
 515     reused = Container.find_reusable(common_attrs)
 516     assert_not_nil reused
 517     assert_equal reused.uuid, c_locked.uuid
 518   end
 519
 520   test "find_reusable method should not select failed container" do
 521     set_user_from_auth :active
 522     attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed"}})
 523     c, _ = minimal_new(attrs)
 524     set_user_from_auth :dispatch1
 525     c.update!({state: Container::Locked})
 526     c.update!({state: Container::Running})
 527     c.update!({state: Container::Complete,
 528                           exit_code: 33})
 529     reused = Container.find_reusable(attrs)
 530     assert_nil reused
 531   end
 532
 533   [[false, false, true],
 534    [false, true, true],
 535    [true, false, false],
 536    [true, true, true]
 537   ].each do |c1_preemptible, c2_preemptible, should_reuse|
 538     [[Container::Queued, 1],
 539      [Container::Locked, 1],
 540      [Container::Running, 0],   # not cancelled yet, but obviously will be soon
 541     ].each do |c1_state, c1_priority|
 542       test "find_reusable for #{c2_preemptible ? '' : 'non-'}preemptible req should #{should_reuse ? '' : 'not'} reuse a #{c1_state} #{c1_preemptible ? '' : 'non-'}preemptible container with priority #{c1_priority}" do
 543         configure_preemptible_instance_type
 544         set_user_from_auth :active
 545         c1_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"test" => name, "state" => c1_state}, scheduling_parameters: {"preemptible" => c1_preemptible}})
 546         c1, _ = minimal_new(c1_attrs)
 547         set_user_from_auth :dispatch1
 548         c1.update!({state: Container::Locked}) if c1_state != Container::Queued
 549         c1.update!({state: Container::Running, priority: c1_priority}) if c1_state == Container::Running
 550         c2_attrs = c1_attrs.merge({scheduling_parameters: {"preemptible" => c2_preemptible}})
 551         reused = Container.find_reusable(c2_attrs)
 552         if should_reuse && c1_priority > 0
 553           assert_not_nil reused
 554         else
 555           assert_nil reused
 556         end
 557       end
 558     end
 559   end
 560
 561   test "find_reusable with logging disabled" do
 562     set_user_from_auth :active
 563     Rails.logger.expects(:info).never
 564     Container.find_reusable(REUSABLE_COMMON_ATTRS)
 565   end
 566
 567   test "find_reusable with logging enabled" do
 568     set_user_from_auth :active
 569     Rails.configuration.Containers.LogReuseDecisions = true
 570     Rails.logger.expects(:info).at_least(3)
 571     Container.find_reusable(REUSABLE_COMMON_ATTRS)
 572   end
 573
 574   def runtime_token_attr tok
 575     auth = api_client_authorizations(tok)
 576     {runtime_user_uuid: User.find_by_id(auth.user_id).uuid,
 577      runtime_auth_scopes: auth.scopes,
 578      runtime_token: auth.token}
 579   end
 580
 581   test "find_reusable method with same runtime_token" do
 582     set_user_from_auth :active
 583     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 584     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:container_runtime_token).token}))
 585     assert_equal Container::Queued, c1.state
 586     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 587     assert_not_nil reused
 588     assert_equal reused.uuid, c1.uuid
 589   end
 590
 591   test "find_reusable method with different runtime_token, same user" do
 592     set_user_from_auth :active
 593     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 594     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:crt_user).token}))
 595     assert_equal Container::Queued, c1.state
 596     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 597     assert_not_nil reused
 598     assert_equal reused.uuid, c1.uuid
 599   end
 600
 601   test "find_reusable method with nil runtime_token, then runtime_token with same user" do
 602     set_user_from_auth :crt_user
 603     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 604     c1, _ = minimal_new(common_attrs)
 605     assert_equal Container::Queued, c1.state
 606     assert_equal users(:container_runtime_token_user).uuid, c1.runtime_user_uuid
 607     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 608     assert_not_nil reused
 609     assert_equal reused.uuid, c1.uuid
 610   end
 611
 612   test "find_reusable method with different runtime_token, different user" do
 613     set_user_from_auth :crt_user
 614     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 615     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:active).token}))
 616     assert_equal Container::Queued, c1.state
 617     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 618     # See #14584
 619     assert_not_nil reused
 620     assert_equal c1.uuid, reused.uuid
 621   end
 622
 623   test "find_reusable method with nil runtime_token, then runtime_token with different user" do
 624     set_user_from_auth :active
 625     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 626     c1, _ = minimal_new(common_attrs.merge({runtime_token: nil}))
 627     assert_equal Container::Queued, c1.state
 628     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 629     # See #14584
 630     assert_not_nil reused
 631     assert_equal c1.uuid, reused.uuid
 632   end
 633
 634   test "find_reusable method with different runtime_token, different scope, same user" do
 635     set_user_from_auth :active
 636     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 637     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:runtime_token_limited_scope).token}))
 638     assert_equal Container::Queued, c1.state
 639     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 640     # See #14584
 641     assert_not_nil reused
 642     assert_equal c1.uuid, reused.uuid
 643   end
 644
 645   test "find_reusable method with cuda" do
 646     set_user_from_auth :active
 647     # No cuda
 648     no_cuda_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"},
 649                                                 runtime_constraints: {"vcpus" => 1, "ram" => 1, "keep_cache_disk"=>0, "keep_cache_ram"=>268435456, "API" => false,
 650                                                                       "cuda" => {"device_count":0, "driver_version": "", "hardware_capability": ""}},})
 651     c1, _ = minimal_new(no_cuda_attrs)
 652     assert_equal Container::Queued, c1.state
 653
 654     # has cuda
 655     cuda_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"},
 656                                                 runtime_constraints: {"vcpus" => 1, "ram" => 1, "keep_cache_disk"=>0, "keep_cache_ram"=>268435456, "API" => false,
 657                                                                       "cuda" => {"device_count":1, "driver_version": "11.0", "hardware_capability": "9.0"}},})
 658     c2, _ = minimal_new(cuda_attrs)
 659     assert_equal Container::Queued, c2.state
 660
 661     # should find the no cuda one
 662     reused = Container.find_reusable(no_cuda_attrs)
 663     assert_not_nil reused
 664     assert_equal reused.uuid, c1.uuid
 665
 666     # should find the cuda one
 667     reused = Container.find_reusable(cuda_attrs)
 668     assert_not_nil reused
 669     assert_equal reused.uuid, c2.uuid
 670   end
 671
 672   test "Container running" do
 673     set_user_from_auth :active
 674     c, _ = minimal_new priority: 1
 675
 676     set_user_from_auth :dispatch1
 677     check_illegal_updates c, [{state: Container::Running},
 678                               {state: Container::Complete}]
 679
 680     c.lock
 681     c.update! state: Container::Running
 682
 683     check_illegal_modify c
 684     check_bogus_states c
 685
 686     check_illegal_updates c, [{state: Container::Queued}]
 687     c.reload
 688
 689     c.update! priority: 3
 690   end
 691
 692   test "Lock and unlock" do
 693     set_user_from_auth :active
 694     c, cr = minimal_new priority: 0
 695
 696     set_user_from_auth :dispatch1
 697     assert_equal Container::Queued, c.state
 698
 699     assert_raise(ArvadosModel::LockFailedError) do
 700       # "no priority"
 701       c.lock
 702     end
 703     c.reload
 704     assert cr.update priority: 1
 705
 706     refute c.update(state: Container::Running), "not locked"
 707     c.reload
 708     refute c.update(state: Container::Complete), "not locked"
 709     c.reload
 710
 711     assert c.lock, show_errors(c)
 712     assert c.locked_by_uuid
 713     assert c.auth_uuid
 714
 715     assert_raise(ArvadosModel::LockFailedError) {c.lock}
 716     c.reload
 717
 718     assert c.unlock, show_errors(c)
 719     refute c.locked_by_uuid
 720     refute c.auth_uuid
 721
 722     refute c.update(state: Container::Running), "not locked"
 723     c.reload
 724     refute c.locked_by_uuid
 725     refute c.auth_uuid
 726
 727     assert c.lock, show_errors(c)
 728     assert c.update(state: Container::Running), show_errors(c)
 729     assert c.locked_by_uuid
 730     assert c.auth_uuid
 731
 732     auth_uuid_was = c.auth_uuid
 733
 734     assert_raise(ArvadosModel::LockFailedError) do
 735       # Running to Locked is not allowed
 736       c.lock
 737     end
 738     c.reload
 739     assert_raise(ArvadosModel::InvalidStateTransitionError) do
 740       # Running to Queued is not allowed
 741       c.unlock
 742     end
 743     c.reload
 744
 745     assert c.update(state: Container::Complete), show_errors(c)
 746     refute c.locked_by_uuid
 747     refute c.auth_uuid
 748
 749     auth_exp = ApiClientAuthorization.find_by_uuid(auth_uuid_was).expires_at
 750     assert_operator auth_exp, :<, db_current_time
 751
 752     assert_nil ApiClientAuthorization.validate(token: ApiClientAuthorization.find_by_uuid(auth_uuid_was).token)
 753   end
 754
 755   test "Exceed maximum lock-unlock cycles" do
 756     Rails.configuration.Containers.MaxDispatchAttempts = 3
 757
 758     set_user_from_auth :active
 759     c, cr = minimal_new
 760
 761     set_user_from_auth :dispatch1
 762     assert_equal Container::Queued, c.state
 763     assert_equal 0, c.lock_count
 764
 765     c.lock
 766     c.reload
 767     assert_equal 1, c.lock_count
 768     assert_equal Container::Locked, c.state
 769
 770     c.unlock
 771     c.reload
 772     assert_equal 1, c.lock_count
 773     assert_equal Container::Queued, c.state
 774
 775     c.lock
 776     c.reload
 777     assert_equal 2, c.lock_count
 778     assert_equal Container::Locked, c.state
 779
 780     c.unlock
 781     c.reload
 782     assert_equal 2, c.lock_count
 783     assert_equal Container::Queued, c.state
 784
 785     c.lock
 786     c.reload
 787     assert_equal 3, c.lock_count
 788     assert_equal Container::Locked, c.state
 789
 790     c.unlock
 791     c.reload
 792     assert_equal 3, c.lock_count
 793     assert_equal Container::Cancelled, c.state
 794
 795     assert_raise(ArvadosModel::LockFailedError) do
 796       # Cancelled to Locked is not allowed
 797       c.lock
 798     end
 799   end
 800
 801   test "Container queued cancel" do
 802     set_user_from_auth :active
 803     c, cr = minimal_new({container_count_max: 1})
 804     set_user_from_auth :dispatch1
 805     assert c.update(state: Container::Cancelled), show_errors(c)
 806     check_no_change_from_cancelled c
 807     cr.reload
 808     assert_equal ContainerRequest::Final, cr.state
 809   end
 810
 811   test "Container queued count" do
 812     assert_equal 1, Container.readable_by(users(:active)).where(state: "Queued").count
 813   end
 814
 815   test "Containers with no matching request are readable by admin" do
 816     uuids = Container.includes('container_requests').where(container_requests: {uuid: nil}).collect(&:uuid)
 817     assert_not_empty uuids
 818     assert_empty Container.readable_by(users(:active)).where(uuid: uuids)
 819     assert_not_empty Container.readable_by(users(:admin)).where(uuid: uuids)
 820     assert_equal uuids.count, Container.readable_by(users(:admin)).where(uuid: uuids).count
 821   end
 822
 823   test "Container locked cancel" do
 824     set_user_from_auth :active
 825     c, _ = minimal_new
 826     set_user_from_auth :dispatch1
 827     assert c.lock, show_errors(c)
 828     assert c.update(state: Container::Cancelled), show_errors(c)
 829     check_no_change_from_cancelled c
 830   end
 831
 832   test "Container locked with non-expiring token" do
 833     Rails.configuration.API.TokenMaxLifetime = 1.hour
 834     set_user_from_auth :active
 835     c, _ = minimal_new
 836     set_user_from_auth :dispatch1
 837     assert c.lock, show_errors(c)
 838     refute c.auth.nil?
 839     assert c.auth.expires_at.nil?
 840     assert c.auth.user_id == User.find_by_uuid(users(:active).uuid).id
 841   end
 842
 843   test "Container locked cancel with log" do
 844     set_user_from_auth :active
 845     c, _ = minimal_new
 846     set_user_from_auth :dispatch1
 847     assert c.lock, show_errors(c)
 848     assert c.update(
 849              state: Container::Cancelled,
 850              log: collections(:log_collection).portable_data_hash,
 851            ), show_errors(c)
 852     check_no_change_from_cancelled c
 853   end
 854
 855   test "Container running cancel" do
 856     set_user_from_auth :active
 857     c, _ = minimal_new
 858     set_user_from_auth :dispatch1
 859     c.lock
 860     c.update! state: Container::Running
 861     c.update! state: Container::Cancelled
 862     check_no_change_from_cancelled c
 863   end
 864
 865   test "Container create forbidden for non-admin" do
 866     set_user_from_auth :active_trustedclient
 867     c = Container.new DEFAULT_ATTRS
 868     c.environment = {}
 869     c.mounts = {"BAR" => "FOO"}
 870     c.output_path = "/tmp"
 871     c.priority = 1
 872     c.runtime_constraints = {}
 873     assert_raises(ArvadosModel::PermissionDeniedError) do
 874       c.save!
 875     end
 876   end
 877
 878   [
 879     [Container::Queued, {state: Container::Locked}],
 880     [Container::Queued, {state: Container::Running}],
 881     [Container::Queued, {state: Container::Complete}],
 882     [Container::Queued, {state: Container::Cancelled}],
 883     [Container::Queued, {priority: 123456789}],
 884     [Container::Queued, {runtime_status: {'error' => 'oops'}}],
 885     [Container::Queued, {cwd: '/'}],
 886     [Container::Locked, {state: Container::Running}],
 887     [Container::Locked, {state: Container::Queued}],
 888     [Container::Locked, {priority: 123456789}],
 889     [Container::Locked, {runtime_status: {'error' => 'oops'}}],
 890     [Container::Locked, {cwd: '/'}],
 891     [Container::Running, {state: Container::Complete}],
 892     [Container::Running, {state: Container::Cancelled}],
 893     [Container::Running, {priority: 123456789}],
 894     [Container::Running, {runtime_status: {'error' => 'oops'}}],
 895     [Container::Running, {cwd: '/'}],
 896     [Container::Running, {gateway_address: "172.16.0.1:12345"}],
 897     [Container::Running, {interactive_session_started: true}],
 898     [Container::Complete, {state: Container::Cancelled}],
 899     [Container::Complete, {priority: 123456789}],
 900     [Container::Complete, {runtime_status: {'error' => 'oops'}}],
 901     [Container::Complete, {cwd: '/'}],
 902     [Container::Cancelled, {cwd: '/'}],
 903   ].each do |start_state, updates|
 904     test "Container update #{updates.inspect} when #{start_state} forbidden for non-admin" do
 905       set_user_from_auth :active
 906       c, _ = minimal_new
 907       if start_state != Container::Queued
 908         set_user_from_auth :dispatch1
 909         c.lock
 910         if start_state != Container::Locked
 911           c.update! state: Container::Running
 912           if start_state != Container::Running
 913             c.update! state: start_state
 914           end
 915         end
 916       end
 917       assert_equal c.state, start_state
 918       set_user_from_auth :active
 919       assert_raises(ArvadosModel::PermissionDeniedError) do
 920         c.update! updates
 921       end
 922     end
 923   end
 924
 925   test "can only change exit code while running and at completion" do
 926     set_user_from_auth :active
 927     c, _ = minimal_new
 928     set_user_from_auth :dispatch1
 929     c.lock
 930     check_illegal_updates c, [{exit_code: 1}]
 931     c.update! state: Container::Running
 932     assert c.update(exit_code: 1)
 933     assert c.update(exit_code: 1, state: Container::Complete)
 934   end
 935
 936   test "locked_by_uuid can update log when locked/running, and output when running" do
 937     set_user_from_auth :active
 938     logcoll = collections(:container_log_collection)
 939     c, cr1 = minimal_new
 940     cr2 = ContainerRequest.new(DEFAULT_ATTRS)
 941     cr2.state = ContainerRequest::Committed
 942     act_as_user users(:active) do
 943       cr2.save!
 944     end
 945     assert_equal cr1.container_uuid, cr2.container_uuid
 946
 947     logpdh_time1 = logcoll.portable_data_hash
 948
 949     set_user_from_auth :dispatch1
 950     c.lock
 951     assert_equal c.locked_by_uuid, Thread.current[:api_client_authorization].uuid
 952     c.update!(log: logpdh_time1)
 953     c.update!(state: Container::Running)
 954     cr1.reload
 955     cr2.reload
 956     cr1log_uuid = cr1.log_uuid
 957     cr2log_uuid = cr2.log_uuid
 958     assert_not_nil cr1log_uuid
 959     assert_not_nil cr2log_uuid
 960     assert_not_equal logcoll.uuid, cr1log_uuid
 961     assert_not_equal logcoll.uuid, cr2log_uuid
 962     assert_not_equal cr1log_uuid, cr2log_uuid
 963
 964     logcoll.update!(manifest_text: logcoll.manifest_text + ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n")
 965     logpdh_time2 = logcoll.portable_data_hash
 966
 967     assert c.update(output: collections(:collection_owned_by_active).portable_data_hash)
 968     assert c.update(log: logpdh_time2)
 969     assert c.update(state: Container::Complete, log: logcoll.portable_data_hash)
 970     c.reload
 971     assert_equal collections(:collection_owned_by_active).portable_data_hash, c.output
 972     assert_equal logpdh_time2, c.log
 973     refute c.update(output: nil)
 974     refute c.update(log: nil)
 975     cr1.reload
 976     cr2.reload
 977     assert_equal cr1log_uuid, cr1.log_uuid
 978     assert_equal cr2log_uuid, cr2.log_uuid
 979     assert_equal 1, Collection.where(uuid: [cr1log_uuid, cr2log_uuid]).to_a.collect(&:portable_data_hash).uniq.length
 980     assert_equal ". 8c12f5f5297b7337598170c6f531fcee+7882 acbd18db4cc2f85cedef654fccc4a4d8+3 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 7882:3:foo.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt
 981 ./log\\040for\\040container\\040#{cr1.container_uuid} 8c12f5f5297b7337598170c6f531fcee+7882 acbd18db4cc2f85cedef654fccc4a4d8+3 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 7882:3:foo.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt
 982 ", Collection.find_by_uuid(cr1log_uuid).manifest_text
 983   end
 984
 985   ["auth_uuid", "runtime_token"].each do |tok|
 986     test "#{tok} can set output, progress, runtime_status, state, exit_code on running container -- but not log" do
 987       if tok == "runtime_token"
 988         set_user_from_auth :spectator
 989         c, _ = minimal_new(container_image: "9ae44d5792468c58bcf85ce7353c7027+124",
 990                            runtime_token: api_client_authorizations(:active).token)
 991       else
 992         set_user_from_auth :active
 993         c, _ = minimal_new
 994       end
 995       set_user_from_auth :dispatch1
 996       c.lock
 997       c.update! state: Container::Running
 998
 999       if tok == "runtime_token"
1000         auth = ApiClientAuthorization.validate(token: c.runtime_token)
1001         Thread.current[:api_client_authorization] = auth
1002         Thread.current[:api_client] = auth.api_client
1003         Thread.current[:token] = auth.token
1004         Thread.current[:user] = auth.user
1005       else
1006         auth = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
1007         Thread.current[:api_client_authorization] = auth
1008         Thread.current[:api_client] = auth.api_client
1009         Thread.current[:token] = auth.token
1010         Thread.current[:user] = auth.user
1011       end
1012
1013       assert c.update(gateway_address: "127.0.0.1:9")
1014       assert c.update(output: collections(:collection_owned_by_active).portable_data_hash)
1015       assert c.update(runtime_status: {'warning' => 'something happened'})
1016       assert c.update(progress: 0.5)
1017       assert c.update(exit_code: 0)
1018       refute c.update(log: collections(:log_collection).portable_data_hash)
1019       c.reload
1020       assert c.update(state: Container::Complete, exit_code: 0)
1021     end
1022   end
1023
1024   test "not allowed to set output that is not readable by current user" do
1025     set_user_from_auth :active
1026     c, _ = minimal_new
1027     set_user_from_auth :dispatch1
1028     c.lock
1029     c.update! state: Container::Running
1030
1031     Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
1032     Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
1033
1034     assert_raises ActiveRecord::RecordInvalid do
1035       c.update! output: collections(:collection_not_readable_by_active).portable_data_hash
1036     end
1037   end
1038
1039   test "other token cannot set output on running container" do
1040     set_user_from_auth :active
1041     c, _ = minimal_new
1042     set_user_from_auth :dispatch1
1043     c.lock
1044     c.update! state: Container::Running
1045
1046     set_user_from_auth :running_to_be_deleted_container_auth
1047     assert_raises(ArvadosModel::PermissionDeniedError) do
1048       c.update(output: collections(:foo_file).portable_data_hash)
1049     end
1050   end
1051
1052   test "can set trashed output on running container" do
1053     set_user_from_auth :active
1054     c, _ = minimal_new
1055     set_user_from_auth :dispatch1
1056     c.lock
1057     c.update! state: Container::Running
1058
1059     output = Collection.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jk')
1060
1061     assert output.is_trashed
1062     assert c.update output: output.portable_data_hash
1063     assert c.update! state: Container::Complete
1064   end
1065
1066   test "not allowed to set trashed output that is not readable by current user" do
1067     set_user_from_auth :active
1068     c, _ = minimal_new
1069     set_user_from_auth :dispatch1
1070     c.lock
1071     c.update! state: Container::Running
1072
1073     output = Collection.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jr')
1074
1075     Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
1076     Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
1077
1078     assert_raises ActiveRecord::RecordInvalid do
1079       c.update! output: output.portable_data_hash
1080     end
1081   end
1082
1083   test "user cannot delete" do
1084     set_user_from_auth :active
1085     c, _ = minimal_new
1086     assert_raises ArvadosModel::PermissionDeniedError do
1087       c.destroy
1088     end
1089     assert Container.find_by_uuid(c.uuid)
1090   end
1091
1092   [
1093     {state: Container::Complete, exit_code: 0, output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'},
1094     {state: Container::Cancelled},
1095   ].each do |final_attrs|
1096     test "secret_mounts and runtime_token are null after container is #{final_attrs[:state]}" do
1097       set_user_from_auth :active
1098       c, cr = minimal_new(secret_mounts: {'/secret' => {'kind' => 'text', 'content' => 'foo'}},
1099                           container_count_max: 1, runtime_token: api_client_authorizations(:active).token)
1100       set_user_from_auth :dispatch1
1101       c.lock
1102       c.update!(state: Container::Running)
1103       c.reload
1104       assert c.secret_mounts.has_key?('/secret')
1105       assert_equal api_client_authorizations(:active).token, c.runtime_token
1106
1107       c.update!(final_attrs)
1108       c.reload
1109       assert_equal({}, c.secret_mounts)
1110       assert_nil c.runtime_token
1111       cr.reload
1112       assert_equal({}, cr.secret_mounts)
1113       assert_nil cr.runtime_token
1114       assert_no_secrets_logged
1115     end
1116   end
1117
1118   def configure_preemptible_instance_type
1119     Rails.configuration.InstanceTypes = ConfigLoader.to_OrderedOptions({
1120       "a1.small.pre" => {
1121         "Preemptible" => true,
1122         "Price" => 0.1,
1123         "ProviderType" => "a1.small",
1124         "VCPUs" => 1,
1125         "RAM" => 1000000000,
1126       },
1127     })
1128   end
1129
1130   def vary_parameters(**kwargs)
1131     # kwargs is a hash that maps parameters to an array of values.
1132     # This function enumerates every possible hash where each key has one of
1133     # the values from its array.
1134     # The output keys are strings since that's what container hash attributes
1135     # want.
1136     # A nil value yields a hash without that key.
1137     [[:_, nil]].product(
1138       *kwargs.map { |(key, values)| [key.to_s].product(values) },
1139     ).map { |param_pairs| Hash[param_pairs].compact }
1140   end
1141
1142   def retry_with_scheduling_parameters(param_hashes)
1143     set_user_from_auth :admin
1144     containers = {}
1145     requests = []
1146     param_hashes.each do |scheduling_parameters|
1147       container, request = minimal_new(scheduling_parameters: scheduling_parameters)
1148       containers[container.uuid] = container
1149       requests << request
1150     end
1151     refute(containers.empty?, "buggy test: no scheduling parameters enumerated")
1152     assert_equal(1, containers.length)
1153     _, container1 = containers.shift
1154     container1.lock
1155     container1.update!(state: Container::Cancelled)
1156     container1.reload
1157     request1 = requests.shift
1158     request1.reload
1159     assert_not_equal(container1.uuid, request1.container_uuid)
1160     requests.each do |request|
1161       request.reload
1162       assert_equal(request1.container_uuid, request.container_uuid)
1163     end
1164     container2 = Container.find_by_uuid(request1.container_uuid)
1165     assert_not_nil(container2)
1166     return container2
1167   end
1168
1169   preemptible_values = [true, false, nil]
1170   preemptible_values.permutation(1).chain(
1171     preemptible_values.product(preemptible_values),
1172     preemptible_values.product(preemptible_values, preemptible_values),
1173   ).each do |preemptible_a|
1174     # If the first req has preemptible=true but a subsequent req
1175     # doesn't, we want to avoid reusing the first container, so this
1176     # test isn't appropriate.
1177     next if preemptible_a[0] &&
1178             ((preemptible_a.length > 1 && !preemptible_a[1]) ||
1179              (preemptible_a.length > 2 && !preemptible_a[2]))
1180     test "retry requests scheduled with preemptible=#{preemptible_a}" do
1181       configure_preemptible_instance_type
1182       param_hashes = vary_parameters(preemptible: preemptible_a)
1183       container = retry_with_scheduling_parameters(param_hashes)
1184       assert_equal(preemptible_a.all?,
1185                    container.scheduling_parameters["preemptible"] || false)
1186     end
1187   end
1188
1189   partition_values = [nil, [], ["alpha"], ["alpha", "bravo"], ["bravo", "charlie"]]
1190   partition_values.permutation(1).chain(
1191     partition_values.permutation(2),
1192   ).each do |partitions_a|
1193     test "retry requests scheduled with partitions=#{partitions_a}" do
1194       param_hashes = vary_parameters(partitions: partitions_a)
1195       container = retry_with_scheduling_parameters(param_hashes)
1196       expected = if partitions_a.any? { |value| value.nil? or value.empty? }
1197                    []
1198                  else
1199                    partitions_a.flatten.uniq
1200                  end
1201       actual = container.scheduling_parameters["partitions"] || []
1202       assert_equal(expected.sort, actual.sort)
1203     end
1204   end
1205
1206   runtime_values = [nil, 0, 1, 2, 3]
1207   runtime_values.permutation(1).chain(
1208     runtime_values.permutation(2),
1209     runtime_values.permutation(3),
1210   ).each do |max_run_time_a|
1211     test "retry requests scheduled with max_run_time=#{max_run_time_a}" do
1212       param_hashes = vary_parameters(max_run_time: max_run_time_a)
1213       container = retry_with_scheduling_parameters(param_hashes)
1214       expected = if max_run_time_a.any? { |value| value.nil? or value == 0 }
1215                    0
1216                  else
1217                    max_run_time_a.max
1218                  end
1219       actual = container.scheduling_parameters["max_run_time"] || 0
1220       assert_equal(expected, actual)
1221     end
1222   end
1223
1224   test "retry requests with multi-varied scheduling parameters" do
1225     configure_preemptible_instance_type
1226     param_hashes = [{
1227                      "partitions": ["alpha", "bravo"],
1228                      "preemptible": false,
1229                      "max_run_time": 10,
1230                     }, {
1231                      "partitions": ["alpha", "charlie"],
1232                      "max_run_time": 20,
1233                     }, {
1234                      "partitions": ["bravo", "charlie"],
1235                      "preemptible": true,
1236                      "max_run_time": 30,
1237                     }]
1238     container = retry_with_scheduling_parameters(param_hashes)
1239     actual = container.scheduling_parameters
1240     assert_equal(["alpha", "bravo", "charlie"], actual["partitions"]&.sort)
1241     assert_equal(false, actual["preemptible"] || false)
1242     assert_equal(30, actual["max_run_time"])
1243   end
1244
1245   test "retry requests with unset scheduling parameters" do
1246     configure_preemptible_instance_type
1247     param_hashes = vary_parameters(
1248       preemptible: [nil, true],
1249       partitions: [nil, ["alpha"]],
1250       max_run_time: [nil, 5],
1251     )
1252     container = retry_with_scheduling_parameters(param_hashes)
1253     actual = container.scheduling_parameters
1254     assert_equal([], actual["partitions"] || [])
1255     assert_equal(false, actual["preemptible"] || false)
1256     assert_equal(0, actual["max_run_time"] || 0)
1257   end
1258
1259   test "retry requests with default scheduling parameters" do
1260     configure_preemptible_instance_type
1261     param_hashes = vary_parameters(
1262       preemptible: [false, true],
1263       partitions: [[], ["bravo"]],
1264       max_run_time: [0, 1],
1265     )
1266     container = retry_with_scheduling_parameters(param_hashes)
1267     actual = container.scheduling_parameters
1268     assert_equal([], actual["partitions"] || [])
1269     assert_equal(false, actual["preemptible"] || false)
1270     assert_equal(0, actual["max_run_time"] || 0)
1271   end
1272
1273   def run_container(request_params, final_attrs)
1274     final_attrs[:state] ||= Container::Complete
1275     if final_attrs[:state] == Container::Complete
1276       final_attrs[:exit_code] ||= 0
1277       final_attrs[:log] ||= collections(:log_collection).portable_data_hash
1278       final_attrs[:output] ||= collections(:multilevel_collection_1).portable_data_hash
1279     end
1280     container, request = minimal_new(request_params)
1281     container.lock
1282     container.update!(state: Container::Running)
1283     container.update!(final_attrs)
1284     return container, request
1285   end
1286
1287   def check_reuse_with_variations(default_keep_cache_ram, vary_attr, start_value, variations)
1288     container_params = REUSABLE_ATTRS_SLIM.merge(vary_attr => start_value)
1289     orig_default = Rails.configuration.Containers.DefaultKeepCacheRAM
1290     begin
1291       Rails.configuration.Containers.DefaultKeepCacheRAM = default_keep_cache_ram
1292       set_user_from_auth :admin
1293       expected, _ = run_container(container_params, {})
1294       variations.each do |variation|
1295         full_variation = REUSABLE_ATTRS_SLIM[vary_attr].merge(variation)
1296         parameters = REUSABLE_ATTRS_SLIM.merge(vary_attr => full_variation)
1297         actual = Container.find_reusable(parameters)
1298         assert_equal(expected.uuid, actual&.uuid,
1299                      "request with #{vary_attr}=#{variation} did not reuse container")
1300       end
1301     ensure
1302       Rails.configuration.Containers.DefaultKeepCacheRAM = orig_default
1303     end
1304   end
1305
1306   # Test that we can reuse a container with a known keep_cache_ram constraint,
1307   # no matter what keep_cache_* constraints the new request uses.
1308   [0, 2 << 30, 4 << 30].product(
1309     [0, 1],
1310     [true, false],
1311   ).each do |(default_keep_cache_ram, multiplier, keep_disk_constraint)|
1312     test "reuse request with DefaultKeepCacheRAM=#{default_keep_cache_ram}, keep_cache_ram*=#{multiplier}, keep_cache_disk=#{keep_disk_constraint}" do
1313       runtime_constraints = REUSABLE_ATTRS_SLIM[:runtime_constraints].merge(
1314         "keep_cache_ram" => default_keep_cache_ram * multiplier,
1315       )
1316       if not keep_disk_constraint
1317         # Simulate a container that predates keep_cache_disk by deleting
1318         # the constraint entirely.
1319         runtime_constraints.delete("keep_cache_disk")
1320       end
1321       # Important values are:
1322       # * 0
1323       # * 2GiB, the minimum default keep_cache_disk
1324       # * 8GiB, the default keep_cache_disk based on container ram
1325       # * 32GiB, the maximum default keep_cache_disk
1326       # Check these values and values in between.
1327       vary_values = [0, 1, 2, 6, 8, 10, 32, 33].map { |v| v << 30 }.to_a
1328       variations = vary_parameters(keep_cache_ram: vary_values)
1329                      .chain(vary_parameters(keep_cache_disk: vary_values))
1330       check_reuse_with_variations(
1331         default_keep_cache_ram,
1332         :runtime_constraints,
1333         runtime_constraints,
1334         variations,
1335       )
1336     end
1337   end
1338
1339   # Test that we can reuse a container with a known keep_cache_disk constraint,
1340   # no matter what keep_cache_* constraints the new request uses.
1341   # keep_cache_disk values are the important values discussed in the test above.
1342   [0, 2 << 30, 4 << 30]
1343     .product([0, 2 << 30, 8 << 30, 32 << 30])
1344     .each do |(default_keep_cache_ram, keep_cache_disk)|
1345     test "reuse request with DefaultKeepCacheRAM=#{default_keep_cache_ram} and keep_cache_disk=#{keep_cache_disk}" do
1346       runtime_constraints = REUSABLE_ATTRS_SLIM[:runtime_constraints].merge(
1347         "keep_cache_disk" => keep_cache_disk,
1348       )
1349       vary_values = [0, 1, 2, 6, 8, 10, 32, 33].map { |v| v << 30 }.to_a
1350       variations = vary_parameters(keep_cache_ram: vary_values)
1351                      .chain(vary_parameters(keep_cache_disk: vary_values))
1352       check_reuse_with_variations(
1353         default_keep_cache_ram,
1354         :runtime_constraints,
1355         runtime_constraints,
1356         variations,
1357       )
1358     end
1359   end
1360
1361   # Test that a container request can reuse a container with an exactly
1362   # matching keep_cache_* constraint, no matter what the defaults.
1363   [0, 2 << 30, 4 << 30].product(
1364     ["keep_cache_disk", "keep_cache_ram"],
1365     [135790, 13 << 30, 135 << 30],
1366   ).each do |(default_keep_cache_ram, constraint_key, constraint_value)|
1367     test "reuse request with #{constraint_key}=#{constraint_value} and DefaultKeepCacheRAM=#{default_keep_cache_ram}" do
1368       runtime_constraints = REUSABLE_ATTRS_SLIM[:runtime_constraints].merge(
1369         constraint_key => constraint_value,
1370       )
1371       check_reuse_with_variations(
1372         default_keep_cache_ram,
1373         :runtime_constraints,
1374         runtime_constraints,
1375         [runtime_constraints],
1376       )
1377     end
1378   end
1379 end