services/api/test/unit/container_test.rb

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 require 'test_helper'
   6 require 'helpers/container_test_helper'
   7
   8 class ContainerTest < ActiveSupport::TestCase
   9   include DbCurrentTime
  10   include ContainerTestHelper
  11
  12   DEFAULT_ATTRS = {
  13     command: ['echo', 'foo'],
  14     container_image: 'fa3c1a9cb6783f85f2ecda037e07b8c3+167',
  15     output_path: '/tmp',
  16     priority: 1,
  17     runtime_constraints: {"vcpus" => 1, "ram" => 1, "cuda" => {"device_count":0, "driver_version": "", "hardware_capability": ""}},
  18   }
  19
  20   REUSABLE_COMMON_ATTRS = {
  21     container_image: "9ae44d5792468c58bcf85ce7353c7027+124",
  22     cwd: "test",
  23     command: ["echo", "hello"],
  24     output_path: "test",
  25     runtime_constraints: {
  26       "API" => false,
  27       "keep_cache_disk" => 0,
  28       "keep_cache_ram" => 0,
  29       "ram" => 12000000000,
  30       "vcpus" => 4
  31     },
  32     mounts: {
  33       "test" => {"kind" => "json"},
  34     },
  35     environment: {
  36       "var" => "val",
  37     },
  38     secret_mounts: {},
  39     runtime_user_uuid: "zzzzz-tpzed-xurymjxw79nv3jz",
  40     runtime_auth_scopes: ["all"]
  41   }
  42
  43   REUSABLE_ATTRS_SLIM = {
  44     command: ["echo", "slim"],
  45     container_image: "9ae44d5792468c58bcf85ce7353c7027+124",
  46     cwd: "test",
  47     environment: {},
  48     mounts: {},
  49     output_path: "test",
  50     runtime_auth_scopes: ["all"],
  51     runtime_constraints: {
  52       "API" => false,
  53       "keep_cache_disk" => 0,
  54       "keep_cache_ram" => 0,
  55       "ram" => 8 << 30,
  56       "vcpus" => 4
  57     },
  58     runtime_user_uuid: "zzzzz-tpzed-xurymjxw79nv3jz",
  59     secret_mounts: {},
  60   }
  61
  62   def request_only attrs
  63     attrs.reject {|k| [:runtime_user_uuid, :runtime_auth_scopes].include? k}
  64   end
  65
  66   def minimal_new attrs={}
  67     cr = ContainerRequest.new request_only(DEFAULT_ATTRS.merge(attrs))
  68     cr.state = ContainerRequest::Committed
  69     cr.save!
  70     c = Container.find_by_uuid cr.container_uuid
  71     assert_not_nil c
  72     return c, cr
  73   end
  74
  75   def check_illegal_updates c, bad_updates
  76     bad_updates.each do |u|
  77       refute c.update_attributes(u), u.inspect
  78       refute c.valid?, u.inspect
  79       c.reload
  80     end
  81   end
  82
  83   def check_illegal_modify c
  84     check_illegal_updates c, [{command: ["echo", "bar"]},
  85                               {container_image: "arvados/apitestfixture:june10"},
  86                               {cwd: "/tmp2"},
  87                               {environment: {"FOO" => "BAR"}},
  88                               {mounts: {"FOO" => "BAR"}},
  89                               {output_path: "/tmp3"},
  90                               {locked_by_uuid: "zzzzz-gj3su-027z32aux8dg2s1"},
  91                               {auth_uuid: "zzzzz-gj3su-017z32aux8dg2s1"},
  92                               {runtime_constraints: {"FOO" => "BAR"}}]
  93   end
  94
  95   def check_bogus_states c
  96     check_illegal_updates c, [{state: nil},
  97                               {state: "Flubber"}]
  98   end
  99
 100   def check_no_change_from_cancelled c
 101     check_illegal_modify c
 102     check_bogus_states c
 103     check_illegal_updates c, [{ priority: 3 },
 104                               { state: Container::Queued },
 105                               { state: Container::Locked },
 106                               { state: Container::Running },
 107                               { state: Container::Complete }]
 108   end
 109
 110   test "Container create" do
 111     act_as_system_user do
 112       c, _ = minimal_new(environment: {},
 113                       mounts: {"BAR" => {"kind" => "FOO"}},
 114                       output_path: "/tmp",
 115                       priority: 1,
 116                       runtime_constraints: {"vcpus" => 1, "ram" => 1})
 117
 118       check_illegal_modify c
 119       check_bogus_states c
 120
 121       c.reload
 122       c.priority = 2
 123       c.save!
 124     end
 125   end
 126
 127   test "Container valid priority" do
 128     act_as_system_user do
 129       c, _ = minimal_new(environment: {},
 130                       mounts: {"BAR" => {"kind" => "FOO"}},
 131                       output_path: "/tmp",
 132                       priority: 1,
 133                       runtime_constraints: {"vcpus" => 1, "ram" => 1})
 134
 135       assert_raises(ActiveRecord::RecordInvalid) do
 136         c.priority = -1
 137         c.save!
 138       end
 139
 140       c.priority = 0
 141       c.save!
 142
 143       c.priority = 1
 144       c.save!
 145
 146       c.priority = 500
 147       c.save!
 148
 149       c.priority = 999
 150       c.save!
 151
 152       c.priority = 1000
 153       c.save!
 154
 155       c.priority = 1000 << 50
 156       c.save!
 157     end
 158   end
 159
 160   test "Container runtime_status data types" do
 161     set_user_from_auth :active
 162     attrs = {
 163       environment: {},
 164       mounts: {"BAR" => {"kind" => "FOO"}},
 165       output_path: "/tmp",
 166       priority: 1,
 167       runtime_constraints: {"vcpus" => 1, "ram" => 1}
 168     }
 169     c, _ = minimal_new(attrs)
 170     assert_equal c.runtime_status, {}
 171     assert_equal Container::Queued, c.state
 172
 173     set_user_from_auth :dispatch1
 174     c.update_attributes! state: Container::Locked
 175     c.update_attributes! state: Container::Running
 176
 177     [
 178       'error', 'errorDetail', 'warning', 'warningDetail', 'activity'
 179     ].each do |k|
 180       # String type is allowed
 181       string_val = 'A string is accepted'
 182       c.update_attributes! runtime_status: {k => string_val}
 183       assert_equal string_val, c.runtime_status[k]
 184
 185       # Other types aren't allowed
 186       [
 187         42, false, [], {}, nil
 188       ].each do |unallowed_val|
 189         assert_raises ActiveRecord::RecordInvalid do
 190           c.update_attributes! runtime_status: {k => unallowed_val}
 191         end
 192       end
 193     end
 194   end
 195
 196   test "Container runtime_status updates" do
 197     set_user_from_auth :active
 198     attrs = {
 199       environment: {},
 200       mounts: {"BAR" => {"kind" => "FOO"}},
 201       output_path: "/tmp",
 202       priority: 1,
 203       runtime_constraints: {"vcpus" => 1, "ram" => 1}
 204     }
 205     c1, _ = minimal_new(attrs)
 206     assert_equal c1.runtime_status, {}
 207
 208     assert_equal Container::Queued, c1.state
 209     assert_raises ArvadosModel::PermissionDeniedError do
 210       c1.update_attributes! runtime_status: {'error' => 'Oops!'}
 211     end
 212
 213     set_user_from_auth :dispatch1
 214
 215     # Allow updates when state = Locked
 216     c1.update_attributes! state: Container::Locked
 217     c1.update_attributes! runtime_status: {'error' => 'Oops!'}
 218     assert c1.runtime_status.key? 'error'
 219
 220     # Reset when transitioning from Locked to Queued
 221     c1.update_attributes! state: Container::Queued
 222     assert_equal c1.runtime_status, {}
 223
 224     # Allow updates when state = Running
 225     c1.update_attributes! state: Container::Locked
 226     c1.update_attributes! state: Container::Running
 227     c1.update_attributes! runtime_status: {'error' => 'Oops!'}
 228     assert c1.runtime_status.key? 'error'
 229
 230     # Don't allow updates on other states
 231     c1.update_attributes! state: Container::Complete
 232     assert_raises ActiveRecord::RecordInvalid do
 233       c1.update_attributes! runtime_status: {'error' => 'Some other error'}
 234     end
 235
 236     set_user_from_auth :active
 237     c2, _ = minimal_new(attrs)
 238     assert_equal c2.runtime_status, {}
 239     set_user_from_auth :dispatch1
 240     c2.update_attributes! state: Container::Locked
 241     c2.update_attributes! state: Container::Running
 242     c2.update_attributes! state: Container::Cancelled
 243     assert_raises ActiveRecord::RecordInvalid do
 244       c2.update_attributes! runtime_status: {'error' => 'Oops!'}
 245     end
 246   end
 247
 248   test "Container serialized hash attributes sorted before save" do
 249     set_user_from_auth :active
 250     env = {"C" => "3", "B" => "2", "A" => "1"}
 251     m = {"F" => {"kind" => "3"}, "E" => {"kind" => "2"}, "D" => {"kind" => "1"}}
 252     rc = {"vcpus" => 1, "ram" => 1, "keep_cache_ram" => 1, "keep_cache_disk" => 0, "API" => true, "cuda" => {"device_count":0, "driver_version": "", "hardware_capability": ""}}
 253     c, _ = minimal_new(environment: env, mounts: m, runtime_constraints: rc)
 254     c.reload
 255     assert_equal Container.deep_sort_hash(env).to_json, c.environment.to_json
 256     assert_equal Container.deep_sort_hash(m).to_json, c.mounts.to_json
 257     assert_equal Container.deep_sort_hash(rc).to_json, c.runtime_constraints.to_json
 258   end
 259
 260   test 'deep_sort_hash on array of hashes' do
 261     a = {'z' => [[{'a' => 'a', 'b' => 'b'}]]}
 262     b = {'z' => [[{'b' => 'b', 'a' => 'a'}]]}
 263     assert_equal Container.deep_sort_hash(a).to_json, Container.deep_sort_hash(b).to_json
 264   end
 265
 266   test "find_reusable method should select higher priority queued container" do
 267         Rails.configuration.Containers.LogReuseDecisions = true
 268     set_user_from_auth :active
 269     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment:{"var" => "queued"}})
 270     c_low_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:1}))
 271     c_high_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:2}))
 272     assert_not_equal c_low_priority.uuid, c_high_priority.uuid
 273     assert_equal Container::Queued, c_low_priority.state
 274     assert_equal Container::Queued, c_high_priority.state
 275     reused = Container.find_reusable(common_attrs)
 276     assert_not_nil reused
 277     assert_equal reused.uuid, c_high_priority.uuid
 278   end
 279
 280   test "find_reusable method should select latest completed container" do
 281     set_user_from_auth :active
 282     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "complete"}})
 283     completed_attrs = {
 284       state: Container::Complete,
 285       exit_code: 0,
 286       log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 287       output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
 288     }
 289
 290     c_older, _ = minimal_new(common_attrs.merge({use_existing: false}))
 291     c_recent, _ = minimal_new(common_attrs.merge({use_existing: false}))
 292     assert_not_equal c_older.uuid, c_recent.uuid
 293
 294     set_user_from_auth :dispatch1
 295     c_older.update_attributes!({state: Container::Locked})
 296     c_older.update_attributes!({state: Container::Running})
 297     c_older.update_attributes!(completed_attrs)
 298
 299     c_recent.update_attributes!({state: Container::Locked})
 300     c_recent.update_attributes!({state: Container::Running})
 301     c_recent.update_attributes!(completed_attrs)
 302
 303     reused = Container.find_reusable(common_attrs)
 304     assert_not_nil reused
 305     assert_equal reused.uuid, c_older.uuid
 306   end
 307
 308   test "find_reusable method should select oldest completed container when inconsistent outputs exist" do
 309     set_user_from_auth :active
 310     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "complete"}, priority: 1})
 311     completed_attrs = {
 312       state: Container::Complete,
 313       exit_code: 0,
 314       log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 315     }
 316
 317     cr = ContainerRequest.new request_only(common_attrs)
 318     cr.use_existing = false
 319     cr.state = ContainerRequest::Committed
 320     cr.save!
 321     c_output1 = Container.where(uuid: cr.container_uuid).first
 322
 323     cr = ContainerRequest.new request_only(common_attrs)
 324     cr.use_existing = false
 325     cr.state = ContainerRequest::Committed
 326     cr.save!
 327     c_output2 = Container.where(uuid: cr.container_uuid).first
 328
 329     assert_not_equal c_output1.uuid, c_output2.uuid
 330
 331     set_user_from_auth :dispatch1
 332
 333     out1 = '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
 334     log1 = collections(:real_log_collection).portable_data_hash
 335     c_output1.update_attributes!({state: Container::Locked})
 336     c_output1.update_attributes!({state: Container::Running})
 337     c_output1.update_attributes!(completed_attrs.merge({log: log1, output: out1}))
 338
 339     out2 = 'fa7aeb5140e2848d39b416daeef4ffc5+45'
 340     c_output2.update_attributes!({state: Container::Locked})
 341     c_output2.update_attributes!({state: Container::Running})
 342     c_output2.update_attributes!(completed_attrs.merge({log: log1, output: out2}))
 343
 344     set_user_from_auth :active
 345     reused = Container.resolve(ContainerRequest.new(request_only(common_attrs)))
 346     assert_equal c_output1.uuid, reused.uuid
 347   end
 348
 349   test "find_reusable method should select running container by start date" do
 350     set_user_from_auth :active
 351     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running"}})
 352     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
 353     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
 354     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
 355     # Confirm the 3 container UUIDs are different.
 356     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
 357     set_user_from_auth :dispatch1
 358     c_slower.update_attributes!({state: Container::Locked})
 359     c_slower.update_attributes!({state: Container::Running,
 360                                  progress: 0.1})
 361     c_faster_started_first.update_attributes!({state: Container::Locked})
 362     c_faster_started_first.update_attributes!({state: Container::Running,
 363                                                progress: 0.15})
 364     c_faster_started_second.update_attributes!({state: Container::Locked})
 365     c_faster_started_second.update_attributes!({state: Container::Running,
 366                                                 progress: 0.15})
 367     reused = Container.find_reusable(common_attrs)
 368     assert_not_nil reused
 369     # Selected container is the one that started first
 370     assert_equal reused.uuid, c_faster_started_first.uuid
 371   end
 372
 373   test "find_reusable method should select running container by progress" do
 374     set_user_from_auth :active
 375     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running2"}})
 376     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
 377     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
 378     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
 379     # Confirm the 3 container UUIDs are different.
 380     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
 381     set_user_from_auth :dispatch1
 382     c_slower.update_attributes!({state: Container::Locked})
 383     c_slower.update_attributes!({state: Container::Running,
 384                                  progress: 0.1})
 385     c_faster_started_first.update_attributes!({state: Container::Locked})
 386     c_faster_started_first.update_attributes!({state: Container::Running,
 387                                                progress: 0.15})
 388     c_faster_started_second.update_attributes!({state: Container::Locked})
 389     c_faster_started_second.update_attributes!({state: Container::Running,
 390                                                 progress: 0.2})
 391     reused = Container.find_reusable(common_attrs)
 392     assert_not_nil reused
 393     # Selected container is the one with most progress done
 394     assert_equal reused.uuid, c_faster_started_second.uuid
 395   end
 396
 397   test "find_reusable method should select non-failing running container" do
 398     set_user_from_auth :active
 399     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running2"}})
 400     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
 401     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
 402     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
 403     # Confirm the 3 container UUIDs are different.
 404     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
 405     set_user_from_auth :dispatch1
 406     c_slower.update_attributes!({state: Container::Locked})
 407     c_slower.update_attributes!({state: Container::Running,
 408                                  progress: 0.1})
 409     c_faster_started_first.update_attributes!({state: Container::Locked})
 410     c_faster_started_first.update_attributes!({state: Container::Running,
 411                                                runtime_status: {'warning' => 'This is not an error'},
 412                                                progress: 0.15})
 413     c_faster_started_second.update_attributes!({state: Container::Locked})
 414     assert_equal 0, Container.where("runtime_status->'error' is not null").count
 415     c_faster_started_second.update_attributes!({state: Container::Running,
 416                                                 runtime_status: {'error' => 'Something bad happened'},
 417                                                 progress: 0.2})
 418     assert_equal 1, Container.where("runtime_status->'error' is not null").count
 419     reused = Container.find_reusable(common_attrs)
 420     assert_not_nil reused
 421     # Selected the non-failing container even if it's the one with less progress done
 422     assert_equal reused.uuid, c_faster_started_first.uuid
 423   end
 424
 425   test "find_reusable method should select locked container most likely to start sooner" do
 426     set_user_from_auth :active
 427     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "locked"}})
 428     c_low_priority, _ = minimal_new(common_attrs.merge({use_existing: false}))
 429     c_high_priority_older, _ = minimal_new(common_attrs.merge({use_existing: false}))
 430     c_high_priority_newer, _ = minimal_new(common_attrs.merge({use_existing: false}))
 431     # Confirm the 3 container UUIDs are different.
 432     assert_equal 3, [c_low_priority.uuid, c_high_priority_older.uuid, c_high_priority_newer.uuid].uniq.length
 433     set_user_from_auth :dispatch1
 434     c_low_priority.update_attributes!({state: Container::Locked,
 435                                        priority: 1})
 436     c_high_priority_older.update_attributes!({state: Container::Locked,
 437                                               priority: 2})
 438     c_high_priority_newer.update_attributes!({state: Container::Locked,
 439                                               priority: 2})
 440     reused = Container.find_reusable(common_attrs)
 441     assert_not_nil reused
 442     assert_equal reused.uuid, c_high_priority_older.uuid
 443   end
 444
 445   test "find_reusable method should select running over failed container" do
 446     set_user_from_auth :active
 447     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed_vs_running"}})
 448     c_failed, _ = minimal_new(common_attrs.merge({use_existing: false}))
 449     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
 450     assert_not_equal c_failed.uuid, c_running.uuid
 451     set_user_from_auth :dispatch1
 452     c_failed.update_attributes!({state: Container::Locked})
 453     c_failed.update_attributes!({state: Container::Running})
 454     c_failed.update_attributes!({state: Container::Complete,
 455                                  exit_code: 42,
 456                                  log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 457                                  output: 'ea10d51bcf88862dbcc36eb292017dfd+45'})
 458     c_running.update_attributes!({state: Container::Locked})
 459     c_running.update_attributes!({state: Container::Running,
 460                                   progress: 0.15})
 461     reused = Container.find_reusable(common_attrs)
 462     assert_not_nil reused
 463     assert_equal reused.uuid, c_running.uuid
 464   end
 465
 466   test "find_reusable method should select complete over running container" do
 467     set_user_from_auth :active
 468     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "completed_vs_running"}})
 469     c_completed, _ = minimal_new(common_attrs.merge({use_existing: false}))
 470     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
 471     assert_not_equal c_completed.uuid, c_running.uuid
 472     set_user_from_auth :dispatch1
 473     c_completed.update_attributes!({state: Container::Locked})
 474     c_completed.update_attributes!({state: Container::Running})
 475     c_completed.update_attributes!({state: Container::Complete,
 476                                     exit_code: 0,
 477                                     log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 478                                     output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'})
 479     c_running.update_attributes!({state: Container::Locked})
 480     c_running.update_attributes!({state: Container::Running,
 481                                   progress: 0.15})
 482     reused = Container.find_reusable(common_attrs)
 483     assert_not_nil reused
 484     assert_equal c_completed.uuid, reused.uuid
 485   end
 486
 487   test "find_reusable method should select running over locked container" do
 488     set_user_from_auth :active
 489     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}})
 490     c_locked, _ = minimal_new(common_attrs.merge({use_existing: false}))
 491     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
 492     assert_not_equal c_running.uuid, c_locked.uuid
 493     set_user_from_auth :dispatch1
 494     c_locked.update_attributes!({state: Container::Locked})
 495     c_running.update_attributes!({state: Container::Locked})
 496     c_running.update_attributes!({state: Container::Running,
 497                                   progress: 0.15})
 498     reused = Container.find_reusable(common_attrs)
 499     assert_not_nil reused
 500     assert_equal reused.uuid, c_running.uuid
 501   end
 502
 503   test "find_reusable method should select locked over queued container" do
 504     set_user_from_auth :active
 505     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}})
 506     c_locked, _ = minimal_new(common_attrs.merge({use_existing: false}))
 507     c_queued, _ = minimal_new(common_attrs.merge({use_existing: false}))
 508     assert_not_equal c_queued.uuid, c_locked.uuid
 509     set_user_from_auth :dispatch1
 510     c_locked.update_attributes!({state: Container::Locked})
 511     reused = Container.find_reusable(common_attrs)
 512     assert_not_nil reused
 513     assert_equal reused.uuid, c_locked.uuid
 514   end
 515
 516   test "find_reusable method should not select failed container" do
 517     set_user_from_auth :active
 518     attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed"}})
 519     c, _ = minimal_new(attrs)
 520     set_user_from_auth :dispatch1
 521     c.update_attributes!({state: Container::Locked})
 522     c.update_attributes!({state: Container::Running})
 523     c.update_attributes!({state: Container::Complete,
 524                           exit_code: 33})
 525     reused = Container.find_reusable(attrs)
 526     assert_nil reused
 527   end
 528
 529   test "find_reusable with logging disabled" do
 530     set_user_from_auth :active
 531     Rails.logger.expects(:info).never
 532     Container.find_reusable(REUSABLE_COMMON_ATTRS)
 533   end
 534
 535   test "find_reusable with logging enabled" do
 536     set_user_from_auth :active
 537     Rails.configuration.Containers.LogReuseDecisions = true
 538     Rails.logger.expects(:info).at_least(3)
 539     Container.find_reusable(REUSABLE_COMMON_ATTRS)
 540   end
 541
 542   def runtime_token_attr tok
 543     auth = api_client_authorizations(tok)
 544     {runtime_user_uuid: User.find_by_id(auth.user_id).uuid,
 545      runtime_auth_scopes: auth.scopes,
 546      runtime_token: auth.token}
 547   end
 548
 549   test "find_reusable method with same runtime_token" do
 550     set_user_from_auth :active
 551     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 552     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:container_runtime_token).token}))
 553     assert_equal Container::Queued, c1.state
 554     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 555     assert_not_nil reused
 556     assert_equal reused.uuid, c1.uuid
 557   end
 558
 559   test "find_reusable method with different runtime_token, same user" do
 560     set_user_from_auth :active
 561     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 562     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:crt_user).token}))
 563     assert_equal Container::Queued, c1.state
 564     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 565     assert_not_nil reused
 566     assert_equal reused.uuid, c1.uuid
 567   end
 568
 569   test "find_reusable method with nil runtime_token, then runtime_token with same user" do
 570     set_user_from_auth :crt_user
 571     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 572     c1, _ = minimal_new(common_attrs)
 573     assert_equal Container::Queued, c1.state
 574     assert_equal users(:container_runtime_token_user).uuid, c1.runtime_user_uuid
 575     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 576     assert_not_nil reused
 577     assert_equal reused.uuid, c1.uuid
 578   end
 579
 580   test "find_reusable method with different runtime_token, different user" do
 581     set_user_from_auth :crt_user
 582     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 583     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:active).token}))
 584     assert_equal Container::Queued, c1.state
 585     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 586     # See #14584
 587     assert_not_nil reused
 588     assert_equal c1.uuid, reused.uuid
 589   end
 590
 591   test "find_reusable method with nil runtime_token, then runtime_token with different user" do
 592     set_user_from_auth :active
 593     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 594     c1, _ = minimal_new(common_attrs.merge({runtime_token: nil}))
 595     assert_equal Container::Queued, c1.state
 596     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 597     # See #14584
 598     assert_not_nil reused
 599     assert_equal c1.uuid, reused.uuid
 600   end
 601
 602   test "find_reusable method with different runtime_token, different scope, same user" do
 603     set_user_from_auth :active
 604     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 605     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:runtime_token_limited_scope).token}))
 606     assert_equal Container::Queued, c1.state
 607     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 608     # See #14584
 609     assert_not_nil reused
 610     assert_equal c1.uuid, reused.uuid
 611   end
 612
 613   test "find_reusable method with cuda" do
 614     set_user_from_auth :active
 615     # No cuda
 616     no_cuda_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"},
 617                                                 runtime_constraints: {"vcpus" => 1, "ram" => 1, "keep_cache_disk"=>0, "keep_cache_ram"=>268435456, "API" => false,
 618                                                                       "cuda" => {"device_count":0, "driver_version": "", "hardware_capability": ""}},})
 619     c1, _ = minimal_new(no_cuda_attrs)
 620     assert_equal Container::Queued, c1.state
 621
 622     # has cuda
 623     cuda_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"},
 624                                                 runtime_constraints: {"vcpus" => 1, "ram" => 1, "keep_cache_disk"=>0, "keep_cache_ram"=>268435456, "API" => false,
 625                                                                       "cuda" => {"device_count":1, "driver_version": "11.0", "hardware_capability": "9.0"}},})
 626     c2, _ = minimal_new(cuda_attrs)
 627     assert_equal Container::Queued, c2.state
 628
 629     # should find the no cuda one
 630     reused = Container.find_reusable(no_cuda_attrs)
 631     assert_not_nil reused
 632     assert_equal reused.uuid, c1.uuid
 633
 634     # should find the cuda one
 635     reused = Container.find_reusable(cuda_attrs)
 636     assert_not_nil reused
 637     assert_equal reused.uuid, c2.uuid
 638   end
 639
 640   test "Container running" do
 641     set_user_from_auth :active
 642     c, _ = minimal_new priority: 1
 643
 644     set_user_from_auth :dispatch1
 645     check_illegal_updates c, [{state: Container::Running},
 646                               {state: Container::Complete}]
 647
 648     c.lock
 649     c.update_attributes! state: Container::Running
 650
 651     check_illegal_modify c
 652     check_bogus_states c
 653
 654     check_illegal_updates c, [{state: Container::Queued}]
 655     c.reload
 656
 657     c.update_attributes! priority: 3
 658   end
 659
 660   test "Lock and unlock" do
 661     set_user_from_auth :active
 662     c, cr = minimal_new priority: 0
 663
 664     set_user_from_auth :dispatch1
 665     assert_equal Container::Queued, c.state
 666
 667     assert_raise(ArvadosModel::LockFailedError) do
 668       # "no priority"
 669       c.lock
 670     end
 671     c.reload
 672     assert cr.update_attributes priority: 1
 673
 674     refute c.update_attributes(state: Container::Running), "not locked"
 675     c.reload
 676     refute c.update_attributes(state: Container::Complete), "not locked"
 677     c.reload
 678
 679     assert c.lock, show_errors(c)
 680     assert c.locked_by_uuid
 681     assert c.auth_uuid
 682
 683     assert_raise(ArvadosModel::LockFailedError) {c.lock}
 684     c.reload
 685
 686     assert c.unlock, show_errors(c)
 687     refute c.locked_by_uuid
 688     refute c.auth_uuid
 689
 690     refute c.update_attributes(state: Container::Running), "not locked"
 691     c.reload
 692     refute c.locked_by_uuid
 693     refute c.auth_uuid
 694
 695     assert c.lock, show_errors(c)
 696     assert c.update_attributes(state: Container::Running), show_errors(c)
 697     assert c.locked_by_uuid
 698     assert c.auth_uuid
 699
 700     auth_uuid_was = c.auth_uuid
 701
 702     assert_raise(ArvadosModel::LockFailedError) do
 703       # Running to Locked is not allowed
 704       c.lock
 705     end
 706     c.reload
 707     assert_raise(ArvadosModel::InvalidStateTransitionError) do
 708       # Running to Queued is not allowed
 709       c.unlock
 710     end
 711     c.reload
 712
 713     assert c.update_attributes(state: Container::Complete), show_errors(c)
 714     refute c.locked_by_uuid
 715     refute c.auth_uuid
 716
 717     auth_exp = ApiClientAuthorization.find_by_uuid(auth_uuid_was).expires_at
 718     assert_operator auth_exp, :<, db_current_time
 719
 720     assert_nil ApiClientAuthorization.validate(token: ApiClientAuthorization.find_by_uuid(auth_uuid_was).token)
 721   end
 722
 723   test "Exceed maximum lock-unlock cycles" do
 724     Rails.configuration.Containers.MaxDispatchAttempts = 3
 725
 726     set_user_from_auth :active
 727     c, cr = minimal_new
 728
 729     set_user_from_auth :dispatch1
 730     assert_equal Container::Queued, c.state
 731     assert_equal 0, c.lock_count
 732
 733     c.lock
 734     c.reload
 735     assert_equal 1, c.lock_count
 736     assert_equal Container::Locked, c.state
 737
 738     c.unlock
 739     c.reload
 740     assert_equal 1, c.lock_count
 741     assert_equal Container::Queued, c.state
 742
 743     c.lock
 744     c.reload
 745     assert_equal 2, c.lock_count
 746     assert_equal Container::Locked, c.state
 747
 748     c.unlock
 749     c.reload
 750     assert_equal 2, c.lock_count
 751     assert_equal Container::Queued, c.state
 752
 753     c.lock
 754     c.reload
 755     assert_equal 3, c.lock_count
 756     assert_equal Container::Locked, c.state
 757
 758     c.unlock
 759     c.reload
 760     assert_equal 3, c.lock_count
 761     assert_equal Container::Cancelled, c.state
 762
 763     assert_raise(ArvadosModel::LockFailedError) do
 764       # Cancelled to Locked is not allowed
 765       c.lock
 766     end
 767   end
 768
 769   test "Container queued cancel" do
 770     set_user_from_auth :active
 771     c, cr = minimal_new({container_count_max: 1})
 772     set_user_from_auth :dispatch1
 773     assert c.update_attributes(state: Container::Cancelled), show_errors(c)
 774     check_no_change_from_cancelled c
 775     cr.reload
 776     assert_equal ContainerRequest::Final, cr.state
 777   end
 778
 779   test "Container queued count" do
 780     assert_equal 1, Container.readable_by(users(:active)).where(state: "Queued").count
 781   end
 782
 783   test "Containers with no matching request are readable by admin" do
 784     uuids = Container.includes('container_requests').where(container_requests: {uuid: nil}).collect(&:uuid)
 785     assert_not_empty uuids
 786     assert_empty Container.readable_by(users(:active)).where(uuid: uuids)
 787     assert_not_empty Container.readable_by(users(:admin)).where(uuid: uuids)
 788     assert_equal uuids.count, Container.readable_by(users(:admin)).where(uuid: uuids).count
 789   end
 790
 791   test "Container locked cancel" do
 792     set_user_from_auth :active
 793     c, _ = minimal_new
 794     set_user_from_auth :dispatch1
 795     assert c.lock, show_errors(c)
 796     assert c.update_attributes(state: Container::Cancelled), show_errors(c)
 797     check_no_change_from_cancelled c
 798   end
 799
 800   test "Container locked with non-expiring token" do
 801     Rails.configuration.API.TokenMaxLifetime = 1.hour
 802     set_user_from_auth :active
 803     c, _ = minimal_new
 804     set_user_from_auth :dispatch1
 805     assert c.lock, show_errors(c)
 806     refute c.auth.nil?
 807     assert c.auth.expires_at.nil?
 808     assert c.auth.user_id == User.find_by_uuid(users(:active).uuid).id
 809   end
 810
 811   test "Container locked cancel with log" do
 812     set_user_from_auth :active
 813     c, _ = minimal_new
 814     set_user_from_auth :dispatch1
 815     assert c.lock, show_errors(c)
 816     assert c.update_attributes(
 817              state: Container::Cancelled,
 818              log: collections(:real_log_collection).portable_data_hash,
 819            ), show_errors(c)
 820     check_no_change_from_cancelled c
 821   end
 822
 823   test "Container running cancel" do
 824     set_user_from_auth :active
 825     c, _ = minimal_new
 826     set_user_from_auth :dispatch1
 827     c.lock
 828     c.update_attributes! state: Container::Running
 829     c.update_attributes! state: Container::Cancelled
 830     check_no_change_from_cancelled c
 831   end
 832
 833   test "Container create forbidden for non-admin" do
 834     set_user_from_auth :active_trustedclient
 835     c = Container.new DEFAULT_ATTRS
 836     c.environment = {}
 837     c.mounts = {"BAR" => "FOO"}
 838     c.output_path = "/tmp"
 839     c.priority = 1
 840     c.runtime_constraints = {}
 841     assert_raises(ArvadosModel::PermissionDeniedError) do
 842       c.save!
 843     end
 844   end
 845
 846   [
 847     [Container::Queued, {state: Container::Locked}],
 848     [Container::Queued, {state: Container::Running}],
 849     [Container::Queued, {state: Container::Complete}],
 850     [Container::Queued, {state: Container::Cancelled}],
 851     [Container::Queued, {priority: 123456789}],
 852     [Container::Queued, {runtime_status: {'error' => 'oops'}}],
 853     [Container::Queued, {cwd: '/'}],
 854     [Container::Locked, {state: Container::Running}],
 855     [Container::Locked, {state: Container::Queued}],
 856     [Container::Locked, {priority: 123456789}],
 857     [Container::Locked, {runtime_status: {'error' => 'oops'}}],
 858     [Container::Locked, {cwd: '/'}],
 859     [Container::Running, {state: Container::Complete}],
 860     [Container::Running, {state: Container::Cancelled}],
 861     [Container::Running, {priority: 123456789}],
 862     [Container::Running, {runtime_status: {'error' => 'oops'}}],
 863     [Container::Running, {cwd: '/'}],
 864     [Container::Running, {gateway_address: "172.16.0.1:12345"}],
 865     [Container::Running, {interactive_session_started: true}],
 866     [Container::Complete, {state: Container::Cancelled}],
 867     [Container::Complete, {priority: 123456789}],
 868     [Container::Complete, {runtime_status: {'error' => 'oops'}}],
 869     [Container::Complete, {cwd: '/'}],
 870     [Container::Cancelled, {cwd: '/'}],
 871   ].each do |start_state, updates|
 872     test "Container update #{updates.inspect} when #{start_state} forbidden for non-admin" do
 873       set_user_from_auth :active
 874       c, _ = minimal_new
 875       if start_state != Container::Queued
 876         set_user_from_auth :dispatch1
 877         c.lock
 878         if start_state != Container::Locked
 879           c.update_attributes! state: Container::Running
 880           if start_state != Container::Running
 881             c.update_attributes! state: start_state
 882           end
 883         end
 884       end
 885       assert_equal c.state, start_state
 886       set_user_from_auth :active
 887       assert_raises(ArvadosModel::PermissionDeniedError) do
 888         c.update_attributes! updates
 889       end
 890     end
 891   end
 892
 893   test "can only change exit code while running and at completion" do
 894     set_user_from_auth :active
 895     c, _ = minimal_new
 896     set_user_from_auth :dispatch1
 897     c.lock
 898     check_illegal_updates c, [{exit_code: 1}]
 899     c.update_attributes! state: Container::Running
 900     assert c.update_attributes(exit_code: 1)
 901     assert c.update_attributes(exit_code: 1, state: Container::Complete)
 902   end
 903
 904   test "locked_by_uuid can update log when locked/running, and output when running" do
 905     set_user_from_auth :active
 906     logcoll = collections(:real_log_collection)
 907     c, cr1 = minimal_new
 908     cr2 = ContainerRequest.new(DEFAULT_ATTRS)
 909     cr2.state = ContainerRequest::Committed
 910     act_as_user users(:active) do
 911       cr2.save!
 912     end
 913     assert_equal cr1.container_uuid, cr2.container_uuid
 914
 915     logpdh_time1 = logcoll.portable_data_hash
 916
 917     set_user_from_auth :dispatch1
 918     c.lock
 919     assert_equal c.locked_by_uuid, Thread.current[:api_client_authorization].uuid
 920     c.update_attributes!(log: logpdh_time1)
 921     c.update_attributes!(state: Container::Running)
 922     cr1.reload
 923     cr2.reload
 924     cr1log_uuid = cr1.log_uuid
 925     cr2log_uuid = cr2.log_uuid
 926     assert_not_nil cr1log_uuid
 927     assert_not_nil cr2log_uuid
 928     assert_not_equal logcoll.uuid, cr1log_uuid
 929     assert_not_equal logcoll.uuid, cr2log_uuid
 930     assert_not_equal cr1log_uuid, cr2log_uuid
 931
 932     logcoll.update_attributes!(manifest_text: logcoll.manifest_text + ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n")
 933     logpdh_time2 = logcoll.portable_data_hash
 934
 935     assert c.update_attributes(output: collections(:collection_owned_by_active).portable_data_hash)
 936     assert c.update_attributes(log: logpdh_time2)
 937     assert c.update_attributes(state: Container::Complete, log: logcoll.portable_data_hash)
 938     c.reload
 939     assert_equal collections(:collection_owned_by_active).portable_data_hash, c.output
 940     assert_equal logpdh_time2, c.log
 941     refute c.update_attributes(output: nil)
 942     refute c.update_attributes(log: nil)
 943     cr1.reload
 944     cr2.reload
 945     assert_equal cr1log_uuid, cr1.log_uuid
 946     assert_equal cr2log_uuid, cr2.log_uuid
 947     assert_equal 1, Collection.where(uuid: [cr1log_uuid, cr2log_uuid]).to_a.collect(&:portable_data_hash).uniq.length
 948     assert_equal ". acbd18db4cc2f85cedef654fccc4a4d8+3 cdd549ae79fe6640fa3d5c6261d8303c+195 0:3:foo.txt 3:195:zzzzz-8i9sb-0vsrcqi7whchuil.log.txt
 949 ./log\\040for\\040container\\040#{cr1.container_uuid} acbd18db4cc2f85cedef654fccc4a4d8+3 cdd549ae79fe6640fa3d5c6261d8303c+195 0:3:foo.txt 3:195:zzzzz-8i9sb-0vsrcqi7whchuil.log.txt
 950 ", Collection.find_by_uuid(cr1log_uuid).manifest_text
 951   end
 952
 953   ["auth_uuid", "runtime_token"].each do |tok|
 954     test "#{tok} can set output, progress, runtime_status, state, exit_code on running container -- but not log" do
 955       if tok == "runtime_token"
 956         set_user_from_auth :spectator
 957         c, _ = minimal_new(container_image: "9ae44d5792468c58bcf85ce7353c7027+124",
 958                            runtime_token: api_client_authorizations(:active).token)
 959       else
 960         set_user_from_auth :active
 961         c, _ = minimal_new
 962       end
 963       set_user_from_auth :dispatch1
 964       c.lock
 965       c.update_attributes! state: Container::Running
 966
 967       if tok == "runtime_token"
 968         auth = ApiClientAuthorization.validate(token: c.runtime_token)
 969         Thread.current[:api_client_authorization] = auth
 970         Thread.current[:api_client] = auth.api_client
 971         Thread.current[:token] = auth.token
 972         Thread.current[:user] = auth.user
 973       else
 974         auth = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
 975         Thread.current[:api_client_authorization] = auth
 976         Thread.current[:api_client] = auth.api_client
 977         Thread.current[:token] = auth.token
 978         Thread.current[:user] = auth.user
 979       end
 980
 981       assert c.update_attributes(gateway_address: "127.0.0.1:9")
 982       assert c.update_attributes(output: collections(:collection_owned_by_active).portable_data_hash)
 983       assert c.update_attributes(runtime_status: {'warning' => 'something happened'})
 984       assert c.update_attributes(progress: 0.5)
 985       assert c.update_attributes(exit_code: 0)
 986       refute c.update_attributes(log: collections(:real_log_collection).portable_data_hash)
 987       c.reload
 988       assert c.update_attributes(state: Container::Complete, exit_code: 0)
 989     end
 990   end
 991
 992   test "not allowed to set output that is not readable by current user" do
 993     set_user_from_auth :active
 994     c, _ = minimal_new
 995     set_user_from_auth :dispatch1
 996     c.lock
 997     c.update_attributes! state: Container::Running
 998
 999     Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
1000     Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
1001
1002     assert_raises ActiveRecord::RecordInvalid do
1003       c.update_attributes! output: collections(:collection_not_readable_by_active).portable_data_hash
1004     end
1005   end
1006
1007   test "other token cannot set output on running container" do
1008     set_user_from_auth :active
1009     c, _ = minimal_new
1010     set_user_from_auth :dispatch1
1011     c.lock
1012     c.update_attributes! state: Container::Running
1013
1014     set_user_from_auth :running_to_be_deleted_container_auth
1015     assert_raises(ArvadosModel::PermissionDeniedError) do
1016       c.update_attributes(output: collections(:foo_file).portable_data_hash)
1017     end
1018   end
1019
1020   test "can set trashed output on running container" do
1021     set_user_from_auth :active
1022     c, _ = minimal_new
1023     set_user_from_auth :dispatch1
1024     c.lock
1025     c.update_attributes! state: Container::Running
1026
1027     output = Collection.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jk')
1028
1029     assert output.is_trashed
1030     assert c.update_attributes output: output.portable_data_hash
1031     assert c.update_attributes! state: Container::Complete
1032   end
1033
1034   test "not allowed to set trashed output that is not readable by current user" do
1035     set_user_from_auth :active
1036     c, _ = minimal_new
1037     set_user_from_auth :dispatch1
1038     c.lock
1039     c.update_attributes! state: Container::Running
1040
1041     output = Collection.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jr')
1042
1043     Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
1044     Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
1045
1046     assert_raises ActiveRecord::RecordInvalid do
1047       c.update_attributes! output: output.portable_data_hash
1048     end
1049   end
1050
1051   test "user cannot delete" do
1052     set_user_from_auth :active
1053     c, _ = minimal_new
1054     assert_raises ArvadosModel::PermissionDeniedError do
1055       c.destroy
1056     end
1057     assert Container.find_by_uuid(c.uuid)
1058   end
1059
1060   [
1061     {state: Container::Complete, exit_code: 0, output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'},
1062     {state: Container::Cancelled},
1063   ].each do |final_attrs|
1064     test "secret_mounts and runtime_token are null after container is #{final_attrs[:state]}" do
1065       set_user_from_auth :active
1066       c, cr = minimal_new(secret_mounts: {'/secret' => {'kind' => 'text', 'content' => 'foo'}},
1067                           container_count_max: 1, runtime_token: api_client_authorizations(:active).token)
1068       set_user_from_auth :dispatch1
1069       c.lock
1070       c.update_attributes!(state: Container::Running)
1071       c.reload
1072       assert c.secret_mounts.has_key?('/secret')
1073       assert_equal api_client_authorizations(:active).token, c.runtime_token
1074
1075       c.update_attributes!(final_attrs)
1076       c.reload
1077       assert_equal({}, c.secret_mounts)
1078       assert_nil c.runtime_token
1079       cr.reload
1080       assert_equal({}, cr.secret_mounts)
1081       assert_nil cr.runtime_token
1082       assert_no_secrets_logged
1083     end
1084   end
1085
1086   def configure_preemptible_instance_type
1087     Rails.configuration.InstanceTypes = ConfigLoader.to_OrderedOptions({
1088       "a1.small.pre" => {
1089         "Preemptible" => true,
1090         "Price" => 0.1,
1091         "ProviderType" => "a1.small",
1092         "VCPUs" => 1,
1093         "RAM" => 1000000000,
1094       },
1095     })
1096   end
1097
1098   def vary_parameters(**kwargs)
1099     # kwargs is a hash that maps parameters to an array of values.
1100     # This function enumerates every possible hash where each key has one of
1101     # the values from its array.
1102     # The output keys are strings since that's what container hash attributes
1103     # want.
1104     # A nil value yields a hash without that key.
1105     [[:_, nil]].product(
1106       *kwargs.map { |(key, values)| [key.to_s].product(values) },
1107     ).map { |param_pairs| Hash[param_pairs].compact }
1108   end
1109
1110   def retry_with_scheduling_parameters(param_hashes)
1111     set_user_from_auth :admin
1112     containers = {}
1113     requests = []
1114     param_hashes.each do |scheduling_parameters|
1115       container, request = minimal_new(scheduling_parameters: scheduling_parameters)
1116       containers[container.uuid] = container
1117       requests << request
1118     end
1119     refute(containers.empty?, "buggy test: no scheduling parameters enumerated")
1120     assert_equal(1, containers.length)
1121     _, container1 = containers.shift
1122     container1.lock
1123     container1.update_attributes!(state: Container::Cancelled)
1124     container1.reload
1125     request1 = requests.shift
1126     request1.reload
1127     assert_not_equal(container1.uuid, request1.container_uuid)
1128     requests.each do |request|
1129       request.reload
1130       assert_equal(request1.container_uuid, request.container_uuid)
1131     end
1132     container2 = Container.find_by_uuid(request1.container_uuid)
1133     assert_not_nil(container2)
1134     return container2
1135   end
1136
1137   preemptible_values = [true, false, nil]
1138   preemptible_values.permutation(1).chain(
1139     preemptible_values.product(preemptible_values),
1140     preemptible_values.product(preemptible_values, preemptible_values),
1141   ).each do |preemptible_a|
1142     test "retry requests scheduled with preemptible=#{preemptible_a}" do
1143       configure_preemptible_instance_type
1144       param_hashes = vary_parameters(preemptible: preemptible_a)
1145       container = retry_with_scheduling_parameters(param_hashes)
1146       assert_equal(preemptible_a.all?,
1147                    container.scheduling_parameters["preemptible"] || false)
1148     end
1149   end
1150
1151   partition_values = [nil, [], ["alpha"], ["alpha", "bravo"], ["bravo", "charlie"]]
1152   partition_values.permutation(1).chain(
1153     partition_values.permutation(2),
1154   ).each do |partitions_a|
1155     test "retry requests scheduled with partitions=#{partitions_a}" do
1156       param_hashes = vary_parameters(partitions: partitions_a)
1157       container = retry_with_scheduling_parameters(param_hashes)
1158       expected = if partitions_a.any? { |value| value.nil? or value.empty? }
1159                    []
1160                  else
1161                    partitions_a.flatten.uniq
1162                  end
1163       actual = container.scheduling_parameters["partitions"] || []
1164       assert_equal(expected.sort, actual.sort)
1165     end
1166   end
1167
1168   runtime_values = [nil, 0, 1, 2, 3]
1169   runtime_values.permutation(1).chain(
1170     runtime_values.permutation(2),
1171     runtime_values.permutation(3),
1172   ).each do |max_run_time_a|
1173     test "retry requests scheduled with max_run_time=#{max_run_time_a}" do
1174       param_hashes = vary_parameters(max_run_time: max_run_time_a)
1175       container = retry_with_scheduling_parameters(param_hashes)
1176       expected = if max_run_time_a.any? { |value| value.nil? or value == 0 }
1177                    0
1178                  else
1179                    max_run_time_a.max
1180                  end
1181       actual = container.scheduling_parameters["max_run_time"] || 0
1182       assert_equal(expected, actual)
1183     end
1184   end
1185
1186   test "retry requests with multi-varied scheduling parameters" do
1187     configure_preemptible_instance_type
1188     param_hashes = [{
1189                      "partitions": ["alpha", "bravo"],
1190                      "preemptible": true,
1191                      "max_run_time": 10,
1192                     }, {
1193                      "partitions": ["alpha", "charlie"],
1194                      "max_run_time": 20,
1195                     }, {
1196                      "partitions": ["bravo", "charlie"],
1197                      "preemptible": false,
1198                      "max_run_time": 30,
1199                     }]
1200     container = retry_with_scheduling_parameters(param_hashes)
1201     actual = container.scheduling_parameters
1202     assert_equal(["alpha", "bravo", "charlie"], actual["partitions"]&.sort)
1203     assert_equal(false, actual["preemptible"] || false)
1204     assert_equal(30, actual["max_run_time"])
1205   end
1206
1207   test "retry requests with unset scheduling parameters" do
1208     configure_preemptible_instance_type
1209     param_hashes = vary_parameters(
1210       preemptible: [nil, true],
1211       partitions: [nil, ["alpha"]],
1212       max_run_time: [nil, 5],
1213     )
1214     container = retry_with_scheduling_parameters(param_hashes)
1215     actual = container.scheduling_parameters
1216     assert_equal([], actual["partitions"] || [])
1217     assert_equal(false, actual["preemptible"] || false)
1218     assert_equal(0, actual["max_run_time"] || 0)
1219   end
1220
1221   test "retry requests with default scheduling parameters" do
1222     configure_preemptible_instance_type
1223     param_hashes = vary_parameters(
1224       preemptible: [false, true],
1225       partitions: [[], ["bravo"]],
1226       max_run_time: [0, 1],
1227     )
1228     container = retry_with_scheduling_parameters(param_hashes)
1229     actual = container.scheduling_parameters
1230     assert_equal([], actual["partitions"] || [])
1231     assert_equal(false, actual["preemptible"] || false)
1232     assert_equal(0, actual["max_run_time"] || 0)
1233   end
1234
1235   def run_container(request_params, final_attrs)
1236     final_attrs[:state] ||= Container::Complete
1237     if final_attrs[:state] == Container::Complete
1238       final_attrs[:exit_code] ||= 0
1239       final_attrs[:log] ||= collections(:log_collection).portable_data_hash
1240       final_attrs[:output] ||= collections(:multilevel_collection_1).portable_data_hash
1241     end
1242     container, request = minimal_new(request_params)
1243     container.lock
1244     container.update_attributes!(state: Container::Running)
1245     container.update_attributes!(final_attrs)
1246     return container, request
1247   end
1248
1249   def check_reuse_with_variations(default_keep_cache_ram, vary_attr, start_value, variations)
1250     container_params = REUSABLE_ATTRS_SLIM.merge(vary_attr => start_value)
1251     orig_default = Rails.configuration.Containers.DefaultKeepCacheRAM
1252     begin
1253       Rails.configuration.Containers.DefaultKeepCacheRAM = default_keep_cache_ram
1254       set_user_from_auth :admin
1255       expected, _ = run_container(container_params, {})
1256       variations.each do |variation|
1257         full_variation = REUSABLE_ATTRS_SLIM[vary_attr].merge(variation)
1258         parameters = REUSABLE_ATTRS_SLIM.merge(vary_attr => full_variation)
1259         actual = Container.find_reusable(parameters)
1260         assert_equal(expected.uuid, actual&.uuid,
1261                      "request with #{vary_attr}=#{variation} did not reuse container")
1262       end
1263     ensure
1264       Rails.configuration.Containers.DefaultKeepCacheRAM = orig_default
1265     end
1266   end
1267
1268   # Test that we can reuse a container with a known keep_cache_ram constraint,
1269   # no matter what keep_cache_* constraints the new request uses.
1270   [0, 2 << 30, 4 << 30].product(
1271     [0, 1],
1272     [true, false],
1273   ).each do |(default_keep_cache_ram, multiplier, keep_disk_constraint)|
1274     test "reuse request with DefaultKeepCacheRAM=#{default_keep_cache_ram}, keep_cache_ram*=#{multiplier}, keep_cache_disk=#{keep_disk_constraint}" do
1275       runtime_constraints = REUSABLE_ATTRS_SLIM[:runtime_constraints].merge(
1276         "keep_cache_ram" => default_keep_cache_ram * multiplier,
1277       )
1278       if not keep_disk_constraint
1279         # Simulate a container that predates keep_cache_disk by deleting
1280         # the constraint entirely.
1281         runtime_constraints.delete("keep_cache_disk")
1282       end
1283       # Important values are:
1284       # * 0
1285       # * 2GiB, the minimum default keep_cache_disk
1286       # * 8GiB, the default keep_cache_disk based on container ram
1287       # * 32GiB, the maximum default keep_cache_disk
1288       # Check these values and values in between.
1289       vary_values = [0, 1, 2, 6, 8, 10, 32, 33].map { |v| v << 30 }.to_a
1290       variations = vary_parameters(keep_cache_ram: vary_values)
1291                      .chain(vary_parameters(keep_cache_disk: vary_values))
1292       check_reuse_with_variations(
1293         default_keep_cache_ram,
1294         :runtime_constraints,
1295         runtime_constraints,
1296         variations,
1297       )
1298     end
1299   end
1300
1301   # Test that we can reuse a container with a known keep_cache_disk constraint,
1302   # no matter what keep_cache_* constraints the new request uses.
1303   # keep_cache_disk values are the important values discussed in the test above.
1304   [0, 2 << 30, 4 << 30]
1305     .product([0, 2 << 30, 8 << 30, 32 << 30])
1306     .each do |(default_keep_cache_ram, keep_cache_disk)|
1307     test "reuse request with DefaultKeepCacheRAM=#{default_keep_cache_ram} and keep_cache_disk=#{keep_cache_disk}" do
1308       runtime_constraints = REUSABLE_ATTRS_SLIM[:runtime_constraints].merge(
1309         "keep_cache_disk" => keep_cache_disk,
1310       )
1311       vary_values = [0, 1, 2, 6, 8, 10, 32, 33].map { |v| v << 30 }.to_a
1312       variations = vary_parameters(keep_cache_ram: vary_values)
1313                      .chain(vary_parameters(keep_cache_disk: vary_values))
1314       check_reuse_with_variations(
1315         default_keep_cache_ram,
1316         :runtime_constraints,
1317         runtime_constraints,
1318         variations,
1319       )
1320     end
1321   end
1322
1323   # Test that a container request can reuse a container with an exactly
1324   # matching keep_cache_* constraint, no matter what the defaults.
1325   [0, 2 << 30, 4 << 30].product(
1326     ["keep_cache_disk", "keep_cache_ram"],
1327     [135790, 13 << 30, 135 << 30],
1328   ).each do |(default_keep_cache_ram, constraint_key, constraint_value)|
1329     test "reuse request with #{constraint_key}=#{constraint_value} and DefaultKeepCacheRAM=#{default_keep_cache_ram}" do
1330       runtime_constraints = REUSABLE_ATTRS_SLIM[:runtime_constraints].merge(
1331         constraint_key => constraint_value,
1332       )
1333       check_reuse_with_variations(
1334         default_keep_cache_ram,
1335         :runtime_constraints,
1336         runtime_constraints,
1337         [runtime_constraints],
1338       )
1339     end
1340   end
1341 end