services/api/test/unit/container_test.rb

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 require 'test_helper'
   6 require 'helpers/container_test_helper'
   7
   8 class ContainerTest < ActiveSupport::TestCase
   9   include DbCurrentTime
  10   include ContainerTestHelper
  11
  12   DEFAULT_ATTRS = {
  13     command: ['echo', 'foo'],
  14     container_image: 'fa3c1a9cb6783f85f2ecda037e07b8c3+167',
  15     output_path: '/tmp',
  16     priority: 1,
  17     runtime_constraints: {"vcpus" => 1, "ram" => 1, "cuda" => {"device_count":0, "driver_version": "", "hardware_capability": ""}},
  18   }
  19
  20   REUSABLE_COMMON_ATTRS = {
  21     container_image: "9ae44d5792468c58bcf85ce7353c7027+124",
  22     cwd: "test",
  23     command: ["echo", "hello"],
  24     output_path: "test",
  25     output_glob: [],
  26     runtime_constraints: {
  27       "API" => false,
  28       "keep_cache_disk" => 0,
  29       "keep_cache_ram" => 0,
  30       "ram" => 12000000000,
  31       "vcpus" => 4
  32     },
  33     mounts: {
  34       "test" => {"kind" => "json"},
  35     },
  36     environment: {
  37       "var" => "val",
  38     },
  39     secret_mounts: {},
  40     runtime_user_uuid: "zzzzz-tpzed-xurymjxw79nv3jz",
  41     runtime_auth_scopes: ["all"],
  42     scheduling_parameters: {},
  43   }
  44
  45   REUSABLE_ATTRS_SLIM = {
  46     command: ["echo", "slim"],
  47     container_image: "9ae44d5792468c58bcf85ce7353c7027+124",
  48     cwd: "test",
  49     environment: {},
  50     mounts: {},
  51     output_path: "test",
  52     output_glob: [],
  53     runtime_auth_scopes: ["all"],
  54     runtime_constraints: {
  55       "API" => false,
  56       "keep_cache_disk" => 0,
  57       "keep_cache_ram" => 0,
  58       "ram" => 8 << 30,
  59       "vcpus" => 4
  60     },
  61     runtime_user_uuid: "zzzzz-tpzed-xurymjxw79nv3jz",
  62     secret_mounts: {},
  63     scheduling_parameters: {},
  64   }
  65
  66   def request_only attrs
  67     attrs.reject {|k| [:runtime_user_uuid, :runtime_auth_scopes].include? k}
  68   end
  69
  70   def minimal_new attrs={}
  71     cr = ContainerRequest.new request_only(DEFAULT_ATTRS.merge(attrs))
  72     cr.state = ContainerRequest::Committed
  73     cr.save!
  74     c = Container.find_by_uuid cr.container_uuid
  75     assert_not_nil c
  76     return c, cr
  77   end
  78
  79   def check_illegal_updates c, bad_updates
  80     bad_updates.each do |u|
  81       refute c.update(u), u.inspect
  82       refute c.valid?, u.inspect
  83       c.reload
  84     end
  85   end
  86
  87   def check_illegal_modify c
  88     check_illegal_updates c, [{command: ["echo", "bar"]},
  89                               {container_image: "arvados/apitestfixture:june10"},
  90                               {cwd: "/tmp2"},
  91                               {environment: {"FOO" => "BAR"}},
  92                               {mounts: {"FOO" => "BAR"}},
  93                               {output_path: "/tmp3"},
  94                               {locked_by_uuid: api_client_authorizations(:admin).uuid},
  95                               {auth_uuid: api_client_authorizations(:system_user).uuid},
  96                               {runtime_constraints: {"FOO" => "BAR"}}]
  97   end
  98
  99   def check_bogus_states c
 100     check_illegal_updates c, [{state: nil},
 101                               {state: "Flubber"}]
 102   end
 103
 104   def check_no_change_from_cancelled c
 105     check_illegal_modify c
 106     check_bogus_states c
 107     check_illegal_updates c, [{ priority: 3 },
 108                               { state: Container::Queued },
 109                               { state: Container::Locked },
 110                               { state: Container::Running },
 111                               { state: Container::Complete }]
 112   end
 113
 114   test "Container create" do
 115     act_as_system_user do
 116       c, _ = minimal_new(environment: {},
 117                       mounts: {"BAR" => {"kind" => "FOO"}},
 118                       output_path: "/tmp",
 119                       priority: 1,
 120                       runtime_constraints: {"vcpus" => 1, "ram" => 1})
 121
 122       check_illegal_modify c
 123       check_bogus_states c
 124
 125       c.reload
 126       c.priority = 2
 127       c.save!
 128     end
 129   end
 130
 131   test "Container valid priority" do
 132     act_as_system_user do
 133       c, _ = minimal_new(environment: {},
 134                       mounts: {"BAR" => {"kind" => "FOO"}},
 135                       output_path: "/tmp",
 136                       priority: 1,
 137                       runtime_constraints: {"vcpus" => 1, "ram" => 1})
 138
 139       assert_raises(ActiveRecord::RecordInvalid) do
 140         c.priority = -1
 141         c.save!
 142       end
 143
 144       c.priority = 0
 145       c.save!
 146
 147       c.priority = 1
 148       c.save!
 149
 150       c.priority = 500
 151       c.save!
 152
 153       c.priority = 999
 154       c.save!
 155
 156       c.priority = 1000
 157       c.save!
 158
 159       c.priority = 1000 << 50
 160       c.save!
 161     end
 162   end
 163
 164   test "Container runtime_status data types" do
 165     set_user_from_auth :active
 166     attrs = {
 167       environment: {},
 168       mounts: {"BAR" => {"kind" => "FOO"}},
 169       output_path: "/tmp",
 170       priority: 1,
 171       runtime_constraints: {"vcpus" => 1, "ram" => 1}
 172     }
 173     c, _ = minimal_new(attrs)
 174     assert_equal c.runtime_status, {}
 175     assert_equal Container::Queued, c.state
 176
 177     set_user_from_auth :system_user
 178     c.update! state: Container::Locked
 179     c.update! state: Container::Running
 180
 181     [
 182       'error', 'errorDetail', 'warning', 'warningDetail', 'activity'
 183     ].each do |k|
 184       # String type is allowed
 185       string_val = 'A string is accepted'
 186       c.update! runtime_status: {k => string_val}
 187       assert_equal string_val, c.runtime_status[k]
 188
 189       # Other types aren't allowed
 190       [
 191         42, false, [], {}, nil
 192       ].each do |unallowed_val|
 193         assert_raises ActiveRecord::RecordInvalid do
 194           c.update! runtime_status: {k => unallowed_val}
 195         end
 196       end
 197     end
 198   end
 199
 200   test "Container runtime_status updates" do
 201     set_user_from_auth :active
 202     attrs = {
 203       environment: {},
 204       mounts: {"BAR" => {"kind" => "FOO"}},
 205       output_path: "/tmp",
 206       priority: 1,
 207       runtime_constraints: {"vcpus" => 1, "ram" => 1}
 208     }
 209     c1, _ = minimal_new(attrs)
 210     assert_equal c1.runtime_status, {}
 211
 212     assert_equal Container::Queued, c1.state
 213     assert_raises ArvadosModel::PermissionDeniedError do
 214       c1.update! runtime_status: {'error' => 'Oops!'}
 215     end
 216
 217     set_user_from_auth :system_user
 218
 219     # Allow updates when state = Locked
 220     c1.update! state: Container::Locked
 221     c1.update! runtime_status: {'error' => 'Oops!'}
 222     assert c1.runtime_status.key? 'error'
 223
 224     # Reset when transitioning from Locked to Queued
 225     c1.update! state: Container::Queued
 226     assert_equal c1.runtime_status, {}
 227
 228     # Allow updates when state = Running
 229     c1.update! state: Container::Locked
 230     c1.update! state: Container::Running
 231     c1.update! runtime_status: {'error' => 'Oops!'}
 232     assert c1.runtime_status.key? 'error'
 233
 234     # Don't allow updates on other states
 235     c1.update! state: Container::Complete
 236     assert_raises ActiveRecord::RecordInvalid do
 237       c1.update! runtime_status: {'error' => 'Some other error'}
 238     end
 239
 240     set_user_from_auth :active
 241     c2, _ = minimal_new(attrs)
 242     assert_equal c2.runtime_status, {}
 243     set_user_from_auth :system_user
 244     c2.update! state: Container::Locked
 245     c2.update! state: Container::Running
 246     c2.update! state: Container::Cancelled
 247     assert_raises ActiveRecord::RecordInvalid do
 248       c2.update! runtime_status: {'error' => 'Oops!'}
 249     end
 250   end
 251
 252   test "Container serialized hash attributes sorted before save" do
 253     set_user_from_auth :active
 254     env = {"C" => "3", "B" => "2", "A" => "1"}
 255     m = {"F" => {"kind" => "3"}, "E" => {"kind" => "2"}, "D" => {"kind" => "1"}}
 256     rc = {"vcpus" => 1, "ram" => 1, "keep_cache_ram" => 1, "keep_cache_disk" => 0, "API" => true, "gpu" => {"stack": "", "device_count":0, "driver_version": "", "hardware_target": [], "vram": 0}}
 257     c, _ = minimal_new(environment: env, mounts: m, runtime_constraints: rc)
 258     c.reload
 259     assert_equal Container.deep_sort_hash(env).to_json, c.environment.to_json
 260     assert_equal Container.deep_sort_hash(m).to_json, c.mounts.to_json
 261     assert_equal Container.deep_sort_hash(rc).to_json, c.runtime_constraints.to_json
 262   end
 263
 264   test 'deep_sort_hash on array of hashes' do
 265     a = {'z' => [[{'a' => 'a', 'b' => 'b'}]]}
 266     b = {'z' => [[{'b' => 'b', 'a' => 'a'}]]}
 267     assert_equal Container.deep_sort_hash(a).to_json, Container.deep_sort_hash(b).to_json
 268   end
 269
 270   test "find_reusable method should select higher priority queued container" do
 271         Rails.configuration.Containers.LogReuseDecisions = true
 272     set_user_from_auth :active
 273     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment:{"var" => "queued"}})
 274     c_low_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:1}))
 275     c_high_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:2}))
 276     assert_not_equal c_low_priority.uuid, c_high_priority.uuid
 277     assert_equal Container::Queued, c_low_priority.state
 278     assert_equal Container::Queued, c_high_priority.state
 279     reused = Container.find_reusable(common_attrs)
 280     assert_not_nil reused
 281     assert_equal reused.uuid, c_high_priority.uuid
 282   end
 283
 284   test "find_reusable method should select latest completed container" do
 285     set_user_from_auth :active
 286     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "complete"}})
 287     completed_attrs = {
 288       state: Container::Complete,
 289       exit_code: 0,
 290       log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 291       output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
 292     }
 293
 294     c_older, _ = minimal_new(common_attrs.merge({use_existing: false}))
 295     c_recent, _ = minimal_new(common_attrs.merge({use_existing: false}))
 296     assert_not_equal c_older.uuid, c_recent.uuid
 297
 298     set_user_from_auth :system_user
 299     c_older.update!({state: Container::Locked})
 300     c_older.update!({state: Container::Running})
 301     c_older.update!(completed_attrs)
 302
 303     c_recent.update!({state: Container::Locked})
 304     c_recent.update!({state: Container::Running})
 305     c_recent.update!(completed_attrs)
 306
 307     reused = Container.find_reusable(common_attrs)
 308     assert_not_nil reused
 309     assert_equal reused.uuid, c_older.uuid
 310   end
 311
 312   test "find_reusable method should select oldest completed container when inconsistent outputs exist" do
 313     set_user_from_auth :active
 314     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "complete"}, priority: 1})
 315     completed_attrs = {
 316       state: Container::Complete,
 317       exit_code: 0,
 318       log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 319     }
 320
 321     cr = ContainerRequest.new request_only(common_attrs)
 322     cr.use_existing = false
 323     cr.state = ContainerRequest::Committed
 324     cr.save!
 325     c_output1 = Container.where(uuid: cr.container_uuid).first
 326
 327     cr = ContainerRequest.new request_only(common_attrs)
 328     cr.use_existing = false
 329     cr.state = ContainerRequest::Committed
 330     cr.save!
 331     c_output2 = Container.where(uuid: cr.container_uuid).first
 332
 333     assert_not_equal c_output1.uuid, c_output2.uuid
 334
 335     set_user_from_auth :system_user
 336
 337     out1 = '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
 338     log1 = collections(:log_collection).portable_data_hash
 339     c_output1.update!({state: Container::Locked})
 340     c_output1.update!({state: Container::Running})
 341     c_output1.update!(completed_attrs.merge({log: log1, output: out1}))
 342
 343     out2 = 'fa7aeb5140e2848d39b416daeef4ffc5+45'
 344     c_output2.update!({state: Container::Locked})
 345     c_output2.update!({state: Container::Running})
 346     c_output2.update!(completed_attrs.merge({log: log1, output: out2}))
 347
 348     set_user_from_auth :active
 349     reused = Container.resolve(ContainerRequest.new(request_only(common_attrs)))
 350     assert_equal c_output1.uuid, reused.uuid
 351   end
 352
 353   test "find_reusable method should select running container by start date" do
 354     set_user_from_auth :active
 355     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running"}})
 356     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
 357     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
 358     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
 359     # Confirm the 3 container UUIDs are different.
 360     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
 361     set_user_from_auth :system_user
 362     c_slower.update!({state: Container::Locked})
 363     c_slower.update!({state: Container::Running,
 364                                  progress: 0.1})
 365     c_faster_started_first.update!({state: Container::Locked})
 366     c_faster_started_first.update!({state: Container::Running,
 367                                                progress: 0.15})
 368     c_faster_started_second.update!({state: Container::Locked})
 369     c_faster_started_second.update!({state: Container::Running,
 370                                                 progress: 0.15})
 371     reused = Container.find_reusable(common_attrs)
 372     assert_not_nil reused
 373     # Selected container is the one that started first
 374     assert_equal reused.uuid, c_faster_started_first.uuid
 375   end
 376
 377   test "find_reusable method should select running container by progress" do
 378     set_user_from_auth :active
 379     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running2"}})
 380     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
 381     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
 382     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
 383     # Confirm the 3 container UUIDs are different.
 384     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
 385     set_user_from_auth :system_user
 386     c_slower.update!({state: Container::Locked})
 387     c_slower.update!({state: Container::Running,
 388                                  progress: 0.1})
 389     c_faster_started_first.update!({state: Container::Locked})
 390     c_faster_started_first.update!({state: Container::Running,
 391                                                progress: 0.15})
 392     c_faster_started_second.update!({state: Container::Locked})
 393     c_faster_started_second.update!({state: Container::Running,
 394                                                 progress: 0.2})
 395     reused = Container.find_reusable(common_attrs)
 396     assert_not_nil reused
 397     # Selected container is the one with most progress done
 398     assert_equal reused.uuid, c_faster_started_second.uuid
 399   end
 400
 401   test "find_reusable method should select non-failing running container" do
 402     set_user_from_auth :active
 403     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running2"}})
 404     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
 405     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
 406     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
 407     # Confirm the 3 container UUIDs are different.
 408     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
 409     set_user_from_auth :system_user
 410     c_slower.update!({state: Container::Locked})
 411     c_slower.update!({state: Container::Running,
 412                                  progress: 0.1})
 413     c_faster_started_first.update!({state: Container::Locked})
 414     c_faster_started_first.update!({state: Container::Running,
 415                                                runtime_status: {'warning' => 'This is not an error'},
 416                                                progress: 0.15})
 417     c_faster_started_second.update!({state: Container::Locked})
 418     assert_equal 0, Container.where("runtime_status->'error' is not null").count
 419     c_faster_started_second.update!({state: Container::Running,
 420                                                 runtime_status: {'error' => 'Something bad happened'},
 421                                                 progress: 0.2})
 422     assert_equal 1, Container.where("runtime_status->'error' is not null").count
 423     reused = Container.find_reusable(common_attrs)
 424     assert_not_nil reused
 425     # Selected the non-failing container even if it's the one with less progress done
 426     assert_equal reused.uuid, c_faster_started_first.uuid
 427   end
 428
 429   test "find_reusable method should select locked container most likely to start sooner" do
 430     set_user_from_auth :active
 431     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "locked"}})
 432     c_low_priority, _ = minimal_new(common_attrs.merge({use_existing: false}))
 433     c_high_priority_older, _ = minimal_new(common_attrs.merge({use_existing: false}))
 434     c_high_priority_newer, _ = minimal_new(common_attrs.merge({use_existing: false}))
 435     # Confirm the 3 container UUIDs are different.
 436     assert_equal 3, [c_low_priority.uuid, c_high_priority_older.uuid, c_high_priority_newer.uuid].uniq.length
 437     set_user_from_auth :system_user
 438     c_low_priority.update!({state: Container::Locked,
 439                                        priority: 1})
 440     c_high_priority_older.update!({state: Container::Locked,
 441                                               priority: 2})
 442     c_high_priority_newer.update!({state: Container::Locked,
 443                                               priority: 2})
 444     reused = Container.find_reusable(common_attrs)
 445     assert_not_nil reused
 446     assert_equal reused.uuid, c_high_priority_older.uuid
 447   end
 448
 449   test "find_reusable method should select running over failed container" do
 450     set_user_from_auth :active
 451     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed_vs_running"}})
 452     c_failed, _ = minimal_new(common_attrs.merge({use_existing: false}))
 453     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
 454     assert_not_equal c_failed.uuid, c_running.uuid
 455     set_user_from_auth :system_user
 456     c_failed.update!({state: Container::Locked})
 457     c_failed.update!({state: Container::Running})
 458     c_failed.update!({state: Container::Complete,
 459                                  exit_code: 42,
 460                                  log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 461                                  output: 'ea10d51bcf88862dbcc36eb292017dfd+45'})
 462     c_running.update!({state: Container::Locked})
 463     c_running.update!({state: Container::Running,
 464                                   progress: 0.15})
 465     reused = Container.find_reusable(common_attrs)
 466     assert_not_nil reused
 467     assert_equal reused.uuid, c_running.uuid
 468   end
 469
 470   test "find_reusable method should select complete over running container" do
 471     set_user_from_auth :active
 472     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "completed_vs_running"}})
 473     c_completed, _ = minimal_new(common_attrs.merge({use_existing: false}))
 474     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
 475     assert_not_equal c_completed.uuid, c_running.uuid
 476     set_user_from_auth :system_user
 477     c_completed.update!({state: Container::Locked})
 478     c_completed.update!({state: Container::Running})
 479     c_completed.update!({state: Container::Complete,
 480                                     exit_code: 0,
 481                                     log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 482                                     output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'})
 483     c_running.update!({state: Container::Locked})
 484     c_running.update!({state: Container::Running,
 485                                   progress: 0.15})
 486     reused = Container.find_reusable(common_attrs)
 487     assert_not_nil reused
 488     assert_equal c_completed.uuid, reused.uuid
 489   end
 490
 491   test "find_reusable method should select running over locked container" do
 492     set_user_from_auth :active
 493     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}})
 494     c_locked, _ = minimal_new(common_attrs.merge({use_existing: false}))
 495     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
 496     assert_not_equal c_running.uuid, c_locked.uuid
 497     set_user_from_auth :system_user
 498     c_locked.update!({state: Container::Locked})
 499     c_running.update!({state: Container::Locked})
 500     c_running.update!({state: Container::Running,
 501                                   progress: 0.15})
 502     reused = Container.find_reusable(common_attrs)
 503     assert_not_nil reused
 504     assert_equal reused.uuid, c_running.uuid
 505   end
 506
 507   test "find_reusable method should select locked over queued container" do
 508     set_user_from_auth :active
 509     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}})
 510     c_locked, _ = minimal_new(common_attrs.merge({use_existing: false}))
 511     c_queued, _ = minimal_new(common_attrs.merge({use_existing: false}))
 512     assert_not_equal c_queued.uuid, c_locked.uuid
 513     set_user_from_auth :system_user
 514     c_locked.update!({state: Container::Locked})
 515     reused = Container.find_reusable(common_attrs)
 516     assert_not_nil reused
 517     assert_equal reused.uuid, c_locked.uuid
 518   end
 519
 520   test "find_reusable method should not select failed container" do
 521     set_user_from_auth :active
 522     attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed"}})
 523     c, _ = minimal_new(attrs)
 524     set_user_from_auth :system_user
 525     c.update!({state: Container::Locked})
 526     c.update!({state: Container::Running})
 527     c.update!({state: Container::Complete,
 528                           exit_code: 33})
 529     reused = Container.find_reusable(attrs)
 530     assert_nil reused
 531   end
 532
 533   [[false, false, true],
 534    [false, true, true],
 535    [true, false, false],
 536    [true, true, true]
 537   ].each do |c1_preemptible, c2_preemptible, should_reuse|
 538     [[Container::Queued, 1],
 539      [Container::Locked, 1],
 540      [Container::Running, 0],   # not cancelled yet, but obviously will be soon
 541     ].each do |c1_state, c1_priority|
 542       test "find_reusable for #{c2_preemptible ? '' : 'non-'}preemptible req should #{should_reuse ? '' : 'not'} reuse a #{c1_state} #{c1_preemptible ? '' : 'non-'}preemptible container with priority #{c1_priority}" do
 543         configure_preemptible_instance_type
 544         set_user_from_auth :active
 545         c1_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"test" => name, "state" => c1_state}, scheduling_parameters: {"preemptible" => c1_preemptible}})
 546         c1, _ = minimal_new(c1_attrs)
 547         set_user_from_auth :system_user
 548         c1.update!({state: Container::Locked}) if c1_state != Container::Queued
 549         c1.update!({state: Container::Running, priority: c1_priority}) if c1_state == Container::Running
 550         c2_attrs = c1_attrs.merge({scheduling_parameters: {"preemptible" => c2_preemptible}})
 551         reused = Container.find_reusable(c2_attrs)
 552         if should_reuse && c1_priority > 0
 553           assert_not_nil reused
 554         else
 555           assert_nil reused
 556         end
 557       end
 558     end
 559   end
 560
 561   test "find_reusable with logging disabled" do
 562     set_user_from_auth :active
 563     Rails.logger.expects(:info).never
 564     Container.find_reusable(REUSABLE_COMMON_ATTRS)
 565   end
 566
 567   test "find_reusable with logging enabled" do
 568     set_user_from_auth :active
 569     Rails.configuration.Containers.LogReuseDecisions = true
 570     Rails.logger.expects(:info).at_least(3)
 571     Container.find_reusable(REUSABLE_COMMON_ATTRS)
 572   end
 573
 574   def runtime_token_attr tok
 575     auth = api_client_authorizations(tok)
 576     {runtime_user_uuid: User.find_by_id(auth.user_id).uuid,
 577      runtime_auth_scopes: auth.scopes,
 578      runtime_token: auth.token}
 579   end
 580
 581   test "find_reusable method with same runtime_token" do
 582     set_user_from_auth :active
 583     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 584     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:container_runtime_token).token}))
 585     assert_equal Container::Queued, c1.state
 586     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 587     assert_not_nil reused
 588     assert_equal reused.uuid, c1.uuid
 589   end
 590
 591   test "find_reusable method with different runtime_token, same user" do
 592     set_user_from_auth :active
 593     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 594     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:crt_user).token}))
 595     assert_equal Container::Queued, c1.state
 596     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 597     assert_not_nil reused
 598     assert_equal reused.uuid, c1.uuid
 599   end
 600
 601   test "find_reusable method with nil runtime_token, then runtime_token with same user" do
 602     set_user_from_auth :crt_user
 603     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 604     c1, _ = minimal_new(common_attrs)
 605     assert_equal Container::Queued, c1.state
 606     assert_equal users(:container_runtime_token_user).uuid, c1.runtime_user_uuid
 607     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 608     assert_not_nil reused
 609     assert_equal reused.uuid, c1.uuid
 610   end
 611
 612   test "find_reusable method with different runtime_token, different user" do
 613     set_user_from_auth :crt_user
 614     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 615     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:active).token}))
 616     assert_equal Container::Queued, c1.state
 617     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 618     # See #14584
 619     assert_not_nil reused
 620     assert_equal c1.uuid, reused.uuid
 621   end
 622
 623   test "find_reusable method with nil runtime_token, then runtime_token with different user" do
 624     set_user_from_auth :active
 625     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 626     c1, _ = minimal_new(common_attrs.merge({runtime_token: nil}))
 627     assert_equal Container::Queued, c1.state
 628     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 629     # See #14584
 630     assert_not_nil reused
 631     assert_equal c1.uuid, reused.uuid
 632   end
 633
 634   test "find_reusable method with different runtime_token, different scope, same user" do
 635     set_user_from_auth :active
 636     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 637     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:runtime_token_limited_scope).token}))
 638     assert_equal Container::Queued, c1.state
 639     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 640     # See #14584
 641     assert_not_nil reused
 642     assert_equal c1.uuid, reused.uuid
 643   end
 644
 645   test "find_reusable method with cuda" do
 646     set_user_from_auth :active
 647     # No cuda
 648     no_cuda_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"},
 649                                                 runtime_constraints: {"vcpus" => 1, "ram" => 1, "keep_cache_disk"=>0, "keep_cache_ram"=>268435456, "API" => false,
 650                                                                       "cuda" => {"device_count" => 0, "driver_version" => "", "hardware_capability" => ""}},})
 651     c1, _ = minimal_new(no_cuda_attrs)
 652     assert_equal Container::Queued, c1.state
 653
 654     # has cuda
 655     cuda_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"},
 656                                                 runtime_constraints: {"vcpus" => 1, "ram" => 1, "keep_cache_disk"=>0, "keep_cache_ram"=>268435456, "API" => false,
 657                                                                       "cuda" => {"device_count" => 1, "driver_version" => "11.0", "hardware_capability" => "9.0"}},})
 658     c2, _ = minimal_new(cuda_attrs)
 659     assert_equal Container::Queued, c2.state
 660
 661     no_cuda_attrs[:runtime_constraints] = Container.resolve_runtime_constraints(no_cuda_attrs[:runtime_constraints])
 662     cuda_attrs[:runtime_constraints] = Container.resolve_runtime_constraints(cuda_attrs[:runtime_constraints])
 663
 664     # should find the no cuda one
 665     reused = Container.find_reusable(no_cuda_attrs)
 666     assert_not_nil reused
 667     assert_equal reused.uuid, c1.uuid
 668
 669     # should find the cuda one
 670     reused = Container.find_reusable(cuda_attrs)
 671     assert_not_nil reused
 672     assert_equal reused.uuid, c2.uuid
 673   end
 674
 675   test "find_reusable with legacy cuda" do
 676     set_user_from_auth :active
 677
 678     # has cuda
 679
 680     cuda_attrs = {
 681       command: ["echo", "hello", "/bin/sh", "-c", "'cat' '/keep/fa7aeb5140e2848d39b416daeef4ffc5+45/foobar' '/keep/fa7aeb5140e2848d39b416daeef4ffc5+45/baz' '|' 'gzip' '>' '/dev/null'"],
 682       cwd: "test",
 683       environment: {},
 684       output_path: "test",
 685       output_glob: [],
 686       container_image: "fa3c1a9cb6783f85f2ecda037e07b8c3+167",
 687       mounts: {},
 688       runtime_constraints: Container.resolve_runtime_constraints({
 689         "cuda" => {
 690           "device_count" => 1,
 691           "driver_version" => "11.0",
 692           "hardware_capability" => "9.0",
 693         },
 694         "ram" => 12000000000,
 695         "vcpus" => 4,
 696       }),
 697       scheduling_parameters: {},
 698       secret_mounts: {},
 699     }
 700
 701     Rails.configuration.Containers.LogReuseDecisions = true
 702     # should find the gpu one
 703     reused = Container.find_reusable(cuda_attrs)
 704     assert_not_nil reused
 705     assert_equal reused.uuid, containers(:legacy_cuda_container).uuid
 706
 707   end
 708
 709   test "find_reusable method with gpu" do
 710     set_user_from_auth :active
 711     # No gpu
 712     no_gpu_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"},
 713                                                 runtime_constraints: {"vcpus" => 1, "ram" => 1, "keep_cache_disk"=>0, "keep_cache_ram"=>268435456, "API" => false,
 714                                                                       "gpu" => {"device_count" => 0, "driver_version" => "",
 715                                                                                 "hardware_target" => [], "stack" => "", "vram" => 0}},})
 716     c1, _ = minimal_new(no_gpu_attrs)
 717     assert_equal Container::Queued, c1.state
 718
 719     # wants gpu
 720     gpu_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"},
 721                                                 runtime_constraints: {"vcpus" => 1, "ram" => 1, "keep_cache_disk"=>0, "keep_cache_ram"=>268435456, "API" => false,
 722                                                                       "gpu" => {"device_count" => 1, "driver_version" => "11.0",
 723                                                                                 "hardware_target" => ["9.0"], "stack" => "cuda",
 724                                                                                 "vram" => 2000000000}},})
 725     c2, _ = minimal_new(gpu_attrs)
 726     assert_equal Container::Queued, c2.state
 727
 728     no_gpu_attrs[:runtime_constraints] = Container.resolve_runtime_constraints(no_gpu_attrs[:runtime_constraints])
 729     gpu_attrs[:runtime_constraints] = Container.resolve_runtime_constraints(gpu_attrs[:runtime_constraints])
 730
 731     # should find the no gpu one
 732     reused = Container.find_reusable(no_gpu_attrs)
 733     assert_not_nil reused
 734     assert_equal reused.uuid, c1.uuid
 735
 736     # should find the gpu one
 737     reused = Container.find_reusable(gpu_attrs)
 738     assert_not_nil reused
 739     assert_equal reused.uuid, c2.uuid
 740   end
 741
 742   test "Container running" do
 743     set_user_from_auth :active
 744     c, _ = minimal_new priority: 1
 745
 746     set_user_from_auth :system_user
 747     check_illegal_updates c, [{state: Container::Running},
 748                               {state: Container::Complete}]
 749
 750     c.lock
 751     c.update! state: Container::Running
 752
 753     check_illegal_modify c
 754     check_bogus_states c
 755
 756     check_illegal_updates c, [{state: Container::Queued}]
 757     c.reload
 758
 759     c.update! priority: 3
 760   end
 761
 762   test "Lock and unlock" do
 763     set_user_from_auth :active
 764     c, cr = minimal_new priority: 0
 765
 766     set_user_from_auth :system_user
 767     assert_equal Container::Queued, c.state
 768
 769     assert_raise(ArvadosModel::LockFailedError) do
 770       # "no priority"
 771       c.lock
 772     end
 773     c.reload
 774     assert cr.update priority: 1
 775
 776     refute c.update(state: Container::Running), "not locked"
 777     c.reload
 778     refute c.update(state: Container::Complete), "not locked"
 779     c.reload
 780
 781     assert c.lock, show_errors(c)
 782     assert c.locked_by_uuid
 783     assert c.auth_uuid
 784
 785     assert_raise(ArvadosModel::LockFailedError) {c.lock}
 786     c.reload
 787
 788     assert c.unlock, show_errors(c)
 789     refute c.locked_by_uuid
 790     refute c.auth_uuid
 791
 792     refute c.update(state: Container::Running), "not locked"
 793     c.reload
 794     refute c.locked_by_uuid
 795     refute c.auth_uuid
 796
 797     assert c.lock, show_errors(c)
 798     assert c.update(state: Container::Running), show_errors(c)
 799     assert c.locked_by_uuid
 800     assert c.auth_uuid
 801
 802     auth_uuid_was = c.auth_uuid
 803
 804     assert_raise(ArvadosModel::LockFailedError) do
 805       # Running to Locked is not allowed
 806       c.lock
 807     end
 808     c.reload
 809     assert_raise(ArvadosModel::InvalidStateTransitionError) do
 810       # Running to Queued is not allowed
 811       c.unlock
 812     end
 813     c.reload
 814
 815     assert c.update(state: Container::Complete), show_errors(c)
 816     refute c.locked_by_uuid
 817     refute c.auth_uuid
 818
 819     auth_exp = ApiClientAuthorization.find_by_uuid(auth_uuid_was).expires_at
 820     assert_operator auth_exp, :<, db_current_time
 821
 822     assert_nil ApiClientAuthorization.validate(token: ApiClientAuthorization.find_by_uuid(auth_uuid_was).token)
 823   end
 824
 825   test "Exceed maximum lock-unlock cycles" do
 826     Rails.configuration.Containers.MaxDispatchAttempts = 3
 827
 828     set_user_from_auth :active
 829     c, cr = minimal_new
 830
 831     set_user_from_auth :system_user
 832     assert_equal Container::Queued, c.state
 833     assert_equal 0, c.lock_count
 834
 835     c.lock
 836     c.reload
 837     assert_equal 1, c.lock_count
 838     assert_equal Container::Locked, c.state
 839
 840     c.unlock
 841     c.reload
 842     assert_equal 1, c.lock_count
 843     assert_equal Container::Queued, c.state
 844
 845     c.lock
 846     c.reload
 847     assert_equal 2, c.lock_count
 848     assert_equal Container::Locked, c.state
 849
 850     c.unlock
 851     c.reload
 852     assert_equal 2, c.lock_count
 853     assert_equal Container::Queued, c.state
 854
 855     c.lock
 856     c.reload
 857     assert_equal 3, c.lock_count
 858     assert_equal Container::Locked, c.state
 859
 860     c.unlock
 861     c.reload
 862     assert_equal 3, c.lock_count
 863     assert_equal Container::Cancelled, c.state
 864
 865     assert_raise(ArvadosModel::LockFailedError) do
 866       # Cancelled to Locked is not allowed
 867       c.lock
 868     end
 869   end
 870
 871   test "Container queued cancel" do
 872     set_user_from_auth :active
 873     c, cr = minimal_new({container_count_max: 1})
 874     set_user_from_auth :system_user
 875     assert c.update(state: Container::Cancelled), show_errors(c)
 876     check_no_change_from_cancelled c
 877     cr.reload
 878     assert_equal ContainerRequest::Final, cr.state
 879   end
 880
 881   test "Container queued count" do
 882     assert_equal 1, Container.readable_by(users(:active)).where(state: "Queued").count
 883   end
 884
 885   test "Containers with no matching request are readable by admin" do
 886     uuids = Container.includes('container_requests').where(container_requests: {uuid: nil}).collect(&:uuid)
 887     assert_not_empty uuids
 888     assert_empty Container.readable_by(users(:active)).where(uuid: uuids)
 889     assert_not_empty Container.readable_by(users(:admin)).where(uuid: uuids)
 890     assert_equal uuids.count, Container.readable_by(users(:admin)).where(uuid: uuids).count
 891   end
 892
 893   test "Container locked cancel" do
 894     set_user_from_auth :active
 895     c, _ = minimal_new
 896     set_user_from_auth :system_user
 897     assert c.lock, show_errors(c)
 898     assert c.update(state: Container::Cancelled), show_errors(c)
 899     check_no_change_from_cancelled c
 900   end
 901
 902   test "Container locked with non-expiring token" do
 903     Rails.configuration.API.TokenMaxLifetime = 1.hour
 904     set_user_from_auth :active
 905     c, _ = minimal_new
 906     set_user_from_auth :system_user
 907     assert c.lock, show_errors(c)
 908     refute c.auth.nil?
 909     assert c.auth.expires_at.nil?
 910     assert c.auth.user_id == User.find_by_uuid(users(:active).uuid).id
 911   end
 912
 913   test "Container locked cancel with log" do
 914     set_user_from_auth :active
 915     c, _ = minimal_new
 916     set_user_from_auth :system_user
 917     assert c.lock, show_errors(c)
 918     assert c.update(
 919              state: Container::Cancelled,
 920              log: collections(:log_collection).portable_data_hash,
 921            ), show_errors(c)
 922     check_no_change_from_cancelled c
 923   end
 924
 925   test "Container running cancel" do
 926     set_user_from_auth :active
 927     c, _ = minimal_new
 928     set_user_from_auth :system_user
 929     c.lock
 930     c.update! state: Container::Running
 931     c.update! state: Container::Cancelled
 932     check_no_change_from_cancelled c
 933   end
 934
 935   test "Container create forbidden for non-admin" do
 936     set_user_from_auth :active_trustedclient
 937     c = Container.new DEFAULT_ATTRS
 938     c.environment = {}
 939     c.mounts = {"BAR" => "FOO"}
 940     c.output_path = "/tmp"
 941     c.priority = 1
 942     c.runtime_constraints = {}
 943     assert_raises(ArvadosModel::PermissionDeniedError) do
 944       c.save!
 945     end
 946   end
 947
 948   [
 949     [Container::Queued, {state: Container::Locked}],
 950     [Container::Queued, {state: Container::Running}],
 951     [Container::Queued, {state: Container::Complete}],
 952     [Container::Queued, {state: Container::Cancelled}],
 953     [Container::Queued, {priority: 123456789}],
 954     [Container::Queued, {runtime_status: {'error' => 'oops'}}],
 955     [Container::Queued, {cwd: '/'}],
 956     [Container::Locked, {state: Container::Running}],
 957     [Container::Locked, {state: Container::Queued}],
 958     [Container::Locked, {priority: 123456789}],
 959     [Container::Locked, {runtime_status: {'error' => 'oops'}}],
 960     [Container::Locked, {cwd: '/'}],
 961     [Container::Running, {state: Container::Complete}],
 962     [Container::Running, {state: Container::Cancelled}],
 963     [Container::Running, {priority: 123456789}],
 964     [Container::Running, {runtime_status: {'error' => 'oops'}}],
 965     [Container::Running, {cwd: '/'}],
 966     [Container::Running, {gateway_address: "172.16.0.1:12345"}],
 967     [Container::Running, {interactive_session_started: true}],
 968     [Container::Complete, {state: Container::Cancelled}],
 969     [Container::Complete, {priority: 123456789}],
 970     [Container::Complete, {runtime_status: {'error' => 'oops'}}],
 971     [Container::Complete, {cwd: '/'}],
 972     [Container::Cancelled, {cwd: '/'}],
 973   ].each do |start_state, updates|
 974     test "Container update #{updates.inspect} when #{start_state} forbidden for non-admin" do
 975       set_user_from_auth :active
 976       c, _ = minimal_new
 977       if start_state != Container::Queued
 978         set_user_from_auth :system_user
 979         c.lock
 980         if start_state != Container::Locked
 981           c.update! state: Container::Running
 982           if start_state != Container::Running
 983             c.update! state: start_state
 984           end
 985         end
 986       end
 987       assert_equal c.state, start_state
 988       set_user_from_auth :active
 989       assert_raises(ArvadosModel::PermissionDeniedError) do
 990         c.update! updates
 991       end
 992     end
 993   end
 994
 995   test "can only change exit code while running and at completion" do
 996     set_user_from_auth :active
 997     c, _ = minimal_new
 998     set_user_from_auth :system_user
 999     c.lock
1000     check_illegal_updates c, [{exit_code: 1}]
1001     c.update! state: Container::Running
1002     assert c.update(exit_code: 1)
1003     assert c.update(exit_code: 1, state: Container::Complete)
1004   end
1005
1006   test "locked_by_uuid can update log when locked/running, and output when running" do
1007     set_user_from_auth :active
1008     logcoll = collections(:container_log_collection)
1009     c, cr1 = minimal_new
1010     cr2 = ContainerRequest.new(DEFAULT_ATTRS)
1011     cr2.state = ContainerRequest::Committed
1012     act_as_user users(:active) do
1013       cr2.save!
1014     end
1015     assert_equal cr1.container_uuid, cr2.container_uuid
1016
1017     logpdh_time1 = logcoll.portable_data_hash
1018
1019     set_user_from_auth :system_user
1020     c.lock
1021     assert_equal c.locked_by_uuid, Thread.current[:api_client_authorization].uuid
1022     c.update!(log: logpdh_time1)
1023     c.update!(state: Container::Running)
1024     cr1.reload
1025     cr2.reload
1026     cr1log_uuid = cr1.log_uuid
1027     cr2log_uuid = cr2.log_uuid
1028     assert_not_nil cr1log_uuid
1029     assert_not_nil cr2log_uuid
1030     assert_not_equal logcoll.uuid, cr1log_uuid
1031     assert_not_equal logcoll.uuid, cr2log_uuid
1032     assert_not_equal cr1log_uuid, cr2log_uuid
1033
1034     logcoll.update!(manifest_text: logcoll.manifest_text + ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n")
1035     logpdh_time2 = logcoll.portable_data_hash
1036
1037     assert c.update(output: collections(:collection_owned_by_active).portable_data_hash)
1038     assert c.update(log: logpdh_time2)
1039     assert c.update(state: Container::Complete, log: logcoll.portable_data_hash)
1040     c.reload
1041     assert_equal collections(:collection_owned_by_active).portable_data_hash, c.output
1042     assert_equal logpdh_time2, c.log
1043     refute c.update(output: nil)
1044     refute c.update(log: nil)
1045     cr1.reload
1046     cr2.reload
1047     assert_equal cr1log_uuid, cr1.log_uuid
1048     assert_equal cr2log_uuid, cr2.log_uuid
1049     assert_equal 1, Collection.where(uuid: [cr1log_uuid, cr2log_uuid]).to_a.collect(&:portable_data_hash).uniq.length
1050     assert_equal ". 8c12f5f5297b7337598170c6f531fcee+7882 acbd18db4cc2f85cedef654fccc4a4d8+3 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 7882:3:foo.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt
1051 ./log\\040for\\040container\\040#{cr1.container_uuid} 8c12f5f5297b7337598170c6f531fcee+7882 acbd18db4cc2f85cedef654fccc4a4d8+3 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 7882:3:foo.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt
1052 ", Collection.find_by_uuid(cr1log_uuid).manifest_text
1053   end
1054
1055   ["auth_uuid", "runtime_token"].each do |tok|
1056     test "#{tok} can set output, progress, runtime_status, state, exit_code on running container -- but not log" do
1057       if tok == "runtime_token"
1058         set_user_from_auth :spectator
1059         c, _ = minimal_new(container_image: "9ae44d5792468c58bcf85ce7353c7027+124",
1060                            runtime_token: api_client_authorizations(:active).token)
1061       else
1062         set_user_from_auth :active
1063         c, _ = minimal_new
1064       end
1065       set_user_from_auth :system_user
1066       c.lock
1067       c.update! state: Container::Running
1068
1069       if tok == "runtime_token"
1070         auth = ApiClientAuthorization.validate(token: c.runtime_token)
1071         Thread.current[:api_client_authorization] = auth
1072         Thread.current[:token] = auth.token
1073         Thread.current[:user] = auth.user
1074       else
1075         auth = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
1076         Thread.current[:api_client_authorization] = auth
1077         Thread.current[:token] = auth.token
1078         Thread.current[:user] = auth.user
1079       end
1080
1081       assert c.update(gateway_address: "127.0.0.1:9")
1082       assert c.update(output: collections(:collection_owned_by_active).portable_data_hash)
1083       assert c.update(runtime_status: {'warning' => 'something happened'})
1084       assert c.update(progress: 0.5)
1085       assert c.update(exit_code: 0)
1086       refute c.update(log: collections(:log_collection).portable_data_hash)
1087       c.reload
1088       assert c.update(state: Container::Complete, exit_code: 0)
1089     end
1090   end
1091
1092   test "not allowed to set output that is not readable by current user" do
1093     set_user_from_auth :active
1094     c, _ = minimal_new
1095     set_user_from_auth :system_user
1096     c.lock
1097     c.update! state: Container::Running
1098
1099     Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
1100     Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
1101
1102     assert_raises ActiveRecord::RecordInvalid do
1103       c.update! output: collections(:collection_not_readable_by_active).portable_data_hash
1104     end
1105   end
1106
1107   test "other token cannot set output on running container" do
1108     set_user_from_auth :active
1109     c, _ = minimal_new
1110     set_user_from_auth :system_user
1111     c.lock
1112     c.update! state: Container::Running
1113
1114     set_user_from_auth :running_to_be_deleted_container_auth
1115     assert_raises(ArvadosModel::PermissionDeniedError) do
1116       c.update(output: collections(:foo_file).portable_data_hash)
1117     end
1118   end
1119
1120   test "can set trashed output on running container" do
1121     set_user_from_auth :active
1122     c, _ = minimal_new
1123     set_user_from_auth :system_user
1124     c.lock
1125     c.update! state: Container::Running
1126
1127     output = Collection.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jk')
1128
1129     assert output.is_trashed
1130     assert c.update output: output.portable_data_hash
1131     assert c.update! state: Container::Complete
1132   end
1133
1134   test "not allowed to set trashed output that is not readable by current user" do
1135     set_user_from_auth :active
1136     c, _ = minimal_new
1137     set_user_from_auth :system_user
1138     c.lock
1139     c.update! state: Container::Running
1140
1141     output = Collection.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jr')
1142
1143     Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
1144     Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
1145
1146     assert_raises ActiveRecord::RecordInvalid do
1147       c.update! output: output.portable_data_hash
1148     end
1149   end
1150
1151   test "user cannot delete" do
1152     set_user_from_auth :active
1153     c, _ = minimal_new
1154     assert_raises ArvadosModel::PermissionDeniedError do
1155       c.destroy
1156     end
1157     assert Container.find_by_uuid(c.uuid)
1158   end
1159
1160   [
1161     {state: Container::Complete, exit_code: 0, output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'},
1162     {state: Container::Cancelled},
1163   ].each do |final_attrs|
1164     test "secret_mounts and runtime_token are null after container is #{final_attrs[:state]}" do
1165       set_user_from_auth :active
1166       c, cr = minimal_new(secret_mounts: {'/secret' => {'kind' => 'text', 'content' => 'foo'}},
1167                           container_count_max: 1, runtime_token: api_client_authorizations(:active).token)
1168       set_user_from_auth :system_user
1169       c.lock
1170       c.update!(state: Container::Running)
1171       c.reload
1172       assert c.secret_mounts.has_key?('/secret')
1173       assert_equal api_client_authorizations(:active).token, c.runtime_token
1174
1175       c.update!(final_attrs)
1176       c.reload
1177       assert_equal({}, c.secret_mounts)
1178       assert_nil c.runtime_token
1179       cr.reload
1180       assert_equal({}, cr.secret_mounts)
1181       assert_nil cr.runtime_token
1182       assert_no_secrets_logged
1183     end
1184   end
1185
1186   def configure_preemptible_instance_type
1187     Rails.configuration.InstanceTypes = ConfigLoader.to_OrderedOptions({
1188       "a1.small.pre" => {
1189         "Preemptible" => true,
1190         "Price" => 0.1,
1191         "ProviderType" => "a1.small",
1192         "VCPUs" => 1,
1193         "RAM" => 1000000000,
1194       },
1195     })
1196   end
1197
1198   def vary_parameters(**kwargs)
1199     # kwargs is a hash that maps parameters to an array of values.
1200     # This function enumerates every possible hash where each key has one of
1201     # the values from its array.
1202     # The output keys are strings since that's what container hash attributes
1203     # want.
1204     # A nil value yields a hash without that key.
1205     [[:_, nil]].product(
1206       *kwargs.map { |(key, values)| [key.to_s].product(values) },
1207     ).map { |param_pairs| Hash[param_pairs].compact }
1208   end
1209
1210   def retry_with_scheduling_parameters(param_hashes)
1211     set_user_from_auth :admin
1212     containers = {}
1213     requests = []
1214     param_hashes.each do |scheduling_parameters|
1215       container, request = minimal_new(scheduling_parameters: scheduling_parameters)
1216       containers[container.uuid] = container
1217       requests << request
1218     end
1219     refute(containers.empty?, "buggy test: no scheduling parameters enumerated")
1220     assert_equal(1, containers.length)
1221     _, container1 = containers.shift
1222     container1.lock
1223     container1.update!(state: Container::Cancelled)
1224     container1.reload
1225     request1 = requests.shift
1226     request1.reload
1227     assert_not_equal(container1.uuid, request1.container_uuid)
1228     requests.each do |request|
1229       request.reload
1230       assert_equal(request1.container_uuid, request.container_uuid)
1231     end
1232     container2 = Container.find_by_uuid(request1.container_uuid)
1233     assert_not_nil(container2)
1234     return container2
1235   end
1236
1237   preemptible_values = [true, false, nil]
1238   preemptible_values.permutation(1).chain(
1239     preemptible_values.product(preemptible_values),
1240     preemptible_values.product(preemptible_values, preemptible_values),
1241   ).each do |preemptible_a|
1242     # If the first req has preemptible=true but a subsequent req
1243     # doesn't, we want to avoid reusing the first container, so this
1244     # test isn't appropriate.
1245     next if preemptible_a[0] &&
1246             ((preemptible_a.length > 1 && !preemptible_a[1]) ||
1247              (preemptible_a.length > 2 && !preemptible_a[2]))
1248     test "retry requests scheduled with preemptible=#{preemptible_a}" do
1249       configure_preemptible_instance_type
1250       param_hashes = vary_parameters(preemptible: preemptible_a)
1251       container = retry_with_scheduling_parameters(param_hashes)
1252       assert_equal(preemptible_a.all?,
1253                    container.scheduling_parameters["preemptible"] || false)
1254     end
1255   end
1256
1257   partition_values = [nil, [], ["alpha"], ["alpha", "bravo"], ["bravo", "charlie"]]
1258   partition_values.permutation(1).chain(
1259     partition_values.permutation(2),
1260   ).each do |partitions_a|
1261     test "retry requests scheduled with partitions=#{partitions_a}" do
1262       param_hashes = vary_parameters(partitions: partitions_a)
1263       container = retry_with_scheduling_parameters(param_hashes)
1264       expected = if partitions_a.any? { |value| value.nil? or value.empty? }
1265                    []
1266                  else
1267                    partitions_a.flatten.uniq
1268                  end
1269       actual = container.scheduling_parameters["partitions"] || []
1270       assert_equal(expected.sort, actual.sort)
1271     end
1272   end
1273
1274   runtime_values = [nil, 0, 1, 2, 3]
1275   runtime_values.permutation(1).chain(
1276     runtime_values.permutation(2),
1277     runtime_values.permutation(3),
1278   ).each do |max_run_time_a|
1279     test "retry requests scheduled with max_run_time=#{max_run_time_a}" do
1280       param_hashes = vary_parameters(max_run_time: max_run_time_a)
1281       container = retry_with_scheduling_parameters(param_hashes)
1282       expected = if max_run_time_a.any? { |value| value.nil? or value == 0 }
1283                    0
1284                  else
1285                    max_run_time_a.max
1286                  end
1287       actual = container.scheduling_parameters["max_run_time"] || 0
1288       assert_equal(expected, actual)
1289     end
1290   end
1291
1292   test "retry requests with multi-varied scheduling parameters" do
1293     configure_preemptible_instance_type
1294     param_hashes = [{
1295                      "partitions": ["alpha", "bravo"],
1296                      "preemptible": false,
1297                      "max_run_time": 10,
1298                     }, {
1299                      "partitions": ["alpha", "charlie"],
1300                      "max_run_time": 20,
1301                     }, {
1302                      "partitions": ["bravo", "charlie"],
1303                      "preemptible": true,
1304                      "max_run_time": 30,
1305                     }]
1306     container = retry_with_scheduling_parameters(param_hashes)
1307     actual = container.scheduling_parameters
1308     assert_equal(["alpha", "bravo", "charlie"], actual["partitions"]&.sort)
1309     assert_equal(false, actual["preemptible"] || false)
1310     assert_equal(30, actual["max_run_time"])
1311   end
1312
1313   test "retry requests with unset scheduling parameters" do
1314     configure_preemptible_instance_type
1315     param_hashes = vary_parameters(
1316       preemptible: [nil, true],
1317       partitions: [nil, ["alpha"]],
1318       max_run_time: [nil, 5],
1319     )
1320     container = retry_with_scheduling_parameters(param_hashes)
1321     actual = container.scheduling_parameters
1322     assert_equal([], actual["partitions"] || [])
1323     assert_equal(false, actual["preemptible"] || false)
1324     assert_equal(0, actual["max_run_time"] || 0)
1325   end
1326
1327   test "retry requests with default scheduling parameters" do
1328     configure_preemptible_instance_type
1329     param_hashes = vary_parameters(
1330       preemptible: [false, true],
1331       partitions: [[], ["bravo"]],
1332       max_run_time: [0, 1],
1333     )
1334     container = retry_with_scheduling_parameters(param_hashes)
1335     actual = container.scheduling_parameters
1336     assert_equal([], actual["partitions"] || [])
1337     assert_equal(false, actual["preemptible"] || false)
1338     assert_equal(0, actual["max_run_time"] || 0)
1339   end
1340
1341   def run_container(request_params, final_attrs)
1342     final_attrs[:state] ||= Container::Complete
1343     if final_attrs[:state] == Container::Complete
1344       final_attrs[:exit_code] ||= 0
1345       final_attrs[:log] ||= collections(:log_collection).portable_data_hash
1346       final_attrs[:output] ||= collections(:multilevel_collection_1).portable_data_hash
1347     end
1348     container, request = minimal_new(request_params)
1349     container.lock
1350     container.update!(state: Container::Running)
1351     container.update!(final_attrs)
1352     return container, request
1353   end
1354
1355   def check_reuse_with_variations(default_keep_cache_ram, vary_attr, start_value, variations)
1356     container_params = REUSABLE_ATTRS_SLIM.merge(vary_attr => start_value)
1357     orig_default = Rails.configuration.Containers.DefaultKeepCacheRAM
1358     begin
1359       Rails.configuration.Containers.DefaultKeepCacheRAM = default_keep_cache_ram
1360       set_user_from_auth :admin
1361       expected, _ = run_container(container_params, {})
1362       variations.each do |variation|
1363         full_variation = REUSABLE_ATTRS_SLIM[vary_attr].merge(variation)
1364         parameters = REUSABLE_ATTRS_SLIM.merge(vary_attr => full_variation)
1365         actual = Container.find_reusable(parameters)
1366         assert_equal(expected.uuid, actual&.uuid,
1367                      "request with #{vary_attr}=#{variation} did not reuse container")
1368       end
1369     ensure
1370       Rails.configuration.Containers.DefaultKeepCacheRAM = orig_default
1371     end
1372   end
1373
1374   # Test that we can reuse a container with a known keep_cache_ram constraint,
1375   # no matter what keep_cache_* constraints the new request uses.
1376   [0, 2 << 30, 4 << 30].product(
1377     [0, 1],
1378     [true, false],
1379   ).each do |(default_keep_cache_ram, multiplier, keep_disk_constraint)|
1380     test "reuse request with DefaultKeepCacheRAM=#{default_keep_cache_ram}, keep_cache_ram*=#{multiplier}, keep_cache_disk=#{keep_disk_constraint}" do
1381       runtime_constraints = REUSABLE_ATTRS_SLIM[:runtime_constraints].merge(
1382         "keep_cache_ram" => default_keep_cache_ram * multiplier,
1383       )
1384       if not keep_disk_constraint
1385         # Simulate a container that predates keep_cache_disk by deleting
1386         # the constraint entirely.
1387         runtime_constraints.delete("keep_cache_disk")
1388       end
1389       # Important values are:
1390       # * 0
1391       # * 2GiB, the minimum default keep_cache_disk
1392       # * 8GiB, the default keep_cache_disk based on container ram
1393       # * 32GiB, the maximum default keep_cache_disk
1394       # Check these values and values in between.
1395       vary_values = [0, 1, 2, 6, 8, 10, 32, 33].map { |v| v << 30 }.to_a
1396       variations = vary_parameters(keep_cache_ram: vary_values)
1397                      .chain(vary_parameters(keep_cache_disk: vary_values))
1398       check_reuse_with_variations(
1399         default_keep_cache_ram,
1400         :runtime_constraints,
1401         runtime_constraints,
1402         variations,
1403       )
1404     end
1405   end
1406
1407   # Test that we can reuse a container with a known keep_cache_disk constraint,
1408   # no matter what keep_cache_* constraints the new request uses.
1409   # keep_cache_disk values are the important values discussed in the test above.
1410   [0, 2 << 30, 4 << 30]
1411     .product([0, 2 << 30, 8 << 30, 32 << 30])
1412     .each do |(default_keep_cache_ram, keep_cache_disk)|
1413     test "reuse request with DefaultKeepCacheRAM=#{default_keep_cache_ram} and keep_cache_disk=#{keep_cache_disk}" do
1414       runtime_constraints = REUSABLE_ATTRS_SLIM[:runtime_constraints].merge(
1415         "keep_cache_disk" => keep_cache_disk,
1416       )
1417       vary_values = [0, 1, 2, 6, 8, 10, 32, 33].map { |v| v << 30 }.to_a
1418       variations = vary_parameters(keep_cache_ram: vary_values)
1419                      .chain(vary_parameters(keep_cache_disk: vary_values))
1420       check_reuse_with_variations(
1421         default_keep_cache_ram,
1422         :runtime_constraints,
1423         runtime_constraints,
1424         variations,
1425       )
1426     end
1427   end
1428
1429   # Test that a container request can reuse a container with an exactly
1430   # matching keep_cache_* constraint, no matter what the defaults.
1431   [0, 2 << 30, 4 << 30].product(
1432     ["keep_cache_disk", "keep_cache_ram"],
1433     [135790, 13 << 30, 135 << 30],
1434   ).each do |(default_keep_cache_ram, constraint_key, constraint_value)|
1435     test "reuse request with #{constraint_key}=#{constraint_value} and DefaultKeepCacheRAM=#{default_keep_cache_ram}" do
1436       runtime_constraints = REUSABLE_ATTRS_SLIM[:runtime_constraints].merge(
1437         constraint_key => constraint_value,
1438       )
1439       check_reuse_with_variations(
1440         default_keep_cache_ram,
1441         :runtime_constraints,
1442         runtime_constraints,
1443         [runtime_constraints],
1444       )
1445     end
1446   end
1447 end