services/api/test/unit/container_test.rb

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 require 'test_helper'
   6 require 'helpers/container_test_helper'
   7
   8 class ContainerTest < ActiveSupport::TestCase
   9   include DbCurrentTime
  10   include ContainerTestHelper
  11
  12   DEFAULT_ATTRS = {
  13     command: ['echo', 'foo'],
  14     container_image: 'fa3c1a9cb6783f85f2ecda037e07b8c3+167',
  15     output_path: '/tmp',
  16     priority: 1,
  17     runtime_constraints: {"vcpus" => 1, "ram" => 1, "cuda" => {"device_count":0, "driver_version": "", "hardware_capability": ""}},
  18   }
  19
  20   REUSABLE_COMMON_ATTRS = {
  21     container_image: "9ae44d5792468c58bcf85ce7353c7027+124",
  22     cwd: "test",
  23     command: ["echo", "hello"],
  24     output_path: "test",
  25     runtime_constraints: {
  26       "API" => false,
  27       "keep_cache_disk" => 0,
  28       "keep_cache_ram" => 0,
  29       "ram" => 12000000000,
  30       "vcpus" => 4
  31     },
  32     mounts: {
  33       "test" => {"kind" => "json"},
  34     },
  35     environment: {
  36       "var" => "val",
  37     },
  38     secret_mounts: {},
  39     runtime_user_uuid: "zzzzz-tpzed-xurymjxw79nv3jz",
  40     runtime_auth_scopes: ["all"]
  41   }
  42
  43   def request_only attrs
  44     attrs.reject {|k| [:runtime_user_uuid, :runtime_auth_scopes].include? k}
  45   end
  46
  47   def minimal_new attrs={}
  48     cr = ContainerRequest.new request_only(DEFAULT_ATTRS.merge(attrs))
  49     cr.state = ContainerRequest::Committed
  50     cr.save!
  51     c = Container.find_by_uuid cr.container_uuid
  52     assert_not_nil c
  53     return c, cr
  54   end
  55
  56   def check_illegal_updates c, bad_updates
  57     bad_updates.each do |u|
  58       refute c.update_attributes(u), u.inspect
  59       refute c.valid?, u.inspect
  60       c.reload
  61     end
  62   end
  63
  64   def check_illegal_modify c
  65     check_illegal_updates c, [{command: ["echo", "bar"]},
  66                               {container_image: "arvados/apitestfixture:june10"},
  67                               {cwd: "/tmp2"},
  68                               {environment: {"FOO" => "BAR"}},
  69                               {mounts: {"FOO" => "BAR"}},
  70                               {output_path: "/tmp3"},
  71                               {locked_by_uuid: "zzzzz-gj3su-027z32aux8dg2s1"},
  72                               {auth_uuid: "zzzzz-gj3su-017z32aux8dg2s1"},
  73                               {runtime_constraints: {"FOO" => "BAR"}}]
  74   end
  75
  76   def check_bogus_states c
  77     check_illegal_updates c, [{state: nil},
  78                               {state: "Flubber"}]
  79   end
  80
  81   def check_no_change_from_cancelled c
  82     check_illegal_modify c
  83     check_bogus_states c
  84     check_illegal_updates c, [{ priority: 3 },
  85                               { state: Container::Queued },
  86                               { state: Container::Locked },
  87                               { state: Container::Running },
  88                               { state: Container::Complete }]
  89   end
  90
  91   test "Container create" do
  92     act_as_system_user do
  93       c, _ = minimal_new(environment: {},
  94                       mounts: {"BAR" => {"kind" => "FOO"}},
  95                       output_path: "/tmp",
  96                       priority: 1,
  97                       runtime_constraints: {"vcpus" => 1, "ram" => 1})
  98
  99       check_illegal_modify c
 100       check_bogus_states c
 101
 102       c.reload
 103       c.priority = 2
 104       c.save!
 105     end
 106   end
 107
 108   test "Container valid priority" do
 109     act_as_system_user do
 110       c, _ = minimal_new(environment: {},
 111                       mounts: {"BAR" => {"kind" => "FOO"}},
 112                       output_path: "/tmp",
 113                       priority: 1,
 114                       runtime_constraints: {"vcpus" => 1, "ram" => 1})
 115
 116       assert_raises(ActiveRecord::RecordInvalid) do
 117         c.priority = -1
 118         c.save!
 119       end
 120
 121       c.priority = 0
 122       c.save!
 123
 124       c.priority = 1
 125       c.save!
 126
 127       c.priority = 500
 128       c.save!
 129
 130       c.priority = 999
 131       c.save!
 132
 133       c.priority = 1000
 134       c.save!
 135
 136       c.priority = 1000 << 50
 137       c.save!
 138     end
 139   end
 140
 141   test "Container runtime_status data types" do
 142     set_user_from_auth :active
 143     attrs = {
 144       environment: {},
 145       mounts: {"BAR" => {"kind" => "FOO"}},
 146       output_path: "/tmp",
 147       priority: 1,
 148       runtime_constraints: {"vcpus" => 1, "ram" => 1}
 149     }
 150     c, _ = minimal_new(attrs)
 151     assert_equal c.runtime_status, {}
 152     assert_equal Container::Queued, c.state
 153
 154     set_user_from_auth :dispatch1
 155     c.update_attributes! state: Container::Locked
 156     c.update_attributes! state: Container::Running
 157
 158     [
 159       'error', 'errorDetail', 'warning', 'warningDetail', 'activity'
 160     ].each do |k|
 161       # String type is allowed
 162       string_val = 'A string is accepted'
 163       c.update_attributes! runtime_status: {k => string_val}
 164       assert_equal string_val, c.runtime_status[k]
 165
 166       # Other types aren't allowed
 167       [
 168         42, false, [], {}, nil
 169       ].each do |unallowed_val|
 170         assert_raises ActiveRecord::RecordInvalid do
 171           c.update_attributes! runtime_status: {k => unallowed_val}
 172         end
 173       end
 174     end
 175   end
 176
 177   test "Container runtime_status updates" do
 178     set_user_from_auth :active
 179     attrs = {
 180       environment: {},
 181       mounts: {"BAR" => {"kind" => "FOO"}},
 182       output_path: "/tmp",
 183       priority: 1,
 184       runtime_constraints: {"vcpus" => 1, "ram" => 1}
 185     }
 186     c1, _ = minimal_new(attrs)
 187     assert_equal c1.runtime_status, {}
 188
 189     assert_equal Container::Queued, c1.state
 190     assert_raises ArvadosModel::PermissionDeniedError do
 191       c1.update_attributes! runtime_status: {'error' => 'Oops!'}
 192     end
 193
 194     set_user_from_auth :dispatch1
 195
 196     # Allow updates when state = Locked
 197     c1.update_attributes! state: Container::Locked
 198     c1.update_attributes! runtime_status: {'error' => 'Oops!'}
 199     assert c1.runtime_status.key? 'error'
 200
 201     # Reset when transitioning from Locked to Queued
 202     c1.update_attributes! state: Container::Queued
 203     assert_equal c1.runtime_status, {}
 204
 205     # Allow updates when state = Running
 206     c1.update_attributes! state: Container::Locked
 207     c1.update_attributes! state: Container::Running
 208     c1.update_attributes! runtime_status: {'error' => 'Oops!'}
 209     assert c1.runtime_status.key? 'error'
 210
 211     # Don't allow updates on other states
 212     c1.update_attributes! state: Container::Complete
 213     assert_raises ActiveRecord::RecordInvalid do
 214       c1.update_attributes! runtime_status: {'error' => 'Some other error'}
 215     end
 216
 217     set_user_from_auth :active
 218     c2, _ = minimal_new(attrs)
 219     assert_equal c2.runtime_status, {}
 220     set_user_from_auth :dispatch1
 221     c2.update_attributes! state: Container::Locked
 222     c2.update_attributes! state: Container::Running
 223     c2.update_attributes! state: Container::Cancelled
 224     assert_raises ActiveRecord::RecordInvalid do
 225       c2.update_attributes! runtime_status: {'error' => 'Oops!'}
 226     end
 227   end
 228
 229   test "Container serialized hash attributes sorted before save" do
 230     set_user_from_auth :active
 231     env = {"C" => "3", "B" => "2", "A" => "1"}
 232     m = {"F" => {"kind" => "3"}, "E" => {"kind" => "2"}, "D" => {"kind" => "1"}}
 233     rc = {"vcpus" => 1, "ram" => 1, "keep_cache_ram" => 1, "keep_cache_disk" => 0, "API" => true, "cuda" => {"device_count":0, "driver_version": "", "hardware_capability": ""}}
 234     c, _ = minimal_new(environment: env, mounts: m, runtime_constraints: rc)
 235     c.reload
 236     assert_equal Container.deep_sort_hash(env).to_json, c.environment.to_json
 237     assert_equal Container.deep_sort_hash(m).to_json, c.mounts.to_json
 238     assert_equal Container.deep_sort_hash(rc).to_json, c.runtime_constraints.to_json
 239   end
 240
 241   test 'deep_sort_hash on array of hashes' do
 242     a = {'z' => [[{'a' => 'a', 'b' => 'b'}]]}
 243     b = {'z' => [[{'b' => 'b', 'a' => 'a'}]]}
 244     assert_equal Container.deep_sort_hash(a).to_json, Container.deep_sort_hash(b).to_json
 245   end
 246
 247   test "find_reusable method should select higher priority queued container" do
 248         Rails.configuration.Containers.LogReuseDecisions = true
 249     set_user_from_auth :active
 250     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment:{"var" => "queued"}})
 251     c_low_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:1}))
 252     c_high_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:2}))
 253     assert_not_equal c_low_priority.uuid, c_high_priority.uuid
 254     assert_equal Container::Queued, c_low_priority.state
 255     assert_equal Container::Queued, c_high_priority.state
 256     reused = Container.find_reusable(common_attrs)
 257     assert_not_nil reused
 258     assert_equal reused.uuid, c_high_priority.uuid
 259   end
 260
 261   test "find_reusable method should select latest completed container" do
 262     set_user_from_auth :active
 263     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "complete"}})
 264     completed_attrs = {
 265       state: Container::Complete,
 266       exit_code: 0,
 267       log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 268       output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
 269     }
 270
 271     c_older, _ = minimal_new(common_attrs.merge({use_existing: false}))
 272     c_recent, _ = minimal_new(common_attrs.merge({use_existing: false}))
 273     assert_not_equal c_older.uuid, c_recent.uuid
 274
 275     set_user_from_auth :dispatch1
 276     c_older.update_attributes!({state: Container::Locked})
 277     c_older.update_attributes!({state: Container::Running})
 278     c_older.update_attributes!(completed_attrs)
 279
 280     c_recent.update_attributes!({state: Container::Locked})
 281     c_recent.update_attributes!({state: Container::Running})
 282     c_recent.update_attributes!(completed_attrs)
 283
 284     reused = Container.find_reusable(common_attrs)
 285     assert_not_nil reused
 286     assert_equal reused.uuid, c_older.uuid
 287   end
 288
 289   test "find_reusable method should select oldest completed container when inconsistent outputs exist" do
 290     set_user_from_auth :active
 291     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "complete"}, priority: 1})
 292     completed_attrs = {
 293       state: Container::Complete,
 294       exit_code: 0,
 295       log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 296     }
 297
 298     cr = ContainerRequest.new request_only(common_attrs)
 299     cr.use_existing = false
 300     cr.state = ContainerRequest::Committed
 301     cr.save!
 302     c_output1 = Container.where(uuid: cr.container_uuid).first
 303
 304     cr = ContainerRequest.new request_only(common_attrs)
 305     cr.use_existing = false
 306     cr.state = ContainerRequest::Committed
 307     cr.save!
 308     c_output2 = Container.where(uuid: cr.container_uuid).first
 309
 310     assert_not_equal c_output1.uuid, c_output2.uuid
 311
 312     set_user_from_auth :dispatch1
 313
 314     out1 = '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
 315     log1 = collections(:real_log_collection).portable_data_hash
 316     c_output1.update_attributes!({state: Container::Locked})
 317     c_output1.update_attributes!({state: Container::Running})
 318     c_output1.update_attributes!(completed_attrs.merge({log: log1, output: out1}))
 319
 320     out2 = 'fa7aeb5140e2848d39b416daeef4ffc5+45'
 321     c_output2.update_attributes!({state: Container::Locked})
 322     c_output2.update_attributes!({state: Container::Running})
 323     c_output2.update_attributes!(completed_attrs.merge({log: log1, output: out2}))
 324
 325     set_user_from_auth :active
 326     reused = Container.resolve(ContainerRequest.new(request_only(common_attrs)))
 327     assert_equal c_output1.uuid, reused.uuid
 328   end
 329
 330   test "find_reusable method should select running container by start date" do
 331     set_user_from_auth :active
 332     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running"}})
 333     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
 334     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
 335     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
 336     # Confirm the 3 container UUIDs are different.
 337     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
 338     set_user_from_auth :dispatch1
 339     c_slower.update_attributes!({state: Container::Locked})
 340     c_slower.update_attributes!({state: Container::Running,
 341                                  progress: 0.1})
 342     c_faster_started_first.update_attributes!({state: Container::Locked})
 343     c_faster_started_first.update_attributes!({state: Container::Running,
 344                                                progress: 0.15})
 345     c_faster_started_second.update_attributes!({state: Container::Locked})
 346     c_faster_started_second.update_attributes!({state: Container::Running,
 347                                                 progress: 0.15})
 348     reused = Container.find_reusable(common_attrs)
 349     assert_not_nil reused
 350     # Selected container is the one that started first
 351     assert_equal reused.uuid, c_faster_started_first.uuid
 352   end
 353
 354   test "find_reusable method should select running container by progress" do
 355     set_user_from_auth :active
 356     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running2"}})
 357     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
 358     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
 359     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
 360     # Confirm the 3 container UUIDs are different.
 361     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
 362     set_user_from_auth :dispatch1
 363     c_slower.update_attributes!({state: Container::Locked})
 364     c_slower.update_attributes!({state: Container::Running,
 365                                  progress: 0.1})
 366     c_faster_started_first.update_attributes!({state: Container::Locked})
 367     c_faster_started_first.update_attributes!({state: Container::Running,
 368                                                progress: 0.15})
 369     c_faster_started_second.update_attributes!({state: Container::Locked})
 370     c_faster_started_second.update_attributes!({state: Container::Running,
 371                                                 progress: 0.2})
 372     reused = Container.find_reusable(common_attrs)
 373     assert_not_nil reused
 374     # Selected container is the one with most progress done
 375     assert_equal reused.uuid, c_faster_started_second.uuid
 376   end
 377
 378   test "find_reusable method should select non-failing running container" do
 379     set_user_from_auth :active
 380     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running2"}})
 381     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
 382     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
 383     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
 384     # Confirm the 3 container UUIDs are different.
 385     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
 386     set_user_from_auth :dispatch1
 387     c_slower.update_attributes!({state: Container::Locked})
 388     c_slower.update_attributes!({state: Container::Running,
 389                                  progress: 0.1})
 390     c_faster_started_first.update_attributes!({state: Container::Locked})
 391     c_faster_started_first.update_attributes!({state: Container::Running,
 392                                                runtime_status: {'warning' => 'This is not an error'},
 393                                                progress: 0.15})
 394     c_faster_started_second.update_attributes!({state: Container::Locked})
 395     assert_equal 0, Container.where("runtime_status->'error' is not null").count
 396     c_faster_started_second.update_attributes!({state: Container::Running,
 397                                                 runtime_status: {'error' => 'Something bad happened'},
 398                                                 progress: 0.2})
 399     assert_equal 1, Container.where("runtime_status->'error' is not null").count
 400     reused = Container.find_reusable(common_attrs)
 401     assert_not_nil reused
 402     # Selected the non-failing container even if it's the one with less progress done
 403     assert_equal reused.uuid, c_faster_started_first.uuid
 404   end
 405
 406   test "find_reusable method should select locked container most likely to start sooner" do
 407     set_user_from_auth :active
 408     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "locked"}})
 409     c_low_priority, _ = minimal_new(common_attrs.merge({use_existing: false}))
 410     c_high_priority_older, _ = minimal_new(common_attrs.merge({use_existing: false}))
 411     c_high_priority_newer, _ = minimal_new(common_attrs.merge({use_existing: false}))
 412     # Confirm the 3 container UUIDs are different.
 413     assert_equal 3, [c_low_priority.uuid, c_high_priority_older.uuid, c_high_priority_newer.uuid].uniq.length
 414     set_user_from_auth :dispatch1
 415     c_low_priority.update_attributes!({state: Container::Locked,
 416                                        priority: 1})
 417     c_high_priority_older.update_attributes!({state: Container::Locked,
 418                                               priority: 2})
 419     c_high_priority_newer.update_attributes!({state: Container::Locked,
 420                                               priority: 2})
 421     reused = Container.find_reusable(common_attrs)
 422     assert_not_nil reused
 423     assert_equal reused.uuid, c_high_priority_older.uuid
 424   end
 425
 426   test "find_reusable method should select running over failed container" do
 427     set_user_from_auth :active
 428     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed_vs_running"}})
 429     c_failed, _ = minimal_new(common_attrs.merge({use_existing: false}))
 430     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
 431     assert_not_equal c_failed.uuid, c_running.uuid
 432     set_user_from_auth :dispatch1
 433     c_failed.update_attributes!({state: Container::Locked})
 434     c_failed.update_attributes!({state: Container::Running})
 435     c_failed.update_attributes!({state: Container::Complete,
 436                                  exit_code: 42,
 437                                  log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 438                                  output: 'ea10d51bcf88862dbcc36eb292017dfd+45'})
 439     c_running.update_attributes!({state: Container::Locked})
 440     c_running.update_attributes!({state: Container::Running,
 441                                   progress: 0.15})
 442     reused = Container.find_reusable(common_attrs)
 443     assert_not_nil reused
 444     assert_equal reused.uuid, c_running.uuid
 445   end
 446
 447   test "find_reusable method should select complete over running container" do
 448     set_user_from_auth :active
 449     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "completed_vs_running"}})
 450     c_completed, _ = minimal_new(common_attrs.merge({use_existing: false}))
 451     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
 452     assert_not_equal c_completed.uuid, c_running.uuid
 453     set_user_from_auth :dispatch1
 454     c_completed.update_attributes!({state: Container::Locked})
 455     c_completed.update_attributes!({state: Container::Running})
 456     c_completed.update_attributes!({state: Container::Complete,
 457                                     exit_code: 0,
 458                                     log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 459                                     output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'})
 460     c_running.update_attributes!({state: Container::Locked})
 461     c_running.update_attributes!({state: Container::Running,
 462                                   progress: 0.15})
 463     reused = Container.find_reusable(common_attrs)
 464     assert_not_nil reused
 465     assert_equal c_completed.uuid, reused.uuid
 466   end
 467
 468   test "find_reusable method should select running over locked container" do
 469     set_user_from_auth :active
 470     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}})
 471     c_locked, _ = minimal_new(common_attrs.merge({use_existing: false}))
 472     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
 473     assert_not_equal c_running.uuid, c_locked.uuid
 474     set_user_from_auth :dispatch1
 475     c_locked.update_attributes!({state: Container::Locked})
 476     c_running.update_attributes!({state: Container::Locked})
 477     c_running.update_attributes!({state: Container::Running,
 478                                   progress: 0.15})
 479     reused = Container.find_reusable(common_attrs)
 480     assert_not_nil reused
 481     assert_equal reused.uuid, c_running.uuid
 482   end
 483
 484   test "find_reusable method should select locked over queued container" do
 485     set_user_from_auth :active
 486     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}})
 487     c_locked, _ = minimal_new(common_attrs.merge({use_existing: false}))
 488     c_queued, _ = minimal_new(common_attrs.merge({use_existing: false}))
 489     assert_not_equal c_queued.uuid, c_locked.uuid
 490     set_user_from_auth :dispatch1
 491     c_locked.update_attributes!({state: Container::Locked})
 492     reused = Container.find_reusable(common_attrs)
 493     assert_not_nil reused
 494     assert_equal reused.uuid, c_locked.uuid
 495   end
 496
 497   test "find_reusable method should not select failed container" do
 498     set_user_from_auth :active
 499     attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed"}})
 500     c, _ = minimal_new(attrs)
 501     set_user_from_auth :dispatch1
 502     c.update_attributes!({state: Container::Locked})
 503     c.update_attributes!({state: Container::Running})
 504     c.update_attributes!({state: Container::Complete,
 505                           exit_code: 33})
 506     reused = Container.find_reusable(attrs)
 507     assert_nil reused
 508   end
 509
 510   test "find_reusable with logging disabled" do
 511     set_user_from_auth :active
 512     Rails.logger.expects(:info).never
 513     Container.find_reusable(REUSABLE_COMMON_ATTRS)
 514   end
 515
 516   test "find_reusable with logging enabled" do
 517     set_user_from_auth :active
 518     Rails.configuration.Containers.LogReuseDecisions = true
 519     Rails.logger.expects(:info).at_least(3)
 520     Container.find_reusable(REUSABLE_COMMON_ATTRS)
 521   end
 522
 523   def runtime_token_attr tok
 524     auth = api_client_authorizations(tok)
 525     {runtime_user_uuid: User.find_by_id(auth.user_id).uuid,
 526      runtime_auth_scopes: auth.scopes,
 527      runtime_token: auth.token}
 528   end
 529
 530   test "find_reusable method with same runtime_token" do
 531     set_user_from_auth :active
 532     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 533     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:container_runtime_token).token}))
 534     assert_equal Container::Queued, c1.state
 535     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 536     assert_not_nil reused
 537     assert_equal reused.uuid, c1.uuid
 538   end
 539
 540   test "find_reusable method with different runtime_token, same user" do
 541     set_user_from_auth :active
 542     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 543     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:crt_user).token}))
 544     assert_equal Container::Queued, c1.state
 545     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 546     assert_not_nil reused
 547     assert_equal reused.uuid, c1.uuid
 548   end
 549
 550   test "find_reusable method with nil runtime_token, then runtime_token with same user" do
 551     set_user_from_auth :crt_user
 552     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 553     c1, _ = minimal_new(common_attrs)
 554     assert_equal Container::Queued, c1.state
 555     assert_equal users(:container_runtime_token_user).uuid, c1.runtime_user_uuid
 556     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 557     assert_not_nil reused
 558     assert_equal reused.uuid, c1.uuid
 559   end
 560
 561   test "find_reusable method with different runtime_token, different user" do
 562     set_user_from_auth :crt_user
 563     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 564     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:active).token}))
 565     assert_equal Container::Queued, c1.state
 566     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 567     # See #14584
 568     assert_not_nil reused
 569     assert_equal c1.uuid, reused.uuid
 570   end
 571
 572   test "find_reusable method with nil runtime_token, then runtime_token with different user" do
 573     set_user_from_auth :active
 574     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 575     c1, _ = minimal_new(common_attrs.merge({runtime_token: nil}))
 576     assert_equal Container::Queued, c1.state
 577     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 578     # See #14584
 579     assert_not_nil reused
 580     assert_equal c1.uuid, reused.uuid
 581   end
 582
 583   test "find_reusable method with different runtime_token, different scope, same user" do
 584     set_user_from_auth :active
 585     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 586     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:runtime_token_limited_scope).token}))
 587     assert_equal Container::Queued, c1.state
 588     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 589     # See #14584
 590     assert_not_nil reused
 591     assert_equal c1.uuid, reused.uuid
 592   end
 593
 594   test "find_reusable method with cuda" do
 595     set_user_from_auth :active
 596     # No cuda
 597     no_cuda_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"},
 598                                                 runtime_constraints: {"vcpus" => 1, "ram" => 1, "keep_cache_disk"=>0, "keep_cache_ram"=>268435456, "API" => false,
 599                                                                       "cuda" => {"device_count":0, "driver_version": "", "hardware_capability": ""}},})
 600     c1, _ = minimal_new(no_cuda_attrs)
 601     assert_equal Container::Queued, c1.state
 602
 603     # has cuda
 604     cuda_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"},
 605                                                 runtime_constraints: {"vcpus" => 1, "ram" => 1, "keep_cache_disk"=>0, "keep_cache_ram"=>268435456, "API" => false,
 606                                                                       "cuda" => {"device_count":1, "driver_version": "11.0", "hardware_capability": "9.0"}},})
 607     c2, _ = minimal_new(cuda_attrs)
 608     assert_equal Container::Queued, c2.state
 609
 610     # should find the no cuda one
 611     reused = Container.find_reusable(no_cuda_attrs)
 612     assert_not_nil reused
 613     assert_equal reused.uuid, c1.uuid
 614
 615     # should find the cuda one
 616     reused = Container.find_reusable(cuda_attrs)
 617     assert_not_nil reused
 618     assert_equal reused.uuid, c2.uuid
 619   end
 620
 621   test "Container running" do
 622     set_user_from_auth :active
 623     c, _ = minimal_new priority: 1
 624
 625     set_user_from_auth :dispatch1
 626     check_illegal_updates c, [{state: Container::Running},
 627                               {state: Container::Complete}]
 628
 629     c.lock
 630     c.update_attributes! state: Container::Running
 631
 632     check_illegal_modify c
 633     check_bogus_states c
 634
 635     check_illegal_updates c, [{state: Container::Queued}]
 636     c.reload
 637
 638     c.update_attributes! priority: 3
 639   end
 640
 641   test "Lock and unlock" do
 642     set_user_from_auth :active
 643     c, cr = minimal_new priority: 0
 644
 645     set_user_from_auth :dispatch1
 646     assert_equal Container::Queued, c.state
 647
 648     assert_raise(ArvadosModel::LockFailedError) do
 649       # "no priority"
 650       c.lock
 651     end
 652     c.reload
 653     assert cr.update_attributes priority: 1
 654
 655     refute c.update_attributes(state: Container::Running), "not locked"
 656     c.reload
 657     refute c.update_attributes(state: Container::Complete), "not locked"
 658     c.reload
 659
 660     assert c.lock, show_errors(c)
 661     assert c.locked_by_uuid
 662     assert c.auth_uuid
 663
 664     assert_raise(ArvadosModel::LockFailedError) {c.lock}
 665     c.reload
 666
 667     assert c.unlock, show_errors(c)
 668     refute c.locked_by_uuid
 669     refute c.auth_uuid
 670
 671     refute c.update_attributes(state: Container::Running), "not locked"
 672     c.reload
 673     refute c.locked_by_uuid
 674     refute c.auth_uuid
 675
 676     assert c.lock, show_errors(c)
 677     assert c.update_attributes(state: Container::Running), show_errors(c)
 678     assert c.locked_by_uuid
 679     assert c.auth_uuid
 680
 681     auth_uuid_was = c.auth_uuid
 682
 683     assert_raise(ArvadosModel::LockFailedError) do
 684       # Running to Locked is not allowed
 685       c.lock
 686     end
 687     c.reload
 688     assert_raise(ArvadosModel::InvalidStateTransitionError) do
 689       # Running to Queued is not allowed
 690       c.unlock
 691     end
 692     c.reload
 693
 694     assert c.update_attributes(state: Container::Complete), show_errors(c)
 695     refute c.locked_by_uuid
 696     refute c.auth_uuid
 697
 698     auth_exp = ApiClientAuthorization.find_by_uuid(auth_uuid_was).expires_at
 699     assert_operator auth_exp, :<, db_current_time
 700
 701     assert_nil ApiClientAuthorization.validate(token: ApiClientAuthorization.find_by_uuid(auth_uuid_was).token)
 702   end
 703
 704   test "Exceed maximum lock-unlock cycles" do
 705     Rails.configuration.Containers.MaxDispatchAttempts = 3
 706
 707     set_user_from_auth :active
 708     c, cr = minimal_new
 709
 710     set_user_from_auth :dispatch1
 711     assert_equal Container::Queued, c.state
 712     assert_equal 0, c.lock_count
 713
 714     c.lock
 715     c.reload
 716     assert_equal 1, c.lock_count
 717     assert_equal Container::Locked, c.state
 718
 719     c.unlock
 720     c.reload
 721     assert_equal 1, c.lock_count
 722     assert_equal Container::Queued, c.state
 723
 724     c.lock
 725     c.reload
 726     assert_equal 2, c.lock_count
 727     assert_equal Container::Locked, c.state
 728
 729     c.unlock
 730     c.reload
 731     assert_equal 2, c.lock_count
 732     assert_equal Container::Queued, c.state
 733
 734     c.lock
 735     c.reload
 736     assert_equal 3, c.lock_count
 737     assert_equal Container::Locked, c.state
 738
 739     c.unlock
 740     c.reload
 741     assert_equal 3, c.lock_count
 742     assert_equal Container::Cancelled, c.state
 743
 744     assert_raise(ArvadosModel::LockFailedError) do
 745       # Cancelled to Locked is not allowed
 746       c.lock
 747     end
 748   end
 749
 750   test "Container queued cancel" do
 751     set_user_from_auth :active
 752     c, cr = minimal_new({container_count_max: 1})
 753     set_user_from_auth :dispatch1
 754     assert c.update_attributes(state: Container::Cancelled), show_errors(c)
 755     check_no_change_from_cancelled c
 756     cr.reload
 757     assert_equal ContainerRequest::Final, cr.state
 758   end
 759
 760   test "Container queued count" do
 761     assert_equal 1, Container.readable_by(users(:active)).where(state: "Queued").count
 762   end
 763
 764   test "Containers with no matching request are readable by admin" do
 765     uuids = Container.includes('container_requests').where(container_requests: {uuid: nil}).collect(&:uuid)
 766     assert_not_empty uuids
 767     assert_empty Container.readable_by(users(:active)).where(uuid: uuids)
 768     assert_not_empty Container.readable_by(users(:admin)).where(uuid: uuids)
 769     assert_equal uuids.count, Container.readable_by(users(:admin)).where(uuid: uuids).count
 770   end
 771
 772   test "Container locked cancel" do
 773     set_user_from_auth :active
 774     c, _ = minimal_new
 775     set_user_from_auth :dispatch1
 776     assert c.lock, show_errors(c)
 777     assert c.update_attributes(state: Container::Cancelled), show_errors(c)
 778     check_no_change_from_cancelled c
 779   end
 780
 781   test "Container locked with non-expiring token" do
 782     Rails.configuration.API.TokenMaxLifetime = 1.hour
 783     set_user_from_auth :active
 784     c, _ = minimal_new
 785     set_user_from_auth :dispatch1
 786     assert c.lock, show_errors(c)
 787     refute c.auth.nil?
 788     assert c.auth.expires_at.nil?
 789     assert c.auth.user_id == User.find_by_uuid(users(:active).uuid).id
 790   end
 791
 792   test "Container locked cancel with log" do
 793     set_user_from_auth :active
 794     c, _ = minimal_new
 795     set_user_from_auth :dispatch1
 796     assert c.lock, show_errors(c)
 797     assert c.update_attributes(
 798              state: Container::Cancelled,
 799              log: collections(:real_log_collection).portable_data_hash,
 800            ), show_errors(c)
 801     check_no_change_from_cancelled c
 802   end
 803
 804   test "Container running cancel" do
 805     set_user_from_auth :active
 806     c, _ = minimal_new
 807     set_user_from_auth :dispatch1
 808     c.lock
 809     c.update_attributes! state: Container::Running
 810     c.update_attributes! state: Container::Cancelled
 811     check_no_change_from_cancelled c
 812   end
 813
 814   test "Container create forbidden for non-admin" do
 815     set_user_from_auth :active_trustedclient
 816     c = Container.new DEFAULT_ATTRS
 817     c.environment = {}
 818     c.mounts = {"BAR" => "FOO"}
 819     c.output_path = "/tmp"
 820     c.priority = 1
 821     c.runtime_constraints = {}
 822     assert_raises(ArvadosModel::PermissionDeniedError) do
 823       c.save!
 824     end
 825   end
 826
 827   [
 828     [Container::Queued, {state: Container::Locked}],
 829     [Container::Queued, {state: Container::Running}],
 830     [Container::Queued, {state: Container::Complete}],
 831     [Container::Queued, {state: Container::Cancelled}],
 832     [Container::Queued, {priority: 123456789}],
 833     [Container::Queued, {runtime_status: {'error' => 'oops'}}],
 834     [Container::Queued, {cwd: '/'}],
 835     [Container::Locked, {state: Container::Running}],
 836     [Container::Locked, {state: Container::Queued}],
 837     [Container::Locked, {priority: 123456789}],
 838     [Container::Locked, {runtime_status: {'error' => 'oops'}}],
 839     [Container::Locked, {cwd: '/'}],
 840     [Container::Running, {state: Container::Complete}],
 841     [Container::Running, {state: Container::Cancelled}],
 842     [Container::Running, {priority: 123456789}],
 843     [Container::Running, {runtime_status: {'error' => 'oops'}}],
 844     [Container::Running, {cwd: '/'}],
 845     [Container::Running, {gateway_address: "172.16.0.1:12345"}],
 846     [Container::Running, {interactive_session_started: true}],
 847     [Container::Complete, {state: Container::Cancelled}],
 848     [Container::Complete, {priority: 123456789}],
 849     [Container::Complete, {runtime_status: {'error' => 'oops'}}],
 850     [Container::Complete, {cwd: '/'}],
 851     [Container::Cancelled, {cwd: '/'}],
 852   ].each do |start_state, updates|
 853     test "Container update #{updates.inspect} when #{start_state} forbidden for non-admin" do
 854       set_user_from_auth :active
 855       c, _ = minimal_new
 856       if start_state != Container::Queued
 857         set_user_from_auth :dispatch1
 858         c.lock
 859         if start_state != Container::Locked
 860           c.update_attributes! state: Container::Running
 861           if start_state != Container::Running
 862             c.update_attributes! state: start_state
 863           end
 864         end
 865       end
 866       assert_equal c.state, start_state
 867       set_user_from_auth :active
 868       assert_raises(ArvadosModel::PermissionDeniedError) do
 869         c.update_attributes! updates
 870       end
 871     end
 872   end
 873
 874   test "can only change exit code while running and at completion" do
 875     set_user_from_auth :active
 876     c, _ = minimal_new
 877     set_user_from_auth :dispatch1
 878     c.lock
 879     check_illegal_updates c, [{exit_code: 1}]
 880     c.update_attributes! state: Container::Running
 881     assert c.update_attributes(exit_code: 1)
 882     assert c.update_attributes(exit_code: 1, state: Container::Complete)
 883   end
 884
 885   test "locked_by_uuid can update log when locked/running, and output when running" do
 886     set_user_from_auth :active
 887     logcoll = collections(:real_log_collection)
 888     c, cr1 = minimal_new
 889     cr2 = ContainerRequest.new(DEFAULT_ATTRS)
 890     cr2.state = ContainerRequest::Committed
 891     act_as_user users(:active) do
 892       cr2.save!
 893     end
 894     assert_equal cr1.container_uuid, cr2.container_uuid
 895
 896     logpdh_time1 = logcoll.portable_data_hash
 897
 898     set_user_from_auth :dispatch1
 899     c.lock
 900     assert_equal c.locked_by_uuid, Thread.current[:api_client_authorization].uuid
 901     c.update_attributes!(log: logpdh_time1)
 902     c.update_attributes!(state: Container::Running)
 903     cr1.reload
 904     cr2.reload
 905     cr1log_uuid = cr1.log_uuid
 906     cr2log_uuid = cr2.log_uuid
 907     assert_not_nil cr1log_uuid
 908     assert_not_nil cr2log_uuid
 909     assert_not_equal logcoll.uuid, cr1log_uuid
 910     assert_not_equal logcoll.uuid, cr2log_uuid
 911     assert_not_equal cr1log_uuid, cr2log_uuid
 912
 913     logcoll.update_attributes!(manifest_text: logcoll.manifest_text + ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n")
 914     logpdh_time2 = logcoll.portable_data_hash
 915
 916     assert c.update_attributes(output: collections(:collection_owned_by_active).portable_data_hash)
 917     assert c.update_attributes(log: logpdh_time2)
 918     assert c.update_attributes(state: Container::Complete, log: logcoll.portable_data_hash)
 919     c.reload
 920     assert_equal collections(:collection_owned_by_active).portable_data_hash, c.output
 921     assert_equal logpdh_time2, c.log
 922     refute c.update_attributes(output: nil)
 923     refute c.update_attributes(log: nil)
 924     cr1.reload
 925     cr2.reload
 926     assert_equal cr1log_uuid, cr1.log_uuid
 927     assert_equal cr2log_uuid, cr2.log_uuid
 928     assert_equal 1, Collection.where(uuid: [cr1log_uuid, cr2log_uuid]).to_a.collect(&:portable_data_hash).uniq.length
 929     assert_equal ". acbd18db4cc2f85cedef654fccc4a4d8+3 cdd549ae79fe6640fa3d5c6261d8303c+195 0:3:foo.txt 3:195:zzzzz-8i9sb-0vsrcqi7whchuil.log.txt
 930 ./log\\040for\\040container\\040#{cr1.container_uuid} acbd18db4cc2f85cedef654fccc4a4d8+3 cdd549ae79fe6640fa3d5c6261d8303c+195 0:3:foo.txt 3:195:zzzzz-8i9sb-0vsrcqi7whchuil.log.txt
 931 ", Collection.find_by_uuid(cr1log_uuid).manifest_text
 932   end
 933
 934   ["auth_uuid", "runtime_token"].each do |tok|
 935     test "#{tok} can set output, progress, runtime_status, state, exit_code on running container -- but not log" do
 936       if tok == "runtime_token"
 937         set_user_from_auth :spectator
 938         c, _ = minimal_new(container_image: "9ae44d5792468c58bcf85ce7353c7027+124",
 939                            runtime_token: api_client_authorizations(:active).token)
 940       else
 941         set_user_from_auth :active
 942         c, _ = minimal_new
 943       end
 944       set_user_from_auth :dispatch1
 945       c.lock
 946       c.update_attributes! state: Container::Running
 947
 948       if tok == "runtime_token"
 949         auth = ApiClientAuthorization.validate(token: c.runtime_token)
 950         Thread.current[:api_client_authorization] = auth
 951         Thread.current[:api_client] = auth.api_client
 952         Thread.current[:token] = auth.token
 953         Thread.current[:user] = auth.user
 954       else
 955         auth = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
 956         Thread.current[:api_client_authorization] = auth
 957         Thread.current[:api_client] = auth.api_client
 958         Thread.current[:token] = auth.token
 959         Thread.current[:user] = auth.user
 960       end
 961
 962       assert c.update_attributes(gateway_address: "127.0.0.1:9")
 963       assert c.update_attributes(output: collections(:collection_owned_by_active).portable_data_hash)
 964       assert c.update_attributes(runtime_status: {'warning' => 'something happened'})
 965       assert c.update_attributes(progress: 0.5)
 966       assert c.update_attributes(exit_code: 0)
 967       refute c.update_attributes(log: collections(:real_log_collection).portable_data_hash)
 968       c.reload
 969       assert c.update_attributes(state: Container::Complete, exit_code: 0)
 970     end
 971   end
 972
 973   test "not allowed to set output that is not readable by current user" do
 974     set_user_from_auth :active
 975     c, _ = minimal_new
 976     set_user_from_auth :dispatch1
 977     c.lock
 978     c.update_attributes! state: Container::Running
 979
 980     Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
 981     Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
 982
 983     assert_raises ActiveRecord::RecordInvalid do
 984       c.update_attributes! output: collections(:collection_not_readable_by_active).portable_data_hash
 985     end
 986   end
 987
 988   test "other token cannot set output on running container" do
 989     set_user_from_auth :active
 990     c, _ = minimal_new
 991     set_user_from_auth :dispatch1
 992     c.lock
 993     c.update_attributes! state: Container::Running
 994
 995     set_user_from_auth :running_to_be_deleted_container_auth
 996     assert_raises(ArvadosModel::PermissionDeniedError) do
 997       c.update_attributes(output: collections(:foo_file).portable_data_hash)
 998     end
 999   end
1000
1001   test "can set trashed output on running container" do
1002     set_user_from_auth :active
1003     c, _ = minimal_new
1004     set_user_from_auth :dispatch1
1005     c.lock
1006     c.update_attributes! state: Container::Running
1007
1008     output = Collection.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jk')
1009
1010     assert output.is_trashed
1011     assert c.update_attributes output: output.portable_data_hash
1012     assert c.update_attributes! state: Container::Complete
1013   end
1014
1015   test "not allowed to set trashed output that is not readable by current user" do
1016     set_user_from_auth :active
1017     c, _ = minimal_new
1018     set_user_from_auth :dispatch1
1019     c.lock
1020     c.update_attributes! state: Container::Running
1021
1022     output = Collection.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jr')
1023
1024     Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
1025     Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
1026
1027     assert_raises ActiveRecord::RecordInvalid do
1028       c.update_attributes! output: output.portable_data_hash
1029     end
1030   end
1031
1032   test "user cannot delete" do
1033     set_user_from_auth :active
1034     c, _ = minimal_new
1035     assert_raises ArvadosModel::PermissionDeniedError do
1036       c.destroy
1037     end
1038     assert Container.find_by_uuid(c.uuid)
1039   end
1040
1041   [
1042     {state: Container::Complete, exit_code: 0, output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'},
1043     {state: Container::Cancelled},
1044   ].each do |final_attrs|
1045     test "secret_mounts and runtime_token are null after container is #{final_attrs[:state]}" do
1046       set_user_from_auth :active
1047       c, cr = minimal_new(secret_mounts: {'/secret' => {'kind' => 'text', 'content' => 'foo'}},
1048                           container_count_max: 1, runtime_token: api_client_authorizations(:active).token)
1049       set_user_from_auth :dispatch1
1050       c.lock
1051       c.update_attributes!(state: Container::Running)
1052       c.reload
1053       assert c.secret_mounts.has_key?('/secret')
1054       assert_equal api_client_authorizations(:active).token, c.runtime_token
1055
1056       c.update_attributes!(final_attrs)
1057       c.reload
1058       assert_equal({}, c.secret_mounts)
1059       assert_nil c.runtime_token
1060       cr.reload
1061       assert_equal({}, cr.secret_mounts)
1062       assert_nil cr.runtime_token
1063       assert_no_secrets_logged
1064     end
1065   end
1066 end