services/api/test/unit/container_test.rb

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 require 'test_helper'
   6 require 'helpers/container_test_helper'
   7
   8 class ContainerTest < ActiveSupport::TestCase
   9   include DbCurrentTime
  10   include ContainerTestHelper
  11
  12   DEFAULT_ATTRS = {
  13     command: ['echo', 'foo'],
  14     container_image: 'fa3c1a9cb6783f85f2ecda037e07b8c3+167',
  15     output_path: '/tmp',
  16     priority: 1,
  17     runtime_constraints: {"vcpus" => 1, "ram" => 1},
  18   }
  19
  20   REUSABLE_COMMON_ATTRS = {
  21     container_image: "9ae44d5792468c58bcf85ce7353c7027+124",
  22     cwd: "test",
  23     command: ["echo", "hello"],
  24     output_path: "test",
  25     runtime_constraints: {
  26       "ram" => 12000000000,
  27       "vcpus" => 4,
  28     },
  29     mounts: {
  30       "test" => {"kind" => "json"},
  31     },
  32     environment: {
  33       "var" => "val",
  34     },
  35     secret_mounts: {},
  36     runtime_user_uuid: "zzzzz-tpzed-xurymjxw79nv3jz",
  37     runtime_auth_scopes: ["all"]
  38   }
  39
  40   def request_only attrs
  41     attrs.reject {|k| [:runtime_user_uuid, :runtime_auth_scopes].include? k}
  42   end
  43
  44   def minimal_new attrs={}
  45     cr = ContainerRequest.new request_only(DEFAULT_ATTRS.merge(attrs))
  46     cr.state = ContainerRequest::Committed
  47     cr.save!
  48     c = Container.find_by_uuid cr.container_uuid
  49     assert_not_nil c
  50     return c, cr
  51   end
  52
  53   def check_illegal_updates c, bad_updates
  54     bad_updates.each do |u|
  55       refute c.update_attributes(u), u.inspect
  56       refute c.valid?, u.inspect
  57       c.reload
  58     end
  59   end
  60
  61   def check_illegal_modify c
  62     check_illegal_updates c, [{command: ["echo", "bar"]},
  63                               {container_image: "arvados/apitestfixture:june10"},
  64                               {cwd: "/tmp2"},
  65                               {environment: {"FOO" => "BAR"}},
  66                               {mounts: {"FOO" => "BAR"}},
  67                               {output_path: "/tmp3"},
  68                               {locked_by_uuid: "zzzzz-gj3su-027z32aux8dg2s1"},
  69                               {auth_uuid: "zzzzz-gj3su-017z32aux8dg2s1"},
  70                               {runtime_constraints: {"FOO" => "BAR"}}]
  71   end
  72
  73   def check_bogus_states c
  74     check_illegal_updates c, [{state: nil},
  75                               {state: "Flubber"}]
  76   end
  77
  78   def check_no_change_from_cancelled c
  79     check_illegal_modify c
  80     check_bogus_states c
  81     check_illegal_updates c, [{ priority: 3 },
  82                               { state: Container::Queued },
  83                               { state: Container::Locked },
  84                               { state: Container::Running },
  85                               { state: Container::Complete }]
  86   end
  87
  88   test "Container create" do
  89     act_as_system_user do
  90       c, _ = minimal_new(environment: {},
  91                       mounts: {"BAR" => {"kind" => "FOO"}},
  92                       output_path: "/tmp",
  93                       priority: 1,
  94                       runtime_constraints: {"vcpus" => 1, "ram" => 1})
  95
  96       check_illegal_modify c
  97       check_bogus_states c
  98
  99       c.reload
 100       c.priority = 2
 101       c.save!
 102     end
 103   end
 104
 105   test "Container valid priority" do
 106     act_as_system_user do
 107       c, _ = minimal_new(environment: {},
 108                       mounts: {"BAR" => {"kind" => "FOO"}},
 109                       output_path: "/tmp",
 110                       priority: 1,
 111                       runtime_constraints: {"vcpus" => 1, "ram" => 1})
 112
 113       assert_raises(ActiveRecord::RecordInvalid) do
 114         c.priority = -1
 115         c.save!
 116       end
 117
 118       c.priority = 0
 119       c.save!
 120
 121       c.priority = 1
 122       c.save!
 123
 124       c.priority = 500
 125       c.save!
 126
 127       c.priority = 999
 128       c.save!
 129
 130       c.priority = 1000
 131       c.save!
 132
 133       c.priority = 1000 << 50
 134       c.save!
 135     end
 136   end
 137
 138   test "Container runtime_status data types" do
 139     set_user_from_auth :active
 140     attrs = {
 141       environment: {},
 142       mounts: {"BAR" => {"kind" => "FOO"}},
 143       output_path: "/tmp",
 144       priority: 1,
 145       runtime_constraints: {"vcpus" => 1, "ram" => 1}
 146     }
 147     c, _ = minimal_new(attrs)
 148     assert_equal c.runtime_status, {}
 149     assert_equal Container::Queued, c.state
 150
 151     set_user_from_auth :dispatch1
 152     c.update_attributes! state: Container::Locked
 153     c.update_attributes! state: Container::Running
 154
 155     [
 156       'error', 'errorDetail', 'warning', 'warningDetail', 'activity'
 157     ].each do |k|
 158       # String type is allowed
 159       string_val = 'A string is accepted'
 160       c.update_attributes! runtime_status: {k => string_val}
 161       assert_equal string_val, c.runtime_status[k]
 162
 163       # Other types aren't allowed
 164       [
 165         42, false, [], {}, nil
 166       ].each do |unallowed_val|
 167         assert_raises ActiveRecord::RecordInvalid do
 168           c.update_attributes! runtime_status: {k => unallowed_val}
 169         end
 170       end
 171     end
 172   end
 173
 174   test "Container runtime_status updates" do
 175     set_user_from_auth :active
 176     attrs = {
 177       environment: {},
 178       mounts: {"BAR" => {"kind" => "FOO"}},
 179       output_path: "/tmp",
 180       priority: 1,
 181       runtime_constraints: {"vcpus" => 1, "ram" => 1}
 182     }
 183     c1, _ = minimal_new(attrs)
 184     assert_equal c1.runtime_status, {}
 185
 186     assert_equal Container::Queued, c1.state
 187     assert_raises ArvadosModel::PermissionDeniedError do
 188       c1.update_attributes! runtime_status: {'error' => 'Oops!'}
 189     end
 190
 191     set_user_from_auth :dispatch1
 192
 193     # Allow updates when state = Locked
 194     c1.update_attributes! state: Container::Locked
 195     c1.update_attributes! runtime_status: {'error' => 'Oops!'}
 196     assert c1.runtime_status.key? 'error'
 197
 198     # Reset when transitioning from Locked to Queued
 199     c1.update_attributes! state: Container::Queued
 200     assert_equal c1.runtime_status, {}
 201
 202     # Allow updates when state = Running
 203     c1.update_attributes! state: Container::Locked
 204     c1.update_attributes! state: Container::Running
 205     c1.update_attributes! runtime_status: {'error' => 'Oops!'}
 206     assert c1.runtime_status.key? 'error'
 207
 208     # Don't allow updates on other states
 209     c1.update_attributes! state: Container::Complete
 210     assert_raises ActiveRecord::RecordInvalid do
 211       c1.update_attributes! runtime_status: {'error' => 'Some other error'}
 212     end
 213
 214     set_user_from_auth :active
 215     c2, _ = minimal_new(attrs)
 216     assert_equal c2.runtime_status, {}
 217     set_user_from_auth :dispatch1
 218     c2.update_attributes! state: Container::Locked
 219     c2.update_attributes! state: Container::Running
 220     c2.update_attributes! state: Container::Cancelled
 221     assert_raises ActiveRecord::RecordInvalid do
 222       c2.update_attributes! runtime_status: {'error' => 'Oops!'}
 223     end
 224   end
 225
 226   test "Container serialized hash attributes sorted before save" do
 227     set_user_from_auth :active
 228     env = {"C" => "3", "B" => "2", "A" => "1"}
 229     m = {"F" => {"kind" => "3"}, "E" => {"kind" => "2"}, "D" => {"kind" => "1"}}
 230     rc = {"vcpus" => 1, "ram" => 1, "keep_cache_ram" => 1}
 231     c, _ = minimal_new(environment: env, mounts: m, runtime_constraints: rc)
 232     assert_equal c.environment.to_json, Container.deep_sort_hash(env).to_json
 233     assert_equal c.mounts.to_json, Container.deep_sort_hash(m).to_json
 234     assert_equal c.runtime_constraints.to_json, Container.deep_sort_hash(rc).to_json
 235   end
 236
 237   test 'deep_sort_hash on array of hashes' do
 238     a = {'z' => [[{'a' => 'a', 'b' => 'b'}]]}
 239     b = {'z' => [[{'b' => 'b', 'a' => 'a'}]]}
 240     assert_equal Container.deep_sort_hash(a).to_json, Container.deep_sort_hash(b).to_json
 241   end
 242
 243   test "find_reusable method should select higher priority queued container" do
 244         Rails.configuration.Containers.LogReuseDecisions = true
 245     set_user_from_auth :active
 246     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment:{"var" => "queued"}})
 247     c_low_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:1}))
 248     c_high_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:2}))
 249     assert_not_equal c_low_priority.uuid, c_high_priority.uuid
 250     assert_equal Container::Queued, c_low_priority.state
 251     assert_equal Container::Queued, c_high_priority.state
 252     reused = Container.find_reusable(common_attrs)
 253     assert_not_nil reused
 254     assert_equal reused.uuid, c_high_priority.uuid
 255   end
 256
 257   test "find_reusable method should select latest completed container" do
 258     set_user_from_auth :active
 259     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "complete"}})
 260     completed_attrs = {
 261       state: Container::Complete,
 262       exit_code: 0,
 263       log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 264       output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
 265     }
 266
 267     c_older, _ = minimal_new(common_attrs.merge({use_existing: false}))
 268     c_recent, _ = minimal_new(common_attrs.merge({use_existing: false}))
 269     assert_not_equal c_older.uuid, c_recent.uuid
 270
 271     set_user_from_auth :dispatch1
 272     c_older.update_attributes!({state: Container::Locked})
 273     c_older.update_attributes!({state: Container::Running})
 274     c_older.update_attributes!(completed_attrs)
 275
 276     c_recent.update_attributes!({state: Container::Locked})
 277     c_recent.update_attributes!({state: Container::Running})
 278     c_recent.update_attributes!(completed_attrs)
 279
 280     reused = Container.find_reusable(common_attrs)
 281     assert_not_nil reused
 282     assert_equal reused.uuid, c_older.uuid
 283   end
 284
 285   test "find_reusable method should select oldest completed container when inconsistent outputs exist" do
 286     set_user_from_auth :active
 287     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "complete"}, priority: 1})
 288     completed_attrs = {
 289       state: Container::Complete,
 290       exit_code: 0,
 291       log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 292     }
 293
 294     cr = ContainerRequest.new request_only(common_attrs)
 295     cr.use_existing = false
 296     cr.state = ContainerRequest::Committed
 297     cr.save!
 298     c_output1 = Container.where(uuid: cr.container_uuid).first
 299
 300     cr = ContainerRequest.new request_only(common_attrs)
 301     cr.use_existing = false
 302     cr.state = ContainerRequest::Committed
 303     cr.save!
 304     c_output2 = Container.where(uuid: cr.container_uuid).first
 305
 306     assert_not_equal c_output1.uuid, c_output2.uuid
 307
 308     set_user_from_auth :dispatch1
 309
 310     out1 = '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
 311     log1 = collections(:real_log_collection).portable_data_hash
 312     c_output1.update_attributes!({state: Container::Locked})
 313     c_output1.update_attributes!({state: Container::Running})
 314     c_output1.update_attributes!(completed_attrs.merge({log: log1, output: out1}))
 315
 316     out2 = 'fa7aeb5140e2848d39b416daeef4ffc5+45'
 317     c_output2.update_attributes!({state: Container::Locked})
 318     c_output2.update_attributes!({state: Container::Running})
 319     c_output2.update_attributes!(completed_attrs.merge({log: log1, output: out2}))
 320
 321     set_user_from_auth :active
 322     reused = Container.resolve(ContainerRequest.new(request_only(common_attrs)))
 323     assert_equal c_output1.uuid, reused.uuid
 324   end
 325
 326   test "find_reusable method should select running container by start date" do
 327     set_user_from_auth :active
 328     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running"}})
 329     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
 330     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
 331     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
 332     # Confirm the 3 container UUIDs are different.
 333     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
 334     set_user_from_auth :dispatch1
 335     c_slower.update_attributes!({state: Container::Locked})
 336     c_slower.update_attributes!({state: Container::Running,
 337                                  progress: 0.1})
 338     c_faster_started_first.update_attributes!({state: Container::Locked})
 339     c_faster_started_first.update_attributes!({state: Container::Running,
 340                                                progress: 0.15})
 341     c_faster_started_second.update_attributes!({state: Container::Locked})
 342     c_faster_started_second.update_attributes!({state: Container::Running,
 343                                                 progress: 0.15})
 344     reused = Container.find_reusable(common_attrs)
 345     assert_not_nil reused
 346     # Selected container is the one that started first
 347     assert_equal reused.uuid, c_faster_started_first.uuid
 348   end
 349
 350   test "find_reusable method should select running container by progress" do
 351     set_user_from_auth :active
 352     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running2"}})
 353     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
 354     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
 355     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
 356     # Confirm the 3 container UUIDs are different.
 357     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
 358     set_user_from_auth :dispatch1
 359     c_slower.update_attributes!({state: Container::Locked})
 360     c_slower.update_attributes!({state: Container::Running,
 361                                  progress: 0.1})
 362     c_faster_started_first.update_attributes!({state: Container::Locked})
 363     c_faster_started_first.update_attributes!({state: Container::Running,
 364                                                progress: 0.15})
 365     c_faster_started_second.update_attributes!({state: Container::Locked})
 366     c_faster_started_second.update_attributes!({state: Container::Running,
 367                                                 progress: 0.2})
 368     reused = Container.find_reusable(common_attrs)
 369     assert_not_nil reused
 370     # Selected container is the one with most progress done
 371     assert_equal reused.uuid, c_faster_started_second.uuid
 372   end
 373
 374   test "find_reusable method should select non-failing running container" do
 375     set_user_from_auth :active
 376     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running2"}})
 377     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
 378     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
 379     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
 380     # Confirm the 3 container UUIDs are different.
 381     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
 382     set_user_from_auth :dispatch1
 383     c_slower.update_attributes!({state: Container::Locked})
 384     c_slower.update_attributes!({state: Container::Running,
 385                                  progress: 0.1})
 386     c_faster_started_first.update_attributes!({state: Container::Locked})
 387     c_faster_started_first.update_attributes!({state: Container::Running,
 388                                                runtime_status: {'warning' => 'This is not an error'},
 389                                                progress: 0.15})
 390     c_faster_started_second.update_attributes!({state: Container::Locked})
 391     assert_equal 0, Container.where("runtime_status->'error' is not null").count
 392     c_faster_started_second.update_attributes!({state: Container::Running,
 393                                                 runtime_status: {'error' => 'Something bad happened'},
 394                                                 progress: 0.2})
 395     assert_equal 1, Container.where("runtime_status->'error' is not null").count
 396     reused = Container.find_reusable(common_attrs)
 397     assert_not_nil reused
 398     # Selected the non-failing container even if it's the one with less progress done
 399     assert_equal reused.uuid, c_faster_started_first.uuid
 400   end
 401
 402   test "find_reusable method should select locked container most likely to start sooner" do
 403     set_user_from_auth :active
 404     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "locked"}})
 405     c_low_priority, _ = minimal_new(common_attrs.merge({use_existing: false}))
 406     c_high_priority_older, _ = minimal_new(common_attrs.merge({use_existing: false}))
 407     c_high_priority_newer, _ = minimal_new(common_attrs.merge({use_existing: false}))
 408     # Confirm the 3 container UUIDs are different.
 409     assert_equal 3, [c_low_priority.uuid, c_high_priority_older.uuid, c_high_priority_newer.uuid].uniq.length
 410     set_user_from_auth :dispatch1
 411     c_low_priority.update_attributes!({state: Container::Locked,
 412                                        priority: 1})
 413     c_high_priority_older.update_attributes!({state: Container::Locked,
 414                                               priority: 2})
 415     c_high_priority_newer.update_attributes!({state: Container::Locked,
 416                                               priority: 2})
 417     reused = Container.find_reusable(common_attrs)
 418     assert_not_nil reused
 419     assert_equal reused.uuid, c_high_priority_older.uuid
 420   end
 421
 422   test "find_reusable method should select running over failed container" do
 423     set_user_from_auth :active
 424     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed_vs_running"}})
 425     c_failed, _ = minimal_new(common_attrs.merge({use_existing: false}))
 426     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
 427     assert_not_equal c_failed.uuid, c_running.uuid
 428     set_user_from_auth :dispatch1
 429     c_failed.update_attributes!({state: Container::Locked})
 430     c_failed.update_attributes!({state: Container::Running})
 431     c_failed.update_attributes!({state: Container::Complete,
 432                                  exit_code: 42,
 433                                  log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 434                                  output: 'ea10d51bcf88862dbcc36eb292017dfd+45'})
 435     c_running.update_attributes!({state: Container::Locked})
 436     c_running.update_attributes!({state: Container::Running,
 437                                   progress: 0.15})
 438     reused = Container.find_reusable(common_attrs)
 439     assert_not_nil reused
 440     assert_equal reused.uuid, c_running.uuid
 441   end
 442
 443   test "find_reusable method should select complete over running container" do
 444     set_user_from_auth :active
 445     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "completed_vs_running"}})
 446     c_completed, _ = minimal_new(common_attrs.merge({use_existing: false}))
 447     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
 448     assert_not_equal c_completed.uuid, c_running.uuid
 449     set_user_from_auth :dispatch1
 450     c_completed.update_attributes!({state: Container::Locked})
 451     c_completed.update_attributes!({state: Container::Running})
 452     c_completed.update_attributes!({state: Container::Complete,
 453                                     exit_code: 0,
 454                                     log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 455                                     output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'})
 456     c_running.update_attributes!({state: Container::Locked})
 457     c_running.update_attributes!({state: Container::Running,
 458                                   progress: 0.15})
 459     reused = Container.find_reusable(common_attrs)
 460     assert_not_nil reused
 461     assert_equal c_completed.uuid, reused.uuid
 462   end
 463
 464   test "find_reusable method should select running over locked container" do
 465     set_user_from_auth :active
 466     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}})
 467     c_locked, _ = minimal_new(common_attrs.merge({use_existing: false}))
 468     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
 469     assert_not_equal c_running.uuid, c_locked.uuid
 470     set_user_from_auth :dispatch1
 471     c_locked.update_attributes!({state: Container::Locked})
 472     c_running.update_attributes!({state: Container::Locked})
 473     c_running.update_attributes!({state: Container::Running,
 474                                   progress: 0.15})
 475     reused = Container.find_reusable(common_attrs)
 476     assert_not_nil reused
 477     assert_equal reused.uuid, c_running.uuid
 478   end
 479
 480   test "find_reusable method should select locked over queued container" do
 481     set_user_from_auth :active
 482     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}})
 483     c_locked, _ = minimal_new(common_attrs.merge({use_existing: false}))
 484     c_queued, _ = minimal_new(common_attrs.merge({use_existing: false}))
 485     assert_not_equal c_queued.uuid, c_locked.uuid
 486     set_user_from_auth :dispatch1
 487     c_locked.update_attributes!({state: Container::Locked})
 488     reused = Container.find_reusable(common_attrs)
 489     assert_not_nil reused
 490     assert_equal reused.uuid, c_locked.uuid
 491   end
 492
 493   test "find_reusable method should not select failed container" do
 494     set_user_from_auth :active
 495     attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed"}})
 496     c, _ = minimal_new(attrs)
 497     set_user_from_auth :dispatch1
 498     c.update_attributes!({state: Container::Locked})
 499     c.update_attributes!({state: Container::Running})
 500     c.update_attributes!({state: Container::Complete,
 501                           exit_code: 33})
 502     reused = Container.find_reusable(attrs)
 503     assert_nil reused
 504   end
 505
 506   test "find_reusable with logging disabled" do
 507     set_user_from_auth :active
 508     Rails.logger.expects(:info).never
 509     Container.find_reusable(REUSABLE_COMMON_ATTRS)
 510   end
 511
 512   test "find_reusable with logging enabled" do
 513     set_user_from_auth :active
 514     Rails.configuration.Containers.LogReuseDecisions = true
 515     Rails.logger.expects(:info).at_least(3)
 516     Container.find_reusable(REUSABLE_COMMON_ATTRS)
 517   end
 518
 519   def runtime_token_attr tok
 520     auth = api_client_authorizations(tok)
 521     {runtime_user_uuid: User.find_by_id(auth.user_id).uuid,
 522      runtime_auth_scopes: auth.scopes,
 523      runtime_token: auth.token}
 524   end
 525
 526   test "find_reusable method with same runtime_token" do
 527     set_user_from_auth :active
 528     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 529     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:container_runtime_token).token}))
 530     assert_equal Container::Queued, c1.state
 531     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 532     assert_not_nil reused
 533     assert_equal reused.uuid, c1.uuid
 534   end
 535
 536   test "find_reusable method with different runtime_token, same user" do
 537     set_user_from_auth :active
 538     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 539     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:crt_user).token}))
 540     assert_equal Container::Queued, c1.state
 541     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 542     assert_not_nil reused
 543     assert_equal reused.uuid, c1.uuid
 544   end
 545
 546   test "find_reusable method with nil runtime_token, then runtime_token with same user" do
 547     set_user_from_auth :crt_user
 548     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 549     c1, _ = minimal_new(common_attrs)
 550     assert_equal Container::Queued, c1.state
 551     assert_equal users(:container_runtime_token_user).uuid, c1.runtime_user_uuid
 552     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 553     assert_not_nil reused
 554     assert_equal reused.uuid, c1.uuid
 555   end
 556
 557   test "find_reusable method with different runtime_token, different user" do
 558     set_user_from_auth :crt_user
 559     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 560     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:active).token}))
 561     assert_equal Container::Queued, c1.state
 562     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 563     # See #14584
 564     assert_equal c1.uuid, reused.uuid
 565   end
 566
 567   test "find_reusable method with nil runtime_token, then runtime_token with different user" do
 568     set_user_from_auth :active
 569     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 570     c1, _ = minimal_new(common_attrs.merge({runtime_token: nil}))
 571     assert_equal Container::Queued, c1.state
 572     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 573     # See #14584
 574     assert_equal c1.uuid, reused.uuid
 575   end
 576
 577   test "find_reusable method with different runtime_token, different scope, same user" do
 578     set_user_from_auth :active
 579     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
 580     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:runtime_token_limited_scope).token}))
 581     assert_equal Container::Queued, c1.state
 582     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
 583     # See #14584
 584     assert_equal c1.uuid, reused.uuid
 585   end
 586
 587   test "Container running" do
 588     set_user_from_auth :active
 589     c, _ = minimal_new priority: 1
 590
 591     set_user_from_auth :dispatch1
 592     check_illegal_updates c, [{state: Container::Running},
 593                               {state: Container::Complete}]
 594
 595     c.lock
 596     c.update_attributes! state: Container::Running
 597
 598     check_illegal_modify c
 599     check_bogus_states c
 600
 601     check_illegal_updates c, [{state: Container::Queued}]
 602     c.reload
 603
 604     c.update_attributes! priority: 3
 605   end
 606
 607   test "Lock and unlock" do
 608     # The "token is expired" check (at the end of this test case)
 609     # requires a west-of-UTC time zone in order to be effective.
 610     ActiveRecord::Base.connection.select_value("SET TIME ZONE '-4'")
 611
 612     set_user_from_auth :active
 613     c, cr = minimal_new priority: 0
 614
 615     set_user_from_auth :dispatch1
 616     assert_equal Container::Queued, c.state
 617
 618     assert_raise(ArvadosModel::LockFailedError) do
 619       # "no priority"
 620       c.lock
 621     end
 622     c.reload
 623     assert cr.update_attributes priority: 1
 624
 625     refute c.update_attributes(state: Container::Running), "not locked"
 626     c.reload
 627     refute c.update_attributes(state: Container::Complete), "not locked"
 628     c.reload
 629
 630     assert c.lock, show_errors(c)
 631     assert c.locked_by_uuid
 632     assert c.auth_uuid
 633
 634     assert_raise(ArvadosModel::LockFailedError) {c.lock}
 635     c.reload
 636
 637     assert c.unlock, show_errors(c)
 638     refute c.locked_by_uuid
 639     refute c.auth_uuid
 640
 641     refute c.update_attributes(state: Container::Running), "not locked"
 642     c.reload
 643     refute c.locked_by_uuid
 644     refute c.auth_uuid
 645
 646     assert c.lock, show_errors(c)
 647     assert c.update_attributes(state: Container::Running), show_errors(c)
 648     assert c.locked_by_uuid
 649     assert c.auth_uuid
 650
 651     auth_uuid_was = c.auth_uuid
 652
 653     assert_raise(ArvadosModel::LockFailedError) do
 654       # Running to Locked is not allowed
 655       c.lock
 656     end
 657     c.reload
 658     assert_raise(ArvadosModel::InvalidStateTransitionError) do
 659       # Running to Queued is not allowed
 660       c.unlock
 661     end
 662     c.reload
 663
 664     assert c.update_attributes(state: Container::Complete), show_errors(c)
 665     refute c.locked_by_uuid
 666     refute c.auth_uuid
 667
 668     auth_exp = ApiClientAuthorization.find_by_uuid(auth_uuid_was).expires_at
 669     assert_operator auth_exp, :<, db_current_time
 670
 671     assert_nil ApiClientAuthorization.validate(token: ApiClientAuthorization.find_by_uuid(auth_uuid_was).token)
 672   end
 673
 674   test "Exceed maximum lock-unlock cycles" do
 675     Rails.configuration.Containers.MaxDispatchAttempts = 3
 676
 677     set_user_from_auth :active
 678     c, cr = minimal_new
 679
 680     set_user_from_auth :dispatch1
 681     assert_equal Container::Queued, c.state
 682     assert_equal 0, c.lock_count
 683
 684     c.lock
 685     c.reload
 686     assert_equal 1, c.lock_count
 687     assert_equal Container::Locked, c.state
 688
 689     c.unlock
 690     c.reload
 691     assert_equal 1, c.lock_count
 692     assert_equal Container::Queued, c.state
 693
 694     c.lock
 695     c.reload
 696     assert_equal 2, c.lock_count
 697     assert_equal Container::Locked, c.state
 698
 699     c.unlock
 700     c.reload
 701     assert_equal 2, c.lock_count
 702     assert_equal Container::Queued, c.state
 703
 704     c.lock
 705     c.reload
 706     assert_equal 3, c.lock_count
 707     assert_equal Container::Locked, c.state
 708
 709     c.unlock
 710     c.reload
 711     assert_equal 3, c.lock_count
 712     assert_equal Container::Cancelled, c.state
 713
 714     assert_raise(ArvadosModel::LockFailedError) do
 715       # Cancelled to Locked is not allowed
 716       c.lock
 717     end
 718   end
 719
 720   test "Container queued cancel" do
 721     set_user_from_auth :active
 722     c, cr = minimal_new({container_count_max: 1})
 723     set_user_from_auth :dispatch1
 724     assert c.update_attributes(state: Container::Cancelled), show_errors(c)
 725     check_no_change_from_cancelled c
 726     cr.reload
 727     assert_equal ContainerRequest::Final, cr.state
 728   end
 729
 730   test "Container queued count" do
 731     assert_equal 1, Container.readable_by(users(:active)).where(state: "Queued").count
 732   end
 733
 734   test "Containers with no matching request are readable by admin" do
 735     uuids = Container.includes('container_requests').where(container_requests: {uuid: nil}).collect(&:uuid)
 736     assert_not_empty uuids
 737     assert_empty Container.readable_by(users(:active)).where(uuid: uuids)
 738     assert_not_empty Container.readable_by(users(:admin)).where(uuid: uuids)
 739     assert_equal uuids.count, Container.readable_by(users(:admin)).where(uuid: uuids).count
 740   end
 741
 742   test "Container locked cancel" do
 743     set_user_from_auth :active
 744     c, _ = minimal_new
 745     set_user_from_auth :dispatch1
 746     assert c.lock, show_errors(c)
 747     assert c.update_attributes(state: Container::Cancelled), show_errors(c)
 748     check_no_change_from_cancelled c
 749   end
 750
 751   test "Container locked cancel with log" do
 752     set_user_from_auth :active
 753     c, _ = minimal_new
 754     set_user_from_auth :dispatch1
 755     assert c.lock, show_errors(c)
 756     assert c.update_attributes(
 757              state: Container::Cancelled,
 758              log: collections(:real_log_collection).portable_data_hash,
 759            ), show_errors(c)
 760     check_no_change_from_cancelled c
 761   end
 762
 763   test "Container running cancel" do
 764     set_user_from_auth :active
 765     c, _ = minimal_new
 766     set_user_from_auth :dispatch1
 767     c.lock
 768     c.update_attributes! state: Container::Running
 769     c.update_attributes! state: Container::Cancelled
 770     check_no_change_from_cancelled c
 771   end
 772
 773   test "Container create forbidden for non-admin" do
 774     set_user_from_auth :active_trustedclient
 775     c = Container.new DEFAULT_ATTRS
 776     c.environment = {}
 777     c.mounts = {"BAR" => "FOO"}
 778     c.output_path = "/tmp"
 779     c.priority = 1
 780     c.runtime_constraints = {}
 781     assert_raises(ArvadosModel::PermissionDeniedError) do
 782       c.save!
 783     end
 784   end
 785
 786   [
 787     [Container::Queued, {state: Container::Locked}],
 788     [Container::Queued, {state: Container::Running}],
 789     [Container::Queued, {state: Container::Complete}],
 790     [Container::Queued, {state: Container::Cancelled}],
 791     [Container::Queued, {priority: 123456789}],
 792     [Container::Queued, {runtime_status: {'error' => 'oops'}}],
 793     [Container::Queued, {cwd: '/'}],
 794     [Container::Locked, {state: Container::Running}],
 795     [Container::Locked, {state: Container::Queued}],
 796     [Container::Locked, {priority: 123456789}],
 797     [Container::Locked, {runtime_status: {'error' => 'oops'}}],
 798     [Container::Locked, {cwd: '/'}],
 799     [Container::Running, {state: Container::Complete}],
 800     [Container::Running, {state: Container::Cancelled}],
 801     [Container::Running, {priority: 123456789}],
 802     [Container::Running, {runtime_status: {'error' => 'oops'}}],
 803     [Container::Running, {cwd: '/'}],
 804     [Container::Complete, {state: Container::Cancelled}],
 805     [Container::Complete, {priority: 123456789}],
 806     [Container::Complete, {runtime_status: {'error' => 'oops'}}],
 807     [Container::Complete, {cwd: '/'}],
 808     [Container::Cancelled, {cwd: '/'}],
 809   ].each do |start_state, updates|
 810     test "Container update #{updates.inspect} when #{start_state} forbidden for non-admin" do
 811       set_user_from_auth :active
 812       c, _ = minimal_new
 813       if start_state != Container::Queued
 814         set_user_from_auth :dispatch1
 815         c.lock
 816         if start_state != Container::Locked
 817           c.update_attributes! state: Container::Running
 818           if start_state != Container::Running
 819             c.update_attributes! state: start_state
 820           end
 821         end
 822       end
 823       assert_equal c.state, start_state
 824       set_user_from_auth :active
 825       assert_raises(ArvadosModel::PermissionDeniedError) do
 826         c.update_attributes! updates
 827       end
 828     end
 829   end
 830
 831   test "Container only set exit code on complete" do
 832     set_user_from_auth :active
 833     c, _ = minimal_new
 834     set_user_from_auth :dispatch1
 835     c.lock
 836     c.update_attributes! state: Container::Running
 837
 838     check_illegal_updates c, [{exit_code: 1},
 839                               {exit_code: 1, state: Container::Cancelled}]
 840
 841     assert c.update_attributes(exit_code: 1, state: Container::Complete)
 842   end
 843
 844   test "locked_by_uuid can update log when locked/running, and output when running" do
 845     set_user_from_auth :active
 846     logcoll = collections(:real_log_collection)
 847     c, cr1 = minimal_new
 848     cr2 = ContainerRequest.new(DEFAULT_ATTRS)
 849     cr2.state = ContainerRequest::Committed
 850     act_as_user users(:active) do
 851       cr2.save!
 852     end
 853     assert_equal cr1.container_uuid, cr2.container_uuid
 854
 855     logpdh_time1 = logcoll.portable_data_hash
 856
 857     set_user_from_auth :dispatch1
 858     c.lock
 859     assert_equal c.locked_by_uuid, Thread.current[:api_client_authorization].uuid
 860     c.update_attributes!(log: logpdh_time1)
 861     c.update_attributes!(state: Container::Running)
 862     cr1.reload
 863     cr2.reload
 864     cr1log_uuid = cr1.log_uuid
 865     cr2log_uuid = cr2.log_uuid
 866     assert_not_nil cr1log_uuid
 867     assert_not_nil cr2log_uuid
 868     assert_not_equal logcoll.uuid, cr1log_uuid
 869     assert_not_equal logcoll.uuid, cr2log_uuid
 870     assert_not_equal cr1log_uuid, cr2log_uuid
 871
 872     logcoll.update_attributes!(manifest_text: logcoll.manifest_text + ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n")
 873     logpdh_time2 = logcoll.portable_data_hash
 874
 875     assert c.update_attributes(output: collections(:collection_owned_by_active).portable_data_hash)
 876     assert c.update_attributes(log: logpdh_time2)
 877     assert c.update_attributes(state: Container::Complete, log: logcoll.portable_data_hash)
 878     c.reload
 879     assert_equal collections(:collection_owned_by_active).portable_data_hash, c.output
 880     assert_equal logpdh_time2, c.log
 881     refute c.update_attributes(output: nil)
 882     refute c.update_attributes(log: nil)
 883     cr1.reload
 884     cr2.reload
 885     assert_equal cr1log_uuid, cr1.log_uuid
 886     assert_equal cr2log_uuid, cr2.log_uuid
 887     assert_equal 1, Collection.where(uuid: [cr1log_uuid, cr2log_uuid]).to_a.collect(&:portable_data_hash).uniq.length
 888     assert_equal ". acbd18db4cc2f85cedef654fccc4a4d8+3 cdd549ae79fe6640fa3d5c6261d8303c+195 0:3:foo.txt 3:195:zzzzz-8i9sb-0vsrcqi7whchuil.log.txt
 889 ./log\\040for\\040container\\040#{cr1.container_uuid} acbd18db4cc2f85cedef654fccc4a4d8+3 cdd549ae79fe6640fa3d5c6261d8303c+195 0:3:foo.txt 3:195:zzzzz-8i9sb-0vsrcqi7whchuil.log.txt
 890 ", Collection.find_by_uuid(cr1log_uuid).manifest_text
 891   end
 892
 893   ["auth_uuid", "runtime_token"].each do |tok|
 894     test "#{tok} can set output, progress, runtime_status, state on running container -- but not log" do
 895       if tok == "runtime_token"
 896         set_user_from_auth :spectator
 897         c, _ = minimal_new(container_image: "9ae44d5792468c58bcf85ce7353c7027+124",
 898                            runtime_token: api_client_authorizations(:active).token)
 899       else
 900         set_user_from_auth :active
 901         c, _ = minimal_new
 902       end
 903       set_user_from_auth :dispatch1
 904       c.lock
 905       c.update_attributes! state: Container::Running
 906
 907       if tok == "runtime_token"
 908         auth = ApiClientAuthorization.validate(token: c.runtime_token)
 909         Thread.current[:api_client_authorization] = auth
 910         Thread.current[:api_client] = auth.api_client
 911         Thread.current[:token] = auth.token
 912         Thread.current[:user] = auth.user
 913       else
 914         auth = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
 915         Thread.current[:api_client_authorization] = auth
 916         Thread.current[:api_client] = auth.api_client
 917         Thread.current[:token] = auth.token
 918         Thread.current[:user] = auth.user
 919       end
 920
 921       assert c.update_attributes(output: collections(:collection_owned_by_active).portable_data_hash)
 922       assert c.update_attributes(runtime_status: {'warning' => 'something happened'})
 923       assert c.update_attributes(progress: 0.5)
 924       refute c.update_attributes(log: collections(:real_log_collection).portable_data_hash)
 925       c.reload
 926       assert c.update_attributes(state: Container::Complete, exit_code: 0)
 927     end
 928   end
 929
 930   test "not allowed to set output that is not readable by current user" do
 931     set_user_from_auth :active
 932     c, _ = minimal_new
 933     set_user_from_auth :dispatch1
 934     c.lock
 935     c.update_attributes! state: Container::Running
 936
 937     Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
 938     Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
 939
 940     assert_raises ActiveRecord::RecordInvalid do
 941       c.update_attributes! output: collections(:collection_not_readable_by_active).portable_data_hash
 942     end
 943   end
 944
 945   test "other token cannot set output on running container" do
 946     set_user_from_auth :active
 947     c, _ = minimal_new
 948     set_user_from_auth :dispatch1
 949     c.lock
 950     c.update_attributes! state: Container::Running
 951
 952     set_user_from_auth :running_to_be_deleted_container_auth
 953     assert_raises(ArvadosModel::PermissionDeniedError) do
 954       c.update_attributes(output: collections(:foo_file).portable_data_hash)
 955     end
 956   end
 957
 958   test "can set trashed output on running container" do
 959     set_user_from_auth :active
 960     c, _ = minimal_new
 961     set_user_from_auth :dispatch1
 962     c.lock
 963     c.update_attributes! state: Container::Running
 964
 965     output = Collection.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jk')
 966
 967     assert output.is_trashed
 968     assert c.update_attributes output: output.portable_data_hash
 969     assert c.update_attributes! state: Container::Complete
 970   end
 971
 972   test "not allowed to set trashed output that is not readable by current user" do
 973     set_user_from_auth :active
 974     c, _ = minimal_new
 975     set_user_from_auth :dispatch1
 976     c.lock
 977     c.update_attributes! state: Container::Running
 978
 979     output = Collection.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jr')
 980
 981     Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
 982     Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
 983
 984     assert_raises ActiveRecord::RecordInvalid do
 985       c.update_attributes! output: output.portable_data_hash
 986     end
 987   end
 988
 989   test "user cannot delete" do
 990     set_user_from_auth :active
 991     c, _ = minimal_new
 992     assert_raises ArvadosModel::PermissionDeniedError do
 993       c.destroy
 994     end
 995     assert Container.find_by_uuid(c.uuid)
 996   end
 997
 998   [
 999     {state: Container::Complete, exit_code: 0, output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'},
1000     {state: Container::Cancelled},
1001   ].each do |final_attrs|
1002     test "secret_mounts and runtime_token are null after container is #{final_attrs[:state]}" do
1003       set_user_from_auth :active
1004       c, cr = minimal_new(secret_mounts: {'/secret' => {'kind' => 'text', 'content' => 'foo'}},
1005                           container_count_max: 1, runtime_token: api_client_authorizations(:active).token)
1006       set_user_from_auth :dispatch1
1007       c.lock
1008       c.update_attributes!(state: Container::Running)
1009       c.reload
1010       assert c.secret_mounts.has_key?('/secret')
1011       assert_equal api_client_authorizations(:active).token, c.runtime_token
1012
1013       c.update_attributes!(final_attrs)
1014       c.reload
1015       assert_equal({}, c.secret_mounts)
1016       assert_nil c.runtime_token
1017       cr.reload
1018       assert_equal({}, cr.secret_mounts)
1019       assert_nil cr.runtime_token
1020       assert_no_secrets_logged
1021     end
1022   end
1023 end