services/api/test/unit/container_test.rb

   1 require 'test_helper'
   2
   3 class ContainerTest < ActiveSupport::TestCase
   4   include DbCurrentTime
   5
   6   DEFAULT_ATTRS = {
   7     command: ['echo', 'foo'],
   8     container_image: 'fa3c1a9cb6783f85f2ecda037e07b8c3+167',
   9     output_path: '/tmp',
  10     priority: 1,
  11     runtime_constraints: {"vcpus" => 1, "ram" => 1},
  12   }
  13
  14   REUSABLE_COMMON_ATTRS = {
  15     container_image: "9ae44d5792468c58bcf85ce7353c7027+124",
  16     cwd: "test",
  17     command: ["echo", "hello"],
  18     output_path: "test",
  19     runtime_constraints: {
  20       "ram" => 12000000000,
  21       "vcpus" => 4,
  22     },
  23     mounts: {
  24       "test" => {"kind" => "json"},
  25     },
  26     environment: {
  27       "var" => "val",
  28     },
  29   }
  30
  31   def minimal_new attrs={}
  32     cr = ContainerRequest.new DEFAULT_ATTRS.merge(attrs)
  33     cr.state = ContainerRequest::Committed
  34     act_as_user users(:active) do
  35       cr.save!
  36     end
  37     c = Container.find_by_uuid cr.container_uuid
  38     assert_not_nil c
  39     return c, cr
  40   end
  41
  42   def check_illegal_updates c, bad_updates
  43     bad_updates.each do |u|
  44       refute c.update_attributes(u), u.inspect
  45       refute c.valid?, u.inspect
  46       c.reload
  47     end
  48   end
  49
  50   def check_illegal_modify c
  51     check_illegal_updates c, [{command: ["echo", "bar"]},
  52                               {container_image: "arvados/apitestfixture:june10"},
  53                               {cwd: "/tmp2"},
  54                               {environment: {"FOO" => "BAR"}},
  55                               {mounts: {"FOO" => "BAR"}},
  56                               {output_path: "/tmp3"},
  57                               {locked_by_uuid: "zzzzz-gj3su-027z32aux8dg2s1"},
  58                               {auth_uuid: "zzzzz-gj3su-017z32aux8dg2s1"},
  59                               {runtime_constraints: {"FOO" => "BAR"}}]
  60   end
  61
  62   def check_bogus_states c
  63     check_illegal_updates c, [{state: nil},
  64                               {state: "Flubber"}]
  65   end
  66
  67   def check_no_change_from_cancelled c
  68     check_illegal_modify c
  69     check_bogus_states c
  70     check_illegal_updates c, [{ priority: 3 },
  71                               { state: Container::Queued },
  72                               { state: Container::Locked },
  73                               { state: Container::Running },
  74                               { state: Container::Complete }]
  75   end
  76
  77   test "Container create" do
  78     act_as_system_user do
  79       c, _ = minimal_new(environment: {},
  80                       mounts: {"BAR" => "FOO"},
  81                       output_path: "/tmp",
  82                       priority: 1,
  83                       runtime_constraints: {"vcpus" => 1, "ram" => 1})
  84
  85       check_illegal_modify c
  86       check_bogus_states c
  87
  88       c.reload
  89       c.priority = 2
  90       c.save!
  91     end
  92   end
  93
  94   test "Container serialized hash attributes sorted before save" do
  95     env = {"C" => 3, "B" => 2, "A" => 1}
  96     m = {"F" => {"kind" => 3}, "E" => {"kind" => 2}, "D" => {"kind" => 1}}
  97     rc = {"vcpus" => 1, "ram" => 1, "keep_cache_ram" => 1}
  98     c, _ = minimal_new(environment: env, mounts: m, runtime_constraints: rc)
  99     assert_equal c.environment.to_json, Container.deep_sort_hash(env).to_json
 100     assert_equal c.mounts.to_json, Container.deep_sort_hash(m).to_json
 101     assert_equal c.runtime_constraints.to_json, Container.deep_sort_hash(rc).to_json
 102   end
 103
 104   test 'deep_sort_hash on array of hashes' do
 105     a = {'z' => [[{'a' => 'a', 'b' => 'b'}]]}
 106     b = {'z' => [[{'b' => 'b', 'a' => 'a'}]]}
 107     assert_equal Container.deep_sort_hash(a).to_json, Container.deep_sort_hash(b).to_json
 108   end
 109
 110   test "find_reusable method should select higher priority queued container" do
 111     set_user_from_auth :active
 112     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment:{"var" => "queued"}})
 113     c_low_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:1}))
 114     c_high_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:2}))
 115     assert_not_equal c_low_priority.uuid, c_high_priority.uuid
 116     assert_equal Container::Queued, c_low_priority.state
 117     assert_equal Container::Queued, c_high_priority.state
 118     reused = Container.find_reusable(common_attrs)
 119     assert_not_nil reused
 120     assert_equal reused.uuid, c_high_priority.uuid
 121   end
 122
 123   test "find_reusable method should select latest completed container" do
 124     set_user_from_auth :active
 125     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "complete"}})
 126     completed_attrs = {
 127       state: Container::Complete,
 128       exit_code: 0,
 129       log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 130       output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
 131     }
 132
 133     c_older, _ = minimal_new(common_attrs.merge({use_existing: false}))
 134     c_recent, _ = minimal_new(common_attrs.merge({use_existing: false}))
 135     assert_not_equal c_older.uuid, c_recent.uuid
 136
 137     set_user_from_auth :dispatch1
 138     c_older.update_attributes!({state: Container::Locked})
 139     c_older.update_attributes!({state: Container::Running})
 140     c_older.update_attributes!(completed_attrs)
 141
 142     c_recent.update_attributes!({state: Container::Locked})
 143     c_recent.update_attributes!({state: Container::Running})
 144     c_recent.update_attributes!(completed_attrs)
 145
 146     reused = Container.find_reusable(common_attrs)
 147     assert_not_nil reused
 148     assert_equal reused.uuid, c_older.uuid
 149   end
 150
 151   test "find_reusable method should select oldest completed container when inconsistent outputs exist" do
 152     set_user_from_auth :active
 153     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "complete"}, priority: 1})
 154     completed_attrs = {
 155       state: Container::Complete,
 156       exit_code: 0,
 157       log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 158     }
 159
 160     cr = ContainerRequest.new common_attrs
 161     cr.use_existing = false
 162     cr.state = ContainerRequest::Committed
 163     cr.save!
 164     c_output1 = Container.where(uuid: cr.container_uuid).first
 165
 166     cr = ContainerRequest.new common_attrs
 167     cr.use_existing = false
 168     cr.state = ContainerRequest::Committed
 169     cr.save!
 170     c_output2 = Container.where(uuid: cr.container_uuid).first
 171
 172     assert_not_equal c_output1.uuid, c_output2.uuid
 173
 174     set_user_from_auth :dispatch1
 175
 176     out1 = '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
 177     log1 = collections(:real_log_collection).portable_data_hash
 178     c_output1.update_attributes!({state: Container::Locked})
 179     c_output1.update_attributes!({state: Container::Running})
 180     c_output1.update_attributes!(completed_attrs.merge({log: log1, output: out1}))
 181
 182     out2 = 'fa7aeb5140e2848d39b416daeef4ffc5+45'
 183     c_output2.update_attributes!({state: Container::Locked})
 184     c_output2.update_attributes!({state: Container::Running})
 185     c_output2.update_attributes!(completed_attrs.merge({log: log1, output: out2}))
 186
 187     reused = Container.resolve(ContainerRequest.new(common_attrs))
 188     assert_equal c_output1.uuid, reused.uuid
 189   end
 190
 191   test "find_reusable method should select running container by start date" do
 192     set_user_from_auth :active
 193     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running"}})
 194     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
 195     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
 196     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
 197     # Confirm the 3 container UUIDs are different.
 198     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
 199     set_user_from_auth :dispatch1
 200     c_slower.update_attributes!({state: Container::Locked})
 201     c_slower.update_attributes!({state: Container::Running,
 202                                  progress: 0.1})
 203     c_faster_started_first.update_attributes!({state: Container::Locked})
 204     c_faster_started_first.update_attributes!({state: Container::Running,
 205                                                progress: 0.15})
 206     c_faster_started_second.update_attributes!({state: Container::Locked})
 207     c_faster_started_second.update_attributes!({state: Container::Running,
 208                                                 progress: 0.15})
 209     reused = Container.find_reusable(common_attrs)
 210     assert_not_nil reused
 211     # Selected container is the one that started first
 212     assert_equal reused.uuid, c_faster_started_first.uuid
 213   end
 214
 215   test "find_reusable method should select running container by progress" do
 216     set_user_from_auth :active
 217     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running2"}})
 218     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
 219     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
 220     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
 221     # Confirm the 3 container UUIDs are different.
 222     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
 223     set_user_from_auth :dispatch1
 224     c_slower.update_attributes!({state: Container::Locked})
 225     c_slower.update_attributes!({state: Container::Running,
 226                                  progress: 0.1})
 227     c_faster_started_first.update_attributes!({state: Container::Locked})
 228     c_faster_started_first.update_attributes!({state: Container::Running,
 229                                                progress: 0.15})
 230     c_faster_started_second.update_attributes!({state: Container::Locked})
 231     c_faster_started_second.update_attributes!({state: Container::Running,
 232                                                 progress: 0.2})
 233     reused = Container.find_reusable(common_attrs)
 234     assert_not_nil reused
 235     # Selected container is the one with most progress done
 236     assert_equal reused.uuid, c_faster_started_second.uuid
 237   end
 238
 239   test "find_reusable method should select locked container most likely to start sooner" do
 240     set_user_from_auth :active
 241     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "locked"}})
 242     c_low_priority, _ = minimal_new(common_attrs.merge({use_existing: false}))
 243     c_high_priority_older, _ = minimal_new(common_attrs.merge({use_existing: false}))
 244     c_high_priority_newer, _ = minimal_new(common_attrs.merge({use_existing: false}))
 245     # Confirm the 3 container UUIDs are different.
 246     assert_equal 3, [c_low_priority.uuid, c_high_priority_older.uuid, c_high_priority_newer.uuid].uniq.length
 247     set_user_from_auth :dispatch1
 248     c_low_priority.update_attributes!({state: Container::Locked,
 249                                        priority: 1})
 250     c_high_priority_older.update_attributes!({state: Container::Locked,
 251                                               priority: 2})
 252     c_high_priority_newer.update_attributes!({state: Container::Locked,
 253                                               priority: 2})
 254     reused = Container.find_reusable(common_attrs)
 255     assert_not_nil reused
 256     assert_equal reused.uuid, c_high_priority_older.uuid
 257   end
 258
 259   test "find_reusable method should select running over failed container" do
 260     set_user_from_auth :active
 261     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed_vs_running"}})
 262     c_failed, _ = minimal_new(common_attrs.merge({use_existing: false}))
 263     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
 264     assert_not_equal c_failed.uuid, c_running.uuid
 265     set_user_from_auth :dispatch1
 266     c_failed.update_attributes!({state: Container::Locked})
 267     c_failed.update_attributes!({state: Container::Running})
 268     c_failed.update_attributes!({state: Container::Complete,
 269                                  exit_code: 42,
 270                                  log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 271                                  output: 'ea10d51bcf88862dbcc36eb292017dfd+45'})
 272     c_running.update_attributes!({state: Container::Locked})
 273     c_running.update_attributes!({state: Container::Running,
 274                                   progress: 0.15})
 275     reused = Container.find_reusable(common_attrs)
 276     assert_not_nil reused
 277     assert_equal reused.uuid, c_running.uuid
 278   end
 279
 280   test "find_reusable method should select complete over running container" do
 281     set_user_from_auth :active
 282     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "completed_vs_running"}})
 283     c_completed, _ = minimal_new(common_attrs.merge({use_existing: false}))
 284     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
 285     assert_not_equal c_completed.uuid, c_running.uuid
 286     set_user_from_auth :dispatch1
 287     c_completed.update_attributes!({state: Container::Locked})
 288     c_completed.update_attributes!({state: Container::Running})
 289     c_completed.update_attributes!({state: Container::Complete,
 290                                     exit_code: 0,
 291                                     log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 292                                     output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'})
 293     c_running.update_attributes!({state: Container::Locked})
 294     c_running.update_attributes!({state: Container::Running,
 295                                   progress: 0.15})
 296     reused = Container.find_reusable(common_attrs)
 297     assert_not_nil reused
 298     assert_equal c_completed.uuid, reused.uuid
 299   end
 300
 301   test "find_reusable method should select running over locked container" do
 302     set_user_from_auth :active
 303     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}})
 304     c_locked, _ = minimal_new(common_attrs.merge({use_existing: false}))
 305     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
 306     assert_not_equal c_running.uuid, c_locked.uuid
 307     set_user_from_auth :dispatch1
 308     c_locked.update_attributes!({state: Container::Locked})
 309     c_running.update_attributes!({state: Container::Locked})
 310     c_running.update_attributes!({state: Container::Running,
 311                                   progress: 0.15})
 312     reused = Container.find_reusable(common_attrs)
 313     assert_not_nil reused
 314     assert_equal reused.uuid, c_running.uuid
 315   end
 316
 317   test "find_reusable method should select locked over queued container" do
 318     set_user_from_auth :active
 319     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}})
 320     c_locked, _ = minimal_new(common_attrs.merge({use_existing: false}))
 321     c_queued, _ = minimal_new(common_attrs.merge({use_existing: false}))
 322     assert_not_equal c_queued.uuid, c_locked.uuid
 323     set_user_from_auth :dispatch1
 324     c_locked.update_attributes!({state: Container::Locked})
 325     reused = Container.find_reusable(common_attrs)
 326     assert_not_nil reused
 327     assert_equal reused.uuid, c_locked.uuid
 328   end
 329
 330   test "find_reusable method should not select failed container" do
 331     set_user_from_auth :active
 332     attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed"}})
 333     c, _ = minimal_new(attrs)
 334     set_user_from_auth :dispatch1
 335     c.update_attributes!({state: Container::Locked})
 336     c.update_attributes!({state: Container::Running})
 337     c.update_attributes!({state: Container::Complete,
 338                           exit_code: 33})
 339     reused = Container.find_reusable(attrs)
 340     assert_nil reused
 341   end
 342
 343   test "find_reusable with logging disabled" do
 344     set_user_from_auth :active
 345     Rails.logger.expects(:info).never
 346     Container.find_reusable(REUSABLE_COMMON_ATTRS)
 347   end
 348
 349   test "find_reusable with logging enabled" do
 350     set_user_from_auth :active
 351     Rails.configuration.log_reuse_decisions = true
 352     Rails.logger.expects(:info).at_least(3)
 353     Container.find_reusable(REUSABLE_COMMON_ATTRS)
 354   end
 355
 356   test "Container running" do
 357     c, _ = minimal_new priority: 1
 358
 359     set_user_from_auth :dispatch1
 360     check_illegal_updates c, [{state: Container::Running},
 361                               {state: Container::Complete}]
 362
 363     c.lock
 364     c.update_attributes! state: Container::Running
 365
 366     check_illegal_modify c
 367     check_bogus_states c
 368
 369     check_illegal_updates c, [{state: Container::Queued}]
 370     c.reload
 371
 372     c.update_attributes! priority: 3
 373   end
 374
 375   test "Lock and unlock" do
 376     c, cr = minimal_new priority: 0
 377
 378     set_user_from_auth :dispatch1
 379     assert_equal Container::Queued, c.state
 380
 381     assert_raise(ArvadosModel::LockFailedError) do
 382       # "no priority"
 383       c.lock
 384     end
 385     c.reload
 386     assert cr.update_attributes priority: 1
 387
 388     refute c.update_attributes(state: Container::Running), "not locked"
 389     c.reload
 390     refute c.update_attributes(state: Container::Complete), "not locked"
 391     c.reload
 392
 393     assert c.lock, show_errors(c)
 394     assert c.locked_by_uuid
 395     assert c.auth_uuid
 396
 397     assert_raise(ArvadosModel::LockFailedError) {c.lock}
 398     c.reload
 399
 400     assert c.unlock, show_errors(c)
 401     refute c.locked_by_uuid
 402     refute c.auth_uuid
 403
 404     refute c.update_attributes(state: Container::Running), "not locked"
 405     c.reload
 406     refute c.locked_by_uuid
 407     refute c.auth_uuid
 408
 409     assert c.lock, show_errors(c)
 410     assert c.update_attributes(state: Container::Running), show_errors(c)
 411     assert c.locked_by_uuid
 412     assert c.auth_uuid
 413
 414     auth_uuid_was = c.auth_uuid
 415
 416     assert_raise(ArvadosModel::LockFailedError) do
 417       # Running to Locked is not allowed
 418       c.lock
 419     end
 420     c.reload
 421     assert_raise(ArvadosModel::InvalidStateTransitionError) do
 422       # Running to Queued is not allowed
 423       c.unlock
 424     end
 425     c.reload
 426
 427     assert c.update_attributes(state: Container::Complete), show_errors(c)
 428     refute c.locked_by_uuid
 429     refute c.auth_uuid
 430
 431     auth_exp = ApiClientAuthorization.find_by_uuid(auth_uuid_was).expires_at
 432     assert_operator auth_exp, :<, db_current_time
 433   end
 434
 435   test "Container queued cancel" do
 436     c, _ = minimal_new
 437     set_user_from_auth :dispatch1
 438     assert c.update_attributes(state: Container::Cancelled), show_errors(c)
 439     check_no_change_from_cancelled c
 440   end
 441
 442   test "Container locked cancel" do
 443     c, _ = minimal_new
 444     set_user_from_auth :dispatch1
 445     assert c.lock, show_errors(c)
 446     assert c.update_attributes(state: Container::Cancelled), show_errors(c)
 447     check_no_change_from_cancelled c
 448   end
 449
 450   test "Container running cancel" do
 451     c, _ = minimal_new
 452     set_user_from_auth :dispatch1
 453     c.lock
 454     c.update_attributes! state: Container::Running
 455     c.update_attributes! state: Container::Cancelled
 456     check_no_change_from_cancelled c
 457   end
 458
 459   test "Container create forbidden for non-admin" do
 460     set_user_from_auth :active_trustedclient
 461     c = Container.new DEFAULT_ATTRS
 462     c.environment = {}
 463     c.mounts = {"BAR" => "FOO"}
 464     c.output_path = "/tmp"
 465     c.priority = 1
 466     c.runtime_constraints = {}
 467     assert_raises(ArvadosModel::PermissionDeniedError) do
 468       c.save!
 469     end
 470   end
 471
 472   test "Container only set exit code on complete" do
 473     c, _ = minimal_new
 474     set_user_from_auth :dispatch1
 475     c.lock
 476     c.update_attributes! state: Container::Running
 477
 478     check_illegal_updates c, [{exit_code: 1},
 479                               {exit_code: 1, state: Container::Cancelled}]
 480
 481     assert c.update_attributes(exit_code: 1, state: Container::Complete)
 482   end
 483
 484   test "locked_by_uuid can set output on running container" do
 485     c, _ = minimal_new
 486     set_user_from_auth :dispatch1
 487     c.lock
 488     c.update_attributes! state: Container::Running
 489
 490     assert_equal c.locked_by_uuid, Thread.current[:api_client_authorization].uuid
 491
 492     assert c.update_attributes output: collections(:collection_owned_by_active).portable_data_hash
 493     assert c.update_attributes! state: Container::Complete
 494   end
 495
 496   test "auth_uuid can set output on running container, but not change container state" do
 497     c, _ = minimal_new
 498     set_user_from_auth :dispatch1
 499     c.lock
 500     c.update_attributes! state: Container::Running
 501
 502     Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
 503     Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
 504     assert c.update_attributes output: collections(:collection_owned_by_active).portable_data_hash
 505
 506     assert_raises ArvadosModel::PermissionDeniedError do
 507       # auth_uuid cannot set container state
 508       c.update_attributes state: Container::Complete
 509     end
 510   end
 511
 512   test "not allowed to set output that is not readable by current user" do
 513     c, _ = minimal_new
 514     set_user_from_auth :dispatch1
 515     c.lock
 516     c.update_attributes! state: Container::Running
 517
 518     Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
 519     Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
 520
 521     assert_raises ActiveRecord::RecordInvalid do
 522       c.update_attributes! output: collections(:collection_not_readable_by_active).portable_data_hash
 523     end
 524   end
 525
 526   test "other token cannot set output on running container" do
 527     c, _ = minimal_new
 528     set_user_from_auth :dispatch1
 529     c.lock
 530     c.update_attributes! state: Container::Running
 531
 532     set_user_from_auth :not_running_container_auth
 533     assert_raises ArvadosModel::PermissionDeniedError do
 534       c.update_attributes! output: collections(:foo_file).portable_data_hash
 535     end
 536   end
 537
 538   test "can set trashed output on running container" do
 539     c, _ = minimal_new
 540     set_user_from_auth :dispatch1
 541     c.lock
 542     c.update_attributes! state: Container::Running
 543
 544     output = Collection.unscoped.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jk')
 545
 546     assert output.is_trashed
 547     assert c.update_attributes output: output.portable_data_hash
 548     assert c.update_attributes! state: Container::Complete
 549   end
 550
 551   test "not allowed to set trashed output that is not readable by current user" do
 552     c, _ = minimal_new
 553     set_user_from_auth :dispatch1
 554     c.lock
 555     c.update_attributes! state: Container::Running
 556
 557     output = Collection.unscoped.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jr')
 558
 559     Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
 560     Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
 561
 562     assert_raises ActiveRecord::RecordInvalid do
 563       c.update_attributes! output: output.portable_data_hash
 564     end
 565   end
 566
 567 end