services/api/test/unit/container_test.rb

   1 require 'test_helper'
   2
   3 class ContainerTest < ActiveSupport::TestCase
   4   include DbCurrentTime
   5
   6   DEFAULT_ATTRS = {
   7     command: ['echo', 'foo'],
   8     container_image: 'fa3c1a9cb6783f85f2ecda037e07b8c3+167',
   9     output_path: '/tmp',
  10     priority: 1,
  11     runtime_constraints: {"vcpus" => 1, "ram" => 1},
  12   }
  13
  14   REUSABLE_COMMON_ATTRS = {container_image: "9ae44d5792468c58bcf85ce7353c7027+124",
  15                            cwd: "test",
  16                            command: ["echo", "hello"],
  17                            output_path: "test",
  18                            runtime_constraints: {"vcpus" => 4,
  19                                                  "ram" => 12000000000},
  20                            mounts: {"test" => {"kind" => "json"}},
  21                            environment: {"var" => 'val'}}
  22
  23   def minimal_new attrs={}
  24     cr = ContainerRequest.new DEFAULT_ATTRS.merge(attrs)
  25     cr.state = ContainerRequest::Committed
  26     act_as_user users(:active) do
  27       cr.save!
  28     end
  29     c = Container.find_by_uuid cr.container_uuid
  30     assert_not_nil c
  31     return c, cr
  32   end
  33
  34   def check_illegal_updates c, bad_updates
  35     bad_updates.each do |u|
  36       refute c.update_attributes(u), u.inspect
  37       refute c.valid?, u.inspect
  38       c.reload
  39     end
  40   end
  41
  42   def check_illegal_modify c
  43     check_illegal_updates c, [{command: ["echo", "bar"]},
  44                               {container_image: "arvados/apitestfixture:june10"},
  45                               {cwd: "/tmp2"},
  46                               {environment: {"FOO" => "BAR"}},
  47                               {mounts: {"FOO" => "BAR"}},
  48                               {output_path: "/tmp3"},
  49                               {locked_by_uuid: "zzzzz-gj3su-027z32aux8dg2s1"},
  50                               {auth_uuid: "zzzzz-gj3su-017z32aux8dg2s1"},
  51                               {runtime_constraints: {"FOO" => "BAR"}}]
  52   end
  53
  54   def check_bogus_states c
  55     check_illegal_updates c, [{state: nil},
  56                               {state: "Flubber"}]
  57   end
  58
  59   def check_no_change_from_cancelled c
  60     check_illegal_modify c
  61     check_bogus_states c
  62     check_illegal_updates c, [{ priority: 3 },
  63                               { state: Container::Queued },
  64                               { state: Container::Locked },
  65                               { state: Container::Running },
  66                               { state: Container::Complete }]
  67   end
  68
  69   test "Container create" do
  70     act_as_system_user do
  71       c, _ = minimal_new(environment: {},
  72                       mounts: {"BAR" => "FOO"},
  73                       output_path: "/tmp",
  74                       priority: 1,
  75                       runtime_constraints: {"vcpus" => 1, "ram" => 1})
  76
  77       check_illegal_modify c
  78       check_bogus_states c
  79
  80       c.reload
  81       c.priority = 2
  82       c.save!
  83     end
  84   end
  85
  86   test "Container serialized hash attributes sorted before save" do
  87     env = {"C" => 3, "B" => 2, "A" => 1}
  88     m = {"F" => {"kind" => 3}, "E" => {"kind" => 2}, "D" => {"kind" => 1}}
  89     rc = {"vcpus" => 1, "ram" => 1}
  90     c, _ = minimal_new(environment: env, mounts: m, runtime_constraints: rc)
  91     assert_equal c.environment.to_json, Container.deep_sort_hash(env).to_json
  92     assert_equal c.mounts.to_json, Container.deep_sort_hash(m).to_json
  93     assert_equal c.runtime_constraints.to_json, Container.deep_sort_hash(rc).to_json
  94   end
  95
  96   test 'deep_sort_hash on array of hashes' do
  97     a = {'z' => [[{'a' => 'a', 'b' => 'b'}]]}
  98     b = {'z' => [[{'b' => 'b', 'a' => 'a'}]]}
  99     assert_equal Container.deep_sort_hash(a).to_json, Container.deep_sort_hash(b).to_json
 100   end
 101
 102   test "find_reusable method should select higher priority queued container" do
 103     set_user_from_auth :active
 104     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment:{"var" => "queued"}})
 105     c_low_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:1}))
 106     c_high_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:2}))
 107     assert_not_equal c_low_priority.uuid, c_high_priority.uuid
 108     assert_equal Container::Queued, c_low_priority.state
 109     assert_equal Container::Queued, c_high_priority.state
 110     reused = Container.find_reusable(common_attrs)
 111     assert_not_nil reused
 112     assert_equal reused.uuid, c_high_priority.uuid
 113   end
 114
 115   test "find_reusable method should select latest completed container" do
 116     set_user_from_auth :active
 117     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "complete"}})
 118     completed_attrs = {
 119       state: Container::Complete,
 120       exit_code: 0,
 121       log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 122       output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
 123     }
 124
 125     c_older, _ = minimal_new(common_attrs.merge({use_existing: false}))
 126     c_recent, _ = minimal_new(common_attrs.merge({use_existing: false}))
 127     assert_not_equal c_older.uuid, c_recent.uuid
 128
 129     set_user_from_auth :dispatch1
 130     c_older.update_attributes!({state: Container::Locked})
 131     c_older.update_attributes!({state: Container::Running})
 132     c_older.update_attributes!(completed_attrs)
 133
 134     c_recent.update_attributes!({state: Container::Locked})
 135     c_recent.update_attributes!({state: Container::Running})
 136     c_recent.update_attributes!(completed_attrs)
 137
 138     reused = Container.find_reusable(common_attrs)
 139     assert_not_nil reused
 140     assert_equal reused.uuid, c_older.uuid
 141   end
 142
 143   test "find_reusable method should select oldest completed container when inconsistent outputs exist" do
 144     set_user_from_auth :active
 145     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "complete"}, priority: 1})
 146     completed_attrs = {
 147       state: Container::Complete,
 148       exit_code: 0,
 149       log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 150     }
 151
 152     set_user_from_auth :dispatch1
 153
 154     c_output1 = Container.create common_attrs
 155     c_output2 = Container.create common_attrs
 156     assert_not_equal c_output1.uuid, c_output2.uuid
 157
 158     cr = ContainerRequest.new common_attrs
 159     cr.state = ContainerRequest::Committed
 160     cr.container_uuid = c_output1.uuid
 161     cr.save!
 162
 163     cr = ContainerRequest.new common_attrs
 164     cr.state = ContainerRequest::Committed
 165     cr.container_uuid = c_output2.uuid
 166     cr.save!
 167
 168     out1 = '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
 169     log1 = collections(:real_log_collection).portable_data_hash
 170     c_output1.update_attributes!({state: Container::Locked})
 171     c_output1.update_attributes!({state: Container::Running})
 172     c_output1.update_attributes!(completed_attrs.merge({log: log1, output: out1}))
 173
 174     out2 = 'fa7aeb5140e2848d39b416daeef4ffc5+45'
 175     c_output2.update_attributes!({state: Container::Locked})
 176     c_output2.update_attributes!({state: Container::Running})
 177     c_output2.update_attributes!(completed_attrs.merge({log: log1, output: out2}))
 178
 179     reused = Container.find_reusable(common_attrs)
 180     assert_not_nil reused
 181     assert_equal reused.uuid, c_output1.uuid
 182   end
 183
 184   test "find_reusable method should select running container by start date" do
 185     set_user_from_auth :active
 186     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running"}})
 187     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
 188     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
 189     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
 190     # Confirm the 3 container UUIDs are different.
 191     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
 192     set_user_from_auth :dispatch1
 193     c_slower.update_attributes!({state: Container::Locked})
 194     c_slower.update_attributes!({state: Container::Running,
 195                                  progress: 0.1})
 196     c_faster_started_first.update_attributes!({state: Container::Locked})
 197     c_faster_started_first.update_attributes!({state: Container::Running,
 198                                                progress: 0.15})
 199     c_faster_started_second.update_attributes!({state: Container::Locked})
 200     c_faster_started_second.update_attributes!({state: Container::Running,
 201                                                 progress: 0.15})
 202     reused = Container.find_reusable(common_attrs)
 203     assert_not_nil reused
 204     # Selected container is the one that started first
 205     assert_equal reused.uuid, c_faster_started_first.uuid
 206   end
 207
 208   test "find_reusable method should select running container by progress" do
 209     set_user_from_auth :active
 210     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running2"}})
 211     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
 212     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
 213     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
 214     # Confirm the 3 container UUIDs are different.
 215     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
 216     set_user_from_auth :dispatch1
 217     c_slower.update_attributes!({state: Container::Locked})
 218     c_slower.update_attributes!({state: Container::Running,
 219                                  progress: 0.1})
 220     c_faster_started_first.update_attributes!({state: Container::Locked})
 221     c_faster_started_first.update_attributes!({state: Container::Running,
 222                                                progress: 0.15})
 223     c_faster_started_second.update_attributes!({state: Container::Locked})
 224     c_faster_started_second.update_attributes!({state: Container::Running,
 225                                                 progress: 0.2})
 226     reused = Container.find_reusable(common_attrs)
 227     assert_not_nil reused
 228     # Selected container is the one with most progress done
 229     assert_equal reused.uuid, c_faster_started_second.uuid
 230   end
 231
 232   test "find_reusable method should select locked container most likely to start sooner" do
 233     set_user_from_auth :active
 234     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "locked"}})
 235     c_low_priority, _ = minimal_new(common_attrs.merge({use_existing: false}))
 236     c_high_priority_older, _ = minimal_new(common_attrs.merge({use_existing: false}))
 237     c_high_priority_newer, _ = minimal_new(common_attrs.merge({use_existing: false}))
 238     # Confirm the 3 container UUIDs are different.
 239     assert_equal 3, [c_low_priority.uuid, c_high_priority_older.uuid, c_high_priority_newer.uuid].uniq.length
 240     set_user_from_auth :dispatch1
 241     c_low_priority.update_attributes!({state: Container::Locked,
 242                                        priority: 1})
 243     c_high_priority_older.update_attributes!({state: Container::Locked,
 244                                               priority: 2})
 245     c_high_priority_newer.update_attributes!({state: Container::Locked,
 246                                               priority: 2})
 247     reused = Container.find_reusable(common_attrs)
 248     assert_not_nil reused
 249     assert_equal reused.uuid, c_high_priority_older.uuid
 250   end
 251
 252   test "find_reusable method should select running over failed container" do
 253     set_user_from_auth :active
 254     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed_vs_running"}})
 255     c_failed, _ = minimal_new(common_attrs.merge({use_existing: false}))
 256     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
 257     assert_not_equal c_failed.uuid, c_running.uuid
 258     set_user_from_auth :dispatch1
 259     c_failed.update_attributes!({state: Container::Locked})
 260     c_failed.update_attributes!({state: Container::Running})
 261     c_failed.update_attributes!({state: Container::Complete,
 262                                  exit_code: 42,
 263                                  log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 264                                  output: 'ea10d51bcf88862dbcc36eb292017dfd+45'})
 265     c_running.update_attributes!({state: Container::Locked})
 266     c_running.update_attributes!({state: Container::Running,
 267                                   progress: 0.15})
 268     reused = Container.find_reusable(common_attrs)
 269     assert_not_nil reused
 270     assert_equal reused.uuid, c_running.uuid
 271   end
 272
 273   test "find_reusable method should select complete over running container" do
 274     set_user_from_auth :active
 275     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "completed_vs_running"}})
 276     c_completed, _ = minimal_new(common_attrs.merge({use_existing: false}))
 277     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
 278     assert_not_equal c_completed.uuid, c_running.uuid
 279     set_user_from_auth :dispatch1
 280     c_completed.update_attributes!({state: Container::Locked})
 281     c_completed.update_attributes!({state: Container::Running})
 282     c_completed.update_attributes!({state: Container::Complete,
 283                                     exit_code: 0,
 284                                     log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 285                                     output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'})
 286     c_running.update_attributes!({state: Container::Locked})
 287     c_running.update_attributes!({state: Container::Running,
 288                                   progress: 0.15})
 289     reused = Container.find_reusable(common_attrs)
 290     assert_not_nil reused
 291     assert_equal c_completed.uuid, reused.uuid
 292   end
 293
 294   test "find_reusable method should select running over locked container" do
 295     set_user_from_auth :active
 296     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}})
 297     c_locked, _ = minimal_new(common_attrs.merge({use_existing: false}))
 298     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
 299     assert_not_equal c_running.uuid, c_locked.uuid
 300     set_user_from_auth :dispatch1
 301     c_locked.update_attributes!({state: Container::Locked})
 302     c_running.update_attributes!({state: Container::Locked})
 303     c_running.update_attributes!({state: Container::Running,
 304                                   progress: 0.15})
 305     reused = Container.find_reusable(common_attrs)
 306     assert_not_nil reused
 307     assert_equal reused.uuid, c_running.uuid
 308   end
 309
 310   test "find_reusable method should select locked over queued container" do
 311     set_user_from_auth :active
 312     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}})
 313     c_locked, _ = minimal_new(common_attrs.merge({use_existing: false}))
 314     c_queued, _ = minimal_new(common_attrs.merge({use_existing: false}))
 315     assert_not_equal c_queued.uuid, c_locked.uuid
 316     set_user_from_auth :dispatch1
 317     c_locked.update_attributes!({state: Container::Locked})
 318     reused = Container.find_reusable(common_attrs)
 319     assert_not_nil reused
 320     assert_equal reused.uuid, c_locked.uuid
 321   end
 322
 323   test "find_reusable method should not select failed container" do
 324     set_user_from_auth :active
 325     attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed"}})
 326     c, _ = minimal_new(attrs)
 327     set_user_from_auth :dispatch1
 328     c.update_attributes!({state: Container::Locked})
 329     c.update_attributes!({state: Container::Running})
 330     c.update_attributes!({state: Container::Complete,
 331                           exit_code: 33})
 332     reused = Container.find_reusable(attrs)
 333     assert_nil reused
 334   end
 335
 336   test "Container running" do
 337     c, _ = minimal_new priority: 1
 338
 339     set_user_from_auth :dispatch1
 340     check_illegal_updates c, [{state: Container::Running},
 341                               {state: Container::Complete}]
 342
 343     c.lock
 344     c.update_attributes! state: Container::Running
 345
 346     check_illegal_modify c
 347     check_bogus_states c
 348
 349     check_illegal_updates c, [{state: Container::Queued}]
 350     c.reload
 351
 352     c.update_attributes! priority: 3
 353   end
 354
 355   test "Lock and unlock" do
 356     c, cr = minimal_new priority: 0
 357
 358     set_user_from_auth :dispatch1
 359     assert_equal Container::Queued, c.state
 360
 361     assert_raise(ActiveRecord::RecordInvalid) {c.lock} # "no priority"
 362     c.reload
 363     assert cr.update_attributes priority: 1
 364
 365     refute c.update_attributes(state: Container::Running), "not locked"
 366     c.reload
 367     refute c.update_attributes(state: Container::Complete), "not locked"
 368     c.reload
 369
 370     assert c.lock, show_errors(c)
 371     assert c.locked_by_uuid
 372     assert c.auth_uuid
 373
 374     assert_raise(ArvadosModel::AlreadyLockedError) {c.lock}
 375     c.reload
 376
 377     assert c.unlock, show_errors(c)
 378     refute c.locked_by_uuid
 379     refute c.auth_uuid
 380
 381     refute c.update_attributes(state: Container::Running), "not locked"
 382     c.reload
 383     refute c.locked_by_uuid
 384     refute c.auth_uuid
 385
 386     assert c.lock, show_errors(c)
 387     assert c.update_attributes(state: Container::Running), show_errors(c)
 388     assert c.locked_by_uuid
 389     assert c.auth_uuid
 390
 391     auth_uuid_was = c.auth_uuid
 392
 393     assert_raise(ActiveRecord::RecordInvalid) {c.lock} # Running to Locked is not allowed
 394     c.reload
 395     assert_raise(ActiveRecord::RecordInvalid) {c.unlock} # Running to Queued is not allowed
 396     c.reload
 397
 398     assert c.update_attributes(state: Container::Complete), show_errors(c)
 399     refute c.locked_by_uuid
 400     refute c.auth_uuid
 401
 402     auth_exp = ApiClientAuthorization.find_by_uuid(auth_uuid_was).expires_at
 403     assert_operator auth_exp, :<, db_current_time
 404   end
 405
 406   test "Container queued cancel" do
 407     c, _ = minimal_new
 408     set_user_from_auth :dispatch1
 409     assert c.update_attributes(state: Container::Cancelled), show_errors(c)
 410     check_no_change_from_cancelled c
 411   end
 412
 413   test "Container locked cancel" do
 414     c, _ = minimal_new
 415     set_user_from_auth :dispatch1
 416     assert c.lock, show_errors(c)
 417     assert c.update_attributes(state: Container::Cancelled), show_errors(c)
 418     check_no_change_from_cancelled c
 419   end
 420
 421   test "Container running cancel" do
 422     c, _ = minimal_new
 423     set_user_from_auth :dispatch1
 424     c.lock
 425     c.update_attributes! state: Container::Running
 426     c.update_attributes! state: Container::Cancelled
 427     check_no_change_from_cancelled c
 428   end
 429
 430   test "Container create forbidden for non-admin" do
 431     set_user_from_auth :active_trustedclient
 432     c = Container.new DEFAULT_ATTRS
 433     c.environment = {}
 434     c.mounts = {"BAR" => "FOO"}
 435     c.output_path = "/tmp"
 436     c.priority = 1
 437     c.runtime_constraints = {}
 438     assert_raises(ArvadosModel::PermissionDeniedError) do
 439       c.save!
 440     end
 441   end
 442
 443   test "Container only set exit code on complete" do
 444     c, _ = minimal_new
 445     set_user_from_auth :dispatch1
 446     c.lock
 447     c.update_attributes! state: Container::Running
 448
 449     check_illegal_updates c, [{exit_code: 1},
 450                               {exit_code: 1, state: Container::Cancelled}]
 451
 452     assert c.update_attributes(exit_code: 1, state: Container::Complete)
 453   end
 454
 455   test "locked_by_uuid can set output on running container" do
 456     c, _ = minimal_new
 457     set_user_from_auth :dispatch1
 458     c.lock
 459     c.update_attributes! state: Container::Running
 460
 461     assert_equal c.locked_by_uuid, Thread.current[:api_client_authorization].uuid
 462
 463     assert c.update_attributes output: collections(:collection_owned_by_active).portable_data_hash
 464     assert c.update_attributes! state: Container::Complete
 465   end
 466
 467   test "auth_uuid can set output on running container, but not change container state" do
 468     c, _ = minimal_new
 469     set_user_from_auth :dispatch1
 470     c.lock
 471     c.update_attributes! state: Container::Running
 472
 473     Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
 474     Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
 475     assert c.update_attributes output: collections(:collection_owned_by_active).portable_data_hash
 476
 477     assert_raises ArvadosModel::PermissionDeniedError do
 478       # auth_uuid cannot set container state
 479       c.update_attributes state: Container::Complete
 480     end
 481   end
 482
 483   test "not allowed to set output that is not readable by current user" do
 484     c, _ = minimal_new
 485     set_user_from_auth :dispatch1
 486     c.lock
 487     c.update_attributes! state: Container::Running
 488
 489     Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
 490     Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
 491
 492     assert_raises ActiveRecord::RecordInvalid do
 493       c.update_attributes! output: collections(:collection_not_readable_by_active).portable_data_hash
 494     end
 495   end
 496
 497   test "other token cannot set output on running container" do
 498     c, _ = minimal_new
 499     set_user_from_auth :dispatch1
 500     c.lock
 501     c.update_attributes! state: Container::Running
 502
 503     set_user_from_auth :not_running_container_auth
 504     assert_raises ArvadosModel::PermissionDeniedError do
 505       c.update_attributes! output: collections(:foo_file).portable_data_hash
 506     end
 507   end
 508
 509   test "can set trashed output on running container" do
 510     c, _ = minimal_new
 511     set_user_from_auth :dispatch1
 512     c.lock
 513     c.update_attributes! state: Container::Running
 514
 515     output = Collection.unscoped.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jk')
 516
 517     assert output.is_trashed
 518     assert c.update_attributes output: output.portable_data_hash
 519     assert c.update_attributes! state: Container::Complete
 520   end
 521
 522 end