services/api/test/unit/container_test.rb

   1 require 'test_helper'
   2
   3 class ContainerTest < ActiveSupport::TestCase
   4   include DbCurrentTime
   5
   6   DEFAULT_ATTRS = {
   7     command: ['echo', 'foo'],
   8     container_image: 'fa3c1a9cb6783f85f2ecda037e07b8c3+167',
   9     output_path: '/tmp',
  10     priority: 1,
  11     runtime_constraints: {"vcpus" => 1, "ram" => 1},
  12   }
  13
  14   REUSABLE_COMMON_ATTRS = {container_image: "9ae44d5792468c58bcf85ce7353c7027+124",
  15                            cwd: "test",
  16                            command: ["echo", "hello"],
  17                            output_path: "test",
  18                            runtime_constraints: {"vcpus" => 4,
  19                                                  "ram" => 12000000000},
  20                            mounts: {"test" => {"kind" => "json"}},
  21                            environment: {"var" => 'val'}}
  22
  23   def minimal_new attrs={}
  24     cr = ContainerRequest.new DEFAULT_ATTRS.merge(attrs)
  25     cr.state = ContainerRequest::Committed
  26     act_as_user users(:active) do
  27       cr.save!
  28     end
  29     c = Container.find_by_uuid cr.container_uuid
  30     assert_not_nil c
  31     return c, cr
  32   end
  33
  34   def check_illegal_updates c, bad_updates
  35     bad_updates.each do |u|
  36       refute c.update_attributes(u), u.inspect
  37       refute c.valid?, u.inspect
  38       c.reload
  39     end
  40   end
  41
  42   def check_illegal_modify c
  43     check_illegal_updates c, [{command: ["echo", "bar"]},
  44                               {container_image: "arvados/apitestfixture:june10"},
  45                               {cwd: "/tmp2"},
  46                               {environment: {"FOO" => "BAR"}},
  47                               {mounts: {"FOO" => "BAR"}},
  48                               {output_path: "/tmp3"},
  49                               {locked_by_uuid: "zzzzz-gj3su-027z32aux8dg2s1"},
  50                               {auth_uuid: "zzzzz-gj3su-017z32aux8dg2s1"},
  51                               {runtime_constraints: {"FOO" => "BAR"}}]
  52   end
  53
  54   def check_bogus_states c
  55     check_illegal_updates c, [{state: nil},
  56                               {state: "Flubber"}]
  57   end
  58
  59   def check_no_change_from_cancelled c
  60     check_illegal_modify c
  61     check_bogus_states c
  62     check_illegal_updates c, [{ priority: 3 },
  63                               { state: Container::Queued },
  64                               { state: Container::Locked },
  65                               { state: Container::Running },
  66                               { state: Container::Complete }]
  67   end
  68
  69   test "Container create" do
  70     act_as_system_user do
  71       c, _ = minimal_new(environment: {},
  72                       mounts: {"BAR" => "FOO"},
  73                       output_path: "/tmp",
  74                       priority: 1,
  75                       runtime_constraints: {"vcpus" => 1, "ram" => 1})
  76
  77       check_illegal_modify c
  78       check_bogus_states c
  79
  80       c.reload
  81       c.priority = 2
  82       c.save!
  83     end
  84   end
  85
  86   test "Container serialized hash attributes sorted before save" do
  87     env = {"C" => 3, "B" => 2, "A" => 1}
  88     m = {"F" => {"kind" => 3}, "E" => {"kind" => 2}, "D" => {"kind" => 1}}
  89     rc = {"vcpus" => 1, "ram" => 1}
  90     c, _ = minimal_new(environment: env, mounts: m, runtime_constraints: rc)
  91     assert_equal c.environment.to_json, Container.deep_sort_hash(env).to_json
  92     assert_equal c.mounts.to_json, Container.deep_sort_hash(m).to_json
  93     assert_equal c.runtime_constraints.to_json, Container.deep_sort_hash(rc).to_json
  94   end
  95
  96   test 'deep_sort_hash on array of hashes' do
  97     a = {'z' => [[{'a' => 'a', 'b' => 'b'}]]}
  98     b = {'z' => [[{'b' => 'b', 'a' => 'a'}]]}
  99     assert_equal Container.deep_sort_hash(a).to_json, Container.deep_sort_hash(b).to_json
 100   end
 101
 102   test "find_reusable method should select higher priority queued container" do
 103     set_user_from_auth :active
 104     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment:{"var" => "queued"}})
 105     c_low_priority, _ = minimal_new(common_attrs.merge({priority:1}))
 106     c_high_priority, _ = minimal_new(common_attrs.merge({priority:2}))
 107     assert_equal Container::Queued, c_low_priority.state
 108     assert_equal Container::Queued, c_high_priority.state
 109     reused = Container.find_reusable(common_attrs)
 110     assert_not_nil reused
 111     assert_equal reused.uuid, c_high_priority.uuid
 112   end
 113
 114   test "find_reusable method should select latest completed container" do
 115     set_user_from_auth :active
 116     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "complete"}})
 117     completed_attrs = {
 118       state: Container::Complete,
 119       exit_code: 0,
 120       log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 121       output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
 122     }
 123
 124     c_older, _ = minimal_new(common_attrs)
 125     c_recent, _ = minimal_new(common_attrs)
 126
 127     set_user_from_auth :dispatch1
 128     c_older.update_attributes!({state: Container::Locked})
 129     c_older.update_attributes!({state: Container::Running})
 130     c_older.update_attributes!(completed_attrs)
 131
 132     c_recent.update_attributes!({state: Container::Locked})
 133     c_recent.update_attributes!({state: Container::Running})
 134     c_recent.update_attributes!(completed_attrs)
 135
 136     reused = Container.find_reusable(common_attrs)
 137     assert_not_nil reused
 138     assert_equal reused.uuid, c_older.uuid
 139   end
 140
 141   test "find_reusable method should not select completed container when inconsistent outputs exist" do
 142     set_user_from_auth :active
 143     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "complete"}, priority: 1})
 144     completed_attrs = {
 145       state: Container::Complete,
 146       exit_code: 0,
 147       log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 148     }
 149
 150     set_user_from_auth :dispatch1
 151
 152     c_output1 = Container.create common_attrs
 153     c_output2 = Container.create common_attrs
 154
 155     cr = ContainerRequest.new common_attrs
 156     cr.state = ContainerRequest::Committed
 157     cr.container_uuid = c_output1.uuid
 158     cr.save!
 159
 160     cr = ContainerRequest.new common_attrs
 161     cr.state = ContainerRequest::Committed
 162     cr.container_uuid = c_output2.uuid
 163     cr.save!
 164
 165     c_output1.update_attributes!({state: Container::Locked})
 166     c_output1.update_attributes!({state: Container::Running})
 167     c_output1.update_attributes!(completed_attrs.merge({output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'}))
 168
 169     c_output2.update_attributes!({state: Container::Locked})
 170     c_output2.update_attributes!({state: Container::Running})
 171     c_output2.update_attributes!(completed_attrs.merge({output: 'fa7aeb5140e2848d39b416daeef4ffc5+45'}))
 172
 173     reused = Container.find_reusable(common_attrs)
 174     assert_nil reused
 175   end
 176
 177   test "find_reusable method should select running container by start date" do
 178     set_user_from_auth :active
 179     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running"}})
 180     c_slower, _ = minimal_new(common_attrs)
 181     c_faster_started_first, _ = minimal_new(common_attrs)
 182     c_faster_started_second, _ = minimal_new(common_attrs)
 183     set_user_from_auth :dispatch1
 184     c_slower.update_attributes!({state: Container::Locked})
 185     c_slower.update_attributes!({state: Container::Running,
 186                                  progress: 0.1})
 187     c_faster_started_first.update_attributes!({state: Container::Locked})
 188     c_faster_started_first.update_attributes!({state: Container::Running,
 189                                                progress: 0.15})
 190     c_faster_started_second.update_attributes!({state: Container::Locked})
 191     c_faster_started_second.update_attributes!({state: Container::Running,
 192                                                 progress: 0.15})
 193     reused = Container.find_reusable(common_attrs)
 194     assert_not_nil reused
 195     # Selected container is the one that started first
 196     assert_equal reused.uuid, c_faster_started_first.uuid
 197   end
 198
 199   test "find_reusable method should select running container by progress" do
 200     set_user_from_auth :active
 201     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running2"}})
 202     c_slower, _ = minimal_new(common_attrs)
 203     c_faster_started_first, _ = minimal_new(common_attrs)
 204     c_faster_started_second, _ = minimal_new(common_attrs)
 205     set_user_from_auth :dispatch1
 206     c_slower.update_attributes!({state: Container::Locked})
 207     c_slower.update_attributes!({state: Container::Running,
 208                                  progress: 0.1})
 209     c_faster_started_first.update_attributes!({state: Container::Locked})
 210     c_faster_started_first.update_attributes!({state: Container::Running,
 211                                                progress: 0.15})
 212     c_faster_started_second.update_attributes!({state: Container::Locked})
 213     c_faster_started_second.update_attributes!({state: Container::Running,
 214                                                 progress: 0.2})
 215     reused = Container.find_reusable(common_attrs)
 216     assert_not_nil reused
 217     # Selected container is the one with most progress done
 218     assert_equal reused.uuid, c_faster_started_second.uuid
 219   end
 220
 221   test "find_reusable method should select locked container most likely to start sooner" do
 222     set_user_from_auth :active
 223     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "locked"}})
 224     c_low_priority, _ = minimal_new(common_attrs)
 225     c_high_priority_older, _ = minimal_new(common_attrs)
 226     c_high_priority_newer, _ = minimal_new(common_attrs)
 227     set_user_from_auth :dispatch1
 228     c_low_priority.update_attributes!({state: Container::Locked,
 229                                        priority: 1})
 230     c_high_priority_older.update_attributes!({state: Container::Locked,
 231                                               priority: 2})
 232     c_high_priority_newer.update_attributes!({state: Container::Locked,
 233                                               priority: 2})
 234     reused = Container.find_reusable(common_attrs)
 235     assert_not_nil reused
 236     assert_equal reused.uuid, c_high_priority_older.uuid
 237   end
 238
 239   test "find_reusable method should select running over failed container" do
 240     set_user_from_auth :active
 241     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed_vs_running"}})
 242     c_failed, _ = minimal_new(common_attrs)
 243     c_running, _ = minimal_new(common_attrs)
 244     set_user_from_auth :dispatch1
 245     c_failed.update_attributes!({state: Container::Locked})
 246     c_failed.update_attributes!({state: Container::Running})
 247     c_failed.update_attributes!({state: Container::Complete,
 248                                  exit_code: 42,
 249                                  log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 250                                  output: 'ea10d51bcf88862dbcc36eb292017dfd+45'})
 251     c_running.update_attributes!({state: Container::Locked})
 252     c_running.update_attributes!({state: Container::Running,
 253                                   progress: 0.15})
 254     reused = Container.find_reusable(common_attrs)
 255     assert_not_nil reused
 256     assert_equal reused.uuid, c_running.uuid
 257   end
 258
 259   test "find_reusable method should select complete over running container" do
 260     set_user_from_auth :active
 261     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "completed_vs_running"}})
 262     c_completed, _ = minimal_new(common_attrs)
 263     c_running, _ = minimal_new(common_attrs)
 264     set_user_from_auth :dispatch1
 265     c_completed.update_attributes!({state: Container::Locked})
 266     c_completed.update_attributes!({state: Container::Running})
 267     c_completed.update_attributes!({state: Container::Complete,
 268                                     exit_code: 0,
 269                                     log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 270                                     output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'})
 271     c_running.update_attributes!({state: Container::Locked})
 272     c_running.update_attributes!({state: Container::Running,
 273                                   progress: 0.15})
 274     reused = Container.find_reusable(common_attrs)
 275     assert_not_nil reused
 276     assert_equal c_completed.uuid, reused.uuid
 277   end
 278
 279   test "find_reusable method should select running over locked container" do
 280     set_user_from_auth :active
 281     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}})
 282     c_locked, _ = minimal_new(common_attrs)
 283     c_running, _ = minimal_new(common_attrs)
 284     set_user_from_auth :dispatch1
 285     c_locked.update_attributes!({state: Container::Locked})
 286     c_running.update_attributes!({state: Container::Locked})
 287     c_running.update_attributes!({state: Container::Running,
 288                                   progress: 0.15})
 289     reused = Container.find_reusable(common_attrs)
 290     assert_not_nil reused
 291     assert_equal reused.uuid, c_running.uuid
 292   end
 293
 294   test "find_reusable method should select locked over queued container" do
 295     set_user_from_auth :active
 296     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}})
 297     c_locked, _ = minimal_new(common_attrs)
 298     c_queued, _ = minimal_new(common_attrs)
 299     set_user_from_auth :dispatch1
 300     c_locked.update_attributes!({state: Container::Locked})
 301     reused = Container.find_reusable(common_attrs)
 302     assert_not_nil reused
 303     assert_equal reused.uuid, c_locked.uuid
 304   end
 305
 306   test "find_reusable method should not select failed container" do
 307     set_user_from_auth :active
 308     attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed"}})
 309     c, _ = minimal_new(attrs)
 310     set_user_from_auth :dispatch1
 311     c.update_attributes!({state: Container::Locked})
 312     c.update_attributes!({state: Container::Running})
 313     c.update_attributes!({state: Container::Complete,
 314                           exit_code: 33})
 315     reused = Container.find_reusable(attrs)
 316     assert_nil reused
 317   end
 318
 319   test "Container running" do
 320     c, _ = minimal_new priority: 1
 321
 322     set_user_from_auth :dispatch1
 323     check_illegal_updates c, [{state: Container::Running},
 324                               {state: Container::Complete}]
 325
 326     c.lock
 327     c.update_attributes! state: Container::Running
 328
 329     check_illegal_modify c
 330     check_bogus_states c
 331
 332     check_illegal_updates c, [{state: Container::Queued}]
 333     c.reload
 334
 335     c.update_attributes! priority: 3
 336   end
 337
 338   test "Lock and unlock" do
 339     c, cr = minimal_new priority: 0
 340
 341     set_user_from_auth :dispatch1
 342     assert_equal Container::Queued, c.state
 343
 344     assert_raise(ActiveRecord::RecordInvalid) {c.lock} # "no priority"
 345     c.reload
 346     assert cr.update_attributes priority: 1
 347
 348     refute c.update_attributes(state: Container::Running), "not locked"
 349     c.reload
 350     refute c.update_attributes(state: Container::Complete), "not locked"
 351     c.reload
 352
 353     assert c.lock, show_errors(c)
 354     assert c.locked_by_uuid
 355     assert c.auth_uuid
 356
 357     assert_raise(ArvadosModel::AlreadyLockedError) {c.lock}
 358     c.reload
 359
 360     assert c.unlock, show_errors(c)
 361     refute c.locked_by_uuid
 362     refute c.auth_uuid
 363
 364     refute c.update_attributes(state: Container::Running), "not locked"
 365     c.reload
 366     refute c.locked_by_uuid
 367     refute c.auth_uuid
 368
 369     assert c.lock, show_errors(c)
 370     assert c.update_attributes(state: Container::Running), show_errors(c)
 371     assert c.locked_by_uuid
 372     assert c.auth_uuid
 373
 374     auth_uuid_was = c.auth_uuid
 375
 376     assert_raise(ActiveRecord::RecordInvalid) {c.lock} # Running to Locked is not allowed
 377     c.reload
 378     assert_raise(ActiveRecord::RecordInvalid) {c.unlock} # Running to Queued is not allowed
 379     c.reload
 380
 381     assert c.update_attributes(state: Container::Complete), show_errors(c)
 382     refute c.locked_by_uuid
 383     refute c.auth_uuid
 384
 385     auth_exp = ApiClientAuthorization.find_by_uuid(auth_uuid_was).expires_at
 386     assert_operator auth_exp, :<, db_current_time
 387   end
 388
 389   test "Container queued cancel" do
 390     c, _ = minimal_new
 391     set_user_from_auth :dispatch1
 392     assert c.update_attributes(state: Container::Cancelled), show_errors(c)
 393     check_no_change_from_cancelled c
 394   end
 395
 396   test "Container locked cancel" do
 397     c, _ = minimal_new
 398     set_user_from_auth :dispatch1
 399     assert c.lock, show_errors(c)
 400     assert c.update_attributes(state: Container::Cancelled), show_errors(c)
 401     check_no_change_from_cancelled c
 402   end
 403
 404   test "Container running cancel" do
 405     c, _ = minimal_new
 406     set_user_from_auth :dispatch1
 407     c.lock
 408     c.update_attributes! state: Container::Running
 409     c.update_attributes! state: Container::Cancelled
 410     check_no_change_from_cancelled c
 411   end
 412
 413   test "Container create forbidden for non-admin" do
 414     set_user_from_auth :active_trustedclient
 415     c = Container.new DEFAULT_ATTRS
 416     c.environment = {}
 417     c.mounts = {"BAR" => "FOO"}
 418     c.output_path = "/tmp"
 419     c.priority = 1
 420     c.runtime_constraints = {}
 421     assert_raises(ArvadosModel::PermissionDeniedError) do
 422       c.save!
 423     end
 424   end
 425
 426   test "Container only set exit code on complete" do
 427     c, _ = minimal_new
 428     set_user_from_auth :dispatch1
 429     c.lock
 430     c.update_attributes! state: Container::Running
 431
 432     check_illegal_updates c, [{exit_code: 1},
 433                               {exit_code: 1, state: Container::Cancelled}]
 434
 435     assert c.update_attributes(exit_code: 1, state: Container::Complete)
 436   end
 437
 438   test "locked_by_uuid can set output on running container" do
 439     c, _ = minimal_new
 440     set_user_from_auth :dispatch1
 441     c.lock
 442     c.update_attributes! state: Container::Running
 443
 444     assert_equal c.locked_by_uuid, Thread.current[:api_client_authorization].uuid
 445
 446     assert c.update_attributes output: collections(:collection_owned_by_active).portable_data_hash
 447     assert c.update_attributes! state: Container::Complete
 448   end
 449
 450   test "auth_uuid can set output on running container, but not change container state" do
 451     c, _ = minimal_new
 452     set_user_from_auth :dispatch1
 453     c.lock
 454     c.update_attributes! state: Container::Running
 455
 456     Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
 457     Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
 458     assert c.update_attributes output: collections(:collection_owned_by_active).portable_data_hash
 459
 460     assert_raises ArvadosModel::PermissionDeniedError do
 461       # auth_uuid cannot set container state
 462       c.update_attributes state: Container::Complete
 463     end
 464   end
 465
 466   test "not allowed to set output that is not readable by current user" do
 467     c, _ = minimal_new
 468     set_user_from_auth :dispatch1
 469     c.lock
 470     c.update_attributes! state: Container::Running
 471
 472     Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
 473     Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
 474
 475     assert_raises ActiveRecord::RecordInvalid do
 476       c.update_attributes! output: collections(:collection_not_readable_by_active).portable_data_hash
 477     end
 478   end
 479
 480   test "other token cannot set output on running container" do
 481     c, _ = minimal_new
 482     set_user_from_auth :dispatch1
 483     c.lock
 484     c.update_attributes! state: Container::Running
 485
 486     set_user_from_auth :not_running_container_auth
 487     assert_raises ArvadosModel::PermissionDeniedError do
 488       c.update_attributes! output: collections(:foo_file).portable_data_hash
 489     end
 490   end
 491
 492 end