services/api/test/unit/container_test.rb

   1 require 'test_helper'
   2
   3 class ContainerTest < ActiveSupport::TestCase
   4   include DbCurrentTime
   5
   6   DEFAULT_ATTRS = {
   7     command: ['echo', 'foo'],
   8     container_image: 'img',
   9     output_path: '/tmp',
  10     priority: 1,
  11     runtime_constraints: {"vcpus" => 1, "ram" => 1},
  12   }
  13
  14   REUSABLE_COMMON_ATTRS = {container_image: "test",
  15                            cwd: "test",
  16                            command: ["echo", "hello"],
  17                            output_path: "test",
  18                            runtime_constraints: {"vcpus" => 4,
  19                                                  "ram" => 12000000000},
  20                            mounts: {"test" => {"kind" => "json"}},
  21                            environment: {"var" => 'val'}}
  22
  23   def minimal_new attrs={}
  24     cr = ContainerRequest.new DEFAULT_ATTRS.merge(attrs)
  25     act_as_user users(:active) do
  26       cr.save!
  27     end
  28     c = Container.new DEFAULT_ATTRS.merge(attrs)
  29     act_as_system_user do
  30       c.save!
  31       assert cr.update_attributes(container_uuid: c.uuid,
  32                                   state: ContainerRequest::Committed,
  33                                   ), show_errors(cr)
  34     end
  35     return c, cr
  36   end
  37
  38   def check_illegal_updates c, bad_updates
  39     bad_updates.each do |u|
  40       refute c.update_attributes(u), u.inspect
  41       refute c.valid?, u.inspect
  42       c.reload
  43     end
  44   end
  45
  46   def check_illegal_modify c
  47     check_illegal_updates c, [{command: ["echo", "bar"]},
  48                               {container_image: "img2"},
  49                               {cwd: "/tmp2"},
  50                               {environment: {"FOO" => "BAR"}},
  51                               {mounts: {"FOO" => "BAR"}},
  52                               {output_path: "/tmp3"},
  53                               {locked_by_uuid: "zzzzz-gj3su-027z32aux8dg2s1"},
  54                               {auth_uuid: "zzzzz-gj3su-017z32aux8dg2s1"},
  55                               {runtime_constraints: {"FOO" => "BAR"}}]
  56   end
  57
  58   def check_bogus_states c
  59     check_illegal_updates c, [{state: nil},
  60                               {state: "Flubber"}]
  61   end
  62
  63   def check_no_change_from_cancelled c
  64     check_illegal_modify c
  65     check_bogus_states c
  66     check_illegal_updates c, [{ priority: 3 },
  67                               { state: Container::Queued },
  68                               { state: Container::Locked },
  69                               { state: Container::Running },
  70                               { state: Container::Complete }]
  71   end
  72
  73   test "Container create" do
  74     act_as_system_user do
  75       c, _ = minimal_new(environment: {},
  76                       mounts: {"BAR" => "FOO"},
  77                       output_path: "/tmp",
  78                       priority: 1,
  79                       runtime_constraints: {"vcpus" => 1, "ram" => 1})
  80
  81       check_illegal_modify c
  82       check_bogus_states c
  83
  84       c.reload
  85       c.priority = 2
  86       c.save!
  87     end
  88   end
  89
  90   test "Container serialized hash attributes sorted before save" do
  91     env = {"C" => 3, "B" => 2, "A" => 1}
  92     m = {"F" => 3, "E" => 2, "D" => 1}
  93     rc = {"vcpus" => 1, "ram" => 1}
  94     c, _ = minimal_new(environment: env, mounts: m, runtime_constraints: rc)
  95     assert_equal c.environment.to_json, Container.deep_sort_hash(env).to_json
  96     assert_equal c.mounts.to_json, Container.deep_sort_hash(m).to_json
  97     assert_equal c.runtime_constraints.to_json, Container.deep_sort_hash(rc).to_json
  98   end
  99
 100   test 'deep_sort_hash on array of hashes' do
 101     a = {'z' => [[{'a' => 'a', 'b' => 'b'}]]}
 102     b = {'z' => [[{'b' => 'b', 'a' => 'a'}]]}
 103     assert_equal Container.deep_sort_hash(a).to_json, Container.deep_sort_hash(b).to_json
 104   end
 105
 106   test "find_reusable method should select higher priority queued container" do
 107     set_user_from_auth :active
 108     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment:{"var" => "queued"}})
 109     c_low_priority, _ = minimal_new(common_attrs.merge({priority:1}))
 110     c_high_priority, _ = minimal_new(common_attrs.merge({priority:2}))
 111     assert_equal Container::Queued, c_low_priority.state
 112     assert_equal Container::Queued, c_high_priority.state
 113     reused = Container.find_reusable(common_attrs)
 114     assert_not_nil reused
 115     assert_equal reused.uuid, c_high_priority.uuid
 116   end
 117
 118   test "find_reusable method should select latest completed container" do
 119     set_user_from_auth :active
 120     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "complete"}})
 121     completed_attrs = {
 122       state: Container::Complete,
 123       exit_code: 0,
 124       log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 125       output: 'zzzzz-4zz18-znfnqtbbv4spc3w'
 126     }
 127
 128     c_older, _ = minimal_new(common_attrs)
 129     c_recent, _ = minimal_new(common_attrs)
 130
 131     set_user_from_auth :dispatch1
 132     c_older.update_attributes!({state: Container::Locked})
 133     c_older.update_attributes!({state: Container::Running})
 134     c_older.update_attributes!(completed_attrs)
 135
 136     c_recent.update_attributes!({state: Container::Locked})
 137     c_recent.update_attributes!({state: Container::Running})
 138     c_recent.update_attributes!(completed_attrs)
 139
 140     reused = Container.find_reusable(common_attrs)
 141     assert_not_nil reused
 142     assert_equal reused.uuid, c_recent.uuid
 143   end
 144
 145   test "find_reusable method should not select completed container when inconsistent outputs exist" do
 146     set_user_from_auth :active
 147     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "complete"}})
 148     completed_attrs = {
 149       state: Container::Complete,
 150       exit_code: 0,
 151       log: 'test',
 152     }
 153
 154     c_older, _ = minimal_new(common_attrs)
 155     c_recent, _ = minimal_new(common_attrs)
 156
 157     set_user_from_auth :dispatch1
 158     c_older.update_attributes!({state: Container::Locked})
 159     c_older.update_attributes!({state: Container::Running})
 160     c_older.update_attributes!(completed_attrs.merge({output: 'output 1'}))
 161
 162     c_recent.update_attributes!({state: Container::Locked})
 163     c_recent.update_attributes!({state: Container::Running})
 164     c_recent.update_attributes!(completed_attrs.merge({output: 'output 2'}))
 165
 166     reused = Container.find_reusable(common_attrs)
 167     assert_nil reused
 168   end
 169
 170   test "find_reusable method should select running container by start date" do
 171     set_user_from_auth :active
 172     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running"}})
 173     c_slower, _ = minimal_new(common_attrs)
 174     c_faster_started_first, _ = minimal_new(common_attrs)
 175     c_faster_started_second, _ = minimal_new(common_attrs)
 176     set_user_from_auth :dispatch1
 177     c_slower.update_attributes!({state: Container::Locked})
 178     c_slower.update_attributes!({state: Container::Running,
 179                                  progress: 0.1})
 180     c_faster_started_first.update_attributes!({state: Container::Locked})
 181     c_faster_started_first.update_attributes!({state: Container::Running,
 182                                                progress: 0.15})
 183     c_faster_started_second.update_attributes!({state: Container::Locked})
 184     c_faster_started_second.update_attributes!({state: Container::Running,
 185                                                 progress: 0.15})
 186     reused = Container.find_reusable(common_attrs)
 187     assert_not_nil reused
 188     # Winner is the one that started first
 189     assert_equal reused.uuid, c_faster_started_first.uuid
 190   end
 191
 192   test "find_reusable method should select running container by progress" do
 193     set_user_from_auth :active
 194     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running2"}})
 195     c_slower, _ = minimal_new(common_attrs)
 196     c_faster_started_first, _ = minimal_new(common_attrs)
 197     c_faster_started_second, _ = minimal_new(common_attrs)
 198     set_user_from_auth :dispatch1
 199     c_slower.update_attributes!({state: Container::Locked})
 200     c_slower.update_attributes!({state: Container::Running,
 201                                  progress: 0.1})
 202     c_faster_started_first.update_attributes!({state: Container::Locked})
 203     c_faster_started_first.update_attributes!({state: Container::Running,
 204                                                progress: 0.15})
 205     c_faster_started_second.update_attributes!({state: Container::Locked})
 206     c_faster_started_second.update_attributes!({state: Container::Running,
 207                                                 progress: 0.2})
 208     reused = Container.find_reusable(common_attrs)
 209     assert_not_nil reused
 210     # Winner is the one with most progress done
 211     assert_equal reused.uuid, c_faster_started_second.uuid
 212   end
 213
 214   test "find_reusable method should select locked container most likely to start sooner" do
 215     set_user_from_auth :active
 216     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "locked"}})
 217     c_low_priority, _ = minimal_new(common_attrs)
 218     c_high_priority_older, _ = minimal_new(common_attrs)
 219     c_high_priority_newer, _ = minimal_new(common_attrs)
 220     set_user_from_auth :dispatch1
 221     c_low_priority.update_attributes!({state: Container::Locked,
 222                                        priority: 1})
 223     c_high_priority_older.update_attributes!({state: Container::Locked,
 224                                               priority: 2})
 225     c_high_priority_newer.update_attributes!({state: Container::Locked,
 226                                               priority: 2})
 227     reused = Container.find_reusable(common_attrs)
 228     assert_not_nil reused
 229     assert_equal reused.uuid, c_high_priority_older.uuid
 230   end
 231
 232   test "find_reusable method should select running over failed container" do
 233     set_user_from_auth :active
 234     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed_vs_running"}})
 235     c_failed, _ = minimal_new(common_attrs)
 236     c_running, _ = minimal_new(common_attrs)
 237     set_user_from_auth :dispatch1
 238     c_failed.update_attributes!({state: Container::Locked})
 239     c_failed.update_attributes!({state: Container::Running})
 240     c_failed.update_attributes!({state: Container::Complete,
 241                                  exit_code: 42,
 242                                  log: "test",
 243                                  output: "test"})
 244     c_running.update_attributes!({state: Container::Locked})
 245     c_running.update_attributes!({state: Container::Running,
 246                                   progress: 0.15})
 247     reused = Container.find_reusable(common_attrs)
 248     assert_not_nil reused
 249     assert_equal reused.uuid, c_running.uuid
 250   end
 251
 252   test "find_reusable method should select complete over running container" do
 253     set_user_from_auth :active
 254     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "completed_vs_running"}})
 255     c_completed, _ = minimal_new(common_attrs)
 256     c_running, _ = minimal_new(common_attrs)
 257     set_user_from_auth :dispatch1
 258     c_completed.update_attributes!({state: Container::Locked})
 259     c_completed.update_attributes!({state: Container::Running})
 260     c_completed.update_attributes!({state: Container::Complete,
 261                                     exit_code: 0,
 262                                     log: "ea10d51bcf88862dbcc36eb292017dfd+45",
 263                                     output: "zzzzz-4zz18-znfnqtbbv4spc3w"})
 264     c_running.update_attributes!({state: Container::Locked})
 265     c_running.update_attributes!({state: Container::Running,
 266                                   progress: 0.15})
 267     reused = Container.find_reusable(common_attrs)
 268     assert_not_nil reused
 269     assert_equal reused.uuid, c_completed.uuid
 270   end
 271
 272   test "find_reusable method should select running over locked container" do
 273     set_user_from_auth :active
 274     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}})
 275     c_locked, _ = minimal_new(common_attrs)
 276     c_running, _ = minimal_new(common_attrs)
 277     set_user_from_auth :dispatch1
 278     c_locked.update_attributes!({state: Container::Locked})
 279     c_running.update_attributes!({state: Container::Locked})
 280     c_running.update_attributes!({state: Container::Running,
 281                                   progress: 0.15})
 282     reused = Container.find_reusable(common_attrs)
 283     assert_not_nil reused
 284     assert_equal reused.uuid, c_running.uuid
 285   end
 286
 287   test "find_reusable method should select locked over queued container" do
 288     set_user_from_auth :active
 289     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}})
 290     c_locked, _ = minimal_new(common_attrs)
 291     c_queued, _ = minimal_new(common_attrs)
 292     set_user_from_auth :dispatch1
 293     c_locked.update_attributes!({state: Container::Locked})
 294     reused = Container.find_reusable(common_attrs)
 295     assert_not_nil reused
 296     assert_equal reused.uuid, c_locked.uuid
 297   end
 298
 299   test "find_reusable method should not select failed container" do
 300     set_user_from_auth :active
 301     attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed"}})
 302     c, _ = minimal_new(attrs)
 303     set_user_from_auth :dispatch1
 304     c.update_attributes!({state: Container::Locked})
 305     c.update_attributes!({state: Container::Running})
 306     c.update_attributes!({state: Container::Complete,
 307                           exit_code: 33})
 308     reused = Container.find_reusable(attrs)
 309     assert_nil reused
 310   end
 311
 312   test "Container running" do
 313     c, _ = minimal_new priority: 1
 314
 315     set_user_from_auth :dispatch1
 316     check_illegal_updates c, [{state: Container::Running},
 317                               {state: Container::Complete}]
 318
 319     c.lock
 320     c.update_attributes! state: Container::Running
 321
 322     check_illegal_modify c
 323     check_bogus_states c
 324
 325     check_illegal_updates c, [{state: Container::Queued}]
 326     c.reload
 327
 328     c.update_attributes! priority: 3
 329   end
 330
 331   test "Lock and unlock" do
 332     c, cr = minimal_new priority: 0
 333
 334     set_user_from_auth :dispatch1
 335     assert_equal Container::Queued, c.state
 336
 337     assert_raise(ActiveRecord::RecordInvalid) {c.lock} # "no priority"
 338     c.reload
 339     assert cr.update_attributes priority: 1
 340
 341     refute c.update_attributes(state: Container::Running), "not locked"
 342     c.reload
 343     refute c.update_attributes(state: Container::Complete), "not locked"
 344     c.reload
 345
 346     assert c.lock, show_errors(c)
 347     assert c.locked_by_uuid
 348     assert c.auth_uuid
 349
 350     assert_raise(ArvadosModel::AlreadyLockedError) {c.lock}
 351     c.reload
 352
 353     assert c.unlock, show_errors(c)
 354     refute c.locked_by_uuid
 355     refute c.auth_uuid
 356
 357     refute c.update_attributes(state: Container::Running), "not locked"
 358     c.reload
 359     refute c.locked_by_uuid
 360     refute c.auth_uuid
 361
 362     assert c.lock, show_errors(c)
 363     assert c.update_attributes(state: Container::Running), show_errors(c)
 364     assert c.locked_by_uuid
 365     assert c.auth_uuid
 366
 367     auth_uuid_was = c.auth_uuid
 368
 369     assert_raise(ActiveRecord::RecordInvalid) {c.lock} # Running to Locked is not allowed
 370     c.reload
 371     assert_raise(ActiveRecord::RecordInvalid) {c.unlock} # Running to Queued is not allowed
 372     c.reload
 373
 374     assert c.update_attributes(state: Container::Complete), show_errors(c)
 375     refute c.locked_by_uuid
 376     refute c.auth_uuid
 377
 378     auth_exp = ApiClientAuthorization.find_by_uuid(auth_uuid_was).expires_at
 379     assert_operator auth_exp, :<, db_current_time
 380   end
 381
 382   test "Container queued cancel" do
 383     c, _ = minimal_new
 384     set_user_from_auth :dispatch1
 385     assert c.update_attributes(state: Container::Cancelled), show_errors(c)
 386     check_no_change_from_cancelled c
 387   end
 388
 389   test "Container locked cancel" do
 390     c, _ = minimal_new
 391     set_user_from_auth :dispatch1
 392     assert c.lock, show_errors(c)
 393     assert c.update_attributes(state: Container::Cancelled), show_errors(c)
 394     check_no_change_from_cancelled c
 395   end
 396
 397   test "Container running cancel" do
 398     c, _ = minimal_new
 399     set_user_from_auth :dispatch1
 400     c.lock
 401     c.update_attributes! state: Container::Running
 402     c.update_attributes! state: Container::Cancelled
 403     check_no_change_from_cancelled c
 404   end
 405
 406   test "Container create forbidden for non-admin" do
 407     set_user_from_auth :active_trustedclient
 408     c = Container.new DEFAULT_ATTRS
 409     c.environment = {}
 410     c.mounts = {"BAR" => "FOO"}
 411     c.output_path = "/tmp"
 412     c.priority = 1
 413     c.runtime_constraints = {}
 414     assert_raises(ArvadosModel::PermissionDeniedError) do
 415       c.save!
 416     end
 417   end
 418
 419   test "Container only set exit code on complete" do
 420     c, _ = minimal_new
 421     set_user_from_auth :dispatch1
 422     c.lock
 423     c.update_attributes! state: Container::Running
 424
 425     check_illegal_updates c, [{exit_code: 1},
 426                               {exit_code: 1, state: Container::Cancelled}]
 427
 428     assert c.update_attributes(exit_code: 1, state: Container::Complete)
 429   end
 430 end