services/api/test/unit/container_request_test.rb

   1 require 'test_helper'
   2
   3 class ContainerRequestTest < ActiveSupport::TestCase
   4   def create_minimal_req! attrs={}
   5     defaults = {
   6       command: ["echo", "foo"],
   7       container_image: links(:docker_image_collection_tag).name,
   8       cwd: "/tmp",
   9       environment: {},
  10       mounts: {"/out" => {"kind" => "tmp", "capacity" => 1000000}},
  11       output_path: "/out",
  12       runtime_constraints: {"vcpus" => 1, "ram" => 2},
  13       name: "foo",
  14       description: "bar",
  15     }
  16     cr = ContainerRequest.create!(defaults.merge(attrs))
  17     cr.reload
  18     return cr
  19   end
  20
  21   def check_bogus_states cr
  22     [nil, "Flubber"].each do |state|
  23       assert_raises(ActiveRecord::RecordInvalid) do
  24         cr.state = state
  25         cr.save!
  26       end
  27       cr.reload
  28     end
  29   end
  30
  31   test "Container request create" do
  32     set_user_from_auth :active
  33     cr = create_minimal_req!
  34
  35     assert_nil cr.container_uuid
  36     assert_nil cr.priority
  37
  38     check_bogus_states cr
  39
  40     # Ensure we can modify all attributes
  41     cr.command = ["echo", "foo3"]
  42     cr.container_image = "img3"
  43     cr.cwd = "/tmp3"
  44     cr.environment = {"BUP" => "BOP"}
  45     cr.mounts = {"BAR" => "BAZ"}
  46     cr.output_path = "/tmp4"
  47     cr.priority = 2
  48     cr.runtime_constraints = {"vcpus" => 4}
  49     cr.name = "foo3"
  50     cr.description = "bar3"
  51     cr.save!
  52
  53     assert_nil cr.container_uuid
  54   end
  55
  56   [
  57     {"vcpus" => 1},
  58     {"vcpus" => 1, "ram" => nil},
  59     {"vcpus" => 0, "ram" => 123},
  60     {"vcpus" => "1", "ram" => "123"}
  61   ].each do |invalid_constraints|
  62     test "Create with #{invalid_constraints}" do
  63       set_user_from_auth :active
  64       assert_raises(ActiveRecord::RecordInvalid) do
  65         cr = create_minimal_req!(state: "Committed",
  66                                  priority: 1,
  67                                  runtime_constraints: invalid_constraints)
  68         cr.save!
  69       end
  70     end
  71
  72     test "Update with #{invalid_constraints}" do
  73       set_user_from_auth :active
  74       cr = create_minimal_req!(state: "Uncommitted", priority: 1)
  75       cr.save!
  76       assert_raises(ActiveRecord::RecordInvalid) do
  77         cr = ContainerRequest.find_by_uuid cr.uuid
  78         cr.update_attributes!(state: "Committed",
  79                               runtime_constraints: invalid_constraints)
  80       end
  81     end
  82   end
  83
  84   test "Update from fixture" do
  85     set_user_from_auth :active
  86     cr = ContainerRequest.find_by_uuid(container_requests(:running).uuid)
  87     cr.update_attributes!(description: "New description")
  88     assert_equal "New description", cr.description
  89   end
  90
  91   test "Update with valid runtime constraints" do
  92       set_user_from_auth :active
  93       cr = create_minimal_req!(state: "Uncommitted", priority: 1)
  94       cr.save!
  95       cr = ContainerRequest.find_by_uuid cr.uuid
  96       cr.update_attributes!(state: "Committed",
  97                             runtime_constraints: {"vcpus" => 1, "ram" => 23})
  98       assert_not_nil cr.container_uuid
  99   end
 100
 101   test "Container request priority must be non-nil" do
 102     set_user_from_auth :active
 103     cr = create_minimal_req!(priority: nil)
 104     cr.state = "Committed"
 105     assert_raises(ActiveRecord::RecordInvalid) do
 106       cr.save!
 107     end
 108   end
 109
 110   test "Container request commit" do
 111     set_user_from_auth :active
 112     cr = create_minimal_req!(runtime_constraints: {"vcpus" => 2, "ram" => 30})
 113
 114     assert_nil cr.container_uuid
 115
 116     cr.reload
 117     cr.state = "Committed"
 118     cr.priority = 1
 119     cr.save!
 120
 121     cr.reload
 122
 123     assert_not_nil cr.container_uuid
 124     c = Container.find_by_uuid cr.container_uuid
 125     assert_not_nil c
 126     assert_equal ["echo", "foo"], c.command
 127     assert_equal collections(:docker_image).portable_data_hash, c.container_image
 128     assert_equal "/tmp", c.cwd
 129     assert_equal({}, c.environment)
 130     assert_equal({"/out" => {"kind"=>"tmp", "capacity"=>1000000}}, c.mounts)
 131     assert_equal "/out", c.output_path
 132     assert_equal({"vcpus" => 2, "ram" => 30}, c.runtime_constraints)
 133     assert_equal 1, c.priority
 134
 135     assert_raises(ActiveRecord::RecordInvalid) do
 136       cr.priority = nil
 137       cr.save!
 138     end
 139
 140     cr.priority = 0
 141     cr.save!
 142
 143     cr.reload
 144     c.reload
 145     assert_equal 0, cr.priority
 146     assert_equal 0, c.priority
 147   end
 148
 149
 150   test "Container request max priority" do
 151     set_user_from_auth :active
 152     cr = create_minimal_req!(priority: 5, state: "Committed")
 153
 154     c = Container.find_by_uuid cr.container_uuid
 155     assert_equal 5, c.priority
 156
 157     cr2 = create_minimal_req!
 158     cr2.priority = 10
 159     cr2.state = "Committed"
 160     cr2.container_uuid = cr.container_uuid
 161     act_as_system_user do
 162       cr2.save!
 163     end
 164
 165     # cr and cr2 have priority 5 and 10, and are being satisfied by
 166     # the same container c, so c's priority should be
 167     # max(priority)=10.
 168     c.reload
 169     assert_equal 10, c.priority
 170
 171     cr2.update_attributes!(priority: 0)
 172
 173     c.reload
 174     assert_equal 5, c.priority
 175
 176     cr.update_attributes!(priority: 0)
 177
 178     c.reload
 179     assert_equal 0, c.priority
 180   end
 181
 182
 183   test "Independent container requests" do
 184     set_user_from_auth :active
 185     cr1 = create_minimal_req!(command: ["foo", "1"], priority: 5, state: "Committed")
 186     cr2 = create_minimal_req!(command: ["foo", "2"], priority: 10, state: "Committed")
 187
 188     c1 = Container.find_by_uuid cr1.container_uuid
 189     assert_equal 5, c1.priority
 190
 191     c2 = Container.find_by_uuid cr2.container_uuid
 192     assert_equal 10, c2.priority
 193
 194     cr1.update_attributes!(priority: 0)
 195
 196     c1.reload
 197     assert_equal 0, c1.priority
 198
 199     c2.reload
 200     assert_equal 10, c2.priority
 201   end
 202
 203   test "Request is finalized when its container is cancelled" do
 204     set_user_from_auth :active
 205     cr = create_minimal_req!(priority: 1, state: "Committed", container_count_max: 1)
 206
 207     act_as_system_user do
 208       Container.find_by_uuid(cr.container_uuid).
 209         update_attributes!(state: Container::Cancelled)
 210     end
 211
 212     cr.reload
 213     assert_equal "Final", cr.state
 214   end
 215
 216   test "Request is finalized when its container is completed" do
 217     set_user_from_auth :active
 218     project = groups(:private)
 219     cr = create_minimal_req!(owner_uuid: project.uuid,
 220                              priority: 1,
 221                              state: "Committed")
 222
 223     c = act_as_system_user do
 224       c = Container.find_by_uuid(cr.container_uuid)
 225       c.update_attributes!(state: Container::Locked)
 226       c.update_attributes!(state: Container::Running)
 227       c
 228     end
 229
 230     cr.reload
 231     assert_equal "Committed", cr.state
 232
 233     act_as_system_user do
 234       c.update_attributes!(state: Container::Complete,
 235                            output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45',
 236                            log: 'fa7aeb5140e2848d39b416daeef4ffc5+45')
 237     end
 238
 239     cr.reload
 240     assert_equal "Final", cr.state
 241     ['output', 'log'].each do |out_type|
 242       pdh = Container.find_by_uuid(cr.container_uuid).send(out_type)
 243       assert_equal(1, Collection.where(portable_data_hash: pdh,
 244                                        owner_uuid: project.uuid).count,
 245                    "Container #{out_type} should be copied to #{project.uuid}")
 246     end
 247   end
 248
 249   test "Container makes container request, then is cancelled" do
 250     set_user_from_auth :active
 251     cr = create_minimal_req!(priority: 5, state: "Committed", container_count_max: 1)
 252
 253     c = Container.find_by_uuid cr.container_uuid
 254     assert_equal 5, c.priority
 255
 256     cr2 = create_minimal_req!
 257     cr2.update_attributes!(priority: 10, state: "Committed", requesting_container_uuid: c.uuid, command: ["echo", "foo2"], container_count_max: 1)
 258     cr2.reload
 259
 260     c2 = Container.find_by_uuid cr2.container_uuid
 261     assert_equal 10, c2.priority
 262
 263     act_as_system_user do
 264       c.state = "Cancelled"
 265       c.save!
 266     end
 267
 268     cr.reload
 269     assert_equal "Final", cr.state
 270
 271     cr2.reload
 272     assert_equal 0, cr2.priority
 273
 274     c2.reload
 275     assert_equal 0, c2.priority
 276   end
 277
 278   [
 279     ['running_container_auth', 'zzzzz-dz642-runningcontainr'],
 280     ['active_no_prefs', nil],
 281   ].each do |token, expected|
 282     test "create as #{token} and expect requesting_container_uuid to be #{expected}" do
 283       set_user_from_auth token
 284       cr = ContainerRequest.create(container_image: "img", output_path: "/tmp", command: ["echo", "foo"])
 285       assert_not_nil cr.uuid, 'uuid should be set for newly created container_request'
 286       assert_equal expected, cr.requesting_container_uuid
 287     end
 288   end
 289
 290   [[{"vcpus" => [2, nil]},
 291     lambda { |resolved| resolved["vcpus"] == 2 }],
 292    [{"vcpus" => [3, 7]},
 293     lambda { |resolved| resolved["vcpus"] == 3 }],
 294    [{"vcpus" => 4},
 295     lambda { |resolved| resolved["vcpus"] == 4 }],
 296    [{"ram" => [1000000000, 2000000000]},
 297     lambda { |resolved| resolved["ram"] == 1000000000 }],
 298    [{"ram" => [1234234234]},
 299     lambda { |resolved| resolved["ram"] == 1234234234 }],
 300   ].each do |rc, okfunc|
 301     test "resolve runtime constraint range #{rc} to values" do
 302       cr = ContainerRequest.new(runtime_constraints: rc)
 303       resolved = cr.send :runtime_constraints_for_container
 304       assert(okfunc.call(resolved),
 305              "container runtime_constraints was #{resolved.inspect}")
 306     end
 307   end
 308
 309   [[{"/out" => {
 310         "kind" => "collection",
 311         "uuid" => "zzzzz-4zz18-znfnqtbbv4spc3w",
 312         "path" => "/foo"}},
 313     lambda do |resolved|
 314       resolved["/out"] == {
 315         "portable_data_hash" => "1f4b0bc7583c2a7f9102c395f4ffc5e3+45",
 316         "kind" => "collection",
 317         "path" => "/foo",
 318       }
 319     end],
 320    [{"/out" => {
 321         "kind" => "collection",
 322         "uuid" => "zzzzz-4zz18-znfnqtbbv4spc3w",
 323         "portable_data_hash" => "1f4b0bc7583c2a7f9102c395f4ffc5e3+45",
 324         "path" => "/foo"}},
 325     lambda do |resolved|
 326       resolved["/out"] == {
 327         "portable_data_hash" => "1f4b0bc7583c2a7f9102c395f4ffc5e3+45",
 328         "kind" => "collection",
 329         "path" => "/foo",
 330       }
 331     end],
 332   ].each do |mounts, okfunc|
 333     test "resolve mounts #{mounts.inspect} to values" do
 334       set_user_from_auth :active
 335       cr = ContainerRequest.new(mounts: mounts)
 336       resolved = cr.send :mounts_for_container
 337       assert(okfunc.call(resolved),
 338              "mounts_for_container returned #{resolved.inspect}")
 339     end
 340   end
 341
 342   test 'mount unreadable collection' do
 343     set_user_from_auth :spectator
 344     m = {
 345       "/foo" => {
 346         "kind" => "collection",
 347         "uuid" => "zzzzz-4zz18-znfnqtbbv4spc3w",
 348         "path" => "/foo",
 349       },
 350     }
 351     cr = ContainerRequest.new(mounts: m)
 352     assert_raises(ArvadosModel::UnresolvableContainerError) do
 353       cr.send :mounts_for_container
 354     end
 355   end
 356
 357   test 'mount collection with mismatched UUID and PDH' do
 358     set_user_from_auth :active
 359     m = {
 360       "/foo" => {
 361         "kind" => "collection",
 362         "uuid" => "zzzzz-4zz18-znfnqtbbv4spc3w",
 363         "portable_data_hash" => "fa7aeb5140e2848d39b416daeef4ffc5+45",
 364         "path" => "/foo",
 365       },
 366     }
 367     cr = ContainerRequest.new(mounts: m)
 368     assert_raises(ArgumentError) do
 369       cr.send :mounts_for_container
 370     end
 371   end
 372
 373   ['arvados/apitestfixture:latest',
 374    'arvados/apitestfixture',
 375    'd8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678',
 376   ].each do |tag|
 377     test "container_image_for_container(#{tag.inspect})" do
 378       set_user_from_auth :active
 379       cr = ContainerRequest.new(container_image: tag)
 380       resolved = cr.send :container_image_for_container
 381       assert_equal resolved, collections(:docker_image).portable_data_hash
 382     end
 383   end
 384
 385   test "container_image_for_container(pdh)" do
 386     set_user_from_auth :active
 387     [:docker_image, :docker_image_1_12].each do |coll|
 388       pdh = collections(coll).portable_data_hash
 389       cr = ContainerRequest.new(container_image: pdh)
 390       resolved = cr.send :container_image_for_container
 391       assert_equal resolved, pdh
 392     end
 393   end
 394
 395   ['acbd18db4cc2f85cedef654fccc4a4d8+3',
 396    'ENOEXIST',
 397    'arvados/apitestfixture:ENOEXIST',
 398   ].each do |img|
 399     test "container_image_for_container(#{img.inspect}) => 422" do
 400       set_user_from_auth :active
 401       cr = ContainerRequest.new(container_image: img)
 402       assert_raises(ArvadosModel::UnresolvableContainerError) do
 403         cr.send :container_image_for_container
 404       end
 405     end
 406   end
 407
 408   test "requestor can retrieve container owned by dispatch" do
 409     assert_not_empty Container.readable_by(users(:admin)).where(uuid: containers(:running).uuid)
 410     assert_not_empty Container.readable_by(users(:active)).where(uuid: containers(:running).uuid)
 411     assert_empty Container.readable_by(users(:spectator)).where(uuid: containers(:running).uuid)
 412   end
 413
 414   [
 415     [{"var" => "value1"}, {"var" => "value1"}, nil],
 416     [{"var" => "value1"}, {"var" => "value1"}, true],
 417     [{"var" => "value1"}, {"var" => "value1"}, false],
 418     [{"var" => "value1"}, {"var" => "value2"}, nil],
 419   ].each do |env1, env2, use_existing|
 420     test "Container request #{((env1 == env2) and (use_existing.nil? or use_existing == true)) ? 'does' : 'does not'} reuse container when committed#{use_existing.nil? ? '' : use_existing ? ' and use_existing == true' : ' and use_existing == false'}" do
 421       common_attrs = {cwd: "test",
 422                       priority: 1,
 423                       command: ["echo", "hello"],
 424                       output_path: "test",
 425                       runtime_constraints: {"vcpus" => 4,
 426                                             "ram" => 12000000000,
 427                                             "keep_cache_ram" => 268435456},
 428                       mounts: {"test" => {"kind" => "json"}}}
 429       set_user_from_auth :active
 430       cr1 = create_minimal_req!(common_attrs.merge({state: ContainerRequest::Committed,
 431                                                     environment: env1}))
 432       if use_existing.nil?
 433         # Testing with use_existing default value
 434         cr2 = create_minimal_req!(common_attrs.merge({state: ContainerRequest::Uncommitted,
 435                                                       environment: env2}))
 436       else
 437
 438         cr2 = create_minimal_req!(common_attrs.merge({state: ContainerRequest::Uncommitted,
 439                                                       environment: env2,
 440                                                       use_existing: use_existing}))
 441       end
 442       assert_not_nil cr1.container_uuid
 443       assert_nil cr2.container_uuid
 444
 445       # Update cr2 to commited state and check for container equality on different cases:
 446       # * When env1 and env2 are equal and use_existing is true, the same container
 447       #   should be assigned.
 448       # * When use_existing is false, a different container should be assigned.
 449       # * When env1 and env2 are different, a different container should be assigned.
 450       cr2.update_attributes!({state: ContainerRequest::Committed})
 451       assert_equal (cr2.use_existing == true and (env1 == env2)),
 452                    (cr1.container_uuid == cr2.container_uuid)
 453     end
 454   end
 455
 456   test "requesting_container_uuid at create is not allowed" do
 457     set_user_from_auth :active
 458     assert_raises(ActiveRecord::RecordNotSaved) do
 459       create_minimal_req!(state: "Uncommitted", priority: 1, requesting_container_uuid: 'youcantdothat')
 460     end
 461   end
 462
 463   test "Retry on container cancelled" do
 464     set_user_from_auth :active
 465     cr = create_minimal_req!(priority: 1, state: "Committed", container_count_max: 2)
 466     prev_container_uuid = cr.container_uuid
 467
 468     c = act_as_system_user do
 469       c = Container.find_by_uuid(cr.container_uuid)
 470       c.update_attributes!(state: Container::Locked)
 471       c.update_attributes!(state: Container::Running)
 472       c
 473     end
 474
 475     cr.reload
 476     assert_equal "Committed", cr.state
 477     assert_equal prev_container_uuid, cr.container_uuid
 478     prev_container_uuid = cr.container_uuid
 479
 480     act_as_system_user do
 481       c.update_attributes!(state: Container::Cancelled)
 482     end
 483
 484     cr.reload
 485     assert_equal "Committed", cr.state
 486     assert_not_equal prev_container_uuid, cr.container_uuid
 487     prev_container_uuid = cr.container_uuid
 488
 489     c = act_as_system_user do
 490       c = Container.find_by_uuid(cr.container_uuid)
 491       c.update_attributes!(state: Container::Cancelled)
 492       c
 493     end
 494
 495     cr.reload
 496     assert_equal "Final", cr.state
 497     assert_equal prev_container_uuid, cr.container_uuid
 498   end
 499
 500   test "Finalize committed request when reusing a finished container" do
 501     set_user_from_auth :active
 502     cr = create_minimal_req!(priority: 1, state: ContainerRequest::Committed)
 503     cr.reload
 504     assert_equal ContainerRequest::Committed, cr.state
 505     act_as_system_user do
 506       c = Container.find_by_uuid(cr.container_uuid)
 507       c.update_attributes!(state: Container::Locked)
 508       c.update_attributes!(state: Container::Running)
 509       c.update_attributes!(state: Container::Complete,
 510                            exit_code: 0,
 511                            output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45',
 512                            log: 'fa7aeb5140e2848d39b416daeef4ffc5+45')
 513     end
 514     cr.reload
 515     assert_equal ContainerRequest::Final, cr.state
 516
 517     cr2 = create_minimal_req!(priority: 1, state: ContainerRequest::Committed)
 518     assert_equal cr.container_uuid, cr2.container_uuid
 519     assert_equal ContainerRequest::Final, cr2.state
 520
 521     cr3 = create_minimal_req!(priority: 1, state: ContainerRequest::Uncommitted)
 522     assert_equal ContainerRequest::Uncommitted, cr3.state
 523     cr3.update_attributes!(state: ContainerRequest::Committed)
 524     assert_equal cr.container_uuid, cr3.container_uuid
 525     assert_equal ContainerRequest::Final, cr3.state
 526   end
 527
 528   [
 529     [{"vcpus" => 1, "ram" => 123, "keep_cache_ram" => 100}, ContainerRequest::Committed, 100],
 530     [{"vcpus" => 1, "ram" => 123}, ContainerRequest::Uncommitted],
 531     [{"vcpus" => 1, "ram" => 123}, ContainerRequest::Committed],
 532     [{"vcpus" => 1, "ram" => 123, "keep_cache_ram" => -1}, ContainerRequest::Committed, ActiveRecord::RecordInvalid],
 533     [{"vcpus" => 1, "ram" => 123, "keep_cache_ram" => '123'}, ContainerRequest::Committed, ActiveRecord::RecordInvalid],
 534   ].each do |rc, state, expected|
 535     test "create container request with #{rc} in state #{state} and verify keep_cache_ram #{expected}" do
 536       common_attrs = {cwd: "test",
 537                       priority: 1,
 538                       command: ["echo", "hello"],
 539                       output_path: "test",
 540                       runtime_constraints: rc,
 541                       mounts: {"test" => {"kind" => "json"}}}
 542       set_user_from_auth :active
 543
 544       if expected == ActiveRecord::RecordInvalid
 545         assert_raises(ActiveRecord::RecordInvalid) do
 546           create_minimal_req!(common_attrs.merge({state: state}))
 547         end
 548       else
 549         cr = create_minimal_req!(common_attrs.merge({state: state}))
 550         expected = Rails.configuration.container_default_keep_cache_ram if state == ContainerRequest::Committed and expected.nil?
 551         assert_equal expected, cr.runtime_constraints['keep_cache_ram']
 552       end
 553     end
 554   end
 555 end