services/api/test/unit/container_request_test.rb

   1 require 'test_helper'
   2
   3 class ContainerRequestTest < ActiveSupport::TestCase
   4   def create_minimal_req! attrs={}
   5     defaults = {
   6       command: ["echo", "foo"],
   7       container_image: links(:docker_image_collection_tag).name,
   8       cwd: "/tmp",
   9       environment: {},
  10       mounts: {"/out" => {"kind" => "tmp", "capacity" => 1000000}},
  11       output_path: "/out",
  12       runtime_constraints: {"vcpus" => 1, "ram" => 2},
  13       name: "foo",
  14       description: "bar",
  15     }
  16     cr = ContainerRequest.create!(defaults.merge(attrs))
  17     cr.reload
  18     return cr
  19   end
  20
  21   def check_bogus_states cr
  22     [nil, "Flubber"].each do |state|
  23       assert_raises(ActiveRecord::RecordInvalid) do
  24         cr.state = state
  25         cr.save!
  26       end
  27       cr.reload
  28     end
  29   end
  30
  31   test "Container request create" do
  32     set_user_from_auth :active
  33     cr = create_minimal_req!
  34
  35     assert_nil cr.container_uuid
  36     assert_nil cr.priority
  37
  38     check_bogus_states cr
  39
  40     # Ensure we can modify all attributes
  41     cr.command = ["echo", "foo3"]
  42     cr.container_image = "img3"
  43     cr.cwd = "/tmp3"
  44     cr.environment = {"BUP" => "BOP"}
  45     cr.mounts = {"BAR" => "BAZ"}
  46     cr.output_path = "/tmp4"
  47     cr.priority = 2
  48     cr.runtime_constraints = {"vcpus" => 4}
  49     cr.name = "foo3"
  50     cr.description = "bar3"
  51     cr.save!
  52
  53     assert_nil cr.container_uuid
  54   end
  55
  56   [
  57     {"vcpus" => 1},
  58     {"vcpus" => 1, "ram" => nil},
  59     {"vcpus" => 0, "ram" => 123},
  60     {"vcpus" => "1", "ram" => "123"}
  61   ].each do |invalid_constraints|
  62     test "Create with #{invalid_constraints}" do
  63       set_user_from_auth :active
  64       assert_raises(ActiveRecord::RecordInvalid) do
  65         cr = create_minimal_req!(state: "Committed",
  66                                  priority: 1,
  67                                  runtime_constraints: invalid_constraints)
  68         cr.save!
  69       end
  70     end
  71
  72     test "Update with #{invalid_constraints}" do
  73       set_user_from_auth :active
  74       cr = create_minimal_req!(state: "Uncommitted", priority: 1)
  75       cr.save!
  76       assert_raises(ActiveRecord::RecordInvalid) do
  77         cr = ContainerRequest.find_by_uuid cr.uuid
  78         cr.update_attributes!(state: "Committed",
  79                               runtime_constraints: invalid_constraints)
  80       end
  81     end
  82   end
  83
  84   test "Update from fixture" do
  85     set_user_from_auth :active
  86     cr = ContainerRequest.find_by_uuid(container_requests(:running).uuid)
  87     cr.update_attributes!(description: "New description")
  88     assert_equal "New description", cr.description
  89   end
  90
  91   test "Update with valid runtime constraints" do
  92       set_user_from_auth :active
  93       cr = create_minimal_req!(state: "Uncommitted", priority: 1)
  94       cr.save!
  95       cr = ContainerRequest.find_by_uuid cr.uuid
  96       cr.update_attributes!(state: "Committed",
  97                             runtime_constraints: {"vcpus" => 1, "ram" => 23})
  98       assert_not_nil cr.container_uuid
  99   end
 100
 101   test "Container request priority must be non-nil" do
 102     set_user_from_auth :active
 103     cr = create_minimal_req!(priority: nil)
 104     cr.state = "Committed"
 105     assert_raises(ActiveRecord::RecordInvalid) do
 106       cr.save!
 107     end
 108   end
 109
 110   test "Container request commit" do
 111     set_user_from_auth :active
 112     cr = create_minimal_req!(runtime_constraints: {"vcpus" => 2, "ram" => 30})
 113
 114     assert_nil cr.container_uuid
 115
 116     cr.reload
 117     cr.state = "Committed"
 118     cr.priority = 1
 119     cr.save!
 120
 121     cr.reload
 122
 123     assert_not_nil cr.container_uuid
 124     c = Container.find_by_uuid cr.container_uuid
 125     assert_not_nil c
 126     assert_equal ["echo", "foo"], c.command
 127     assert_equal collections(:docker_image).portable_data_hash, c.container_image
 128     assert_equal "/tmp", c.cwd
 129     assert_equal({}, c.environment)
 130     assert_equal({"/out" => {"kind"=>"tmp", "capacity"=>1000000}}, c.mounts)
 131     assert_equal "/out", c.output_path
 132     assert_equal({"keep_cache_ram"=>268435456, "vcpus" => 2, "ram" => 30}, c.runtime_constraints)
 133     assert_equal 1, c.priority
 134
 135     assert_raises(ActiveRecord::RecordInvalid) do
 136       cr.priority = nil
 137       cr.save!
 138     end
 139
 140     cr.priority = 0
 141     cr.save!
 142
 143     cr.reload
 144     c.reload
 145     assert_equal 0, cr.priority
 146     assert_equal 0, c.priority
 147   end
 148
 149
 150   test "Container request max priority" do
 151     set_user_from_auth :active
 152     cr = create_minimal_req!(priority: 5, state: "Committed")
 153
 154     c = Container.find_by_uuid cr.container_uuid
 155     assert_equal 5, c.priority
 156
 157     cr2 = create_minimal_req!
 158     cr2.priority = 10
 159     cr2.state = "Committed"
 160     cr2.container_uuid = cr.container_uuid
 161     act_as_system_user do
 162       cr2.save!
 163     end
 164
 165     # cr and cr2 have priority 5 and 10, and are being satisfied by
 166     # the same container c, so c's priority should be
 167     # max(priority)=10.
 168     c.reload
 169     assert_equal 10, c.priority
 170
 171     cr2.update_attributes!(priority: 0)
 172
 173     c.reload
 174     assert_equal 5, c.priority
 175
 176     cr.update_attributes!(priority: 0)
 177
 178     c.reload
 179     assert_equal 0, c.priority
 180   end
 181
 182
 183   test "Independent container requests" do
 184     set_user_from_auth :active
 185     cr1 = create_minimal_req!(command: ["foo", "1"], priority: 5, state: "Committed")
 186     cr2 = create_minimal_req!(command: ["foo", "2"], priority: 10, state: "Committed")
 187
 188     c1 = Container.find_by_uuid cr1.container_uuid
 189     assert_equal 5, c1.priority
 190
 191     c2 = Container.find_by_uuid cr2.container_uuid
 192     assert_equal 10, c2.priority
 193
 194     cr1.update_attributes!(priority: 0)
 195
 196     c1.reload
 197     assert_equal 0, c1.priority
 198
 199     c2.reload
 200     assert_equal 10, c2.priority
 201   end
 202
 203   test "Request is finalized when its container is cancelled" do
 204     set_user_from_auth :active
 205     cr = create_minimal_req!(priority: 1, state: "Committed", container_count_max: 1)
 206
 207     act_as_system_user do
 208       Container.find_by_uuid(cr.container_uuid).
 209         update_attributes!(state: Container::Cancelled)
 210     end
 211
 212     cr.reload
 213     assert_equal "Final", cr.state
 214   end
 215
 216   test "Request is finalized when its container is completed" do
 217     set_user_from_auth :active
 218     project = groups(:private)
 219     cr = create_minimal_req!(owner_uuid: project.uuid,
 220                              priority: 1,
 221                              state: "Committed")
 222
 223     c = act_as_system_user do
 224       c = Container.find_by_uuid(cr.container_uuid)
 225       c.update_attributes!(state: Container::Locked)
 226       c.update_attributes!(state: Container::Running)
 227       c
 228     end
 229
 230     cr.reload
 231     assert_equal "Committed", cr.state
 232
 233     output_pdh = '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
 234     log_pdh = 'fa7aeb5140e2848d39b416daeef4ffc5+45'
 235     act_as_system_user do
 236       c.update_attributes!(state: Container::Complete,
 237                            output: output_pdh,
 238                            log: log_pdh)
 239     end
 240
 241     cr.reload
 242     assert_equal "Final", cr.state
 243     ['output', 'log'].each do |out_type|
 244       pdh = Container.find_by_uuid(cr.container_uuid).send(out_type)
 245       assert_equal(1, Collection.where(portable_data_hash: pdh,
 246                                        owner_uuid: project.uuid).count,
 247                    "Container #{out_type} should be copied to #{project.uuid}")
 248     end
 249     assert_not_nil cr.output_uuid
 250     assert_not_nil cr.log_uuid
 251     output = Collection.find_by_uuid cr.output_uuid
 252     assert_equal output_pdh, output.portable_data_hash
 253     log = Collection.find_by_uuid cr.log_uuid
 254     assert_equal log_pdh, log.portable_data_hash
 255   end
 256
 257   test "Container makes container request, then is cancelled" do
 258     set_user_from_auth :active
 259     cr = create_minimal_req!(priority: 5, state: "Committed", container_count_max: 1)
 260
 261     c = Container.find_by_uuid cr.container_uuid
 262     assert_equal 5, c.priority
 263
 264     cr2 = create_minimal_req!
 265     cr2.update_attributes!(priority: 10, state: "Committed", requesting_container_uuid: c.uuid, command: ["echo", "foo2"], container_count_max: 1)
 266     cr2.reload
 267
 268     c2 = Container.find_by_uuid cr2.container_uuid
 269     assert_equal 10, c2.priority
 270
 271     act_as_system_user do
 272       c.state = "Cancelled"
 273       c.save!
 274     end
 275
 276     cr.reload
 277     assert_equal "Final", cr.state
 278
 279     cr2.reload
 280     assert_equal 0, cr2.priority
 281
 282     c2.reload
 283     assert_equal 0, c2.priority
 284   end
 285
 286   [
 287     ['running_container_auth', 'zzzzz-dz642-runningcontainr'],
 288     ['active_no_prefs', nil],
 289   ].each do |token, expected|
 290     test "create as #{token} and expect requesting_container_uuid to be #{expected}" do
 291       set_user_from_auth token
 292       cr = ContainerRequest.create(container_image: "img", output_path: "/tmp", command: ["echo", "foo"])
 293       assert_not_nil cr.uuid, 'uuid should be set for newly created container_request'
 294       assert_equal expected, cr.requesting_container_uuid
 295     end
 296   end
 297
 298   [[{"vcpus" => [2, nil]},
 299     lambda { |resolved| resolved["vcpus"] == 2 }],
 300    [{"vcpus" => [3, 7]},
 301     lambda { |resolved| resolved["vcpus"] == 3 }],
 302    [{"vcpus" => 4},
 303     lambda { |resolved| resolved["vcpus"] == 4 }],
 304    [{"ram" => [1000000000, 2000000000]},
 305     lambda { |resolved| resolved["ram"] == 1000000000 }],
 306    [{"ram" => [1234234234]},
 307     lambda { |resolved| resolved["ram"] == 1234234234 }],
 308   ].each do |rc, okfunc|
 309     test "resolve runtime constraint range #{rc} to values" do
 310       cr = ContainerRequest.new(runtime_constraints: rc)
 311       resolved = cr.send :runtime_constraints_for_container
 312       assert(okfunc.call(resolved),
 313              "container runtime_constraints was #{resolved.inspect}")
 314     end
 315   end
 316
 317   [[{"/out" => {
 318         "kind" => "collection",
 319         "uuid" => "zzzzz-4zz18-znfnqtbbv4spc3w",
 320         "path" => "/foo"}},
 321     lambda do |resolved|
 322       resolved["/out"] == {
 323         "portable_data_hash" => "1f4b0bc7583c2a7f9102c395f4ffc5e3+45",
 324         "kind" => "collection",
 325         "path" => "/foo",
 326       }
 327     end],
 328    [{"/out" => {
 329         "kind" => "collection",
 330         "uuid" => "zzzzz-4zz18-znfnqtbbv4spc3w",
 331         "portable_data_hash" => "1f4b0bc7583c2a7f9102c395f4ffc5e3+45",
 332         "path" => "/foo"}},
 333     lambda do |resolved|
 334       resolved["/out"] == {
 335         "portable_data_hash" => "1f4b0bc7583c2a7f9102c395f4ffc5e3+45",
 336         "kind" => "collection",
 337         "path" => "/foo",
 338       }
 339     end],
 340   ].each do |mounts, okfunc|
 341     test "resolve mounts #{mounts.inspect} to values" do
 342       set_user_from_auth :active
 343       cr = ContainerRequest.new(mounts: mounts)
 344       resolved = cr.send :mounts_for_container
 345       assert(okfunc.call(resolved),
 346              "mounts_for_container returned #{resolved.inspect}")
 347     end
 348   end
 349
 350   test 'mount unreadable collection' do
 351     set_user_from_auth :spectator
 352     m = {
 353       "/foo" => {
 354         "kind" => "collection",
 355         "uuid" => "zzzzz-4zz18-znfnqtbbv4spc3w",
 356         "path" => "/foo",
 357       },
 358     }
 359     cr = ContainerRequest.new(mounts: m)
 360     assert_raises(ArvadosModel::UnresolvableContainerError) do
 361       cr.send :mounts_for_container
 362     end
 363   end
 364
 365   test 'mount collection with mismatched UUID and PDH' do
 366     set_user_from_auth :active
 367     m = {
 368       "/foo" => {
 369         "kind" => "collection",
 370         "uuid" => "zzzzz-4zz18-znfnqtbbv4spc3w",
 371         "portable_data_hash" => "fa7aeb5140e2848d39b416daeef4ffc5+45",
 372         "path" => "/foo",
 373       },
 374     }
 375     cr = ContainerRequest.new(mounts: m)
 376     assert_raises(ArgumentError) do
 377       cr.send :mounts_for_container
 378     end
 379   end
 380
 381   ['arvados/apitestfixture:latest',
 382    'arvados/apitestfixture',
 383    'd8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678',
 384   ].each do |tag|
 385     test "container_image_for_container(#{tag.inspect})" do
 386       set_user_from_auth :active
 387       cr = ContainerRequest.new(container_image: tag)
 388       resolved = cr.send :container_image_for_container
 389       assert_equal resolved, collections(:docker_image).portable_data_hash
 390     end
 391   end
 392
 393   test "container_image_for_container(pdh)" do
 394     set_user_from_auth :active
 395     [:docker_image, :docker_image_1_12].each do |coll|
 396       pdh = collections(coll).portable_data_hash
 397       cr = ContainerRequest.new(container_image: pdh)
 398       resolved = cr.send :container_image_for_container
 399       assert_equal resolved, pdh
 400     end
 401   end
 402
 403   ['acbd18db4cc2f85cedef654fccc4a4d8+3',
 404    'ENOEXIST',
 405    'arvados/apitestfixture:ENOEXIST',
 406   ].each do |img|
 407     test "container_image_for_container(#{img.inspect}) => 422" do
 408       set_user_from_auth :active
 409       cr = ContainerRequest.new(container_image: img)
 410       assert_raises(ArvadosModel::UnresolvableContainerError) do
 411         cr.send :container_image_for_container
 412       end
 413     end
 414   end
 415
 416   test "requestor can retrieve container owned by dispatch" do
 417     assert_not_empty Container.readable_by(users(:admin)).where(uuid: containers(:running).uuid)
 418     assert_not_empty Container.readable_by(users(:active)).where(uuid: containers(:running).uuid)
 419     assert_empty Container.readable_by(users(:spectator)).where(uuid: containers(:running).uuid)
 420   end
 421
 422   [
 423     [{"var" => "value1"}, {"var" => "value1"}, nil],
 424     [{"var" => "value1"}, {"var" => "value1"}, true],
 425     [{"var" => "value1"}, {"var" => "value1"}, false],
 426     [{"var" => "value1"}, {"var" => "value2"}, nil],
 427   ].each do |env1, env2, use_existing|
 428     test "Container request #{((env1 == env2) and (use_existing.nil? or use_existing == true)) ? 'does' : 'does not'} reuse container when committed#{use_existing.nil? ? '' : use_existing ? ' and use_existing == true' : ' and use_existing == false'}" do
 429       common_attrs = {cwd: "test",
 430                       priority: 1,
 431                       command: ["echo", "hello"],
 432                       output_path: "test",
 433                       runtime_constraints: {"vcpus" => 4,
 434                                             "ram" => 12000000000,
 435                                             "keep_cache_ram" => 268435456},
 436                       mounts: {"test" => {"kind" => "json"}}}
 437       set_user_from_auth :active
 438       cr1 = create_minimal_req!(common_attrs.merge({state: ContainerRequest::Committed,
 439                                                     environment: env1}))
 440       if use_existing.nil?
 441         # Testing with use_existing default value
 442         cr2 = create_minimal_req!(common_attrs.merge({state: ContainerRequest::Uncommitted,
 443                                                       environment: env2}))
 444       else
 445
 446         cr2 = create_minimal_req!(common_attrs.merge({state: ContainerRequest::Uncommitted,
 447                                                       environment: env2,
 448                                                       use_existing: use_existing}))
 449       end
 450       assert_not_nil cr1.container_uuid
 451       assert_nil cr2.container_uuid
 452
 453       # Update cr2 to commited state and check for container equality on different cases:
 454       # * When env1 and env2 are equal and use_existing is true, the same container
 455       #   should be assigned.
 456       # * When use_existing is false, a different container should be assigned.
 457       # * When env1 and env2 are different, a different container should be assigned.
 458       cr2.update_attributes!({state: ContainerRequest::Committed})
 459       assert_equal (cr2.use_existing == true and (env1 == env2)),
 460                    (cr1.container_uuid == cr2.container_uuid)
 461     end
 462   end
 463
 464   test "requesting_container_uuid at create is not allowed" do
 465     set_user_from_auth :active
 466     assert_raises(ActiveRecord::RecordNotSaved) do
 467       create_minimal_req!(state: "Uncommitted", priority: 1, requesting_container_uuid: 'youcantdothat')
 468     end
 469   end
 470
 471   test "Retry on container cancelled" do
 472     set_user_from_auth :active
 473     cr = create_minimal_req!(priority: 1, state: "Committed", container_count_max: 2)
 474     prev_container_uuid = cr.container_uuid
 475
 476     c = act_as_system_user do
 477       c = Container.find_by_uuid(cr.container_uuid)
 478       c.update_attributes!(state: Container::Locked)
 479       c.update_attributes!(state: Container::Running)
 480       c
 481     end
 482
 483     cr.reload
 484     assert_equal "Committed", cr.state
 485     assert_equal prev_container_uuid, cr.container_uuid
 486     prev_container_uuid = cr.container_uuid
 487
 488     act_as_system_user do
 489       c.update_attributes!(state: Container::Cancelled)
 490     end
 491
 492     cr.reload
 493     assert_equal "Committed", cr.state
 494     assert_not_equal prev_container_uuid, cr.container_uuid
 495     prev_container_uuid = cr.container_uuid
 496
 497     c = act_as_system_user do
 498       c = Container.find_by_uuid(cr.container_uuid)
 499       c.update_attributes!(state: Container::Cancelled)
 500       c
 501     end
 502
 503     cr.reload
 504     assert_equal "Final", cr.state
 505     assert_equal prev_container_uuid, cr.container_uuid
 506   end
 507
 508   test "Finalize committed request when reusing a finished container" do
 509     set_user_from_auth :active
 510     cr = create_minimal_req!(priority: 1, state: ContainerRequest::Committed)
 511     cr.reload
 512     assert_equal ContainerRequest::Committed, cr.state
 513     act_as_system_user do
 514       c = Container.find_by_uuid(cr.container_uuid)
 515       c.update_attributes!(state: Container::Locked)
 516       c.update_attributes!(state: Container::Running)
 517       c.update_attributes!(state: Container::Complete,
 518                            exit_code: 0,
 519                            output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45',
 520                            log: 'fa7aeb5140e2848d39b416daeef4ffc5+45')
 521     end
 522     cr.reload
 523     assert_equal ContainerRequest::Final, cr.state
 524
 525     cr2 = create_minimal_req!(priority: 1, state: ContainerRequest::Committed)
 526     assert_equal cr.container_uuid, cr2.container_uuid
 527     assert_equal ContainerRequest::Final, cr2.state
 528
 529     cr3 = create_minimal_req!(priority: 1, state: ContainerRequest::Uncommitted)
 530     assert_equal ContainerRequest::Uncommitted, cr3.state
 531     cr3.update_attributes!(state: ContainerRequest::Committed)
 532     assert_equal cr.container_uuid, cr3.container_uuid
 533     assert_equal ContainerRequest::Final, cr3.state
 534   end
 535
 536   [
 537     [{"vcpus" => 1, "ram" => 123, "keep_cache_ram" => 100}, ContainerRequest::Committed, 100],
 538     [{"vcpus" => 1, "ram" => 123}, ContainerRequest::Uncommitted],
 539     [{"vcpus" => 1, "ram" => 123}, ContainerRequest::Committed],
 540     [{"vcpus" => 1, "ram" => 123, "keep_cache_ram" => -1}, ContainerRequest::Committed, ActiveRecord::RecordInvalid],
 541     [{"vcpus" => 1, "ram" => 123, "keep_cache_ram" => '123'}, ContainerRequest::Committed, ActiveRecord::RecordInvalid],
 542   ].each do |rc, state, expected|
 543     test "create container request with #{rc} in state #{state} and verify keep_cache_ram #{expected}" do
 544       common_attrs = {cwd: "test",
 545                       priority: 1,
 546                       command: ["echo", "hello"],
 547                       output_path: "test",
 548                       runtime_constraints: rc,
 549                       mounts: {"test" => {"kind" => "json"}}}
 550       set_user_from_auth :active
 551
 552       if expected == ActiveRecord::RecordInvalid
 553         assert_raises(ActiveRecord::RecordInvalid) do
 554           create_minimal_req!(common_attrs.merge({state: state}))
 555         end
 556       else
 557         cr = create_minimal_req!(common_attrs.merge({state: state}))
 558         expected = Rails.configuration.container_default_keep_cache_ram if state == ContainerRequest::Committed and expected.nil?
 559         assert_equal expected, cr.runtime_constraints['keep_cache_ram']
 560       end
 561     end
 562   end
 563
 564   [
 565     [{"partitions" => ["fastcpu","vfastcpu", 100]}, ContainerRequest::Committed, ActiveRecord::RecordInvalid],
 566     [{"partitions" => ["fastcpu","vfastcpu", 100]}, ContainerRequest::Uncommitted],
 567     [{"partitions" => "fastcpu"}, ContainerRequest::Committed, ActiveRecord::RecordInvalid],
 568     [{"partitions" => "fastcpu"}, ContainerRequest::Uncommitted],
 569     [{"partitions" => ["fastcpu","vfastcpu"]}, ContainerRequest::Committed],
 570   ].each do |sp, state, expected|
 571     test "create container request with scheduling_parameters #{sp} in state #{state} and verify #{expected}" do
 572       common_attrs = {cwd: "test",
 573                       priority: 1,
 574                       command: ["echo", "hello"],
 575                       output_path: "test",
 576                       scheduling_parameters: sp,
 577                       mounts: {"test" => {"kind" => "json"}}}
 578       set_user_from_auth :active
 579
 580       if expected == ActiveRecord::RecordInvalid
 581         assert_raises(ActiveRecord::RecordInvalid) do
 582           create_minimal_req!(common_attrs.merge({state: state}))
 583         end
 584       else
 585         cr = create_minimal_req!(common_attrs.merge({state: state}))
 586         assert_equal sp, cr.scheduling_parameters
 587
 588         if state == ContainerRequest::Committed
 589           c = Container.find_by_uuid(cr.container_uuid)
 590           assert_equal sp, c.scheduling_parameters
 591         end
 592       end
 593     end
 594   end
 595 end