services/api/test/unit/container_request_test.rb

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 require 'test_helper'
   6 require 'helpers/container_test_helper'
   7 require 'helpers/docker_migration_helper'
   8 require 'arvados/collection'
   9
  10 class ContainerRequestTest < ActiveSupport::TestCase
  11   include DockerMigrationHelper
  12   include DbCurrentTime
  13   include ContainerTestHelper
  14
  15   def with_container_auth(ctr)
  16     auth_was = Thread.current[:api_client_authorization]
  17     client_was = Thread.current[:api_client]
  18     token_was = Thread.current[:token]
  19     user_was = Thread.current[:user]
  20     auth = ApiClientAuthorization.find_by_uuid(ctr.auth_uuid)
  21     Thread.current[:api_client_authorization] = auth
  22     Thread.current[:api_client] = auth.api_client
  23     Thread.current[:token] = auth.token
  24     Thread.current[:user] = auth.user
  25     begin
  26       yield
  27     ensure
  28       Thread.current[:api_client_authorization] = auth_was
  29       Thread.current[:api_client] = client_was
  30       Thread.current[:token] = token_was
  31       Thread.current[:user] = user_was
  32     end
  33   end
  34
  35   def lock_and_run(ctr)
  36       act_as_system_user do
  37         ctr.update_attributes!(state: Container::Locked)
  38         ctr.update_attributes!(state: Container::Running)
  39       end
  40   end
  41
  42   def create_minimal_req! attrs={}
  43     defaults = {
  44       command: ["echo", "foo"],
  45       container_image: links(:docker_image_collection_tag).name,
  46       cwd: "/tmp",
  47       environment: {},
  48       mounts: {"/out" => {"kind" => "tmp", "capacity" => 1000000}},
  49       output_path: "/out",
  50       runtime_constraints: {"vcpus" => 1, "ram" => 2},
  51       name: "foo",
  52       description: "bar",
  53     }
  54     cr = ContainerRequest.create!(defaults.merge(attrs))
  55     cr.reload
  56     return cr
  57   end
  58
  59   def check_bogus_states cr
  60     [nil, "Flubber"].each do |state|
  61       assert_raises(ActiveRecord::RecordInvalid) do
  62         cr.state = state
  63         cr.save!
  64       end
  65       cr.reload
  66     end
  67   end
  68
  69   def configure_preemptible_instance_type
  70     Rails.configuration.InstanceTypes = ConfigLoader.to_OrderedOptions({
  71       "a1.small.pre" => {
  72         "Preemptible" => true,
  73         "Price" => 0.1,
  74         "ProviderType" => "a1.small",
  75         "VCPUs" => 1,
  76         "RAM" => 1000000000,
  77       },
  78     })
  79   end
  80
  81   test "Container request create" do
  82     set_user_from_auth :active
  83     cr = create_minimal_req!
  84
  85     assert_nil cr.container_uuid
  86     assert_equal 0, cr.priority
  87
  88     check_bogus_states cr
  89
  90     # Ensure we can modify all attributes
  91     cr.command = ["echo", "foo3"]
  92     cr.container_image = "img3"
  93     cr.cwd = "/tmp3"
  94     cr.environment = {"BUP" => "BOP"}
  95     cr.mounts = {"BAR" => {"kind" => "BAZ"}}
  96     cr.output_path = "/tmp4"
  97     cr.priority = 2
  98     cr.runtime_constraints = {"vcpus" => 4}
  99     cr.name = "foo3"
 100     cr.description = "bar3"
 101     cr.save!
 102
 103     assert_nil cr.container_uuid
 104   end
 105
 106   [
 107     {"runtime_constraints" => {"vcpus" => 1}},
 108     {"runtime_constraints" => {"vcpus" => 1, "ram" => nil}},
 109     {"runtime_constraints" => {"vcpus" => 0, "ram" => 123}},
 110     {"runtime_constraints" => {"vcpus" => "1", "ram" => -1}},
 111     {"mounts" => {"FOO" => "BAR"}},
 112     {"mounts" => {"FOO" => {}}},
 113     {"mounts" => {"FOO" => {"kind" => "tmp", "capacity" => 42.222}}},
 114     {"command" => ["echo", 55]},
 115     {"environment" => {"FOO" => 55}}
 116   ].each do |value|
 117     test "Create with invalid #{value}" do
 118       set_user_from_auth :active
 119       assert_raises(ActiveRecord::RecordInvalid) do
 120         cr = create_minimal_req!({state: "Committed",
 121                priority: 1}.merge(value))
 122         cr.save!
 123       end
 124     end
 125
 126     test "Update with invalid #{value}" do
 127       set_user_from_auth :active
 128       cr = create_minimal_req!(state: "Uncommitted", priority: 1)
 129       cr.save!
 130       assert_raises(ActiveRecord::RecordInvalid) do
 131         cr = ContainerRequest.find_by_uuid cr.uuid
 132         cr.update_attributes!({state: "Committed",
 133                                priority: 1}.merge(value))
 134       end
 135     end
 136   end
 137
 138   test "Update from fixture" do
 139     set_user_from_auth :active
 140     cr = ContainerRequest.find_by_uuid(container_requests(:running).uuid)
 141     cr.update_attributes!(description: "New description")
 142     assert_equal "New description", cr.description
 143   end
 144
 145   test "Update with valid runtime constraints" do
 146       set_user_from_auth :active
 147       cr = create_minimal_req!(state: "Uncommitted", priority: 1)
 148       cr.save!
 149       cr = ContainerRequest.find_by_uuid cr.uuid
 150       cr.update_attributes!(state: "Committed",
 151                             runtime_constraints: {"vcpus" => 1, "ram" => 23})
 152       assert_not_nil cr.container_uuid
 153   end
 154
 155   test "Container request priority must be non-nil" do
 156     set_user_from_auth :active
 157     cr = create_minimal_req!
 158     cr.priority = nil
 159     cr.state = "Committed"
 160     assert_raises(ActiveRecord::RecordInvalid) do
 161       cr.save!
 162     end
 163   end
 164
 165   test "Container request commit" do
 166     set_user_from_auth :active
 167     cr = create_minimal_req!(runtime_constraints: {"vcpus" => 2, "ram" => 300000000})
 168
 169     assert_nil cr.container_uuid
 170
 171     cr.reload
 172     cr.state = "Committed"
 173     cr.priority = 1
 174     cr.save!
 175
 176     cr.reload
 177
 178     assert_empty({"vcpus" => 2, "ram" => 300000000}.to_a - cr.runtime_constraints.to_a)
 179
 180     assert_equal 0, Rails.configuration.Containers.DefaultKeepCacheRAM
 181
 182     assert_not_nil cr.container_uuid
 183     c = Container.find_by_uuid cr.container_uuid
 184     assert_not_nil c
 185     assert_equal ["echo", "foo"], c.command
 186     assert_equal collections(:docker_image).portable_data_hash, c.container_image
 187     assert_equal "/tmp", c.cwd
 188     assert_equal({}, c.environment)
 189     assert_equal({"/out" => {"kind"=>"tmp", "capacity"=>1000000}}, c.mounts)
 190     assert_equal "/out", c.output_path
 191     assert ({"keep_cache_disk" => 2<<30, "keep_cache_ram" => 0, "vcpus" => 2, "ram" => 300000000}.to_a - c.runtime_constraints.to_a).empty?
 192     assert_operator 0, :<, c.priority
 193
 194     assert_raises(ActiveRecord::RecordInvalid) do
 195       cr.priority = nil
 196       cr.save!
 197     end
 198
 199     cr.priority = 0
 200     cr.save!
 201
 202     cr.reload
 203     c.reload
 204     assert_equal 0, cr.priority
 205     assert_equal 0, c.priority
 206   end
 207
 208   test "Independent container requests" do
 209     set_user_from_auth :active
 210     cr1 = create_minimal_req!(command: ["foo", "1"], priority: 5, state: "Committed")
 211     cr2 = create_minimal_req!(command: ["foo", "2"], priority: 10, state: "Committed")
 212
 213     c1 = Container.find_by_uuid cr1.container_uuid
 214     assert_operator 0, :<, c1.priority
 215
 216     c2 = Container.find_by_uuid cr2.container_uuid
 217     assert_operator c1.priority, :<, c2.priority
 218     c2priority_was = c2.priority
 219
 220     cr1.update_attributes!(priority: 0)
 221
 222     c1.reload
 223     assert_equal 0, c1.priority
 224
 225     c2.reload
 226     assert_equal c2priority_was, c2.priority
 227   end
 228
 229   test "Request is finalized when its container is cancelled" do
 230     set_user_from_auth :active
 231     cr = create_minimal_req!(priority: 1, state: "Committed", container_count_max: 1)
 232     assert_equal users(:active).uuid, cr.modified_by_user_uuid
 233
 234     act_as_system_user do
 235       Container.find_by_uuid(cr.container_uuid).
 236         update_attributes!(state: Container::Cancelled, cost: 1.25)
 237     end
 238
 239     cr.reload
 240     assert_equal "Final", cr.state
 241     assert_equal 1.25, cr.cumulative_cost
 242     assert_equal users(:active).uuid, cr.modified_by_user_uuid
 243   end
 244
 245   test "Request is finalized when its container is completed" do
 246     set_user_from_auth :active
 247     project = groups(:private)
 248     cr = create_minimal_req!(owner_uuid: project.uuid,
 249                              priority: 1,
 250                              state: "Committed")
 251     assert_equal users(:active).uuid, cr.modified_by_user_uuid
 252
 253     c = act_as_system_user do
 254       c = Container.find_by_uuid(cr.container_uuid)
 255       c.update_attributes!(state: Container::Locked)
 256       c.update_attributes!(state: Container::Running)
 257       c
 258     end
 259
 260     cr.reload
 261     assert_equal "Committed", cr.state
 262
 263     output_pdh = '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
 264     log_pdh = 'fa7aeb5140e2848d39b416daeef4ffc5+45'
 265     act_as_system_user do
 266       c.update_attributes!(state: Container::Complete,
 267                            cost: 1.25,
 268                            output: output_pdh,
 269                            log: log_pdh)
 270     end
 271
 272     cr.reload
 273     assert_equal "Final", cr.state
 274     assert_equal 1.25, cr.cumulative_cost
 275     assert_equal users(:active).uuid, cr.modified_by_user_uuid
 276
 277     assert_not_nil cr.output_uuid
 278     assert_not_nil cr.log_uuid
 279     output = Collection.find_by_uuid cr.output_uuid
 280     assert_equal output_pdh, output.portable_data_hash
 281     assert_equal output.owner_uuid, project.uuid, "Container output should be copied to #{project.uuid}"
 282     assert_not_nil output.modified_at
 283
 284     log = Collection.find_by_uuid cr.log_uuid
 285     assert_equal log.manifest_text, ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar
 286 ./log\\040for\\040container\\040#{cr.container_uuid} 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"
 287
 288     assert_equal log.owner_uuid, project.uuid, "Container log should be copied to #{project.uuid}"
 289   end
 290
 291   # This tests bug report #16144
 292   test "Request is finalized when its container is completed even when log & output don't exist" do
 293     set_user_from_auth :active
 294     project = groups(:private)
 295     cr = create_minimal_req!(owner_uuid: project.uuid,
 296                              priority: 1,
 297                              state: "Committed")
 298     assert_equal users(:active).uuid, cr.modified_by_user_uuid
 299
 300     output_pdh = '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
 301     log_pdh = 'fa7aeb5140e2848d39b416daeef4ffc5+45'
 302
 303     c = act_as_system_user do
 304       c = Container.find_by_uuid(cr.container_uuid)
 305       c.update_attributes!(state: Container::Locked)
 306       c.update_attributes!(state: Container::Running,
 307                            output: output_pdh,
 308                            log: log_pdh)
 309       c
 310     end
 311
 312     cr.reload
 313     assert_equal "Committed", cr.state
 314
 315     act_as_system_user do
 316       Collection.where(portable_data_hash: output_pdh).delete_all
 317       Collection.where(portable_data_hash: log_pdh).delete_all
 318       c.update_attributes!(state: Container::Complete)
 319     end
 320
 321     cr.reload
 322     assert_equal "Final", cr.state
 323   end
 324
 325   # This tests bug report #16144
 326   test "Can destroy CR even if its container doesn't exist" do
 327     set_user_from_auth :active
 328     project = groups(:private)
 329     cr = create_minimal_req!(owner_uuid: project.uuid,
 330                              priority: 1,
 331                              state: "Committed")
 332     assert_equal users(:active).uuid, cr.modified_by_user_uuid
 333
 334     c = act_as_system_user do
 335       c = Container.find_by_uuid(cr.container_uuid)
 336       c.update_attributes!(state: Container::Locked)
 337       c.update_attributes!(state: Container::Running)
 338       c
 339     end
 340
 341     cr.reload
 342     assert_equal "Committed", cr.state
 343
 344     cr_uuid = cr.uuid
 345     act_as_system_user do
 346       Container.find_by_uuid(cr.container_uuid).destroy
 347       cr.destroy
 348     end
 349     assert_nil ContainerRequest.find_by_uuid(cr_uuid)
 350   end
 351
 352   test "Container makes container request, then is cancelled" do
 353     set_user_from_auth :active
 354     cr = create_minimal_req!(priority: 5, state: "Committed", container_count_max: 1)
 355
 356     c = Container.find_by_uuid cr.container_uuid
 357     assert_operator 0, :<, c.priority
 358     lock_and_run(c)
 359
 360     cr2 = with_container_auth(c) do
 361       create_minimal_req!(priority: 10, state: "Committed", container_count_max: 1, command: ["echo", "foo2"])
 362     end
 363     assert_equal c.uuid, cr2.requesting_container_uuid
 364     assert_equal users(:active).uuid, cr2.modified_by_user_uuid
 365
 366     c2 = Container.find_by_uuid cr2.container_uuid
 367     assert_operator 0, :<, c2.priority
 368
 369     act_as_system_user do
 370       c.state = "Cancelled"
 371       c.save!
 372     end
 373
 374     cr.reload
 375     assert_equal "Final", cr.state
 376
 377     cr2.reload
 378     assert_equal 0, cr2.priority
 379     assert_equal users(:active).uuid, cr2.modified_by_user_uuid
 380
 381     c2.reload
 382     assert_equal 0, c2.priority
 383   end
 384
 385   test "child container priority follows same ordering as corresponding top-level ancestors" do
 386     findctr = lambda { |cr| Container.find_by_uuid(cr.container_uuid) }
 387
 388     set_user_from_auth :active
 389
 390     toplevel_crs = [
 391       create_minimal_req!(priority: 5, state: "Committed", environment: {"workflow" => "0"}),
 392       create_minimal_req!(priority: 5, state: "Committed", environment: {"workflow" => "1"}),
 393       create_minimal_req!(priority: 5, state: "Committed", environment: {"workflow" => "2"}),
 394     ]
 395     parents = toplevel_crs.map(&findctr)
 396
 397     children = parents.map do |parent|
 398       lock_and_run(parent)
 399       with_container_auth(parent) do
 400         create_minimal_req!(state: "Committed",
 401                             priority: 1,
 402                             environment: {"child" => parent.environment["workflow"]})
 403       end
 404     end.map(&findctr)
 405
 406     grandchildren = children.reverse.map do |child|
 407       lock_and_run(child)
 408       with_container_auth(child) do
 409         create_minimal_req!(state: "Committed",
 410                             priority: 1,
 411                             environment: {"grandchild" => child.environment["child"]})
 412       end
 413     end.reverse.map(&findctr)
 414
 415     shared_grandchildren = children.map do |child|
 416       with_container_auth(child) do
 417         create_minimal_req!(state: "Committed",
 418                             priority: 1,
 419                             environment: {"grandchild" => "shared"})
 420       end
 421     end.map(&findctr)
 422
 423     assert_equal shared_grandchildren[0].uuid, shared_grandchildren[1].uuid
 424     assert_equal shared_grandchildren[0].uuid, shared_grandchildren[2].uuid
 425     shared_grandchild = shared_grandchildren[0]
 426
 427     set_user_from_auth :active
 428
 429     # parents should be prioritized by submit time.
 430     assert_operator parents[0].priority, :>, parents[1].priority
 431     assert_operator parents[1].priority, :>, parents[2].priority
 432
 433     # children should be prioritized in same order as their respective
 434     # parents.
 435     assert_operator children[0].priority, :>, children[1].priority
 436     assert_operator children[1].priority, :>, children[2].priority
 437
 438     # grandchildren should also be prioritized in the same order,
 439     # despite having been submitted in the opposite order.
 440     assert_operator grandchildren[0].priority, :>, grandchildren[1].priority
 441     assert_operator grandchildren[1].priority, :>, grandchildren[2].priority
 442
 443     # shared grandchild container should be prioritized above
 444     # everything that isn't needed by parents[0], but not above
 445     # earlier-submitted descendants of parents[0]
 446     assert_operator shared_grandchild.priority, :>, grandchildren[1].priority
 447     assert_operator shared_grandchild.priority, :>, children[1].priority
 448     assert_operator shared_grandchild.priority, :>, parents[1].priority
 449     assert_operator shared_grandchild.priority, :<=, grandchildren[0].priority
 450     assert_operator shared_grandchild.priority, :<=, children[0].priority
 451     assert_operator shared_grandchild.priority, :<=, parents[0].priority
 452
 453     # increasing priority of the most recent toplevel container should
 454     # reprioritize all of its descendants (including the shared
 455     # grandchild) above everything else.
 456     toplevel_crs[2].update_attributes!(priority: 72)
 457     (parents + children + grandchildren + [shared_grandchild]).map(&:reload)
 458     assert_operator shared_grandchild.priority, :>, grandchildren[0].priority
 459     assert_operator shared_grandchild.priority, :>, children[0].priority
 460     assert_operator shared_grandchild.priority, :>, parents[0].priority
 461     assert_operator shared_grandchild.priority, :>, grandchildren[1].priority
 462     assert_operator shared_grandchild.priority, :>, children[1].priority
 463     assert_operator shared_grandchild.priority, :>, parents[1].priority
 464     # ...but the shared container should not have higher priority than
 465     # the earlier-submitted descendants of the high-priority workflow.
 466     assert_operator shared_grandchild.priority, :<=, grandchildren[2].priority
 467     assert_operator shared_grandchild.priority, :<=, children[2].priority
 468     assert_operator shared_grandchild.priority, :<=, parents[2].priority
 469   end
 470
 471   [
 472     ['running_container_auth', 'zzzzz-dz642-runningcontainr', 501],
 473     ['active_no_prefs', nil, 0]
 474   ].each do |token, expected, expected_priority|
 475     test "create as #{token} and expect requesting_container_uuid to be #{expected}" do
 476       set_user_from_auth token
 477       cr = create_minimal_req!
 478       assert_not_nil cr.uuid, 'uuid should be set for newly created container_request'
 479       assert_equal expected, cr.requesting_container_uuid
 480       assert_equal expected_priority, cr.priority
 481     end
 482   end
 483
 484   [
 485     ['running_container_auth', 'zzzzz-dz642-runningcontainr', 501],
 486   ].each do |token, expected, expected_priority|
 487     test "create as #{token} with requesting_container_uuid set and expect output to be intermediate" do
 488       set_user_from_auth token
 489       cr = create_minimal_req!
 490       assert_not_nil cr.uuid, 'uuid should be set for newly created container_request'
 491       assert_equal expected, cr.requesting_container_uuid
 492       assert_equal expected_priority, cr.priority
 493
 494       cr.state = ContainerRequest::Committed
 495       cr.save!
 496
 497       run_container(cr)
 498       cr.reload
 499       output = Collection.find_by_uuid(cr.output_uuid)
 500       props = {"type": "intermediate", "container_request": cr.uuid}
 501       assert_equal props.symbolize_keys, output.properties.symbolize_keys
 502     end
 503   end
 504
 505   test "create as container_runtime_token and expect requesting_container_uuid to be zzzzz-dz642-20isqbkl8xwnsao" do
 506     set_user_from_auth :container_runtime_token
 507     Thread.current[:token] = "#{Thread.current[:token]}/zzzzz-dz642-20isqbkl8xwnsao"
 508     cr = ContainerRequest.create(container_image: "img", output_path: "/tmp", command: ["echo", "foo"])
 509     assert_not_nil cr.uuid, 'uuid should be set for newly created container_request'
 510     assert_equal 'zzzzz-dz642-20isqbkl8xwnsao', cr.requesting_container_uuid
 511     assert_equal 1, cr.priority
 512   end
 513
 514   [[{"vcpus" => [2, nil]},
 515     lambda { |resolved| resolved["vcpus"] == 2 }],
 516    [{"vcpus" => [3, 7]},
 517     lambda { |resolved| resolved["vcpus"] == 3 }],
 518    [{"vcpus" => 4},
 519     lambda { |resolved| resolved["vcpus"] == 4 }],
 520    [{"ram" => [1000000000, 2000000000]},
 521     lambda { |resolved| resolved["ram"] == 1000000000 }],
 522    [{"ram" => [1234234234]},
 523     lambda { |resolved| resolved["ram"] == 1234234234 }],
 524   ].each do |rc, okfunc|
 525     test "resolve runtime constraint range #{rc} to values" do
 526       resolved = Container.resolve_runtime_constraints(rc)
 527       assert(okfunc.call(resolved),
 528              "container runtime_constraints was #{resolved.inspect}")
 529     end
 530   end
 531
 532   [[{"/out" => {
 533         "kind" => "collection",
 534         "uuid" => "zzzzz-4zz18-znfnqtbbv4spc3w",
 535         "path" => "/foo"}},
 536     lambda do |resolved|
 537       resolved["/out"] == {
 538         "portable_data_hash" => "1f4b0bc7583c2a7f9102c395f4ffc5e3+45",
 539         "kind" => "collection",
 540         "path" => "/foo",
 541       }
 542     end],
 543    [{"/out" => {
 544         "kind" => "collection",
 545         "uuid" => "zzzzz-4zz18-znfnqtbbv4spc3w",
 546         "portable_data_hash" => "1f4b0bc7583c2a7f9102c395f4ffc5e3+45",
 547         "path" => "/foo"}},
 548     lambda do |resolved|
 549       resolved["/out"] == {
 550         "portable_data_hash" => "1f4b0bc7583c2a7f9102c395f4ffc5e3+45",
 551         "kind" => "collection",
 552         "path" => "/foo",
 553       }
 554     end],
 555    [{"/out" => {
 556       "kind" => "collection",
 557       "portable_data_hash" => "1f4b0bc7583c2a7f9102c395f4ffc5e3+45",
 558       "path" => "/foo"}},
 559     lambda do |resolved|
 560       resolved["/out"] == {
 561         "portable_data_hash" => "1f4b0bc7583c2a7f9102c395f4ffc5e3+45",
 562         "kind" => "collection",
 563         "path" => "/foo",
 564       }
 565     end],
 566     # Empty collection
 567     [{"/out" => {
 568       "kind" => "collection",
 569       "path" => "/foo"}},
 570     lambda do |resolved|
 571       resolved["/out"] == {
 572         "kind" => "collection",
 573         "path" => "/foo",
 574       }
 575     end],
 576   ].each do |mounts, okfunc|
 577     test "resolve mounts #{mounts.inspect} to values" do
 578       set_user_from_auth :active
 579       resolved = Container.resolve_mounts(mounts)
 580       assert(okfunc.call(resolved),
 581              "Container.resolve_mounts returned #{resolved.inspect}")
 582     end
 583   end
 584
 585   test 'mount unreadable collection' do
 586     set_user_from_auth :spectator
 587     m = {
 588       "/foo" => {
 589         "kind" => "collection",
 590         "uuid" => "zzzzz-4zz18-znfnqtbbv4spc3w",
 591         "path" => "/foo",
 592       },
 593     }
 594     assert_raises(ArvadosModel::UnresolvableContainerError) do
 595       Container.resolve_mounts(m)
 596     end
 597   end
 598
 599   test 'mount collection with mismatched UUID and PDH' do
 600     set_user_from_auth :active
 601     m = {
 602       "/foo" => {
 603         "kind" => "collection",
 604         "uuid" => "zzzzz-4zz18-znfnqtbbv4spc3w",
 605         "portable_data_hash" => "fa7aeb5140e2848d39b416daeef4ffc5+45",
 606         "path" => "/foo",
 607       },
 608     }
 609     resolved_mounts = Container.resolve_mounts(m)
 610     assert_equal m['portable_data_hash'], resolved_mounts['portable_data_hash']
 611   end
 612
 613   ['arvados/apitestfixture:latest',
 614    'arvados/apitestfixture',
 615    'd8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678',
 616   ].each do |tag|
 617     test "Container.resolve_container_image(#{tag.inspect})" do
 618       set_user_from_auth :active
 619       resolved = Container.resolve_container_image(tag)
 620       assert_equal resolved, collections(:docker_image).portable_data_hash
 621     end
 622   end
 623
 624   test "Container.resolve_container_image(pdh)" do
 625     set_user_from_auth :active
 626     [[:docker_image, 'v1'], [:docker_image_1_12, 'v2']].each do |coll, ver|
 627       Rails.configuration.Containers.SupportedDockerImageFormats = ConfigLoader.to_OrderedOptions({ver=>{}})
 628       pdh = collections(coll).portable_data_hash
 629       resolved = Container.resolve_container_image(pdh)
 630       assert_equal resolved, pdh
 631     end
 632   end
 633
 634   ['acbd18db4cc2f85cedef654fccc4a4d8+3',
 635    'ENOEXIST',
 636    'arvados/apitestfixture:ENOEXIST',
 637   ].each do |img|
 638     test "container_image_for_container(#{img.inspect}) => 422" do
 639       set_user_from_auth :active
 640       assert_raises(ArvadosModel::UnresolvableContainerError) do
 641         Container.resolve_container_image(img)
 642       end
 643     end
 644   end
 645
 646   test "allow unrecognized container when there are remote_hosts" do
 647     set_user_from_auth :active
 648     Rails.configuration.RemoteClusters = Rails.configuration.RemoteClusters.merge({foooo: ActiveSupport::InheritableOptions.new({Host: "bar.com"})})
 649     Container.resolve_container_image('acbd18db4cc2f85cedef654fccc4a4d8+3')
 650   end
 651
 652   test "migrated docker image" do
 653     Rails.configuration.Containers.SupportedDockerImageFormats = ConfigLoader.to_OrderedOptions({'v2'=>{}})
 654     add_docker19_migration_link
 655
 656     # Test that it returns only v2 images even though request is for v1 image.
 657
 658     set_user_from_auth :active
 659     cr = create_minimal_req!(command: ["true", "1"],
 660                              container_image: collections(:docker_image).portable_data_hash)
 661     assert_equal(Container.resolve_container_image(cr.container_image),
 662                  collections(:docker_image_1_12).portable_data_hash)
 663
 664     cr = create_minimal_req!(command: ["true", "2"],
 665                              container_image: links(:docker_image_collection_tag).name)
 666     assert_equal(Container.resolve_container_image(cr.container_image),
 667                  collections(:docker_image_1_12).portable_data_hash)
 668   end
 669
 670   test "use unmigrated docker image" do
 671     Rails.configuration.Containers.SupportedDockerImageFormats = ConfigLoader.to_OrderedOptions({'v1'=>{}})
 672     add_docker19_migration_link
 673
 674     # Test that it returns only supported v1 images even though there is a
 675     # migration link.
 676
 677     set_user_from_auth :active
 678     cr = create_minimal_req!(command: ["true", "1"],
 679                              container_image: collections(:docker_image).portable_data_hash)
 680     assert_equal(Container.resolve_container_image(cr.container_image),
 681                  collections(:docker_image).portable_data_hash)
 682
 683     cr = create_minimal_req!(command: ["true", "2"],
 684                              container_image: links(:docker_image_collection_tag).name)
 685     assert_equal(Container.resolve_container_image(cr.container_image),
 686                  collections(:docker_image).portable_data_hash)
 687   end
 688
 689   test "incompatible docker image v1" do
 690     Rails.configuration.Containers.SupportedDockerImageFormats = ConfigLoader.to_OrderedOptions({'v1'=>{}})
 691     add_docker19_migration_link
 692
 693     # Don't return unsupported v2 image even if we ask for it directly.
 694     set_user_from_auth :active
 695     cr = create_minimal_req!(command: ["true", "1"],
 696                              container_image: collections(:docker_image_1_12).portable_data_hash)
 697     assert_raises(ArvadosModel::UnresolvableContainerError) do
 698       Container.resolve_container_image(cr.container_image)
 699     end
 700   end
 701
 702   test "incompatible docker image v2" do
 703     Rails.configuration.Containers.SupportedDockerImageFormats = ConfigLoader.to_OrderedOptions({'v2'=>{}})
 704     # No migration link, don't return unsupported v1 image,
 705
 706     set_user_from_auth :active
 707     cr = create_minimal_req!(command: ["true", "1"],
 708                              container_image: collections(:docker_image).portable_data_hash)
 709     assert_raises(ArvadosModel::UnresolvableContainerError) do
 710       Container.resolve_container_image(cr.container_image)
 711     end
 712     cr = create_minimal_req!(command: ["true", "2"],
 713                              container_image: links(:docker_image_collection_tag).name)
 714     assert_raises(ArvadosModel::UnresolvableContainerError) do
 715       Container.resolve_container_image(cr.container_image)
 716     end
 717   end
 718
 719   test "requestor can retrieve container owned by dispatch" do
 720     assert_not_empty Container.readable_by(users(:admin)).where(uuid: containers(:running).uuid)
 721     assert_not_empty Container.readable_by(users(:active)).where(uuid: containers(:running).uuid)
 722     assert_empty Container.readable_by(users(:spectator)).where(uuid: containers(:running).uuid)
 723   end
 724
 725   [
 726     [{"var" => "value1"}, {"var" => "value1"}, nil],
 727     [{"var" => "value1"}, {"var" => "value1"}, true],
 728     [{"var" => "value1"}, {"var" => "value1"}, false],
 729     [{"var" => "value1"}, {"var" => "value2"}, nil],
 730   ].each do |env1, env2, use_existing|
 731     test "Container request #{((env1 == env2) and (use_existing.nil? or use_existing == true)) ? 'does' : 'does not'} reuse container when committed#{use_existing.nil? ? '' : use_existing ? ' and use_existing == true' : ' and use_existing == false'}" do
 732       common_attrs = {cwd: "test",
 733                       priority: 1,
 734                       command: ["echo", "hello"],
 735                       output_path: "test",
 736                       runtime_constraints: {"vcpus" => 4,
 737                                             "ram" => 12000000000},
 738                       mounts: {"test" => {"kind" => "json"}}}
 739       set_user_from_auth :active
 740       cr1 = create_minimal_req!(common_attrs.merge({state: ContainerRequest::Committed,
 741                                                     environment: env1}))
 742       run_container(cr1)
 743       cr1.reload
 744       if use_existing.nil?
 745         # Testing with use_existing default value
 746         cr2 = create_minimal_req!(common_attrs.merge({state: ContainerRequest::Uncommitted,
 747                                                       environment: env2}))
 748       else
 749
 750         cr2 = create_minimal_req!(common_attrs.merge({state: ContainerRequest::Uncommitted,
 751                                                       environment: env2,
 752                                                       use_existing: use_existing}))
 753       end
 754       assert_not_nil cr1.container_uuid
 755       assert_nil cr2.container_uuid
 756
 757       # Update cr2 to commited state and check for container equality on different cases:
 758       # * When env1 and env2 are equal and use_existing is true, the same container
 759       #   should be assigned.
 760       # * When use_existing is false, a different container should be assigned.
 761       # * When env1 and env2 are different, a different container should be assigned.
 762       cr2.update_attributes!({state: ContainerRequest::Committed})
 763       assert_equal (cr2.use_existing == true and (env1 == env2)),
 764                    (cr1.container_uuid == cr2.container_uuid)
 765     end
 766   end
 767
 768   test "requesting_container_uuid at create is not allowed" do
 769     set_user_from_auth :active
 770     assert_raises(ActiveRecord::RecordInvalid) do
 771       create_minimal_req!(state: "Uncommitted", priority: 1, requesting_container_uuid: 'youcantdothat')
 772     end
 773   end
 774
 775   test "Retry on container cancelled" do
 776     set_user_from_auth :active
 777     cr = create_minimal_req!(priority: 1, state: "Committed", container_count_max: 2)
 778     cr2 = create_minimal_req!(priority: 1, state: "Committed", container_count_max: 2, command: ["echo", "baz"])
 779     prev_container_uuid = cr.container_uuid
 780
 781     c = act_as_system_user do
 782       c = Container.find_by_uuid(cr.container_uuid)
 783       c.update_attributes!(state: Container::Locked)
 784       c.update_attributes!(state: Container::Running)
 785       c
 786     end
 787
 788     cr.reload
 789     cr2.reload
 790     assert_equal "Committed", cr.state
 791     assert_equal prev_container_uuid, cr.container_uuid
 792     assert_not_equal cr2.container_uuid, cr.container_uuid
 793     prev_container_uuid = cr.container_uuid
 794
 795     act_as_system_user do
 796       c.update_attributes!(cost: 0.5, subrequests_cost: 1.25)
 797       c.update_attributes!(state: Container::Cancelled)
 798     end
 799
 800     cr.reload
 801     cr2.reload
 802     assert_equal "Committed", cr.state
 803     assert_not_equal prev_container_uuid, cr.container_uuid
 804     assert_not_equal cr2.container_uuid, cr.container_uuid
 805     prev_container_uuid = cr.container_uuid
 806
 807     c = act_as_system_user do
 808       c = Container.find_by_uuid(cr.container_uuid)
 809       c.update_attributes!(state: Container::Locked)
 810       c.update_attributes!(state: Container::Running)
 811       c.update_attributes!(cost: 0.125)
 812       c.update_attributes!(state: Container::Cancelled)
 813       c
 814     end
 815
 816     cr.reload
 817     cr2.reload
 818     assert_equal "Final", cr.state
 819     assert_equal prev_container_uuid, cr.container_uuid
 820     assert_not_equal cr2.container_uuid, cr.container_uuid
 821     assert_equal 1.875, cr.cumulative_cost
 822   end
 823
 824   test "Retry on container cancelled with runtime_token" do
 825     set_user_from_auth :spectator
 826     spec = api_client_authorizations(:active)
 827     cr = create_minimal_req!(priority: 1, state: "Committed",
 828                              runtime_token: spec.token,
 829                              container_count_max: 2)
 830     prev_container_uuid = cr.container_uuid
 831
 832     c = act_as_system_user do
 833       c = Container.find_by_uuid(cr.container_uuid)
 834       assert_equal spec.token, c.runtime_token
 835       c.update_attributes!(state: Container::Locked)
 836       c.update_attributes!(state: Container::Running)
 837       c
 838     end
 839
 840     cr.reload
 841     assert_equal "Committed", cr.state
 842     assert_equal prev_container_uuid, cr.container_uuid
 843     prev_container_uuid = cr.container_uuid
 844
 845     act_as_system_user do
 846       c.update_attributes!(state: Container::Cancelled)
 847     end
 848
 849     cr.reload
 850     assert_equal "Committed", cr.state
 851     assert_not_equal prev_container_uuid, cr.container_uuid
 852     prev_container_uuid = cr.container_uuid
 853
 854     c = act_as_system_user do
 855       c = Container.find_by_uuid(cr.container_uuid)
 856       assert_equal spec.token, c.runtime_token
 857       c.update_attributes!(state: Container::Cancelled)
 858       c
 859     end
 860
 861     cr.reload
 862     assert_equal "Final", cr.state
 863     assert_equal prev_container_uuid, cr.container_uuid
 864   end
 865
 866
 867   test "Retry saves logs from previous attempts" do
 868     set_user_from_auth :active
 869     cr = create_minimal_req!(priority: 1, state: "Committed", container_count_max: 3)
 870
 871     c = act_as_system_user do
 872       c = Container.find_by_uuid(cr.container_uuid)
 873       c.update_attributes!(state: Container::Locked)
 874       c.update_attributes!(state: Container::Running)
 875       c
 876     end
 877
 878     container_uuids = []
 879
 880     [0, 1, 2].each do
 881       cr.reload
 882       assert_equal "Committed", cr.state
 883       container_uuids << cr.container_uuid
 884
 885       c = act_as_system_user do
 886         logc = Collection.new(manifest_text: ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n")
 887         logc.save!
 888         c = Container.find_by_uuid(cr.container_uuid)
 889         c.update_attributes!(state: Container::Cancelled, log: logc.portable_data_hash)
 890         c
 891       end
 892     end
 893
 894     container_uuids.sort!
 895
 896     cr.reload
 897     assert_equal "Final", cr.state
 898     assert_equal 3, cr.container_count
 899     assert_equal ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar
 900 ./log\\040for\\040container\\040#{container_uuids[0]} 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar
 901 ./log\\040for\\040container\\040#{container_uuids[1]} 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar
 902 ./log\\040for\\040container\\040#{container_uuids[2]} 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar
 903 " , Collection.find_by_uuid(cr.log_uuid).manifest_text
 904
 905   end
 906
 907   test "Output collection name setting using output_name with name collision resolution" do
 908     set_user_from_auth :active
 909     output_name = 'unimaginative name'
 910     Collection.create!(name: output_name)
 911
 912     cr = create_minimal_req!(priority: 1,
 913                              state: ContainerRequest::Committed,
 914                              output_name: output_name)
 915     run_container(cr)
 916     cr.reload
 917     assert_equal ContainerRequest::Final, cr.state
 918     output_coll = Collection.find_by_uuid(cr.output_uuid)
 919     # Make sure the resulting output collection name include the original name
 920     # plus the date
 921     assert_not_equal output_name, output_coll.name,
 922                      "more than one collection with the same owner and name"
 923     assert output_coll.name.include?(output_name),
 924            "New name should include original name"
 925     assert_match /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z/, output_coll.name,
 926                  "New name should include ISO8601 date"
 927   end
 928
 929   [[0, :check_output_ttl_0],
 930    [1, :check_output_ttl_1s],
 931    [365*86400, :check_output_ttl_1y],
 932   ].each do |ttl, checker|
 933     test "output_ttl=#{ttl}" do
 934       act_as_user users(:active) do
 935         cr = create_minimal_req!(priority: 1,
 936                                  state: ContainerRequest::Committed,
 937                                  output_name: 'foo',
 938                                  output_ttl: ttl)
 939         run_container(cr)
 940         cr.reload
 941         output = Collection.find_by_uuid(cr.output_uuid)
 942         send(checker, db_current_time, output.trash_at, output.delete_at)
 943       end
 944     end
 945   end
 946
 947   def check_output_ttl_0(now, trash, delete)
 948     assert_nil(trash)
 949     assert_nil(delete)
 950   end
 951
 952   def check_output_ttl_1s(now, trash, delete)
 953     assert_not_nil(trash)
 954     assert_not_nil(delete)
 955     assert_in_delta(trash, now + 1.second, 10)
 956     assert_in_delta(delete, now + Rails.configuration.Collections.BlobSigningTTL, 10)
 957   end
 958
 959   def check_output_ttl_1y(now, trash, delete)
 960     year = (86400*365).second
 961     assert_not_nil(trash)
 962     assert_not_nil(delete)
 963     assert_in_delta(trash, now + year, 10)
 964     assert_in_delta(delete, now + year, 10)
 965   end
 966
 967   def run_container(cr)
 968     act_as_system_user do
 969       logc = Collection.new(owner_uuid: system_user_uuid,
 970                             manifest_text: ". ef772b2f28e2c8ca84de45466ed19ee9+7815 0:0:arv-mount.txt\n")
 971       logc.save!
 972
 973       c = Container.find_by_uuid(cr.container_uuid)
 974       c.update_attributes!(state: Container::Locked)
 975       c.update_attributes!(state: Container::Running)
 976       c.update_attributes!(state: Container::Complete,
 977                            exit_code: 0,
 978                            output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45',
 979                            log: logc.portable_data_hash)
 980       logc.destroy
 981       c
 982     end
 983   end
 984
 985   test "Finalize committed request when reusing a finished container" do
 986     set_user_from_auth :active
 987     cr = create_minimal_req!(priority: 1, state: ContainerRequest::Committed)
 988     cr.reload
 989     assert_equal ContainerRequest::Committed, cr.state
 990     run_container(cr)
 991     cr.reload
 992     assert_equal ContainerRequest::Final, cr.state
 993
 994     cr2 = create_minimal_req!(priority: 1, state: ContainerRequest::Committed)
 995     assert_equal cr.container_uuid, cr2.container_uuid
 996     assert_equal ContainerRequest::Final, cr2.state
 997
 998     cr3 = create_minimal_req!(priority: 1, state: ContainerRequest::Uncommitted)
 999     assert_equal ContainerRequest::Uncommitted, cr3.state
1000     cr3.update_attributes!(state: ContainerRequest::Committed)
1001     assert_equal cr.container_uuid, cr3.container_uuid
1002     assert_equal ContainerRequest::Final, cr3.state
1003   end
1004
1005   [
1006     # client requests preemptible, but types are not configured
1007     [false, false, false, true, ActiveRecord::RecordInvalid],
1008     [true, false, false, true, ActiveRecord::RecordInvalid],
1009     # client requests preemptible, types are configured
1010     [false, true, false, true, true],
1011     [true, true, false, true, true],
1012     # client requests non-preemptible for top-level container
1013     [false, false, false, false, false],
1014     [true, false, false, false, false],
1015     [false, true, false, false, false],
1016     [true, true, false, false, false],
1017     # client requests non-preemptible for child container, preemptible
1018     # is enabled anyway if AlwaysUsePreemptibleInstances and instance types
1019     # are configured.
1020     [false, false, true, false, false],
1021     [true, false, true, false, false],
1022     [false, true, true, false, false],
1023     [true, true, true, false, true],
1024   ].each do |use_preemptible, have_preemptible, is_child, ask, expect|
1025     test "with AlwaysUsePreemptibleInstances=#{use_preemptible} and preemptible types #{have_preemptible ? '' : 'not '}configured, create #{is_child ? 'child' : 'top-level'} container request with preemptible=#{ask} and expect #{expect}" do
1026       Rails.configuration.Containers.AlwaysUsePreemptibleInstances = use_preemptible
1027       if have_preemptible
1028         configure_preemptible_instance_type
1029       end
1030       common_attrs = {
1031         cwd: "test",
1032         priority: 1,
1033         command: ["echo", "hello"],
1034         output_path: "test",
1035         scheduling_parameters: {"preemptible" => ask},
1036         mounts: {"test" => {"kind" => "json"}},
1037       }
1038       set_user_from_auth :active
1039
1040       if is_child
1041         cr = with_container_auth(containers(:running)) do
1042           create_minimal_req!(common_attrs)
1043         end
1044       else
1045         cr = create_minimal_req!(common_attrs)
1046       end
1047
1048       cr.reload
1049       cr.state = ContainerRequest::Committed
1050
1051       if expect == true || expect == false
1052         cr.save!
1053         assert_equal expect, cr.scheduling_parameters["preemptible"]
1054       else
1055         assert_raises(expect) do
1056           cr.save!
1057         end
1058       end
1059     end
1060   end
1061
1062   test "config update does not flip preemptible flag on already-committed container requests" do
1063     parent = containers(:running_container_with_logs)
1064     attrs_p = {
1065       scheduling_parameters: {"preemptible" => true},
1066       "state" => "Committed",
1067       "priority" => 1,
1068     }
1069     attrs_nonp = {
1070       scheduling_parameters: {"preemptible" => false},
1071       "state" => "Committed",
1072       "priority" => 1,
1073     }
1074     expect = {false => [], true => []}
1075
1076     with_container_auth(parent) do
1077       configure_preemptible_instance_type
1078       Rails.configuration.Containers.AlwaysUsePreemptibleInstances = false
1079
1080       expect[true].push create_minimal_req!(attrs_p)
1081       expect[false].push create_minimal_req!(attrs_nonp)
1082
1083       Rails.configuration.Containers.AlwaysUsePreemptibleInstances = true
1084
1085       expect[true].push create_minimal_req!(attrs_p)
1086       expect[true].push create_minimal_req!(attrs_nonp)
1087       commit_later = create_minimal_req!()
1088
1089       Rails.configuration.InstanceTypes = ConfigLoader.to_OrderedOptions({})
1090
1091       expect[false].push create_minimal_req!(attrs_nonp)
1092
1093       # Even though preemptible is not allowed, we should be able to
1094       # commit a CR that was created earlier when preemptible was the
1095       # default.
1096       commit_later.update_attributes!(priority: 1, state: "Committed")
1097       expect[false].push commit_later
1098     end
1099
1100     set_user_from_auth :active
1101     [false, true].each do |pflag|
1102       expect[pflag].each do |cr|
1103         cr.reload
1104         assert_equal pflag, cr.scheduling_parameters['preemptible']
1105       end
1106     end
1107
1108     act_as_system_user do
1109       # Cancelling the parent used to fail while updating the child
1110       # containers' priority, because the child containers' unchanged
1111       # preemptible fields caused validation to fail.
1112       parent.update_attributes!(state: 'Cancelled')
1113
1114       [false, true].each do |pflag|
1115         expect[pflag].each do |cr|
1116           cr.reload
1117           assert_equal 0, cr.priority, "unexpected non-zero priority #{cr.priority} for #{cr.uuid}"
1118         end
1119       end
1120     end
1121   end
1122
1123   [
1124     [{"partitions" => ["fastcpu","vfastcpu", 100]}, ContainerRequest::Committed, ActiveRecord::RecordInvalid],
1125     [{"partitions" => ["fastcpu","vfastcpu", 100]}, ContainerRequest::Uncommitted],
1126     [{"partitions" => "fastcpu"}, ContainerRequest::Committed, ActiveRecord::RecordInvalid],
1127     [{"partitions" => "fastcpu"}, ContainerRequest::Uncommitted],
1128     [{"partitions" => ["fastcpu","vfastcpu"]}, ContainerRequest::Committed],
1129     [{"max_run_time" => "one day"}, ContainerRequest::Committed, ActiveRecord::RecordInvalid],
1130     [{"max_run_time" => "one day"}, ContainerRequest::Uncommitted],
1131     [{"max_run_time" => -1}, ContainerRequest::Committed, ActiveRecord::RecordInvalid],
1132     [{"max_run_time" => -1}, ContainerRequest::Uncommitted],
1133     [{"max_run_time" => 86400}, ContainerRequest::Committed],
1134   ].each do |sp, state, expected|
1135     test "create container request with scheduling_parameters #{sp} in state #{state} and verify #{expected}" do
1136       common_attrs = {cwd: "test",
1137                       priority: 1,
1138                       command: ["echo", "hello"],
1139                       output_path: "test",
1140                       scheduling_parameters: sp,
1141                       mounts: {"test" => {"kind" => "json"}}}
1142       set_user_from_auth :active
1143
1144       if expected == ActiveRecord::RecordInvalid
1145         assert_raises(ActiveRecord::RecordInvalid) do
1146           create_minimal_req!(common_attrs.merge({state: state}))
1147         end
1148       else
1149         cr = create_minimal_req!(common_attrs.merge({state: state}))
1150         assert (sp.to_a - cr.scheduling_parameters.to_a).empty?
1151
1152         if state == ContainerRequest::Committed
1153           c = Container.find_by_uuid(cr.container_uuid)
1154           assert (sp.to_a - c.scheduling_parameters.to_a).empty?
1155         end
1156       end
1157     end
1158   end
1159
1160   test "AlwaysUsePreemptibleInstances makes child containers preemptible" do
1161     Rails.configuration.Containers.AlwaysUsePreemptibleInstances = true
1162     common_attrs = {cwd: "test",
1163                     priority: 1,
1164                     command: ["echo", "hello"],
1165                     output_path: "test",
1166                     state: ContainerRequest::Committed,
1167                     mounts: {"test" => {"kind" => "json"}}}
1168     set_user_from_auth :active
1169     configure_preemptible_instance_type
1170
1171     cr = with_container_auth(Container.find_by_uuid 'zzzzz-dz642-runningcontainr') do
1172       create_minimal_req!(common_attrs)
1173     end
1174     assert_equal 'zzzzz-dz642-runningcontainr', cr.requesting_container_uuid
1175     assert_equal true, cr.scheduling_parameters["preemptible"]
1176
1177     c = Container.find_by_uuid(cr.container_uuid)
1178     assert_equal true, c.scheduling_parameters["preemptible"]
1179   end
1180
1181   [['Committed', true, {name: "foobar", priority: 123}],
1182    ['Committed', false, {container_count: 2}],
1183    ['Committed', false, {container_count: 0}],
1184    ['Committed', false, {container_count: nil}],
1185    ['Committed', true, {priority: 0, mounts: {"/out" => {"kind" => "tmp", "capacity" => 1000000}}}],
1186    ['Committed', true, {priority: 0, mounts: {"/out" => {"capacity" => 1000000, "kind" => "tmp"}}}],
1187    # Addition of default values for mounts / runtime_constraints /
1188    # scheduling_parameters, as happens in a round-trip through
1189    # controller, does not have any real effect and should be
1190    # accepted/ignored rather than causing an error when the CR state
1191    # dictates those attributes are not allowed to change.
1192    ['Committed', true, {priority: 0, mounts: {"/out" => {"capacity" => 0, "kind" => "tmp"}}}, {mounts: {"/out" => {"kind" => "tmp"}}}],
1193    ['Committed', true, {priority: 0, mounts: {"/out" => {"capacity" => 1000000, "kind" => "tmp", "exclude_from_output": false}}}],
1194    ['Committed', true, {priority: 0, mounts: {"/out" => {"capacity" => 1000000, "kind" => "tmp", "repository_name": ""}}}],
1195    ['Committed', true, {priority: 0, mounts: {"/out" => {"capacity" => 1000000, "kind" => "tmp", "content": nil}}}],
1196    ['Committed', false, {priority: 0, mounts: {"/out" => {"capacity" => 1000000, "kind" => "tmp", "content": {}}}}],
1197    ['Committed', false, {priority: 0, mounts: {"/out" => {"capacity" => 1000000, "kind" => "tmp", "repository_name": "foo"}}}],
1198    ['Committed', false, {priority: 0, mounts: {"/out" => {"kind" => "tmp", "capacity" => 1234567}}}],
1199    ['Committed', false, {priority: 0, mounts: {}}],
1200    ['Committed', true, {priority: 0, runtime_constraints: {"vcpus" => 1, "ram" => 2}}],
1201    ['Committed', true, {priority: 0, runtime_constraints: {"vcpus" => 1, "ram" => 2, "keep_cache_ram" => 0}}],
1202    ['Committed', true, {priority: 0, runtime_constraints: {"vcpus" => 1, "ram" => 2, "API" => false}}],
1203    ['Committed', false, {priority: 0, runtime_constraints: {"vcpus" => 1, "ram" => 2, "keep_cache_ram" => 1}}],
1204    ['Committed', false, {priority: 0, runtime_constraints: {"vcpus" => 1, "ram" => 2, "API" => true}}],
1205    ['Committed', true, {priority: 0, scheduling_parameters: {"preemptible" => false}}],
1206    ['Committed', true, {priority: 0, scheduling_parameters: {"partitions" => []}}],
1207    ['Committed', true, {priority: 0, scheduling_parameters: {"max_run_time" => 0}}],
1208    ['Committed', false, {priority: 0, scheduling_parameters: {"preemptible" => true}}],
1209    ['Committed', false, {priority: 0, scheduling_parameters: {"partitions" => ["foo"]}}],
1210    ['Committed', false, {priority: 0, scheduling_parameters: {"max_run_time" => 1}}],
1211    ['Final', false, {state: ContainerRequest::Committed, name: "foobar"}],
1212    ['Final', false, {name: "foobar", priority: 123}],
1213    ['Final', false, {name: "foobar", output_uuid: "zzzzz-4zz18-znfnqtbbv4spc3w"}],
1214    ['Final', false, {name: "foobar", log_uuid: "zzzzz-4zz18-znfnqtbbv4spc3w"}],
1215    ['Final', false, {log_uuid: "zzzzz-4zz18-znfnqtbbv4spc3w"}],
1216    ['Final', false, {priority: 123}],
1217    ['Final', false, {mounts: {}}],
1218    ['Final', false, {container_count: 2}],
1219    ['Final', true, {name: "foobar"}],
1220    ['Final', true, {name: "foobar", description: "baz"}],
1221   ].each do |state, permitted, updates, create_attrs|
1222     test "state=#{state} can#{'not' if !permitted} update #{updates.inspect}" do
1223       act_as_user users(:active) do
1224         attrs = {
1225           priority: 1,
1226           state: "Committed",
1227           container_count_max: 1
1228         }
1229         if !create_attrs.nil?
1230           attrs.merge!(create_attrs)
1231         end
1232         cr = create_minimal_req!(attrs)
1233         case state
1234         when 'Committed'
1235           # already done
1236         when 'Final'
1237           act_as_system_user do
1238             Container.find_by_uuid(cr.container_uuid).
1239               update_attributes!(state: Container::Cancelled)
1240           end
1241           cr.reload
1242         else
1243           raise 'broken test case'
1244         end
1245         assert_equal state, cr.state
1246         if permitted
1247           assert cr.update_attributes!(updates)
1248         else
1249           assert_raises(ActiveRecord::RecordInvalid) do
1250             cr.update_attributes!(updates)
1251           end
1252         end
1253       end
1254     end
1255   end
1256
1257   test "delete container_request and check its container's priority" do
1258     act_as_user users(:active) do
1259       cr = ContainerRequest.find_by_uuid container_requests(:running_to_be_deleted).uuid
1260
1261       # initially the cr's container has priority > 0
1262       c = Container.find_by_uuid(cr.container_uuid)
1263       assert_equal 1, c.priority
1264
1265       cr.destroy
1266
1267       # the cr's container now has priority of 0
1268       c = Container.find_by_uuid(cr.container_uuid)
1269       assert_equal 0, c.priority
1270     end
1271   end
1272
1273   test "delete container_request in final state and expect no error due to before_destroy callback" do
1274     act_as_user users(:active) do
1275       cr = ContainerRequest.find_by_uuid container_requests(:completed).uuid
1276       assert_nothing_raised {cr.destroy}
1277     end
1278   end
1279
1280   test "Container request valid priority" do
1281     set_user_from_auth :active
1282     cr = create_minimal_req!
1283
1284     assert_raises(ActiveRecord::RecordInvalid) do
1285       cr.priority = -1
1286       cr.save!
1287     end
1288
1289     cr.priority = 0
1290     cr.save!
1291
1292     cr.priority = 1
1293     cr.save!
1294
1295     cr.priority = 500
1296     cr.save!
1297
1298     cr.priority = 999
1299     cr.save!
1300
1301     cr.priority = 1000
1302     cr.save!
1303
1304     assert_raises(ActiveRecord::RecordInvalid) do
1305       cr.priority = 1001
1306       cr.save!
1307     end
1308   end
1309
1310   # Note: some of these tests might look redundant because they test
1311   # that out-of-order spellings of hashes are still considered equal
1312   # regardless of whether the existing (container) or new (container
1313   # request) hash needs to be re-ordered.
1314   secrets = {"/foo" => {"kind" => "text", "content" => "xyzzy"}}
1315   same_secrets = {"/foo" => {"content" => "xyzzy", "kind" => "text"}}
1316   different_secrets = {"/foo" => {"kind" => "text", "content" => "something completely different"}}
1317   [
1318     [true, nil, nil],
1319     [true, nil, {}],
1320     [true, {}, nil],
1321     [true, {}, {}],
1322     [true, secrets, same_secrets],
1323     [true, same_secrets, secrets],
1324     [false, nil, secrets],
1325     [false, {}, secrets],
1326     [false, secrets, {}],
1327     [false, secrets, nil],
1328     [false, secrets, different_secrets],
1329   ].each do |expect_reuse, sm1, sm2|
1330     test "container reuse secret_mounts #{sm1.inspect}, #{sm2.inspect}" do
1331       set_user_from_auth :active
1332       cr1 = create_minimal_req!(state: "Committed", priority: 1, secret_mounts: sm1)
1333       cr2 = create_minimal_req!(state: "Committed", priority: 1, secret_mounts: sm2)
1334       assert_not_nil cr1.container_uuid
1335       assert_not_nil cr2.container_uuid
1336       if expect_reuse
1337         assert_equal cr1.container_uuid, cr2.container_uuid
1338       else
1339         assert_not_equal cr1.container_uuid, cr2.container_uuid
1340       end
1341     end
1342   end
1343
1344   test "scrub secret_mounts but reuse container for request with identical secret_mounts" do
1345     set_user_from_auth :active
1346     sm = {'/secret/foo' => {'kind' => 'text', 'content' => secret_string}}
1347     cr1 = create_minimal_req!(state: "Committed", priority: 1, secret_mounts: sm.dup)
1348     run_container(cr1)
1349     cr1.reload
1350
1351     # secret_mounts scrubbed from db
1352     c = Container.where(uuid: cr1.container_uuid).first
1353     assert_equal({}, c.secret_mounts)
1354     assert_equal({}, cr1.secret_mounts)
1355
1356     # can reuse container if secret_mounts match
1357     cr2 = create_minimal_req!(state: "Committed", priority: 1, secret_mounts: sm.dup)
1358     assert_equal cr1.container_uuid, cr2.container_uuid
1359
1360     # don't reuse container if secret_mounts don't match
1361     cr3 = create_minimal_req!(state: "Committed", priority: 1, secret_mounts: {})
1362     assert_not_equal cr1.container_uuid, cr3.container_uuid
1363
1364     assert_no_secrets_logged
1365   end
1366
1367   test "conflicting key in mounts and secret_mounts" do
1368     sm = {'/secret/foo' => {'kind' => 'text', 'content' => secret_string}}
1369     set_user_from_auth :active
1370     cr = create_minimal_req!
1371     assert_equal false, cr.update_attributes(state: "Committed",
1372                                              priority: 1,
1373                                              mounts: cr.mounts.merge(sm),
1374                                              secret_mounts: sm)
1375     assert_equal [:secret_mounts], cr.errors.messages.keys
1376   end
1377
1378   test "using runtime_token" do
1379     set_user_from_auth :spectator
1380     spec = api_client_authorizations(:active)
1381     cr = create_minimal_req!(state: "Committed", runtime_token: spec.token, priority: 1)
1382     cr.save!
1383     c = Container.find_by_uuid cr.container_uuid
1384     lock_and_run c
1385     assert_nil c.auth_uuid
1386     assert_equal c.runtime_token, spec.token
1387
1388     assert_not_nil ApiClientAuthorization.find_by_uuid(spec.uuid)
1389
1390     act_as_system_user do
1391       c.update_attributes!(state: Container::Complete,
1392                            exit_code: 0,
1393                            output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45',
1394                            log: 'fa7aeb5140e2848d39b416daeef4ffc5+45')
1395     end
1396
1397     cr.reload
1398     c.reload
1399     assert_nil cr.runtime_token
1400     assert_nil c.runtime_token
1401   end
1402
1403   test "invalid runtime_token" do
1404     set_user_from_auth :active
1405     spec = api_client_authorizations(:spectator)
1406     assert_raises(ArgumentError) do
1407       cr = create_minimal_req!(state: "Committed", runtime_token: "#{spec.token}xx")
1408       cr.save!
1409     end
1410   end
1411
1412   test "default output_storage_classes" do
1413     saved = Rails.configuration.DefaultStorageClasses
1414     Rails.configuration.DefaultStorageClasses = ["foo"]
1415     begin
1416       act_as_user users(:active) do
1417         cr = create_minimal_req!(priority: 1,
1418                                  state: ContainerRequest::Committed,
1419                                  output_name: 'foo')
1420         run_container(cr)
1421         cr.reload
1422         output = Collection.find_by_uuid(cr.output_uuid)
1423         assert_equal ["foo"], output.storage_classes_desired
1424       end
1425     ensure
1426       Rails.configuration.DefaultStorageClasses = saved
1427     end
1428   end
1429
1430   test "setting output_storage_classes" do
1431     act_as_user users(:active) do
1432       cr = create_minimal_req!(priority: 1,
1433                                state: ContainerRequest::Committed,
1434                                output_name: 'foo',
1435                                output_storage_classes: ["foo_storage_class", "bar_storage_class"])
1436       run_container(cr)
1437       cr.reload
1438       output = Collection.find_by_uuid(cr.output_uuid)
1439       assert_equal ["foo_storage_class", "bar_storage_class"], output.storage_classes_desired
1440       log = Collection.find_by_uuid(cr.log_uuid)
1441       assert_equal ["foo_storage_class", "bar_storage_class"], log.storage_classes_desired
1442     end
1443   end
1444
1445   test "reusing container with different container_request.output_storage_classes" do
1446     common_attrs = {cwd: "test",
1447                     priority: 1,
1448                     command: ["echo", "hello"],
1449                     output_path: "test",
1450                     runtime_constraints: {"vcpus" => 4,
1451                                           "ram" => 12000000000},
1452                     mounts: {"test" => {"kind" => "json"}},
1453                     environment: {"var" => "value1"},
1454                     output_storage_classes: ["foo_storage_class"]}
1455     set_user_from_auth :active
1456     cr1 = create_minimal_req!(common_attrs.merge({state: ContainerRequest::Committed}))
1457     cont1 = run_container(cr1)
1458     cr1.reload
1459
1460     output1 = Collection.find_by_uuid(cr1.output_uuid)
1461
1462     # Testing with use_existing default value
1463     cr2 = create_minimal_req!(common_attrs.merge({state: ContainerRequest::Uncommitted,
1464                                                   output_storage_classes: ["bar_storage_class"]}))
1465
1466     assert_not_nil cr1.container_uuid
1467     assert_nil cr2.container_uuid
1468
1469     # Update cr2 to commited state, check for reuse, then run it
1470     cr2.update_attributes!({state: ContainerRequest::Committed})
1471     assert_equal cr1.container_uuid, cr2.container_uuid
1472
1473     cr2.reload
1474     output2 = Collection.find_by_uuid(cr2.output_uuid)
1475
1476     # the original CR output has the original storage class,
1477     # but the second CR output has the new storage class.
1478     assert_equal ["foo_storage_class"], cont1.output_storage_classes
1479     assert_equal ["foo_storage_class"], output1.storage_classes_desired
1480     assert_equal ["bar_storage_class"], output2.storage_classes_desired
1481   end
1482
1483   [
1484     [{},               {},           {"type": "output"}],
1485     [{"a1": "b1"},     {},           {"type": "output", "a1": "b1"}],
1486     [{},               {"a1": "b1"}, {"type": "output", "a1": "b1"}],
1487     [{"a1": "b1"},     {"a1": "c1"}, {"type": "output", "a1": "b1"}],
1488     [{"a1": "b1"},     {"a2": "c2"}, {"type": "output", "a1": "b1", "a2": "c2"}],
1489     [{"type": "blah"}, {},           {"type": "blah"}],
1490   ].each do |cr_prop, container_prop, expect_prop|
1491     test "setting output_properties #{cr_prop} #{container_prop} on current container" do
1492       act_as_user users(:active) do
1493         cr = create_minimal_req!(priority: 1,
1494                                  state: ContainerRequest::Committed,
1495                                  output_name: 'foo',
1496                                  output_properties: cr_prop)
1497
1498         act_as_system_user do
1499           logc = Collection.new(owner_uuid: system_user_uuid,
1500                                 manifest_text: ". ef772b2f28e2c8ca84de45466ed19ee9+7815 0:0:arv-mount.txt\n")
1501           logc.save!
1502
1503           c = Container.find_by_uuid(cr.container_uuid)
1504           c.update_attributes!(state: Container::Locked)
1505           c.update_attributes!(state: Container::Running)
1506
1507           c.update_attributes!(output_properties: container_prop)
1508
1509           c.update_attributes!(state: Container::Complete,
1510                                exit_code: 0,
1511                                output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45',
1512                                log: logc.portable_data_hash)
1513           logc.destroy
1514         end
1515
1516         cr.reload
1517         expect_prop["container_request"] = cr.uuid
1518         output = Collection.find_by_uuid(cr.output_uuid)
1519         assert_equal expect_prop.symbolize_keys, output.properties.symbolize_keys
1520       end
1521     end
1522   end
1523
1524   test "Cumulative cost includes retried attempts but not reused containers" do
1525     set_user_from_auth :active
1526     cr = create_minimal_req!(priority: 5, state: "Committed", container_count_max: 3)
1527     c = Container.find_by_uuid cr.container_uuid
1528     act_as_system_user do
1529       c.update_attributes!(state: Container::Locked)
1530       c.update_attributes!(state: Container::Running)
1531       c.update_attributes!(state: Container::Cancelled, cost: 3)
1532     end
1533     cr.reload
1534     assert_equal 3, cr.cumulative_cost
1535
1536     c = Container.find_by_uuid cr.container_uuid
1537     lock_and_run c
1538     c.reload
1539     assert_equal 0, c.subrequests_cost
1540
1541     # cr2 is a child/subrequest
1542     cr2 = with_container_auth(c) do
1543       create_minimal_req!(priority: 10, state: "Committed", container_count_max: 1, command: ["echo", "foo2"])
1544     end
1545     assert_equal c.uuid, cr2.requesting_container_uuid
1546     c2 = Container.find_by_uuid cr2.container_uuid
1547     act_as_system_user do
1548       c2.update_attributes!(state: Container::Locked)
1549       c2.update_attributes!(state: Container::Running)
1550       logc = Collection.new(owner_uuid: system_user_uuid,
1551                             manifest_text: ". ef772b2f28e2c8ca84de45466ed19ee9+7815 0:0:arv-mount.txt\n")
1552       logc.save!
1553       c2.update_attributes!(state: Container::Complete,
1554                             exit_code: 0,
1555                             output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45',
1556                             log: logc.portable_data_hash,
1557                             cost: 7)
1558     end
1559     c.reload
1560     assert_equal 7, c.subrequests_cost
1561
1562     # cr3 is an identical child/subrequest, will reuse c2
1563     cr3 = with_container_auth(c) do
1564       create_minimal_req!(priority: 10, state: "Committed", container_count_max: 1, command: ["echo", "foo2"])
1565     end
1566     assert_equal c.uuid, cr3.requesting_container_uuid
1567     c3 = Container.find_by_uuid cr3.container_uuid
1568     assert_equal c2.uuid, c3.uuid
1569     assert_equal Container::Complete, c3.state
1570     c.reload
1571     assert_equal 7, c.subrequests_cost
1572
1573     act_as_system_user do
1574       c.update_attributes!(state: Container::Complete, exit_code: 0, cost: 9)
1575     end
1576
1577     c.reload
1578     assert_equal 7, c.subrequests_cost
1579     cr.reload
1580     assert_equal 3+7+9, cr.cumulative_cost
1581   end
1582
1583 end