Merge branch '14260-runtime-token' refs #14260
[arvados.git] / services / api / test / unit / container_request_test.rb
1 # Copyright (C) The Arvados Authors. All rights reserved.
2 #
3 # SPDX-License-Identifier: AGPL-3.0
4
5 require 'test_helper'
6 require 'helpers/container_test_helper'
7 require 'helpers/docker_migration_helper'
8
9 class ContainerRequestTest < ActiveSupport::TestCase
10   include DockerMigrationHelper
11   include DbCurrentTime
12   include ContainerTestHelper
13
14   def with_container_auth(ctr)
15     auth_was = Thread.current[:api_client_authorization]
16     Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(ctr.auth_uuid)
17     begin
18       yield
19     ensure
20       Thread.current[:api_client_authorization] = auth_was
21     end
22   end
23
24   def lock_and_run(ctr)
25       act_as_system_user do
26         ctr.update_attributes!(state: Container::Locked)
27         ctr.update_attributes!(state: Container::Running)
28       end
29   end
30
31   def create_minimal_req! attrs={}
32     defaults = {
33       command: ["echo", "foo"],
34       container_image: links(:docker_image_collection_tag).name,
35       cwd: "/tmp",
36       environment: {},
37       mounts: {"/out" => {"kind" => "tmp", "capacity" => 1000000}},
38       output_path: "/out",
39       runtime_constraints: {"vcpus" => 1, "ram" => 2},
40       name: "foo",
41       description: "bar",
42     }
43     cr = ContainerRequest.create!(defaults.merge(attrs))
44     cr.reload
45     return cr
46   end
47
48   def check_bogus_states cr
49     [nil, "Flubber"].each do |state|
50       assert_raises(ActiveRecord::RecordInvalid) do
51         cr.state = state
52         cr.save!
53       end
54       cr.reload
55     end
56   end
57
58   test "Container request create" do
59     set_user_from_auth :active
60     cr = create_minimal_req!
61
62     assert_nil cr.container_uuid
63     assert_equal 0, cr.priority
64
65     check_bogus_states cr
66
67     # Ensure we can modify all attributes
68     cr.command = ["echo", "foo3"]
69     cr.container_image = "img3"
70     cr.cwd = "/tmp3"
71     cr.environment = {"BUP" => "BOP"}
72     cr.mounts = {"BAR" => {"kind" => "BAZ"}}
73     cr.output_path = "/tmp4"
74     cr.priority = 2
75     cr.runtime_constraints = {"vcpus" => 4}
76     cr.name = "foo3"
77     cr.description = "bar3"
78     cr.save!
79
80     assert_nil cr.container_uuid
81   end
82
83   [
84     {"runtime_constraints" => {"vcpus" => 1}},
85     {"runtime_constraints" => {"vcpus" => 1, "ram" => nil}},
86     {"runtime_constraints" => {"vcpus" => 0, "ram" => 123}},
87     {"runtime_constraints" => {"vcpus" => "1", "ram" => "123"}},
88     {"mounts" => {"FOO" => "BAR"}},
89     {"mounts" => {"FOO" => {}}},
90     {"mounts" => {"FOO" => {"kind" => "tmp", "capacity" => 42.222}}},
91     {"command" => ["echo", 55]},
92     {"environment" => {"FOO" => 55}}
93   ].each do |value|
94     test "Create with invalid #{value}" do
95       set_user_from_auth :active
96       assert_raises(ActiveRecord::RecordInvalid) do
97         cr = create_minimal_req!({state: "Committed",
98                priority: 1}.merge(value))
99         cr.save!
100       end
101     end
102
103     test "Update with invalid #{value}" do
104       set_user_from_auth :active
105       cr = create_minimal_req!(state: "Uncommitted", priority: 1)
106       cr.save!
107       assert_raises(ActiveRecord::RecordInvalid) do
108         cr = ContainerRequest.find_by_uuid cr.uuid
109         cr.update_attributes!({state: "Committed",
110                                priority: 1}.merge(value))
111       end
112     end
113   end
114
115   test "Update from fixture" do
116     set_user_from_auth :active
117     cr = ContainerRequest.find_by_uuid(container_requests(:running).uuid)
118     cr.update_attributes!(description: "New description")
119     assert_equal "New description", cr.description
120   end
121
122   test "Update with valid runtime constraints" do
123       set_user_from_auth :active
124       cr = create_minimal_req!(state: "Uncommitted", priority: 1)
125       cr.save!
126       cr = ContainerRequest.find_by_uuid cr.uuid
127       cr.update_attributes!(state: "Committed",
128                             runtime_constraints: {"vcpus" => 1, "ram" => 23})
129       assert_not_nil cr.container_uuid
130   end
131
132   test "Container request priority must be non-nil" do
133     set_user_from_auth :active
134     cr = create_minimal_req!
135     cr.priority = nil
136     cr.state = "Committed"
137     assert_raises(ActiveRecord::RecordInvalid) do
138       cr.save!
139     end
140   end
141
142   test "Container request commit" do
143     set_user_from_auth :active
144     cr = create_minimal_req!(runtime_constraints: {"vcpus" => 2, "ram" => 30})
145
146     assert_nil cr.container_uuid
147
148     cr.reload
149     cr.state = "Committed"
150     cr.priority = 1
151     cr.save!
152
153     cr.reload
154
155     assert_equal({"vcpus" => 2, "ram" => 30}, cr.runtime_constraints)
156
157     assert_not_nil cr.container_uuid
158     c = Container.find_by_uuid cr.container_uuid
159     assert_not_nil c
160     assert_equal ["echo", "foo"], c.command
161     assert_equal collections(:docker_image).portable_data_hash, c.container_image
162     assert_equal "/tmp", c.cwd
163     assert_equal({}, c.environment)
164     assert_equal({"/out" => {"kind"=>"tmp", "capacity"=>1000000}}, c.mounts)
165     assert_equal "/out", c.output_path
166     assert_equal({"keep_cache_ram"=>268435456, "vcpus" => 2, "ram" => 30}, c.runtime_constraints)
167     assert_operator 0, :<, c.priority
168
169     assert_raises(ActiveRecord::RecordInvalid) do
170       cr.priority = nil
171       cr.save!
172     end
173
174     cr.priority = 0
175     cr.save!
176
177     cr.reload
178     c.reload
179     assert_equal 0, cr.priority
180     assert_equal 0, c.priority
181   end
182
183   test "Independent container requests" do
184     set_user_from_auth :active
185     cr1 = create_minimal_req!(command: ["foo", "1"], priority: 5, state: "Committed")
186     cr2 = create_minimal_req!(command: ["foo", "2"], priority: 10, state: "Committed")
187
188     c1 = Container.find_by_uuid cr1.container_uuid
189     assert_operator 0, :<, c1.priority
190
191     c2 = Container.find_by_uuid cr2.container_uuid
192     assert_operator c1.priority, :<, c2.priority
193     c2priority_was = c2.priority
194
195     cr1.update_attributes!(priority: 0)
196
197     c1.reload
198     assert_equal 0, c1.priority
199
200     c2.reload
201     assert_equal c2priority_was, c2.priority
202   end
203
204   test "Request is finalized when its container is cancelled" do
205     set_user_from_auth :active
206     cr = create_minimal_req!(priority: 1, state: "Committed", container_count_max: 1)
207     assert_equal users(:active).uuid, cr.modified_by_user_uuid
208
209     act_as_system_user do
210       Container.find_by_uuid(cr.container_uuid).
211         update_attributes!(state: Container::Cancelled)
212     end
213
214     cr.reload
215     assert_equal "Final", cr.state
216     assert_equal users(:active).uuid, cr.modified_by_user_uuid
217   end
218
219   test "Request is finalized when its container is completed" do
220     set_user_from_auth :active
221     project = groups(:private)
222     cr = create_minimal_req!(owner_uuid: project.uuid,
223                              priority: 1,
224                              state: "Committed")
225     assert_equal users(:active).uuid, cr.modified_by_user_uuid
226
227     c = act_as_system_user do
228       c = Container.find_by_uuid(cr.container_uuid)
229       c.update_attributes!(state: Container::Locked)
230       c.update_attributes!(state: Container::Running)
231       c
232     end
233
234     cr.reload
235     assert_equal "Committed", cr.state
236
237     output_pdh = '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
238     log_pdh = 'fa7aeb5140e2848d39b416daeef4ffc5+45'
239     act_as_system_user do
240       c.update_attributes!(state: Container::Complete,
241                            output: output_pdh,
242                            log: log_pdh)
243     end
244
245     cr.reload
246     assert_equal "Final", cr.state
247     assert_equal users(:active).uuid, cr.modified_by_user_uuid
248     ['output', 'log'].each do |out_type|
249       pdh = Container.find_by_uuid(cr.container_uuid).send(out_type)
250       assert_equal(1, Collection.where(portable_data_hash: pdh,
251                                        owner_uuid: project.uuid).count,
252                    "Container #{out_type} should be copied to #{project.uuid}")
253     end
254     assert_not_nil cr.output_uuid
255     assert_not_nil cr.log_uuid
256     output = Collection.find_by_uuid cr.output_uuid
257     assert_equal output_pdh, output.portable_data_hash
258     log = Collection.find_by_uuid cr.log_uuid
259     assert_equal log_pdh, log.portable_data_hash
260   end
261
262   test "Container makes container request, then is cancelled" do
263     set_user_from_auth :active
264     cr = create_minimal_req!(priority: 5, state: "Committed", container_count_max: 1)
265
266     c = Container.find_by_uuid cr.container_uuid
267     assert_operator 0, :<, c.priority
268     lock_and_run(c)
269
270     cr2 = with_container_auth(c) do
271       create_minimal_req!(priority: 10, state: "Committed", container_count_max: 1, command: ["echo", "foo2"])
272     end
273     assert_not_nil cr2.requesting_container_uuid
274     assert_equal users(:active).uuid, cr2.modified_by_user_uuid
275
276     c2 = Container.find_by_uuid cr2.container_uuid
277     assert_operator 0, :<, c2.priority
278
279     act_as_system_user do
280       c.state = "Cancelled"
281       c.save!
282     end
283
284     cr.reload
285     assert_equal "Final", cr.state
286
287     cr2.reload
288     assert_equal 0, cr2.priority
289     assert_equal users(:active).uuid, cr2.modified_by_user_uuid
290
291     c2.reload
292     assert_equal 0, c2.priority
293   end
294
295   test "child container priority follows same ordering as corresponding top-level ancestors" do
296     findctr = lambda { |cr| Container.find_by_uuid(cr.container_uuid) }
297
298     set_user_from_auth :active
299
300     toplevel_crs = [
301       create_minimal_req!(priority: 5, state: "Committed", environment: {"workflow" => "0"}),
302       create_minimal_req!(priority: 5, state: "Committed", environment: {"workflow" => "1"}),
303       create_minimal_req!(priority: 5, state: "Committed", environment: {"workflow" => "2"}),
304     ]
305     parents = toplevel_crs.map(&findctr)
306
307     children = parents.map do |parent|
308       lock_and_run(parent)
309       with_container_auth(parent) do
310         create_minimal_req!(state: "Committed",
311                             priority: 1,
312                             environment: {"child" => parent.environment["workflow"]})
313       end
314     end.map(&findctr)
315
316     grandchildren = children.reverse.map do |child|
317       lock_and_run(child)
318       with_container_auth(child) do
319         create_minimal_req!(state: "Committed",
320                             priority: 1,
321                             environment: {"grandchild" => child.environment["child"]})
322       end
323     end.reverse.map(&findctr)
324
325     shared_grandchildren = children.map do |child|
326       with_container_auth(child) do
327         create_minimal_req!(state: "Committed",
328                             priority: 1,
329                             environment: {"grandchild" => "shared"})
330       end
331     end.map(&findctr)
332
333     assert_equal shared_grandchildren[0].uuid, shared_grandchildren[1].uuid
334     assert_equal shared_grandchildren[0].uuid, shared_grandchildren[2].uuid
335     shared_grandchild = shared_grandchildren[0]
336
337     set_user_from_auth :active
338
339     # parents should be prioritized by submit time.
340     assert_operator parents[0].priority, :>, parents[1].priority
341     assert_operator parents[1].priority, :>, parents[2].priority
342
343     # children should be prioritized in same order as their respective
344     # parents.
345     assert_operator children[0].priority, :>, children[1].priority
346     assert_operator children[1].priority, :>, children[2].priority
347
348     # grandchildren should also be prioritized in the same order,
349     # despite having been submitted in the opposite order.
350     assert_operator grandchildren[0].priority, :>, grandchildren[1].priority
351     assert_operator grandchildren[1].priority, :>, grandchildren[2].priority
352
353     # shared grandchild container should be prioritized above
354     # everything that isn't needed by parents[0], but not above
355     # earlier-submitted descendants of parents[0]
356     assert_operator shared_grandchild.priority, :>, grandchildren[1].priority
357     assert_operator shared_grandchild.priority, :>, children[1].priority
358     assert_operator shared_grandchild.priority, :>, parents[1].priority
359     assert_operator shared_grandchild.priority, :<=, grandchildren[0].priority
360     assert_operator shared_grandchild.priority, :<=, children[0].priority
361     assert_operator shared_grandchild.priority, :<=, parents[0].priority
362
363     # increasing priority of the most recent toplevel container should
364     # reprioritize all of its descendants (including the shared
365     # grandchild) above everything else.
366     toplevel_crs[2].update_attributes!(priority: 72)
367     (parents + children + grandchildren + [shared_grandchild]).map(&:reload)
368     assert_operator shared_grandchild.priority, :>, grandchildren[0].priority
369     assert_operator shared_grandchild.priority, :>, children[0].priority
370     assert_operator shared_grandchild.priority, :>, parents[0].priority
371     assert_operator shared_grandchild.priority, :>, grandchildren[1].priority
372     assert_operator shared_grandchild.priority, :>, children[1].priority
373     assert_operator shared_grandchild.priority, :>, parents[1].priority
374     # ...but the shared container should not have higher priority than
375     # the earlier-submitted descendants of the high-priority workflow.
376     assert_operator shared_grandchild.priority, :<=, grandchildren[2].priority
377     assert_operator shared_grandchild.priority, :<=, children[2].priority
378     assert_operator shared_grandchild.priority, :<=, parents[2].priority
379   end
380
381   [
382     ['running_container_auth', 'zzzzz-dz642-runningcontainr', 501],
383     ['active_no_prefs', nil, 0]
384   ].each do |token, expected, expected_priority|
385     test "create as #{token} and expect requesting_container_uuid to be #{expected}" do
386       set_user_from_auth token
387       cr = ContainerRequest.create(container_image: "img", output_path: "/tmp", command: ["echo", "foo"])
388       assert_not_nil cr.uuid, 'uuid should be set for newly created container_request'
389       assert_equal expected, cr.requesting_container_uuid
390       assert_equal expected_priority, cr.priority
391     end
392   end
393
394   test "create as container_runtime_token and expect requesting_container_uuid to be zzzzz-dz642-20isqbkl8xwnsao" do
395     set_user_from_auth :container_runtime_token
396     Thread.current[:token] = "#{Thread.current[:token]}/zzzzz-dz642-20isqbkl8xwnsao"
397     cr = ContainerRequest.create(container_image: "img", output_path: "/tmp", command: ["echo", "foo"])
398     assert_not_nil cr.uuid, 'uuid should be set for newly created container_request'
399     assert_equal 'zzzzz-dz642-20isqbkl8xwnsao', cr.requesting_container_uuid
400     assert_equal 1, cr.priority
401   end
402
403   [[{"vcpus" => [2, nil]},
404     lambda { |resolved| resolved["vcpus"] == 2 }],
405    [{"vcpus" => [3, 7]},
406     lambda { |resolved| resolved["vcpus"] == 3 }],
407    [{"vcpus" => 4},
408     lambda { |resolved| resolved["vcpus"] == 4 }],
409    [{"ram" => [1000000000, 2000000000]},
410     lambda { |resolved| resolved["ram"] == 1000000000 }],
411    [{"ram" => [1234234234]},
412     lambda { |resolved| resolved["ram"] == 1234234234 }],
413   ].each do |rc, okfunc|
414     test "resolve runtime constraint range #{rc} to values" do
415       resolved = Container.resolve_runtime_constraints(rc)
416       assert(okfunc.call(resolved),
417              "container runtime_constraints was #{resolved.inspect}")
418     end
419   end
420
421   [[{"/out" => {
422         "kind" => "collection",
423         "uuid" => "zzzzz-4zz18-znfnqtbbv4spc3w",
424         "path" => "/foo"}},
425     lambda do |resolved|
426       resolved["/out"] == {
427         "portable_data_hash" => "1f4b0bc7583c2a7f9102c395f4ffc5e3+45",
428         "kind" => "collection",
429         "path" => "/foo",
430       }
431     end],
432    [{"/out" => {
433         "kind" => "collection",
434         "uuid" => "zzzzz-4zz18-znfnqtbbv4spc3w",
435         "portable_data_hash" => "1f4b0bc7583c2a7f9102c395f4ffc5e3+45",
436         "path" => "/foo"}},
437     lambda do |resolved|
438       resolved["/out"] == {
439         "portable_data_hash" => "1f4b0bc7583c2a7f9102c395f4ffc5e3+45",
440         "kind" => "collection",
441         "path" => "/foo",
442       }
443     end],
444   ].each do |mounts, okfunc|
445     test "resolve mounts #{mounts.inspect} to values" do
446       set_user_from_auth :active
447       resolved = Container.resolve_mounts(mounts)
448       assert(okfunc.call(resolved),
449              "Container.resolve_mounts returned #{resolved.inspect}")
450     end
451   end
452
453   test 'mount unreadable collection' do
454     set_user_from_auth :spectator
455     m = {
456       "/foo" => {
457         "kind" => "collection",
458         "uuid" => "zzzzz-4zz18-znfnqtbbv4spc3w",
459         "path" => "/foo",
460       },
461     }
462     assert_raises(ArvadosModel::UnresolvableContainerError) do
463       Container.resolve_mounts(m)
464     end
465   end
466
467   test 'mount collection with mismatched UUID and PDH' do
468     set_user_from_auth :active
469     m = {
470       "/foo" => {
471         "kind" => "collection",
472         "uuid" => "zzzzz-4zz18-znfnqtbbv4spc3w",
473         "portable_data_hash" => "fa7aeb5140e2848d39b416daeef4ffc5+45",
474         "path" => "/foo",
475       },
476     }
477     assert_raises(ArgumentError) do
478       Container.resolve_mounts(m)
479     end
480   end
481
482   ['arvados/apitestfixture:latest',
483    'arvados/apitestfixture',
484    'd8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678',
485   ].each do |tag|
486     test "Container.resolve_container_image(#{tag.inspect})" do
487       set_user_from_auth :active
488       resolved = Container.resolve_container_image(tag)
489       assert_equal resolved, collections(:docker_image).portable_data_hash
490     end
491   end
492
493   test "Container.resolve_container_image(pdh)" do
494     set_user_from_auth :active
495     [[:docker_image, 'v1'], [:docker_image_1_12, 'v2']].each do |coll, ver|
496       Rails.configuration.docker_image_formats = [ver]
497       pdh = collections(coll).portable_data_hash
498       resolved = Container.resolve_container_image(pdh)
499       assert_equal resolved, pdh
500     end
501   end
502
503   ['acbd18db4cc2f85cedef654fccc4a4d8+3',
504    'ENOEXIST',
505    'arvados/apitestfixture:ENOEXIST',
506   ].each do |img|
507     test "container_image_for_container(#{img.inspect}) => 422" do
508       set_user_from_auth :active
509       assert_raises(ArvadosModel::UnresolvableContainerError) do
510         Container.resolve_container_image(img)
511       end
512     end
513   end
514
515   test "migrated docker image" do
516     Rails.configuration.docker_image_formats = ['v2']
517     add_docker19_migration_link
518
519     # Test that it returns only v2 images even though request is for v1 image.
520
521     set_user_from_auth :active
522     cr = create_minimal_req!(command: ["true", "1"],
523                              container_image: collections(:docker_image).portable_data_hash)
524     assert_equal(Container.resolve_container_image(cr.container_image),
525                  collections(:docker_image_1_12).portable_data_hash)
526
527     cr = create_minimal_req!(command: ["true", "2"],
528                              container_image: links(:docker_image_collection_tag).name)
529     assert_equal(Container.resolve_container_image(cr.container_image),
530                  collections(:docker_image_1_12).portable_data_hash)
531   end
532
533   test "use unmigrated docker image" do
534     Rails.configuration.docker_image_formats = ['v1']
535     add_docker19_migration_link
536
537     # Test that it returns only supported v1 images even though there is a
538     # migration link.
539
540     set_user_from_auth :active
541     cr = create_minimal_req!(command: ["true", "1"],
542                              container_image: collections(:docker_image).portable_data_hash)
543     assert_equal(Container.resolve_container_image(cr.container_image),
544                  collections(:docker_image).portable_data_hash)
545
546     cr = create_minimal_req!(command: ["true", "2"],
547                              container_image: links(:docker_image_collection_tag).name)
548     assert_equal(Container.resolve_container_image(cr.container_image),
549                  collections(:docker_image).portable_data_hash)
550   end
551
552   test "incompatible docker image v1" do
553     Rails.configuration.docker_image_formats = ['v1']
554     add_docker19_migration_link
555
556     # Don't return unsupported v2 image even if we ask for it directly.
557     set_user_from_auth :active
558     cr = create_minimal_req!(command: ["true", "1"],
559                              container_image: collections(:docker_image_1_12).portable_data_hash)
560     assert_raises(ArvadosModel::UnresolvableContainerError) do
561       Container.resolve_container_image(cr.container_image)
562     end
563   end
564
565   test "incompatible docker image v2" do
566     Rails.configuration.docker_image_formats = ['v2']
567     # No migration link, don't return unsupported v1 image,
568
569     set_user_from_auth :active
570     cr = create_minimal_req!(command: ["true", "1"],
571                              container_image: collections(:docker_image).portable_data_hash)
572     assert_raises(ArvadosModel::UnresolvableContainerError) do
573       Container.resolve_container_image(cr.container_image)
574     end
575     cr = create_minimal_req!(command: ["true", "2"],
576                              container_image: links(:docker_image_collection_tag).name)
577     assert_raises(ArvadosModel::UnresolvableContainerError) do
578       Container.resolve_container_image(cr.container_image)
579     end
580   end
581
582   test "requestor can retrieve container owned by dispatch" do
583     assert_not_empty Container.readable_by(users(:admin)).where(uuid: containers(:running).uuid)
584     assert_not_empty Container.readable_by(users(:active)).where(uuid: containers(:running).uuid)
585     assert_empty Container.readable_by(users(:spectator)).where(uuid: containers(:running).uuid)
586   end
587
588   [
589     [{"var" => "value1"}, {"var" => "value1"}, nil],
590     [{"var" => "value1"}, {"var" => "value1"}, true],
591     [{"var" => "value1"}, {"var" => "value1"}, false],
592     [{"var" => "value1"}, {"var" => "value2"}, nil],
593   ].each do |env1, env2, use_existing|
594     test "Container request #{((env1 == env2) and (use_existing.nil? or use_existing == true)) ? 'does' : 'does not'} reuse container when committed#{use_existing.nil? ? '' : use_existing ? ' and use_existing == true' : ' and use_existing == false'}" do
595       common_attrs = {cwd: "test",
596                       priority: 1,
597                       command: ["echo", "hello"],
598                       output_path: "test",
599                       runtime_constraints: {"vcpus" => 4,
600                                             "ram" => 12000000000},
601                       mounts: {"test" => {"kind" => "json"}}}
602       set_user_from_auth :active
603       cr1 = create_minimal_req!(common_attrs.merge({state: ContainerRequest::Committed,
604                                                     environment: env1}))
605       if use_existing.nil?
606         # Testing with use_existing default value
607         cr2 = create_minimal_req!(common_attrs.merge({state: ContainerRequest::Uncommitted,
608                                                       environment: env2}))
609       else
610
611         cr2 = create_minimal_req!(common_attrs.merge({state: ContainerRequest::Uncommitted,
612                                                       environment: env2,
613                                                       use_existing: use_existing}))
614       end
615       assert_not_nil cr1.container_uuid
616       assert_nil cr2.container_uuid
617
618       # Update cr2 to commited state and check for container equality on different cases:
619       # * When env1 and env2 are equal and use_existing is true, the same container
620       #   should be assigned.
621       # * When use_existing is false, a different container should be assigned.
622       # * When env1 and env2 are different, a different container should be assigned.
623       cr2.update_attributes!({state: ContainerRequest::Committed})
624       assert_equal (cr2.use_existing == true and (env1 == env2)),
625                    (cr1.container_uuid == cr2.container_uuid)
626     end
627   end
628
629   test "requesting_container_uuid at create is not allowed" do
630     set_user_from_auth :active
631     assert_raises(ActiveRecord::RecordInvalid) do
632       create_minimal_req!(state: "Uncommitted", priority: 1, requesting_container_uuid: 'youcantdothat')
633     end
634   end
635
636   test "Retry on container cancelled" do
637     set_user_from_auth :active
638     cr = create_minimal_req!(priority: 1, state: "Committed", container_count_max: 2)
639     cr2 = create_minimal_req!(priority: 1, state: "Committed", container_count_max: 2, command: ["echo", "baz"])
640     prev_container_uuid = cr.container_uuid
641
642     c = act_as_system_user do
643       c = Container.find_by_uuid(cr.container_uuid)
644       c.update_attributes!(state: Container::Locked)
645       c.update_attributes!(state: Container::Running)
646       c
647     end
648
649     cr.reload
650     cr2.reload
651     assert_equal "Committed", cr.state
652     assert_equal prev_container_uuid, cr.container_uuid
653     assert_not_equal cr2.container_uuid, cr.container_uuid
654     prev_container_uuid = cr.container_uuid
655
656     act_as_system_user do
657       c.update_attributes!(state: Container::Cancelled)
658     end
659
660     cr.reload
661     cr2.reload
662     assert_equal "Committed", cr.state
663     assert_not_equal prev_container_uuid, cr.container_uuid
664     assert_not_equal cr2.container_uuid, cr.container_uuid
665     prev_container_uuid = cr.container_uuid
666
667     c = act_as_system_user do
668       c = Container.find_by_uuid(cr.container_uuid)
669       c.update_attributes!(state: Container::Cancelled)
670       c
671     end
672
673     cr.reload
674     cr2.reload
675     assert_equal "Final", cr.state
676     assert_equal prev_container_uuid, cr.container_uuid
677     assert_not_equal cr2.container_uuid, cr.container_uuid
678   end
679
680   test "Retry on container cancelled with runtime_token" do
681     set_user_from_auth :spectator
682     spec = api_client_authorizations(:active)
683     cr = create_minimal_req!(priority: 1, state: "Committed",
684                              runtime_token: spec.token,
685                              container_count_max: 2)
686     prev_container_uuid = cr.container_uuid
687
688     c = act_as_system_user do
689       c = Container.find_by_uuid(cr.container_uuid)
690       assert_equal spec.token, c.runtime_token
691       c.update_attributes!(state: Container::Locked)
692       c.update_attributes!(state: Container::Running)
693       c
694     end
695
696     cr.reload
697     assert_equal "Committed", cr.state
698     assert_equal prev_container_uuid, cr.container_uuid
699     prev_container_uuid = cr.container_uuid
700
701     act_as_system_user do
702       c.update_attributes!(state: Container::Cancelled)
703     end
704
705     cr.reload
706     assert_equal "Committed", cr.state
707     assert_not_equal prev_container_uuid, cr.container_uuid
708     prev_container_uuid = cr.container_uuid
709
710     c = act_as_system_user do
711       c = Container.find_by_uuid(cr.container_uuid)
712       assert_equal spec.token, c.runtime_token
713       c.update_attributes!(state: Container::Cancelled)
714       c
715     end
716
717     cr.reload
718     assert_equal "Final", cr.state
719     assert_equal prev_container_uuid, cr.container_uuid
720
721   end
722
723   test "Output collection name setting using output_name with name collision resolution" do
724     set_user_from_auth :active
725     output_name = 'unimaginative name'
726     Collection.create!(name: output_name)
727
728     cr = create_minimal_req!(priority: 1,
729                              state: ContainerRequest::Committed,
730                              output_name: output_name)
731     run_container(cr)
732     cr.reload
733     assert_equal ContainerRequest::Final, cr.state
734     output_coll = Collection.find_by_uuid(cr.output_uuid)
735     # Make sure the resulting output collection name include the original name
736     # plus the date
737     assert_not_equal output_name, output_coll.name,
738                      "more than one collection with the same owner and name"
739     assert output_coll.name.include?(output_name),
740            "New name should include original name"
741     assert_match /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z/, output_coll.name,
742                  "New name should include ISO8601 date"
743   end
744
745   [[0, :check_output_ttl_0],
746    [1, :check_output_ttl_1s],
747    [365*86400, :check_output_ttl_1y],
748   ].each do |ttl, checker|
749     test "output_ttl=#{ttl}" do
750       act_as_user users(:active) do
751         cr = create_minimal_req!(priority: 1,
752                                  state: ContainerRequest::Committed,
753                                  output_name: 'foo',
754                                  output_ttl: ttl)
755         run_container(cr)
756         cr.reload
757         output = Collection.find_by_uuid(cr.output_uuid)
758         send(checker, db_current_time, output.trash_at, output.delete_at)
759       end
760     end
761   end
762
763   def check_output_ttl_0(now, trash, delete)
764     assert_nil(trash)
765     assert_nil(delete)
766   end
767
768   def check_output_ttl_1s(now, trash, delete)
769     assert_not_nil(trash)
770     assert_not_nil(delete)
771     assert_in_delta(trash, now + 1.second, 10)
772     assert_in_delta(delete, now + Rails.configuration.blob_signature_ttl.second, 10)
773   end
774
775   def check_output_ttl_1y(now, trash, delete)
776     year = (86400*365).second
777     assert_not_nil(trash)
778     assert_not_nil(delete)
779     assert_in_delta(trash, now + year, 10)
780     assert_in_delta(delete, now + year, 10)
781   end
782
783   def run_container(cr)
784     act_as_system_user do
785       c = Container.find_by_uuid(cr.container_uuid)
786       c.update_attributes!(state: Container::Locked)
787       c.update_attributes!(state: Container::Running)
788       c.update_attributes!(state: Container::Complete,
789                            exit_code: 0,
790                            output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45',
791                            log: 'fa7aeb5140e2848d39b416daeef4ffc5+45')
792       c
793     end
794   end
795
796   test "Finalize committed request when reusing a finished container" do
797     set_user_from_auth :active
798     cr = create_minimal_req!(priority: 1, state: ContainerRequest::Committed)
799     cr.reload
800     assert_equal ContainerRequest::Committed, cr.state
801     run_container(cr)
802     cr.reload
803     assert_equal ContainerRequest::Final, cr.state
804
805     cr2 = create_minimal_req!(priority: 1, state: ContainerRequest::Committed)
806     assert_equal cr.container_uuid, cr2.container_uuid
807     assert_equal ContainerRequest::Final, cr2.state
808
809     cr3 = create_minimal_req!(priority: 1, state: ContainerRequest::Uncommitted)
810     assert_equal ContainerRequest::Uncommitted, cr3.state
811     cr3.update_attributes!(state: ContainerRequest::Committed)
812     assert_equal cr.container_uuid, cr3.container_uuid
813     assert_equal ContainerRequest::Final, cr3.state
814   end
815
816   [
817     [false, ActiveRecord::RecordInvalid],
818     [true, nil],
819   ].each do |preemptible_conf, expected|
820     test "having Rails.configuration.preemptible_instances=#{preemptible_conf}, create preemptible container request and verify #{expected}" do
821       sp = {"preemptible" => true}
822       common_attrs = {cwd: "test",
823                       priority: 1,
824                       command: ["echo", "hello"],
825                       output_path: "test",
826                       scheduling_parameters: sp,
827                       mounts: {"test" => {"kind" => "json"}}}
828       Rails.configuration.preemptible_instances = preemptible_conf
829       set_user_from_auth :active
830
831       cr = create_minimal_req!(common_attrs)
832       cr.state = ContainerRequest::Committed
833
834       if !expected.nil?
835         assert_raises(expected) do
836           cr.save!
837         end
838       else
839         cr.save!
840         assert_equal sp, cr.scheduling_parameters
841       end
842     end
843   end
844
845   [
846     'zzzzz-dz642-runningcontainr',
847     nil,
848   ].each do |requesting_c|
849     test "having preemptible instances active on the API server, a committed #{requesting_c.nil? ? 'non-':''}child CR should not ask for preemptible instance if parameter already set to false" do
850       common_attrs = {cwd: "test",
851                       priority: 1,
852                       command: ["echo", "hello"],
853                       output_path: "test",
854                       scheduling_parameters: {"preemptible" => false},
855                       mounts: {"test" => {"kind" => "json"}}}
856
857       Rails.configuration.preemptible_instances = true
858       set_user_from_auth :active
859
860       if requesting_c
861         cr = with_container_auth(Container.find_by_uuid requesting_c) do
862           create_minimal_req!(common_attrs)
863         end
864         assert_not_nil cr.requesting_container_uuid
865       else
866         cr = create_minimal_req!(common_attrs)
867       end
868
869       cr.state = ContainerRequest::Committed
870       cr.save!
871
872       assert_equal false, cr.scheduling_parameters['preemptible']
873     end
874   end
875
876   [
877     [true, 'zzzzz-dz642-runningcontainr', true],
878     [true, nil, nil],
879     [false, 'zzzzz-dz642-runningcontainr', nil],
880     [false, nil, nil],
881   ].each do |preemptible_conf, requesting_c, schedule_preemptible|
882     test "having Rails.configuration.preemptible_instances=#{preemptible_conf}, #{requesting_c.nil? ? 'non-':''}child CR should #{schedule_preemptible ? '':'not'} ask for preemptible instance by default" do
883       common_attrs = {cwd: "test",
884                       priority: 1,
885                       command: ["echo", "hello"],
886                       output_path: "test",
887                       mounts: {"test" => {"kind" => "json"}}}
888
889       Rails.configuration.preemptible_instances = preemptible_conf
890       set_user_from_auth :active
891
892       if requesting_c
893         cr = with_container_auth(Container.find_by_uuid requesting_c) do
894           create_minimal_req!(common_attrs)
895         end
896         assert_not_nil cr.requesting_container_uuid
897       else
898         cr = create_minimal_req!(common_attrs)
899       end
900
901       cr.state = ContainerRequest::Committed
902       cr.save!
903
904       assert_equal schedule_preemptible, cr.scheduling_parameters['preemptible']
905     end
906   end
907
908   [
909     [{"partitions" => ["fastcpu","vfastcpu", 100]}, ContainerRequest::Committed, ActiveRecord::RecordInvalid],
910     [{"partitions" => ["fastcpu","vfastcpu", 100]}, ContainerRequest::Uncommitted],
911     [{"partitions" => "fastcpu"}, ContainerRequest::Committed, ActiveRecord::RecordInvalid],
912     [{"partitions" => "fastcpu"}, ContainerRequest::Uncommitted],
913     [{"partitions" => ["fastcpu","vfastcpu"]}, ContainerRequest::Committed],
914     [{"max_run_time" => "one day"}, ContainerRequest::Committed, ActiveRecord::RecordInvalid],
915     [{"max_run_time" => "one day"}, ContainerRequest::Uncommitted],
916     [{"max_run_time" => -1}, ContainerRequest::Committed, ActiveRecord::RecordInvalid],
917     [{"max_run_time" => -1}, ContainerRequest::Uncommitted],
918     [{"max_run_time" => 86400}, ContainerRequest::Committed],
919   ].each do |sp, state, expected|
920     test "create container request with scheduling_parameters #{sp} in state #{state} and verify #{expected}" do
921       common_attrs = {cwd: "test",
922                       priority: 1,
923                       command: ["echo", "hello"],
924                       output_path: "test",
925                       scheduling_parameters: sp,
926                       mounts: {"test" => {"kind" => "json"}}}
927       set_user_from_auth :active
928
929       if expected == ActiveRecord::RecordInvalid
930         assert_raises(ActiveRecord::RecordInvalid) do
931           create_minimal_req!(common_attrs.merge({state: state}))
932         end
933       else
934         cr = create_minimal_req!(common_attrs.merge({state: state}))
935         assert_equal sp, cr.scheduling_parameters
936
937         if state == ContainerRequest::Committed
938           c = Container.find_by_uuid(cr.container_uuid)
939           assert_equal sp, c.scheduling_parameters
940         end
941       end
942     end
943   end
944
945   test "Having preemptible_instances=true create a committed child container request and verify the scheduling parameter of its container" do
946     common_attrs = {cwd: "test",
947                     priority: 1,
948                     command: ["echo", "hello"],
949                     output_path: "test",
950                     state: ContainerRequest::Committed,
951                     mounts: {"test" => {"kind" => "json"}}}
952     set_user_from_auth :active
953     Rails.configuration.preemptible_instances = true
954
955     cr = with_container_auth(Container.find_by_uuid 'zzzzz-dz642-runningcontainr') do
956       create_minimal_req!(common_attrs)
957     end
958     assert_equal 'zzzzz-dz642-runningcontainr', cr.requesting_container_uuid
959     assert_equal true, cr.scheduling_parameters["preemptible"]
960
961     c = Container.find_by_uuid(cr.container_uuid)
962     assert_equal true, c.scheduling_parameters["preemptible"]
963   end
964
965   [['Committed', true, {name: "foobar", priority: 123}],
966    ['Committed', false, {container_count: 2}],
967    ['Committed', false, {container_count: 0}],
968    ['Committed', false, {container_count: nil}],
969    ['Final', false, {state: ContainerRequest::Committed, name: "foobar"}],
970    ['Final', false, {name: "foobar", priority: 123}],
971    ['Final', false, {name: "foobar", output_uuid: "zzzzz-4zz18-znfnqtbbv4spc3w"}],
972    ['Final', false, {name: "foobar", log_uuid: "zzzzz-4zz18-znfnqtbbv4spc3w"}],
973    ['Final', false, {log_uuid: "zzzzz-4zz18-znfnqtbbv4spc3w"}],
974    ['Final', false, {priority: 123}],
975    ['Final', false, {mounts: {}}],
976    ['Final', false, {container_count: 2}],
977    ['Final', true, {name: "foobar"}],
978    ['Final', true, {name: "foobar", description: "baz"}],
979   ].each do |state, permitted, updates|
980     test "state=#{state} can#{'not' if !permitted} update #{updates.inspect}" do
981       act_as_user users(:active) do
982         cr = create_minimal_req!(priority: 1,
983                                  state: "Committed",
984                                  container_count_max: 1)
985         case state
986         when 'Committed'
987           # already done
988         when 'Final'
989           act_as_system_user do
990             Container.find_by_uuid(cr.container_uuid).
991               update_attributes!(state: Container::Cancelled)
992           end
993           cr.reload
994         else
995           raise 'broken test case'
996         end
997         assert_equal state, cr.state
998         if permitted
999           assert cr.update_attributes!(updates)
1000         else
1001           assert_raises(ActiveRecord::RecordInvalid) do
1002             cr.update_attributes!(updates)
1003           end
1004         end
1005       end
1006     end
1007   end
1008
1009   test "delete container_request and check its container's priority" do
1010     act_as_user users(:active) do
1011       cr = ContainerRequest.find_by_uuid container_requests(:running_to_be_deleted).uuid
1012
1013       # initially the cr's container has priority > 0
1014       c = Container.find_by_uuid(cr.container_uuid)
1015       assert_equal 1, c.priority
1016
1017       cr.destroy
1018
1019       # the cr's container now has priority of 0
1020       c = Container.find_by_uuid(cr.container_uuid)
1021       assert_equal 0, c.priority
1022     end
1023   end
1024
1025   test "delete container_request in final state and expect no error due to before_destroy callback" do
1026     act_as_user users(:active) do
1027       cr = ContainerRequest.find_by_uuid container_requests(:completed).uuid
1028       assert_nothing_raised {cr.destroy}
1029     end
1030   end
1031
1032   test "Container request valid priority" do
1033     set_user_from_auth :active
1034     cr = create_minimal_req!
1035
1036     assert_raises(ActiveRecord::RecordInvalid) do
1037       cr.priority = -1
1038       cr.save!
1039     end
1040
1041     cr.priority = 0
1042     cr.save!
1043
1044     cr.priority = 1
1045     cr.save!
1046
1047     cr.priority = 500
1048     cr.save!
1049
1050     cr.priority = 999
1051     cr.save!
1052
1053     cr.priority = 1000
1054     cr.save!
1055
1056     assert_raises(ActiveRecord::RecordInvalid) do
1057       cr.priority = 1001
1058       cr.save!
1059     end
1060   end
1061
1062   # Note: some of these tests might look redundant because they test
1063   # that out-of-order spellings of hashes are still considered equal
1064   # regardless of whether the existing (container) or new (container
1065   # request) hash needs to be re-ordered.
1066   secrets = {"/foo" => {"kind" => "text", "content" => "xyzzy"}}
1067   same_secrets = {"/foo" => {"content" => "xyzzy", "kind" => "text"}}
1068   different_secrets = {"/foo" => {"kind" => "text", "content" => "something completely different"}}
1069   [
1070     [true, nil, nil],
1071     [true, nil, {}],
1072     [true, {}, nil],
1073     [true, {}, {}],
1074     [true, secrets, same_secrets],
1075     [true, same_secrets, secrets],
1076     [false, nil, secrets],
1077     [false, {}, secrets],
1078     [false, secrets, {}],
1079     [false, secrets, nil],
1080     [false, secrets, different_secrets],
1081   ].each do |expect_reuse, sm1, sm2|
1082     test "container reuse secret_mounts #{sm1.inspect}, #{sm2.inspect}" do
1083       set_user_from_auth :active
1084       cr1 = create_minimal_req!(state: "Committed", priority: 1, secret_mounts: sm1)
1085       cr2 = create_minimal_req!(state: "Committed", priority: 1, secret_mounts: sm2)
1086       assert_not_nil cr1.container_uuid
1087       assert_not_nil cr2.container_uuid
1088       if expect_reuse
1089         assert_equal cr1.container_uuid, cr2.container_uuid
1090       else
1091         assert_not_equal cr1.container_uuid, cr2.container_uuid
1092       end
1093     end
1094   end
1095
1096   test "scrub secret_mounts but reuse container for request with identical secret_mounts" do
1097     set_user_from_auth :active
1098     sm = {'/secret/foo' => {'kind' => 'text', 'content' => secret_string}}
1099     cr1 = create_minimal_req!(state: "Committed", priority: 1, secret_mounts: sm.dup)
1100     run_container(cr1)
1101     cr1.reload
1102
1103     # secret_mounts scrubbed from db
1104     c = Container.where(uuid: cr1.container_uuid).first
1105     assert_equal({}, c.secret_mounts)
1106     assert_equal({}, cr1.secret_mounts)
1107
1108     # can reuse container if secret_mounts match
1109     cr2 = create_minimal_req!(state: "Committed", priority: 1, secret_mounts: sm.dup)
1110     assert_equal cr1.container_uuid, cr2.container_uuid
1111
1112     # don't reuse container if secret_mounts don't match
1113     cr3 = create_minimal_req!(state: "Committed", priority: 1, secret_mounts: {})
1114     assert_not_equal cr1.container_uuid, cr3.container_uuid
1115
1116     assert_no_secrets_logged
1117   end
1118
1119   test "conflicting key in mounts and secret_mounts" do
1120     sm = {'/secret/foo' => {'kind' => 'text', 'content' => secret_string}}
1121     set_user_from_auth :active
1122     cr = create_minimal_req!
1123     assert_equal false, cr.update_attributes(state: "Committed",
1124                                              priority: 1,
1125                                              mounts: cr.mounts.merge(sm),
1126                                              secret_mounts: sm)
1127     assert_equal [:secret_mounts], cr.errors.messages.keys
1128   end
1129
1130   test "using runtime_token" do
1131     set_user_from_auth :spectator
1132     spec = api_client_authorizations(:active)
1133     cr = create_minimal_req!(state: "Committed", runtime_token: spec.token, priority: 1)
1134     cr.save!
1135     c = Container.find_by_uuid cr.container_uuid
1136     lock_and_run c
1137     assert_nil c.auth_uuid
1138     assert_equal c.runtime_token, spec.token
1139
1140     assert_not_nil ApiClientAuthorization.find_by_uuid(spec.uuid)
1141
1142     act_as_system_user do
1143       c.update_attributes!(state: Container::Complete,
1144                            exit_code: 0,
1145                            output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45',
1146                            log: 'fa7aeb5140e2848d39b416daeef4ffc5+45')
1147     end
1148
1149     cr.reload
1150     c.reload
1151     assert_nil cr.runtime_token
1152     assert_nil c.runtime_token
1153   end
1154
1155   test "invalid runtime_token" do
1156     set_user_from_auth :active
1157     spec = api_client_authorizations(:spectator)
1158     assert_raises(ArgumentError) do
1159       cr = create_minimal_req!(state: "Committed", runtime_token: "#{spec.token}xx")
1160       cr.save!
1161     end
1162   end
1163 end