2 require 'helpers/git_test_helper'
4 class Arvados::V1::JobReuseControllerTest < ActionController::TestCase
5 fixtures :repositories, :users, :jobs, :links, :collections
7 # See git_setup.rb for the commit log for test.git.tar
11 @controller = Arvados::V1::JobsController.new
12 authorize_with :active
15 test "reuse job with no_reuse=false" do
19 script_version: "4fe459abe02d9b365932b8f5dc419439ab4e2577",
20 repository: "active/foo",
23 input: 'fa7aeb5140e2848d39b416daeef4ffc5+45'
26 assert_response :success
27 assert_not_nil assigns(:object)
28 new_job = JSON.parse(@response.body)
29 assert_equal 'zzzzz-8i9sb-cjs4pklxxjykqqq', new_job['uuid']
30 assert_equal '4fe459abe02d9b365932b8f5dc419439ab4e2577', new_job['script_version']
33 test "reuse job with find_or_create=true" do
37 script_version: "4fe459abe02d9b365932b8f5dc419439ab4e2577",
38 repository: "active/foo",
40 input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
46 assert_response :success
47 assert_not_nil assigns(:object)
48 new_job = JSON.parse(@response.body)
49 assert_equal 'zzzzz-8i9sb-cjs4pklxxjykqqq', new_job['uuid']
50 assert_equal '4fe459abe02d9b365932b8f5dc419439ab4e2577', new_job['script_version']
53 test "no reuse job with null log" do
57 script_version: "4fe459abe02d9b365932b8f5dc419439ab4e2577",
58 repository: "active/foo",
60 input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
66 assert_response :success
67 assert_not_nil assigns(:object)
68 new_job = JSON.parse(@response.body)
69 assert_not_equal 'zzzzz-8i9sb-cjs4pklxxjykqq3', new_job['uuid']
70 assert_equal '4fe459abe02d9b365932b8f5dc419439ab4e2577', new_job['script_version']
73 test "reuse job with symbolic script_version" do
77 script_version: "tag1",
78 repository: "active/foo",
80 input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
86 assert_response :success
87 assert_not_nil assigns(:object)
88 new_job = JSON.parse(@response.body)
89 assert_equal 'zzzzz-8i9sb-cjs4pklxxjykqqq', new_job['uuid']
90 assert_equal '4fe459abe02d9b365932b8f5dc419439ab4e2577', new_job['script_version']
93 test "do not reuse job because no_reuse=true" do
98 script_version: "4fe459abe02d9b365932b8f5dc419439ab4e2577",
99 repository: "active/foo",
101 input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
106 assert_response :success
107 assert_not_nil assigns(:object)
108 new_job = JSON.parse(@response.body)
109 assert_not_equal 'zzzzz-8i9sb-cjs4pklxxjykqqq', new_job['uuid']
110 assert_equal '4fe459abe02d9b365932b8f5dc419439ab4e2577', new_job['script_version']
113 [false, "false"].each do |whichfalse|
114 test "do not reuse job because find_or_create=#{whichfalse.inspect}" do
118 script_version: "4fe459abe02d9b365932b8f5dc419439ab4e2577",
119 repository: "active/foo",
121 input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
125 find_or_create: whichfalse
127 assert_response :success
128 assert_not_nil assigns(:object)
129 new_job = JSON.parse(@response.body)
130 assert_not_equal 'zzzzz-8i9sb-cjs4pklxxjykqqq', new_job['uuid']
131 assert_equal '4fe459abe02d9b365932b8f5dc419439ab4e2577', new_job['script_version']
135 test "do not reuse job because output is not readable by user" do
136 authorize_with :job_reader
140 script_version: "4fe459abe02d9b365932b8f5dc419439ab4e2577",
141 repository: "active/foo",
143 input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
149 assert_response :success
150 assert_not_nil assigns(:object)
151 new_job = JSON.parse(@response.body)
152 assert_not_equal 'zzzzz-8i9sb-cjs4pklxxjykqqq', new_job['uuid']
153 assert_equal '4fe459abe02d9b365932b8f5dc419439ab4e2577', new_job['script_version']
156 test "test_cannot_reuse_job_no_output" do
160 script_version: "4fe459abe02d9b365932b8f5dc419439ab4e2577",
161 repository: "active/foo",
163 input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
167 assert_response :success
168 assert_not_nil assigns(:object)
169 new_job = JSON.parse(@response.body)
170 assert_not_equal 'zzzzz-8i9sb-cjs4pklxxjykppp', new_job['uuid']
173 test "test_reuse_job_range" do
177 minimum_script_version: "tag1",
178 script_version: "master",
179 repository: "active/foo",
181 input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
185 assert_response :success
186 assert_not_nil assigns(:object)
187 new_job = JSON.parse(@response.body)
188 assert_equal 'zzzzz-8i9sb-cjs4pklxxjykqqq', new_job['uuid']
189 assert_equal '4fe459abe02d9b365932b8f5dc419439ab4e2577', new_job['script_version']
192 test "cannot_reuse_job_no_minimum_given_so_must_use_specified_commit" do
196 script_version: "master",
197 repository: "active/foo",
199 input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
203 assert_response :success
204 assert_not_nil assigns(:object)
205 new_job = JSON.parse(@response.body)
206 assert_not_equal 'zzzzz-8i9sb-cjs4pklxxjykqqq', new_job['uuid']
207 assert_equal '077ba2ad3ea24a929091a9e6ce545c93199b8e57', new_job['script_version']
210 test "test_cannot_reuse_job_different_input" do
214 script_version: "4fe459abe02d9b365932b8f5dc419439ab4e2577",
215 repository: "active/foo",
217 input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
221 assert_response :success
222 assert_not_nil assigns(:object)
223 new_job = JSON.parse(@response.body)
224 assert_not_equal 'zzzzz-8i9sb-cjs4pklxxjykqqq', new_job['uuid']
225 assert_equal '4fe459abe02d9b365932b8f5dc419439ab4e2577', new_job['script_version']
228 test "test_cannot_reuse_job_different_version" do
232 script_version: "master",
233 repository: "active/foo",
235 input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
239 assert_response :success
240 assert_not_nil assigns(:object)
241 new_job = JSON.parse(@response.body)
242 assert_not_equal 'zzzzz-8i9sb-cjs4pklxxjykqqq', new_job['uuid']
243 assert_equal '077ba2ad3ea24a929091a9e6ce545c93199b8e57', new_job['script_version']
246 test "test_can_reuse_job_submitted_nondeterministic" do
250 script_version: "4fe459abe02d9b365932b8f5dc419439ab4e2577",
251 repository: "active/foo",
253 input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
256 nondeterministic: true
258 assert_response :success
259 assert_not_nil assigns(:object)
260 new_job = JSON.parse(@response.body)
261 assert_equal 'zzzzz-8i9sb-cjs4pklxxjykqqq', new_job['uuid']
262 assert_equal '4fe459abe02d9b365932b8f5dc419439ab4e2577', new_job['script_version']
265 test "test_cannot_reuse_job_past_nondeterministic" do
269 script_version: "4fe459abe02d9b365932b8f5dc419439ab4e2577",
270 repository: "active/foo",
272 input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
276 assert_response :success
277 assert_not_nil assigns(:object)
278 new_job = JSON.parse(@response.body)
279 assert_not_equal 'zzzzz-8i9sb-cjs4pklxxjykyyy', new_job['uuid']
280 assert_equal '4fe459abe02d9b365932b8f5dc419439ab4e2577', new_job['script_version']
283 test "test_cannot_reuse_job_no_permission" do
284 authorize_with :spectator
288 script_version: "4fe459abe02d9b365932b8f5dc419439ab4e2577",
289 repository: "active/foo",
291 input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
295 assert_response :success
296 assert_not_nil assigns(:object)
297 new_job = JSON.parse(@response.body)
298 assert_not_equal 'zzzzz-8i9sb-cjs4pklxxjykqqq', new_job['uuid']
299 assert_equal '4fe459abe02d9b365932b8f5dc419439ab4e2577', new_job['script_version']
302 test "test_cannot_reuse_job_excluded" do
306 minimum_script_version: "31ce37fe365b3dc204300a3e4c396ad333ed0556",
307 script_version: "master",
308 repository: "active/foo",
309 exclude_script_versions: ["tag1"],
311 input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
315 assert_response :success
316 assert_not_nil assigns(:object)
317 new_job = JSON.parse(@response.body)
318 assert_not_equal 'zzzzz-8i9sb-cjs4pklxxjykqqq', new_job['uuid']
319 assert_not_equal('4fe459abe02d9b365932b8f5dc419439ab4e2577',
320 new_job['script_version'])
323 test "cannot reuse job with find_or_create but excluded version" do
327 script_version: "master",
328 repository: "active/foo",
330 input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
334 find_or_create: true,
335 minimum_script_version: "31ce37fe365b3dc204300a3e4c396ad333ed0556",
336 exclude_script_versions: ["tag1"],
338 assert_response :success
339 assert_not_nil assigns(:object)
340 new_job = JSON.parse(@response.body)
341 assert_not_equal 'zzzzz-8i9sb-cjs4pklxxjykqqq', new_job['uuid']
342 assert_not_equal('4fe459abe02d9b365932b8f5dc419439ab4e2577',
343 new_job['script_version'])
346 test "cannot reuse job when hash-like branch includes newer commit" do
347 check_new_job_created_from({job: {script_version: "738783"}},
348 :previous_job_run_superseded_by_hash_branch)
352 'repository' => ['=', 'active/foo'],
353 'script' => ['=', 'hash'],
354 'script_version' => ['in git', 'master'],
355 'docker_image_locator' => ['=', nil],
356 'arvados_sdk_version' => ['=', nil],
359 def filters_from_hash(hash)
360 hash.each_pair.map { |name, filter| [name] + filter }
363 test "can reuse a Job based on filters" do
364 filters_hash = BASE_FILTERS.
365 merge('script_version' => ['in git', 'tag1'])
369 script_version: "master",
370 repository: "active/foo",
372 input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
376 filters: filters_from_hash(filters_hash),
377 find_or_create: true,
379 assert_response :success
380 assert_not_nil assigns(:object)
381 new_job = JSON.parse(@response.body)
382 assert_equal 'zzzzz-8i9sb-cjs4pklxxjykqqq', new_job['uuid']
383 assert_equal '4fe459abe02d9b365932b8f5dc419439ab4e2577', new_job['script_version']
386 test "can not reuse a Job based on filters" do
387 filters = filters_from_hash(BASE_FILTERS
388 .reject { |k| k == 'script_version' })
389 filters += [["script_version", "in git",
390 "31ce37fe365b3dc204300a3e4c396ad333ed0556"],
391 ["script_version", "not in git", ["tag1"]]]
395 script_version: "master",
396 repository: "active/foo",
398 input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
403 find_or_create: true,
405 assert_response :success
406 assert_not_nil assigns(:object)
407 new_job = JSON.parse(@response.body)
408 assert_not_equal 'zzzzz-8i9sb-cjs4pklxxjykqqq', new_job['uuid']
409 assert_equal '077ba2ad3ea24a929091a9e6ce545c93199b8e57', new_job['script_version']
412 test "can not reuse a Job based on arbitrary filters" do
413 filters_hash = BASE_FILTERS.
414 merge("created_at" => ["<", "2010-01-01T00:00:00Z"])
418 script_version: "4fe459abe02d9b365932b8f5dc419439ab4e2577",
419 repository: "active/foo",
421 input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
425 filters: filters_from_hash(filters_hash),
426 find_or_create: true,
428 assert_response :success
429 assert_not_nil assigns(:object)
430 new_job = JSON.parse(@response.body)
431 assert_not_equal 'zzzzz-8i9sb-cjs4pklxxjykqqq', new_job['uuid']
432 assert_equal '4fe459abe02d9b365932b8f5dc419439ab4e2577', new_job['script_version']
435 test "can reuse a Job with a Docker image" do
439 script_version: "4fe459abe02d9b365932b8f5dc419439ab4e2577",
440 repository: "active/foo",
442 input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
445 runtime_constraints: {
446 docker_image: 'arvados/apitestfixture',
449 find_or_create: true,
451 assert_response :success
452 new_job = assigns(:object)
453 assert_not_nil new_job
454 target_job = jobs(:previous_docker_job_run)
455 [:uuid, :script_version, :docker_image_locator].each do |attr|
456 assert_equal(target_job.send(attr), new_job.send(attr))
460 test "can reuse a Job with a Docker image hash filter" do
461 filters_hash = BASE_FILTERS.
462 merge("script_version" =>
463 ["=", "4fe459abe02d9b365932b8f5dc419439ab4e2577"],
464 "docker_image_locator" =>
465 ["in docker", links(:docker_image_collection_hash).name])
469 script_version: "4fe459abe02d9b365932b8f5dc419439ab4e2577",
470 repository: "active/foo",
472 input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
476 filters: filters_from_hash(filters_hash),
477 find_or_create: true,
479 assert_response :success
480 new_job = assigns(:object)
481 assert_not_nil new_job
482 target_job = jobs(:previous_docker_job_run)
483 [:uuid, :script_version, :docker_image_locator].each do |attr|
484 assert_equal(target_job.send(attr), new_job.send(attr))
488 test "reuse Job with Docker image repo+tag" do
489 filters_hash = BASE_FILTERS.
490 merge("script_version" =>
491 ["=", "4fe459abe02d9b365932b8f5dc419439ab4e2577"],
492 "docker_image_locator" =>
493 ["in docker", links(:docker_image_collection_tag2).name])
497 script_version: "4fe459abe02d9b365932b8f5dc419439ab4e2577",
498 repository: "active/foo",
500 input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
504 filters: filters_from_hash(filters_hash),
505 find_or_create: true,
507 assert_response :success
508 new_job = assigns(:object)
509 assert_not_nil new_job
510 target_job = jobs(:previous_docker_job_run)
511 [:uuid, :script_version, :docker_image_locator].each do |attr|
512 assert_equal(target_job.send(attr), new_job.send(attr))
516 test "new job with unknown Docker image filter" do
517 filters_hash = BASE_FILTERS.
518 merge("docker_image_locator" => ["in docker", "_nonesuchname_"])
522 script_version: "4fe459abe02d9b365932b8f5dc419439ab4e2577",
523 repository: "active/foo",
525 input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
529 filters: filters_from_hash(filters_hash),
530 find_or_create: true,
532 assert_response :success
533 new_job = assigns(:object)
534 assert_not_nil new_job
535 assert_not_equal(jobs(:previous_docker_job_run).uuid, new_job.uuid)
538 test "don't reuse job using older Docker image of same name" do
539 jobspec = {runtime_constraints: {
540 docker_image: "arvados/apitestfixture",
542 check_new_job_created_from({job: jobspec},
543 :previous_ancient_docker_image_job_run)
546 test "reuse job with Docker image that has hash name" do
547 jobspec = {runtime_constraints: {
548 docker_image: "a" * 64,
550 check_job_reused_from(jobspec, :previous_docker_job_run)
553 ["repository", "script"].each do |skip_key|
554 test "missing #{skip_key} filter raises an error" do
555 filters = filters_from_hash(BASE_FILTERS.reject { |k| k == skip_key })
559 script_version: "master",
560 repository: "active/foo",
562 input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
567 find_or_create: true,
569 assert_includes(405..599, @response.code.to_i,
570 "bad status code with missing #{skip_key} filter")
574 test "find Job with script version range" do
575 get :index, filters: [["repository", "=", "active/foo"],
576 ["script", "=", "hash"],
577 ["script_version", "in git", "tag1"]]
578 assert_response :success
579 assert_not_nil assigns(:objects)
580 assert_includes(assigns(:objects).map { |job| job.uuid },
581 jobs(:previous_job_run).uuid)
584 test "find Job with script version range exclusions" do
585 get :index, filters: [["repository", "=", "active/foo"],
586 ["script", "=", "hash"],
587 ["script_version", "not in git", "tag1"]]
588 assert_response :success
589 assert_not_nil assigns(:objects)
590 refute_includes(assigns(:objects).map { |job| job.uuid },
591 jobs(:previous_job_run).uuid)
594 test "find Job with Docker image range" do
595 get :index, filters: [["docker_image_locator", "in docker",
596 "arvados/apitestfixture"]]
597 assert_response :success
598 assert_not_nil assigns(:objects)
599 assert_includes(assigns(:objects).map { |job| job.uuid },
600 jobs(:previous_docker_job_run).uuid)
601 refute_includes(assigns(:objects).map { |job| job.uuid },
602 jobs(:previous_job_run).uuid)
605 test "find Job with Docker image using reader tokens" do
606 authorize_with :inactive
608 filters: [["docker_image_locator", "in docker",
609 "arvados/apitestfixture"]],
610 reader_tokens: [api_token(:active)],
612 assert_response :success
613 assert_not_nil assigns(:objects)
614 assert_includes(assigns(:objects).map { |job| job.uuid },
615 jobs(:previous_docker_job_run).uuid)
616 refute_includes(assigns(:objects).map { |job| job.uuid },
617 jobs(:previous_job_run).uuid)
620 test "'in docker' filter accepts arrays" do
621 get :index, filters: [["docker_image_locator", "in docker",
622 ["_nonesuchname_", "arvados/apitestfixture"]]]
623 assert_response :success
624 assert_not_nil assigns(:objects)
625 assert_includes(assigns(:objects).map { |job| job.uuid },
626 jobs(:previous_docker_job_run).uuid)
627 refute_includes(assigns(:objects).map { |job| job.uuid },
628 jobs(:previous_job_run).uuid)
631 test "'not in docker' filter accepts arrays" do
632 get :index, filters: [["docker_image_locator", "not in docker",
633 ["_nonesuchname_", "arvados/apitestfixture"]]]
634 assert_response :success
635 assert_not_nil assigns(:objects)
636 assert_includes(assigns(:objects).map { |job| job.uuid },
637 jobs(:previous_job_run).uuid)
638 refute_includes(assigns(:objects).map { |job| job.uuid },
639 jobs(:previous_docker_job_run).uuid)
642 JOB_SUBMIT_KEYS = [:script, :script_parameters, :script_version, :repository]
643 DEFAULT_START_JOB = :previous_job_run
645 def create_job_params(params, start_from=DEFAULT_START_JOB)
646 if not params.has_key?(:find_or_create)
647 params[:find_or_create] = true
649 job_attrs = params.delete(:job) || {}
650 start_job = jobs(start_from)
651 params[:job] = Hash[JOB_SUBMIT_KEYS.map do |key|
652 [key, start_job.send(key)]
654 params[:job][:runtime_constraints] =
655 job_attrs.delete(:runtime_constraints) || {}
656 { arvados_sdk_version: :arvados_sdk_version,
657 docker_image_locator: :docker_image }.each do |method, constraint_key|
658 if constraint_value = start_job.send(method)
659 params[:job][:runtime_constraints][constraint_key] ||= constraint_value
662 params[:job].merge!(job_attrs)
666 def create_job_from(params, start_from)
667 post(:create, create_job_params(params, start_from))
668 assert_response :success
669 new_job = assigns(:object)
670 assert_not_nil new_job
674 def check_new_job_created_from(params, start_from=DEFAULT_START_JOB)
675 start_time = Time.now
676 new_job = create_job_from(params, start_from)
677 assert_operator(start_time, :<=, new_job.created_at)
681 def check_job_reused_from(params, start_from)
682 new_job = create_job_from(params, start_from)
683 assert_equal(jobs(start_from).uuid, new_job.uuid)
686 def check_errors_from(params, start_from=DEFAULT_START_JOB)
687 post(:create, create_job_params(params, start_from))
688 assert_includes(405..499, @response.code.to_i)
689 errors = json_response.fetch("errors", [])
690 assert(errors.any?, "no errors assigned from #{params}")
691 refute(errors.any? { |msg| msg =~ /^#<[A-Za-z]+: / },
692 "errors include raw exception: #{errors.inspect}")
696 # 1de84a8 is on the b1 branch, after master's tip.
697 test "new job created from unsatisfiable minimum version filter" do
698 filters_hash = BASE_FILTERS.merge("script_version" => ["in git", "1de84a8"])
699 check_new_job_created_from(filters: filters_from_hash(filters_hash))
702 test "new job created from unsatisfiable minimum version parameter" do
703 check_new_job_created_from(minimum_script_version: "1de84a8")
706 test "new job created from unsatisfiable minimum version attribute" do
707 check_new_job_created_from(job: {minimum_script_version: "1de84a8"})
710 test "graceful error from nonexistent minimum version filter" do
711 filters_hash = BASE_FILTERS.merge("script_version" =>
712 ["in git", "__nosuchbranch__"])
713 errors = check_errors_from(filters: filters_from_hash(filters_hash))
714 assert(errors.any? { |msg| msg.include? "__nosuchbranch__" },
715 "bad refspec not mentioned in error message")
718 test "graceful error from nonexistent minimum version parameter" do
719 errors = check_errors_from(minimum_script_version: "__nosuchbranch__")
720 assert(errors.any? { |msg| msg.include? "__nosuchbranch__" },
721 "bad refspec not mentioned in error message")
724 test "graceful error from nonexistent minimum version attribute" do
725 errors = check_errors_from(job: {minimum_script_version: "__nosuchbranch__"})
726 assert(errors.any? { |msg| msg.include? "__nosuchbranch__" },
727 "bad refspec not mentioned in error message")
730 test "don't reuse job with older Arvados SDK version specified by branch" do
731 jobspec = {runtime_constraints: {
732 arvados_sdk_version: "master",
734 check_new_job_created_from({job: jobspec},
735 :previous_job_run_with_arvados_sdk_version)
738 test "don't reuse job with older Arvados SDK version specified by commit" do
739 jobspec = {runtime_constraints: {
740 arvados_sdk_version: "ca68b24e51992e790f29df5cc4bc54ce1da4a1c2",
742 check_new_job_created_from({job: jobspec},
743 :previous_job_run_with_arvados_sdk_version)
746 test "don't reuse job with newer Arvados SDK version specified by commit" do
747 jobspec = {runtime_constraints: {
748 arvados_sdk_version: "436637c87a1d2bdbf4b624008304064b6cf0e30c",
750 check_new_job_created_from({job: jobspec},
751 :previous_job_run_with_arvados_sdk_version)
754 test "reuse job from arvados_sdk_version git filters" do
755 prev_job = jobs(:previous_job_run_with_arvados_sdk_version)
756 filters_hash = BASE_FILTERS.
757 merge("arvados_sdk_version" => ["in git", "commit2"],
758 "docker_image_locator" => ["=", prev_job.docker_image_locator])
759 filters_hash.delete("script_version")
760 params = create_job_params(filters: filters_from_hash(filters_hash))
761 post(:create, params)
762 assert_response :success
763 assert_equal(prev_job.uuid, assigns(:object).uuid)
766 test "create new job because of arvados_sdk_version 'not in git' filters" do
767 filters_hash = BASE_FILTERS.reject { |k| k == "script_version" }
768 filters = filters_from_hash(filters_hash)
769 # Allow anything from the root commit, but before commit 2.
770 filters += [["arvados_sdk_version", "in git", "436637c8"],
771 ["arvados_sdk_version", "not in git", "00634b2b"]]
772 check_new_job_created_from(filters: filters)