1 class Arvados::V1::JobsController < ApplicationController
2 accept_attribute_as_json :script_parameters, Hash
3 accept_attribute_as_json :runtime_constraints, Hash
4 accept_attribute_as_json :tasks_summary, Hash
5 skip_before_filter :find_object_by_uuid, :only => :queue
6 skip_before_filter :render_404_if_no_object, :only => :queue
9 [:repository, :script, :script_version, :script_parameters].each do |r|
11 return send_error("#{r} attribute must be specified",
12 status: :unprocessable_entity)
16 # We used to ask for the minimum_, exclude_, and no_reuse params
17 # in the job resource. Now we advertise them as flags that alter
18 # the behavior of the create action.
19 [:minimum_script_version, :exclude_script_versions].each do |attr|
20 if resource_attrs.has_key? attr
21 params[attr] = resource_attrs.delete attr
24 if resource_attrs.has_key? :no_reuse
25 params[:find_or_create] = !resource_attrs.delete(:no_reuse)
28 if params[:find_or_create]
29 return if false.equal?(load_filters_param)
30 if @filters.empty? # Translate older creation parameters into filters.
32 [["repository", "=", resource_attrs[:repository]],
33 ["script", "=", resource_attrs[:script]],
34 ["script_version", "in git",
35 params[:minimum_script_version] || resource_attrs[:script_version]],
36 ["script_version", "not in git", params[:exclude_script_versions]],
37 ].reject { |filter| filter.last.nil? }
38 if image_search = resource_attrs[:runtime_constraints].andand["docker_image"]
39 if image_tag = resource_attrs[:runtime_constraints]["docker_image_tag"]
40 image_search += ":#{image_tag}"
42 @filters.append(["docker_image_locator", "in docker", image_search])
44 @filters.append(["docker_image_locator", "=", nil])
47 load_job_specific_filters
48 rescue ArgumentError => error
49 return send_error(error.message)
53 # Check specified filters for some reasonableness.
54 filter_names = @filters.map { |f| f.first }.uniq
55 ["repository", "script"].each do |req_filter|
56 if not filter_names.include?(req_filter)
57 return send_error("#{req_filter} filter required")
61 # Search for a reusable Job, and return it if found.
62 @objects = Job.readable_by(current_user)
67 if j.nondeterministic != true and
68 ((j.success == true and j.output != nil) or j.running == true) and
69 j.script_parameters == resource_attrs[:script_parameters]
71 # We'll use this if we don't find a job that has completed
74 # Record the first job in the list
78 # Ensure that all candidate jobs actually did produce the same output
79 if @object.output != j.output
85 @object ||= incomplete_job
96 reload_object_before_update
97 @object.update_attributes! cancelled_at: Time.now
102 Q_UPDATE_INTERVAL = 12
103 def initialize(job, opts={})
109 yield "#{@job.uuid} finished at #{@job.finished_at}\n"
112 while not @job.started_at
113 # send a summary (job queue + available nodes) to the client
114 # every few seconds while waiting for the job to start
115 last_ack_at ||= Time.now - Q_UPDATE_INTERVAL - 1
116 if Time.now - last_ack_at >= Q_UPDATE_INTERVAL
117 nodes_in_state = {idle: 0, alloc: 0}
118 ActiveRecord::Base.uncached do
119 Node.where('hostname is not ?', nil).collect do |n|
120 if n.info[:slurm_state]
121 nodes_in_state[n.info[:slurm_state]] ||= 0
122 nodes_in_state[n.info[:slurm_state]] += 1
126 job_queue = Job.queue
127 n_queued_before_me = 0
128 job_queue.each do |j|
129 break if j.uuid == @job.uuid
130 n_queued_before_me += 1
132 yield "#{Time.now}" \
133 " job #{@job.uuid}" \
134 " queue_position #{n_queued_before_me}" \
135 " queue_size #{job_queue.size}" \
136 " nodes_idle #{nodes_in_state[:idle]}" \
137 " nodes_alloc #{nodes_in_state[:alloc]}\n"
138 last_ack_at = Time.now
141 ActiveRecord::Base.uncached do
149 params[:order] ||= ['priority desc', 'created_at']
150 load_limit_offset_order_params
154 is_locked_by_uuid: nil,
158 return if false.equal?(load_filters_param)
159 find_objects_for_index
163 def self._queue_requires_parameters
164 self._index_requires_parameters
169 def load_job_specific_filters
170 # Convert Job-specific @filters entries into general SQL filters.
171 script_info = {"repository" => nil, "script" => nil}
172 script_range = {"exclude_versions" => []}
173 @filters.select! do |filter|
174 if (script_info.has_key? filter[0]) and (filter[1] == "=")
175 if script_info[filter[0]].nil?
176 script_info[filter[0]] = filter[2]
177 elsif script_info[filter[0]] != filter[2]
178 raise ArgumentError.new("incompatible #{filter[0]} filters")
182 when ["script_version", "in git"]
183 script_range["min_version"] = filter.last
185 when ["script_version", "not in git"]
187 script_range["exclude_versions"] += filter.last
189 script_range["exclude_versions"] << filter.last
192 when ["docker_image_locator", "in docker"], ["docker_image_locator", "not in docker"]
193 filter[1].sub!(/ docker$/, '')
194 search_list = filter[2].is_a?(Enumerable) ? filter[2] : [filter[2]]
195 filter[2] = search_list.flat_map do |search_term|
196 image_search, image_tag = search_term.split(':', 2)
197 Collection.uuids_for_docker_image(image_search, image_tag, @read_users)
205 # Build a real script_version filter from any "not? in git" filters.
206 if (script_range.size > 1) or script_range["exclude_versions"].any?
207 script_info.each_pair do |key, value|
209 raise ArgumentError.new("script_version filter needs #{key} filter")
212 last_version = begin resource_attrs[:script_version] rescue "HEAD" end
213 version_range = Commit.find_commit_range(current_user,
214 script_info["repository"],
215 script_range["min_version"],
217 script_range["exclude_versions"])
218 if version_range.nil?
220 new(["error searching #{script_info['repository']} from",
221 "#{script_range['min_version']} to #{last_version},",
222 "excluding #{script_range['exclude_versions']}"].join(" "))
224 @filters.append(["script_version", "in", version_range])
228 def load_filters_param
231 load_job_specific_filters
232 rescue ArgumentError => error
233 send_error(error.message)