projects
/
arvados.git
/ blobdiff
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
3036: Test fixing work in progress.
[arvados.git]
/
services
/
api
/
script
/
crunch-dispatch.rb
diff --git
a/services/api/script/crunch-dispatch.rb
b/services/api/script/crunch-dispatch.rb
index 6f3aeda413bac1b5e8a2ce29bf6a842d8467a871..5a990f0cb41ad30bd8832399a0d40c1839f2460e 100755
(executable)
--- a/
services/api/script/crunch-dispatch.rb
+++ b/
services/api/script/crunch-dispatch.rb
@@
-78,7
+78,7
@@
class Dispatcher
# update our database (and cache) when a node's state changes
if @node_state[re[1]] != re[2]
@node_state[re[1]] = re[2]
# update our database (and cache) when a node's state changes
if @node_state[re[1]] != re[2]
@node_state[re[1]] = re[2]
- node = Node.where('hostname=?', re[1]).
fir
st
+ node = Node.where('hostname=?', re[1]).
order(:last_ping_at).la
st
if node
$stderr.puts "dispatch: update #{re[1]} state to #{re[2]}"
node.info['slurm_state'] = re[2]
if node
$stderr.puts "dispatch: update #{re[1]} state to #{re[2]}"
node.info['slurm_state'] = re[2]
@@
-166,6
+166,10
@@
class Dispatcher
cmd_args = nil
case Server::Application.config.crunch_job_wrapper
when :none
cmd_args = nil
case Server::Application.config.crunch_job_wrapper
when :none
+ if @running.size > 0
+ # Don't run more than one at a time.
+ return
+ end
cmd_args = []
when :slurm_immediate
nodelist = nodes_available_for_job(job)
cmd_args = []
when :slurm_immediate
nodelist = nodes_available_for_job(job)
@@
-226,7
+230,7
@@
class Dispatcher
next
end
next
end
- $stderr.puts `cd #{arvados_internal.shellescape} && git fetch
--no-tags
#{src_repo.shellescape} && git tag #{job.uuid.shellescape} #{job.script_version.shellescape}`
+ $stderr.puts `cd #{arvados_internal.shellescape} && git fetch
-pack --all
#{src_repo.shellescape} && git tag #{job.uuid.shellescape} #{job.script_version.shellescape}`
cmd_args << crunch_job_bin
cmd_args << '--job-api-token'
cmd_args << crunch_job_bin
cmd_args << '--job-api-token'
@@
-263,6
+267,8
@@
class Dispatcher
job_auth: job_auth,
stderr_buf_to_flush: '',
stderr_flushed_at: 0,
job_auth: job_auth,
stderr_buf_to_flush: '',
stderr_flushed_at: 0,
+ bytes_logged: 0,
+ events_logged: 0,
log_truncated: false
}
i.close
log_truncated: false
}
i.close
@@
-448,17
+454,13
@@
class Dispatcher
protected
protected
- def too_many_bytes_logged_for_job(running_job)
- bytes_logged = Log.where(uuid: running_job[:uuid]).map { |event|
- (event.properties["text"] || "").length
- }.sum
- return (bytes_logged + j[:stderr_buf_to_flush].size >
+ def too_many_bytes_logged_for_job(j)
+ return (j[:bytes_logged] + j[:stderr_buf_to_flush].size >
Rails.configuration.crunch_limit_log_event_bytes_per_job)
end
Rails.configuration.crunch_limit_log_event_bytes_per_job)
end
- def too_many_events_logged_for_job(running_job)
- log_count = Log.where(uuid: running_job[:uuid]).count
- return (log_count >= Rails.configuration.crunch_limit_log_events_per_job)
+ def too_many_events_logged_for_job(j)
+ return (j[:events_logged] >= Rails.configuration.crunch_limit_log_events_per_job)
end
def did_recently(thing, min_interval)
end
def did_recently(thing, min_interval)
@@
-493,6
+495,8
@@
class Dispatcher
owner_uuid: running_job[:job].owner_uuid,
properties: {"text" => running_job[:stderr_buf_to_flush]})
log.save!
owner_uuid: running_job[:job].owner_uuid,
properties: {"text" => running_job[:stderr_buf_to_flush]})
log.save!
+ running_job[:bytes_logged] += running_job[:stderr_buf_to_flush].size
+ running_job[:events_logged] += 1
running_job[:stderr_buf_to_flush] = ''
running_job[:stderr_flushed_at] = Time.now.to_i
end
running_job[:stderr_buf_to_flush] = ''
running_job[:stderr_flushed_at] = Time.now.to_i
end