#
# [--template uuid] Use the specified pipeline template.
#
+# [--template path] Load the pipeline template from the specified
+# local file.
+#
# [--instance uuid] Use the specified pipeline instance.
#
# [-n, --dry-run] Do not start any new jobs or wait for existing jobs
# this program, that is, not the pipeline component parameters).
p = Trollop::Parser.new do
+ version __FILE__
opt(:dry_run,
"Do not start any new jobs or wait for existing jobs to finish. Just find out whether jobs are finished, queued, or running for each component.",
:type => :boolean,
:short => :none,
:type => :integer)
opt(:template,
- "UUID of pipeline template.",
+ "UUID of pipeline template, or path to local pipeline template file.",
:short => :none,
:type => :string)
opt(:instance,
def self.find(uuid)
result = $client.execute(:api_method => $arvados.pipeline_instances.get,
:parameters => {
- :api_token => ENV['ARVADOS_API_TOKEN'],
:uuid => uuid
},
+ :body => {
+ :api_token => ENV['ARVADOS_API_TOKEN']
+ },
:authenticated => false)
j = JSON.parse result.body, :symbolize_names => true
unless j.is_a? Hash and j[:uuid]
end
def self.create(attributes)
result = $client.execute(:api_method => $arvados.pipeline_instances.create,
- :parameters => {
+ :body => {
:api_token => ENV['ARVADOS_API_TOKEN'],
- :pipeline_instance => attributes.to_json
+ :pipeline_instance => attributes
},
:authenticated => false)
j = JSON.parse result.body, :symbolize_names => true
def save
result = $client.execute(:api_method => $arvados.pipeline_instances.update,
:parameters => {
+ :uuid => @pi[:uuid]
+ },
+ :body => {
:api_token => ENV['ARVADOS_API_TOKEN'],
- :uuid => @pi[:uuid],
:pipeline_instance => @attributes_to_update.to_json
},
:authenticated => false)
@options = _options
end
- def fetch_template(template_uuid)
- result = $client.execute(:api_method => $arvados.pipeline_templates.get,
- :parameters => {
- :api_token => ENV['ARVADOS_API_TOKEN'],
- :uuid => template_uuid
- },
- :authenticated => false)
- @template = JSON.parse result.body, :symbolize_names => true
- if !@template[:uuid]
- abort "#{$0}: fatal: failed to retrieve pipeline template #{template_uuid} #{@template[:errors].inspect rescue nil}"
+ def fetch_template(template)
+ if template.match /[^-0-9a-z]/
+ # Doesn't look like a uuid -- use it as a filename.
+ @template = JSON.parse File.read(template), :symbolize_names => true
+ if !@template[:components]
+ abort ("#{$0}: Template loaded from #{template} " +
+ "does not have a \"components\" key")
+ end
+ else
+ result = $client.execute(:api_method => $arvados.pipeline_templates.get,
+ :parameters => {
+ :api_token => ENV['ARVADOS_API_TOKEN'],
+ :uuid => template
+ },
+ :authenticated => false)
+ @template = JSON.parse result.body, :symbolize_names => true
+ if !@template[:uuid]
+ abort "#{$0}: fatal: failed to retrieve pipeline template #{template} #{@template[:errors].inspect rescue nil}"
+ end
end
self
end
(@options[:no_reuse] ? [] : JobCache.
where(script: c[:script],
script_parameters: c[:script_parameters],
- script_version_descends_from: c[:script_version_descends_from])
+ script_version_descends_from: c[:script_version])
).each do |candidate_job|
candidate_params_downcase = Hash[candidate_job[:script_parameters].
map { |k,v| [k.downcase,v] }]
next
end
+ if c[:script_version] !=
+ candidate_job[:script_version][0,c[:script_version].length]
+ debuglog "component #{cname} would be satisfied by job #{candidate_job[:uuid]} if script_version matched.", 2
+ next
+ end
+
unless candidate_job[:success] || candidate_job[:running] ||
(!candidate_job[:started_at] && !candidate_job[:cancelled_at])
debuglog "component #{cname} would be satisfied by job #{candidate_job[:uuid]} if it were running or successful.", 2
if candidate_job[:success]
unless @options[:no_reuse_finished]
job = candidate_job
- debuglog "component #{cname} satisfied by job #{job[:uuid]} version #{job[:script_version]}"
+ $stderr.puts "using #{job[:uuid]} (finished at #{job[:finished_at]}) for component #{cname}"
c[:job] = job
end
else
end
if not c[:job] and second_place_job
job = second_place_job
- debuglog "component #{cname} satisfied by job #{job[:uuid]} version #{job[:script_version]}"
+ $stderr.puts "using #{job[:uuid]} (running since #{job[:started_at]}) for component #{cname}"
c[:job] = job
end
if not c[:job]
debuglog "component #{cname} new job."
job = JobCache.create(:script => c[:script],
:script_parameters => c[:script_parameters],
- :resource_limits => c[:resource_limits] || {},
+ :runtime_constraints => c[:runtime_constraints] || {},
:script_version => c[:script_version] || 'master')
if job
debuglog "component #{cname} new job #{job[:uuid]}"
c[:wait] = true
end
if c[:job] and c[:job][:uuid]
- if not c[:job][:finished_at] and not c[:job][:cancelled_at]
+ if (c[:job][:running] or
+ not (c[:job][:finished_at] or c[:job][:cancelled_at]))
c[:job] = JobCache.get(c[:job][:uuid])
end
if c[:job][:success]
if @options[:status_text] != '/dev/null'
File.open(@options[:status_text], 'w') do |f|
+ f.puts ""
f.puts "#{Time.now} -- pipeline_instance #{@instance[:uuid]}"
namewidth = @components.collect { |cname, c| cname.size }.max
@components.each do |cname, c|
projects / arvados.git / blobdiff