X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/e80fcd1a64c60849112e6af470b620f517eeb996..6d59f29ba37c608e4b01b27a2fa78bc065fdc2a6:/sdk/cli/bin/arv-run-pipeline-instance diff --git a/sdk/cli/bin/arv-run-pipeline-instance b/sdk/cli/bin/arv-run-pipeline-instance index e36ac0f031..d2b1109e16 100755 --- a/sdk/cli/bin/arv-run-pipeline-instance +++ b/sdk/cli/bin/arv-run-pipeline-instance @@ -12,6 +12,9 @@ # # [--template uuid] Use the specified pipeline template. # +# [--template path] Load the pipeline template from the specified +# local file. +# # [--instance uuid] Use the specified pipeline instance. # # [-n, --dry-run] Do not start any new jobs or wait for existing jobs @@ -128,6 +131,7 @@ end # this program, that is, not the pipeline component parameters). p = Trollop::Parser.new do + version __FILE__ opt(:dry_run, "Do not start any new jobs or wait for existing jobs to finish. Just find out whether jobs are finished, queued, or running for each component.", :type => :boolean, @@ -162,7 +166,7 @@ p = Trollop::Parser.new do :short => :none, :type => :integer) opt(:template, - "UUID of pipeline template.", + "UUID of pipeline template, or path to local pipeline template file.", :short => :none, :type => :string) opt(:instance, @@ -201,9 +205,11 @@ class PipelineInstance def self.find(uuid) result = $client.execute(:api_method => $arvados.pipeline_instances.get, :parameters => { - :api_token => ENV['ARVADOS_API_TOKEN'], :uuid => uuid }, + :body => { + :api_token => ENV['ARVADOS_API_TOKEN'] + }, :authenticated => false) j = JSON.parse result.body, :symbolize_names => true unless j.is_a? Hash and j[:uuid] @@ -216,9 +222,9 @@ class PipelineInstance end def self.create(attributes) result = $client.execute(:api_method => $arvados.pipeline_instances.create, - :parameters => { + :body => { :api_token => ENV['ARVADOS_API_TOKEN'], - :pipeline_instance => attributes.to_json + :pipeline_instance => attributes }, :authenticated => false) j = JSON.parse result.body, :symbolize_names => true @@ -231,8 +237,10 @@ class PipelineInstance def save result = $client.execute(:api_method => $arvados.pipeline_instances.update, :parameters => { + :uuid => @pi[:uuid] + }, + :body => { :api_token => ENV['ARVADOS_API_TOKEN'], - :uuid => @pi[:uuid], :pipeline_instance => @attributes_to_update.to_json }, :authenticated => false) @@ -310,16 +318,25 @@ class WhRunPipelineInstance @options = _options end - def fetch_template(template_uuid) - result = $client.execute(:api_method => $arvados.pipeline_templates.get, - :parameters => { - :api_token => ENV['ARVADOS_API_TOKEN'], - :uuid => template_uuid - }, - :authenticated => false) - @template = JSON.parse result.body, :symbolize_names => true - if !@template[:uuid] - abort "#{$0}: fatal: failed to retrieve pipeline template #{template_uuid} #{@template[:errors].inspect rescue nil}" + def fetch_template(template) + if template.match /[^-0-9a-z]/ + # Doesn't look like a uuid -- use it as a filename. + @template = JSON.parse File.read(template), :symbolize_names => true + if !@template[:components] + abort ("#{$0}: Template loaded from #{template} " + + "does not have a \"components\" key") + end + else + result = $client.execute(:api_method => $arvados.pipeline_templates.get, + :parameters => { + :api_token => ENV['ARVADOS_API_TOKEN'], + :uuid => template + }, + :authenticated => false) + @template = JSON.parse result.body, :symbolize_names => true + if !@template[:uuid] + abort "#{$0}: fatal: failed to retrieve pipeline template #{template} #{@template[:errors].inspect rescue nil}" + end end self end @@ -403,7 +420,7 @@ class WhRunPipelineInstance (@options[:no_reuse] ? [] : JobCache. where(script: c[:script], script_parameters: c[:script_parameters], - script_version_descends_from: c[:script_version_descends_from]) + script_version_descends_from: c[:script_version]) ).each do |candidate_job| candidate_params_downcase = Hash[candidate_job[:script_parameters]. map { |k,v| [k.downcase,v] }] @@ -416,6 +433,12 @@ class WhRunPipelineInstance next end + if c[:script_version] != + candidate_job[:script_version][0,c[:script_version].length] + debuglog "component #{cname} would be satisfied by job #{candidate_job[:uuid]} if script_version matched.", 2 + next + end + unless candidate_job[:success] || candidate_job[:running] || (!candidate_job[:started_at] && !candidate_job[:cancelled_at]) debuglog "component #{cname} would be satisfied by job #{candidate_job[:uuid]} if it were running or successful.", 2 @@ -425,7 +448,7 @@ class WhRunPipelineInstance if candidate_job[:success] unless @options[:no_reuse_finished] job = candidate_job - debuglog "component #{cname} satisfied by job #{job[:uuid]} version #{job[:script_version]}" + $stderr.puts "using #{job[:uuid]} (finished at #{job[:finished_at]}) for component #{cname}" c[:job] = job end else @@ -435,7 +458,7 @@ class WhRunPipelineInstance end if not c[:job] and second_place_job job = second_place_job - debuglog "component #{cname} satisfied by job #{job[:uuid]} version #{job[:script_version]}" + $stderr.puts "using #{job[:uuid]} (running since #{job[:started_at]}) for component #{cname}" c[:job] = job end if not c[:job] @@ -444,7 +467,7 @@ class WhRunPipelineInstance debuglog "component #{cname} new job." job = JobCache.create(:script => c[:script], :script_parameters => c[:script_parameters], - :resource_limits => c[:resource_limits] || {}, + :runtime_constraints => c[:runtime_constraints] || {}, :script_version => c[:script_version] || 'master') if job debuglog "component #{cname} new job #{job[:uuid]}" @@ -458,7 +481,8 @@ class WhRunPipelineInstance c[:wait] = true end if c[:job] and c[:job][:uuid] - if not c[:job][:finished_at] and not c[:job][:cancelled_at] + if (c[:job][:running] or + not (c[:job][:finished_at] or c[:job][:cancelled_at])) c[:job] = JobCache.get(c[:job][:uuid]) end if c[:job][:success] @@ -520,6 +544,7 @@ class WhRunPipelineInstance if @options[:status_text] != '/dev/null' File.open(@options[:status_text], 'w') do |f| + f.puts "" f.puts "#{Time.now} -- pipeline_instance #{@instance[:uuid]}" namewidth = @components.collect { |cname, c| cname.size }.max @components.each do |cname, c|