skip_before_filter :find_object_by_uuid, only: :compare
before_filter :find_objects_by_uuid, only: :compare
include PipelineInstancesHelper
+ include PipelineComponentsHelper
def copy
@object = @object.dup
if @object and @object.state.in? ['New', 'Ready']
panes = %w(Inputs) + panes
end
- if not @object.components.values.collect { |x| x[:job] }.compact.any?
+ if not @object.components.values.any? { |x| x[:job] rescue false }
panes -= ['Graph']
end
panes
class PipelineTemplatesController < ApplicationController
-
+ include PipelineComponentsHelper
+
def show
@objects = PipelineInstance.where(pipeline_template_uuid: @object.uuid)
super
--- /dev/null
+module PipelineComponentsHelper
+ def render_pipeline_components(template_suffix, fallback=nil, locals={})
+ begin
+ render(partial: "pipeline_instances/show_components_#{template_suffix}",
+ locals: locals)
+ rescue
+ case fallback
+ when :json
+ render(partial: "pipeline_instances/show_components_json")
+ end
+ end
+ end
+end
object.components.each do |cname, c|
i += 1
pj = {index: i, name: cname}
+ if not c.is_a?(Hash)
+ ret << pj
+ next
+ end
pj[:job] = c[:job].is_a?(Hash) ? c[:job] : {}
pj[:percent_done] = 0
pj[:percent_running] = 0
<% component_frac = 1.0 / p.components.length %>
<div class="progress">
<% p.components.each do |k,c| %>
- <% if c[:job] %>
+ <% if c.is_a?(Hash) and c[:job] %>
<%= render partial: "job_progress", locals: {:j => c[:job], :scaleby => component_frac } %>
<% end %>
<% end %>
<% elsif p.state == 'RunningOnServer' || p.state == 'RunningOnClient' %>
<span class="label label-info">running</span>
<% else %>
- <% if (p.components.select do |k,v| v[:job] end).length == 0 %>
+ <% if not p.components.values.any? { |c| c[:job] rescue false } %>
<span class="label label-default">not started</span>
<% else %>
<span class="label label-default">not running</span>
Current state: <span class="badge badge-info" data-pipeline-state="<%= @object.state %>"><%= @object.state.sub('OnServer', '') %></span>
</div>
- <table class="table pipeline-components-table">
- <colgroup>
- <col style="width: 15%" />
- <col style="width: 25%" />
- <col style="width: 8%" />
- <col style="width: 13%" />
- <col style="width: 12%" />
- <col style="width: 14%" />
- <col style="width: 13%" />
- </colgroup>
- <thead>
- <tr>
- <th colspan="2">
- component
- </th><th colspan="5">
- job
- <%# format:'js' here helps browsers avoid using the cached js
- content in html context (e.g., duplicate tab -> see
- javascript) %>
- <%= link_to '(refresh)', {format: :js}, {class: 'refresh hide', remote: true, method: 'get'} %>
- </th>
- </tr>
- </thead>
- <tbody>
- <% render_pipeline_jobs.each do |pj| %>
- <% if pj[:job].andand[:uuid]
- pipeline_job_uuids << pj[:job][:uuid]
- end %>
- <tr>
- <td>
- <%= pj[:name] %>
- </td><td>
- <%= pj[:script] %>
- <br /><span class="deemphasize"><%= pj[:script_version] %></span>
- </td><td>
- <%= render(partial: 'job_status_label', locals: { j: pj[:job] }) %>
- </td><td>
- <%= pj[:progress_bar] %>
- </td>
- <% current_job = Job.find(pj[:job][:uuid]) rescue nil %>
- <td>
- <% if current_job %>
- <%= render partial: 'show_object_button', locals: {object: current_job, size: 'xs', link_text: 'Show job details'} %>
- <% end %>
- </td><td>
- <% if current_job.andand[:log] %>
- <% fixup = /([a-f0-9]{32}\+\d+)(\+?.*)/.match(current_job[:log])%>
- <% Collection.limit(1).where(uuid: fixup[1]).each do |c| %>
- <% c.files.first.andand do |file| %>
- <%= link_to url_for(controller: 'collections', action: 'show_file', uuid: current_job[:log], file: "#{file[0]}/#{file[1]}", disposition: 'inline', size: file[2]), class: 'btn btn-default btn-xs' do %>
- <i class="fa fa-fw fa-info"></i> Show log messages
- <% end %>
- <% end %>
- <% end %>
- <% end %>
- </td><td>
- <% if current_job.andand[:output] %>
- <%= link_to_if_arvados_object current_job[:output], {thumbnail: true, link_text: raw('<i class="fa fa-fw fa-archive"></i> Show output files')}, {class: 'btn btn-default btn-xs'} %>
- <% end %>
- </td>
- </tr>
- <% end %>
- </tbody>
- <tfoot>
- <tr><td colspan="7"></td></tr>
- </tfoot>
- </table>
+ <%= render_pipeline_components("running", :json, pipeline_job_uuids: pipeline_job_uuids) %>
<% if @object.state.in? %w(RunningOnServer RunningOnClient Failed) %>
<% else %>
<%# state is either New or Ready %>
+ <p><i>Here are all of the pipeline's components (jobs that will need to run in order to complete the pipeline). If you know what you're doing (or you're experimenting) you can modify these parameters before starting the pipeline. Usually, you only need to edit the settings presented on the "Inputs" tab above.</i></p>
- <% if @object.state.in? %w(New Ready) %>
- <p><i>Here are all of the pipeline's components (jobs that will need to run in order to complete the pipeline). If you know what you're doing (or you're experimenting) you can modify these parameters before starting the pipeline. Usually, you only need to edit the settings presented on the "Inputs" tab above.</i></p>
- <% end %>
-
- <% if @object.state.in? ['New', 'Ready'] %>
- <%= render partial: 'show_components_editable', locals: {editable: true} %>
- <% else %>
- <%= render partial: 'show_components_editable', locals: {editable: false} %>
- <% end %>
+ <%= render_pipeline_components("editable", :json, editable: true) %>
<% end %>
--- /dev/null
+<p>The components of this pipeline are in a format that Workbench does not recognize.</p>
+
+ <div id="components-accordion" class="panel panel-default">
+ <div class="panel-heading">
+ <h4 class="panel-title">
+ <a data-toggle="collapse" data-parent="#components-accordion" href="#components-json">
+ Show components JSON
+ </a>
+ </h4>
+ </div>
+ <div id="components-json" class="panel-collapse collapse">
+ <div class="panel-body">
+ <pre><%= Oj.dump(@object.components, indent: 2) %></pre>
+ </div>
+ </div>
+ </div>
--- /dev/null
+ <table class="table pipeline-components-table">
+ <colgroup>
+ <col style="width: 15%" />
+ <col style="width: 25%" />
+ <col style="width: 8%" />
+ <col style="width: 13%" />
+ <col style="width: 12%" />
+ <col style="width: 14%" />
+ <col style="width: 13%" />
+ </colgroup>
+ <thead>
+ <tr>
+ <th colspan="2">
+ component
+ </th><th colspan="5">
+ job
+ <%# format:'js' here helps browsers avoid using the cached js
+ content in html context (e.g., duplicate tab -> see
+ javascript) %>
+ <%= link_to '(refresh)', {format: :js}, {class: 'refresh hide', remote: true, method: 'get'} %>
+ </th>
+ </tr>
+ </thead>
+ <tbody>
+ <% render_pipeline_jobs.each do |pj| %>
+ <% if pj[:job].andand[:uuid]
+ pipeline_job_uuids << pj[:job][:uuid]
+ end %>
+ <tr>
+ <td>
+ <%= pj[:name] %>
+ </td><td>
+ <%= pj[:script] %>
+ <br /><span class="deemphasize"><%= pj[:script_version] %></span>
+ </td><td>
+ <%= render(partial: 'job_status_label', locals: { j: pj[:job] }) %>
+ </td><td>
+ <%= pj[:progress_bar] %>
+ </td>
+ <% current_job = Job.find(pj[:job][:uuid]) rescue nil %>
+ <td>
+ <% if current_job %>
+ <%= render partial: 'show_object_button', locals: {object: current_job, size: 'xs', link_text: 'Show job details'} %>
+ <% end %>
+ </td><td>
+ <% if current_job.andand[:log] %>
+ <% fixup = /([a-f0-9]{32}\+\d+)(\+?.*)/.match(current_job[:log])%>
+ <% Collection.limit(1).where(uuid: fixup[1]).each do |c| %>
+ <% c.files.first.andand do |file| %>
+ <%= link_to url_for(controller: 'collections', action: 'show_file', uuid: current_job[:log], file: "#{file[0]}/#{file[1]}", disposition: 'inline', size: file[2]), class: 'btn btn-default btn-xs' do %>
+ <i class="fa fa-fw fa-info"></i> Show log messages
+ <% end %>
+ <% end %>
+ <% end %>
+ <% end %>
+ </td><td>
+ <% if current_job.andand[:output] %>
+ <%= link_to_if_arvados_object current_job[:output], {thumbnail: true, link_text: raw('<i class="fa fa-fw fa-archive"></i> Show output files')}, {class: 'btn btn-default btn-xs'} %>
+ <% end %>
+ </td>
+ </tr>
+ <% end %>
+ </tbody>
+ <tfoot>
+ <tr><td colspan="7"></td></tr>
+ </tfoot>
+ </table>
</td>
<td style="border-top: 0; opacity: 0.5;" colspan="6">
<% ob.components.each do |cname, c| %>
- <% if c[:job] %>
+ <% if c.is_a?(Hash) and c[:job] %>
<%= render partial: "job_status_label", locals: {:j => c[:job], :title => cname.to_s } %>
<% else %>
- <span class="label label-default"><%= cname.to_s %></span>
+ <span class="label label-default"><%= cname.to_s %></span>
<% end %>
<% end %>
</td>
}.to_json),
{ class: "btn btn-primary btn-sm", remote: true, method: 'get' }
) do %>
- Run this pipeline
+ Run this pipeline
<% end %>
<% end %>
-<%= render partial: 'pipeline_instances/show_components_editable', locals: {editable: false} %>
+<%= render_pipeline_components("editable", :json, editable: false) %>
end
end
end
+
+ test "component rendering copes with unexpeceted components format" do
+ get(:show,
+ {id: api_fixture("pipeline_instances")["components_is_jobspec"]["uuid"]},
+ session_for(:active))
+ assert_response :success
+ end
end
require 'test_helper'
class PipelineTemplatesControllerTest < ActionController::TestCase
+ test "component rendering copes with unexpeceted components format" do
+ get(:show,
+ {id: api_fixture("pipeline_templates")["components_is_jobspec"]["uuid"]},
+ session_for(:active))
+ assert_response :success
+ end
end
find('.btn', text: 'Run a pipeline').click
within('.modal-dialog') do
assert page.has_text? 'Two Part Pipeline Template'
- find('.fa-gear').click
+ find('.selectable', text: 'Two Part Pipeline Template').click
find('.btn', text: 'Next: choose inputs').click
end
assert page.has_text? 'script_version'
end
+ test "JSON popup available for strange components" do
+ uuid = api_fixture("pipeline_instances")["components_is_jobspec"]["uuid"]
+ visit page_with_token("active", "/pipeline_instances/#{uuid}")
+ click_on "Components"
+ assert(page.has_no_text?("script_parameters"),
+ "components JSON visible without popup")
+ click_on "Show components JSON"
+ assert(page.has_text?("script_parameters"),
+ "components JSON not found")
+ end
end
{% include 'alert_stub' %}
# Set up a cluster, or use Amazon
-# Create and mount Keep volumes
+# "Install Keep":install-keep.html
# "Install the Single Sign On (SSO) server":install-sso.html
# "Install the Arvados REST API server":install-api-server.html
# "Install the Arvados workbench application":install-workbench-app.html
--- /dev/null
+---
+layout: default
+navsection: installguide
+title: Install Keep
+...
+
+This installation guide assumes you are on a 64 bit Debian or Ubuntu system.
+
+First add the Arvados apt repository, and then install the Keep package.
+
+<notextile>
+<pre><code>~$ <span class="userinput">echo "# apt.arvados.org" > /etc/apt/sources.list.d/apt.arvados.org.list</span>
+~$ <span class="userinput">echo "deb http://apt.arvados.org/ wheezy main" >> /etc/apt/sources.list.d/apt.arvados.org.list</span>
+~$ <span class="userinput">/usr/bin/apt-key adv --keyserver pgp.mit.edu --recv 1078ECD7</span>
+~$ <span class="userinput">/usr/bin/apt-get update</span>
+~$ <span class="userinput">/usr/bin/apt-get install keep</span>
+</code></pre>
+</notextile>
+
+Verify that Keep is functional:
+
+<notextile>
+<pre><code>~$ <span class="userinput">keep -h</span>
+keep -h
+2014/07/24 15:38:27 Keep started: pid 13606
+Usage of keep:
+ -data-manager-token-file="": File with the API token used by the Data Manager. All DELETE requests or GET /index requests must carry this token.
+ -enforce-permissions=false: Enforce permission signatures on requests.
+ -listen=":25107": Interface on which to listen for requests, in the format ipaddr:port. e.g. -listen=10.0.1.24:8000. Use -listen=:port to listen on all network interfaces.
+ -permission-key-file="": File containing the secret key for generating and verifying permission signatures.
+ -permission-ttl=1209600: Expiration time (in seconds) for newly generated permission signatures.
+ -pid="": Path to write pid file
+ -serialize=false: If set, all read and write operations on local Keep volumes will be serialized.
+ -volumes="": Comma-separated list of directories to use for Keep volumes, e.g. -volumes=/var/keep1,/var/keep2. If empty or not supplied, Keep will scan mounted filesystems for volumes with a /keep top-level directory.
+</code></pre>
+</notextile>
+
+Prepare one or more volumes for Keep to use. Simply create a /keep directory on all the partitions you would like Keep to use, and then start Keep. For example, using 2 tmpfs volumes:
+
+<notextile>
+<pre><code>~$ <span class="userinput">keep</span>
+2014/07/24 11:41:37 Keep started: pid 20736
+2014/07/24 11:41:37 adding Keep volume: /tmp/tmp.vwSCtUCyeH/keep
+2014/07/24 11:41:37 adding Keep volume: /tmp/tmp.Lsn4w8N3Xv/keep
+2014/07/24 11:41:37 Running without a PermissionSecret. Block locators returned by this server will not be signed, and will be rejected by a server that enforces permissions.
+2014/07/24 11:41:37 To fix this, run Keep with --permission-key-file=<path> to define the location of a file containing the permission key.
+
+</code></pre>
+</notextile>
+
+It's recommended to run Keep under "runit":https://packages.debian.org/search?keywords=runit or something similar.
+
use IO::Select;
use File::Temp;
use Fcntl ':flock';
+use File::Path qw( make_path );
$ENV{"TMPDIR"} ||= "/tmp";
unless (defined $ENV{"CRUNCH_TMP"}) {
$ENV{"CRUNCH_TMP"} .= "-$<";
}
}
+
+# Create the tmp directory if it does not exist
+if ( ! -d $ENV{"CRUNCH_TMP"} ) {
+ make_path $ENV{"CRUNCH_TMP"} or die "Failed to create temporary working directory: " . $ENV{"CRUNCH_TMP"};
+}
+
$ENV{"JOB_WORK"} = $ENV{"CRUNCH_TMP"} . "/work";
$ENV{"CRUNCH_INSTALL"} = "$ENV{CRUNCH_TMP}/opt";
$ENV{"CRUNCH_WORK"} = $ENV{"JOB_WORK"}; # deprecated
state: Ready
uuid: zzzzz-d1hrv-1yfj6xkidf2muk3
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
- owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
components:
foo:
script: foo
uuid: zzzzz-8i9sb-pshmckwoma9plh7,
script_version: master
}
+
+components_is_jobspec:
+ # Helps test that clients cope with funny-shaped components.
+ # For an example, see #3321.
+ uuid: zzzzz-d1hrv-jobspeccomponts
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2014-04-14 12:35:04 -0400
+ updated_at: 2014-04-14 12:35:04 -0400
+ modified_at: 2014-04-14 12:35:04 -0400
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: RunningOnServer
+ components:
+ script: foo
+ script_version: master
+ script_parameters:
+ input:
+ required: true
+ dataclass: Collection
+ title: "Foo/bar pair"
+ description: "Provide a collection containing at least two files."
default: [1,1,2,3,5]
array_with_value: # important to test repeating values in the array!
value: [1,1,2,3,5]
+
+components_is_jobspec:
+ # Helps test that clients cope with funny-shaped components.
+ # For an example, see #3321.
+ uuid: zzzzz-p5p6p-jobspeccomponts
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2014-04-14 12:35:04 -0400
+ updated_at: 2014-04-14 12:35:04 -0400
+ modified_at: 2014-04-14 12:35:04 -0400
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ name: Pipeline Template with Jobspec Components
+ components:
+ script: foo
+ script_version: master
+ script_parameters:
+ input:
+ required: true
+ dataclass: Collection
+ title: "Foo/bar pair"
+ description: "Provide a collection containing at least two files."
projects / arvados.git / commitdiff