/config/environments/development.rb
/config/environments/test.rb
/config/environments/production.rb
+/config/application.yml
/config/piwik.yml
# gem 'capistrano'
# To use debugger
-# gem 'debugger'
+#gem 'byebug'
gem 'rvm-capistrano', :group => :test
gem 'piwik_analytics'
gem 'httpclient'
gem 'themes_for_rails'
+gem "deep_merge", :require => 'deep_merge/rails_compat'
\ No newline at end of file
coffee-script-source (1.6.3)
commonjs (0.2.7)
daemon_controller (1.1.7)
+ deep_merge (1.0.1)
erubis (2.7.0)
execjs (2.0.2)
highline (1.6.20)
bootstrap-sass (~> 3.1.0)
bootstrap-x-editable-rails
coffee-rails (~> 3.2.0)
+ deep_merge
httpclient
jquery-rails
less
}
targets.fadeToggle(200);
});
+
$(document).
on('ajax:send', function(e, xhr) {
$('.loading').fadeTo('fast', 1);
fixer.duplicateTheadTr();
fixer.fixThead();
});
-})(jQuery);
+});
$.fn.editable.defaults.ajaxOptions = {type: 'put', dataType: 'json'};
$.fn.editable.defaults.send = 'always';
+
+// Default for editing is popup. I experimented with inline which is a little
+// nicer in that it shows up right under the mouse instead of nearby. However,
+// the inline box is taller than the regular content, which causes the page
+// layout to shift unless we make the table rows tall, which leaves a lot of
+// wasted space when not editing. Also inline can get cut off if the page is
+// too narrow, when the popup box will just move to do the right thing.
+//$.fn.editable.defaults.mode = 'inline';
+
$.fn.editable.defaults.params = function (params) {
var a = {};
var key = params.pk.key;
a[key] = {};
a[key][params.name] = params.value;
return a;
-};
\ No newline at end of file
+};
+
+$.fn.editable.defaults.validate = function (value) {
+ if (value == "***invalid***") {
+ return "Invalid selection";
+ }
+}
--- /dev/null
+
+(function() {
+ var run_pipeline_button_state = function() {
+ var a = $('a.editable.required.editable-empty');
+ if (a.length > 0) {
+ $("#run-pipeline-button").addClass("disabled");
+ }
+ else {
+ $("#run-pipeline-button").removeClass("disabled");
+ }
+ }
+
+ $.fn.editable.defaults.success = function (response, newValue) {
+ var tag = $(this);
+ if (tag.hasClass("required")) {
+ if (newValue && newValue.trim() != "") {
+ tag.removeClass("editable-empty");
+ tag.parent().css("background-color", "");
+ tag.parent().prev().css("background-color", "");
+ }
+ else {
+ tag.addClass("editable-empty");
+ tag.parent().css("background-color", "#ffdddd");
+ tag.parent().prev().css("background-color", "#ffdddd");
+ }
+ }
+ run_pipeline_button_state();
+ }
+
+ $(window).on('load', function() {
+ var a = $('a.editable.required');
+ for (var i = 0; i < a.length; i++) {
+ var tag = $(a[i]);
+ if (tag.hasClass("editable-empty")) {
+ tag.parent().css("background-color", "#ffdddd");
+ tag.parent().prev().css("background-color", "#ffdddd");
+ }
+ else {
+ tag.parent().css("background-color", "");
+ tag.parent().prev().css("background-color", "");
+ }
+ }
+ run_pipeline_button_state();
+ } );
+
+})();
+++ /dev/null
-# Place all the behaviors and hooks related to the matching controller here.
-# All this logic will automatically be available in application.js.
-# You can use CoffeeScript in this file: http://jashkenas.github.com/coffee-script/
--- /dev/null
+//= require jquery
+//= require jquery_ujs
+
+/** Javascript for local persistent selection. */
+
+get_selection_list = null;
+form_selection_sources = {};
+
+jQuery(function($){
+ var storage = localStorage; // sessionStorage
+
+ get_selection_list = function() {
+ if (!storage.persistentSelection) {
+ storage.persistentSelection = JSON.stringify([]);
+ }
+ return JSON.parse(storage.persistentSelection);
+ }
+
+ var put_storage = function(lst) {
+ storage.persistentSelection = JSON.stringify(lst);
+ }
+
+ var add_selection = function(uuid, name, href, type) {
+ var lst = get_selection_list();
+ lst.push({"uuid": uuid, "name": name, "href": href, "type": type});
+ put_storage(lst);
+ update_count();
+ };
+
+ var remove_selection = function(uuid) {
+ var lst = get_selection_list();
+ for (var i = 0; i < lst.length; i++) {
+ if (lst[i].uuid == uuid) {
+ lst.splice(i, 1);
+ i--;
+ }
+ }
+ put_storage(lst);
+ update_count();
+ };
+
+ var remove_selection_click = function(e) {
+ remove_selection($(this).val());
+ };
+
+ var clear_selections = function() {
+ put_storage([]);
+ update_count();
+ }
+
+ var update_count = function(e) {
+ var lst = get_selection_list();
+ $("#persistent-selection-count").text(lst.length);
+ if (lst.length > 0) {
+ $('#selection-form-content').html(
+ '<li><a href="#" id="clear_selections_button">Clear selections</a></li>'
+ + '<li><input type="submit" name="combine_selected_files_into_collection" '
+ + ' id="combine_selected_files_into_collection" '
+ + ' value="Combine selected collections and files into a new collection" /></li>'
+ + '<li class="notification"><table style="width: 100%"></table></li>');
+
+ for (var i = 0; i < lst.length; i++) {
+ $('#selection-form-content > li > table').append("<tr>"
+ + "<td>"
+ + "<input class='remove-selection' name='selection[]' type='checkbox' value='" + lst[i].uuid + "' checked='true' data-stoppropagation='true' />"
+ + "</td>"
+
+ + "<td>"
+ + "<div style='padding-left: 1em'><a href=\"" + lst[i].href + "\">" + lst[i].name + "</a></div>"
+ + "</td>"
+
+ + "<td style=\"vertical-align: top\">"
+ + "<span style=\"padding-right: 1em\">" + lst[i].type + "</span>"
+ + "</td>"
+
+ + "</tr>");
+ }
+ } else {
+ $('#selection-form-content').html("<li class='notification empty'>No selections.</li>");
+ }
+
+ var checkboxes = $('.persistent-selection:checkbox');
+ for (i = 0; i < checkboxes.length; i++) {
+ for (var j = 0; j < lst.length; j++) {
+ if (lst[j].uuid == $(checkboxes[i]).val()) {
+ checkboxes[i].checked = true;
+ break;
+ }
+ }
+ if (j == lst.length) {
+ checkboxes[i].checked = false;
+ }
+ }
+
+ $('.remove-selection').on('click', remove_selection_click);
+ $('#clear_selections_button').on('click', clear_selections);
+ };
+
+ $(document).
+ on('change', '.persistent-selection:checkbox', function(e) {
+ //console.log($(this));
+ //console.log($(this).val());
+
+ var inc = 0;
+ if ($(this).is(":checked")) {
+ add_selection($(this).val(), $(this).attr('friendly_name'), $(this).attr('href'), $(this).attr('friendly_type'));
+ }
+ else {
+ remove_selection($(this).val());
+ }
+ });
+
+
+ $(window).on('load storage', update_count);
+
+ $('#selection-form-content').on("click", function(e) {
+ e.stopPropagation();
+ });
+});
+
+add_form_selection_sources = null;
+select_form_sources = null;
+
+(function() {
+ var form_selection_sources = {};
+ add_form_selection_sources = function (src) {
+ for (var i = 0; i < src.length; i++) {
+ var t = form_selection_sources[src[i].type];
+ if (!t) {
+ t = form_selection_sources[src[i].type] = {};
+ }
+ if (!t[src[i].uuid]) {
+ t[src[i].uuid] = src[i];
+ }
+ }
+ };
+
+ select_form_sources = function(type) {
+ var ret = [];
+
+ if (get_selection_list) {
+ var lst = get_selection_list();
+ if (lst.length > 0) {
+ var text = "― Selections ―";
+ var span = document.createElement('span');
+ span.innerHTML = text;
+ ret.push({text: span.innerHTML, value: "***invalid***"});
+
+ for (var i = 0; i < lst.length; i++) {
+ if (lst[i].type == type) {
+ ret.push({text: lst[i].name, value: lst[i].uuid})
+ }
+ }
+ }
+ }
+
+ var text = "― Recent ―";
+ var span = document.createElement('span');
+ span.innerHTML = text;
+ ret.push({text: span.innerHTML, value: "***invalid***"});
+
+ var t = form_selection_sources[type];
+ for (var key in t) {
+ if (t.hasOwnProperty(key)) {
+ var obj = t[key];
+ ret.push({text: obj.name, value: obj.uuid})
+ }
+ }
+ return ret;
+ };
+})();
+
}
function smart_scroll_fixup(s) {
- console.log(s);
+ //console.log(s);
if (s != null && s.type == 'shown.bs.tab') {
s = [s.target];
}
else {
s = $(".smart-scroll");
}
- console.log(s);
+ //console.log(s);
for (var i = 0; i < s.length; i++) {
a = s[i];
var h = window.innerHeight - a.getBoundingClientRect().top - 20;
}
.inline-progress-container {
- width: 100px;
+ width: 100%;
display:inline-block;
}
position:relative;
top:1.5em;
}
+
+/* Setting the height needs to be fixed with javascript. */
+.dropdown-menu {
+ padding-right: 20px;
+ max-height: 440px;
+ width: 400px;
+ overflow-y: auto;
+}
+
// Place all the styles related to the PipelineTemplates controller here.
// They will automatically be included in application.css.
// You can use Sass (SCSS) here: http://sass-lang.com/
+
+.pipeline_color_legend {
+ padding-left: 1em;
+ padding-right: 1em;
+}
+
+table.pipeline-components-table {
+ width: 100%;
+ table-layout: fixed;
+ overflow: hidden;
+}
+
+table.pipeline-components-table thead th {
+ text-align: bottom;
+}
+table.pipeline-components-table div.progress {
+ margin-bottom: 0;
+}
+
+table.pipeline-components-table td {
+ overflow: hidden;
+ text-overflow: ellipsis;
+}
+
+td.required {
+ background: #ffdddd;
+}
--- /dev/null
+#persistent-selection-list {
+ width: 500px;
+}
+
+#selection-form-content > li > a, #selection-form-content > li > input {
+ display: block;
+ padding: 3px 20px;
+ clear: both;
+ font-weight: normal;
+ line-height: 1.42857;
+ color: rgb(51, 51, 51);
+ white-space: nowrap;
+ border: none;
+ background: transparent;
+ width: 100%;
+ text-align: left;
+}
+
+#selection-form-content li table tr {
+ padding: 3px 20px;
+ line-height: 1.42857;
+ border-top: 1px solid rgb(221, 221, 221);
+}
+
+#selection-form-content a:hover, #selection-form-content a:focus, #selection-form-content input:hover, #selection-form-content input:focus, #selection-form-content tr:hover {
+ text-decoration: none;
+ color: rgb(38, 38, 38);
+ background-color: whitesmoke;
+}
\ No newline at end of file
--- /dev/null
+class ActionsController < ApplicationController
+
+ skip_before_filter :find_object_by_uuid, only: :post
+
+ def combine_selected_files_into_collection
+ lst = []
+ files = []
+ params["selection"].each do |s|
+ m = CollectionsHelper.match(s)
+ if m and m[1] and m[2]
+ lst.append(m[1] + m[2])
+ files.append(m)
+ end
+ end
+
+ collections = Collection.where(uuid: lst)
+
+ chash = {}
+ collections.each do |c|
+ c.reload()
+ chash[c.uuid] = c
+ end
+
+ combined = ""
+ files.each do |m|
+ mt = chash[m[1]+m[2]].manifest_text
+ if m[4]
+ IO.popen(['arv-normalize', '--extract', m[4][1..-1]], 'w+b') do |io|
+ io.write mt
+ io.close_write
+ while buf = io.read(2**20)
+ combined += buf
+ end
+ end
+ else
+ combined += chash[m[1]+m[2]].manifest_text
+ end
+ end
+
+ normalized = ''
+ IO.popen(['arv-normalize'], 'w+b') do |io|
+ io.write combined
+ io.close_write
+ while buf = io.read(2**20)
+ normalized += buf
+ end
+ end
+
+ require 'digest/md5'
+
+ d = Digest::MD5.new()
+ d << normalized
+ newuuid = "#{d.hexdigest}+#{normalized.length}"
+
+ env = Hash[ENV].
+ merge({
+ 'ARVADOS_API_HOST' =>
+ $arvados_api_client.arvados_v1_base.
+ sub(/\/arvados\/v1/, '').
+ sub(/^https?:\/\//, ''),
+ 'ARVADOS_API_TOKEN' => Thread.current[:arvados_api_token],
+ 'ARVADOS_API_HOST_INSECURE' =>
+ Rails.configuration.arvados_insecure_https ? 'true' : 'false'
+ })
+
+ IO.popen([env, 'arv-put', '--raw'], 'w+b') do |io|
+ io.write normalized
+ io.close_write
+ while buf = io.read(2**20)
+
+ end
+ end
+
+ newc = Collection.new({:uuid => newuuid, :manifest_text => normalized})
+ newc.save!
+
+ chash.each do |k,v|
+ l = Link.new({
+ tail_kind: "arvados#collection",
+ tail_uuid: k,
+ head_kind: "arvados#collection",
+ head_uuid: newuuid,
+ link_class: "provenance",
+ name: "provided"
+ })
+ l.save!
+ end
+
+ redirect_to controller: 'collections', action: :show, id: newc.uuid
+ end
+
+ def post
+ if params["combine_selected_files_into_collection"]
+ combine_selected_files_into_collection
+ else
+ redirect_to :back
+ end
+ end
+end
def unprocessable(message=nil)
@errors ||= []
+
@errors << message if message
render_error status: 422
end
def create
@object ||= model_class.new params[model_class.to_s.underscore.singularize]
@object.save!
+
respond_to do |f|
f.json { render json: @object }
f.html {
end
Collection.where(uuid: @object.uuid).each do |u|
- @prov_svg = ProvenanceHelper::create_provenance_graph u.provenance, "provenance_svg", {:direction => :top_down, :combine_jobs => :script_only} rescue nil
- @used_by_svg = ProvenanceHelper::create_provenance_graph u.used_by, "used_by_svg", {:direction => :top_down, :combine_jobs => :script_only, :pdata_only => true} rescue nil
+ puts request
+ @prov_svg = ProvenanceHelper::create_provenance_graph(u.provenance, "provenance_svg",
+ {:request => request,
+ :direction => :top_down,
+ :combine_jobs => :script_only}) rescue nil
+ @used_by_svg = ProvenanceHelper::create_provenance_graph(u.used_by, "used_by_svg",
+ {:request => request,
+ :direction => :top_down,
+ :combine_jobs => :script_only,
+ :pdata_only => true}) rescue nil
end
end
nodes << c
end
- @svg = ProvenanceHelper::create_provenance_graph nodes, "provenance_svg", {:all_script_parameters => true, :script_version_nodes => true}
+ @svg = ProvenanceHelper::create_provenance_graph nodes, "provenance_svg", {
+ :request => request,
+ :all_script_parameters => true,
+ :script_version_nodes => true}
end
def index
class KeepDisksController < ApplicationController
+ def create
+ defaults = { is_readable: true, is_writable: true }
+ @object = KeepDisk.new defaults.merge(params[:keep_disk] || {})
+ super
+ end
end
end
def show
+ if @object.components.empty? and @object.pipeline_template_uuid
+ template = PipelineTemplate.find(@object.pipeline_template_uuid)
+ pipeline = {}
+ template.components.each do |component_name, component_props|
+ pipeline[component_name] = {}
+ component_props.each do |k, v|
+ if k == :script_parameters
+ pipeline[component_name][:script_parameters] = {}
+ v.each do |param_name, param_value|
+ if param_value.is_a? Hash
+ if param_value[:value]
+ pipeline[component_name][:script_parameters][param_name] = param_value[:value]
+ elsif param_value[:default]
+ pipeline[component_name][:script_parameters][param_name] = param_value[:default]
+ elsif param_value[:optional] != nil or param_value[:required] != nil or param_value[:dataclass] != nil
+ pipeline[component_name][:script_parameters][param_name] = ""
+ else
+ pipeline[component_name][:script_parameters][param_name] = param_value
+ end
+ else
+ pipeline[component_name][:script_parameters][param_name] = param_value
+ end
+ end
+ else
+ pipeline[component_name][k] = v
+ end
+ end
+ end
+ @object.components= pipeline
+ @object.save
+ end
+
@pipelines = [@object]
if params[:compare]
provenance, pips = graph(@pipelines)
@prov_svg = ProvenanceHelper::create_provenance_graph provenance, "provenance_svg", {
+ :request => request,
:all_script_parameters => true,
:combine_jobs => :script_and_version,
:script_version_nodes => true,
@pipelines = @objects
@prov_svg = ProvenanceHelper::create_provenance_graph provenance, "provenance_svg", {
+ :request => request,
:all_script_parameters => true,
:combine_jobs => :script_and_version,
:script_version_nodes => true,
%w(Compare Graph)
end
+ def update
+ updates = params[@object.class.to_s.underscore.singularize.to_sym]
+ if updates["components"]
+ require 'deep_merge/rails_compat'
+ updates["components"] = updates["components"].deeper_merge(@object.components)
+ end
+ super
+ end
+
+ def index
+ @objects ||= model_class.limit(20).all
+ super
+ end
+
protected
def for_comparison v
if v.is_a? Hash or v.is_a? Array
class PipelineTemplatesController < ApplicationController
+
+ def show
+ @objects = []
+ PipelineInstance.where(pipeline_template_uuid: @object.uuid).each do |pipeline|
+ @objects.push(pipeline)
+ end
+ super
+ end
+
+ def show_pane_list
+ %w(Components Pipelines Attributes Metadata JSON API)
+ end
+
end
class UsersController < ApplicationController
skip_before_filter :find_object_by_uuid, :only => :welcome
skip_around_filter :thread_with_mandatory_api_token, :only => :welcome
+ before_filter :ensure_current_user_is_admin, only: :sudo
def welcome
if current_user
end
end
+ def show_pane_list
+ if current_user.andand.is_admin
+ super | %w(Admin)
+ else
+ super
+ end
+ end
+
+ def sudo
+ resp = $arvados_api_client.api(ApiClientAuthorization, '', {
+ api_client_authorization: {
+ owner_uuid: @object.uuid
+ }
+ })
+ redirect_to root_url(api_token: resp[:api_token])
+ end
+
def home
@showallalerts = false
@my_ssh_keys = AuthorizedKey.where(authorized_user_uuid: current_user.uuid)
controller.current_user
end
+ def self.match_uuid(uuid)
+ /^([0-9a-z]{5})-([0-9a-z]{5})-([0-9a-z]{15})$/.match(uuid.to_s)
+ end
+
def current_api_host
Rails.configuration.arvados_v1_base.gsub /https?:\/\/|\/arvados\/v1/,''
end
end
end
style_opts[:class] = (style_opts[:class] || '') + ' nowrap'
- link_to link_name, { controller: resource_class.to_s.underscore.pluralize, action: 'show', id: link_uuid }, style_opts
+ link_to link_name, { controller: resource_class.to_s.tableize, action: 'show', id: link_uuid }, style_opts
else
attrvalue
end
:class => "editable"
}.merge(htmloptions)
end
+
+ def render_editable_subattribute(object, attr, subattr, template, htmloptions={})
+ if object
+ attrvalue = object.send(attr)
+ subattr.each do |k|
+ if attrvalue and attrvalue.is_a? Hash
+ attrvalue = attrvalue[k]
+ else
+ break
+ end
+ end
+ end
+
+ datatype = nil
+ required = true
+ if template
+ #puts "Template is #{template.class} #{template.is_a? Hash} #{template}"
+ if template.is_a? Hash
+ if template[:output_of]
+ return raw("<span class='label label-default'>#{template[:output_of]}</span>")
+ end
+ if template[:dataclass]
+ dataclass = template[:dataclass]
+ end
+ if template[:optional] != nil
+ required = (template[:optional] != "true")
+ end
+ if template[:required] != nil
+ required = template[:required]
+ end
+ end
+ end
+
+ rsc = template
+ if template.is_a? Hash
+ if template[:value]
+ rsc = template[:value]
+ elsif template[:default]
+ rsc = template[:default]
+ end
+ end
+
+ return link_to_if_arvados_object(rsc) if !object
+ return link_to_if_arvados_object(attrvalue) if !object.attribute_editable? attr
+
+ if dataclass
+ begin
+ dataclass = dataclass.constantize
+ rescue NameError
+ end
+ else
+ dataclass = ArvadosBase.resource_class_for_uuid(rsc)
+ end
+
+ if dataclass && dataclass.is_a?(Class)
+ datatype = 'select'
+ elsif dataclass == 'number'
+ datatype = 'number'
+ else
+ if template.is_a? Array
+ # ?!?
+ elsif template.is_a? String
+ if /^\d+$/.match(template)
+ datatype = 'number'
+ else
+ datatype = 'text'
+ end
+ end
+ end
+
+ id = "#{object.uuid}-#{subattr.join('-')}"
+ dn = "[#{attr}]"
+ subattr.each do |a|
+ dn += "[#{a}]"
+ end
+
+ if attrvalue.is_a? String
+ attrvalue = attrvalue.strip
+ end
+
+ if dataclass and dataclass.is_a? Class
+ items = []
+ if attrvalue and !attrvalue.empty?
+ items.append({name: attrvalue, uuid: attrvalue, type: dataclass.to_s})
+ end
+ #dataclass.where(uuid: attrvalue).each do |item|
+ # items.append({name: item.uuid, uuid: item.uuid, type: dataclass.to_s})
+ #end
+ dataclass.limit(10).each do |item|
+ items.append({name: item.uuid, uuid: item.uuid, type: dataclass.to_s})
+ end
+ end
+
+ lt = link_to attrvalue, '#', {
+ "data-emptytext" => "none",
+ "data-placement" => "bottom",
+ "data-type" => datatype,
+ "data-url" => url_for(action: "update", id: object.uuid, controller: object.class.to_s.pluralize.underscore),
+ "data-title" => "Set value for #{subattr[-1].to_s}",
+ "data-name" => dn,
+ "data-pk" => "{id: \"#{object.uuid}\", key: \"#{object.class.to_s.underscore}\"}",
+ "data-showbuttons" => "false",
+ "data-value" => attrvalue,
+ :class => "editable #{'required' if required}",
+ :id => id
+ }.merge(htmloptions)
+
+ lt += raw("\n<script>")
+
+ if items and items.length > 0
+ lt += raw("add_form_selection_sources(#{items.to_json});\n")
+ end
+
+ lt += raw("$('##{id}').editable({source: function() { return select_form_sources('#{dataclass}'); } });\n")
+
+ lt += raw("</script>")
+
+ lt
+ end
end
{source: x.tail_uuid, target: x.head_uuid, type: x.name}
end
end
+
+ def self.match(uuid)
+ /^([a-f0-9]{32})(\+[0-9]+)?(\+.*?)?(\/.*)?$/.match(uuid.to_s)
+ end
end
module PipelineInstancesHelper
- def pipeline_summary object=nil
- object ||= @object
- ret = {todo:0, running:0, queued:0, done:0, failed:0, total:0}
- object.components.values.each do |c|
- ret[:total] += 1
- case
- when !c[:job]
- ret[:todo] += 1
- when c[:job][:success]
- ret[:done] += 1
- when c[:job][:failed]
- ret[:failed] += 1
- when c[:job][:finished_at]
- ret[:running] += 1 # XXX finished but !success and !failed??
- when c[:job][:started_at]
- ret[:running] += 1
- else
- ret[:queued] += 1
- end
- end
- ret.merge! Hash[ret.collect do |k,v|
- [('percent_' + k.to_s).to_sym,
- ret[:total]<1 ? 0 : (100.0*v/ret[:total]).floor]
- end]
- ret
- end
def pipeline_jobs object=nil
object ||= @object
end
def render_pipeline_job pj
- if pj[:percent_done]
- pj[:progress_bar] = raw("<div class=\"progress\" style=\"width:100px\"><span class=\"progress-bar progress-bar-success\" style=\"width:#{pj[:percent_done]}%\"></span><span class=\"progress-bar\" style=\"width:#{pj[:percent_running]}%\"></span></div>")
- elsif pj[:progress]
- raw("<div class=\"progress\" style=\"width:100px\"><span class=\"progress-bar\" style=\"width:#{pj[:progress]*100}%\"></span></div>")
- end
+ pj[:progress_bar] = render partial: 'job_progress', locals: {:j => pj[:job]}
pj[:output_link] = link_to_if_arvados_object pj[:output]
pj[:job_link] = link_to_if_arvados_object pj[:job][:uuid]
pj
end
+
protected
def pipeline_jobs_newschool object
ret = []
i = -1
- object.components.each do |cname, c|
+
+ comp = []
+
+ template = PipelineTemplate.find(@object.pipeline_template_uuid) rescue nil
+ if template
+ order = PipelineTemplatesHelper::sort_components(template.components)
+ order.each do |k|
+ if object.components[k]
+ comp.push([k, object.components[k]])
+ end
+ end
+ else
+ object.components.each do |k, v|
+ comp.push([k, v])
+ end
+ end
+
+ comp.each do |cname, c|
+ puts cname, c
i += 1
pj = {index: i, name: cname}
pj[:job] = c[:job].is_a?(Hash) ? c[:job] : {}
+require 'tsort'
+
+class Hash
+ include TSort
+ def tsort_each_node(&block)
+ keys.sort.each(&block)
+ end
+
+ def tsort_each_child(node)
+ if self[node]
+ self[node][:script_parameters].sort.map do |k, v|
+ if v.is_a? Hash and v[:output_of]
+ yield v[:output_of].to_sym
+ end
+ end
+ end
+ end
+end
+
module PipelineTemplatesHelper
+ def self.sort_components(components)
+ components.tsort
+ end
end
@visited = {}
@jobs = {}
end
-
+
def self.collection_uuid(uuid)
- m = /^([a-f0-9]{32}(\+[0-9]+)?)(\+.*)?$/.match(uuid.to_s)
+ m = CollectionsHelper.match(uuid)
if m
- #if m[2]
- return m[1]
- #else
+ if m[2]
+ return m[1]+m[2]
+ else
+ return m[1]
+ end
# Collection.where(uuid: ['contains', m[1]]).each do |u|
# puts "fixup #{uuid} to #{u.uuid}"
# return u.uuid
end
def describe_node(uuid)
+ uuid = uuid.to_sym
bgcolor = determine_fillcolor @opts[:pips][uuid] if @opts[:pips]
rsc = ArvadosBase::resource_class_for_uuid uuid.to_s
if rsc
- href = "/#{rsc.to_s.underscore.pluralize rsc}/#{uuid}"
+ href = Rails.application.routes.url_helpers.url_for ({:controller => rsc.to_s.tableize,
+ :action => :show,
+ :id => uuid.to_s,
+ :host => @opts[:request].host,
+ :port => @opts[:request].port})
#"\"#{uuid}\" [label=\"#{rsc}\\n#{uuid}\",href=\"#{href}\"];\n"
if rsc == Collection
#puts "empty!"
return "\"#{uuid}\" [label=\"(empty collection)\"];\n"
end
+ puts "#{uuid.class} #{@pdata[uuid]}"
if @pdata[uuid]
#puts @pdata[uuid]
if @pdata[uuid][:name]
return "\"#{uuid}\" [label=\"#{@pdata[uuid][:name]}\",href=\"#{href}\",shape=oval,#{bgcolor}];\n"
- else
+ else
files = nil
if @pdata[uuid].respond_to? :files
files = @pdata[uuid].files
if i < files.length
label += "\\n⋮"
end
+ #puts "#{uuid} #{label} #{files}"
return "\"#{uuid}\" [label=\"#{label}\",href=\"#{href}\",shape=oval,#{bgcolor}];\n"
end
end
gr = "\"#{head}\" -> \"#{tail}\""
end
if extra.length > 0
- gr += "["
+ gr += " ["
extra.each do |k, v|
gr += "#{k}=\"#{v}\","
end
gr += edge(job_uuid(job), job[:script_version], {:label => "script_version"})
end
end
+ elsif rsc == Link
+ # do nothing
else
gr += describe_node(uuid)
end
@pdata.each do |k, link|
if link[:head_uuid] == uuid.to_s and link[:link_class] == "provenance"
+ href = Rails.application.routes.url_helpers.url_for ({:controller => Link.to_s.tableize,
+ :action => :show,
+ :id => link[:uuid],
+ :host => @opts[:request].host,
+ :port => @opts[:request].port})
+
gr += describe_node(link[:tail_uuid])
- gr += edge(link[:head_uuid], link[:tail_uuid], {:label => link[:name], :href => "/links/#{link[:uuid]}"})
+ gr += edge(link[:head_uuid], link[:tail_uuid], {:label => link[:name], :href => href})
gr += generate_provenance_edges(link[:tail_uuid])
end
end
def describe_jobs
gr = ""
@jobs.each do |k, v|
- gr += "\"#{k}\" [href=\"/jobs?"
+ href = Rails.application.routes.url_helpers.url_for ({:controller => Job.to_s.tableize,
+ :action => :index,
+ :host => @opts[:request].host,
+ :port => @opts[:request].port})
+
+ gr += "\"#{k}\" [href=\"#{href}?"
n = 0
v.each do |u|
gr += "\",label=\""
if @opts[:combine_jobs] == :script_only
- gr += uuid = "#{v[0][:script]}"
+ gr += "#{v[0][:script]}"
elsif @opts[:combine_jobs] == :script_and_version
- gr += uuid = "#{v[0][:script]}"
+ gr += "#{v[0][:script]}" # Just show the name but the nodes will be distinct
else
- gr += uuid = "#{v[0][:script]}\\n#{v[0][:finished_at]}"
+ gr += "#{v[0][:script]}\\n#{v[0][:finished_at]}"
end
gr += "\",#{determine_fillcolor n}];\n"
end
gr += "}"
svg = ""
- #puts gr
-
+ puts gr
+
require 'open3'
Open3.popen2("dot", "-Tsvg") do |stdin, stdout, wait_thr|
attr_reader :kind
@columns
end
+
def self.column(name, sql_type = nil, default = nil, null = true)
ActiveRecord::ConnectionAdapters::Column.new(name.to_s, default, sql_type.to_s, null)
end
+
def self.attribute_info
self.columns
@attribute_info
end
+
def self.find(uuid, opts={})
if uuid.class != String or uuid.length < 27 then
raise 'argument to find() must be a uuid string. Acceptable formats: warehouse locator or string with format xxxxx-xxxxx-xxxxxxxxxxxxxxx'
end
new.private_reload(hash)
end
+
def self.order(*args)
ArvadosResourceList.new(self).order(*args)
end
+
def self.where(*args)
ArvadosResourceList.new(self).where(*args)
end
+
def self.limit(*args)
ArvadosResourceList.new(self).limit(*args)
end
+
def self.eager(*args)
ArvadosResourceList.new(self).eager(*args)
end
+
def self.all(*args)
ArvadosResourceList.new(self).all(*args)
end
+
def save
obdata = {}
self.class.columns.each do |col|
end
end
+ @new_record = false
+
self
end
+
def save!
self.save or raise Exception.new("Save failed")
end
@links = $arvados_api_client.api Link, '', { _method: 'GET', where: o, eager: true }
@links = $arvados_api_client.unpack_api_response(@links)
end
+
def all_links
return @all_links if @all_links
res = $arvados_api_client.api Link, '', {
}
@all_links = $arvados_api_client.unpack_api_response(res)
end
+
def reload
private_reload(self.uuid)
end
+
def private_reload(uuid_or_hash)
raise "No such object" if !uuid_or_hash
if uuid_or_hash.is_a? Hash
end
end
@all_links = nil
+ @new_record = false
self
end
+
+ def to_param
+ uuid
+ end
+
def dup
super.forget_uuid!
end
(name if self.respond_to? :name) || uuid
end
+ def selection_label
+ friendly_link_name
+ end
+
protected
def forget_uuid!
class Collection < ArvadosBase
+
def total_bytes
if files
tot = 0
def used_by
$arvados_api_client.api "collections/#{self.uuid}/", "used_by"
end
+
end
end
end
end
-
+
def attribute_editable?(attr)
- attr == 'name'
+ attr.to_sym == :name || (attr.to_sym == :components and self.active == nil)
end
def attributes_for_display
<% end %>
<% content_for :js do %>
- $(window).on('load', function() {
- $('ul.nav-tabs > li > a').on('shown.bs.tab', smart_scroll_fixup);
- });
+ $(window).on('load', smart_scroll_fixup);
+ $(document).on('shown.bs.tab', 'ul.nav-tabs > li > a', smart_scroll_fixup);
<% end %>
--- /dev/null
+<% percent_total_tasks = 100 / (j[:tasks_summary][:done] + j[:tasks_summary][:running] + j[:tasks_summary][:failed] + j[:tasks_summary][:todo]) rescue 0 %>
+
+<% if defined? scaleby %>
+ <% percent_total_tasks *= scaleby %>
+<% end %>
+
+<% if not defined? scaleby %>
+ <div class="progress">
+<% end %>
+
+<span class="progress-bar progress-bar-success" style="width: <%= j[:tasks_summary][:done] * percent_total_tasks rescue 0 %>%;">
+</span>
+<span class="progress-bar progress-bar-danger" style="width: <%= j[:tasks_summary][:failed] * percent_total_tasks rescue 0 %>%;">
+</span>
+<span class="progress-bar" style="width: <%= j[:tasks_summary][:running] * percent_total_tasks rescue 0 %>%;">
+</span>
+
+<% if not defined? scaleby %>
+</div>
+<% end %>
--- /dev/null
+<% if j[:success] %>
+ <span class="label label-success"><%= if defined? title then title else 'success' end %></span>
+<% elsif j[:success] == false %>
+ <span class="label label-danger"><%= if defined? title then title else 'failed' end %></span>
+<% elsif j[:finished_at] %>
+ <span class="label label-default"><%= if defined? title then title else 'finished' end %></span>
+<% elsif j[:started_at] %>
+ <span class="label label-info"><%= if defined? title then title else 'running' end %></span>
+<% else %>
+ <span class="label label-default"><%= if defined? title then title else 'not running' end %></span>
+<% end %>
--- /dev/null
+<% component_frac = 1.0 / p.components.length %>
+<div class="progress">
+ <% p.components.each do |k,c| %>
+ <% if c[:job] %>
+ <%= render partial: "job_progress", locals: {:j => c[:job], :scaleby => component_frac } %>
+ <% end %>
+ <% end %>
+</div>
--- /dev/null
+<% if p.success %>
+ <span class="label label-success">finished</span>
+<% elsif p.success == false %>
+ <span class="label label-danger">failed</span>
+<% elsif p.active %>
+ <span class="label label-info">running</span>
+<% else %>
+ <% if (p.components.select do |k,v| v[:job] end).length == 0 %>
+ <span class="label label-default">not started</span>
+ <% else %>
+ <span class="label label-default">not running</span>
+ <% end %>
+<% end %>
--- /dev/null
+<%if object %>
+<%= check_box_tag 'uuids[]', object.uuid, false, {
+ :class => 'persistent-selection',
+ :friendly_type => object.class.name,
+ :friendly_name => object.selection_label,
+ :href => "#{url_for controller: object.class.name.tableize, action: 'show', id: object.uuid }"
+} %>
+<% end %>
<% attr_blacklist = ' created_at modified_at modified_by_user_uuid modified_by_client_uuid updated_at' %>
+<%= form_tag do |f| %>
+
<table class="table table-condensed arv-index">
<thead>
<tr>
+ <th></th>
<% @objects.first.attributes_for_display.each do |attr, attrvalue| %>
<% next if attr_blacklist.index(" "+attr) %>
<th class="arv-attr-<%= attr %>">
<tbody>
<% @objects.each do |object| %>
<tr data-object-uuid="<%= object.uuid %>">
+ <td>
+ <%= render :partial => "selection_checkbox", :locals => {:object => object} %>
+ </td>
+
<% object.attributes_for_display.each do |attr, attrvalue| %>
<% next if attr_blacklist.index(" "+attr) %>
<td class="arv-object-<%= object.class.to_s %> arv-attr-<%= attr %>">
</table>
<% end %>
+
+<% end %>
<% @collections.each do |c| %>
<tr class="collection" data-object-uuid="<%= c.uuid %>">
+ <td>
+ <%= render :partial => "selection_checkbox", :locals => {:object => c} %>
+ </td>
<td>
<%= link_to_if_arvados_object c.uuid %>
</td>
<table class="table table-condensed table-fixedlayout">
<colgroup>
+ <col width="4%" />
<col width="35%" />
<col width="40%" />
<col width="15%" />
</colgroup>
<thead>
<tr>
+ <th></th>
<th>path</th>
<th>file</th>
<th style="text-align:right">size</th>
</tr>
</thead><tbody>
<% if @object then @object.files.sort_by{|f|[f[0],f[1]]}.each do |file| %>
- <% file_path = "#{file[0]}/#{file[1]}" %>
- <tr>
- <td>
- <%= file[0] %>
- </td>
+ <% f0 = file[0] %>
+ <% f0 = '' if f0 == '.' %>
+ <% f0 = f0[2..-1] if f0[0..1] == './' %>
+ <% f0 += '/' if not f0.empty? %>
+ <% file_path = "#{f0}#{file[1]}" %>
+ <tr>
+ <td>
+ <%= check_box_tag 'uuids[]', @object.uuid+'/'+file_path, false, {
+ :class => 'persistent-selection',
+ :friendly_type => "File",
+ :friendly_name => "#{@object.uuid}/#{file_path}",
+ :href => "#{url_for controller: 'collections', action: 'show', id: @object.uuid }/#{file_path}"
+ } %>
+ </td>
+ <td>
+ <%= file[0] %>
+ </td>
- <td>
- <%= link_to file[1], {controller: 'collections', action: 'show_file', uuid: @object.uuid, file: file_path, size: file[2], disposition: 'inline'}, {title: 'View in browser'} %>
- </td>
+ <td>
+ <%= link_to file[1], {controller: 'collections', action: 'show_file', uuid: @object.uuid, file: file_path, size: file[2], disposition: 'inline'}, {title: 'View in browser'} %>
+ </td>
- <td style="text-align:right">
- <%= raw(human_readable_bytes_html(file[2])) %>
- </td>
+ <td style="text-align:right">
+ <%= raw(human_readable_bytes_html(file[2])) %>
+ </td>
- <td>
- <div style="display:inline-block">
- <%= link_to raw('<i class="glyphicon glyphicon-download-alt"></i>'), {controller: 'collections', action: 'show_file', uuid: @object.uuid, file: file_path, size: file[2], disposition: 'attachment'}, {class: 'btn btn-info btn-sm', title: 'Download'} %>
- </div>
- </td>
- </tr>
+ <td>
+
<div style="display:inline-block">
+ <%= link_to raw('<i class="glyphicon glyphicon-download-alt"></i>'), {controller: 'collections', action: 'show_file', uuid: @object.uuid, file: file_path, size: file[2], disposition: 'attachment'}, {class: 'btn btn-info btn-sm', title: 'Download'} %>
+ </div>
+ </td>
+ </tr>
<% end; end %>
</tbody>
</table>
<div style="padding-right: 1em">
+<%= form_tag do |f| %>
+
<table id="collections-index" class="topalign table table-condensed table-fixedlayout table-fixed-header-row">
<colgroup>
+ <col width="4%" />
<col width="10%" />
<col width="36%" />
<col width="15%" />
</colgroup>
<thead>
<tr class="contain-align-left">
+ <th></th>
<th>uuid</th>
<th>contents</th>
<th>owner</th>
<%= render partial: 'index_tbody' %>
</tbody>
</table>
+
+<% end %>
+
</div>
<% content_for :footer_js do %>
<i class="icon-plus-sign expand-collapse-row" data-id="<%= j.uuid %>" style="cursor: pointer"></i>
</td>
<td>
- <% if j.success == false %>
- <span class="badge badge-warning" title="fail">✖</span>
- <% elsif j.success %>
- <span class="badge badge-success" title="success">✔</span>
- <% elsif j.running %>
- <span class="badge badge-info" title="running">✈</span>
- <% else %>
- <span class="badge" title="queued">✉</span>
- <% end %>
+ <%= render partial: 'job_status_label', locals: {:j => j} %>
</td>
<td>
- <% if j.started_at and not j.finished_at %>
- <% percent_total_tasks = 100 / (j.tasks_summary[:running] + j.tasks_summary[:done] + j.tasks_summary[:todo]) rescue 0 %>
- <div class="progress" style="margin-bottom: 0">
- <div class="bar bar-success" style="width: <%= j.tasks_summary[:done] * percent_total_tasks rescue 0 %>%;"></div>
- <div class="bar" style="width: <%= j.tasks_summary[:running] * percent_total_tasks rescue 0 %>%; opacity: 0.3"></div>
+ <div class="inline-progress-container">
+ <%= render partial: 'job_progress', locals: {:j => j} %>
</div>
- <% end %>
</td>
<td>
<%= link_to_if_arvados_object j.uuid %>
padding-top: 1.25em;
}
- /* Setting the height needs to be fixed with javascript. */
- .dropdown-menu {
- padding-right: 20px;
- max-height: 440px;
- width: 400px;
- overflow-y: auto;
- }
-
@media (min-width: 768px) {
.left-nav {
position: fixed;
<span class="glyphicon glyphicon-arrow-right"></span>
</li>
<li>
-<%= link_to controller.breadcrumb_page_name, request.fullpath %>
+ <%= link_to controller.breadcrumb_page_name, request.fullpath %>
+ </li>
+ <li style="padding: 14px 0 14px">
+ <%= form_tag do |f| %>
+ <%= render :partial => "selection_checkbox", :locals => {:object => @object} %>
+ <% end %>
</li>
<% end %>
<% end %>
</li>
-->
- <!-- XXX placeholder for this when persistent selection is implemented
- <li class="dropdown">
+ <li class="dropdown notification-menu">
<a href="#" class="dropdown-toggle" data-toggle="dropdown">
<span class="glyphicon glyphicon-paperclip"></span>
- <span class="badge badge-alert"><%= @selection_count %></span>
+ <span class="badge" id="persistent-selection-count"></span>
<span class="caret"></span>
</a>
- <ul class="dropdown-menu" role="menu">
- <li style="padding: 10px">No selections.</li>
+ <ul class="dropdown-menu" role="menu" id="persistent-selection-list">
+ <%= form_tag '/actions' do %>
+ <div id="selection-form-content"></div>
+ <% end %>
</ul>
</li>
- -->
<% if current_user.is_active %>
<li class="dropdown notification-menu">
<% content_for :css do %>
- .pipeline_color_legend {
- padding-left: 1em;
- padding-right: 1em;
- }
-table.pipeline-components-table thead th {
- text-align: bottom;
-}
-table.pipeline-components-table div.progress {
- margin-bottom: 0;
-}
+
<% end %>
-<br />
+<% template = PipelineTemplate.find(@object.pipeline_template_uuid) rescue nil %>
+
+<%= content_for :content_top do %>
+ <% if template %>
+ <h2><%= template.name %></h2>
+ <% end %>
+<% end %>
+
+<% if @object.active != nil %>
<table class="table pipeline-components-table">
<colgroup>
- <col width="15%" />
- <col width="15%" />
- <col width="35%" />
- <col width="35%" />
+ <col style="width: 15%" />
+ <col style="width: 20%" />
+ <col style="width: 12%" />
+ <col style="width: 12%" />
+ <col style="width: 45%" />
</colgroup>
<thead>
<tr>
<th>
component
+ </th><th>
+ script, version
</th><th>
progress
<%= link_to '(refresh)', request.fullpath, class: 'refresh', remote: true, method: 'get' %>
</th><th>
- script, version
</th><th>
output
</th>
<% render_pipeline_jobs.each do |pj| %>
<tr>
<td>
+ <% job_status = render(partial: 'job_status_label',
+ locals: { :j => pj[:job], :title => pj[:name] }) %>
<% if pj[:job].andand[:uuid] %>
- <%= link_to pj[:name], job_url(id: pj[:job][:uuid]) %>
+ <%= link_to(job_status, job_url(id: pj[:job][:uuid])) %>
<% else %>
- <%= pj[:name] %>
- <% end %>
- </td><td>
- <%= pj[:progress_bar] %>
- <% if pj[:job].andand[:cancelled_at] %>
- <span class="pull-right label label-warning">cancelled</span>
- <% elsif pj[:failed] %>
- <span class="pull-right label label-warning">failed</span>
- <% elsif pj[:result] == 'queued' %>
- <span class="pull-right label">queued</span>
+ <%= job_status %>
<% end %>
</td><td>
<%= pj[:script] %>
<br /><span class="deemphasize"><%= pj[:script_version] %></span>
+ </td><td>
+ <%= pj[:progress_bar] %>
+ </td><td>
+ <%= render(partial: 'job_status_label',
+ locals: { :j => pj[:job] }) %>
</td><td>
<%= link_to_if_arvados_object pj[:output] %>
</td>
<% end %>
</tbody>
<tfoot>
- <tr><td colspan="4"></td></tr>
+ <tr><td colspan="5"></td></tr>
</tfoot>
</table>
<% if @object.active %>
<% content_for :js do %>
-setInterval(function(){$('a.refresh').click()}, 30000);
+setInterval(function(){$('a.refresh').click()}, 15000);
<% end %>
<% end %>
-<pre><%= JSON.pretty_generate @object.attributes %></pre>
+<% else %>
+
+ <p>Please set the desired input parameters for the components of this pipeline. Parameters highlighted in red are required.</p>
+
+ <% content_for :tab_line_buttons do %>
+ <%= form_tag @object, :method => :put do |f| %>
+
+ <%= hidden_field @object.class.to_s.underscore.singularize.to_sym, :active, :value => true %>
+
+ <%= button_tag "Run pipeline", {class: 'btn btn-primary pull-right', id: "run-pipeline-button"} %>
+ <% end %>
+ <% end %>
+
+ <%= render partial: 'pipeline_templates/show_components_template', locals: {:template => template, :obj => @object} %>
+
+<% end %>
<table class="table table-condensed table-fixedlayout">
<colgroup>
<col width="5%" />
- <col width="10%" />
+ <col width="15%" />
+ <col width="25%" />
<col width="20%" />
- <col width="10%" />
- <col width="30%" />
<col width="15%" />
- <col width="10%" />
+ <col width="20%" />
</colgroup>
<thead>
<tr class="contain-align-left">
Status
</th><th>
Instance
- </th><th colspan="2">
+ </th><th>
Template
</th><th>
Owner
<tr data-object-uuid="<%= ob.uuid %>">
<td>
- <%= check_box_tag 'uuids[]', ob.uuid, false %>
+ <%= check_box_tag 'uuids[]', ob.uuid, false, :class => 'persistent-selection' %>
</td><td>
- <% if ob.success %>
- <span class="label label-success">success</span>
- <% elsif ob.active %>
- <span class="label label-info">active</span>
- <% end %>
- </td><td colspan="2">
+ <%= render partial: 'pipeline_status_label', locals: {:p => ob} %>
+ </td><td colspan="1">
<%= link_to_if_arvados_object ob, friendly_name: true %>
</td><td>
<%= link_to_if_arvados_object ob.pipeline_template_uuid, friendly_name: true %>
</td>
</tr>
<tr>
- <td style="border-top: 0;" colspan="3">
+ <td style="border-top: 0;" colspan="2">
</td>
- <td style="border-top: 0; opacity: 0.5;" colspan="4">
+ <td style="border-top: 0; opacity: 0.5;" colspan="5">
<% ob.components.each do |cname, c| %>
- <% status = if !(c.is_a?(Hash) && c[:job].is_a?(Hash))
- nil
- elsif c[:job][:success]
- 'success'
- elsif c[:job][:running]
- 'info'
- else
- 'warning'
- end %>
- <span class="label label-<%= status || 'default' %>"><%= cname.to_s %></span>
+ <% if c[:job] %>
+ <%= render partial: "job_status_label", locals: {:j => c[:job], :title => cname.to_s } %>
+ <% else %>
+ <span class="label label-default"><%= cname.to_s %></span>
+ <% end %>
<% end %>
</td>
</tr>
var new_content = "<%= escape_javascript(render template: 'pipeline_instances/show') %>";
if ($('div.body-content').html() != new_content)
$('div.body-content').html(new_content);
+$(document).trigger('ajax:complete');
+<%= content_for :content_top do %>
+ <h2>Template '<%= @object.name %>'</h2>
+<% end %>
+
<table class="table topalign">
<thead>
</thead>
<tbody>
<% @object.attributes_for_display.each do |attr, attrvalue| %>
- <% if attr == 'components' and attrvalue.is_a? Hash and attrvalue[:steps].is_a? Array %>
-
- <tr><td>components[steps]</td><td>
- <table class="table">
- <% attrvalue[:steps].each_with_index do |s, i| %>
- <tr>
- <td><%= i %></td>
- <% %w(name function min_revision max_steps_per_node params).each do |key| %>
- <td>
- <% if key == 'params' %>
- <% s[key.to_sym].each do |p| %>
- <%= p[:name] %>
- <% if p[:data_locator] || p[:value] %>
- ← <%= p[:data_locator] || p[:value] %>
- <% end %>
- <%= '(optional)' if p[:optional] %>
- <br />
- <% end %>
- <% else %>
- <%= s[key.to_sym] %>
- <% end %>
- </td>
- <% end %>
- </tr>
- <% end %>
- </table>
- </td></tr>
-
- <% else %>
- <%= render partial: 'application/arvados_object_attr', locals: { attr: attr, attrvalue: attrvalue } %>
- <% end %>
+ <% if attr != 'components' %>
+ <%= render partial: 'application/arvados_object_attr', locals: { attr: attr, attrvalue: attrvalue } %>
+ <% end %>
<% end %>
</tbody>
</table>
--- /dev/null
+<% content_for :tab_line_buttons do %>
+ <%= form_tag '/pipeline_instances' do |f| %>
+ <%= hidden_field :pipeline_instance, :pipeline_template_uuid, :value => @object.uuid %>
+ <%= button_tag "Create pipeline", {class: 'btn btn-primary pull-right', id: "run-pipeline-button"} %>
+<% end %>
+<% end %>
+
+<%= render partial: 'pipeline_templates/show_components_template', locals: {:template => @object, :obj => nil} %>
--- /dev/null
+<table class="table pipeline-components-table" style="margin-top: -.1em">
+ <colgroup>
+ <col style="width: 15%" />
+ <col style="width: 20%" />
+ <col style="width: 20%" />
+ <col style="width: 45%" />
+ </colgroup>
+
+ <thead>
+ <tr>
+ <th>
+ component
+ </th><th>
+ script
+ </th><th>
+ parameter
+ </th><th>
+ value
+ </th>
+ </tr>
+ </thead>
+ <tbody>
+ <% order = PipelineTemplatesHelper::sort_components(template.components) %>
+ <% puts "order is #{order}" %>
+ <% order.each do |k| %>
+ <% template_value = template.components[k] %>
+ <% puts "#{k} #{template_value}" %>
+ <% if not template_value then next end %>
+ <tr>
+ <td><span class="label label-default"><%= k %></span></td>
+
+ <td><%= render_editable_subattribute obj, :components, [k, :script], template_value[:script] %></td>
+
+ <td>script version</td>
+
+ <td>
+ <%= render_editable_subattribute obj, :components, [k, :script_version], template_value[:script_version] %>
+ </td>
+ </tr>
+
+ <% if template_value[:script_parameters].length > 0 %>
+ <% template_value[:script_parameters].each do |p, tv| %>
+ <tr>
+ <td style="border-top: none"></td>
+ <td style="border-top: none"></td>
+
+ <td class="property-edit-row"><%= p %></td>
+ <td class="property-edit-row"><%= render_editable_subattribute obj, :components, [k, :script_parameters, p.to_sym], tv %></td>
+ <% end %>
+ </tr>
+ <% end %>
+ <% end %>
+ </tbody>
+</table>
--- /dev/null
+
+ <%= render partial: 'pipeline_instances/show_recent' %>
+<% content_for :css do %>
+ .playbutton {
+ color: white;
+ background: rgb(91, 192, 222);
+ border: 0px;
+ border-radius: 3px;
+ padding: 0px 3px;
+ }
+ .playbutton:hover {
+ background: rgb(57, 179, 215);
+ }
+<% end %>
+
<table class="table table-hover">
<thead>
<tr class="contain-align-left">
<th>
+ </th><th>
id
</th><th>
name
<% @objects.sort_by { |ob| ob[:created_at] }.reverse.each do |ob| %>
<tr>
+ <td>
+ <%= form_tag '/pipeline_instances' do |f| %>
+ <%= hidden_field :pipeline_instance, :pipeline_template_uuid, :value => ob.uuid %>
+ <%= button_tag nil, {class: 'playbutton', title: "Run #{ob.name}"} do %>
+ <span class="glyphicon glyphicon-play"></span>
+ <% end %>
+ <% end %>
+ </td>
<td>
<%= link_to_if_arvados_object ob %>
</td><td>
--- /dev/null
+<p>As an admin, you can log in as this user. When you’ve
+finished, you will need to log out and log in again with your own
+account.</p>
+
+<blockquote>
+<%= button_to "Log in as #{@object.full_name}", sudo_user_url(id: @object.uuid), class: 'btn btn-primary' %>
+</blockquote>
<col width="20%" />
<col width="20%" />
<col width="13%" />
- <col width="27%" />
+ <col width="13%" />
+ <col width="20%" />
</colgroup>
<tr>
<th>Log</th>
<th>Age</th>
<th>Status</th>
+ <th>Progress</th>
</tr>
<% @my_jobs[0..6].each do |j| %>
</td>
<td>
- <% if j.success %>
- <span class="label label-success">finished</span>
- <% elsif j.success == false %>
- <span class="label label-danger">failed</span>
- <% elsif j.finished_at %>
- <span class="label">finished?</span>
- <% elsif j.started_at %>
- <span class="label label-info">running</span>
- <% else %>
- <span class="label">queued</span>
- <% end %>
- <% percent_total_tasks = 100 / (j.tasks_summary[:running] + j.tasks_summary[:done] + j.tasks_summary[:todo]) rescue 0 %>
- <div class="inline-progress-container pull-right">
- <div class="progress">
- <span class="progress-bar progress-bar-success" style="width: <%= j.tasks_summary[:done] * percent_total_tasks rescue 0 %>%;">
- </span>
- <span class="progress-bar" style="width: <%= j.tasks_summary[:running] * percent_total_tasks rescue 0 %>%;">
- </span>
- <% if j.success == false %>
- <span class="progress-bar progress-bar-danger" style="width: <%= tasks_summary[:failed] * percent_total_tasks rescue 0 %>%;">
- </span>
- <% end %>
- </div>
+ <%= render partial: 'job_status_label', locals: {:j => j} %>
+</td>
+<td>
+ <div class="inline-progress-container">
+ <%= render partial: 'job_progress', locals: {:j => j} %>
</div>
</td>
<col width="30%" />
<col width="30%" />
<col width="13%" />
- <col width="27%" />
+ <col width="13%" />
+ <col width="20%" />
</colgroup>
<tr>
<th>Template</th>
<th>Age</th>
<th>Status</th>
+ <th>Progress</th>
</tr>
<% @my_pipelines[0..6].each do |p| %>
</td>
<td>
- <% if p.success %>
- <span class="label label-success">finished</span>
- <% elsif p.success == false %>
- <span class="label label-danger">failed</span>
- <% elsif p.active and p.modified_at < 30.minutes.ago %>
- <span class="label label-info">stopped</span>
- <% elsif p.active %>
- <span class="label label-info">running</span>
- <% else %>
- <span class="label">queued</span>
- <% end %>
+ <%= render partial: 'pipeline_status_label', locals: {:p => p} %>
+ </td>
- <% summary = pipeline_summary p %>
- <div class="inline-progress-container pull-right">
- <div class="progress">
- <span class="progress-bar progress-bar-success" style="width: <%= summary[:percent_done] %>%;">
- </span>
- <% if p.success == false %>
- <span class="progress-bar progress-bar-danger" style="width: <%= 100.0 - summary[:percent_done] %>%;">
- </span>
- <% else %>
- <span class="progress-bar" style="width: <%= summary[:percent_running] %>%;">
- </span>
- <span class="progress-bar progress-bar-info" style="width: <%= summary[:percent_queued] %>%;">
- </span>
- <span class="progress-bar progress-bar-danger" style="width: <%= summary[:percent_failed] %>%;">
- </span>
- <% end %>
- </div>
+ <td>
+ <div class="inline-progress-container">
+ <%= render partial: 'pipeline_progress', locals: {:p => p} %>
</div>
</td>
-
</tr>
<% end %>
</table>
--- /dev/null
+# Do not use this file for site configuration. Create config.yml
+# instead (see application.yml.example).
+
+development:
+ cache_classes: false
+ whiny_nils: true
+ consider_all_requests_local: true
+ action_controller.perform_caching: false
+ action_mailer.raise_delivery_errors: false
+ active_support.deprecation: :log
+ action_dispatch.best_standards_support: :builtin
+ active_record.mass_assignment_sanitizer: :strict
+ active_record.auto_explain_threshold_in_seconds: 0.5
+ assets.compress: false
+ assets.debug: true
+ profiling_enabled: true
+ site_name: Workbench:dev
+
+production:
+ force_ssl: true
+ cache_classes: true
+ consider_all_requests_local: false
+ action_controller.perform_caching: true
+ serve_static_assets: false
+ assets.compress: true
+ assets.compile: false
+ assets.digest: true
+ i18n.fallbacks: true
+ active_support.deprecation: :notify
+ profiling_enabled: false
+
+ arvados_insecure_https: false
+
+ data_import_dir: /data/arvados-workbench-upload/data
+ data_export_dir: /data/arvados-workbench-download/data
+
+ site_name: Arvados Workbench
+
+test:
+ cache_classes: true
+ serve_static_assets: true
+ static_cache_control: public, max-age=3600
+ whiny_nils: true
+ consider_all_requests_local: true
+ action_controller.perform_caching: false
+ action_dispatch.show_exceptions: false
+ action_controller.allow_forgery_protection: false
+ action_mailer.delivery_method: :test
+ active_record.mass_assignment_sanitizer: :strict
+ active_support.deprecation: :stderr
+ profiling_enabled: false
+ secret_token: <%= rand(2**256).to_s(36) %>
+
+ site_name: Workbench:test
+
+common:
+ data_import_dir: /tmp/arvados-workbench-upload
+ data_export_dir: /tmp/arvados-workbench-download
+ arvados_login_base: https://arvados.local/login
+ arvados_v1_base: https://arvados.local/arvados/v1
+ arvados_insecure_https: true
+ activation_contact_link: mailto:info@arvados.org
+ arvados_docsite: http://doc.arvados.org
+ arvados_theme: default
+ show_user_agreement_inline: false
+ secret_token: ~
--- /dev/null
+# Copy this file to application.yml and edit to suit.
+#
+# Consult application.default.yml for the full list of configuration
+# settings.
+#
+# The order of precedence is:
+# 1. config/environments/{RAILS_ENV}.rb (deprecated)
+# 2. Section in application.yml corresponding to RAILS_ENV (e.g., development)
+# 3. Section in application.yml called "common"
+# 4. Section in application.default.yml corresponding to RAILS_ENV
+# 5. Section in application.default.yml called "common"
+
+common:
+ # At minimum, you need a nice long randomly generated secret_token here.
+ secret_token: ~
+
+ # You probably also want to point to your API server.
+ arvados_login_base: https://arvados.local:3000/login
+ arvados_v1_base: https://arvados.local:3000/arvados/v1
+ arvados_insecure_https: true
--- /dev/null
+$application_config = {}
+
+%w(application.default application).each do |cfgfile|
+ path = "#{::Rails.root.to_s}/config/#{cfgfile}.yml"
+ if File.exists? path
+ yaml = ERB.new(IO.read path).result(binding)
+ confs = YAML.load(yaml)
+ $application_config.merge!(confs['common'] || {})
+ $application_config.merge!(confs[::Rails.env.to_s] || {})
+ end
+end
+
+ArvadosWorkbench::Application.configure do
+ nils = []
+ $application_config.each do |k, v|
+ # "foo.bar: baz" --> { config.foo.bar = baz }
+ cfg = config
+ ks = k.split '.'
+ k = ks.pop
+ ks.each do |kk|
+ cfg = cfg.send(kk)
+ end
+ if cfg.respond_to?(k.to_sym) and !cfg.send(k).nil?
+ # Config must have been set already in environments/*.rb.
+ #
+ # After config files have been migrated, this mechanism should
+ # be deprecated, then removed.
+ elsif v.nil?
+ # Config variables are not allowed to be nil. Make a "naughty"
+ # list, and present it below.
+ nils << k
+ else
+ cfg.send "#{k}=", v
+ end
+ end
+ if !nils.empty?
+ raise <<EOS
+Refusing to start in #{::Rails.env.to_s} mode with missing configuration.
+
+The following configuration settings must be specified in
+config/application.yml:
+* #{nils.join "\n* "}
+
+EOS
+ end
+end
resources :users do
get 'home', :on => :member
get 'welcome', :on => :collection
+ post 'sudo', :on => :member
end
resources :logs
resources :factory_jobs
match '/collections/graph' => 'collections#graph'
resources :collections
get '/collections/:uuid/*file' => 'collections#show_file', :format => false
+
+ post 'actions' => 'actions#post'
+
root :to => 'users#welcome'
# Send unroutable requests to an arbitrary controller
--- /dev/null
+namespace :config do
+ desc 'Ensure site configuration has all required settings'
+ task check: :environment do
+ $application_config.sort.each do |k, v|
+ $stderr.puts "%-32s %s" % [k, eval("Rails.configuration.#{k}")]
+ end
+ end
+end
See "REST methods for working with Arvados resources":/api/methods.html
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/api_client_authorization@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/api_client_authorizations@
h2. Creation
See "REST methods for working with Arvados resources":/api/methods.html
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/authorized_key@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/authorized_keys@
h2. Creation
See "REST methods for working with Arvados resources":/api/methods.html
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/commit@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/commits@
h2. Creation
See "REST methods for working with Arvados resources":/api/methods.html
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/commit_ancestor@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/commit_ancestors@
h2. Creation
See "REST methods for working with Arvados resources":/api/methods.html
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/group@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/groups@
h2. Creation
See "REST methods for working with Arvados resources":/api/methods.html
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/human@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/humans@
h2. Creation
See "REST methods for working with Arvados resources":/api/methods.html
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/keep_disk@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/keep_disks@
h2. Creation
See "REST methods for working with Arvados resources":/api/methods.html
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/log@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/logs@
h2. Creation
See "REST methods for working with Arvados resources":/api/methods.html
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/node@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/nodes@
h2. Creation
See "REST methods for working with Arvados resources":/api/methods.html
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/pipeline_instance@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/pipeline_instances@
h2. Creation
See "REST methods for working with Arvados resources":/api/methods.html
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/pipeline_template@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/pipeline_templates@
h2. Creation
See "REST methods for working with Arvados resources":/api/methods.html
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/repository@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/repositories@
h2. Creation
See "REST methods for working with Arvados resources":/api/methods.html
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/specimen@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/specimens@
h2. Creation
See "REST methods for working with Arvados resources":/api/methods.html
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/trait@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/traits@
h2. Creation
See "REST methods for working with Arvados resources":/api/methods.html
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/user@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/users@
h2. Creation
See "REST methods for working with Arvados resources":/api/methods.html
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/virtual_machine@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/virtual_machines@
h2. Creation
navorder: 1
...
-{% include 'alert_stub' %}
-
# API server setup
-## Prerequisites
+## Prerequisites:
-1. A GNU/linux (virtual) machine
+1. A GNU/Linux (virtual) machine
2. A domain name for your api server
+3. Ruby >= 2.0.0
+4. Bundler
## Download the source tree
-Please follow the instructions on the [Download page](https://arvados.org/projects/arvados/wiki/Download) in the wiki.
+ git clone https://github.com/curoverse/arvados.git
+
+See also: [Downloading the source code](https://arvados.org/projects/arvados/wiki/Download) on the Arvados wiki.
## Configure the API server
Next, configure the database:
- cp config/database.yml.sample config/database.yml
+ cp -i config/database.yml.sample config/database.yml
-Edit database.yml to your liking and make sure the database and db user exist.
+Edit `database.yml` to your liking and make sure the database and db user exist.
Then set up the database:
RAILS_ENV=production rake db:setup
Then set up omniauth:
- cp config/initializers/omniauth.rb.example config/initializers/omniauth.rb
+ cp -i config/initializers/omniauth.rb.example config/initializers/omniauth.rb
-Edit config/initializers/omniauth.rb. Choose an *APP_SECRET* and *APP_ID*. Also set
-*CUSTOM_PROVIDER_URL*.
+Edit `config/initializers/omniauth.rb`. Choose an *APP_SECRET* and
+*APP_ID*. Also set *CUSTOM_PROVIDER_URL*.
Make sure your Omniauth provider knows about your *APP_ID* and *APP_SECRET*
combination.
-You also need to update config/initializers/secret_token.rb. Generate a new secret with
+You also need to update `config/initializers/secret_token.rb`.
+Generate a new secret with
rake secret
-and put it in config/initializers/secret_token.rb:
+and put it in `config/initializers/secret_token.rb`:
Server::Application.config.secret_token = 'your-new-secret-here'
-Finally, edit the main configuration:
+Edit the main configuration:
- cp config/environments/production.rb.example config/environments/production.rb
+ cp -i config/config.yml.example config/config.yml
-First, you want to make sure that
+First, you want to make sure that
- config.uuid_prefix
+ uuid_prefix
-is set to a unique 5-digit hex string. You can replace the 'cfi-aws-0' string
-with a string of your choice to make that happen.
+is set to a unique 5-character alphanumeric string. An example is
+given that generates a 5-character string based on a hash of your
+hostname.
-The *config.uuid_prefix* string is a unique identifier for your API server. It
-also serves as the first part of the hostname for your API server, for instance
+The `uuid_prefix` is a unique identifier for your API server. It also
+serves as the first part of the hostname for your API server, for
+instance
{{ site.arvados_api_host }}
-You should use your own domain instead of arvadosapi.com
-
-Second, unless you are running on AWS, you will want to change the definition of
-
- config.compute_node_nameservers
+For a development site, use your own domain instead of arvadosapi.com.
-If you know your nameservers and they are fixed, you can hardcode them, and
-make sure to remove the code that tries to look them up from the AWS metadata:
+You will also want to change `compute_node_nameservers` to suit your
+environment.
- config.compute_node_nameservers = ['1.2.3.4','2.3.4.5','3.4.5.6']
- #require 'net/http'
- #config.compute_node_nameservers = ['local', 'public'].collect do |iface|
- # Net::HTTP.get(URI("http://169.254.169.254/latest/meta-data/#{iface}-ipv4")).match(/^[\d\.]+$/)[0]
- #end << '172.16.0.23'
+Consult `config.defaults.yml` for a full list of configuration
+options. Always put your local configuration in `config.yml` instead
+of editing `config.defaults.yml`.
## Apache/Passenger
<notextile>
<pre>
-$ <code class="userinput">sudo apt-get install python-dev libattr1-dev libfuse-dev pkg-config</code>
+$ <code class="userinput">sudo apt-get install python-pip python-dev libattr1-dev libfuse-dev pkg-config</code>
$ <code class="userinput">sudo pip install arvados-python-client</code>
</pre>
</notextile>
+_If your version of @pip@ is 1.4 or newer, the @pip install@ command might give an error: "Could not find a version that satisfies the requirement arvados-python-client". If this happens, fix it by adding a @--pre@ flag:_
+
+<notextile>
+<pre>
+$ <code class="userinput">sudo pip install --pre arvados-python-client</code>
+</pre>
+</notextile>
+
h4. Option 2: build and install from source
<notextile>
# == Synopsis
#
-# wh-run-pipeline-instance --template pipeline-template-uuid [options] [--] [parameters]
-# wh-run-pipeline-instance --instance pipeline-instance-uuid [options]
+# arv-run-pipeline-instance --template pipeline-template-uuid [options] [--] [parameters]
+# arv-run-pipeline-instance --instance pipeline-instance-uuid [options]
#
# Satisfy a pipeline template by finding or submitting a mapreduce job
# for each pipeline component.
# to finish. Just find out whether jobs are finished,
# queued, or running for each component
#
-# [--create-instance-only] Do not try to satisfy any components. Just
+# [--submit] Do not try to satisfy any components. Just
# create an instance, print its UUID to
# stdout, and exit.
#
begin
require 'rubygems'
- require 'google/api_client'
require 'json'
require 'pp'
require 'trollop'
-rescue LoadError
+ require 'google/api_client'
+rescue LoadError => l
+ puts $:
abort <<-EOS
-#{$0}: fatal: some runtime dependencies are missing.
+#{$0}: fatal: #{l.message}
+Some runtime dependencies may be missing.
Try: gem install pp google-api-client json trollop
EOS
end
"UUID of pipeline instance.",
:short => :none,
:type => :string)
- opt(:create_instance_only,
+ opt(:submit,
"Do not try to satisfy any components. Just create a pipeline instance and output its UUID.",
:short => :none,
:type => :boolean)
+ opt(:run_here,
+ "Manage the pipeline in process.",
+ :short => :none,
+ :type => :boolean)
stop_on [:'--']
end
$options = Trollop::with_standard_exception_handling p do
$debuglevel = $options[:debug_level] || ($options[:debug] && 1) || 0
if $options[:instance]
- if $options[:template] or $options[:create_instance_only]
- abort "#{$0}: syntax error: --instance cannot be combined with --template or --create-instance-only."
+ if $options[:template] or $options[:submit]
+ abort "#{$0}: syntax error: --instance cannot be combined with --template or --submit."
end
elsif not $options[:template]
abort "#{$0}: syntax error: you must supply a --template or --instance."
end
+if $options[:run_here] == $options[:submit]
+ abort "#{$0}: syntax error: you must supply either --run-here or --submit."
+end
+
+# Suppress SSL certificate checks if ARVADOS_API_HOST_INSECURE
+
+module Kernel
+ def suppress_warnings
+ original_verbosity = $VERBOSE
+ $VERBOSE = nil
+ result = yield
+ $VERBOSE = original_verbosity
+ return result
+ end
+end
+
+if ENV['ARVADOS_API_HOST_INSECURE']
+ suppress_warnings { OpenSSL::SSL::VERIFY_PEER = OpenSSL::SSL::VERIFY_NONE }
+end
+
# Set up the API client.
$client ||= Google::APIClient.
end
elsif c[:job][:running] ||
(!c[:job][:started_at] && !c[:job][:cancelled_at])
- moretodo ||= !@options[:no_wait]
+ moretodo ||= true
elsif c[:job][:cancelled_at]
debuglog "component #{cname} job #{c[:job][:uuid]} cancelled."
end
@instance[:components] = @components
@instance[:active] = moretodo
report_status
+
+ if @options[:no_wait]
+ moretodo = false
+ end
+
if moretodo
begin
sleep 10
end
end
end
- @instance[:success] = @components.reject { |cname,c| c[:job] and c[:job][:success] }.empty?
+
+ ended = 0
+ succeeded = 0
+ failed = 0
+ @components.each do |cname, c|
+ if c[:job]
+ if c[:job][:finished_at]
+ ended += 1
+ if c[:job][:success] == true
+ succeeded += 1
+ end
+ end
+ end
+ end
+
+ if ended == @components.length
+ @instance[:active] = false
+ @instance[:success] = (succeeded == @components.length)
+ end
+
@instance.save
end
end
runner.apply_parameters(p.leftovers)
runner.setup_instance
- if $options[:create_instance_only]
+ if $options[:submit]
runner.instance.save
puts runner.instance[:uuid]
else
from stream import *
import config
import errors
+import util
def normalize_stream(s, stream):
stream_tokens = [s]
parser = argparse.ArgumentParser(
description='Read manifest on standard input and put normalized manifest on standard output.')
+parser.add_argument('--extract', type=str, help="The file to extract from the input manifest")
+
args = parser.parse_args()
import arvados
cr = arvados.CollectionReader(r)
-print cr.manifest_text()
+if args.extract:
+ i = args.extract.rfind('/')
+ if i == -1:
+ stream = '.'
+ fn = args.extract
+ else:
+ stream = args.extract[:i]
+ fn = args.extract[(i+1):]
+ for s in cr.all_streams():
+ if s.name() == stream:
+ if fn in s.files():
+ sys.stdout.write(s.files()[fn].as_manifest())
+else:
+ sys.stdout.write(cr.manifest_text())
/config/api.clinicalfuture.com.*
/config/database.yml
/config/initializers/omniauth.rb
+/config/application.yml
# asset cache
/public/assets/
gem 'redis'
gem 'test_after_commit', :group => :test
+
+gem 'google-api-client', '~> 0.6.3'
+gem 'trollop'
+
+gem 'arvados-cli', '>= 0.1.20140310170846'
activemodel (>= 3.0.0)
activesupport (>= 3.0.0)
rack (>= 1.1.0)
+ addressable (2.3.5)
andand (1.3.3)
arel (3.0.2)
+ arvados-cli (0.1.20140310170846)
+ activesupport (~> 3.2, >= 3.2.13)
+ andand (~> 1.3, >= 1.3.3)
+ curb (~> 0.8)
+ google-api-client (~> 0.6.3)
+ json (~> 1.7, >= 1.7.7)
+ oj (~> 2.0, >= 2.0.3)
+ trollop (~> 2.0)
+ autoparse (0.3.3)
+ addressable (>= 2.3.1)
+ extlib (>= 0.9.15)
+ multi_json (>= 1.0.0)
builder (3.0.4)
capistrano (2.15.5)
highline
coffee-script-source
execjs
coffee-script-source (1.6.3)
+ curb (0.8.5)
daemon_controller (1.1.7)
erubis (2.7.0)
execjs (2.0.2)
+ extlib (0.9.16)
faraday (0.8.8)
multipart-post (~> 1.2.0)
+ google-api-client (0.6.4)
+ addressable (>= 2.3.2)
+ autoparse (>= 0.3.3)
+ extlib (>= 0.9.15)
+ faraday (~> 0.8.4)
+ jwt (>= 0.1.5)
+ launchy (>= 2.1.1)
+ multi_json (>= 1.0.0)
+ signet (~> 0.4.5)
+ uuidtools (>= 2.1.0)
hashie (1.2.0)
highline (1.6.20)
hike (1.2.3)
json (1.8.1)
jwt (0.1.8)
multi_json (>= 1.5)
+ launchy (2.4.2)
+ addressable (~> 2.3)
libv8 (3.16.14.3)
mail (2.5.4)
mime-types (~> 1.16)
railties (~> 3.2.0)
sass (>= 3.1.10)
tilt (~> 1.3)
+ signet (0.4.5)
+ addressable (>= 2.2.3)
+ faraday (~> 0.8.1)
+ jwt (>= 0.1.5)
+ multi_json (>= 1.0.0)
sprockets (2.2.2)
hike (~> 1.2)
multi_json (~> 1.0)
treetop (1.4.15)
polyglot
polyglot (>= 0.3.1)
+ trollop (2.0)
tzinfo (0.3.38)
uglifier (2.3.0)
execjs (>= 0.3.0)
json (>= 1.8.0)
+ uuidtools (2.1.4)
PLATFORMS
ruby
DEPENDENCIES
acts_as_api
andand
+ arvados-cli (>= 0.1.20140310170846)
coffee-rails (~> 3.2.0)
+ google-api-client (~> 0.6.3)
jquery-rails
multi_json
oj
sass-rails (>= 3.2.0)
test_after_commit
therubyracer
+ trollop
uglifier (>= 1.0.3)
resource_attrs[:user_id] =
User.where(uuid: resource_attrs.delete(:owner_uuid)).first.andand.id
end
+ resource_attrs[:api_client_id] = Thread.current[:api_client].id
super
end
def update_modified_by_fields
self.created_at ||= Time.now
- self.owner_uuid ||= current_default_owner
+ self.owner_uuid ||= current_default_owner if self.respond_to? :owner_uuid=
self.modified_at = Time.now
self.modified_by_user_uuid = current_user ? current_user.uuid : nil
self.modified_by_client_uuid = current_api_client ? current_api_client.uuid : nil
MAX_SLOTS = 64
- @@confdir = if Rails.configuration.respond_to? :dnsmasq_conf_dir
- Rails.configuration.dnsmasq_conf_dir
- elsif File.exists? '/etc/dnsmasq.d/.'
- '/etc/dnsmasq.d'
- else
- nil
- end
+ @@confdir = Rails.configuration.dnsmasq_conf_dir
@@domain = Rails.configuration.compute_node_domain rescue `hostname --domain`.strip
@@nameservers = Rails.configuration.compute_node_nameservers
def start!(ping_url_method)
ensure_permission_to_update
ping_url = ping_url_method.call({ uuid: self.uuid, ping_secret: self.info[:ping_secret] })
- if (Rails.configuration.compute_node_ec2run_args rescue false) and
- (Rails.configuration.compute_node_ami rescue false)
+ if (Rails.configuration.compute_node_ec2run_args and
+ Rails.configuration.compute_node_ami)
ec2_args = ["--user-data '#{ping_url}'",
"-t c1.xlarge -n 1",
Rails.configuration.compute_node_ec2run_args,
t.collect { |r| r[2] }.inject(0.0) { |sum,a| sum += a } / t.size
end
+ def self.queue
+ self.where('active = true')
+ end
+
protected
def bootstrap_components
if pipeline_template and (!components or components.empty?)
--- /dev/null
+# Do not use this file for site configuration. Create application.yml
+# instead (see application.yml.example).
+
+development:
+ force_ssl: false
+ cache_classes: false
+ whiny_nils: true
+ consider_all_requests_local: true
+ action_controller.perform_caching: false
+ action_mailer.raise_delivery_errors: false
+ action_mailer.perform_deliveries: false
+ active_support.deprecation: :log
+ action_dispatch.best_standards_support: :builtin
+ active_record.mass_assignment_sanitizer: :strict
+ active_record.auto_explain_threshold_in_seconds: 0.5
+ assets.compress: false
+ assets.debug: true
+
+production:
+ force_ssl: true
+ cache_classes: true
+ consider_all_requests_local: false
+ action_controller.perform_caching: true
+ serve_static_assets: false
+ assets.compress: true
+ assets.compile: false
+ assets.digest: true
+
+test:
+ force_ssl: false
+ cache_classes: true
+ serve_static_assets: true
+ static_cache_control: public, max-age=3600
+ whiny_nils: true
+ consider_all_requests_local: true
+ action_controller.perform_caching: false
+ action_dispatch.show_exceptions: false
+ action_controller.allow_forgery_protection: false
+ action_mailer.delivery_method: :test
+ active_support.deprecation: :stderr
+ active_record.mass_assignment_sanitizer: :strict
+
+common:
+ secret_token: ~
+ uuid_prefix: <%= Digest::MD5.hexdigest(`hostname`).to_i(16).to_s(36)[0..4] %>
+
+ git_repositories_dir: /var/cache/git
+
+ # :none or :slurm_immediate
+ crunch_job_wrapper: :none
+
+ # username, or false = do not set uid when running jobs.
+ crunch_job_user: crunch
+
+ # The web service must be able to create/write this file, and
+ # crunch-job must be able to stat() it.
+ crunch_refresh_trigger: /tmp/crunch_refresh_trigger
+
+ # Path to /etc/dnsmasq.d, or false = do not update dnsmasq data.
+ dnsmasq_conf_dir: false
+
+ # Set to AMI id (ami-123456) to auto-start nodes. See app/models/node.rb
+ compute_node_ami: false
+ compute_node_ec2run_args: -g arvados-compute
+ compute_node_spot_bid: 0.11
+
+ compute_node_domain: <%= `hostname`.split('.')[1..-1].join('.').strip %>
+ compute_node_nameservers:
+ - 192.168.1.1
+ compute_node_ec2_tag_enable: false
+
+ accept_api_token: {}
+
+ new_users_are_active: false
+ admin_notifier_email_from: arvados@example.com
+ email_subject_prefix: "[ARVADOS] "
+
+ # Visitors to the API server will be redirected to the workbench
+ workbench_address: https://workbench.local:3001/
+
+ # The e-mail address of the user you would like to become marked as an admin
+ # user on their first login.
+ # In the default configuration, authentication happens through the Arvados SSO
+ # server, which uses openid against Google's servers, so in that case this
+ # should be an address associated with a Google account.
+ auto_admin_user: ~
--- /dev/null
+# Copy this file to application.yml and edit to suit.
+#
+# Consult application.default.yml for the full list of configuration
+# settings.
+#
+# The order of precedence is:
+# 1. config/environments/{RAILS_ENV}.rb (deprecated)
+# 2. Section in application.yml corresponding to RAILS_ENV (e.g., development)
+# 3. Section in application.yml called "common"
+# 4. Section in application.default.yml corresponding to RAILS_ENV
+# 5. Section in application.default.yml called "common"
+
+development:
+
+production:
+ # At minimum, you need a nice long randomly generated secret_token here.
+ secret_token: ~
+
+ uuid_prefix: bogus
+
+ # This is suitable for AWS; see common section below for a static example.
+ compute_node_nameservers: <%=
+ require 'net/http'
+ ['local', 'public'].collect do |iface|
+ Net::HTTP.get(URI("http://169.254.169.254/latest/meta-data/#{iface}-ipv4")).match(/^[\d\.]+$/)[0]
+ end << '172.16.0.23'
+ %>
+ # You must customize these. See config.defaults.yml for information.
+ compute_node_ami: ~
+ compute_node_ec2_tag_enable: ~
+ compute_node_domain: ~
+ compute_node_spot_bid: ~
+
+test:
+ uuid_prefix: zzzzz
+
+common:
+ secret_token: ~
+ compute_node_nameservers:
+ - 192.168.0.1
+ - 172.16.0.1
development:
- adapter: mysql
+ adapter: postgresql
encoding: utf8
database: arvados_development
username: arvados
host: localhost
test:
- adapter: mysql
+ adapter: postgresql
encoding: utf8
database: arvados_test
username: arvados
host: localhost
production:
- adapter: mysql
+ adapter: postgresql
encoding: utf8
database: arvados_production
username: arvados
# Visitors to the API server will be redirected to the workbench
config.workbench_address = "http://localhost:3000/"
+
+ # The e-mail address of the user you would like to become marked as an admin
+ # user on their first login.
+ # In the default configuration, authentication happens through the Arvados SSO
+ # server, which uses openid against Google's servers, so in that case this
+ # should be an address associated with a Google account.
+ config.auto_admin_user = ''
end
--- /dev/null
+$application_config = {}
+
+%w(application.default application).each do |cfgfile|
+ path = "#{::Rails.root.to_s}/config/#{cfgfile}.yml"
+ if File.exists? path
+ yaml = ERB.new(IO.read path).result(binding)
+ confs = YAML.load(yaml)
+ $application_config.merge!(confs['common'] || {})
+ $application_config.merge!(confs[::Rails.env.to_s] || {})
+ end
+end
+
+Server::Application.configure do
+ nils = []
+ $application_config.each do |k, v|
+ # "foo.bar: baz" --> { config.foo.bar = baz }
+ cfg = config
+ ks = k.split '.'
+ k = ks.pop
+ ks.each do |kk|
+ cfg = cfg.send(kk)
+ end
+ if cfg.respond_to?(k.to_sym) and !cfg.send(k).nil?
+ # Config must have been set already in environments/*.rb.
+ #
+ # After config files have been migrated, this mechanism should
+ # be deprecated, then removed.
+ elsif v.nil?
+ # Config variables are not allowed to be nil. Make a "naughty"
+ # list, and present it below.
+ nils << k
+ else
+ cfg.send "#{k}=", v
+ end
+ end
+ if !nils.empty?
+ raise <<EOS
+Refusing to start in #{::Rails.env.to_s} mode with missing configuration.
+
+The following configuration settings must be specified in
+config/application.yml:
+* #{nils.join "\n* "}
+
+EOS
+ end
+end
--- /dev/null
+namespace :config do
+ desc 'Ensure site configuration has all required settings'
+ task check: :environment do
+ $application_config.sort.each do |k, v|
+ $stderr.puts "%-32s %s" % [k, eval("Rails.configuration.#{k}")]
+ end
+ end
+end
def refresh_todo
@todo = Job.queue
+ @todo_pipelines = PipelineInstance.queue
end
def sinfo
@running.delete job_done.uuid
end
+ def update_pipelines
+ @todo_pipelines.each do |p|
+ pipe_auth = ApiClientAuthorization.
+ new(user: User.where('uuid=?', p.modified_by_user_uuid).first,
+ api_client_id: 0)
+ pipe_auth.save
+
+ puts `export ARVADOS_API_TOKEN=#{pipe_auth.api_token} && arv-run-pipeline-instance --run-here --no-wait --instance #{p.uuid}`
+ end
+ end
+
def run
act_as_system_user
@running ||= {}
unless @todo.empty? or did_recently(:start_jobs, 1.0) or $signal[:term]
start_jobs
end
+ unless @todo_pipelines.empty? or did_recently(:update_pipelines, 5.0)
+ update_pipelines
+ end
end
reap_children
select(@running.values.collect { |j| [j[:stdout], j[:stderr]] }.flatten,
end
end
+
+
protected
def did_recently(thing, min_interval)
assert_response :success
end
+ test "create token for different user" do
+ post "/arvados/v1/api_client_authorizations", {
+ :format => :json,
+ :api_client_authorization => {
+ :owner_uuid => users(:spectator).uuid
+ }
+ }, {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:admin_trustedclient).api_token}"}
+ assert_response :success
+
+ get "/arvados/v1/users/current", {
+ :format => :json
+ }, {'HTTP_AUTHORIZATION' => "OAuth2 #{jresponse['api_token']}"}
+ @jresponse = nil
+ assert_equal users(:spectator).uuid, jresponse['uuid']
+ end
+
+ test "refuse to create token for different user if not trusted client" do
+ post "/arvados/v1/api_client_authorizations", {
+ :format => :json,
+ :api_client_authorization => {
+ :owner_uuid => users(:spectator).uuid
+ }
+ }, {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:admin).api_token}"}
+ assert_response 403
+ end
+
+ test "refuse to create token for different user if not admin" do
+ post "/arvados/v1/api_client_authorizations", {
+ :format => :json,
+ :api_client_authorization => {
+ :owner_uuid => users(:spectator).uuid
+ }
+ }, {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:active_trustedclient).api_token}"}
+ assert_response 403
+ end
+
end
projects / arvados.git / commitdiff