--- /dev/null
+function provenance_sizing_fixup(divId, svgId) {
+ var a = document.getElementById(divId);
+ var b = document.getElementById("_" + divId + "_padding");
+ var c = document.getElementById("_" + divId + "_center");
+ var g = document.getElementById(svgId);
+
+ var h = window.innerHeight - a.getBoundingClientRect().top - 2;
+ var max = window.innerHeight - 100;
+ var gh = parseFloat(g.getAttribute("height"));
+ //var sh = (a.scrollTopMax > 0) ? a.scrollHeight : 0;
+ //gh = (gh > sh) ? gh : sh;
+ gh += 25;
+ if (h < 0) { h = max; }
+ var height;
+ if (gh < h) {
+ if (gh > max) { gh = max; }
+ height = String(gh) + "px";
+ }
+ else {
+ if (h > max) { h = max; }
+ height = String(h) + "px";
+ }
+ a.style.height = height;
+
+ if (gh < max) {
+ b.style.paddingTop = String(gh) + "px";
+ }
+ else {
+ b.style.paddingTop = String(max) + "px";
+ }
+
+ w = window.innerWidth - 25;
+ a.style.width = String(w) + "px";
+ gw = parseFloat(g.getAttribute("width"));
+ if (gw < w) {
+ c.style.paddingLeft = String((w - gw)/2.0) + "px";
+ }
+ else {
+ c.style.paddingLeft = "0px";
+ }
+}
+
+function graph_zoom(divId, svgId, scale) {
+ var pg = document.getElementById(divId);
+ vcenter = (pg.scrollTop + (pg.scrollHeight - pg.scrollTopMax)/2.0) / pg.scrollHeight;
+ hcenter = (pg.scrollLeft + (pg.scrollWidth - pg.scrollLeftMax)/2.0) / pg.scrollWidth;
+ var g = document.getElementById(svgId);
+ g.setAttribute("height", parseFloat(g.getAttribute("height")) * scale);
+ g.setAttribute("width", parseFloat(g.getAttribute("width")) * scale);
+ pg.scrollTop = (vcenter * pg.scrollHeight) - (pg.scrollHeight - pg.scrollTopMax)/2.0;
+ pg.scrollLeft = (hcenter * pg.scrollWidth) - (pg.scrollWidth - pg.scrollLeftMax)/2.0;
+ provenance_sizing_fixup(divId, svgId);
+}
class CollectionsController < ApplicationController
- skip_before_filter :find_object_by_uuid, :only => [:graph]
+ skip_before_filter :find_object_by_uuid, :only => [:provenance]
skip_before_filter :check_user_agreements, :only => [:show_file]
- def graph
- index
- end
-
def index
if params[:search].andand.length.andand > 0
tags = Link.where(any: ['contains', params[:search]])
self.response_body = FileStreamer.new opts
end
+
def show
return super if !@object
@provenance = []
@sourcedata[collection.uuid][:collection] = collection
end
end
+
+ Collection.where(uuid: @object.uuid).each do |u|
+ @prov_svg = ProvenanceHelper::create_provenance_graph u.provenance, "provenance_svg", {:direction => :bottom_up, :combine_jobs => :script_only}
+ end
end
protected
class JobsController < ApplicationController
+
+ def generate_provenance(jobs)
+ nodes = []
+ collections = []
+ jobs.each do |j|
+ nodes << j
+ collections << j[:output]
+ collections.concat(ProvenanceHelper::find_collections(j[:script_parameters]))
+ nodes << {:uuid => j[:script_version]}
+ end
+
+ Collection.where(uuid: collections).each do |c|
+ nodes << c
+ end
+
+ @svg = ProvenanceHelper::create_provenance_graph nodes, "provenance_svg", {:all_script_parameters => true, :script_version_nodes => true}
+ end
+
def index
- @jobs = Job.all
+ @svg = ""
+ if params[:uuid]
+ @jobs = Job.where(uuid: params[:uuid])
+ generate_provenance(@jobs)
+ else
+ @jobs = Job.all
+ end
+ end
+
+ def show
+ generate_provenance([@object])
end
end
before_filter :find_objects_by_uuid, only: :compare
include PipelineInstancesHelper
+ def show
+ pipelines = [@object]
+
+ if params[:compare]
+ PipelineInstance.where(uuid: params[:compare]).each do |p|
+ pipelines << p
+ end
+ end
+
+ count = {}
+ provenance = {}
+ pips = {}
+ n = 1
+
+ pipelines.each do |p|
+ collections = []
+
+ p.components.each do |k, v|
+ j = v[:job]
+
+ uuid = j[:uuid].intern
+ provenance[uuid] = j
+ pips[uuid] = 0 unless pips[uuid] != nil
+ pips[uuid] |= n
+
+ collections << j[:output]
+ ProvenanceHelper::find_collections(j[:script_parameters]).each do |k|
+ collections << k
+ end
+
+ uuid = j[:script_version].intern
+ provenance[uuid] = {:uuid => uuid}
+ pips[uuid] = 0 unless pips[uuid] != nil
+ pips[uuid] |= n
+ end
+
+ Collection.where(uuid: collections).each do |c|
+ uuid = c.uuid.intern
+ provenance[uuid] = c
+ pips[uuid] = 0 unless pips[uuid] != nil
+ pips[uuid] |= n
+ end
+
+ n = n << 1
+ end
+
+ @prov_svg = ProvenanceHelper::create_provenance_graph provenance, "provenance_svg", {
+ :all_script_parameters => true,
+ :combine_jobs => :script_and_version,
+ :script_version_nodes => true,
+ :pips => pips }
+ end
+
def compare
@breadcrumb_page_name = 'compare'
--- /dev/null
+module ProvenanceHelper
+
+ class GenerateGraph
+ def initialize(pdata, opts)
+ @pdata = pdata
+ @opts = opts
+ @visited = {}
+ @jobs = {}
+ end
+
+ def self.collection_uuid(uuid)
+ m = /^([a-f0-9]{32}(\+[0-9]+)?)(\+.*)?$/.match(uuid.to_s)
+ if m
+ #if m[2]
+ return m[1]
+ #else
+ # Collection.where(uuid: ['contains', m[1]]).each do |u|
+ # puts "fixup #{uuid} to #{u.uuid}"
+ # return u.uuid
+ # end
+ #end
+ else
+ nil
+ end
+ end
+
+ def determine_fillcolor(n)
+ bgcolor = ""
+ case n
+ when 1
+ bgcolor = "style=filled,fillcolor=\"#88ff88\""
+ when 2
+ bgcolor = "style=filled,fillcolor=\"#8888ff\""
+ when 3
+ bgcolor = "style=filled,fillcolor=\"#88ffff\""
+ end
+ bgcolor
+ end
+
+ def describe_node(uuid)
+ bgcolor = determine_fillcolor @opts[:pips][uuid] if @opts[:pips]
+
+ rsc = ArvadosBase::resource_class_for_uuid uuid.to_s
+ if rsc
+ href = "/#{rsc.to_s.underscore.pluralize rsc}/#{uuid}"
+
+ #"\"#{uuid}\" [label=\"#{rsc}\\n#{uuid}\",href=\"#{href}\"];\n"
+ if rsc == Collection
+ if @pdata[uuid]
+ #puts @pdata[uuid]
+ if @pdata[uuid][:name]
+ return "\"#{uuid}\" [label=\"#{@pdata[uuid][:name]}\",href=\"#{href}\",shape=oval,#{bgcolor}];\n"
+ else
+ files = nil
+ if @pdata[uuid].respond_to? :files
+ files = @pdata[uuid].files
+ elsif @pdata[uuid][:files]
+ files = @pdata[uuid][:files]
+ end
+
+ if files
+ i = 0
+ label = ""
+ while i < 3 and i < files.length
+ label += "\\n" unless label == ""
+ label += files[i][1]
+ i += 1
+ end
+ if i < files.length
+ label += "\\n⋮"
+ end
+ return "\"#{uuid}\" [label=\"#{label}\",href=\"#{href}\",shape=oval,#{bgcolor}];\n"
+ end
+ end
+ end
+ return "\"#{uuid}\" [label=\"#{rsc}\",href=\"#{href}\",#{bgcolor}];\n"
+ end
+ end
+ "\"#{uuid}\" [#{bgcolor}];\n"
+ end
+
+ def job_uuid(job)
+ if @opts[:combine_jobs] == :script_only
+ uuid = "#{job[:script]}"
+ elsif @opts[:combine_jobs] == :script_and_version
+ uuid = "#{job[:script]}_#{job[:script_version]}"
+ else
+ uuid = "#{job[:uuid]}"
+ end
+
+ @jobs[uuid] = [] unless @jobs[uuid]
+ @jobs[uuid] << job unless @jobs[uuid].include? job
+
+ uuid
+ end
+
+ def edge(tail, head, extra)
+ if @opts[:direction] == :bottom_up
+ gr = "\"#{tail}\" -> \"#{head}\""
+ else
+ gr = "\"#{head}\" -> \"#{tail}\""
+ end
+ if extra.length > 0
+ gr += "["
+ extra.each do |k, v|
+ gr += "#{k}=\"#{v}\","
+ end
+ gr += "]"
+ end
+ gr += ";\n"
+ gr
+ end
+
+ def script_param_edges(job, prefix, sp)
+ gr = ""
+ if sp and not sp.empty?
+ case sp
+ when Hash
+ sp.each do |k, v|
+ if prefix.size > 0
+ k = prefix + "::" + k.to_s
+ end
+ gr += script_param_edges(job, k.to_s, v)
+ end
+ when Array
+ i = 0
+ node = ""
+ sp.each do |v|
+ if GenerateGraph::collection_uuid(v)
+ gr += script_param_edges(job, "#{prefix}[#{i}]", v)
+ else
+ node += "', '" unless node == ""
+ node = "['" if node == ""
+ node += "#{v}"
+ end
+ i += 1
+ end
+ unless node == ""
+ node += "']"
+ #puts node
+ #id = "#{job[:uuid]}_#{prefix}"
+ gr += "\"#{node}\" [label=\"#{node}\"];\n"
+ gr += edge(job_uuid(job), node, {:label => prefix})
+ end
+ else
+ m = GenerateGraph::collection_uuid(sp)
+ if m
+ gr += edge(job_uuid(job), m, {:label => prefix})
+ gr += generate_provenance_edges(m)
+ elsif @opts[:all_script_parameters]
+ #id = "#{job[:uuid]}_#{prefix}"
+ gr += "\"#{sp}\" [label=\"#{sp}\"];\n"
+ gr += edge(job_uuid(job), sp, {:label => prefix})
+ end
+ end
+ end
+ gr
+ end
+
+ def generate_provenance_edges(uuid)
+ gr = ""
+ m = GenerateGraph::collection_uuid(uuid)
+ uuid = m if m
+
+ uuid = uuid.intern if uuid
+
+ if (not uuid) or uuid.empty? or @visited[uuid]
+
+ #puts "already @visited #{uuid}"
+ return ""
+ end
+
+ if not @pdata[uuid] then
+ return describe_node(uuid)
+ else
+ @visited[uuid] = true
+ end
+
+ #puts "visiting #{uuid}"
+
+ if m
+ # uuid is a collection
+ gr += describe_node(uuid)
+
+ @pdata.each do |k, job|
+ if job[:output] == uuid.to_s
+ gr += edge(uuid, job_uuid(job), {:label => "output"})
+ gr += generate_provenance_edges(job[:uuid])
+ end
+ if job[:log] == uuid.to_s
+ gr += edge(uuid, job_uuid(job), {:label => "log"})
+ gr += generate_provenance_edges(job[:uuid])
+ end
+ end
+ else
+ # uuid is something else
+ rsc = ArvadosBase::resource_class_for_uuid uuid.to_s
+
+ if rsc == Job
+ job = @pdata[uuid]
+ if job
+ gr += script_param_edges(job, "", job[:script_parameters])
+
+ if @opts[:script_version_nodes]
+ gr += describe_node(job[:script_version])
+ gr += edge(job_uuid(job), job[:script_version], {:label => "script_version"})
+ end
+ end
+ else
+ gr += describe_node(uuid)
+ end
+ end
+
+ @pdata.each do |k, link|
+ if link[:head_uuid] == uuid.to_s and link[:link_class] == "provenance"
+ gr += describe_node(link[:tail_uuid])
+ gr += edge(link[:head_uuid], link[:tail_uuid], {:label => link[:name], :href => "/links/#{link[:uuid]}"})
+ gr += generate_provenance_edges(link[:tail_uuid])
+ end
+ end
+
+ #puts "finished #{uuid}"
+
+ gr
+ end
+
+ def describe_jobs
+ gr = ""
+ @jobs.each do |k, v|
+ gr += "\"#{k}\" [href=\"/jobs?"
+
+ n = 0
+ v.each do |u|
+ gr += "uuid%5b%5d=#{u[:uuid]}&"
+ n |= @opts[:pips][u[:uuid].intern] if @opts[:pips] and @opts[:pips][u[:uuid].intern]
+ end
+
+ gr += "\",label=\""
+
+ if @opts[:combine_jobs] == :script_only
+ gr += uuid = "#{v[0][:script]}"
+ elsif @opts[:combine_jobs] == :script_and_version
+ gr += uuid = "#{v[0][:script]}"
+ else
+ gr += uuid = "#{v[0][:script]}\\n#{v[0][:finished_at]}"
+ end
+ gr += "\",#{determine_fillcolor n}];\n"
+ end
+ gr
+ end
+
+ end
+
+ def self.create_provenance_graph(pdata, svgId, opts={})
+ if pdata.is_a? Array or pdata.is_a? ArvadosResourceList
+ p2 = {}
+ pdata.each do |k|
+ p2[k[:uuid].intern] = k if k[:uuid]
+ end
+ pdata = p2
+ end
+
+ unless pdata.is_a? Hash
+ raise "create_provenance_graph accepts Array or Hash for pdata only, pdata is #{pdata.class}"
+ end
+
+ gr = """strict digraph {
+node [fontsize=8,shape=box];
+edge [fontsize=8];
+"""
+
+ if opts[:direction] == :bottom_up
+ gr += "edge [dir=back];"
+ end
+
+ #puts "@pdata is #{pdata}"
+
+ g = GenerateGraph.new(pdata, opts)
+
+ pdata.each do |k, v|
+ gr += g.generate_provenance_edges(k)
+ end
+
+ gr += g.describe_jobs
+
+ gr += "}"
+ svg = ""
+
+ #puts gr
+
+ require 'open3'
+
+ Open3.popen2("dot", "-Tsvg") do |stdin, stdout, wait_thr|
+ stdin.print(gr)
+ stdin.close
+ svg = stdout.read()
+ wait_thr.value
+ stdout.close()
+ end
+
+ svg = svg.sub(/<\?xml.*?\?>/m, "")
+ svg = svg.sub(/<!DOCTYPE.*?>/m, "")
+ svg = svg.sub(/<svg /, "<svg id=\"#{svgId}\" ")
+ end
+
+ def self.find_collections(sp)
+ c = []
+ if sp and not sp.empty?
+ case sp
+ when Hash
+ sp.each do |k, v|
+ c.concat(find_collections(v))
+ end
+ when Array
+ sp.each do |v|
+ c.concat(find_collections(v))
+ end
+ else
+ m = GenerateGraph::collection_uuid(sp)
+ if m
+ c << m
+ end
+ end
+ end
+ c
+ end
+end
def attribute_editable?(attr)
false
end
+
+ def provenance
+ $arvados_api_client.api "collections/#{self.uuid}/", "provenance"
+ end
end
--- /dev/null
+<%= content_for :css do %>
+#<%= divId %> {
+ padding-left: 3px;
+ overflow: auto;
+ border: solid;
+ border-width: 1px;
+ border-color: gray;
+ position: absolute;
+ left: 1px;
+
+}
+<% end %>
+
+<div id="_<%= divId %>_container" style="padding-top: 41px; margin-top: -41px">
+ <script>
+ (function() {
+ var fn = function () { graph_zoom("<%= divId %>", "<%=svgId %>", 1) };
+ $(window).resize(fn);
+ $(window).load(fn);
+ $(window).scroll(fn);
+ })();
+ </script>
+ <div style="text-align: right">
+ <a style="cursor: pointer"><span class="icon-zoom-out" onclick="graph_zoom('<%= divId %>', '<%= svgId %>', .9)"></span></a>
+ <a style="cursor: pointer"><span class="icon-zoom-in" onclick="graph_zoom('<%= divId %>', '<%= svgId %>', 1./.9)"></span></a>
+ </div>
+
+ <div id="<%= divId %>">
+ <span id="_<%= divId %>_center" style="padding-left: 300px"/>
+ <%= raw(svg) %>
+ </div>
+ <div id="_<%= divId %>_padding" style="padding-bottom: 1em" />
+</div>
-<%= content_for :head do %>
-<style>
+<%= content_for :css do %>
<%# https://github.com/mbostock/d3/wiki/Ordinal-Scales %>
<% n=-1; %w(#1f77b4 #ff7f0e #2ca02c #d62728 #9467bd #8c564b #e377c2 #7f7f7f #bcbd22 #17becf).each do |color| %>
.colorseries-10-<%= n += 1 %>, .btn.colorseries-10-<%= n %>:hover, .label.colorseries-10-<%= n %>:hover {
.label a {
color: inherit;
}
-</style>
+
<% end %>
<%#= render :partial => 'nav' %>
+<div class="tabbable">
<ul class="nav nav-tabs">
<li class="active"><a href="#files" data-toggle="tab">Files (<%= @object.files ? @object.files.size : 0 %>)</a></li>
<li><a href="#provenance" data-toggle="tab">Provenance (<%= @provenance.size %>)</a></li>
<li><a href="#jobs" data-toggle="tab">Jobs (<%= @provenance.size %>)</a></li>
<li><a href="#sourcedata" data-toggle="tab">Source data (<%= @sourcedata.size %>)</a></li>
<li><a href="#owner-groups-resources" data-toggle="tab">Owner, groups, resources</a></li>
+ <li><a href="#provenance2" data-toggle="tab" id="prov-tab">Provenance graph</a></li>
+ <li><a href="#used-by" data-toggle="tab">Used by</a></li>
</ul>
<div class="tab-content">
</tbody>
</table>
</div>
+ <div id="provenance2" class="tab-pane fade">
+<script>
+ $(function() {
+ $('#prov-tab').on('shown', function() { provenance_sizing_fixup("provenance_graph", "provenance_svg"); });
+ })
+</script>
+ <%= render partial: 'application/svg_div', locals: {
+ divId: "provenance_graph",
+ svgId: "provenance_svg",
+ svg: @prov_svg } %>
+ </div>
<div id="jobs" class="tab-pane fade">
<table class="topalign table table-bordered">
<thead>
</td><td>
<%= j.script_version %>
</td><td>
- <span class="label <%= if j.success then 'label-success'; elsif j.active then 'label-primary'; else 'label-warning'; end %>">
- <%= j.success || j.active ? 'ok' : 'failed' %>
+ <span class="label <%= if j.success then 'label-success'; elsif j.running then 'label-primary'; else 'label-warning'; end %>">
+ <%= j.success || j.running ? 'ok' : 'failed' %>
</span>
</td><td>
<%= j.started_at %>
<%= render :partial => 'application/arvados_object' %>
</div>
</div>
+</div>
</tbody>
</table>
+
+<%= render partial: 'application/svg_div', locals: {
+ divId: "provenance_graph",
+ svgId: "provenance_svg",
+ svg: @svg } %>
+
--- /dev/null
+<%= render partial: 'arvados_object' %>
+
+<%= render partial: 'application/svg_div', locals: {
+ divId: "provenance_graph",
+ svgId: "provenance_svg",
+ svg: @svg } %>
+<% content_for :css do %>
+ .pipeline_color_legend {
+ padding-left: 1em;
+ padding-right: 1em;
+ }
+<% end %>
+
<table class="table table-condensed table-hover topalign">
<thead>
</thead>
<% end %>
</tbody>
</table>
+
+<div style="text-align: center">
+<span class="pipeline_color_legend" style="background: #88ff88">This pipeline</span>
+<span class="pipeline_color_legend" style="background: #8888ff">Comparison pipeline</span>
+<span class="pipeline_color_legend" style="background: #88ffff">Shared by both pipelines</span>
+</div>
+
+ <%= render partial: 'application/svg_div', locals: {
+ divId: "provenance_graph",
+ svgId: "provenance_svg",
+ svg: @prov_svg } %>
+
<pre>
<%= JSON.pretty_generate(@object.attributes) rescue nil %>
</pre>
show
end
- protected
+ def collection_uuid(uuid)
+ m = /([a-f0-9]{32}(\+[0-9]+)?)(\+.*)?/.match(uuid)
+ if m
+ m[1]
+ else
+ nil
+ end
+ end
+
+ def script_param_edges(visited, sp)
+ if sp and not sp.empty?
+ case sp
+ when Hash
+ sp.each do |k, v|
+ script_param_edges(visited, v)
+ end
+ when Array
+ sp.each do |v|
+ script_param_edges(visited, v)
+ end
+ else
+ m = collection_uuid(sp)
+ if m
+ generate_provenance_edges(visited, m)
+ end
+ end
+ end
+ end
+
+ def generate_provenance_edges(visited, uuid)
+ m = collection_uuid(uuid)
+ uuid = m if m
+
+ if not uuid or uuid.empty? or visited[uuid]
+ return ""
+ end
+
+ #puts "visiting #{uuid}"
+
+ if m
+ # uuid is a collection
+ Collection.where(uuid: uuid).each do |c|
+ visited[uuid] = c.as_api_response
+ visited[uuid][:files] = []
+ c.files.each do |f|
+ visited[uuid][:files] << f
+ end
+ end
+ Job.where(output: uuid).each do |job|
+ generate_provenance_edges(visited, job.uuid)
+ end
+
+ Job.where(log: uuid).each do |job|
+ generate_provenance_edges(visited, job.uuid)
+ end
+
+ else
+ # uuid is something else
+ rsc = ArvadosModel::resource_class_for_uuid uuid
+ if rsc == Job
+ Job.where(uuid: uuid).each do |job|
+ visited[uuid] = job.as_api_response
+ script_param_edges(visited, job.script_parameters)
+ end
+ elsif rsc != nil
+ rsc.where(uuid: uuid).each do |r|
+ visited[uuid] = r.as_api_response
+ end
+ end
+ end
+
+ Link.where(head_uuid: uuid, link_class: "provenance").each do |link|
+ visited[link.uuid] = link.as_api_response
+ generate_provenance_edges(visited, link.tail_uuid)
+ end
+
+ #puts "finished #{uuid}"
+ end
+
+ def provenance
+ visited = {}
+ generate_provenance_edges(visited, @object[:uuid])
+ render json: visited
+ end
+
+
+ protected
def find_object_by_uuid
super
if !@object and !params[:uuid].match(/^[0-9a-f]+\+\d+$/)
end
end
end
+
end
+require 'assign_uuid'
class ArvadosModel < ActiveRecord::Base
self.abstract_class = true
end
end
end
+
+ def self.resource_class_for_uuid(uuid)
+ if uuid.is_a? ArvadosModel
+ return uuid.class
+ end
+ unless uuid.is_a? String
+ return nil
+ end
+ if uuid.match /^[0-9a-f]{32}(\+[^,]+)*(,[0-9a-f]{32}(\+[^,]+)*)*$/
+ return Collection
+ end
+ resource_class = nil
+
+ Rails.application.eager_load!
+ uuid.match /^[0-9a-z]{5}-([0-9a-z]{5})-[0-9a-z]{15}$/ do |re|
+ ActiveRecord::Base.descendants.reject(&:abstract_class?).each do |k|
+ if k.respond_to?(:uuid_prefix)
+ if k.uuid_prefix == re[1]
+ return k
+ end
+ end
+ end
+ end
+ nil
+ end
+
end
match '/repositories/get_all_permissions' => 'repositories#get_all_permissions'
get '/user_agreements/signatures' => 'user_agreements#signatures'
post '/user_agreements/sign' => 'user_agreements#sign'
+ get '/collections/:uuid/provenance' => 'collections#provenance'
resources :collections
resources :links
resources :nodes
projects / arvados.git / commitdiff