From: Peter Amstutz <peter.amstutz@curoverse.com>
Date: Thu, 30 Jan 2014 21:18:43 +0000 (-0500)
Subject: Further improvements to provenance graphs, added ability to select a few jobs and... 
X-Git-Tag: 1.1.0~2722^2~34
X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/aa66a90eecda1fc79f869ceee36eaed12d806145

Further improvements to provenance graphs, added ability to select a few jobs and shows the graph for those jobs.
---

diff --git a/apps/workbench/app/controllers/collections_controller.rb b/apps/workbench/app/controllers/collections_controller.rb
index 27f5bb7594..caf49baa74 100644
--- a/apps/workbench/app/controllers/collections_controller.rb
+++ b/apps/workbench/app/controllers/collections_controller.rb
@@ -99,7 +99,7 @@ class CollectionsController < ApplicationController
     end
     
     Collection.where(uuid: @object.uuid).each do |u|
-      @prov_svg = ProvenanceHelper::create_provenance_graph u.provenance, u.uuid, {:direction => :bottom_up}
+      @prov_svg = ProvenanceHelper::create_provenance_graph u.provenance, {:direction => :bottom_up, :combine_jobs => true}
     end
   end
 
diff --git a/apps/workbench/app/controllers/jobs_controller.rb b/apps/workbench/app/controllers/jobs_controller.rb
index 1cd3c3e28e..061db6de86 100644
--- a/apps/workbench/app/controllers/jobs_controller.rb
+++ b/apps/workbench/app/controllers/jobs_controller.rb
@@ -1,5 +1,24 @@
 class JobsController < ApplicationController
   def index
-    @jobs = Job.all
+    @svg = ""
+    if params[:uuid]
+        @jobs = Job.where(uuid: params[:uuid])
+        nodes = []
+        collections = []
+        @jobs.each do |j|
+          nodes << j
+          collections << j[:output]
+          collections.concat(ProvenanceHelper::find_collections(j[:script_parameters]))
+          nodes << {:uuid => j[:script_version]}
+        end
+
+        Collection.where(uuid: collections).each do |c|
+          nodes << c
+        end
+
+      @svg = ProvenanceHelper::create_provenance_graph(nodes, {:all_script_parameters => true, :script_version_nodes => true})
+    else
+      @jobs = Job.all
+    end
   end
 end
diff --git a/apps/workbench/app/controllers/pipeline_instances_controller.rb b/apps/workbench/app/controllers/pipeline_instances_controller.rb
index 37e743f99e..a92694010f 100644
--- a/apps/workbench/app/controllers/pipeline_instances_controller.rb
+++ b/apps/workbench/app/controllers/pipeline_instances_controller.rb
@@ -19,9 +19,7 @@ class PipelineInstancesController < ApplicationController
       provenance[c.uuid.intern] = c
     end
 
-    PipelineInstance.where(uuid: @object.uuid).each do |u|
-      @prov_svg = ProvenanceHelper::create_provenance_graph provenance, collections, {:all_script_parameters => true}
-    end
+    @prov_svg = ProvenanceHelper::create_provenance_graph provenance, {:all_script_parameters => true}
   end
 
 end
diff --git a/apps/workbench/app/helpers/provenance_helper.rb b/apps/workbench/app/helpers/provenance_helper.rb
index dbc7dbb976..9d9f44b335 100644
--- a/apps/workbench/app/helpers/provenance_helper.rb
+++ b/apps/workbench/app/helpers/provenance_helper.rb
@@ -1,210 +1,268 @@
 module ProvenanceHelper
-  def self.describe_node(pdata, uuid)
-    rsc = ArvadosBase::resource_class_for_uuid uuid.to_s
-    if rsc
-      href = "/#{rsc.to_s.underscore.pluralize rsc}/#{uuid}"
-
-      #"\"#{uuid}\" [label=\"#{rsc}\\n#{uuid}\",href=\"#{href}\"];\n"
-      if rsc == Collection
-        if pdata[uuid] 
-          #puts pdata[uuid]
-          if pdata[uuid][:name]
-            return "\"#{uuid}\" [label=\"#{pdata[uuid][:name]}\",href=\"#{href}\",shape=oval];\n"
-          else
-            files = nil
-            if pdata[uuid].respond_to? :files
-              files = pdata[uuid].files
-            elsif pdata[uuid][:files]
-              files = pdata[uuid][:files]
-            end
-            
-            if files
-              i = 0
-              label = ""
-              while i < 3 and i < files.length
-                label += "\\n" unless label == ""
-                label += files[i][1]
-                i += 1
+
+  class GenerateGraph
+    def initialize(pdata, opts)
+      @pdata = pdata
+      @opts = opts
+      @visited = {}
+      @jobs = {}
+    end
+
+    def self.collection_uuid(uuid)
+      m = /^([a-f0-9]{32}(\+[0-9]+)?)(\+.*)?$/.match(uuid.to_s)
+      if m
+        #if m[2]
+        return m[1]
+        #else
+        #  Collection.where(uuid: ['contains', m[1]]).each do |u|
+        #    puts "fixup #{uuid} to #{u.uuid}"
+        #    return u.uuid
+        #  end
+        #end
+      else
+        nil
+      end
+    end
+
+    def describe_node(uuid)
+      rsc = ArvadosBase::resource_class_for_uuid uuid.to_s
+      if rsc
+        href = "/#{rsc.to_s.underscore.pluralize rsc}/#{uuid}"
+
+        #"\"#{uuid}\" [label=\"#{rsc}\\n#{uuid}\",href=\"#{href}\"];\n"
+        if rsc == Collection
+          if @pdata[uuid] 
+            #puts @pdata[uuid]
+            if @pdata[uuid][:name]
+              return "\"#{uuid}\" [label=\"#{@pdata[uuid][:name]}\",href=\"#{href}\",shape=oval];\n"
+            else
+              files = nil
+              if @pdata[uuid].respond_to? :files
+                files = @pdata[uuid].files
+              elsif @pdata[uuid][:files]
+                files = @pdata[uuid][:files]
               end
-              if i < files.length
-                label += "\\n&vellip;"
+              
+              if files
+                i = 0
+                label = ""
+                while i < 3 and i < files.length
+                  label += "\\n" unless label == ""
+                  label += files[i][1]
+                  i += 1
+                end
+                if i < files.length
+                  label += "\\n&vellip;"
+                end
+                return "\"#{uuid}\" [label=\"#{label}\",href=\"#{href}\",shape=oval];\n"
               end
-              return "\"#{uuid}\" [label=\"#{label}\",href=\"#{href}\",shape=oval];\n"
-            end
-          end  
+            end  
+          end
+          return "\"#{uuid}\" [label=\"#{rsc}\",href=\"#{href}\"];\n"
         end
-        return "\"#{uuid}\" [label=\"#{rsc}\",href=\"#{href}\"];\n"
       end
+      return ""
     end
-    return ""
-  end
 
-  def self.job_uuid(job)
-    # "#{job[:script]}\\n#{job[:script_version]}"
-    "#{job[:script]}"
-  end
+    def job_uuid(job)
+      if @opts[:combine_jobs]
+        uuid = "#{job[:script]}"
+      else
+        uuid = "#{job[:uuid]}"
+      end
 
-  def self.collection_uuid(uuid)
-    m = /([a-f0-9]{32}(\+[0-9]+)?)(\+.*)?/.match(uuid.to_s)
-    if m
-      m[1]
-    else
-      nil
-    end
-  end
+      @jobs[uuid] = [] unless @jobs[uuid]
+      @jobs[uuid] << job unless @jobs[uuid].include? job
 
-  def self.edge(tail, head, extra, opts)
-    if opts[:direction] == :bottom_up
-      gr = "\"#{tail}\" -> \"#{head}\""
-    else
-      gr = "\"#{head}\" -> \"#{tail}\""
+      uuid
     end
-    if extra.length > 0
-      gr += "["
-      extra.each do |k, v|
-        gr += "#{k}=\"#{v}\","
+
+    def edge(tail, head, extra)
+      if @opts[:direction] == :bottom_up
+        gr = "\"#{tail}\" -> \"#{head}\""
+      else
+        gr = "\"#{head}\" -> \"#{tail}\""
       end
-      gr += "]"
+      if extra.length > 0
+        gr += "["
+        extra.each do |k, v|
+          gr += "#{k}=\"#{v}\","
+        end
+        gr += "]"
+      end
+      gr += ";\n"
+      gr
     end
-    gr += ";\n"
-    gr
-  end
 
-  def self.script_param_edges(pdata, visited, job, prefix, sp, opts)
-    gr = ""
-    if sp and not sp.empty?
-      case sp
-      when Hash
-        sp.each do |k, v|
-          if prefix.size > 0
-            k = prefix + "::" + k.to_s
+    def script_param_edges(job, prefix, sp)
+      gr = ""
+      if sp and not sp.empty?
+        case sp
+        when Hash
+          sp.each do |k, v|
+            if prefix.size > 0
+              k = prefix + "::" + k.to_s
+            end
+            gr += script_param_edges(job, k.to_s, v)
           end
-          gr += ProvenanceHelper::script_param_edges(pdata, visited, job, k.to_s, v, opts)
-        end
-      when Array
-        i = 0
-        node = ""
-        sp.each do |v|
-          if collection_uuid(v)
-            gr += ProvenanceHelper::script_param_edges(pdata, visited, job, "#{prefix}[#{i}]", v, opts)
-          else
-            node += "', '" unless node == ""
-            node = "['" if node == ""
-            node += "#{v}"
+        when Array
+          i = 0
+          node = ""
+          sp.each do |v|
+            if GenerateGraph::collection_uuid(v)
+              gr += script_param_edges(job, "#{prefix}[#{i}]", v)
+            else
+              node += "', '" unless node == ""
+              node = "['" if node == ""
+              node += "#{v}"
+            end
+            i += 1
+          end
+          unless node == ""
+            node += "']"
+            #puts node
+            #id = "#{job[:uuid]}_#{prefix}"
+            gr += "\"#{node}\" [label=\"#{node}\"];\n"
+            gr += edge(job_uuid(job), node, {:label => prefix})        
+          end
+        else
+          m = GenerateGraph::collection_uuid(sp)
+          if m
+            gr += edge(job_uuid(job), m, {:label => prefix})
+            gr += generate_provenance_edges(m)
+          elsif @opts[:all_script_parameters]
+            #id = "#{job[:uuid]}_#{prefix}"
+            gr += "\"#{sp}\" [label=\"#{sp}\"];\n"
+            gr += edge(job_uuid(job), sp, {:label => prefix})
           end
-          i += 1
-        end
-        unless node == ""
-          node += "']"
-          #puts node
-          #id = "#{job[:uuid]}_#{prefix}"
-          gr += "\"#{node}\" [label=\"#{node}\"];\n"
-          gr += edge(job_uuid(job), node, {:label => prefix}, opts)        
-        end
-      else
-        m = collection_uuid(sp)
-        if m
-          gr += edge(job_uuid(job), m, {:label => prefix}, opts)
-          gr += ProvenanceHelper::generate_provenance_edges(pdata, visited, m, opts)
-        elsif opts[:all_script_parameters]
-          #id = "#{job[:uuid]}_#{prefix}"
-          gr += "\"#{sp}\" [label=\"#{sp}\"];\n"
-          gr += edge(job_uuid(job), sp, {:label => prefix}, opts)
         end
       end
+      gr
     end
-    gr
-  end
 
-  def self.generate_provenance_edges(pdata, visited, uuid, opts)
-    gr = ""
-    m = ProvenanceHelper::collection_uuid(uuid)
-    uuid = m if m
+    def generate_provenance_edges(uuid)
+      gr = ""
+      m = GenerateGraph::collection_uuid(uuid)
+      uuid = m if m
 
-    uuid = uuid.intern if uuid
+      uuid = uuid.intern if uuid
 
-    if (not uuid) or uuid.empty? or visited[uuid]
+      if (not uuid) or uuid.empty? or @visited[uuid]
 
-      #puts "already visited #{uuid}"
-      return ""
-    end
+        #puts "already @visited #{uuid}"
+        return ""
+      end
 
-    if not pdata[uuid] then 
-      return ProvenanceHelper::describe_node(pdata, uuid)
-    else
-      visited[uuid] = true
-    end
+      if not @pdata[uuid] then 
+        return describe_node(uuid)
+      else
+        @visited[uuid] = true
+      end
 
-    #puts "visiting #{uuid}"
+      #puts "visiting #{uuid}"
 
-    if m  
-      # uuid is a collection
-      gr += ProvenanceHelper::describe_node(pdata, uuid)
+      if m  
+        # uuid is a collection
+        gr += describe_node(uuid)
 
-      pdata.each do |k, job|
-        if job[:output] == uuid.to_s
-          gr += self.edge(uuid, job_uuid(job), {:label => "output"}, opts)
-          gr += ProvenanceHelper::generate_provenance_edges(pdata, visited, job[:uuid], opts)
+        @pdata.each do |k, job|
+          if job[:output] == uuid.to_s
+            gr += edge(uuid, job_uuid(job), {:label => "output"})
+            gr += generate_provenance_edges(job[:uuid])
+          end
+          if job[:log] == uuid.to_s
+            gr += edge(uuid, job_uuid(job), {:label => "log"})
+            gr += generate_provenance_edges(job[:uuid])
+          end
         end
-        if job[:log] == uuid.to_s
-          gr += edge(uuid, job_uuid(job), {:label => "log"}, opts)
-          gr += ProvenanceHelper::generate_provenance_edges(pdata, visited, job[:uuid], opts)
+      else
+        # uuid is something else
+        rsc = ArvadosBase::resource_class_for_uuid uuid.to_s
+
+        if rsc == Job
+          job = @pdata[uuid]
+          if job
+            gr += script_param_edges(job, "", job[:script_parameters])
+
+            if @opts[:script_version_nodes]
+              gr += edge(job_uuid(job), job[:script_version], {:label => "script_version"})
+            end
+          end
+        else
+          gr += describe_node(uuid)
         end
       end
-    else
-      # uuid is something else
-      rsc = ArvadosBase::resource_class_for_uuid uuid.to_s
 
-      if rsc == Job
-        job = pdata[uuid]
-        if job
-          gr += ProvenanceHelper::script_param_edges(pdata, visited, job, "", job[:script_parameters], opts)
+      @pdata.each do |k, link|
+        if link[:head_uuid] == uuid.to_s and link[:link_class] == "provenance"
+          gr += describe_node(link[:tail_uuid])
+          gr += edge(link[:head_uuid], link[:tail_uuid], {:label => link[:name], :href => "/links/#{link[:uuid]}"}) 
+          gr += generate_provenance_edges(link[:tail_uuid])
         end
-      else
-        gr += ProvenanceHelper::describe_node(pdata, uuid)
       end
+
+      #puts "finished #{uuid}"
+
+      gr
     end
 
-    pdata.each do |k, link|
-      if link[:head_uuid] == uuid.to_s and link[:link_class] == "provenance"
-        gr += ProvenanceHelper::describe_node(pdata, link[:tail_uuid])
-        gr += edge(link[:head_uuid], link[:tail_uuid], {:label => link[:name], :href => "/links/#{link[:uuid]}"}, opts) 
-        gr += ProvenanceHelper::generate_provenance_edges(pdata, visited, link[:tail_uuid], opts)
+    def add_jobs_href
+      gr = ""
+      @jobs.each do |k, v|
+        gr += "\"#{k}\" [href=\"/jobs?"
+        script = ""
+        v.each do |u|
+          gr += "uuid%5b%5d=#{u[:uuid]}&"
+          script = u[:script]
+        end
+        gr += "\",label=\""
+        gr += if @opts[:combine_jobs] then "#{script}" else "#{script}\\n#{v[0][:finished_at]}" end
+        gr += "\"];\n"
       end
+      gr
     end
 
-    #puts "finished #{uuid}"
-
-    gr
   end
 
-  def self.create_provenance_graph(pdata, uuid, opts={})
-    require 'open3'
+  def self.create_provenance_graph(pdata, opts={})
+    if pdata.is_a? Array or pdata.is_a? ArvadosResourceList
+      p2 = {}
+      pdata.each do |k|
+        p2[k[:uuid].intern] = k if k[:uuid]
+      end
+      pdata = p2
+    end
+
+    unless pdata.is_a? Hash
+      raise "create_provenance_graph accepts Array or Hash for pdata only, pdata is #{pdata.class}"
+    end
     
     gr = """strict digraph {
 node [fontsize=8,shape=box];
-edge [fontsize=8];"""
+edge [fontsize=8];
+"""
 
     if opts[:direction] == :bottom_up
       gr += "edge [dir=back];"
     end
 
-    #puts "pdata is #{pdata}"
+    #puts "@pdata is #{pdata}"
 
-    visited = {}
-    if uuid.respond_to? :each
-      uuid.each do |u|
-        gr += ProvenanceHelper::generate_provenance_edges(pdata, visited, u, opts)
-      end
-    else
-      gr += ProvenanceHelper::generate_provenance_edges(pdata, visited, uuid, opts)
+    g = GenerateGraph.new(pdata, opts)
+
+    pdata.each do |k, v|
+      gr += g.generate_provenance_edges(k)
     end
 
+    gr += g.add_jobs_href
+
     gr += "}"
     svg = ""
 
     #puts gr
 
+    require 'open3'
+
     Open3.popen2("dot", "-Tsvg") do |stdin, stdout, wait_thr|
       stdin.print(gr)
       stdin.close
@@ -216,4 +274,26 @@ edge [fontsize=8];"""
     svg = svg.sub(/<\?xml.*?\?>/m, "")
     svg = svg.sub(/<!DOCTYPE.*?>/m, "")
   end
+
+  def self.find_collections(sp)
+    c = []
+    if sp and not sp.empty?
+      case sp
+      when Hash
+        sp.each do |k, v|
+          c.concat(find_collections(v))
+        end
+      when Array
+        sp.each do |v|
+          c.concat(find_collections(v))
+        end
+      else
+        m = GenerateGraph::collection_uuid(sp)
+        if m
+          c << m
+        end
+      end
+    end
+    c
+  end
 end
diff --git a/apps/workbench/app/views/jobs/index.html.erb b/apps/workbench/app/views/jobs/index.html.erb
index 2188a3bcf6..2c473ebd94 100644
--- a/apps/workbench/app/views/jobs/index.html.erb
+++ b/apps/workbench/app/views/jobs/index.html.erb
@@ -135,3 +135,5 @@
 
   </tbody>
 </table>
+
+<%= raw(@svg) %>