apps/workbench/app/helpers/provenance_helper.rb

   1 module ProvenanceHelper
   2
   3   class GenerateGraph
   4     def initialize(pdata, opts)
   5       @pdata = pdata
   6       @opts = opts
   7       @visited = {}
   8       @jobs = {}
   9     end
  10
  11     def self.collection_uuid(uuid)
  12       m = CollectionsHelper.match(uuid)
  13       if m
  14         if m[2]
  15           return m[1]+m[2]
  16         else
  17           return m[1]
  18         end
  19         #  Collection.where(uuid: ['contains', m[1]]).each do |u|
  20         #    puts "fixup #{uuid} to #{u.uuid}"
  21         #    return u.uuid
  22         #  end
  23         #end
  24       else
  25         nil
  26       end
  27     end
  28
  29     def determine_fillcolor(n)
  30       fillcolor = %w(aaaaaa aaffaa aaaaff aaaaaa ffaaaa)[n || 0] || 'aaaaaa'
  31       "style=filled,fillcolor=\"##{fillcolor}\""
  32     end
  33
  34     def describe_node(uuid)
  35       uuid = uuid.to_sym
  36       bgcolor = determine_fillcolor @opts[:pips][uuid] if @opts[:pips]
  37
  38       rsc = ArvadosBase::resource_class_for_uuid uuid.to_s
  39       if rsc
  40         href = Rails.application.routes.url_helpers.url_for ({:controller => rsc.to_s.tableize,
  41                                                                :action => :show,
  42                                                                :id => uuid.to_s,
  43                                                                :host => @opts[:request].host,
  44                                                                :port => @opts[:request].port})
  45
  46         #"\"#{uuid}\" [label=\"#{rsc}\\n#{uuid}\",href=\"#{href}\"];\n"
  47         if rsc == Collection
  48           #puts uuid
  49           if uuid == :"d41d8cd98f00b204e9800998ecf8427e+0"
  50             # special case
  51             #puts "empty!"
  52             return "\"#{uuid}\" [label=\"(empty collection)\"];\n"
  53           end
  54           puts "#{uuid.class} #{@pdata[uuid]}"
  55           if @pdata[uuid]
  56             #puts @pdata[uuid]
  57             if @pdata[uuid][:name]
  58               return "\"#{uuid}\" [label=\"#{@pdata[uuid][:name]}\",href=\"#{href}\",shape=oval,#{bgcolor}];\n"
  59             else
  60               files = nil
  61               if @pdata[uuid].respond_to? :files
  62                 files = @pdata[uuid].files
  63               elsif @pdata[uuid][:files]
  64                 files = @pdata[uuid][:files]
  65               end
  66
  67               if files
  68                 i = 0
  69                 label = ""
  70                 while i < 3 and i < files.length
  71                   label += "\\n" unless label == ""
  72                   label += files[i][1]
  73                   i += 1
  74                 end
  75                 if i < files.length
  76                   label += "\\n&vellip;"
  77                 end
  78                 #puts "#{uuid} #{label} #{files}"
  79                 return "\"#{uuid}\" [label=\"#{label}\",href=\"#{href}\",shape=oval,#{bgcolor}];\n"
  80               end
  81             end
  82           end
  83           return "\"#{uuid}\" [label=\"#{rsc}\",href=\"#{href}\",#{bgcolor}];\n"
  84         end
  85       end
  86       "\"#{uuid}\" [#{bgcolor}];\n"
  87     end
  88
  89     def job_uuid(job)
  90       if @opts[:combine_jobs] == :script_only
  91         uuid = "#{job[:script]}"
  92       elsif @opts[:combine_jobs] == :script_and_version
  93         uuid = "#{job[:script]}_#{job[:script_version]}"
  94       else
  95         uuid = "#{job[:uuid]}"
  96       end
  97
  98       @jobs[uuid] = [] unless @jobs[uuid]
  99       @jobs[uuid] << job unless @jobs[uuid].include? job
 100
 101       uuid
 102     end
 103
 104     def edge(tail, head, extra)
 105       if @opts[:direction] == :bottom_up
 106         gr = "\"#{tail}\" -> \"#{head}\""
 107       else
 108         gr = "\"#{head}\" -> \"#{tail}\""
 109       end
 110       if extra.length > 0
 111         gr += " ["
 112         extra.each do |k, v|
 113           gr += "#{k}=\"#{v}\","
 114         end
 115         gr += "]"
 116       end
 117       gr += ";\n"
 118       gr
 119     end
 120
 121     def script_param_edges(job, prefix, sp)
 122       gr = ""
 123       case sp
 124       when Hash
 125         sp.each do |k, v|
 126           if prefix.size > 0
 127             k = prefix + "::" + k.to_s
 128           end
 129           gr += script_param_edges(job, k.to_s, v)
 130         end
 131       when Array
 132         i = 0
 133         node = ""
 134         sp.each do |v|
 135           if GenerateGraph::collection_uuid(v)
 136             gr += script_param_edges(job, "#{prefix}[#{i}]", v)
 137           elsif @opts[:all_script_parameters]
 138             node += "', '" unless node == ""
 139             node = "['" if node == ""
 140             node += "#{v}"
 141           end
 142           i += 1
 143         end
 144         unless node == ""
 145           node += "']"
 146           #puts node
 147           #id = "#{job[:uuid]}_#{prefix}"
 148           gr += "\"#{node}\" [label=\"#{node}\"];\n"
 149           gr += edge(job_uuid(job), node, {:label => prefix})
 150         end
 151       when String
 152         return '' if sp.empty?
 153         m = GenerateGraph::collection_uuid(sp)
 154         #puts "#{m} pdata is #{@pdata[m.intern]}"
 155         if m and (@pdata[m.intern] or (not @opts[:pdata_only]))
 156           gr += edge(job_uuid(job), m, {:label => prefix})
 157           gr += generate_provenance_edges(m)
 158         elsif @opts[:all_script_parameters]
 159           #id = "#{job[:uuid]}_#{prefix}"
 160           gr += "\"#{sp}\" [label=\"#{sp}\"];\n"
 161           gr += edge(job_uuid(job), sp, {:label => prefix})
 162         end
 163       end
 164       gr
 165     end
 166
 167     def generate_provenance_edges(uuid)
 168       gr = ""
 169       m = GenerateGraph::collection_uuid(uuid)
 170       uuid = m if m
 171
 172       uuid = uuid.intern if uuid
 173
 174       if (not uuid) or uuid.empty? or @visited[uuid]
 175
 176         #puts "already @visited #{uuid}"
 177         return ""
 178       end
 179
 180       if not @pdata[uuid] then
 181         return describe_node(uuid)
 182       else
 183         @visited[uuid] = true
 184       end
 185
 186       #puts "visiting #{uuid}"
 187
 188       if m
 189         # uuid is a collection
 190         gr += describe_node(uuid)
 191
 192         if m == :"d41d8cd98f00b204e9800998ecf8427e+0"
 193           # empty collection, don't follow any further
 194           return gr
 195         end
 196
 197         @pdata.each do |k, job|
 198           if job[:output] == uuid.to_s
 199             gr += edge(uuid, job_uuid(job), {:label => "output"})
 200             gr += generate_provenance_edges(job[:uuid])
 201           end
 202           if job[:log] == uuid.to_s
 203             gr += edge(uuid, job_uuid(job), {:label => "log"})
 204             gr += generate_provenance_edges(job[:uuid])
 205           end
 206         end
 207       else
 208         # uuid is something else
 209         rsc = ArvadosBase::resource_class_for_uuid uuid.to_s
 210
 211         if rsc == Job
 212           job = @pdata[uuid]
 213           if job
 214             gr += script_param_edges(job, "", job[:script_parameters])
 215
 216             if @opts[:script_version_nodes]
 217               gr += describe_node(job[:script_version])
 218               gr += edge(job_uuid(job), job[:script_version], {:label => "script_version"})
 219             end
 220           end
 221         elsif rsc == Link
 222           # do nothing
 223         else
 224           gr += describe_node(uuid)
 225         end
 226       end
 227
 228       @pdata.each do |k, link|
 229         if link[:head_uuid] == uuid.to_s and link[:link_class] == "provenance"
 230           href = Rails.application.routes.url_helpers.url_for ({:controller => Link.to_s.tableize,
 231                                                                  :action => :show,
 232                                                                  :id => link[:uuid],
 233                                                                  :host => @opts[:request].host,
 234                                                                  :port => @opts[:request].port})
 235
 236           gr += describe_node(link[:tail_uuid])
 237           gr += edge(link[:head_uuid], link[:tail_uuid], {:label => link[:name], :href => href})
 238           gr += generate_provenance_edges(link[:tail_uuid])
 239         end
 240       end
 241
 242       #puts "finished #{uuid}"
 243
 244       gr
 245     end
 246
 247     def describe_jobs
 248       gr = ""
 249       @jobs.each do |k, v|
 250         href = Rails.application.routes.url_helpers.url_for ({:controller => Job.to_s.tableize,
 251                                                                :action => :index,
 252                                                                :host => @opts[:request].host,
 253                                                                :port => @opts[:request].port})
 254
 255         gr += "\"#{k}\" [href=\"#{href}?"
 256
 257         n = 0
 258         v.each do |u|
 259           gr += "uuid%5b%5d=#{u[:uuid]}&"
 260           n |= @opts[:pips][u[:uuid].intern] if @opts[:pips] and @opts[:pips][u[:uuid].intern]
 261         end
 262
 263         gr += "\",label=\""
 264
 265         if @opts[:combine_jobs] == :script_only
 266           gr += "#{v[0][:script]}"
 267         elsif @opts[:combine_jobs] == :script_and_version
 268           gr += "#{v[0][:script]}" # Just show the name but the nodes will be distinct
 269         else
 270           gr += "#{v[0][:script]}\\n#{v[0][:finished_at]}"
 271         end
 272         gr += "\",#{determine_fillcolor n}];\n"
 273       end
 274       gr
 275     end
 276
 277   end
 278
 279   def self.create_provenance_graph(pdata, svgId, opts={})
 280     if pdata.is_a? Array or pdata.is_a? ArvadosResourceList
 281       p2 = {}
 282       pdata.each do |k|
 283         p2[k[:uuid].intern] = k if k[:uuid]
 284       end
 285       pdata = p2
 286     end
 287
 288     unless pdata.is_a? Hash
 289       raise "create_provenance_graph accepts Array or Hash for pdata only, pdata is #{pdata.class}"
 290     end
 291
 292     gr = """strict digraph {
 293 node [fontsize=10,shape=box];
 294 edge [fontsize=10];
 295 """
 296
 297     if opts[:direction] == :bottom_up
 298       gr += "edge [dir=back];"
 299     end
 300
 301     #puts "@pdata is #{pdata}"
 302
 303     g = GenerateGraph.new(pdata, opts)
 304
 305     pdata.each do |k, v|
 306       gr += g.generate_provenance_edges(k)
 307     end
 308
 309     gr += g.describe_jobs
 310
 311     gr += "}"
 312     svg = ""
 313
 314     puts gr
 315
 316     require 'open3'
 317
 318     Open3.popen2("dot", "-Tsvg") do |stdin, stdout, wait_thr|
 319       stdin.print(gr)
 320       stdin.close
 321       svg = stdout.read()
 322       wait_thr.value
 323       stdout.close()
 324     end
 325
 326     svg = svg.sub(/<\?xml.*?\?>/m, "")
 327     svg = svg.sub(/<!DOCTYPE.*?>/m, "")
 328     svg = svg.sub(/<svg /, "<svg id=\"#{svgId}\" ")
 329   end
 330
 331   def self.find_collections(sp)
 332     c = []
 333     case sp
 334     when Hash
 335       sp.each do |k, v|
 336         c.concat(find_collections(v))
 337       end
 338     when Array
 339       sp.each do |v|
 340         c.concat(find_collections(v))
 341       end
 342     when String
 343       if !sp.empty?
 344         m = GenerateGraph::collection_uuid(sp)
 345         if m
 346           c << m
 347         end
 348       end
 349     end
 350     c
 351   end
 352 end