apps/workbench/app/controllers/collections_controller.rb

   1 class CollectionsController < ApplicationController
   2   skip_before_filter :find_object_by_uuid, :only => [:provenance]
   3   skip_before_filter :check_user_agreements, :only => [:show_file]
   4
   5   def index
   6     if params[:search].andand.length.andand > 0
   7       tags = Link.where(any: ['contains', params[:search]])
   8       @collections = (Collection.where(uuid: tags.collect(&:head_uuid)) |
   9                       Collection.where(any: ['contains', params[:search]])).
  10         uniq { |c| c.uuid }
  11     else
  12       @collections = Collection.limit(100)
  13     end
  14     @links = Link.limit(1000).
  15       where(head_uuid: @collections.collect(&:uuid))
  16     @collection_info = {}
  17     @collections.each do |c|
  18       @collection_info[c.uuid] = {
  19         tags: [],
  20         wanted: false,
  21         wanted_by_me: false,
  22         provenance: [],
  23         links: []
  24       }
  25     end
  26     @links.each do |link|
  27       @collection_info[link.head_uuid] ||= {}
  28       info = @collection_info[link.head_uuid]
  29       case link.link_class
  30       when 'tag'
  31         info[:tags] << link.name
  32       when 'resources'
  33         info[:wanted] = true
  34         info[:wanted_by_me] ||= link.tail_uuid == current_user.uuid
  35       when 'provenance'
  36         info[:provenance] << link.name
  37       end
  38       info[:links] << link
  39     end
  40     @request_url = request.url
  41   end
  42
  43   def show_file
  44     opts = params.merge(arvados_api_token: Thread.current[:arvados_api_token])
  45     if r = params[:file].match(/(\.\w+)/)
  46       ext = r[1]
  47     end
  48     self.response.headers['Content-Type'] =
  49       Rack::Mime::MIME_TYPES[ext] || 'application/octet-stream'
  50     self.response.headers['Content-Length'] = params[:size] if params[:size]
  51     self.response.headers['Content-Disposition'] = params[:disposition] if params[:disposition]
  52     self.response_body = FileStreamer.new opts
  53   end
  54
  55   def describe_node(uuid)
  56     rsc = ArvadosBase::resource_class_for_uuid uuid
  57     if rsc
  58       "\"#{uuid}\" [label=\"#{rsc}\\n#{uuid}\",href=\"#{url_for rsc}/#{uuid}\"];"
  59     else
  60       ""
  61     end
  62   end
  63
  64   def describe_script(job)
  65     #"""\"#{job.script_version}\" [label=\"#{job.script}: #{job.script_version}\"];
  66     #   \"#{job.uuid}\" -> \"#{job.script_version}\" [label=\"script\"];"""
  67     "\"#{job.uuid}\" [label=\"#{job.script}\\n#{job.script_version}\"];"
  68   end
  69
  70   def job_uuid(job)
  71     "#{job.script}\\n#{job.script_version}"
  72   end
  73
  74   def collection_uuid(uuid)
  75     m = /([a-f0-9]{32}(\+[0-9]+)?)(\+.*)?/.match(uuid)
  76     if m
  77       m[1]
  78     else
  79       nil
  80     end
  81   end
  82
  83   def script_param_edges(visited, job, prefix, sp)
  84     gr = ""
  85     if sp and not sp.empty?
  86       case sp
  87       when Hash
  88         sp.each do |k, v|
  89           if prefix.size > 0
  90             k = prefix + "::" + k.to_s
  91           end
  92           gr += script_param_edges(visited, job, k.to_s, v)
  93         end
  94       when Array
  95         sp.each do |v|
  96           gr += script_param_edges(visited, job, prefix, v)
  97         end
  98       else
  99         m = collection_uuid(sp)
 100         if m
 101           gr += "\"#{job_uuid(job)}\" -> \"#{m}\" [label=\" #{prefix}\"];"
 102           gr += generate_provenance_edges(visited, m)
 103         end
 104       end
 105     end
 106     gr
 107   end
 108
 109   def generate_provenance_edges(visited, uuid)
 110     gr = ""
 111     m = collection_uuid(uuid)
 112
 113     if not uuid or uuid.empty? or visited[uuid] or visited[m]
 114       return ""
 115     end
 116
 117     #puts "visiting #{uuid}"
 118
 119     if m
 120       # uuid is a collection
 121       uuid = m
 122       visited[uuid] = true
 123
 124       gr += describe_node(uuid)
 125
 126       Job.where(output: uuid).each do |job|
 127         #gr += describe_node(job_uuid(job))
 128         gr += "\"#{uuid}\" -> \"#{job_uuid(job)}\" [label=\" output\"];"
 129         gr += generate_provenance_edges(visited, job.uuid)
 130       end
 131
 132       Job.where(log: uuid).each do |job|
 133         #gr += describe_node(job_uuid(job))
 134         gr += "\"#{uuid}\" -> \"#{job_uuid(job)}\" [label=\" log\"];"
 135         gr += generate_provenance_edges(visited, job.uuid)
 136       end
 137
 138     else
 139       visited[uuid] = true
 140
 141       # uuid is something else
 142       rsc = ArvadosBase::resource_class_for_uuid uuid
 143
 144       if rsc == Job
 145         Job.where(uuid: uuid).each do |job|
 146           gr += script_param_edges(visited, job, "", job.script_parameters)
 147           #gr += describe_script(job)
 148         end
 149       else
 150         gr += describe_node(uuid)
 151       end
 152     end
 153
 154     Link.where(head_uuid: uuid, link_class: "provenance").each do |link|
 155       gr += describe_node(link.tail_uuid)
 156       gr += "\"#{link.head_uuid}\" -> \"#{link.tail_uuid}\" [label=\" #{link.name}\", href=\"/links/#{link.uuid}\"];"
 157       gr += generate_provenance_edges(visited, link.tail_uuid)
 158     end
 159
 160     #puts "finished #{uuid}"
 161
 162     gr
 163   end
 164
 165   def create_provenance_graph(uuid)
 166     require 'open3'
 167
 168     gr = """strict digraph {
 169 //rankdir=LR;
 170 node [fontsize=8,shape=box];
 171 edge [dir=back,fontsize=8];"""
 172
 173     visited = {}
 174     gr += generate_provenance_edges(visited, uuid)
 175
 176     gr += "}"
 177     svg = ""
 178
 179     Open3.popen2("dot", "-Tsvg") do |stdin, stdout, wait_thr|
 180       stdin.print(gr)
 181       stdin.close
 182       svg = stdout.read()
 183       wait_thr.value
 184       stdout.close()
 185     end
 186
 187     svg = svg.sub(/<\?xml.*?\?>/m, "")
 188     svg = svg.sub(/<!DOCTYPE.*?>/m, "")
 189   end
 190
 191   def show
 192     return super if !@object
 193     @provenance = []
 194     @output2job = {}
 195     @output2colorindex = {}
 196     @sourcedata = {params[:uuid] => {uuid: params[:uuid]}}
 197     @protected = {}
 198
 199     colorindex = -1
 200     any_hope_left = true
 201     while any_hope_left
 202       any_hope_left = false
 203       Job.where(output: @sourcedata.keys).sort_by { |a| a.finished_at || a.created_at }.reverse.each do |job|
 204         if !@output2colorindex[job.output]
 205           any_hope_left = true
 206           @output2colorindex[job.output] = (colorindex += 1) % 10
 207           @provenance << {job: job, output: job.output}
 208           @sourcedata.delete job.output
 209           @output2job[job.output] = job
 210           job.dependencies.each do |new_source_data|
 211             unless @output2colorindex[new_source_data]
 212               @sourcedata[new_source_data] = {uuid: new_source_data}
 213             end
 214           end
 215         end
 216       end
 217     end
 218
 219     Link.where(head_uuid: @sourcedata.keys | @output2job.keys).each do |link|
 220       if link.link_class == 'resources' and link.name == 'wants'
 221         @protected[link.head_uuid] = true
 222       end
 223     end
 224     Link.where(tail_uuid: @sourcedata.keys).each do |link|
 225       if link.link_class == 'data_origin'
 226         @sourcedata[link.tail_uuid][:data_origins] ||= []
 227         @sourcedata[link.tail_uuid][:data_origins] << [link.name, link.head_kind, link.head_uuid]
 228       end
 229     end
 230     Collection.where(uuid: @sourcedata.keys).each do |collection|
 231       if @sourcedata[collection.uuid]
 232         @sourcedata[collection.uuid][:collection] = collection
 233       end
 234     end
 235
 236     @prov_svg = create_provenance_graph(@object.uuid)
 237   end
 238
 239   protected
 240   class FileStreamer
 241     def initialize(opts={})
 242       @opts = opts
 243     end
 244     def each
 245       return unless @opts[:uuid] && @opts[:file]
 246       env = Hash[ENV].
 247         merge({
 248                 'ARVADOS_API_HOST' =>
 249                 $arvados_api_client.arvados_v1_base.
 250                 sub(/\/arvados\/v1/, '').
 251                 sub(/^https?:\/\//, ''),
 252                 'ARVADOS_API_TOKEN' =>
 253                 @opts[:arvados_api_token],
 254                 'ARVADOS_API_HOST_INSECURE' =>
 255                 Rails.configuration.arvados_insecure_https ? 'true' : 'false'
 256               })
 257       IO.popen([env, 'arv-get', "#{@opts[:uuid]}/#{@opts[:file]}"],
 258                'rb') do |io|
 259         while buf = io.read(2**20)
 260           yield buf
 261         end
 262       end
 263       Rails.logger.warn("#{@opts[:uuid]}/#{@opts[:file]}: #{$?}") if $? != 0
 264     end
 265   end
 266 end