apps/workbench/app/controllers/collections_controller.rb

   1 class CollectionsController < ApplicationController
   2   skip_before_filter :find_object_by_uuid, :only => [:provenance]
   3   skip_before_filter :check_user_agreements, :only => [:show_file]
   4
   5   def index
   6     if params[:search].andand.length.andand > 0
   7       tags = Link.where(any: ['contains', params[:search]])
   8       @collections = (Collection.where(uuid: tags.collect(&:head_uuid)) |
   9                       Collection.where(any: ['contains', params[:search]])).
  10         uniq { |c| c.uuid }
  11     else
  12       @collections = Collection.limit(100)
  13     end
  14     @links = Link.limit(1000).
  15       where(head_uuid: @collections.collect(&:uuid))
  16     @collection_info = {}
  17     @collections.each do |c|
  18       @collection_info[c.uuid] = {
  19         tags: [],
  20         wanted: false,
  21         wanted_by_me: false,
  22         provenance: [],
  23         links: []
  24       }
  25     end
  26     @links.each do |link|
  27       @collection_info[link.head_uuid] ||= {}
  28       info = @collection_info[link.head_uuid]
  29       case link.link_class
  30       when 'tag'
  31         info[:tags] << link.name
  32       when 'resources'
  33         info[:wanted] = true
  34         info[:wanted_by_me] ||= link.tail_uuid == current_user.uuid
  35       when 'provenance'
  36         info[:provenance] << link.name
  37       end
  38       info[:links] << link
  39     end
  40     @request_url = request.url
  41   end
  42
  43   def show_file
  44     opts = params.merge(arvados_api_token: Thread.current[:arvados_api_token])
  45     if r = params[:file].match(/(\.\w+)/)
  46       ext = r[1]
  47     end
  48     self.response.headers['Content-Type'] =
  49       Rack::Mime::MIME_TYPES[ext] || 'application/octet-stream'
  50     self.response.headers['Content-Length'] = params[:size] if params[:size]
  51     self.response.headers['Content-Disposition'] = params[:disposition] if params[:disposition]
  52     self.response_body = FileStreamer.new opts
  53   end
  54
  55   def self.describe_node(uuid)
  56     uuid = uuid.to_s
  57     rsc = ArvadosBase::resource_class_for_uuid uuid
  58     if rsc
  59       "\"#{uuid}\" [label=\"#{rsc}\\n#{uuid}\",href=\"/#{rsc.to_s.underscore.pluralize rsc}/#{uuid}\"];"
  60     else
  61       ""
  62     end
  63   end
  64
  65   def self.job_uuid(job)
  66     # "#{job[:script]}\\n#{job[:script_version]}"
  67     "#{job[:script]}"
  68   end
  69
  70   def self.collection_uuid(uuid)
  71     m = /([a-f0-9]{32}(\+[0-9]+)?)(\+.*)?/.match(uuid.to_s)
  72     if m
  73       m[1]
  74     else
  75       nil
  76     end
  77   end
  78
  79   def self.script_param_edges(visited, job, prefix, sp, opts)
  80     gr = ""
  81     if sp and not sp.empty?
  82       case sp
  83       when Hash
  84         sp.each do |k, v|
  85           if prefix.size > 0
  86             k = prefix + "::" + k.to_s
  87           end
  88           gr += CollectionsController::script_param_edges(visited, job, k.to_s, v, opts)
  89         end
  90       when Array
  91         sp.each do |v|
  92           gr += CollectionsController::script_param_edges(visited, job, prefix, v, opts)
  93         end
  94       else
  95         m = collection_uuid(sp)
  96         if m
  97           gr += "\"#{job_uuid(job)}\" -> \"#{m}\" [label=\" #{prefix}\"];"
  98           gr += CollectionsController::generate_provenance_edges(visited, m, opts)
  99         end
 100       end
 101     end
 102     gr
 103   end
 104
 105   def self.generate_provenance_edges(pdata, uuid, opts)
 106     gr = ""
 107     m = CollectionsController::collection_uuid(uuid)
 108     uuid = m if m
 109
 110     uuid = uuid.intern if uuid
 111
 112     if (not uuid) or uuid.empty? \
 113       or (pdata[uuid] and pdata[uuid][:_visited])
 114
 115       #puts "already visited #{uuid}"
 116       return ""
 117     end
 118
 119     if not pdata[uuid] then
 120       return CollectionsController::describe_node(uuid)
 121     else
 122       pdata[uuid][:_visited] = true
 123     end
 124
 125     #puts "visiting #{uuid}"
 126
 127     if m
 128       # uuid is a collection
 129       gr += CollectionsController::describe_node(uuid)
 130
 131       pdata.each do |k, job|
 132         if job[:output] == uuid.to_s
 133           gr += "\"#{uuid}\" -> \"#{job_uuid(job)}\" [label=\"output\"];"
 134           gr += CollectionsController::generate_provenance_edges(pdata, job[:uuid])
 135         end
 136         if job[:log] == uuid.to_s
 137           gr += "\"#{uuid}\" -> \"#{job_uuid(job)}\" [label=\"log\"];"
 138           gr += CollectionsController::generate_provenance_edges(pdata, job[:uuid])
 139         end
 140       end
 141     else
 142       # uuid is something else
 143       rsc = ArvadosBase::resource_class_for_uuid uuid.to_s
 144
 145       if rsc == Job
 146         job = pdata[uuid]
 147         if job
 148           gr += CollectionsController::script_param_edges(pdata, job, "", job[:script_parameters], opts)
 149         end
 150       else
 151         gr += CollectionsController::describe_node(uuid)
 152       end
 153     end
 154
 155     pdata.each do |k, link|
 156       if link[:head_uuid] == uuid.to_s and link[:link_class] == "provenance"
 157         gr += CollectionsController::describe_node(link[:tail_uuid])
 158         gr += "\"#{link[:head_uuid]}\" -> \"#{link[:tail_uuid]}\" [label=\" #{link[:name]}\", href=\"/links/#{link[:uuid]}\"];"
 159         gr += CollectionsController::generate_provenance_edges(pdata, link[:tail_uuid], opts)
 160       end
 161     end
 162
 163     #puts "finished #{uuid}"
 164
 165     gr
 166   end
 167
 168   def self.create_provenance_graph(pdata, uuid, opts={})
 169     require 'open3'
 170
 171     gr = """strict digraph {
 172 node [fontsize=8,shape=box];
 173 edge [dir=back,fontsize=8];"""
 174
 175     #puts "pdata is #{pdata}"
 176
 177     gr += CollectionsController::generate_provenance_edges(pdata, uuid, opts)
 178
 179     gr += "}"
 180     svg = ""
 181
 182     Open3.popen2("dot", "-Tsvg") do |stdin, stdout, wait_thr|
 183       stdin.print(gr)
 184       stdin.close
 185       svg = stdout.read()
 186       wait_thr.value
 187       stdout.close()
 188     end
 189
 190     svg = svg.sub(/<\?xml.*?\?>/m, "")
 191     svg = svg.sub(/<!DOCTYPE.*?>/m, "")
 192   end
 193
 194   def show
 195     return super if !@object
 196     @provenance = []
 197     @output2job = {}
 198     @output2colorindex = {}
 199     @sourcedata = {params[:uuid] => {uuid: params[:uuid]}}
 200     @protected = {}
 201
 202     colorindex = -1
 203     any_hope_left = true
 204     while any_hope_left
 205       any_hope_left = false
 206       Job.where(output: @sourcedata.keys).sort_by { |a| a.finished_at || a.created_at }.reverse.each do |job|
 207         if !@output2colorindex[job.output]
 208           any_hope_left = true
 209           @output2colorindex[job.output] = (colorindex += 1) % 10
 210           @provenance << {job: job, output: job.output}
 211           @sourcedata.delete job.output
 212           @output2job[job.output] = job
 213           job.dependencies.each do |new_source_data|
 214             unless @output2colorindex[new_source_data]
 215               @sourcedata[new_source_data] = {uuid: new_source_data}
 216             end
 217           end
 218         end
 219       end
 220     end
 221
 222     Link.where(head_uuid: @sourcedata.keys | @output2job.keys).each do |link|
 223       if link.link_class == 'resources' and link.name == 'wants'
 224         @protected[link.head_uuid] = true
 225       end
 226     end
 227     Link.where(tail_uuid: @sourcedata.keys).each do |link|
 228       if link.link_class == 'data_origin'
 229         @sourcedata[link.tail_uuid][:data_origins] ||= []
 230         @sourcedata[link.tail_uuid][:data_origins] << [link.name, link.head_kind, link.head_uuid]
 231       end
 232     end
 233     Collection.where(uuid: @sourcedata.keys).each do |collection|
 234       if @sourcedata[collection.uuid]
 235         @sourcedata[collection.uuid][:collection] = collection
 236       end
 237     end
 238
 239     Collection.where(uuid: @object.uuid).each do |u|
 240       @prov_svg = CollectionsController::create_provenance_graph u.provenance, u.uuid
 241     end
 242   end
 243
 244   protected
 245   class FileStreamer
 246     def initialize(opts={})
 247       @opts = opts
 248     end
 249     def each
 250       return unless @opts[:uuid] && @opts[:file]
 251       env = Hash[ENV].
 252         merge({
 253                 'ARVADOS_API_HOST' =>
 254                 $arvados_api_client.arvados_v1_base.
 255                 sub(/\/arvados\/v1/, '').
 256                 sub(/^https?:\/\//, ''),
 257                 'ARVADOS_API_TOKEN' =>
 258                 @opts[:arvados_api_token],
 259                 'ARVADOS_API_HOST_INSECURE' =>
 260                 Rails.configuration.arvados_insecure_https ? 'true' : 'false'
 261               })
 262       IO.popen([env, 'arv-get', "#{@opts[:uuid]}/#{@opts[:file]}"],
 263                'rb') do |io|
 264         while buf = io.read(2**20)
 265           yield buf
 266         end
 267       end
 268       Rails.logger.warn("#{@opts[:uuid]}/#{@opts[:file]}: #{$?}") if $? != 0
 269     end
 270   end
 271 end