Merge branch '11507-deleted-bufferblocks' refs #11507
[arvados.git] / apps / workbench / app / controllers / pipeline_instances_controller.rb
1 class PipelineInstancesController < ApplicationController
2   skip_before_filter :find_object_by_uuid, only: :compare
3   before_filter :find_objects_by_uuid, only: :compare
4   skip_around_filter :require_thread_api_token, if: proc { |ctrl|
5     Rails.configuration.anonymous_user_token and
6     'show' == ctrl.action_name
7   }
8
9   include PipelineInstancesHelper
10   include PipelineComponentsHelper
11
12   def copy
13     template = PipelineTemplate.find?(@object.pipeline_template_uuid)
14
15     source = @object
16     @object = PipelineInstance.new
17     @object.pipeline_template_uuid = source.pipeline_template_uuid
18
19     if params['components'] == 'use_latest' and template
20       @object.components = template.components.deep_dup
21       @object.components.each do |cname, component|
22         # Go through the script parameters of each component
23         # that are marked as user input and copy them over.
24         # Skip any components that are not present in the
25         # source instance (there's nothing to copy)
26         if source.components.include? cname
27           component[:script_parameters].each do |pname, val|
28             if val.is_a? Hash and val[:dataclass]
29               # this is user-inputtable, so check the value from the source pipeline
30               srcvalue = source.components[cname][:script_parameters][pname]
31               if not srcvalue.nil?
32                 component[:script_parameters][pname] = srcvalue
33               end
34             end
35           end
36         end
37       end
38     else
39       @object.components = source.components.deep_dup
40     end
41
42     if params['script'] == 'use_same'
43       # Go through each component and copy the script_version from each job.
44       @object.components.each do |cname, component|
45         if source.components.include? cname and source.components[cname][:job]
46           component[:script_version] = source.components[cname][:job][:script_version]
47         end
48       end
49     end
50
51     @object.components.each do |cname, component|
52       component.delete :job
53     end
54     @object.state = 'New'
55
56     # set owner_uuid to that of source, provided it is a project and writable by current user
57     current_project = Group.find(source.owner_uuid) rescue nil
58     if (current_project && current_project.writable_by.andand.include?(current_user.uuid))
59       @object.owner_uuid = source.owner_uuid
60     end
61
62     super
63   end
64
65   def update
66     @updates ||= params[@object.class.to_s.underscore.singularize.to_sym]
67     if (components = @updates[:components])
68       components.each do |cname, component|
69         if component[:script_parameters]
70           component[:script_parameters].each do |param, value_info|
71             if value_info.is_a? Hash
72               value_info_partitioned = value_info[:value].partition('/') if value_info[:value].andand.class.eql?(String)
73               value_info_value = value_info_partitioned ? value_info_partitioned[0] : value_info[:value]
74               value_info_class = resource_class_for_uuid value_info_value
75               if value_info_class == Link
76                 # Use the link target, not the link itself, as script
77                 # parameter; but keep the link info around as well.
78                 link = Link.find value_info[:value]
79                 value_info[:value] = link.head_uuid
80                 value_info[:link_uuid] = link.uuid
81                 value_info[:link_name] = link.name
82               else
83                 # Delete stale link_uuid and link_name data.
84                 value_info[:link_uuid] = nil
85                 value_info[:link_name] = nil
86               end
87               if value_info_class == Collection
88                 # to ensure reproducibility, the script_parameter for a
89                 # collection should be the portable_data_hash
90                 # keep the collection name and uuid for human-readability
91                 obj = Collection.find value_info_value
92                 if value_info_partitioned
93                   value_info[:value] = obj.portable_data_hash + value_info_partitioned[1] + value_info_partitioned[2]
94                   value_info[:selection_name] = obj.name ? obj.name + value_info_partitioned[1] + value_info_partitioned[2] : obj.name
95                 else
96                   value_info[:value] = obj.portable_data_hash
97                   value_info[:selection_name] = obj.name
98                 end
99                 value_info[:selection_uuid] = obj.uuid
100               end
101             end
102           end
103         end
104       end
105     end
106     super
107   end
108
109   def graph(pipelines)
110     return nil, nil if params['tab_pane'] != "Graph"
111
112     provenance = {}
113     pips = {}
114     n = 1
115
116     # When comparing more than one pipeline, "pips" stores bit fields that
117     # indicates which objects are part of which pipelines.
118
119     pipelines.each do |p|
120       collections = []
121       hashes = []
122       jobs = []
123
124       p[:components].each do |k, v|
125         provenance["component_#{p[:uuid]}_#{k}"] = v
126
127         collections << v[:output_uuid] if v[:output_uuid]
128         jobs << v[:job][:uuid] if v[:job]
129       end
130
131       jobs = jobs.compact.uniq
132       if jobs.any?
133         Job.where(uuid: jobs).each do |j|
134           job_uuid = j.uuid
135
136           provenance[job_uuid] = j
137           pips[job_uuid] = 0 unless pips[job_uuid] != nil
138           pips[job_uuid] |= n
139
140           hashes << j[:output] if j[:output]
141           ProvenanceHelper::find_collections(j) do |hash, uuid|
142             collections << uuid if uuid
143             hashes << hash if hash
144           end
145
146           if j[:script_version]
147             script_uuid = j[:script_version]
148             provenance[script_uuid] = {:uuid => script_uuid}
149             pips[script_uuid] = 0 unless pips[script_uuid] != nil
150             pips[script_uuid] |= n
151           end
152         end
153       end
154
155       hashes = hashes.compact.uniq
156       if hashes.any?
157         Collection.where(portable_data_hash: hashes).each do |c|
158           hash_uuid = c.portable_data_hash
159           provenance[hash_uuid] = c
160           pips[hash_uuid] = 0 unless pips[hash_uuid] != nil
161           pips[hash_uuid] |= n
162         end
163       end
164
165       collections = collections.compact.uniq
166       if collections.any?
167         Collection.where(uuid: collections).each do |c|
168           collection_uuid = c.uuid
169           provenance[collection_uuid] = c
170           pips[collection_uuid] = 0 unless pips[collection_uuid] != nil
171           pips[collection_uuid] |= n
172         end
173       end
174
175       n = n << 1
176     end
177
178     return provenance, pips
179   end
180
181   def show
182     # the #show action can also be called by #compare, which does its own work to set up @pipelines
183     unless defined? @pipelines
184       @pipelines = [@object]
185     end
186
187     provenance, pips = graph(@pipelines)
188     if provenance
189       @prov_svg = ProvenanceHelper::create_provenance_graph provenance, "provenance_svg", {
190         :request => request,
191         :direction => :top_down,
192         :all_script_parameters => true,
193         :combine_jobs => :script_and_version,
194         :pips => pips,
195         :only_components => true,
196         :no_docker => true,
197         :no_log => true}
198     end
199
200     super
201   end
202
203   def compare
204     @breadcrumb_page_name = 'compare'
205
206     @rows = []          # each is {name: S, components: [...]}
207
208     if params['tab_pane'] == "Compare" or params['tab_pane'].nil?
209       # Build a table: x=pipeline y=component
210       @objects.each_with_index do |pi, pi_index|
211         pipeline_jobs(pi).each do |component|
212           # Find a cell with the same name as this component but no
213           # entry for this pipeline
214           target_row = nil
215           @rows.each_with_index do |row, row_index|
216             if row[:name] == component[:name] and !row[:components][pi_index]
217               target_row = row
218             end
219           end
220           if !target_row
221             target_row = {name: component[:name], components: []}
222             @rows << target_row
223           end
224           target_row[:components][pi_index] = component
225         end
226       end
227
228       @rows.each do |row|
229         # Build a "normal" pseudo-component for this row by picking the
230         # most common value for each attribute. If all values are
231         # equally common, there is no "normal".
232         normal = {}              # attr => most common value
233         highscore = {}           # attr => how common "normal" is
234         score = {}               # attr => { value => how common }
235         row[:components].each do |pj|
236           next if pj.nil?
237           pj.each do |k,v|
238             vstr = for_comparison v
239             score[k] ||= {}
240             score[k][vstr] = (score[k][vstr] || 0) + 1
241             highscore[k] ||= 0
242             if score[k][vstr] == highscore[k]
243               # tie for first place = no "normal"
244               normal.delete k
245             elsif score[k][vstr] == highscore[k] + 1
246               # more pipelines have v than anything else
247               highscore[k] = score[k][vstr]
248               normal[k] = vstr
249             end
250           end
251         end
252
253         # Add a hash in component[:is_normal]: { attr => is_the_value_normal? }
254         row[:components].each do |pj|
255           next if pj.nil?
256           pj[:is_normal] = {}
257           pj.each do |k,v|
258             pj[:is_normal][k] = (normal.has_key?(k) && normal[k] == for_comparison(v))
259           end
260         end
261       end
262     end
263
264     if params['tab_pane'] == "Graph"
265       @pipelines = @objects
266     end
267
268     @object = @objects.first
269
270     show
271   end
272
273   def show_pane_list
274     panes = %w(Components Log Graph Advanced)
275     if @object and @object.state.in? ['New', 'Ready']
276       panes = %w(Inputs) + panes - %w(Log)
277     end
278     if not @object.components.values.any? { |x| x[:job] rescue false }
279       panes -= ['Graph']
280     end
281     panes
282   end
283
284   def compare_pane_list
285     %w(Compare Graph)
286   end
287
288   helper_method :unreadable_inputs_present?
289   def unreadable_inputs_present?
290     unless @unreadable_inputs_present.nil?
291       return @unreadable_inputs_present
292     end
293
294     input_uuids = []
295     input_pdhs = []
296     @object.components.each do |k, component|
297       next if !component
298       component[:script_parameters].andand.each do |p, tv|
299         if (tv.is_a? Hash) and ((tv[:dataclass] == "Collection") || (tv[:dataclass] == "File"))
300           if tv[:value]
301             value = tv[:value]
302           elsif tv[:default]
303             value = tv[:default]
304           else
305             value = ''
306           end
307           if value.present?
308             split = value.split '/'
309             if CollectionsHelper.match(split[0])
310               input_pdhs << split[0]
311             else
312               input_uuids << split[0]
313             end
314           end
315         end
316       end
317     end
318
319     input_pdhs = input_pdhs.uniq
320     input_uuids = input_uuids.uniq
321
322     preload_collections_for_objects input_uuids if input_uuids.any?
323     preload_for_pdhs input_pdhs if input_pdhs.any?
324
325     @unreadable_inputs_present = false
326     input_uuids.each do |uuid|
327       if !collections_for_object(uuid).any?
328         @unreadable_inputs_present = true
329         break
330       end
331     end
332     if !@unreadable_inputs_present
333       input_pdhs.each do |pdh|
334         if !collection_for_pdh(pdh).any?
335           @unreadable_inputs_present = true
336           break
337         end
338       end
339     end
340
341     @unreadable_inputs_present
342   end
343
344   def cancel
345     @object.cancel
346     if params[:return_to]
347       redirect_to params[:return_to]
348     else
349       redirect_to @object
350     end
351   end
352
353   protected
354   def for_comparison v
355     if v.is_a? Hash or v.is_a? Array
356       v.to_json
357     else
358       v.to_s
359     end
360   end
361
362   def load_filters_and_paging_params
363     params[:limit] = 20
364     super
365   end
366
367   def find_objects_by_uuid
368     @objects = model_class.where(uuid: params[:uuids])
369   end
370 end