From 54bc567a6667783decce679e2ea77b93d5f68322 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Fri, 12 Jul 2019 11:41:02 -0400 Subject: [PATCH] 15422: provenance and used_by endpoints traverse containers Fix up workbench rendering of provenance as well. Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- .../app/controllers/collections_controller.rb | 4 +- .../container_requests_controller.rb | 31 ++-- .../app/controllers/jobs_controller.rb | 1 - .../pipeline_instances_controller.rb | 1 - .../app/helpers/provenance_helper.rb | 137 +++++++++++------- .../arvados/v1/collections_controller.rb | 67 ++++++--- 6 files changed, 151 insertions(+), 90 deletions(-) diff --git a/apps/workbench/app/controllers/collections_controller.rb b/apps/workbench/app/controllers/collections_controller.rb index 5141012443..de20b8858a 100644 --- a/apps/workbench/app/controllers/collections_controller.rb +++ b/apps/workbench/app/controllers/collections_controller.rb @@ -176,7 +176,7 @@ class CollectionsController < ApplicationController if params["tab_pane"] == "Provenance_graph" @prov_svg = ProvenanceHelper::create_provenance_graph(@object.provenance, "provenance_svg", {:request => request, - :direction => :top_down, + :direction => "RL", :combine_jobs => :script_only}) rescue nil end @@ -217,7 +217,7 @@ class CollectionsController < ApplicationController if params["tab_pane"] == "Used_by" @used_by_svg = ProvenanceHelper::create_provenance_graph(@object.used_by, "used_by_svg", {:request => request, - :direction => :top_down, + :direction => "LR", :combine_jobs => :script_only, :pdata_only => true}) rescue nil end diff --git a/apps/workbench/app/controllers/container_requests_controller.rb b/apps/workbench/app/controllers/container_requests_controller.rb index 385d9dc6d7..587819dd44 100644 --- a/apps/workbench/app/controllers/container_requests_controller.rb +++ b/apps/workbench/app/controllers/container_requests_controller.rb @@ -11,7 +11,7 @@ class ContainerRequestsController < ApplicationController def generate_provenance(cr) return if params['tab_pane'] != "Provenance" - nodes = {cr[:uuid] => cr} + nodes = {} child_crs = [] col_uuids = [] col_pdhs = [] @@ -29,38 +29,33 @@ class ContainerRequestsController < ApplicationController end end - output_cols = {} # Indexed by UUID - input_cols = {} # Indexed by PDH + if nodes.length == 0 + nodes[cr[:uuid]] = cr + end + + pdh_to_col = {} # Indexed by PDH output_pdhs = [] # Batch requests to get all related collections # First fetch output collections by UUID. Collection.filter([['uuid', 'in', col_uuids.uniq]]).each do |c| - output_cols[c[:uuid]] = c output_pdhs << c[:portable_data_hash] + pdh_to_col[c[:portable_data_hash]] = c + nodes[c[:uuid]] = c end - # Then, get only input collections by PDH. There could be more than one collection - # per PDH: the number of collections is used on the collection node label. + # Next, get input collections by PDH. Collection.filter( [['portable_data_hash', 'in', col_pdhs - output_pdhs]]).each do |c| - if input_cols[c[:portable_data_hash]] - input_cols[c[:portable_data_hash]] << c - else - input_cols[c[:portable_data_hash]] = [c] - end + nodes[c[:portable_data_hash]] = c end @svg = ProvenanceHelper::create_provenance_graph( nodes, "provenance_svg", { :request => request, - :direction => :top_down, - :output_collections => output_cols, - :input_collections => input_cols, - :cr_children_of => { - cr[:uuid] => child_crs.select{|child| child[:uuid]}, - }, - }) + :pdh_to_uuid => pdh_to_col, + } + ) end def show_pane_list diff --git a/apps/workbench/app/controllers/jobs_controller.rb b/apps/workbench/app/controllers/jobs_controller.rb index 4f7bfcee53..bac1530d39 100644 --- a/apps/workbench/app/controllers/jobs_controller.rb +++ b/apps/workbench/app/controllers/jobs_controller.rb @@ -34,7 +34,6 @@ class JobsController < ApplicationController @svg = ProvenanceHelper::create_provenance_graph nodes, "provenance_svg", { :request => request, - :direction => :top_down, :all_script_parameters => true, :script_version_nodes => true} end diff --git a/apps/workbench/app/controllers/pipeline_instances_controller.rb b/apps/workbench/app/controllers/pipeline_instances_controller.rb index c8863653a1..6431057ddf 100644 --- a/apps/workbench/app/controllers/pipeline_instances_controller.rb +++ b/apps/workbench/app/controllers/pipeline_instances_controller.rb @@ -192,7 +192,6 @@ class PipelineInstancesController < ApplicationController if provenance @prov_svg = ProvenanceHelper::create_provenance_graph provenance, "provenance_svg", { :request => request, - :direction => :top_down, :all_script_parameters => true, :combine_jobs => :script_and_version, :pips => pips, diff --git a/apps/workbench/app/helpers/provenance_helper.rb b/apps/workbench/app/helpers/provenance_helper.rb index c06d83ae8b..8a8337fe12 100644 --- a/apps/workbench/app/helpers/provenance_helper.rb +++ b/apps/workbench/app/helpers/provenance_helper.rb @@ -151,56 +151,97 @@ module ProvenanceHelper gr end - def cr_edges cr, edge_opts={} + # def cr_edges cr, edge_opts={} + # gr = "" + + # gr += describe_node(cr[:uuid], {href: {controller: 'container_requests', + # id: cr[:uuid]}, + # label: cr[:name], + # shape: 'oval'}) + # # Connect child CRs + # children = @opts[:cr_children_of].andand[cr[:uuid]] + # if children + # children.each do |child| + # gr += edge(child[:uuid], cr[:uuid], {label: 'child'}) + # end + # end + # # Output collection node + # if cr[:output_uuid] and @opts[:output_collections][cr[:output_uuid]] + # c = @opts[:output_collections][cr[:output_uuid]] + # gr += describe_node(c[:portable_data_hash], + # { + # label: c[:name], + # col_uuid: c[:uuid], + # }) + # gr += edge(cr[:uuid], + # c[:portable_data_hash], + # {label: 'output'}) + # end + # # Input collection nodes + # output_pdhs = @opts[:output_collections].values.collect{|oc| + # oc[:portable_data_hash]} + # ProvenanceHelper::cr_input_pdhs(cr).each do |pdh| + # if not output_pdhs.include?(pdh) + # # Search for collections on the same project first + # cols = @opts[:input_collections][pdh].andand.select{|ic| + # ic[:owner_uuid] == cr[:owner_uuid]} + # if not cols or cols.empty? + # # Search for any collection with this PDH + # cols = @opts[:input_collections][pdh] + # end + # if cols + # names = cols.collect{|x| x[:name]}.uniq + # else + # names = ['(collection not found)'] + # end + # input_name = names.first + # if names.length > 1 + # input_name += " + #{names.length - 1} more" + # end + # gr += describe_node(pdh, {label: input_name}) + # end + # gr += edge(pdh, cr[:uuid], {label: 'input'}) + # end + + # gr + # end + + def cr_edges cont, edge_opts={} + uuid = cont[:uuid] gr = "" - gr += describe_node(cr[:uuid], {href: {controller: 'container_requests', - id: cr[:uuid]}, - label: cr[:name], - shape: 'oval'}) - # Connect child CRs - children = @opts[:cr_children_of].andand[cr[:uuid]] - if children - children.each do |child| - gr += edge(child[:uuid], cr[:uuid], {label: 'child'}) + gr += describe_node(cont[:uuid], {href: {controller: 'container_requests', + id: cont[:uuid]}, + shape: 'oval', + label: cont[:name]}) + + ProvenanceHelper::find_collections cont[:mounts] do |collection_hash, collection_uuid, key| + if @opts[:pdh_to_uuid] and @opts[:pdh_to_uuid][collection_hash] + collection_uuid = @opts[:pdh_to_uuid][collection_hash].uuid + collection_hash = nil + end + if collection_uuid and @pdata[collection_uuid] + gr += describe_node(collection_uuid) + gr += edge(collection_uuid, uuid, {:label => key}) + elsif collection_hash and @pdata[collection_hash] + gr += describe_node(collection_hash) + gr += edge(collection_hash, uuid, {:label => key}) end end - # Output collection node - if cr[:output_uuid] and @opts[:output_collections][cr[:output_uuid]] - c = @opts[:output_collections][cr[:output_uuid]] - gr += describe_node(c[:portable_data_hash], - { - label: c[:name], - col_uuid: c[:uuid], - }) - gr += edge(cr[:uuid], - c[:portable_data_hash], - {label: 'output'}) + + if cont[:container_image] and !@opts[:no_docker] and @pdata[cont[:container_image]] + gr += describe_node(cont[:container_image], {label: cont[:container_image]}) + gr += edge(cont[:container_image], uuid, {label: "docker_image"}) end - # Input collection nodes - output_pdhs = @opts[:output_collections].values.collect{|oc| - oc[:portable_data_hash]} - ProvenanceHelper::cr_input_pdhs(cr).each do |pdh| - if not output_pdhs.include?(pdh) - # Search for collections on the same project first - cols = @opts[:input_collections][pdh].andand.select{|ic| - ic[:owner_uuid] == cr[:owner_uuid]} - if not cols or cols.empty? - # Search for any collection with this PDH - cols = @opts[:input_collections][pdh] - end - if cols - names = cols.collect{|x| x[:name]}.uniq - else - names = ['(collection not found)'] - end - input_name = names.first - if names.length > 1 - input_name += " + #{names.length - 1} more" - end - gr += describe_node(pdh, {label: input_name}) - end - gr += edge(pdh, cr[:uuid], {label: 'input'}) + + if cont[:output_uuid] and !edge_opts[:no_output] and @pdata[cont[:output_uuid]] + gr += describe_node(cont[:output_uuid]) + gr += edge(uuid, cont[:output_uuid], {label: "output" }) + end + + if cont[:log_uuid] and !edge_opts[:no_log] and @pdata[cont[:log_uuid]] + gr += describe_node(cont[:log_uuid]) + gr += edge(uuid, cont[:log_uuid], {label: "log"}) end gr @@ -364,11 +405,9 @@ module ProvenanceHelper gr = """strict digraph { node [fontsize=10,fontname=\"Helvetica,Arial,sans-serif\"]; edge [fontsize=10,fontname=\"Helvetica,Arial,sans-serif\"]; -rankdir=RL; """ - - if opts[:direction] == :bottom_up - gr += "edge [dir=back];" + if ["LR", "RL"].include? opts[:direction] + gr += "rankdir=#{opts[:direction]};" end begin diff --git a/services/api/app/controllers/arvados/v1/collections_controller.rb b/services/api/app/controllers/arvados/v1/collections_controller.rb index 51a47f0186..ede6fbd899 100644 --- a/services/api/app/controllers/arvados/v1/collections_controller.rb +++ b/services/api/app/controllers/arvados/v1/collections_controller.rb @@ -96,11 +96,11 @@ class Arvados::V1::CollectionsController < ApplicationController end - def find_collections(visited, sp, &b) + def find_collections(visited, sp, ignore_columns=[], &b) case sp when ArvadosModel sp.class.columns.each do |c| - find_collections(visited, sp[c.name.to_sym], &b) if c.name != "log" + find_collections(visited, sp[c.name.to_sym], &b) if !ignore_columns.include?(c.name) end when Hash sp.each do |k, v| @@ -129,8 +129,6 @@ class Arvados::V1::CollectionsController < ApplicationController return if visited[loc.to_s] end - logger.debug "visiting #{uuid}" - if loc # uuid is a portable_data_hash collections = Collection.readable_by(*@read_users).where(portable_data_hash: loc.to_s) @@ -188,14 +186,11 @@ class Arvados::V1::CollectionsController < ApplicationController end end - Container.readable_by(*@read_users).where(["mounts like ?", "%#{loc.to_s}%"]).each do |c| - search_edges(visited, c.uuid, :search_down) - end - - Container.readable_by(*@read_users).where(["container_image = '#{loc.to_s}'"]).each do |c| - search_edges(visited, c.uuid, :search_down) + Container.readable_by(*@read_users).where([Container.full_text_trgm + " like ?", "%#{loc.to_s}%"]).each do |c| + if c.output != loc.to_s && c.log != loc.to_s + search_edges(visited, c.uuid, :search_down) + end end - end else # uuid is a regular Arvados UUID @@ -215,7 +210,8 @@ class Arvados::V1::CollectionsController < ApplicationController end end elsif rsc == Container - Container.readable_by(*@read_users).where(uuid: uuid).each do |c| + c = Container.readable_by(*@read_users).where(uuid: uuid).limit(1).first + if c visited[uuid] = c.as_api_response if direction == :search_up # Follow upstream collections referenced in the script parameters @@ -228,10 +224,37 @@ class Arvados::V1::CollectionsController < ApplicationController search_edges(visited, c.output, direction) end end + elsif rsc == ContainerRequest + c = ContainerRequest.readable_by(*@read_users).where(uuid: uuid).limit(1).first + if c + visited[uuid] = c.as_api_response + if direction == :search_up + # Follow upstream collections + find_collections(visited, c, ignore_columns=["log_uuid"]) do |hash, col_uuid| + search_edges(visited, hash, :search_up) if hash + search_edges(visited, col_uuid, :search_up) if col_uuid + end + elsif direction == :search_down + # Follow downstream job output + search_edges(visited, c.output_uuid, direction) + end + end elsif rsc == Collection - if c = Collection.readable_by(*@read_users).where(uuid: uuid).limit(1).first - search_edges(visited, c.portable_data_hash, direction) - visited[c.portable_data_hash] = c.as_api_response + c = Collection.readable_by(*@read_users).where(uuid: uuid).limit(1).first + if c + if direction == :search_up + visited[c.uuid] = c.as_api_response + + ContainerRequest.readable_by(*@read_users).where(output_uuid: uuid).each do |cr| + search_edges(visited, cr.uuid, :search_up) + end + + ContainerRequest.readable_by(*@read_users).where(log_uuid: uuid).each do |cr| + search_edges(visited, cr.uuid, :search_up) + end + elsif direction == :search_down + search_edges(visited, c.portable_data_hash, direction) + end end elsif rsc != nil rsc.where(uuid: uuid).each do |r| @@ -261,15 +284,21 @@ class Arvados::V1::CollectionsController < ApplicationController def provenance visited = {} - search_edges(visited, @object[:portable_data_hash], :search_up) - search_edges(visited, @object[:uuid], :search_up) + if @object[:uuid] + search_edges(visited, @object[:uuid], :search_up) + else + search_edges(visited, @object[:portable_data_hash], :search_up) + end send_json visited end def used_by visited = {} - search_edges(visited, @object[:uuid], :search_down) - search_edges(visited, @object[:portable_data_hash], :search_down) + if @object[:uuid] + search_edges(visited, @object[:uuid], :search_down) + else + search_edges(visited, @object[:portable_data_hash], :search_down) + end send_json visited end -- 2.30.2