From b6546202f3830243ece6bfe13b83f09da3eb8c0e Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Thu, 11 Jul 2019 11:25:22 -0400 Subject: [PATCH] 15422: Provenance for collections handles containers Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- .../app/helpers/provenance_helper.rb | 146 ++++++++++++------ .../arvados/v1/collections_controller.rb | 6 +- 2 files changed, 101 insertions(+), 51 deletions(-) diff --git a/apps/workbench/app/helpers/provenance_helper.rb b/apps/workbench/app/helpers/provenance_helper.rb index 75261adbda..c06d83ae8b 100644 --- a/apps/workbench/app/helpers/provenance_helper.rb +++ b/apps/workbench/app/helpers/provenance_helper.rb @@ -151,6 +151,97 @@ module ProvenanceHelper gr end + def cr_edges cr, edge_opts={} + gr = "" + + gr += describe_node(cr[:uuid], {href: {controller: 'container_requests', + id: cr[:uuid]}, + label: cr[:name], + shape: 'oval'}) + # Connect child CRs + children = @opts[:cr_children_of].andand[cr[:uuid]] + if children + children.each do |child| + gr += edge(child[:uuid], cr[:uuid], {label: 'child'}) + end + end + # Output collection node + if cr[:output_uuid] and @opts[:output_collections][cr[:output_uuid]] + c = @opts[:output_collections][cr[:output_uuid]] + gr += describe_node(c[:portable_data_hash], + { + label: c[:name], + col_uuid: c[:uuid], + }) + gr += edge(cr[:uuid], + c[:portable_data_hash], + {label: 'output'}) + end + # Input collection nodes + output_pdhs = @opts[:output_collections].values.collect{|oc| + oc[:portable_data_hash]} + ProvenanceHelper::cr_input_pdhs(cr).each do |pdh| + if not output_pdhs.include?(pdh) + # Search for collections on the same project first + cols = @opts[:input_collections][pdh].andand.select{|ic| + ic[:owner_uuid] == cr[:owner_uuid]} + if not cols or cols.empty? + # Search for any collection with this PDH + cols = @opts[:input_collections][pdh] + end + if cols + names = cols.collect{|x| x[:name]}.uniq + else + names = ['(collection not found)'] + end + input_name = names.first + if names.length > 1 + input_name += " + #{names.length - 1} more" + end + gr += describe_node(pdh, {label: input_name}) + end + gr += edge(pdh, cr[:uuid], {label: 'input'}) + end + + gr + end + + def container_edges cont, edge_opts={} + uuid = cont[:uuid] + gr = "" + + gr += describe_node(cont[:uuid], {href: {controller: 'containers', + id: cont[:uuid]}, + shape: 'oval'}) + + ProvenanceHelper::find_collections cont[:mounts] do |collection_hash, collection_uuid, key| + if collection_uuid and @pdata[collection_uuid] + gr += describe_node(collection_uuid) + gr += edge(collection_uuid, uuid, {:label => key}) + elsif collection_hash and @pdata[collection_hash] + gr += describe_node(collection_hash) + gr += edge(collection_hash, uuid, {:label => key}) + end + end + + if cont[:container_image] and !@opts[:no_docker] and @pdata[cont[:container_image]] + gr += describe_node(cont[:container_image], {label: cont[:container_image]}) + gr += edge(cont[:container_image], uuid, {label: "docker_image"}) + end + + if cont[:output] and !edge_opts[:no_output] and @pdata[cont[:output]] + gr += describe_node(cont[:output]) + gr += edge(uuid, cont[:output], {label: "output" }) + end + + if cont[:log] and !edge_opts[:no_log] and @pdata[cont[:log]] + gr += describe_node(cont[:log]) + gr += edge(uuid, cont[:log], {label: "log"}) + end + + gr + end + def generate_provenance_edges(uuid) gr = "" m = GenerateGraph::collection_uuid(uuid) @@ -196,56 +287,10 @@ module ProvenanceHelper gr += job_edges job if job elsif rsc == ContainerRequest cr = @pdata[uuid] - if cr - gr += describe_node(cr[:uuid], {href: {controller: 'container_requests', - id: cr[:uuid]}, - label: cr[:name], - shape: 'oval'}) - # Connect child CRs - children = @opts[:cr_children_of].andand[cr[:uuid]] - if children - children.each do |child| - gr += edge(child[:uuid], cr[:uuid], {label: 'child'}) - end - end - # Output collection node - if cr[:output_uuid] and @opts[:output_collections][cr[:output_uuid]] - c = @opts[:output_collections][cr[:output_uuid]] - gr += describe_node(c[:portable_data_hash], - { - label: c[:name], - col_uuid: c[:uuid], - }) - gr += edge(cr[:uuid], - c[:portable_data_hash], - {label: 'output'}) - end - # Input collection nodes - output_pdhs = @opts[:output_collections].values.collect{|oc| - oc[:portable_data_hash]} - ProvenanceHelper::cr_input_pdhs(cr).each do |pdh| - if not output_pdhs.include?(pdh) - # Search for collections on the same project first - cols = @opts[:input_collections][pdh].andand.select{|ic| - ic[:owner_uuid] == cr[:owner_uuid]} - if not cols or cols.empty? - # Search for any collection with this PDH - cols = @opts[:input_collections][pdh] - end - if cols - names = cols.collect{|x| x[:name]}.uniq - else - names = ['(collection not found)'] - end - input_name = names.first - if names.length > 1 - input_name += " + #{names.length - 1} more" - end - gr += describe_node(pdh, {label: input_name}) - end - gr += edge(pdh, cr[:uuid], {label: 'input'}) - end - end + gr += cr_edges cr if cr + elsif rsc == Container + cr = @pdata[uuid] + gr += container_edges cr if cr end end @@ -319,6 +364,7 @@ module ProvenanceHelper gr = """strict digraph { node [fontsize=10,fontname=\"Helvetica,Arial,sans-serif\"]; edge [fontsize=10,fontname=\"Helvetica,Arial,sans-serif\"]; +rankdir=RL; """ if opts[:direction] == :bottom_up diff --git a/services/api/app/controllers/arvados/v1/collections_controller.rb b/services/api/app/controllers/arvados/v1/collections_controller.rb index 3d0c6a4d33..51a47f0186 100644 --- a/services/api/app/controllers/arvados/v1/collections_controller.rb +++ b/services/api/app/controllers/arvados/v1/collections_controller.rb @@ -188,7 +188,11 @@ class Arvados::V1::CollectionsController < ApplicationController end end - Container.readable_by(*@read_users).where(["any", "like", "%#{loc.to_s}%"]).each do |c| + Container.readable_by(*@read_users).where(["mounts like ?", "%#{loc.to_s}%"]).each do |c| + search_edges(visited, c.uuid, :search_down) + end + + Container.readable_by(*@read_users).where(["container_image = '#{loc.to_s}'"]).each do |c| search_edges(visited, c.uuid, :search_down) end -- 2.39.5