services/api/app/controllers/arvados/v1/collections_controller.rb

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 require "arvados/keep"
   6 require "trashable"
   7
   8 class Arvados::V1::CollectionsController < ApplicationController
   9   include DbCurrentTime
  10   include TrashableController
  11
  12   def self._index_requires_parameters
  13     (super rescue {}).
  14       merge({
  15         include_trash: {
  16           type: 'boolean', required: false, default: false, description: "Include collections whose is_trashed attribute is true.",
  17         },
  18         include_old_versions: {
  19           type: 'boolean', required: false, default: false, description: "Include past collection versions.",
  20         },
  21       })
  22   end
  23
  24   def self._show_requires_parameters
  25     (super rescue {}).
  26       merge({
  27         include_trash: {
  28           type: 'boolean', required: false, default: false, description: "Show collection even if its is_trashed attribute is true.",
  29         },
  30         include_old_versions: {
  31           type: 'boolean', required: false, default: true, description: "Include past collection versions.",
  32         },
  33       })
  34   end
  35
  36   def create
  37     if resource_attrs[:uuid] and (loc = Keep::Locator.parse(resource_attrs[:uuid]))
  38       resource_attrs[:portable_data_hash] = loc.to_s
  39       resource_attrs.delete :uuid
  40     end
  41     resource_attrs.delete :version
  42     resource_attrs.delete :current_version_uuid
  43     super
  44   end
  45
  46   def update
  47     # preserve_version should be disabled unless explicitly asked otherwise.
  48     if !resource_attrs[:preserve_version]
  49       resource_attrs[:preserve_version] = false
  50     end
  51     super
  52   end
  53
  54   def find_objects_for_index
  55     opts = {
  56       include_trash: params[:include_trash] || ['destroy', 'trash', 'untrash'].include?(action_name),
  57       include_old_versions: params[:include_old_versions] || false,
  58     }
  59     @objects = Collection.readable_by(*@read_users, opts) if !opts.empty?
  60     super
  61   end
  62
  63   def find_object_by_uuid(with_lock: false)
  64     if loc = Keep::Locator.parse(params[:id])
  65       loc.strip_hints!
  66
  67       opts = {
  68         include_trash: params[:include_trash],
  69         include_old_versions: params[:include_old_versions],
  70       }
  71
  72       # It matters which Collection object we pick because blob
  73       # signatures depend on the value of trash_at.
  74       #
  75       # From postgres doc: "By default, null values sort as if larger
  76       # than any non-null value; that is, NULLS FIRST is the default
  77       # for DESC order, and NULLS LAST otherwise."
  78       #
  79       # "trash_at desc" sorts null first, then latest to earliest, so
  80       # it will select the Collection object with the longest
  81       # available lifetime.
  82
  83       select_attrs = (@select || ["manifest_text"]) | ["portable_data_hash", "trash_at"]
  84       model = Collection
  85       if with_lock && Rails.configuration.API.LockBeforeUpdate
  86         model = model.lock
  87       end
  88       if c = model.
  89                readable_by(*@read_users, opts).
  90                where({ portable_data_hash: loc.to_s }).
  91                order("trash_at desc").
  92                select(select_attrs.join(", ")).
  93                limit(1).
  94                first
  95         @object = {
  96           uuid: c.portable_data_hash,
  97           portable_data_hash: c.portable_data_hash,
  98           trash_at: c.trash_at,
  99         }
 100         if select_attrs.index("manifest_text")
 101           @object[:manifest_text] = c.manifest_text
 102         end
 103       end
 104     else
 105       super(with_lock: with_lock)
 106     end
 107   end
 108
 109   def show
 110     if @object.is_a? Collection
 111       # Omit unsigned_manifest_text
 112       @select ||= model_class.selectable_attributes - ["unsigned_manifest_text"]
 113       super
 114     else
 115       send_json @object
 116     end
 117   end
 118
 119
 120   def find_collections(visited, sp, ignore_columns=[], &b)
 121     case sp
 122     when ArvadosModel
 123       sp.class.columns.each do |c|
 124         find_collections(visited, sp[c.name.to_sym], &b) if !ignore_columns.include?(c.name)
 125       end
 126     when Hash
 127       sp.each do |k, v|
 128         find_collections(visited, v, &b)
 129       end
 130     when Array
 131       sp.each do |v|
 132         find_collections(visited, v, &b)
 133       end
 134     when String
 135       if m = /[a-f0-9]{32}\+\d+/.match(sp)
 136         yield m[0], nil
 137       elsif m = Collection.uuid_regex.match(sp)
 138         yield nil, m[0]
 139       end
 140     end
 141   end
 142
 143   def search_edges(visited, uuid, direction)
 144     if uuid.nil? or uuid.empty? or visited[uuid]
 145       return
 146     end
 147
 148     if loc = Keep::Locator.parse(uuid)
 149       loc.strip_hints!
 150       return if visited[loc.to_s]
 151     end
 152
 153     if loc
 154       # uuid is a portable_data_hash
 155       collections = Collection.readable_by(*@read_users).where(portable_data_hash: loc.to_s)
 156       c = collections.limit(2).all
 157       if c.size == 1
 158         visited[loc.to_s] = c[0]
 159       elsif c.size > 1
 160         name = collections.limit(1).where("name <> ''").first
 161         if name
 162           visited[loc.to_s] = {
 163             portable_data_hash: c[0].portable_data_hash,
 164             name: "#{name.name} + #{collections.count-1} more"
 165           }
 166         else
 167           visited[loc.to_s] = {
 168             portable_data_hash: c[0].portable_data_hash,
 169             name: loc.to_s
 170           }
 171         end
 172       end
 173
 174       if direction == :search_up
 175         # Search upstream for jobs where this locator is the output of some container
 176         Container.readable_by(*@read_users).where(output: loc.to_s).pluck(:uuid).each do |c_uuid|
 177           search_edges(visited, c_uuid, :search_up)
 178         end
 179
 180         Container.readable_by(*@read_users).where(log: loc.to_s).pluck(:uuid).each do |c_uuid|
 181           search_edges(visited, c_uuid, :search_up)
 182         end
 183       elsif direction == :search_down
 184         if loc.to_s == "d41d8cd98f00b204e9800998ecf8427e+0"
 185           # Special case, don't follow the empty collection.
 186           return
 187         end
 188
 189         # Search downstream for jobs where this locator is in mounts
 190         Container.readable_by(*@read_users).where([Container.full_text_trgm + " like ?", "%#{loc.to_s}%"]).select("output, log, uuid").each do |c|
 191           if c.output != loc.to_s && c.log != loc.to_s
 192             search_edges(visited, c.uuid, :search_down)
 193           end
 194         end
 195       end
 196     else
 197       # uuid is a regular Arvados UUID
 198       rsc = ArvadosModel::resource_class_for_uuid uuid
 199       if rsc == Container
 200         c = Container.readable_by(*@read_users).where(uuid: uuid).limit(1).first
 201         if c
 202           visited[uuid] = c.as_api_response
 203           if direction == :search_up
 204             # Follow upstream collections referenced in the script parameters
 205             find_collections(visited, c, ignore_columns=["log", "output"]) do |hash, col_uuid|
 206               search_edges(visited, hash, :search_up) if hash
 207               search_edges(visited, col_uuid, :search_up) if col_uuid
 208             end
 209           elsif direction == :search_down
 210             # Follow downstream job output
 211             search_edges(visited, c.output, :search_down)
 212           end
 213         end
 214       elsif rsc == ContainerRequest
 215         c = ContainerRequest.readable_by(*@read_users).where(uuid: uuid).limit(1).first
 216         if c
 217           visited[uuid] = c.as_api_response
 218           if direction == :search_up
 219             # Follow upstream collections
 220             find_collections(visited, c, ignore_columns=["log_uuid", "output_uuid"]) do |hash, col_uuid|
 221               search_edges(visited, hash, :search_up) if hash
 222               search_edges(visited, col_uuid, :search_up) if col_uuid
 223             end
 224           elsif direction == :search_down
 225             # Follow downstream job output
 226             search_edges(visited, c.output_uuid, :search_down)
 227           end
 228         end
 229       elsif rsc == Collection
 230         c = Collection.readable_by(*@read_users).where(uuid: uuid).limit(1).first
 231         if c
 232           if direction == :search_up
 233             visited[c.uuid] = c.as_api_response
 234
 235             ContainerRequest.readable_by(*@read_users).where(output_uuid: uuid).pluck(:uuid).each do |cr_uuid|
 236               search_edges(visited, cr_uuid, :search_up)
 237             end
 238
 239             ContainerRequest.readable_by(*@read_users).where(log_uuid: uuid).pluck(:uuid).each do |cr_uuid|
 240               search_edges(visited, cr_uuid, :search_up)
 241             end
 242           elsif direction == :search_down
 243             search_edges(visited, c.portable_data_hash, :search_down)
 244           end
 245         end
 246       elsif rsc != nil
 247         rsc.where(uuid: uuid).each do |r|
 248           visited[uuid] = r.as_api_response
 249         end
 250       end
 251     end
 252
 253     if direction == :search_up
 254       # Search for provenance links pointing to the current uuid
 255       Link.readable_by(*@read_users).
 256         where(head_uuid: uuid, link_class: "provenance").
 257         each do |link|
 258         visited[link.uuid] = link.as_api_response
 259         search_edges(visited, link.tail_uuid, direction)
 260       end
 261     elsif direction == :search_down
 262       # Search for provenance links emanating from the current uuid
 263       Link.readable_by(current_user).
 264         where(tail_uuid: uuid, link_class: "provenance").
 265         each do |link|
 266         visited[link.uuid] = link.as_api_response
 267         search_edges(visited, link.head_uuid, direction)
 268       end
 269     end
 270   end
 271
 272   def provenance
 273     visited = {}
 274     if @object[:uuid]
 275       search_edges(visited, @object[:uuid], :search_up)
 276     else
 277       search_edges(visited, @object[:portable_data_hash], :search_up)
 278     end
 279     send_json visited
 280   end
 281
 282   def used_by
 283     visited = {}
 284     if @object[:uuid]
 285       search_edges(visited, @object[:uuid], :search_down)
 286     else
 287       search_edges(visited, @object[:portable_data_hash], :search_down)
 288     end
 289     send_json visited
 290   end
 291
 292   protected
 293
 294   def load_select_param *args
 295     super
 296     if action_name == 'index'
 297       # Omit manifest_text and unsigned_manifest_text from index results unless expressly selected.
 298       @select ||= model_class.selectable_attributes - ["manifest_text", "unsigned_manifest_text"]
 299     end
 300   end
 301 end