services/api/app/controllers/arvados/v1/collections_controller.rb

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 require "arvados/keep"
   6 require "trashable"
   7
   8 class Arvados::V1::CollectionsController < ApplicationController
   9   include DbCurrentTime
  10   include TrashableController
  11
  12   def self._index_requires_parameters
  13     (super rescue {}).
  14       merge({
  15         include_trash: {
  16           type: 'boolean', required: false, default: false, description: "Include collections whose is_trashed attribute is true.",
  17         },
  18         include_old_versions: {
  19           type: 'boolean', required: false, default: false, description: "Include past collection versions.",
  20         },
  21       })
  22   end
  23
  24   def self._show_requires_parameters
  25     (super rescue {}).
  26       merge({
  27         include_trash: {
  28           type: 'boolean', required: false, default: false, description: "Show collection even if its is_trashed attribute is true.",
  29         },
  30         include_old_versions: {
  31           type: 'boolean', required: false, default: true, description: "Include past collection versions.",
  32         },
  33       })
  34   end
  35
  36   def create
  37     if resource_attrs[:uuid] and (loc = Keep::Locator.parse(resource_attrs[:uuid]))
  38       resource_attrs[:portable_data_hash] = loc.to_s
  39       resource_attrs.delete :uuid
  40     end
  41     resource_attrs.delete :version
  42     resource_attrs.delete :current_version_uuid
  43     super
  44   end
  45
  46   def update
  47     # preserve_version should be disabled unless explicitly asked otherwise.
  48     if !resource_attrs[:preserve_version]
  49       resource_attrs[:preserve_version] = false
  50     end
  51     super
  52   end
  53
  54   def find_objects_for_index
  55     opts = {
  56       include_trash: params[:include_trash] || ['destroy', 'trash', 'untrash'].include?(action_name),
  57       include_old_versions: params[:include_old_versions] || false,
  58     }
  59     @objects = Collection.readable_by(*@read_users, opts) if !opts.empty?
  60     super
  61   end
  62
  63   def find_object_by_uuid
  64     if loc = Keep::Locator.parse(params[:id])
  65       loc.strip_hints!
  66
  67       opts = {
  68         include_trash: params[:include_trash],
  69         include_old_versions: params[:include_old_versions],
  70       }
  71
  72       # It matters which Collection object we pick because blob
  73       # signatures depend on the value of trash_at.
  74       #
  75       # From postgres doc: "By default, null values sort as if larger
  76       # than any non-null value; that is, NULLS FIRST is the default
  77       # for DESC order, and NULLS LAST otherwise."
  78       #
  79       # "trash_at desc" sorts null first, then latest to earliest, so
  80       # it will select the Collection object with the longest
  81       # available lifetime.
  82
  83       select_attrs = (@select || ["manifest_text"]) | ["portable_data_hash", "trash_at"]
  84       if c = Collection.
  85                readable_by(*@read_users, opts).
  86                where({ portable_data_hash: loc.to_s }).
  87                order("trash_at desc").
  88                select(select_attrs.join(", ")).
  89                limit(1).
  90                first
  91         @object = {
  92           uuid: c.portable_data_hash,
  93           portable_data_hash: c.portable_data_hash,
  94           trash_at: c.trash_at,
  95         }
  96         if select_attrs.index("manifest_text")
  97           @object[:manifest_text] = c.manifest_text
  98         end
  99       end
 100     else
 101       super
 102     end
 103   end
 104
 105   def show
 106     if @object.is_a? Collection
 107       # Omit unsigned_manifest_text
 108       @select ||= model_class.selectable_attributes - ["unsigned_manifest_text"]
 109       super
 110     else
 111       send_json @object
 112     end
 113   end
 114
 115
 116   def find_collections(visited, sp, ignore_columns=[], &b)
 117     case sp
 118     when ArvadosModel
 119       sp.class.columns.each do |c|
 120         find_collections(visited, sp[c.name.to_sym], &b) if !ignore_columns.include?(c.name)
 121       end
 122     when Hash
 123       sp.each do |k, v|
 124         find_collections(visited, v, &b)
 125       end
 126     when Array
 127       sp.each do |v|
 128         find_collections(visited, v, &b)
 129       end
 130     when String
 131       if m = /[a-f0-9]{32}\+\d+/.match(sp)
 132         yield m[0], nil
 133       elsif m = Collection.uuid_regex.match(sp)
 134         yield nil, m[0]
 135       end
 136     end
 137   end
 138
 139   def search_edges(visited, uuid, direction)
 140     if uuid.nil? or uuid.empty? or visited[uuid]
 141       return
 142     end
 143
 144     if loc = Keep::Locator.parse(uuid)
 145       loc.strip_hints!
 146       return if visited[loc.to_s]
 147     end
 148
 149     if loc
 150       # uuid is a portable_data_hash
 151       collections = Collection.readable_by(*@read_users).where(portable_data_hash: loc.to_s)
 152       c = collections.limit(2).all
 153       if c.size == 1
 154         visited[loc.to_s] = c[0]
 155       elsif c.size > 1
 156         name = collections.limit(1).where("name <> ''").first
 157         if name
 158           visited[loc.to_s] = {
 159             portable_data_hash: c[0].portable_data_hash,
 160             name: "#{name.name} + #{collections.count-1} more"
 161           }
 162         else
 163           visited[loc.to_s] = {
 164             portable_data_hash: c[0].portable_data_hash,
 165             name: loc.to_s
 166           }
 167         end
 168       end
 169
 170       if direction == :search_up
 171         # Search upstream for jobs where this locator is the output of some job
 172         if !Rails.configuration.API.DisabledAPIs["jobs.list"]
 173           Job.readable_by(*@read_users).where(output: loc.to_s).each do |job|
 174             search_edges(visited, job.uuid, :search_up)
 175           end
 176
 177           Job.readable_by(*@read_users).where(log: loc.to_s).each do |job|
 178             search_edges(visited, job.uuid, :search_up)
 179           end
 180         end
 181
 182         Container.readable_by(*@read_users).where(output: loc.to_s).each do |c|
 183           search_edges(visited, c.uuid, :search_up)
 184         end
 185
 186         Container.readable_by(*@read_users).where(log: loc.to_s).each do |c|
 187           search_edges(visited, c.uuid, :search_up)
 188         end
 189       elsif direction == :search_down
 190         if loc.to_s == "d41d8cd98f00b204e9800998ecf8427e+0"
 191           # Special case, don't follow the empty collection.
 192           return
 193         end
 194
 195         # Search downstream for jobs where this locator is in script_parameters
 196         if !Rails.configuration.API.DisabledAPIs["jobs.list"]
 197           Job.readable_by(*@read_users).where(["jobs.script_parameters like ?", "%#{loc.to_s}%"]).each do |job|
 198             search_edges(visited, job.uuid, :search_down)
 199           end
 200
 201           Job.readable_by(*@read_users).where(["jobs.docker_image_locator = ?", "#{loc.to_s}"]).each do |job|
 202             search_edges(visited, job.uuid, :search_down)
 203           end
 204         end
 205
 206         Container.readable_by(*@read_users).where([Container.full_text_trgm + " like ?", "%#{loc.to_s}%"]).each do |c|
 207           if c.output != loc.to_s && c.log != loc.to_s
 208             search_edges(visited, c.uuid, :search_down)
 209           end
 210         end
 211       end
 212     else
 213       # uuid is a regular Arvados UUID
 214       rsc = ArvadosModel::resource_class_for_uuid uuid
 215       if rsc == Job
 216         Job.readable_by(*@read_users).where(uuid: uuid).each do |job|
 217           visited[uuid] = job.as_api_response
 218           if direction == :search_up
 219             # Follow upstream collections referenced in the script parameters
 220             find_collections(visited, job) do |hash, col_uuid|
 221               search_edges(visited, hash, :search_up) if hash
 222               search_edges(visited, col_uuid, :search_up) if col_uuid
 223             end
 224           elsif direction == :search_down
 225             # Follow downstream job output
 226             search_edges(visited, job.output, direction)
 227           end
 228         end
 229       elsif rsc == Container
 230         c = Container.readable_by(*@read_users).where(uuid: uuid).limit(1).first
 231         if c
 232           visited[uuid] = c.as_api_response
 233           if direction == :search_up
 234             # Follow upstream collections referenced in the script parameters
 235             find_collections(visited, c, ignore_columns=["log", "output"]) do |hash, col_uuid|
 236               search_edges(visited, hash, :search_up) if hash
 237               search_edges(visited, col_uuid, :search_up) if col_uuid
 238             end
 239           elsif direction == :search_down
 240             # Follow downstream job output
 241             search_edges(visited, c.output, :search_down)
 242           end
 243         end
 244       elsif rsc == ContainerRequest
 245         c = ContainerRequest.readable_by(*@read_users).where(uuid: uuid).limit(1).first
 246         if c
 247           visited[uuid] = c.as_api_response
 248           if direction == :search_up
 249             # Follow upstream collections
 250             find_collections(visited, c, ignore_columns=["log_uuid", "output_uuid"]) do |hash, col_uuid|
 251               search_edges(visited, hash, :search_up) if hash
 252               search_edges(visited, col_uuid, :search_up) if col_uuid
 253             end
 254           elsif direction == :search_down
 255             # Follow downstream job output
 256             search_edges(visited, c.output_uuid, :search_down)
 257           end
 258         end
 259       elsif rsc == Collection
 260         c = Collection.readable_by(*@read_users).where(uuid: uuid).limit(1).first
 261         if c
 262           if direction == :search_up
 263             visited[c.uuid] = c.as_api_response
 264
 265             if !Rails.configuration.API.DisabledAPIs["jobs.list"]
 266               Job.readable_by(*@read_users).where(output: c.portable_data_hash).each do |job|
 267                 search_edges(visited, job.uuid, :search_up)
 268               end
 269
 270               Job.readable_by(*@read_users).where(log: c.portable_data_hash).each do |job|
 271                 search_edges(visited, job.uuid, :search_up)
 272               end
 273             end
 274
 275             ContainerRequest.readable_by(*@read_users).where(output_uuid: uuid).each do |cr|
 276               search_edges(visited, cr.uuid, :search_up)
 277             end
 278
 279             ContainerRequest.readable_by(*@read_users).where(log_uuid: uuid).each do |cr|
 280               search_edges(visited, cr.uuid, :search_up)
 281             end
 282           elsif direction == :search_down
 283             search_edges(visited, c.portable_data_hash, :search_down)
 284           end
 285         end
 286       elsif rsc != nil
 287         rsc.where(uuid: uuid).each do |r|
 288           visited[uuid] = r.as_api_response
 289         end
 290       end
 291     end
 292
 293     if direction == :search_up
 294       # Search for provenance links pointing to the current uuid
 295       Link.readable_by(*@read_users).
 296         where(head_uuid: uuid, link_class: "provenance").
 297         each do |link|
 298         visited[link.uuid] = link.as_api_response
 299         search_edges(visited, link.tail_uuid, direction)
 300       end
 301     elsif direction == :search_down
 302       # Search for provenance links emanating from the current uuid
 303       Link.readable_by(current_user).
 304         where(tail_uuid: uuid, link_class: "provenance").
 305         each do |link|
 306         visited[link.uuid] = link.as_api_response
 307         search_edges(visited, link.head_uuid, direction)
 308       end
 309     end
 310   end
 311
 312   def provenance
 313     visited = {}
 314     if @object[:uuid]
 315       search_edges(visited, @object[:uuid], :search_up)
 316     else
 317       search_edges(visited, @object[:portable_data_hash], :search_up)
 318     end
 319     send_json visited
 320   end
 321
 322   def used_by
 323     visited = {}
 324     if @object[:uuid]
 325       search_edges(visited, @object[:uuid], :search_down)
 326     else
 327       search_edges(visited, @object[:portable_data_hash], :search_down)
 328     end
 329     send_json visited
 330   end
 331
 332   protected
 333
 334   def load_select_param *args
 335     super
 336     if action_name == 'index'
 337       # Omit manifest_text and unsigned_manifest_text from index results unless expressly selected.
 338       @select ||= model_class.selectable_attributes - ["manifest_text", "unsigned_manifest_text"]
 339     end
 340   end
 341 end