services/api/app/controllers/arvados/v1/collections_controller.rb

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 require "arvados/keep"
   6 require "trashable"
   7
   8 class Arvados::V1::CollectionsController < ApplicationController
   9   include DbCurrentTime
  10   include TrashableController
  11
  12   def self._index_requires_parameters
  13     (super rescue {}).
  14       merge({
  15         include_trash: {
  16           type: 'boolean', required: false, default: false, description: "Include collections whose is_trashed attribute is true.",
  17         },
  18         include_old_versions: {
  19           type: 'boolean', required: false, default: false, description: "Include past collection versions.",
  20         },
  21       })
  22   end
  23
  24   def self._show_requires_parameters
  25     (super rescue {}).
  26       merge({
  27         include_trash: {
  28           type: 'boolean', required: false, default: false, description: "Show collection even if its is_trashed attribute is true.",
  29         },
  30         include_old_versions: {
  31           type: 'boolean', required: false, default: true, description: "Include past collection versions.",
  32         },
  33       })
  34   end
  35
  36   def create
  37     if resource_attrs[:uuid] and (loc = Keep::Locator.parse(resource_attrs[:uuid]))
  38       resource_attrs[:portable_data_hash] = loc.to_s
  39       resource_attrs.delete :uuid
  40     end
  41     resource_attrs.delete :version
  42     resource_attrs.delete :current_version_uuid
  43     super
  44   end
  45
  46   def update
  47     # preserve_version should be disabled unless explicitly asked otherwise.
  48     if !resource_attrs[:preserve_version]
  49       resource_attrs[:preserve_version] = false
  50     end
  51     super
  52   end
  53
  54   def find_objects_for_index
  55     opts = {
  56       include_trash: params[:include_trash] || ['destroy', 'trash', 'untrash'].include?(action_name),
  57       include_old_versions: params[:include_old_versions] || false,
  58     }
  59     @objects = Collection.readable_by(*@read_users, opts) if !opts.empty?
  60     super
  61   end
  62
  63   def find_object_by_uuid(with_lock: false)
  64     if loc = Keep::Locator.parse(params[:id])
  65       loc.strip_hints!
  66
  67       opts = {
  68         include_trash: params[:include_trash],
  69         include_old_versions: params[:include_old_versions],
  70       }
  71
  72       # It matters which Collection object we pick because blob
  73       # signatures depend on the value of trash_at.
  74       #
  75       # From postgres doc: "By default, null values sort as if larger
  76       # than any non-null value; that is, NULLS FIRST is the default
  77       # for DESC order, and NULLS LAST otherwise."
  78       #
  79       # "trash_at desc" sorts null first, then latest to earliest, so
  80       # it will select the Collection object with the longest
  81       # available lifetime.
  82
  83       select_attrs = (@select || ["manifest_text"]) | ["portable_data_hash", "trash_at"]
  84       model = Collection
  85       if with_lock && Rails.configuration.API.LockBeforeUpdate
  86         model = model.lock
  87       end
  88       if c = model.
  89                readable_by(*@read_users, opts).
  90                where({ portable_data_hash: loc.to_s }).
  91                order("trash_at desc").
  92                select(select_attrs.join(", ")).
  93                limit(1).
  94                first
  95         @object = {
  96           uuid: c.portable_data_hash,
  97           portable_data_hash: c.portable_data_hash,
  98           trash_at: c.trash_at,
  99         }
 100         if select_attrs.index("manifest_text")
 101           @object[:manifest_text] = c.manifest_text
 102         end
 103       end
 104     else
 105       super(with_lock: with_lock)
 106     end
 107   end
 108
 109   def show
 110     if @object.is_a? Collection
 111       # Omit unsigned_manifest_text
 112       @select ||= model_class.selectable_attributes - ["unsigned_manifest_text"]
 113       super
 114     else
 115       send_json @object
 116     end
 117   end
 118
 119
 120   def find_collections(visited, sp, ignore_columns=[], &b)
 121     case sp
 122     when ArvadosModel
 123       sp.class.columns.each do |c|
 124         find_collections(visited, sp[c.name.to_sym], &b) if !ignore_columns.include?(c.name)
 125       end
 126     when Hash
 127       sp.each do |k, v|
 128         find_collections(visited, v, &b)
 129       end
 130     when Array
 131       sp.each do |v|
 132         find_collections(visited, v, &b)
 133       end
 134     when String
 135       if m = /[a-f0-9]{32}\+\d+/.match(sp)
 136         yield m[0], nil
 137       elsif m = Collection.uuid_regex.match(sp)
 138         yield nil, m[0]
 139       end
 140     end
 141   end
 142
 143   def search_edges(visited, uuid, direction)
 144     if uuid.nil? or uuid.empty? or visited[uuid]
 145       return
 146     end
 147
 148     if loc = Keep::Locator.parse(uuid)
 149       loc.strip_hints!
 150       return if visited[loc.to_s]
 151     end
 152
 153     if loc
 154       # uuid is a portable_data_hash
 155       collections = Collection.readable_by(*@read_users).where(portable_data_hash: loc.to_s)
 156       c = collections.limit(2).all
 157       if c.size == 1
 158         visited[loc.to_s] = c[0]
 159       elsif c.size > 1
 160         name = collections.limit(1).where("name <> ''").first
 161         if name
 162           visited[loc.to_s] = {
 163             portable_data_hash: c[0].portable_data_hash,
 164             name: "#{name.name} + #{collections.count-1} more"
 165           }
 166         else
 167           visited[loc.to_s] = {
 168             portable_data_hash: c[0].portable_data_hash,
 169             name: loc.to_s
 170           }
 171         end
 172       end
 173
 174       if direction == :search_up
 175         # Search upstream for jobs where this locator is the output of some job
 176         if !Rails.configuration.API.DisabledAPIs["jobs.list"]
 177           Job.readable_by(*@read_users).where(output: loc.to_s).each do |job|
 178             search_edges(visited, job.uuid, :search_up)
 179           end
 180
 181           Job.readable_by(*@read_users).where(log: loc.to_s).each do |job|
 182             search_edges(visited, job.uuid, :search_up)
 183           end
 184         end
 185
 186         Container.readable_by(*@read_users).where(output: loc.to_s).pluck(:uuid).each do |c_uuid|
 187           search_edges(visited, c_uuid, :search_up)
 188         end
 189
 190         Container.readable_by(*@read_users).where(log: loc.to_s).pluck(:uuid).each do |c_uuid|
 191           search_edges(visited, c_uuid, :search_up)
 192         end
 193       elsif direction == :search_down
 194         if loc.to_s == "d41d8cd98f00b204e9800998ecf8427e+0"
 195           # Special case, don't follow the empty collection.
 196           return
 197         end
 198
 199         # Search downstream for jobs where this locator is in script_parameters
 200         if !Rails.configuration.API.DisabledAPIs["jobs.list"]
 201           Job.readable_by(*@read_users).where(["jobs.script_parameters like ?", "%#{loc.to_s}%"]).each do |job|
 202             search_edges(visited, job.uuid, :search_down)
 203           end
 204
 205           Job.readable_by(*@read_users).where(["jobs.docker_image_locator = ?", "#{loc.to_s}"]).each do |job|
 206             search_edges(visited, job.uuid, :search_down)
 207           end
 208         end
 209
 210         Container.readable_by(*@read_users).where([Container.full_text_trgm + " like ?", "%#{loc.to_s}%"]).select("output, log, uuid").each do |c|
 211           if c.output != loc.to_s && c.log != loc.to_s
 212             search_edges(visited, c.uuid, :search_down)
 213           end
 214         end
 215       end
 216     else
 217       # uuid is a regular Arvados UUID
 218       rsc = ArvadosModel::resource_class_for_uuid uuid
 219       if rsc == Job
 220         Job.readable_by(*@read_users).where(uuid: uuid).each do |job|
 221           visited[uuid] = job.as_api_response
 222           if direction == :search_up
 223             # Follow upstream collections referenced in the script parameters
 224             find_collections(visited, job) do |hash, col_uuid|
 225               search_edges(visited, hash, :search_up) if hash
 226               search_edges(visited, col_uuid, :search_up) if col_uuid
 227             end
 228           elsif direction == :search_down
 229             # Follow downstream job output
 230             search_edges(visited, job.output, direction)
 231           end
 232         end
 233       elsif rsc == Container
 234         c = Container.readable_by(*@read_users).where(uuid: uuid).limit(1).first
 235         if c
 236           visited[uuid] = c.as_api_response
 237           if direction == :search_up
 238             # Follow upstream collections referenced in the script parameters
 239             find_collections(visited, c, ignore_columns=["log", "output"]) do |hash, col_uuid|
 240               search_edges(visited, hash, :search_up) if hash
 241               search_edges(visited, col_uuid, :search_up) if col_uuid
 242             end
 243           elsif direction == :search_down
 244             # Follow downstream job output
 245             search_edges(visited, c.output, :search_down)
 246           end
 247         end
 248       elsif rsc == ContainerRequest
 249         c = ContainerRequest.readable_by(*@read_users).where(uuid: uuid).limit(1).first
 250         if c
 251           visited[uuid] = c.as_api_response
 252           if direction == :search_up
 253             # Follow upstream collections
 254             find_collections(visited, c, ignore_columns=["log_uuid", "output_uuid"]) do |hash, col_uuid|
 255               search_edges(visited, hash, :search_up) if hash
 256               search_edges(visited, col_uuid, :search_up) if col_uuid
 257             end
 258           elsif direction == :search_down
 259             # Follow downstream job output
 260             search_edges(visited, c.output_uuid, :search_down)
 261           end
 262         end
 263       elsif rsc == Collection
 264         c = Collection.readable_by(*@read_users).where(uuid: uuid).limit(1).first
 265         if c
 266           if direction == :search_up
 267             visited[c.uuid] = c.as_api_response
 268
 269             if !Rails.configuration.API.DisabledAPIs["jobs.list"]
 270               Job.readable_by(*@read_users).where(output: c.portable_data_hash).each do |job|
 271                 search_edges(visited, job.uuid, :search_up)
 272               end
 273
 274               Job.readable_by(*@read_users).where(log: c.portable_data_hash).each do |job|
 275                 search_edges(visited, job.uuid, :search_up)
 276               end
 277             end
 278
 279             ContainerRequest.readable_by(*@read_users).where(output_uuid: uuid).pluck(:uuid).each do |cr_uuid|
 280               search_edges(visited, cr_uuid, :search_up)
 281             end
 282
 283             ContainerRequest.readable_by(*@read_users).where(log_uuid: uuid).pluck(:uuid).each do |cr_uuid|
 284               search_edges(visited, cr_uuid, :search_up)
 285             end
 286           elsif direction == :search_down
 287             search_edges(visited, c.portable_data_hash, :search_down)
 288           end
 289         end
 290       elsif rsc != nil
 291         rsc.where(uuid: uuid).each do |r|
 292           visited[uuid] = r.as_api_response
 293         end
 294       end
 295     end
 296
 297     if direction == :search_up
 298       # Search for provenance links pointing to the current uuid
 299       Link.readable_by(*@read_users).
 300         where(head_uuid: uuid, link_class: "provenance").
 301         each do |link|
 302         visited[link.uuid] = link.as_api_response
 303         search_edges(visited, link.tail_uuid, direction)
 304       end
 305     elsif direction == :search_down
 306       # Search for provenance links emanating from the current uuid
 307       Link.readable_by(current_user).
 308         where(tail_uuid: uuid, link_class: "provenance").
 309         each do |link|
 310         visited[link.uuid] = link.as_api_response
 311         search_edges(visited, link.head_uuid, direction)
 312       end
 313     end
 314   end
 315
 316   def provenance
 317     visited = {}
 318     if @object[:uuid]
 319       search_edges(visited, @object[:uuid], :search_up)
 320     else
 321       search_edges(visited, @object[:portable_data_hash], :search_up)
 322     end
 323     send_json visited
 324   end
 325
 326   def used_by
 327     visited = {}
 328     if @object[:uuid]
 329       search_edges(visited, @object[:uuid], :search_down)
 330     else
 331       search_edges(visited, @object[:portable_data_hash], :search_down)
 332     end
 333     send_json visited
 334   end
 335
 336   protected
 337
 338   def load_select_param *args
 339     super
 340     if action_name == 'index'
 341       # Omit manifest_text and unsigned_manifest_text from index results unless expressly selected.
 342       @select ||= model_class.selectable_attributes - ["manifest_text", "unsigned_manifest_text"]
 343     end
 344   end
 345 end