Merge branch '20470-contents-select' refs #20470
authorPeter Amstutz <peter.amstutz@curii.com>
Fri, 5 May 2023 17:33:57 +0000 (13:33 -0400)
committerPeter Amstutz <peter.amstutz@curii.com>
Fri, 5 May 2023 17:33:57 +0000 (13:33 -0400)
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz@curii.com>

sdk/python/arvados-v1-discovery.json
services/api/app/controllers/application_controller.rb
services/api/app/controllers/arvados/v1/containers_controller.rb
services/api/app/controllers/arvados/v1/groups_controller.rb
services/api/app/controllers/arvados/v1/schema_controller.rb
services/api/app/controllers/arvados/v1/users_controller.rb
services/api/app/models/group.rb
services/api/test/functional/arvados/v1/groups_controller_test.rb
services/api/test/functional/arvados/v1/schema_controller_test.rb

index 2fb9678c3472fde9512ed2bffb1f09e3e96b8656..5d4666bf9435440e7774f0899e90e21e116a8c38 100644 (file)
               "description": "",
               "location": "query"
             },
+            "select": {
+              "type": "array",
+              "description": "Attributes of each object to return in the response.",
+              "required": false,
+              "location": "query"
+            },
             "distinct": {
               "type": "boolean",
               "required": false,
         "created_at": {
           "type": "datetime"
         },
-        "updated_at": {
-          "type": "datetime"
-        },
         "is_locked_by_uuid": {
           "type": "string"
         },
         },
         "components": {
           "type": "Hash"
-        },
-        "script_parameters_digest": {
-          "type": "string"
         }
       }
     },
         "created_at": {
           "type": "datetime"
         },
-        "updated_at": {
-          "type": "datetime"
-        },
         "is_trusted": {
           "type": "boolean"
         }
         "created_at": {
           "type": "datetime"
         },
-        "updated_at": {
-          "type": "datetime"
-        },
         "default_owner_uuid": {
           "type": "string"
         },
         },
         "created_at": {
           "type": "datetime"
-        },
-        "updated_at": {
-          "type": "datetime"
         }
       }
     },
         "replication_confirmed": {
           "type": "integer"
         },
-        "updated_at": {
-          "type": "datetime"
-        },
         "manifest_text": {
           "type": "text"
         },
         "delete_at": {
           "type": "datetime"
         },
-        "file_names": {
-          "type": "text"
-        },
         "trash_at": {
           "type": "datetime"
         },
         "priority": {
           "type": "integer"
         },
-        "updated_at": {
-          "type": "datetime"
-        },
         "exit_code": {
           "type": "integer"
         },
         "runtime_auth_scopes": {
           "type": "Array"
         },
-        "runtime_token": {
-          "type": "text"
-        },
         "lock_count": {
           "type": "integer"
         },
         "filters": {
           "type": "text"
         },
-        "updated_at": {
-          "type": "datetime"
-        },
         "container_count": {
           "type": "integer"
         },
         "output_ttl": {
           "type": "integer"
         },
-        "runtime_token": {
-          "type": "text"
-        },
         "output_storage_classes": {
           "type": "Array"
         },
         "description": {
           "type": "string"
         },
-        "updated_at": {
-          "type": "datetime"
-        },
         "group_class": {
           "type": "string"
         },
         },
         "created_at": {
           "type": "datetime"
-        },
-        "updated_at": {
-          "type": "datetime"
         }
       }
     },
         "created_at": {
           "type": "datetime"
         },
-        "updated_at": {
-          "type": "datetime"
-        },
         "created_by_job_task_uuid": {
           "type": "string"
         },
         "modified_at": {
           "type": "datetime"
         },
-        "ping_secret": {
-          "type": "string"
-        },
         "node_uuid": {
           "type": "string"
         },
         "created_at": {
           "type": "datetime"
         },
-        "updated_at": {
-          "type": "datetime"
-        },
         "keep_service_uuid": {
           "type": "string"
         }
         "created_at": {
           "type": "datetime"
         },
-        "updated_at": {
-          "type": "datetime"
-        },
         "read_only": {
           "type": "boolean"
         }
         },
         "properties": {
           "type": "Hash"
-        },
-        "updated_at": {
-          "type": "datetime"
         }
       }
     },
           "type": "string",
           "description": "Object version."
         },
+        "id": {
+          "type": "integer"
+        },
         "owner_uuid": {
           "type": "string"
         },
         "created_at": {
           "type": "datetime"
         },
-        "updated_at": {
-          "type": "datetime"
-        },
         "modified_at": {
           "type": "datetime"
         },
         "ip_address": {
           "type": "string"
         },
-        "first_ping_at": {
-          "type": "datetime"
-        },
         "last_ping_at": {
           "type": "datetime"
         },
-        "info": {
-          "type": "Hash"
-        },
-        "updated_at": {
-          "type": "datetime"
-        },
         "properties": {
           "type": "Hash"
         },
         "prefs": {
           "type": "Hash"
         },
-        "updated_at": {
-          "type": "datetime"
-        },
-        "default_owner_uuid": {
-          "type": "string"
-        },
         "is_active": {
           "type": "boolean"
         },
         "username": {
           "type": "string"
-        },
-        "redirect_to_user_uuid": {
-          "type": "string"
         }
       }
     },
         "components": {
           "type": "Hash"
         },
-        "updated_at": {
-          "type": "datetime"
-        },
         "properties": {
           "type": "Hash"
         },
         "components": {
           "type": "Hash"
         },
-        "updated_at": {
-          "type": "datetime"
-        },
         "description": {
           "type": "string"
         }
         },
         "created_at": {
           "type": "datetime"
-        },
-        "updated_at": {
-          "type": "datetime"
         }
       }
     },
         "material": {
           "type": "string"
         },
-        "updated_at": {
-          "type": "datetime"
-        },
         "properties": {
           "type": "Hash"
         }
         },
         "created_at": {
           "type": "datetime"
-        },
-        "updated_at": {
-          "type": "datetime"
         }
       }
     },
         },
         "created_at": {
           "type": "datetime"
-        },
-        "updated_at": {
-          "type": "datetime"
         }
       }
     },
         },
         "definition": {
           "type": "text"
-        },
-        "updated_at": {
-          "type": "datetime"
         }
       }
     },
         "replication_confirmed": {
           "type": "integer"
         },
-        "updated_at": {
-          "type": "datetime"
-        },
         "manifest_text": {
           "type": "text"
         },
         "delete_at": {
           "type": "datetime"
         },
-        "file_names": {
-          "type": "text"
-        },
         "trash_at": {
           "type": "datetime"
         },
index cf7271bbffa12bcf113a46ba7555010d125ef43a..b1915502404811177dfef15c605ff42db52df2d6 100644 (file)
@@ -101,7 +101,7 @@ class ApplicationController < ActionController::Base
   end
 
   def show
-    send_json @object.as_api_response(nil, select: @select)
+    send_json @object.as_api_response(nil, select: select_for_klass(@select, model_class))
   end
 
   def create
@@ -228,6 +228,24 @@ class ApplicationController < ActionController::Base
     @objects = model_class.apply_filters(@objects, @filters)
   end
 
+  def select_for_klass sel, model_class, raise_unknown=true
+    return nil if sel.nil?
+    # Filter the select fields to only the ones that apply to the
+    # given class.
+    sel.map do |column|
+      sp = column.split(".")
+      if sp.length == 2 && sp[0] == model_class.table_name && model_class.selectable_attributes.include?(sp[1])
+        sp[1]
+      elsif model_class.selectable_attributes.include? column
+        column
+      elsif raise_unknown
+        raise ArgumentError.new("Invalid attribute '#{column}' of #{model_class.name} in select parameter")
+      else
+        nil
+      end
+    end.compact
+  end
+
   def apply_where_limit_order_params model_class=nil
     model_class ||= self.model_class
     apply_filters model_class
@@ -291,7 +309,7 @@ class ApplicationController < ActionController::Base
         # Map attribute names in @select to real column names, resolve
         # those to fully-qualified SQL column names, and pass the
         # resulting string to the select method.
-        columns_list = model_class.columns_for_attributes(@select).
+        columns_list = model_class.columns_for_attributes(select_for_klass @select, model_class).
           map { |s| "#{ar_table_name}.#{ActiveRecord::Base.connection.quote_column_name s}" }
         @objects = @objects.select(columns_list.join(", "))
       end
@@ -317,7 +335,7 @@ class ApplicationController < ActionController::Base
     return if @limit == 0 || @limit == 1
     model_class ||= self.model_class
     limit_columns = model_class.limit_index_columns_read
-    limit_columns &= model_class.columns_for_attributes(@select) if @select
+    limit_columns &= model_class.columns_for_attributes(select_for_klass @select, model_class) if @select
     return if limit_columns.empty?
     model_class.transaction do
       limit_query = @objects.
@@ -479,12 +497,23 @@ class ApplicationController < ActionController::Base
     @orders = []
     @filters = []
     @objects = nil
+
+    # This is a little hacky but sometimes the fields the user wants
+    # to selecting on are unrelated to the object being loaded here,
+    # for example groups#contents, so filter the fields that will be
+    # used in find_objects_for_index and then reset afterwards.  In
+    # some cases, code that modifies the @select list needs to set
+    # @preserve_select.
+    @preserve_select = @select
+    @select = select_for_klass(@select, self.model_class, false)
+
     find_objects_for_index
     if with_lock && Rails.configuration.API.LockBeforeUpdate
       @object = @objects.lock.first
     else
       @object = @objects.first
     end
+    @select = @preserve_select
   end
 
   def nullable_attributes
index ceb40348bbdfc723c30a5e9f99a76d0132ff6fe1..17970ce4cd271c13d6f66c2ee59b1a04da76ce3a 100644 (file)
@@ -51,7 +51,13 @@ class Arvados::V1::ContainersController < ApplicationController
     if action_name == 'lock' || action_name == 'unlock'
       # Avoid loading more fields than we need
       @objects = @objects.select(:id, :uuid, :state, :priority, :auth_uuid, :locked_by_uuid, :lock_count)
-      @select = %w(uuid state priority auth_uuid locked_by_uuid)
+      # This gets called from within find_object_by_uuid.
+      # find_object_by_uuid stores the original value of @select in
+      # @preserve_select, edits the value of @select, calls
+      # find_objects_for_index, then restores @select from the value
+      # of @preserve_select.  So if we want our updated value of
+      # @select here to stick, we have to set @preserve_select.
+      @select = @preserve_select = %w(uuid state priority auth_uuid locked_by_uuid)
     elsif action_name == 'update_priority'
       # We're going to reload in update_priority!, which will select
       # all attributes, but will fail if we don't select :id now.
index e9bc006a36664bb1a929bf72dccc9730fa9b049c..efcc43db26ff5df549f7933c5b84998be88af80e 100644 (file)
@@ -46,7 +46,6 @@ class Arvados::V1::GroupsController < ApplicationController
                 type: 'boolean', required: false, default: false, description: 'Include past collection versions.',
               }
             })
-    params.delete(:select)
     params
   end
 
@@ -260,6 +259,20 @@ class Arvados::V1::GroupsController < ApplicationController
       end
     end
 
+    # Check that any fields in @select are valid for at least one class
+    if @select
+      all_attributes = []
+      klasses.each do |klass|
+        all_attributes.concat klass.selectable_attributes
+      end
+      @select.each do |check|
+        if !all_attributes.include? check
+          raise ArgumentError.new "Invalid attribute '#{check}' in select"
+        end
+      end
+    end
+    any_selections = @select
+
     included_by_uuid = {}
 
     seen_last_class = false
@@ -291,14 +304,21 @@ class Arvados::V1::GroupsController < ApplicationController
         request_orders.andand.find { |r| r =~ /^#{klass.table_name}\./i || r !~ /\./ } ||
         klass.default_orders.join(", ")
 
-      @select = nil
+      @select = select_for_klass any_selections, klass, false
+
       where_conds = filter_by_owner
-      if klass == Collection
+      if klass == Collection && @select.nil?
         @select = klass.selectable_attributes - ["manifest_text", "unsigned_manifest_text"]
       elsif klass == Group
         where_conds = where_conds.merge(group_class: ["project","filter"])
       end
 
+      # Make signed manifest_text not selectable because controller
+      # currently doesn't know to sign it.
+      if @select
+        @select = @select - ["manifest_text"]
+      end
+
       @filters = request_filters.map do |col, op, val|
         if !col.index('.')
           [col, op, val]
index 34dfe966b0a38a7d347eb7427706d35b12465557..4d15cb121524e530adf0008d8c62deac6b8cad91 100644 (file)
@@ -127,7 +127,7 @@ class Arvados::V1::SchemaController < ApplicationController
       end
       object_properties = {}
       k.columns.
-        select { |col| col.name != 'id' && !col.name.start_with?('secret_') }.
+        select { |col| k.selectable_attributes.include? col.name }.
         collect do |col|
         if k.serialized_attributes.has_key? col.name
           object_properties[col.name] = {
index 507cb4ac339fe5fd63fbbf7fb3013411fc44b5e9..ded86aa66dc02aa140b754c761e83270681cad62 100644 (file)
@@ -274,7 +274,7 @@ class Arvados::V1::UsersController < ApplicationController
     return super if @read_users.any?(&:is_admin)
     if params[:uuid] != current_user.andand.uuid
       # Non-admin index/show returns very basic information about readable users.
-      safe_attrs = ["uuid", "is_active", "email", "first_name", "last_name", "username", "can_write", "can_manage"]
+      safe_attrs = ["uuid", "is_active", "email", "first_name", "last_name", "username", "can_write", "can_manage", "kind"]
       if @select
         @select = @select & safe_attrs
       else
@@ -282,6 +282,13 @@ class Arvados::V1::UsersController < ApplicationController
       end
       @filters += [['is_active', '=', true]]
     end
+    # This gets called from within find_object_by_uuid.
+    # find_object_by_uuid stores the original value of @select in
+    # @preserve_select, edits the value of @select, calls
+    # find_objects_for_index, then restores @select from the value
+    # of @preserve_select.  So if we want our updated value of
+    # @select here to stick, we have to set @preserve_select.
+    @preserve_select = @select
     super
   end
 
index 85855fda97271a2cbfc855fef5d0862fa2a7122e..aa3a19bf87004f950e7e5f390650ce9ac964619f 100644 (file)
@@ -54,6 +54,7 @@ class Group < ArvadosModel
     super.merge(
                 'can_write' => ['owner_uuid', 'uuid'],
                 'can_manage' => ['owner_uuid', 'uuid'],
+                'writable_by' => ['owner_uuid', 'uuid'],
                 )
   end
 
index cfcb33d40a743c21cbbd8ae0ff1ec7dd15c5945e..a64ea766921472c2426274c246ac3112dfb25ee3 100644 (file)
@@ -330,6 +330,38 @@ class Arvados::V1::GroupsControllerTest < ActionController::TestCase
     assert_equal 0, json_response['items'].count
   end
 
+  test 'get group-owned objects with select' do
+    authorize_with :active
+    get :contents, params: {
+      id: groups(:aproject).uuid,
+      limit: 100,
+      format: :json,
+      select: ["uuid", "storage_classes_desired"]
+    }
+    assert_response :success
+    assert_equal 17, json_response['items_available']
+    assert_equal 17, json_response['items'].count
+    json_response['items'].each do |item|
+      # Expect collections to have a storage_classes field, other items should not.
+      if item["kind"] == "arvados#collection"
+        assert !item["storage_classes_desired"].nil?
+      else
+        assert item["storage_classes_desired"].nil?
+      end
+    end
+  end
+
+  test 'get group-owned objects with invalid field in select' do
+    authorize_with :active
+    get :contents, params: {
+      id: groups(:aproject).uuid,
+      limit: 100,
+      format: :json,
+      select: ["uuid", "storage_classes_desire"]
+    }
+    assert_response 422
+  end
+
   test 'get group-owned objects with additional filter matching nothing' do
     authorize_with :active
     get :contents, params: {
index f96f1af53746e6399efe911d19fb3d7976e95961..65a2b64b8a4be509aee4eba716e310477d88a43b 100644 (file)
@@ -83,7 +83,7 @@ class Arvados::V1::SchemaControllerTest < ActionController::TestCase
     group_index_params = discovery_doc['resources']['groups']['methods']['index']['parameters']
     group_contents_params = discovery_doc['resources']['groups']['methods']['contents']['parameters']
 
-    assert_equal group_contents_params.keys.sort, (group_index_params.keys - ['select'] + ['uuid', 'recursive', 'include', 'include_old_versions']).sort
+    assert_equal group_contents_params.keys.sort, (group_index_params.keys + ['uuid', 'recursive', 'include', 'include_old_versions']).sort
 
     recursive_param = group_contents_params['recursive']
     assert_equal 'boolean', recursive_param['type']