services/api/test/functional/arvados/v1/filters_test.rb

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 require 'test_helper'
   6
   7 class Arvados::V1::FiltersTest < ActionController::TestCase
   8   test '"not in" filter passes null values' do
   9     @controller = Arvados::V1::GroupsController.new
  10     authorize_with :admin
  11     get :index, params: {
  12       filters: [ ['group_class', 'not in', ['project']] ],
  13       controller: 'groups',
  14     }
  15     assert_response :success
  16     found = assigns(:objects)
  17     assert_includes(found.collect(&:group_class), nil,
  18                     "'group_class not in ['project']' filter should pass null")
  19   end
  20
  21   test 'error message for non-array element in filters array' do
  22     @controller = Arvados::V1::CollectionsController.new
  23     authorize_with :active
  24     get :index, params: {
  25       filters: [{bogus: 'filter'}],
  26     }
  27     assert_response 422
  28     assert_match(/Invalid element in filters array/,
  29                  json_response['errors'].join(' '))
  30   end
  31
  32   test 'error message for full text search on a specific column' do
  33     @controller = Arvados::V1::CollectionsController.new
  34     authorize_with :active
  35     get :index, params: {
  36       filters: [['uuid', '@@', 'abcdef']],
  37     }
  38     assert_response 422
  39     assert_match(/not supported/, json_response['errors'].join(' '))
  40   end
  41
  42   test 'difficult characters in full text search' do
  43     @controller = Arvados::V1::CollectionsController.new
  44     authorize_with :active
  45     get :index, params: {
  46       filters: [['any', '@@', 'a|b"c']],
  47     }
  48     assert_response :success
  49     # (Doesn't matter so much which results are returned.)
  50   end
  51
  52   test 'array operand in full text search' do
  53     @controller = Arvados::V1::CollectionsController.new
  54     authorize_with :active
  55     get :index, params: {
  56       filters: [['any', '@@', ['abc', 'def']]],
  57     }
  58     assert_response 422
  59     assert_match(/not supported/, json_response['errors'].join(' '))
  60   end
  61
  62   test 'api responses provide timestamps with nanoseconds' do
  63     @controller = Arvados::V1::CollectionsController.new
  64     authorize_with :active
  65     get :index
  66     assert_response :success
  67     assert_not_empty json_response['items']
  68     json_response['items'].each do |item|
  69       %w(created_at modified_at).each do |attr|
  70         # Pass fixtures with null timestamps.
  71         next if item[attr].nil?
  72         assert_match(/^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d.\d{9}Z$/, item[attr])
  73       end
  74     end
  75   end
  76
  77   %w(< > <= >= =).each do |operator|
  78     test "timestamp #{operator} filters work with nanosecond precision" do
  79       # Python clients like Node Manager rely on this exact format.
  80       # If you must change this format for some reason, make sure you
  81       # coordinate the change with them.
  82       expect_match = !!operator.index('=')
  83       mine = act_as_user users(:active) do
  84         Collection.create!(manifest_text: '')
  85       end
  86       timestamp = mine.modified_at.strftime('%Y-%m-%dT%H:%M:%S.%NZ')
  87       @controller = Arvados::V1::CollectionsController.new
  88       authorize_with :active
  89       get :index, params: {
  90         filters: [['modified_at', operator, timestamp],
  91                   ['uuid', '=', mine.uuid]],
  92       }
  93       assert_response :success
  94       uuids = json_response['items'].map { |item| item['uuid'] }
  95       if expect_match
  96         assert_includes uuids, mine.uuid
  97       else
  98         assert_not_includes uuids, mine.uuid
  99       end
 100     end
 101   end
 102
 103   test "full text search with count='none'" do
 104     @controller = Arvados::V1::GroupsController.new
 105     authorize_with :admin
 106
 107     get :contents, params: {
 108       format: :json,
 109       count: 'none',
 110       limit: 1000,
 111       filters: [['any', '@@', Rails.configuration.ClusterID]],
 112     }
 113
 114     assert_response :success
 115
 116     all_objects = Hash.new(0)
 117     json_response['items'].map{|o| o['kind']}.each{|t| all_objects[t] += 1}
 118
 119     assert_equal true, all_objects['arvados#group']>0
 120     assert_equal true, all_objects['arvados#job']>0
 121     assert_equal true, all_objects['arvados#pipelineInstance']>0
 122     assert_equal true, all_objects['arvados#pipelineTemplate']>0
 123
 124     # Perform test again mimicking a second page request with:
 125     # last_object_class = PipelineInstance
 126     #   and hence groups and jobs should not be included in the response
 127     # offset = 5, which means first 5 pipeline instances were already received in page 1
 128     #   and hence the remaining pipeline instances and all other object types should be included in the response
 129
 130     @test_counter = 0  # Reset executed action counter
 131
 132     @controller = Arvados::V1::GroupsController.new
 133
 134     get :contents, params: {
 135       format: :json,
 136       count: 'none',
 137       limit: 1000,
 138       offset: '5',
 139       last_object_class: 'PipelineInstance',
 140       filters: [['any', '@@', Rails.configuration.ClusterID]],
 141     }
 142
 143     assert_response :success
 144
 145     second_page = Hash.new(0)
 146     json_response['items'].map{|o| o['kind']}.each{|t| second_page[t] += 1}
 147
 148     assert_equal false, second_page.include?('arvados#group')
 149     assert_equal false, second_page.include?('arvados#job')
 150     assert_equal true, second_page['arvados#pipelineInstance']>0
 151     assert_equal all_objects['arvados#pipelineInstance'], second_page['arvados#pipelineInstance']+5
 152     assert_equal true, second_page['arvados#pipelineTemplate']>0
 153   end
 154
 155   [['prop1', '=', 'value1', [:collection_with_prop1_value1], [:collection_with_prop1_value2, :collection_with_prop2_1]],
 156    ['prop1', '!=', 'value1', [:collection_with_prop1_value2, :collection_with_prop2_1], [:collection_with_prop1_value1]],
 157    ['prop1', 'exists', true, [:collection_with_prop1_value1, :collection_with_prop1_value2, :collection_with_prop1_value3, :collection_with_prop1_other1], [:collection_with_prop2_1]],
 158    ['prop1', 'exists', false, [:collection_with_prop2_1], [:collection_with_prop1_value1, :collection_with_prop1_value2, :collection_with_prop1_value3, :collection_with_prop1_other1]],
 159    ['prop1', 'in', ['value1', 'value2'], [:collection_with_prop1_value1, :collection_with_prop1_value2], [:collection_with_prop1_value3, :collection_with_prop2_1]],
 160    ['prop1', 'in', ['value1', 'valueX'], [:collection_with_prop1_value1], [:collection_with_prop1_value3, :collection_with_prop2_1]],
 161    ['prop1', 'not in', ['value1', 'value2'], [:collection_with_prop1_value3, :collection_with_prop1_other1, :collection_with_prop2_1], [:collection_with_prop1_value1, :collection_with_prop1_value2]],
 162    ['prop1', 'not in', ['value1', 'valueX'], [:collection_with_prop1_value2, :collection_with_prop1_value3, :collection_with_prop1_other1, :collection_with_prop2_1], [:collection_with_prop1_value1]],
 163    ['prop1', '>', 'value2', [:collection_with_prop1_value3], [:collection_with_prop1_other1, :collection_with_prop1_value1]],
 164    ['prop1', '<', 'value2', [:collection_with_prop1_other1, :collection_with_prop1_value1], [:collection_with_prop1_value2, :collection_with_prop1_value2]],
 165    ['prop1', '<=', 'value2', [:collection_with_prop1_other1, :collection_with_prop1_value1, :collection_with_prop1_value2], [:collection_with_prop1_value3]],
 166    ['prop1', '>=', 'value2', [:collection_with_prop1_value2, :collection_with_prop1_value3], [:collection_with_prop1_other1, :collection_with_prop1_value1]],
 167    ['prop1', 'like', 'value%', [:collection_with_prop1_value1, :collection_with_prop1_value2, :collection_with_prop1_value3], [:collection_with_prop1_other1]],
 168    ['prop1', 'like', '%1', [:collection_with_prop1_value1, :collection_with_prop1_other1], [:collection_with_prop1_value2, :collection_with_prop1_value3]],
 169    ['prop1', 'ilike', 'VALUE%', [:collection_with_prop1_value1, :collection_with_prop1_value2, :collection_with_prop1_value3], [:collection_with_prop1_other1]],
 170    ['prop2', '>',  1, [:collection_with_prop2_5], [:collection_with_prop2_1]],
 171    ['prop2', '<',  5, [:collection_with_prop2_1], [:collection_with_prop2_5]],
 172    ['prop2', '<=', 5, [:collection_with_prop2_1, :collection_with_prop2_5], []],
 173    ['prop2', '>=', 1, [:collection_with_prop2_1, :collection_with_prop2_5], []],
 174    ['<http://schema.org/example>', '=', "value1", [:collection_with_uri_prop], []],
 175    ['listprop', 'contains', 'elem1', [:collection_with_list_prop_odd, :collection_with_listprop_elem1], [:collection_with_list_prop_even]],
 176    ['listprop', '=', 'elem1', [:collection_with_listprop_elem1], [:collection_with_list_prop_odd]],
 177    ['listprop', 'contains', 5, [:collection_with_list_prop_odd], [:collection_with_list_prop_even, :collection_with_listprop_elem1]],
 178    ['listprop', 'contains', 'elem2', [:collection_with_list_prop_even], [:collection_with_list_prop_odd, :collection_with_listprop_elem1]],
 179    ['listprop', 'contains', 4, [:collection_with_list_prop_even], [:collection_with_list_prop_odd, :collection_with_listprop_elem1]],
 180   ].each do |prop, op, opr, inc, ex|
 181     test "jsonb filter properties.#{prop} #{op} #{opr})" do
 182       @controller = Arvados::V1::CollectionsController.new
 183       authorize_with :admin
 184       get :index, params: {
 185             filters: SafeJSON.dump([ ["properties.#{prop}", op, opr] ]),
 186             limit: 1000
 187           }
 188       assert_response :success
 189       found = assigns(:objects).collect(&:uuid)
 190
 191       inc.each do |i|
 192         assert_includes(found, collections(i).uuid)
 193       end
 194
 195       ex.each do |e|
 196         assert_not_includes(found, collections(e).uuid)
 197       end
 198     end
 199   end
 200
 201   test "jsonb hash 'exists' and '!=' filter" do
 202     @controller = Arvados::V1::CollectionsController.new
 203     authorize_with :admin
 204     get :index, params: {
 205       filters: [ ['properties.prop1', 'exists', true], ['properties.prop1', '!=', 'value1'] ]
 206     }
 207     assert_response :success
 208     found = assigns(:objects).collect(&:uuid)
 209     assert_equal found.length, 3
 210     assert_not_includes(found, collections(:collection_with_prop1_value1).uuid)
 211     assert_includes(found, collections(:collection_with_prop1_value2).uuid)
 212     assert_includes(found, collections(:collection_with_prop1_value3).uuid)
 213     assert_includes(found, collections(:collection_with_prop1_other1).uuid)
 214   end
 215
 216   test "jsonb array 'exists'" do
 217     @controller = Arvados::V1::CollectionsController.new
 218     authorize_with :admin
 219     get :index, params: {
 220       filters: [ ['storage_classes_confirmed.default', 'exists', true] ]
 221     }
 222     assert_response :success
 223     found = assigns(:objects).collect(&:uuid)
 224     assert_equal 2, found.length
 225     assert_not_includes(found,
 226       collections(:storage_classes_desired_default_unconfirmed).uuid)
 227     assert_includes(found,
 228       collections(:storage_classes_desired_default_confirmed_default).uuid)
 229     assert_includes(found,
 230       collections(:storage_classes_desired_archive_confirmed_default).uuid)
 231   end
 232
 233   test "jsonb hash alternate form 'exists' and '!=' filter" do
 234     @controller = Arvados::V1::CollectionsController.new
 235     authorize_with :admin
 236     get :index, params: {
 237       filters: [ ['properties', 'exists', 'prop1'], ['properties.prop1', '!=', 'value1'] ]
 238     }
 239     assert_response :success
 240     found = assigns(:objects).collect(&:uuid)
 241     assert_equal found.length, 3
 242     assert_not_includes(found, collections(:collection_with_prop1_value1).uuid)
 243     assert_includes(found, collections(:collection_with_prop1_value2).uuid)
 244     assert_includes(found, collections(:collection_with_prop1_value3).uuid)
 245     assert_includes(found, collections(:collection_with_prop1_other1).uuid)
 246   end
 247
 248   test "jsonb array alternate form 'exists' filter" do
 249     @controller = Arvados::V1::CollectionsController.new
 250     authorize_with :admin
 251     get :index, params: {
 252       filters: [ ['storage_classes_confirmed', 'exists', 'default'] ]
 253     }
 254     assert_response :success
 255     found = assigns(:objects).collect(&:uuid)
 256     assert_equal 2, found.length
 257     assert_not_includes(found,
 258       collections(:storage_classes_desired_default_unconfirmed).uuid)
 259     assert_includes(found,
 260       collections(:storage_classes_desired_default_confirmed_default).uuid)
 261     assert_includes(found,
 262       collections(:storage_classes_desired_archive_confirmed_default).uuid)
 263   end
 264
 265   test "jsonb 'exists' must be boolean" do
 266     @controller = Arvados::V1::CollectionsController.new
 267     authorize_with :admin
 268     get :index, params: {
 269       filters: [ ['properties.prop1', 'exists', nil] ]
 270     }
 271     assert_response 422
 272     assert_match(/Invalid operand '' for 'exists' must be true or false/,
 273                  json_response['errors'].join(' '))
 274   end
 275
 276   test "jsonb checks column exists" do
 277     @controller = Arvados::V1::CollectionsController.new
 278     authorize_with :admin
 279     get :index, params: {
 280       filters: [ ['puppies.prop1', '=', 'value1'] ]
 281     }
 282     assert_response 422
 283     assert_match(/Invalid attribute 'puppies' for subproperty filter/,
 284                  json_response['errors'].join(' '))
 285   end
 286
 287   test "jsonb checks column is valid" do
 288     @controller = Arvados::V1::CollectionsController.new
 289     authorize_with :admin
 290     get :index, params: {
 291       filters: [ ['name.prop1', '=', 'value1'] ]
 292     }
 293     assert_response 422
 294     assert_match(/Invalid attribute 'name' for subproperty filter/,
 295                  json_response['errors'].join(' '))
 296   end
 297
 298   test "jsonb invalid operator" do
 299     @controller = Arvados::V1::CollectionsController.new
 300     authorize_with :admin
 301     get :index, params: {
 302       filters: [ ['properties.prop1', '###', 'value1'] ]
 303     }
 304     assert_response 422
 305     assert_match(/Invalid operator for subproperty search '###'/,
 306                  json_response['errors'].join(' '))
 307   end
 308
 309   test "replication_desired = 2" do
 310     @controller = Arvados::V1::CollectionsController.new
 311     authorize_with :admin
 312     get :index, params: {
 313       filters: SafeJSON.dump([ ['replication_desired', '=', 2] ])
 314     }
 315     assert_response :success
 316     found = assigns(:objects).collect(&:uuid)
 317     assert_includes(found, collections(:replication_desired_2_unconfirmed).uuid)
 318     assert_includes(found, collections(:replication_desired_2_confirmed_2).uuid)
 319   end
 320 end