17994: Support filtering by element/key presence in jsonb columns.

Remove storage_classes_* from searchable_columns so that filters with
the magic attr "any" don't try to search them.

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@curii.com>

diff --git a/services/api/app/models/collection.rb b/services/api/app/models/collection.rb
index f4c717999e8b0a2a134d641e6f0d018c83d62fc4..1bbe8cc6614b2becdfeac1afef32cd8fcfb4beea 100644 (file)
--- a/services/api/app/models/collection.rb
+++ b/services/api/app/models/collection.rb
@@ -613,7 +613,7 @@ class Collection < ArvadosModel
   end
 
   def self.searchable_columns operator
-    super - ["manifest_text"] + ["storage_classes_desired", "storage_classes_confirmed"]
+    super - ["manifest_text"]
   end
 
   def self.full_text_searchable_columns

diff --git a/services/api/lib/record_filters.rb b/services/api/lib/record_filters.rb
index 5688ca6140f17fcef94ed112481cddb06e75c668..337197c2c5bc931c078b89c78280757bce5bd420 100644 (file)
--- a/services/api/lib/record_filters.rb
+++ b/services/api/lib/record_filters.rb
@@ -44,9 +44,10 @@ module RecordFilters
         raise ArgumentError.new("Invalid operator '#{operator}' (#{operator.class}) in filter")
       end
 
+      operator = operator.downcase
       cond_out = []
 
-      if attrs_in == 'any' && (operator.casecmp('ilike').zero? || operator.casecmp('like').zero?) && (operand.is_a? String) && operand.match('^[%].*[%]$')
+      if attrs_in == 'any' && (operator == 'ilike' || operator == 'like') && (operand.is_a? String) && operand.match('^[%].*[%]$')
         # Trigram index search
         cond_out << model_class.full_text_trgm + " #{operator} ?"
         param_out << operand
@@ -98,9 +99,9 @@ module RecordFilters
           end
 
           # jsonb search
-          case operator.downcase
+          case operator
           when '=', '!='
-            not_in = if operator.downcase == "!=" then "NOT " else "" end
+            not_in = if operator == "!=" then "NOT " else "" end
             cond_out << "#{not_in}(#{attr_table_name}.#{attr} @> ?::jsonb)"
             param_out << SafeJSON.dump({proppath => operand})
           when 'in'
@@ -147,7 +148,7 @@ module RecordFilters
           else
             raise ArgumentError.new("Invalid operator for subproperty search '#{operator}'")
           end
-        elsif operator.downcase == "exists"
+        elsif operator == "exists"
           if col.type != :jsonb
             raise ArgumentError.new("Invalid attribute '#{attr}' for operator '#{operator}' in filter")
           end
@@ -155,11 +156,12 @@ module RecordFilters
           cond_out << "jsonb_exists(#{attr_table_name}.#{attr}, ?)"
           param_out << operand
         else
-          if !attr_model_class.searchable_columns(operator).index attr
+          if !attr_model_class.searchable_columns(operator).index(attr) &&
+             !(col.andand.type == :jsonb && ['contains', '=', '<>', '!='].index(operator))
             raise ArgumentError.new("Invalid attribute '#{attr}' in filter")
           end
 
-          case operator.downcase
+          case operator
           when '=', '<', '<=', '>', '>=', '!=', 'like', 'ilike'
             attr_type = attr_model_class.attribute_column(attr).type
             operator = '<>' if operator == '!='
@@ -240,6 +242,23 @@ module RecordFilters
               end
             end
             cond_out << cond.join(' OR ')
+          when 'contains'
+            if col.andand.type != :jsonb
+              raise ArgumentError.new("Invalid attribute '#{attr}' for '#{operator}' operator")
+            end
+            if operand == []
+              raise ArgumentError.new("Invalid operand '#{operand.inspect}' for '#{operator}' operator")
+            end
+            operand = [operand] unless operand.is_a? Array
+            operand.each do |op|
+              if !op.is_a?(String)
+                raise ArgumentError.new("Invalid element #{operand.inspect} in operand for #{operator.inspect} operator (operand must be a string or array of strings)")
+              end
+            end
+            q = operand.map { |s| ActiveRecord::Base.connection.quote(s) }.join(',')
+            # We use jsonb_exists_all(a,b) instead of "a ?& b" because
+            # the pg gem thinks "?" is a bind var.
+            cond_out << "jsonb_exists_all(#{attr_table_name}.#{attr}, array[#{q}])"
           else
             raise ArgumentError.new("Invalid operator '#{operator}'")
           end

diff --git a/services/api/test/functional/arvados/v1/filters_test.rb b/services/api/test/functional/arvados/v1/filters_test.rb
index 26270b1c3c9c9b4da0ec4c03f6a8d6fd861fbe70..04d0329b26caac6ce30f500b091602a727922190 100644 (file)
--- a/services/api/test/functional/arvados/v1/filters_test.rb
+++ b/services/api/test/functional/arvados/v1/filters_test.rb
@@ -319,4 +319,51 @@ class Arvados::V1::FiltersTest < ActionController::TestCase
     assert_includes(found, collections(:replication_desired_2_unconfirmed).uuid)
     assert_includes(found, collections(:replication_desired_2_confirmed_2).uuid)
   end
+
+  [
+    [1, "foo"],
+    [1, ["foo"]],
+    [1, ["bar"]],
+    [1, ["bar", "foo"]],
+    [0, ["foo", "qux"]],
+    [0, ["qux"]],
+    [nil, []],
+    [nil, [[]]],
+    [nil, [["bogus"]]],
+    [nil, [{"foo" => "bar"}]],
+    [nil, {"foo" => "bar"}],
+  ].each do |results, operand|
+    test "storage_classes_desired contains #{operand.inspect}" do
+      @controller = Arvados::V1::CollectionsController.new
+      authorize_with(:active)
+      c = Collection.create!(
+        manifest_text: "",
+        storage_classes_desired: ["foo", "bar", "baz"])
+      get :index, params: {
+            filters: [["storage_classes_desired", "contains", operand]],
+          }
+      if results.nil?
+        assert_response 422
+        next
+      end
+      assert_response :success
+      assert_equal results, json_response["items"].length
+      if results > 0
+        assert_equal c.uuid, json_response["items"][0]["uuid"]
+      end
+    end
+  end
+
+  test "collections properties contains top level key" do
+    @controller = Arvados::V1::CollectionsController.new
+    authorize_with(:active)
+    get :index, params: {
+          filters: [["properties", "contains", "prop1"]],
+        }
+    assert_response :success
+    assert_not_empty json_response["items"]
+    json_response["items"].each do |c|
+      assert c["properties"].has_key?("prop1")
+    end
+  end
 end