Merge branch '4019-query-properties' closes #4019
authorPeter Amstutz <pamstutz@veritasgenetics.com>
Wed, 13 Dec 2017 16:12:56 +0000 (11:12 -0500)
committerPeter Amstutz <pamstutz@veritasgenetics.com>
Wed, 13 Dec 2017 16:12:58 +0000 (11:12 -0500)
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz@veritasgenetics.com>

doc/api/methods.html.textile.liquid
services/api/db/migrate/20171212153352_add_gin_index_to_collection_properties.rb [new file with mode: 0644]
services/api/db/structure.sql
services/api/lib/record_filters.rb
services/api/test/fixtures/collections.yml
services/api/test/functional/arvados/v1/filters_test.rb

index a24f34c1ac2ab32957b4c3d80f9c901a56ae36b1..00c120d9f8f1be4aad90022b514fe37024618dc3 100644 (file)
@@ -90,12 +90,27 @@ table(table table-bordered table-condensed).
 The following operators are available.
 
 table(table table-bordered table-condensed).
-|_. Operator|_. Operand type|_. Example|
-|@<@, @<=@, @>=@, @>@, @like@, @ilike@|string|@["script_version","like","d00220fb%"]@|
-|@=@, @!=@|string or null|@["tail_uuid","=","xyzzy-j7d0g-fffffffffffffff"]@
-@["tail_uuid","!=",null]@|
-|@in@, @not in@|array of strings|@["script_version","in",["master","d00220fb38d4b85ca8fc28a8151702a2b9d1dec5"]]@|
-|@is_a@|string|@["head_uuid","is_a","arvados#pipelineInstance"]@|
+|_. Operator|_. Operand type|_. Description|_. Example|
+|@=@, @!=@|string, number, timestamp, or null|Equality comparison|@["tail_uuid","=","xyzzy-j7d0g-fffffffffffffff"]@ @["tail_uuid","!=",null]@|
+|@<@, @<=@, @>=@, @>@|string, number, or timestamp|Ordering comparison|@["script_version",">","123"]@|
+|@like@, @ilike@|string|SQL pattern match.  Single character match is @_@ and wildcard is @%@. The @ilike@ operator is case-insensitive|@["script_version","like","d00220fb%"]@|
+|@in@, @not in@|array of strings|Set membership|@["script_version","in",["master","d00220fb38d4b85ca8fc28a8151702a2b9d1dec5"]]@|
+|@is_a@|string|Arvados object type|@["head_uuid","is_a","arvados#collection"]@|
+|@exists@|string|Test if a subproperty is present.|@["properties","exists","my_subproperty"]@|
+
+h4. Filtering on subproperties
+
+Some record type have an additional @properties@ attribute that allows recording and filtering on additional key-value pairs.  To filter on a subproperty, the value in the @attribute@ position has the form @properties.user_property@.  You may also use JSON-LD / RDF style URIs for property keys by enclosing them in @<...>@ for example @properties.<http://example.com/user_property>@.  Alternately you may also provide a JSON-LD "@context" field, however at this time JSON-LD contexts are not interpreted by Arvados.
+
+table(table table-bordered table-condensed).
+|_. Operator|_. Operand type|_. Description|_. Example|
+|@=@, @!=@|string, number or boolean|Equality comparison|@["properties.my_subproperty", "=", "fizzy whizy sparkle pop"]@|
+|@<@, @<=@, @>=@, @>@|string or number|Ordering comparison|@["properties.my_subproperty", "<", 3]@|
+|@like@, @ilike@|string|SQL pattern match, single character match is @_@ and wildcard is @%@, ilike is case-insensitive|@["properties.my_subproperty", "like", "d00220fb%"]@|
+|@in@, @not in@|array of strings|Set membership|@["properties.my_subproperty", "in", ["fizz", "buzz"]]@|
+|@exists@|boolean|Test if a subproperty is present or not (determined by operand).|@["properties.my_subproperty", "exists", true]@|
+
+Note that exclusion filters @!=@ and @not in@ will return records for which the property is not defined at all.  To restrict filtering to records on which the subproperty is defined, combine with an @exists@ filter.
 
 h3. Results of list method
 
diff --git a/services/api/db/migrate/20171212153352_add_gin_index_to_collection_properties.rb b/services/api/db/migrate/20171212153352_add_gin_index_to_collection_properties.rb
new file mode 100644 (file)
index 0000000..ce2403e
--- /dev/null
@@ -0,0 +1,8 @@
+class AddGinIndexToCollectionProperties < ActiveRecord::Migration
+  def up
+    ActiveRecord::Base.connection.execute("CREATE INDEX collection_index_on_properties ON collections USING gin (properties);")
+  end
+  def down
+    ActiveRecord::Base.connection.execute("DROP INDEX collection_index_on_properties")
+  end
+end
index 60fd88a98bb243c8ee11c99e1b18d90e4f0c0ca3..14729d31bc91a558dbead7de381e80f85ffb0cfe 100644 (file)
@@ -1605,6 +1605,13 @@ CREATE INDEX api_clients_search_index ON api_clients USING btree (uuid, owner_uu
 CREATE INDEX authorized_keys_search_index ON authorized_keys USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, name, key_type, authorized_user_uuid);
 
 
+--
+-- Name: collection_index_on_properties; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX collection_index_on_properties ON collections USING gin (properties);
+
+
 --
 -- Name: collections_full_text_search_idx; Type: INDEX; Schema: public; Owner: -
 --
@@ -3039,3 +3046,6 @@ INSERT INTO schema_migrations (version) VALUES ('20170906224040');
 INSERT INTO schema_migrations (version) VALUES ('20171027183824');
 
 INSERT INTO schema_migrations (version) VALUES ('20171208203841');
+
+INSERT INTO schema_migrations (version) VALUES ('20171212153352');
+
index eb8d09b74c88e118af371f35ab58d15bb1a704b7..dc427c12c1f82cfc76d8b53a13ad1d7b8a88c032 100644 (file)
@@ -9,6 +9,9 @@
 #   model_class
 # Operates on:
 #   @objects
+
+require 'safe_json'
+
 module RecordFilters
 
   # Input:
@@ -58,80 +61,152 @@ module RecordFilters
         param_out << operand.split.join(' & ')
       end
       attrs.each do |attr|
-        if !model_class.searchable_columns(operator).index attr.to_s
-          raise ArgumentError.new("Invalid attribute '#{attr}' in filter")
-        end
-        case operator.downcase
-        when '=', '<', '<=', '>', '>=', '!=', 'like', 'ilike'
-          attr_type = model_class.attribute_column(attr).type
-          operator = '<>' if operator == '!='
-          if operand.is_a? String
-            if attr_type == :boolean
-              if not ['=', '<>'].include?(operator)
-                raise ArgumentError.new("Invalid operator '#{operator}' for " \
-                                        "boolean attribute '#{attr}'")
-              end
-              case operand.downcase
-              when '1', 't', 'true', 'y', 'yes'
-                operand = true
-              when '0', 'f', 'false', 'n', 'no'
-                operand = false
-              else
-                raise ArgumentError("Invalid operand '#{operand}' for " \
-                                    "boolean attribute '#{attr}'")
+        subproperty = attr.split(".", 2)
+
+        col = model_class.columns.select { |c| c.name == subproperty[0] }.first
+
+        if subproperty.length == 2
+          if col.nil? or col.type != :jsonb
+            raise ArgumentError.new("Invalid attribute '#{subproperty[0]}' for subproperty filter")
+          end
+
+          if subproperty[1][0] == "<" and subproperty[1][-1] == ">"
+            subproperty[1] = subproperty[1][1..-2]
+          end
+
+        # jsonb search
+          case operator.downcase
+          when '=', '!='
+            not_in = if operator.downcase == "!=" then "NOT " else "" end
+            cond_out << "#{not_in}(#{ar_table_name}.#{subproperty[0]} @> ?::jsonb)"
+            param_out << SafeJSON.dump({subproperty[1] => operand})
+          when 'in'
+            if operand.is_a? Array
+              operand.each do |opr|
+                cond_out << "#{ar_table_name}.#{subproperty[0]} @> ?::jsonb"
+                param_out << SafeJSON.dump({subproperty[1] => opr})
               end
-            end
-            if operator == '<>'
-              # explicitly allow NULL
-              cond_out << "#{ar_table_name}.#{attr} #{operator} ? OR #{ar_table_name}.#{attr} IS NULL"
             else
-              cond_out << "#{ar_table_name}.#{attr} #{operator} ?"
-            end
-            if (# any operator that operates on value rather than
-                # representation:
-                operator.match(/[<=>]/) and (attr_type == :datetime))
-              operand = Time.parse operand
+              raise ArgumentError.new("Invalid operand type '#{operand.class}' "\
+                                      "for '#{operator}' operator in filters")
             end
+          when '<', '<=', '>', '>='
+            cond_out << "#{ar_table_name}.#{subproperty[0]}->? #{operator} ?::jsonb"
+            param_out << subproperty[1]
+            param_out << SafeJSON.dump(operand)
+          when 'like', 'ilike'
+            cond_out << "#{ar_table_name}.#{subproperty[0]}->>? #{operator} ?"
+            param_out << subproperty[1]
             param_out << operand
-          elsif operand.nil? and operator == '='
-            cond_out << "#{ar_table_name}.#{attr} is null"
-          elsif operand.nil? and operator == '<>'
-            cond_out << "#{ar_table_name}.#{attr} is not null"
-          elsif (attr_type == :boolean) and ['=', '<>'].include?(operator) and
-              [true, false].include?(operand)
-            cond_out << "#{ar_table_name}.#{attr} #{operator} ?"
-            param_out << operand
+          when 'not in'
+            if operand.is_a? Array
+              cond_out << "#{ar_table_name}.#{subproperty[0]}->>? NOT IN (?) OR #{ar_table_name}.#{subproperty[0]}->>? IS NULL"
+              param_out << subproperty[1]
+              param_out << operand
+              param_out << subproperty[1]
+            else
+              raise ArgumentError.new("Invalid operand type '#{operand.class}' "\
+                                      "for '#{operator}' operator in filters")
+            end
+          when 'exists'
+          if operand == true
+            cond_out << "jsonb_exists(#{ar_table_name}.#{subproperty[0]}, ?)"
+          elsif operand == false
+            cond_out << "(NOT jsonb_exists(#{ar_table_name}.#{subproperty[0]}, ?)) OR #{ar_table_name}.#{subproperty[0]} is NULL"
           else
-            raise ArgumentError.new("Invalid operand type '#{operand.class}' "\
-                                    "for '#{operator}' operator in filters")
+            raise ArgumentError.new("Invalid operand '#{operand}' for '#{operator}' must be true or false")
           end
-        when 'in', 'not in'
-          if operand.is_a? Array
-            cond_out << "#{ar_table_name}.#{attr} #{operator} (?)"
-            param_out << operand
-            if operator == 'not in' and not operand.include?(nil)
-              # explicitly allow NULL
-              cond_out[-1] = "(#{cond_out[-1]} OR #{ar_table_name}.#{attr} IS NULL)"
-            end
+          param_out << subproperty[1]
           else
-            raise ArgumentError.new("Invalid operand type '#{operand.class}' "\
-                                    "for '#{operator}' operator in filters")
+            raise ArgumentError.new("Invalid operator for subproperty search '#{operator}'")
           end
-        when 'is_a'
-          operand = [operand] unless operand.is_a? Array
-          cond = []
-          operand.each do |op|
-            cl = ArvadosModel::kind_class op
-            if cl
-              cond << "#{ar_table_name}.#{attr} like ?"
-              param_out << cl.uuid_like_pattern
+        elsif operator.downcase == "exists"
+          if col.type != :jsonb
+            raise ArgumentError.new("Invalid attribute '#{subproperty[0]}' for operator '#{operator}' in filter")
+          end
+
+          cond_out << "jsonb_exists(#{ar_table_name}.#{subproperty[0]}, ?)"
+          param_out << operand
+        else
+          if !model_class.searchable_columns(operator).index subproperty[0]
+            raise ArgumentError.new("Invalid attribute '#{subproperty[0]}' in filter")
+          end
+
+          case operator.downcase
+          when '=', '<', '<=', '>', '>=', '!=', 'like', 'ilike'
+            attr_type = model_class.attribute_column(attr).type
+            operator = '<>' if operator == '!='
+            if operand.is_a? String
+              if attr_type == :boolean
+                if not ['=', '<>'].include?(operator)
+                  raise ArgumentError.new("Invalid operator '#{operator}' for " \
+                                          "boolean attribute '#{attr}'")
+                end
+                case operand.downcase
+                when '1', 't', 'true', 'y', 'yes'
+                  operand = true
+                when '0', 'f', 'false', 'n', 'no'
+                  operand = false
+                else
+                  raise ArgumentError("Invalid operand '#{operand}' for " \
+                                      "boolean attribute '#{attr}'")
+                end
+              end
+              if operator == '<>'
+                # explicitly allow NULL
+                cond_out << "#{ar_table_name}.#{attr} #{operator} ? OR #{ar_table_name}.#{attr} IS NULL"
+              else
+                cond_out << "#{ar_table_name}.#{attr} #{operator} ?"
+              end
+              if (# any operator that operates on value rather than
+                # representation:
+                operator.match(/[<=>]/) and (attr_type == :datetime))
+                operand = Time.parse operand
+              end
+              param_out << operand
+            elsif operand.nil? and operator == '='
+              cond_out << "#{ar_table_name}.#{attr} is null"
+            elsif operand.nil? and operator == '<>'
+              cond_out << "#{ar_table_name}.#{attr} is not null"
+            elsif (attr_type == :boolean) and ['=', '<>'].include?(operator) and
+                 [true, false].include?(operand)
+              cond_out << "#{ar_table_name}.#{attr} #{operator} ?"
+              param_out << operand
+            elsif (attr_type == :integer)
+              cond_out << "#{ar_table_name}.#{attr} #{operator} ?"
+              param_out << operand
+            else
+              raise ArgumentError.new("Invalid operand type '#{operand.class}' "\
+                                      "for '#{operator}' operator in filters")
+            end
+          when 'in', 'not in'
+            if operand.is_a? Array
+              cond_out << "#{ar_table_name}.#{attr} #{operator} (?)"
+              param_out << operand
+              if operator == 'not in' and not operand.include?(nil)
+                # explicitly allow NULL
+                cond_out[-1] = "(#{cond_out[-1]} OR #{ar_table_name}.#{attr} IS NULL)"
+              end
             else
-              cond << "1=0"
+              raise ArgumentError.new("Invalid operand type '#{operand.class}' "\
+                                      "for '#{operator}' operator in filters")
             end
+          when 'is_a'
+            operand = [operand] unless operand.is_a? Array
+            cond = []
+            operand.each do |op|
+              cl = ArvadosModel::kind_class op
+              if cl
+                cond << "#{ar_table_name}.#{attr} like ?"
+                param_out << cl.uuid_like_pattern
+              else
+                cond << "1=0"
+              end
+            end
+            cond_out << cond.join(' OR ')
+          else
+            raise ArgumentError.new("Invalid operator '#{operator}'")
           end
-          cond_out << cond.join(' OR ')
-        else
-          raise ArgumentError.new("Invalid operator '#{operator}'")
         end
       end
       conds_out << cond_out.join(' OR ') if cond_out.any?
index 802350320107b4df069ba6bea969efca50531475..ea87cca36fb5d266ad1bf7a93ae690e5783f6747 100644 (file)
@@ -715,6 +715,104 @@ collection_in_trashed_subproject:
   manifest_text: ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:file1 0:0:file2\n"
   name: collection in trashed subproject
 
+collection_with_prop1_value1:
+  uuid: zzzzz-4zz18-withprop1value1
+  portable_data_hash: fa7aeb5140e2848d39b416daeef4ffc5+45
+  owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+  created_at: 2015-02-13T17:22:54Z
+  modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+  modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+  modified_at: 2015-02-13T17:22:54Z
+  updated_at: 2015-02-13T17:22:54Z
+  manifest_text: ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"
+  name: collection with prop1 value1
+  properties:
+    prop1: value1
+
+collection_with_prop1_value2:
+  uuid: zzzzz-4zz18-withprop1value2
+  portable_data_hash: fa7aeb5140e2848d39b416daeef4ffc5+45
+  owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+  created_at: 2015-02-13T17:22:54Z
+  modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+  modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+  modified_at: 2015-02-13T17:22:54Z
+  updated_at: 2015-02-13T17:22:54Z
+  manifest_text: ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"
+  name: collection with prop1 value2
+  properties:
+    prop1: value2
+
+collection_with_prop1_value3:
+  uuid: zzzzz-4zz18-withprop1value3
+  portable_data_hash: fa7aeb5140e2848d39b416daeef4ffc5+45
+  owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+  created_at: 2015-02-13T17:22:54Z
+  modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+  modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+  modified_at: 2015-02-13T17:22:54Z
+  updated_at: 2015-02-13T17:22:54Z
+  manifest_text: ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"
+  name: collection with prop1 value3
+  properties:
+    prop1: value3
+
+collection_with_prop1_other1:
+  uuid: zzzzz-4zz18-withprop1other1
+  portable_data_hash: fa7aeb5140e2848d39b416daeef4ffc5+45
+  owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+  created_at: 2015-02-13T17:22:54Z
+  modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+  modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+  modified_at: 2015-02-13T17:22:54Z
+  updated_at: 2015-02-13T17:22:54Z
+  manifest_text: ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"
+  name: collection with prop1 other1
+  properties:
+    prop1: other1
+
+collection_with_prop2_1:
+  uuid: zzzzz-4zz18-withprop2value1
+  portable_data_hash: fa7aeb5140e2848d39b416daeef4ffc5+45
+  owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+  created_at: 2015-02-13T17:22:54Z
+  modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+  modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+  modified_at: 2015-02-13T17:22:54Z
+  updated_at: 2015-02-13T17:22:54Z
+  manifest_text: ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"
+  name: collection with prop1 1
+  properties:
+    prop2: 1
+
+collection_with_prop2_5:
+  uuid: zzzzz-4zz18-withprop2value5
+  portable_data_hash: fa7aeb5140e2848d39b416daeef4ffc5+45
+  owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+  created_at: 2015-02-13T17:22:54Z
+  modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+  modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+  modified_at: 2015-02-13T17:22:54Z
+  updated_at: 2015-02-13T17:22:54Z
+  manifest_text: ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"
+  name: collection with prop1 5
+  properties:
+    prop2: 5
+
+collection_with_uri_prop:
+  uuid: zzzzz-4zz18-withuripropval1
+  portable_data_hash: fa7aeb5140e2848d39b416daeef4ffc5+45
+  owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+  created_at: 2015-02-13T17:22:54Z
+  modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+  modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+  modified_at: 2015-02-13T17:22:54Z
+  updated_at: 2015-02-13T17:22:54Z
+  manifest_text: ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"
+  name: collection with RDF-style URI property key
+  properties:
+    "http://schema.org/example": "value1"
+
 # Test Helper trims the rest of the file
 
 # Do not add your fixtures below this line as the rest of this file will be trimmed by test_helper
index 2c7427cba824b7346577cdd6d3cd6791afebcade..ef120b1ca86b79ff54b028fc3c1cf6c5a5e6806a 100644 (file)
@@ -151,4 +151,132 @@ class Arvados::V1::FiltersTest < ActionController::TestCase
     assert_equal all_objects['arvados#pipelineInstance'], second_page['arvados#pipelineInstance']+5
     assert_equal true, second_page['arvados#pipelineTemplate']>0
   end
+
+  [['prop1', '=', 'value1', [:collection_with_prop1_value1], [:collection_with_prop1_value2, :collection_with_prop2_1]],
+   ['prop1', '!=', 'value1', [:collection_with_prop1_value2, :collection_with_prop2_1], [:collection_with_prop1_value1]],
+   ['prop1', 'exists', true, [:collection_with_prop1_value1, :collection_with_prop1_value2, :collection_with_prop1_value3, :collection_with_prop1_other1], [:collection_with_prop2_1]],
+   ['prop1', 'exists', false, [:collection_with_prop2_1], [:collection_with_prop1_value1, :collection_with_prop1_value2, :collection_with_prop1_value3, :collection_with_prop1_other1]],
+   ['prop1', 'in', ['value1', 'value2'], [:collection_with_prop1_value1, :collection_with_prop1_value2], [:collection_with_prop1_value3, :collection_with_prop2_1]],
+   ['prop1', 'in', ['value1', 'valueX'], [:collection_with_prop1_value1], [:collection_with_prop1_value3, :collection_with_prop2_1]],
+   ['prop1', 'not in', ['value1', 'value2'], [:collection_with_prop1_value3, :collection_with_prop1_other1, :collection_with_prop2_1], [:collection_with_prop1_value1, :collection_with_prop1_value2]],
+   ['prop1', 'not in', ['value1', 'valueX'], [:collection_with_prop1_value2, :collection_with_prop1_value3, :collection_with_prop1_other1, :collection_with_prop2_1], [:collection_with_prop1_value1]],
+   ['prop1', '>', 'value2', [:collection_with_prop1_value3], [:collection_with_prop1_other1, :collection_with_prop1_value1]],
+   ['prop1', '<', 'value2', [:collection_with_prop1_other1, :collection_with_prop1_value1], [:collection_with_prop1_value2, :collection_with_prop1_value2]],
+   ['prop1', '<=', 'value2', [:collection_with_prop1_other1, :collection_with_prop1_value1, :collection_with_prop1_value2], [:collection_with_prop1_value3]],
+   ['prop1', '>=', 'value2', [:collection_with_prop1_value2, :collection_with_prop1_value3], [:collection_with_prop1_other1, :collection_with_prop1_value1]],
+   ['prop1', 'like', 'value%', [:collection_with_prop1_value1, :collection_with_prop1_value2, :collection_with_prop1_value3], [:collection_with_prop1_other1]],
+   ['prop1', 'like', '%1', [:collection_with_prop1_value1, :collection_with_prop1_other1], [:collection_with_prop1_value2, :collection_with_prop1_value3]],
+   ['prop1', 'ilike', 'VALUE%', [:collection_with_prop1_value1, :collection_with_prop1_value2, :collection_with_prop1_value3], [:collection_with_prop1_other1]],
+   ['prop2', '>',  1, [:collection_with_prop2_5], [:collection_with_prop2_1]],
+   ['prop2', '<',  5, [:collection_with_prop2_1], [:collection_with_prop2_5]],
+   ['prop2', '<=', 5, [:collection_with_prop2_1, :collection_with_prop2_5], []],
+   ['prop2', '>=', 1, [:collection_with_prop2_1, :collection_with_prop2_5], []],
+   ['<http://schema.org/example>', '=', "value1", [:collection_with_uri_prop], []],
+  ].each do |prop, op, opr, inc, ex|
+    test "jsonb filter properties.#{prop} #{op} #{opr})" do
+      @controller = Arvados::V1::CollectionsController.new
+      authorize_with :admin
+      get :index, {
+            filters: SafeJSON.dump([ ["properties.#{prop}", op, opr] ]),
+            limit: 1000
+          }
+      assert_response :success
+      found = assigns(:objects).collect(&:uuid)
+
+      inc.each do |i|
+        assert_includes(found, collections(i).uuid)
+      end
+
+      ex.each do |e|
+        assert_not_includes(found, collections(e).uuid)
+      end
+    end
+  end
+
+  test "jsonb 'exists' and '!=' filter" do
+    @controller = Arvados::V1::CollectionsController.new
+    authorize_with :admin
+    get :index, {
+      filters: [ ['properties.prop1', 'exists', true], ['properties.prop1', '!=', 'value1'] ]
+    }
+    assert_response :success
+    found = assigns(:objects).collect(&:uuid)
+    assert_equal found.length, 3
+    assert_not_includes(found, collections(:collection_with_prop1_value1).uuid)
+    assert_includes(found, collections(:collection_with_prop1_value2).uuid)
+    assert_includes(found, collections(:collection_with_prop1_value3).uuid)
+    assert_includes(found, collections(:collection_with_prop1_other1).uuid)
+  end
+
+  test "jsonb alternate form 'exists' and '!=' filter" do
+    @controller = Arvados::V1::CollectionsController.new
+    authorize_with :admin
+    get :index, {
+      filters: [ ['properties', 'exists', 'prop1'], ['properties.prop1', '!=', 'value1'] ]
+    }
+    assert_response :success
+    found = assigns(:objects).collect(&:uuid)
+    assert_equal found.length, 3
+    assert_not_includes(found, collections(:collection_with_prop1_value1).uuid)
+    assert_includes(found, collections(:collection_with_prop1_value2).uuid)
+    assert_includes(found, collections(:collection_with_prop1_value3).uuid)
+    assert_includes(found, collections(:collection_with_prop1_other1).uuid)
+  end
+
+  test "jsonb 'exists' must be boolean" do
+    @controller = Arvados::V1::CollectionsController.new
+    authorize_with :admin
+    get :index, {
+      filters: [ ['properties.prop1', 'exists', nil] ]
+    }
+    assert_response 422
+    assert_match(/Invalid operand '' for 'exists' must be true or false/,
+                 json_response['errors'].join(' '))
+  end
+
+  test "jsonb checks column exists" do
+    @controller = Arvados::V1::CollectionsController.new
+    authorize_with :admin
+    get :index, {
+      filters: [ ['puppies.prop1', '=', 'value1'] ]
+    }
+    assert_response 422
+    assert_match(/Invalid attribute 'puppies' for subproperty filter/,
+                 json_response['errors'].join(' '))
+  end
+
+  test "jsonb checks column is valid" do
+    @controller = Arvados::V1::CollectionsController.new
+    authorize_with :admin
+    get :index, {
+      filters: [ ['name.prop1', '=', 'value1'] ]
+    }
+    assert_response 422
+    assert_match(/Invalid attribute 'name' for subproperty filter/,
+                 json_response['errors'].join(' '))
+  end
+
+  test "jsonb invalid operator" do
+    @controller = Arvados::V1::CollectionsController.new
+    authorize_with :admin
+    get :index, {
+      filters: [ ['properties.prop1', '###', 'value1'] ]
+    }
+    assert_response 422
+    assert_match(/Invalid operator for subproperty search '###'/,
+                 json_response['errors'].join(' '))
+  end
+
+  test "replication_desired = 2" do
+    @controller = Arvados::V1::CollectionsController.new
+    authorize_with :admin
+    get :index, {
+      filters: SafeJSON.dump([ ['replication_desired', '=', 2] ])
+    }
+    assert_response :success
+    found = assigns(:objects).collect(&:uuid)
+    assert_includes(found, collections(:replication_desired_2_unconfirmed).uuid)
+    assert_includes(found, collections(:replication_desired_2_confirmed_2).uuid)
+  end
+
 end