table(table table-bordered table-condensed).
|_. Operator|_. Operand type|_. Description|_. Example|
- |@=@, @!=@|string, number, timestamp, or null|Equality comparison|@["tail_uuid","=","xyzzy-j7d0g-fffffffffffffff"]@ @["tail_uuid","!=",null]@|
+ |@=@, @!=@, @<>@|string, number, timestamp, JSON-encoded array, JSON-encoded object, or null|Equality comparison|@["tail_uuid","=","xyzzy-j7d0g-fffffffffffffff"]@
+ @["tail_uuid","!=",null]@
+ @["storage_classes_desired","=","[\"default\"]"]@|
|@<@, @<=@, @>=@, @>@|string, number, or timestamp|Ordering comparison|@["script_version",">","123"]@|
|@like@, @ilike@|string|SQL pattern match. Single character match is @_@ and wildcard is @%@. The @ilike@ operator is case-insensitive|@["script_version","like","d00220fb%"]@|
|@in@, @not in@|array of strings|Set membership|@["script_version","in",["main","d00220fb38d4b85ca8fc28a8151702a2b9d1dec5"]]@|
|@is_a@|string|Arvados object type|@["head_uuid","is_a","arvados#collection"]@|
- |@exists@|string|Test if a subproperty is present.|@["properties","exists","my_subproperty"]@|
+ |@exists@|string|Presence of subproperty|@["properties","exists","my_subproperty"]@|
+ |@contains@|string, array of strings|Presence of one or more keys or array elements|@["storage_classes_desired", "contains", ["foo", "bar"]]@ (matches both @["foo", "bar"]@ and @["foo", "bar", "baz"]@)
+ (note @[..., "contains", "foo"]@ is also accepted, and is equivalent to @[..., "contains", ["foo"]]@)|
+h4(#filterexpression). Filtering using boolean expressions
+
+In place of an attribute, the first element of the three-element filter array can be a boolean expression. The following restrictions apply:
+* The expression must contain exactly one operator.
+* The operator must be @=@, @<@, @<=@, @>@, or @>=@.
+* There must be exactly one pair of parentheses, surrounding the entire expression.
+* Each operand must be the name of a numeric attribute like @replication_desired@ (literal values like @3@ and non-numeric attributes like @uuid@ are not accepted).
+* The expression must not contain whitespace other than an ASCII space (newline and tab characters are not accepted).
+* The second and third elements of the filter array must be @"="@ and @true@ respectively.
+
+Examples:
+* @["(replication_desired < replication_confirmed)", "=", true]@
+* @["(replication_desired = replication_confirmed)", "=", true]@
+
h4(#substringsearchfilter). Filtering using substring search
Resources can also be filtered by searching for a substring in attributes of type @string@, @array of strings@, @text@, and @hash@, which are indexed in the database specifically for search. To use substring search, the filter must:
|@like@, @ilike@|string|SQL pattern match, single character match is @_@ and wildcard is @%@, ilike is case-insensitive|@["properties.my_subproperty", "like", "d00220fb%"]@|
|@in@, @not in@|array of strings|Set membership|@["properties.my_subproperty", "in", ["fizz", "buzz"]]@|
|@exists@|boolean|Test if a subproperty is present or not (determined by operand).|@["properties.my_subproperty", "exists", true]@|
- |@contains@|string, number|Filter where subproperty has a value either by exact match or value is element of subproperty list.|@["foo", "contains", "bar"]@ will find both @{"foo": "bar"}@ and @{"foo": ["bar", "baz"]}@.|
+ |@contains@|string, number|Filter where subproperty has a value either by exact match or value is element of subproperty list.|@["properties.foo", "contains", "bar"]@ will find both @{"foo": "bar"}@ and @{"foo": ["bar", "baz"]}@.|
Note that exclusion filters @!=@ and @not in@ will return records for which the property is not defined at all. To restrict filtering to records on which the subproperty is defined, combine with an @exists@ filter.
raise ArgumentError.new("Invalid operator '#{operator}' (#{operator.class}) in filter")
end
+ operator = operator.downcase
cond_out = []
- if attrs_in == 'any' && (operator.casecmp('ilike').zero? || operator.casecmp('like').zero?) && (operand.is_a? String) && operand.match('^[%].*[%]$')
+ if attrs_in == 'any' && (operator == 'ilike' || operator == 'like') && (operand.is_a? String) && operand.match('^[%].*[%]$')
# Trigram index search
cond_out << model_class.full_text_trgm + " #{operator} ?"
param_out << operand
end
# jsonb search
- case operator.downcase
+ case operator
when '=', '!='
- not_in = if operator.downcase == "!=" then "NOT " else "" end
+ not_in = if operator == "!=" then "NOT " else "" end
cond_out << "#{not_in}(#{attr_table_name}.#{attr} @> ?::jsonb)"
param_out << SafeJSON.dump({proppath => operand})
when 'in'
else
raise ArgumentError.new("Invalid operator for subproperty search '#{operator}'")
end
- elsif operator.downcase == "exists"
+ elsif operator == "exists"
if col.type != :jsonb
raise ArgumentError.new("Invalid attribute '#{attr}' for operator '#{operator}' in filter")
end
cond_out << "jsonb_exists(#{attr_table_name}.#{attr}, ?)"
param_out << operand
+ elsif expr = /^ *\( *(\w+) *(<=?|>=?|=) *(\w+) *\) *$/.match(attr)
+ if operator != '=' || ![true,"true"].index(operand)
+ raise ArgumentError.new("Invalid expression filter '#{attr}': subsequent elements must be [\"=\", true]")
+ end
+ operator = expr[2]
+ attr1, attr2 = expr[1], expr[3]
+ allowed = attr_model_class.searchable_columns(operator)
+ [attr1, attr2].each do |tok|
+ if !allowed.index(tok)
+ raise ArgumentError.new("Invalid attribute in expression: '#{tok}'")
+ end
+ col = attr_model_class.columns.select { |c| c.name == tok }.first
+ if col.type != :integer
+ raise ArgumentError.new("Non-numeric attribute in expression: '#{tok}'")
+ end
+ end
+ cond_out << "#{attr1} #{operator} #{attr2}"
else
- if !attr_model_class.searchable_columns(operator).index attr
+ if !attr_model_class.searchable_columns(operator).index(attr) &&
+ !(col.andand.type == :jsonb && ['contains', '=', '<>', '!='].index(operator))
raise ArgumentError.new("Invalid attribute '#{attr}' in filter")
end
- case operator.downcase
+ case operator
when '=', '<', '<=', '>', '>=', '!=', 'like', 'ilike'
attr_type = attr_model_class.attribute_column(attr).type
operator = '<>' if operator == '!='
end
end
cond_out << cond.join(' OR ')
+ when 'contains'
+ if col.andand.type != :jsonb
+ raise ArgumentError.new("Invalid attribute '#{attr}' for '#{operator}' operator")
+ end
+ if operand == []
+ raise ArgumentError.new("Invalid operand '#{operand.inspect}' for '#{operator}' operator")
+ end
+ operand = [operand] unless operand.is_a? Array
+ operand.each do |op|
+ if !op.is_a?(String)
+ raise ArgumentError.new("Invalid element #{operand.inspect} in operand for #{operator.inspect} operator (operand must be a string or array of strings)")
+ end
+ end
+ # We use jsonb_exists_all(a,b) instead of "a ?& b" because
+ # the pg gem thinks "?" is a bind var. And we use string
+ # interpolation instead of param_out because the pg gem
+ # flattens param_out / doesn't support passing arrays as
+ # bind vars.
+ q = operand.map { |s| ActiveRecord::Base.connection.quote(s) }.join(',')
+ cond_out << "jsonb_exists_all(#{attr_table_name}.#{attr}, array[#{q}])"
else
raise ArgumentError.new("Invalid operator '#{operator}'")
end
end
end
- def assert_unsigned_manifest resp, label=''
- txt = resp['unsigned_manifest_text']
+ def assert_unsigned_manifest txt, label=''
assert_not_nil(txt, "#{label} unsigned_manifest_text was nil")
locs = 0
txt.scan(/ [[:xdigit:]]{32}\S*/) do |tok|
"past version not included on index")
end
- test "collections.get returns signed locators, and no unsigned_manifest_text" do
+ test "collections.get returns unsigned locators, and no unsigned_manifest_text" do
permit_unsigned_manifests
authorize_with :active
get :show, params: {id: collections(:foo_file).uuid}
assert_response :success
- assert_signed_manifest json_response['manifest_text'], 'foo_file'
+ assert_unsigned_manifest json_response["manifest_text"], 'foo_file'
refute_includes json_response, 'unsigned_manifest_text'
end
['v1token', 'v2token'].each do |token_method|
- test "correct signatures are given for #{token_method}" do
- token = api_client_authorizations(:active).send(token_method)
- authorize_with_token token
- get :show, params: {id: collections(:foo_file).uuid}
- assert_response :success
- assert_signed_manifest json_response['manifest_text'], 'foo_file', token: token
- end
-
test "signatures with #{token_method} are accepted" do
token = api_client_authorizations(:active).send(token_method)
signed = Blob.sign_locator(
},
}
assert_response :success
- assert_signed_manifest json_response['manifest_text'], 'updated', token: token
+ assert_unsigned_manifest json_response['manifest_text'], 'updated'
end
end
- test "index with manifest_text selected returns signed locators" do
+ test "index with manifest_text selected returns unsigned locators" do
columns = %w(uuid owner_uuid manifest_text)
authorize_with :active
get :index, params: {select: columns}
json_response["items"].each do |coll|
assert_equal(coll.keys - ['kind'], columns,
"Collections index did not respect selected columns")
- assert_signed_manifest coll['manifest_text'], coll['uuid']
+ assert_unsigned_manifest coll['manifest_text'], coll['uuid']
end
end
json_response["items"].each do |coll|
assert_equal(coll.keys - ['kind'], ['unsigned_manifest_text'],
"Collections index did not respect selected columns")
- locs += assert_unsigned_manifest coll, coll['uuid']
+ assert_nil coll['manifest_text']
+ locs += assert_unsigned_manifest coll['unsigned_manifest_text'], coll['uuid']
end
assert_operator locs, :>, 0, "no locators found in any manifests"
end
assert_not_nil assigns(:object)
resp = assigns(:object)
assert_equal foo_collection[:portable_data_hash], resp[:portable_data_hash]
- assert_signed_manifest resp[:manifest_text]
+ assert_unsigned_manifest resp[:manifest_text]
# The manifest in the response will have had permission hints added.
# Remove any permission hints in the response before comparing it to the source.
authorize_with :active
manifest_text = ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:0:foo.txt\n"
if !unsigned
- manifest_text = Collection.sign_manifest manifest_text, api_token(:active)
+ manifest_text = Collection.sign_manifest_only_for_tests manifest_text, api_token(:active)
end
post :create, params: {
collection: {
assert_not_nil assigns(:object)
resp = JSON.parse(@response.body)
assert_equal manifest_uuid, resp['portable_data_hash']
- # All of the locators in the output must be signed.
+ # All of the signatures in the output must be valid.
resp['manifest_text'].lines.each do |entry|
m = /([[:xdigit:]]{32}\+\S+)/.match(entry)
- if m
+ if m && m[0].index('+A')
assert Blob.verify_signature m[0], signing_opts
end
end
assert_not_nil assigns(:object)
resp = JSON.parse(@response.body)
assert_equal manifest_uuid, resp['portable_data_hash']
- # All of the locators in the output must be signed.
+ # All of the signatures in the output must be valid.
resp['manifest_text'].lines.each do |entry|
m = /([[:xdigit:]]{32}\+\S+)/.match(entry)
- if m
+ if m && m[0].index('+A')
assert Blob.verify_signature m[0], signing_opts
end
end
assert_equal manifest_text, stripped_manifest
end
- test "multiple signed locators per line" do
- permit_unsigned_manifests
- authorize_with :active
- locators = %w(
- d41d8cd98f00b204e9800998ecf8427e+0
- acbd18db4cc2f85cedef654fccc4a4d8+3
- ea10d51bcf88862dbcc36eb292017dfd+45)
-
- signing_opts = {
- key: Rails.configuration.Collections.BlobSigningKey,
- api_token: api_token(:active),
- }
-
- unsigned_manifest = [".", *locators, "0:0:foo.txt\n"].join(" ")
- manifest_uuid = Digest::MD5.hexdigest(unsigned_manifest) +
- '+' +
- unsigned_manifest.length.to_s
-
- signed_locators = locators.map { |loc| Blob.sign_locator loc, signing_opts }
- signed_manifest = [".", *signed_locators, "0:0:foo.txt\n"].join(" ")
-
- post :create, params: {
- collection: {
- manifest_text: signed_manifest,
- portable_data_hash: manifest_uuid,
- }
- }
- assert_response :success
- assert_not_nil assigns(:object)
- resp = JSON.parse(@response.body)
- assert_equal manifest_uuid, resp['portable_data_hash']
- # All of the locators in the output must be signed.
- # Each line is of the form "path locator locator ... 0:0:file.txt"
- # entry.split[1..-2] will yield just the tokens in the middle of the line
- returned_locator_count = 0
- resp['manifest_text'].lines.each do |entry|
- entry.split[1..-2].each do |tok|
- returned_locator_count += 1
- assert Blob.verify_signature tok, signing_opts
- end
- end
- assert_equal locators.count, returned_locator_count
- end
-
test 'Reject manifest with unsigned blob' do
permit_unsigned_manifests false
authorize_with :active
assert_equal col.version, json_response['version'], 'Trashing a collection should not create a new version'
end
+ [['<', :<],
+ ['<=', :<=],
+ ['>', :>],
+ ['>=', :>=],
+ ['=', :==]].each do |op, rubyop|
+ test "filter collections by replication_desired #{op} replication_confirmed" do
+ authorize_with(:active)
+ get :index, params: {
+ filters: [["(replication_desired #{op} replication_confirmed)", "=", true]],
+ }
+ assert_response :success
+ json_response["items"].each do |c|
+ assert_operator(c["replication_desired"], rubyop, c["replication_confirmed"])
+ end
+ end
+ end
+
+ ["(replication_desired < bogus)",
+ "replication_desired < replication_confirmed",
+ "(replication_desired < replication_confirmed",
+ "(replication_desired ! replication_confirmed)",
+ "(replication_desired <)",
+ "(replication_desired < manifest_text)",
+ "(manifest_text < manifest_text)", # currently only numeric attrs are supported
+ "(replication_desired < 2)", # currently only attrs are supported, not literals
+ "(1 < 2)",
+ ].each do |expr|
+ test "invalid filter expression #{expr}" do
+ authorize_with(:active)
+ get :index, params: {
+ filters: [[expr, "=", true]],
+ }
+ assert_response 422
+ end
+ end
+
+ test "invalid op/arg with filter expression" do
+ authorize_with(:active)
+ get :index, params: {
+ filters: [["replication_desired < replication_confirmed", "!=", false]],
+ }
+ assert_response 422
+ end
++
+ ["storage_classes_desired", "storage_classes_confirmed"].each do |attr|
+ test "filter collections by #{attr}" do
+ authorize_with(:active)
+ get :index, params: {
+ filters: [[attr, "=", '["default"]']]
+ }
+ assert_response :success
+ assert_not_equal 0, json_response["items"].length
+ json_response["items"].each do |c|
+ assert_equal ["default"], c[attr]
+ end
+ end
+ end
++>>>>>>> main
end
projects / arvados.git / commitdiff