function updateFilterableQueryNow($target) {
var newquery = $target.data('filterable-query-new');
var params = $target.data('infinite-content-params-filterable') || {};
- params.filters = [['any', 'ilike', '%' + newquery + '%']];
+ if (newquery == null || newquery == '') {
+ params.filters = [];
+ } else {
+ params.filters = [['any', '@@', newquery.concat(':*')]];
+ }
$target.data('infinite-content-params-filterable', params);
$target.data('filterable-query', newquery);
}
# unused ?search=foo param to pre-populate the search field.
test 'no double-load if text input has a value at page load time' do
visit page_with_token('admin', '/pipeline_instances')
- assert_text 'pipeline_2'
- visit page_with_token('admin', '/pipeline_instances?search=pipeline_1')
+ assert_text 'pipeline_with_job'
+ visit page_with_token('admin', '/pipeline_instances?search=pipeline_with_tagged')
# Horrible hack to ensure the search results can't load correctly
# on the second attempt.
assert_selector '#recent-pipeline-instances'
assert page.evaluate_script('$("#recent-pipeline-instances[data-infinite-content-href0]").attr("data-infinite-content-href0","/give-me-an-error").length == 1')
# Wait for the first page of results to appear.
- assert_text 'pipeline_1'
+ assert_text 'pipeline_with_tagged_collection_input'
# Make sure the results are filtered.
- assert_no_text 'pipeline_2'
+ assert_no_text 'pipeline_with_job'
# Make sure pipeline_2 didn't disappear merely because the results
# were replaced with an error message.
- assert_text 'pipeline_1'
+ assert_text 'pipeline_with_tagged_collection_input'
end
end
end
test 'Create and run a pipeline' do
- visit page_with_token('active_trustedclient')
+ visit page_with_token('active')
visit '/pipeline_templates'
within('tr', text: 'Two Part Pipeline Template') do
# Create a pipeline instance from within a project and run
test 'Create pipeline inside a project and run' do
- visit page_with_token('active_trustedclient')
+ visit page_with_token('active')
# Add this collection to the project using collections menu from top nav
visit '/projects'
end
test 'view pipeline with job and see graph' do
- visit page_with_token('active_trustedclient')
-
- visit '/pipeline_instances'
+ visit page_with_token('active_trustedclient', '/pipeline_instances')
assert page.has_text? 'pipeline_with_job'
find('a', text: 'pipeline_with_job').click
end
test 'pipeline description' do
- visit page_with_token('active_trustedclient')
-
- visit '/pipeline_instances'
+ visit page_with_token('active_trustedclient', '/pipeline_instances')
assert page.has_text? 'pipeline_with_job'
find('a', text: 'pipeline_with_job').click
[
['fuse', nil, 2, 20], # has 2 as of 11-07-2014
- ['fuse', 'FUSE project', 1, 1], # 1 with this name
- ['user1_with_load', nil, 30, 100], # has 37 as of 11-07-2014
- ['user1_with_load', 'pipeline_10', 2, 2], # 2 with this name
- ['user1_with_load', '000010pipelines', 10, 10], # owned_by the project zzzzz-j7d0g-000010pipelines
['user1_with_load', '000025pipelines', 25, 25], # owned_by the project zzzzz-j7d0g-000025pipelines, two pages
- ['admin', nil, 40, 200],
- ['admin', 'FUSE project', 1, 1],
- ['admin', 'pipeline_10', 2, 2],
- ['active', 'containing at least two', 2, 100],
- ['active', nil, 10, 100],
+ ['admin', 'pipeline_20', 1, 1],
['active', 'no such match', 0, 0],
].each do |user, search_filter, expected_min, expected_max|
test "scroll pipeline instances page for #{user} with search filter #{search_filter}
attributes
end
+ def self.full_text_searchable_columns
+ self.columns.select do |col|
+ if col.type == :string or col.type == :text
+ true
+ end
+ end.map(&:name)
+ end
+
+ def self.full_text_tsvector
+ tsvector_str = "to_tsvector('english', "
+ first = true
+ self.full_text_searchable_columns.each do |column|
+ tsvector_str += " || ' ' || " if not first
+ tsvector_str += "coalesce(#{column},'')"
+ first = false
+ end
+ tsvector_str += ")"
+ end
+
protected
def ensure_ownership_path_leads_to_user
super - ["manifest_text"]
end
+ def self.full_text_searchable_columns
+ super - ["manifest_text"]
+ end
+
protected
def portable_manifest_text
portable_manifest = self[:manifest_text].dup
--- /dev/null
+class FullTextSearch < ActiveRecord::Migration
+
+ def up
+ execute "CREATE INDEX collections_full_text_search_idx ON collections USING gin(#{Collection.full_text_tsvector});"
+ execute "CREATE INDEX groups_full_text_search_idx ON groups USING gin(#{Group.full_text_tsvector});"
+ execute "CREATE INDEX jobs_full_text_search_idx ON jobs USING gin(#{Job.full_text_tsvector});"
+ execute "CREATE INDEX pipeline_instances_full_text_search_idx ON pipeline_instances USING gin(#{PipelineInstance.full_text_tsvector});"
+ execute "CREATE INDEX pipeline_templates_full_text_search_idx ON pipeline_templates USING gin(#{PipelineTemplate.full_text_tsvector});"
+ end
+
+ def down
+ remove_index :pipeline_templates, :name => 'pipeline_templates_full_text_search_idx'
+ remove_index :pipeline_instances, :name => 'pipeline_instances_full_text_search_idx'
+ remove_index :jobs, :name => 'jobs_full_text_search_idx'
+ remove_index :groups, :name => 'groups_full_text_search_idx'
+ remove_index :collections, :name => 'collections_full_text_search_idx'
+ end
+end
CREATE UNIQUE INDEX collection_owner_uuid_name_unique ON collections USING btree (owner_uuid, name);
+--
+-- Name: collections_full_text_search_idx; Type: INDEX; Schema: public; Owner: -; Tablespace:
+--
+
+CREATE INDEX collections_full_text_search_idx ON collections USING gin (to_tsvector('english'::regconfig, (((((((((((((((((((COALESCE(owner_uuid, ''::character varying))::text || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(portable_data_hash, ''::character varying))::text) || ' '::text) || (COALESCE(redundancy_confirmed_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || (COALESCE(description, ''::character varying))::text) || ' '::text) || COALESCE(properties, ''::text)) || ' '::text) || (COALESCE(file_names, ''::character varying))::text)));
+
+
--
-- Name: collections_search_index; Type: INDEX; Schema: public; Owner: -; Tablespace:
--
CREATE INDEX collections_search_index ON collections USING btree (owner_uuid, modified_by_client_uuid, modified_by_user_uuid, portable_data_hash, redundancy_confirmed_by_client_uuid, uuid, name, file_names);
+--
+-- Name: groups_full_text_search_idx; Type: INDEX; Schema: public; Owner: -; Tablespace:
+--
+
+CREATE INDEX groups_full_text_search_idx ON groups USING gin (to_tsvector('english'::regconfig, (((((((((((((COALESCE(uuid, ''::character varying))::text || ' '::text) || (COALESCE(owner_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || (COALESCE(description, ''::character varying))::text) || ' '::text) || (COALESCE(group_class, ''::character varying))::text)));
+
+
--
-- Name: groups_owner_uuid_name_unique; Type: INDEX; Schema: public; Owner: -; Tablespace:
--
CREATE INDEX job_tasks_search_index ON job_tasks USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, job_uuid, created_by_job_task_uuid);
+--
+-- Name: jobs_full_text_search_idx; Type: INDEX; Schema: public; Owner: -; Tablespace:
+--
+
+CREATE INDEX jobs_full_text_search_idx ON jobs USING gin (to_tsvector('english'::regconfig, (((((((((((((((((((((((((((((((((((((((((COALESCE(uuid, ''::character varying))::text || ' '::text) || (COALESCE(owner_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(submit_id, ''::character varying))::text) || ' '::text) || (COALESCE(script, ''::character varying))::text) || ' '::text) || (COALESCE(script_version, ''::character varying))::text) || ' '::text) || COALESCE(script_parameters, ''::text)) || ' '::text) || (COALESCE(cancelled_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(cancelled_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(output, ''::character varying))::text) || ' '::text) || (COALESCE(is_locked_by_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(log, ''::character varying))::text) || ' '::text) || COALESCE(tasks_summary, ''::text)) || ' '::text) || COALESCE(runtime_constraints, ''::text)) || ' '::text) || (COALESCE(repository, ''::character varying))::text) || ' '::text) || (COALESCE(supplied_script_version, ''::character varying))::text) || ' '::text) || (COALESCE(docker_image_locator, ''::character varying))::text) || ' '::text) || (COALESCE(description, ''::character varying))::text) || ' '::text) || (COALESCE(state, ''::character varying))::text) || ' '::text) || (COALESCE(arvados_sdk_version, ''::character varying))::text)));
+
+
--
-- Name: jobs_search_index; Type: INDEX; Schema: public; Owner: -; Tablespace:
--
CREATE INDEX nodes_search_index ON nodes USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, hostname, domain, ip_address, job_uuid);
+--
+-- Name: pipeline_instances_full_text_search_idx; Type: INDEX; Schema: public; Owner: -; Tablespace:
+--
+
+CREATE INDEX pipeline_instances_full_text_search_idx ON pipeline_instances USING gin (to_tsvector('english'::regconfig, (((((((((((((((((((((COALESCE(uuid, ''::character varying))::text || ' '::text) || (COALESCE(owner_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(pipeline_template_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || COALESCE(components, ''::text)) || ' '::text) || COALESCE(properties, ''::text)) || ' '::text) || (COALESCE(state, ''::character varying))::text) || ' '::text) || COALESCE(components_summary, ''::text)) || ' '::text) || (COALESCE(description, ''::character varying))::text)));
+
+
--
-- Name: pipeline_instances_search_index; Type: INDEX; Schema: public; Owner: -; Tablespace:
--
CREATE UNIQUE INDEX pipeline_template_owner_uuid_name_unique ON pipeline_templates USING btree (owner_uuid, name);
+--
+-- Name: pipeline_templates_full_text_search_idx; Type: INDEX; Schema: public; Owner: -; Tablespace:
+--
+
+CREATE INDEX pipeline_templates_full_text_search_idx ON pipeline_templates USING gin (to_tsvector('english'::regconfig, (((((((((((((COALESCE(uuid, ''::character varying))::text || ' '::text) || (COALESCE(owner_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || COALESCE(components, ''::text)) || ' '::text) || (COALESCE(description, ''::character varying))::text)));
+
+
--
-- Name: pipeline_templates_search_index; Type: INDEX; Schema: public; Owner: -; Tablespace:
--
INSERT INTO schema_migrations (version) VALUES ('20150122175935');
+INSERT INTO schema_migrations (version) VALUES ('20150123142953');
+
INSERT INTO schema_migrations (version) VALUES ('20150203180223');
\ No newline at end of file
ar_table_name = model_class.table_name
filters.each do |filter|
attrs_in, operator, operand = filter
- if attrs_in == 'any'
+ if attrs_in == 'any' && operator != '@@'
attrs = model_class.searchable_columns(operator)
elsif attrs_in.is_a? Array
attrs = attrs_in
elsif !operator.is_a? String
raise ArgumentError.new("Invalid operator '#{operator}' (#{operator.class}) in filter")
end
+
cond_out = []
+
+ if operator == '@@'
+ # Full-text search
+ if attrs_in != 'any'
+ raise ArgumentError.new("Full text search on individual columns is not supported")
+ end
+ if operand.is_a? Array
+ raise ArgumentError.new("Full text search not supported for array operands")
+ end
+
+ # Skip the generic per-column operator loop below
+ attrs = []
+ # Use to_tsquery since plainto_tsquery does not support prefix
+ # search. And, split operand and join the words with ' & '
+ cond_out << model_class.full_text_tsvector+" @@ to_tsquery(?)"
+ param_out << operand.split.join(' & ')
+ end
attrs.each do |attr|
if !model_class.searchable_columns(operator).index attr.to_s
raise ArgumentError.new("Invalid attribute '#{attr}' in filter")
manifest_text: ''
name: upload sandbox
+collection_with_unique_words_to_test_full_text_search:
+ uuid: zzzzz-4zz18-mnt690klmb51aud
+ portable_data_hash: fa7aeb5140e2848d39b416daeef4ffc5+45
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2014-02-03T17:22:54Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2014-02-03T17:22:54Z
+ updated_at: 2014-02-03T17:22:54Z
+ manifest_text: ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"
+ name: collection_with_some_unique_words
+ description: The quick_brown_fox jumps over the lazy_dog
+
# Test Helper trims the rest of the file
# Do not add your fixtures below this line as the rest of this file will be trimmed by test_helper
assert_includes(found.collect(&:group_class), nil,
"'group_class not in ['project']' filter should pass null")
end
+
+ test 'error message for non-array element in filters array' do
+ @controller = Arvados::V1::CollectionsController.new
+ authorize_with :active
+ get :index, {
+ filters: [{bogus: 'filter'}],
+ }
+ assert_response 422
+ assert_match(/Invalid element in filters array/,
+ json_response['errors'].join(' '))
+ end
+
+ test 'error message for full text search on a specific column' do
+ @controller = Arvados::V1::CollectionsController.new
+ authorize_with :active
+ get :index, {
+ filters: [['uuid', '@@', 'abcdef']],
+ }
+ assert_response 422
+ assert_match /not supported/, json_response['errors'].join(' ')
+ end
+
+ test 'difficult characters in full text search' do
+ @controller = Arvados::V1::CollectionsController.new
+ authorize_with :active
+ get :index, {
+ filters: [['any', '@@', 'a|b"c']],
+ }
+ assert_response :success
+ # (Doesn't matter so much which results are returned.)
+ end
+
+ test 'array operand in full text search' do
+ @controller = Arvados::V1::CollectionsController.new
+ authorize_with :active
+ get :index, {
+ filters: [['any', '@@', ['abc', 'def']]],
+ }
+ assert_response 422
+ assert_match /not supported/, json_response['errors'].join(' ')
+ end
end
}, auth(:active)
assert_response :success
assert_equal true, json_response['manifest_text'].include?('my_test_file.txt')
+ assert_includes json_response['manifest_text'], 'my_test_file.txt'
created = json_response
}, auth(:active)
assert_response :success
assert_equal created['uuid'], json_response['uuid']
- assert_equal true, json_response['manifest_text'].include?('my_updated_test_file.txt')
- assert_equal false, json_response['manifest_text'].include?('my_test_file.txt')
+ assert_includes json_response['manifest_text'], 'my_updated_test_file.txt'
+ assert_not_includes json_response['manifest_text'], 'my_test_file.txt'
# search using the new filename
search_using_filter 'my_updated_test_file.txt', 1
response_items = json_response['items']
assert_not_nil response_items
if expected_items == 0
- assert_equal 0, json_response['items_available']
- assert_equal 0, response_items.size
+ assert_empty response_items
else
- assert_equal expected_items, response_items.size
+ refute_empty response_items
first_item = response_items.first
assert_not_nil first_item
end
end
+
+ test "search collection using full text search" do
+ # create collection to be searched for
+ signed_manifest = Collection.sign_manifest(". 85877ca2d7e05498dd3d109baf2df106+95+A3a4e26a366ee7e4ed3e476ccf05354761be2e4ae@545a9920 0:95:file_in_subdir1\n./subdir2/subdir3 2bbc341c702df4d8f42ec31f16c10120+64+A315d7e7bad2ce937e711fc454fae2d1194d14d64@545a9920 0:32:file1_in_subdir3.txt 32:32:file2_in_subdir3.txt\n./subdir2/subdir3/subdir4 2bbc341c702df4d8f42ec31f16c10120+64+A315d7e7bad2ce937e711fc454fae2d1194d14d64@545a9920 0:32:file3_in_subdir4.txt 32:32:file4_in_subdir4.txt\n", api_token(:active))
+ post "/arvados/v1/collections", {
+ format: :json,
+ collection: {description: 'specific collection description', manifest_text: signed_manifest}.to_json,
+ }, auth(:active)
+ assert_response :success
+ assert_equal true, json_response['manifest_text'].include?('file4_in_subdir4.txt')
+
+ created = json_response
+
+ # search using the filename
+ search_using_full_text_search 'subdir2', 0
+ search_using_full_text_search 'subdir2:*', 1
+ search_using_full_text_search 'subdir2/subdir3/subdir4', 1
+ search_using_full_text_search 'file4:*', 1
+ search_using_full_text_search 'file4_in_subdir4.txt', 1
+ search_using_full_text_search 'subdir2 file4:*', 0 # first word is incomplete
+ search_using_full_text_search 'subdir2/subdir3/subdir4 file4:*', 1
+ search_using_full_text_search 'subdir2/subdir3/subdir4 file4_in_subdir4.txt', 1
+ search_using_full_text_search 'ile4', 0 # not a prefix match
+ end
+
+ def search_using_full_text_search search_filter, expected_items
+ get '/arvados/v1/collections', {
+ :filters => [['any', '@@', search_filter]].to_json
+ }, auth(:active)
+ assert_response :success
+ response_items = json_response['items']
+ assert_not_nil response_items
+ if expected_items == 0
+ assert_empty response_items
+ else
+ refute_empty response_items
+ first_item = response_items.first
+ assert_not_nil first_item
+ end
+ end
+
+ # search for the filename in the file_names column and expect error
+ test "full text search not supported for individual columns" do
+ get '/arvados/v1/collections', {
+ :filters => [['name', '@@', 'General']].to_json
+ }, auth(:active)
+ assert_response 422
+ end
+
+ [
+ 'quick fox',
+ 'quick_brown fox',
+ 'brown_ fox',
+ 'fox dogs',
+ ].each do |search_filter|
+ test "full text search ignores special characters and finds with filter #{search_filter}" do
+ # description: The quick_brown_fox jumps over the lazy_dog
+ # full text search treats '_' as space apparently
+ get '/arvados/v1/collections', {
+ :filters => [['any', '@@', search_filter]].to_json
+ }, auth(:active)
+ assert_response 200
+ response_items = json_response['items']
+ assert_not_nil response_items
+ first_item = response_items.first
+ refute_empty first_item
+ assert_equal first_item['description'], 'The quick_brown_fox jumps over the lazy_dog'
+ end
+ end
end
end
end
+ [
+ ['Collection_', true], # collections and pipelines templates
+ ['hash', true], # pipeline templates
+ ['fa7aeb5140e2848d39b', false], # script_parameter of pipeline instances
+ ['fa7aeb5140e2848d39b:*', true], # script_parameter of pipeline instances
+ ['project pipeline', true], # finds "Completed pipeline in A Project"
+ ['project pipeli:*', true], # finds "Completed pipeline in A Project"
+ ['proje pipeli:*', false], # first word is incomplete, so no prefix match
+ ['no-such-thing', false], # script_parameter of pipeline instances
+ ].each do |search_filter, expect_results|
+ test "full text search of group-owned objects for #{search_filter}" do
+ get "/arvados/v1/groups/contents", {
+ id: groups(:aproject).uuid,
+ limit: 5,
+ :filters => [['any', '@@', search_filter]].to_json
+ }, auth(:active)
+ assert_response :success
+ if expect_results
+ refute_empty json_response['items']
+ json_response['items'].each do |item|
+ assert item['uuid']
+ assert_equal groups(:aproject).uuid, item['owner_uuid']
+ end
+ else
+ assert_empty json_response['items']
+ end
+ end
+ end
+
+ test "full text search is not supported for individual columns" do
+ get "/arvados/v1/groups/contents", {
+ :filters => [['name', '@@', 'Private']].to_json
+ }, auth(:active)
+ assert_response 422
+ end
end
end
end
+ test "full text search for collections" do
+ # file_names column does not get populated when fixtures are loaded, hence setup test data
+ act_as_system_user do
+ Collection.create(manifest_text: ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo\n")
+ Collection.create(manifest_text: ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n")
+ Collection.create(manifest_text: ". 85877ca2d7e05498dd3d109baf2df106+95+A3a4e26a366ee7e4ed3e476ccf05354761be2e4ae@545a9920 0:95:file_in_subdir1\n./subdir2/subdir3 2bbc341c702df4d8f42ec31f16c10120+64+A315d7e7bad2ce937e711fc454fae2d1194d14d64@545a9920 0:32:file1.txt 32:32:file2.txt\n./subdir2/subdir3/subdir4 2bbc341c702df4d8f42ec31f16c10120+64+A315d7e7bad2ce937e711fc454fae2d1194d14d64@545a9920 0:32:file3.txt 32:32:file4.txt")
+ end
+
+ [
+ ['foo', true],
+ ['foo bar', false], # no collection matching both
+ ['foo&bar', false], # no collection matching both
+ ['foo|bar', true], # works only no spaces between the words
+ ['Gnu public', true], # both prefixes found, though not consecutively
+ ['Gnu&public', true], # both prefixes found, though not consecutively
+ ['file4', true], # prefix match
+ ['file4.txt', true], # whole string match
+ ['filex', false], # no such prefix
+ ['subdir', true], # prefix matches
+ ['subdir2', true],
+ ['subdir2/', true],
+ ['subdir2/subdir3', true],
+ ['subdir2/subdir3/subdir4', true],
+ ['subdir2 file4', true], # look for both prefixes
+ ['subdir4', false], # not a prefix match
+ ].each do |search_filter, expect_results|
+ search_filters = search_filter.split.each {|s| s.concat(':*')}.join('&')
+ results = Collection.where("#{Collection.full_text_tsvector} @@ to_tsquery(?)",
+ "#{search_filters}")
+ if expect_results
+ refute_empty results
+ else
+ assert_empty results
+ end
+ end
+ end
+
[0, 2, 4, nil].each do |ask|
test "replication_desired reports #{ask or 2} if redundancy is #{ask}" do
act_as_user users(:active) do
projects / arvados.git / commitdiff