From 9655bf0df07d5bde6946f6d20b279e85d07db3a6 Mon Sep 17 00:00:00 2001 From: Asad Jibran Ahmed Date: Mon, 10 Apr 2017 20:14:52 +0400 Subject: [PATCH 1/5] Added suggestion API --- app/controllers/v3/suggest_controller.rb | 23 +++++ config/routes.rb | 1 + lib/search/client.rb | 16 +-- lib/search/suggest.rb | 98 +++++++++++++++++++ .../controllers/v3/suggest_controller_spec.rb | 12 +++ 5 files changed, 136 insertions(+), 14 deletions(-) create mode 100644 app/controllers/v3/suggest_controller.rb create mode 100644 lib/search/suggest.rb create mode 100644 spec/controllers/v3/suggest_controller_spec.rb diff --git a/app/controllers/v3/suggest_controller.rb b/app/controllers/v3/suggest_controller.rb new file mode 100644 index 00000000..2f4bb06a --- /dev/null +++ b/app/controllers/v3/suggest_controller.rb @@ -0,0 +1,23 @@ +class V3::SuggestController < ApplicationController + def index + if query.nil? + render status: 400, json: { + 'message': 'No query parameter provided' + } + else + render json: get_suggestions + end + end + + def get_suggestions + Search::Suggest.new(query, lang: language).get_suggestions + end + + def language + params[:l] || params[:language] || 'en' + end + + def query + params[:q] || params[:query] + end +end diff --git a/config/routes.rb b/config/routes.rb index 7a47bf2f..741f8c37 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -33,6 +33,7 @@ end get 'search', to: 'search#index' + get 'suggest', to: 'suggest#index' end end diff --git a/lib/search/client.rb b/lib/search/client.rb index 547abe91..fca13473 100644 --- a/lib/search/client.rb +++ b/lib/search/client.rb @@ -22,23 +22,11 @@ def search_defination end def search_query - #hash = if @query.is_arabic? - # [verse_query(@query.query), words_query(@query.query)] - # else - # ids = Translation.where(resource_type: 'Verse').pluck(:language_id).uniq - # available_languages = Language.find(ids).pluck(:iso_code) - # available_languages.map {|lang| trans_query(lang, @query.query)} - # end - #ids = Translation.where(resource_type: 'Verse').pluck(:language_id).uniq.first(10) - #available_languages = Language.find(ids).pluck(:iso_code) - trans_query = [] unless /[:\/]/.match(@query.query) - available_languages = [ "ml", "en", "bs", "az", "cs", "fr", "hi", "es", "fi", "id", "it", "ko", "dv", "bn", "ku", - "de", "am", "al", "fa", "ha", "mrn", "ms", "pl", "ja", "nl", "tr", "ur", "th", "no", "tg", - "ug", "ru", "pt", "ro", "sq", "sw", "so", "sv", "ta", "uz", "zh", "tt" - ] + available_languages = %w[ml en bs az cs fr hi es fi id it ko dv bn ku de am al fa ha mrn ms pl ja nl tr ur th + no tg ug ru pt ro sq sw so sv ta uz zh tt] trans_query = available_languages.map {|lang| trans_query(lang, @query.query)} end diff --git a/lib/search/suggest.rb b/lib/search/suggest.rb new file mode 100644 index 00000000..872eead2 --- /dev/null +++ b/lib/search/suggest.rb @@ -0,0 +1,98 @@ +module Search + class Suggest + attr_accessor :query, :lang, :size + + def initialize(query, lang: 'en', size: 30) + @query = Search::Query.new(query) + @lang = lang + @size = size + end + + def get_suggestions + q = search_params + process_results Verse.search(q).results.results + end + + def translation_path + "trans_#{lang}" + end + + def translation_text_field_name + "#{translation_path}.text" + end + + def search_params + { + size: @size, + query: query_dict, + highlight: highlight + } + end + + def query_dict + if @query.is_arabic? + base_query = { + match_phrase_prefix: { + translation_text_field_name => @query.query, + } + } + else + base_query = { + match_phrase: { + translation_text_field_name => { + query: @query.query, + operator: 'and' + } + } + } + end + + { + nested: { + path: translation_path, + query: base_query + } + } + end + + def highlight + { + fields: { + translation_text_field_name => { + number_of_fragments: 0, # just highlight the entire string instead of breaking it down into sentence fragments, that's easier for now + pre_tags: [ '' ], + post_tags: [ '' ], + type: 'plain' + } + } + } + end + + def process_results(es_response) + processed = [] + seen = {} + + es_response.each do |hit| + if hit.key?('highlight') + text = hit['highlight'][translation_text_field_name][0] + else + text = hit['_source'][translation_path][0].text + end + ayah = "#{hit['_source']['verse_key']}" + href = "/#{hit['_source']['verse_key'].gsub(/:/,'/')}" + + unless seen.key?(ayah) + seen[ayah] = true + item = { + text: text, + href: href, + ayah: ayah + } + processed.push( item ) + end + end + + processed[0, @size] + end + end +end \ No newline at end of file diff --git a/spec/controllers/v3/suggest_controller_spec.rb b/spec/controllers/v3/suggest_controller_spec.rb new file mode 100644 index 00000000..7c6c4ff4 --- /dev/null +++ b/spec/controllers/v3/suggest_controller_spec.rb @@ -0,0 +1,12 @@ +require 'rails_helper' + +RSpec.describe V3::SuggestController, type: :controller do + + describe "GET #index" do + it "returns http success" do + get :index + expect(response).to have_http_status(:success) + end + end + +end From 3e3cac9b9cef80a04e5e9b99249627b32ce6fe89 Mon Sep 17 00:00:00 2001 From: Naveed Ahmad Date: Thu, 13 Apr 2017 17:18:01 +0500 Subject: [PATCH 2/5] bare minimum suggestions --- app/controllers/v3/suggest_controller.rb | 4 +- app/models/concerns/searchable.rb | 4 +- lib/search/results.rb | 2 + lib/search/suggest.rb | 185 ++++++++++++++--------- 4 files changed, 121 insertions(+), 74 deletions(-) diff --git a/app/controllers/v3/suggest_controller.rb b/app/controllers/v3/suggest_controller.rb index 2f4bb06a..d8282aed 100644 --- a/app/controllers/v3/suggest_controller.rb +++ b/app/controllers/v3/suggest_controller.rb @@ -1,9 +1,7 @@ class V3::SuggestController < ApplicationController def index if query.nil? - render status: 400, json: { - 'message': 'No query parameter provided' - } + render status: 400, json: [].to_json else render json: get_suggestions end diff --git a/app/models/concerns/searchable.rb b/app/models/concerns/searchable.rb index f8a1f8ad..0206e7ee 100644 --- a/app/models/concerns/searchable.rb +++ b/app/models/concerns/searchable.rb @@ -29,7 +29,7 @@ def verse_path def as_indexed_json(options={}) hash = self.as_json( only: [:id, :verse_key, :text_madani, :text_indopak, :text_simple], - methods: :verse_path #chapter_names + methods: [:verse_path, :chapter_names] ) hash[:words] = words.where.not(text_madani: nil).map do |w| @@ -81,7 +81,7 @@ def as_indexed_json(options={}) indexes :verse_path, type: 'keyword' # allow user to search by path e.g 1/2, 2/29 etc - # indexes :chapter_names + indexes :chapter_names indexes "words", type: 'nested' do indexes :madani, type: 'text', diff --git a/lib/search/results.rb b/lib/search/results.rb index 37625c5c..3a16b51d 100644 --- a/lib/search/results.rb +++ b/lib/search/results.rb @@ -62,6 +62,8 @@ def prepare_translation(trans, key) end def word_hightlight_class(hit) + return 'hlt1' unless hit['highlight'] + highlight = hit['highlight'].values.first.first if matched = highlight.match(/(hlt\ds*)/) diff --git a/lib/search/suggest.rb b/lib/search/suggest.rb index 872eead2..cceae9d5 100644 --- a/lib/search/suggest.rb +++ b/lib/search/suggest.rb @@ -1,98 +1,145 @@ module Search class Suggest attr_accessor :query, :lang, :size - - def initialize(query, lang: 'en', size: 30) + + def initialize(query, lang: 'en', size: 10) @query = Search::Query.new(query) - @lang = lang - @size = size + @lang = lang + @size = size end - + def get_suggestions - q = search_params - process_results Verse.search(q).results.results + process_results Verse.search(search_defination).page(1).per(10) end - - def translation_path - "trans_#{lang}" + + protected + def search_defination + { + _source: source_attributes, + query: suggest_query, + highlight: highlight + } end - - def translation_text_field_name - "#{translation_path}.text" + + def suggest_query + trans_query = [] + + unless /[:\/]/.match(@query.query) + available_languages = %w[ml en bs az cs fr hi es fi id it ko dv bn ku de am al fa ha mrn ms pl ja nl tr ur th + no tg ug ru pt ro sq sw so sv ta uz zh tt] + + trans_query = available_languages.map { |lang| trans_query(lang, @query.query) } + end + + _verse_query = [verse_query(@query.query)] + + { bool: { should: _verse_query + trans_query } } end - - def search_params + + def verse_query(query) { - size: @size, - query: query_dict, - highlight: highlight + multi_match: { + query: query, + fields: [ + 'verse_key', + 'verse_path', + 'text_madani.text', + 'text_simple.text', + 'text_indopak.text', + 'chapter_names' + ] + } } end - - def query_dict - if @query.is_arabic? - base_query = { - match_phrase_prefix: { - translation_text_field_name => @query.query, - } - } - else - base_query = { - match_phrase: { - translation_text_field_name => { - query: @query.query, - operator: 'and' - } - } - } + + def trans_query(lang, query) + # We boost the results if the query matched a translation of the same language as the user requested + lang_boost = 1 + if lang == @language + lang_boost = 2 end - + { - nested: { - path: translation_path, - query: base_query - } + nested: { + path: "trans_#{lang}", + query: { + match: { + "trans_#{lang}.text": { + query: query, + boost: lang_boost + } + } + }, + inner_hits: nested_highlight("trans_#{lang}.text") + } } end - + + def source_attributes + ["verse_key", "id"] + end + def highlight { - fields: { - translation_text_field_name => { - number_of_fragments: 0, # just highlight the entire string instead of breaking it down into sentence fragments, that's easier for now - pre_tags: [ '' ], - post_tags: [ '' ], - type: 'plain' - } + fields: { + "text_madani.text" => { + type: 'fvh'.freeze } + }, + tags_schema: 'styled'.freeze } end - + + def nested_highlight(filed) + { + highlight: { + fields: { + filed => { type: 'plain'.freeze }, + }, + tags_schema: 'styled'.freeze, + number_of_fragments: 1, # Don't highlight entire translation + fragment_size: 90, # return 90 chars around matched word + pre_tags: [''], + post_tags: [''] + } + } + end + def process_results(es_response) processed = [] - seen = {} - - es_response.each do |hit| - if hit.key?('highlight') - text = hit['highlight'][translation_text_field_name][0] - else - text = hit['_source'][translation_path][0].text - end - ayah = "#{hit['_source']['verse_key']}" - href = "/#{hit['_source']['verse_key'].gsub(/:/,'/')}" - - unless seen.key?(ayah) - seen[ayah] = true - item = { - text: text, - href: href, - ayah: ayah + + results = es_response.results.results + + results.map do |hit| + matched_values = if hit['highlight'].present? + [hit['highlight']['text_madani.text'].first] + else + trans = hit['inner_hits'].detect do |key, value| + value['hits']['total'] > 0 + end + + next unless trans + trans.last['hits']['hits'].map do |trans_match| + text = trans_match['highlight'].first.last.last + translation_id = trans_match['_source']['resource_content_id'] + + [text, translation_id] + end + end + + verse_key = "#{hit['_source']['verse_key']}" + href = "/#{verse_key.tr ':', '/'}" + + matched_values.each do |match| + processed << { + text: match.is_a?(Array) ? match[0] : match, + href: match.is_a?(Array) ? "#{href}?translations=#{match[1]}" : href, + ayah: verse_key } - processed.push( item ) end end - - processed[0, @size] + + processed end end end \ No newline at end of file From fb7a42738f036f7889295b6e16c8abc24a9086e2 Mon Sep 17 00:00:00 2001 From: Naveed Ahmad Date: Sat, 15 Apr 2017 03:24:42 +0500 Subject: [PATCH 3/5] enable get transaltion using slug --- app/controllers/v3/verses_controller.rb | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/app/controllers/v3/verses_controller.rb b/app/controllers/v3/verses_controller.rb index 6edeaa85..f1248dc3 100644 --- a/app/controllers/v3/verses_controller.rb +++ b/app/controllers/v3/verses_controller.rb @@ -1,4 +1,5 @@ class V3::VersesController < ApplicationController + before_action :fix_resource_content # GET /verses def index verses_index @@ -100,4 +101,13 @@ def load_translations .where(translations: translations) .eager_load(:translations) end + + def fix_resource_content + # user can get translation using ID or Slug + if render_translations? + translation = params[:translations] + + params[:translations] = ResourceContent.where(id: translation).or(ResourceContent.where(slug: translation)).first&.id + end + end end From ca0e74a68f8b89114fec14fea95e0a8f729c3175 Mon Sep 17 00:00:00 2001 From: Naveed Ahmad Date: Sat, 15 Apr 2017 03:25:59 +0500 Subject: [PATCH 4/5] increase highlight size --- lib/search/client.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/search/client.rb b/lib/search/client.rb index fca13473..1eb4e9c5 100644 --- a/lib/search/client.rb +++ b/lib/search/client.rb @@ -109,7 +109,8 @@ def nested_highlight(filed) filed => { type: 'fvh'.freeze } }, tags_schema: 'styled'.freeze - } + }, + size: 200 } end end From c1a28bfba48848eb96a30483e9952594d4fcc193 Mon Sep 17 00:00:00 2001 From: Naveed Ahmad Date: Sat, 15 Apr 2017 03:42:40 +0500 Subject: [PATCH 5/5] we don't need to highlight ayah text for search! --- lib/search/client.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/search/client.rb b/lib/search/client.rb index 1eb4e9c5..7799cdb8 100644 --- a/lib/search/client.rb +++ b/lib/search/client.rb @@ -16,8 +16,8 @@ def search def search_defination { _source: source_attributes, - query: search_query, - highlight: highlight + query: search_query #, + # highlight: highlight } end @@ -110,7 +110,7 @@ def nested_highlight(filed) }, tags_schema: 'styled'.freeze }, - size: 200 + size: 500 } end end