Merge pull request #2 from alphagov/replace-tokenizer-separator

Add configuration option to replace the default tokenizer separator

lib/middleman-search/extension.rb

@@ -9,6 +9,7 @@ class SearchExtension < Middleman::Extension
     option :index_path, 'search.json', 'Index file path'
     option :pipeline, {}, 'Javascript pipeline functions to use in lunr index'
     option :pipeline_remove, {}, 'Default pipeline functions to remove'
+    option :tokenizer_separator, {}, 'Replace the default tokizer separator'
     option :cache, false, 'Avoid the cache to be rebuilt on every request in development mode'
     option :language, 'en', 'Language code ("es", "fr") to use when indexing site\'s content'
     option :lunr_dirs, [], 'Directories in which to look for custom lunr.js files'

lib/middleman-search/search-index-resource.rb

@@ -9,6 +9,7 @@ def initialize(store, path, options)
         @callback = options[:before_index]
         @pipeline = options[:pipeline]
         @pipeline_remove = options[:pipeline_remove]
+        @tokenizer_separator = options[:tokenizer_separator]
         @cache_index = options[:cache]
         @language = options[:language]
         @lunr_dirs = options[:lunr_dirs] + [File.expand_path("../../../vendor/assets/javascripts/", __FILE__)]
@@ -63,6 +64,10 @@ def build_index
           source << "this.pipeline.add(lunr.Pipeline.registeredFunctions.#{name});"
         end
 
+        if @tokenizer_separator
+          source << "this.tokenizerFn.seperator = #{@tokenizer_separator}"
+        end
+
         # Use language if set
         source << "this.use(lunr.#{@language});" if @language != 'en'