Skip to content

Commit

Permalink
Move the config options around
Browse files Browse the repository at this point in the history
  • Loading branch information
robsears committed Apr 27, 2016
1 parent 0951991 commit 485495c
Show file tree
Hide file tree
Showing 2 changed files with 40 additions and 48 deletions.
43 changes: 30 additions & 13 deletions lib/searchyou/generator.rb
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,6 @@

module Searchyou

# Allow access to Searchyou.configuration hash
class << self
attr_accessor :configuration
end

class Generator < Jekyll::Generator

safe true
Expand All @@ -16,10 +11,10 @@ class Generator < Jekyll::Generator
def generate(site)

# Gather the configuration options
Searchyou.configure(site)
configuration = Configuration.new(site)

# Prepare the indexer
indexer = Searchyou::Indexer.new(Searchyou.configuration)
indexer = Searchyou::Indexer.new(configuration)
indexer.start

# Iterate through the site contents and send to indexer
Expand All @@ -43,16 +38,38 @@ def generate(site)

end

# Create a configuration for the site
def self.configure(site)
self.configuration ||= Configuration.new(site)
end

# Class containing configuration options
class Configuration
attr_accessor :site
def initialize(site)
@site = site.config
self.site = site
end

# Determine a URL for the cluster, or fail with error
def elasticsearch_url
ENV['BONSAI_URL'] || ENV['ELASTICSEARCH_URL'] ||
((site.config||{})['elasticsearch']||{})['url'] ||
raise(ArgumentError, "No Elasticsearch URL present, skipping indexing")
end

# Getter for the number of primary shards
def elasticsearch_number_of_shards
site.config['elasticsearch']['number_of_shards'] || 1
end

# Getter for the number of replicas
def elasticsearch_number_of_replicas
site.config['elasticsearch']['number_of_replicas'] || 1
end

# Getter for the index name
def elasticsearch_index_base_name
site.config['elasticsearch']['index_name'] || "jekyll"
end

# Getter for the default type
def elasticsearch_default_type
site.config['elasticsearch']['default_type'] || 'post'
end
end
end
45 changes: 10 additions & 35 deletions lib/searchyou/indexer.rb
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ class Indexer

BATCH_SIZE = 50

attr_accessor :configuration
attr_accessor :indexer_thread
attr_accessor :old_indices
attr_accessor :queue
Expand All @@ -14,39 +15,13 @@ class Indexer
attr_accessor :working

def initialize(configuration)
self.uri = URI(elasticsearch_url)
self.configuration = configuration
self.uri = URI(configuration.elasticsearch_url)
self.queue = Queue.new
self.working = true
self.timestamp = Time.now
end

# Determine a URL for the cluster, or fail with error
def elasticsearch_url
ENV['BONSAI_URL'] || ENV['ELASTICSEARCH_URL'] ||
((Searchyou.configuration.site||{})['elasticsearch']||{})['url'] ||
raise(ArgumentError, "No Elasticsearch URL present, skipping indexing")
end

# Getter for the number of primary shards
def elasticsearch_number_of_shards
Searchyou.configuration.site['elasticsearch']['number_of_shards'] || 1
end

# Getter for the number of replicas
def elasticsearch_number_of_replicas
Searchyou.configuration.site['elasticsearch']['number_of_replicas'] || 1
end

# Getter for the index name
def elasticsearch_index_base_name
Searchyou.configuration.site['elasticsearch']['index_name'] || "jekyll"
end

# Getter for the default type
def elasticsearch_default_type
Searchyou.configuration.site['elasticsearch']['default_type'] || 'post'
end

# Public: Add new documents for batch indexing.
def <<(doc)
self.queue << doc
Expand All @@ -60,7 +35,7 @@ def working?
# A versioned index name, based on the time of the indexing run.
# Will be later added to an alias for hot reindexing.
def elasticsearch_index_name
"#{elasticsearch_index_base_name}-#{timestamp.strftime('%Y%m%d%H%M%S')}"
"#{configuration.elasticsearch_index_base_name}-#{timestamp.strftime('%Y%m%d%H%M%S')}"
end

# Prepare an HTTP connection
Expand All @@ -77,7 +52,7 @@ def prepare_index
create_index = http_post("/#{elasticsearch_index_name}")
create_index.body = {
index: {
number_of_shards: elasticsearch_number_of_shards,
number_of_shards: configuration.elasticsearch_number_of_shards,
number_of_replicas: 0,
refresh_interval: -1
}
Expand Down Expand Up @@ -131,7 +106,7 @@ def http_request(klass, path)
# using its Bulk Update API.
# https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html
def es_bulk_insert!(http, batch)
bulk_insert = http_post("/#{elasticsearch_index_name}/#{elasticsearch_default_type}/_bulk")
bulk_insert = http_post("/#{elasticsearch_index_name}/#{configuration.elasticsearch_default_type}/_bulk")
bulk_insert.body = batch.map do |doc|
[ { :index => {} }.to_json, doc.to_json ].join("\n")
end.join("\n") + "\n"
Expand Down Expand Up @@ -160,7 +135,7 @@ def finish
def old_indices
resp = http_start { |h| h.request(http_get("/_cat/indices?h=index")) }
indices = JSON.parse(resp.body).map{|i|i['index']}
indices = indices.select{|i| i =~ /\A#{elasticsearch_index_base_name}/ }
indices = indices.select{|i| i =~ /\A#{configuration.elasticsearch_index_base_name}/ }
indices = indices - [ elasticsearch_index_name ]
self.old_indices = indices
end
Expand All @@ -173,14 +148,14 @@ def finalize!

# add replication to the new index
add_replication = http_put("/#{elasticsearch_index_name}/_settings")
add_replication.body = { index: { number_of_replicas: elasticsearch_number_of_replicas }}.to_json
add_replication.body = { index: { number_of_replicas: configuration.elasticsearch_number_of_replicas }}.to_json

# hot swap the index into the canonical alias
update_aliases = http_post("/_aliases")
update_aliases.body = {
"actions": [
{ "remove": { "index": old_indices.join(','), "alias": elasticsearch_index_base_name }},
{ "add": { "index": elasticsearch_index_name, "alias": elasticsearch_index_base_name }}
{ "remove": { "index": old_indices.join(','), "alias": configuration.elasticsearch_index_base_name }},
{ "add": { "index": elasticsearch_index_name, "alias": configuration.elasticsearch_index_base_name }}
]
}.to_json

Expand Down

0 comments on commit 485495c

Please sign in to comment.