diff --git a/app/models/quran/ayah.rb b/app/models/quran/ayah.rb deleted file mode 100644 index 75c5b5a7..00000000 --- a/app/models/quran/ayah.rb +++ /dev/null @@ -1,120 +0,0 @@ -# == Schema Information -# -# Table name: ayah -# -# ayah_index :integer not null -# surah_id :integer -# ayah_num :integer -# page_num :integer -# juz_num :integer -# hizb_num :integer -# rub_num :integer -# text :text -# ayah_key :text not null, primary key -# sajdah :text -# - -# vim: ts=4 sw=4 expandtab -class Quran::Ayah < ActiveRecord::Base - extend Quran - - self.table_name = 'ayah' - self.primary_key = 'ayah_key' - - belongs_to :surah, class_name: 'Quran::Surah' - - has_many :words, class_name: 'Quran::Word', foreign_key: 'ayah_key' - has_many :tokens, class_name: 'Quran::Token', through: :words - has_many :stems, class_name: 'Quran::Stem', through: :words - has_many :lemmas, class_name: 'Quran::Lemma', through: :words - has_many :roots, class_name: 'Quran::Root', through: :words - - has_many :media_resources, class_name: 'Media::Resource', through: :media_content, source: :resource - has_many :media_content, class_name: 'Media::Content', foreign_key: 'ayah_key' - - has_many :_tafsir_ayah, class_name: 'Content::TafsirAyah', foreign_key: 'ayah_key' - has_many :tafsirs, class_name: 'Content::Tafsir', through: :_tafsir_ayah - - has_many :translations, class_name: 'Content::Translation', foreign_key: 'ayah_key' - has_one :transliteration, class_name: 'Content::Transliteration', foreign_key: 'ayah_key' - - has_many :audio, class_name: 'Audio::File', foreign_key: 'ayah_key' - has_many :texts, class_name: 'Quran::Text', foreign_key: 'ayah_key' - has_one :text_tashkeel, -> { where(resource_id: 12) }, class_name: 'Quran::Text', foreign_key: 'ayah_key' - has_many :images, class_name: 'Quran::Image', foreign_key: 'ayah_key' - has_many :glyphs, -> {order('position asc') }, class_name: 'Quran::WordFont', foreign_key: 'ayah_key' - - # NOTE the relationships below were created as database-side views for use with elasticsearch - has_many :text_roots, class_name: 'Quran::TextRoot', foreign_key: 'ayah_key' - has_many :text_lemmas, class_name: 'Quran::TextLemma', foreign_key: 'ayah_key' - has_many :text_stems, class_name: 'Quran::TextStem', foreign_key: 'ayah_key' - has_one :text_token, class_name: 'Quran::TextToken', foreign_key: 'ayah_key' - - def self.by_range(surah_id, from, to) - where('quran.ayah.surah_id = ?', surah_id) - .where('quran.ayah.ayah_num >= ?', from) - .where('quran.ayah.ayah_num <= ?', to) - .order(:surah_id, :ayah_num) - end - - def self.by_array(ayahs_keys_array) - order = sanitize_sql_array( - ["position(ayah_key::text in ?)", ayahs_keys_array.join(',')] - ) - where(ayah_key: ayahs_keys_array).order(order) - end - - def self.get_ayahs_by_page(page) - where(page_num: page).order('quran.ayah.surah_id, quran.ayah.ayah_num') - end - - def self.import_options ( options = {} ) - transform = lambda do |a| - data = a.__elasticsearch__.as_indexed_json - data.delete( 'text' ) # NOTE we exclude text because it serves no value in the parent mapping - { index: { _id: "#{a.ayah_key}", data: data } } - end - options = { transform: transform, batch_size: 6236 }.merge(options) - - return options - end - - def self.import (options = {}) - self.importing( self.import_options( options ) ) - end - - def as_json(options = {}) - super(options) - .merge(words: glyphs.sort.as_json) - .merge(text_tashkeel: text_tashkeel ? text_tashkeel.text : '') - end - - def self.as_json_with_resources(ayahs, options = {}) - keys = ayahs.map(&:ayah_key) - - if audio_option = options[:audio] - audio = - Audio::File - .where(ayah_key: keys, recitation_id: audio_option, is_enabled: true, format: 'mp3') - .order(:ayah_key) - .group_by(&:ayah_key) - end - - if content_option = options[:content] - content = - Content::Translation - .preload(:resource).where(ayah_key: keys, resource_id: content_option) - .order(:ayah_key) - .group_by(&:ayah_key) - end - - ayahs.map do |ayah| - ayah_json = ayah.as_json(include: {media_content: {include: :resource}}) - ayah_json.merge({ - content: content_option && content[ayah.ayah_key] ? - content[ayah.ayah_key] : [], - audio: audio_option && audio[ayah.ayah_key] ? audio[ayah.ayah_key].first : {}, - }) - end - end -end diff --git a/app/models/quran/lemma.rb b/app/models/quran/lemma.rb deleted file mode 100644 index 4d746a7e..00000000 --- a/app/models/quran/lemma.rb +++ /dev/null @@ -1,20 +0,0 @@ -# == Schema Information -# -# Table name: lemma -# -# lemma_id :integer not null, primary key -# value :string(50) not null -# clean :string(50) not null -# - -class Quran::Lemma < ActiveRecord::Base - self.table_name = 'lemma' - - has_many :_word_lemma, class_name: 'Quran::WordLemma', foreign_key: 'lemma_id' - has_many :words, class_name: 'Quran::Word', through: :_word_lemma - has_many :tokens, class_name: 'Quran::Token', through: :words - has_many :stems, class_name: 'Quran::Stem', through: :words - has_many :roots, class_name: 'Quran::Root', through: :words - has_many :ayahs, class_name: 'Quran::Ayah', through: :words -end - diff --git a/app/models/quran/root.rb b/app/models/quran/root.rb deleted file mode 100644 index 0d0d1a1b..00000000 --- a/app/models/quran/root.rb +++ /dev/null @@ -1,29 +0,0 @@ -# == Schema Information -# -# Table name: root -# -# root_id :integer not null, primary key -# value :string(50) not null -# - -class Quran::Root < ActiveRecord::Base - self.table_name = 'root' - - has_many :_word_root, class_name: 'Quran::WordRoot', foreign_key: 'root_id' - - has_many :words, class_name: 'Quran::Word', through: :_word_root - has_many :tokens, class_name: 'Quran::Token', through: :words - has_many :stems, class_name: 'Quran::Stem', through: :words - has_many :lemmas, class_name: 'Quran::Lemma', through: :words - has_many :ayahs, class_name: 'Quran::Ayah', through: :words - -# def self.import(options = {}) -# transform = lambda do |a| -# {index: {_id: "#{a.resource_id},#{a.ayah_key}", _parent: a.ayah_key, data: a.__elasticsearch__.as_indexed_json}} -# end -# options = {transform: transform}.merge(options) -# self.importing options -# end - - -end diff --git a/app/models/quran/stem.rb b/app/models/quran/stem.rb deleted file mode 100644 index b20533f1..00000000 --- a/app/models/quran/stem.rb +++ /dev/null @@ -1,20 +0,0 @@ -# == Schema Information -# -# Table name: stem -# -# stem_id :integer not null, primary key -# value :string(50) not null -# clean :string(50) not null -# - -class Quran::Stem < ActiveRecord::Base - self.table_name = 'stem' - - has_many :_word_stem, class_name: 'Quran::WordStem', foreign_key: 'stem_id' - - has_many :words, class_name: 'Quran::Word', through: :_word_stem - has_many :tokens, class_name: 'Quran::Token', through: :words # uniq ? - has_many :lemmas, class_name: 'Quran::Lemma', through: :words - has_many :roots, class_name: 'Quran::Root', through: :words - has_many :ayahs, class_name: 'Quran::Ayah', through: :words -end diff --git a/app/models/quran/surah.rb b/app/models/quran/surah.rb deleted file mode 100644 index 86366a77..00000000 --- a/app/models/quran/surah.rb +++ /dev/null @@ -1,48 +0,0 @@ -# == Schema Information -# -# Table name: surah -# -# surah_id :integer not null, primary key -# ayat :integer not null -# bismillah_pre :boolean not null -# revelation_order :integer not null -# revelation_place :text not null -# page :integer not null, is an Array -# name_complex :text not null -# name_simple :text not null -# name_english :text not null -# name_arabic :text not null -# - -class Quran::Surah < ActiveRecord::Base - self.table_name = 'surah' - - has_many :ayahs, class_name: 'Quran::Ayah', foreign_key: 'surah_id' - has_many :surah_infos, class_name: 'Content::SurahInfo' - - def get_surah_info_for_language(language_code) - language = Locale::Language.find_by_language_code(language_code) || Locale::Language.find('en') - - surah_infos.find_by(language: language) - end - - def name - { - complex: name_complex, - simple: name_simple, - english: name_english, - arabic: name_arabic - } - end - - def revelation - { - place: revelation_place, - order: revelation_order - } - end - - def as_json(options = {}) - super(methods: [:name, :revelation]).merge(id: surah_id) - end -end diff --git a/app/models/quran/text.rb b/app/models/quran/text.rb deleted file mode 100644 index e9d70d27..00000000 --- a/app/models/quran/text.rb +++ /dev/null @@ -1,78 +0,0 @@ -# == Schema Information -# -# Table name: text -# -# resource_id :integer not null -# ayah_key :text not null -# text :text not null -# - -# vim: ts=4 sw=4 expandtab -class Quran::Text < ActiveRecord::Base - # extend Batchelor - - self.table_name = 'text' - - # relationships - belongs_to :resource, class_name: 'Content::Resource' - belongs_to :ayah, class_name: 'Quran::Ayah', foreign_key: 'ayah_key' - - # scope - default_scope { where resource_id: 6 } # limit to Simple Enhanced - - #settings YAML.load( - # File.read( - # File.expand_path( - # "#{Rails.root}/config/elasticsearch/settings.yml", __FILE__ - # ) - # ) - #) - -=begin - mappings _all: { enabled: false } do - indexes :text, type: 'multi_field' do - indexes :text, - type: 'string', - similarity: 'my_bm25', - term_vector: 'with_positions_offsets_payloads', - analyzer: 'arabic_normalized' - indexes :stemmed, - type: 'string', - similarity: 'my_bm25', - term_vector: 'with_positions_offsets_payloads', - search_analyzer: 'arabic_normalized', - analyzer: 'arabic_ngram' - indexes :autocomplete, - type: 'string', - analyzer: 'autocomplete_arabic', - search_analyzer: 'arabic_normalized', - index_options: 'offsets' - end - end -=end - - def as_indexed_json(options = {}) - as_json(include: :resource) - end - - def self.import( options = {} ) - transform = lambda do |a| - this_data = a.__elasticsearch__.as_indexed_json - ayah_data = a.ayah.__elasticsearch__.as_indexed_json - - { - index: { - _id: "#{a.resource_id}_#{a.ayah_key.gsub!(/:/, '_')}", - data: this_data.merge({ - ayah_key: ayah_data['ayah_key'].gsub!(/:/, '_'), - text: ayah_data['text'] - }) - } - } - end - - options = { transform: transform, batch_size: 6236 }.merge(options) - - self.importing options - end -end diff --git a/app/models/quran/text_lemma.rb b/app/models/quran/text_lemma.rb deleted file mode 100644 index 9651f450..00000000 --- a/app/models/quran/text_lemma.rb +++ /dev/null @@ -1,43 +0,0 @@ -# == Schema Information -# -# Table name: text_lemma -# -# id :text -# ayah_key :text -# surah_id :integer -# ayah_num :integer -# is_hidden :boolean -# text :text -# - -# vim: ts=4 sw=4 expandtab -class Quran::TextLemma < ActiveRecord::Base - - self.table_name = 'text_lemma' - - belongs_to :ayah, class_name: 'Quran::Ayah', foreign_key: 'ayah_key' - - # scope - # default_scope { where surah_id: -1 } - - # elasticsearch index name - #index_name "text-lemma" - - def self.import( options = {} ) - Quran::TextLemma.connection.cache do - transform = lambda do |a| - this_data = a.__elasticsearch__.as_indexed_json - ayah_data = a.ayah.__elasticsearch__.as_indexed_json - this_data.delete( 'ayah_key' ) - ayah_data.delete( 'text' ) - ayah_data[ 'ayah_key' ].gsub!( /:/, '_' ) - { index: { - _id: "#{this_data['id'].gsub!( /:/, '_' )}", - data: this_data.merge( { 'ayah' => ayah_data } ) - } } - end - options = { transform: transform, batch_size: 6236 }.merge( options ) - self.importing options - end - end -end diff --git a/app/models/quran/text_root.rb b/app/models/quran/text_root.rb deleted file mode 100644 index 9839ca22..00000000 --- a/app/models/quran/text_root.rb +++ /dev/null @@ -1,42 +0,0 @@ -# == Schema Information -# -# Table name: text_root -# -# id :text -# ayah_key :text -# surah_id :integer -# ayah_num :integer -# is_hidden :boolean -# text :text -# - -# vim: ts=4 sw=4 expandtab -class Quran::TextRoot < ActiveRecord::Base - self.table_name = 'text_root' - - # relationships - belongs_to :ayah, class_name: 'Quran::Ayah', foreign_key: 'ayah_key' - - # scope - # default_scope { where surah_id: -1 } - - #index_name 'text-root' - - def self.import( options = {} ) - Quran::TextRoot.connection.cache do - transform = lambda do |a| - this_data = a.__elasticsearch__.as_indexed_json - ayah_data = a.ayah.__elasticsearch__.as_indexed_json - this_data.delete( 'ayah_key' ) - ayah_data.delete( 'text' ) - ayah_data[ 'ayah_key' ].gsub!( /:/, '_' ) - { index: { - _id: "#{this_data['id'].gsub!( /:/, '_' )}", - data: this_data.merge( { 'ayah' => ayah_data } ) - } } - end - options = { transform: transform, batch_size: 6236 }.merge( options ) - self.importing options - end - end -end diff --git a/app/models/quran/text_stem.rb b/app/models/quran/text_stem.rb deleted file mode 100644 index 10715439..00000000 --- a/app/models/quran/text_stem.rb +++ /dev/null @@ -1,41 +0,0 @@ -# == Schema Information -# -# Table name: text_stem -# -# id :text -# ayah_key :text -# surah_id :integer -# ayah_num :integer -# is_hidden :boolean -# text :text -# - -# vim: ts=4 sw=4 expandtab -class Quran::TextStem < ActiveRecord::Base - self.table_name = 'text_stem' - - belongs_to :ayah, class_name: 'Quran::Ayah', foreign_key: 'ayah_key' - - # scope - # default_scope { where surah_id: -1 } - - #index_name 'text-stem' - - def self.import( options = { }) - Quran::TextStem.connection.cache do - transform = lambda do |a| - this_data = a.__elasticsearch__.as_indexed_json - ayah_data = a.ayah.__elasticsearch__.as_indexed_json - this_data.delete( 'ayah_key' ) - ayah_data.delete( 'text' ) - ayah_data[ 'ayah_key' ].gsub!( /:/, '_' ) - { index: { - _id: "#{this_data['id'].gsub!( /:/, '_' )}", - data: this_data.merge( { 'ayah' => ayah_data } ) - } } - end - options = { transform: transform, batch_size: 6236 }.merge( options ) - self.importing options - end - end -end diff --git a/app/models/quran/text_token.rb b/app/models/quran/text_token.rb deleted file mode 100644 index 6e778c88..00000000 --- a/app/models/quran/text_token.rb +++ /dev/null @@ -1,44 +0,0 @@ -# == Schema Information -# -# Table name: text_token -# -# id :text -# ayah_key :text -# surah_id :integer -# ayah_num :integer -# is_hidden :boolean -# text :text -# - -# vim: ts=4 sw=4 expandtab -class Quran::TextToken < ActiveRecord::Base - #extend Quran - - self.table_name = 'text_token' - #self.primary_key = 'id' - - belongs_to :ayah, class_name: 'Quran::Ayah', foreign_key: 'ayah_key' - - # scope - # default_scope { where surah_id: -1 } - - # index_name 'text-token' - - def self.import( options = {} ) - Quran::TextToken.connection.cache do - transform = lambda do |a| - this_data = a.__elasticsearch__.as_indexed_json - ayah_data = a.ayah.__elasticsearch__.as_indexed_json - this_data.delete( 'ayah_key' ) - ayah_data.delete( 'text' ) - ayah_data[ 'ayah_key' ].gsub!( /:/, '_' ) - { index: { - _id: "#{this_data['id'].gsub!( /:/, '_' )}", - data: this_data.merge( { 'ayah' => ayah_data } ) - } } - end - options = { transform: transform, batch_size: 6236 }.merge( options ) - self.importing options - end - end -end diff --git a/app/models/quran/token.rb b/app/models/quran/token.rb deleted file mode 100644 index 2db666e5..00000000 --- a/app/models/quran/token.rb +++ /dev/null @@ -1,19 +0,0 @@ -# == Schema Information -# -# Table name: token -# -# token_id :integer not null, primary key -# value :string(50) not null -# clean :string(50) not null -# - -class Quran::Token < ActiveRecord::Base - self.table_name = 'token' - #self.primary_key = 'token_id' - - has_many :words, class_name: 'Quran::Word', foreign_key: 'token_id' - has_many :stems, class_name: 'Quran::Stem', through: :words - has_many :lemmas, class_name: 'Quran::Lemma', through: :words - has_many :roots, class_name: 'Quran::Root', through: :words - has_many :ayahs, class_name: 'Quran::Ayah', through: :words -end diff --git a/app/models/quran/word.rb b/app/models/quran/word.rb deleted file mode 100644 index 293949c8..00000000 --- a/app/models/quran/word.rb +++ /dev/null @@ -1,40 +0,0 @@ -# == Schema Information -# -# Table name: word -# -# word_id :integer not null, primary key -# ayah_key :text not null -# position :integer not null -# token_id :integer not null -# translation :string -# transliteration :string -# - -class Quran::Word < ActiveRecord::Base - extend Quran - - self.table_name = 'word' - - belongs_to :ayah, class_name: 'Quran::Ayah' - belongs_to :token, class_name: 'Quran::Token' - - has_one :corpus, class_name: 'Quran::WordCorpus', foreign_key: :word_id - - has_many :glyphs, class_name: 'Quran::WordFont', foreign_key: :word_id - - has_many :_word_lemma, class_name: 'Quran::WordLemma', foreign_key: :word_id - has_many :_word_root, class_name: 'Quran::WordRoot', foreign_key: :word_id - has_many :_word_stem, class_name: 'Quran::WordStem', foreign_key: :word_id - - has_many :stems, class_name: 'Quran::Stem', through: :_word_stem - has_many :lemmas, class_name: 'Quran::Lemma', through: :_word_lemma - has_many :roots, class_name: 'Quran::Root', through: :_word_root - - def as_json(options = {}) - super(include: { - corpus: { - only: [:description, :image_src, :segment] - } - }) - end -end diff --git a/app/models/quran/word_corpus.rb b/app/models/quran/word_corpus.rb deleted file mode 100644 index fc24e1e4..00000000 --- a/app/models/quran/word_corpus.rb +++ /dev/null @@ -1,21 +0,0 @@ -# == Schema Information -# -# Table name: word_corpus -# -# corpus_id :integer not null, primary key -# word_id :integer -# location :string -# description :string -# transliteration :string -# image_src :string -# segment :json -# -# Indexes -# -# index_quran.word_corpus_on_word_id (word_id) -# - -class Quran::WordCorpus < ActiveRecord::Base - self.table_name = 'word_corpus' - belongs_to :word -end diff --git a/app/models/quran/word_font.rb b/app/models/quran/word_font.rb deleted file mode 100644 index f6e8383a..00000000 --- a/app/models/quran/word_font.rb +++ /dev/null @@ -1,51 +0,0 @@ -# == Schema Information -# -# Table name: word_font -# -# resource_id :integer not null -# ayah_key :text not null -# position :integer not null -# word_id :integer -# page_num :integer not null -# line_num :integer not null -# code_dec :integer not null -# code_hex :text not null -# char_type_id :integer not null -# - -#TODO: marked for deletion -class Quran::WordFont < ActiveRecord::Base - extend Quran - - self.table_name = 'word_font' - #self.primary_keys = :resource_id, :ayah_key, :position - - belongs_to :ayah, class_name: 'Quran::Ayah' - belongs_to :char_type, class_name: 'Quran::CharType' - belongs_to :resource, class_name: 'Content::Resource' - belongs_to :word, class_name: 'Quran::Word' - - def class_name - "p#{page_num}" - end - - def code - "&#x#{code_hex}" - end - - def translation - word && word.translation - end - - def transliteration - word && word.transliteration - end - - def arabic - word && word.token.value - end - - def as_json(options = {}) - super(methods: [:class_name, :code, :translation, :transliteration, :arabic]) - end -end diff --git a/app/models/quran/word_lemma.rb b/app/models/quran/word_lemma.rb deleted file mode 100644 index 12bd6db0..00000000 --- a/app/models/quran/word_lemma.rb +++ /dev/null @@ -1,17 +0,0 @@ -# == Schema Information -# -# Table name: word_lemma -# -# word_id :integer not null -# lemma_id :integer not null -# position :integer default(1), not null -# - -class Quran::WordLemma < ActiveRecord::Base - self.table_name = 'word_lemma' - - belongs_to :word, class_name: 'Quran::Word' - belongs_to :lemma, class_name: 'Quran::Lemma' -end - - diff --git a/app/models/quran/word_root.rb b/app/models/quran/word_root.rb deleted file mode 100644 index 2194281a..00000000 --- a/app/models/quran/word_root.rb +++ /dev/null @@ -1,16 +0,0 @@ -# == Schema Information -# -# Table name: word_root -# -# word_id :integer not null -# root_id :integer not null -# position :integer default(1), not null -# - -class Quran::WordRoot < ActiveRecord::Base - self.table_name = 'word_root' - - belongs_to :word, class_name: 'Quran::Word' - belongs_to :root, class_name: 'Quran::Root' -end - diff --git a/app/models/quran/word_stem.rb b/app/models/quran/word_stem.rb deleted file mode 100644 index 512b20e8..00000000 --- a/app/models/quran/word_stem.rb +++ /dev/null @@ -1,15 +0,0 @@ -# == Schema Information -# -# Table name: word_stem -# -# word_id :integer not null -# stem_id :integer not null -# position :integer default(1), not null -# - -class Quran::WordStem < ActiveRecord::Base - self.table_name = 'word_stem' - - belongs_to :word, class_name: 'Quran::Word' - belongs_to :stem, class_name: 'Quran::Stem' -end diff --git a/config/initializers/keenio.rb b/config/initializers/keenio.rb deleted file mode 100644 index 7c09589d..00000000 --- a/config/initializers/keenio.rb +++ /dev/null @@ -1,4 +0,0 @@ -ENV['KEEN_PROJECT_ID']='54d67a02c1e0ab788bb53d0b' -ENV['KEEN_MASTER_KEY']='8313EE2C23727619417C21F38E1DC81F' -ENV['KEEN_WRITE_KEY']='7f633087c2fa7a18f3f04768746ff1d3e6e6e7b09897fbf543fa26b8056b9f851eaba53e4a228b5be74e4f3bacff537994b5e5e1db6cc2f4ae51197116b6c27b68fd9772bb47906ed2481aec06ba6d5b4e67525b6aa746b8b02a4f24c4e770860497a4e6a7ec9bcd262fbdfdd4498e2a' -ENV['KEEN_READ_KEY']='596c35534abcd246c3784520493175700a4e58c6ad9d43a366d2918873a297c2867948975128c847492a609baa5d1e072d40fdc2e6416e03da64dca3c7a9ca7e52b53b834820cd7e389e9cb8825a1cd6e956b932f8f674f1cfdc9c4d9eff0e6496da918eeb985fcc99b10789b22e0184' \ No newline at end of file diff --git a/lib/tasks/db_pg_dump.rake b/lib/tasks/db_pg_dump.rake index f2ad9041..3a43dcfd 100644 --- a/lib/tasks/db_pg_dump.rake +++ b/lib/tasks/db_pg_dump.rake @@ -1,12 +1,12 @@ namespace :db do desc 'Dump the database' task pg_dump: :environment do - previous_version = Dir['db/dumps/**'].map{ |dir| dir.split('/').last }.sort_by{|version| version.to_i}.last.to_i + 1 + previous_version = Dir['dumps/**'].map{ |dir| dir.split('/').last }.sort_by{|version| version.to_i}.last.to_i + 1 sh "mkdir db/dumps/#{previous_version}" sh "mv db/quran_dev.psql.bz2 db/dumps/#{previous_version}" sh "touch db/dumps/#{previous_version}/CHANGES.md" - sh "pg_dump quran_dev -U quran_dev > db/quran_dev_v3.psql" - sh "bzip2 db/quran_dev_v3.psql" + sh "pg_dump quran_dev -U quran_dev > db/quran_dev.psql" + sh "bzip2 db/quran_dev.psql" end task load_pg_dump: :environment do diff --git a/lib/tasks/one_time.rake b/lib/tasks/one_time.rake deleted file mode 100644 index 271be087..00000000 --- a/lib/tasks/one_time.rake +++ /dev/null @@ -1,195 +0,0 @@ -namespace :one_time do - task import_malayalam_data: :environment do - require 'httparty' - language = Language.find_by_name('Malayalam') - - source = DataSource.where(name: "Tafhim al-Qur'an", url: "http://www.tafheem.net/").first_or_create - author = Author.where(name: "Sayyid Abul Ala Maududi").first_or_create - resource_content = ResourceContent.where(name: "Chapter Info", author: author, language: language).first_or_create(author_name: author.name, cardinality_type: ResourceContent::CardinalityType::OneChapter, resource_type: ResourceContent::ResourceType::Content, sub_type: 'info') - resource_content.data_source = source - resource_content.description = "Sayyid Abul Ala Maududi - Tafhim al-Qur'an - The Meaning of the Quran" - resource_content.save - - chapter_names = JSON.parse(HTTParty.get('http://www.thafheem.net/assets/suranames.json').body) - chapter_names.each do |item| - chapter = Chapter.find(item['SuraID']) - - chapter.translated_names.where(language: language, name: item['MSuraName']).first_or_create - - info_data = JSON.parse(HTTParty.get("http://www.thafheem.net/thaf-api/preface/#{chapter.chapter_number}/M").body) - - short_text = info_data.first['PrefObj'].first['PrefaceText'] - text = info_data.first['PrefObj'].map do |section| - "

#{section['PrefaceSubTitle']}

#{section['PrefaceText'] }

" - end.join('
') - chapter_info = chapter.chapter_infos.where(language: language).first_or_create - - chapter_info.text = text - chapter_info.short_text = short_text - chapter_info.source = "www.thafheem.net" - chapter_info.resource_content = resource_content - chapter_info.save - end - - resource_content = ResourceContent.where(name: "Word translation", author: author, language: language).first_or_create(author_name: author.name, cardinality_type: ResourceContent::CardinalityType::OneWord, resource_type: ResourceContent::ResourceType::Content, sub_type: ResourceContent::SubType::Translation) - resource_content.data_source = source - resource_content.description = "Word-by-word translation in Malayalam language by Sayyid Abul Ala Maududi" - resource_content.save - end - - task :import_tamil do - - language = Language.find_by_name('Tamil') - source = DataSource.where(name: "Tafhim al-Qur'an", url: "http://truevisionmedia.org/").first_or_create - author = Author.where(name: "Sayyid Abul Ala Maududi").first_or_create - resource_content = ResourceContent.where(name: "Chapter Info", author: author, language: language).first_or_create(author_name: author.name, cardinality_type: ResourceContent::CardinalityType::OneChapter, resource_type: ResourceContent::ResourceType::Content, sub_type: 'info') - resource_content.data_source = source - resource_content.description = "Sayyid Abul Ala Maududi - Tafhim al-Qur'an - The Meaning of the Quran" - resource_content.save - - 1.upto(114) do |number| - chapter = Chapter.find(number) - - chapter_info = chapter.chapter_infos.where(language: language).first_or_create - chapter_info.text = File.open("data/tamil_info/#{number}.html").read.to_s.strip - chapter_info.source ="Sayyid Abul Ala Maududi - Tafhim al-Qur'an - The Meaning of the Quran" - chapter_info.resource_content = resource_content - chapter_info.save - end - end - - task import_urdu_data: :environment do - language = Language.find_by_name('Urdu') - - source = DataSource.where(name: "Tafhim al-Qur'an", url: "http://www.tafheemulquran.net/").first_or_create - author = Author.where(name: "Sayyid Abul Ala Maududi").first_or_create - resource_content = ResourceContent.where(name: "Chapter Info", author: author, language: language).first_or_create(author_name: author.name, cardinality_type: ResourceContent::CardinalityType::OneChapter, resource_type: ResourceContent::ResourceType::Content, sub_type: 'info') - resource_content.data_source = source - resource_content.description = "Sayyid Abul Ala Maududi - Tafhim al-Qur'an - The Meaning of the Quran" - resource_content.save - - 1.upto(114) do |number| - chapter = Chapter.find(number) - chapter_number = number.to_s.rjust(3, '0') - - chapter_info = chapter.chapter_infos.where(language: language).first_or_create - - chapter_info.text = File.open("data/urdu_final/#{chapter_number}.html").read.to_s.strip - chapter_info.source = "سید ابو اعلیٰ مودودیؒ - تفہیم القرآن" - chapter_info.resource_content = resource_content - chapter_info.save - end - end - - task :add_chapter_translated_name do - #Create translated name of languages - Language.find_each do |lan| - lan.translated_names.where(language: Language.default, name: lan.name).first_or_create - end - - [['Chinese', 'chinese.json'], ['French', 'france.json'], ['Urdu', 'urdu.json'], ['Azeri', 'azirbijan.json'], - ['Bosnian', 'bonski.json'], ['Dutch', 'neatherland.json'], ['Russian', 'ru.json'], ['Spanish', 'spanish.json'], - ['Swedish', 'swedish.json'], ['Indonesian', 'id.json'] - ].each do |item| - lang, file_name = item - language = Language.find_by_name(lang) - - json = JSON.parse(File.open("data/translated_names/#{file_name}", 'r').read) - - json.each do |c| - name = c['name'].to_s.split(',').first - chapter = Chapter.find_by_chapter_number(c['id']) - chapter.translated_names.where(language: language).first_or_create.update_attributes(name: name) - end - end - end -end - -=begin - -agent = Mechanize.new - -1.upto(114).each do |chapter_number| - chapter_number = chapter_number.to_s.rjust(3, '0') - url = "http://www.tafheemulquran.net/1_Tafheem/Suraes/#{chapter_number}/S#{chapter_number}.html" - info = agent.get(url).search-v2("#intro td p").inner_html - File.open("data/urdu_info/#{chapter_number}.html", 'wb') do |file| - file << info - end - - frag.xpath("//span[@class='style24']").each do |node| - node.name = 'h2' - node.remove_attribute 'class' - end - - frag.xpath('/html/body/text()').wrap('

') - - text = File.open('data/urdu_info/002.html').read - frag = Nokogiri::HTML(text) - - -end - -def is_blank?(node) - (node.text? && node.content.strip == '') || (node.element? && node.name == 'br') -end - -def all_children_are_blank?(node) - node.children.all?{|child| is_blank?(child) } - # Here you see the convenience of monkeypatching... sometimes. -end - - -1.upto(114).each do |n| -n = n.to_s.rjust(3, '0') - -frag = Nokogiri::HTML(File.open("data/urdu_info/#{n}.html").read) - -#1 move style24 to body -frag.css('.style24').each do |node| - while (node.parent && node.parent.name != 'body') do - p = node.parent - puts "replaces #{p.name}" - p.replace(p.children.to_html.to_s.strip) - node = p - end -end - -#replace all tag expect span.style24 -frag.search-v2('//*/*').each do |node| - next if ['a', 'body'].include?(node.name) || (node.name == 'span' && ['style24', 'style41'].include?(node.attr("class"))) - - puts "replacing #{node.name}" - node.replace(node.content.to_s.strip) -end - -#replace span with h2 -frag.xpath("//span[@class='style24']").each do |node| - node.name = 'h2' - node.remove_attribute 'class' -end - -#update style41 to indopak -frag.xpath("//span[@class='style41']").each do |node| - node.set_attribute 'class', 'ar indopak' -end - -#wrap all text with p -frag.xpath('/html/body/text()').wrap('

') - -#remove empty -frag.css("p").find_all{|p| all_children_are_blank?(p) }.each do |p| - p.remove -end - -File.open "data/urdu_fixed6/#{n}.html", 'wb' do |file| file << frag.search-v2('body').children.to_html end -end - -#Tamil -1.upto(114) do |number| - content = agent.get("http://truevisionmedia.org/ift-chennai/tafheem-tamil/intro/intro3.htm").search-v2("body p")[1].to_html - - File.open("data/tamil_info/#{number}.html", 'wb') do |file| file << content - end - end -=end \ No newline at end of file