diff --git a/scraper/src/config/config_validator.py b/scraper/src/config/config_validator.py index f432d0d..70d2291 100644 --- a/scraper/src/config/config_validator.py +++ b/scraper/src/config/config_validator.py @@ -25,6 +25,11 @@ def validate(self): list): raise Exception('stop_urls should be list') + # Custom settings must be a dict + if self.config.custom_settings and not isinstance(self.config.custom_settings, + dict): + raise Exception('custom_settings must be a dictionary') + if self.config.js_render and not isinstance(self.config.js_render, bool): raise Exception('js_render should be boolean') diff --git a/scraper/src/index.py b/scraper/src/index.py index 2b06928..7472544 100644 --- a/scraper/src/index.py +++ b/scraper/src/index.py @@ -40,6 +40,7 @@ def run_config(config): config.app_id, config.api_key, config.index_uid, + config.custom_settings ) root_module = 'src.' if __name__ == '__main__' else 'scraper.src.' diff --git a/scraper/src/meilisearch_helper.py b/scraper/src/meilisearch_helper.py index 1e40e06..3383f90 100644 --- a/scraper/src/meilisearch_helper.py +++ b/scraper/src/meilisearch_helper.py @@ -101,11 +101,12 @@ class MeiliSearchHelper: 'acceptNewFields': False } - def __init__(self, host_url, api_key, index_uid): + def __init__(self, host_url, api_key, index_uid, custom_settings): self.meilisearch_client = meilisearch.Client(host_url, api_key) self.__delete_and_create_index(index_uid) self.meilisearch_index = self.__delete_and_create_index(index_uid) - self.meilisearch_index.update_settings(MeiliSearchHelper.SETTINGS) + settings = {**MeiliSearchHelper.SETTINGS, **custom_settings} + self.meilisearch_index.update_settings(settings) def add_records(self, records, url, from_sitemap): """Add new records to the index""" diff --git a/scraper/src/tests/config_loader/abstract.py b/scraper/src/tests/config_loader/abstract.py index 198cf6e..c9f230f 100644 --- a/scraper/src/tests/config_loader/abstract.py +++ b/scraper/src/tests/config_loader/abstract.py @@ -7,7 +7,7 @@ def config(additional_config={}): 'allowed_domains': 'allowed_domains', 'api_key': 'api_key', 'app_id': 'app_id', - 'custom_settings': 'custom_settings', + 'custom_settings': {}, 'hash_strategy': 'hash_strategy', 'index_uid': 'index_uid', 'selectors': [],