diff --git a/es/basesqlalchemy.py b/es/basesqlalchemy.py index 4aa6900..47431d2 100644 --- a/es/basesqlalchemy.py +++ b/es/basesqlalchemy.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals import logging +from typing import List import es from es import exceptions @@ -145,10 +146,16 @@ def get_schema_names(self, connection, **kwargs): def has_table(self, connection, table_name, schema=None): return table_name in self.get_table_names(connection, schema) - def get_table_names(self, connection, schema=None, **kwargs): + def get_table_names(self, connection, schema=None, **kwargs) -> List[str]: query = "SHOW TABLES" result = connection.execute(query) - return [row.name for row in result if row.name[0] != "."] + # return a list of table names exclude hidden and empty indexes + return [ + table.name + for table in result + if table.name[0] != "." + and len(self.get_columns(connection, table.name)) > 0 + ] def get_view_names(self, connection, schema=None, **kwargs): return [] diff --git a/es/elastic/api.py b/es/elastic/api.py index 645541d..94dd317 100644 --- a/es/elastic/api.py +++ b/es/elastic/api.py @@ -4,7 +4,7 @@ from __future__ import unicode_literals import re -from typing import Dict +from typing import Any, Dict, Optional from elasticsearch import Elasticsearch, exceptions as es_exceptions from es import exceptions @@ -12,14 +12,14 @@ def connect( - host="localhost", - port=9200, - path="", - scheme="http", - user=None, - password=None, - context=None, - **kwargs, + host: str = "localhost", + port: int = 9200, + path: str = "", + scheme: str = "http", + user: Optional[str] = None, + password: Optional[str] = None, + context: Optional[Dict] = None, + **kwargs: Any, ): """ Constructor for creating a connection to the database. @@ -156,7 +156,7 @@ def get_array_type_columns(self, table_name: str) -> "Cursor": """ array_columns = [] try: - resp = self.es.search(index=table_name, size=1) + response = self.es.search(index=table_name, size=1) except es_exceptions.ConnectionError as e: raise exceptions.OperationalError( f"Error connecting to {self.url}: {e.info}" @@ -166,12 +166,15 @@ def get_array_type_columns(self, table_name: str) -> "Cursor": f"Error ({e.error}): {e.info['error']['reason']}" ) try: - _source = resp["hits"]["hits"][0]["_source"] + if response["hits"]["total"]["value"] == 0: + source = {} + else: + source = response["hits"]["hits"][0]["_source"] except KeyError as e: raise exceptions.DataError( f"Error inferring array type columns {self.url}: {e}" ) - for col_name, value in _source.items(): + for col_name, value in source.items(): # If it's a list (ES Array add to cursor) if isinstance(value, list): if len(value) > 0: diff --git a/es/opendistro/api.py b/es/opendistro/api.py index 4cca0f9..7706adb 100644 --- a/es/opendistro/api.py +++ b/es/opendistro/api.py @@ -5,6 +5,7 @@ import csv import re +from typing import Any, Dict, Optional # pragma: no cover from elasticsearch import Elasticsearch from es import exceptions @@ -13,14 +14,14 @@ def connect( - host="localhost", - port=443, - path="", - scheme="https", - user=None, - password=None, - context=None, - **kwargs, + host: str = "localhost", + port: int = 443, + path: str = "", + scheme: str = "https", + user: Optional[str] = None, + password: Optional[str] = None, + context: Optional[Dict] = None, + **kwargs: Any, ): # pragma: no cover """ Constructor for creating a connection to the database. diff --git a/es/tests/fixtures/fixtures.py b/es/tests/fixtures/fixtures.py index 896767d..7157469 100644 --- a/es/tests/fixtures/fixtures.py +++ b/es/tests/fixtures/fixtures.py @@ -103,3 +103,7 @@ def import_flights(base_url): def import_data1(base_url): path = os.path.join(os.path.dirname(__file__), "data1.json") import_file_to_es(base_url, path, "data1") + + +def import_empty_index(base_url): + set_index_replica_zero(base_url, "empty_index") diff --git a/es/tests/test_0_fixtures.py b/es/tests/test_0_fixtures.py index 3cd2ab6..9208f16 100644 --- a/es/tests/test_0_fixtures.py +++ b/es/tests/test_0_fixtures.py @@ -1,7 +1,12 @@ import os import unittest -from .fixtures.fixtures import delete_index, import_data1, import_flights +from .fixtures.fixtures import ( + delete_index, + import_data1, + import_empty_index, + import_flights, +) BASE_URL = "http://localhost:9200" @@ -17,3 +22,7 @@ def test_data_flights(self): def test_data_data1(self): delete_index(self.base_url, "data1") import_data1(self.base_url) + + def test_data_empty_index(self): + delete_index(self.base_url, "empty_index") + import_empty_index(self.base_url)