From cdf3f5707cefc800e1653e586cf2b77b451a3524 Mon Sep 17 00:00:00 2001 From: Bas Verlooy Date: Mon, 4 Dec 2023 15:28:32 +0100 Subject: [PATCH] Remove progress bar from datasource.py --- src/alexandria3k/data_source.py | 46 ++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/src/alexandria3k/data_source.py b/src/alexandria3k/data_source.py index 6c6b0794..c1627b80 100644 --- a/src/alexandria3k/data_source.py +++ b/src/alexandria3k/data_source.py @@ -61,7 +61,9 @@ class StreamingTable: """An apsw table streaming over data of the supplied table metadata""" - def __init__(self, table_meta, table_dict, data_source, sample=lambda x: True): + def __init__( + self, table_meta, table_dict, data_source, sample=lambda x: True + ): """Not part of the apsw VTTable interface""" self.table_meta = table_meta self.table_dict = table_dict @@ -139,7 +141,8 @@ def BestIndex(self, constraints, _orderbys): used_constraints.append((0, False)) index_number |= CONTAINER_INDEX elif ( - column == ROWID_COLUMN and operation == apsw.SQLITE_INDEX_CONSTRAINT_EQ + column == ROWID_COLUMN + and operation == apsw.SQLITE_INDEX_CONSTRAINT_EQ ): # Pass value to Filter as constraint_arg[1], and do not # require the engine to perform extra checks (exact match) @@ -157,7 +160,8 @@ def BestIndex(self, constraints, _orderbys): index_number, None, # index name (second argument to Filter) False, # results are not in orderbys order - 2000 / index_number, # about 2000 disk i/o (8M file / 4k block) + 2000 + / index_number, # about 2000 disk i/o (8M file / 4k block) ) return None @@ -298,7 +302,9 @@ def Next(self): return self.file_index += 1 - self.items = self.get_file_cache().read(self.table.data_source[self.file_index]) + self.items = self.get_file_cache().read( + self.table.data_source[self.file_index] + ) self.eof = False # The single file has been read. Set EOF in next Next call self.file_read = True @@ -322,7 +328,9 @@ def create_index(self, table, column): return self.database.execute( - log_sql(f"""CREATE INDEX {table}_{column}_idx ON {table}({column})""") + log_sql( + f"""CREATE INDEX {table}_{column}_idx ON {table}({column})""" + ) ) self.indexes.add((table, column)) @@ -396,7 +404,9 @@ def __init__( try: (db_name, db_path) = db_spec.split(":", 1) except ValueError: - fail(f"Invalid database specification: '{db_spec}'; expected name:path") + fail( + f"Invalid database specification: '{db_spec}'; expected name:path" + ) attach_command = f"ATTACH DATABASE '{db_path}' AS {db_name}" self.vdb.execute(log_sql(attach_command)) self.attached_databases.append(db_name) @@ -405,7 +415,9 @@ def __init__( # Create virtual table placeholders for table in tables: self.vdb.execute( - log_sql(f"CREATE VIRTUAL TABLE {table.get_name()} USING filesource()") + log_sql( + f"CREATE VIRTUAL TABLE {table.get_name()} USING filesource()" + ) ) def get_table_meta_by_name(self, name): @@ -623,7 +635,9 @@ def set_join_columns(): table_name = parent_table_name # print("ADD COLUMNS ", to_add) for table, column in to_add: - DataSource.add_column(self.query_and_population_columns, table, column) + DataSource.add_column( + self.query_and_population_columns, table, column + ) def query_and_population_tables(): """Return a sequence consisting of the tables required @@ -665,7 +679,9 @@ def joined_tables(table_names, rename_temp): self.index_manager.create_index( f"temp_{parent_table_name}", primary_key ) - self.index_manager.create_index(f"temp_{table_name}", foreign_key) + self.index_manager.create_index( + f"temp_{table_name}", foreign_key + ) return result def partition_condition(partition_index): @@ -678,7 +694,9 @@ def partition_condition(partition_index): else f"{table}.container_id = {partition_index}" ) - def populate_only_root_table(table, partition_index, selection_condition): + def populate_only_root_table( + table, partition_index, selection_condition + ): """Populate the root table, when no other tables will be needed""" columns = ", ".join( @@ -760,7 +778,9 @@ def create_database_schema(columns): pdb = sqlite3.connect(database_path) pdb.close() - self.vdb.execute(log_sql(f"ATTACH DATABASE '{database_path}' AS populated")) + self.vdb.execute( + log_sql(f"ATTACH DATABASE '{database_path}' AS populated") + ) set_fast_writing(self.vdb, "populated") self.index_manager = _IndexManager(self.vdb, self.root_name) @@ -807,7 +827,9 @@ def create_matched_tables(matched_tables): columns = set.union(columns, query_columns_of_table) column_list = ", ".join(columns) - self.vdb.execute(log_sql(f"""DROP TABLE IF EXISTS temp_{table}""")) + self.vdb.execute( + log_sql(f"""DROP TABLE IF EXISTS temp_{table}""") + ) create = f"""CREATE TEMP TABLE temp_{table} AS SELECT {column_list} FROM {table} WHERE container_id = {i}"""