From 7bc85047e6634281a67815b003f01d6b3c183b8b Mon Sep 17 00:00:00 2001 From: Rick Klomp Date: Mon, 29 Aug 2022 12:02:56 +0200 Subject: [PATCH 1/5] bugfix: --main-schema wasn't working; db procedures relied on the default --- sql/common-tables.sql | 36 +++++++++---------- sql/templates/create-changes-functions.sql | 4 +-- .../create-entrypoint-changes-functions.sql | 8 ++--- sql/templates/create-function-shortcuts.sql | 8 ++--- sql/templates/create-snapshot-functions.sql | 2 +- sql/templates/repopulate-changes-derived.sql | 14 ++++---- sql/templates/repopulate-snapshot-derived.sql | 8 ++--- sql/templates/update-changes-derived.sql | 18 +++++----- sql/templates/update-snapshot-derived.sql | 8 ++--- src/sql/db.rs | 27 +++++++++----- src/sql/inserter.rs | 2 +- src/sql/postgresql_generator.rs | 18 ++++++++-- 12 files changed, 89 insertions(+), 64 deletions(-) diff --git a/sql/common-tables.sql b/sql/common-tables.sql index fa2dd60..4b50c99 100644 --- a/sql/common-tables.sql +++ b/sql/common-tables.sql @@ -92,10 +92,10 @@ CREATE VIEW txs_ordered AS ( ctx.level, meta.baked_at as level_timestamp, tx.* - FROM que_pasa.txs tx - JOIN que_pasa.tx_contexts ctx + FROM txs tx + JOIN tx_contexts ctx ON ctx.id = tx.tx_context_id - JOIN que_pasa.levels meta + JOIN levels meta ON meta.level = ctx.level ORDER BY ordering ); @@ -136,7 +136,7 @@ CREATE TABLE bigmap_keys( ); -CREATE OR REPLACE FUNCTION que_pasa.last_context_at(lvl INT) RETURNS TABLE (tx_context_id BIGINT, level INT, operation_group_number INT, operation_number INT, content_number INT, internal_number INT) +CREATE OR REPLACE FUNCTION "{main_schema}".last_context_at(lvl INT) RETURNS TABLE (tx_context_id BIGINT, level INT, operation_group_number INT, operation_number INT, content_number INT, internal_number INT) AS $$ SELECT ctx.id as tx_context_id, @@ -145,17 +145,17 @@ AS $$ operation_number, content_number, internal_number - FROM que_pasa.tx_contexts AS ctx + FROM "{main_schema}".tx_contexts AS ctx WHERE id = ( SELECT id - FROM que_pasa.tx_contexts + FROM "{main_schema}".tx_contexts WHERE level <= lvl ORDER BY level DESC, operation_group_number DESC, operation_number DESC, content_number DESC, COALESCE(internal_number, -1) DESC LIMIT 1 ) $$ LANGUAGE SQL; -CREATE OR REPLACE FUNCTION que_pasa.last_context_at(lvl INT, op_grp INT) RETURNS TABLE (tx_context_id BIGINT, level INT, operation_group_number INT, operation_number INT, content_number INT, internal_number INT) +CREATE OR REPLACE FUNCTION "{main_schema}".last_context_at(lvl INT, op_grp INT) RETURNS TABLE (tx_context_id BIGINT, level INT, operation_group_number INT, operation_number INT, content_number INT, internal_number INT) AS $$ SELECT ctx.id AS tx_context_id, @@ -164,10 +164,10 @@ AS $$ operation_number, content_number, internal_number - FROM que_pasa.tx_contexts AS ctx + FROM "{main_schema}".tx_contexts AS ctx WHERE id = ( SELECT id - FROM que_pasa.tx_contexts + FROM "{main_schema}".tx_contexts WHERE ARRAY[ level, operation_group_number] @@ -180,7 +180,7 @@ AS $$ ) $$ LANGUAGE SQL; -CREATE OR REPLACE FUNCTION que_pasa.last_context_at(lvl INT, op_grp INT, op INT) RETURNS TABLE (tx_context_id BIGINT, level INT, operation_group_number INT, operation_number INT, content_number INT, internal_number INT) +CREATE OR REPLACE FUNCTION "{main_schema}".last_context_at(lvl INT, op_grp INT, op INT) RETURNS TABLE (tx_context_id BIGINT, level INT, operation_group_number INT, operation_number INT, content_number INT, internal_number INT) AS $$ SELECT ctx.id, @@ -189,10 +189,10 @@ AS $$ operation_number, content_number, internal_number - FROM que_pasa.tx_contexts AS ctx + FROM "{main_schema}".tx_contexts AS ctx WHERE id = ( SELECT id - FROM que_pasa.tx_contexts + FROM "{main_schema}".tx_contexts WHERE ARRAY[ level, operation_group_number, @@ -207,7 +207,7 @@ AS $$ ) $$ LANGUAGE SQL; -CREATE OR REPLACE FUNCTION que_pasa.last_context_at(lvl INT, op_grp INT, op INT, content INT) RETURNS TABLE (tx_context_id BIGINT, level INT, operation_group_number INT, operation_number INT, content_number INT, internal_number INT) +CREATE OR REPLACE FUNCTION "{main_schema}".last_context_at(lvl INT, op_grp INT, op INT, content INT) RETURNS TABLE (tx_context_id BIGINT, level INT, operation_group_number INT, operation_number INT, content_number INT, internal_number INT) AS $$ SELECT ctx.id AS tx_context_id, @@ -216,10 +216,10 @@ AS $$ operation_number, content_number, internal_number - FROM que_pasa.tx_contexts AS ctx + FROM "{main_schema}".tx_contexts AS ctx WHERE id = ( SELECT id - FROM que_pasa.tx_contexts + FROM "{main_schema}".tx_contexts WHERE ARRAY[ level, operation_group_number, @@ -236,7 +236,7 @@ AS $$ ) $$ LANGUAGE SQL; -CREATE OR REPLACE FUNCTION que_pasa.last_context_at(lvl INT, op_grp INT, op INT, content INT, internal INT) RETURNS TABLE (tx_context_id BIGINT, level INT, operation_group_number INT, operation_number INT, content_number INT, internal_number INT) +CREATE OR REPLACE FUNCTION "{main_schema}".last_context_at(lvl INT, op_grp INT, op INT, content INT, internal INT) RETURNS TABLE (tx_context_id BIGINT, level INT, operation_group_number INT, operation_number INT, content_number INT, internal_number INT) AS $$ SELECT ctx.id AS tx_context_id, @@ -245,10 +245,10 @@ AS $$ operation_number, content_number, internal_number - FROM que_pasa.tx_contexts AS ctx + FROM "{main_schema}".tx_contexts AS ctx WHERE id = ( SELECT id - FROM que_pasa.tx_contexts + FROM "{main_schema}".tx_contexts WHERE ARRAY[ level, operation_group_number, diff --git a/sql/templates/create-changes-functions.sql b/sql/templates/create-changes-functions.sql index df57a15..76717e3 100644 --- a/sql/templates/create-changes-functions.sql +++ b/sql/templates/create-changes-functions.sql @@ -17,8 +17,8 @@ AS $$ , LAST_VALUE(t.{{ col }}) OVER w AS {{ col }} {%- endfor %} FROM "{{ contract_schema }}"."{{ table }}_ordered" AS t - CROSS JOIN que_pasa.contracts AS contract - JOIN que_pasa.tx_contexts ctx + CROSS JOIN "{{ main_schema }}".contracts AS contract + JOIN "{{ main_schema }}".tx_contexts ctx ON ctx.id = t.tx_context_id AND ctx.contract = contract.address WHERE contract.name = '{{ contract_schema }}' diff --git a/sql/templates/create-entrypoint-changes-functions.sql b/sql/templates/create-entrypoint-changes-functions.sql index a503a90..4fb888b 100644 --- a/sql/templates/create-entrypoint-changes-functions.sql +++ b/sql/templates/create-entrypoint-changes-functions.sql @@ -28,16 +28,16 @@ BEGIN value->>'contract_address' AS address, value->>'table' AS "table" INTO source - FROM bigmap_meta_actions AS meta + FROM "{{ main_schema }}".bigmap_meta_actions AS meta WHERE action = 'alloc' AND meta.bigmap_id = ( SELECT (value->'source')::INT - FROM bigmap_meta_actions AS meta + FROM "{{ main_schema }}".bigmap_meta_actions AS meta WHERE meta.action = 'copy' AND meta.bigmap_id = bigmap_target AND tx_context_id = ( SELECT ctx.id - FROM tx_contexts AS ctx + FROM "{{ main_schema }}".tx_contexts AS ctx JOIN "{{ contract_schema }}"."{{ table }}" AS t ON t.tx_context_id = ctx.id WHERE ARRAY[ @@ -60,7 +60,7 @@ BEGIN SELECT name INTO source_schema - FROM contracts + FROM "{{ main_schema }}".contracts WHERE address = source.address; in_schema := source_schema; diff --git a/sql/templates/create-function-shortcuts.sql b/sql/templates/create-function-shortcuts.sql index 8dc0fba..f1646ef 100644 --- a/sql/templates/create-function-shortcuts.sql +++ b/sql/templates/create-function-shortcuts.sql @@ -13,7 +13,7 @@ AS $$ FROM ( SELECT "{{ contract_schema }}"."{{ table }}_{{ function_postfix }}"(ctx.level, ctx.operation_group_number, ctx.operation_number, ctx.content_number, ctx.internal_number) - FROM que_pasa.last_context_at(lvl) AS ctx + FROM "{{ main_schema }}".last_context_at(lvl) AS ctx ) q $$ LANGUAGE SQL; @@ -21,19 +21,19 @@ CREATE OR REPLACE FUNCTION "{{ contract_schema }}"."{{ table }}_{{ function_post AS $$ SELECT "{{ contract_schema }}"."{{ table }}_{{ function_postfix }}"(ctx.level, ctx.operation_group_number, ctx.operation_number, ctx.content_number, ctx.internal_number) - FROM que_pasa.last_context_at(lvl, op_grp) AS ctx + FROM "{{ main_schema }}".last_context_at(lvl, op_grp) AS ctx $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION "{{ contract_schema }}"."{{ table }}_{{ function_postfix }}"(lvl INT, op_grp INT, op INT) RETURNS TABLE ({% call unfold(typed_columns, "", false) %}) AS $$ SELECT "{{ contract_schema }}"."{{ table }}_{{ function_postfix }}"(ctx.level, ctx.operation_group_number, ctx.operation_number, ctx.content_number, ctx.internal_number) - FROM que_pasa.last_context_at(lvl, op_grp, op) AS ctx + FROM "{{ main_schema }}".last_context_at(lvl, op_grp, op) AS ctx $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION "{{ contract_schema }}"."{{ table }}_{{ function_postfix }}"(lvl INT, op_grp INT, op INT, content INT) RETURNS TABLE ({% call unfold(typed_columns, "", false) %}) AS $$ SELECT "{{ contract_schema }}"."{{ table }}_{{ function_postfix }}"(ctx.level, ctx.operation_group_number, ctx.operation_number, ctx.content_number, ctx.internal_number) - FROM que_pasa.last_context_at(lvl, op_grp, op, content) AS ctx + FROM "{{ main_schema }}".last_context_at(lvl, op_grp, op, content) AS ctx $$ LANGUAGE SQL; diff --git a/sql/templates/create-snapshot-functions.sql b/sql/templates/create-snapshot-functions.sql index 29aade9..f2afbaf 100644 --- a/sql/templates/create-snapshot-functions.sql +++ b/sql/templates/create-snapshot-functions.sql @@ -12,7 +12,7 @@ AS $$ SELECT ctx.id AS tx_context_id FROM "{{ contract_schema }}"."{{ table }}" AS t - JOIN que_pasa.tx_contexts ctx + JOIN "{{ main_schema }}".tx_contexts ctx ON ctx.id = t.tx_context_id WHERE ARRAY[ ctx.level, diff --git a/sql/templates/repopulate-changes-derived.sql b/sql/templates/repopulate-changes-derived.sql index 7bd9e00..e3b7a52 100644 --- a/sql/templates/repopulate-changes-derived.sql +++ b/sql/templates/repopulate-changes-derived.sql @@ -32,12 +32,12 @@ FROM ( t.* FROM "{{ contract_schema }}"."{{ table }}" t WHERE t.bigmap_id NOT IN ( - SELECT bigmap_id FROM que_pasa.bigmap_meta_actions WHERE action = 'clear' + SELECT bigmap_id FROM "{{ main_schema }}".bigmap_meta_actions WHERE action = 'clear' ) ) t - JOIN tx_contexts ctx + JOIN "{{ main_schema }}".tx_contexts ctx ON ctx.id = t.tx_context_id - JOIN levels level_meta + JOIN "{{ main_schema }}".levels level_meta ON level_meta.level = ctx.level ORDER BY {% call unfold(indices, "t", false) %}, @@ -95,10 +95,10 @@ FROM ( {%- for col in columns %} , LAST_VALUE(t.{{ col }}) OVER w AS {{ col }} {%- endfor %} - FROM que_pasa.bigmap_meta_actions AS bigmap_meta + FROM "{{ main_schema }}".bigmap_meta_actions AS bigmap_meta JOIN "{{ contract_schema }}"."{{ table }}" t ON t.bigmap_id = bigmap_meta.bigmap_id - JOIN tx_contexts ctx + JOIN "{{ main_schema }}".tx_contexts ctx ON ctx.id = t.tx_context_id WHERE bigmap_meta.action = 'clear' WINDOW w AS ( @@ -120,9 +120,9 @@ FROM ( WHERE NOT t.latest_deleted AND t2 IS NULL ) t -- t with bigmap clears unfolded - JOIN tx_contexts ctx + JOIN "{{ main_schema }}".tx_contexts ctx ON ctx.id = t.tx_context_id - JOIN levels level_meta + JOIN "{{ main_schema }}".levels level_meta ON level_meta.level = ctx.level ) q; diff --git a/sql/templates/repopulate-snapshot-derived.sql b/sql/templates/repopulate-snapshot-derived.sql index b3f990d..247b914 100644 --- a/sql/templates/repopulate-snapshot-derived.sql +++ b/sql/templates/repopulate-snapshot-derived.sql @@ -26,7 +26,7 @@ FROM ( ctx.id, ctx.level FROM "{{ contract_schema }}"."{{ parent_table }}" t - JOIN tx_contexts ctx + JOIN "{{ main_schema }}".tx_contexts ctx ON ctx.id = t.tx_context_id ORDER BY ctx.level DESC, @@ -36,7 +36,7 @@ FROM ( COALESCE(ctx.internal_number, -1) DESC LIMIT 1 ) last_ctx - JOIN levels level_meta + JOIN "{{ main_schema }}".levels level_meta ON level_meta.level = last_ctx.level WHERE t.tx_context_id = last_ctx.id ) q; @@ -64,8 +64,8 @@ FROM ( t.tx_context_id {% call unfold(columns, "t", true) %} FROM "{{ contract_schema }}"."{{ table }}" t - JOIN tx_contexts ctx + JOIN "{{ main_schema }}".tx_contexts ctx ON ctx.id = t.tx_context_id - JOIN levels level_meta + JOIN "{{ main_schema }}".levels level_meta ON level_meta.level = ctx.level ) q; diff --git a/sql/templates/update-changes-derived.sql b/sql/templates/update-changes-derived.sql index 3a644df..f708e4d 100644 --- a/sql/templates/update-changes-derived.sql +++ b/sql/templates/update-changes-derived.sql @@ -12,7 +12,7 @@ DELETE FROM "{{ contract_schema }}"."{{ table }}_live" WHERE bigmap_id IN ( SELECT bigmap_id - FROM que_pasa.bigmap_meta_actions + FROM "{{ main_schema }}".bigmap_meta_actions WHERE tx_context_id in ({% call unfold(tx_context_ids, "", false) %}) AND action = 'clear' ); @@ -25,9 +25,9 @@ WHERE id IN ( SELECT DISTINCT {% call unfold(indices, "t", false) %} FROM "{{ contract_schema }}"."{{ table }}" t - JOIN tx_contexts ctx + JOIN "{{ main_schema }}".tx_contexts ctx ON ctx.id = t.tx_context_id - JOIN levels level_meta + JOIN "{{ main_schema }}".levels level_meta ON level_meta.level = ctx.level WHERE t.tx_context_id IN ({% call unfold(tx_context_ids, "", false) %}) ) as overwritten_indices @@ -55,9 +55,9 @@ FROM ( level_meta.baked_at AS level_timestamp, t.* FROM "{{ contract_schema }}"."{{ table }}" t - JOIN tx_contexts ctx + JOIN "{{ main_schema }}".tx_contexts ctx ON ctx.id = t.tx_context_id - JOIN levels level_meta + JOIN "{{ main_schema }}".levels level_meta ON level_meta.level = ctx.level WHERE t.tx_context_id IN ({% call unfold(tx_context_ids, "", false) %}) ORDER BY @@ -122,10 +122,10 @@ FROM ( {%- for col in columns %} , LAST_VALUE(t.{{ col }}) OVER w as {{ col }} {%- endfor %} - FROM que_pasa.bigmap_meta_actions bigmap_meta + FROM "{{ main_schema }}".bigmap_meta_actions bigmap_meta JOIN "{{ contract_schema }}"."{{ table }}" t ON t.bigmap_id = bigmap_meta.bigmap_id - JOIN tx_contexts ctx + JOIN "{{ main_schema }}".tx_contexts ctx ON ctx.id = t.tx_context_id WHERE bigmap_meta.tx_context_id IN ({% call unfold(tx_context_ids, "", false) %}) AND bigmap_meta.action = 'clear' @@ -148,8 +148,8 @@ FROM ( WHERE NOT t.latest_deleted AND t2 IS NULL ) t -- t with bigmap clears unfolded - JOIN tx_contexts ctx + JOIN "{{ main_schema }}".tx_contexts ctx ON ctx.id = t.tx_context_id - JOIN levels level_meta + JOIN "{{ main_schema }}".levels level_meta ON level_meta.level = ctx.level ) t; diff --git a/sql/templates/update-snapshot-derived.sql b/sql/templates/update-snapshot-derived.sql index 970f4c0..c897ab3 100644 --- a/sql/templates/update-snapshot-derived.sql +++ b/sql/templates/update-snapshot-derived.sql @@ -26,7 +26,7 @@ FROM ( ctx.id, ctx.level FROM "{{ contract_schema }}"."{{ parent_table }}" t - JOIN tx_contexts ctx + JOIN "{{ main_schema }}".tx_contexts ctx ON ctx.id = t.tx_context_id ORDER BY ctx.level DESC, @@ -36,7 +36,7 @@ FROM ( COALESCE(ctx.internal_number, -1) DESC LIMIT 1 ) last_ctx - JOIN levels level_meta + JOIN "{{ main_schema }}".levels level_meta ON level_meta.level = last_ctx.level WHERE t.tx_context_id = last_ctx.id ) t; @@ -68,9 +68,9 @@ FROM ( t.tx_context_id {% call unfold(columns, "t", true) %} FROM "{{ contract_schema }}"."{{ table }}" t - JOIN tx_contexts ctx + JOIN "{{ main_schema }}".tx_contexts ctx ON ctx.id = t.tx_context_id - JOIN levels level_meta + JOIN "{{ main_schema }}".levels level_meta ON level_meta.level = ctx.level WHERE t.tx_context_id IN ({% call unfold(tx_context_ids, "", false) %}) ) t; diff --git a/src/sql/db.rs b/src/sql/db.rs index dffe3de..145c1eb 100644 --- a/src/sql/db.rs +++ b/src/sql/db.rs @@ -32,6 +32,7 @@ pub(crate) enum IndexerMode { #[derive(Template)] #[template(path = "repopulate-snapshot-derived.sql", escape = "none")] struct RepopulateSnapshotDerivedTmpl<'a> { + main_schema: &'a str, contract_schema: &'a str, table: &'a str, parent_table: &'a str, @@ -40,6 +41,7 @@ struct RepopulateSnapshotDerivedTmpl<'a> { #[derive(Template)] #[template(path = "repopulate-changes-derived.sql", escape = "none")] struct RepopulateChangesDerivedTmpl<'a> { + main_schema: &'a str, contract_schema: &'a str, table: &'a str, columns: &'a [String], @@ -48,6 +50,7 @@ struct RepopulateChangesDerivedTmpl<'a> { #[derive(Template)] #[template(path = "update-snapshot-derived.sql", escape = "none")] struct UpdateSnapshotDerivedTmpl<'a> { + main_schema: &'a str, contract_schema: &'a str, table: &'a str, parent_table: &'a str, @@ -57,6 +60,7 @@ struct UpdateSnapshotDerivedTmpl<'a> { #[derive(Template)] #[template(path = "update-changes-derived.sql", escape = "none")] struct UpdateChangesDerivedTmpl<'a> { + main_schema: &'a str, contract_schema: &'a str, table: &'a str, columns: &'a [String], @@ -129,7 +133,8 @@ FROM indexer_state .as_str(), )?; conn.simple_query( - PostgresqlGenerator::create_common_tables().as_str(), + PostgresqlGenerator::create_common_tables(&self.main_schema) + .as_str(), )?; Ok(()) } @@ -175,11 +180,7 @@ WHERE table_schema = $1 table_i = i, table_total = tables.len(), ); - DBClient::repopulate_derived_table( - &mut tx, - &contract.cid, - table, - )?; + self.repopulate_derived_table(&mut tx, &contract.cid, table)?; } } tx.commit()?; @@ -187,6 +188,7 @@ WHERE table_schema = $1 } fn repopulate_derived_table( + &self, tx: &mut Transaction, contract_id: &ContractID, table: &Table, @@ -198,6 +200,7 @@ WHERE table_schema = $1 PostgresqlGenerator::table_parent_name(table) .unwrap_or(table.name.clone()); let tmpl = RepopulateSnapshotDerivedTmpl { + main_schema: &self.main_schema, contract_schema: &contract_id.name, table: &table.name, parent_table: &parent_table, @@ -206,6 +209,7 @@ WHERE table_schema = $1 tx.simple_query(&tmpl.render()?)?; } else { let tmpl = RepopulateChangesDerivedTmpl { + main_schema: &self.main_schema, contract_schema: &contract_id.name, table: &table.name, columns: &columns, @@ -218,6 +222,7 @@ WHERE table_schema = $1 } pub(crate) fn update_derived_tables( + &self, tx: &mut Transaction, contract: &relational::Contract, tx_contexts: &[TxContext], @@ -239,7 +244,7 @@ WHERE table_schema = $1 .iter() .any(|prefix| table.name.starts_with(prefix)) { - DBClient::update_derived_table( + self.update_derived_table( tx, &contract.cid, table, @@ -251,6 +256,7 @@ WHERE table_schema = $1 } fn update_derived_table( + &self, tx: &mut Transaction, contract_id: &ContractID, table: &Table, @@ -268,6 +274,7 @@ WHERE table_schema = $1 PostgresqlGenerator::table_parent_name(table) .unwrap_or(table.name.clone()); let tmpl = UpdateSnapshotDerivedTmpl { + main_schema: &self.main_schema, contract_schema: &contract_id.name, table: &table.name, parent_table: &parent_table, @@ -277,6 +284,7 @@ WHERE table_schema = $1 tx.simple_query(&tmpl.render()?)?; } else { let tmpl = UpdateChangesDerivedTmpl { + main_schema: &self.main_schema, contract_schema: &contract_id.name, table: &table.name, columns: &columns, @@ -351,7 +359,10 @@ CREATE SCHEMA IF NOT EXISTS "{contract_schema}"; contract_schema = contract.cid.name )); - let generator = PostgresqlGenerator::new(&contract.cid); + let generator = PostgresqlGenerator::new( + self.main_schema.clone(), + &contract.cid, + ); for table in &tables { let table_def = generator.create_table_definition(table)?; diff --git a/src/sql/inserter.rs b/src/sql/inserter.rs index 241efd3..a1ebc8e 100644 --- a/src/sql/inserter.rs +++ b/src/sql/inserter.rs @@ -136,7 +136,7 @@ fn insert_batch( if update_derived_tables { for (contract_id, (contract, ctxs)) in &batch.contract_tx_contexts { - DBClient::update_derived_tables( + dbcli.update_derived_tables( &mut db_tx, &contract, ctxs, diff --git a/src/sql/postgresql_generator.rs b/src/sql/postgresql_generator.rs index 42d3644..2382059 100644 --- a/src/sql/postgresql_generator.rs +++ b/src/sql/postgresql_generator.rs @@ -9,6 +9,7 @@ use crate::storage_structure::typing::{ExprTy, SimpleExprTy}; #[derive(Template)] #[template(path = "create-changes-functions.sql", escape = "none")] struct CreateChangesFunctionsTmpl<'a> { + main_schema: &'a str, contract_schema: &'a str, table: &'a str, columns: &'a [String], @@ -19,6 +20,7 @@ struct CreateChangesFunctionsTmpl<'a> { #[derive(Template)] #[template(path = "create-snapshot-functions.sql", escape = "none")] struct CreateSnapshotFunctionsTmpl<'a> { + main_schema: &'a str, contract_schema: &'a str, table: &'a str, columns: &'a [String], @@ -28,6 +30,7 @@ struct CreateSnapshotFunctionsTmpl<'a> { #[derive(Template)] #[template(path = "create-entrypoint-changes-functions.sql", escape = "none")] struct CreateEntrypointChangesFunctionsTmpl<'a> { + main_schema: &'a str, contract_schema: &'a str, table: &'a str, columns: &'a [String], @@ -37,6 +40,7 @@ struct CreateEntrypointChangesFunctionsTmpl<'a> { #[derive(Template)] #[template(path = "create-function-shortcuts.sql", escape = "none")] struct CreateFunctionShortcutsTmpl<'a> { + main_schema: &'a str, contract_schema: &'a str, table: &'a str, function_postfix: &'a str, @@ -45,12 +49,14 @@ struct CreateFunctionShortcutsTmpl<'a> { #[derive(Clone, Debug)] pub struct PostgresqlGenerator { + main_schema: String, contract_id: ContractID, } impl PostgresqlGenerator { - pub(crate) fn new(contract_id: &ContractID) -> Self { + pub(crate) fn new(main_schema: String, contract_id: &ContractID) -> Self { Self { + main_schema, contract_id: contract_id.clone(), } } @@ -167,12 +173,14 @@ impl PostgresqlGenerator { if table.contains_pointers() { let shallow_tmpl = CreateSnapshotFunctionsTmpl { + main_schema: &self.main_schema, contract_schema: contract_schema, table: &table.name, columns: &columns, typed_columns: &typed_columns, }; let shallow_shortcuts = CreateFunctionShortcutsTmpl { + main_schema: &self.main_schema, contract_schema: contract_schema, table: &table.name, function_postfix: "at", @@ -187,12 +195,14 @@ impl PostgresqlGenerator { deep_typed_columns.insert(0, "in_table TEXT".to_string()); deep_typed_columns.insert(0, "in_schema TEXT".to_string()); let deep_tmpl = CreateEntrypointChangesFunctionsTmpl { + main_schema: &self.main_schema, contract_schema: contract_schema, table: &table.name, columns: &columns, typed_columns: &deep_typed_columns, }; let deep_shortcuts = CreateFunctionShortcutsTmpl { + main_schema: &self.main_schema, contract_schema: contract_schema, table: &table.name, function_postfix: "at_deref", @@ -208,6 +218,7 @@ impl PostgresqlGenerator { } let shortcuts = CreateFunctionShortcutsTmpl { + main_schema: &self.main_schema, contract_schema: contract_schema, table: &table.name, function_postfix: "at", @@ -216,6 +227,7 @@ impl PostgresqlGenerator { if table.contains_snapshots() { let tmpl = CreateSnapshotFunctionsTmpl { + main_schema: &self.main_schema, contract_schema: contract_schema, table: &table.name, columns: &columns, @@ -225,6 +237,7 @@ impl PostgresqlGenerator { } let tmpl = CreateChangesFunctionsTmpl { + main_schema: &self.main_schema, contract_schema: contract_schema, table: &table.name, columns: &columns, @@ -383,9 +396,10 @@ impl PostgresqlGenerator { }).collect::>() } - pub(crate) fn create_common_tables() -> String { + pub(crate) fn create_common_tables(main_schema: &str) -> String { format!( include_str!("../../sql/common-tables.sql"), + main_schema = main_schema, quepasa_version = QUEPASA_VERSION, ) } From aa21a4d561f8e5be1432fa4b5f5d000e64c027ff Mon Sep 17 00:00:00 2001 From: Rick Klomp Date: Mon, 29 Aug 2022 12:04:18 +0200 Subject: [PATCH 2/5] adjust regression tests to use --main-schema feature --- script/regression_tests.bash | 22 +++++++++++----------- test/regression/1.query | 2 +- test/regression/2.query | 2 +- test/regression/3.query | 2 +- test/regression/4.query | 2 +- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/script/regression_tests.bash b/script/regression_tests.bash index 861ace1..302537a 100755 --- a/script/regression_tests.bash +++ b/script/regression_tests.bash @@ -53,10 +53,10 @@ function query { function assert { query_id=0 - query 'select count(1) from que_pasa.tx_contexts' || exit 1 - query 'select count(1) from que_pasa.contracts' || exit 1 - query 'select count(1) from que_pasa.contract_levels' || exit 1 - query 'select count(1) from que_pasa.contract_deps' || exit 1 + query 'select count(1) from "custom_Main_Schema".tx_contexts' || exit 1 + query 'select count(1) from "custom_Main_Schema".contracts' || exit 1 + query 'select count(1) from "custom_Main_Schema".contract_levels' || exit 1 + query 'select count(1) from "custom_Main_Schema".contract_deps' || exit 1 query 'select administrator, all_tokens, paused, level, level_timestamp from "KT1RJ6PbjHpwc3M5rw5s2Nbmefwbuwbdxton"."storage_live"' || exit 1 query 'select level, level_timestamp, idx_address, idx_nat, nat from "KT1RJ6PbjHpwc3M5rw5s2Nbmefwbuwbdxton"."storage.ledger_live" order by idx_address, idx_nat' || exit 1 @@ -118,15 +118,15 @@ EOF export RUST_BACKTRACE=1 -cargo run -- --index-all-contracts -l 1500000-1500001 || exit 1 -cargo run --features regression_force_update_derived -- --index-all-contracts -l 1500002-1500005 || exit 1 -cargo run --features regression_force_update_derived -- --index-all-contracts -l 1700002-1700005 || exit 1 +cargo run -- --main-schema custom_Main_Schema --index-all-contracts -l 1500000-1500001 || exit 1 +cargo run --features regression_force_update_derived -- --main-schema custom_Main_Schema --index-all-contracts -l 1500002-1500005 || exit 1 +cargo run --features regression_force_update_derived -- --main-schema custom_Main_Schema --index-all-contracts -l 1700002-1700005 || exit 1 # the latter has a delete bigmap, the first 3 have rows indexed of the deleted bigmap -cargo run --features regression_force_update_derived -- --index-all-contracts -l 1768431 || exit 1 -cargo run --features regression_force_update_derived -- --index-all-contracts -l 1768503 || exit 1 -cargo run --features regression_force_update_derived -- --index-all-contracts -l 1768506 || exit 1 -cargo run --features regression_force_update_derived -- --index-all-contracts -l 1768606 || exit 1 +cargo run --features regression_force_update_derived -- --main-schema custom_Main_Schema --index-all-contracts -l 1768431 || exit 1 +cargo run --features regression_force_update_derived -- --main-schema custom_Main_Schema --index-all-contracts -l 1768503 || exit 1 +cargo run --features regression_force_update_derived -- --main-schema custom_Main_Schema --index-all-contracts -l 1768506 || exit 1 +cargo run --features regression_force_update_derived -- --main-schema custom_Main_Schema --index-all-contracts -l 1768606 || exit 1 if [[ "$MODE" == "inspect" ]]; then psql diff --git a/test/regression/1.query b/test/regression/1.query index 10cc8c6..b7d295b 100644 --- a/test/regression/1.query +++ b/test/regression/1.query @@ -1,4 +1,4 @@ -select count(1) from que_pasa.tx_contexts; +select count(1) from "custom_Main_Schema".tx_contexts; count ------- 1132 diff --git a/test/regression/2.query b/test/regression/2.query index 93a658a..279ac0f 100644 --- a/test/regression/2.query +++ b/test/regression/2.query @@ -1,4 +1,4 @@ -select count(1) from que_pasa.contracts; +select count(1) from "custom_Main_Schema".contracts; count ------- 147 diff --git a/test/regression/3.query b/test/regression/3.query index 7b5af3d..ee049ec 100644 --- a/test/regression/3.query +++ b/test/regression/3.query @@ -1,4 +1,4 @@ -select count(1) from que_pasa.contract_levels; +select count(1) from "custom_Main_Schema".contract_levels; count ------- 313 diff --git a/test/regression/4.query b/test/regression/4.query index 93469b6..b203f81 100644 --- a/test/regression/4.query +++ b/test/regression/4.query @@ -1,4 +1,4 @@ -select count(1) from que_pasa.contract_deps; +select count(1) from "custom_Main_Schema".contract_deps; count ------- 1 From 5b4f1acf77abe84462994eeb0d3bbaf77b2a0002 Mon Sep 17 00:00:00 2001 From: Rick Klomp Date: Mon, 29 Aug 2022 12:06:03 +0200 Subject: [PATCH 3/5] bump que pasa version up --- Cargo.lock | 2 +- Cargo.toml | 2 +- run_dockerized.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 67e685c..561c4af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1361,7 +1361,7 @@ dependencies = [ [[package]] name = "que-pasa" -version = "1.2.5" +version = "1.2.6" dependencies = [ "anyhow", "askama", diff --git a/Cargo.toml b/Cargo.toml index d53f643..536ca39 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "que-pasa" -version = "1.2.5" +version = "1.2.6" authors = ["Rick Klomp "] edition = "2018" diff --git a/run_dockerized.sh b/run_dockerized.sh index a9c44c0..00dd96a 100755 --- a/run_dockerized.sh +++ b/run_dockerized.sh @@ -2,5 +2,5 @@ docker run \ --network host \ - ghcr.io/tzconnectberlin/que-pasa:1.2.5 \ + ghcr.io/tzconnectberlin/que-pasa:1.2.6 \ "$@" From f17b9459dcd6e648ca632111dcf3b7414ab43b0b Mon Sep 17 00:00:00 2001 From: Rick Klomp Date: Mon, 29 Aug 2022 12:08:49 +0200 Subject: [PATCH 4/5] fix tests --- src/highlevel.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/highlevel.rs b/src/highlevel.rs index 1826b7b..cd386af 100644 --- a/src/highlevel.rs +++ b/src/highlevel.rs @@ -1221,10 +1221,13 @@ fn test_generate() { .build_relational_ast(&context, &type_ast) .unwrap(); println!("{:#?}", rel_ast); - let generator = PostgresqlGenerator::new(&ContractID { - name: "testcontract".to_string(), - address: "".to_string(), - }); + let generator = PostgresqlGenerator::new( + "some_main_schema".to_string(), + &ContractID { + name: "testcontract".to_string(), + address: "".to_string(), + }, + ); let mut builder = crate::sql::table_builder::TableBuilder::new("storage"); builder.populate(&rel_ast); let mut sorted_tables: Vec<_> = builder.tables.iter().collect(); From 742223a5bbccd3a9ef9b358ca5083fcbc8dddb27 Mon Sep 17 00:00:00 2001 From: Rick Klomp Date: Mon, 29 Aug 2022 12:21:49 +0200 Subject: [PATCH 5/5] fix linter warnings --- script/regression_tests.bash | 2 +- src/config.rs | 2 +- src/highlevel.rs | 14 ++++++++------ src/main.rs | 2 +- src/octez/block.rs | 6 ++---- src/octez/node.rs | 4 ++-- src/sql/db.rs | 4 ++-- src/sql/inserter.rs | 4 ++-- src/sql/postgresql_generator.rs | 16 ++++++++-------- src/sql/table_builder.rs | 11 ++++------- src/storage_structure/relational.rs | 2 +- src/storage_update/processor.rs | 6 +++--- 12 files changed, 35 insertions(+), 38 deletions(-) diff --git a/script/regression_tests.bash b/script/regression_tests.bash index 302537a..76dfb4f 100755 --- a/script/regression_tests.bash +++ b/script/regression_tests.bash @@ -143,7 +143,7 @@ if [[ "$MODE" == "generate" ]]; then fi # verifying here that the repopulate also works with deleted bigmap rows -cargo run -- --index-all-contracts -l 1768606 || exit 1 +cargo run -- --main-schema custom_Main_Schema --index-all-contracts -l 1768606 || exit 1 assert diff --git a/src/config.rs b/src/config.rs index e8f2265..3c85e8f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -216,7 +216,7 @@ unfortunately.") if matches.is_present("contracts") { config.contracts.extend( matches.values_of("contracts").unwrap().map(|s| { - match s.split_once("=") { + match s.split_once('=') { Some((name, address)) => ContractID { name: name.to_string(), address: address.to_string(), diff --git a/src/highlevel.rs b/src/highlevel.rs index cd386af..3b381a7 100644 --- a/src/highlevel.rs +++ b/src/highlevel.rs @@ -652,7 +652,11 @@ impl Executor { ); } - for (_, contract) in &self.mutexed_state.get_contracts()? { + for contract in self + .mutexed_state + .get_contracts()? + .values() + { self.dbcli .repopulate_derived_tables(contract)?; } @@ -845,8 +849,7 @@ impl Executor { let prev = self.dbcli.get_level(level - 1)?; if let Some(db_prev_hash) = prev .as_ref() - .map(|l| l.hash.as_ref()) - .flatten() + .and_then(|l| l.hash.as_ref()) { if db_prev_hash != prev_hash { let forked_lvl = prev.as_ref().unwrap().level; @@ -861,8 +864,7 @@ impl Executor { let next = self.dbcli.get_level(level + 1)?; if let Some(db_next_prev_hash) = next .as_ref() - .map(|l| l.prev_hash.as_ref()) - .flatten() + .and_then(|l| l.prev_hash.as_ref()) { if db_next_prev_hash != hash { let forked_lvl = next.as_ref().unwrap().level; @@ -983,7 +985,7 @@ impl Executor { let mut storage_processor = self.get_storage_processor()?; storage_processor.set_stats_logger(self.stats.clone()); storage_processor - .process_block(block, diffs, &contract) + .process_block(block, diffs, contract) .with_context(|| { format!( "execute failed (level={}, contract={}): could not process block", diff --git a/src/main.rs b/src/main.rs index a344d36..72354b2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -238,7 +238,7 @@ fn schema_version(v: &str) -> String { }; // Minor version bumps (_._.x) have same db schemas v.to_string() - .rsplit_once(".") + .rsplit_once('.') .map(|(db_ver, _)| db_ver.to_string()) .unwrap_or_else(|| "".to_string()) } diff --git a/src/octez/block.rs b/src/octez/block.rs index 8d0910d..c3e5f0c 100644 --- a/src/octez/block.rs +++ b/src/octez/block.rs @@ -219,8 +219,7 @@ impl Block { entrypoint_args: content .parameters .clone() - .map(|p| p.value) - .flatten(), + .and_then(|p| p.value), amount: Self::parse_option_pgnumeric( content.amount.as_ref(), @@ -309,8 +308,7 @@ impl Block { entrypoint_args: internal_op .parameters .clone() - .map(|p| p.value) - .flatten(), + .and_then(|p| p.value), amount: Self::parse_option_pgnumeric( internal_op.amount.as_ref(), diff --git a/src/octez/node.rs b/src/octez/node.rs index bfe8bcb..36f6a10 100644 --- a/src/octez/node.rs +++ b/src/octez/node.rs @@ -160,8 +160,8 @@ impl NodeClient { } Ok(( - get_prim(&code_def, "storage")?, - get_prim(&code_def, "parameter")?, + get_prim(code_def, "storage")?, + get_prim(code_def, "parameter")?, )) } diff --git a/src/sql/db.rs b/src/sql/db.rs index 145c1eb..8b1a75e 100644 --- a/src/sql/db.rs +++ b/src/sql/db.rs @@ -198,7 +198,7 @@ WHERE table_schema = $1 if table.contains_snapshots() { let parent_table: String = PostgresqlGenerator::table_parent_name(table) - .unwrap_or(table.name.clone()); + .unwrap_or_else(|| table.name.clone()); let tmpl = RepopulateSnapshotDerivedTmpl { main_schema: &self.main_schema, contract_schema: &contract_id.name, @@ -272,7 +272,7 @@ WHERE table_schema = $1 if table.contains_snapshots() { let parent_table: String = PostgresqlGenerator::table_parent_name(table) - .unwrap_or(table.name.clone()); + .unwrap_or_else(|| table.name.clone()); let tmpl = UpdateSnapshotDerivedTmpl { main_schema: &self.main_schema, contract_schema: &contract_id.name, diff --git a/src/sql/inserter.rs b/src/sql/inserter.rs index a1ebc8e..2e44ee3 100644 --- a/src/sql/inserter.rs +++ b/src/sql/inserter.rs @@ -138,7 +138,7 @@ fn insert_batch( for (contract_id, (contract, ctxs)) in &batch.contract_tx_contexts { dbcli.update_derived_tables( &mut db_tx, - &contract, + contract, ctxs, ).with_context(|| { format!( @@ -218,7 +218,7 @@ impl ProcessedContractBlock { let shifted = k.1.id.unwrap() + offset; k.1.id = Some(shifted); max = std::cmp::max(shifted, max); - return (k, v); + (k, v) }) .collect(); diff --git a/src/sql/postgresql_generator.rs b/src/sql/postgresql_generator.rs index 2382059..edb2c51 100644 --- a/src/sql/postgresql_generator.rs +++ b/src/sql/postgresql_generator.rs @@ -174,14 +174,14 @@ impl PostgresqlGenerator { if table.contains_pointers() { let shallow_tmpl = CreateSnapshotFunctionsTmpl { main_schema: &self.main_schema, - contract_schema: contract_schema, + contract_schema, table: &table.name, columns: &columns, typed_columns: &typed_columns, }; let shallow_shortcuts = CreateFunctionShortcutsTmpl { main_schema: &self.main_schema, - contract_schema: contract_schema, + contract_schema, table: &table.name, function_postfix: "at", typed_columns: &typed_columns, @@ -196,14 +196,14 @@ impl PostgresqlGenerator { deep_typed_columns.insert(0, "in_schema TEXT".to_string()); let deep_tmpl = CreateEntrypointChangesFunctionsTmpl { main_schema: &self.main_schema, - contract_schema: contract_schema, + contract_schema, table: &table.name, columns: &columns, typed_columns: &deep_typed_columns, }; let deep_shortcuts = CreateFunctionShortcutsTmpl { main_schema: &self.main_schema, - contract_schema: contract_schema, + contract_schema, table: &table.name, function_postfix: "at_deref", typed_columns: &deep_typed_columns, @@ -219,7 +219,7 @@ impl PostgresqlGenerator { let shortcuts = CreateFunctionShortcutsTmpl { main_schema: &self.main_schema, - contract_schema: contract_schema, + contract_schema, table: &table.name, function_postfix: "at", typed_columns: &typed_columns, @@ -228,7 +228,7 @@ impl PostgresqlGenerator { if table.contains_snapshots() { let tmpl = CreateSnapshotFunctionsTmpl { main_schema: &self.main_schema, - contract_schema: contract_schema, + contract_schema, table: &table.name, columns: &columns, typed_columns: &typed_columns, @@ -238,7 +238,7 @@ impl PostgresqlGenerator { let tmpl = CreateChangesFunctionsTmpl { main_schema: &self.main_schema, - contract_schema: contract_schema, + contract_schema, table: &table.name, columns: &columns, typed_columns: &typed_columns, @@ -358,7 +358,7 @@ impl PostgresqlGenerator { } pub(crate) fn parent_name(name: &str) -> Option { - if name.starts_with("entry.") && name.matches(".").count() == 1 { + if name.starts_with("entry.") && name.matches('.').count() == 1 { return None; } name.rfind('.') diff --git a/src/sql/table_builder.rs b/src/sql/table_builder.rs index a4306b3..9d47e92 100644 --- a/src/sql/table_builder.rs +++ b/src/sql/table_builder.rs @@ -89,13 +89,10 @@ impl TableBuilder { None => { let mut t = Table::new(name.to_string()); t.add_index( - &"tx_context_id".to_string(), - &ExprTy::SimpleExprTy(SimpleExprTy::Int), - ); - t.add_column( - &"id".to_string(), + "tx_context_id", &ExprTy::SimpleExprTy(SimpleExprTy::Int), ); + t.add_column("id", &ExprTy::SimpleExprTy(SimpleExprTy::Int)); t } } @@ -132,14 +129,14 @@ impl TableBuilder { let mut t = self.get_table(table); t.add_index( - &"bigmap_id".to_string(), + "bigmap_id", &ExprTy::SimpleExprTy(SimpleExprTy::Int), ); if *has_memory { t.tracks_changes(); t.add_column( - &"deleted".to_string(), + "deleted", &ExprTy::SimpleExprTy(SimpleExprTy::Bool), ); } else { diff --git a/src/storage_structure/relational.rs b/src/storage_structure/relational.rs index 71d1441..b71823b 100644 --- a/src/storage_structure/relational.rs +++ b/src/storage_structure/relational.rs @@ -185,7 +185,7 @@ impl ASTBuilder { pub(crate) fn memoryless_bigmaps(&mut self) -> &mut Self { self.bigmaps_retain = false; - return self; + self } fn start_table(&mut self, ctx: &Context, ele: &Ele) -> Context { diff --git a/src/storage_update/processor.rs b/src/storage_update/processor.rs index 8c63a4c..d307156 100644 --- a/src/storage_update/processor.rs +++ b/src/storage_update/processor.rs @@ -530,7 +530,7 @@ where None => self.sql_add_cell( ctx, &table, - &"deleted".to_string(), + "deleted", insert::Value::Bool(true), tx_context, ), @@ -546,7 +546,7 @@ where self.sql_add_cell( ctx, &table, - &"bigmap_id".to_string(), + "bigmap_id", insert::Value::Int(*bigmap), tx_context, ); @@ -600,7 +600,7 @@ where self.sql_add_cell( ctx, &table, - &"bigmap_id".to_string(), + "bigmap_id", insert::Value::Int(*bigmap), tx_context, );