From 1f0dede86f9d901452da32030078e96afdc559a0 Mon Sep 17 00:00:00 2001 From: urvisavla Date: Tue, 17 Oct 2023 23:24:30 -0700 Subject: [PATCH] 5061/remove unused indexes (#5081) * services/horizon: remove unused indexes from Horizon database * Add notransaction * Update CHANGELOG --- services/horizon/CHANGELOG.md | 2 ++ .../horizon/internal/db2/schema/bindata.go | 23 +++++++++++++ .../migrations/65_remove_unused_indexes.sql | 34 +++++++++++++++++++ 3 files changed, 59 insertions(+) create mode 100644 services/horizon/internal/db2/schema/migrations/65_remove_unused_indexes.sql diff --git a/services/horizon/CHANGELOG.md b/services/horizon/CHANGELOG.md index 8bf83b5ad6..84b897e2e6 100644 --- a/services/horizon/CHANGELOG.md +++ b/services/horizon/CHANGELOG.md @@ -18,6 +18,8 @@ file. This project adheres to [Semantic Versioning](http://semver.org/). - Limit the display of global flags on command line help `-h` output ([5077](https://github.com/stellar/go/pull/5077)). - Add a deprecation warning for using command-line flags when running Horizon ([5051](https://github.com/stellar/go/pull/5051)) +### DB Schema Migration +- Drop unused indices from the Horizon database. For the database with full history, the migration is anticipated to take up to an hour and is expected to free up approximately 1.3TB of storage ([5081](https://github.com/stellar/go/pull/5081)). ## 2.26.1 diff --git a/services/horizon/internal/db2/schema/bindata.go b/services/horizon/internal/db2/schema/bindata.go index 46546303d1..46dc3ba3c3 100644 --- a/services/horizon/internal/db2/schema/bindata.go +++ b/services/horizon/internal/db2/schema/bindata.go @@ -61,6 +61,7 @@ // migrations/62_claimable_balance_claimants.sql (1.428kB) // migrations/63_add_contract_id_to_asset_stats.sql (153B) // migrations/64_add_payment_flag_history_ops.sql (300B) +// migrations/65_remove_unused_indexes.sql (2.897kB) // migrations/6_create_assets_table.sql (366B) // migrations/7_modify_trades_table.sql (2.303kB) // migrations/8_add_aggregators.sql (907B) @@ -1354,6 +1355,26 @@ func migrations64_add_payment_flag_history_opsSql() (*asset, error) { return a, nil } +var _migrations65_remove_unused_indexesSql = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xa4\x55\xc9\x6e\xdb\x30\x10\xbd\xe7\x2b\x78\x8b\x8d\xc6\xa7\xdc\x12\x34\x40\x5b\xab\xad\x2f\x76\xeb\x05\xcd\x8d\xa0\xc4\xb1\xc5\x46\xe2\x08\xe4\x28\x8d\xff\xbe\x90\xe5\x85\x92\x45\x2d\xc9\x4d\x10\xdf\x9b\x37\x9c\xe5\x71\x32\x61\x9f\x52\xb5\x33\x82\x80\x6d\x32\xa6\x91\x8c\xd0\x56\x44\xa4\x50\xdf\xdc\x4c\x97\x8b\x5f\x6c\x36\x9f\x06\xcf\x6c\xf6\x9d\x05\xcf\xb3\xd5\x7a\xc5\x44\x14\x61\xae\xc9\xf2\x18\x53\xe0\x12\x53\xa1\xf4\x63\x07\x54\xe9\x6d\x22\x8a\x98\x5c\x82\x25\xa5\x0f\xdf\x3e\x92\xb5\x40\x3c\xdc\x73\x65\x6d\x0e\xc6\x83\x8a\x12\xa1\x52\x11\x8a\x30\x01\x1e\x8a\x44\xe8\x08\x6c\x41\x2a\xff\x6b\xb2\x1e\x5e\x4c\x46\x72\x52\x29\xf0\x04\xf1\x25\xcf\x3c\x30\xa5\x25\xbc\xf1\x58\x59\x42\xb3\xe7\xb0\xdd\x42\x44\x96\xa3\xe6\xb4\xcf\xa0\x17\x27\x01\xb9\x03\x73\xe0\x44\x09\x5a\x90\x5c\xd0\x50\xa2\x4a\x33\x34\x04\x86\xbf\x82\xb1\xfe\x8a\x79\xf9\xe5\x27\x8f\x85\x8d\x87\x52\x33\x03\xaf\x0a\x73\x3b\x38\x06\x66\x60\xca\x56\x1f\x5b\xe4\x36\xa8\xb8\x93\xf4\xb5\xc6\x13\x67\x58\xd9\x9d\xf9\x1d\x9c\x00\x19\x21\xe1\xdc\xec\x70\xcf\xd1\x48\x30\x3c\x44\x7c\xf1\x32\x72\x4b\x3c\x51\xba\x1c\xbe\xd3\xc4\xde\xb8\x6b\x35\xc5\x7f\xba\xb6\x58\xdf\x96\xc1\x97\x75\x70\x89\x36\x5f\xac\xdb\xb6\x8b\x2d\xe6\xe7\xff\x6c\xb3\x9a\xcd\x7f\xb0\xaf\xeb\x65\x10\x8c\x1c\xcc\xf8\xb1\x57\xd4\xc6\x45\xf4\xc6\x6f\x44\x77\x28\x55\xb7\xb7\x08\x7d\x6a\xcd\xe1\xe8\x24\x10\x92\x01\x60\xa3\x12\x5e\x62\xdb\x03\x77\x2e\x7c\x21\x75\xdd\xf1\xa3\xdc\x4e\xe9\xd1\x05\xf9\xd7\xa2\x0e\x79\x26\x28\xe6\x98\xd9\x76\xdd\xba\x61\xb8\x37\x3a\x0c\x4c\xb5\x64\xc7\x75\x39\xaf\x7c\x7b\xf0\x56\x9b\x71\x95\x8e\x47\xd5\xe2\x15\xa0\x21\xf1\x9b\x2c\xc9\x15\x39\x9e\x57\x45\xde\x75\x93\x16\x0f\xeb\x14\xac\x13\x2e\xba\x9b\xf9\xec\xf7\x66\xa0\xbc\x63\x5f\x9d\xca\x0e\xf6\x63\xa2\x4d\xe6\xd9\xa9\xde\x44\x7a\x47\x1a\x3d\xfc\xd7\x4d\xa5\x0d\x5e\xcd\xef\x9a\xa1\x24\xbb\x3b\x07\xba\xa2\x73\x25\x87\x8c\xcc\xb5\xdf\x37\xa6\xd9\xba\x02\xbd\x8b\xd4\xeb\x95\xa8\x6d\x7a\x0b\xa1\xb9\x50\x2e\xe7\x83\xa5\x6a\x79\x99\x3a\x5d\x62\x34\x92\x40\x42\x25\x96\x4d\x9e\x9e\xd8\xad\xc5\x44\x96\x56\x7c\x28\xf2\xed\xc3\x03\xc1\x1b\x8d\xc7\x77\xcc\x0f\x8c\x50\xf6\x03\x96\x46\xee\x87\x86\x98\xef\x62\xea\x25\x5f\x81\xb6\x27\x50\x81\xd6\x52\x18\xb3\x3f\x3f\x83\x65\x50\x4e\x0a\xfb\xcc\xee\xef\xbb\x2a\xdd\xf0\xa2\x17\x35\x76\x0e\x2a\x86\x5f\x7b\xc1\xfe\x07\x00\x00\xff\xff\x6e\x06\xca\xb9\x51\x0b\x00\x00") + +func migrations65_remove_unused_indexesSqlBytes() ([]byte, error) { + return bindataRead( + _migrations65_remove_unused_indexesSql, + "migrations/65_remove_unused_indexes.sql", + ) +} + +func migrations65_remove_unused_indexesSql() (*asset, error) { + bytes, err := migrations65_remove_unused_indexesSqlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "migrations/65_remove_unused_indexes.sql", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x99, 0x6, 0xc9, 0x3, 0x51, 0x52, 0x77, 0x87, 0x7f, 0x5, 0xe2, 0xdd, 0xd2, 0x49, 0xd0, 0xd1, 0x1b, 0x9a, 0x9, 0x2f, 0x10, 0xb7, 0x68, 0x44, 0xd1, 0x12, 0xce, 0x43, 0x50, 0x2e, 0xbd, 0xef}} + return a, nil +} + var _migrations6_create_assets_tableSql = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x6c\x90\x3d\x4f\xc3\x30\x18\x84\x77\xff\x8a\x1b\x1d\x91\x0e\x20\xe8\x92\xc9\x34\x16\x58\x18\xa7\xb8\x31\xa2\x53\xe5\x26\x16\x78\x80\x54\xb6\x11\xca\xbf\x47\xaa\x28\xf9\x50\xe6\x7b\xf4\xbc\xef\xdd\x6a\x85\xab\x4f\xff\x1e\x6c\x72\x30\x27\xb2\xd1\x9c\xd5\x1c\x35\xbb\x97\x1c\x1f\x3e\xa6\x2e\xf4\x07\x1b\xa3\x4b\x11\x94\x00\x80\x6f\xb1\xe3\x5a\x30\x89\xad\x16\xcf\x4c\xef\xf1\xc4\xf7\xc8\xcf\xd9\x19\x3c\xa4\xfe\xe4\xf0\xca\xf4\xe6\x91\x69\xba\xbe\xcd\xa0\xaa\x1a\xca\x48\x39\x86\x9a\xae\x1d\xa0\xeb\x9b\x65\xc8\xc7\xf8\xed\xc2\x3f\x76\xb7\x9e\x63\x46\x89\x17\xc3\xe9\xa0\xcc\x47\x3f\xe4\x13\x4b\x46\xb2\x82\x5c\xfa\x09\x55\xf2\xb7\xbf\xf8\xd8\x5f\xee\x54\x6a\x5e\xd9\xec\x84\x7a\xc0\x31\x05\xe7\x40\x27\xb6\x82\x90\xf1\x74\x65\xf7\xf3\x45\x4a\x5d\x6d\x97\xa7\x6b\x6c\x6c\x6c\xeb\x8a\xdf\x00\x00\x00\xff\xff\xfb\x53\x3e\x81\x6e\x01\x00\x00") func migrations6_create_assets_tableSqlBytes() ([]byte, error) { @@ -1606,6 +1627,7 @@ var _bindata = map[string]func() (*asset, error){ "migrations/62_claimable_balance_claimants.sql": migrations62_claimable_balance_claimantsSql, "migrations/63_add_contract_id_to_asset_stats.sql": migrations63_add_contract_id_to_asset_statsSql, "migrations/64_add_payment_flag_history_ops.sql": migrations64_add_payment_flag_history_opsSql, + "migrations/65_remove_unused_indexes.sql": migrations65_remove_unused_indexesSql, "migrations/6_create_assets_table.sql": migrations6_create_assets_tableSql, "migrations/7_modify_trades_table.sql": migrations7_modify_trades_tableSql, "migrations/8_add_aggregators.sql": migrations8_add_aggregatorsSql, @@ -1718,6 +1740,7 @@ var _bintree = &bintree{nil, map[string]*bintree{ "62_claimable_balance_claimants.sql": {migrations62_claimable_balance_claimantsSql, map[string]*bintree{}}, "63_add_contract_id_to_asset_stats.sql": {migrations63_add_contract_id_to_asset_statsSql, map[string]*bintree{}}, "64_add_payment_flag_history_ops.sql": {migrations64_add_payment_flag_history_opsSql, map[string]*bintree{}}, + "65_remove_unused_indexes.sql": {migrations65_remove_unused_indexesSql, map[string]*bintree{}}, "6_create_assets_table.sql": {migrations6_create_assets_tableSql, map[string]*bintree{}}, "7_modify_trades_table.sql": {migrations7_modify_trades_tableSql, map[string]*bintree{}}, "8_add_aggregators.sql": {migrations8_add_aggregatorsSql, map[string]*bintree{}}, diff --git a/services/horizon/internal/db2/schema/migrations/65_remove_unused_indexes.sql b/services/horizon/internal/db2/schema/migrations/65_remove_unused_indexes.sql new file mode 100644 index 0000000000..3b9584c7ab --- /dev/null +++ b/services/horizon/internal/db2/schema/migrations/65_remove_unused_indexes.sql @@ -0,0 +1,34 @@ +-- +migrate Up notransaction + +DROP INDEX IF EXISTS accounts_home_domain; +DROP INDEX IF EXISTS accounts_inflation_destination; +DROP INDEX IF EXISTS asset_by_issuer; +DROP INDEX IF EXISTS claimabable_balances_by_claimants; +DROP INDEX IF EXISTS htrd_time_lookup; +DROP INDEX IF EXISTS index_history_effects_on_type; +DROP INDEX IF EXISTS index_history_ledgers_on_closed_at; +DROP INDEX IF EXISTS index_history_ledgers_on_importer_version; +DROP INDEX IF EXISTS index_history_ledgers_on_ledger_hash; +DROP INDEX IF EXISTS index_history_ledgers_on_previous_ledger_hash; +DROP INDEX IF EXISTS index_history_operation_claimable_balances_on_ids; +DROP INDEX IF EXISTS index_history_operations_on_type; +DROP INDEX IF EXISTS index_history_transaction_claimable_balances_on_ids; +DROP INDEX IF EXISTS trade_effects_by_order_book; +DROP INDEX IF EXISTS trust_lines_by_issuer; + +-- +migrate Down notransaction +CREATE INDEX IF NOT EXISTS accounts_home_domain ON accounts USING BTREE(home_domain); +CREATE INDEX IF NOT EXISTS accounts_inflation_destination ON accounts USING BTREE(inflation_destination); +CREATE INDEX IF NOT EXISTS asset_by_issuer ON history_assets USING btree (asset_issuer); +CREATE INDEX IF NOT EXISTS claimabable_balances_by_claimants ON claimable_balances USING gin(claimants jsonb_path_ops); +CREATE INDEX IF NOT EXISTS htrd_time_lookup ON history_trades USING BTREE(ledger_closed_at); +CREATE INDEX IF NOT EXISTS index_history_effects_on_type ON history_effects USING btree (type); +CREATE INDEX IF NOT EXISTS index_history_ledgers_on_closed_at ON history_ledgers USING btree (closed_at); +CREATE INDEX IF NOT EXISTS index_history_ledgers_on_importer_version ON history_ledgers USING btree (importer_version); +CREATE UNIQUE INDEX IF NOT EXISTS index_history_ledgers_on_ledger_hash ON history_ledgers USING btree (ledger_hash); +CREATE UNIQUE INDEX IF NOT EXISTS index_history_ledgers_on_previous_ledger_hash ON history_ledgers USING btree (previous_ledger_hash); +CREATE UNIQUE INDEX IF NOT EXISTS index_history_operation_claimable_balances_on_ids ON history_operation_claimable_balances USING btree (history_operation_id , history_claimable_balance_id); +CREATE INDEX IF NOT EXISTS index_history_operations_on_type ON history_operations USING btree (type); +CREATE UNIQUE INDEX IF NOT EXISTS index_history_transaction_claimable_balances_on_ids ON history_transaction_claimable_balances USING btree (history_transaction_id , history_claimable_balance_id); +CREATE INDEX IF NOT EXISTS trade_effects_by_order_book ON history_effects USING btree (((details ->> 'sold_asset_type'::text)), ((details ->> 'sold_asset_code'::text)), ((details ->> 'sold_asset_issuer'::text)), ((details ->> 'bought_asset_type'::text)), ((details ->> 'bought_asset_code'::text)), ((details ->> 'bought_asset_issuer'::text))) WHERE (type = 33); +CREATE INDEX IF NOT EXISTS trust_lines_by_issuer ON trust_lines USING BTREE(asset_issuer);