-
Notifications
You must be signed in to change notification settings - Fork 3.4k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #6032 from ryanpetrello/bigint
migrate event table primary keys from integer to bigint Reviewed-by: Ryan Petrello https://github.com/ryanpetrello
- Loading branch information
Showing
6 changed files
with
203 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,118 @@ | ||
# Generated by Django 2.2.8 on 2020-02-21 16:31 | ||
|
||
from django.db import migrations, models, connection | ||
|
||
|
||
def migrate_event_data(apps, schema_editor): | ||
# see: https://github.com/ansible/awx/issues/6010 | ||
# | ||
# the goal of this function is to end with event tables (e.g., main_jobevent) | ||
# that have a bigint primary key (because the old usage of an integer | ||
# numeric isn't enough, as its range is about 2.1B, see: | ||
# https://www.postgresql.org/docs/9.1/datatype-numeric.html) | ||
|
||
# unfortunately, we can't do this with a simple ALTER TABLE, because | ||
# for tables with hundreds of millions or billions of rows, the ALTER TABLE | ||
# can take *hours* on modest hardware. | ||
# | ||
# the approach in this migration means that post-migration, event data will | ||
# *not* immediately show up, but will be repopulated over time progressively | ||
# the trade-off here is not having to wait hours for the full data migration | ||
# before you can start and run AWX again (including new playbook runs) | ||
for tblname in ( | ||
'main_jobevent', 'main_inventoryupdateevent', | ||
'main_projectupdateevent', 'main_adhoccommandevent', | ||
'main_systemjobevent' | ||
): | ||
with connection.cursor() as cursor: | ||
# rename the current event table | ||
cursor.execute( | ||
f'ALTER TABLE {tblname} RENAME TO _old_{tblname};' | ||
) | ||
# create a *new* table with the same schema | ||
cursor.execute( | ||
f'CREATE TABLE {tblname} (LIKE _old_{tblname} INCLUDING ALL);' | ||
) | ||
# alter the *new* table so that the primary key is a big int | ||
cursor.execute( | ||
f'ALTER TABLE {tblname} ALTER COLUMN id TYPE bigint USING id::bigint;' | ||
) | ||
|
||
# recreate counter for the new table's primary key to | ||
# start where the *old* table left off (we have to do this because the | ||
# counter changed from an int to a bigint) | ||
cursor.execute(f'DROP SEQUENCE IF EXISTS "{tblname}_id_seq" CASCADE;') | ||
cursor.execute(f'CREATE SEQUENCE "{tblname}_id_seq";') | ||
cursor.execute( | ||
f'ALTER TABLE "{tblname}" ALTER COLUMN "id" ' | ||
f"SET DEFAULT nextval('{tblname}_id_seq');" | ||
) | ||
cursor.execute( | ||
f"SELECT setval('{tblname}_id_seq', (SELECT MAX(id) FROM _old_{tblname}), true);" | ||
) | ||
|
||
# replace the BTREE index on main_jobevent.job_id with | ||
# a BRIN index to drastically improve per-UJ lookup performance | ||
# see: https://info.crunchydata.com/blog/postgresql-brin-indexes-big-data-performance-with-minimal-storage | ||
if tblname == 'main_jobevent': | ||
cursor.execute("SELECT indexname FROM pg_indexes WHERE tablename='main_jobevent' AND indexdef LIKE '%USING btree (job_id)';") | ||
old_index = cursor.fetchone()[0] | ||
cursor.execute(f'DROP INDEX {old_index}') | ||
cursor.execute('CREATE INDEX main_jobevent_job_id_brin_idx ON main_jobevent USING brin (job_id);') | ||
|
||
# remove all of the indexes and constraints from the old table | ||
# (they just slow down the data migration) | ||
cursor.execute(f"SELECT indexname, indexdef FROM pg_indexes WHERE tablename='_old_{tblname}' AND indexname != '{tblname}_pkey';") | ||
indexes = cursor.fetchall() | ||
|
||
cursor.execute(f"SELECT conname, contype, pg_catalog.pg_get_constraintdef(r.oid, true) as condef FROM pg_catalog.pg_constraint r WHERE r.conrelid = '_old_{tblname}'::regclass AND conname != '{tblname}_pkey';") | ||
constraints = cursor.fetchall() | ||
|
||
for indexname, indexdef in indexes: | ||
cursor.execute(f'DROP INDEX IF EXISTS {indexname}') | ||
for conname, contype, condef in constraints: | ||
cursor.execute(f'ALTER TABLE _old_{tblname} DROP CONSTRAINT IF EXISTS {conname}') | ||
|
||
|
||
class FakeAlterField(migrations.AlterField): | ||
|
||
def database_forwards(self, *args): | ||
# this is intentionally left blank, because we're | ||
# going to accomplish the migration with some custom raw SQL | ||
pass | ||
|
||
|
||
class Migration(migrations.Migration): | ||
|
||
dependencies = [ | ||
('main', '0112_v370_workflow_node_identifier'), | ||
] | ||
|
||
operations = [ | ||
migrations.RunPython(migrate_event_data), | ||
FakeAlterField( | ||
model_name='adhoccommandevent', | ||
name='id', | ||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), | ||
), | ||
FakeAlterField( | ||
model_name='inventoryupdateevent', | ||
name='id', | ||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), | ||
), | ||
FakeAlterField( | ||
model_name='jobevent', | ||
name='id', | ||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), | ||
), | ||
FakeAlterField( | ||
model_name='projectupdateevent', | ||
name='id', | ||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), | ||
), | ||
FakeAlterField( | ||
model_name='systemjobevent', | ||
name='id', | ||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), | ||
), | ||
] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters