Skip to content

Commit

Permalink
Split migration into two parts
Browse files Browse the repository at this point in the history
Unit tests remain to be split.
  • Loading branch information
rod-glover committed Feb 4, 2025
1 parent 6a1c0fb commit 6688d63
Show file tree
Hide file tree
Showing 2 changed files with 47 additions and 14 deletions.
17 changes: 3 additions & 14 deletions pycds/alembic/versions/8c05da87cb79_add_hx_tkg_to_obs_raw.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
"""Add history tracking to obs_raw
"""Add history tracking to obs_raw, part 1 (create hx table)
Revision ID: 8c05da87cb79
Revises: a59d64cf16ca
Expand Down Expand Up @@ -49,7 +49,6 @@ def upgrade():

# Primary table
####

add_history_cols_to_primary(
table_name,
columns=(
Expand All @@ -65,24 +64,14 @@ def upgrade():

# History table
####

create_history_table(table_name, foreign_keys)

# Populate the history table, then update its history FKs in bulk.
# If we let the FK trigger do this work, fired row-by-row on ~1e9 records,
# it requires an unfeasible amount of time, so we do it in bulk.
grant_standard_table_privileges(hx_table_name(table_name), schema=schema_name)
populate_history_table(table_name, primary_key_name)
update_obs_raw_history_FKs()

# History table triggers must be created after the table is populated.
create_history_table_triggers(table_name, foreign_keys)
# History table hx foreign keys are updated in bulk in next migration.

create_history_table_indexes(table_name, primary_key_name, foreign_keys)
grant_standard_table_privileges(table_name, schema=schema_name)


def downgrade():
drop_history_triggers(table_name)
drop_history_table(table_name)
drop_history_cols_from_primary(table_name, columns=("mod_user",))
op.execute(
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
"""Add history tracking to obs_raw, part 2 (update FKs)
Revision ID: 976071a4f6e8
Revises: 8c05da87cb79
Create Date: 2025-02-04 10:09:18.094153
"""
from alembic import op
import sqlalchemy as sa

from pycds import get_schema_name
from pycds.alembic.change_history_utils import update_obs_raw_history_FKs, create_history_table_triggers, \
create_history_table_indexes, drop_history_triggers

# revision identifiers, used by Alembic.
revision = "976071a4f6e8"
down_revision = "8c05da87cb79"
branch_labels = None
depends_on = None


schema_name = get_schema_name()

# TODO: These are the same as in the previous migration, but can't be accessed due
# to module naming. Use __import__()?
table_name = "obs_raw"
primary_key_name = "obs_raw_id"
foreign_keys = [("meta_history", "history_id"), ("meta_vars", "vars_id")]


def upgrade():
# If we let the FK trigger update FKs, fired row-by-row on ~1e9 records,
# it requires an unfeasible amount of time, so we do it in bulk.
update_obs_raw_history_FKs()
# History table triggers must be created after the table is populated.
create_history_table_triggers(table_name, foreign_keys)
create_history_table_indexes(table_name, primary_key_name, foreign_keys)


def downgrade():
drop_history_triggers(table_name)
# We could set the hx FKs to NULL here, but there's not much point given this
# migration is paired with the previous one, whose downgrade immediately drops
# the entire table.

0 comments on commit 6688d63

Please sign in to comment.