From 2d7ff0ceafe29529e9b7cf026686f22b4869c607 Mon Sep 17 00:00:00 2001 From: Jodi Jang Date: Wed, 15 Jan 2025 12:05:35 -0500 Subject: [PATCH 1/5] [WIP] chore(sourcemaps): Add sourcemap subcolumn to errors --- .../configuration/events/storages/errors.yaml | 5 ++ ...sourcemapped_subcolumn_exception_frames.py | 80 +++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 snuba/snuba_migrations/events/0025_add_sourcemapped_subcolumn_exception_frames.py diff --git a/snuba/datasets/configuration/events/storages/errors.yaml b/snuba/datasets/configuration/events/storages/errors.yaml index 26afb5ad48b..2b1a37fc58c 100644 --- a/snuba/datasets/configuration/events/storages/errors.yaml +++ b/snuba/datasets/configuration/events/storages/errors.yaml @@ -194,6 +194,11 @@ schema: type: UInt, args: { schema_modifiers: [nullable], size: 16 }, }, + { + name: sourcemapped, + type: UInt, + args: { schema_modifiers: [nullable], size: 8 }, + }, ], }, }, diff --git a/snuba/snuba_migrations/events/0025_add_sourcemapped_subcolumn_exception_frames.py b/snuba/snuba_migrations/events/0025_add_sourcemapped_subcolumn_exception_frames.py new file mode 100644 index 00000000000..f1209bb71d2 --- /dev/null +++ b/snuba/snuba_migrations/events/0025_add_sourcemapped_subcolumn_exception_frames.py @@ -0,0 +1,80 @@ +from typing import Sequence + +from snuba.clickhouse.columns import Column, String, UInt +from snuba.clusters.storage_sets import StorageSetKey +from snuba.migrations import migration, operations +from snuba.migrations.columns import MigrationModifiers as Modifiers +from snuba.migrations.operations import OperationTarget +from snuba.utils.schemas import Nested + + +class Migration(migration.ClickhouseNodeMigration): + blocking = False + + storage_set_key = StorageSetKey.EVENTS + + local_table_name = "errors_local" + dist_table_name = "errors_dist" + + column_before = Column( + "exception_frames", + Nested( + [ + ("abs_path", String(Modifiers(nullable=True))), + ("colno", UInt(32, Modifiers(nullable=True))), + ("filename", String(Modifiers(nullable=True))), + ("function", String(Modifiers(nullable=True))), + ("lineno", UInt(32, Modifiers(nullable=True))), + ("in_app", UInt(8, Modifiers(nullable=True))), + ("package", String(Modifiers(nullable=True))), + ("module", String(Modifiers(nullable=True))), + ("stack_level", UInt(16, Modifiers(nullable=True))), + ] + ), + ) + + column_after = Column( + "exception_frames", + Nested( + [ + ("abs_path", String(Modifiers(nullable=True))), + ("colno", UInt(32, Modifiers(nullable=True))), + ("filename", String(Modifiers(nullable=True))), + ("function", String(Modifiers(nullable=True))), + ("lineno", UInt(32, Modifiers(nullable=True))), + ("in_app", UInt(8, Modifiers(nullable=True))), + ("package", String(Modifiers(nullable=True))), + ("module", String(Modifiers(nullable=True))), + ("stack_level", UInt(16, Modifiers(nullable=True))), + ("sourcemapped", UInt(8, Modifiers(nullable=True))), + ] + ), + ) + + def forwards_ops(self) -> Sequence[operations.SqlOperation]: + return [ + operations.ModifyColumn( + storage_set=self.storage_set_key, + table_name=table_name, + column=self.column_after, + target=target, + ) + for table_name, target in [ + (self.local_table_name, OperationTarget.LOCAL), + (self.dist_table_name, OperationTarget.DISTRIBUTED), + ] + ] + + def backwards_ops(self) -> Sequence[operations.SqlOperation]: + return [ + operations.ModifyColumn( + storage_set=self.storage_set_key, + table_name=table_name, + column=self.column_before, + target=target, + ) + for table_name, target in [ + (self.local_table_name, OperationTarget.LOCAL), + (self.dist_table_name, OperationTarget.DISTRIBUTED), + ] + ] From d888e34b9d3c69dc318f4c40cd339d8572dfac69 Mon Sep 17 00:00:00 2001 From: Jodi Jang Date: Wed, 15 Jan 2025 13:24:05 -0500 Subject: [PATCH 2/5] fix: Remove yaml change --- snuba/datasets/configuration/events/storages/errors.yaml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/snuba/datasets/configuration/events/storages/errors.yaml b/snuba/datasets/configuration/events/storages/errors.yaml index 2b1a37fc58c..cb1dd0357c7 100644 --- a/snuba/datasets/configuration/events/storages/errors.yaml +++ b/snuba/datasets/configuration/events/storages/errors.yaml @@ -193,12 +193,7 @@ schema: name: stack_level, type: UInt, args: { schema_modifiers: [nullable], size: 16 }, - }, - { - name: sourcemapped, - type: UInt, - args: { schema_modifiers: [nullable], size: 8 }, - }, + } ], }, }, From 3bf22d0a67dec1b1c1d8878050e92e5a46cc091a Mon Sep 17 00:00:00 2001 From: Jodi Jang Date: Wed, 15 Jan 2025 14:07:05 -0500 Subject: [PATCH 3/5] fix: Add column --- ...sourcemapped_subcolumn_exception_frames.py | 51 ++++--------------- 1 file changed, 9 insertions(+), 42 deletions(-) diff --git a/snuba/snuba_migrations/events/0025_add_sourcemapped_subcolumn_exception_frames.py b/snuba/snuba_migrations/events/0025_add_sourcemapped_subcolumn_exception_frames.py index f1209bb71d2..428a0357386 100644 --- a/snuba/snuba_migrations/events/0025_add_sourcemapped_subcolumn_exception_frames.py +++ b/snuba/snuba_migrations/events/0025_add_sourcemapped_subcolumn_exception_frames.py @@ -1,11 +1,10 @@ from typing import Sequence -from snuba.clickhouse.columns import Column, String, UInt +from snuba.clickhouse.columns import Column, UInt from snuba.clusters.storage_sets import StorageSetKey from snuba.migrations import migration, operations from snuba.migrations.columns import MigrationModifiers as Modifiers from snuba.migrations.operations import OperationTarget -from snuba.utils.schemas import Nested class Migration(migration.ClickhouseNodeMigration): @@ -16,47 +15,15 @@ class Migration(migration.ClickhouseNodeMigration): local_table_name = "errors_local" dist_table_name = "errors_dist" - column_before = Column( - "exception_frames", - Nested( - [ - ("abs_path", String(Modifiers(nullable=True))), - ("colno", UInt(32, Modifiers(nullable=True))), - ("filename", String(Modifiers(nullable=True))), - ("function", String(Modifiers(nullable=True))), - ("lineno", UInt(32, Modifiers(nullable=True))), - ("in_app", UInt(8, Modifiers(nullable=True))), - ("package", String(Modifiers(nullable=True))), - ("module", String(Modifiers(nullable=True))), - ("stack_level", UInt(16, Modifiers(nullable=True))), - ] - ), - ) - - column_after = Column( - "exception_frames", - Nested( - [ - ("abs_path", String(Modifiers(nullable=True))), - ("colno", UInt(32, Modifiers(nullable=True))), - ("filename", String(Modifiers(nullable=True))), - ("function", String(Modifiers(nullable=True))), - ("lineno", UInt(32, Modifiers(nullable=True))), - ("in_app", UInt(8, Modifiers(nullable=True))), - ("package", String(Modifiers(nullable=True))), - ("module", String(Modifiers(nullable=True))), - ("stack_level", UInt(16, Modifiers(nullable=True))), - ("sourcemapped", UInt(8, Modifiers(nullable=True))), - ] - ), - ) - def forwards_ops(self) -> Sequence[operations.SqlOperation]: return [ - operations.ModifyColumn( + operations.AddColumn( storage_set=self.storage_set_key, table_name=table_name, - column=self.column_after, + column=Column( + "exception_frames.sourcemapped", UInt(8, Modifiers(nullable=True)) + ), + after="exception_frames.stack_level", target=target, ) for table_name, target in [ @@ -67,14 +34,14 @@ def forwards_ops(self) -> Sequence[operations.SqlOperation]: def backwards_ops(self) -> Sequence[operations.SqlOperation]: return [ - operations.ModifyColumn( + operations.DropColumn( storage_set=self.storage_set_key, table_name=table_name, - column=self.column_before, + column_name="exception_frames.sourcemapped", target=target, ) for table_name, target in [ - (self.local_table_name, OperationTarget.LOCAL), (self.dist_table_name, OperationTarget.DISTRIBUTED), + (self.local_table_name, OperationTarget.LOCAL), ] ] From 00c9435897d414c227044953adc7582d14f8bac8 Mon Sep 17 00:00:00 2001 From: Jodi Jang Date: Wed, 15 Jan 2025 14:08:26 -0500 Subject: [PATCH 4/5] fix: Typo --- snuba/datasets/configuration/events/storages/errors.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snuba/datasets/configuration/events/storages/errors.yaml b/snuba/datasets/configuration/events/storages/errors.yaml index cb1dd0357c7..26afb5ad48b 100644 --- a/snuba/datasets/configuration/events/storages/errors.yaml +++ b/snuba/datasets/configuration/events/storages/errors.yaml @@ -193,7 +193,7 @@ schema: name: stack_level, type: UInt, args: { schema_modifiers: [nullable], size: 16 }, - } + }, ], }, }, From 807ad136060a7c6e60443b4cbdaa3e25fd807134 Mon Sep 17 00:00:00 2001 From: Jodi Jang Date: Thu, 16 Jan 2025 13:41:34 -0500 Subject: [PATCH 5/5] fix: Change column type --- .../0025_add_sourcemapped_subcolumn_exception_frames.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/snuba/snuba_migrations/events/0025_add_sourcemapped_subcolumn_exception_frames.py b/snuba/snuba_migrations/events/0025_add_sourcemapped_subcolumn_exception_frames.py index 428a0357386..d9a583e14fe 100644 --- a/snuba/snuba_migrations/events/0025_add_sourcemapped_subcolumn_exception_frames.py +++ b/snuba/snuba_migrations/events/0025_add_sourcemapped_subcolumn_exception_frames.py @@ -1,6 +1,6 @@ from typing import Sequence -from snuba.clickhouse.columns import Column, UInt +from snuba.clickhouse.columns import Array, Column, UInt from snuba.clusters.storage_sets import StorageSetKey from snuba.migrations import migration, operations from snuba.migrations.columns import MigrationModifiers as Modifiers @@ -21,7 +21,8 @@ def forwards_ops(self) -> Sequence[operations.SqlOperation]: storage_set=self.storage_set_key, table_name=table_name, column=Column( - "exception_frames.sourcemapped", UInt(8, Modifiers(nullable=True)) + "exception_frames.sourcemapped", + Array(UInt(8, Modifiers(nullable=True))), ), after="exception_frames.stack_level", target=target,