Skip to content

Commit

Permalink
4376 message verification fails bc of duplicated missing leaves in th…
Browse files Browse the repository at this point in the history 
…e message table (#4440)

* feat: replace pk on messages tb

* fix: duplicated insert of messages

* test: fix tests of client

* chore: zapatos
  • Loading branch information
@liu-zhipeng
liu-zhipeng authored Jun 9, 2023
1 parent 8b7ba3e commit 10ceea9
Show file tree
Hide file tree
Showing 5 changed files with 128 additions and 16 deletions.
8 changes: 8 additions & 0 deletions packages/adapters/database/db/migrations/20230608135754_replace_pk_messages.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
-- migrate:up
ALTER TABLE public.messages DROP CONSTRAINT messages_pkey;
ALTER TABLE public.messages ADD PRIMARY KEY (origin_domain, index);

CREATE INDEX messages_domain_leaf_idx ON public.messages (origin_domain, leaf);

-- migrate:down
DROP INDEX IF EXISTS messages_domain_leaf_idx;
14 changes: 11 additions & 3 deletions packages/adapters/database/db/schema.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -592,7 +592,7 @@ CREATE TABLE public.messages (
leaf character(66) NOT NULL,
origin_domain character varying(255) NOT NULL,
destination_domain character varying(255),
index numeric,
index numeric NOT NULL,
root character(66),
message character varying,
processed boolean DEFAULT false,
Expand Down Expand Up @@ -1057,7 +1057,7 @@ ALTER TABLE ONLY public.merkle_cache
--

ALTER TABLE ONLY public.messages
ADD CONSTRAINT messages_pkey PRIMARY KEY (leaf);
ADD CONSTRAINT messages_pkey PRIMARY KEY (origin_domain, index);


--
Expand Down Expand Up @@ -1199,6 +1199,13 @@ CREATE INDEX idx_daily_transfer_volume_transfer_date ON public.daily_transfer_vo
CREATE INDEX idx_hourly_transfer_volume_transfer_hour ON public.hourly_transfer_volume USING btree (transfer_hour);


--
-- Name: messages_domain_leaf_idx; Type: INDEX; Schema: public; Owner: -
--

CREATE INDEX messages_domain_leaf_idx ON public.messages USING btree (origin_domain, leaf);


--
-- Name: messages_processed_index_idx; Type: INDEX; Schema: public; Owner: -
--
Expand Down Expand Up @@ -1351,4 +1358,5 @@ INSERT INTO public.schema_migrations (version) VALUES
('20230510210620'),
('20230519155643'),
('20230523134345'),
('20230530074124');
('20230530074124'),
('20230608135754');
2 changes: 1 addition & 1 deletion packages/adapters/database/src/client.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -319,7 +319,7 @@ export const saveMessages = async (
const poolToUse = _pool ?? pool;
const messages: s.messages.Insertable[] = xMessages.map(convertToDbMessage).map(sanitizeNull);

await db.upsert("messages", messages, ["leaf"]).run(poolToUse);
await db.upsert("messages", messages, ["origin_domain", "index"]).run(poolToUse);
};

export const saveSentRootMessages = async (
Expand Down
108 changes: 99 additions & 9 deletions packages/adapters/database/src/zapatos/schema.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2626,9 +2626,9 @@ declare module 'zapatos/schema' {
/**
* **messages.index**
* - `numeric` in database
* - Nullable, no default
* - `NOT NULL`, no default
*/
index: number | null;
index: number;
/**
* **messages.root**
* - `bpchar` in database
Expand Down Expand Up @@ -2682,9 +2682,9 @@ declare module 'zapatos/schema' {
/**
* **messages.index**
* - `numeric` in database
* - Nullable, no default
* - `NOT NULL`, no default
*/
index: number | null;
index: number;
/**
* **messages.root**
* - `bpchar` in database
Expand Down Expand Up @@ -2738,7 +2738,7 @@ declare module 'zapatos/schema' {
/**
* **messages.index**
* - `numeric` in database
* - Nullable, no default
* - `NOT NULL`, no default
*/
index?: number | db.Parameter<number> | db.SQLFragment | db.ParentColumn | db.SQLFragment<any, number | db.Parameter<number> | db.SQLFragment | db.ParentColumn>;
/**
Expand Down Expand Up @@ -2794,9 +2794,9 @@ declare module 'zapatos/schema' {
/**
* **messages.index**
* - `numeric` in database
* - Nullable, no default
* - `NOT NULL`, no default
*/
index?: number | db.Parameter<number> | null | db.DefaultType | db.SQLFragment;
index: number | db.Parameter<number> | db.SQLFragment;
/**
* **messages.root**
* - `bpchar` in database
Expand Down Expand Up @@ -2850,9 +2850,9 @@ declare module 'zapatos/schema' {
/**
* **messages.index**
* - `numeric` in database
* - Nullable, no default
* - `NOT NULL`, no default
*/
index?: number | db.Parameter<number> | null | db.DefaultType | db.SQLFragment | db.SQLFragment<any, number | db.Parameter<number> | null | db.DefaultType | db.SQLFragment>;
index?: number | db.Parameter<number> | db.SQLFragment | db.SQLFragment<any, number | db.Parameter<number> | db.SQLFragment>;
/**
* **messages.root**
* - `bpchar` in database
Expand Down Expand Up @@ -7170,6 +7170,24 @@ declare module 'zapatos/schema' {
* - Nullable, no default
*/
relayer_fees: db.JSONValue | null;
/**
* **transfers.xcall_tx_nonce**
* - `numeric` in database
* - `NOT NULL`, default: `0`
*/
xcall_tx_nonce: number;
/**
* **transfers.execute_tx_nonce**
* - `numeric` in database
* - `NOT NULL`, default: `0`
*/
execute_tx_nonce: number;
/**
* **transfers.reconcile_tx_nonce**
* - `numeric` in database
* - `NOT NULL`, default: `0`
*/
reconcile_tx_nonce: number;
}
export interface JSONSelectable {
/**
Expand Down Expand Up @@ -7544,6 +7562,24 @@ declare module 'zapatos/schema' {
* - Nullable, no default
*/
relayer_fees: db.JSONValue | null;
/**
* **transfers.xcall_tx_nonce**
* - `numeric` in database
* - `NOT NULL`, default: `0`
*/
xcall_tx_nonce: number;
/**
* **transfers.execute_tx_nonce**
* - `numeric` in database
* - `NOT NULL`, default: `0`
*/
execute_tx_nonce: number;
/**
* **transfers.reconcile_tx_nonce**
* - `numeric` in database
* - `NOT NULL`, default: `0`
*/
reconcile_tx_nonce: number;
}
export interface Whereable {
/**
Expand Down Expand Up @@ -7918,6 +7954,24 @@ declare module 'zapatos/schema' {
* - Nullable, no default
*/
relayer_fees?: db.JSONValue | db.Parameter<db.JSONValue> | db.SQLFragment | db.ParentColumn | db.SQLFragment<any, db.JSONValue | db.Parameter<db.JSONValue> | db.SQLFragment | db.ParentColumn>;
/**
* **transfers.xcall_tx_nonce**
* - `numeric` in database
* - `NOT NULL`, default: `0`
*/
xcall_tx_nonce?: number | db.Parameter<number> | db.SQLFragment | db.ParentColumn | db.SQLFragment<any, number | db.Parameter<number> | db.SQLFragment | db.ParentColumn>;
/**
* **transfers.execute_tx_nonce**
* - `numeric` in database
* - `NOT NULL`, default: `0`
*/
execute_tx_nonce?: number | db.Parameter<number> | db.SQLFragment | db.ParentColumn | db.SQLFragment<any, number | db.Parameter<number> | db.SQLFragment | db.ParentColumn>;
/**
* **transfers.reconcile_tx_nonce**
* - `numeric` in database
* - `NOT NULL`, default: `0`
*/
reconcile_tx_nonce?: number | db.Parameter<number> | db.SQLFragment | db.ParentColumn | db.SQLFragment<any, number | db.Parameter<number> | db.SQLFragment | db.ParentColumn>;
}
export interface Insertable {
/**
Expand Down Expand Up @@ -8292,6 +8346,24 @@ declare module 'zapatos/schema' {
* - Nullable, no default
*/
relayer_fees?: db.JSONValue | db.Parameter<db.JSONValue> | null | db.DefaultType | db.SQLFragment;
/**
* **transfers.xcall_tx_nonce**
* - `numeric` in database
* - `NOT NULL`, default: `0`
*/
xcall_tx_nonce?: number | db.Parameter<number> | db.DefaultType | db.SQLFragment;
/**
* **transfers.execute_tx_nonce**
* - `numeric` in database
* - `NOT NULL`, default: `0`
*/
execute_tx_nonce?: number | db.Parameter<number> | db.DefaultType | db.SQLFragment;
/**
* **transfers.reconcile_tx_nonce**
* - `numeric` in database
* - `NOT NULL`, default: `0`
*/
reconcile_tx_nonce?: number | db.Parameter<number> | db.DefaultType | db.SQLFragment;
}
export interface Updatable {
/**
Expand Down Expand Up @@ -8666,6 +8738,24 @@ declare module 'zapatos/schema' {
* - Nullable, no default
*/
relayer_fees?: db.JSONValue | db.Parameter<db.JSONValue> | null | db.DefaultType | db.SQLFragment | db.SQLFragment<any, db.JSONValue | db.Parameter<db.JSONValue> | null | db.DefaultType | db.SQLFragment>;
/**
* **transfers.xcall_tx_nonce**
* - `numeric` in database
* - `NOT NULL`, default: `0`
*/
xcall_tx_nonce?: number | db.Parameter<number> | db.DefaultType | db.SQLFragment | db.SQLFragment<any, number | db.Parameter<number> | db.DefaultType | db.SQLFragment>;
/**
* **transfers.execute_tx_nonce**
* - `numeric` in database
* - `NOT NULL`, default: `0`
*/
execute_tx_nonce?: number | db.Parameter<number> | db.DefaultType | db.SQLFragment | db.SQLFragment<any, number | db.Parameter<number> | db.DefaultType | db.SQLFragment>;
/**
* **transfers.reconcile_tx_nonce**
* - `numeric` in database
* - `NOT NULL`, default: `0`
*/
reconcile_tx_nonce?: number | db.Parameter<number> | db.DefaultType | db.SQLFragment | db.SQLFragment<any, number | db.Parameter<number> | db.DefaultType | db.SQLFragment>;
}
export type UniqueIndex = 'transfers_pkey';
export type Column = keyof Selectable;
Expand Down
12 changes: 9 additions & 3 deletions packages/adapters/database/test/client.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -674,15 +674,19 @@ describe("Database client", () => {
it("should save multiple messages", async () => {
const messages: XMessage[] = [];
for (var _i = 0; _i < batchSize; _i++) {
messages.push(mock.entity.xMessage());
let message = mock.entity.xMessage();
message.origin.index = _i;
messages.push(message);
}
await saveMessages(messages, pool);
});

it("should upsert multiple messages", async () => {
const messages: XMessage[] = [];
for (var _i = 0; _i < batchSize; _i++) {
messages.push(mock.entity.xMessage());
let message = mock.entity.xMessage();
message.origin.index = _i;
messages.push(message);
}
await saveMessages(messages, pool);
for (let message of messages) {
Expand Down Expand Up @@ -803,7 +807,9 @@ describe("Database client", () => {
it("should getMessageByLeaf", async () => {
const messages: XMessage[] = [];
for (var _i = 0; _i < batchSize; _i++) {
messages.push(mock.entity.xMessage());
let message = mock.entity.xMessage();
message.origin.index = _i;
messages.push(message);
}
await saveMessages(messages, pool);

Expand Down

0 comments on commit 10ceea9

Please sign in to comment.