From 99c1d5871859f056e6290fe85e0065126dc6578e Mon Sep 17 00:00:00 2001 From: gabrielmer <101006718+gabrielmer@users.noreply.github.com> Date: Fri, 19 Apr 2024 13:10:43 +0300 Subject: [PATCH 1/7] chore(release): update changelog for v0.27.0 release (#2596) --- CHANGELOG.md | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32ec36146c..2a4fa4c07f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,80 @@ +## v0.27.0 (2024-04-19) + +> **Note:** + +> - Filter v1 protocol and its REST-API access have been deprecated. +> - A new field of the `WakuMetadataRequest` protobuf for shards was introduced. The old shards field (2) will be deprecated in 2 releases time +> - CLI flags `--requestRateLimit` and `--requestRatePeriod` have been added for rate limiting configuration. Period is measured in seconds. Limits are measured per protocol per period of time. Over limit will result in TOO_MANY_REQUEST (429) response. + +## What's Changed + +Release highlights: + +* Introduced configurable rate limiting for lightpush and store requests +* Sync time has been considerably reduced for node initialization +* Significant refactors were made to node initialization and `WakuArchive` logic as work towards C-bindings and Store V3 features + +### Features + +- Added simple, configurable rate limit for lightpush and store-query ([#2390](https://github.com/waku-org/nwaku/issues/2390)) ([a00f350c](https://github.com/waku-org/nwaku/commit/a00f350c)) +- examples/golang/waku.go add new example ([#2559](https://github.com/waku-org/nwaku/issues/2559)) ([8d66a548](https://github.com/waku-org/nwaku/commit/8d66a548)) +- **c-bindings:** rln relay ([#2544](https://github.com/waku-org/nwaku/issues/2544)) ([2aa835e3](https://github.com/waku-org/nwaku/commit/2aa835e3)) +- **incentivization:** add codec for eligibility proof and status ([#2419](https://github.com/waku-org/nwaku/issues/2419)) ([65530264](https://github.com/waku-org/nwaku/commit/65530264)) +- **rest:** add support to ephemeral field ([#2525](https://github.com/waku-org/nwaku/issues/2525)) ([c734f60d](https://github.com/waku-org/nwaku/commit/c734f60d)) +- archive update for store v3 ([#2451](https://github.com/waku-org/nwaku/issues/2451)) ([505479b8](https://github.com/waku-org/nwaku/commit/505479b8)) +- **c-bindings:** add function to dealloc nodes ([#2499](https://github.com/waku-org/nwaku/issues/2499)) ([8341864d](https://github.com/waku-org/nwaku/commit/8341864d)) + +### Bug Fixes + +- **rln-relay:** reduce sync time ([#2577](https://github.com/waku-org/nwaku/issues/2577)) ([480a62fa](https://github.com/waku-org/nwaku/commit/480a62fa)) +- rest store: content_topic -> contentTopic in the response ([#2584](https://github.com/waku-org/nwaku/issues/2584)) ([d2578553](https://github.com/waku-org/nwaku/commit/d2578553)) +- **c-bindings:** rln credential path key ([#2564](https://github.com/waku-org/nwaku/issues/2564)) ([3d752b11](https://github.com/waku-org/nwaku/commit/3d752b11)) +- cluster-id 0 disc5 issue ([#2562](https://github.com/waku-org/nwaku/issues/2562)) ([a76c9587](https://github.com/waku-org/nwaku/commit/a76c9587)) +- regex for rpc endpoint ([#2563](https://github.com/waku-org/nwaku/issues/2563)) ([c87545d5](https://github.com/waku-org/nwaku/commit/c87545d5)) +- **rln:** set a minimum epoch gap ([#2555](https://github.com/waku-org/nwaku/issues/2555)) ([b5e4795f](https://github.com/waku-org/nwaku/commit/b5e4795f)) +- fix regresion + remove deprecated flag ([#2556](https://github.com/waku-org/nwaku/issues/2556)) ([47ad0fb0](https://github.com/waku-org/nwaku/commit/47ad0fb0)) +- **networkmanager:** regularly disconnect from random peers ([#2553](https://github.com/waku-org/nwaku/issues/2553)) ([70c53fc0](https://github.com/waku-org/nwaku/commit/70c53fc0)) +- remove subscription queue limit ([#2551](https://github.com/waku-org/nwaku/issues/2551)) ([94ff5eab](https://github.com/waku-org/nwaku/commit/94ff5eab)) +- peer_manager - extend the number of connection requests to known peers ([#2534](https://github.com/waku-org/nwaku/issues/2534)) ([2173fe22](https://github.com/waku-org/nwaku/commit/2173fe22)) +- **2491:** Fix metadata protocol disconnecting light nodes ([#2533](https://github.com/waku-org/nwaku/issues/2533)) ([33774fad](https://github.com/waku-org/nwaku/commit/33774fad)) +- **rest:** filter/v2/subscriptions response ([#2529](https://github.com/waku-org/nwaku/issues/2529)) ([7aea2d4f](https://github.com/waku-org/nwaku/commit/7aea2d4f)) +- **store:** retention policy regex ([#2532](https://github.com/waku-org/nwaku/issues/2532)) ([23a291b3](https://github.com/waku-org/nwaku/commit/23a291b3)) +- enable autosharding in any cluster ([#2505](https://github.com/waku-org/nwaku/issues/2505)) ([5a225809](https://github.com/waku-org/nwaku/commit/5a225809)) +- introduce new field for shards in metadata protocol ([#2511](https://github.com/waku-org/nwaku/issues/2511)) ([f9f92b7d](https://github.com/waku-org/nwaku/commit/f9f92b7d)) +- **rln-relay:** handle empty metadata returned by getMetadata proc ([#2516](https://github.com/waku-org/nwaku/issues/2516)) ([1274b15d](https://github.com/waku-org/nwaku/commit/1274b15d)) + +### Changes + +- adding migration script adding i_query index ([#2578](https://github.com/waku-org/nwaku/issues/2578)) ([4117fe65](https://github.com/waku-org/nwaku/commit/4117fe65)) +- bumping chronicles version ([#2583](https://github.com/waku-org/nwaku/issues/2583)) ([a04e0d99](https://github.com/waku-org/nwaku/commit/a04e0d99)) +- add ARM64 support for Linux/MacOS ([#2580](https://github.com/waku-org/nwaku/issues/2580)) ([269139cf](https://github.com/waku-org/nwaku/commit/269139cf)) +- **rln:** update submodule + rln patch version ([#2574](https://github.com/waku-org/nwaku/issues/2574)) ([24f6fed8](https://github.com/waku-org/nwaku/commit/24f6fed8)) +- bumping dependencies for 0.27.0 ([#2572](https://github.com/waku-org/nwaku/issues/2572)) ([f68ac792](https://github.com/waku-org/nwaku/commit/f68ac792)) +- **c-bindings:** node initialization ([#2547](https://github.com/waku-org/nwaku/issues/2547)) ([6d0f6d82](https://github.com/waku-org/nwaku/commit/6d0f6d82)) +- remove deprecated legacy filter protocol ([#2507](https://github.com/waku-org/nwaku/issues/2507)) ([e8613172](https://github.com/waku-org/nwaku/commit/e8613172)) +- switch wakuv2 to waku fleet ([#2519](https://github.com/waku-org/nwaku/issues/2519)) ([18a05359](https://github.com/waku-org/nwaku/commit/18a05359)) +- create nph.md ([#2536](https://github.com/waku-org/nwaku/issues/2536)) ([a576e624](https://github.com/waku-org/nwaku/commit/a576e624)) +- Better postgres duplicate insert ([#2535](https://github.com/waku-org/nwaku/issues/2535)) ([693a1778](https://github.com/waku-org/nwaku/commit/693a1778)) +- add 150 kB to msg size histogram metric ([#2430](https://github.com/waku-org/nwaku/issues/2430)) ([2c1391d3](https://github.com/waku-org/nwaku/commit/2c1391d3)) +- content_script_version_2: add simple protection and rename messages_backup if exists ([#2531](https://github.com/waku-org/nwaku/issues/2531)) ([c6c376b5](https://github.com/waku-org/nwaku/commit/c6c376b5)) +- **vendor:** update nim-libp2p path ([#2527](https://github.com/waku-org/nwaku/issues/2527)) ([3c823756](https://github.com/waku-org/nwaku/commit/3c823756)) +- adding node factory tests ([#2524](https://github.com/waku-org/nwaku/issues/2524)) ([a1b3e090](https://github.com/waku-org/nwaku/commit/a1b3e090)) +- factory cleanup ([#2523](https://github.com/waku-org/nwaku/issues/2523)) ([8d7eb3a6](https://github.com/waku-org/nwaku/commit/8d7eb3a6)) +- **rln-relay-v2:** wakunode testing + improvements ([#2501](https://github.com/waku-org/nwaku/issues/2501)) ([059cb975](https://github.com/waku-org/nwaku/commit/059cb975)) +- update CHANGELOG for v0.26.0 release ([#2518](https://github.com/waku-org/nwaku/issues/2518)) ([097cb362](https://github.com/waku-org/nwaku/commit/097cb362)) +- migrating logic from wakunode2.nim to node_factory.nim ([#2504](https://github.com/waku-org/nwaku/issues/2504)) ([dcc88ee0](https://github.com/waku-org/nwaku/commit/dcc88ee0)) + +This release supports the following [libp2p protocols](https://docs.libp2p.io/concepts/protocols/): +| Protocol | Spec status | Protocol id | +| ---: | :---: | :--- | +| [`11/WAKU2-RELAY`](https://rfc.vac.dev/spec/11/) | `stable` | `/vac/waku/relay/2.0.0` | +| [`12/WAKU2-FILTER`](https://rfc.vac.dev/spec/12/) | `draft` | `/vac/waku/filter/2.0.0-beta1`
`/vac/waku/filter-subscribe/2.0.0-beta1`
`/vac/waku/filter-push/2.0.0-beta1` | +| [`13/WAKU2-STORE`](https://rfc.vac.dev/spec/13/) | `draft` | `/vac/waku/store/2.0.0-beta4` | +| [`19/WAKU2-LIGHTPUSH`](https://rfc.vac.dev/spec/19/) | `draft` | `/vac/waku/lightpush/2.0.0-beta1` | +| [`66/WAKU2-METADATA`](https://rfc.vac.dev/spec/66/) | `raw` | `/vac/waku/metadata/1.0.0` | + +The Waku v1 implementation has been removed from this repository and can be found in a separate [Waku Legacy](https://github.com/waku-org/waku-legacy) repository. + ## v0.26.0 (2024-03-07) > **Note:** From 21c0e8807210fa02cc8bb45d18266d6af4b31644 Mon Sep 17 00:00:00 2001 From: Gabriel mermelstein Date: Fri, 19 Apr 2024 17:24:11 +0300 Subject: [PATCH 2/7] Adding meta field to db schema --- .../content_script_version_4.nim | 71 +++++++++++++++++++ .../pg_migration_manager.nim | 5 +- .../driver/postgres_driver/migrations.nim | 2 +- 3 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 migrations/message_store_postgres/content_script_version_4.nim diff --git a/migrations/message_store_postgres/content_script_version_4.nim b/migrations/message_store_postgres/content_script_version_4.nim new file mode 100644 index 0000000000..d3a6ea7cb3 --- /dev/null +++ b/migrations/message_store_postgres/content_script_version_4.nim @@ -0,0 +1,71 @@ +const ContentScriptVersion_4* = + """ +ALTER TABLE IF EXISTS messages_backup RENAME TO messages; +ALTER TABLE messages RENAME TO messages_backup; +ALTER TABLE messages_backup DROP CONSTRAINT messageIndex; + +CREATE TABLE IF NOT EXISTS messages ( + pubsubTopic VARCHAR NOT NULL, + contentTopic VARCHAR NOT NULL, + payload VARCHAR, + version INTEGER NOT NULL, + timestamp BIGINT NOT NULL, + id VARCHAR NOT NULL, + messageHash VARCHAR NOT NULL, + storedAt BIGINT NOT NULL, + meta VARCHAR, + CONSTRAINT messageIndex PRIMARY KEY (messageHash, storedAt) + ) PARTITION BY RANGE (storedAt); + +DO $$ +DECLARE + min_storedAt numeric; + max_storedAt numeric; + min_storedAtSeconds integer = 0; + max_storedAtSeconds integer = 0; + partition_name TEXT; + create_partition_stmt TEXT; +BEGIN + SELECT MIN(storedAt) into min_storedAt + FROM messages_backup; + + SELECT MAX(storedAt) into max_storedAt + FROM messages_backup; + + min_storedAtSeconds := min_storedAt / 1000000000; + max_storedAtSeconds := max_storedAt / 1000000000; + + partition_name := 'messages_' || min_storedAtSeconds || '_' || max_storedAtSeconds; + create_partition_stmt := 'CREATE TABLE ' || partition_name || + ' PARTITION OF messages FOR VALUES FROM (' || + min_storedAt || ') TO (' || (max_storedAt + 1) || ')'; + IF min_storedAtSeconds > 0 AND max_storedAtSeconds > 0 THEN + EXECUTE create_partition_stmt USING partition_name, min_storedAt, max_storedAt; + END IF; +END $$; + +INSERT INTO messages ( + pubsubTopic, + contentTopic, + payload, + version, + timestamp, + id, + messageHash, + storedAt + ) + SELECT pubsubTopic, + contentTopic, + payload, + version, + timestamp, + id, + messageHash, + storedAt + FROM messages_backup; + +DROP TABLE messages_backup; + +UPDATE version SET version = 4 WHERE version = 3; + +""" diff --git a/migrations/message_store_postgres/pg_migration_manager.nim b/migrations/message_store_postgres/pg_migration_manager.nim index d4de18f6d9..052bc20fee 100644 --- a/migrations/message_store_postgres/pg_migration_manager.nim +++ b/migrations/message_store_postgres/pg_migration_manager.nim @@ -1,4 +1,6 @@ -import content_script_version_1, content_script_version_2, content_script_version_3 +import + content_script_version_1, content_script_version_2, content_script_version_3, + content_script_version_4 type MigrationScript* = object version*: int @@ -12,6 +14,7 @@ const PgMigrationScripts* = MigrationScript(version: 1, scriptContent: ContentScriptVersion_1), MigrationScript(version: 2, scriptContent: ContentScriptVersion_2), MigrationScript(version: 3, scriptContent: ContentScriptVersion_3), + MigrationScript(version: 3, scriptContent: ContentScriptVersion_4), ] proc getMigrationScripts*(currentVersion: int64, targetVersion: int64): seq[string] = diff --git a/waku/waku_archive/driver/postgres_driver/migrations.nim b/waku/waku_archive/driver/postgres_driver/migrations.nim index a7f4814792..a3e376e0ae 100644 --- a/waku/waku_archive/driver/postgres_driver/migrations.nim +++ b/waku/waku_archive/driver/postgres_driver/migrations.nim @@ -9,7 +9,7 @@ import logScope: topics = "waku archive migration" -const SchemaVersion* = 3 # increase this when there is an update in the database schema +const SchemaVersion* = 4 # increase this when there is an update in the database schema proc breakIntoStatements*(script: string): seq[string] = ## Given a full migration script, that can potentially contain a list From 13369fdd73749761d283f3f8d69c0fa3d0831d4c Mon Sep 17 00:00:00 2001 From: Gabriel mermelstein Date: Fri, 19 Apr 2024 17:41:31 +0300 Subject: [PATCH 3/7] adding meta field to postgres driver --- .../postgres_driver/postgres_driver.nim | 35 +++++++++++++------ 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/waku/waku_archive/driver/postgres_driver/postgres_driver.nim b/waku/waku_archive/driver/postgres_driver/postgres_driver.nim index 756590cf1a..6f1d672817 100644 --- a/waku/waku_archive/driver/postgres_driver/postgres_driver.nim +++ b/waku/waku_archive/driver/postgres_driver/postgres_driver.nim @@ -31,11 +31,11 @@ type PostgresDriver* = ref object of ArchiveDriver const InsertRowStmtName = "InsertRow" const InsertRowStmtDefinition = # TODO: get the sql queries from a file """INSERT INTO messages (id, messageHash, storedAt, contentTopic, payload, pubsubTopic, - version, timestamp) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT DO NOTHING;""" + version, timestamp, meta) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT DO NOTHING;""" const SelectNoCursorAscStmtName = "SelectWithoutCursorAsc" const SelectNoCursorAscStmtDef = - """SELECT storedAt, contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash FROM messages + """SELECT storedAt, contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta FROM messages WHERE contentTopic IN ($1) AND pubsubTopic = $2 AND storedAt >= $3 AND @@ -44,7 +44,7 @@ const SelectNoCursorAscStmtDef = const SelectNoCursorDescStmtName = "SelectWithoutCursorDesc" const SelectNoCursorDescStmtDef = - """SELECT storedAt, contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash FROM messages + """SELECT storedAt, contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta FROM messages WHERE contentTopic IN ($1) AND pubsubTopic = $2 AND storedAt >= $3 AND @@ -53,7 +53,7 @@ const SelectNoCursorDescStmtDef = const SelectWithCursorDescStmtName = "SelectWithCursorDesc" const SelectWithCursorDescStmtDef = - """SELECT storedAt, contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash FROM messages + """SELECT storedAt, contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta FROM messages WHERE contentTopic IN ($1) AND pubsubTopic = $2 AND (storedAt, id) < ($3,$4) AND @@ -63,7 +63,7 @@ const SelectWithCursorDescStmtDef = const SelectWithCursorAscStmtName = "SelectWithCursorAsc" const SelectWithCursorAscStmtDef = - """SELECT storedAt, contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash FROM messages + """SELECT storedAt, contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta FROM messages WHERE contentTopic IN ($1) AND pubsubTopic = $2 AND (storedAt, id) > ($3,$4) AND @@ -119,7 +119,7 @@ proc rowCallbackImpl( ## outRows - seq of Store-rows. This is populated from the info contained in pqResult let numFields = pqResult.pqnfields() - if numFields != 8: + if numFields != 9: error "Wrong number of fields" return @@ -134,6 +134,7 @@ proc rowCallbackImpl( var payload: string var hashHex: string var msgHash: WakuMessageHash + var meta: string try: storedAt = parseInt($(pqgetvalue(pqResult, iRow, 0))) @@ -144,6 +145,7 @@ proc rowCallbackImpl( timestamp = parseInt($(pqgetvalue(pqResult, iRow, 5))) digest = parseHexStr($(pqgetvalue(pqResult, iRow, 6))) hashHex = parseHexStr($(pqgetvalue(pqResult, iRow, 7))) + meta = parseHexStr($(pqgetvalue(pqResult, iRow, 8))) msgHash = fromBytes(hashHex.toOpenArrayByte(0, 31)) except ValueError: error "could not parse correctly", error = getCurrentExceptionMsg() @@ -152,6 +154,7 @@ proc rowCallbackImpl( wakuMessage.version = uint32(version) wakuMessage.contentTopic = contentTopic wakuMessage.payload = @(payload.toOpenArrayByte(0, payload.high)) + wakuMessage.meta = @(meta.toOpenArrayByte(0, meta.high)) outRows.add( ( @@ -178,6 +181,7 @@ method put*( let payload = toHex(message.payload) let version = $message.version let timestamp = $message.timestamp + let meta = toHex(message.meta) debug "put PostgresDriver", timestamp = timestamp @@ -186,7 +190,7 @@ method put*( InsertRowStmtDefinition, @[ digest, messageHash, rxTime, contentTopic, payload, pubsubTopic, version, - timestamp, + timestamp, meta, ], @[ int32(digest.len), @@ -197,8 +201,19 @@ method put*( int32(pubsubTopic.len), int32(version.len), int32(timestamp.len), + int32(meta.len), + ], + @[ + int32(0), + int32(0), + int32(0), + int32(0), + int32(0), + int32(0), + int32(0), + int32(0), + int32(0), ], - @[int32(0), int32(0), int32(0), int32(0), int32(0), int32(0), int32(0), int32(0)], ) method getAllMessages*( @@ -214,7 +229,7 @@ method getAllMessages*( await s.readConnPool.pgQuery( """SELECT storedAt, contentTopic, payload, pubsubTopic, version, timestamp, - id, messageHash FROM messages ORDER BY storedAt ASC""", + id, messageHash, meta FROM messages ORDER BY storedAt ASC""", newSeq[string](0), rowCallback, ) @@ -269,7 +284,7 @@ proc getMessagesArbitraryQuery( ## This proc allows to handle atypical queries. We don't use prepared statements for those. var query = - """SELECT storedAt, contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash FROM messages""" + """SELECT storedAt, contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta FROM messages""" var statements: seq[string] var args: seq[string] From 35132e8aad1354335144148742c228ee2d7a3ec5 Mon Sep 17 00:00:00 2001 From: Gabriel mermelstein Date: Fri, 19 Apr 2024 18:12:22 +0300 Subject: [PATCH 4/7] adding meta field to sqlite --- waku/factory/validator_signed.nim | 1 + .../driver/sqlite_driver/queries.nim | 36 +++++++++++++------ .../driver/sqlite_driver/sqlite_driver.nim | 1 + 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/waku/factory/validator_signed.nim b/waku/factory/validator_signed.nim index ddb45102f0..108629bff8 100644 --- a/waku/factory/validator_signed.nim +++ b/waku/factory/validator_signed.nim @@ -34,6 +34,7 @@ proc msgHash*(pubSubTopic: string, msg: WakuMessage): array[32, byte] = ctx.update(msg.payload) ctx.update(msg.contentTopic.toBytes()) ctx.update(msg.timestamp.uint64.toBytes(Endianness.littleEndian)) + ctx.update(msg.meta) ctx.update( if msg.ephemeral: @[1.byte] diff --git a/waku/waku_archive/driver/sqlite_driver/queries.nim b/waku/waku_archive/driver/sqlite_driver/queries.nim index ab1fa1e9f1..54c233f1b8 100644 --- a/waku/waku_archive/driver/sqlite_driver/queries.nim +++ b/waku/waku_archive/driver/sqlite_driver/queries.nim @@ -18,7 +18,7 @@ type SqlQueryStr = string proc queryRowWakuMessageCallback( s: ptr sqlite3_stmt, - contentTopicCol, payloadCol, versionCol, senderTimestampCol: cint, + contentTopicCol, payloadCol, versionCol, senderTimestampCol, metaCol: cint, ): WakuMessage = let topic = cast[ptr UncheckedArray[byte]](sqlite3_column_blob(s, contentTopicCol)) @@ -26,17 +26,21 @@ proc queryRowWakuMessageCallback( contentTopic = string.fromBytes(@(toOpenArray(topic, 0, topicLength - 1))) p = cast[ptr UncheckedArray[byte]](sqlite3_column_blob(s, payloadCol)) + m = cast[ptr UncheckedArray[byte]](sqlite3_column_blob(s, metaCol)) - length = sqlite3_column_bytes(s, payloadCol) - payload = @(toOpenArray(p, 0, length - 1)) + payloadLength = sqlite3_column_bytes(s, payloadCol) + metaLength = sqlite3_column_bytes(s, metaCol) + payload = @(toOpenArray(p, 0, payloadLength - 1)) version = sqlite3_column_int64(s, versionCol) senderTimestamp = sqlite3_column_int64(s, senderTimestampCol) + meta = @(toOpenArray(m, 0, metaLength - 1)) return WakuMessage( contentTopic: ContentTopic(contentTopic), payload: payload, version: uint32(version), timestamp: Timestamp(senderTimestamp), + meta: meta, ) proc queryRowReceiverTimestampCallback( @@ -83,8 +87,8 @@ proc createTableQuery(table: string): SqlQueryStr = "CREATE TABLE IF NOT EXISTS " & table & " (" & " pubsubTopic BLOB NOT NULL," & " contentTopic BLOB NOT NULL," & " payload BLOB," & " version INTEGER NOT NULL," & " timestamp INTEGER NOT NULL," & " id BLOB," & " messageHash BLOB," & - " storedAt INTEGER NOT NULL," & " CONSTRAINT messageIndex PRIMARY KEY (messageHash)" & - ") WITHOUT ROWID;" + " storedAt INTEGER NOT NULL," & " meta BLOB" & + " CONSTRAINT messageIndex PRIMARY KEY (messageHash)" & ") WITHOUT ROWID;" proc createTable*(db: SqliteDatabase): DatabaseResult[void] = let query = createTableQuery(DbTable) @@ -135,8 +139,8 @@ type InsertMessageParams* = proc insertMessageQuery(table: string): SqlQueryStr = return "INSERT INTO " & table & - "(id, messageHash, storedAt, contentTopic, payload, pubsubTopic, version, timestamp)" & - " VALUES (?, ?, ?, ?, ?, ?, ?, ?);" + "(id, messageHash, storedAt, contentTopic, payload, pubsubTopic, version, timestamp, meta)" & + " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?);" proc prepareInsertMessageStmt*( db: SqliteDatabase @@ -244,7 +248,7 @@ proc deleteOldestMessagesNotWithinLimit*( proc selectAllMessagesQuery(table: string): SqlQueryStr = return - "SELECT storedAt, contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash" & + "SELECT storedAt, contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta" & " FROM " & table & " ORDER BY storedAt ASC" proc selectAllMessages*( @@ -258,7 +262,12 @@ proc selectAllMessages*( let pubsubTopic = queryRowPubsubTopicCallback(s, pubsubTopicCol = 3) wakuMessage = queryRowWakuMessageCallback( - s, contentTopicCol = 1, payloadCol = 2, versionCol = 4, senderTimestampCol = 5 + s, + contentTopicCol = 1, + payloadCol = 2, + versionCol = 4, + senderTimestampCol = 5, + metaCol = 8, ) digest = queryRowDigestCallback(s, digestCol = 6) storedAt = queryRowReceiverTimestampCallback(s, storedAtCol = 0) @@ -354,7 +363,7 @@ proc selectMessagesWithLimitQuery( var query: string query = - "SELECT storedAt, contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash" + "SELECT storedAt, contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta" query &= " FROM " & table if where.isSome(): @@ -457,7 +466,12 @@ proc selectMessagesByHistoryQueryWithLimit*( let pubsubTopic = queryRowPubsubTopicCallback(s, pubsubTopicCol = 3) message = queryRowWakuMessageCallback( - s, contentTopicCol = 1, payloadCol = 2, versionCol = 4, senderTimestampCol = 5 + s, + contentTopicCol = 1, + payloadCol = 2, + versionCol = 4, + senderTimestampCol = 5, + metaCol = 8, ) digest = queryRowDigestCallback(s, digestCol = 6) storedAt = queryRowReceiverTimestampCallback(s, storedAtCol = 0) diff --git a/waku/waku_archive/driver/sqlite_driver/sqlite_driver.nim b/waku/waku_archive/driver/sqlite_driver/sqlite_driver.nim index 5331f2ea47..2ce31fe0fb 100644 --- a/waku/waku_archive/driver/sqlite_driver/sqlite_driver.nim +++ b/waku/waku_archive/driver/sqlite_driver/sqlite_driver.nim @@ -72,6 +72,7 @@ method put*( toBytes(pubsubTopic), # pubsubTopic int64(message.version), # version message.timestamp, # senderTimestamp + message.meta, # meta ) ) From dd3d2f8ffc59d906cc4931f5b8cfbc4af0740b5e Mon Sep 17 00:00:00 2001 From: Gabriel mermelstein Date: Fri, 19 Apr 2024 18:19:20 +0300 Subject: [PATCH 5/7] fix compilation error --- waku/waku_archive/driver/sqlite_driver/queries.nim | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/waku/waku_archive/driver/sqlite_driver/queries.nim b/waku/waku_archive/driver/sqlite_driver/queries.nim index 54c233f1b8..812a919829 100644 --- a/waku/waku_archive/driver/sqlite_driver/queries.nim +++ b/waku/waku_archive/driver/sqlite_driver/queries.nim @@ -133,8 +133,17 @@ proc createHistoryQueryIndex*(db: SqliteDatabase): DatabaseResult[void] = return ok() ## Insert message -type InsertMessageParams* = - (seq[byte], seq[byte], Timestamp, seq[byte], seq[byte], seq[byte], int64, Timestamp) +type InsertMessageParams* = ( + seq[byte], + seq[byte], + Timestamp, + seq[byte], + seq[byte], + seq[byte], + int64, + Timestamp, + seq[byte], +) proc insertMessageQuery(table: string): SqlQueryStr = return From 4964a79c6db71de70e8f046ddad7625799f68761 Mon Sep 17 00:00:00 2001 From: Gabriel mermelstein Date: Mon, 22 Apr 2024 10:40:24 +0300 Subject: [PATCH 6/7] adding missing comma --- waku/waku_archive/driver/sqlite_driver/queries.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/waku/waku_archive/driver/sqlite_driver/queries.nim b/waku/waku_archive/driver/sqlite_driver/queries.nim index 812a919829..8c16f66095 100644 --- a/waku/waku_archive/driver/sqlite_driver/queries.nim +++ b/waku/waku_archive/driver/sqlite_driver/queries.nim @@ -87,7 +87,7 @@ proc createTableQuery(table: string): SqlQueryStr = "CREATE TABLE IF NOT EXISTS " & table & " (" & " pubsubTopic BLOB NOT NULL," & " contentTopic BLOB NOT NULL," & " payload BLOB," & " version INTEGER NOT NULL," & " timestamp INTEGER NOT NULL," & " id BLOB," & " messageHash BLOB," & - " storedAt INTEGER NOT NULL," & " meta BLOB" & + " storedAt INTEGER NOT NULL," & " meta BLOB," & " CONSTRAINT messageIndex PRIMARY KEY (messageHash)" & ") WITHOUT ROWID;" proc createTable*(db: SqliteDatabase): DatabaseResult[void] = From d92459a196691a30a475fa651ed030bbba03f34a Mon Sep 17 00:00:00 2001 From: Gabriel mermelstein Date: Mon, 22 Apr 2024 11:34:05 +0300 Subject: [PATCH 7/7] not including meta field in hash for signed topics --- waku/factory/validator_signed.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/waku/factory/validator_signed.nim b/waku/factory/validator_signed.nim index 108629bff8..f33d362dee 100644 --- a/waku/factory/validator_signed.nim +++ b/waku/factory/validator_signed.nim @@ -34,7 +34,7 @@ proc msgHash*(pubSubTopic: string, msg: WakuMessage): array[32, byte] = ctx.update(msg.payload) ctx.update(msg.contentTopic.toBytes()) ctx.update(msg.timestamp.uint64.toBytes(Endianness.littleEndian)) - ctx.update(msg.meta) + # ctx.update(msg.meta) meta is not included in the message hash, as the signature goes in the meta field ctx.update( if msg.ephemeral: @[1.byte]