From 4ed576c2ac415be60d2d04373ea3489656518eb1 Mon Sep 17 00:00:00 2001 From: Ivan Folgueira Bande Date: Thu, 21 Dec 2023 21:59:23 +0100 Subject: [PATCH] wakunode2 app now accepts max-num-bytes-msg-size with KiB, KB, or B units --- apps/wakunode2/app.nim | 8 +- apps/wakunode2/external_config.nim | 6 +- tests/common/test_all.nim | 3 +- tests/common/test_parse_size.nim | 107 ++++++++++++++++++++++ waku/common/utils/parse_size_units.nim | 41 +++++++++ waku/waku_core/message/default_values.nim | 1 + 6 files changed, 161 insertions(+), 5 deletions(-) create mode 100644 tests/common/test_parse_size.nim create mode 100644 waku/common/utils/parse_size_units.nim diff --git a/apps/wakunode2/app.nim b/apps/wakunode2/app.nim index 8da049d2c3..46de84a70b 100644 --- a/apps/wakunode2/app.nim +++ b/apps/wakunode2/app.nim @@ -21,6 +21,7 @@ import metrics/chronos_httpserver import ../../waku/common/utils/nat, + ../../waku/common/utils/parse_size_units, ../../waku/common/databases/db_sqlite, ../../waku/waku_archive/driver/builder, ../../waku/waku_archive/retention_policy/builder, @@ -440,9 +441,14 @@ proc setupProtocols(node: WakuNode, else: conf.topics + let parsedMaxMsgSize = parseMsgSize(conf.maxMessageSize).valueOr: + return err("failed to parse 'max-num-bytes-msg-size' param: " & $error) + + debug "Setting max message size", num_bytes=parsedMaxMsgSize + try: await mountRelay(node, pubsubTopics, peerExchangeHandler = peerExchangeHandler, - conf.maxMessageSize) + parsedMaxMsgSize) except CatchableError: return err("failed to mount waku relay protocol: " & getCurrentExceptionMsg()) diff --git a/apps/wakunode2/external_config.nim b/apps/wakunode2/external_config.nim index 799c6f6589..08192f82d1 100644 --- a/apps/wakunode2/external_config.nim +++ b/apps/wakunode2/external_config.nim @@ -80,9 +80,9 @@ type name: "rln-relay-eth-private-key" }: string maxMessageSize* {. - desc: "Maximum message size in bytes. e.j. to set a 1 MiB, set this to 1048576 (1024*1024)." - defaultValue: MaxWakuMessageSize - name: "max-num-bytes-msg-size" }: int + desc: "Maximum message size. Accepted units: KiB, KB, and B. e.g. 1024KiB; 1500 B; etc." + defaultValue: DefaultMaxWakuMessageSizeStr + name: "max-num-bytes-msg-size" }: string case cmd* {. command diff --git a/tests/common/test_all.nim b/tests/common/test_all.nim index d1573c5894..fbc8ada09f 100644 --- a/tests/common/test_all.nim +++ b/tests/common/test_all.nim @@ -4,4 +4,5 @@ import ./test_enr_builder, ./test_envvar_serialization, ./test_protobuf_validation, - ./test_sqlite_migrations + ./test_sqlite_migrations, + ./test_parse_size diff --git a/tests/common/test_parse_size.nim b/tests/common/test_parse_size.nim new file mode 100644 index 0000000000..b7e343f241 --- /dev/null +++ b/tests/common/test_parse_size.nim @@ -0,0 +1,107 @@ +{.used.} + +import + testutils/unittests, + stew/results +import + ../../waku/common/utils/parse_size_units + +suite "Size serialization test": + test "parse normal sizes": + var sizeInBytesRes = parseMsgSize("15 KiB") + assert sizeInBytesRes.isOk(), sizeInBytesRes.error + check sizeInBytesRes.get() == 15360 + + sizeInBytesRes = parseMsgSize(" 1048576 B") + assert sizeInBytesRes.isOk(), sizeInBytesRes.error + check sizeInBytesRes.get() == 1048576 + + sizeInBytesRes = parseMsgSize("150 B") + assert sizeInBytesRes.isOk(), sizeInBytesRes.error + check sizeInBytesRes.get() == 150 + + sizeInBytesRes = parseMsgSize("150 b") + assert sizeInBytesRes.isOk(), sizeInBytesRes.error + check sizeInBytesRes.get() == 150 + + sizeInBytesRes = parseMsgSize("150b") + assert sizeInBytesRes.isOk(), sizeInBytesRes.error + check sizeInBytesRes.get() == 150 + + sizeInBytesRes = parseMsgSize("1024kib") + assert sizeInBytesRes.isOk(), sizeInBytesRes.error + check sizeInBytesRes.get() == 1048576 + + sizeInBytesRes = parseMsgSize("1024KiB") + assert sizeInBytesRes.isOk(), sizeInBytesRes.error + check sizeInBytesRes.get() == 1048576 + + sizeInBytesRes = parseMsgSize("1024KB") + assert sizeInBytesRes.isOk(), sizeInBytesRes.error + check sizeInBytesRes.get() == 1024000 + + sizeInBytesRes = parseMsgSize("1024kb") + assert sizeInBytesRes.isOk(), sizeInBytesRes.error + check sizeInBytesRes.get() == 1024000 + + sizeInBytesRes = parseMsgSize("1.5 kib") + assert sizeInBytesRes.isOk(), sizeInBytesRes.error + check sizeInBytesRes.get() == 1536 + + sizeInBytesRes = parseMsgSize("1,5 kb") + assert sizeInBytesRes.isOk(), sizeInBytesRes.error + check sizeInBytesRes.get() == 1500 + + sizeInBytesRes = parseMsgSize("0,5 kb") + assert sizeInBytesRes.isOk(), sizeInBytesRes.error + check sizeInBytesRes.get() == 500 + + sizeInBytesRes = parseMsgSize("1.5 kb") + assert sizeInBytesRes.isOk(), sizeInBytesRes.error + check sizeInBytesRes.get() == 1500 + + sizeInBytesRes = parseMsgSize("0.5 kb") + assert sizeInBytesRes.isOk(), sizeInBytesRes.error + check sizeInBytesRes.get() == 500 + + sizeInBytesRes = parseMsgSize(" 1.5 KB") + assert sizeInBytesRes.isOk(), sizeInBytesRes.error + check sizeInBytesRes.get() == 1500 + + sizeInBytesRes = parseMsgSize(" 0.5 kb") + assert sizeInBytesRes.isOk(), sizeInBytesRes.error + check sizeInBytesRes.get() == 500 + + sizeInBytesRes = parseMsgSize(" 1024 kib") + assert sizeInBytesRes.isOk(), sizeInBytesRes.error + check sizeInBytesRes.get() == (1024 * 1024) + + test "parse wrong sizes": + var sizeInBytesRes = parseMsgSize("150K") + assert sizeInBytesRes.isErr(), "The size should be considered incorrect" + + sizeInBytesRes = parseMsgSize("150 iB") + assert sizeInBytesRes.isErr(), "The size should be considered incorrect" + + sizeInBytesRes = parseMsgSize("150 ib") + assert sizeInBytesRes.isErr(), "The size should be considered incorrect" + + sizeInBytesRes = parseMsgSize("150 MB") + assert sizeInBytesRes.isErr(), "The size should be considered incorrect" + + ## notice that we don't allow MB units explicitly. If someone want to set 1MiB, the + ## s/he should use 1024 KiB + sizeInBytesRes = parseMsgSize("150 MiB") + assert sizeInBytesRes.isErr(), "The size should be considered incorrect" + + sizeInBytesRes = parseMsgSize("150MiB") + assert sizeInBytesRes.isErr(), "The size should be considered incorrect" + + sizeInBytesRes = parseMsgSize("150K") + assert sizeInBytesRes.isErr(), "The size should be considered incorrect" + + sizeInBytesRes = parseMsgSize("150 K") + assert sizeInBytesRes.isErr(), "The size should be considered incorrect" + + sizeInBytesRes = parseMsgSize("15..0 KiB") + assert sizeInBytesRes.isErr(), "The size should be considered incorrect" \ No newline at end of file diff --git a/waku/common/utils/parse_size_units.nim b/waku/common/utils/parse_size_units.nim new file mode 100644 index 0000000000..e2a22b5b47 --- /dev/null +++ b/waku/common/utils/parse_size_units.nim @@ -0,0 +1,41 @@ + +import + std/strutils, + stew/results, + regex + +proc parseMsgSize*(input: string): Result[int, string] = + ## Parses size strings such as "1.2 KiB" or "3Kb" and returns the equivalent number of bytes + ## if the parse task goes well. If not, it returns an error describing the problem. + + const RegexDef = """\s*(\d+([\,\.]\d*)?)\s*([Kk]{0,1}[i]?[Bb]{1})""" + const RegexParseSize = re2(RegexDef) + + var m: RegexMatch2 + if input.match(RegexParseSize, m) == false: + return err("error in parseSize. regex is not matching: " & RegexDef) + + var value: float + + try: + value = parseFloat(input[m.captures[0]].replace(",", ".")) + except ValueError: + return err("invalid size in parseSize: " & getCurrentExceptionMsg() & + " error parsing: " & input[m.captures[0]] & " KKK : " & $m) + + let units = input[m.captures[2]].toLowerAscii() # units is "kib", or "kb", or "b". + + var multiplier: float + case units: + of "kb": + multiplier = 1000 + of "kib": + multiplier = 1024 + of "ib": + return err("wrong units. ib or iB aren't allowed.") + else: ## bytes + multiplier = 1 + + value = value * multiplier + + return ok(int(value)) diff --git a/waku/waku_core/message/default_values.nim b/waku/waku_core/message/default_values.nim index 984d9bb046..0b22cdf472 100644 --- a/waku/waku_core/message/default_values.nim +++ b/waku/waku_core/message/default_values.nim @@ -3,4 +3,5 @@ const ## https://rfc.vac.dev/spec/64/#message-size MaxWakuMessageSize* = 150 * 1024 # Remember that 1 MiB is the PubSub default + DefaultMaxWakuMessageSizeStr* = "150KiB" DefaultSafetyBufferProtocolOverhead* = 64 * 1024 # overhead measured in bytes