From 5cc4e3af6af2d6a9008740c2e41fe96a116c1046 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 19 May 2023 17:46:29 +0200 Subject: [PATCH] tests: only warn about "invalid" chunker params, fixes #7590 we previously allowed some weird chunker params, so we just warn about them instead of rejecting them. e.g. one could use super small chunk sizes. borg can do that, but you'll end up with a huge amount of chunks and very large hash tables (RAM usage) to manage them. also, one can violate the min <= mask <= max chunker param condition as in #7590 and it will somehow work, but will likely not dedup as good as when not violating that. borg2 **will** reject such strange chunker params, see #7586 for some ideas how to "fix" your repos. --- src/borg/helpers/parseformat.py | 24 ++++++++++++------------ src/borg/testsuite/helpers.py | 16 ++++++++++------ 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py index 9dfda8d24e..d96637b4d4 100644 --- a/src/borg/helpers/parseformat.py +++ b/src/borg/helpers/parseformat.py @@ -95,10 +95,16 @@ def interval(s): def ChunkerParams(s): + def reject_or_warn(msg, reject): + if reject: + raise argparse.ArgumentTypeError(msg) + else: + logger.warning(msg) + params = s.strip().split(',') count = len(params) if count == 0: - raise argparse.ArgumentTypeError('no chunker params given') + reject_or_warn('no chunker params given', True) algo = params[0].lower() if algo == CH_FIXED and 2 <= count <= 3: # fixed, block_size[, header_size] block_size = int(params[1]) @@ -109,11 +115,9 @@ def ChunkerParams(s): # or in-memory chunk management. # choose the block (chunk) size wisely: if you have a lot of data and you cut # it into very small chunks, you are asking for trouble! - raise argparse.ArgumentTypeError('block_size must not be less than 64 Bytes') + reject_or_warn('block_size must not be less than 64 Bytes', False) if block_size > MAX_DATA_SIZE or header_size > MAX_DATA_SIZE: - raise argparse.ArgumentTypeError( - 'block_size and header_size must not exceed MAX_DATA_SIZE [%d]' % MAX_DATA_SIZE - ) + reject_or_warn('block_size and header_size must not exceed MAX_DATA_SIZE [%d]' % MAX_DATA_SIZE, True) return algo, block_size, header_size if algo == 'default' and count == 1: # default return CHUNKER_PARAMS @@ -121,16 +125,12 @@ def ChunkerParams(s): if algo == CH_BUZHASH and count == 5 or count == 4: # [buzhash, ]chunk_min, chunk_max, chunk_mask, window_size chunk_min, chunk_max, chunk_mask, window_size = (int(p) for p in params[count - 4:]) if not (chunk_min <= chunk_mask <= chunk_max): - raise argparse.ArgumentTypeError('required: chunk_min <= chunk_mask <= chunk_max') + reject_or_warn('required: chunk_min <= chunk_mask <= chunk_max', False) if chunk_min < 6: # see comment in 'fixed' algo check - raise argparse.ArgumentTypeError( - 'min. chunk size exponent must not be less than 6 (2^6 = 64B min. chunk size)' - ) + reject_or_warn('min. chunk size exponent must not be less than 6 (2^6 = 64B min. chunk size)', False) if chunk_max > 23: - raise argparse.ArgumentTypeError( - 'max. chunk size exponent must not be more than 23 (2^23 = 8MiB max. chunk size)' - ) + reject_or_warn('max. chunk size exponent must not be more than 23 (2^23 = 8MiB max. chunk size)', True) return CH_BUZHASH, chunk_min, chunk_max, chunk_mask, window_size raise argparse.ArgumentTypeError('invalid chunker params') diff --git a/src/borg/testsuite/helpers.py b/src/borg/testsuite/helpers.py index 99057a3d29..ed7140e5ae 100644 --- a/src/borg/testsuite/helpers.py +++ b/src/borg/testsuite/helpers.py @@ -349,16 +349,20 @@ def test_chunkerparams(): assert ChunkerParams('fixed,4096') == ('fixed', 4096, 0) assert ChunkerParams('fixed,4096,200') == ('fixed', 4096, 200) # invalid values checking + borg2 = False # for borg < 2, we only emit a warning, but do not raise ArgumentTypeError for some cases with pytest.raises(ArgumentTypeError): ChunkerParams('crap,1,2,3,4') # invalid algo - with pytest.raises(ArgumentTypeError): - ChunkerParams('buzhash,5,7,6,4095') # too small min. size + if borg2: + with pytest.raises(ArgumentTypeError): + ChunkerParams('buzhash,5,7,6,4095') # too small min. size with pytest.raises(ArgumentTypeError): ChunkerParams('buzhash,19,24,21,4095') # too big max. size - with pytest.raises(ArgumentTypeError): - ChunkerParams('buzhash,23,19,21,4095') # violates min <= mask <= max - with pytest.raises(ArgumentTypeError): - ChunkerParams('fixed,63') # too small block size + if borg2: + with pytest.raises(ArgumentTypeError): + ChunkerParams('buzhash,23,19,21,4095') # violates min <= mask <= max + if borg2: + with pytest.raises(ArgumentTypeError): + ChunkerParams('fixed,63') # too small block size with pytest.raises(ArgumentTypeError): ChunkerParams('fixed,%d,%d' % (MAX_DATA_SIZE + 1, 4096)) # too big block size with pytest.raises(ArgumentTypeError):