diff --git a/.github/workflows/libfuzzer.yaml b/.github/workflows/libfuzzer.yaml index ac08fc7b7ec..ebe4354fb8b 100644 --- a/.github/workflows/libfuzzer.yaml +++ b/.github/workflows/libfuzzer.yaml @@ -49,7 +49,7 @@ jobs: # leading to a tainted cache - name: Cache PostgreSQL id: cache-postgresql - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/${{ env.PG_SRC_DIR }} key: "postgresql-libfuzzer-${{ steps.get-date.outputs.date }}-${{ hashFiles('.github/**') }}" @@ -145,7 +145,7 @@ jobs: uses: actions/checkout@v4 - name: Download the installation directory - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: fuzzing-install-dir @@ -240,9 +240,6 @@ jobs: exit 1 fi - # Check that the server is still alive. - psql -c "select 1" - ls db/corpus | wc -l fn="ts_read_compressed_data_directory('${{ matrix.case.algo }}', @@ -257,7 +254,7 @@ jobs: # Save interesting cases because the caches are not available for download from UI mkdir -p interesting psql -qtAX -c "select distinct on (location) 'db/' || path from $fn - order by location, bytes + order by location, bytes, path " | xargs cp -t interesting # Check that we don't have any internal errors @@ -267,10 +264,12 @@ jobs: echo "Internal program errors: $errors" [ $errors -eq 0 ] || exit 1 - # Shouldn't have any WARNINGS in the log. ! grep -F "] WARNING: " postgres.log + # Check that the server is still alive. + psql -c "select 1" + - name: Collect the logs if: always() id: collectlogs diff --git a/.github/workflows/sanitizer-build-and-test.yaml b/.github/workflows/sanitizer-build-and-test.yaml index ee1bb324f7d..261455cd48f 100644 --- a/.github/workflows/sanitizer-build-and-test.yaml +++ b/.github/workflows/sanitizer-build-and-test.yaml @@ -204,6 +204,11 @@ jobs: ./scripts/bundle_coredumps.sh grep -C40 "was terminated by signal" postgres.log > postgres-failure.log ||: + - name: Show sanitizer logs + if: always() + run: | + tail -vn +1 sanitizer* || : + - name: Coredumps if: always() && steps.collectlogs.outputs.coredumps == 'true' uses: actions/upload-artifact@v4 @@ -211,12 +216,14 @@ jobs: name: Coredumps ${{ matrix.os }} ${{ env.name }} ${{ matrix.pg }} path: coredumps - - name: sanitizer logs + - name: Upload sanitizer logs if: always() uses: actions/upload-artifact@v4 with: name: sanitizer logs ${{ matrix.os }} ${{ env.name }} ${{ matrix.pg }} - path: ${{ github.workspace }}/sanitizer + # The log_path sanitizer option means "Write logs to 'log_path.pid'". + # https://github.com/google/sanitizers/wiki/SanitizerCommonFlags + path: ${{ github.workspace }}/sanitizer* - name: Upload test results to the database if: always() diff --git a/scripts/upload_ci_stats.sh b/scripts/upload_ci_stats.sh index 38c0ff734da..e6b6ba7e85a 100755 --- a/scripts/upload_ci_stats.sh +++ b/scripts/upload_ci_stats.sh @@ -150,7 +150,8 @@ do done # Upload the logs. -for x in sanitizer/* {sqlsmith/sqlsmith,sanitizer,stacktrace,postgres-failure}.log *.diff +# Note that the sanitizer setting log_path means "write logs to 'log_path.pid'". +for x in sanitizer* sanitizer/* {sqlsmith/sqlsmith,sanitizer,stacktrace,postgres-failure}.log *.diff do if ! [ -e "$x" ]; then continue ; fi "${PSQL[@]}" <<<" diff --git a/tsl/src/compression/array.c b/tsl/src/compression/array.c index 78b8e6b2aa3..d73287c8a73 100644 --- a/tsl/src/compression/array.c +++ b/tsl/src/compression/array.c @@ -493,12 +493,18 @@ text_array_decompress_all_serialized_no_header(StringInfo si, bool has_nulls, Simple8bRleSerialized *sizes_serialized = bytes_deserialize_simple8b_and_advance(si); - uint32 sizes[GLOBAL_MAX_ROWS_PER_COMPRESSION]; + /* + * We need a quite significant padding of 63 elements, not bytes, after the + * last element, because we work in Simple8B blocks which can contain up to + * 64 elements. + */ + uint32 sizes[GLOBAL_MAX_ROWS_PER_COMPRESSION + 63]; const uint16 n_notnull = simple8brle_decompress_all_buf_uint32(sizes_serialized, sizes, sizeof(sizes) / sizeof(sizes[0])); const int n_total = has_nulls ? nulls_serialized->num_elements : n_notnull; + CheckCompressedData(n_total >= n_notnull); uint32 *offsets = (uint32 *) MemoryContextAllocZero(dest_mctx, @@ -509,22 +515,56 @@ text_array_decompress_all_serialized_no_header(StringInfo si, bool has_nulls, uint32 offset = 0; for (int i = 0; i < n_notnull; i++) { - void *vardata = consumeCompressedData(si, sizes[i]); + void *unaligned = consumeCompressedData(si, sizes[i]); + + /* + * We start reading from the end of previous datum, but this pointer + * might be not aligned as required for varlena-4b struct. We have to + * align it here. Note that sizes[i] includes the alignment as well in + * addition to the varlena size. + * + * See the corresponding row-by-row code in bytes_to_datum_and_advance(). + */ + void *vardata = DatumGetPointer(att_align_pointer(unaligned, TYPALIGN_INT, -1, unaligned)); + /* * Check for potentially corrupt varlena headers since we're reading them - * directly from compressed data. We can only have a plain datum - * with 1-byte or 4-byte header here, no TOAST or compressed data. + * directly from compressed data. */ - CheckCompressedData(VARATT_IS_4B_U(vardata) || - (VARATT_IS_1B(vardata) && !VARATT_IS_1B_E(vardata))); + if (VARATT_IS_4B_U(vardata)) + { + /* + * Full varsize must be larger or equal than the header size so that + * the calculation of size without header doesn't overflow. + */ + CheckCompressedData(VARSIZE_4B(vardata) >= VARHDRSZ); + } + else if (VARATT_IS_1B(vardata)) + { + /* Can't have a TOAST pointer here. */ + CheckCompressedData(!VARATT_IS_1B_E(vardata)); + + /* + * Full varsize must be larger or equal than the header size so that + * the calculation of size without header doesn't overflow. + */ + CheckCompressedData(VARSIZE_1B(vardata) >= VARHDRSZ_SHORT); + } + else + { + /* + * Can only have an uncompressed datum with 1-byte or 4-byte header + * here, no TOAST or compressed data. + */ + CheckCompressedData(false); + } + /* - * Full varsize must be larger or equal than the header size so that the - * calculation of size without header doesn't overflow. + * Size of varlena plus alignment must match the size stored in the + * sizes array for this element. */ - CheckCompressedData((VARATT_IS_1B(vardata) && VARSIZE_1B(vardata) >= VARHDRSZ_SHORT) || - (VARSIZE_4B(vardata) >= VARHDRSZ)); - /* Varsize must match the size stored in the sizes array for this element. */ - CheckCompressedData(VARSIZE_ANY(vardata) == sizes[i]); + const Datum alignment_bytes = PointerGetDatum(vardata) - PointerGetDatum(unaligned); + CheckCompressedData(VARSIZE_ANY(vardata) + alignment_bytes == sizes[i]); const uint32 textlen = VARSIZE_ANY_EXHDR(vardata); memcpy(&arrow_bodies[offset], VARDATA_ANY(vardata), textlen); diff --git a/tsl/src/compression/compression.h b/tsl/src/compression/compression.h index 7c391c0a854..5f917eaa7c1 100644 --- a/tsl/src/compression/compression.h +++ b/tsl/src/compression/compression.h @@ -377,7 +377,7 @@ extern enum CompressionAlgorithms compress_get_default_algorithm(Oid typeoid); */ #ifndef TS_COMPRESSION_FUZZING #define CORRUPT_DATA_MESSAGE(X) \ - (errmsg("the compressed data is corrupt"), errdetail(X), errcode(ERRCODE_DATA_CORRUPTED)) + (errmsg("the compressed data is corrupt"), errdetail("%s", X), errcode(ERRCODE_DATA_CORRUPTED)) #else #define CORRUPT_DATA_MESSAGE(X) (errcode(ERRCODE_DATA_CORRUPTED)) #endif diff --git a/tsl/src/compression/dictionary.c b/tsl/src/compression/dictionary.c index 63fa38104b2..2147105c7d7 100644 --- a/tsl/src/compression/dictionary.c +++ b/tsl/src/compression/dictionary.c @@ -426,6 +426,7 @@ tsl_text_dictionary_decompress_all(Datum compressed, Oid element_type, MemoryCon const uint16 n_notnull = indices_serialized->num_elements; const uint16 n_total = header->has_nulls ? nulls_serialized->num_elements : n_notnull; + CheckCompressedData(n_total >= n_notnull); const uint16 n_padded = n_total + 63; /* This is the padding requirement of simple8brle_decompress_all. */ int16 *restrict indices = MemoryContextAlloc(dest_mctx, sizeof(int16) * n_padded); diff --git a/tsl/src/compression/gorilla_impl.c b/tsl/src/compression/gorilla_impl.c index 41f98a31345..1fdd1d1a956 100644 --- a/tsl/src/compression/gorilla_impl.c +++ b/tsl/src/compression/gorilla_impl.c @@ -51,11 +51,11 @@ FUNCTION_NAME(gorilla_decompress_all, ELEMENT_TYPE)(CompressedGorillaData *goril const uint16 leading_zeros_padded = unpack_leading_zeros_array(&gorilla_data->leading_zeros, all_leading_zeros); - uint8 bit_widths[MAX_NUM_LEADING_ZEROS_PADDED_N64]; + uint8 bit_widths[GLOBAL_MAX_ROWS_PER_COMPRESSION + 63]; const uint16 num_bit_widths = simple8brle_decompress_all_buf_uint8(gorilla_data->num_bits_used_per_xor, bit_widths, - MAX_NUM_LEADING_ZEROS_PADDED_N64); + sizeof(bit_widths) / sizeof(bit_widths[0])); BitArray xors_bitarray = gorilla_data->xors; BitArrayIterator xors_iterator; diff --git a/tsl/src/compression/simple8b_rle_decompress_all.h b/tsl/src/compression/simple8b_rle_decompress_all.h index 52f8de8499b..8d36cd19087 100644 --- a/tsl/src/compression/simple8b_rle_decompress_all.h +++ b/tsl/src/compression/simple8b_rle_decompress_all.h @@ -21,6 +21,11 @@ FUNCTION_NAME(simple8brle_decompress_all_buf, { const uint16 n_total_values = compressed->num_elements; + /* + * Caller must have allocated a properly sized buffer, see the comment above. + */ + Assert(n_buffer_elements >= n_total_values + 63); + const uint16 num_selector_slots = simple8brle_num_selector_slots_for_num_blocks(compressed->num_blocks); const uint16 num_blocks = compressed->num_blocks; diff --git a/tsl/test/expected/compression_algos.out b/tsl/test/expected/compression_algos.out index fef1cb4444a..27cde6265a4 100644 --- a/tsl/test/expected/compression_algos.out +++ b/tsl/test/expected/compression_algos.out @@ -1595,10 +1595,10 @@ group by 2, 3 order by 1 desc ; count | bulk_result | rowbyrow_result -------+-------------+----------------- - 21 | XX001 | XX001 - 6 | 08P01 | 08P01 + 17 | XX001 | XX001 + 15 | true | true + 7 | 08P01 | 08P01 2 | 3F000 | 3F000 - 2 | true | true 1 | 22021 | 22021 1 | false | false (6 rows) @@ -1613,11 +1613,11 @@ group by 2, 3 order by 1 desc ; count | bulk_result | rowbyrow_result -------+-------------+----------------- - 51 | XX001 | XX001 - 4 | 08P01 | 08P01 - 4 | XX001 | true - 2 | true | true - 2 | 22021 | 22021 + 80 | XX001 | XX001 + 5 | 08P01 | 08P01 + 5 | XX001 | true + 4 | true | true + 3 | 22021 | 22021 1 | 3F000 | 3F000 1 | false | false (7 rows) diff --git a/tsl/test/expected/decompress_vector_qual.out b/tsl/test/expected/decompress_vector_qual.out index a567d375015..759f516740e 100644 --- a/tsl/test/expected/decompress_vector_qual.out +++ b/tsl/test/expected/decompress_vector_qual.out @@ -1053,3 +1053,71 @@ select * from date_table where ts < CURRENT_DATE; 01-01-2021 (3 rows) +-- Text columns. Only tests bulk decompression for now. +create table text_table(ts int, d int); +select create_hypertable('text_table', 'ts'); +NOTICE: adding not-null constraint to column "ts" + create_hypertable +------------------------- + (9,public,text_table,t) +(1 row) + +alter table text_table set (timescaledb.compress, timescaledb.compress_segmentby = 'd'); +insert into text_table select x, 0 /*, default */ from generate_series(1, 1000) x; +select count(compress_chunk(x, true)) from show_chunks('text_table') x; + count +------- + 1 +(1 row) + +alter table text_table add column a text default 'default'; +insert into text_table select x, 1, '' from generate_series(1, 1000) x; +insert into text_table select x, 2, 'same' from generate_series(1, 1000) x; +insert into text_table select x, 3, 'different' || x from generate_series(1, 1000) x; +insert into text_table select x, 4, case when x % 2 = 0 then null else 'same-with-nulls' end from generate_series(1, 1000) x; +insert into text_table select x, 5, case when x % 2 = 0 then null else 'different-with-nulls' || x end from generate_series(1, 1000) x; +insert into text_table select x, 6, 'одинаковый' from generate_series(1, 1000) x; +insert into text_table select x, 7, '異なる' || x from generate_series(1, 1000) x; +-- Some text values with varying lengths in a single batch. They are all different +-- to prevent dictionary compression, because we want to test particular orders +-- here as well. +insert into text_table select x, 8, repeat( x::text || 'a', x) from generate_series(1, 100) x; +insert into text_table select x + 100, 8, repeat((101 - x)::text || 'b', (101 - x)) from generate_series(1, 100) x; +insert into text_table select x + 200, 8, repeat((101 - x)::text || 'c', (101 - x)) from generate_series(1, 100) x; +insert into text_table select x + 300, 8, repeat( x::text || 'd', x) from generate_series(1, 100) x; +set timescaledb.debug_require_vector_qual to 'forbid'; +select sum(length(a)) from text_table; + sum +-------- + 118551 +(1 row) + +select count(distinct a) from text_table; + count +------- + 2905 +(1 row) + +select count(compress_chunk(x, true)) from show_chunks('text_table') x; +NOTICE: chunk "_hyper_9_17_chunk" is already compressed + count +------- + 1 +(1 row) + +select format('call recompress_chunk(''%s'')', x) from show_chunks('text_table') x \gexec +call recompress_chunk('_timescaledb_internal._hyper_9_17_chunk') +set timescaledb.enable_bulk_decompression to on; +set timescaledb.debug_require_vector_qual to 'forbid'; +select sum(length(a)) from text_table; + sum +-------- + 118551 +(1 row) + +select count(distinct a) from text_table; + count +------- + 2905 +(1 row) + diff --git a/tsl/test/fuzzing/compression/array-text/01accf1c403c681e8ccc10349c97a28ef2afbdd3 b/tsl/test/fuzzing/compression/array-text/01accf1c403c681e8ccc10349c97a28ef2afbdd3 new file mode 100644 index 00000000000..00a5f7bbffa Binary files /dev/null and b/tsl/test/fuzzing/compression/array-text/01accf1c403c681e8ccc10349c97a28ef2afbdd3 differ diff --git a/tsl/test/fuzzing/compression/array-text/3f82cf837a5f3fae26f7cbb25ed3d903eed71687 b/tsl/test/fuzzing/compression/array-text/3f82cf837a5f3fae26f7cbb25ed3d903eed71687 new file mode 100644 index 00000000000..1f88b03407d Binary files /dev/null and b/tsl/test/fuzzing/compression/array-text/3f82cf837a5f3fae26f7cbb25ed3d903eed71687 differ diff --git a/tsl/test/fuzzing/compression/array-text/5d1be7e9dda1ee8896be5b7e34a85ee16452a7b4 b/tsl/test/fuzzing/compression/array-text/5d1be7e9dda1ee8896be5b7e34a85ee16452a7b4 new file mode 100644 index 00000000000..303e398c82e --- /dev/null +++ b/tsl/test/fuzzing/compression/array-text/5d1be7e9dda1ee8896be5b7e34a85ee16452a7b4 @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/tsl/test/fuzzing/compression/array-text/6a126964d691ada7de1d25c976c4e7b481858665 b/tsl/test/fuzzing/compression/array-text/6a126964d691ada7de1d25c976c4e7b481858665 new file mode 100644 index 00000000000..2031608b329 Binary files /dev/null and b/tsl/test/fuzzing/compression/array-text/6a126964d691ada7de1d25c976c4e7b481858665 differ diff --git a/tsl/test/fuzzing/compression/array-text/bc6960f7ed1b4b216c5cbc2721ea5136ffb80aae b/tsl/test/fuzzing/compression/array-text/bc6960f7ed1b4b216c5cbc2721ea5136ffb80aae new file mode 100644 index 00000000000..a0ba87a1079 Binary files /dev/null and b/tsl/test/fuzzing/compression/array-text/bc6960f7ed1b4b216c5cbc2721ea5136ffb80aae differ diff --git a/tsl/test/fuzzing/compression/array-text/c2588a11e70caad0b9c91272f81d48348b94f938 b/tsl/test/fuzzing/compression/array-text/c2588a11e70caad0b9c91272f81d48348b94f938 new file mode 100644 index 00000000000..e85116c554b Binary files /dev/null and b/tsl/test/fuzzing/compression/array-text/c2588a11e70caad0b9c91272f81d48348b94f938 differ diff --git a/tsl/test/fuzzing/compression/array-text/d57ef126bce1cf2bb5a63addf172a9cf9bedb7da b/tsl/test/fuzzing/compression/array-text/d57ef126bce1cf2bb5a63addf172a9cf9bedb7da new file mode 100644 index 00000000000..269d95e5d72 Binary files /dev/null and b/tsl/test/fuzzing/compression/array-text/d57ef126bce1cf2bb5a63addf172a9cf9bedb7da differ diff --git a/tsl/test/fuzzing/compression/array-text/ebddebf24b791e82c4ac007197581ec4fa2886ee b/tsl/test/fuzzing/compression/array-text/ebddebf24b791e82c4ac007197581ec4fa2886ee new file mode 100644 index 00000000000..c32cf608e96 Binary files /dev/null and b/tsl/test/fuzzing/compression/array-text/ebddebf24b791e82c4ac007197581ec4fa2886ee differ diff --git a/tsl/test/fuzzing/compression/array-text/ec536d9d30f08137531d4bfe676a545d897ae98b b/tsl/test/fuzzing/compression/array-text/ec536d9d30f08137531d4bfe676a545d897ae98b new file mode 100644 index 00000000000..be853c86fe7 Binary files /dev/null and b/tsl/test/fuzzing/compression/array-text/ec536d9d30f08137531d4bfe676a545d897ae98b differ diff --git a/tsl/test/fuzzing/compression/array-text/length-saw b/tsl/test/fuzzing/compression/array-text/length-saw new file mode 100644 index 00000000000..600db2b823d Binary files /dev/null and b/tsl/test/fuzzing/compression/array-text/length-saw differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/03be2aaf02e2f78738dcd55209eb660c63460f73 b/tsl/test/fuzzing/compression/dictionary-text/03be2aaf02e2f78738dcd55209eb660c63460f73 new file mode 100644 index 00000000000..ff21c25502c Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/03be2aaf02e2f78738dcd55209eb660c63460f73 differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/0ce9675bea488114cd61e0902df0f65bb266b414 b/tsl/test/fuzzing/compression/dictionary-text/0ce9675bea488114cd61e0902df0f65bb266b414 new file mode 100644 index 00000000000..e0ceba082c0 Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/0ce9675bea488114cd61e0902df0f65bb266b414 differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/193ff1ad23aaeaa8feafcbe7168f259e5063e715 b/tsl/test/fuzzing/compression/dictionary-text/193ff1ad23aaeaa8feafcbe7168f259e5063e715 new file mode 100644 index 00000000000..492ae25446f Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/193ff1ad23aaeaa8feafcbe7168f259e5063e715 differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/24a09483ef45200b6ed894e43934767755155e3d b/tsl/test/fuzzing/compression/dictionary-text/24a09483ef45200b6ed894e43934767755155e3d new file mode 100644 index 00000000000..b9af7a27b03 Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/24a09483ef45200b6ed894e43934767755155e3d differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/2dd04d16222832a133c966eb5e9c4e7ae4d31059 b/tsl/test/fuzzing/compression/dictionary-text/2dd04d16222832a133c966eb5e9c4e7ae4d31059 new file mode 100644 index 00000000000..09862945ae9 Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/2dd04d16222832a133c966eb5e9c4e7ae4d31059 differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/3902e1ec9e8894b6befc97144a957e69f696bc75 b/tsl/test/fuzzing/compression/dictionary-text/3902e1ec9e8894b6befc97144a957e69f696bc75 new file mode 100644 index 00000000000..734c2990566 Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/3902e1ec9e8894b6befc97144a957e69f696bc75 differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/39e6777ccb030519cefc62205e2df23e703aa9fa b/tsl/test/fuzzing/compression/dictionary-text/39e6777ccb030519cefc62205e2df23e703aa9fa new file mode 100644 index 00000000000..f7c4c973617 Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/39e6777ccb030519cefc62205e2df23e703aa9fa differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/3b2185a90d32decad650ceb443751a4023368cdd b/tsl/test/fuzzing/compression/dictionary-text/3b2185a90d32decad650ceb443751a4023368cdd new file mode 100644 index 00000000000..4cf12c61b2a Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/3b2185a90d32decad650ceb443751a4023368cdd differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/446dbbeac1d19ca3ac51def5be4e9fcccf97cdc6 b/tsl/test/fuzzing/compression/dictionary-text/446dbbeac1d19ca3ac51def5be4e9fcccf97cdc6 new file mode 100644 index 00000000000..0eab8660ae7 Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/446dbbeac1d19ca3ac51def5be4e9fcccf97cdc6 differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/48ddda607e4778f35289784b3f6c80234660856d b/tsl/test/fuzzing/compression/dictionary-text/48ddda607e4778f35289784b3f6c80234660856d new file mode 100644 index 00000000000..f2ffb07080f Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/48ddda607e4778f35289784b3f6c80234660856d differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/4979455aba5b4c7d302af47c4fd941214e16d3a9 b/tsl/test/fuzzing/compression/dictionary-text/4979455aba5b4c7d302af47c4fd941214e16d3a9 new file mode 100644 index 00000000000..701645ad3d8 Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/4979455aba5b4c7d302af47c4fd941214e16d3a9 differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/5e81672e813bd1e06ecba224f520328498f27ea8 b/tsl/test/fuzzing/compression/dictionary-text/5e81672e813bd1e06ecba224f520328498f27ea8 new file mode 100644 index 00000000000..2d0a6cd31a0 Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/5e81672e813bd1e06ecba224f520328498f27ea8 differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/626bf1b65f1a0c5fba061fac07a711f970b07a80 b/tsl/test/fuzzing/compression/dictionary-text/626bf1b65f1a0c5fba061fac07a711f970b07a80 new file mode 100644 index 00000000000..ee91a2b682b Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/626bf1b65f1a0c5fba061fac07a711f970b07a80 differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/68c94392b09d47edea2a48f62808073bf83448f3 b/tsl/test/fuzzing/compression/dictionary-text/68c94392b09d47edea2a48f62808073bf83448f3 new file mode 100644 index 00000000000..c367012e07d Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/68c94392b09d47edea2a48f62808073bf83448f3 differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/75fc076b6fc8dac4f65c31fa4fd34ad236530422 b/tsl/test/fuzzing/compression/dictionary-text/75fc076b6fc8dac4f65c31fa4fd34ad236530422 new file mode 100644 index 00000000000..cd21b9ed2b8 Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/75fc076b6fc8dac4f65c31fa4fd34ad236530422 differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/76fa9dc37fc42934404c72df57d296c663ee807d b/tsl/test/fuzzing/compression/dictionary-text/76fa9dc37fc42934404c72df57d296c663ee807d new file mode 100644 index 00000000000..80396130f9b Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/76fa9dc37fc42934404c72df57d296c663ee807d differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/79ef4a8ba594c7a2b611bc33fc8b83fe8ac14998 b/tsl/test/fuzzing/compression/dictionary-text/79ef4a8ba594c7a2b611bc33fc8b83fe8ac14998 new file mode 100644 index 00000000000..ec70dbd3d3a Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/79ef4a8ba594c7a2b611bc33fc8b83fe8ac14998 differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/7bbc7585698a7c3375bea9c3bcff8838722d8f64 b/tsl/test/fuzzing/compression/dictionary-text/7bbc7585698a7c3375bea9c3bcff8838722d8f64 new file mode 100644 index 00000000000..f6e3ac22f9c Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/7bbc7585698a7c3375bea9c3bcff8838722d8f64 differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/7be90a9058961ac6ee07f764618805dfe8b6cfae b/tsl/test/fuzzing/compression/dictionary-text/7be90a9058961ac6ee07f764618805dfe8b6cfae new file mode 100644 index 00000000000..5076d4c666a Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/7be90a9058961ac6ee07f764618805dfe8b6cfae differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/8426e28aabe6684eb7200bd032ff0dad5a5169af b/tsl/test/fuzzing/compression/dictionary-text/8426e28aabe6684eb7200bd032ff0dad5a5169af new file mode 100644 index 00000000000..b14a908aa6f Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/8426e28aabe6684eb7200bd032ff0dad5a5169af differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/8a5c3216797d7a54adeafc86b708b942a9894b2f b/tsl/test/fuzzing/compression/dictionary-text/8a5c3216797d7a54adeafc86b708b942a9894b2f new file mode 100644 index 00000000000..2b37c0ddc60 Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/8a5c3216797d7a54adeafc86b708b942a9894b2f differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/9a78211436f6d425ec38f5c4e02270801f3524f8 b/tsl/test/fuzzing/compression/dictionary-text/9a78211436f6d425ec38f5c4e02270801f3524f8 new file mode 100644 index 00000000000..b516b2c489f --- /dev/null +++ b/tsl/test/fuzzing/compression/dictionary-text/9a78211436f6d425ec38f5c4e02270801f3524f8 @@ -0,0 +1 @@ +@ \ No newline at end of file diff --git a/tsl/test/fuzzing/compression/dictionary-text/9b99593353a610c4bee0d6a94a01a3296080c0fb b/tsl/test/fuzzing/compression/dictionary-text/9b99593353a610c4bee0d6a94a01a3296080c0fb new file mode 100644 index 00000000000..5407bf3ddf8 Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/9b99593353a610c4bee0d6a94a01a3296080c0fb differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/a54a56388dd751dc1ea1727f8e2965b349b54800 b/tsl/test/fuzzing/compression/dictionary-text/a54a56388dd751dc1ea1727f8e2965b349b54800 new file mode 100644 index 00000000000..5c727e2feee Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/a54a56388dd751dc1ea1727f8e2965b349b54800 differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/ae02ec1f395c202f6cd2965ea34d73dc35e10fdf b/tsl/test/fuzzing/compression/dictionary-text/ae02ec1f395c202f6cd2965ea34d73dc35e10fdf new file mode 100644 index 00000000000..4c3399746ad Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/ae02ec1f395c202f6cd2965ea34d73dc35e10fdf differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/af094ea4a3607992332e87dbe90f1a0f0cf82e09 b/tsl/test/fuzzing/compression/dictionary-text/af094ea4a3607992332e87dbe90f1a0f0cf82e09 new file mode 100644 index 00000000000..46f0878ad9d Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/af094ea4a3607992332e87dbe90f1a0f0cf82e09 differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/b18ecac8feda2826b91131b386b8842a1fca17e5 b/tsl/test/fuzzing/compression/dictionary-text/b18ecac8feda2826b91131b386b8842a1fca17e5 new file mode 100644 index 00000000000..314e37e727c Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/b18ecac8feda2826b91131b386b8842a1fca17e5 differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/b931131d935fb27ebf7977285299dcf11bb52eb4 b/tsl/test/fuzzing/compression/dictionary-text/b931131d935fb27ebf7977285299dcf11bb52eb4 new file mode 100644 index 00000000000..3312fe547a3 Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/b931131d935fb27ebf7977285299dcf11bb52eb4 differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/e767ec96033f7d9da306711adab0a6557fd4b71e b/tsl/test/fuzzing/compression/dictionary-text/e767ec96033f7d9da306711adab0a6557fd4b71e new file mode 100644 index 00000000000..343f09135c0 Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/e767ec96033f7d9da306711adab0a6557fd4b71e differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/f0f2e7efda77af51c83fe7870bd04d1b93556116 b/tsl/test/fuzzing/compression/dictionary-text/f0f2e7efda77af51c83fe7870bd04d1b93556116 new file mode 100644 index 00000000000..2fbea9bb8d8 Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/f0f2e7efda77af51c83fe7870bd04d1b93556116 differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/f2d42e12fb2ed451e7ba0b270a9065916a391fb1 b/tsl/test/fuzzing/compression/dictionary-text/f2d42e12fb2ed451e7ba0b270a9065916a391fb1 new file mode 100644 index 00000000000..7a86d09d8a0 Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/f2d42e12fb2ed451e7ba0b270a9065916a391fb1 differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/f6377e2f32fd888f1877518c26bf6be4e24f92bd b/tsl/test/fuzzing/compression/dictionary-text/f6377e2f32fd888f1877518c26bf6be4e24f92bd new file mode 100644 index 00000000000..69a70de609c Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/f6377e2f32fd888f1877518c26bf6be4e24f92bd differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/fc8f69146bf6f556444c5acd8053537a292db418 b/tsl/test/fuzzing/compression/dictionary-text/fc8f69146bf6f556444c5acd8053537a292db418 new file mode 100644 index 00000000000..0d10f5366f9 Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/fc8f69146bf6f556444c5acd8053537a292db418 differ diff --git a/tsl/test/fuzzing/compression/dictionary-text/fee17c65913f4eddcae3f69d1c2b6f318b938af2 b/tsl/test/fuzzing/compression/dictionary-text/fee17c65913f4eddcae3f69d1c2b6f318b938af2 new file mode 100644 index 00000000000..f7ca816fb87 Binary files /dev/null and b/tsl/test/fuzzing/compression/dictionary-text/fee17c65913f4eddcae3f69d1c2b6f318b938af2 differ diff --git a/tsl/test/sql/decompress_vector_qual.sql b/tsl/test/sql/decompress_vector_qual.sql index b9105ec4ec8..d68d05db828 100644 --- a/tsl/test/sql/decompress_vector_qual.sql +++ b/tsl/test/sql/decompress_vector_qual.sql @@ -352,3 +352,40 @@ select * from date_table where ts <= '2021-01-02'; select * from date_table where ts < '2021-01-02'; select * from date_table where ts < CURRENT_DATE; +-- Text columns. Only tests bulk decompression for now. +create table text_table(ts int, d int); +select create_hypertable('text_table', 'ts'); +alter table text_table set (timescaledb.compress, timescaledb.compress_segmentby = 'd'); + +insert into text_table select x, 0 /*, default */ from generate_series(1, 1000) x; +select count(compress_chunk(x, true)) from show_chunks('text_table') x; +alter table text_table add column a text default 'default'; + +insert into text_table select x, 1, '' from generate_series(1, 1000) x; +insert into text_table select x, 2, 'same' from generate_series(1, 1000) x; +insert into text_table select x, 3, 'different' || x from generate_series(1, 1000) x; +insert into text_table select x, 4, case when x % 2 = 0 then null else 'same-with-nulls' end from generate_series(1, 1000) x; +insert into text_table select x, 5, case when x % 2 = 0 then null else 'different-with-nulls' || x end from generate_series(1, 1000) x; +insert into text_table select x, 6, 'одинаковый' from generate_series(1, 1000) x; +insert into text_table select x, 7, '異なる' || x from generate_series(1, 1000) x; + +-- Some text values with varying lengths in a single batch. They are all different +-- to prevent dictionary compression, because we want to test particular orders +-- here as well. +insert into text_table select x, 8, repeat( x::text || 'a', x) from generate_series(1, 100) x; +insert into text_table select x + 100, 8, repeat((101 - x)::text || 'b', (101 - x)) from generate_series(1, 100) x; +insert into text_table select x + 200, 8, repeat((101 - x)::text || 'c', (101 - x)) from generate_series(1, 100) x; +insert into text_table select x + 300, 8, repeat( x::text || 'd', x) from generate_series(1, 100) x; + +set timescaledb.debug_require_vector_qual to 'forbid'; +select sum(length(a)) from text_table; +select count(distinct a) from text_table; + +select count(compress_chunk(x, true)) from show_chunks('text_table') x; +select format('call recompress_chunk(''%s'')', x) from show_chunks('text_table') x \gexec + +set timescaledb.enable_bulk_decompression to on; +set timescaledb.debug_require_vector_qual to 'forbid'; + +select sum(length(a)) from text_table; +select count(distinct a) from text_table; diff --git a/tsl/test/src/decompress_arithmetic_test_impl.c b/tsl/test/src/decompress_arithmetic_test_impl.c index c8445095d05..83db75b0a73 100644 --- a/tsl/test/src/decompress_arithmetic_test_impl.c +++ b/tsl/test/src/decompress_arithmetic_test_impl.c @@ -200,7 +200,17 @@ FUNCTION_NAME3(decompress, ALGO, PG_TYPE_PREFIX)(const uint8 *Data, size_t Size, * 3) The bulk decompression must absolutely work on the correct compressed * data we've just generated. */ - arrow = decompress_all(compressed_data, PG_TYPE_OID, CurrentMemoryContext); + PG_TRY(); + { + arrow = decompress_all(compressed_data, PG_TYPE_OID, CurrentMemoryContext); + } + PG_CATCH(); + { + EmitErrorReport(); + elog(PANIC, "bulk decompression failed for data that we've just compressed"); + } + PG_END_TRY(); + FUNCTION_NAME2(check_arrow, CTYPE)(arrow, PANIC, results, n); return n; diff --git a/tsl/test/src/decompress_text_test_impl.c b/tsl/test/src/decompress_text_test_impl.c index e4741af4d44..da9a158704b 100644 --- a/tsl/test/src/decompress_text_test_impl.c +++ b/tsl/test/src/decompress_text_test_impl.c @@ -220,7 +220,17 @@ decompress_generic_text(const uint8 *Data, size_t Size, bool bulk, int requested * 3) The bulk decompression must absolutely work on the correct compressed * data we've just generated. */ - arrow = decompress_all(compressed_data, TEXTOID, CurrentMemoryContext); + PG_TRY(); + { + arrow = decompress_all(compressed_data, TEXTOID, CurrentMemoryContext); + } + PG_CATCH(); + { + EmitErrorReport(); + elog(PANIC, "bulk decompression failed for data that we've just compressed"); + } + PG_END_TRY(); + decompress_generic_text_check_arrow(arrow, PANIC, results, n); return n;