Skip to content

Commit

Permalink
BLOSC_WARN envvar introduced. Return 0 when input size is exceeded.
Browse files Browse the repository at this point in the history
  • Loading branch information
@FrancescAlted
FrancescAlted committed May 10, 2020
1 parent 4f16db5 commit 403d50c
Show file tree
Hide file tree
Showing 3 changed files with 54 additions and 17 deletions.
28 changes: 20 additions & 8 deletions blosc/blosc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1070,6 +1070,8 @@ static int initialize_context_compression(struct blosc_context* context,
int32_t blocksize,
int32_t numthreads)
{
char *envvar;
int warnlvl;
/* Set parameters */
context->compress = 1;
context->src = (const uint8_t*)src;
Expand All @@ -1083,19 +1085,29 @@ static int initialize_context_compression(struct blosc_context* context,
context->end_threads = 0;
context->clevel = clevel;

envvar = getenv("BLOSC_WARN");

This comment has been minimized.

Sign in to view

Copy link
@kalvdans

kalvdans Aug 22, 2024

Contributor

getenv is not threadsafe, see #384
if (envvar != NULL) {
warnlvl = strtol(envvar, NULL, 10);
}

/* Check buffer size limits */
if (sourcesize > BLOSC_MAX_BUFFERSIZE) {
fprintf(stderr, "Input buffer size cannot exceed %d bytes\n",
BLOSC_MAX_BUFFERSIZE);
return -1;
if (warnlvl > 0) {
fprintf(stderr, "Input buffer size cannot exceed %d bytes\n",
BLOSC_MAX_BUFFERSIZE);
}
return 0;
}
if (destsize < BLOSC_MAX_OVERHEAD) {
if (warnlvl > 0) {
fprintf(stderr, "Output buffer size should be larger than %d bytes\n",
BLOSC_MAX_OVERHEAD);
}
return 0;
}

/* Compression level */
if (clevel < 0 || clevel > 9) {
/* If clevel not in 0..9, print an error */
fprintf(stderr, "`clevel` parameter must be between 0 and 9!\n");
return -10;
}
Expand Down Expand Up @@ -1271,10 +1283,10 @@ int blosc_compress_ctx(int clevel, int doshuffle, size_t typesize,
nbytes, src, dest, destsize,
blosc_compname_to_compcode(compressor),
blocksize, numinternalthreads);
if (error < 0) { return error; }
if (error <= 0) { return error; }

error = write_compression_header(&context, clevel, doshuffle);
if (error < 0) { return error; }
if (error <= 0) { return error; }

result = blosc_compress_context(&context);

Expand Down Expand Up @@ -1393,10 +1405,10 @@ int blosc_compress(int clevel, int doshuffle, size_t typesize, size_t nbytes,
typesize, nbytes, src, dest, destsize,
g_compressor, g_force_blocksize,
g_threads);
if (result < 0) { break; }
if (result <= 0) { break; }

result = write_compression_header(g_global_context, clevel, doshuffle);
if (result < 0) { break; }
if (result <= 0) { break; }

result = blosc_compress_context(g_global_context);
} while (0);
Expand Down
4 changes: 4 additions & 0 deletions blosc/blosc.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,10 @@ BLOSC_EXPORT void blosc_destroy(void);
This will call blosc_set_splitmode() with the different supported values.
See blosc_set_splitmode() docstrings for more info on each mode.
BLOSC_WARN=(INTEGER): This will print some warning message on stderr
showing more info in situations where data inputs cannot be compressed.
The values can range from 1 (less verbose) to 10 (full verbose). 0 is
the same as if the BLOSC_WARN envvar was not defined.
*/
BLOSC_EXPORT int blosc_compress(int clevel, int doshuffle, size_t typesize,
size_t nbytes, const void *src, void *dest,
Expand Down
39 changes: 30 additions & 9 deletions tests/test_maxout.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,24 @@ size_t typesize = 4;
size_t size = 1000; /* must be divisible by 4 */


/* Check input size > BLOSC_MAX_BUFFERSIZE */
static const char *test_input_too_large(void) {

/* Get a compressed buffer */
cbytes = blosc_compress(clevel, doshuffle, typesize, BLOSC_MAX_BUFFERSIZE + 1, src,
dest, size + BLOSC_MAX_OVERHEAD - 1);
mu_assert("ERROR: cbytes is not 0", cbytes == 0);

return 0;
}


/* Check maxout with maxout < size */
static const char *test_maxout_less(void) {

/* Get a compressed buffer */
cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest, size + BLOSC_MAX_OVERHEAD - 1);
cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest,
size + BLOSC_MAX_OVERHEAD - 1);
mu_assert("ERROR: cbytes is not 0", cbytes == 0);

return 0;
Expand All @@ -37,7 +50,8 @@ static const char *test_maxout_less(void) {
static const char *test_maxout_less_memcpy(void) {

/* Get a compressed buffer */
cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest, size + BLOSC_MAX_OVERHEAD - 1);
cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest,
size + BLOSC_MAX_OVERHEAD - 1);
mu_assert("ERROR: cbytes is not 0", cbytes == 0);

return 0;
Expand All @@ -48,7 +62,8 @@ static const char *test_maxout_less_memcpy(void) {
static const char *test_maxout_equal(void) {

/* Get a compressed buffer */
cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest, size + BLOSC_MAX_OVERHEAD);
cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest,
size + BLOSC_MAX_OVERHEAD);
mu_assert("ERROR: cbytes is not correct", cbytes == size + BLOSC_MAX_OVERHEAD);

/* Decompress the buffer */
Expand All @@ -63,7 +78,8 @@ static const char *test_maxout_equal(void) {
static const char *test_maxout_equal_memcpy(void) {

/* Get a compressed buffer */
cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest, size + BLOSC_MAX_OVERHEAD);
cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest,
size + BLOSC_MAX_OVERHEAD);
mu_assert("ERROR: cbytes is not correct", cbytes == size + BLOSC_MAX_OVERHEAD);

/* Decompress the buffer */
Expand All @@ -77,7 +93,8 @@ static const char *test_maxout_equal_memcpy(void) {
/* Check maxout with maxout > size */
static const char *test_maxout_great(void) {
/* Get a compressed buffer */
cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest, size + BLOSC_MAX_OVERHEAD + 1);
cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest,
size + BLOSC_MAX_OVERHEAD + 1);
mu_assert("ERROR: cbytes is not correct", cbytes == size + BLOSC_MAX_OVERHEAD);

/* Decompress the buffer */
Expand All @@ -91,7 +108,8 @@ static const char *test_maxout_great(void) {
/* Check maxout with maxout > size (memcpy version) */
static const char *test_maxout_great_memcpy(void) {
/* Get a compressed buffer */
cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest, size + BLOSC_MAX_OVERHEAD + 1);
cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest,
size + BLOSC_MAX_OVERHEAD + 1);
mu_assert("ERROR: cbytes is not correct", cbytes == size + BLOSC_MAX_OVERHEAD);

/* Decompress the buffer */
Expand All @@ -104,12 +122,14 @@ static const char *test_maxout_great_memcpy(void) {
/* Check maxout with maxout < BLOSC_MAX_OVERHEAD */
static const char *test_max_overhead(void) {
blosc_init();
cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest, BLOSC_MAX_OVERHEAD - 1);
cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest,
BLOSC_MAX_OVERHEAD - 1);
mu_assert("ERROR: cbytes is not correct", cbytes == 0);
blosc_destroy();

blosc_init();
cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest, BLOSC_MAX_OVERHEAD - 2);
cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest,
BLOSC_MAX_OVERHEAD - 2);
mu_assert("ERROR: cbytes is not correct", cbytes == 0);
blosc_destroy();

Expand All @@ -123,6 +143,7 @@ static const char *test_max_overhead(void) {


static const char *all_tests(void) {
mu_run_test(test_input_too_large);
mu_run_test(test_maxout_less);
mu_run_test(test_maxout_less_memcpy);
mu_run_test(test_maxout_equal);
Expand Down Expand Up @@ -175,4 +196,4 @@ int main(int argc, char **argv) {
blosc_destroy();

return result != 0;
}
}

1 comment on commit 403d50c

@kalvdans
Copy link
Contributor

@kalvdans kalvdans commented on 403d50c May 10, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please add a note about BLOSC_MAX_BUFFERSIZE in the documentation for blosc_compress too.

Please sign in to comment.