diff --git a/c/common/shared_dictionary.c b/c/common/shared_dictionary.c index 3ca40c069..49f1c9b07 100644 --- a/c/common/shared_dictionary.c +++ b/c/common/shared_dictionary.c @@ -20,6 +20,8 @@ extern "C" { #endif +#if defined(BROTLI_EXPERIMENTAL) + #define BROTLI_NUM_ENCODED_LENGTHS (SHARED_BROTLI_MAX_DICTIONARY_WORD_LENGTH \ - SHARED_BROTLI_MIN_DICTIONARY_WORD_LENGTH + 1) @@ -442,6 +444,8 @@ static BROTLI_BOOL DecodeSharedDictionary( return ParseDictionary(encoded, size, dict); } +#endif /* BROTLI_EXPERIMENTAL */ + void BrotliSharedDictionaryDestroyInstance( BrotliSharedDictionary* dict) { if (!dict) { @@ -464,9 +468,12 @@ BROTLI_BOOL BrotliSharedDictionaryAttach( if (!dict) { return BROTLI_FALSE; } +#if defined(BROTLI_EXPERIMENTAL) if (type == BROTLI_SHARED_DICTIONARY_SERIALIZED) { return DecodeSharedDictionary(data, data_size, dict); - } else if (type == BROTLI_SHARED_DICTIONARY_RAW) { + } +#endif /* BROTLI_EXPERIMENTAL */ + if (type == BROTLI_SHARED_DICTIONARY_RAW) { if (dict->num_prefix >= SHARED_BROTLI_MAX_COMPOUND_DICTS) { return BROTLI_FALSE; } @@ -474,9 +481,8 @@ BROTLI_BOOL BrotliSharedDictionaryAttach( dict->prefix[dict->num_prefix] = data; dict->num_prefix++; return BROTLI_TRUE; - } else { - return BROTLI_FALSE; } + return BROTLI_FALSE; } BrotliSharedDictionary* BrotliSharedDictionaryCreateInstance( diff --git a/c/enc/encode.c b/c/enc/encode.c index 0303094af..81ea40e31 100644 --- a/c/enc/encode.c +++ b/c/enc/encode.c @@ -7,6 +7,8 @@ /* Implementation of Brotli compressor. */ #include +#include +#include #include /* free, malloc */ #include /* memcpy, memset */ @@ -1707,8 +1709,12 @@ BrotliEncoderPreparedDictionary* BrotliEncoderPrepareDictionary( const uint8_t data[BROTLI_ARRAY_PARAM(size)], int quality, brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque) { ManagedDictionary* managed_dictionary = NULL; - if (type != BROTLI_SHARED_DICTIONARY_RAW && - type != BROTLI_SHARED_DICTIONARY_SERIALIZED) { + BROTLI_BOOL type_is_known = BROTLI_FALSE; + type_is_known |= (type == BROTLI_SHARED_DICTIONARY_RAW); +#if defined(BROTLI_EXPERIMENTAL) + type_is_known |= (type == BROTLI_SHARED_DICTIONARY_SERIALIZED); +#endif /* BROTLI_EXPERIMENTAL */ + if (!type_is_known) { return NULL; } managed_dictionary = @@ -1719,7 +1725,9 @@ BrotliEncoderPreparedDictionary* BrotliEncoderPrepareDictionary( if (type == BROTLI_SHARED_DICTIONARY_RAW) { managed_dictionary->dictionary = (uint32_t*)CreatePreparedDictionary( &managed_dictionary->memory_manager_, data, size); - } else { + } +#if defined(BROTLI_EXPERIMENTAL) + if (type == BROTLI_SHARED_DICTIONARY_SERIALIZED) { SharedEncoderDictionary* dict = (SharedEncoderDictionary*)BrotliAllocate( &managed_dictionary->memory_manager_, sizeof(SharedEncoderDictionary)); managed_dictionary->dictionary = (uint32_t*)dict; @@ -1732,6 +1740,9 @@ BrotliEncoderPreparedDictionary* BrotliEncoderPrepareDictionary( } } } +#else /* BROTLI_EXPERIMENTAL */ + (void)quality; +#endif /* BROTLI_EXPERIMENTAL */ if (managed_dictionary->dictionary == NULL) { BrotliDestroyManagedDictionary(managed_dictionary); return NULL; diff --git a/c/enc/encoder_dict.c b/c/enc/encoder_dict.c index 0c93e009a..6602c55a2 100644 --- a/c/enc/encoder_dict.c +++ b/c/enc/encoder_dict.c @@ -77,6 +77,7 @@ static void BrotliDestroyEncoderDictionary(MemoryManager* m, BrotliTrieFree(m, &dict->trie); } +#if defined(BROTLI_EXPERIMENTAL) /* Word length must be at least 4 bytes */ static uint32_t Hash(const uint8_t* data, int bits) { uint32_t h = BROTLI_UNALIGNED_LOAD32LE(data) * kHashMul32; @@ -481,6 +482,7 @@ static BROTLI_BOOL ComputeDictionary(MemoryManager* m, int quality, return BROTLI_TRUE; } +#endif /* BROTLI_EXPERIMENTAL */ void BrotliInitSharedEncoderDictionary(SharedEncoderDictionary* dict) { dict->magic = kSharedDictionaryMagic; @@ -501,6 +503,7 @@ void BrotliInitSharedEncoderDictionary(SharedEncoderDictionary* dict) { dict->max_quality = BROTLI_MAX_QUALITY; } +#if defined(BROTLI_EXPERIMENTAL) /* TODO(eustas): make sure that tooling will warn user if not all the cutoff transforms are available (for low-quality encoder). */ static BROTLI_BOOL InitCustomSharedEncoderDictionary( @@ -586,6 +589,7 @@ BROTLI_BOOL BrotliInitCustomSharedEncoderDictionary( BrotliSharedDictionaryDestroyInstance(decoded_dict); return success; } +#endif /* BROTLI_EXPERIMENTAL */ void BrotliCleanupSharedEncoderDictionary(MemoryManager* m, SharedEncoderDictionary* dict) { diff --git a/c/enc/encoder_dict.h b/c/enc/encoder_dict.h index b291f98b4..27dcbcd2f 100644 --- a/c/enc/encoder_dict.h +++ b/c/enc/encoder_dict.h @@ -52,8 +52,11 @@ typedef struct BrotliTrie { BrotliTrieNode root; } BrotliTrie; +#if defined(BROTLI_EXPERIMENTAL) BROTLI_INTERNAL const BrotliTrieNode* BrotliTrieSub(const BrotliTrie* trie, const BrotliTrieNode* node, uint8_t c); +#endif /* BROTLI_EXPERIMENTAL */ + /* Dictionary data (words and transforms) for 1 possible context */ typedef struct BrotliEncoderDictionary { const BrotliDictionary* words; @@ -129,12 +132,14 @@ typedef struct ManagedDictionary { BROTLI_INTERNAL void BrotliInitSharedEncoderDictionary( SharedEncoderDictionary* dict); +#if defined(BROTLI_EXPERIMENTAL) /* Initializes to shared dictionary that will be parsed from encoded_dict. Requires that you keep the encoded_dict buffer around, parts of data will point to it. */ BROTLI_INTERNAL BROTLI_BOOL BrotliInitCustomSharedEncoderDictionary( MemoryManager* m, const uint8_t* encoded_dict, size_t size, int quality, SharedEncoderDictionary* dict); +#endif /* BROTLI_EXPERIMENTAL */ BROTLI_INTERNAL void BrotliCleanupSharedEncoderDictionary( MemoryManager* m, SharedEncoderDictionary* dict); diff --git a/c/enc/memory.c b/c/enc/memory.c index 51e1b7f18..8811821f8 100644 --- a/c/enc/memory.c +++ b/c/enc/memory.c @@ -20,9 +20,11 @@ extern "C" { #endif -#define MAX_PERM_ALLOCATED 128 +/* TODO(eustas): fine-tune for "many slots" case */ #define MAX_NEW_ALLOCATED 64 #define MAX_NEW_FREED 64 +#define MAX_PERM_ALLOCATED \ + (BROTLI_ENCODER_MEMORY_MANAGER_SLOTS - MAX_NEW_ALLOCATED - MAX_NEW_FREED) #define PERM_ALLOCATED_OFFSET 0 #define NEW_ALLOCATED_OFFSET MAX_PERM_ALLOCATED @@ -68,6 +70,7 @@ void BrotliWipeOutMemoryManager(MemoryManager* m) { static void SortPointers(void** items, const size_t n) { /* Shell sort. */ + /* TODO(eustas): fine-tune for "many slots" case */ static const size_t gaps[] = {23, 10, 4, 1}; int g = 0; for (; g < 4; ++g) { diff --git a/c/enc/memory.h b/c/enc/memory.h index cbe4e309a..9dbed55de 100644 --- a/c/enc/memory.h +++ b/c/enc/memory.h @@ -24,6 +24,14 @@ extern "C" { #define BROTLI_ENCODER_EXIT_ON_OOM #endif +#if !defined(BROTLI_ENCODER_EXIT_ON_OOM) +#if defined(BROTLI_EXPERIMENTAL) +#define BROTLI_ENCODER_MEMORY_MANAGER_SLOTS 6144 +#else /* BROTLI_EXPERIMENTAL */ +#define BROTLI_ENCODER_MEMORY_MANAGER_SLOTS 256 +#endif /* BROTLI_EXPERIMENTAL */ +#endif /* BROTLI_ENCODER_EXIT_ON_OOM */ + typedef struct MemoryManager { brotli_alloc_func alloc_func; brotli_free_func free_func; @@ -33,7 +41,7 @@ typedef struct MemoryManager { size_t perm_allocated; size_t new_allocated; size_t new_freed; - void* pointers[256]; + void* pointers[BROTLI_ENCODER_MEMORY_MANAGER_SLOTS]; #endif /* BROTLI_ENCODER_EXIT_ON_OOM */ } MemoryManager; diff --git a/c/enc/static_dict.c b/c/enc/static_dict.c index 9e6f27043..291d28335 100644 --- a/c/enc/static_dict.c +++ b/c/enc/static_dict.c @@ -79,6 +79,7 @@ static BROTLI_BOOL BrotliFindAllStaticDictionaryMatchesFor( const BrotliEncoderDictionary* dictionary, const uint8_t* data, size_t min_length, size_t max_length, uint32_t* matches) { BROTLI_BOOL has_found_match = BROTLI_FALSE; +#if defined(BROTLI_EXPERIMENTAL) if (dictionary->has_words_heavy) { const BrotliTrieNode* node = &dictionary->trie.root; size_t l = 0; @@ -93,6 +94,7 @@ static BROTLI_BOOL BrotliFindAllStaticDictionaryMatchesFor( } return has_found_match; } +#endif /* BROTLI_EXPERIMENTAL */ { size_t offset = dictionary->buckets[Hash(data)]; BROTLI_BOOL end = !offset; diff --git a/c/include/brotli/shared_dictionary.h b/c/include/brotli/shared_dictionary.h index ceb6cf1cd..2970c2dcc 100644 --- a/c/include/brotli/shared_dictionary.h +++ b/c/include/brotli/shared_dictionary.h @@ -35,7 +35,10 @@ typedef struct BrotliSharedDictionaryStruct BrotliSharedDictionary; typedef enum BrotliSharedDictionaryType { /** Raw LZ77 prefix dictionary. */ BROTLI_SHARED_DICTIONARY_RAW = 0, - /** Serialized shared dictionary. */ + /** Serialized shared dictionary. + * + * DO NOT USE: methods accepting this value will fail. + */ BROTLI_SHARED_DICTIONARY_SERIALIZED = 1 } BrotliSharedDictionaryType;