Skip to content

Commit

Permalink
Refactor components in namepsaces
Browse files Browse the repository at this point in the history
  • Loading branch information
@cal-pratt
cal-pratt committed Jan 28, 2025
1 parent eaec127 commit 0406aa5
Show file tree
Hide file tree
Showing 112 changed files with 479 additions and 282 deletions.
3 changes: 3 additions & 0 deletions src/benchmarks/benchmarkBitReader.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@
#include <Statistics.hpp>


using namespace rapidgzip;


template<bool MOST_SIGNIFICANT_BITS_FIRST,
typename BitBuffer>
[[nodiscard]] std::pair<double, uint64_t>
Expand Down
4 changes: 4 additions & 0 deletions src/benchmarks/benchmarkBitStringFinder.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,10 @@

//#define BENCHMARK


using namespace rapidgzip;


namespace
{
constexpr uint64_t bitStringToFind = 0x314159265359; /* bcd(pi) */
Expand Down
3 changes: 3 additions & 0 deletions src/benchmarks/benchmarkCRC32.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@
#endif


using namespace rapidgzip;


constexpr size_t REPEAT_COUNT{ 10 };


Expand Down
3 changes: 3 additions & 0 deletions src/benchmarks/benchmarkCalculations.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@
#include <Statistics.hpp>


using namespace rapidgzip;


[[nodiscard]] std::string
formatBandwidth( const std::vector<double>& times,
size_t byteCount )
Expand Down
3 changes: 3 additions & 0 deletions src/benchmarks/benchmarkGzip.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@
#include <TestHelpers.hpp>


using namespace rapidgzip;


class GzipWrapper
{
public:
Expand Down
67 changes: 35 additions & 32 deletions src/benchmarks/benchmarkGzipBlockFinder.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,9 @@ GZIP file format specification version 4.3
#include <TestHelpers.hpp>


using namespace rapidgzip;


std::ostream&
operator<<( std::ostream& out, std::vector<size_t> vector )
{
Expand Down Expand Up @@ -431,7 +434,7 @@ findDeflateBlocksZlib( BufferedFileReader::AlignedBuffer buffer )
[[nodiscard]] std::vector<size_t>
findDeflateBlocksZlibOptimized( BufferedFileReader::AlignedBuffer buffer )
{
rapidgzip::BitReader bitReader( std::make_unique<BufferedFileReader>( buffer ) );
gzip::BitReader bitReader( std::make_unique<BufferedFileReader>( buffer ) );

/**
* Deflate Block:
Expand Down Expand Up @@ -572,7 +575,7 @@ findDeflateBlocksRapidgzip( BufferedFileReader::AlignedBuffer buffer )
using DeflateBlock = rapidgzip::deflate::Block<>;

const auto nBitsToTest = buffer.size() * CHAR_BIT;
rapidgzip::BitReader bitReader( std::make_unique<BufferedFileReader>( std::move( buffer ) ) );
gzip::BitReader bitReader( std::make_unique<BufferedFileReader>( std::move( buffer ) ) );

std::vector<size_t> bitOffsets;

Expand Down Expand Up @@ -621,7 +624,7 @@ findDeflateBlocksRapidgzip( BufferedFileReader::AlignedBuffer buffer )
* decoded data if we do decide that it is a valid block. The number of checks during reading is also
* pretty few because there almost are no wasted / invalid symbols. */
bitOffsets.push_back( offset );
} catch ( const rapidgzip::BitReader::EndOfFileReached& ) {
} catch ( const gzip::BitReader::EndOfFileReached& ) {
break;
}
}
Expand All @@ -634,7 +637,7 @@ template<uint8_t CACHED_BIT_COUNT>
countDeflateBlocksPreselection( BufferedFileReader::AlignedBuffer data )
{
const size_t nBitsToTest = data.size() * CHAR_BIT;
rapidgzip::BitReader bitReader( std::make_unique<BufferedFileReader>( std::move( data ) ) );
gzip::BitReader bitReader( std::make_unique<BufferedFileReader>( std::move( data ) ) );

uint64_t candidateCount{ 0 };

Expand All @@ -661,7 +664,7 @@ countDeflateBlocksPreselection( BufferedFileReader::AlignedBuffer data )

++candidateCount;
++offset;
} catch ( const rapidgzip::BitReader::EndOfFileReached& ) {
} catch ( const gzip::BitReader::EndOfFileReached& ) {
/* This might happen when calling readDynamicHuffmanCoding quite some bytes before the end! */
break;
}
Expand All @@ -676,7 +679,7 @@ template<uint8_t CACHED_BIT_COUNT>
countDeflateBlocksPreselectionManualSlidingBuffer( BufferedFileReader::AlignedBuffer data )
{
const size_t nBitsToTest = data.size() * CHAR_BIT;
rapidgzip::BitReader bitReader( std::make_unique<BufferedFileReader>( std::move( data ) ) );
gzip::BitReader bitReader( std::make_unique<BufferedFileReader>( std::move( data ) ) );

uint64_t candidateCount{ 0 };

Expand Down Expand Up @@ -706,7 +709,7 @@ countDeflateBlocksPreselectionManualSlidingBuffer( BufferedFileReader::AlignedBu
<< static_cast<uint8_t>( CACHED_BIT_COUNT - nextPosition );
offset += nextPosition;
}
} catch ( const rapidgzip::BitReader::EndOfFileReached& ) {
} catch ( const gzip::BitReader::EndOfFileReached& ) {
/* This might happen when calling readDynamicHuffmanCoding quite some bytes before the end! */
}

Expand Down Expand Up @@ -894,10 +897,10 @@ checkPrecode( const uint64_t next4Bits,
/* Without "forceinline", I observed a ~10% performance degradation! */
template<CheckPrecodeMethod CHECK_PRECODE_METHOD>
[[nodiscard]] forceinline rapidgzip::Error
checkDeflateBlock( const uint64_t bitBufferForLUT,
const uint64_t bitBufferPrecodeBits,
const size_t offset,
rapidgzip::BitReader& bitReader )
checkDeflateBlock( const uint64_t bitBufferForLUT,
const uint64_t bitBufferPrecodeBits,
const size_t offset,
gzip::BitReader& bitReader )
{
using namespace rapidgzip;
using namespace deflate;
Expand Down Expand Up @@ -1105,10 +1108,10 @@ checkAndGetValidHistogramID( const uint64_t precodeBits )

template<>
[[nodiscard]] forceinline rapidgzip::Error
checkDeflateBlock<CheckPrecodeMethod::SINGLE_COMPRESSED_LUT>( const uint64_t bitBufferForLUT,
const uint64_t bitBufferPrecodeBits,
const size_t offset,
rapidgzip::BitReader& bitReader )
checkDeflateBlock<CheckPrecodeMethod::SINGLE_COMPRESSED_LUT>( const uint64_t bitBufferForLUT,
const uint64_t bitBufferPrecodeBits,
const size_t offset,
gzip::BitReader& bitReader )
{
using namespace rapidgzip;
using namespace deflate;
Expand Down Expand Up @@ -1189,7 +1192,7 @@ template<uint8_t CACHED_BIT_COUNT,
findDeflateBlocksRapidgzipLUT( BufferedFileReader::AlignedBuffer data )
{
const size_t nBitsToTest = data.size() * CHAR_BIT;
rapidgzip::BitReader bitReader( std::make_unique<BufferedFileReader>( std::move( data ) ) );
gzip::BitReader bitReader( std::make_unique<BufferedFileReader>( std::move( data ) ) );

std::vector<size_t> bitOffsets;

Expand All @@ -1212,7 +1215,7 @@ findDeflateBlocksRapidgzipLUT( BufferedFileReader::AlignedBuffer data )
constexpr auto ALL_PRECODE_BITS = PRECODE_COUNT_BITS + MAX_PRECODE_COUNT * PRECODE_BITS;
static_assert( ( ALL_PRECODE_BITS == 61 ) && ( ALL_PRECODE_BITS >= CACHED_BIT_COUNT )
&& ( ALL_PRECODE_BITS <= std::numeric_limits<uint64_t>::digits )
&& ( ALL_PRECODE_BITS <= rapidgzip::BitReader::MAX_BIT_BUFFER_SIZE ),
&& ( ALL_PRECODE_BITS <= gzip::BitReader::MAX_BIT_BUFFER_SIZE ),
"It must fit into 64-bit and it also must fit the largest possible jump in the LUT." );
auto bitBufferPrecodeBits = bitReader.read<ALL_PRECODE_BITS>();

Expand Down Expand Up @@ -1260,7 +1263,7 @@ findDeflateBlocksRapidgzipLUT( BufferedFileReader::AlignedBuffer data )

offset += nextPosition;
}
} catch ( const rapidgzip::BitReader::EndOfFileReached& ) {
} catch ( const gzip::BitReader::EndOfFileReached& ) {
/* This might happen when calling readDynamicHuffmanCoding quite some bytes before the end! */
}

Expand All @@ -1272,7 +1275,7 @@ findDeflateBlocksRapidgzipLUT( BufferedFileReader::AlignedBuffer data )
countFilterEfficiencies( BufferedFileReader::AlignedBuffer data )
{
const size_t nBitsToTest = data.size() * CHAR_BIT;
rapidgzip::BitReader bitReader( std::make_unique<BufferedFileReader>( std::move( data ) ) );
gzip::BitReader bitReader( std::make_unique<BufferedFileReader>( std::move( data ) ) );

std::vector<size_t> bitOffsets;

Expand Down Expand Up @@ -1304,7 +1307,7 @@ countFilterEfficiencies( BufferedFileReader::AlignedBuffer data )
const auto next57Bits = bitReader.peek( rapidgzip::deflate::MAX_PRECODE_COUNT
* rapidgzip::deflate::PRECODE_BITS );
static_assert( rapidgzip::deflate::MAX_PRECODE_COUNT * rapidgzip::deflate::PRECODE_BITS
<= rapidgzip::BitReader::MAX_BIT_BUFFER_SIZE,
<= gzip::BitReader::MAX_BIT_BUFFER_SIZE,
"This optimization requires a larger BitBuffer inside BitReader!" );
using rapidgzip::PrecodeCheck::WalkTreeLUT::checkPrecode;
const auto precodeError = checkPrecode( next4Bits, next57Bits );
Expand All @@ -1331,7 +1334,7 @@ countFilterEfficiencies( BufferedFileReader::AlignedBuffer data )

bitOffsets.push_back( offset );
++offset;
} catch ( const rapidgzip::BitReader::EndOfFileReached& ) {
} catch ( const gzip::BitReader::EndOfFileReached& ) {
/* This might happen when calling readDynamicHuffmanCoding quite some bytes before the end! */
break;
}
Expand Down Expand Up @@ -1473,7 +1476,7 @@ findDeflateBlocksRapidgzipLUTTwoPass( BufferedFileReader::AlignedBuffer data )
"code length check, to avoid duplicate checks in the precode check!" );

const size_t nBitsToTest = data.size() * CHAR_BIT;
rapidgzip::BitReader bitReader( std::make_unique<BufferedFileReader>( std::move( data ) ) );
gzip::BitReader bitReader( std::make_unique<BufferedFileReader>( std::move( data ) ) );

std::vector<size_t> bitOffsetCandidates;

Expand All @@ -1492,7 +1495,7 @@ findDeflateBlocksRapidgzipLUTTwoPass( BufferedFileReader::AlignedBuffer data )
offset += nextPosition;
bitReader.seekAfterPeek( nextPosition );
}
} catch ( const rapidgzip::BitReader::EndOfFileReached& ) {
} catch ( const gzip::BitReader::EndOfFileReached& ) {
break;
}
}
Expand All @@ -1517,7 +1520,7 @@ findDeflateBlocksRapidgzipLUTTwoPass( BufferedFileReader::AlignedBuffer data )
const auto next57Bits = bitReader.peek( rapidgzip::deflate::MAX_PRECODE_COUNT
* rapidgzip::deflate::PRECODE_BITS );
static_assert( rapidgzip::deflate::MAX_PRECODE_COUNT * rapidgzip::deflate::PRECODE_BITS
<= rapidgzip::BitReader::MAX_BIT_BUFFER_SIZE,
<= gzip::BitReader::MAX_BIT_BUFFER_SIZE,
"This optimization requires a larger BitBuffer inside BitReader!" );

auto error = rapidgzip::Error::NONE;
Expand All @@ -1538,12 +1541,12 @@ findDeflateBlocksRapidgzipLUTTwoPass( BufferedFileReader::AlignedBuffer data )
if ( error != rapidgzip::Error::NONE ) {
return false;
}
} catch ( const rapidgzip::BitReader::EndOfFileReached& ) {}
} catch ( const gzip::BitReader::EndOfFileReached& ) {}

try {
bitReader.seek( static_cast<long long int>( offset ) + 3 );
return block.readDynamicHuffmanCoding( bitReader ) == rapidgzip::Error::NONE;
} catch ( const rapidgzip::BitReader::EndOfFileReached& ) {}
} catch ( const gzip::BitReader::EndOfFileReached& ) {}
return false;
};

Expand Down Expand Up @@ -1573,7 +1576,7 @@ findDeflateBlocksRapidgzipLUTTwoPassWithPrecode( BufferedFileReader::AlignedBuff
"code length check, to avoid duplicate checks in the precode check!" );

const size_t nBitsToTest = data.size() * CHAR_BIT;
rapidgzip::BitReader bitReader( std::make_unique<BufferedFileReader>( std::move( data ) ) );
gzip::BitReader bitReader( std::make_unique<BufferedFileReader>( std::move( data ) ) );

std::vector<size_t> bitOffsetCandidates;

Expand All @@ -1594,7 +1597,7 @@ findDeflateBlocksRapidgzipLUTTwoPassWithPrecode( BufferedFileReader::AlignedBuff
constexpr auto ALL_PRECODE_BITS = PRECODE_COUNT_BITS + MAX_PRECODE_COUNT * PRECODE_BITS;
static_assert( ( ALL_PRECODE_BITS == 61 ) && ( ALL_PRECODE_BITS >= CACHED_BIT_COUNT )
&& ( ALL_PRECODE_BITS <= std::numeric_limits<uint64_t>::digits )
&& ( ALL_PRECODE_BITS <= rapidgzip::BitReader::MAX_BIT_BUFFER_SIZE ),
&& ( ALL_PRECODE_BITS <= gzip::BitReader::MAX_BIT_BUFFER_SIZE ),
"It must fit into 64-bit and it also must fit the largest possible jump in the LUT." );
auto bitBufferPrecodeBits = bitReader.read<ALL_PRECODE_BITS>();

Expand Down Expand Up @@ -1650,7 +1653,7 @@ findDeflateBlocksRapidgzipLUTTwoPassWithPrecode( BufferedFileReader::AlignedBuff

offset += nextPosition;
}
} catch ( const rapidgzip::BitReader::EndOfFileReached& ) {
} catch ( const gzip::BitReader::EndOfFileReached& ) {
/* Might happen when testing close to the end. */
}

Expand All @@ -1674,7 +1677,7 @@ findDeflateBlocksRapidgzipLUTTwoPassWithPrecode( BufferedFileReader::AlignedBuff
const auto next57Bits = bitReader.peek( rapidgzip::deflate::MAX_PRECODE_COUNT
* rapidgzip::deflate::PRECODE_BITS );
static_assert( rapidgzip::deflate::MAX_PRECODE_COUNT * rapidgzip::deflate::PRECODE_BITS
<= rapidgzip::BitReader::MAX_BIT_BUFFER_SIZE,
<= gzip::BitReader::MAX_BIT_BUFFER_SIZE,
"This optimization requires a larger BitBuffer inside BitReader!" );

using rapidgzip::PrecodeCheck::WalkTreeLUT::checkPrecode;
Expand All @@ -1683,12 +1686,12 @@ findDeflateBlocksRapidgzipLUTTwoPassWithPrecode( BufferedFileReader::AlignedBuff
if ( error != rapidgzip::Error::NONE ) {
return false;
}
} catch ( const rapidgzip::BitReader::EndOfFileReached& ) {}
} catch ( const gzip::BitReader::EndOfFileReached& ) {}

try {
bitReader.seek( static_cast<long long int>( offset ) + 3 );
return block.readDynamicHuffmanCoding( bitReader ) == rapidgzip::Error::NONE;
} catch ( const rapidgzip::BitReader::EndOfFileReached& ) {}
} catch ( const gzip::BitReader::EndOfFileReached& ) {}
return false;
};

Expand Down
4 changes: 2 additions & 2 deletions src/benchmarks/benchmarkHuffmanCoding.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ template<typename HuffmanCoding>
benchmarkHuffmanCoding( const std::vector<typename HuffmanCoding::BitCount>& codeLengths,
const BufferedFileReader::AlignedBuffer& encoded )
{
rapidgzip::BitReader bitReader( std::make_unique<BufferedFileReader>( encoded ) );
gzip::BitReader bitReader( std::make_unique<BufferedFileReader>( encoded ) );

Result result;
result.encodedSizeInBits = bitReader.size().value();
Expand All @@ -67,7 +67,7 @@ benchmarkHuffmanCoding( const std::vector<typename HuffmanCoding::BitCount>& cod
break;
}
++count;
} catch ( const rapidgzip::BitReader::EndOfFileReached& ) {
} catch ( const gzip::BitReader::EndOfFileReached& ) {
break;
}
}
Expand Down
3 changes: 3 additions & 0 deletions src/benchmarks/benchmarkIORead.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@
#include <FileUtils.hpp>


using namespace rapidgzip;


/* Create a temporary file for benchmarking that is cleaned up with RAII. */
struct TemporaryFile
{
Expand Down
3 changes: 3 additions & 0 deletions src/benchmarks/benchmarkIOWrite.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@
#include <ThreadPool.hpp>


using namespace rapidgzip;


/* Alignment to the filesystem block size is necessary for direct I/O. */
using DataBuffer = std::vector<char, AlignedAllocator<char, 4096> >;

Expand Down
5 changes: 4 additions & 1 deletion src/benchmarks/benchmarkIndexCompression.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@
#include <ParallelGzipReader.hpp>


using namespace rapidgzip;


template<typename ResultContainer = std::vector<uint8_t> >
[[nodiscard]] ResultContainer
compress( const VectorView<uint8_t> toCompress )
Expand Down Expand Up @@ -64,7 +67,7 @@ main( int argc,

std::array<uint8_t, 64_Ki> windowPatches;

rapidgzip::BitReader bitReader( file->clone() );
gzip::BitReader bitReader( file->clone() );
WindowMap windows;
for ( const auto& checkpoint : index.checkpoints ) {
windowCount++;
Expand Down
5 changes: 4 additions & 1 deletion src/benchmarks/benchmarkInflate.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@
#include <TestHelpers.hpp>


using namespace rapidgzip;


[[nodiscard]] size_t
decompressWithRapidgzip( UniqueFileReader fileReader )
{
Expand Down Expand Up @@ -125,7 +128,7 @@ compressWithZlib( const std::vector<std::byte>& toCompress,
output.resize( output.size() + CHUNK_SIZE );
stream.next_out = reinterpret_cast<Bytef*>( output.data() + output.size() - CHUNK_SIZE );
stream.avail_out = CHUNK_SIZE;
status = deflate( &stream, Z_FINISH );
status = ::deflate( &stream, Z_FINISH );
}

deflateEnd( &stream );
Expand Down
3 changes: 3 additions & 0 deletions src/benchmarks/benchmarkMarkerReplacement.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@
#include <common.hpp>


using namespace rapidgzip;


void
replaceInPlace( std::vector<std::uint16_t>& buffer,
const std::vector<std::uint8_t>& window )
Expand Down
3 changes: 3 additions & 0 deletions src/benchmarks/benchmarkPigzBlockFinder.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@
#include <TestHelpers.hpp>


using namespace rapidgzip;


struct BenchmarkResults
{
double duration{ 0 };
Expand Down
Loading

0 comments on commit 0406aa5

Please sign in to comment.