Skip to content

Commit

Permalink
feat(tianmu): Improve the readbility of stonedb. (#11)
Browse files Browse the repository at this point in the history 
[summary]
1 class member variable refactor：compress directory;
2 class member variable refactor：types directory;
  • Loading branch information
@mergify
lujiashun authored and mergify[bot] committed Oct 28, 2022
1 parent 32a763c commit a7da00e
Show file tree
Hide file tree
Showing 57 changed files with 1,886 additions and 1,758 deletions.
125 changes: 63 additions & 62 deletions storage/tianmu/compress/arith_coder.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,38 +23,39 @@ namespace Tianmu {
namespace compress {

void ArithCoder::InitCompress() {
low = 0;
high = 0xffff;
underflow_bits = 0;
low_ = 0;
high_ = 0xffff;
underflow_bits_ = 0;
}

// if possible, make normalization and send bits to the 'dest'
CprsErr ArithCoder::ScaleRange(BitStream *dest, BaseT s_low, BaseT s_high, BaseT total) {
range = (WideT)(high - low) + 1;
high = low + (BaseT)((range * s_high) / total - 1);
low = low + (BaseT)((range * s_low) / total);
if (high < low) return CprsErr::CPRS_ERR_SUM;
range_ = (WideT)(high_ - low_) + 1;
high_ = low_ + (BaseT)((range_ * s_high) / total - 1);
low_ = low_ + (BaseT)((range_ * s_low) / total);
if (high_ < low_)
return CprsErr::CPRS_ERR_SUM;

for (;;) {
// the same MS bits
if ((high & 0x8000) == (low & 0x8000)) {
dest->PutBit(high >> 15);
while (underflow_bits > 0) {
dest->PutBit((~high & 0x8000) >> 15);
underflow_bits--;
if ((high_ & 0x8000) == (low_ & 0x8000)) {
dest->PutBit(high_ >> 15);
while (underflow_bits_ > 0) {
dest->PutBit((~high_ & 0x8000) >> 15);
underflow_bits_--;
}
}
// low=01... high=10...
else if ((low & 0x4000) && !(high & 0x4000)) {
underflow_bits++;
low &= 0x3fff;
high |= 0x4000;
// low_=01... high_=10...
else if ((low_ & 0x4000) && !(high_ & 0x4000)) {
underflow_bits_++;
low_ &= 0x3fff;
high_ |= 0x4000;
} else
return CprsErr::CPRS_SUCCESS;

low <<= 1;
high <<= 1;
high |= 1;
low_ <<= 1;
high_ <<= 1;
high_ |= 1;
}
}

Expand All @@ -67,17 +68,17 @@ CprsErr ArithCoder::EncodeUniform(BitStream *dest, T val, T maxval, uint bitmax)
// encode groups of 'uni_nbit' bits, from the least significant
BaseT v;
CprsErr err;
DEBUG_ASSERT(uni_total <= MAX_TOTAL);
while (bitmax > uni_nbit) {
v = (BaseT)(val & uni_mask);
err = ScaleRange(dest, v, v + (BaseT)1, uni_total);
DEBUG_ASSERT(uni_total_ <= MAX_TOTAL_);
while (bitmax > uni_nbit_) {
v = (BaseT)(val & uni_mask_);
err = ScaleRange(dest, v, v + (BaseT)1, uni_total_);
if (static_cast<int>(err)) return err;
val >>= uni_nbit;
maxval >>= uni_nbit;
bitmax -= uni_nbit;
val >>= uni_nbit_;
maxval >>= uni_nbit_;
bitmax -= uni_nbit_;
}
// encode the most significant group
DEBUG_ASSERT(maxval < MAX_TOTAL);
DEBUG_ASSERT(maxval < MAX_TOTAL_);
err = ScaleRange(dest, (BaseT)val, (BaseT)val + (BaseT)1, (BaseT)maxval + (BaseT)1);
if (static_cast<int>(err)) return err;

Expand All @@ -86,9 +87,9 @@ CprsErr ArithCoder::EncodeUniform(BitStream *dest, T val, T maxval, uint bitmax)

// TODO: it was
void ArithCoder::EndCompress(BitStream *dest) {
dest->PutBit((low & 0x4000) > 0);
underflow_bits++;
while (underflow_bits-- > 0) dest->PutBit(((~low) & 0x4000) > 0);
dest->PutBit((low_ & 0x4000) > 0);
underflow_bits_++;
while (underflow_bits_-- > 0) dest->PutBit(((~low_) & 0x4000) > 0);
}

CprsErr ArithCoder::CompressBytes(BitStream *dest, char *src, int slen, BaseT *sum, BaseT total) {
Expand All @@ -102,7 +103,7 @@ CprsErr ArithCoder::CompressBytes(BitStream *dest, char *src, int slen, BaseT *s
for (; slen > 0; slen--) {
c = *(src++);
err = ScaleRange(dest, sum[c], sum[c + 1],
total); // rescale high and low, send bits to dest
total); // rescale high_ and low_, send bits to dest
if (static_cast<int>(static_cast<int>(err))) return err;
}

Expand All @@ -121,7 +122,7 @@ CprsErr ArithCoder::CompressBits(BitStream *dest, BitStream *src, BaseT *sum, Ba
while (src->CanRead()) {
c = src->GetBit();
err = ScaleRange(dest, sum[c], sum[c + 1],
total); // rescale high and low, send bits to dest
total); // rescale high_ and low_, send bits to dest
if (static_cast<int>(static_cast<int>(err))) return err;
}

Expand All @@ -130,44 +131,44 @@ CprsErr ArithCoder::CompressBits(BitStream *dest, BitStream *src, BaseT *sum, Ba
}

void ArithCoder::InitDecompress(BitStream *src) {
low = 0;
high = 0xffff;
code = 0;
added = 0;
low_ = 0;
high_ = 0xffff;
code_ = 0;
added_ = 0;

for (int i = 0; i < 16; i++) {
code <<= 1;
code_ <<= 1;
if (src->CanRead())
code |= src->GetBit();
code_ |= src->GetBit();
else
added++;
added_++;
}
}

// remove the symbol from the input
CprsErr ArithCoder::RemoveSymbol(BitStream *src, BaseT s_low, BaseT s_high, BaseT total) {
high = low + (BaseT)((range * s_high) / total - 1); // TODO: optimize for decompression of bits
low = low + (BaseT)((range * s_low) / total);
high_ = low_ + (BaseT)((range_ * s_high) / total - 1); // TODO: optimize for decompression of bits
low_ = low_ + (BaseT)((range_ * s_low) / total);
for (;;) {
// the same MS bits
if ((high & 0x8000) == (low & 0x8000)) {
if ((high_ & 0x8000) == (low_ & 0x8000)) {
}
// low=01... high=10...
else if ((low & 0x4000) && !(high & 0x4000)) {
code ^= 0x4000;
low &= 0x3fff;
high |= 0x4000;
// low_=01... high_=10...
else if ((low_ & 0x4000) && !(high_ & 0x4000)) {
code_ ^= 0x4000;
low_ &= 0x3fff;
high_ |= 0x4000;
} else
return CprsErr::CPRS_SUCCESS;

low <<= 1;
high <<= 1;
high |= 1;
low_ <<= 1;
high_ <<= 1;
high_ |= 1;

code <<= 1;
code_ <<= 1;
if (src->CanRead())
code |= src->GetBit();
else if (++added > sizeof(BaseT) * 8)
code_ |= src->GetBit();
else if (++added_ > sizeof(BaseT) * 8)
return CprsErr::CPRS_ERR_BUF;
}
}
Expand All @@ -181,20 +182,20 @@ CprsErr ArithCoder::DecodeUniform(BitStream *src, T &val, T maxval, uint bitmax)
// decode groups of 'uni_nbit' bits, from the least significant
BaseT v;
CprsErr err;
DEBUG_ASSERT(uni_total <= MAX_TOTAL);
DEBUG_ASSERT(uni_total_ <= MAX_TOTAL_);
uint shift = 0;
while (shift + uni_nbit < bitmax) {
v = GetCount(uni_total);
err = RemoveSymbol(src, v, v + (BaseT)1, uni_total);
while (shift + uni_nbit_ < bitmax) {
v = GetCount(uni_total_);
err = RemoveSymbol(src, v, v + (BaseT)1, uni_total_);
if (static_cast<int>(err)) return err;
DEBUG_ASSERT(shift < 64);
val |= (uint64_t)v << shift;
shift += uni_nbit;
shift += uni_nbit_;
}

// decode the most significant group
BaseT total = (BaseT)(maxval _SHR_ shift) + (BaseT)1;
DEBUG_ASSERT(total <= MAX_TOTAL);
DEBUG_ASSERT(total <= MAX_TOTAL_);
v = GetCount(total);
err = RemoveSymbol(src, v, v + (BaseT)1, total);
if (static_cast<int>(err)) return err;
Expand Down Expand Up @@ -260,8 +261,8 @@ CprsErr ArithCoder::DecompressBits(BitStream *dest, BitStream *src, BaseT *sum,
}

ArithCoder::BaseT ArithCoder::GetCount(BaseT total) {
range = (WideT)(high - low) + 1;
return (BaseT)((((WideT)(code - low) + 1) * total - 1) / range);
range_ = (WideT)(high_ - low_) + 1;
return (BaseT)((((WideT)(code_ - low_) + 1) * total - 1) / range_);
}

template <class T>
Expand Down
30 changes: 15 additions & 15 deletions storage/tianmu/compress/arith_coder.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,12 @@ class ArithCoder {
public:
using BaseT = unsigned short;
using WideT = unsigned long;
static const BaseT MAX_TOTAL = USHRT_MAX / 4;
static const BaseT MAX_TOTAL_ = USHRT_MAX / 4;

// Saves encoded data to 'dest'.
// sum - array of cumulated counts, starting with value 0.
// Its size = highest_index_of_character_to_encode + 2 (1 for total)
// (contains 'low' for every character plus total),
// (contains 'low_' for every character plus total),
// e.g. for characters 0 and 1 it is: { low0, low1, total }
// total - total sum of counts
CprsErr CompressBytes(BitStream *dest, char *src, int slen, BaseT *sum, BaseT total);
Expand All @@ -60,24 +60,24 @@ class ArithCoder {
// num0, uint num1);

private:
BaseT low;
BaseT high;
BaseT code;
WideT range;
int underflow_bits;
unsigned int added; // number of '0' bits virtually "added" to the source
// during decompr. (added > 16, means error)
BaseT low_;
BaseT high_;
BaseT code_;
WideT range_;
int underflow_bits_;
unsigned int added_; // number of '0' bits virtually "added_" to the source
// during decompr. (added_ > 16, means error)

// constants for uniform encoding:
static const uint uni_nbit = 13;
static const BaseT uni_mask = (1 << uni_nbit) - 1;
static const BaseT uni_total = 1 << uni_nbit;
static const uint uni_nbit_ = 13;
static const BaseT uni_mask_ = (1 << uni_nbit_) - 1;
static const BaseT uni_total_ = 1 << uni_nbit_;

public:
ArithCoder() {
low = high = code = 0;
range = 0;
underflow_bits = added = 0;
low_ = high_ = code_ = 0;
range_ = 0;
underflow_bits_ = added_ = 0;
}
~ArithCoder() {}
// compression methods
Expand Down
Loading

0 comments on commit a7da00e

Please sign in to comment.