Skip to content

Commit

Permalink
tighten up progressive read code a little bit
Browse files Browse the repository at this point in the history
  • Loading branch information
Samuel Li committed Sep 15, 2023
1 parent 99e5d89 commit c71a156
Show file tree
Hide file tree
Showing 4 changed files with 56 additions and 81 deletions.
4 changes: 4 additions & 0 deletions include/SPERR3D_Stream_Tools.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,10 @@ class SPERR3D_Stream_Tools {
private:
const size_t m_header_magic_nchunks = 20;
const size_t m_header_magic_1chunk = 14;

// To simplify logic with progressive read, we set a minimum number of bytes to read from
// a chunk, unless the chunk doesn't have that many bytes (e.g., a constant chunk).
const size_t m_progressive_min_chunk_bytes = 64;
};

} // End of namespace sperr
Expand Down
14 changes: 7 additions & 7 deletions src/SPERR3D_OMP_C.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,17 +193,17 @@ auto sperr::SPERR3D_OMP_C::m_generate_header() const -> sperr::vec8_type
header[pos++] = sperr::pack_8_booleans(b8);

// Volume dimensions
const uint32_t vdim[3] = {static_cast<uint32_t>(m_dims[0]), static_cast<uint32_t>(m_dims[1]),
static_cast<uint32_t>(m_dims[2])};
std::memcpy(&header[pos], vdim, sizeof(vdim));
const auto vdim = std::array{static_cast<uint32_t>(m_dims[0]), static_cast<uint32_t>(m_dims[1]),
static_cast<uint32_t>(m_dims[2])};
std::memcpy(&header[pos], vdim.data(), sizeof(vdim));
pos += sizeof(vdim);

// Chunk dimensions, if there are more than one chunk.
if (num_chunks > 1) {
const uint16_t vcdim[3] = {static_cast<uint16_t>(m_chunk_dims[0]),
static_cast<uint16_t>(m_chunk_dims[1]),
static_cast<uint16_t>(m_chunk_dims[2])};
std::memcpy(&header[pos], vcdim, sizeof(vcdim));
auto vcdim =
std::array{static_cast<uint16_t>(m_chunk_dims[0]), static_cast<uint16_t>(m_chunk_dims[1]),
static_cast<uint16_t>(m_chunk_dims[2])};
std::memcpy(&header[pos], vcdim.data(), sizeof(vcdim));
pos += sizeof(vcdim);
}

Expand Down
54 changes: 16 additions & 38 deletions src/SPERR3D_Stream_Tools.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,57 +114,35 @@ auto sperr::SPERR3D_Stream_Tools::progressive_read(std::string filename, uint32_
return complete_stream;
}

// Calculate how many bytes to allocate to each chunk. There is a consideration:
// a truncated chunk should always be as long as a Conditioner bitstream (17 bytes).
auto condi_array = sperr::condi_type(); // temporary use
// Calculate how many bytes to allocate to each chunk, with `m_progressive_min_chunk_bytes`
// being the minimal length. The only exception is that when the chunk itself has less
// bytes, e.g., when it's a constant chunk.
assert(chunk_offsets.size() % 2 == 0);
auto nchunks = chunk_offsets.size() / 2;
auto chunk_offsets_new = chunk_offsets;

for (size_t i = 0; i < nchunks; i++) {
auto chunk_orig_len = chunk_offsets[i * 2 + 1];
auto request_len = static_cast<size_t>(double(pct) / 100.0 * double(chunk_orig_len));
request_len = std::max(condi_array.size(), request_len);
chunk_offsets_new[i * 2 + 1] = request_len;
auto orig_len = chunk_offsets[i * 2 + 1];
// Only touch the value stored at `chunk_offsets_new[i * 2 + 1]` if it's bigger than
// the minimal number of bytes to keep.
if (orig_len > m_progressive_min_chunk_bytes) {
auto request_len = static_cast<size_t>(double(pct) / 100.0 * double(orig_len));
request_len = std::max(m_progressive_min_chunk_bytes, request_len);
chunk_offsets_new[i * 2 + 1] = request_len;
}
}

// Calculate the total length of the new bitstream, and read it from disk!
auto total_len_new = header.size();
auto total_len_new = header_len;
for (size_t i = 0; i < nchunks; i++)
total_len_new += chunk_offsets_new[i * 2 + 1];
auto stream_new = vec8_type();
stream_new.reserve(total_len_new);
stream_new.resize(header.size()); // Occupy the space for a new header.
stream_new.resize(header_len); // Occupy the space for a new header.
auto rtn = sperr::read_sections(filename, chunk_offsets_new, stream_new);
if (rtn != RTNType::Good) {
vec20.clear(); // Just reuse an old variable.
return vec20;
}

// Need to verify each chunk: if the chunk is not constant, then it should also include
// the SPECK stream header (9 bytes), so that chunk should have 26 bytes at least.
// If it doesn't, we go back to the disk and read again!
// Note: this situation should happen VERY rarely though!
const auto chunk_min = condi_array.size() + SPECK_INT<uint8_t>::header_size; // 26
auto conditioner = sperr::Conditioner();
bool reread = false;
for (size_t i = 1; i < nchunks; i++)
chunk_offsets_new[i * 2] = chunk_offsets_new[i * 2 - 2] + chunk_offsets_new[i * 2 - 1];
for (size_t i = 0; i < nchunks; i++) {
auto byte = stream_new[chunk_offsets_new[i * 2]];
if (!conditioner.is_constant(byte) && chunk_offsets_new[i * 2 + 1] < chunk_min) {
chunk_offsets_new[i * 2 + 1] = chunk_min;
reread = true;
}
}
if (reread) {
for (size_t i = 0; i < nchunks; i++)
chunk_offsets_new[i * 2] = chunk_offsets[i * 2];
stream_new.resize(header.size());
auto rtn = sperr::read_sections(filename, chunk_offsets_new, stream_new);
if (rtn != RTNType::Good) {
vec20.clear(); // Just reuse an old variable.
return vec20;
}
stream_new.clear();
return stream_new;
}

// Finally, create a proper new header.
Expand Down
65 changes: 29 additions & 36 deletions test_scripts/stream_tools_unit_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,45 +10,44 @@ using sperr::RTNType;
// Test constant field
TEST(stream_tools, constant_1chunk)
{
// Produce a bigstream to disk
auto filename = std::string("./test.tmp");
// Produce a bitstream to disk
auto input = sperr::read_whole_file<float>("../test_data/const32x20x16.float");
assert(!input.empty());
auto encoder = sperr::SPERR3D_OMP_C();
encoder.set_dims_and_chunks({32, 20, 16}, {32, 20, 16});
encoder.set_psnr(99.0);
encoder.compress(input.data(), input.size());
auto stream = encoder.get_encoded_bitstream();
auto filename = std::string("./test.tmp");
sperr::write_n_bytes(filename, stream.size(), stream.data());

// Test progressive read!
auto tools = sperr::SPERR3D_Stream_Tools();
auto part = tools.progressive_read(filename, 100); // also populates data fields inside of `tools`

// The returned bitstream should remain the same, because we requested way more bit budget
// than the compressed bitstream has.
// The returned bitstream should remain the same.
EXPECT_EQ(part, stream);
}

TEST(stream_tools, constant_nchunks)
{
// Produce a bigstream to disk
auto filename = std::string("./test.tmp");
// Produce a bitstream to disk
auto input = sperr::read_whole_file<float>("../test_data/const32x20x16.float");
assert(!input.empty());
auto encoder = sperr::SPERR3D_OMP_C();
encoder.set_dims_and_chunks({32, 20, 16}, {10, 10, 8});
encoder.set_psnr(99.0);
encoder.compress(input.data(), input.size());
auto stream = encoder.get_encoded_bitstream();
auto filename = std::string("./test.tmp");
sperr::write_n_bytes(filename, stream.size(), stream.data());

// Test progressive read!
auto tools = sperr::SPERR3D_Stream_Tools();
auto part = tools.progressive_read(filename, 50); // also populates data fields inside of `tools`.

// The returned bitstream should (largely) remain the same, because the header of a constant
// is always recorded.
// The returned bitstream should still remain the same, except than one bit, because
// each chunk is so small that it's still kept in whole.
EXPECT_EQ(part.size(), stream.size());
EXPECT_EQ(part[0], stream[0]);
EXPECT_EQ(part[1], stream[1] + 128);
Expand All @@ -67,7 +66,7 @@ TEST(stream_tools, regular_1chunk)
assert(!input.empty());
auto encoder = sperr::SPERR3D_OMP_C();
encoder.set_dims_and_chunks({128, 128, 41}, {128, 128, 41});
encoder.set_psnr(100.0); // Resulting about 10bpp.
encoder.set_psnr(100.0); // Resulting about 9.2bpp.
encoder.compress(input.data(), input.size());
auto stream = encoder.get_encoded_bitstream();
sperr::write_n_bytes(filename, stream.size(), stream.data());
Expand All @@ -82,18 +81,16 @@ TEST(stream_tools, regular_1chunk)
for (size_t i = 2; i < tools.header_len - 4; i++) // Exclude the last 4 bytes (chunk len).
EXPECT_EQ(part[i], stream[i]);

// The header of each chunk (first 26 bytes) should also remain the same.
// To know offsets of each chunk of the new portioned bitstream, we use another
// stream tool to parse it.
auto tools_part = sperr::SPERR3D_Stream_Tools();
tools_part.populate_stream_info(part.data());
EXPECT_EQ(tools.chunk_offsets.size(), tools_part.chunk_offsets.size());
// The remaining bytes of each chunk should also remain the same. To know offsets of each chunk
// in the new portioned bitstream, we use another stream tool to parse it.
auto tools2 = sperr::SPERR3D_Stream_Tools();
tools2.populate_stream_info(part.data());
EXPECT_EQ(tools.chunk_offsets.size(), tools2.chunk_offsets.size());

for (size_t i = 0; i < tools.chunk_offsets.size() / 2; i++) {
auto orig_start = tools.chunk_offsets[i * 2];
auto part_start = tools_part.chunk_offsets[i * 2];
// We actually test first 90 bytes. There must be 90 bytes there because of the 50% bytes request.
for (size_t j = 0; j < 90; j++)
auto part_start = tools2.chunk_offsets[i * 2];
for (size_t j = 0; j < tools2.chunk_offsets[i * 2 + 1]; j++)
EXPECT_EQ(stream[orig_start + j], part[part_start + j]);
}
}
Expand All @@ -119,18 +116,16 @@ TEST(stream_tools, regular_nchunks)
EXPECT_EQ(part[0], stream[0]);
EXPECT_EQ(part[1], stream[1] + 128);

// The header of each chunk (first 26 bytes) should also remain the same.
// To know offsets of each chunk of the new portioned bitstream, we use another
// stream tool to parse it.
auto tools_part = sperr::SPERR3D_Stream_Tools();
tools_part.populate_stream_info(part.data());
EXPECT_EQ(tools.chunk_offsets.size(), tools_part.chunk_offsets.size());
// The remaining bytes should also remain the same. To know offsets of each chunk in the
// new portioned bitstream, we use another stream tool to parse it.
auto tools2 = sperr::SPERR3D_Stream_Tools();
tools2.populate_stream_info(part.data());
EXPECT_EQ(tools.chunk_offsets.size(), tools2.chunk_offsets.size());

for (size_t i = 0; i < tools.chunk_offsets.size() / 2; i++) {
auto orig_start = tools.chunk_offsets[i * 2];
auto part_start = tools_part.chunk_offsets[i * 2];
// We actually test first 30 bytes. There must be 30 bytes there because of the 35% bytes request.
for (size_t j = 0; j < 30; j++)
auto part_start = tools2.chunk_offsets[i * 2];
for (size_t j = 0; j < tools2.chunk_offsets[i * 2 + 1]; j++)
EXPECT_EQ(stream[orig_start + j], part[part_start + j]);
}
}
Expand All @@ -157,18 +152,16 @@ TEST(stream_tools, min_chunk_len)
EXPECT_EQ(part[0], stream[0]);
EXPECT_EQ(part[1], stream[1] + 128);

// The header of each chunk (first 26 bytes) should also remain the same.
// To know offsets of each chunk of the new portioned bitstream, we use another
// stream tool to parse it.
auto tools_part = sperr::SPERR3D_Stream_Tools();
tools_part.populate_stream_info(part.data());
EXPECT_EQ(tools.chunk_offsets.size(), tools_part.chunk_offsets.size());
// The header of each chunk (first 26 bytes) should also remain the same. To know offsets of
// each chunk in the new portioned bitstream, we use another stream tool to parse it.
auto tools2 = sperr::SPERR3D_Stream_Tools();
tools2.populate_stream_info(part.data());
EXPECT_EQ(tools.chunk_offsets.size(), tools2.chunk_offsets.size());

for (size_t i = 0; i < tools.chunk_offsets.size() / 2; i++) {
auto orig_start = tools.chunk_offsets[i * 2];
auto part_start = tools_part.chunk_offsets[i * 2];
// We actually test first 26 bytes.
for (size_t j = 0; j < 26; j++)
auto part_start = tools2.chunk_offsets[i * 2];
for (size_t j = 0; j < tools2.chunk_offsets[i * 2 + 1]; j++)
EXPECT_EQ(stream[orig_start + j], part[part_start + j]);
}
}
Expand Down

0 comments on commit c71a156

Please sign in to comment.