Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add mode autodetection + update mode toggle to function as autodetect off/on #32

Merged
merged 8 commits into from
Mar 3, 2024
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions app/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ android {
applicationId "org.cimbar.camerafilecopy"
minSdkVersion 21
targetSdkVersion 30
versionCode 13
versionName "0.6.0f"
versionCode 14
versionName "0.6.1"
testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner"
externalNativeBuild {
cmake {
Expand Down
65 changes: 53 additions & 12 deletions app/src/cpp/cfc-cpp/MultiThreadedDecoder.h
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,9 @@
class MultiThreadedDecoder
{
public:
MultiThreadedDecoder(std::string data_path, bool legacy_mode);
MultiThreadedDecoder(std::string data_path, int mode_val);

inline static clock_t count = 0;
inline static clock_t bytes = 0;
inline static clock_t perfect = 0;
inline static clock_t decoded = 0;
Expand All @@ -29,7 +30,10 @@ class MultiThreadedDecoder

void stop();

bool legacy_mode() const;
int mode() const;
bool set_mode(int mode_val);
int detected_mode() const;
Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This interface is to allow us to switch modes while keeping the MultiThreadedDecoder around when the mode doesn't switch (rather than tearing it down, resetting progress). So we can click the toggle button to our heart's content...


unsigned num_threads() const;
unsigned backlog() const;
unsigned files_in_flight() const;
Expand All @@ -41,21 +45,26 @@ class MultiThreadedDecoder
int do_extract(const cv::Mat& mat, cv::Mat& img);
void save(const cv::Mat& img);

static unsigned fountain_chunk_size(int mode_val);

protected:
bool _legacyMode;
int _modeVal;
int _detectedMode;

Decoder _dec;
unsigned _numThreads;
turbo::thread_pool _pool;
concurrent_fountain_decoder_sink<cimbar::zstd_decompressor<std::ofstream>> _writer;
std::string _dataPath;
};

inline MultiThreadedDecoder::MultiThreadedDecoder(std::string data_path, bool legacy_mode)
: _legacyMode(legacy_mode)
, _dec(cimbar::Config::ecc_bytes(), cimbar::Config::color_bits(), legacy_mode? 0 : 1, legacy_mode)
inline MultiThreadedDecoder::MultiThreadedDecoder(std::string data_path, int mode_val)
: _modeVal(mode_val)
, _detectedMode(0)
, _dec(cimbar::Config::ecc_bytes(), cimbar::Config::color_bits())
, _numThreads(std::max<int>(((int)std::thread::hardware_concurrency()/2), 1))
, _pool(_numThreads, 1)
, _writer(data_path, cimbar::Config::fountain_chunk_size(cimbar::Config::ecc_bytes(), cimbar::Config::symbol_bits() + cimbar::Config::color_bits(), legacy_mode))
, _writer(data_path, fountain_chunk_size(mode_val))
, _dataPath(data_path)
{
FountainInit::init();
Expand Down Expand Up @@ -87,7 +96,9 @@ inline int MultiThreadedDecoder::do_extract(const cv::Mat& mat, cv::Mat& img)

inline bool MultiThreadedDecoder::add(cv::Mat mat)
{
return _pool.try_execute( [&, mat] () {
++count;
bool legacy_mode = _modeVal == 4 or (_modeVal == 0 and count%2 == 0);
Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Crucially, we do this count -> legacy_mode calculation on the main thread, not on the background threads. 😶

return _pool.try_execute( [&, mat, legacy_mode] () {
cv::Mat img;
int res = do_extract(mat, img);
if (res == Extractor::FAILURE)
Expand All @@ -96,12 +107,16 @@ inline bool MultiThreadedDecoder::add(cv::Mat mat)
// if extracted image is small, we'll need to run some filters on it
clock_t begin = clock();
bool should_preprocess = (res == Extractor::NEEDS_SHARPEN);
int color_correction = _legacyMode? 1 : 2;
unsigned decodeRes = _dec.decode_fountain(img, _writer, should_preprocess, color_correction);
int color_correction = legacy_mode? 1 : 2;
unsigned color_mode = legacy_mode? 0 : 1;
unsigned decodeRes = _dec.decode_fountain(img, _writer, color_mode, should_preprocess, color_correction);
bytes += decodeRes;
++decoded;
decodeTicks += clock() - begin;

if (decodeRes and _modeVal == 0)
_detectedMode = legacy_mode? 4 : 68;

if (decodeRes >= 6900)
++perfect;
} );
Expand All @@ -121,9 +136,35 @@ inline void MultiThreadedDecoder::stop()
_pool.stop();
}

inline bool MultiThreadedDecoder::legacy_mode() const
unsigned MultiThreadedDecoder::fountain_chunk_size(int mode_val)
{
return cimbar::Config::fountain_chunk_size(cimbar::Config::ecc_bytes(), cimbar::Config::symbol_bits() + cimbar::Config::color_bits(), mode_val==4);
}

inline int MultiThreadedDecoder::mode() const
{
return _modeVal;
}

inline bool MultiThreadedDecoder::set_mode(int mode_val)
{
if (_modeVal == mode_val)
return true;

if (mode_val != 0 and _writer.chunk_size() != fountain_chunk_size(mode_val))
return false; // if so, we need to reset to change it
Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the past (for example when from 4-color to 8-color, in 0.5.x), we had to throw away the MultiThreadedDecoder object and get a new one any time we changed anything. Since we have an "autodetect mode" now, we want to only do that when we must (since it currently requires throwing away progress).

The fountain_chunk_size being different is the condition. Which is to say, if the modes had the same fountain_chunk_size, we would be able to make continuous progress on the stream no matter what mode we were in. But there are other things that might prevent that, and in any case there are good reasons mode B has to use a different number.


// reset detectedMode iff we're switching back to autodetect
if (mode_val == 0)
_detectedMode = 0;
Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

0 is our "it's not a real mode, it's autodetect mode" value.


_modeVal = mode_val;
return true;
}

inline int MultiThreadedDecoder::detected_mode() const
{
return _legacyMode;
return _detectedMode;
}

inline unsigned MultiThreadedDecoder::num_threads() const
Expand Down
13 changes: 8 additions & 5 deletions app/src/cpp/cfc-cpp/jni.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
#include "cimb_translator/CimbReader.h"
#include "encoder/Decoder.h"
#include "extractor/Scanner.h"
#include "serialize/format.h"

#include <jni.h>
#include <android/log.h>
Expand Down Expand Up @@ -123,8 +124,8 @@ namespace {
void drawDebugInfo(cv::Mat& mat, MultiThreadedDecoder& proc)
{
std::stringstream sstop;
sstop << "cfc using " << proc.num_threads() << " thread(s). " << proc.legacy_mode() << "..." << proc.backlog() << "? ";
sstop << (MultiThreadedDecoder::bytes / std::max<double>(1, MultiThreadedDecoder::decoded)) << "b v0.6.0f";
sstop << "cfc using " << proc.num_threads() << " thread(s). " << proc.mode() << ":" << proc.detected_mode() << "..." << proc.backlog() << "? ";
sstop << (MultiThreadedDecoder::bytes / std::max<double>(1, MultiThreadedDecoder::decoded)) << "b v0.6.1";
std::stringstream ssmid;
ssmid << "#: " << MultiThreadedDecoder::perfect << " / " << MultiThreadedDecoder::decoded << " / " << MultiThreadedDecoder::scanned << " / " << _calls;
std::stringstream ssperf;
Expand Down Expand Up @@ -170,13 +171,12 @@ Java_org_cimbar_camerafilecopy_MainActivity_processImageJNI(JNIEnv *env, jobject
Mat &mat = *(Mat *) matAddr;
string dataPath = jstring_to_cppstr(env, dataPathObj);
int modeVal = (int)modeInt;
bool legacyMode = modeVal <= 8; // current scheme: old 4c = 4, old 8c = 8, new = bigger number

std::shared_ptr<MultiThreadedDecoder> proc;
{
std::lock_guard<std::mutex> lock(_mutex);
if (!_proc or _proc->legacy_mode() != legacyMode)
_proc = std::make_shared<MultiThreadedDecoder>(dataPath, legacyMode);
if (!_proc or !_proc->set_mode(modeVal))
_proc = std::make_shared<MultiThreadedDecoder>(dataPath, modeVal);
proc = _proc;
}

Expand Down Expand Up @@ -205,6 +205,9 @@ Java_org_cimbar_camerafilecopy_MainActivity_processImageJNI(JNIEnv *env, jobject

// return a decoded file to prompt the user to save it, if there is a new one
string result;
if (proc->detected_mode()) // repurpose str for special message passing
result = fmt::format("/{}", proc->detected_mode());
Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a bit ridiculous. But it also works. 🤔


std::vector<string> all_decodes = proc->get_done();
for (string& s : all_decodes)
if (_completed.find(s) == _completed.end())
Expand Down
27 changes: 13 additions & 14 deletions app/src/cpp/libcimbar/src/exe/cimbar/cimbar.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ int encode(const FilenameIterable& infiles, const std::string& outpath, int ecc,
}

template <typename FilenameIterable>
int decode(const FilenameIterable& infiles, const std::function<int(cv::UMat, bool, int)>& decodefun, bool no_deskew, bool undistort, int preprocess, int color_correct)
int decode(const FilenameIterable& infiles, const std::function<int(cv::UMat, unsigned, bool, int)>& decodefun, bool no_deskew, bool undistort, unsigned color_mode, int preprocess, int color_correct)
{
int err = 0;
for (const string& inf : infiles)
Expand Down Expand Up @@ -152,7 +152,7 @@ int decode(const FilenameIterable& infiles, const std::function<int(cv::UMat, bo
shouldPreprocess = true;
}

int bytes = decodefun(img, shouldPreprocess, color_correct);
int bytes = decodefun(img, color_mode, shouldPreprocess, color_correct);
if (!bytes)
err |= 4;
}
Expand All @@ -162,10 +162,10 @@ int decode(const FilenameIterable& infiles, const std::function<int(cv::UMat, bo
// see also "decodefun" for non-fountain decodes, defined as a lambda inline below.
// this one needs its own function since it's a template (:
template <typename SINK>
std::function<int(cv::UMat,bool,int)> fountain_decode_fun(SINK& sink, Decoder& d)
std::function<int(cv::UMat,unsigned,bool,int)> fountain_decode_fun(SINK& sink, Decoder& d)
{
return [&sink, &d] (cv::UMat m, bool pre, int cc) {
return d.decode_fountain(m, sink, pre, cc);
return [&sink, &d] (cv::UMat m, unsigned cm, bool pre, int cc) {
return d.decode_fountain(m, sink, cm, pre, cc);
};
}

Expand Down Expand Up @@ -247,8 +247,7 @@ int main(int argc, char** argv)
int preprocess = result["preprocess"].as<int>();

unsigned color_mode = legacy_mode? 0 : 1;
bool coupled = legacy_mode;
Decoder d(ecc, colorBits, color_mode, coupled);
Decoder d(ecc, colorBits);

if (no_fountain)
{
Expand All @@ -257,13 +256,13 @@ int main(int argc, char** argv)

// simpler encoding, just the basics + ECC. No compression, fountain codes, etc.
std::ofstream f(outpath);
std::function<int(cv::UMat,bool,int)> decodefun = [&f, &d] (cv::UMat m, bool pre, int cc) {
return d.decode(m, f, pre, cc);
std::function<int(cv::UMat,unsigned,bool,int)> decodefun = [&f, &d] (cv::UMat m, unsigned cm, bool pre, int cc) {
return d.decode(m, f, cm, pre, cc);
};
if (useStdin)
return decode(StdinLineReader(), decodefun, no_deskew, undistort, preprocess, color_correct);
return decode(StdinLineReader(), decodefun, no_deskew, undistort, color_mode, preprocess, color_correct);
else
return decode(infiles, decodefun, no_deskew, undistort, preprocess, color_correct);
return decode(infiles, decodefun, no_deskew, undistort, color_mode, preprocess, color_correct);
}

// else, the good stuff
Expand All @@ -273,16 +272,16 @@ int main(int argc, char** argv)
if (compressionLevel <= 0)
{
fountain_decoder_sink<std::ofstream> sink(outpath, chunkSize, true);
res = decode(infiles, fountain_decode_fun(sink, d), no_deskew, undistort, preprocess, color_correct);
res = decode(infiles, fountain_decode_fun(sink, d), no_deskew, undistort, color_mode, preprocess, color_correct);
}
else // default case, all bells and whistles
{
fountain_decoder_sink<cimbar::zstd_decompressor<std::ofstream>> sink(outpath, chunkSize, true);

if (useStdin)
res = decode(StdinLineReader(), fountain_decode_fun(sink, d), no_deskew, undistort, preprocess, color_correct);
res = decode(StdinLineReader(), fountain_decode_fun(sink, d), no_deskew, undistort, color_mode, preprocess, color_correct);
else
res = decode(infiles, fountain_decode_fun(sink, d), no_deskew, undistort, preprocess, color_correct);
res = decode(infiles, fountain_decode_fun(sink, d), no_deskew, undistort, color_mode, preprocess, color_correct);
}
if (not color_correction_file.empty())
d.save_ccm(color_correction_file);
Expand Down
6 changes: 3 additions & 3 deletions app/src/cpp/libcimbar/src/exe/cimbar_recv/recv.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ int main(int argc, char** argv)
string mode = result["mode"].as<string>();
legacy_mode = (mode == "4c") or (mode == "4C");
}
unsigned color_mode = legacy_mode? 0 : 1;

unsigned fps = result["fps"].as<unsigned>();
if (fps == 0)
Expand Down Expand Up @@ -104,8 +105,7 @@ int main(int argc, char** argv)
window.auto_scale_to_window();

Extractor ext;
unsigned color_mode = legacy_mode? 0 : 1;
Decoder dec(-1, -1, color_mode, legacy_mode);
Decoder dec(-1, -1);

unsigned chunkSize = cimbar::Config::fountain_chunk_size(ecc, colorBits+cimbar::Config::symbol_bits(), legacy_mode);
fountain_decoder_sink<cimbar::zstd_decompressor<std::ofstream>> sink(outpath, chunkSize);
Expand Down Expand Up @@ -147,7 +147,7 @@ int main(int argc, char** argv)
shouldPreprocess = true;

// decode
int bytes = dec.decode_fountain(img, sink, shouldPreprocess);
int bytes = dec.decode_fountain(img, sink, color_mode, shouldPreprocess);
if (bytes > 0)
std::cerr << "got some bytes " << bytes << std::endl;
}
Expand Down
2 changes: 1 addition & 1 deletion app/src/cpp/libcimbar/src/exe/cimbar_send/send.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ int main(int argc, char** argv)
("c,colorbits", "Color bits. [0-3]", cxxopts::value<int>()->default_value(turbo::str::str(colorBits)))
("e,ecc", "ECC level", cxxopts::value<unsigned>()->default_value(turbo::str::str(ecc)))
("f,fps", "Target FPS", cxxopts::value<unsigned>()->default_value(turbo::str::str(defaultFps)))
("m,mode", "Select a cimbar mode. B (the default) is new to 0.6.x. 4C is the 0.5.x config. [B,4C]", cxxopts::value<string>()->default_value("B"))
("m,mode", "Select a cimbar mode. B is new to 0.6.x. 4C is the 0.5.x config. [B,4C]", cxxopts::value<string>()->default_value("4C"))
("z,compression", "Compression level. 0 == no compression.", cxxopts::value<int>()->default_value(turbo::str::str(compressionLevel)))
("h,help", "Print usage")
;
Expand Down
15 changes: 7 additions & 8 deletions app/src/cpp/libcimbar/src/lib/cimb_translator/CimbDecoder.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -54,11 +54,10 @@ namespace {
}
}

CimbDecoder::CimbDecoder(unsigned symbol_bits, unsigned color_bits, unsigned color_mode, bool dark, uchar ahashThreshold)
CimbDecoder::CimbDecoder(unsigned symbol_bits, unsigned color_bits, bool dark, uchar ahashThreshold)
: _symbolBits(symbol_bits)
, _numSymbols(1 << symbol_bits)
, _numColors(1 << color_bits)
, _colorMode(color_mode)
, _dark(dark)
, _ahashThreshold(ahashThreshold)
{
Expand Down Expand Up @@ -160,12 +159,12 @@ unsigned CimbDecoder::check_color_distance(std::tuple<uchar,uchar,uchar> a, std:
return color_diff(a, b);
}

std::tuple<uchar,uchar,uchar> CimbDecoder::get_color(int i) const
std::tuple<uchar,uchar,uchar> CimbDecoder::get_color(int i, unsigned color_mode) const
{
return cimbar::getColor(i, _numColors, _colorMode);
return cimbar::getColor(i, _numColors, color_mode);
}

unsigned CimbDecoder::get_best_color(float r, float g, float b) const
unsigned CimbDecoder::get_best_color(float r, float g, float b, unsigned color_mode) const
{
// transform color with ccm
if (internal_ccm().active())
Expand All @@ -188,7 +187,7 @@ unsigned CimbDecoder::get_best_color(float r, float g, float b) const
float best_distance = 1000000;
for (unsigned i = 0; i < _numColors; ++i)
{
std::tuple<uchar,uchar,uchar> candidate = get_color(i);
std::tuple<uchar,uchar,uchar> candidate = get_color(i, color_mode);
unsigned distance = check_color_distance(c, candidate);
if (distance < best_distance)
{
Expand All @@ -208,12 +207,12 @@ std::tuple<uchar,uchar,uchar> CimbDecoder::avg_color(const Cell& color_cell) con
return center.mean_rgb();
}

unsigned CimbDecoder::decode_color(const Cell& color_cell) const
unsigned CimbDecoder::decode_color(const Cell& color_cell, unsigned color_mode) const
{
if (_numColors <= 1)
return 0;
auto [r, g, b] = avg_color(color_cell);
return get_best_color(r, g, b);
return get_best_color(r, g, b, color_mode);
}

bool CimbDecoder::expects_binary_threshold() const
Expand Down
9 changes: 4 additions & 5 deletions app/src/cpp/libcimbar/src/lib/cimb_translator/CimbDecoder.h
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
class CimbDecoder
{
public:
CimbDecoder(unsigned symbol_bits, unsigned color_bits, unsigned color_mode=1, bool dark=true, uchar ahashThreshold=0);
CimbDecoder(unsigned symbol_bits, unsigned color_bits, bool dark=true, uchar ahashThreshold=0);

const color_correction& get_ccm() const;
void update_color_correction(cv::Matx<float, 3, 3>&& ccm);
Expand All @@ -22,10 +22,10 @@ class CimbDecoder
unsigned decode_symbol(const cv::Mat& cell, unsigned& drift_offset, unsigned& best_distance, unsigned cooldown=0xFF) const;
unsigned decode_symbol(const bitmatrix& cell, unsigned& drift_offset, unsigned& best_distance, unsigned cooldown=0xFF) const;

std::tuple<uchar,uchar,uchar> get_color(int i) const;
std::tuple<uchar,uchar,uchar> get_color(int i, unsigned color_mode) const;
std::tuple<uchar,uchar,uchar> avg_color(const Cell& color_cell) const;
unsigned get_best_color(float r, float g, float b) const;
unsigned decode_color(const Cell& cell) const;
unsigned get_best_color(float r, float g, float b, unsigned color_mode) const;
unsigned decode_color(const Cell& cell, unsigned color_mode) const;

bool expects_binary_threshold() const;
unsigned symbol_bits() const;
Expand All @@ -44,7 +44,6 @@ class CimbDecoder
unsigned _symbolBits;
unsigned _numSymbols;
unsigned _numColors;
unsigned _colorMode;
bool _dark;
uchar _ahashThreshold;
};
Loading