Misc bugfixes + dependency upgrades

CMakeLists.txt

@@ -16,7 +16,7 @@ if("${CMAKE_BUILD_TYPE}" STREQUAL "")
 	set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-Wall -g -O2 -fPIC")
 endif()
 
-if(DEFINED USE_WASM)
+if(DEFINED USE_WASM)  # wasm build needs OPENCV_DIR defined
 	set(DISABLE_TESTS true)
 	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DGF256_TARGET_MOBILE")
 	set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-Os")
@@ -28,10 +28,13 @@ if(DEFINED USE_WASM)
 		${OPENCV_DIR}/opencv-build-wasm/build_wasm/
 		${opencv_include_modules}
 	)
+else()  # if not wasm, go find opencv. 3 or 4 should both work
+	find_package(OpenCV REQUIRED)
+	include_directories(${OpenCV_INCLUDE_DIRS})
 endif()
 
 if(NOT DEFINED OPENCV_LIBS)
-	set(OPENCV_LIBS "opencv_core" "opencv_imgcodecs" "opencv_imgproc" "opencv_photo")
+	set(OPENCV_LIBS "opencv_calib3d" "opencv_imgcodecs" "opencv_imgproc" "opencv_photo" "opencv_core")
 endif()
 
 if(NOT DEFINED CPPFILESYSTEM)

src/exe/cimbar_send/send.cpp

@@ -70,7 +70,7 @@ int main(int argc, char** argv)
 	if (!initialize_GL(window_size, window_size))
 	{
 		std::cerr << "failed to create window :(" << std::endl;
-		return 50;
+		return 70;
 	}
 
 	configure(colorBits, ecc, compressionLevel);

src/lib/cimb_translator/CellPositions.h

@@ -1,8 +1,9 @@
 /* This code is subject to the terms of the Mozilla Public License, v.2.0. http://mozilla.org/MPL/2.0/. */
 #pragma once
 
-#include <vector>
+#include <cstddef>
 #include <utility>
+#include <vector>
 
 class CellPositions
 {

src/lib/compression/zstd_compressor.h

@@ -32,7 +32,7 @@ class zstd_compressor : public STREAM
 			size_t compressedBytes = ZSTD_compressCCtx(_cctx, _compBuff.data(), _compBuff.size(), data, writeLen, _compressionLevel);
 			if (ZSTD_isError(compressedBytes))
 			{
-				std::cout << "error? " << ZSTD_getErrorName(compressedBytes) << std::endl;
+				std::cerr << "error? " << ZSTD_getErrorName(compressedBytes) << std::endl;
 				return false;
 			}
 			STREAM::write(_compBuff.data(), compressedBytes);

src/lib/encoder/test/DecoderTest.cpp

@@ -56,5 +56,8 @@ TEST_CASE( "DecoderTest/testDecode.Sample", "[unit]" )
 	unsigned bytesDecoded = dec.decode(TestCimbar::getSample("6bit/4_30_f0_627_extract.jpg"), decodedFile);
 	assertEquals( 9300, bytesDecoded );
 
-	assertEquals( "3de927c8aa0221807a2784210160cdc17567eb587bf01233d166900aadf14bf5", get_hash(decodedFile) );
+	if (CV_VERSION_MAJOR == 3)
+		assertEquals( "3de927c8aa0221807a2784210160cdc17567eb587bf01233d166900aadf14bf5", get_hash(decodedFile) );
+	else // # cv4
+		assertEquals( "59ddb2516b4ff5a528aebe538a22b736a6714263a454d20e146e1ffbba36c5ae", get_hash(decodedFile) );
 }

src/lib/fountain/FountainMetadata.h

@@ -15,7 +15,7 @@ class FountainMetadata
 	}
 
 public:
-	FountainMetadata(uint64_t id)
+	FountainMetadata(uint32_t id)
 	    : _data(id)
 	{
 	}
@@ -33,7 +33,7 @@ class FountainMetadata
 		to_uint8_arr(encode_id, size, d);
 	}
 
-	unsigned id() const
+	uint32_t id() const
 	{
 		return _data;
 	}
@@ -65,5 +65,5 @@ class FountainMetadata
 	}
 
 protected:
-	uint64_t _data; // might invert this and only generate the uint64_t when we need it
+	uint32_t _data; // might invert this and only generate the uint32_t when we need it
 };

src/lib/fountain/fountain_decoder_sink.h

@@ -39,10 +39,10 @@ class fountain_decoder_sink
 		return true;
 	}
 
-	void mark_done(uint64_t id)
+	void mark_done(const FountainMetadata& md)
 	{
-		_done.insert(id);
-		auto it = _streams.find(id);
+		_done.insert(md.id());
+		auto it = _streams.find(stream_slot(md));
 		if (it != _streams.end())
 			_streams.erase(it);
 	}
@@ -60,7 +60,7 @@ class fountain_decoder_sink
 	std::vector<std::string> get_done() const
 	{
 		std::vector<std::string> done;
-		for (uint64_t id : _done)
+		for (uint32_t id : _done)
 			done.push_back( get_filename(FountainMetadata(id)) );
 		return done;
 	}
@@ -77,7 +77,7 @@ class fountain_decoder_sink
 		return progress;
 	}
 
-	bool is_done(uint64_t id) const
+	bool is_done(uint32_t id) const
 	{
 		return _done.find(id) != _done.end();
 	}
@@ -106,7 +106,7 @@ class fountain_decoder_sink
 			return false;
 
 		if (store(md, *finished))
-			mark_done(md.id());
+			mark_done(md);
 		return true;
 	}
 
@@ -122,7 +122,7 @@ class fountain_decoder_sink
 	}
 
 protected:
-	// streams is limited to at most 8 decoders at a time. Current, we just use the lower bits of the encode_id.
+	// streams is limited to at most 8 decoders at a time. Currently, we just use the lower bits of the encode_id.
 	uint8_t stream_slot(const FountainMetadata& md) const
 	{
 		return md.encode_id() & 0x7;
@@ -137,7 +137,10 @@ class fountain_decoder_sink
 	std::string _dataDir;
 	unsigned _chunkSize;
 
+	// maybe instead of unordered_map+set, something where we can "age out" old streams?
+	// e.g. most recent 16/8, or something?
+	// question is what happens to _done/_streams when we wrap for continuous data streaming...
 	std::unordered_map<uint8_t, fountain_decoder_stream> _streams;
-	// track the uint64_t combo of (encode_id,size) to avoid redundant work
-	std::set<uint64_t> _done;
+	// track the uint32_t combo of (encode_id,size) to avoid redundant work
+	std::set<uint32_t> _done;
 };

src/lib/image_hash/ahash_result.h

@@ -2,7 +2,7 @@
 #pragma once
 
 #include "bit_extractor.h"
-#include "intx/int128.hpp"
+#include "intx/intx.hpp"
 #include <array>
 #include <iostream>
 #include <utility>

src/lib/image_hash/average_hash.h

@@ -6,7 +6,7 @@
 #include "bit_file/bitmatrix.h"
 #include "cimb_translator/Cell.h"
 
-#include "intx/int128.hpp"
+#include "intx/intx.hpp"
 #include <opencv2/opencv.hpp>
 
 #include <array>

src/lib/image_hash/test/bitExtractorTest.cpp

@@ -2,7 +2,7 @@
 #include "unittest.h"
 
 #include "bit_extractor.h"
-#include "intx/int128.hpp"
+#include "intx/intx.hpp"
 
 #include <bitset>
 #include <iostream>
@@ -36,7 +36,9 @@ TEST_CASE( "bitExtractorTest/testLargerValue.1", "[unit]" )
 
 TEST_CASE( "bitExtractorTest/testLargerValue.2", "[unit]" )
 {
-	intx::uint128 bits{0xFFBFCFE3FULL, 0xF83C0E030080000ULL};
+	intx::uint128 bits{0xF83C0E030080000ULL, 0xFFBFCFE3FULL};
+	assertEquals( "ffbfcfe3f0f83c0e030080000", intx::hex(bits) );  // sanity check
+
 	bit_extractor<intx::uint128, 100> be(bits);
 	uint64_t res = be.extract(1, 11, 21, 31);
 	assertEquals( 0xfffefcf8, res );

src/third_party_lib/PicoSHA2/picosha2.h

@@ -93,7 +93,7 @@ void hash256_block(RaIter1 message_digest, RaIter2 first, RaIter2 last) {
     assert(first + 64 == last);
     static_cast<void>(last);  // for avoiding unused-variable warning
     word_t w[64];
-    std::fill(w, w + 64, 0);
+    std::fill(w, w + 64, word_t(0));
     for (std::size_t i = 0; i < 16; ++i) {
         w[i] = (static_cast<word_t>(mask_8bit(*(first + i * 4))) << 24) |
                (static_cast<word_t>(mask_8bit(*(first + i * 4 + 1))) << 16) |
@@ -185,7 +185,7 @@ class hash256_one_by_one {
 
     void init() {
         buffer_.clear();
-        std::fill(data_length_digits_, data_length_digits_ + 4, 0);
+        std::fill(data_length_digits_, data_length_digits_ + 4, word_t(0));
         std::copy(detail::initial_message_digest,
                   detail::initial_message_digest + 8, h_);
     }
@@ -204,17 +204,17 @@ class hash256_one_by_one {
 
     void finish() {
         byte_t temp[64];
-        std::fill(temp, temp + 64, 0);
+        std::fill(temp, temp + 64, byte_t(0));
         std::size_t remains = buffer_.size();
         std::copy(buffer_.begin(), buffer_.end(), temp);
         temp[remains] = 0x80;
 
         if (remains > 55) {
-            std::fill(temp + remains + 1, temp + 64, 0);
+            std::fill(temp + remains + 1, temp + 64, byte_t(0));
             detail::hash256_block(h_, temp, temp + 64);
-            std::fill(temp, temp + 64 - 4, 0);
+            std::fill(temp, temp + 64 - 4, byte_t(0));
         } else {
-            std::fill(temp + remains + 1, temp + 64 - 4, 0);
+            std::fill(temp + remains + 1, temp + 64 - 4, byte_t(0));
         }
 
         write_data_bit_length(&(temp[56]));

src/third_party_lib/concurrentqueue/LICENSE.md

@@ -1,6 +1,7 @@
 This license file applies to everything in this repository except that which
 is explicitly annotated as being written by other authors, i.e. the Boost
 queue (included in the benchmarks for comparison), Intel's TBB library (ditto),
+dlib::pipe (ditto),
 the CDSChecker tool (used for verification), the Relacy model checker (ditto),
 and Jeff Preshing's semaphore implementation (used in the blocking queue) which
 has a zlib license (embedded in lightweightsempahore.h).

src/third_party_lib/concurrentqueue/concurrentqueue.h

@@ -1688,7 +1688,7 @@ class ConcurrentQueue
 		{
 		}
 
-		virtual ~ProducerBase() { };
+		virtual ~ProducerBase() { }
 
 		template<typename U>
 		inline bool dequeue(U& element)
@@ -1897,7 +1897,7 @@ class ConcurrentQueue
 					++pr_blockIndexSlotsUsed;
 				}
 
-				MOODYCAMEL_CONSTEXPR_IF (!MOODYCAMEL_NOEXCEPT_CTOR(T, U, new ((T*)nullptr) T(std::forward<U>(element)))) {
+				MOODYCAMEL_CONSTEXPR_IF (!MOODYCAMEL_NOEXCEPT_CTOR(T, U, new (static_cast<T*>(nullptr)) T(std::forward<U>(element)))) {
 					// The constructor may throw. We want the element not to appear in the queue in
 					// that case (without corrupting the queue):
 					MOODYCAMEL_TRY {
@@ -1923,7 +1923,7 @@ class ConcurrentQueue
 				blockIndex.load(std::memory_order_relaxed)->front.store(pr_blockIndexFront, std::memory_order_release);
 				pr_blockIndexFront = (pr_blockIndexFront + 1) & (pr_blockIndexSize - 1);
 
-				MOODYCAMEL_CONSTEXPR_IF (!MOODYCAMEL_NOEXCEPT_CTOR(T, U, new ((T*)nullptr) T(std::forward<U>(element)))) {
+				MOODYCAMEL_CONSTEXPR_IF (!MOODYCAMEL_NOEXCEPT_CTOR(T, U, new (static_cast<T*>(nullptr)) T(std::forward<U>(element)))) {
 					this->tailIndex.store(newTailIndex, std::memory_order_release);
 					return true;
 				}
@@ -2139,7 +2139,7 @@ class ConcurrentQueue
 					block = block->next;
 				}
 
-				MOODYCAMEL_CONSTEXPR_IF (MOODYCAMEL_NOEXCEPT_CTOR(T, decltype(*itemFirst), new ((T*)nullptr) T(details::deref_noexcept(itemFirst)))) {
+				MOODYCAMEL_CONSTEXPR_IF (MOODYCAMEL_NOEXCEPT_CTOR(T, decltype(*itemFirst), new (static_cast<T*>(nullptr)) T(details::deref_noexcept(itemFirst)))) {
 					blockIndex.load(std::memory_order_relaxed)->front.store((pr_blockIndexFront - 1) & (pr_blockIndexSize - 1), std::memory_order_release);
 				}
 			}
@@ -2158,7 +2158,7 @@ class ConcurrentQueue
 				if (details::circular_less_than<index_t>(newTailIndex, stopIndex)) {
 					stopIndex = newTailIndex;
 				}
-				MOODYCAMEL_CONSTEXPR_IF (MOODYCAMEL_NOEXCEPT_CTOR(T, decltype(*itemFirst), new ((T*)nullptr) T(details::deref_noexcept(itemFirst)))) {
+				MOODYCAMEL_CONSTEXPR_IF (MOODYCAMEL_NOEXCEPT_CTOR(T, decltype(*itemFirst), new (static_cast<T*>(nullptr)) T(details::deref_noexcept(itemFirst)))) {
 					while (currentTailIndex != stopIndex) {
 						new ((*this->tailBlock)[currentTailIndex++]) T(*itemFirst++);
 					}
@@ -2173,7 +2173,7 @@ class ConcurrentQueue
 							// may only define a (noexcept) move constructor, and so calls to the
 							// cctor will not compile, even if they are in an if branch that will never
 							// be executed
-							new ((*this->tailBlock)[currentTailIndex]) T(details::nomove_if<(bool)!MOODYCAMEL_NOEXCEPT_CTOR(T, decltype(*itemFirst), new ((T*)nullptr) T(details::deref_noexcept(itemFirst)))>::eval(*itemFirst));
+							new ((*this->tailBlock)[currentTailIndex]) T(details::nomove_if<!MOODYCAMEL_NOEXCEPT_CTOR(T, decltype(*itemFirst), new (static_cast<T*>(nullptr)) T(details::deref_noexcept(itemFirst)))>::eval(*itemFirst));
 							++currentTailIndex;
 							++itemFirst;
 						}
@@ -2220,7 +2220,7 @@ class ConcurrentQueue
 				this->tailBlock = this->tailBlock->next;
 			}
 
-			MOODYCAMEL_CONSTEXPR_IF (!MOODYCAMEL_NOEXCEPT_CTOR(T, decltype(*itemFirst), new ((T*)nullptr) T(details::deref_noexcept(itemFirst)))) {
+			MOODYCAMEL_CONSTEXPR_IF (!MOODYCAMEL_NOEXCEPT_CTOR(T, decltype(*itemFirst), new (static_cast<T*>(nullptr)) T(details::deref_noexcept(itemFirst)))) {
 				if (firstAllocatedBlock != nullptr)
 					blockIndex.load(std::memory_order_relaxed)->front.store((pr_blockIndexFront - 1) & (pr_blockIndexSize - 1), std::memory_order_release);
 			}
@@ -2239,7 +2239,7 @@ class ConcurrentQueue
 				desiredCount = desiredCount < max ? desiredCount : max;
 				std::atomic_thread_fence(std::memory_order_acquire);
 
-				auto myDequeueCount = this->dequeueOptimisticCount.fetch_add(desiredCount, std::memory_order_relaxed);;
+				auto myDequeueCount = this->dequeueOptimisticCount.fetch_add(desiredCount, std::memory_order_relaxed);
 
 				tail = this->tailIndex.load(std::memory_order_acquire);
 				auto actualCount = static_cast<size_t>(tail - (myDequeueCount - overcommit));
@@ -2501,7 +2501,7 @@ class ConcurrentQueue
 #endif
 				newBlock->ConcurrentQueue::Block::template reset_empty<implicit_context>();
 
-				MOODYCAMEL_CONSTEXPR_IF (!MOODYCAMEL_NOEXCEPT_CTOR(T, U, new ((T*)nullptr) T(std::forward<U>(element)))) {
+				MOODYCAMEL_CONSTEXPR_IF (!MOODYCAMEL_NOEXCEPT_CTOR(T, U, new (static_cast<T*>(nullptr)) T(std::forward<U>(element)))) {
 					// May throw, try to insert now before we publish the fact that we have this new block
 					MOODYCAMEL_TRY {
 						new ((*newBlock)[currentTailIndex]) T(std::forward<U>(element));
@@ -2519,7 +2519,7 @@ class ConcurrentQueue
 
 				this->tailBlock = newBlock;
 
-				MOODYCAMEL_CONSTEXPR_IF (!MOODYCAMEL_NOEXCEPT_CTOR(T, U, new ((T*)nullptr) T(std::forward<U>(element)))) {
+				MOODYCAMEL_CONSTEXPR_IF (!MOODYCAMEL_NOEXCEPT_CTOR(T, U, new (static_cast<T*>(nullptr)) T(std::forward<U>(element)))) {
 					this->tailIndex.store(newTailIndex, std::memory_order_release);
 					return true;
 				}
@@ -2697,15 +2697,15 @@ class ConcurrentQueue
 				if (details::circular_less_than<index_t>(newTailIndex, stopIndex)) {
 					stopIndex = newTailIndex;
 				}
-				MOODYCAMEL_CONSTEXPR_IF (MOODYCAMEL_NOEXCEPT_CTOR(T, decltype(*itemFirst), new ((T*)nullptr) T(details::deref_noexcept(itemFirst)))) {
+				MOODYCAMEL_CONSTEXPR_IF (MOODYCAMEL_NOEXCEPT_CTOR(T, decltype(*itemFirst), new (static_cast<T*>(nullptr)) T(details::deref_noexcept(itemFirst)))) {
 					while (currentTailIndex != stopIndex) {
 						new ((*this->tailBlock)[currentTailIndex++]) T(*itemFirst++);
 					}
 				}
 				else {
 					MOODYCAMEL_TRY {
 						while (currentTailIndex != stopIndex) {
-							new ((*this->tailBlock)[currentTailIndex]) T(details::nomove_if<(bool)!MOODYCAMEL_NOEXCEPT_CTOR(T, decltype(*itemFirst), new ((T*)nullptr) T(details::deref_noexcept(itemFirst)))>::eval(*itemFirst));
+							new ((*this->tailBlock)[currentTailIndex]) T(details::nomove_if<!MOODYCAMEL_NOEXCEPT_CTOR(T, decltype(*itemFirst), new (static_cast<T*>(nullptr)) T(details::deref_noexcept(itemFirst)))>::eval(*itemFirst));
 							++currentTailIndex;
 							++itemFirst;
 						}
@@ -3459,7 +3459,7 @@ class ConcurrentQueue
 					}
 
 					auto newHash = new (raw) ImplicitProducerHash;
-					newHash->capacity = (size_t)newCapacity;
+					newHash->capacity = static_cast<size_t>(newCapacity);
 					newHash->entries = reinterpret_cast<ImplicitProducerKVP*>(details::align_for<ImplicitProducerKVP>(raw + sizeof(ImplicitProducerHash)));
 					for (size_t i = 0; i != newCapacity; ++i) {
 						new (newHash->entries + i) ImplicitProducerKVP;
@@ -3698,15 +3698,15 @@ ConsumerToken::ConsumerToken(ConcurrentQueue<T, Traits>& queue)
 	: itemsConsumedFromCurrent(0), currentProducer(nullptr), desiredProducer(nullptr)
 {
 	initialOffset = queue.nextExplicitConsumerId.fetch_add(1, std::memory_order_release);
-	lastKnownGlobalOffset = (std::uint32_t)-1;
+	lastKnownGlobalOffset = static_cast<std::uint32_t>(-1);
 }
 
 template<typename T, typename Traits>
 ConsumerToken::ConsumerToken(BlockingConcurrentQueue<T, Traits>& queue)
 	: itemsConsumedFromCurrent(0), currentProducer(nullptr), desiredProducer(nullptr)
 {
 	initialOffset = reinterpret_cast<ConcurrentQueue<T, Traits>*>(&queue)->nextExplicitConsumerId.fetch_add(1, std::memory_order_release);
-	lastKnownGlobalOffset = (std::uint32_t)-1;
+	lastKnownGlobalOffset = static_cast<std::uint32_t>(-1);
 }
 
 template<typename T, typename Traits>