diff --git a/drivers/CoreVideo/include/CoreJPEG.hpp b/drivers/CoreVideo/include/CoreJPEG.hpp index c392aeb9c6..c74e167814 100644 --- a/drivers/CoreVideo/include/CoreJPEG.hpp +++ b/drivers/CoreVideo/include/CoreJPEG.hpp @@ -27,7 +27,8 @@ class CoreJPEG : public interface::JPEGBase auto getHandle() -> JPEG_HandleTypeDef & final; - void decodeImage(interface::File &file) final; + auto decodeImage(interface::File &file) -> size_t final; + auto findSOIMarker(interface::File &file, size_t start_index) -> size_t final; auto getImageProperties() -> JPEGImageProperties final; diff --git a/drivers/CoreVideo/include/CoreJPEGModeDMA.hpp b/drivers/CoreVideo/include/CoreJPEGModeDMA.hpp index 860ba22725..c61ac89938 100644 --- a/drivers/CoreVideo/include/CoreJPEGModeDMA.hpp +++ b/drivers/CoreVideo/include/CoreJPEGModeDMA.hpp @@ -26,7 +26,7 @@ struct CoreJPEGModeDMA : interface::JPEGMode { void onDataAvailableCallback(JPEG_HandleTypeDef *hjpeg, uint32_t decoded_datasize) final; void onDecodeCompleteCallback(JPEG_HandleTypeDef *hjpeg) final; - auto decode(JPEG_HandleTypeDef *hjpeg, interface::File &file) -> HAL_StatusTypeDef final; + auto decode(JPEG_HandleTypeDef *hjpeg, interface::File &file) -> size_t final; private: static std::array BIG_CHUNGUS_OF_MEMORY_IN; @@ -66,7 +66,7 @@ struct CoreJPEGModeDMA : interface::JPEGMode { // color conversion function pointer, set by onInfoReadyCallback JPEG_YCbCrToRGB_Convert_Function convertMCUBlocks; - uint32_t _image_size = 0; + size_t _image_size = 0; uint32_t _mcu_number = 0; uint32_t _mcu_block_index = 0; bool _hw_decode_ended = false; diff --git a/drivers/CoreVideo/include/CoreJPEGModePolling.hpp b/drivers/CoreVideo/include/CoreJPEGModePolling.hpp index 76f191d026..a91c945d16 100644 --- a/drivers/CoreVideo/include/CoreJPEGModePolling.hpp +++ b/drivers/CoreVideo/include/CoreJPEGModePolling.hpp @@ -24,7 +24,7 @@ class CoreJPEGModePolling : public interface::JPEGMode void onDataAvailableCallback(JPEG_HandleTypeDef *hjpeg, uint32_t decoded_datasize) final; void onDecodeCompleteCallback(JPEG_HandleTypeDef *hjpeg) final; - auto decode(JPEG_HandleTypeDef *hjpeg, interface::File &file) -> HAL_StatusTypeDef final; + auto decode(JPEG_HandleTypeDef *hjpeg, interface::File &file) -> size_t final; private: struct JPEGDataBuffer { @@ -35,6 +35,7 @@ class CoreJPEGModePolling : public interface::JPEGMode interface::STM32Hal &_hal; interface::File *_file {}; + size_t _image_size = 0; uint32_t _mcu_number = 0; uint32_t _mcu_block_index = 0; uint32_t _input_file_offset = 0; diff --git a/drivers/CoreVideo/include/CoreVideo.hpp b/drivers/CoreVideo/include/CoreVideo.hpp index 59c4827e3f..8634036c90 100644 --- a/drivers/CoreVideo/include/CoreVideo.hpp +++ b/drivers/CoreVideo/include/CoreVideo.hpp @@ -34,6 +34,7 @@ class CoreVideo void clearScreen(CGColor color = CGColor::white); void displayRectangle(interface::Graphics::FilledRectangle rectangle, CGColor color); void displayImage(interface::File &file); + void playVideo(interface::File &file); void displayText(const char *text, uint32_t size, uint32_t starting_line, CGColor foreground = CGColor::black, CGColor background = CGColor::white); diff --git a/drivers/CoreVideo/include/interface/JPEG.hpp b/drivers/CoreVideo/include/interface/JPEG.hpp index 8031bd1d98..4fec336ac9 100644 --- a/drivers/CoreVideo/include/interface/JPEG.hpp +++ b/drivers/CoreVideo/include/interface/JPEG.hpp @@ -19,7 +19,8 @@ class JPEGBase virtual auto getHandle() -> JPEG_HandleTypeDef & = 0; - virtual void decodeImage(interface::File &) = 0; + virtual auto decodeImage(interface::File &) -> size_t = 0; + virtual auto findSOIMarker(interface::File &file, size_t start_index) -> size_t = 0; virtual auto getImageProperties() -> JPEGImageProperties = 0; }; diff --git a/drivers/CoreVideo/include/interface/JPEGMode.hpp b/drivers/CoreVideo/include/interface/JPEGMode.hpp index b0b35e3d12..3d5cadce37 100644 --- a/drivers/CoreVideo/include/interface/JPEGMode.hpp +++ b/drivers/CoreVideo/include/interface/JPEGMode.hpp @@ -19,8 +19,7 @@ struct JPEGMode { virtual void onDataAvailableCallback(JPEG_HandleTypeDef *hjpeg, uint32_t decoded_datasize) = 0; virtual void onDecodeCompleteCallback(JPEG_HandleTypeDef *hjpeg) = 0; - // TODO(@yann): Update Return type with something else than HAL status - virtual auto decode(JPEG_HandleTypeDef *hjpeg, interface::File &file) -> HAL_StatusTypeDef = 0; + virtual auto decode(JPEG_HandleTypeDef *hjpeg, interface::File &file) -> size_t = 0; }; } // namespace leka::interface diff --git a/drivers/CoreVideo/include/internal/corevideo_config.h b/drivers/CoreVideo/include/internal/corevideo_config.h index 0ab27460c2..fa36c68bac 100644 --- a/drivers/CoreVideo/include/internal/corevideo_config.h +++ b/drivers/CoreVideo/include/internal/corevideo_config.h @@ -71,6 +71,8 @@ namespace jpeg { inline constexpr uint8_t JPEG_RGB_FORMAT = JPEG_ARGB8888; // Select RGB format: ARGB8888, RGB888, RBG565 inline constexpr uint8_t JPEG_SWAP_RB = 0; // Change color order to BGR + inline constexpr uint16_t JPEG_SOI_MARKER = 0xFFD8; // JPEG Start of Image marker + } // namespace jpeg namespace graphics { diff --git a/drivers/CoreVideo/source/CoreJPEG.cpp b/drivers/CoreVideo/source/CoreJPEG.cpp index ce65ffdd50..e75c8fd3e5 100644 --- a/drivers/CoreVideo/source/CoreJPEG.cpp +++ b/drivers/CoreVideo/source/CoreJPEG.cpp @@ -71,7 +71,33 @@ void CoreJPEG::registerProcessCallbacks() [](JPEG_HandleTypeDef *hjpeg) { self._mode.onErrorCallback(hjpeg); }); } -void CoreJPEG::decodeImage(interface::File &file) +auto CoreJPEG::decodeImage(interface::File &file) -> size_t { - _mode.decode(&_hjpeg, file); + return _mode.decode(&_hjpeg, file); +} + +auto CoreJPEG::findSOIMarker(interface::File &file, size_t start_index) -> size_t +{ + auto buffer = std::array {}; + + auto is_jpeg_soi_marker_predicate = [](uint8_t val1, uint8_t val2) { + return (static_cast((val1 << 8) + val2)) == jpeg::JPEG_SOI_MARKER; + }; + + while (file.size() > start_index) { + file.seek(start_index, SEEK_SET); + auto bytes_read = file.read(buffer.data(), buffer.size()); + + auto *buffer_end_index = std::begin(buffer) + bytes_read; + auto *find_index = std::adjacent_find(std::begin(buffer), buffer_end_index, is_jpeg_soi_marker_predicate); + + if (auto marker_found = (find_index != buffer_end_index)) { + auto marker_index = start_index + std::distance(std::begin(buffer), find_index); + return marker_index; + } + + start_index += bytes_read; + } + + return 0; } diff --git a/drivers/CoreVideo/source/CoreJPEGModeDMA.cpp b/drivers/CoreVideo/source/CoreJPEGModeDMA.cpp index 1282871a4e..5d7c46e4c3 100644 --- a/drivers/CoreVideo/source/CoreJPEGModeDMA.cpp +++ b/drivers/CoreVideo/source/CoreJPEGModeDMA.cpp @@ -148,7 +148,7 @@ void CoreJPEGModeDMA::onDecodeCompleteCallback(JPEG_HandleTypeDef *hjpeg) _hw_decode_ended = true; } -auto CoreJPEGModeDMA::decode(JPEG_HandleTypeDef *hjpeg, interface::File &file) -> HAL_StatusTypeDef +auto CoreJPEGModeDMA::decode(JPEG_HandleTypeDef *hjpeg, interface::File &file) -> size_t { reset(); @@ -171,12 +171,7 @@ auto CoreJPEGModeDMA::decode(JPEG_HandleTypeDef *hjpeg, interface::File &file) - process_ended = decoderOutputHandler(hjpeg); } while (!process_ended); - auto ret = HAL_OK; - if (_image_size == 0) { - ret = HAL_ERROR; - } - - return ret; + return _image_size; } std::array CoreJPEGModeDMA::BIG_CHUNGUS_OF_MEMORY_IN; diff --git a/drivers/CoreVideo/source/CoreJPEGModePolling.cpp b/drivers/CoreVideo/source/CoreJPEGModePolling.cpp index 5c047f8acb..dc61498b64 100644 --- a/drivers/CoreVideo/source/CoreJPEGModePolling.cpp +++ b/drivers/CoreVideo/source/CoreJPEGModePolling.cpp @@ -75,6 +75,8 @@ void CoreJPEGModePolling::onDataAvailableCallback(JPEG_HandleTypeDef *hjpeg, uin } else { // TODO(@yann): handle error } + + _image_size += _jpeg_input_buffer.size; } void CoreJPEGModePolling::onDecodeCompleteCallback(JPEG_HandleTypeDef *hjpeg) @@ -82,11 +84,12 @@ void CoreJPEGModePolling::onDecodeCompleteCallback(JPEG_HandleTypeDef *hjpeg) // TODO(@yann): implement flag } -auto CoreJPEGModePolling::decode(JPEG_HandleTypeDef *hjpeg, interface::File &file) -> HAL_StatusTypeDef +auto CoreJPEGModePolling::decode(JPEG_HandleTypeDef *hjpeg, interface::File &file) -> size_t { _file = &file; // WARNING: DO NOT REMOVE + _image_size = 0; _mcu_block_index = 0; if (_jpeg_input_buffer.size = _file->read(_jpeg_input_buffer.data, leka::jpeg::input_chunk_size); @@ -99,5 +102,5 @@ auto CoreJPEGModePolling::decode(JPEG_HandleTypeDef *hjpeg, interface::File &fil _hal.HAL_JPEG_Decode(hjpeg, _jpeg_input_buffer.data, _jpeg_input_buffer.size, _mcu_data_output_buffer.data(), leka::jpeg::output_chunk_size, HAL_MAX_DELAY); - return HAL_OK; + return _image_size; } diff --git a/drivers/CoreVideo/source/CoreVideo.cpp b/drivers/CoreVideo/source/CoreVideo.cpp index a6bf42856d..d83edeb938 100644 --- a/drivers/CoreVideo/source/CoreVideo.cpp +++ b/drivers/CoreVideo/source/CoreVideo.cpp @@ -80,6 +80,23 @@ void CoreVideo::displayImage(interface::File &file) image_properties.getWidthOffset()); } +void CoreVideo::playVideo(interface::File &file) +{ + auto config = _corejpeg.getImageProperties(); + + auto frame_index = _corejpeg.findSOIMarker(file, 0); + auto frame_size = size_t {0}; + + while (frame_index != 0) { + file.seek(frame_index, SEEK_SET); + + frame_size = _corejpeg.decodeImage(file); + _coredma2d.transferImage(config.ImageWidth, config.ImageHeight, config.getWidthOffset()); + + frame_index = _corejpeg.findSOIMarker(file, frame_index + frame_size); + } +} + void CoreVideo::displayText(const char *text, uint32_t size, uint32_t starting_line, CGColor foreground, CGColor background) { diff --git a/drivers/CoreVideo/tests/CoreJPEG_test.cpp b/drivers/CoreVideo/tests/CoreJPEG_test.cpp index a7b9ac32df..bf1673716d 100644 --- a/drivers/CoreVideo/tests/CoreJPEG_test.cpp +++ b/drivers/CoreVideo/tests/CoreJPEG_test.cpp @@ -13,10 +13,13 @@ using namespace leka; using ::testing::_; +using ::testing::AnyNumber; using ::testing::DoAll; using ::testing::InSequence; +using ::testing::Matcher; using ::testing::Return; using ::testing::SetArgPointee; +using ::testing::SetArrayArgument; class CoreJPEGTest : public ::testing::Test { @@ -91,3 +94,135 @@ TEST_F(CoreJPEGTest, decodeImage) corejpeg.decodeImage(filemock); } + +TEST_F(CoreJPEGTest, findSOIMarkerFileIsEmpty) +{ + EXPECT_CALL(filemock, size).WillOnce(Return(0)); + + corejpeg.findSOIMarker(filemock, 0); +} + +TEST_F(CoreJPEGTest, findSOIMarkerBase) +{ + auto start_index = 0; + auto expected_frame_index = 0; + + EXPECT_CALL(filemock, size).WillRepeatedly(Return(1024)); + // EXPECT_CALL(filemock, seek(start_index, _)).Times(AnyNumber()); + EXPECT_CALL(filemock, seek).Times(AnyNumber()); + + EXPECT_CALL(filemock, read(Matcher(_), _)).WillRepeatedly(Return(512)); + + auto actual_frame_index = corejpeg.findSOIMarker(filemock, start_index); + EXPECT_EQ(actual_frame_index, expected_frame_index); +} + +TEST_F(CoreJPEGTest, findSOIMarkerPresent) +{ + auto start_index = 0; + + std::array data {}; + std::fill(std::begin(data), std::end(data), 0); + + const auto expected_frame_index = size_t {218}; + data.at(expected_frame_index) = (jpeg::JPEG_SOI_MARKER >> 8) & 0xFF; + data.at(expected_frame_index + 1) = jpeg::JPEG_SOI_MARKER & 0xFF; + + EXPECT_CALL(filemock, size).Times(1).WillRepeatedly(Return(1024)); + EXPECT_CALL(filemock, seek(start_index, _)).Times(1); + + auto read_bytes = 512; + EXPECT_CALL(filemock, read(Matcher(_), _)) + .Times(1) + .WillRepeatedly( + DoAll(SetArrayArgument<0>(std::begin(data), std::begin(data) + read_bytes), Return(read_bytes))); + + auto actual_frame_index = corejpeg.findSOIMarker(filemock, start_index); + EXPECT_EQ(actual_frame_index, expected_frame_index); +} + +TEST_F(CoreJPEGTest, findSOIMarkerAbsent) +{ + auto start_index = 0; + + std::array data {}; + std::fill(std::begin(data), std::end(data), 0); + + const auto expected_frame_index = size_t {0}; + + EXPECT_CALL(filemock, size).Times(AnyNumber()).WillRepeatedly(Return(1024)); + EXPECT_CALL(filemock, seek).Times(AnyNumber()); + + auto read_bytes = 256; + EXPECT_CALL(filemock, read(Matcher(_), _)) + .Times(4) + .WillOnce(DoAll(SetArrayArgument<0>(std::begin(data) + 0 * read_bytes, std::begin(data) + 1 * read_bytes), + Return(read_bytes))) + .WillOnce(DoAll(SetArrayArgument<0>(std::begin(data) + 1 * read_bytes, std::begin(data) + 2 * read_bytes), + Return(read_bytes))) + .WillOnce(DoAll(SetArrayArgument<0>(std::begin(data) + 2 * read_bytes, std::begin(data) + 3 * read_bytes), + Return(read_bytes))) + .WillOnce(DoAll(SetArrayArgument<0>(std::begin(data) + 3 * read_bytes, std::begin(data) + 4 * read_bytes), + Return(read_bytes))); + + auto actual_frame_index = corejpeg.findSOIMarker(filemock, start_index); + EXPECT_EQ(actual_frame_index, expected_frame_index); +} + +TEST_F(CoreJPEGTest, findSOIMarkerPresentInLastPacket) +{ + auto start_index = 0; + + std::array data {}; + std::fill(std::begin(data), std::end(data), 0); + + const auto expected_frame_index = size_t {1001}; + data.at(expected_frame_index) = (jpeg::JPEG_SOI_MARKER >> 8) & 0xFF; + data.at(expected_frame_index + 1) = jpeg::JPEG_SOI_MARKER & 0xFF; + + EXPECT_CALL(filemock, size).Times(AnyNumber()).WillRepeatedly(Return(1024)); + EXPECT_CALL(filemock, seek).Times(AnyNumber()); + + auto read_bytes = 256; + EXPECT_CALL(filemock, read(Matcher(_), _)) + .Times(4) + .WillOnce(DoAll(SetArrayArgument<0>(std::begin(data) + 0 * read_bytes, std::begin(data) + 1 * read_bytes), + Return(read_bytes))) + .WillOnce(DoAll(SetArrayArgument<0>(std::begin(data) + 1 * read_bytes, std::begin(data) + 2 * read_bytes), + Return(read_bytes))) + .WillOnce(DoAll(SetArrayArgument<0>(std::begin(data) + 2 * read_bytes, std::begin(data) + 3 * read_bytes), + Return(read_bytes))) + .WillOnce(DoAll(SetArrayArgument<0>(std::begin(data) + 3 * read_bytes, std::begin(data) + 4 * read_bytes), + Return(read_bytes))); + + auto actual_frame_index = corejpeg.findSOIMarker(filemock, start_index); + EXPECT_EQ(actual_frame_index, expected_frame_index); +} + +TEST_F(CoreJPEGTest, findSOIMarkerPresentInLastPacketStartingIndexWithOffset) +{ + auto start_index = 512; + + std::array data {}; + std::fill(std::begin(data), std::end(data), 0); + + const auto expected_frame_index = size_t {1001}; + data.at(expected_frame_index) = (jpeg::JPEG_SOI_MARKER >> 8) & 0xFF; + data.at(expected_frame_index + 1) = jpeg::JPEG_SOI_MARKER & 0xFF; + + EXPECT_CALL(filemock, size).Times(AnyNumber()).WillRepeatedly(Return(1024)); + EXPECT_CALL(filemock, seek).Times(AnyNumber()); + + auto read_bytes = 256; + EXPECT_CALL(filemock, read(Matcher(_), _)) + .Times(2) + .WillOnce(DoAll(SetArrayArgument<0>(std::begin(data) + start_index + 0 * read_bytes, + std::begin(data) + start_index + 1 * read_bytes), + Return(read_bytes))) + .WillOnce(DoAll(SetArrayArgument<0>(std::begin(data) + start_index + 1 * read_bytes, + std::begin(data) + start_index + 2 * read_bytes), + Return(read_bytes))); + + auto actual_frame_index = corejpeg.findSOIMarker(filemock, start_index); + EXPECT_EQ(actual_frame_index, expected_frame_index); +} diff --git a/drivers/CoreVideo/tests/CoreVideo_test.cpp b/drivers/CoreVideo/tests/CoreVideo_test.cpp index 11f6da554c..e3a482892f 100644 --- a/drivers/CoreVideo/tests/CoreVideo_test.cpp +++ b/drivers/CoreVideo/tests/CoreVideo_test.cpp @@ -19,6 +19,7 @@ using namespace leka; using ::testing::_; using ::testing::InSequence; +using ::testing::Return; class CoreVideoTest : public ::testing::Test { @@ -198,3 +199,25 @@ TEST_F(CoreVideoTest, displayTextWithColor) corevideo.displayText(buff, text_length, starting_line, foreground_color, background_color); } + +TEST_F(CoreVideoTest, playVideo) +{ + const auto any_frame_index = 218; + const auto any_frame_size = 27; + + { + InSequence seq; + + EXPECT_CALL(jpegmock, getImageProperties); + EXPECT_CALL(jpegmock, findSOIMarker).WillOnce(Return(any_frame_index)); + + EXPECT_CALL(filemock, seek(any_frame_index, SEEK_SET)); + + EXPECT_CALL(jpegmock, decodeImage).WillOnce(Return(any_frame_size)); + EXPECT_CALL(dma2dmock, transferImage); + + EXPECT_CALL(jpegmock, findSOIMarker(_, any_frame_index + any_frame_size)).WillOnce(Return(0)); + } + + corevideo.playVideo(filemock); +} diff --git a/spikes/lk_lcd/main.cpp b/spikes/lk_lcd/main.cpp index f1741fe473..ff42f0dbc9 100644 --- a/spikes/lk_lcd/main.cpp +++ b/spikes/lk_lcd/main.cpp @@ -52,6 +52,10 @@ CoreVideo corevideo(hal, coresdram, coredma2d, coredsi, coreltdc, corelcd, coreg auto file = FileManagerKit::File {}; auto images = std::to_array({"/fs/images/logo.jpg", "/fs/images/robot-emotion-happy.jpg"}); +auto videos = std::to_array({ + "/fs/videos/2022_02_14-animation-face-state-happy-without-eyebrows.avi", + "/fs/videos/2022_01_17-animation-face-state-yawning-sleeping_without_eyebrows.avi", +}); extern "C" { void DMA2D_IRQHandler(void) @@ -147,6 +151,7 @@ auto main() -> int corevideo.setBrightness(0.9F); corevideo.turnOn(); + for (const auto &image_name: images) { if (file.open(image_name)) { log_info("File opened"); @@ -156,6 +161,14 @@ auto main() -> int } } + for (const auto &video_name: videos) { + if (file.open(video_name)) { + corevideo.playVideo(file); + file.close(); + rtos::ThisThread::sleep_for(2s); + } + } + corevideo.turnOff(); } } diff --git a/tests/unit/mocks/mocks/leka/CoreJPEG.h b/tests/unit/mocks/mocks/leka/CoreJPEG.h index 443ba479a5..3d9d70134c 100644 --- a/tests/unit/mocks/mocks/leka/CoreJPEG.h +++ b/tests/unit/mocks/mocks/leka/CoreJPEG.h @@ -16,7 +16,8 @@ class CoreJPEG : public interface::JPEGBase MOCK_METHOD(JPEG_HandleTypeDef &, getHandle, (), (override)); - MOCK_METHOD(void, decodeImage, (interface::File &), (override)); + MOCK_METHOD(size_t, decodeImage, (interface::File &), (override)); + MOCK_METHOD(size_t, findSOIMarker, (interface::File &, size_t), (override)); MOCK_METHOD(JPEGImageProperties, getImageProperties, (), (override)); }; diff --git a/tests/unit/mocks/mocks/leka/CoreJPEGMode.h b/tests/unit/mocks/mocks/leka/CoreJPEGMode.h index 151f0abc4f..eedc475b96 100644 --- a/tests/unit/mocks/mocks/leka/CoreJPEGMode.h +++ b/tests/unit/mocks/mocks/leka/CoreJPEGMode.h @@ -19,7 +19,7 @@ class CoreJPEGMode : public interface::JPEGMode MOCK_METHOD(void, onDataAvailableCallback, (JPEG_HandleTypeDef *, uint32_t), (override)); MOCK_METHOD(void, onDecodeCompleteCallback, (JPEG_HandleTypeDef *), (override)); - MOCK_METHOD(HAL_StatusTypeDef, decode, (JPEG_HandleTypeDef *, interface::File &), (override)); + MOCK_METHOD(size_t, decode, (JPEG_HandleTypeDef *, interface::File &), (override)); }; } // namespace leka::mock