From e61e6c9f36c84e6fac7f115ddf815d9211d35b17 Mon Sep 17 00:00:00 2001 From: Matt Leotta Date: Mon, 1 Jun 2020 10:34:46 -0400 Subject: [PATCH 1/4] Implemented a faster alternative to counting number of frames in FFMPEG. Instead of advancing through the entire video and counting every frame, the new algorithm seeks to the last keyframe it can find and then advances to the end to discover the frame number of the last frame. With this change the number of frames is accessible in constant time rather than linear in the number of frames of video. --- arrows/ffmpeg/ffmpeg_video_input.cxx | 83 +++++++++++++++++++++++++++- 1 file changed, 81 insertions(+), 2 deletions(-) diff --git a/arrows/ffmpeg/ffmpeg_video_input.cxx b/arrows/ffmpeg/ffmpeg_video_input.cxx index bbf0e63124..a1ecb6842c 100644 --- a/arrows/ffmpeg/ffmpeg_video_input.cxx +++ b/arrows/ffmpeg/ffmpeg_video_input.cxx @@ -85,7 +85,8 @@ class ffmpeg_video_input::priv frame_advanced(0), end_of_video(true), number_of_frames(0), - have_loop_vars(false) + have_loop_vars(false), + estimated_num_frames(false) { f_packet.data = nullptr; } @@ -144,6 +145,7 @@ class ffmpeg_video_input::priv bool end_of_video; size_t number_of_frames; bool have_loop_vars; + bool estimated_num_frames; // ================================================================== /* @@ -669,6 +671,82 @@ class ffmpeg_video_input::priv } } + + // ================================================================== + /* + * @brief Seek to the end of the video to estimate number of frames + */ + void estimate_num_frames() + { + // is stream open? + if (!this->is_opened()) + { + VITAL_THROW(vital::file_not_read_exception, video_path, "Video not open"); + } + + if (!estimated_num_frames && !have_loop_vars) + { + std::lock_guard< std::mutex > lock(open_mutex); + + auto initial_frame_number = this->frame_number(); + + if (!frame_advanced && !end_of_video) + { + initial_frame_number = 0; + } + + bool advance_successful = false; + // Seek to as close as possibly to the end of the video + int64_t frame_ts = this->f_video_stream->duration + this->f_start_time; + do + { + auto seek_rslt = av_seek_frame(this->f_format_context, + this->f_video_index, + frame_ts, + AVSEEK_FLAG_BACKWARD); + avcodec_flush_buffers(this->f_video_encoding); + + if (seek_rslt < 0) + { + break; + } + + advance_successful = this->advance(); + + // Continue to make seek request further back until we land at a valid frame + frame_ts -= this->f_backstep_size * this->stream_time_base_to_frame(); + } while (!advance_successful); + + LOG_DEBUG(this->logger, "Seeked to near end frame: " << this->frame_number()); + + // Step through the end of the video to find the last valid frame number + do + { + number_of_frames = this->frame_number(); + } + while (this->advance()); + // The number of frames is one greater than the last valid frame number + ++number_of_frames; + + LOG_DEBUG(this->logger, "Found " << number_of_frames << " video frames"); + + // Set this flag so we do not have a count frames next time + estimated_num_frames = true; + + // Return the video to its state before seeking to the end + if (initial_frame_number == 0) + { + // Close and reopen to reset + this->close(); + this->open(video_path); + } + else + { + this->seek(initial_frame_number); + } + } + } + }; // end of internal class. // static open interlocking mutex @@ -784,6 +862,7 @@ ::close() d->end_of_video = true; d->number_of_frames = 0; d->have_loop_vars = false; + d->estimated_num_frames = false; d->metadata.clear(); } @@ -1008,7 +1087,7 @@ size_t ffmpeg_video_input ::num_frames() const { - d->process_loop_dependencies(); + d->estimate_num_frames(); return d->number_of_frames; } From 04f546a892a2fe5609f8173cd56297b82fef40bc Mon Sep 17 00:00:00 2001 From: Matt Leotta Date: Mon, 1 Jun 2020 10:45:46 -0400 Subject: [PATCH 2/4] Renamed "loop dependencies" to be specific to metadata collection --- arrows/ffmpeg/ffmpeg_video_input.cxx | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/arrows/ffmpeg/ffmpeg_video_input.cxx b/arrows/ffmpeg/ffmpeg_video_input.cxx index a1ecb6842c..6e7566d059 100644 --- a/arrows/ffmpeg/ffmpeg_video_input.cxx +++ b/arrows/ffmpeg/ffmpeg_video_input.cxx @@ -85,7 +85,7 @@ class ffmpeg_video_input::priv frame_advanced(0), end_of_video(true), number_of_frames(0), - have_loop_vars(false), + collected_all_metadata(false), estimated_num_frames(false) { f_packet.data = nullptr; @@ -144,7 +144,7 @@ class ffmpeg_video_input::priv int frame_advanced; // This is a boolean check value really bool end_of_video; size_t number_of_frames; - bool have_loop_vars; + bool collected_all_metadata; bool estimated_num_frames; // ================================================================== @@ -614,11 +614,9 @@ class ffmpeg_video_input::priv // ================================================================== /* - * @brief Loop over all frames to collect metadata and exact frame count - * - * @return \b Current frame number. + * @brief Loop over all frames to collect metadata */ - void process_loop_dependencies() + void collect_all_metadata() { // is stream open? if ( ! this->is_opened() ) @@ -626,7 +624,7 @@ class ffmpeg_video_input::priv VITAL_THROW( vital::file_not_read_exception, video_path, "Video not open" ); } - if ( !have_loop_vars ) + if ( !collected_all_metadata ) { std::lock_guard< std::mutex > lock( open_mutex ); @@ -667,7 +665,7 @@ class ffmpeg_video_input::priv std::make_pair( this->frame_number(), this->current_metadata() ) ); } - have_loop_vars = true; + collected_all_metadata = true; } } @@ -684,7 +682,7 @@ class ffmpeg_video_input::priv VITAL_THROW(vital::file_not_read_exception, video_path, "Video not open"); } - if (!estimated_num_frames && !have_loop_vars) + if (!estimated_num_frames && !collected_all_metadata) { std::lock_guard< std::mutex > lock(open_mutex); @@ -861,7 +859,7 @@ ::close() d->frame_advanced = 0; d->end_of_video = true; d->number_of_frames = 0; - d->have_loop_vars = false; + d->collected_all_metadata = false; d->estimated_num_frames = false; d->metadata.clear(); } @@ -1050,7 +1048,7 @@ kwiver::vital::metadata_map_sptr ffmpeg_video_input ::metadata_map() { - d->process_loop_dependencies(); + d->collect_all_metadata(); return std::make_shared(d->metadata_map); } From 5ae143131265cea1c528e7dc7dca091187623dda Mon Sep 17 00:00:00 2001 From: Matt Leotta Date: Mon, 1 Jun 2020 10:52:11 -0400 Subject: [PATCH 3/4] separate frame counting from metadata collection --- arrows/ffmpeg/ffmpeg_video_input.cxx | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/arrows/ffmpeg/ffmpeg_video_input.cxx b/arrows/ffmpeg/ffmpeg_video_input.cxx index 6e7566d059..2af12d8dbc 100644 --- a/arrows/ffmpeg/ffmpeg_video_input.cxx +++ b/arrows/ffmpeg/ffmpeg_video_input.cxx @@ -638,7 +638,6 @@ class ffmpeg_video_input::priv // Add metadata for current frame if ( frame_advanced ) { - number_of_frames++; this->metadata_map.insert( std::make_pair( this->frame_number(), this->current_metadata() ) ); } @@ -646,7 +645,6 @@ class ffmpeg_video_input::priv // Advance video stream to end while ( this->advance() ) { - number_of_frames++; this->metadata_map.insert( std::make_pair( this->frame_number(), this->current_metadata() ) ); } @@ -659,7 +657,6 @@ class ffmpeg_video_input::priv unsigned int frame_num = 0; while ( frame_num < initial_frame_number && this->advance() ) { - number_of_frames++; ++frame_num; this->metadata_map.insert( std::make_pair( this->frame_number(), this->current_metadata() ) ); @@ -682,7 +679,7 @@ class ffmpeg_video_input::priv VITAL_THROW(vital::file_not_read_exception, video_path, "Video not open"); } - if (!estimated_num_frames && !collected_all_metadata) + if ( !estimated_num_frames ) { std::lock_guard< std::mutex > lock(open_mutex); From d07fc28b868f2bd2001186e5cbede6c393e2eba4 Mon Sep 17 00:00:00 2001 From: Matt Leotta Date: Mon, 1 Jun 2020 13:02:32 -0400 Subject: [PATCH 4/4] added release note --- doc/release-notes/1.5.0.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/release-notes/1.5.0.txt b/doc/release-notes/1.5.0.txt index 01feb68789..b97096c689 100644 --- a/doc/release-notes/1.5.0.txt +++ b/doc/release-notes/1.5.0.txt @@ -207,6 +207,11 @@ Arrows: CUDA * CUDA depth map integration now uses optional weight maps to down-weight the contribution of some rays. +Arrows: FFMPEG + + * Optimized the num_frames() call to produce a count of the number of frames + in constant time without scanning the entire video and counting all frames. + Arrows: Serialization * Updated serialization algorithms to use PLUGIN_INFO macro and added