Merge pull request #899 from OpenShot/improved-captions

Improved Caption Effect

.github/workflows/ci.yml

@@ -202,6 +202,9 @@ jobs:
 
       - name: Test libopenshot
         run: |
+          # Allow unit tests which require a display screen
+          export DISPLAY=:0.0
+          export QT_QPA_PLATFORM=offscreen
           cmake --build build --target coverage -- VERBOSE=1
 
       - name: Install libopenshot

src/Clip.cpp

@@ -164,9 +164,9 @@ Clip::Clip(std::string path) : resampler(NULL), reader(NULL), allocated_reader(N
 	std::string ext = get_file_extension(path);
 	std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower);
 
-	// Determine if common video formats
+	// Determine if common video formats (or image sequences)
 	if (ext=="avi" || ext=="mov" || ext=="mkv" ||  ext=="mpg" || ext=="mpeg" || ext=="mp3" || ext=="mp4" || ext=="mts" ||
-		ext=="ogg" || ext=="wav" || ext=="wmv" || ext=="webm" || ext=="vob")
+		ext=="ogg" || ext=="wav" || ext=="wmv" || ext=="webm" || ext=="vob" || path.find("%") != std::string::npos)
 	{
 		try
 		{
@@ -443,6 +443,11 @@ std::shared_ptr<Frame> Clip::GetFrame(std::shared_ptr<openshot::Frame> backgroun
 		// Return the frame's number so the correct keyframes are applied.
 		original_frame->number = frame_number;
 
+        // Apply waveform image (if any)
+        if (Waveform()) {
+            apply_waveform(original_frame, background_frame->GetImage());
+        }
+
 		// Apply local effects to the frame (if any)
 		apply_effects(original_frame);
 
@@ -1265,36 +1270,14 @@ bool Clip::isEqual(double a, double b)
 // Apply keyframes to the source frame (if any)
 void Clip::apply_keyframes(std::shared_ptr<Frame> frame, std::shared_ptr<QImage> background_canvas) {
     // Skip out if video was disabled or only an audio frame (no visualisation in use)
-    if (!Waveform() && !Reader()->info.has_video) {
+    if (!frame->has_image_data) {
         // Skip the rest of the image processing for performance reasons
         return;
     }
 
     // Get image from clip
     std::shared_ptr<QImage> source_image = frame->GetImage();
 
-    /* REPLACE IMAGE WITH WAVEFORM IMAGE (IF NEEDED) */
-    if (Waveform())
-    {
-        // Debug output
-        ZmqLogger::Instance()->AppendDebugMethod(
-            "Clip::get_transform (Generate Waveform Image)",
-            "frame->number", frame->number,
-            "Waveform()", Waveform(),
-            "background_canvas->width()", background_canvas->width(),
-            "background_canvas->height()", background_canvas->height());
-
-        // Get the color of the waveform
-        int red = wave_color.red.GetInt(frame->number);
-        int green = wave_color.green.GetInt(frame->number);
-        int blue = wave_color.blue.GetInt(frame->number);
-        int alpha = wave_color.alpha.GetInt(frame->number);
-
-        // Generate Waveform Dynamically (the size of the timeline)
-        source_image = frame->GetWaveform(background_canvas->width(), background_canvas->height(), red, green, blue, alpha);
-        frame->AddImage(source_image);
-    }
-
     // Get transform from clip's keyframes
     QTransform transform = get_transform(frame, background_canvas->width(), background_canvas->height());
 
@@ -1351,6 +1334,30 @@ void Clip::apply_keyframes(std::shared_ptr<Frame> frame, std::shared_ptr<QImage>
     frame->AddImage(background_canvas);
 }
 
+// Apply apply_waveform image to the source frame (if any)
+void Clip::apply_waveform(std::shared_ptr<Frame> frame, std::shared_ptr<QImage> background_canvas) {
+    // Get image from clip
+    std::shared_ptr<QImage> source_image = frame->GetImage();
+
+    // Debug output
+    ZmqLogger::Instance()->AppendDebugMethod(
+            "Clip::apply_waveform (Generate Waveform Image)",
+            "frame->number", frame->number,
+            "Waveform()", Waveform(),
+            "background_canvas->width()", background_canvas->width(),
+            "background_canvas->height()", background_canvas->height());
+
+    // Get the color of the waveform
+    int red = wave_color.red.GetInt(frame->number);
+    int green = wave_color.green.GetInt(frame->number);
+    int blue = wave_color.blue.GetInt(frame->number);
+    int alpha = wave_color.alpha.GetInt(frame->number);
+
+    // Generate Waveform Dynamically (the size of the timeline)
+    source_image = frame->GetWaveform(background_canvas->width(), background_canvas->height(), red, green, blue, alpha);
+    frame->AddImage(source_image);
+}
+
 // Apply keyframes to the source frame (if any)
 QTransform Clip::get_transform(std::shared_ptr<Frame> frame, int width, int height)
 {

src/Clip.h

@@ -129,6 +129,9 @@ namespace openshot {
         /// Apply keyframes to an openshot::Frame and use an existing QImage as a background image (if any)
         void apply_keyframes(std::shared_ptr<Frame> frame, std::shared_ptr<QImage> background_canvas);
 
+        /// Apply waveform image to an openshot::Frame and use an existing QImage as a background image (if any)
+        void apply_waveform(std::shared_ptr<Frame> frame, std::shared_ptr<QImage> background_canvas);
+
         /// Get QTransform from keyframes
         QTransform get_transform(std::shared_ptr<Frame> frame, int width, int height);
 

src/FFmpegReader.cpp

@@ -670,13 +670,20 @@ bool FFmpegReader::HasAlbumArt() {
 }
 
 void FFmpegReader::UpdateAudioInfo() {
+	// Set default audio channel layout (if needed)
+	if (AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout == 0)
+		AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout = av_get_default_channel_layout(AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels);
+
+	if (info.sample_rate > 0) {
+		// Skip init - if info struct already populated
+		return;
+	}
+
 	// Set values of FileInfo struct
 	info.has_audio = true;
 	info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
 	info.acodec = aCodecCtx->codec->name;
 	info.channels = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels;
-	if (AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout == 0)
-		AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout = av_get_default_channel_layout(AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels);
 	info.channel_layout = (ChannelLayout) AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout;
 	info.sample_rate = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->sample_rate;
 	info.audio_bit_rate = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->bit_rate;
@@ -720,6 +727,16 @@ void FFmpegReader::UpdateAudioInfo() {
 		info.video_length = info.duration * info.fps.ToDouble();
 		info.width = 720;
 		info.height = 480;
+
+		// Use timeline to set correct width & height (if any)
+		Clip *parent = (Clip *) ParentClip();
+		if (parent) {
+			if (parent->ParentTimeline()) {
+				// Set max width/height based on parent clip's timeline (if attached to a timeline)
+				info.width = parent->ParentTimeline()->preview_width;
+				info.height = parent->ParentTimeline()->preview_height;
+			}
+		}
 	}
 
 	// Fix invalid video lengths for certain types of files (MP3 for example)
@@ -737,6 +754,11 @@ void FFmpegReader::UpdateAudioInfo() {
 }
 
 void FFmpegReader::UpdateVideoInfo() {
+	if (info.vcodec.length() > 0) {
+		// Skip init - if info struct already populated
+		return;
+	}
+
 	// Set values of FileInfo struct
 	info.has_video = true;
 	info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;

src/FrameMapper.cpp

@@ -727,6 +727,9 @@ Json::Value FrameMapper::JsonValue() const {
 	// Create root json object
 	Json::Value root = ReaderBase::JsonValue(); // get parent properties
 	root["type"] = "FrameMapper";
+	if (reader) {
+		root["reader"] = reader->JsonValue();
+	}
 
 	// return JsonValue
 	return root;
@@ -741,6 +744,12 @@ void FrameMapper::SetJson(const std::string value) {
 		const Json::Value root = openshot::stringToJson(value);
 		// Set all values that match
 		SetJsonValue(root);
+
+		if (!root["reader"].isNull()) // does Json contain a reader?
+		{
+			// Placeholder to load reader
+			// TODO: need a createReader method for this and Clip JSON methods
+		}
 	}
 	catch (const std::exception& e)
 	{

src/effects/Caption.cpp

@@ -88,8 +88,8 @@ void Caption::process_regex() {
 			caption_prepared.append("\n\n");
 		}
 
-		// Parse regex and find all matches
-		QRegularExpression allPathsRegex(QStringLiteral("(\\d{2})?:*(\\d{2}):(\\d{2}).(\\d{2,3})\\s*-->\\s*(\\d{2})?:*(\\d{2}):(\\d{2}).(\\d{2,3})([\\s\\S]*?)\\n(.*?)(?=\\n\\d{2,3}|\\Z)"), QRegularExpression::MultilineOption);
+		// Parse regex and find all matches (i.e. 00:00.000 --> 00:10.000\ncaption-text)
+		QRegularExpression allPathsRegex(QStringLiteral("(\\d{2})?:*(\\d{2}):(\\d{2}).(\\d{2,3})\\s*-->\\s*(\\d{2})?:*(\\d{2}):(\\d{2}).(\\d{2,3})([\\s\\S]*?)(.*?)(?=\\d{2}.\\d{2,3}|\\Z)"), QRegularExpression::MultilineOption);
 		QRegularExpressionMatchIterator i = allPathsRegex.globalMatch(caption_prepared);
 		while (i.hasNext()) {
 			QRegularExpressionMatch match = i.next();
@@ -200,6 +200,13 @@ std::shared_ptr<openshot::Frame> Caption::GetFrame(std::shared_ptr<openshot::Fra
 
 				// Loop through words, and find word-wrap boundaries
 				QStringList words = line.split(" ");
+
+				// Wrap languages which do not use spaces
+				bool use_spaces = true;
+				if (line.length() > 20 && words.length() == 1) {
+					words = line.split("");
+					use_spaces = false;
+				}
 				int words_remaining = words.length();
 				while (words_remaining > 0) {
 					bool words_displayed = false;
@@ -215,7 +222,12 @@ std::shared_ptr<openshot::Frame> Caption::GetFrame(std::shared_ptr<openshot::Fra
 
 							// Create path and add text to it (for correct border and fill)
 							QPainterPath path1;
-							QString fitting_line = words.mid(0, word_index).join(" ");
+							QString fitting_line;
+							if (use_spaces) {
+								fitting_line = words.mid(0, word_index).join(" ");
+							} else {
+								fitting_line = words.mid(0, word_index).join("");
+							}
 							path1.addText(p, font, fitting_line);
 							text_paths.push_back(path1);
 

tests/CMakeLists.txt

@@ -24,6 +24,7 @@ set(OPENSHOT_TESTS
   AudioWaveformer
   CacheDisk
   CacheMemory
+  Caption
   Clip
   Color
   Coordinate