diff --git a/example/api2-samples/api2-decode-overlay-encode.cpp b/example/api2-samples/api2-decode-overlay-encode.cpp new file mode 100644 index 00000000..170f9b1d --- /dev/null +++ b/example/api2-samples/api2-decode-overlay-encode.cpp @@ -0,0 +1,389 @@ +#include +#include +#include +#include +#include + +#include "audioresampler.h" +#include "av.h" +#include "avutils.h" +#include "codec.h" +#include "ffmpeg.h" +#include "packet.h" +#include "videorescaler.h" + +// API2 +#include "codec.h" +#include "codeccontext.h" +#include "filters/buffersink.h" +#include "filters/buffersrc.h" +#include "filters/filtergraph.h" +#include "format.h" +#include "formatcontext.h" + +using namespace std; +using namespace av; + +int main(int argc, char **argv) +{ + if (argc < 3) + return 1; + + av::init(); + av::setFFmpegLoggingLevel(AV_LOG_DEBUG); + + string uri{argv[1]}; + string image{argv[2]}; + string out{argv[3]}; + + error_code ec; + + // + // INPUT + // + FormatContext ictx; + ssize_t videoStream = -1; + VideoDecoderContext vdec; + Stream vst; + + int count = 0; + + ictx.openInput(uri, ec); + if (ec) + { + cerr << "Can't open input\n"; + return 1; + } + + ictx.findStreamInfo(); + + for (size_t i = 0; i < ictx.streamsCount(); ++i) + { + auto st = ictx.stream(i); + if (st.mediaType() == AVMEDIA_TYPE_VIDEO) + { + videoStream = i; + vst = st; + break; + } + } + + if (vst.isNull()) + { + cerr << "Video stream not found\n"; + return 1; + } + + if (vst.isValid()) + { + vdec = VideoDecoderContext(vst); + vdec.setRefCountedFrames(true); + + vdec.open(Codec(), ec); + if (ec) + { + cerr << "Can't open codec\n"; + return 1; + } + } + + // + // STATIC IMAGE (more or less identical to INPUT) + // + FormatContext static_image_ctx; + ssize_t imageStream = -1; + VideoDecoderContext static_image_dec; + Stream image_st; + + static_image_ctx.openInput(image, ec); + if (ec) + { + cerr << "Can't open static image\n"; + return 1; + } + + static_image_ctx.findStreamInfo(); + + for (size_t i = 0; i < static_image_ctx.streamsCount(); ++i) + { + auto st = static_image_ctx.stream(i); + // An image is still a video stream in the ffmpeg world + if (st.mediaType() == AVMEDIA_TYPE_VIDEO) + { + imageStream = i; + image_st = st; + break; + } + } + + if (image_st.isNull()) + { + cerr << "Image stream not found\n"; + return 1; + } + + if (image_st.isValid()) + { + static_image_dec = VideoDecoderContext(image_st); + static_image_dec.setRefCountedFrames(true); + + static_image_dec.open(Codec(), ec); + if (ec) + { + cerr << "Can't open codec\n"; + return 1; + } + } + + // + // OUTPUT + // + OutputFormat ofrmt; + FormatContext octx; + + ofrmt.setFormat(string(), out); + octx.setFormat(ofrmt); + + Codec ocodec = findEncodingCodec(ofrmt); + VideoEncoderContext encoder{ocodec}; + + // Settings + encoder.setWidth(vdec.width()); + encoder.setHeight(vdec.height()); + if (vdec.pixelFormat() > -1) + encoder.setPixelFormat(vdec.pixelFormat()); + encoder.setTimeBase(Rational{1, 1000}); + encoder.setBitRate(vdec.bitRate()); + + encoder.open(Codec(), ec); + if (ec) + { + cerr << "Can't opent encodec\n"; + return 1; + } + + Stream ost = octx.addStream(encoder); + ost.setFrameRate(vst.frameRate()); + + octx.openOutput(out, ec); + if (ec) + { + cerr << "Can't open output\n"; + return 1; + } + + octx.dump(); + octx.writeHeader(); + octx.flush(); + + // FILTER (OVERLAY) + FilterGraph filter_graph; + + // Construct the filter descriptor + stringstream filter_desc; + + // * for the video input + filter_desc << "buffer@in=video_size=" << vdec.width() << "x" << vdec.height() << ":pix_fmt=" << vdec.pixelFormat() + << ":time_base=" << encoder.timeBase() << ":pixel_aspect=" << vdec.sampleAspectRatio() << " [in]; "; + // * for the fixed image input + filter_desc << "buffer@image=video_size=" << static_image_dec.width() << "x" << static_image_dec.height() + << ":pix_fmt=" << static_image_dec.pixelFormat() << ":time_base=" << encoder.timeBase() + << ":pixel_aspect=" << static_image_dec.sampleAspectRatio() << " [image_unscaled]; "; + filter_desc << "[image_unscaled] scale=" << vdec.width() / 4 << "x" << vdec.height() / 4 << " [image]; "; + // * for the output + filter_desc << "[in][image] overlay=x=50:y=50:repeatlast=1 [out]; "; + filter_desc << "[out] buffersink@out"; + + // Setup the filter chain + clog << "Filter descriptor: " << filter_desc.str() << endl; + filter_graph.parse(filter_desc.str(), ec); + if (ec) + { + clog << "Error parsing filter chain: " << ec << ", " << ec.message() << endl; + return 1; + } + filter_graph.config(ec); + if (ec) + { + clog << "Error configuring filter chain: " << ec << ", " << ec.message() << endl; + return 1; + } + + // Setup the entry/exit points + BufferSrcFilterContext buffer_video_src{filter_graph.filter("buffer@in")}; + BufferSrcFilterContext buffer_image_src{filter_graph.filter("buffer@image")}; + BufferSinkFilterContext buffer_sink{filter_graph.filter("buffersink@out")}; + + // Load the static image into buffer_image_src, it won't change + { + auto image_pkt = static_image_ctx.readPacket(ec); + if (ec) + { + clog << "Image reading error: " << ec << ", " << ec.message() << endl; + return 1; + } + auto image_frame = static_image_dec.decode(image_pkt, ec); + if (ec) + { + clog << "Image decoding error: " << ec << ", " << ec.message() << endl; + return 1; + } + image_frame.setTimeBase(encoder.timeBase()); + image_frame.setStreamIndex(0); + image_frame.setPictureType(); + clog << "Frame for overlaying: pts=" << image_frame.pts() << " / " << image_frame.pts().seconds() << " / " + << image_frame.timeBase() << ", " << image_frame.width() << "x" << image_frame.height() + << ", size=" << image_frame.size() << ", ref=" << image_frame.isReferenced() << ":" + << image_frame.refCount() << " / type: " << image_frame.pictureType() << endl; + buffer_image_src.addVideoFrame(image_frame, ec); + if (ec) + { + clog << "Image processing error: " << ec << ", " << ec.message() << endl; + return 1; + } + // Very important! Signal that there are no more frames after this one! + // ffmpeg expects a full video, this will allow the repeatlast=1 in + // overlay to kick in + buffer_image_src.writeVideoFrame(VideoFrame::null()); + } + + // + // PROCESS + // + while (true) + { + + // READING + Packet pkt = ictx.readPacket(ec); + if (ec) + { + clog << "Packet reading error: " << ec << ", " << ec.message() << endl; + break; + } + + bool flushDecoder = false; + // !EOF + if (pkt) + { + if (pkt.streamIndex() != videoStream) + { + continue; + } + + clog << "Read packet: pts=" << pkt.pts() << ", dts=" << pkt.dts() << " / " << pkt.pts().seconds() << " / " + << pkt.timeBase() << " / st: " << pkt.streamIndex() << endl; + } + else + { + flushDecoder = true; + } + + do + { + // DECODING + auto frame = vdec.decode(pkt, ec); + + count++; + // if (count > 200) + // break; + + bool flushEncoder = false; + + if (ec) + { + cerr << "Decoding error: " << ec << endl; + return 1; + } + else if (!frame) + { + if (flushDecoder) + { + flushEncoder = true; + } + } + + if (frame) + { + clog << "Frame from decoder: pts=" << frame.pts() << " / " << frame.pts().seconds() << " / " + << frame.timeBase() << ", " << frame.width() << "x" << frame.height() << ", size=" << frame.size() + << ", ref=" << frame.isReferenced() << ":" << frame.refCount() + << " / type: " << frame.pictureType() << endl; + + frame.setTimeBase(encoder.timeBase()); + frame.setStreamIndex(0); + frame.setPictureType(); + + // Push into the filter + buffer_video_src.addVideoFrame(frame, ec); + if (ec) + { + clog << "Filter push error: " << ec << ", " << ec.message() << endl; + return 1; + } + } + + // Pull from the filter until there are no more incoming frames + VideoFrame filtered; + while (buffer_sink.getVideoFrame(filtered, ec)) + { + // Change timebase + filtered.setTimeBase(encoder.timeBase()); + filtered.setStreamIndex(0); + filtered.setPictureType(); + clog << "Frame from filter: pts=" << filtered.pts() << " / " << filtered.pts().seconds() << " / " + << filtered.timeBase() << ", " << filtered.width() << "x" << filtered.height() + << ", size=" << filtered.size() << ", ref=" << filtered.isReferenced() << ":" + << filtered.refCount() << " / type: " << filtered.pictureType() << endl; + + if (filtered || flushEncoder) + { + do + { + // Encode + Packet opkt = filtered ? encoder.encode(filtered, ec) : encoder.encode(ec); + if (ec) + { + cerr << "Encoding error: " << ec << endl; + return 1; + } + else if (!opkt) + { + // cerr << "Empty packet\n"; + // continue; + break; + } + + // Only one output stream + opkt.setStreamIndex(0); + + clog << "Write packet: pts=" << opkt.pts() << ", dts=" << opkt.dts() << " / " + << opkt.pts().seconds() << " / " << opkt.timeBase() << " / st: " << opkt.streamIndex() + << endl; + + octx.writePacket(opkt, ec); + if (ec) + { + cerr << "Error write packet: " << ec << ", " << ec.message() << endl; + return 1; + } + } while (flushEncoder); + } + } + if (ec && ec.value() != -EAGAIN) + { + cerr << "Error reading frame from the filter chain: " << ec << ", " << ec.message() << endl; + return 1; + } + + if (flushEncoder) + break; + + } while (flushDecoder); + + if (flushDecoder) + break; + } + + clog << "done" << endl; + octx.writeTrailer(); +} diff --git a/src/filters/buffersink.cpp b/src/filters/buffersink.cpp index d803764e..dee2843b 100644 --- a/src/filters/buffersink.cpp +++ b/src/filters/buffersink.cpp @@ -36,7 +36,9 @@ bool BufferSinkFilterContext::getVideoFrame(VideoFrame &frame, int flags, Option return false; } - return getFrame(frame.raw(), flags, ec); + bool sts = getFrame(frame.raw(), flags, ec); + frame.setComplete(sts); + return sts; } bool BufferSinkFilterContext::getVideoFrame(VideoFrame &frame, OptionalErrorCode ec) @@ -50,7 +52,9 @@ bool BufferSinkFilterContext::getAudioFrame(AudioSamples &samples, int flags, Op throws_if(ec, Errors::IncorrectBufferSinkMediaType); return false; } - return getFrame(samples.raw(), flags, ec); + bool sts = getFrame(samples.raw(), flags, ec); + samples.setComplete(sts); + return sts; } bool BufferSinkFilterContext::getAudioFrame(AudioSamples &samples, OptionalErrorCode ec) diff --git a/src/filters/filtergraph.cpp b/src/filters/filtergraph.cpp index 591780f2..37e5327a 100644 --- a/src/filters/filtergraph.cpp +++ b/src/filters/filtergraph.cpp @@ -199,6 +199,29 @@ void FilterGraph::parse(const string &graphDescription, } } +void FilterGraph::parse(const string &graphDescription, OptionalErrorCode ec) { + clear_if(ec); + + if (!m_raw) { + throws_if(ec, Errors::Unallocated); + return; + } + + AVFilterInOut *inputs; + AVFilterInOut *outputs; + + if (graphDescription.empty()) { + fflog(AV_LOG_ERROR, "Empty graph description"); + throws_if(ec, Errors::FilterGraphDescriptionEmpty); + return; + } else { + int sts = avfilter_graph_parse2(m_raw, graphDescription.c_str(), &inputs, &outputs); + if (sts < 0) { + throws_if(ec, sts, ffmpeg_category()); + return; + } + } +} void FilterGraph::config(OptionalErrorCode ec) { diff --git a/src/filters/filtergraph.h b/src/filters/filtergraph.h index 53e58a5e..497980f1 100644 --- a/src/filters/filtergraph.h +++ b/src/filters/filtergraph.h @@ -47,6 +47,8 @@ class FilterGraph : FFWrapperPtr FilterContext &sinkFilterCtx, OptionalErrorCode ec = throws()); + void parse(const std::string &graphDescription, OptionalErrorCode ec = throws()); + void config(OptionalErrorCode ec = throws()); std::string dump(bool doPrint = true, const std::string& options = std::string());