for #738, parse avcc/sps/pps/asc from mp4.

trunk/research/librtmp/srs_ingest_mp4.c

@@ -180,35 +180,34 @@ int do_proxy(srs_mp4_t mp4, srs_rtmp_t ortmp, int64_t re, int32_t* pstarttime, u
 {
     int ret = 0;
 
-    // packet data
-    char type;
-    int size;
-    char* data = NULL;
-
     srs_human_trace("start ingest mp4 to RTMP stream");
     for (;;) {
-#if 0
-        // tag header
-        if ((ret = srs_flv_read_tag_header(flv, &type, &size, ptimestamp)) != 0) {
-            if (srs_flv_is_eof(ret)) {
-                srs_human_trace("parse completed.");
-                return 0;
-            }
-            srs_human_trace("flv get packet failed. ret=%d", ret);
-            return ret;
-        }
+        // packet data
+        char type;
+        int32_t size;
+        char* data = NULL;
 
-        if (size <= 0) {
-            srs_human_trace("invalid size=%d", size);
-            break;
-        }
-
-        // TODO: FIXME: mem leak when error.
-        data = (char*)malloc(size);
-        if ((ret = srs_flv_read_tag_data(flv, data, size)) != 0) {
-            return ret;
+        // Read a mp4 sample and convert to flv tag.
+        if (1) {
+            srs_mp4_sample_t sample;
+            if ((ret = srs_mp4_read_sample(mp4, &sample)) != 0) {
+                if (srs_mp4_is_eof(ret)) {
+                    srs_human_trace("parse completed.");
+                    return 0;
+                }
+                srs_human_trace("mp4 get sample failed. ret=%d", ret);
+                return ret;
+            }
+
+            size = srs_mp4_sizeof(mp4, &sample);
+            data = (char*)malloc(size);
+
+            if ((ret = srs_mp4_to_flv_tag(mp4, &sample, &type, ptimestamp, data, size)) != 0) {
+                return ret;
+            }
+
+            srs_mp4_free_sample(&sample);
         }
-#endif
         uint32_t timestamp = *ptimestamp;
 
         if ((ret = srs_human_print_rtmp_packet(type, timestamp, data, size)) != 0) {

trunk/src/kernel/srs_kernel_codec.cpp

@@ -162,6 +162,17 @@ string srs_codec_avc_level2str(SrsAvcLevel level)
     }
 }
 
+string srs_codec_audio_samplerate2str(SrsCodecAudioSampleRate v)
+{
+    switch (v) {
+        case SrsCodecAudioSampleRate5512: return "5512";
+        case SrsCodecAudioSampleRate11025: return "11025";
+        case SrsCodecAudioSampleRate22050: return "22050";
+        case SrsCodecAudioSampleRate44100: return "44100";
+        default: return "Other";
+    }
+}
+
 /**
 * the public data, event HLS disable, others can use it.
 */
@@ -288,6 +299,24 @@ bool SrsFlvCodec::video_is_acceptable(char* data, int size)
     return true;
 }
 
+string srs_codec_audio_samplesize2str(SrsCodecAudioSampleSize v)
+{
+    switch (v) {
+        case SrsCodecAudioSampleSize16bit: return "16bits";
+        case SrsCodecAudioSampleSize8bit: return "8bits";
+        default: return "Other";
+    }
+}
+
+string srs_codec_audio_channels2str(SrsCodecAudioSoundType v)
+{
+    switch (v) {
+        case SrsCodecAudioSoundTypeStereo: return "Stereo";
+        case SrsCodecAudioSoundTypeMono: return "Mono";
+        default: return "Other";
+    }
+}
+
 string srs_codec_avc_nalu2str(SrsAvcNaluType nalu_type)
 {
     switch (nalu_type) {

trunk/src/kernel/srs_kernel_codec.hpp

@@ -42,6 +42,7 @@ enum SrsCodecAudioType
 {
     // set to the max value to reserved, for array map.
     SrsCodecAudioTypeReserved = 2,
+    SrsCodecAudioTypeForbidden = 2,
 
     SrsCodecAudioTypeSequenceHeader = 0,
     SrsCodecAudioTypeRawData = 1,
@@ -59,6 +60,7 @@ enum SrsCodecVideoAVCFrame
 {
     // set to the zero to reserved, for array map.
     SrsCodecVideoAVCFrameReserved = 0,
+    SrsCodecVideoAVCFrameForbidden = 0,
     SrsCodecVideoAVCFrameReserved1 = 6,
 
     SrsCodecVideoAVCFrameKeyFrame = 1,
@@ -78,6 +80,7 @@ enum SrsCodecVideoAVCType
 {
     // set to the max value to reserved, for array map.
     SrsCodecVideoAVCTypeReserved = 3,
+    SrsCodecVideoAVCTypeForbidden = 3,
 
     SrsCodecVideoAVCTypeSequenceHeader = 0,
     SrsCodecVideoAVCTypeNALU = 1,
@@ -170,12 +173,14 @@ enum SrsCodecAudioSampleRate
 {
     // set to the max value to reserved, for array map.
     SrsCodecAudioSampleRateReserved = 4,
+    SrsCodecAudioSampleRateForbidden = 4,
 
     SrsCodecAudioSampleRate5512 = 0,
     SrsCodecAudioSampleRate11025 = 1,
     SrsCodecAudioSampleRate22050 = 2,
     SrsCodecAudioSampleRate44100 = 3,
 };
+std::string srs_codec_audio_samplerate2str(SrsCodecAudioSampleRate v);
 
 /**
 * E.4.1 FLV Tag, page 75
@@ -263,10 +268,12 @@ enum SrsCodecAudioSampleSize
 {
     // set to the max value to reserved, for array map.
     SrsCodecAudioSampleSizeReserved = 2,
+    SrsCodecAudioSampleSizeForbidden = 2,
 
     SrsCodecAudioSampleSize8bit = 0,
     SrsCodecAudioSampleSize16bit = 1,
 };
+std::string srs_codec_audio_samplesize2str(SrsCodecAudioSampleSize v);
 
 /**
 * the FLV/RTMP supported audio sound type/channel.
@@ -277,11 +284,13 @@ enum SrsCodecAudioSampleSize
 enum SrsCodecAudioSoundType
 {
     // set to the max value to reserved, for array map.
-    SrsCodecAudioSoundTypeReserved = 2, 
+    SrsCodecAudioSoundTypeReserved = 2,
+    SrsCodecAudioSoundTypeForbidden = 2,
 
     SrsCodecAudioSoundTypeMono = 0,
     SrsCodecAudioSoundTypeStereo = 1,
 };
+std::string srs_codec_audio_channels2str(SrsCodecAudioSoundType v);
 
 /**
  * Table 7-1 - NAL unit type codes, syntax element categories, and NAL unit type classes

trunk/src/kernel/srs_kernel_error.hpp

@@ -250,6 +250,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #define ERROR_MP4_BOX_ILLEGAL_BRAND         3074
 #define ERROR_MP4_ESDS_SL_Config            3075
 #define ERROR_MP4_ILLEGAL_MOOV              3076
+#define ERROR_MP4_ILLEGAL_HANDLER           3077
 
 ///////////////////////////////////////////////////////
 // HTTP/StreamCaster/KAFKA protocol error.

trunk/src/kernel/srs_kernel_mp4.cpp

@@ -2963,12 +2963,22 @@ SrsMp4Decoder::SrsMp4Decoder()
     brand = SrsMp4BoxBrandForbidden;
     buf = new char[SRS_MP4_BUF_SIZE];
     stream = new SrsSimpleStream();
+    vcodec = SrsCodecVideoForbidden;
+    acodec = SrsCodecAudioForbidden;
+    nb_asc = nb_avcc = 0;
+    pasc = pavcc = NULL;
+    asc_written = avcc_written = false;
+    sample_rate = SrsCodecAudioSampleRateForbidden;
+    sound_bits = SrsCodecAudioSampleSizeForbidden;
+    channels = SrsCodecAudioSoundTypeForbidden;
 }
 
 SrsMp4Decoder::~SrsMp4Decoder()
 {
     srs_freepa(buf);
     srs_freep(stream);
+    srs_freepa(pasc);
+    srs_freepa(pavcc);
 }
 
 int SrsMp4Decoder::initialize(ISrsReadSeeker* rs)
@@ -2978,7 +2988,7 @@ int SrsMp4Decoder::initialize(ISrsReadSeeker* rs)
     srs_assert(rs);
     rsio = rs;
 
-    // For mdat before moov, we must reset the io.
+    // For mdat before moov, we must reset the offset to the mdat.
     off_t offset = -1;
 
     while (true) {
@@ -3016,14 +3026,52 @@ int SrsMp4Decoder::initialize(ISrsReadSeeker* rs)
         return ret;
     }
 
-    // Reset the io to the start to reparse the general MP4.
+    // Set the offset to the mdat.
     if (offset >= 0) {
         return rsio->lseek(offset, SEEK_SET, NULL);
     }
 
     return ret;
 }
 
+int SrsMp4Decoder::read_sample(SrsMp4HandlerType* pht,
+    uint16_t* pft, uint16_t* pct, uint32_t* pdts, uint32_t* ppts, uint8_t** psample, uint32_t* pnb_sample)
+{
+    int ret = ERROR_SUCCESS;
+
+    if (!avcc_written && nb_avcc) {
+        avcc_written = true;
+        *pdts = *ppts = 0;
+        *pht = SrsMp4HandlerTypeVIDE;
+
+        uint32_t nb_sample = *pnb_sample = nb_avcc;
+        uint8_t* sample = *psample = new uint8_t[nb_sample];
+        memcpy(sample, pavcc, nb_sample);
+
+        *pft = SrsCodecVideoAVCFrameKeyFrame;
+        *pct = SrsCodecVideoAVCTypeSequenceHeader;
+
+        return ret;
+    }
+
+    if (!asc_written && nb_asc) {
+        asc_written = true;
+        *pdts = *ppts = 0;
+        *pht = SrsMp4HandlerTypeSOUN;
+
+        uint32_t nb_sample = *pnb_sample = nb_asc;
+        uint8_t* sample = *psample = new uint8_t[nb_sample];
+        memcpy(sample, pasc, nb_sample);
+
+        *pft = 0x00;
+        *pct = SrsCodecAudioTypeSequenceHeader;
+
+        return ret;
+    }
+
+    return ret;
+}
+
 int SrsMp4Decoder::parse_ftyp(SrsMp4FileTypeBox* ftyp)
 {
     int ret = ERROR_SUCCESS;
@@ -3069,6 +3117,32 @@ int SrsMp4Decoder::parse_moov(SrsMp4MovieBox* moov)
         return ret;
     }
 
+    SrsMp4AudioSampleEntry* mp4a = soun? soun->mp4a():NULL;
+    if (mp4a) {
+        uint32_t sr = mp4a->samplerate>>16;
+        if (sr >= 44100) {
+            sample_rate = SrsCodecAudioSampleRate44100;
+        } else if (sr >= 22050) {
+            sample_rate = SrsCodecAudioSampleRate22050;
+        } else if (sr >= 11025) {
+            sample_rate = SrsCodecAudioSampleRate11025;
+        } else {
+            sample_rate = SrsCodecAudioSampleRate5512;
+        }
+
+        if (mp4a->samplesize == 16) {
+            sound_bits = SrsCodecAudioSampleSize16bit;
+        } else {
+            sound_bits = SrsCodecAudioSampleSize8bit;
+        }
+
+        if (mp4a->channelcount == 2) {
+            channels = SrsCodecAudioSoundTypeStereo;
+        } else {
+            channels = SrsCodecAudioSoundTypeMono;
+        }
+    }
+
     SrsMp4AvccBox* avcc = vide? vide->avcc():NULL;
     SrsMp4DecoderSpecificInfo* asc = soun? soun->asc():NULL;
     if (vide && !avcc) {
@@ -3082,16 +3156,33 @@ int SrsMp4Decoder::parse_moov(SrsMp4MovieBox* moov)
         return ret;
     }
 
+    vcodec = vide?vide->vide_codec():SrsCodecVideoForbidden;
+    acodec = soun?soun->soun_codec():SrsCodecAudioForbidden;
+
+    if (avcc && avcc->nb_config) {
+        nb_avcc = avcc->nb_config;
+        pavcc = new uint8_t[nb_avcc];
+        memcpy(pavcc, avcc->avc_config, nb_avcc);
+    }
+    if (asc && asc->nb_asc) {
+        nb_asc = asc->nb_asc;
+        pasc = new uint8_t[nb_asc];
+        memcpy(pasc, asc->asc, nb_asc);
+    }
+
     stringstream ss;
     ss << "dur=" << mvhd->duration() << "ms";
     // video codec.
     ss << ", vide=" << moov->nb_vide_tracks() << "("
-        << srs_codec_video2str(vide?vide->vide_codec():SrsCodecVideoForbidden)
-        << "," << (avcc? avcc->nb_config:0) << "BSH" << ")";
+        << srs_codec_video2str(vcodec) << "," << nb_avcc << "BSH"
+        << ")";
     // audio codec.
     ss << ", soun=" << moov->nb_soun_tracks() << "("
-        << srs_codec_audio2str(soun?soun->soun_codec():SrsCodecAudioForbidden)
-        << "," << (asc? asc->nb_asc:0) << "BSH" << ")";
+        << srs_codec_audio2str(acodec) << "," << nb_asc << "BSH"
+        << "," << srs_codec_audio_channels2str(channels)
+        << "," << srs_codec_audio_samplesize2str(sound_bits)
+        << "," << srs_codec_audio_samplerate2str(sample_rate)
+        << ")";
 
     srs_trace("MP4 moov %s", ss.str().c_str());
 

trunk/src/kernel/srs_kernel_mp4.hpp

@@ -400,6 +400,7 @@ class SrsMp4TrackBox : public SrsMp4Box
     virtual SrsMp4SampleTableBox* stbl();
     // Get the sample description box
     virtual SrsMp4SampleDescriptionBox* stsd();
+public:
     // For H.264/AVC, get the avc1 box.
     virtual SrsMp4VisualSampleEntry* avc1();
     // For AAC, get the mp4a box.
@@ -1344,6 +1345,32 @@ class SrsMp4Decoder
 private:
     // The major brand of decoder, parse from ftyp.
     SrsMp4BoxBrand brand;
+public:
+    // The video codec of first track, generally there is zero or one track.
+    // Forbidden if no video stream.
+    SrsCodecVideo vcodec;
+private:
+    // For H.264/AVC, the avcc contains the sps/pps.
+    int nb_avcc;
+    uint8_t* pavcc;
+    // Whether avcc is written to reader.
+    bool avcc_written;
+public:
+    // The audio codec of first track, generally there is zero or one track.
+    // Forbidden if no audio stream.
+    SrsCodecAudio acodec;
+    // The audio sample rate.
+    SrsCodecAudioSampleRate sample_rate;
+    // The audio sound bits.
+    SrsCodecAudioSampleSize sound_bits;
+    // The audio sound type.
+    SrsCodecAudioSoundType channels;
+private:
+    // For AAC, the asc in esds box.
+    int nb_asc;
+    uint8_t* pasc;
+    // Whether asc is written to reader.
+    bool asc_written;
 private:
     // Underlayer reader and seeker.
     // @remark The demuxer must use seeker for general MP4 to seek the moov.
@@ -1363,6 +1390,20 @@ class SrsMp4Decoder
      *      the decoder just read data from the reader.
      */
     virtual int initialize(ISrsReadSeeker* rs);
+    /**
+     * Read a sample from mp4.
+     * @param pht The sample type, audio/soun or video/vide.
+     * @param pft, The frame type. For video, it's SrsCodecVideoAVCFrame.
+     * @param pct, The codec type. For video, it's SrsCodecVideoAVCType. For audio, it's SrsCodecAudioType.
+     * @param pdts The output dts in milliseconds.
+     * @param ppts The output pts in milliseconds.
+     * @param pnb_sample The output size of payload.
+     * @param psample The output payload, user must free it.
+     * @remark The decoder will generate the first two audio/video sequence header.
+     */
+    virtual int read_sample(SrsMp4HandlerType* pht, uint16_t* pft, uint16_t* pct,
+        uint32_t* pdts, uint32_t* ppts, uint8_t** psample, uint32_t* pnb_sample
+    );
 private:
     virtual int parse_ftyp(SrsMp4FileTypeBox* ftyp);
     virtual int parse_moov(SrsMp4MovieBox* moov);