H.264 encoding: frame ordering issues

Philip_Starke — Wed, 19 Sep 2012 13:24:05 GMT

I have implemented a video export function writing H.264 encoded video to an MPEG 4 container using the UMC classes from IPP 6.1.

The resulting video (example attached) plays fine in VLC, but in QuickTime player (on Mac) the frames jump around as though they have been re ordered.

Here is the relevant code. The parameters I use for the codec came from the umc_video_enc_con sample.

Any suggestions?

void CMovieExport::ExportFrames(const QValueVector<AnimData> &anims, const QString &movieName, CProgressReporter &pr) const
{
    QString curFrameName;
    CPtr<CCanvasU8> curFrameCanvas;

   for (unsigned a = 0, an = anims.size(); a != an; ++a)
   {
       if (anims.size())
       {
           curFrameName = anims [0].first;
           break;
       }
   }

    if (curFrameName.isEmpty())
        throw "No files";

    curFrameCanvas.Reset(CCanvasLoader::Instance()->LoadU8(curFrameName));

    if (!curFrameCanvas)
        throw "Frame Failed";

    unsigned int width = curFrameCanvas->Width();
    unsigned int height = curFrameCanvas->Height();

    unsigned nExpectedFrames = 0;
    for (unsigned a = 0, an = anims.size(); a != an; ++a)
        nExpectedFrames += anims .size();

    UMC::Status status;
    UMC::MediaData DataOut;
    UMC::VideoData DataIn;
    UMC::H264EncoderParams Params;
    UMC::H264VideoEncoder H264Encoder;
   UMC::MP4Muxer Muxer;
   UMC::MuxerParams MuxerParams;

    UMC::VideoStreamInfo VideoInfo;

    UMC::FileWriter Writer; UMC::FileWriterParams WriterParams;
    strcpy(WriterParams.m_file_name, VM_STRING(movieName.utf8()));
    Writer.Init(&WriterParams);

    MuxerParams.m_lpDataWriter = &Writer;
   MuxerParams.m_SystemType = UMC::MPEG4_SYSTEM_STREAM;
    MuxerParams.m_lFlags = UMC::FLAG_FRAGMENTED_AT_I_PICTURES;

   MuxerParams.m_nNumberOfTracks = 1;
    MuxerParams.pTrackParams = new UMC::TrackParams[MuxerParams.m_nNumberOfTracks];

    VideoInfo.clip_info.width = width;
    VideoInfo.clip_info.height = height;
    VideoInfo.stream_type = UMC::H264_VIDEO;
   VideoInfo.color_format = UMC::YUV420;
    VideoInfo.interlace_type = UMC::PROGRESSIVE;
   VideoInfo.bitrate=1000000;
    VideoInfo.streamPID = 0;

    MuxerParams.pTrackParams[0].type = UMC::VIDEO_TRACK;
    MuxerParams.pTrackParams[0].info.video = &VideoInfo;
    MuxerParams.pTrackParams[0].bufferParams.m_prefInputBufferSize = 2000000;
    MuxerParams.pTrackParams[0].bufferParams.m_prefOutputBufferSize = 2000000;
    MuxerParams.pTrackParams[0].bufferParams.m_numberOfFrames = nExpectedFrames;

    if((status = Muxer.Init(&MuxerParams))!=UMC::UMC_OK)
        return;

    // Additional params for H.264
    SetNiceH264Params(Params);

    Params.info.clip_info.width = width;
    Params.info.clip_info.height = height;
    Params.info.bitrate = 1000000;
    Params.info.framerate = 30.00;
    Params.numThreads = 1;
    Params.numFramesToEncode = nExpectedFrames;

    if((status = H264Encoder.Init(&Params))!=UMC::UMC_OK)
        return;

    Ipp8u *cYUVData = ippsMalloc_8u(MAXYUVSIZE);

    DataIn.Init(width, height, UMC::YUV420, 8);
    DataOut.Alloc(MAXAFRAMESIZE);

    int nEncodedFrames = 0;
    int nRequestedFrames = 0;
    for (unsigned a = 0, an = anims.size(); a != an; ++a)
    {
        for (unsigned curFrame = 0, numFrames = anims .size(); curFrame < numFrames; ++curFrame)
        {
            FrameData curFrameData = anims [curFrame];

            curFrameCanvas.Reset(CCanvasLoader::Instance()->LoadU8(curFrameData.first));

            if (!curFrameCanvas)
                continue;

            nRequestedFrames++;

            // Convert to YUV 420
            curFrameCanvas->ExportYUV420Data(cYUVData, MAXYUVSIZE);

            // Get it into the UMC input buffer
            size_t YUVBytes = (width * height * 3) / 2;
            DataIn.SetBufferPointer(cYUVData, YUVBytes);
            DataIn.SetDataSize(YUVBytes);
            DataIn.SetTime(nEncodedFrames/Params.info.framerate, (nEncodedFrames + 1)/Params.info.framerate);

            status = H264Encoder.GetFrame(&DataIn, &DataOut);
            if (status == UMC::UMC_OK)
            {
                DataOut.SetTime(nEncodedFrames/Params.info.framerate, (nEncodedFrames + 1)/Params.info.framerate);
                do {
                    status = Muxer.PutVideoData(&DataOut);
                    if (UMC::UMC_ERR_NOT_ENOUGH_BUFFER == status)
                        vm_time_sleep(5);
                } while (UMC::UMC_ERR_NOT_ENOUGH_BUFFER == status);

                nEncodedFrames++;

                DataOut.SetDataSize(0);
            }
            else if (status == UMC::UMC_ERR_NOT_ENOUGH_DATA)
            {
                // do nothing, the codec just needs more input frames before it can spit something out
            }
            else
            {
                throw "GetFrame";
            }

            if (!pr.Update(nEncodedFrames, nExpectedFrames))
                throw "Cancelled";
        }
    }

    // Recover any remaining frames from the buffer
    while ( nEncodedFrames < nRequestedFrames)
    {
        status = H264Encoder.GetFrame(NULL, &DataOut);
        if (status == UMC::UMC_OK) {
            nEncodedFrames++;

            DataOut.SetTime((nEncodedFrames+1)/Params.info.framerate);
            do {
                status = Muxer.PutVideoData(&DataOut);
                if (UMC::UMC_ERR_NOT_ENOUGH_BUFFER == status)
                    vm_time_sleep(5);
            } while (UMC::UMC_ERR_NOT_ENOUGH_BUFFER == status);

            DataOut.SetDataSize(0);
        }
        else
        {
            throw "GetFrame";
        }

        if (!pr.Update(nEncodedFrames, nExpectedFrames))
            throw "Cancelled";
    }

    ippsFree(cYUVData);

   Muxer.Close();
    return;
}

void SetNiceH264Params(UMC::H264EncoderParams &p)
    {
        p.key_frame_controls.method         = UMC::H264_KFCM_INTERVAL;
        p.key_frame_controls.interval       = 200;
        p.key_frame_controls.idr_interval   = 1;
        p.B_frame_rate                      = 3;
        p.treat_B_as_reference              = 1;
        p.num_ref_frames                    = 4;
        p.num_ref_to_start_code_B_slice     = 1;
        p.num_slices                        = 0; // Autoselect
        p.profile_idc                       = UMC::H264_HIGH_PROFILE;
        p.level_idc                         = 0; //Autoselect
        p.chroma_format_idc                 = 1; // YUV 420.
        p.bit_depth_luma                    = 8;
        p.bit_depth_chroma                  = 8;
        p.aux_format_idc                    = 0;
        p.bit_depth_aux                     = 8;
        p.alpha_incr_flag                   = 0;
        p.alpha_opaque_value                = 0;
        p.alpha_transparent_value           = 0;
        p.rate_controls.method              = UMC::H264_RCM_VBR;
        p.rate_controls.quantI              = 0;
        p.rate_controls.quantP              = 0;
        p.rate_controls.quantB              = 0;
        p.info.bitrate                      = 1000000;
        p.mv_search_method                  = 2;
        p.me_split_mode                     = 1;
        p.me_search_x                       = 8;
        p.me_search_y                       = 8;
        p.use_weighted_pred                 = 0;
        p.use_weighted_bipred               = 0;
        p.use_implicit_weighted_bipred      = 0;
        p.direct_pred_mode                  = 1;
        p.use_direct_inference              = 0;
        p.deblocking_filter_idc             = 0;    // 0 is "on". 1 - "off"
        p.deblocking_filter_alpha           = 0;
        p.deblocking_filter_beta            = 0;
        p.transform_8x8_mode_flag           = 1;
        p.use_default_scaling_matrix        = 0;
        p.qpprime_y_zero_transform_bypass_flag =0;
        p.entropy_coding_mode               = 1;
        p.cabac_init_idc                    = 2;
        p.coding_type                       = 0;
        p.numFramesToEncode                 = 0;
        p.m_QualitySpeed                    = 1;
        p.quant_opt_level                   = 0;
    }

I have now solved this: there

Philip_Starke — Wed, 19 Sep 2012 15:58:14 GMT

I have now solved this: there were a few problems. 1. I was overriding the time on each output frame (DataOut) to match the input frame. This was wrong as the encoder was reordering frames so the times were different 2. GetFrames returns UMC_OK even if there is no output data. I needed to also check DataOut.GetDataSize 3. Even when I corrected the above the movie would still not play in QuickTime player: it seems QuickTime player doesn't like H.264 high profile (or my specific settings) - when I left the H264Encoder params on their default settings (main profile) the video was okay

topic I have now solved this: there in Intel® Integrated Performance Primitives

H.264 encoding: frame ordering issues

I have now solved this: there