Community
cancel
Showing results for 
Search instead for 
Did you mean: 
Mohnish_J_
Beginner
51 Views

Composition of different resolution streams using VPP

Hi,

I am developing a sample application for overlaying four NV12 streams of different resolution. For this I am modifying simple_4_vpp_resize_denoise application present in mediasdk-tutorials-0.0.3 and I have followed the procedure as specified in https://software.intel.com/en-us/articles/video-composition-using-intel-media-sdk. I am using Intel(R) Core(TM) i7-4770R CPU @ 3.20GHz, which has Iris Pro Graphics 5200, Linux OS 3.10.0-229.1.2.39163.MSSr4.el7.centos.x86_64,  Media Server Studio Essentials 2015 R6 API version 1.16 for my development.

The application works fine if the input streams are of same resolution i.e. 1920x1080 with the below parameters:

#define NUM_STREAMS 4

#define STREAM_FILE_ONE     "test1.yuv"
#define IN_HEIGHT_ONE       1080
#define IN_WIDTH_ONE        1920
#define CROPH_ONE           1080
#define CROPW_ONE           1920
#define CROPX_ONE           0
#define CROPY_ONE           0
#define DESTX_ONE           0
#define DESTY_ONE           0
#define DESTH_ONE           720
#define DESTW_ONE           1280
#define ALPHA_ONE           255

#define STREAM_FILE_TWO     "test2.yuv"
#define IN_HEIGHT_TWO       1080
#define IN_WIDTH_TWO        1920
#define CROPH_TWO           1080
#define CROPW_TWO           1920
#define CROPX_TWO           0
#define CROPY_TWO           0
#define DESTX_TWO           0
#define DESTY_TWO           0
#define DESTH_TWO           270
#define DESTW_TWO           480
#define ALPHA_TWO           128

#define STREAM_FILE_THREE   "test3.yuv"
#define IN_HEIGHT_THREE     1080
#define IN_WIDTH_THREE      1920
#define CROPH_THREE         1080
#define CROPW_THREE         1920
#define CROPX_THREE         0
#define CROPY_THREE         0
#define DESTX_THREE         500
#define DESTY_THREE         0
#define DESTH_THREE         270
#define DESTW_THREE         480
#define ALPHA_THREE         200

#define STREAM_FILE_FOUR    "test4.yuv"
#define IN_HEIGHT_FOUR      1080
#define IN_WIDTH_FOUR       1920
#define CROPH_FOUR          1080
#define CROPW_FOUR          1920
#define CROPX_FOUR          0
#define CROPY_FOUR          0
#define DESTX_FOUR          100
#define DESTY_FOUR          300
#define DESTH_FOUR          270
#define DESTW_FOUR          480
#define ALPHA_FOUR          256

#define STREAM_FILE_OUT "out.yuv"
#define VPP_OUT_HEIGHT          720
#define VPP_OUT_WIDTH           1280

#define VPP_IN_HEIGHT           1080
#define VPP_IN_WIDTH            1920

and I get the below output:

proper_overlay_out.PNG

Now I change stream3 to 960x540 and accordingly I modify my parameters also as below:

#define NUM_STREAMS 4

#define STREAM_FILE_ONE     "test1.yuv"
#define IN_HEIGHT_ONE       1080
#define IN_WIDTH_ONE        1920
#define CROPH_ONE           1080
#define CROPW_ONE           1920
#define CROPX_ONE           0
#define CROPY_ONE           0
#define DESTX_ONE           0
#define DESTY_ONE           0
#define DESTH_ONE           720
#define DESTW_ONE           1280
#define ALPHA_ONE           255

#define STREAM_FILE_TWO     "test2.yuv"
#define IN_HEIGHT_TWO       1080
#define IN_WIDTH_TWO        1920
#define CROPH_TWO           1080
#define CROPW_TWO           1920
#define CROPX_TWO           0
#define CROPY_TWO           0
#define DESTX_TWO           0
#define DESTY_TWO           0
#define DESTH_TWO           270
#define DESTW_TWO           480
#define ALPHA_TWO           128

#define STREAM_FILE_THREE   "test3.yuv"
#define IN_HEIGHT_THREE     540
#define IN_WIDTH_THREE      960
#define CROPH_THREE         540
#define CROPW_THREE         960
#define CROPX_THREE         0
#define CROPY_THREE         0
#define DESTX_THREE         500
#define DESTY_THREE         0
#define DESTH_THREE         270
#define DESTW_THREE         480
#define ALPHA_THREE         200

#define STREAM_FILE_FOUR    "test4.yuv"
#define IN_HEIGHT_FOUR      1080
#define IN_WIDTH_FOUR       1920
#define CROPH_FOUR          1080
#define CROPW_FOUR          1920
#define CROPX_FOUR          0
#define CROPY_FOUR          0
#define DESTX_FOUR          100
#define DESTY_FOUR          300
#define DESTH_FOUR          270
#define DESTW_FOUR          480
#define ALPHA_FOUR          255

#define STREAM_FILE_OUT "out.yuv"
#define VPP_OUT_HEIGHT          720
#define VPP_OUT_WIDTH           1280

#define VPP_IN_HEIGHT           1920
#define VPP_IN_WIDTH            1080

With this I get the output as:

improper_overlay_0.PNG

I have done these changes in my application:

1. VPP Parameters change for the largest stream:
    // Input data
    VPPParams.vpp.In.FourCC = MFX_FOURCC_NV12;
    VPPParams.vpp.In.ChromaFormat = MFX_CHROMAFORMAT_YUV420;
    VPPParams.vpp.In.CropX = 0;
    VPPParams.vpp.In.CropY = 0;
    VPPParams.vpp.In.CropW = VPP_IN_WIDTH;
    VPPParams.vpp.In.CropH = VPP_IN_HEIGHT;
    VPPParams.vpp.In.PicStruct = MFX_PICSTRUCT_PROGRESSIVE;
    VPPParams.vpp.In.FrameRateExtN = 30;
    VPPParams.vpp.In.FrameRateExtD = 1;
    // width must be a multiple of 16
    // height must be a multiple of 16
    // in case of frame picture and a multiple
    // of 32 in case of field picture
    VPPParams.vpp.In.Width = MSDK_ALIGN16(VPP_IN_WIDTH);
    VPPParams.vpp.In.Height =
        (MFX_PICSTRUCT_PROGRESSIVE == VPPParams.vpp.In.PicStruct) ?
        MSDK_ALIGN16(VPP_IN_HEIGHT) :
        MSDK_ALIGN32(VPP_IN_HEIGHT);

 

    // Output data
    VPPParams.vpp.Out.FourCC = MFX_FOURCC_NV12;
    VPPParams.vpp.Out.ChromaFormat = MFX_CHROMAFORMAT_YUV420;
    VPPParams.vpp.Out.CropX = 0;
    VPPParams.vpp.Out.CropY = 0;
    VPPParams.vpp.Out.CropW = VPP_OUT_WIDTH;
    VPPParams.vpp.Out.CropH = VPP_OUT_HEIGHT;
    VPPParams.vpp.Out.PicStruct = MFX_PICSTRUCT_PROGRESSIVE;
    VPPParams.vpp.Out.FrameRateExtN = 30;
    VPPParams.vpp.Out.FrameRateExtD = 1;
    // width must be a multiple of 16
    // height must be a multiple of 16 in case of frame
    // picture and a multiple of 32 in case of field picture
    VPPParams.vpp.Out.Width = MSDK_ALIGN16(VPP_OUT_WIDTH);
    VPPParams.vpp.Out.Height =
        (MFX_PICSTRUCT_PROGRESSIVE == VPPParams.vpp.Out.PicStruct) ?
        MSDK_ALIGN16(VPP_OUT_HEIGHT) :
        MSDK_ALIGN32(VPP_OUT_HEIGHT);

2. mfxFrameInfo structure for input streams:

 mfxFrameInfo inputStreams[NUM_STREAMS];
    /** Fill frame params in mFrameInfo structures
     * with the above parameters **/
    for (mfxU16 i = 0; i < NUM_STREAMS; i++){
        memcpy(&inputStreams,&(VPPParams.vpp.In), sizeof(mfxFrameInfo));
    }

    inputStreams[0].Width = IN_WIDTH_ONE;
    inputStreams[0].Height = IN_HEIGHT_ONE;
    inputStreams[0].CropX = CROPX_ONE;
    inputStreams[0].CropY = CROPY_ONE;
    inputStreams[0].CropW = CROPW_ONE;
    inputStreams[0].CropH = CROPH_ONE;

    inputStreams[1].Width = IN_WIDTH_TWO;
    inputStreams[1].Height = IN_HEIGHT_TWO;
    inputStreams[1].CropX = CROPX_TWO;
    inputStreams[1].CropY = CROPY_TWO;
    inputStreams[1].CropW = CROPW_TWO;
    inputStreams[1].CropH = CROPH_TWO;

    inputStreams[2].Width = IN_WIDTH_THREE;
    inputStreams[2].Height = IN_HEIGHT_THREE;
    inputStreams[2].CropX = CROPX_THREE;
    inputStreams[2].CropY = CROPY_THREE;
    inputStreams[2].CropW = DESTW_THREE;
    inputStreams[2].CropH = DESTH_THREE;

    inputStreams[3].Width = IN_WIDTH_FOUR;
    inputStreams[3].Height = IN_HEIGHT_FOUR;
    inputStreams[3].CropX = CROPX_FOUR;
    inputStreams[3].CropY = CROPY_FOUR;
    inputStreams[3].CropW = CROPW_FOUR;
    inputStreams[3].CropH = CROPH_FOUR;

3. Initializing Composite extended buffer:

  mfxExtVPPComposite composite;
    memset(&composite, 0, sizeof(composite));
    composite.Header.BufferId = MFX_EXTBUFF_VPP_COMPOSITE;
    composite.Header.BufferSz = sizeof(mfxExtVPPComposite);
    composite.NumInputStream = NUM_STREAMS;
    composite.Y = 10;
    composite.U = 80;
    composite.V = 80;

    composite.InputStream = new mfxVPPCompInputStream[NUM_STREAMS];
    MSDK_CHECK_POINTER(composite.InputStream, MFX_ERR_MEMORY_ALLOC);
    memset(composite.InputStream, 0, sizeof(mfxVPPCompInputStream) * NUM_STREAMS);

    composite.InputStream[0].DstX = (mfxU32)DESTX_ONE;
    composite.InputStream[0].DstY = (mfxU32)DESTY_ONE;
    composite.InputStream[0].DstW = (mfxU32)DESTW_ONE;
    composite.InputStream[0].DstH = (mfxU32)DESTH_ONE;
    composite.InputStream[0].GlobalAlphaEnable = 1;
    composite.InputStream[0].GlobalAlpha = ALPHA_ONE;

    composite.InputStream[1].DstX = (mfxU32)DESTX_TWO;
    composite.InputStream[1].DstY = (mfxU32)DESTY_TWO;
    composite.InputStream[1].DstW = (mfxU32)DESTW_TWO;
    composite.InputStream[1].DstH = (mfxU32)DESTH_TWO;
    composite.InputStream[1].GlobalAlphaEnable = 1;
    composite.InputStream[1].GlobalAlpha = ALPHA_TWO;

    composite.InputStream[2].DstX = (mfxU32)DESTX_THREE;
    composite.InputStream[2].DstY = (mfxU32)DESTY_THREE;
    composite.InputStream[2].DstW = (mfxU32)DESTW_THREE;
    composite.InputStream[2].DstH = (mfxU32)DESTH_THREE;
    composite.InputStream[2].GlobalAlphaEnable = 1;
    composite.InputStream[2].GlobalAlpha = ALPHA_THREE;

    composite.InputStream[3].DstX = (mfxU32)DESTX_FOUR;
    composite.InputStream[3].DstY = (mfxU32)DESTY_FOUR;
    composite.InputStream[3].DstW = (mfxU32)DESTW_FOUR;
    composite.InputStream[3].DstH = (mfxU32)DESTH_FOUR;
    composite.InputStream[3].GlobalAlphaEnable = 1;
    composite.InputStream[3].GlobalAlpha = ALPHA_FOUR;

 

    // Add extended VPP buffers
    mfxExtBuffer* ExtBuffer[1];
    ExtBuffer[0] = (mfxExtBuffer*) &composite;
    VPPParams.NumExtParam = 1;
    VPPParams.ExtParam = (mfxExtBuffer**) &ExtBuffer[0];

4. In the processing loop, after getting surface index modifying surfaces Info and Data fields for 960x540 stream before I do LoadRawFrame call :

        memcpy(&(pVPPSurfacesIn[nSurfIdxIn]->Info), &(inputStreams[streamNum]), sizeof(mfxFrameInfo));
        //pVPPSurfacesIn->Data.Y = &surfaceBuffersIn[surfaceSize * i];

        pVPPSurfacesIn[nSurfIdxIn]->Data.U = pVPPSurfacesIn[nSurfIdxIn]->Data.Y + inputStreams[streamNum].Width*inputStreams[streamNum].Height;
        pVPPSurfacesIn[nSurfIdxIn]->Data.V = pVPPSurfacesIn[nSurfIdxIn]->Data.U + 1;
pVPPSurfacesIn[nSurfIdxIn]->Data.Pitch = inputStreams[streamNum].Width;

        // Load frame from file into surface
        sts = LoadRawFrame(pVPPSurfacesIn[nSurfIdxIn], fSource[streamNum]);
        streamNum++;

OR just modifying the frame info:

        memcpy(&(pVPPSurfacesIn[nSurfIdxIn]->Info), &(inputStreams[streamNum]), sizeof(mfxFrameInfo));
        //pVPPSurfacesIn->Data.Y = &surfaceBuffersIn[surfaceSize * i];

        // Load frame from file into surface
        sts = LoadRawFrame(pVPPSurfacesIn[nSurfIdxIn], fSource[streamNum]);
        streamNum++;

Please help me with this issue. Thanks in advance.

 

0 Kudos
2 Replies
Mohnish_J_
Beginner
51 Views

Hi,

I resolved the issue. Input 2 stream parameters were not proper:

    inputStreams[2].CropW = DESTW_THREE;
    inputStreams[2].CropH = DESTH_THREE;

was to be replaced with

    inputStreams[2].CropW = CROPW_THREE;
    inputStreams[2].CropH = CROPH_THREE;

51 Views

Thanks for the update Mohnish - glad you got the composition working. I am closing this thread.

Reply