Hi Petter,
Here's my trace log from Init to where I get the error in the call to Reset:
INFO :found API function "MFXInit", maxAPIVersion increased == 1.0
INFO :found API function "MFXJoinSession", maxAPIVersion increased == 1.2
INFO :found API function "MFXDisjoinSession", maxAPIVersion increased == 1.4
INFO :minAPIVersion(32767.32767) > maxAPIVersion(1.4), minAPIVersion lowered to (1.4)
INFO :MFXInit(MFX_IMPL_SOFTWARE,ver=1.1,session=0x0000000002602180)
INFO :library loaded succesfully
SDK DLL File Version=3.12.4.12
SDK DLL Product Version=3.5.811.41085
SDK DLL File Size=13840312
MFXInit.impl=SOFTWARE
MFXInit.ver=1.1
MFXInit.status=MFX_ERR_NONE
core.SetFrameAllocator.status=MFX_ERR_NONE
vpp.Init.par.AsyncDepth=0
vpp.Init.par.vpp.In.FourCC=RGB4
vpp.Init.par.vpp.In.Resolution=(1280,720)
vpp.Init.par.vpp.In.Crop=(0,0,1280,720)
vpp.Init.par.vpp.In.FrameRate=30/1
vpp.Init.par.vpp.In.AspectRatio=0:0
vpp.Init.par.vpp.In.PicStruct=PROGRESSIVE
vpp.Init.par.vpp.In.ChromaFormat=4:2:0
vpp.Init.par.vpp.Out.FourCC=NV12
vpp.Init.par.vpp.Out.Resolution=(1280,720)
vpp.Init.par.vpp.Out.Crop=(0,0,1280,720)
vpp.Init.par.vpp.Out.FrameRate=30/1
vpp.Init.par.vpp.Out.AspectRatio=0:0
vpp.Init.par.vpp.Out.PicStruct=PROGRESSIVE
vpp.Init.par.vpp.Out.ChromaFormat=4:2:0
vpp.Init.par.Protected=0
vpp.Init.par.IOPattern=IN_SYSTEM_MEMORY|OUT_SYSTEM_MEMORY
vpp.Init.par.NumExtParam=0
vpp.Init.par.ExtParam=NULL
core.FrameAllocator.Alloc.request.Info.FourCC=NV12
core.FrameAllocator.Alloc.request.Info.Resolution=(1280,720)
core.FrameAllocator.Alloc.request.Info.Crop=(0,0,1280,720)
core.FrameAllocator.Alloc.request.Info.FrameRate=30/1
core.FrameAllocator.Alloc.request.Info.AspectRatio=0:0
core.FrameAllocator.Alloc.request.Info.PicStruct=PROGRESSIVE
core.FrameAllocator.Alloc.request.Info.ChromaFormat=4:2:0
core.FrameAllocator.Alloc.request.Type=SYSTEM_MEMORY|FROM_VPPIN|INTERNAL_FRAME
core.FrameAllocator.Alloc.request.NumFrameMin=1
core.FrameAllocator.Alloc.request.NumFrameSuggested=1
core.FrameAllocator.Alloc.response.NumFrameActual=1
core.FrameAllocator.Alloc.status=MFX_ERR_NONE
vpp.Init.status=MFX_ERR_NONE
encode.Init.par.AsyncDepth=0
encode.Init.par.mfx.BRCParamMultiplier=0
encode.Init.par.mfx.FrameInfo.FourCC=NV12
encode.Init.par.mfx.FrameInfo.Resolution=(1280,720)
encode.Init.par.mfx.FrameInfo.Crop=(0,0,1280,720)
encode.Init.par.mfx.FrameInfo.FrameRate=30/1
encode.Init.par.mfx.FrameInfo.AspectRatio=0:0
encode.Init.par.mfx.FrameInfo.PicStruct=PROGRESSIVE
encode.Init.par.mfx.FrameInfo.ChromaFormat=4:2:0
encode.Init.par.mfx.CodecId=AVC
encode.Init.par.mfx.CodecProfile=66
encode.Init.par.mfx.CodecLevel=0
encode.Init.par.mfx.NumThread=0
encode.Init.par.mfx.TargetUsage=7
encode.Init.par.mfx.GopPicSize=150
encode.Init.par.mfx.GopRefDist=1
encode.Init.par.mfx.GopOptFlag=0
encode.Init.par.mfx.IdrInterval=0
encode.Init.par.mfx.RateControlMethod=AVBR
encode.Init.par.mfx.Accuracy=10
encode.Init.par.mfx.Convergence=1
encode.Init.par.mfx.BufferSizeInKB=375
encode.Init.par.mfx.NumSlice=0
encode.Init.par.mfx.NumRefFrame=0
encode.Init.par.mfx.EncodedOrder=0
encode.Init.par.Protected=0
encode.Init.par.IOPattern=IN_SYSTEM_MEMORY
encode.Init.par.NumExtParam=0
encode.Init.par.ExtParam=NULL
encode.Init.status=MFX_ERR_NONE
core.FrameAllocator.Alloc.request.Info.FourCC=NV12
core.FrameAllocator.Alloc.request.Info.Resolution=(1280,720)
core.FrameAllocator.Alloc.request.Info.Crop=(0,0,1280,720)
core.FrameAllocator.Alloc.request.Info.FrameRate=30/1
core.FrameAllocator.Alloc.request.Info.AspectRatio=0:0
core.FrameAllocator.Alloc.request.Info.PicStruct=PROGRESSIVE
core.FrameAllocator.Alloc.request.Info.ChromaFormat=4:2:0
core.FrameAllocator.Alloc.request.Type=SYSTEM_MEMORY|FROM_ENCODE|EXTERNAL_FRAME
core.FrameAllocator.Alloc.request.NumFrameMin=1
core.FrameAllocator.Alloc.request.NumFrameSuggested=1
core.FrameAllocator.Alloc.response.NumFrameActual=1
core.FrameAllocator.Alloc.status=MFX_ERR_NONE
encode.NumExternalFrames=1
vpp.QueryIOSurf.par.AsyncDepth=0
vpp.QueryIOSurf.par.vpp.In.FourCC=RGB4
vpp.QueryIOSurf.par.vpp.In.Resolution=(1280,720)
vpp.QueryIOSurf.par.vpp.In.Crop=(0,0,1280,720)
vpp.QueryIOSurf.par.vpp.In.FrameRate=30/1
vpp.QueryIOSurf.par.vpp.In.AspectRatio=0:0
vpp.QueryIOSurf.par.vpp.In.PicStruct=PROGRESSIVE
vpp.QueryIOSurf.par.vpp.In.ChromaFormat=4:2:0
vpp.QueryIOSurf.par.vpp.Out.FourCC=NV12
vpp.QueryIOSurf.par.vpp.Out.Resolution=(1280,720)
vpp.QueryIOSurf.par.vpp.Out.Crop=(0,0,1280,720)
vpp.QueryIOSurf.par.vpp.Out.FrameRate=30/1
vpp.QueryIOSurf.par.vpp.Out.AspectRatio=0:0
vpp.QueryIOSurf.par.vpp.Out.PicStruct=PROGRESSIVE
vpp.QueryIOSurf.par.vpp.Out.ChromaFormat=4:2:0
vpp.QueryIOSurf.par.Protected=0
vpp.QueryIOSurf.par.IOPattern=IN_SYSTEM_MEMORY|OUT_SYSTEM_MEMORY
vpp.QueryIOSurf.par.NumExtParam=0
vpp.QueryIOSurf.par.ExtParam=NULL
vpp.QueryIOSurf.request[0].Info.FourCC=RGB4
vpp.QueryIOSurf.request[0].Info.Resolution=(1280,720)
vpp.QueryIOSurf.request[0].Info.Crop=(0,0,1280,720)
vpp.QueryIOSurf.request[0].Info.FrameRate=30/1
vpp.QueryIOSurf.request[0].Info.AspectRatio=0:0
vpp.QueryIOSurf.request[0].Info.PicStruct=PROGRESSIVE
vpp.QueryIOSurf.request[0].Info.ChromaFormat=4:2:0
vpp.QueryIOSurf.request[0].Type=SYSTEM_MEMORY|FROM_VPPIN|EXTERNAL_FRAME
vpp.QueryIOSurf.request[0].NumFrameMin=5
vpp.QueryIOSurf.request[0].NumFrameSuggested=5
vpp.QueryIOSurf.request[1].Info.FourCC=NV12
vpp.QueryIOSurf.request[1].Info.Resolution=(1280,720)
vpp.QueryIOSurf.request[1].Info.Crop=(0,0,1280,720)
vpp.QueryIOSurf.request[1].Info.FrameRate=30/1
vpp.QueryIOSurf.request[1].Info.AspectRatio=0:0
vpp.QueryIOSurf.request[1].Info.PicStruct=PROGRESSIVE
vpp.QueryIOSurf.request[1].Info.ChromaFormat=4:2:0
vpp.QueryIOSurf.request[1].Type=SYSTEM_MEMORY|FROM_VPPOUT|EXTERNAL_FRAME
vpp.QueryIOSurf.request[1].NumFrameMin=5
vpp.QueryIOSurf.request[1].NumFrameSuggested=5
vpp.QueryIOSurf.status=MFX_ERR_NONE
encode.QueryIOSurf.par.AsyncDepth=0
encode.QueryIOSurf.par.mfx.BRCParamMultiplier=0
encode.QueryIOSurf.par.mfx.FrameInfo.FourCC=NV12
encode.QueryIOSurf.par.mfx.FrameInfo.Resolution=(1280,720)
encode.QueryIOSurf.par.mfx.FrameInfo.Crop=(0,0,1280,720)
encode.QueryIOSurf.par.mfx.FrameInfo.FrameRate=30/1
encode.QueryIOSurf.par.mfx.FrameInfo.AspectRatio=0:0
encode.QueryIOSurf.par.mfx.FrameInfo.PicStruct=PROGRESSIVE
encode.QueryIOSurf.par.mfx.FrameInfo.ChromaFormat=4:2:0
encode.QueryIOSurf.par.mfx.CodecId=AVC
encode.QueryIOSurf.par.mfx.CodecProfile=66
encode.QueryIOSurf.par.mfx.CodecLevel=0
encode.QueryIOSurf.par.mfx.NumThread=0
encode.QueryIOSurf.par.mfx.TargetUsage=7
encode.QueryIOSurf.par.mfx.GopPicSize=150
encode.QueryIOSurf.par.mfx.GopRefDist=1
encode.QueryIOSurf.par.mfx.GopOptFlag=0
encode.QueryIOSurf.par.mfx.IdrInterval=0
encode.QueryIOSurf.par.mfx.RateControlMethod=AVBR
encode.QueryIOSurf.par.mfx.Accuracy=10
encode.QueryIOSurf.par.mfx.Convergence=1
encode.QueryIOSurf.par.mfx.BufferSizeInKB=375
encode.QueryIOSurf.par.mfx.NumSlice=0
encode.QueryIOSurf.par.mfx.NumRefFrame=0
encode.QueryIOSurf.par.mfx.EncodedOrder=0
encode.QueryIOSurf.par.Protected=0
encode.QueryIOSurf.par.IOPattern=IN_SYSTEM_MEMORY
encode.QueryIOSurf.par.NumExtParam=0
encode.QueryIOSurf.par.ExtParam=NULL
encode.QueryIOSurf.request.Info.FourCC=NV12
encode.QueryIOSurf.request.Info.Resolution=(1280,720)
encode.QueryIOSurf.request.Info.Crop=(0,0,1280,720)
encode.QueryIOSurf.request.Info.FrameRate=30/1
encode.QueryIOSurf.request.Info.AspectRatio=0:0
encode.QueryIOSurf.request.Info.PicStruct=PROGRESSIVE
encode.QueryIOSurf.request.Info.ChromaFormat=4:2:0
encode.QueryIOSurf.request.Type=SYSTEM_MEMORY|FROM_ENCODE|EXTERNAL_FRAME
encode.QueryIOSurf.request.NumFrameMin=1
encode.QueryIOSurf.request.NumFrameSuggested=1
encode.QueryIOSurf.status=MFX_ERR_NONE
encode.GetVideoParam.par.AsyncDepth=0
encode.GetVideoParam.par.mfx.BRCParamMultiplier=1
encode.GetVideoParam.par.mfx.FrameInfo.FourCC=NV12
encode.GetVideoParam.par.mfx.FrameInfo.Resolution=(1280,720)
encode.GetVideoParam.par.mfx.FrameInfo.Crop=(0,0,1280,720)
encode.GetVideoParam.par.mfx.FrameInfo.FrameRate=30/1
encode.GetVideoParam.par.mfx.FrameInfo.AspectRatio=1:1
encode.GetVideoParam.par.mfx.FrameInfo.PicStruct=PROGRESSIVE
encode.GetVideoParam.par.mfx.FrameInfo.ChromaFormat=4:2:0
encode.GetVideoParam.par.mfx.CodecId=AVC
encode.GetVideoParam.par.mfx.CodecProfile=66
encode.GetVideoParam.par.mfx.CodecLevel=31
encode.GetVideoParam.par.mfx.NumThread=4
encode.GetVideoParam.par.mfx.TargetUsage=7
encode.GetVideoParam.par.mfx.GopPicSize=150
encode.GetVideoParam.par.mfx.GopRefDist=1
encode.GetVideoParam.par.mfx.GopOptFlag=0
encode.GetVideoParam.par.mfx.IdrInterval=0
encode.GetVideoParam.par.mfx.RateControlMethod=AVBR
encode.GetVideoParam.par.mfx.Accuracy=10
encode.GetVideoParam.par.mfx.Convergence=1
encode.GetVideoParam.par.mfx.BufferSizeInKB=375
encode.GetVideoParam.par.mfx.NumSlice=1
encode.GetVideoParam.par.mfx.NumRefFrame=1
encode.GetVideoParam.par.mfx.EncodedOrder=0
encode.GetVideoParam.par.Protected=0
encode.GetVideoParam.par.IOPattern=IN_SYSTEM_MEMORY
encode.GetVideoParam.par.NumExtParam=1
encode.GetVideoParam.par.ExtParam=mfxExtCodingOptionSPSPPS
encode.GetVideoParam.par.ExtParam.mfxExtCodingOptionSPSPPS.SPSId=0
encode.GetVideoParam.par.ExtParam.mfxExtCodingOptionSPSPPS.SPSBufSize=26
encode.GetVideoParam.par.ExtParam.mfxExtCodingOptionSPSPPS.PPSId=0
encode.GetVideoParam.par.ExtParam.mfxExtCodingOptionSPSPPS.PPSBufSize=9
encode.GetVideoParam.status=MFX_ERR_NONE
vpp.Reset.par.AsyncDepth=0
vpp.Reset.par.vpp.In.FourCC=RGB4
vpp.Reset.par.vpp.In.Resolution=(1280,720)
vpp.Reset.par.vpp.In.Crop=(0,0,1280,720)
vpp.Reset.par.vpp.In.FrameRate=30/1
vpp.Reset.par.vpp.In.AspectRatio=0:0
vpp.Reset.par.vpp.In.PicStruct=PROGRESSIVE
vpp.Reset.par.vpp.In.ChromaFormat=4:2:0
vpp.Reset.par.vpp.Out.FourCC=NV12
vpp.Reset.par.vpp.Out.Resolution=(1280,720)
vpp.Reset.par.vpp.Out.Crop=(0,0,1280,720)
vpp.Reset.par.vpp.Out.FrameRate=30/1
vpp.Reset.par.vpp.Out.AspectRatio=0:0
vpp.Reset.par.vpp.Out.PicStruct=PROGRESSIVE
vpp.Reset.par.vpp.Out.ChromaFormat=4:2:0
vpp.Reset.par.Protected=0
vpp.Reset.par.IOPattern=IN_SYSTEM_MEMORY|OUT_SYSTEM_MEMORY
vpp.Reset.par.NumExtParam=0
vpp.Reset.par.ExtParam=NULL
vpp.Reset.status=MFX_ERR_NONE
encode.Reset.par.AsyncDepth=0
encode.Reset.par.mfx.BRCParamMultiplier=0
encode.Reset.par.mfx.FrameInfo.FourCC=NV12
encode.Reset.par.mfx.FrameInfo.Resolution=(1280,720)
encode.Reset.par.mfx.FrameInfo.Crop=(0,0,1280,720)
encode.Reset.par.mfx.FrameInfo.FrameRate=30/1
encode.Reset.par.mfx.FrameInfo.AspectRatio=0:0
encode.Reset.par.mfx.FrameInfo.PicStruct=PROGRESSIVE
encode.Reset.par.mfx.FrameInfo.ChromaFormat=4:2:0
encode.Reset.par.mfx.CodecId=AVC
encode.Reset.par.mfx.CodecProfile=66
encode.Reset.par.mfx.CodecLevel=0
encode.Reset.par.mfx.NumThread=0
encode.Reset.par.mfx.TargetUsage=7
encode.Reset.par.mfx.GopPicSize=150
encode.Reset.par.mfx.GopRefDist=1
encode.Reset.par.mfx.GopOptFlag=0
encode.Reset.par.mfx.IdrInterval=0
encode.Reset.par.mfx.RateControlMethod=AVBR
encode.Reset.par.mfx.Accuracy=10
encode.Reset.par.mfx.Convergence=1
encode.Reset.par.mfx.BufferSizeInKB=375
encode.Reset.par.mfx.NumSlice=0
encode.Reset.par.mfx.NumRefFrame=0
encode.Reset.par.mfx.EncodedOrder=0
encode.Reset.par.Protected=0
encode.Reset.par.IOPattern=IN_SYSTEM_MEMORY
encode.Reset.par.NumExtParam=0
encode.Reset.par.ExtParam=NULL
encode.Reset.status=MFX_ERR_INCOMPATIBLE_VIDEO_PARAM
Thanks for your help,
Farhad