1. 技术文章
  2. Setup of IMTransform Video Processor for color space conversion

Setup of IMTransform Video Processor for color space conversion

由 荒凉一梦 提交于 2021-02-10 12:44:16

问题


I am trying to do some basic color space conversion using Video Processor MFT. My camera natively supports NV12 and I need RGB24, to code some shader which will provide cartoon-like effect.

Here is definition of class Media which is used to perform MF.

class Media : public IMFSourceReaderCallback //this class inhertis from IMFSourceReaderCallback
{
    CRITICAL_SECTION criticalSection;
    long referenceCount;
    WCHAR                   *wSymbolicLink;
    UINT32                  cchSymbolicLink;
    IMFSourceReader* sourceReader;
    MFT_REGISTER_TYPE_INFO *inputVideoTypes;
    MFT_REGISTER_TYPE_INFO *outputVideoTypes;
    IMFMediaType* mediaType = NULL;
    IMFMediaType* streamType = NULL;
    IMFMediaType* streamType2 = NULL;
    IMFMediaType* streamType3 = NULL;
    IMFTransform **VP;

public:
    LONG stride;
    float bytesPerPixel;
    GUID videoFormat;
    UINT height;
    UINT width;
    WCHAR deviceNameString[2048];
    BYTE* rawData;
    UINT32 count;
    DWORD devices_found = 0;
    HRESULT CreateCaptureDevice();
    HRESULT SetSourceReader(IMFActivate *device);
    HRESULT IsMediaTypeSupported(IMFMediaType* type);
    HRESULT GetDefaultStride(IMFMediaType *pType, LONG *plStride);
    HRESULT Close();
    Media();
    ~Media();   

    // the class must implement the methods from IUnknown 
    STDMETHODIMP QueryInterface(REFIID iid, void** ppv);
    STDMETHODIMP_(ULONG) AddRef();
    STDMETHODIMP_(ULONG) Release();

    //  the class must implement the methods from IMFSourceReaderCallback 
    STDMETHODIMP OnReadSample(HRESULT status, DWORD streamIndex, DWORD streamFlags, LONGLONG timeStamp, IMFSample *sample);
    STDMETHODIMP OnEvent(DWORD, IMFMediaEvent *);
    STDMETHODIMP OnFlush(DWORD);

};

I am doing setup of IMFTransform in the following way:

inputVideoTypes = new MFT_REGISTER_TYPE_INFO;
inputVideoTypes->guidMajorType = MFMediaType_Video;
inputVideoTypes->guidSubtype = MFVideoFormat_NV12;
outputVideoTypes = new MFT_REGISTER_TYPE_INFO;
outputVideoTypes->guidMajorType = MFMediaType_Video;
outputVideoTypes->guidSubtype = MFVideoFormat_RGB24;
hr = sourceReader->GetNativeMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, i, &streamType2);
IMFActivate **transformActivateArray = NULL;
UINT32 MFTcount;
hr = MFTEnumEx(MFT_CATEGORY_VIDEO_PROCESSOR, MFT_ENUM_FLAG_ALL, inputVideoTypes, outputVideoTypes, &transformActivateArray, &MFTcount);
if (FAILED(hr))
{
    exit(3);
}
if (MFTcount == 0)
exit(7);
VP = new IMFTransform*[MFTcount];
for (DWORD i = 0; i < MFTcount; i++)
{
    hr = transformActivateArray[i]->ActivateObject(__uuidof(IMFTransform), (void**)&VP[i]);
}
DWORD* inputCount = new DWORD[MFTcount];
DWORD* outputCount = new DWORD[MFTcount];
for (DWORD i = 0; i < MFTcount; i++)
{
    hr = VP[i]->GetStreamCount(&inputCount[i], &outputCount[i]);
}
DWORD **inputids = new DWORD*[MFTcount];
DWORD **outputids = new DWORD*[MFTcount];
for (DWORD i = 0; i < MFTcount; i++)
{
        inputids[i] = new DWORD[inputCount[i]];
        outputids[i] = new DWORD[outputCount[i]];
}
for (DWORD i = 0; i < MFTcount; i++)
{
    VP[i]->GetStreamIDs(inputCount[i], inputids[i], outputCount[i], outputids[i]);
    if (FAILED(hr))
    exit(5);
}
DWORD flag1 = -1;
DWORD flag2 = -1;
for (DWORD i = 0; i < MFTcount; i++)
{
    for (DWORD j = 0; j < inputCount[i]; j++)
    {
        hr = VP[i]->GetInputAvailableType(0, 0, &streamType);
        if (SUCCEEDED(hr))
        {
            flag1 = i;
            flag2 = j;
            break;
        }
    }
}
if (flag1 == -1 && flag2 == -1)
exit(2);
hr = VP[0]->SetInputType(0, streamType2, 0);
hr = VP[0]->GetOutputAvailableType(0, 0, &streamType3);
hr = VP[0]->SetOutputType(0, streamType3, 0);

The problem is that SetOutput method returns: cannot find a demanded attribute, and I dont really get, what is wrong. Can anyone point where I am doing things bad way? Thank you

EDIT: LogMediaType of input:

MF_MT_FRAME_SIZE    1280 x 720
MF_MT_YUV_MATRIX    2
MF_MT_MAJOR_TYPE    MFMediaType_Video
MF_MT_VIDEO_LIGHTING    3
MF_MT_VIDEO_CHROMA_SITING   1
MF_MT_AM_FORMAT_TYPE    {F72A76A0-EB0A-11D0-ACE4-0000C0CC16BA}
MF_MT_FIXED_SIZE_SAMPLES    1
MF_MT_VIDEO_NOMINAL_RANGE   1
MF_MT_FRAME_RATE    30 x 1
MF_MT_PIXEL_ASPECT_RATIO    1 x 1
MF_MT_ALL_SAMPLES_INDEPENDENT   1
MF_MT_FRAME_RATE_RANGE_MIN  128849018881
MF_MT_VIDEO_PRIMARIES   2
MF_MT_INTERLACE_MODE    2
MF_MT_FRAME_RATE_RANGE_MAX  128849018881
{EA031A62-8BBB-43C5-B5C4-572D2D231C18}  1
MF_MT_SUBTYPE   MFVideoFormat_NV12

Logs of Output dosen't work

Exception thrown: read access violation.
**pType** was nullptr.

EDIT2

I have only one VP enumerated by EnumEx method, and it has fixed number of input (1) and output (1) streams, so previous log is the only one log for input

Edit 3

hr = VP->SetInputType(0, streamType2, 0);
//MediaFoundationSamples::LogMediaType(streamType2);
DWORD dwIndex = 4;
hr = VP->GetOutputAvailableType(0, dwIndex, &streamType3);
hr = MFSetAttributeSize(streamType3, MF_MT_FRAME_SIZE, 1280, 720);
hr = streamType3->SetUINT32(MF_MT_FIXED_SIZE_SAMPLES, 1);
hr = MFSetAttributeRatio(streamType3, MF_MT_FRAME_RATE, 30, 1);
hr = MFSetAttributeRatio(streamType3, MF_MT_PIXEL_ASPECT_RATIO, 1, 1);
streamType3->SetUINT32(MF_MT_ALL_SAMPLES_INDEPENDENT, 1);
streamType3->SetUINT32(MF_MT_INTERLACE_MODE, 2);
MediaFoundationSamples::LogMediaType(streamType3);
hr = VP->SetOutputType(0, streamType3, 0);
hr = VP->GetInputStreamInfo(0, &InputInfo);
hr = VP->GetOutputStreamInfo(0, &OutputInfo);

and processing in onReadSample method looks like:

hr = VP->ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, NULL);
hr = VP->ProcessInput(0, sample, 0);
DWORD statusFlags;
hr = VP->GetOutputStatus(&statusFlags);
while (statusFlags == 0)
{
    hr = VP->ProcessInput(0, sample, 0);
    hr = VP->GetOutputStatus(&statusFlags);
}
DWORD outputStatus = 0;
IMFSample* outputSample;
MFCreateSample(&outputSample);
MFT_OUTPUT_DATA_BUFFER outputBuffer = {};
outputBuffer.pSample = outputSample;
hr = VP->ProcessOutput(0, OutputInfo.cbSize, &outputBuffer, &outputStatus);

回答1:


From Mediafoundation samples, you have LogMediaType function LogMediaType

Can you show log for streamType2/streamType3.

Also check existing video media type from here video-format-attributes

It could help you to find the missing attribute.

EDIT1

Beacause i use Windows Seven, i don't have the Video Processor MFT. I use Color Converter DSP, who can also do the color space conversion Color Converter DSP

IMFTransform* pVideoColorConverter = NULL;
IMFMediaType* pVideoOutputType = NULL;
// RGB24 media type at index 10, but can be different on your system
DWORD dwRGB24Index = 10;

hr = CoCreateInstance(CLSID_CColorConvertDMO, NULL, CLSCTX_INPROC_SERVER, IID_IMFTransform, reinterpret_cast<void**>(&pVideoColorConverter);
hr = pVideoColorConverter->SetInputType(0, pVideoInputType, 0);
hr = pVideoColorConverter->GetOutputAvailableType(0, dwRGB24Index, &pVideoOutputType);
hr = pVideoColorConverter->SetOutputType(0, pVideoOutputType, 0);

LogMediaType(pVideoInputType);
LogMediaType(pVideoOutputType);

Here is the log :

pVideoInputType (my camera provide YUY2 not NV12) :

MF_MT_MAJOR_TYPE                MFMediaType_Video
MF_MT_SUBTYPE                   MFVideoFormat_YUY2
MF_MT_FRAME_SIZE                640 x 480
MF_MT_DEFAULT_STRIDE            1280
MF_MT_ALL_SAMPLES_INDEPENDENT   1
MF_MT_FIXED_SIZE_SAMPLES        1
MF_MT_SAMPLE_SIZE               614400
MF_MT_AVG_BITRATE               147456000
MF_MT_FRAME_RATE                30 x 1
MF_MT_PIXEL_ASPECT_RATIO        1 x 1
MF_MT_INTERLACE_MODE            2
MF_MT_AM_FORMAT_TYPE            {05589F80-C356-11CE-BF01-00AA0055595A}
MF_MT_FRAME_RATE_RANGE_MAX      128849018881
MF_MT_FRAME_RATE_RANGE_MIN      42949672961333333


pVideoOutputType :

MF_MT_MAJOR_TYPE                MFMediaType_Video
MF_MT_SUBTYPE                   MFVideoFormat_RGB24
MF_MT_FRAME_SIZE                640 x 480
MF_MT_FRAME_RATE                10000000 x 333333
MF_MT_GEOMETRIC_APERTURE        <<byte array>>
MF_MT_PIXEL_ASPECT_RATIO        1 x 1
MF_MT_INTERLACE_MODE            2
MF_MT_DEFAULT_STRIDE            1920
MF_MT_ALL_SAMPLES_INDEPENDENT   1
MF_MT_FIXED_SIZE_SAMPLES        1
MF_MT_SAMPLE_SIZE               921600

EDIT2

Ok, your video input type seems to be correct.

Can you now log all input type (streamType) from all VP :

DWORD* inputCount = new DWORD[MFTcount];
DWORD* outputCount = new DWORD[MFTcount];

for(DWORD i = 0; i < MFTcount; i++){

    hr = VP[i]->GetStreamCount(&inputCount[i], &outputCount[i]);

    if(FAILED(hr)){
        exit(3);
    }
}

DWORD** inputids = new DWORD*[MFTcount];
DWORD** outputids = new DWORD*[MFTcount];

for(DWORD i = 0; i < MFTcount; i++){

    inputids[i] = new DWORD[inputCount[i]];
    outputids[i] = new DWORD[outputCount[i]];
}

for(DWORD i = 0; i < MFTcount; i++){

    hr = VP[i]->GetStreamIDs(inputCount[i], inputids[i], outputCount[i], outputids[i]);

    // By convention, if an MFT has exactly one fixed input stream and one fixed output stream, it should assign the identifier 0 to both streams
    if(hr == E_NOTIMPL && inputCount[i] == 1 && outputCount[i] == 1){

        inputids[i][0] = 0;
        outputids[i][0] = 0;
    }
    else if(FAILED(hr)){
        exit(4);
    }
}

for(DWORD i = 0; i < MFTcount; i++){

    // todo : log VP = i

    for(DWORD j = 0; j < inputCount[i]; j++){

        // todo : log stream id = inputids[i][j]

        DWORD dwTypeIndex = 0;
        hr = S_OK;

        while(hr == S_OK){

            // todo :log dwTypeIndex

            hr = VP[i]->GetInputAvailableType(inputids[i][j], dwTypeIndex, &streamType);

            if(SUCCEEDED(hr)){

                LogMediaType(streamType);
                SAFE_RELEASE(streamType);
                dwTypeIndex++;
            }
            else{

                // todo : check hr, should be MF_E_NO_MORE_TYPES
                // if hr == E_NOTIMPL/MF_E_INVALIDSTREAMNUMBER... should be error
            }
        }
    }
}

EDIT3

Here is a use of the Video Processor MFT : VideoProcessor MTF sample

It seems that you need to provide a D3DManager before using it (HRESULT DX11VideoRenderer::CPresenter::CreateXVP(void) -> line 1118)

I'm not sure for the D3DManager because i can't test it on my system.

hr = CoCreateInstance(CLSID_VideoProcessorMFT, nullptr, CLSCTX_INPROC_SERVER, IID_IMFTransform, (void**)&m_pXVP);
if (FAILED(hr))
{
    break;
}

hr = m_pXVP->ProcessMessage(MFT_MESSAGE_SET_D3D_MANAGER, ULONG_PTR(m_pDXGIManager));
if (FAILED(hr))
{
    break;
}

// Tell the XVP that we are the swapchain allocator
hr = m_pXVP->GetAttributes(&pAttributes);
if (FAILED(hr))
{
    break;
}

hr = pAttributes->SetUINT32(MF_XVP_PLAYBACK_MODE, TRUE);
if (FAILED(hr))
{
    break;
}

Check all m_pXVP and m_pXVPControl from the code.

EDIT4

According to your inputType, try to manually create the outputType, adding this attributes :

MF_MT_MAJOR_TYPE                MFMediaType_Video
MF_MT_SUBTYPE                   MFVideoFormat_RGB24
MF_MT_FRAME_SIZE                1280 x 720
MF_MT_FIXED_SIZE_SAMPLES        1
MF_MT_FRAME_RATE                30 x 1
MF_MT_PIXEL_ASPECT_RATIO        1 x 1
MF_MT_ALL_SAMPLES_INDEPENDENT   1
MF_MT_INTERLACE_MODE            2

So just after SetInputType, create the video media output type, and then call SetOutputType with this new mediaType.

You can first try only with MF_MT_MAJOR_TYPE/MF_MT_SUBTYPE/MF_MT_FRAME_SIZE, and then add others, one by one.



来源：https://stackoverflow.com/questions/51368961/setup-of-imtransform-video-processor-for-color-space-conversion

标签
visual-studio
image-processing
sdk
ms-media-foundation
color-space
易学教程内所有资源均来自网络或用户发布的内容，如有违反法律规定的内容欢迎反馈！
该文章没有解决你所遇到的问题?点击提问,说说你的问题,让更多的人一起探讨吧!