Camera frame yuv to rgb conversion using GL shader language

Question

I am getting the camera frame from the android camera Preview Callback in Byte array and pass it to jni code. As we can't use byte in c++ so i am converting it to the integer array as follows:

    JNIEXPORT void JNICALL Java_com_omobio_armadillo_Armadillo_onAndroidCameraFrameNative(
            JNIEnv* env, jobject, jbyteArray data, jint dataLen, jint width,
            jint height, jint bitsPerComponent) {
        Armadillo *armadillo = Armadillo::singleton();

        jbyte *jArr = env->GetByteArrayElements(data, NULL);
        int dataChar[dataLen];
        for (int i = 0; i < dataLen; i++) {
            dataChar[i] = (int) jArr[i];
    }

Then I am paasing it to the the CCImage to create a texture as folllows:

 void AppClass::drawAndroidCameraFrame() {

CCLOG("drawAndroidCameraFrame");
int nextBufferIndex = !_bufferIndex;
if (mIsNewFrameReceived) {
    mIsNewFrameReceived = false;
    return;
}
CCLOG("drawAndroidCameraFrame - creating CCImage");
_image[nextBufferIndex] = new CCImage();
_image[nextBufferIndex]->initWithImageData(mFramePData, mFrameDataLen,
        mFrameFormat, mFrameWidth, mFrameHeight, mBitsPerComponent);
if (mIsNewFrameReceived) {
    CCLOG("drawAndroidCameraFrame = relasing frame image");
    _image[nextBufferIndex]->release();
    mIsNewFrameReceived = false;
    CCLOG("camera frame process cancelled 2");
    return;
}
CCLOG("drawAndroidCameraFrame - creating texture2d");
_texture[nextBufferIndex] = new CCTexture2D();
_texture[nextBufferIndex]->initWithImage(_image[nextBufferIndex]);

    if (!_videoSprite) {
    CCLOG("Creating new sprite");

    if (mIsNewFrameReceived) {
        CCLOG("drawAndroidCameraFrame - releasing image an texture");
        _image[nextBufferIndex]->release();
        _texture[nextBufferIndex]->release();
        mIsNewFrameReceived = false;
        CCLOG("camera frame process cancelled 3");
        return;
    }

    CCLOG("drawAndroidCameraFrame - creating video sprite");
    _videoSprite = new CollisionBitmapSprite();
    _videoSprite->initWithTexture(_texture[nextBufferIndex]);

    //get director
    CCDirector *director = CCDirector::sharedDirector();

    // ask director the window size
    CCSize size = director->getWinSize();
    // position the sprite on the center of the screen
    _videoSprite->setPosition(ccp(size.width/2, size.height/2));

    //get scale factor
    CCSize* imageSize = new CCSize(_image[nextBufferIndex]->getWidth(),
            _image[nextBufferIndex]->getHeight());

    CCSize scale = getCameraFrameScaleFactor(*imageSize);
    //      CCLOG ("Scale factor is x=%f and y=%f", scale.width, scale.height);

    _videoSprite->setScaleX(scale.width);
    _videoSprite->setScaleY(scale.height);

    if (mIsNewFrameReceived) {
        _image[nextBufferIndex]->release();
        _texture[nextBufferIndex]->release();
        mIsNewFrameReceived = false;
        CCLOG("camera frame process cancelled 4");
        return;
    }

    _videoSprite->setTexture(_texture[nextBufferIndex]);

                Shaders::addProgram(_videoSprite, (char *)     Shaders::textureVertShader,
    mFrameWidth, mFrameHeight);
    GLuint i =Shaders::addProgram(_videoSprite, (char *) Shaders::vertShader,
            (char *) Shaders::yuvtorgb);
        Shaders::setYuvtorgbParameters(_videoSprite,i);
    addChild(_videoSprite, -1);

} else {
    _videoSprite->setTexture(_texture[nextBufferIndex]);
}
//  CCLOG ("Armadillo::drawCameraFrame completed successfully");
//release memory
if (_image[_bufferIndex]) {
    _image[_bufferIndex]->release();
}

if (_texture[_bufferIndex]) {
    _texture[_bufferIndex]->release();
}

_bufferIndex = nextBufferIndex;

 }

As the image is in YUV(N21) format so I am applying the shader to the frame which can convert the image frame in to rgb. The shader program is as follows:

Fragment Shader:

const char *Shaders::yuvtorgb = MULTI_LINE_STRING(
        precision highp float;
        varying vec2 v_yTexCoord;
        varying vec4 v_effectTexCoord;

        uniform sampler2D y_texture;
        uniform sampler2D u_texture;
        uniform sampler2D v_texture;

        void main()
        {
            float y = texture2D(y_texture, v_yTexCoord).r;
            float u = texture2D( u_texture, v_yTexCoord ).r;
            float v = texture2D( v_texture, v_yTexCoord ).r;


            y = 1.1643 * ( y - 0.0625 );

            u = u - 0.5;
            v = v - 0.5;

            float r = y + 1.5958 * v;
            float g = y - 0.39173 * u - 0.81290 * v;
            float b = y + 2.017 * u;

            gl_FragColor = vec4(r,g,b, 1.0);
        }
);

Vertex Shader:

const char *Shaders::vertShader = MULTI_LINE_STRING(
        attribute vec4 a_position;
        attribute vec2 a_yTexCoord;
        attribute vec4 a_effectTexCoord;

        varying vec2 v_yTexCoord;
        varying vec4 v_effectTexCoord;
        uniform mat4 u_MVPMatrix;
        void main()
        {
            v_yTexCoord = a_yTexCoord;
            v_effectTexCoord = a_effectTexCoord;
            gl_Position = u_MVPMatrix * a_position;
        }
);

Add Program method:

GLuint Shaders::addProgram(CCSprite *sprite, char *vertShader,
            char*fragShader) {
        CCGLProgram *glProgram = new CCGLProgram();
        if (!glProgram->initWithVertexShaderByteArray(vertShader, fragShader)) {
        CCLOG("Shader problem: %s\n %s \n%s", glProgram->vertexShaderLog(), glProgram->fragmentShaderLog(), glProgram->programLog());
    }

    glProgram->addAttribute(kCCAttributeNamePosition, kCCVertexAttrib_Position);
    glProgram->addAttribute(kCCAttributeNameTexCoord,
            kCCVertexAttrib_TexCoords);
    if (!glProgram->link()) {
        CCLOG(
                "Shader problem: %s\n %s \n%s",      glProgram->vertexShaderLog(), glProgram->fragmentShaderLog(), glProgram->programLog());
    }
    glProgram->updateUniforms();

    sprite->setShaderProgram(glProgram);
    return glProgram->getProgram();
}

Then I am applying the shader to the frame sprite:

GLuint i =Shaders::addProgram(_videoSprite, (char *) Shaders::vertShader,
                (char *) Shaders::yuvtorgb);

I am getting a image frame in green and pink color. The dark portion changes to green color and bright portion displays in pink color.

The resulting image url is as follows:

I am stuck over her and not finding any proper solution. Can any one help for solving the issue?

Answer 1

I am not sure if you are trying to get the RGB from the camera in order to use it outside the phone. But maybe you can convert YUV to RGB in the android,a nd then pass the RGB pixel array?

Here is the code I used to convert to RGB:

For openning the camera I used:

try {
    camera = Camera.open();
    cameraParam = camera.getParameters();
    cameraParam.setPreviewFormat(ImageFormat.NV21);
    List<int[]> fps = cameraParam.getSupportedPreviewFpsRange();
    camera.setDisplayOrientation(90);
    camera.setParameters(cameraParam);
    cameraParam = camera.getParameters();
    camera.startPreview();

    // wait for frames to come in
    camera.setPreviewCallback(new PreviewCallback() {
        @Override
        public void onPreviewFrame(byte[] data, Camera camera) {
            frameHeight = camera.getParameters().getPreviewSize().height;
             frameWidth = camera.getParameters().getPreviewSize().width;
             int rgb[] = new int[frameWidth * frameHeight]; // number of pixels
             // the following returns a pixel array in RGB format
             byte[] bytes = decodeYUV420SP(rgb, data, frameWidth, frameHeight);
        }
    });
} catch (Exception e) {
    Log.e("camera", "  error camera  ");
}

The decodeYUV420SP I got from a differnet post you can find it here.

And here is the code itself from the above post:

//  Byte decoder : ---------------------------------------------------------------------
int[] decodeYUV420SP(int[] rgb, byte[] yuv420sp, int width, int height) {
    Log.e("camera", "   decodeYUV420SP  ");
    Log.e("camera", "   Clearing Sums and Pixel Arrays  ");
    sumRED = 0;
    sumGREEN = 0;
    sumBLUE = 0;
    rStandardDeviation.clear();
    gStandardDeviation.clear();
    bStandardDeviation.clear();
    // TODO Auto-generated method stub
    final int frameSize = width * height;

    for (int j = 0, yp = 0; j < height; j++) {
        int uvp = frameSize + (j >> 1) * width, u = 0, v = 0;
        for (int i = 0; i < width; i++, yp++) {
            int y = (0xff & (yuv420sp[yp])) - 16;
            if (y < 0)
                y = 0;
            if ((i & 1) == 0) {
                v = (0xff & yuv420sp[uvp++]) - 128;
                u = (0xff & yuv420sp[uvp++]) - 128;
            }

            int y1192 = 1192 * y;
            int r = (y1192 + 1634 * v);
            int g = (y1192 - 833 * v - 400 * u);
            int b = (y1192 + 2066 * u);

            if (r < 0)
                r = 0;
            else if (r > 262143)
                r = 262143;
            if (g < 0)
                g = 0;
            else if (g > 262143)
                g = 262143;
            if (b < 0)
                b = 0;
            else if (b > 262143)
                b = 262143;

            rgb[yp] = 0xff000000 | ((r << 6) & 0xff0000) | ((g >> 2) & 0xff00) | ((b >> 10) & 0xff);

        }
    }
    return rgb;
} 

Then once you read back the return rgb pixel array by running decodeYUV420SP you can reconstruct the image by:

Bitmap bitmap= BitmapFactory.decodeByteArray(bytes, 0, bytes.length);

Hope it helps. My code might have errors, double check stuff, but in general it works for me.