How to convert & rotate raw NV21 array image (android.media.Image) from front cam portrait mode in onImageAvailable (android Camera2)?

Question

Note: All info in my post only goes for Samsung Galaxy S7 device. I do not know how emulators and other devices behave.

In onImageAvailable I conver

Answer 1

With OpenCV and Android Camera API 2 this task is very fast and you don't need YUV420toNV21 Java conversion, and with OpenCV this convertion is 4x more fast:

Java side:

//Starts a builtin camera with api camera 2
public void startCamera() {
    CameraManager manager = (CameraManager) AppData.getAppContext().getSystemService(Context.CAMERA_SERVICE);
    try {
        String pickedCamera = getCamera(manager);
        manager.openCamera(pickedCamera, cameraStateCallback, null);
        // set image format on YUV
        mImageReader = ImageReader.newInstance(mWidth,mHeight, ImageFormat.YUV_420_888, 4);
        mImageReader.setOnImageAvailableListener(onImageAvailableListener, null);
        Log.d(TAG, "imageReader created");
    } catch (CameraAccessException e) {
        Log.e(TAG, e.getMessage());
    }
}


//Listens for frames and send them to  be processed
protected ImageReader.OnImageAvailableListener onImageAvailableListener = new ImageReader.OnImageAvailableListener() {
    @Override
    public void onImageAvailable(ImageReader reader) {
        Image image = null;
        try {

            image = reader.acquireLatestImage();
            ByteBuffer buffer = image.getPlanes()[0].getBuffer();
            byte[] frameData = new byte[buffer.capacity()];
            buffer.get(frameData);
            // Native process (see below)
            processAndRotateFrame(frameData);
            image.close();

        } catch (Exception e) {
            Logger.e(TAG, "imageReader exception: "+e.getMessage());

        } finally {
            if (image != null) {
                image.close();
            }
        }
    }
};

Native side (NDK or Cmake):

JNIEXPORT jint JNICALL  com_android_mvf_Utils_ProccessAndRotateFrame
    (JNIEnv *env, jobject object, jint width, jint height, jbyteArray frame, jint rotation) {
    // load data from JAVA side
    jbyte *pFrameData = env->GetByteArrayElements(frame, 0);
    // convert array to Mat, for example GRAY or COLOR
    Mat mGray(height, width, cv::IMREAD_GRAYSCALE, (unsigned char *)pFrameData);
    // rotate image
    rotateMat(mGray, rotation);
    int objects = your_function(env, mGray);
    env->ReleaseByteArrayElements(frame, pFrameData, 0);
    return objects;
}

void rotateMat(cv::Mat &matImage, int rotFlag) {
    if (rotFlag != 0 && rotFlag != 360) {
        if (rotFlag == 90) {
            cv::transpose(matImage, matImage);
            cv::flip(matImage, matImage, 1);
        } else if (rotFlag == 270 || rotFlag == -90) {
            cv::transpose(matImage, matImage);
            cv::flip(matImage, matImage, 0);
        } else if (rotFlag == 180) {
            cv::flip(matImage, matImage, -1);
        }
    }
}