Rotate camera preview to Portrait Android OpenCV Camera

Question

I am trying to use OpenCV 2.4.3.2 to create a camera app and do some opencv processing. I would like it to be able to have multiple UI orientations, not just Landscape.

Answer 1

I have the same issue, i have had figure out it!! and there is my solution:

as the part of first, In CameraBridgeViewBase.Java, the two constructor, add initialization of WindowManager:

public CameraBridgeViewBase(Context context, int cameraId) {  
   super(context);  
   mCameraIndex = cameraId;  
   getHolder().addCallback(this);  
   mMaxWidth = MAX_UNSPECIFIED;  
   mMaxHeight = MAX_UNSPECIFIED;  
   windowManager = (WindowManager) context.getSystemService(Context.WINDOW_SERVICE);  
}  


public CameraBridgeViewBase(Context context, AttributeSet attrs) {  
   super(context, attrs);  
   int count = attrs.getAttributeCount();  
   Log.d(TAG, "Attr count: " + Integer.valueOf(count));  

   TypedArray styledAttrs = getContext().obtainStyledAttributes(attrs, R.styleable.CameraBridgeViewBase);  
   if (styledAttrs.getBoolean(R.styleable.CameraBridgeViewBase_show_fps, false))  
       enableFpsMeter();  

   mCameraIndex = styledAttrs.getInt(R.styleable.CameraBridgeViewBase_camera_id, -1);  

   getHolder().addCallback(this);  
   mMaxWidth = MAX_UNSPECIFIED;  
   mMaxHeight = MAX_UNSPECIFIED;  
   windowManager = (WindowManager) context.getSystemService(Context.WINDOW_SERVICE);  
   styledAttrs.recycle();  
}  

then ,You need to replace function deliverAndDrawFrame(CvCameraViewFrame frame) as follows ,

protected void deliverAndDrawFrame(CvCameraViewFrame frame) {  
  Mat modified;  

  if (mListener != null) {  
      modified = mListener.onCameraFrame(frame);  
  } else {  
      modified = frame.rgba();  
  }  

  boolean bmpValid = true;  
  if (modified != null) {  
      try {  
          Utils.matToBitmap(modified, mCacheBitmap);  
      } catch (Exception e) {  
          Log.e(TAG, "Mat type: " + modified);  
          Log.e(TAG, "Bitmap type: " + mCacheBitmap.getWidth() + "*" + mCacheBitmap.getHeight());  
          Log.e(TAG, "Utils.matToBitmap() throws an exception: " + e.getMessage());  
          bmpValid = false;  
      }  
  }  

  if (bmpValid && mCacheBitmap != null) {  
      Canvas canvas = getHolder().lockCanvas();  
      if (canvas != null) {  
          canvas.drawColor(0, android.graphics.PorterDuff.Mode.CLEAR);  
          int rotation = windowManager.getDefaultDisplay().getRotation();  
          int degrees = 0;  
          // config degrees as you need  
          switch (rotation) {  
              case Surface.ROTATION_0:  
                  degrees = 90;  
                  break;  
              case Surface.ROTATION_90:  
                  degrees = 0;  
                  break;  
              case Surface.ROTATION_180:  
                  degrees = 270;  
                  break;  
              case Surface.ROTATION_270:  
                  degrees = 180;  
                  break;  
          }  

          Matrix matrix = new Matrix();  
          matrix.postRotate(degrees);  
          Bitmap outputBitmap = Bitmap.createBitmap(mCacheBitmap, 0, 0, mCacheBitmap.getWidth(), mCacheBitmap.getHeight(), matrix, true);  

          if (outputBitmap.getWidth() <= canvas.getWidth()) {  
              mScale = getRatio(outputBitmap.getWidth(), outputBitmap.getHeight(), canvas.getWidth(), canvas.getHeight());  
          } else {  
              mScale = getRatio(canvas.getWidth(), canvas.getHeight(), outputBitmap.getWidth(), outputBitmap.getHeight());  
          }  

          if (mScale != 0) {  
              canvas.scale(mScale, mScale, 0, 0);  
          }  
          Log.d(TAG, "mStretch value: " + mScale);  

          canvas.drawBitmap(outputBitmap, 0, 0, null);  

          if (mFpsMeter != null) {  
              mFpsMeter.measure();  
              mFpsMeter.draw(canvas, 20, 30);  
          }  
          getHolder().unlockCanvasAndPost(canvas);
      }  
   }  
}  

and add this function extra,

private float getRatio(int widthSource, int heightSource, int widthTarget, int heightTarget) {  
   if (widthTarget <= heightTarget) {  
       return (float) heightTarget / (float) heightSource;  
   } else {  
       return (float) widthTarget / (float) widthSource;  
   }  
}  

it's okey ,and If this answer useful to you, please mark 'accepted' Help Reputation

Answer 2

Another solution. i think this better

protected void deliverAndDrawFrame(CvCameraViewFrame frame) {
    Mat modified;

    if (mListener != null) {
        modified = mListener.onCameraFrame(frame);
    } else {
        modified = frame.rgba();
    }

    boolean bmpValid = true;
    if (modified != null) {
        try {
            Utils.matToBitmap(modified, mCacheBitmap);
        } catch(Exception e) {
            Log.e(TAG, "Mat type: " + modified);
            Log.e(TAG, "Bitmap type: " + mCacheBitmap.getWidth() + "*" + mCacheBitmap.getHeight());
            Log.e(TAG, "Utils.matToBitmap() throws an exception: " + e.getMessage());
            bmpValid = false;
        }
    }

    if (bmpValid && mCacheBitmap != null) {
        Canvas canvas = getHolder().lockCanvas();
        if (canvas != null) {
            canvas.drawColor(0, android.graphics.PorterDuff.Mode.CLEAR);
            if (BuildConfig.DEBUG)
                Log.d(TAG, "mStretch value: " + mScale);

            int currentOrientation = getResources().getConfiguration().orientation;
            if (currentOrientation == Configuration.ORIENTATION_LANDSCAPE) {
                if (mScale != 0) {
                    canvas.drawBitmap(mCacheBitmap, new Rect(0,0,mCacheBitmap.getWidth(), mCacheBitmap.getHeight()),
                            new Rect((int)((canvas.getWidth() - mScale*mCacheBitmap.getWidth()) / 2),
                                    (int)((canvas.getHeight() - mScale*mCacheBitmap.getHeight()) / 2),
                                    (int)((canvas.getWidth() - mScale*mCacheBitmap.getWidth()) / 2 + mScale*mCacheBitmap.getWidth()),
                                    (int)((canvas.getHeight() - mScale*mCacheBitmap.getHeight()) / 2 + mScale*mCacheBitmap.getHeight())), null);
                } else {
                    canvas.drawBitmap(mCacheBitmap, new Rect(0,0,mCacheBitmap.getWidth(), mCacheBitmap.getHeight()),
                            new Rect((canvas.getWidth() - mCacheBitmap.getWidth()) / 2,
                                    (canvas.getHeight() - mCacheBitmap.getHeight()) / 2,
                                    (canvas.getWidth() - mCacheBitmap.getWidth()) / 2 + mCacheBitmap.getWidth(),
                                    (canvas.getHeight() - mCacheBitmap.getHeight()) / 2 + mCacheBitmap.getHeight()), null);
                }
            } else {

                if (mScale != 0) {
                    Bitmap bitmap = Bitmap.createScaledBitmap(mCacheBitmap, canvas.getHeight(), canvas.getWidth(), true);
                    canvas.drawBitmap(bitmap, new Rect(0,0,bitmap.getWidth(), bitmap.getHeight()), new Rect(
                            (int)((canvas.getWidth() - mScale*bitmap.getWidth()) / 2),
                            (int)(0),
                            (int)((canvas.getWidth() - mScale*bitmap.getWidth()) / 2 + mScale*bitmap.getWidth()),
                            (int)((canvas.getHeight()))), null);
                } else {
                    Bitmap bitmap = Bitmap.createScaledBitmap(mCacheBitmap, canvas.getHeight(), canvas.getWidth(), true);
                    canvas.drawBitmap(bitmap, new Rect(0,0,bitmap.getWidth(), bitmap.getHeight()), new Rect(
                            (int)((canvas.getWidth() - bitmap.getWidth()) / 2),
                            (int)(0),
                            (int)((canvas.getWidth() - bitmap.getWidth()) / 2 + bitmap.getWidth()),
                            (int)(canvas.getHeight())), null);
                }
            }

            if (mFpsMeter != null) {
                mFpsMeter.measure();
                mFpsMeter.draw(canvas, 20, 30);
            }
            getHolder().unlockCanvasAndPost(canvas);
        }
    }
}

and...

@Override
public Mat onCameraFrame(CameraBridgeViewBase.CvCameraViewFrame inputFrame) {
    MatOfRect faces = new MatOfRect();
    int currentOrientation = getResources().getConfiguration().orientation;
    if (currentOrientation == Configuration.ORIENTATION_LANDSCAPE) { 
        mRgba = inputFrame.rgba();
        mGray = inputFrame.gray();

        int height = mGray.rows();
        if (Math.round(height * 0.2) > 0) {
            mFaceSize = (int) Math.round(height * 0.2);
        }

        cascadeClassifier.detectMultiScale(mGray, faces, 1.1, 3, 2,
                new Size(mFaceSize, mFaceSize));
        Rect[] facesArray = faces.toArray();
        for (int i = 0; i < facesArray.length; i++) {
            Point center = new Point(facesArray[i].x + facesArray[i].width / 2,
                    facesArray[i].y + facesArray[i].height / 2);
            rectangle(mRgba, facesArray[i].tl(), facesArray[i].br(), FACE_RECT_COLOR, 3);
        }

    } else {
        mRgba = inputFrame.rgba();
        mGray = inputFrame.gray();

        Mat rotImage = Imgproc.getRotationMatrix2D(new Point(mRgba.cols() / 2,
                mRgba.rows() / 2), 90, 1.0);

        Imgproc.warpAffine(mRgba, mRgba, rotImage, mRgba.size());
        Imgproc.warpAffine(mGray, mGray, rotImage, mRgba.size());

        Core.flip(mRgba, mRgba, 1);
        Core.flip(mGray, mGray, 1);

        int height = mGray.rows();
        if (Math.round(height * 0.2) > 0) {
            mFaceSize = (int) Math.round(height * 0.2);
        }

        cascadeClassifier.detectMultiScale(mGray, faces, 1.1, 3, 2,
                new Size(mFaceSize, mFaceSize));
        Rect[] facesArray = faces.toArray();
        for (int i = 0; i < facesArray.length; i++) {
            Point center = new Point(facesArray[i].x + facesArray[i].width / 2,
                    facesArray[i].y + facesArray[i].height / 2);
            rectangle(mRgba, facesArray[i].tl(), facesArray[i].br(), FACE_RECT_COLOR, 3);
        }
    }

    return mRgba;

Answer 3

All answers here are hacks. i prefer this solution :

change in JavaCameraView code :

mBuffer = new byte[size];
mCamera.setDisplayOrientation(90); //add this
mCamera.addCallbackBuffer(mBuffer);

Second Change :

//                    if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.HONEYCOMB) {
//                        mSurfaceTexture = new SurfaceTexture(MAGIC_TEXTURE_ID);
//                        mCamera.setPreviewTexture(mSurfaceTexture);
//                    } else
//                       mCamera.setPreviewDisplay(null);
                    mCamera.setPreviewDisplay(getHolder());

Answer 4

I don't know well, but camera size is decided from screen width. Because screen width is low, camera height also is decided by low in the portrait orientation. Therefore, camera resolution is decided by low too. And preview image lay down (preview image rotation is decided as camera image's width and height in CameraBridgeViewBase.java).

As the solution, use landscape orientation (decide landscape mode in the manifest.xml as Activity). As a result, because screen width is high, height will be high also and your app decide high resolution. Also, you don't have to rotate the camera image and always full-screen mode. But the disadvantage is the origin point is different. I tried variety method about high-resolution image as the portrait orientation but I couldn't find a way.

My app : portrait orientation

my camera image is 720, 480 / landscape orientation 1280, 1080.

Answer 5

Maybe this help anyone. tested on android 9 with opencv343. Now this full screen and DETECT face in portrait and landscape modes. small changes in CameraBridgeViewBase class:

private final Matrix matrix = new Matrix();

...... change deliverAndDrawFrame() method:

protected void deliverAndDrawFrame(CvCameraViewFrame frame) {
    Mat modified;

    if (mListener != null) {
        modified = mListener.onCameraFrame(frame);
    } else {
        modified = frame.rgba();
    }

    boolean bmpValid = true;
    if (modified != null) {
        try {
            Utils.matToBitmap(modified, mCacheBitmap);
        } catch(Exception e) {
            Log.e(TAG, "Mat type: " + modified);
            Log.e(TAG, "Bitmap type: " + mCacheBitmap.getWidth() + "*" + mCacheBitmap.getHeight());
            Log.e(TAG, "Utils.matToBitmap() throws an exception: " + e.getMessage());
            bmpValid = false;
        }
    }

    if (bmpValid && mCacheBitmap != null) {
        int currentOrientation = getResources().getConfiguration().orientation;
        if (currentOrientation == Configuration.ORIENTATION_LANDSCAPE) {
            Canvas canvas = getHolder().lockCanvas();
            if (canvas != null) {
                canvas.drawColor(0, android.graphics.PorterDuff.Mode.CLEAR);
                if (BuildConfig.DEBUG)
                    Log.d(TAG, "mStretch value: " + mScale);

                if (mScale != 0) {
                    canvas.drawBitmap(mCacheBitmap, new Rect(0,0,mCacheBitmap.getWidth(), mCacheBitmap.getHeight()),
                            new Rect((int)((canvas.getWidth() - mScale*mCacheBitmap.getWidth()) / 2),
                                    (int)((canvas.getHeight() - mScale*mCacheBitmap.getHeight()) / 2),
                                    (int)((canvas.getWidth() - mScale*mCacheBitmap.getWidth()) / 2 + mScale*mCacheBitmap.getWidth()),
                                    (int)((canvas.getHeight() - mScale*mCacheBitmap.getHeight()) / 2 + mScale*mCacheBitmap.getHeight())), null);

                } else {
                    canvas.drawBitmap(mCacheBitmap, new Rect(0,0,mCacheBitmap.getWidth(), mCacheBitmap.getHeight()),
                            new Rect((canvas.getWidth() - mCacheBitmap.getWidth()) / 2,
                                    (canvas.getHeight() - mCacheBitmap.getHeight()) / 2,
                                    (canvas.getWidth() - mCacheBitmap.getWidth()) / 2 + mCacheBitmap.getWidth(),
                                    (canvas.getHeight() - mCacheBitmap.getHeight()) / 2 + mCacheBitmap.getHeight()), null);
                }

                if (mFpsMeter != null) {
                    mFpsMeter.measure();
                    mFpsMeter.draw(canvas, 20, 30);
                }
                getHolder().unlockCanvasAndPost(canvas);
            }
        } else {
            Canvas canvas = getHolder().lockCanvas();
            if (canvas != null) {
                int saveCount = canvas.save();
                canvas.setMatrix(matrix);
                mScale = Math.max((float) canvas.getHeight() / mCacheBitmap.getWidth(), (float) canvas.getWidth() / mCacheBitmap.getHeight());
                canvas.drawColor(0, android.graphics.PorterDuff.Mode.CLEAR);

                if (mScale != 0) {
                    canvas.drawBitmap(mCacheBitmap, new Rect(0,0,mCacheBitmap.getWidth(), mCacheBitmap.getHeight()),
                            new Rect((int)((canvas.getWidth() - mCacheBitmap.getWidth()) - mCacheBitmap.getWidth())/2,
                                    (int)(canvas.getHeight() - mScale*mCacheBitmap.getHeight() - mScale*mCacheBitmap.getHeight()/2),
                                    (int)((canvas.getWidth() - mScale*mCacheBitmap.getWidth()) / 2 + mScale*mCacheBitmap.getWidth()),
                                    (int)((canvas.getHeight() - mCacheBitmap.getHeight()) / 2 + mScale*mCacheBitmap.getHeight())), null);

                } else {
                    canvas.drawBitmap(mCacheBitmap, new Rect(0, 0, mCacheBitmap.getWidth(), mCacheBitmap.getHeight()),
                            new Rect((canvas.getWidth() - mCacheBitmap.getWidth()) / 2,
                                    (canvas.getHeight() - mCacheBitmap.getHeight()) / 2,
                                    (canvas.getWidth() - mCacheBitmap.getWidth()) / 2 + mCacheBitmap.getWidth(),
                                    (canvas.getHeight() - mCacheBitmap.getHeight()) / 2 + mCacheBitmap.getHeight()), null);
                }
                canvas.restoreToCount(saveCount);

                if (mFpsMeter != null) {
                    mFpsMeter.measure();
                    mFpsMeter.draw(canvas, 20, 30);
                }
                getHolder().unlockCanvasAndPost(canvas);
            }
        }
    }
}

and in MainActivity:

public Mat rotateMat(Mat matImage) {
    Mat rotated = matImage.t();
    Core.flip(rotated, rotated, 1);
    return rotated;
}

@Override
public Mat onCameraFrame(CameraBridgeViewBase.CvCameraViewFrame inputFrame) {
    MatOfRect faces = new MatOfRect();
    int currentOrientation = getResources().getConfiguration().orientation;
    if (currentOrientation == Configuration.ORIENTATION_LANDSCAPE) {
        mRgba = inputFrame.rgba();
        mGray = inputFrame.gray();
        int height = mGray.rows();
        if (Math.round(height * 0.2) > 0) {
            mFaceSize = (int) Math.round(height * 0.2);
        }

        cascadeClassifier.detectMultiScale(mGray, faces, 1.1, 3, 2,
                new Size(mFaceSize, mFaceSize));
        Rect[] facesArray = faces.toArray();
        for (int i = 0; i < facesArray.length; i++) {
            rectangle(mRgba, facesArray[i].tl(), facesArray[i].br(), FACE_RECT_COLOR, 3);
        }
    } else {

        mRgba = inputFrame.rgba();
        mGray = rotateMat(inputFrame.gray());

        if (mFaceSize == 0) {
            int height = mGray.cols();
            if (Math.round(height * 0.2) > 0) {
                mFaceSize = (int) Math.round(height * 0.2);
            }
        }
        Mat newMat = rotateMat(mRgba);
        if(!isBackCameraOn){
            flip(newMat, newMat, -1);
            flip(mGray, mGray, -1);
        }
        if (cascadeClassifier != null)
            cascadeClassifier.detectMultiScale(mGray, faces, 1.1, 3, 2, new Size(mFaceSize, mFaceSize));
        mGray.release();

        Rect[] facesArray = faces.toArray();
        for (int i = 0; i < facesArray.length; i++) {
            rectangle(newMat, facesArray[i].tl(), facesArray[i].br(), FACE_RECT_COLOR, 3);
        }

        Imgproc.resize(newMat, mRgba, new Size(mRgba.width(), mRgba.height()));

        newMat.release();
    }

    if(!isBackCameraOn){
        flip(mRgba, mRgba, 1);
        flip(mGray, mGray, 1);
    }


    return mRgba;
}

Answer 6

As in other answers, I've written my personal version of deliverAndDrawFrame (I've also notified through comments where my code starts and ends):

protected void deliverAndDrawFrame(CvCameraViewFrame frame) {
    Mat modified;

    if (mListener != null) {
        modified = mListener.onCameraFrame(frame);
    } else {
        modified = frame.rgba();
    }

    boolean bmpValid = true;
    if (modified != null) {
        try {
            Utils.matToBitmap(modified, mCacheBitmap);
        } catch(Exception e) {
            Log.e(TAG, "Mat type: " + modified);
            Log.e(TAG, "Bitmap type: " + mCacheBitmap.getWidth() + "*" + mCacheBitmap.getHeight());
            Log.e(TAG, "Utils.matToBitmap() throws an exception: " + e.getMessage());
            bmpValid = false;
        }
    }

    if (bmpValid && mCacheBitmap != null) {
        Canvas canvas = getHolder().lockCanvas();
        if (canvas != null) {
            canvas.drawColor(0, android.graphics.PorterDuff.Mode.CLEAR);
            if (BuildConfig.DEBUG) {
                Log.d(TAG, "mStretch value: " + mScale);
            }

            // Start of the fix
            Matrix matrix = new Matrix();
            matrix.preTranslate( ( canvas.getWidth() - mCacheBitmap.getWidth() ) / 2f, ( canvas.getHeight() - mCacheBitmap.getHeight() ) / 2f );
            matrix.postRotate( 90f, ( canvas.getWidth()) / 2f, canvas.getHeight() / 2f );
            float scale = (float) canvas.getWidth() / (float) mCacheBitmap.getHeight();
            matrix.postScale(scale, scale, canvas.getWidth() / 2f , canvas.getHeight() / 2f );
            canvas.drawBitmap( mCacheBitmap, matrix, null );

            // Back to original OpenCV code
            if (mFpsMeter != null) {
                mFpsMeter.measure();
                mFpsMeter.draw(canvas, 20, 30);
            }

            getHolder().unlockCanvasAndPost(canvas);
        }
    }

}

Preview is now in Portrait mode, as you can see: