Skip to content

Instantly share code, notes, and snippets.

@liberaid2

liberaid2/CustomCameraBridge.kt

Created December 7, 2019 04:18
Show Gist options
  • Save liberaid2/29311f7dfaeb4565240fd1d45c4d2f77 to your computer and use it in GitHub Desktop.
Save liberaid2/29311f7dfaeb4565240fd1d45c4d2f77 to your computer and use it in GitHub Desktop.
Download ZIP
Custom bridge between camera and OpenCV for Android. This approach uses Camera2 API. Also there is a snippet for handling wrong camera rotation.
Raw
CustomCameraBridge.kt
import android.Manifest
import android.annotation.SuppressLint
import android.content.Context
import android.graphics.Bitmap
import android.graphics.ImageFormat
import android.hardware.camera2.CameraCaptureSession
import android.hardware.camera2.CameraCharacteristics
import android.hardware.camera2.CameraDevice
import android.hardware.camera2.CameraManager
import android.media.ImageReader
import android.os.Handler
import android.os.HandlerThread
import org.opencv.android.CameraBridgeViewBase.CvCameraViewFrame
import org.opencv.android.Utils
import org.opencv.core.Mat
import pub.devrel.easypermissions.EasyPermissions
import timber.log.Timber
class CustomCameraBridge(private val context: Context, private val changeListener: IChangeListener) {
companion object {
private const val IMG_FORMAT = ImageFormat.YUV_420_888
private const val MAX_IMAGES = 8
}
private var mWidth = -1
private var mHeight = -1
private var bitmapPreview: Bitmap? = null
private val bgThread = HandlerThread("CameraBridgeThread").apply { start() }
private val bgHandler = Handler(bgThread.looper)
private val mCameraCallback = object : CameraDevice.StateCallback() {
override fun onOpened(camera: CameraDevice) {
Timber.d("Camera opened, id=${camera.id}")
setupCameraPreview(camera)
}
override fun onDisconnected(camera: CameraDevice) {
camera.close()
Timber.d("Camera disconnected, id=${camera.id}")
}
override fun onError(camera: CameraDevice, error: Int) {
Timber.w("Camera error, id=${camera.id}, error=$error")
}
}
fun stopCamera() {
bgThread.quitSafely()
try{
bgThread.join()
} catch (e: Exception) {
Timber.e(e, "Cannot join background thread")
}
}
@SuppressLint("MissingPermission")
fun runCamera(): Boolean {
val manager = context.getSystemService(Context.CAMERA_SERVICE) as CameraManager
val cameraIds = try {
manager.cameraIdList
} catch (e: Exception) {
Timber.w(e)
return false
}
Timber.d("Number of cameras: ${cameraIds.size}")
check(cameraIds.isNotEmpty()) { "Cannot find camera for this device" }
val backCameraId = cameraIds.find {
val chars = manager.getCameraCharacteristics(it)
chars[CameraCharacteristics.LENS_FACING] == CameraCharacteristics.LENS_FACING_BACK
} ?: throw RuntimeException("Cannot find back facing camera")
if(!EasyPermissions.hasPermissions(context, Manifest.permission.CAMERA))
throw IllegalStateException("Camera permission is not granted")
val backCameraChars = manager.getCameraCharacteristics(backCameraId)
val sizes = backCameraChars.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)
?.getOutputSizes(IMG_FORMAT)
?: throw RuntimeException("Cannot get sizes for camera, format=${IMG_FORMAT}")
Timber.d("Available sizes: ${sizes.toList()}")
check(sizes.isNotEmpty()) { "There are no sizes for format=${IMG_FORMAT}" }
mWidth = sizes[16].width
mHeight = sizes[16].height
manager.openCamera(backCameraId, mCameraCallback, bgHandler)
return true
}
private fun setupCameraPreview(camera: CameraDevice) {
val width = mWidth
val height = mHeight
check(width > 0 && height > 0) { "Invalid size, width=$width, height=$height" }
bitmapPreview = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888)
val reader = ImageReader.newInstance(width, height, IMG_FORMAT, MAX_IMAGES)
val surface = reader.surface
val surfaces = mutableListOf(surface)
val request = camera.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW).apply {
surfaces.forEach { addTarget(it) }
}
camera.createCaptureSession(surfaces, object : CameraCaptureSession.StateCallback() {
override fun onConfigureFailed(session: CameraCaptureSession) {
Timber.w("Failed to configure camera, session=$session")
}
override fun onConfigured(session: CameraCaptureSession) {
Timber.d("Camera session configured, session=$session")
session.setRepeatingRequest(request.build(), null, null)
}
}, bgHandler)
reader.setOnImageAvailableListener({ image ->
image?.acquireLatestImage()?.use {
val frame = JavaCamera2Frame(it)
val mat = changeListener.onFrameChanged(frame)
Utils.matToBitmap(mat, bitmapPreview)
frame.release()
}
changeListener.onBitmap(bitmapPreview!!)
}, bgHandler)
}
interface IChangeListener {
fun onFrameChanged(frame: CvCameraViewFrame): Mat
fun onBitmap(bitmap: Bitmap)
}
}
Raw
IChangeListenerExample.kt
val bridge = CustomCameraBridge(this@MainActivity, object : CustomCameraBridge.IChangeListener{
var transformMatrix: Matrix? = null
var srcBitmap = getBitmapFromAssets()
val srcDescriptors = Mat()
val frameDescriptors = Mat()
init {
val mat = Mat()
Utils.bitmapToMat(srcBitmap, mat)
JniBridge.setupDescriptors(mat.nativeObjAddr)
Utils.matToBitmap(mat, srcBitmap)
}
override fun onFrameChanged(frame: CameraBridgeViewBase.CvCameraViewFrame): Mat {
val gray = frame.gray()
val rgba = frame.rgba()
JniBridge.findMatches(gray.nativeObjAddr, rgba.nativeObjAddr, frameDescriptors.nativeObjAddr);
return rgba
}
override fun onBitmap(bitmap: Bitmap) {
val canvas = ivPreview.lockCanvas() ?: return
canvas.drawColor(Color.GRAY)
transformMatrix = transformMatrix ?: Matrix().apply {
val cx = ivPreview.width / 2f
val cy = ivPreview.height / 2f
val bx = bitmap.width / 2f
val by = bitmap.height / 2f
setTranslate(-bx + cx, -by + cy)
postRotate(90f, cx, cy)
}
canvas.drawBitmap(bitmap, transformMatrix!!, null)
ivPreview.unlockCanvasAndPost(canvas)
}
})
bridge.runCamera()
Raw
JavaCamera2Frame.java
import android.media.Image;
import org.opencv.android.CameraBridgeViewBase;
import org.opencv.core.CvType;
import org.opencv.core.Mat;
import org.opencv.imgproc.Imgproc;
import java.nio.ByteBuffer;
final class JavaCamera2Frame implements CameraBridgeViewBase.CvCameraViewFrame {
@Override
public Mat gray() {
Image.Plane[] planes = mImage.getPlanes();
int w = mImage.getWidth();
int h = mImage.getHeight();
ByteBuffer y_plane = planes[0].getBuffer();
mGray = new Mat(h, w, CvType.CV_8UC1, y_plane);
return mGray;
}
@Override
public Mat rgba() {
Image.Plane[] planes = mImage.getPlanes();
int w = mImage.getWidth();
int h = mImage.getHeight();
int chromaPixelStride = planes[1].getPixelStride();
if (chromaPixelStride == 2) { // Chroma channels are interleaved
assert(planes[0].getPixelStride() == 1);
assert(planes[2].getPixelStride() == 2);
ByteBuffer y_plane = planes[0].getBuffer();
ByteBuffer uv_plane1 = planes[1].getBuffer();
ByteBuffer uv_plane2 = planes[2].getBuffer();
Mat y_mat = new Mat(h, w, CvType.CV_8UC1, y_plane);
Mat uv_mat1 = new Mat(h / 2, w / 2, CvType.CV_8UC2, uv_plane1);
Mat uv_mat2 = new Mat(h / 2, w / 2, CvType.CV_8UC2, uv_plane2);
long addr_diff = uv_mat2.dataAddr() - uv_mat1.dataAddr();
if (addr_diff > 0) {
assert(addr_diff == 1);
Imgproc.cvtColorTwoPlane(y_mat, uv_mat1, mRgba, Imgproc.COLOR_YUV2RGBA_NV12);
} else {
assert(addr_diff == -1);
Imgproc.cvtColorTwoPlane(y_mat, uv_mat2, mRgba, Imgproc.COLOR_YUV2RGBA_NV21);
}
return mRgba;
} else { // Chroma channels are not interleaved
byte[] yuv_bytes = new byte[w*(h+h/2)];
ByteBuffer y_plane = planes[0].getBuffer();
ByteBuffer u_plane = planes[1].getBuffer();
ByteBuffer v_plane = planes[2].getBuffer();
y_plane.get(yuv_bytes, 0, w*h);
int chromaRowStride = planes[1].getRowStride();
int chromaRowPadding = chromaRowStride - w/2;
int offset = w*h;
if (chromaRowPadding == 0){
// When the row stride of the chroma channels equals their width, we can copy
// the entire channels in one go
u_plane.get(yuv_bytes, offset, w*h/4);
offset += w*h/4;
v_plane.get(yuv_bytes, offset, w*h/4);
} else {
// When not equal, we need to copy the channels row by row
for (int i = 0; i < h/2; i++){
u_plane.get(yuv_bytes, offset, w/2);
offset += w/2;
if (i < h/2-1){
u_plane.position(u_plane.position() + chromaRowPadding);
}
}
for (int i = 0; i < h/2; i++){
v_plane.get(yuv_bytes, offset, w/2);
offset += w/2;
if (i < h/2-1){
v_plane.position(v_plane.position() + chromaRowPadding);
}
}
}
Mat yuv_mat = new Mat(h+h/2, w, CvType.CV_8UC1);
yuv_mat.put(0, 0, yuv_bytes);
Imgproc.cvtColor(yuv_mat, mRgba, Imgproc.COLOR_YUV2RGBA_I420, 4);
return mRgba;
}
}
public JavaCamera2Frame(Image image) {
super();
mImage = image;
mRgba = new Mat();
mGray = new Mat();
}
public void release() {
mRgba.release();
mGray.release();
}
private Image mImage;
private Mat mRgba;
private Mat mGray;
};
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment