Skip to content

Instantly share code, notes, and snippets.

@BoHellgren

BoHellgren/main.dart

Last active October 8, 2021 14:34
Show Gist options
  • Save BoHellgren/00390ba963933249218e513e41dfede4 to your computer and use it in GitHub Desktop.
Save BoHellgren/00390ba963933249218e513e41dfede4 to your computer and use it in GitHub Desktop.
Download ZIP
main.dart for MLKit version
Raw
main.dart
import 'package:flutter/material.dart';
import 'package:flutter/services.dart';
import 'package:camera/camera.dart';
import 'package:soundpool/soundpool.dart';
import 'dart:ui' as ui;
import 'dart:typed_data';
import 'package:image/image.dart' as imglib;
import 'package:flutter/foundation.dart';
import 'breedinfo.dart';
import 'package:image_gallery_saver/image_gallery_saver.dart';
import 'package:permission_handler/permission_handler.dart';
import 'package:intent/intent.dart' as android_intent;
import 'package:intent/action.dart' as android_action;
import 'package:intent/category.dart' as android_category;
import 'package:mini_ml/mini_ml.dart';
import 'package:csv/csv.dart';
void main() => runApp(MyApp());
class MyApp extends StatelessWidget {
@override
Widget build(BuildContext context) {
return MaterialApp(
home: MyHomePage(),
);
}
}
class MyHomePage extends StatefulWidget {
MyHomePage({Key key}) : super(key: key);
@override
_MyHomePageState createState() => _MyHomePageState();
}
class _MyHomePageState extends State<MyHomePage> {
CameraController _controller;
bool _cameraInitialized = false;
bool _isDetecting = false;
String _topText = '';
Soundpool _pool;
int _soundId;
CameraImage _savedImage;
Rect _savedRect;
Uint8List _snapShot;
ui.Image _buttonImage;
bool _showingWiki = false;
bool _showSnapshot = false;
List breedTable;
FirebaseVisionObjectDetector objectDetector =
FirebaseVisionObjectDetector.instance;
FirebaseVisionLabelDetector labelDetector =
FirebaseVisionLabelDetector.instance;
@override
void initState() {
super.initState();
SystemChrome.setEnabledSystemUIOverlays([SystemUiOverlay.bottom]);
SystemChrome.setSystemUIOverlayStyle(SystemUiOverlayStyle.dark);
_initializeApp();
}
void _initializeApp() async {
await PermissionHandler().requestPermissions(<PermissionGroup>[
PermissionGroup.camera,
PermissionGroup.storage,
]);
_pool = Soundpool(streamType: StreamType.notification);
_soundId = await rootBundle
.load("assets/178186__snapper4298__camera-click-nikon.wav")
.then((ByteData soundData) {
return _pool.load(soundData);
});
List<CameraDescription> cameras = await availableCameras();
_controller = CameraController(cameras[0], ResolutionPreset.medium);
_controller.initialize().then((_) async {
_cameraInitialized = true;
await _controller
.startImageStream((CameraImage image) => _processCameraImage(image));
setState(() {});
});
String csvTable = await rootBundle.loadString("assets/Breeds.csv");
breedTable = CsvToListConverter(fieldDelimiter: ',').convert(csvTable);
}
void _processCameraImage(CameraImage image) async {
if (_showingWiki) return;
if (_isDetecting) return;
_isDetecting = true;
Future findDogFuture = _findDog(image);
List results = await Future.wait(
[findDogFuture, Future.delayed(Duration(milliseconds: 500))]);
setState(() {
_savedImage = image;
_savedRect = results[0];
});
_isDetecting = false;
}
@override
void dispose() {
_controller?.dispose();
super.dispose();
}
@override
Widget build(BuildContext context) {
return Scaffold(
body: GestureDetector(
onTapDown: (TapDownDetails details) async {
double mediaHeight = MediaQuery.of(context).size.height;
double mediaWidth = MediaQuery.of(context).size.width;
int marginToScreen =
((mediaHeight * _controller.value.aspectRatio - mediaWidth) / 2)
.round();
if (details.localPosition.dy < mediaHeight * 0.8)
return; // Tapped above button area
// double mediaWidth = MediaQuery.of(context).size.width;
double xTap = details.localPosition.dx;
if (xTap < mediaWidth * 0.35) {
// Left button tapped: show Gallery.
var intent = android_intent.Intent();
intent.setAction(android_action.Action.ACTION_MAIN);
intent.addCategory(android_category.Category.CATEGORY_APP_GALLERY);
intent.startActivity().catchError((e) => print(e));
} else if (xTap < mediaWidth * 0.65) {
//
// Middle button tapped: process dog inside [_savedRect] in [_savedImage]
if (_showSnapshot) {
// Stop showing snapshot if tapped while doing so
_showSnapshot = false;
return;
} else {
_pool.play(_soundId);
imglib.Image img = _convertCameraImage(_savedImage);
imglib.Image convertedImage = imglib.copyRotate(img, 90);
imglib.Image fullImage =
imglib.copyResize(convertedImage, height: mediaHeight.round());
imglib.Image croppedImage = fullImage;
if (_savedRect != null) {
_topText = await _classifyDog(img);
imglib.drawString(
fullImage, imglib.arial_24, marginToScreen, 20, _topText);
croppedImage = imglib.copyCrop(
convertedImage,
_savedRect.left.round(),
_savedRect.top.round(),
_savedRect.width.round(),
_savedRect.height.round());
}
_snapShot = imglib.encodePng(fullImage);
imglib.Image button = imglib.copyResizeCropSquare(croppedImage, 40);
Uint8List buttonPng = imglib.encodePng(button);
ui.Codec codec = await ui.instantiateImageCodec(buttonPng);
ui.FrameInfo fi = await codec.getNextFrame();
_buttonImage = fi.image;
await ImageGallerySaver.saveImage(_snapShot);
// Show the snapshot with text for four seconds
setState(() {
_showSnapshot = true;
});
Future.delayed(const Duration(seconds: 4), () {
setState(() {
_showSnapshot = false;
});
});
}
} else {
//
// Right button tapped: show Wikipedia info about the dog's breed.
_showingWiki = true;
String breed = _topText.split('(')[0];
await Navigator.push(context,
MaterialPageRoute(builder: (context) => BreedInfo(breed: breed)));
_showingWiki = false;
}
},
child: Container(
child: _cameraInitialized
? OverflowBox(
maxWidth: double.infinity,
child: AspectRatio(
aspectRatio: _controller.value.aspectRatio,
child: _showSnapshot
? Stack(fit: StackFit.expand, children: <Widget>[
_snapShot != null
? Image.memory(
_snapShot,
)
: Text('wait'),
CustomPaint(painter: ButtonsPainter(_buttonImage)),
])
: Stack(fit: StackFit.expand, children: <Widget>[
CameraPreview(_controller),
CustomPaint(painter: ButtonsPainter(_buttonImage)),
CustomPaint(
painter: RectPainter(
_savedRect, _controller.value.previewSize))
// RectPainter(_savedRect, Size(MediaQuery.of(context).size.height, MediaQuery.of(context).size.width)))
])))
: Text(
' Waiting for camera initialization',
style: TextStyle(fontSize: 20),
),
),
));
}
Future<Rect> _findDog(CameraImage image) async {
imglib.Image img = _convertCameraImage(image);
Uint8List png = imglib.encodePng(img);
List<VisionLabel> _onDeviceLabels =
await labelDetector.detectFromBinary(png, false);
bool foundDog = false;
for (int i = 0; i < _onDeviceLabels.length; i++) {
if (_onDeviceLabels[i].label == "Dog") foundDog = true;
}
if (foundDog) {
List<VisionObject> _foundObjects =
await objectDetector.detectFromBinary(png);
if (_foundObjects.length > 0)
return _foundObjects[0].bounds;
else
return Rect.fromLTRB(50.0, 25.0, img.height - 50.0, img.width - 25.0);
} else
return null;
}
static imglib.Image _convertCameraImage(CameraImage image) {
int width = image.width;
int height = image.height;
var img = imglib.Image(width, height); // Create Image buffer
const int hexFF = 0xFF000000;
final int uvyButtonStride = image.planes[1].bytesPerRow;
final int uvPixelStride = image.planes[1].bytesPerPixel;
for (int x = 0; x < width; x++) {
for (int y = 0; y < height; y++) {
final int uvIndex =
uvPixelStride * (x / 2).floor() + uvyButtonStride * (y / 2).floor();
final int index = y * width + x;
final yp = image.planes[0].bytes[index];
final up = image.planes[1].bytes[uvIndex];
final vp = image.planes[2].bytes[uvIndex];
// Calculate pixel color
int r = (yp + vp * 1436 / 1024 - 179).round().clamp(0, 255);
int g = (yp - up * 46549 / 131072 + 44 - vp * 93604 / 131072 + 91)
.round()
.clamp(0, 255);
int b = (yp + up * 1814 / 1024 - 227).round().clamp(0, 255);
// color: 0x FF FF FF FF
// A B G R
img.data[index] = hexFF | (b << 16) | (g << 8) | r;
}
}
return img;
}
Future<String> _classifyDog(imglib.Image img) async {
Uint8List png = imglib.encodePng(img);
List<VisionLabel> _cloudLabels =
await labelDetector.detectFromBinary(png, true);
for (int i = 0; i < _cloudLabels.length; i++) {
String foundLabel = _cloudLabels[i].label;
if (isBreed(foundLabel)) {
String conf = (_cloudLabels[i].confidence * 100).toStringAsFixed(0);
return (foundLabel + ' (' + conf + '%)');
}
}
return null;
}
bool isBreed(String label) {
List labelWords = label.split(' ');
for (int i = 0; i < labelWords.length; i++) {
String word = labelWords[i];
List<String> ignoreLabels = [
'Dog',
'Dog breed',
'Canidae',
'Vertebrate',
'Carnivore',
'Mammal',
'Snout',
'Puppy',
'Grass',
'Sky'
];
if (!ignoreLabels.contains(word)) {
for (int i = 0; i < breedTable.length; i++) {
String aBreed = breedTable[i][0];
if (aBreed.contains(word.toUpperCase())) return true;
}
}
}
return false;
}
}
class RectPainter extends CustomPainter {
Rect rect;
Size imageSize;
RectPainter(this.rect, this.imageSize);
@override
void paint(Canvas canvas, Size size) {
if (rect != null) {
final paint = Paint();
paint.color = Colors.yellow;
paint.style = PaintingStyle.stroke;
paint.strokeWidth = 2.0;
final _heightRatio = imageSize.width / size.height;
double _widthRatio = imageSize.height / size.width;
final rect1 = Rect.fromLTRB(
(rect.left) / _widthRatio,
rect.top / _heightRatio,
(rect.right) / _widthRatio,
rect.bottom / _heightRatio);
canvas.drawRect(rect1, paint);
}
}
@override
bool shouldRepaint(RectPainter oldDelegate) => oldDelegate.rect != rect;
}
class ButtonsPainter extends CustomPainter {
ui.Image buttonImage;
ButtonsPainter(this.buttonImage);
@override
void paint(Canvas canvas, Size size) {
var paint = Paint();
// First paint black field around buttons with low opacity
paint.color = Colors.black.withOpacity(0.1);
Rect rect =
Offset(0.0, size.height * 0.8) & Size(size.width, size.height * 0.2);
canvas.drawRect(rect, paint);
// Draw buttons at 10% from the bottom
final double yButton = size.height * 0.9;
paint.style = PaintingStyle.fill;
paint.color = Colors.grey;
final double canvasWidth = size.width;
double xButton;
var icon;
// Paint left button if no buttonImage supplied
if (buttonImage == null) {
xButton = canvasWidth * 0.3;
icon = Icons.photo_library;
canvas.drawCircle(Offset(xButton, yButton), 22.0, paint);
var builder = ui.ParagraphBuilder(ui.ParagraphStyle(
fontFamily: icon.fontFamily,
fontSize: 25.0,
))
..addText(String.fromCharCode(icon.codePoint));
var para = builder.build();
para.layout(const ui.ParagraphConstraints(width: 100.0));
canvas.drawParagraph(para, Offset(xButton - 12.5, yButton - 12.5));
}
//Paint middle button.
xButton = canvasWidth * 0.5;
canvas.drawCircle(Offset(xButton, yButton), 32.0, paint);
paint.color = Colors.white;
canvas.drawCircle(Offset(xButton, yButton), 28.0, paint);
// Paint right button.
xButton = canvasWidth * 0.7;
icon = Icons.info_outline;
paint.color = Colors.grey;
canvas.drawCircle(Offset(xButton, yButton), 22.0, paint);
var builder = ui.ParagraphBuilder(ui.ParagraphStyle(
fontFamily: icon.fontFamily,
fontSize: 25.0,
))
..addText(String.fromCharCode(icon.codePoint));
var para = builder.build();
para.layout(const ui.ParagraphConstraints(width: 100.0));
canvas.drawParagraph(para, Offset(xButton - 12.5, yButton - 12.5));
// Paint image on left button
if (buttonImage != null) {
xButton = canvasWidth * 0.3;
// First set up a round clipping area.
double radius = 22.0;
double l, t, r, b;
l = xButton - radius;
r = xButton + radius;
t = yButton - radius;
b = yButton + radius;
ui.Rect clippingRect = Rect.fromLTRB(l, t, r, b);
RRect clippingArea =
RRect.fromRectAndRadius(clippingRect, Radius.circular(radius));
canvas.clipRRect(clippingArea);
// Then draw the square button image over the round clipping area
double x, y = 0.0;
x = xButton - buttonImage.height / 2.0;
y = yButton - buttonImage.width / 2.0;
Offset buttonOffset = Offset(x, y);
canvas.drawImage(buttonImage, buttonOffset, Paint());
}
}
@override
bool shouldRepaint(ButtonsPainter oldDelegate) =>
oldDelegate.buttonImage != buttonImage;
}
@klivin
Copy link

klivin commented Mar 13, 2020

Hi Bo, I have a pixel 3a i am testing on, and the image from the _convertCameraImage function returns a vertically striped image, see attached. I'd appreciate your thoughts, maybe there are other yuv formats that should be taken into account?
Screenshot_20200313-114321

@BoHellgren
Copy link
Author

BoHellgren commented Mar 13, 2020 •
edited
Loading

Hi Kevin! Thanks for your report. I know that the _convertCameraImage function does not work on the old Samsung Galaxy S4, but I was hoping all newer phones were OK. I don't have access to a Pixel 3a so it is difficult for me to debug this. Does it work with the front-facing camera?

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment