From 9c5587df030e996d8e2602b73157788696b3930c Mon Sep 17 00:00:00 2001 From: ibnubatutah Date: Wed, 15 May 2024 00:29:08 +0700 Subject: [PATCH] Adding isolate --- lib/data/remote/models/detection_model.dart | 22 ++++ lib/data/remote/models/recipe_model.g.dart | 6 +- lib/data/remote/models/utensil_model.g.dart | 4 +- .../remote/responses/base_response.g.dart | 2 +- .../recipe_detection_view_model.dart | 28 ++--- lib/utils/detection/isolate_inference.dart | 114 ++++++++++++++++++ lib/utils/detection/nms.dart | 1 + lib/utils/detection/yolo.dart | 22 +++- 8 files changed, 174 insertions(+), 25 deletions(-) create mode 100644 lib/data/remote/models/detection_model.dart create mode 100644 lib/utils/detection/isolate_inference.dart diff --git a/lib/data/remote/models/detection_model.dart b/lib/data/remote/models/detection_model.dart new file mode 100644 index 0000000..464a2fe --- /dev/null +++ b/lib/data/remote/models/detection_model.dart @@ -0,0 +1,22 @@ +import 'package:json_annotation/json_annotation.dart'; + +part 'detection_model.g.dart'; + +@JsonSerializable() +class DetectionModel { + List finalClasses; + List> finalBboxes; + List finalScores; + + + DetectionModel({ + required this.finalClasses, + required this.finalBboxes, + required this.finalScores, + }); + + Map toJson() => _$DetectionModelToJson(this); + + factory DetectionModel.fromJson(Map json) => + _$DetectionModelFromJson(json); +} \ No newline at end of file diff --git a/lib/data/remote/models/recipe_model.g.dart b/lib/data/remote/models/recipe_model.g.dart index e584b6b..b863f0c 100644 --- a/lib/data/remote/models/recipe_model.g.dart +++ b/lib/data/remote/models/recipe_model.g.dart @@ -17,9 +17,9 @@ RecipeModel _$RecipeModelFromJson(Map json) => RecipeModel( instructions: (json['instructions'] as List?) ?.map((e) => e as String) .toList(), - prepTime: json['prepTime'] as int?, - cookTime: json['cookTime'] as int?, - servings: json['servings'] as int?, + prepTime: (json['prepTime'] as num?)?.toInt(), + cookTime: (json['cookTime'] as num?)?.toInt(), + servings: (json['servings'] as num?)?.toInt(), utensils: (json['utensils'] as List?) ?.map((e) => e as String) .toList(), diff --git a/lib/data/remote/models/utensil_model.g.dart b/lib/data/remote/models/utensil_model.g.dart index 36552b3..443ecba 100644 --- a/lib/data/remote/models/utensil_model.g.dart +++ b/lib/data/remote/models/utensil_model.g.dart @@ -7,9 +7,9 @@ part of 'utensil_model.dart'; // ************************************************************************** Utensil _$UtensilFromJson(Map json) => Utensil( - id: json['id'] as int?, + id: (json['id'] as num?)?.toInt(), name: json['name'] as String?, - isSelected: json['isSelected'] as int?, + isSelected: (json['isSelected'] as num?)?.toInt(), ); Map _$UtensilToJson(Utensil instance) => { diff --git a/lib/data/remote/responses/base_response.g.dart b/lib/data/remote/responses/base_response.g.dart index 27c418c..e8ada4c 100644 --- a/lib/data/remote/responses/base_response.g.dart +++ b/lib/data/remote/responses/base_response.g.dart @@ -13,7 +13,7 @@ BaseResponse _$BaseResponseFromJson( BaseResponse( data: _$nullableGenericFromJson(json['data'], fromJsonT), ) - ..code = json['code'] as int? + ..code = (json['code'] as num?)?.toInt() ..message = json['message'] as String?; Map _$BaseResponseToJson( diff --git a/lib/presentation/recipe_detection/view_model/recipe_detection_view_model.dart b/lib/presentation/recipe_detection/view_model/recipe_detection_view_model.dart index 30afdf6..2a6784b 100644 --- a/lib/presentation/recipe_detection/view_model/recipe_detection_view_model.dart +++ b/lib/presentation/recipe_detection/view_model/recipe_detection_view_model.dart @@ -65,9 +65,14 @@ class RecipeDetectionViewModel extends BaseViewModel { hideLoadingContainer(); } + void _resetBounding(){ + classes.clear(); + bboxes.clear(); + scores.clear(); + } + Future updatePostProcess() async { detectedIngredients.clear(); - print("MASUK 1"); if (inferenceOutput.isEmpty) { return; } @@ -76,8 +81,6 @@ class RecipeDetectionViewModel extends BaseViewModel { List> newBboxes = []; List newScores = []; - print("MASUK 2"); - /// Wait this process with loading (newClasses, newBboxes, newScores) = await model.postprocess( inferenceOutput, @@ -86,7 +89,6 @@ class RecipeDetectionViewModel extends BaseViewModel { confidenceThreshold: confidenceThreshold, iouThreshold: iouThreshold, ); - print("MASUK 3"); debugPrint('Detected ${newClasses} classes'); debugPrint('Detected ${newBboxes.length} boxed'); @@ -132,17 +134,11 @@ class RecipeDetectionViewModel extends BaseViewModel { const CustomCameraWidget(compressionQuality: 80), ), ); - print("MASUK 9"); - if (data != null) { - print("MASUK 10"); - imageFile.value = data; // _startTimer(); - await Future.delayed(const Duration(milliseconds: 500)); - print("MASUK 11"); - + _resetBounding(); _detectIngredients(); } } @@ -152,18 +148,14 @@ class RecipeDetectionViewModel extends BaseViewModel { } void _detectIngredients() async { - print("MASUK SINI 1"); - - // showLoadingDialog(); - + showLoadingDialog(); final image = img.decodeImage(await imageFile.value!.readAsBytes())!; - print("MASUK SINI 2"); imageHeight.value = image.height; imageWidth.value = image.width; - inferenceOutput.value = model.infer(image); + inferenceOutput.value = await model.inferenceImage(image); + closeLoadingDialog(); - print("MASUK SINI 3"); updatePostProcess(); diff --git a/lib/utils/detection/isolate_inference.dart b/lib/utils/detection/isolate_inference.dart new file mode 100644 index 0000000..c8c3158 --- /dev/null +++ b/lib/utils/detection/isolate_inference.dart @@ -0,0 +1,114 @@ +import 'dart:isolate'; +import 'dart:math'; + +import 'package:flutter/foundation.dart'; +import 'package:image/image.dart' as image_lib; +import 'package:tflite_flutter/tflite_flutter.dart'; + +class IsolateInference { + static const String _debugName = "TFLITE_INFERENCE"; + final ReceivePort _receivePort = ReceivePort(); + late Isolate _isolate; + late SendPort _sendPort; + + SendPort get sendPort => _sendPort; + + Future start() async { + _isolate = await Isolate.spawn(entryPoint, _receivePort.sendPort, + debugName: _debugName); + _sendPort = await _receivePort.first; + } + + Future close() async { + _isolate.kill(); + _receivePort.close(); + } + + static void entryPoint(SendPort sendPort) async { + final port = ReceivePort(); + sendPort.send(port.sendPort); + + await for (final InferenceModel isolateModel in port) { + image_lib.Image? image; + image = isolateModel.image; + int inModelWidth = 640; + int inModelHeight = 640; + int numClasses = 6; + + final imgResized = image_lib.copyResize(image!, + width: inModelWidth, height: inModelHeight); + final imgNormalized = List.generate( + inModelHeight, + (y) => List.generate( + inModelWidth, + (x) { + final pixel = imgResized.getPixel(x, y); + return [pixel.rNormalized, pixel.gNormalized, pixel.bNormalized]; + }, + ), + ); + + // output shape: + // 1 : batch size + // 4 + 6: left, top, right, bottom and probabilities for each class + // 8400: num predictions + final output = [ + List>.filled(4 + numClasses, List.filled(8400, 0)) + ]; + int predictionTimeStart = DateTime.now().millisecondsSinceEpoch; + Interpreter interpreter = + Interpreter.fromAddress(isolateModel.interpreterAddress); + + interpreter.run([imgNormalized], output); + debugPrint( + 'Prediction time: ${DateTime.now().millisecondsSinceEpoch - predictionTimeStart} ms'); + + isolateModel.responsePort.send(output[0]); + } + } + + static List xywh2xyxy(List bbox) { + double halfWidth = bbox[2] / 2; + double halfHeight = bbox[3] / 2; + return [ + bbox[0] - halfWidth, + bbox[1] - halfHeight, + bbox[0] + halfWidth, + bbox[1] + halfHeight, + ]; + } + + /// Computes the intersection over union between two bounding boxes encoded with + /// the xyxy format. + static double computeIou(List bbox1, List bbox2) { + assert(bbox1[0] < bbox1[2]); + assert(bbox1[1] < bbox1[3]); + assert(bbox2[0] < bbox2[2]); + assert(bbox2[1] < bbox2[3]); + + // Determine the coordinate of the intersection rectangle + double xLeft = max(bbox1[0], bbox2[0]); + double yTop = max(bbox1[1], bbox2[1]); + double xRight = min(bbox1[2], bbox2[2]); + double yBottom = min(bbox1[3], bbox2[3]); + + if (xRight < xLeft || yBottom < yTop) { + return 0; + } + double intersectionArea = (xRight - xLeft) * (yBottom - yTop); + double bbox1Area = (bbox1[2] - bbox1[0]) * (bbox1[3] - bbox1[1]); + double bbox2Area = (bbox2[2] - bbox2[0]) * (bbox2[3] - bbox2[1]); + + double iou = intersectionArea / (bbox1Area + bbox2Area - intersectionArea); + assert(iou >= 0 && iou <= 1); + return iou; + } +} + +class InferenceModel { + image_lib.Image? image; + int interpreterAddress; + late SendPort responsePort; + + InferenceModel(this.image, this.interpreterAddress); +} diff --git a/lib/utils/detection/nms.dart b/lib/utils/detection/nms.dart index 6319b60..7c6b958 100644 --- a/lib/utils/detection/nms.dart +++ b/lib/utils/detection/nms.dart @@ -74,6 +74,7 @@ import 'dart:math'; sortedBestScores.removeAt(index); } } + return (finalClasses, finalBboxes, finalScores); } diff --git a/lib/utils/detection/yolo.dart b/lib/utils/detection/yolo.dart index 19c4b5a..8b2234a 100644 --- a/lib/utils/detection/yolo.dart +++ b/lib/utils/detection/yolo.dart @@ -1,7 +1,10 @@ +import 'dart:isolate'; + import 'package:flutter/foundation.dart'; import 'package:tflite_flutter/tflite_flutter.dart'; import 'package:image/image.dart'; +import 'isolate_inference.dart'; import 'nms.dart'; class YoloModel { @@ -10,6 +13,7 @@ class YoloModel { final int inHeight; final int numClasses; Interpreter? _interpreter; + late final IsolateInference isolateInference; YoloModel( this.modelPath, @@ -20,11 +24,12 @@ class YoloModel { Future init() async { _interpreter = await Interpreter.fromAsset(modelPath); + isolateInference = IsolateInference(); + await isolateInference.start(); } List> infer(Image image) { assert(_interpreter != null, 'The model must be initialized'); - final imgResized = copyResize(image, width: inWidth, height: inHeight); final imgNormalized = List.generate( inHeight, @@ -51,6 +56,21 @@ class YoloModel { return output[0]; } + Future>> _inference(InferenceModel inferenceModel) async { + ReceivePort responsePort = ReceivePort(); + isolateInference.sendPort + .send(inferenceModel..responsePort = responsePort.sendPort); + // get inference result. + var results = await responsePort.first; + return results; + } + + // inference still image + Future>> inferenceImage(Image image) async { + var isolateModel = InferenceModel(image, _interpreter?.address ?? 0); + return _inference(isolateModel); + } + Future<(List, List>, List)> postprocess( List> unfilteredBboxes, int imageWidth,