diff --git a/public/model/group1-shard1of1.bin b/public/model/group1-shard1of1.bin index 9da80f4..34556ed 100644 Binary files a/public/model/group1-shard1of1.bin and b/public/model/group1-shard1of1.bin differ diff --git a/public/model/model.json b/public/model/model.json index fb4dc09..b983b3d 100644 --- a/public/model/model.json +++ b/public/model/model.json @@ -1 +1 @@ -{"format": "layers-model", "generatedBy": "keras v2.15.0", "convertedBy": "TensorFlow.js Converter v4.21.0", "modelTopology": {"keras_version": "2.15.0", "backend": "tensorflow", "model_config": {"class_name": "Sequential", "config": {"name": "sequential", "layers": [{"class_name": "InputLayer", "config": {"batch_input_shape": [null, 42], "dtype": "float32", "sparse": false, "ragged": false, "name": "reshape_input"}}, {"class_name": "Reshape", "config": {"name": "reshape", "trainable": true, "dtype": "float32", "batch_input_shape": [null, 42], "target_shape": [21, 2]}}, {"class_name": "LSTM", "config": {"name": "lstm", "trainable": true, "dtype": "float32", "return_sequences": true, "return_state": false, "go_backwards": false, "stateful": false, "unroll": false, "time_major": false, "units": 128, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"module": "keras.initializers", "class_name": "GlorotUniform", "config": {"seed": null}, "registered_name": null}, "recurrent_initializer": {"module": "keras.initializers", "class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}, "registered_name": null}, "bias_initializer": {"module": "keras.initializers", "class_name": "Zeros", "config": {}, "registered_name": null}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}}, {"class_name": "LSTM", "config": {"name": "lstm_1", "trainable": true, "dtype": "float32", "return_sequences": false, "return_state": false, "go_backwards": false, "stateful": false, "unroll": false, "time_major": false, "units": 128, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"module": "keras.initializers", "class_name": "GlorotUniform", "config": {"seed": null}, "registered_name": null}, "recurrent_initializer": {"module": "keras.initializers", "class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}, "registered_name": null}, "bias_initializer": {"module": "keras.initializers", "class_name": "Zeros", "config": {}, "registered_name": null}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}}, {"class_name": "Dense", "config": {"name": "dense", "trainable": true, "dtype": "float32", "units": 128, "activation": "relu", "use_bias": true, "kernel_initializer": {"module": "keras.initializers", "class_name": "GlorotUniform", "config": {"seed": null}, "registered_name": null}, "bias_initializer": {"module": "keras.initializers", "class_name": "Zeros", "config": {}, "registered_name": null}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}, {"class_name": "Dense", "config": {"name": "dense_1", "trainable": true, "dtype": "float32", "units": 24, "activation": "softmax", "use_bias": true, "kernel_initializer": {"module": "keras.initializers", "class_name": "GlorotUniform", "config": {"seed": null}, "registered_name": null}, "bias_initializer": {"module": "keras.initializers", "class_name": "Zeros", "config": {}, "registered_name": null}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}]}}, "training_config": {"loss": "sparse_categorical_crossentropy", "metrics": [[{"class_name": "MeanMetricWrapper", "config": {"name": "accuracy", "dtype": "float32", "fn": "sparse_categorical_accuracy"}}]], "weighted_metrics": null, "loss_weights": null, "optimizer_config": {"class_name": "Custom>Adam", "config": {"name": "Adam", "weight_decay": null, "clipnorm": null, "global_clipnorm": null, "clipvalue": null, "use_ema": false, "ema_momentum": 0.99, "ema_overwrite_frequency": null, "jit_compile": false, "is_legacy_optimizer": false, "learning_rate": 0.0010000000474974513, "beta_1": 0.9, "beta_2": 0.999, "epsilon": 1e-07, "amsgrad": false}}}}, "weightsManifest": [{"paths": ["group1-shard1of1.bin"], "weights": [{"name": "dense/kernel", "shape": [128, 128], "dtype": "float32"}, {"name": "dense/bias", "shape": [128], "dtype": "float32"}, {"name": "dense_1/kernel", "shape": [128, 24], "dtype": "float32"}, {"name": "dense_1/bias", "shape": [24], "dtype": "float32"}, {"name": "lstm/lstm_cell/kernel", "shape": [2, 512], "dtype": "float32"}, {"name": "lstm/lstm_cell/recurrent_kernel", "shape": [128, 512], "dtype": "float32"}, {"name": "lstm/lstm_cell/bias", "shape": [512], "dtype": "float32"}, {"name": "lstm_1/lstm_cell/kernel", "shape": [128, 512], "dtype": "float32"}, {"name": "lstm_1/lstm_cell/recurrent_kernel", "shape": [128, 512], "dtype": "float32"}, {"name": "lstm_1/lstm_cell/bias", "shape": [512], "dtype": "float32"}]}]} \ No newline at end of file +{"format": "layers-model", "generatedBy": "keras v2.15.0", "convertedBy": "TensorFlow.js Converter v4.22.0", "modelTopology": {"keras_version": "2.15.0", "backend": "tensorflow", "model_config": {"class_name": "Sequential", "config": {"name": "sequential", "layers": [{"class_name": "InputLayer", "config": {"batch_input_shape": [null, 42], "dtype": "float32", "sparse": false, "ragged": false, "name": "reshape_input"}}, {"class_name": "Reshape", "config": {"name": "reshape", "trainable": true, "dtype": "float32", "batch_input_shape": [null, 42], "target_shape": [21, 2]}}, {"class_name": "Bidirectional", "config": {"name": "bidirectional", "trainable": true, "dtype": "float32", "layer": {"module": "keras.layers", "class_name": "LSTM", "config": {"name": "lstm", "trainable": true, "dtype": "float32", "return_sequences": true, "return_state": false, "go_backwards": false, "stateful": false, "unroll": false, "time_major": false, "units": 128, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"module": "keras.initializers", "class_name": "GlorotUniform", "config": {"seed": null}, "registered_name": null}, "recurrent_initializer": {"module": "keras.initializers", "class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}, "registered_name": null}, "bias_initializer": {"module": "keras.initializers", "class_name": "Zeros", "config": {}, "registered_name": null}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}, "registered_name": null}, "merge_mode": "concat"}}, {"class_name": "Bidirectional", "config": {"name": "bidirectional_1", "trainable": true, "dtype": "float32", "layer": {"module": "keras.layers", "class_name": "LSTM", "config": {"name": "lstm_1", "trainable": true, "dtype": "float32", "return_sequences": false, "return_state": false, "go_backwards": false, "stateful": false, "unroll": false, "time_major": false, "units": 128, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"module": "keras.initializers", "class_name": "GlorotUniform", "config": {"seed": null}, "registered_name": null}, "recurrent_initializer": {"module": "keras.initializers", "class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}, "registered_name": null}, "bias_initializer": {"module": "keras.initializers", "class_name": "Zeros", "config": {}, "registered_name": null}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}, "registered_name": null}, "merge_mode": "concat"}}, {"class_name": "Dropout", "config": {"name": "dropout", "trainable": true, "dtype": "float32", "rate": 0.2, "noise_shape": null, "seed": null}}, {"class_name": "Dense", "config": {"name": "dense", "trainable": true, "dtype": "float32", "units": 128, "activation": "relu", "use_bias": true, "kernel_initializer": {"module": "keras.initializers", "class_name": "GlorotUniform", "config": {"seed": null}, "registered_name": null}, "bias_initializer": {"module": "keras.initializers", "class_name": "Zeros", "config": {}, "registered_name": null}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}, {"class_name": "Dropout", "config": {"name": "dropout_1", "trainable": true, "dtype": "float32", "rate": 0.2, "noise_shape": null, "seed": null}}, {"class_name": "Dense", "config": {"name": "dense_1", "trainable": true, "dtype": "float32", "units": 24, "activation": "softmax", "use_bias": true, "kernel_initializer": {"module": "keras.initializers", "class_name": "GlorotUniform", "config": {"seed": null}, "registered_name": null}, "bias_initializer": {"module": "keras.initializers", "class_name": "Zeros", "config": {}, "registered_name": null}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}]}}, "training_config": {"loss": "sparse_categorical_crossentropy", "metrics": [[{"class_name": "MeanMetricWrapper", "config": {"name": "accuracy", "dtype": "float32", "fn": "sparse_categorical_accuracy"}}]], "weighted_metrics": null, "loss_weights": null, "optimizer_config": {"class_name": "Custom>Adam", "config": {"name": "Adam", "weight_decay": null, "clipnorm": null, "global_clipnorm": null, "clipvalue": null, "use_ema": false, "ema_momentum": 0.99, "ema_overwrite_frequency": null, "jit_compile": false, "is_legacy_optimizer": false, "learning_rate": 0.0010000000474974513, "beta_1": 0.9, "beta_2": 0.999, "epsilon": 1e-07, "amsgrad": false}}}}, "weightsManifest": [{"paths": ["group1-shard1of1.bin"], "weights": [{"name": "bidirectional/forward_lstm/lstm_cell/kernel", "shape": [2, 512], "dtype": "float32"}, {"name": "bidirectional/forward_lstm/lstm_cell/recurrent_kernel", "shape": [128, 512], "dtype": "float32"}, {"name": "bidirectional/forward_lstm/lstm_cell/bias", "shape": [512], "dtype": "float32"}, {"name": "bidirectional/backward_lstm/lstm_cell/kernel", "shape": [2, 512], "dtype": "float32"}, {"name": "bidirectional/backward_lstm/lstm_cell/recurrent_kernel", "shape": [128, 512], "dtype": "float32"}, {"name": "bidirectional/backward_lstm/lstm_cell/bias", "shape": [512], "dtype": "float32"}, {"name": "bidirectional_1/forward_lstm_1/lstm_cell/kernel", "shape": [256, 512], "dtype": "float32"}, {"name": "bidirectional_1/forward_lstm_1/lstm_cell/recurrent_kernel", "shape": [128, 512], "dtype": "float32"}, {"name": "bidirectional_1/forward_lstm_1/lstm_cell/bias", "shape": [512], "dtype": "float32"}, {"name": "bidirectional_1/backward_lstm_1/lstm_cell/kernel", "shape": [256, 512], "dtype": "float32"}, {"name": "bidirectional_1/backward_lstm_1/lstm_cell/recurrent_kernel", "shape": [128, 512], "dtype": "float32"}, {"name": "bidirectional_1/backward_lstm_1/lstm_cell/bias", "shape": [512], "dtype": "float32"}, {"name": "dense/kernel", "shape": [256, 128], "dtype": "float32"}, {"name": "dense/bias", "shape": [128], "dtype": "float32"}, {"name": "dense_1/kernel", "shape": [128, 24], "dtype": "float32"}, {"name": "dense_1/bias", "shape": [24], "dtype": "float32"}]}]} \ No newline at end of file diff --git a/src/pages/Home.tsx b/src/pages/Home.tsx index 5fdfa06..2cb9b73 100644 --- a/src/pages/Home.tsx +++ b/src/pages/Home.tsx @@ -16,6 +16,7 @@ type PredictResult = { const Home = () => { const videoRef = useRef(null); const [loadCamera, setLoadCamera] = useState(false); + const canvasRef = useRef(null); const [resultPredict, setResultPredict] = useState({ abjad: "", @@ -107,6 +108,29 @@ const Home = () => { prediction.dispose(); }; + const drawLandmarks = (landmarks: any[]) => { + if (!canvasRef.current) return; + const canvas = canvasRef.current; + const ctx = canvas.getContext("2d"); + + if (ctx) { + ctx.clearRect(0, 0, canvas.width, canvas.height); + ctx.fillStyle = "red"; + + landmarks.forEach((landmark) => { + ctx.beginPath(); + ctx.arc( + landmark.x * canvas.width, + landmark.y * canvas.height, + 2, + 0, + 2 * Math.PI + ); + ctx.fill(); + }); + } + }; + const detectHands = async () => { if (videoRef.current && videoRef.current.readyState >= 2) { const detections = handLandmarker.detectForVideo( @@ -115,16 +139,23 @@ const Home = () => { ); setHandPresence(detections.handedness.length > 0); - // Assuming detections.landmarks is an array of landmark objects if (detections.landmarks) { if (detections.handednesses.length > 0) { - const landm = detections.landmarks[0].map((landmark) => landmark); + console.log(detections); - const calt = calcLandmarkList(videoRef.current, landm); - const finalResult = preProcessLandmark(calt); + if (detections.handednesses[0][0].displayName === "Right") { + const landm = detections.landmarks[0].map((landmark) => landmark); - makePrediction(finalResult); + const calt = calcLandmarkList(videoRef.current, landm); + const finalResult = preProcessLandmark(calt); + + // drawLandmarks(landm); + + makePrediction(finalResult); + } else { + setHandPresence(false); + } } } } @@ -142,7 +173,9 @@ const Home = () => { setLoadCamera(true); return () => { - // stop camera + if (handLandmarker) { + handLandmarker.close(); + } }; }, []); @@ -159,6 +192,10 @@ const Home = () => { )} +