diff --git a/src/pages/Home.tsx b/src/pages/Home.tsx index 814a2c6..6b0beed 100644 --- a/src/pages/Home.tsx +++ b/src/pages/Home.tsx @@ -2,12 +2,14 @@ import LayoutPage from "@/components/templates/LayoutPage"; import { useEffect, useRef, useState } from "react"; import { FaCircleCheck } from "react-icons/fa6"; import * as tf from "@tensorflow/tfjs"; -import { FilesetResolver, HandLandmarker } from "@mediapipe/tasks-vision"; +import { HandLandmarker } from "@mediapipe/tasks-vision"; import calcLandmarkList from "@/utils/CalculateLandmark"; import preProcessLandmark from "@/utils/PreProcessLandmark"; import ConvertResult from "@/utils/ConvertResult"; import useNavbarStore from "@/stores/NavbarStore"; import { AnimatePresence, motion } from "framer-motion"; +import { loadTensorFlowModel } from "@/utils/tensorflowModelLoader"; +import { loadHandLandmarker } from "@/utils/handLandmarkerLoader"; type PredictResult = { abjad: String; @@ -50,12 +52,13 @@ const Home = () => { const loadModel = async () => { setLoadCamera(false); try { - const lm = await tf.loadLayersModel("/model/model.json"); + const lm = await loadTensorFlowModel(); model = lm; - const emptyInput = tf.tensor2d([[0, 0]]); + // const emptyInput = tf.tensor2d([[0, 0]]); - model.predict(emptyInput) as tf.Tensor; + // model.predict(emptyInput) as tf.Tensor; + setLoadCamera(true); } catch (error) { @@ -65,17 +68,8 @@ const Home = () => { const initializeHandDetection = async () => { try { - const vision = await FilesetResolver.forVisionTasks( - "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm" - ); - handLandmarker = await HandLandmarker.createFromOptions(vision, { - baseOptions: { - modelAssetPath: `https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task`, - }, - numHands: 2, - runningMode: "VIDEO", - }); - + handLandmarker = await loadHandLandmarker(); + detectHands(); } catch (error) { console.error("Error initializing hand detection:", error); @@ -117,11 +111,13 @@ const Home = () => { performance.now() ); + + setHandPresence(detections.handedness.length > 0); // Assuming detections.landmarks is an array of landmark objects if (detections.landmarks) { if (detections.handednesses.length > 0) { - console.log(detections); + // console.log(detections); if (detections.handednesses[0][0].displayName === "Right") { const landm = detections.landmarks[0].map((landmark) => landmark); @@ -184,10 +180,7 @@ const Home = () => { d="M13 16h-1v-4h-1m1-4h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z" > -

- Gunakan tangan kanan dan pastikan gambar pada kamera terlihat - jelas -

+

Gunakan tangan kanan. Pencahayaan ideal 300 Lux, jarak maksimal 2 meter dari kamera.

+ {/*
{selectedKamus?.badge.map((item, index) => ( { const loadModel = async () => { setLoadCamera(false); try { - const lm = await tf.loadLayersModel("/model/model.json"); + const lm = await loadTensorFlowModel(); model = lm; const emptyInput = tf.tensor2d([[0, 0]]); @@ -76,16 +78,7 @@ const Quiz = () => { const initializeHandDetection = async () => { try { - const vision = await FilesetResolver.forVisionTasks( - "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm" - ); - handLandmarker = await HandLandmarker.createFromOptions(vision, { - baseOptions: { - modelAssetPath: `https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task`, - }, - numHands: 2, - runningMode: "VIDEO", - }); + handLandmarker = await loadHandLandmarker(); detectHands(); } catch (error) { diff --git a/src/pages/Kuis/TebakHuruf/Quiz.tsx b/src/pages/Kuis/TebakHuruf/Quiz.tsx index 3e373dd..2c1e20a 100644 --- a/src/pages/Kuis/TebakHuruf/Quiz.tsx +++ b/src/pages/Kuis/TebakHuruf/Quiz.tsx @@ -1,7 +1,7 @@ import LayoutPage from "@/components/templates/LayoutPage"; import { useEffect, useRef, useState } from "react"; import * as tf from "@tensorflow/tfjs"; -import { FilesetResolver, HandLandmarker } from "@mediapipe/tasks-vision"; +import { HandLandmarker } from "@mediapipe/tasks-vision"; import calcLandmarkList from "@/utils/CalculateLandmark"; import preProcessLandmark from "@/utils/PreProcessLandmark"; import ConvertResult, { abjads } from "@/utils/ConvertResult"; @@ -11,6 +11,8 @@ import { MdOutlineQuiz } from "react-icons/md"; import useTebakHurufStore from "@/stores/TebakHurufStore"; import { useNavigate } from "react-router-dom"; import Swal from "sweetalert2"; +import { loadTensorFlowModel } from "@/utils/tensorflowModelLoader"; +import { loadHandLandmarker } from "@/utils/handLandmarkerLoader"; // type PredictResult = { // abjad: String; @@ -63,7 +65,7 @@ const Quiz = () => { const loadModel = async () => { setLoadCamera(false); try { - const lm = await tf.loadLayersModel("/model/model.json"); + const lm = await loadTensorFlowModel(); model = lm; const emptyInput = tf.tensor2d([[0, 0]]); @@ -78,16 +80,7 @@ const Quiz = () => { const initializeHandDetection = async () => { try { - const vision = await FilesetResolver.forVisionTasks( - "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm" - ); - handLandmarker = await HandLandmarker.createFromOptions(vision, { - baseOptions: { - modelAssetPath: `https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task`, - }, - numHands: 2, - runningMode: "VIDEO", - }); + handLandmarker = await loadHandLandmarker(); detectHands(); } catch (error) { @@ -161,11 +154,13 @@ const Quiz = () => { setTimeout(() => { setShowAnswer(false); - isLoading = false; - setProgress(0); - noSoal++; - previousResult = []; - quizStore.setSoalIndex(noSoal); + setTimeout(() => { + isLoading = false; + setProgress(0); + noSoal++; + previousResult = []; + quizStore.setSoalIndex(noSoal); + }, 500); if (noSoal === quizStore.listSoal.length) { quizStore.setSession(false); @@ -231,7 +226,7 @@ const Quiz = () => { const store = useNavbarStore(); const quizStore = useTebakHurufStore(); - console.log(quizStore.jawaban); + // console.log(quizStore.jawaban); useEffect(() => { if (!quizStore.session) { diff --git a/src/utils/handLandmarkerLoader.ts b/src/utils/handLandmarkerLoader.ts new file mode 100644 index 0000000..5eeadce --- /dev/null +++ b/src/utils/handLandmarkerLoader.ts @@ -0,0 +1,34 @@ +import { FilesetResolver, HandLandmarker } from "@mediapipe/tasks-vision"; +import { getHandLandmarkerModel, saveHandLandmarkerModel } from "./indexedDBHelper"; + +let handLandmarker: HandLandmarker | null = null; + +export async function loadHandLandmarker(): Promise { + if (handLandmarker) return handLandmarker; // Jika model sudah ada, langsung kembalikan + + let modelBlob = await getHandLandmarkerModel(); + + if (!modelBlob) { + console.log("🔄 Model Hand Landmarker tidak ditemukan di cache, mengunduh..."); + const response = await fetch( + "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task" + ); + modelBlob = await response.blob(); + await saveHandLandmarkerModel(modelBlob); + } else { + console.log("✅ Model Hand Landmarker ditemukan di cache, menggunakan model lokal."); + } + + const vision = await FilesetResolver.forVisionTasks( + "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm" + ); + + const modelURL = URL.createObjectURL(modelBlob); + handLandmarker = await HandLandmarker.createFromOptions(vision, { + baseOptions: { modelAssetPath: modelURL }, + numHands: 2, + runningMode: "VIDEO", + }); + + return handLandmarker; +} diff --git a/src/utils/indexedDBHelper.ts b/src/utils/indexedDBHelper.ts new file mode 100644 index 0000000..9d22f3e --- /dev/null +++ b/src/utils/indexedDBHelper.ts @@ -0,0 +1,59 @@ +import * as tf from "@tensorflow/tfjs"; + +const DB_NAME = "ModelCacheDB"; +const STORE_NAME = "models"; +const TENSORFLOW_MODEL_KEY = "tensorflow_model"; +const HAND_LANDMARKER_MODEL_KEY = "hand_landmarker_model"; + +// Membuka IndexedDB +export function openDB(): Promise { + return new Promise((resolve, reject) => { + const request = indexedDB.open(DB_NAME, 1); + + request.onupgradeneeded = (event: IDBVersionChangeEvent) => { + const db = (event.target as IDBOpenDBRequest).result; + if (!db.objectStoreNames.contains(STORE_NAME)) { + db.createObjectStore(STORE_NAME); + } + }; + + request.onsuccess = () => resolve(request.result); + request.onerror = () => reject(request.error); + }); +} + +// 🟢 **Simpan Model TensorFlow.js ke IndexedDB** +export async function saveTensorFlowModel(model: tf.LayersModel): Promise { + await model.save(`indexeddb://${TENSORFLOW_MODEL_KEY}`); +} + +// 🟢 **Ambil Model TensorFlow.js dari IndexedDB** +export async function getTensorFlowModel(): Promise { + try { + return await tf.loadLayersModel(`indexeddb://${TENSORFLOW_MODEL_KEY}`); + } catch (error) { + console.warn("⚠️ Model TensorFlow tidak ditemukan di cache:", error); + return null; + } +} + +// 🟢 **Simpan Model Hand Landmarker ke IndexedDB** +export async function saveHandLandmarkerModel(blob: Blob): Promise { + const db = await openDB(); + const transaction = db.transaction(STORE_NAME, "readwrite"); + const store = transaction.objectStore(STORE_NAME); + store.put(blob, HAND_LANDMARKER_MODEL_KEY); +} + +// 🟢 **Ambil Model Hand Landmarker dari IndexedDB** +export async function getHandLandmarkerModel(): Promise { + const db = await openDB(); + return new Promise((resolve, reject) => { + const transaction = db.transaction(STORE_NAME, "readonly"); + const store = transaction.objectStore(STORE_NAME); + const request = store.get(HAND_LANDMARKER_MODEL_KEY); + + request.onsuccess = () => resolve(request.result as Blob | null); + request.onerror = () => reject(request.error); + }); +} diff --git a/src/utils/tensorflowModelLoader.ts b/src/utils/tensorflowModelLoader.ts new file mode 100644 index 0000000..06af412 --- /dev/null +++ b/src/utils/tensorflowModelLoader.ts @@ -0,0 +1,20 @@ +import * as tf from "@tensorflow/tfjs"; +import { getTensorFlowModel, saveTensorFlowModel } from "./indexedDBHelper"; + +let model: tf.LayersModel | null = null; + +export async function loadTensorFlowModel(): Promise { + if (model) return model; // Jika model sudah dimuat, langsung kembalikan + + model = await getTensorFlowModel(); + + if (!model) { + console.log("🔄 Model TensorFlow tidak ditemukan di cache, mengunduh..."); + model = await tf.loadLayersModel("/model/model.json"); + await saveTensorFlowModel(model); + } else { + console.log("✅ Model TensorFlow ditemukan di cache, menggunakan model lokal."); + } + + return model; +}