Compare commits

...

10 Commits

22 changed files with 287 additions and 90 deletions

View File

@ -1,9 +1,37 @@
<!doctype html>
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<link rel="icon" type="image/svg+xml+png" href="/assets/images/logo.png" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta
name="description"
content="Ksuli SIBI adalah aplikasi yang membantu pengguna mengenal abjad dalam
Sistem Isyarat Bahasa Indonesia (SIBI). Aplikasi ini menyediakan panduan
visual dan interaktif untuk mempelajari huruf-huruf dalam SIBI, sehingga
memudahkan komunikasi bagi penyandang tuli dan orang-orang yang ingin
belajar bahasa isyarat."
/>
<meta
name="keywords"
content="ksuli, kedai susu tuli, mphstar, 'ksuli sibi"
/>
<meta name="robots" content="index, follow" />
<meta property="og:title" content="Ksuli SIBI" />
<meta
property="og:description"
content="Ksuli SIBI adalah aplikasi yang membantu pengguna mengenal abjad dalam
Sistem Isyarat Bahasa Indonesia (SIBI). Aplikasi ini menyediakan panduan
visual dan interaktif untuk mempelajari huruf-huruf dalam SIBI, sehingga
memudahkan komunikasi bagi penyandang tuli dan orang-orang yang ingin
belajar bahasa isyarat."
/>
<meta
property="og:image"
content="https://ksuli-sibi.vercel.app/assets/images/logo.png"
/>
<meta property="og:url" content="https://ksuli-sibi.vercel.app" />
<title>Kedai Susu Tuli - SIBI</title>
</head>
<body>

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 4.7 KiB

After

Width:  |  Height:  |  Size: 12 KiB

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 3.5 KiB

After

Width:  |  Height:  |  Size: 13 KiB

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 14 KiB

After

Width:  |  Height:  |  Size: 7.8 KiB

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 5.0 KiB

After

Width:  |  Height:  |  Size: 4.7 KiB

File diff suppressed because one or more lines are too long

After

Width:  |  Height:  |  Size: 12 KiB

File diff suppressed because one or more lines are too long

After

Width:  |  Height:  |  Size: 14 KiB

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

File diff suppressed because one or more lines are too long

View File

@ -12,21 +12,21 @@ const slides = [
{
title: "Gerakan",
description:
"Gerakkan tangan Anda di depan kamera untuk melanjutkan. Sistem akan mendeteksi gerakan Anda untuk navigasi tanpa sentuhan.",
image: "/assets/images/susun_2.svg",
"Gerakkan tangan Anda di depan kamera. Sistem akan mendeteksi gerakan Anda.",
image: "/assets/images/susun_5.svg",
button: "Next",
},
{
title: "Susun abjad",
description:
"Susun huruf-huruf menjadi kata yang benar. Ini adalah latihan untuk meningkatkan keterampilan kognitif dan kecepatan berpikir Anda.",
image: "/assets/images/susun_3.svg",
"Susun huruf-huruf menjadi kata pada soal.",
image: "/assets/images/susun_6.svg",
button: "Next",
},
{
title: "Score",
description:
"Selesaikan tugas dengan cepat untuk mendapatkan skor tertinggi. Skor Anda akan dibandingkan dengan pengguna lain di papan peringkat.",
"Selesaikan soal dengan cepat untuk mendapatkan skor tertinggi. Skor Anda akan dibandingkan dengan pengguna lain di papan peringkat.",
image: "/assets/images/susun_4.svg",
button: "Get Started",
},
@ -63,7 +63,7 @@ export default function Carousel() {
className="h-60"
/>
{/* <h2 className="mt-4 text-xl font-bold">{slide.title}</h2> */}
<p className="mt-8 text-gray-500 text-sm">
<p className="mt-8 text-gray-800 text-sm">
{slide.description}
</p>
{/* <button

View File

@ -12,7 +12,7 @@ const slides = [
{
title: "Tebak Huruf",
description:
"Tebak huruf sesuai soal yang ditampilkan. Ini adalah latihan untuk meningkatkan keterampilan kognitif dan kecepatan berpikir Anda.",
"Tebak huruf sesuai soal yang ditampilkan.",
image: "/assets/images/susun_2.svg",
button: "Next",
},
@ -59,7 +59,7 @@ export default function Carousel() {
>
<img src={slide.image} alt={slide.title} className="h-60" />
{/* <h2 className="mt-4 text-xl font-bold">{slide.title}</h2> */}
<p className="mt-8 text-gray-500 text-sm">
<p className="mt-8 text-gray-800 text-sm">
{slide.description}
</p>
{/* <button

View File

@ -2,12 +2,14 @@ import LayoutPage from "@/components/templates/LayoutPage";
import { useEffect, useRef, useState } from "react";
import { FaCircleCheck } from "react-icons/fa6";
import * as tf from "@tensorflow/tfjs";
import { FilesetResolver, HandLandmarker } from "@mediapipe/tasks-vision";
import { HandLandmarker } from "@mediapipe/tasks-vision";
import calcLandmarkList from "@/utils/CalculateLandmark";
import preProcessLandmark from "@/utils/PreProcessLandmark";
import ConvertResult from "@/utils/ConvertResult";
import useNavbarStore from "@/stores/NavbarStore";
import { AnimatePresence, motion } from "framer-motion";
import { loadTensorFlowModel } from "@/utils/tensorflowModelLoader";
import { loadHandLandmarker } from "@/utils/handLandmarkerLoader";
type PredictResult = {
abjad: String;
@ -33,6 +35,7 @@ const Home = () => {
try {
const stream = await navigator.mediaDevices.getUserMedia({
video: true,
});
if (videoRef.current) {
@ -49,12 +52,13 @@ const Home = () => {
const loadModel = async () => {
setLoadCamera(false);
try {
const lm = await tf.loadLayersModel("/model/model.json");
const lm = await loadTensorFlowModel();
model = lm;
const emptyInput = tf.tensor2d([[0, 0]]);
// const emptyInput = tf.tensor2d([[0, 0]]);
model.predict(emptyInput) as tf.Tensor;
// model.predict(emptyInput) as tf.Tensor;
setLoadCamera(true);
} catch (error) {
@ -64,17 +68,8 @@ const Home = () => {
const initializeHandDetection = async () => {
try {
const vision = await FilesetResolver.forVisionTasks(
"https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm"
);
handLandmarker = await HandLandmarker.createFromOptions(vision, {
baseOptions: {
modelAssetPath: `https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task`,
},
numHands: 2,
runningMode: "VIDEO",
});
handLandmarker = await loadHandLandmarker();
detectHands();
} catch (error) {
console.error("Error initializing hand detection:", error);
@ -116,11 +111,13 @@ const Home = () => {
performance.now()
);
setHandPresence(detections.handedness.length > 0);
// Assuming detections.landmarks is an array of landmark objects
if (detections.landmarks) {
if (detections.handednesses.length > 0) {
console.log(detections);
// console.log(detections);
if (detections.handednesses[0][0].displayName === "Right") {
const landm = detections.landmarks[0].map((landmark) => landmark);
@ -183,10 +180,7 @@ const Home = () => {
d="M13 16h-1v-4h-1m1-4h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z"
></path>
</svg>
<p className="ml-2">
Gunakan tangan kanan dan pastikan gambar pada kamera terlihat
jelas
</p>
<p className="ml-2">Gunakan tangan kanan. Pencahayaan ideal 120 Lux, jarak maksimal 2 meter dari kamera.</p>
</div>
<button
onClick={() => setInfo(false)}

View File

@ -80,7 +80,13 @@ const Kamus = () => {
<div className="flex flex-col md:flex-row gap-6 overflow-y-auto flex-1">
<div className="flex flex-col order-2 md:order-1">
<p>{selectedKamus?.keterangan}</p>
<div className="flex flex-wrap gap-2 mt-3">
<a href="/">
<button className="btn mt-8 bg-slate-900 text-white hover:bg-slate-950 w-fit">
Coba Sekarang
</button>
</a>
{/* <div className="flex flex-wrap gap-2 mt-3">
{selectedKamus?.badge.map((item, index) => (
<span
key={index}
@ -89,7 +95,7 @@ const Kamus = () => {
{item}
</span>
))}
</div>
</div> */}
</div>
<img
className="h-[250px] object-cover rounded-md order-1 md:order-2"

View File

@ -1,7 +1,7 @@
import LayoutPage from "@/components/templates/LayoutPage";
import { useEffect, useRef, useState } from "react";
import * as tf from "@tensorflow/tfjs";
import { FilesetResolver, HandLandmarker } from "@mediapipe/tasks-vision";
import { HandLandmarker } from "@mediapipe/tasks-vision";
import calcLandmarkList from "@/utils/CalculateLandmark";
import preProcessLandmark from "@/utils/PreProcessLandmark";
import { abjads } from "@/utils/ConvertResult";
@ -10,6 +10,8 @@ import { MdOutlineQuiz } from "react-icons/md";
import useMenyusunHurufStore from "@/stores/MenyusunHurufStore";
import { useNavigate } from "react-router-dom";
import Swal from "sweetalert2";
import { loadTensorFlowModel } from "@/utils/tensorflowModelLoader";
import { loadHandLandmarker } from "@/utils/handLandmarkerLoader";
// type PredictResult = {
// abjad: String;
@ -61,7 +63,7 @@ const Quiz = () => {
const loadModel = async () => {
setLoadCamera(false);
try {
const lm = await tf.loadLayersModel("/model/model.json");
const lm = await loadTensorFlowModel();
model = lm;
const emptyInput = tf.tensor2d([[0, 0]]);
@ -76,16 +78,7 @@ const Quiz = () => {
const initializeHandDetection = async () => {
try {
const vision = await FilesetResolver.forVisionTasks(
"https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm"
);
handLandmarker = await HandLandmarker.createFromOptions(vision, {
baseOptions: {
modelAssetPath: `https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task`,
},
numHands: 2,
runningMode: "VIDEO",
});
handLandmarker = await loadHandLandmarker();
detectHands();
} catch (error) {
@ -180,6 +173,8 @@ const Quiz = () => {
answerTime = answerTime += elapsedTime;
noSoal++;
tempAnswer = "";
setAnswer("");
if (noSoal === quizStore.listSoal.length) {
quizStore.setTime(answerTime);

View File

@ -1,7 +1,7 @@
import LayoutPage from "@/components/templates/LayoutPage";
import { useEffect, useRef, useState } from "react";
import * as tf from "@tensorflow/tfjs";
import { FilesetResolver, HandLandmarker } from "@mediapipe/tasks-vision";
import { HandLandmarker } from "@mediapipe/tasks-vision";
import calcLandmarkList from "@/utils/CalculateLandmark";
import preProcessLandmark from "@/utils/PreProcessLandmark";
import ConvertResult, { abjads } from "@/utils/ConvertResult";
@ -11,12 +11,23 @@ import { MdOutlineQuiz } from "react-icons/md";
import useTebakHurufStore from "@/stores/TebakHurufStore";
import { useNavigate } from "react-router-dom";
import Swal from "sweetalert2";
import { loadTensorFlowModel } from "@/utils/tensorflowModelLoader";
import { loadHandLandmarker } from "@/utils/handLandmarkerLoader";
// type PredictResult = {
// abjad: String;
// acc: String;
// };
const imagesToPreload = ["/assets/gif/betul.gif", "/assets/gif/salah.gif"];
const preloadImages = () => {
imagesToPreload.forEach((src) => {
const img = new Image();
img.src = src;
});
};
const Quiz = () => {
const videoRef = useRef<HTMLVideoElement>(null);
const [loadCamera, setLoadCamera] = useState(false);
@ -54,7 +65,7 @@ const Quiz = () => {
const loadModel = async () => {
setLoadCamera(false);
try {
const lm = await tf.loadLayersModel("/model/model.json");
const lm = await loadTensorFlowModel();
model = lm;
const emptyInput = tf.tensor2d([[0, 0]]);
@ -69,16 +80,7 @@ const Quiz = () => {
const initializeHandDetection = async () => {
try {
const vision = await FilesetResolver.forVisionTasks(
"https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm"
);
handLandmarker = await HandLandmarker.createFromOptions(vision, {
baseOptions: {
modelAssetPath: `https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task`,
},
numHands: 2,
runningMode: "VIDEO",
});
handLandmarker = await loadHandLandmarker();
detectHands();
} catch (error) {
@ -150,28 +152,31 @@ const Quiz = () => {
setShowAnswer(true);
setTimeout(() => {
isLoading = false;
setShowAnswer(false);
setProgress(0);
noSoal++;
previousResult = [];
quizStore.setSoalIndex(noSoal);
if (noSoal === quizStore.listSoal.length) {
quizStore.setSession(false);
quizStore.setIsFinish(true);
setTimeout(() => {
isLoading = false;
setProgress(0);
noSoal++;
previousResult = [];
quizStore.setSoalIndex(noSoal);
Swal.fire({
title: "Loading",
text: "Proses menyimpan data...",
allowOutsideClick: false,
didOpen: () => {
Swal.showLoading();
},
});
if (noSoal === quizStore.listSoal.length) {
quizStore.setSession(false);
quizStore.setIsFinish(true);
navigate("/kuis/tebak-huruf");
}
Swal.fire({
title: "Loading",
text: "Proses menyimpan data...",
allowOutsideClick: false,
didOpen: () => {
Swal.showLoading();
},
});
navigate("/kuis/tebak-huruf");
}
}, 300);
}, 2000);
}
@ -221,7 +226,7 @@ const Quiz = () => {
const store = useNavbarStore();
const quizStore = useTebakHurufStore();
console.log(quizStore.jawaban);
// console.log(quizStore.jawaban);
useEffect(() => {
if (!quizStore.session) {
@ -231,6 +236,7 @@ const Quiz = () => {
useEffect(() => {
store.setNavSelected("kuis");
preloadImages();
loadModel();
startWebcam();
@ -249,22 +255,30 @@ const Quiz = () => {
<div
className={`fixed inset-0 w-screen h-screen bg-black/60 ${
showAnswer ? "opacity-100" : "opacity-0"
} z-[999] flex items-center justify-center pointer-events-none duration-300 ease-in-out`}
} z-[999] flex items-center justify-center pointer-events-none ease-in-out`}
>
<div className="rounded-md px-3 py-2 text-white flex flex-col justify-center items-center gap-3">
<img
className="h-56"
src={`/assets/gif/${
quizStore.jawaban[quizStore.soalIndex]?.isCorrect
? "betul"
: "salah"
}.gif`}
alt={`Jawaban ${
quizStore.jawaban[quizStore.soalIndex]?.isCorrect
? "Benar"
: "Salah"
className={`h-56 ${
quizStore.jawaban[quizStore.soalIndex]?.isCorrect ? "" : "hidden"
}`}
src={`/assets/gif/betul.gif`}
loading="eager"
fetchPriority="high"
alt={`Jawaban Benar`}
/>
<img
className={`h-56 ${
quizStore.jawaban[quizStore.soalIndex]?.isCorrect ? "hidden" : ""
}`}
src={`/assets/gif/salah.gif`}
loading="eager"
fetchPriority="high"
alt={`Jawaban Salah`}
/>
<p className="text-center text-6xl font-bold">
{quizStore.jawaban[quizStore.soalIndex]?.jawaban}
</p>

View File

@ -23,9 +23,14 @@ export const abjads = [
"W",
"X",
"Y",
"Tidak Dikenali",
];
const ConvertResult = (result: number) => {
if (result < 0 || result > 23) {
return "Tidak Dikenali";
}
return `Abjad ${abjads[result]}`;
};

View File

@ -0,0 +1,34 @@
import { FilesetResolver, HandLandmarker } from "@mediapipe/tasks-vision";
import { getHandLandmarkerModel, saveHandLandmarkerModel } from "./indexedDBHelper";
let handLandmarker: HandLandmarker | null = null;
export async function loadHandLandmarker(): Promise<HandLandmarker> {
if (handLandmarker) return handLandmarker; // Jika model sudah ada, langsung kembalikan
let modelBlob = await getHandLandmarkerModel();
if (!modelBlob) {
console.log("🔄 Model Hand Landmarker tidak ditemukan di cache, mengunduh...");
const response = await fetch(
"https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task"
);
modelBlob = await response.blob();
await saveHandLandmarkerModel(modelBlob);
} else {
console.log("✅ Model Hand Landmarker ditemukan di cache, menggunakan model lokal.");
}
const vision = await FilesetResolver.forVisionTasks(
"https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm"
);
const modelURL = URL.createObjectURL(modelBlob);
handLandmarker = await HandLandmarker.createFromOptions(vision, {
baseOptions: { modelAssetPath: modelURL },
numHands: 2,
runningMode: "VIDEO",
});
return handLandmarker;
}

View File

@ -0,0 +1,59 @@
import * as tf from "@tensorflow/tfjs";
const DB_NAME = "ModelCacheDB";
const STORE_NAME = "models";
const TENSORFLOW_MODEL_KEY = "tensorflow_model";
const HAND_LANDMARKER_MODEL_KEY = "hand_landmarker_model";
// Membuka IndexedDB
export function openDB(): Promise<IDBDatabase> {
return new Promise((resolve, reject) => {
const request = indexedDB.open(DB_NAME, 1);
request.onupgradeneeded = (event: IDBVersionChangeEvent) => {
const db = (event.target as IDBOpenDBRequest).result;
if (!db.objectStoreNames.contains(STORE_NAME)) {
db.createObjectStore(STORE_NAME);
}
};
request.onsuccess = () => resolve(request.result);
request.onerror = () => reject(request.error);
});
}
// 🟢 **Simpan Model TensorFlow.js ke IndexedDB**
export async function saveTensorFlowModel(model: tf.LayersModel): Promise<void> {
await model.save(`indexeddb://${TENSORFLOW_MODEL_KEY}`);
}
// 🟢 **Ambil Model TensorFlow.js dari IndexedDB**
export async function getTensorFlowModel(): Promise<tf.LayersModel | null> {
try {
return await tf.loadLayersModel(`indexeddb://${TENSORFLOW_MODEL_KEY}`);
} catch (error) {
console.warn("⚠️ Model TensorFlow tidak ditemukan di cache:", error);
return null;
}
}
// 🟢 **Simpan Model Hand Landmarker ke IndexedDB**
export async function saveHandLandmarkerModel(blob: Blob): Promise<void> {
const db = await openDB();
const transaction = db.transaction(STORE_NAME, "readwrite");
const store = transaction.objectStore(STORE_NAME);
store.put(blob, HAND_LANDMARKER_MODEL_KEY);
}
// 🟢 **Ambil Model Hand Landmarker dari IndexedDB**
export async function getHandLandmarkerModel(): Promise<Blob | null> {
const db = await openDB();
return new Promise((resolve, reject) => {
const transaction = db.transaction(STORE_NAME, "readonly");
const store = transaction.objectStore(STORE_NAME);
const request = store.get(HAND_LANDMARKER_MODEL_KEY);
request.onsuccess = () => resolve(request.result as Blob | null);
request.onerror = () => reject(request.error);
});
}

View File

@ -0,0 +1,20 @@
import * as tf from "@tensorflow/tfjs";
import { getTensorFlowModel, saveTensorFlowModel } from "./indexedDBHelper";
let model: tf.LayersModel | null = null;
export async function loadTensorFlowModel(): Promise<tf.LayersModel> {
if (model) return model; // Jika model sudah dimuat, langsung kembalikan
model = await getTensorFlowModel();
if (!model) {
console.log("🔄 Model TensorFlow tidak ditemukan di cache, mengunduh...");
model = await tf.loadLayersModel("/model/model.json");
await saveTensorFlowModel(model);
} else {
console.log("✅ Model TensorFlow ditemukan di cache, menggunakan model lokal.");
}
return model;
}