commit afe97830d04af47e0725224512fb929fe5c05835 Author: Rafael Dwiky Date: Tue Oct 27 09:52:03 2026 +0700 Skripsi diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3324626 --- /dev/null +++ b/.gitignore @@ -0,0 +1,56 @@ +# Python cache and tooling +__pycache__/ +*.py[cod] +*.pyo +*.pyd +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +.coverage +htmlcov/ + +# Python virtual environments and packages +.venv/ +venv/ +env/ +ENV/ +*.egg-info/ + +# Backend generated data and evaluation results +/backend/dataset/ +/backend/outputs/ + +# Frontend dependencies, build output, and test coverage +/frontend/node_modules/ +/frontend/build/ +/frontend/coverage/ +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* + +# Environment variables and local secrets +.env +.env.* +!.env.example +*.local +*.pem +*.key + +# Operating system files +.DS_Store +Thumbs.db +Desktop.ini + +# IDE and editor files +.vscode/* +!.vscode/extensions.json +.idea/ +*.swp +*.swo +*.code-workspace + +# Logs and temporary files +*.log +*.tmp +*.temp diff --git a/README.md b/README.md new file mode 100644 index 0000000..fc395b6 --- /dev/null +++ b/README.md @@ -0,0 +1,798 @@ +# Aplikasi Penerjemah Bahasa Isyarat Indonesia + +Aplikasi penerjemah bahasa isyarat Indonesia (BISINDO & SIBI) berbasis machine learning menggunakan **MediaPipe Hands**, **MediaPipe Face Mesh**, dan **Temporal Convolutional Network (TCN)**. + +--- + +## πŸ“‹ Daftar Isi + +- [Deskripsi](#deskripsi) +- [Fitur](#fitur) +- [Arsitektur Sistem](#arsitektur-sistem) +- [Struktur Direktori](#struktur-direktori) +- [Instalasi](#instalasi) +- [Library dan Dependensi](#library-dan-dependensi) +- [Pipeline Project](#pipeline-project) +- [Penggunaan](#penggunaan) + - [1. Ekstraksi Landmark](#1-ekstraksi-landmark) + - [2. Normalisasi Landmark](#2-normalisasi-landmark) + - [3. Split Dataset](#3-split-dataset) + - [4. Training Model](#4-training-model) + - [5. Testing Model](#5-testing-model) + - [6. Deteksi Realtime](#6-deteksi-realtime) + - [7. Menjalankan API Backend](#7-menjalankan-api-backend) + - [8. Menjalankan Frontend](#8-menjalankan-frontend) +- [API Reference](#api-reference) +- [Struktur Dataset](#struktur-dataset) +- [Model TCN](#model-tcn) +- [Konfigurasi](#konfigurasi) + +--- + +## πŸ“ Deskripsi + +Project ini merupakan aplikasi penerjemah bahasa isyarat Indonesia yang mendukung: +- **BISINDO** (Bahasa Isyarat Indonesia) +- **SIBI** (Sistem Isyarat Bahasa Indonesia) +- **Angka** +- **Kata/Kosakata** + +### Teknologi yang Digunakan + +| Teknologi | Fungsi | +|-----------|--------| +| **MediaPipe Hands** | Deteksi 21 landmark tangan kiri + 21 landmark tangan kanan | +| **MediaPipe Face Mesh** | Deteksi 60 landmark wajah (dagu, mulut, bibir, hidung, mata, pipi) | +| **Temporal Convolutional Network (TCN)** | Klasifikasi sequence gesture | +| **Flask** | Backend API server | +| **React.js** | Frontend web application | + +--- + +## ✨ Fitur + +- πŸŽ₯ Deteksi realtime gestur bahasa isyarat via webcam +- πŸ€– Model ML untuk 4 kategori (Abjad BISINDO, Abjad SIBI, Angka, Kata) +- πŸ“Š Visualisasi landmark tangan dan wajah +- πŸ“– Kamus isyarat dengan video contoh +- πŸ”„ Temporal smoothing untuk prediksi yang stabil +- πŸ“‘ REST API untuk komunikasi frontend-backend + +--- + +## πŸ—οΈ Arsitektur Sistem + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ FRONTEND (React.js) β”‚ +β”‚ http://localhost:3000 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β”‚ HTTP/JSON (Base64 Frame) + β–Ό +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ BACKEND (Flask API) β”‚ +β”‚ http://localhost:5000 β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ +β”‚ β”‚ /init β”‚ β”‚ /detect β”‚ β”‚ / β”‚ β”‚ +β”‚ β”‚ POST β”‚ β”‚ POST β”‚ β”‚ GET β”‚ β”‚ +β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β–Ό β–Ό β–Ό +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ MediaPipe β”‚ β”‚ Normalizer β”‚ β”‚ TCN Model Inference β”‚ +β”‚ Detector β”‚ β”‚ β”‚ β”‚ β”‚ +β”‚ - Hands β”‚ β”‚ - Translation β”‚ β”‚ - 30 Frame Window β”‚ +β”‚ - Face Mesh β”‚ β”‚ - Scaling β”‚ β”‚ - Temporal Smoothingβ”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +### Alur Data Processing + +``` +Video/ webcam + β”‚ + β–Ό +Extract Landmark (MediaPipe) + β”‚ + β”œβ”€β”€ 21 landmark tangan kiri + β”œβ”€β”€ 21 landmark tangan kanan + └── 60 landmark wajah + β”‚ + β–Ό +Normalisasi Landmark + β”‚ + β”œβ”€β”€ Translation ke wrist/nose + └── Scaling + β”‚ + β–Ό +Sliding Window (30 frame, stride 5) + β”‚ + β–Ό +Model TCN Inference + β”‚ + β–Ό +Prediksi Label + Confidence +``` + +--- + +## πŸ“ Struktur Direktori + +``` +APP BAHASA ISYARAT/ +β”‚ +β”œβ”€β”€ backend/ +β”‚ β”‚ +β”‚ β”œβ”€β”€ app.py # Flask API server +β”‚ β”œβ”€β”€ requirements.txt # Dependensi Python +β”‚ β”‚ +β”‚ β”œβ”€β”€ dataset/ +β”‚ β”‚ β”œβ”€β”€ raw/ # Dataset video mentah +β”‚ β”‚ β”‚ β”œβ”€β”€ abjad_bisindo/ +β”‚ β”‚ β”‚ β”œβ”€β”€ abjad_sibi/ +β”‚ β”‚ β”‚ β”œβ”€β”€ angka/ +β”‚ β”‚ β”‚ └── kata/ +β”‚ β”‚ β”œβ”€β”€ landmark/ # Landmark hasil ekstraksi +β”‚ β”‚ β”œβ”€β”€ normalized/ # Landmark hasil normalisasi +β”‚ β”‚ β”œβ”€β”€ train/ # Data training (80%) +β”‚ β”‚ └── test/ # Data testing (20%) +β”‚ β”‚ +β”‚ β”œβ”€β”€ models/ # Model TCN terlatih +β”‚ β”‚ β”œβ”€β”€ abjad_bisindo/ +β”‚ β”‚ β”‚ β”œβ”€β”€ model_abjad_bisindo.pt +β”‚ β”‚ β”‚ └── label_encoder.json +β”‚ β”‚ β”œβ”€β”€ abjad_sibi/ +β”‚ β”‚ β”œβ”€β”€ angka/ +β”‚ β”‚ └── kata/ +β”‚ β”‚ +β”‚ β”œβ”€β”€ outputs/ # Output training/testing +β”‚ β”‚ β”œβ”€β”€ confusion_matrix/ +β”‚ β”‚ β”œβ”€β”€ classification_report/ +β”‚ β”‚ β”œβ”€β”€ accuracy/ +β”‚ β”‚ └── loss/ +β”‚ β”‚ +β”‚ β”œβ”€β”€ realtime/ # Script deteksi realtime +β”‚ β”‚ β”œβ”€β”€ realtime_abjad_bisindo.py +β”‚ β”‚ β”œβ”€β”€ realtime_abjad_sibi.py +β”‚ β”‚ β”œβ”€β”€ realtime_angka.py +β”‚ β”‚ └── realtime_kata.py +β”‚ β”‚ +β”‚ β”œβ”€β”€ scripts/ # Pipeline preprocessing & training +β”‚ β”‚ β”œβ”€β”€ 01_extract_landmark.py +β”‚ β”‚ β”œβ”€β”€ 02_normalize_landmark.py +β”‚ β”‚ β”œβ”€β”€ 03_split_dataset.py +β”‚ β”‚ β”œβ”€β”€ 04_train_kata.py +β”‚ β”‚ β”œβ”€β”€ 05_test_kata.py +β”‚ β”‚ β”œβ”€β”€ 06_train_abjad_bisindo.py +β”‚ β”‚ β”œβ”€β”€ 07_test_abjad_bisindo.py +β”‚ β”‚ β”œβ”€β”€ 08_train_abjad_sibi.py +β”‚ β”‚ β”œβ”€β”€ 09_test_abjad_sibi.py +β”‚ β”‚ β”œβ”€β”€ 10_train_angka.py +β”‚ β”‚ └── 11_test_angka.py +β”‚ β”‚ +β”‚ └── utils/ # Utility modules +β”‚ β”œβ”€β”€ mediapipe_detector.py # MediaPipe detector +β”‚ β”œβ”€β”€ normalization.py # Normalisasi landmark +β”‚ β”œβ”€β”€ dataset.py # PyTorch Dataset +β”‚ β”œβ”€β”€ tcn.py # Arsitektur TCN +β”‚ β”œβ”€β”€ inference.py # Inference utilities +β”‚ └── config.py # Konfigurasi project +β”‚ +└── frontend/ + β”œβ”€β”€ package.json # Dependensi Node.js + β”œβ”€β”€ public/ + └── src/ + β”œβ”€β”€ App.js # Router utama + β”œβ”€β”€ index.js # Entry point + └── pages/ # Halaman React + β”œβ”€β”€ HomePage.jsx + β”œβ”€β”€ CategoryPage.jsx + β”œβ”€β”€ DetectionPage.jsx # Halaman deteksi webcam + β”œβ”€β”€ DictionaryPage.jsx # Kamus isyarat + β”œβ”€β”€ AngkaPage.jsx + β”œβ”€β”€ SibiPage.jsx + β”œβ”€β”€ BisindoPage.jsx + └── KataPage.jsx +``` + +--- + +## πŸ”§ Instalasi + +### Prasyarat + +- Python 3.8+ +- Node.js 16+ +- npm atau yarn +- Webcam (untuk deteksi realtime) + +### 1. Setup Backend + +```bash +# Masuk ke direktori backend +cd backend + +# Buat virtual environment (opsional tapi disarankan) +python -m venv venv +source venv/bin/activate # Linux/Mac +# atau +.\venv\Scripts\activate # Windows + +# Install dependensi +pip install -r requirements.txt +``` + +### 2. Setup Frontend + +```bash +# Masuk ke direktori frontend +cd frontend + +# Install dependensi +npm install +``` + +--- + +## πŸ“š Library dan Dependensi + +### Backend (Python) + +| Library | Versi Minimal | Fungsi | +|---------|---------------|--------| +| `numpy` | >=1.21.0 | Manipulasi array dan operasi matematika | +| `torch` | >=1.10.0 | Framework deep learning | +| `torchvision` | >=0.11.0 | Transformasi data PyTorch | +| `mediapipe` | >=0.10.0 | Deteksi landmark tangan & wajah | +| `opencv-python` | >=4.8.0 | Processing video | +| `opencv-python-headless` | >=4.8.0 | OpenCV tanpa GUI | +| `scikit-learn` | >=0.24.0 | Evaluasi model (confusion matrix, dll) | +| `matplotlib` | >=3.7.0 | Visualisasi data | +| `seaborn` | >=0.12.0 | Visualisasi statistik | +| `tqdm` | >=4.65.0 | Progress bar | +| `Flask` | >=2.3.0 | Web framework | +| `flask-cors` | >=4.0.0 | CORS support | +| `ml_dtypes` | >=0.3.0 | Dependensi MediaPipe | + +### Frontend (JavaScript/React) + +| Library | Fungsi | +|---------|--------| +| `react` | UI framework | +| `react-dom` | React DOM rendering | +| `react-router-dom` | Routing | +| `react-scripts` | Build tool (Create React App) | +| `@testing-library/*` | Testing utilities | +| `web-vitals` | Performance metrics | + +--- + +## πŸ”„ Pipeline Project + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ PIPELINE MACHINE LEARNING β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + +Raw Video (.mp4) + β”‚ + β–Ό +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ Tahap 1: Extract Landmark β”‚ +β”‚ Script: 01_extract_landmark.py β”‚ +β”‚ Output: dataset/landmark/ β”‚ +β”‚ Input shape: (30, 306) per window β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β–Ό +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ Tahap 2: Normalize Landmark β”‚ +β”‚ Script: 02_normalize_landmark.py β”‚ +β”‚ Output: dataset/normalized/ β”‚ +β”‚ Proses: Translation + Scaling β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β–Ό +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ Tahap 3: Split Dataset β”‚ +β”‚ Script: 03_split_dataset.py β”‚ +β”‚ Output: dataset/train/ & dataset/test/ β”‚ +β”‚ Rasio: 80% train, 20% test β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β–Ό +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ Tahap 4: Training TCN β”‚ +β”‚ Scripts: 04-11_train_*.py β”‚ +β”‚ Output: models/{category}/model_*.pt β”‚ +β”‚ Output: models/{category}/label_encoder β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β–Ό +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ Tahap 5: Testing Model β”‚ +β”‚ Scripts: 05-11_test_*.py β”‚ +β”‚ Output: Confusion Matrix, Classificationβ”‚ +β”‚ Report, Accuracy, F1-Score β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β–Ό +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ Tahap 6: Realtime Detection β”‚ +β”‚ Scripts: realtime/*.py β”‚ +β”‚ Input: Webcam β”‚ +β”‚ Output: Prediksi real-time β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +--- + +## πŸ“– Penggunaan + +### 1. Ekstraksi Landmark + +Ekstrak landmark dari video menggunakan MediaPipe. + +```bash +cd backend + +# Ekstrak semua kategori +python scripts/01_extract_landmark.py + +# Ekstrak kategori tertentu +python scripts/01_extract_landmark.py --categories abjad_bisindo angka + +# Custom parameter +python scripts/01_extract_landmark.py --window-size 30 --stride 5 +``` + +**Parameter:** +| Parameter | Default | Deskripsi | +|-----------|---------|-----------| +| `--input` | `dataset/raw/` | Direktori video mentah | +| `--output` | `dataset/landmark/` | Direktori output landmark | +| `--categories` | Semua | Kategori yang diproses | +| `--window-size` | 30 | Jumlah frame per window | +| `--stride` | 5 | Step size sliding window | + +--- + +### 2. Normalisasi Landmark + +Normalisasi koordinat landmark untuk konsistensi. + +```bash +cd backend + +# Normalisasi semua kategori +python scripts/02_normalize_landmark.py + +# Custom input/output +python scripts/02_normalize_landmark.py --input dataset/landmark --output dataset/normalized +``` + +--- + +### 3. Split Dataset + +Bagi dataset menjadi training (80%) dan testing (20%). + +```bash +cd backend + +# Split semua kategori +python scripts/03_split_dataset.py + +# Custom rasio +python scripts/03_split_dataset.py --train-ratio 0.8 + +# Split kategori tertentu +python scripts/03_split_dataset.py --categories abjad_bisindo kata +``` + +--- + +### 4. Training Model + +Latih model TCN untuk setiap kategori. + +```bash +cd backend + +# Training model kata +python scripts/04_train_kata.py + +# Training model abjad BISINDO +python scripts/06_train_abjad_bisindo.py + +# Training model abjad SIBI +python scripts/08_train_abjad_sibi.py + +# Training model angka +python scripts/10_train_angka.py + +# Custom parameter training +python scripts/04_train_kata.py --epochs 100 --batch-size 32 --lr 0.001 +``` + +**Parameter Training:** +| Parameter | Default | Deskripsi | +|-----------|---------|-----------| +| `--epochs` | 100 | Jumlah epoch | +| `--batch-size` | 32 | Batch size | +| `--lr` | 0.001 | Learning rate | +| `--data-dir` | `dataset/` | Direktori dataset | +| `--model-dir` | `models/` | Direktori output model | + +**Output Training:** +``` +models/{category}/ +β”œβ”€β”€ model_{category}.pt # Model PyTorch +β”œβ”€β”€ label_encoder.json # Mapping label ke index +└── history.json # History training +``` + +--- + +### 5. Testing Model + +Evaluasi model menggunakan data test. + +```bash +cd backend + +# Test model kata +python scripts/05_test_kata.py + +# Test model abjad BISINDO +python scripts/07_test_abjad_bisindo.py + +# Test model abjad SIBI +python scripts/09_test_abjad_sibi.py + +# Test model angka +python scripts/11_test_angka.py +``` + +**Output Testing:** +``` +outputs/ +β”œβ”€β”€ confusion_matrix/ +β”‚ └── {category}_confusion_matrix.png +β”œβ”€β”€ classification_report/ +β”‚ └── {category}_classification_report.txt +└── {category}_test_results.json +``` + +**Metrik Evaluasi:** +- Accuracy +- Precision +- Recall +- F1-Score +- Confusion Matrix + +--- + +### 6. Deteksi Realtime + +Jalankan deteksi realtime menggunakan webcam (tanpa frontend). + +```bash +cd backend + +# Deteksi kata +python realtime/realtime_kata.py + +# Deteksi abjad BISINDO +python realtime/realtime_abjad_bisindo.py + +# Deteksi abjad SIBI +python realtime/realtime_abjad_sibi.py + +# Deteksi angka +python realtime/realtime_angka.py + +# Custom camera index +python realtime/realtime_kata.py --camera 0 +``` + +**Kontrol Keyboard:** +| Key | Fungsi | +|-----|--------| +| `q` | Quit | +| `r` | Reset buffer | +| `l` | Toggle landmark visibility | + +--- + +### 7. Menjalankan API Backend + +Jalankan Flask API server untuk komunikasi dengan frontend. + +```bash +cd backend + +# Jalankan server +python app.py +``` + +Server akan berjalan di `http://localhost:5000` + +**Endpoints:** + +| Endpoint | Method | Deskripsi | +|----------|--------|-----------| +| `/` | GET | Status server | +| `/init` | POST | Inisialisasi session model | +| `/detect` | POST | Proses frame dan prediksi | + +--- + +### 8. Menjalankan Frontend + +Jalankan aplikasi React. + +```bash +cd frontend + +# Development mode +npm start + +# Production build +npm run build +``` + +Frontend akan berjalan di `http://localhost:3000` + +--- + +## πŸ”Œ API Reference + +### GET / + +Cek status server. + +**Response:** +```json +{ + "status": "online", + "message": "Backend Penerjemah Bahasa Isyarat Indonesia (TCN) Siap!", + "categories": ["abjad_bisindo", "abjad_sibi", "angka", "kata"], + "endpoints": { + "/init": "POST - Inisialisasi session model", + "/detect": "POST - Kirim frame base64 dan dapatkan prediksi" + } +} +``` + +### POST /init + +Inisialisasi session inferensi untuk kategori tertentu. + +**Request Body:** +```json +{ + "category": "kata" +} +``` + +**Response:** +```json +{ + "session_id": "uuid-string", + "category": "kata", + "status": "ready" +} +``` + +**Kategori Valid:** +- `abjad_bisindo` +- `abjad_sibi` +- `angka` +- `kata` + +### POST /detect + +Kirim frame webcam dan dapatkan prediksi. + +**Request Body:** +```json +{ + "session_id": "uuid-string", + "frame": "data:image/jpeg;base64,..." +} +``` + +**Response:** +```json +{ + "label": "halo", + "confidence": 0.95, + "buffer_size": 30, + "window_size": 30 +} +``` + +--- + +## πŸ“‚ Struktur Dataset + +### Format Dataset Raw + +``` +dataset/raw/ +β”œβ”€β”€ abjad_bisindo/ +β”‚ β”œβ”€β”€ A/ +β”‚ β”‚ β”œβ”€β”€ video1.mp4 +β”‚ β”‚ └── video2.mp4 +β”‚ β”œβ”€β”€ B/ +β”‚ β”‚ └── ... +β”‚ └── ... +β”œβ”€β”€ abjad_sibi/ +β”‚ └── ... +β”œβ”€β”€ angka/ +β”‚ β”œβ”€β”€ 0/ +β”‚ β”œβ”€β”€ 1/ +β”‚ └── ... +└── kata/ + β”œβ”€β”€ halo/ + β”œβ”€β”€ terima_kasih/ + └── ... +``` + +### Format Landmark + +``` +dataset/landmark/ +β”œβ”€β”€ abjad_bisindo/ +β”‚ β”œβ”€β”€ A/ +β”‚ β”‚ β”œβ”€β”€ video1_w0.npy # Window 0 +β”‚ β”‚ β”œβ”€β”€ video1_w1.npy # Window 1 +β”‚ β”‚ └── ... +β”‚ └── ... +└── ... +``` + +### Shape Data + +| Tahap | Shape | Deskripsi | +|-------|-------|-----------| +| Raw Video | (N, H, W, 3) | N frame | +| Landmark | (window, 306) | 30 frame Γ— 306 fitur | +| Normalized | (window, 306) | Setelah normalisasi | + +**Detail Feature Size (306):** +- Tangan kiri: 21 Γ— 3 = 63 +- Tangan kanan: 21 Γ— 3 = 63 +- Wajah: 60 Γ— 3 = 180 +- **Total: 306** + +--- + +## 🧠 Model TCN + +### Arsitektur + +``` +TCNClassifier( + β”œβ”€β”€ TemporalConvNet + β”‚ β”œβ”€β”€ TemporalBlock (dilation 1) + β”‚ β”œβ”€β”€ TemporalBlock (dilation 2) + β”‚ └── TemporalBlock (dilation 4) + β”‚ + └── FC Layers + β”œβ”€β”€ Linear(256, 512) + β”œβ”€β”€ ReLU() + β”œβ”€β”€ Dropout(0.2) + └── Linear(512, num_classes) +) +``` + +### Hyperparameter Default + +| Parameter | Nilai | +|-----------|-------| +| Input Size | 306 | +| Hidden Channels | [64, 128, 256] | +| Kernel Size | 3 | +| Dropout | 0.2 | +| FC Hidden Size | 512 | +| Window Size | 30 | + +--- + +## βš™οΈ Konfigurasi + +Konfigurasi utama ada di `backend/utils/config.py`: + +```python +# Paths +BASE_DIR = Path(__file__).parent.parent +DATASET_DIR = BASE_DIR / "dataset" +MODEL_DIR = BASE_DIR / "models" + +# Categories +CATEGORIES = ["abjad_bisindo", "abjad_sibi", "angka", "kata"] + +# Sliding Window +WINDOW_SIZE = 30 +STRIDE = 5 + +# Feature Configuration +NUM_HAND_LANDMARKS = 21 +NUM_FACE_LANDMARKS = 60 +FEATURE_SIZE = 306 # 63 + 63 + 180 + +# Training +BATCH_SIZE = 32 +EPOCHS = 100 +LEARNING_RATE = 0.001 +TRAIN_RATIO = 0.8 + +# Session +SESSION_TIMEOUT_SECONDS = 600 # 10 menit +``` + +--- + +## πŸ“Œ Catatan Penting + +### Persyaratan Video +- Format: `.mp4` +- Durasi minimum: 30 frame (untuk window size 30) +- Disarankan: 60-90 frame per video + +### GPU Acceleration +Model akan otomatis menggunakan GPU jika tersedia: +```python +device = 'cuda' if torch.cuda.is_available() else 'cpu' +``` + +### Session Management +- Session akan expire setelah 10 menit tidak aktif +- Frontend akanauto-reconnect saat session expired + +--- + +## πŸ› Troubleshooting + +### Error: "ModuleNotFoundError" +```bash +pip install -r requirements.txt +``` + +### Error: "Camera not found" +```python +# Ganti camera index di realtime script +cap = cv2.VideoCapture(0) # Coba 0, 1, 2... +``` + +### Error: "Model not found" +Pastikan model sudah dilatih: +```bash +python scripts/04_train_kata.py +``` + +### Frontend tidak connect ke Backend +1. Pastikan backend running di port 5000 +2. Cek CORS settings di `app.py` +3. Pastikan URL di frontend sesuai + +--- + +## πŸ“„ Lisensi + +Project ini adalah bagian dari Tugas Akhir (TA) - Aplikasi Penerjemah Bahasa Isyarat Indonesia. + +--- + +## πŸ‘¨β€πŸ’» Kontribusi + +Project ini dikembangkan menggunakan: +- MediaPipe (Google) +- PyTorch +- Flask +- React.js diff --git a/backend/README.md b/backend/README.md new file mode 100644 index 0000000..6043267 --- /dev/null +++ b/backend/README.md @@ -0,0 +1,637 @@ +# Aplikasi Penerjemah Bahasa Isyarat Indonesia Menggunakan MediaPipe Hands, Face Mesh, dan Temporal Convolutional Network (TCN) + +## Deskripsi + +Project ini merupakan backend machine learning untuk aplikasi penerjemah Bahasa Isyarat Indonesia (SIBI dan BISINDO) menggunakan MediaPipe Hands, MediaPipe Face Mesh, dan Temporal Convolutional Network (TCN). + +Model dibagi menjadi empat kategori agar sesuai dengan alur aplikasi frontend. + +- Abjad Bisindo +- Abjad SIbi +- Angka +- Kata + +Setiap kategori memiliki model TCN tersendiri sehingga proses inferensi lebih cepat dan akurasi lebih baik. + +--- + +# Pipeline Project + +``` +Raw Video + β”‚ + β–Ό +Extract Landmark + β”‚ + β–Ό +Normalize Landmark + β”‚ + β–Ό +Split Dataset + β”‚ + β”œβ”€β”€ Train 80% + └── Test 20% + β”‚ + β–Ό +Training TCN + β”‚ + β–Ό +Testing Model + β”‚ + β–Ό +Realtime Camera + β”‚ + β–Ό +Frontend ReactJS +``` + +--- + +# Struktur Folder + +``` +backend/ + +β”‚ +β”œβ”€β”€ dataset/ +β”‚ +β”‚ β”œβ”€β”€ raw/ +β”‚ β”‚ +β”‚ β”‚ β”œβ”€β”€ abjad_bisindo/ +β”‚ β”‚ β”‚ +β”‚ β”‚ β”œβ”€β”€ abjad_sibi/ +β”‚ β”‚ β”‚ +β”‚ β”‚ β”œβ”€β”€ angka/ +β”‚ β”‚ β”‚ +β”‚ β”‚ └── kata/ +β”‚ β”‚ +β”‚ β”œβ”€β”€ landmark/ +β”‚ β”‚ +β”‚ β”œβ”€β”€ normalized/ +β”‚ β”‚ +β”‚ β”œβ”€β”€ train/ +β”‚ β”‚ +β”‚ └── test/ +β”‚ +β”‚ +β”œβ”€β”€ models/ +β”‚ +β”‚ β”œβ”€β”€ abjad_bisindo/ +β”‚ β”‚ +β”‚ β”œβ”€β”€ abjad_sibi/ +β”‚ β”‚ +β”‚ β”œβ”€β”€ angka/ +β”‚ β”‚ +β”‚ └── kata/ +β”‚ +β”‚ +β”œβ”€β”€ outputs/ +β”‚ +β”‚ β”œβ”€β”€ confusion_matrix/ +β”‚ β”‚ +β”‚ β”œβ”€β”€ classification_report/ +β”‚ β”‚ +β”‚ β”œβ”€β”€ accuracy/ +β”‚ β”‚ +β”‚ └── loss/ +β”‚ +β”‚ +β”œβ”€β”€ realtime/ +β”‚ +β”‚ β”œβ”€β”€ realtime_abjad_bisindo.py +β”‚ β”‚ +β”‚ β”œβ”€β”€ realtime_abjad_sibi.py +β”‚ β”‚ +β”‚ β”œβ”€β”€ realtime_angka.py +β”‚ β”‚ +β”‚ └── realtime_kata.py +β”‚ +β”‚ +β”œβ”€β”€ scripts/ +β”‚ +β”‚ β”œβ”€β”€ 01_extract_landmark.py +β”‚ β”‚ +β”‚ β”œβ”€β”€ 02_normalize_landmark.py +β”‚ β”‚ +β”‚ β”œβ”€β”€ 03_split_dataset.py +β”‚ β”‚ +β”‚ β”œβ”€β”€ 04_train_abjad_kata.py +β”‚ β”‚ +β”‚ β”œβ”€β”€ 05_test_abjad_kata.py +β”‚ β”‚ +β”‚ β”œβ”€β”€ 06_train_abjad_bisindo.py +β”‚ β”‚ +β”‚ β”œβ”€β”€ 07_test_abjad_bisindo.py +β”‚ β”‚ +β”‚ β”œβ”€β”€ 08_train_sibi.py +β”‚ β”‚ +β”‚ β”œβ”€β”€ 09_test_sibi.py +β”‚ β”‚ +β”‚ β”œβ”€β”€ 10_train_angka.py +β”‚ β”‚ +β”‚ └── 11_test_angka.py +β”‚ +β”‚ +β”œβ”€β”€ utils/ +β”‚ +β”‚ β”œβ”€β”€ mediapipe_detector.py +β”‚ β”‚ +β”‚ β”œβ”€β”€ normalization.py +β”‚ β”‚ +β”‚ β”œβ”€β”€ dataset.py +β”‚ β”‚ +β”‚ β”œβ”€β”€ tcn.py +β”‚ β”‚ +β”‚ β”œβ”€β”€ inference.py +β”‚ β”‚ +β”‚ └── config.py +β”‚ +β”‚ +β”œβ”€β”€ requirements.txt +β”‚ +└── README.md +``` + +--- + +# Struktur Dataset + +Semua tahapan preprocessing akan mempertahankan struktur folder yang sama seperti folder `raw`. + +``` +dataset/ + +β”œβ”€β”€ raw/ +β”œβ”€β”€ landmark/ +β”œβ”€β”€ normalized/ +β”œβ”€β”€ train/ +└── test/ +``` + +Artinya setiap folder memiliki struktur yang identik sehingga proses preprocessing menjadi lebih mudah, konsisten, dan mudah dikembangkan apabila terdapat penambahan dataset baru. + +--- + +# Struktur Dataset Raw + +``` +dataset/raw/ + +abjad_bisindo/ + + a/ + a_s1_light_on.mp4 + a_s2_light_off.mp4 + ... + + b/ + c/ + ... + +abjad_sibi/ + + a/ + b/ + ... + +angka/ + + 0/ + 1/ + ... + +kata/ + + halo/ + terimakasih/ + ... +``` + +**Format video yang didukung** + +- `.mp4` + +--- + +# Tahap 1 + +## Extract Landmark + +Script + +``` +scripts/01_extract_landmark.py +``` + +Input + +``` +dataset/raw/ +``` + +Output + +``` +dataset/landmark/ +``` + +Proses + +- Membaca seluruh video +- Membaca frame menggunakan OpenCV +- Deteksi MediaPipe Hands 21 landmark tangan kiri, 21 landmark tangan kanan +- Deteksi MediaPipe Face Mesh 60 landmark dari dagu, mulut, bibir, hidung, mata, pipi +- Mengambil landmark +- Membentuk Sliding Window 30 Frame dan 5 Stride +- Menyimpan file .npy + +Output contoh + +``` +dataset/ + +landmark/ + +abjad_bisindo/ + +A/ + +sample_1.npy +sample_2.npy +... +``` + +Shape + +``` +(30, feature_size) +``` + +> **Catatan:** Struktur folder pada `landmark/` dibuat **sama persis** dengan struktur folder `raw/`. + +--- + +# Tahap 2 + +## Normalize Landmark + +Script + +``` +scripts/02_normalize_landmark.py +``` + +Input + +``` +dataset/landmark/ +``` + +Output + +``` +dataset/normalized/ +``` + +Proses + +- Membaca seluruh landmark +- Menggeser koordinat ke titik referensi +- Scaling +- Menyimpan landmark hasil normalisasi + +> **Catatan:** Struktur folder pada `normalized/` akan dibuat **identik** dengan folder `landmark/` dan `raw/`. + +--- + +# Tahap 3 + +## Split Dataset + +Script + +``` +scripts/03_split_dataset.py +``` + +Input + +``` +dataset/normalized/ +``` + +Output + +``` +dataset/train/ + +dataset/test/ +``` + +Persentase + +``` +Train + +80% + +Test + +20% +``` + +Pembagian dilakukan **untuk setiap kelas**, sehingga semua kelas tetap memiliki data train dan data test. + +Seluruh struktur folder pada `train/` dan `test/` akan dibuat **sama persis** dengan struktur folder `normalized/`, `landmark/`, dan `raw/`. + +--- + +# Tahap 4 + +## Training Kata + +Script + +``` +scripts/04_train_kata.py +``` + +Input + +``` +dataset/train/kata/ +``` + +Output + +``` +models/kata/model_kata.pt +``` + +Output tambahan + +``` +Accuracy + +Loss + +Training History +``` + +--- + +# Tahap 5 + +## Testing Kata + +Script + +``` +scripts/05_test_kata.py +``` + +Input + +``` +dataset/test/kata/ +``` + +Model + +``` +model_kata.pt +``` + +Output + +``` +Confusion Matrix + +Classification Report + +Accuracy + +Precision + +Recall + +F1 Score +``` + +--- + +# Tahap 6 + +## Training Abjad BISINDO + +Script + +``` +scripts/06_train_abjad_bisindo.py +``` + +Output + +``` +model_abjad_bisindo.pt +``` + +--- + +# Tahap 7 + +## Testing Abjad BISINDO + +Script + +``` +scripts/07_test_abjad_bisindo.py +``` + +--- + +# Tahap 8 + +## Training Abjad SIBI + +Script + +``` +scripts/08_train_abjad_sibi.py +``` + +Output + +``` +model_abjad_sibi.pt +``` + +--- + +# Tahap 9 + +## Testing Abjad SIBI + +Script + +``` +scripts/09_test_abjad_sibi.py +``` + +--- + +# Tahap 10 + +## Training Angka + +Script + +``` +scripts/10_train_angka.py +``` + +Output + +``` +model_angka.pt +``` + +--- + +# Tahap 11 + +## Testing Angka + +Script + +``` +scripts/11_test_angka.py +``` + +Output + +``` +Accuracy + +Precision + +Recall + +F1 Score + +Confusion Matrix +``` + +--- + +# Tahap 12 + +## Realtime Detection + +Folder + +``` +realtime/ +``` + +Script + +``` +realtime_abjad_bisindo.py + +realtime_abjad_sibi.py + +realtime_angka.py + +realtime_kata.py +``` + +Proses + +``` +Webcam + +↓ + +MediaPipe + +↓ + +Landmark + +↓ + +Normalisasi + +↓ + +Window 30 Frame + +↓ + +Model TCN + +↓ + +Prediksi + +↓ + +Tampilkan hasil +``` + +--- + +# Fungsi Folder Utils + +## mediapipe_detector.py + +Mengambil landmark tangan dan wajah menggunakan MediaPipe. + +--- + +## normalization.py + +Melakukan normalisasi koordinat landmark. + +--- + +## dataset.py + +Membaca dataset .npy menjadi Dataset PyTorch. + +--- + +## tcn.py + +Berisi implementasi model Temporal Convolutional Network. + +--- + +## inference.py + +Melakukan prediksi menggunakan model TCN. + +--- + +## config.py + +Berisi konfigurasi project. + +--- + +# Model + +Project menghasilkan empat model. + +``` +model_abjad_bisindo.pt + +model_abjad_sibi.pt + +model_angka.pt + +model_kata.pt +``` + +--- diff --git a/backend/app.py b/backend/app.py new file mode 100644 index 0000000..a66a683 --- /dev/null +++ b/backend/app.py @@ -0,0 +1,174 @@ +""" +Flask API Server untuk Aplikasi Penerjemah Bahasa Isyarat Indonesia. + +Menyediakan REST API endpoint untuk frontend React: +- POST /init : Inisialisasi session inferensi berdasarkan kategori (angka, abjad_bisindo, abjad_sibi, kata) +- POST /detect : Memproses frame kamera (base64) dari frontend dan mengembalikan hasil prediksi +""" + +import sys +import uuid +import time +import base64 +import numpy as np +import cv2 +from pathlib import Path +from flask import Flask, request, jsonify +from flask_cors import CORS + +# Menambahkan root folder backend ke sys.path +BASE_DIR = Path(__file__).parent +sys.path.insert(0, str(BASE_DIR)) + +from utils.inference import GestureRecognizer +from utils.config import MODEL_DIR, WINDOW_SIZE, CATEGORIES + +app = Flask(__name__) + +# Mengizinkan Cross-Origin Requests (CORS) agar Frontend React (port 3000) bisa terhubung +CORS(app, resources={r"/*": {"origins": "*"}}) + +# Dictionary penyimpanan session inferensi +# Format: { session_id: { "category": str, "recognizer": GestureRecognizer, "last_accessed": float } } +sessions = {} + +# Timeout session (10 menit tanpa aktivitas -> cleanup) +SESSION_TIMEOUT_SECONDS = 600 + + +def cleanup_expired_sessions(): + """Hapus session yang sudah tidak aktif untuk menghemat memory.""" + now = time.time() + expired = [ + sid for sid, sess in sessions.items() + if now - sess.get("last_accessed", 0) > SESSION_TIMEOUT_SECONDS + ] + for sid in expired: + try: + sessions[sid]["recognizer"].close() + except Exception: + pass + del sessions[sid] + + +@app.route("/", methods=["GET"]) +def index(): + return jsonify({ + "status": "online", + "message": "Backend Penerjemah Bahasa Isyarat Indonesia (TCN) Siap!", + "categories": CATEGORIES, + "endpoints": { + "/init": "POST - Inisialisasi session model", + "/detect": "POST - Kirim frame base64 dan dapatkan prediksi" + } + }) + + +@app.route("/init", methods=["POST"]) +def init_session(): + """Inisialisasi session inferensi untuk frontend.""" + cleanup_expired_sessions() + + data = request.get_json() or {} + category = data.get("category", "kata") + + if category not in CATEGORIES: + return jsonify({ + "error": f"Kategori '{category}' tidak valid. Pilihan kategori: {CATEGORIES}" + }), 400 + + model_path = MODEL_DIR / category / f"model_{category}.pt" + encoder_path = MODEL_DIR / category / "label_encoder.json" + + if not model_path.exists(): + return jsonify({ + "error": f"Model untuk kategori '{category}' tidak ditemukan di {model_path}" + }), 404 + + if not encoder_path.exists(): + return jsonify({ + "error": f"Label encoder untuk kategori '{category}' tidak ditemukan di {encoder_path}" + }), 404 + + try: + recognizer = GestureRecognizer( + model_path=model_path, + label_encoder_path=encoder_path, + window_size=WINDOW_SIZE, + draw_landmarks=False + ) + session_id = str(uuid.uuid4()) + sessions[session_id] = { + "category": category, + "recognizer": recognizer, + "last_accessed": time.time() + } + + print(f"[INIT] Session {session_id} dibuat untuk kategori: {category}") + return jsonify({ + "session_id": session_id, + "category": category, + "status": "ready" + }), 200 + + except Exception as e: + print(f"[ERROR] Gagal inisialisasi session: {e}") + return jsonify({"error": f"Gagal inisialisasi model: {str(e)}"}), 500 + + +@app.route("/detect", methods=["POST"]) +def detect_frame(): + """Memproses 1 frame JPEG base64 dari frontend dan mengembalikan hasil prediksi.""" + data = request.get_json() or {} + session_id = data.get("session_id") + frame_data = data.get("frame") + + if not session_id or session_id not in sessions: + return jsonify({"error": "Session ID tidak ditemukan atau expired. Silakan muat ulang halaman."}), 400 + + if not frame_data: + return jsonify({"error": "Data frame gambar tidak boleh kosong."}), 400 + + session = sessions[session_id] + session["last_accessed"] = time.time() + recognizer = session["recognizer"] + + try: + # Decode base64 image dari dataURL format (data:image/jpeg;base64,...) + if "," in frame_data: + frame_data = frame_data.split(",")[1] + + image_bytes = base64.b64decode(frame_data) + nparr = np.frombuffer(image_bytes, np.uint8) + frame = cv2.imdecode(nparr, cv2.IMREAD_COLOR) + + if frame is None: + return jsonify({"error": "Gagal mendekode gambar dari frame data."}), 400 + + # Process frame + label, confidence = recognizer.process_frame(frame) + + buffer_size = len(recognizer.buffer) + window_size = recognizer.window_size + + # Print log untuk debug + if buffer_size > 0: + print(f"[{session['category'].upper()}] Buffer: {buffer_size}/{window_size} | Prediksi: {label if label else '...'}") + + return jsonify({ + "label": label if label is not None else None, + "confidence": float(confidence) if confidence is not None else 0.0, + "buffer_size": buffer_size, + "window_size": window_size + }), 200 + + except Exception as e: + print(f"[ERROR] Exception saat memproses frame: {e}") + return jsonify({"error": f"Error memproses frame: {str(e)}"}), 500 + + +if __name__ == "__main__": + print("=" * 60) + print(" STARTING BACKEND FLASK SERVER (PORT 5000)") + print("=" * 60) + app.run(host="0.0.0.0", port=5000, debug=True) diff --git a/backend/models/abjad_bisindo/history.json b/backend/models/abjad_bisindo/history.json new file mode 100644 index 0000000..20abca2 --- /dev/null +++ b/backend/models/abjad_bisindo/history.json @@ -0,0 +1,410 @@ +{ + "train_loss": [ + 1.5748291508121006, + 1.313897638932123, + 1.2141942426941152, + 1.149300922870131, + 1.0958258122458295, + 1.0510434047776764, + 1.011689099492663, + 0.9716906165419998, + 0.939193170271435, + 0.9123329845336029, + 0.8856088568724818, + 0.8636209451546103, + 0.8387187182145603, + 0.8202173071897636, + 0.796350415736058, + 0.7784183181411887, + 0.7713465532761509, + 0.7500143765519232, + 0.7426115740879865, + 0.726687729219764, + 0.7184128944935687, + 0.709730332478944, + 0.6965001238983567, + 0.6879600547569789, + 0.6812859095490964, + 0.6699685157071483, + 0.6604369327520667, + 0.6549840101482883, + 0.651449440549112, + 0.6466304036157238, + 0.6441302910573402, + 0.63207894296117, + 0.6352141795964059, + 0.6246137397055972, + 0.626480469749392, + 0.6193447640623455, + 0.6153222816412227, + 0.6124987483277159, + 0.6047568864504791, + 0.608962155346585, + 0.6022711705432227, + 0.5458365550098973, + 0.537587986527883, + 0.5390045229556686, + 0.5304915702144094, + 0.5339429753925257, + 0.5287424210210379, + 0.5304791700576384, + 0.5273460065867057, + 0.5266827925563922, + 0.5273894643772619, + 0.5245254044020785, + 0.524798184886409, + 0.5287661519471825, + 0.5200734632427141, + 0.5012993795017345, + 0.5003996313936191, + 0.5001111038300008, + 0.4994928677900427, + 0.4994188940609801, + 0.49994342079988785, + 0.4971893386441279, + 0.4926408766587475, + 0.4909394289243271, + 0.4895352427455491, + 0.4908438322097489, + 0.4898921017929597, + 0.4902769059620014, + 0.48999726226870505, + 0.4895176852120823, + 0.49006391950651496, + 0.48682986277163165, + 0.48677835346524556, + 0.486094390400797, + 0.4862956269855512, + 0.4857775389516881, + 0.4858230252085027, + 0.4851837724622606, + 0.48574044555171025, + 0.4849040159962982, + 0.48537822934034675, + 0.48492309527821403, + 0.4851960905569563, + 0.4847374036914447, + 0.4844478008946623, + 0.484438501255632, + 0.48434210287455554, + 0.4841712624936667, + 0.48406325649642107, + 0.484237751131236, + 0.4844073577035164, + 0.4839994715214173, + 0.4843415894116981, + 0.483532013739146, + 0.48357039407876795, + 0.4842723530559073, + 0.4840719777999179, + 0.4836043946853644, + 0.48381175970708235, + 0.48385722420951083 + ], + "train_acc": [ + 50.84568416312367, + 59.87337916054092, + 62.67864001801301, + 64.61636269718282, + 66.09713778625449, + 67.50639064383253, + 68.45869591131243, + 69.56464152792678, + 70.20569263983259, + 71.05468801737726, + 71.89573648031153, + 72.60830982371095, + 73.38180951245678, + 73.90365690520655, + 74.48643064330274, + 75.18046118594455, + 75.60562111760109, + 76.09303188037244, + 76.36455146289454, + 76.79633382339307, + 77.15526946662959, + 77.43473596376207, + 77.82016132236659, + 78.18174593713991, + 78.51021840770322, + 78.69167295797406, + 79.04398617236858, + 79.17643474920862, + 79.4651726467199, + 79.62146196739116, + 79.81351240380921, + 80.09430338671011, + 80.02940358405849, + 80.43204725765221, + 80.41085548535781, + 80.6784016105747, + 80.66648123865909, + 80.83866438855114, + 81.03071482496921, + 81.06117799764242, + 81.23998357637647, + 83.04128422140104, + 83.2147918570615, + 83.15783896902028, + 83.46909312459438, + 83.41478920808996, + 83.5525357280036, + 83.49160938265719, + 83.68895776214885, + 83.73796373557965, + 83.65187216063363, + 83.80021456669448, + 83.76445345094767, + 83.75783102210566, + 83.8810081985669, + 84.56179388352471, + 84.55782042621952, + 84.55914491198791, + 84.59358154196633, + 84.56576734082992, + 84.59887948503993, + 84.63728957232355, + 84.95119269943444, + 84.92602746983484, + 84.96841101442365, + 84.91013364061402, + 84.97105998596045, + 84.95649064250804, + 84.94059681328724, + 84.98960278671805, + 84.95251718520285, + 85.04390670322248, + 85.00947007324406, + 85.05317810360128, + 85.14986556469451, + 85.00682110170726, + 85.09158819088489, + 85.1247003350949, + 85.08496576204288, + 85.12072687778969, + 85.14191865008411, + 85.1220513635581, + 85.1233758493265, + 85.14324313585252, + 85.17238182275732, + 85.16443490814692, + 85.18695116620972, + 85.16708387968372, + 85.14324313585252, + 85.16046145084171, + 85.17767976583092, + 85.16575939391531, + 85.15516350776811, + 85.19092462351492, + 85.19092462351492, + 85.1260248208633, + 85.15781247930491, + 85.17767976583092, + 85.18562668044132, + 85.20946742427252 + ], + "test_loss": [ + 1.4322998681453933, + 1.2812489121711093, + 1.1776135883305077, + 1.152196538825275, + 1.0826810743382387, + 1.0271916073221437, + 1.0357880473397705, + 0.9983245046059984, + 0.9350582004280314, + 0.9588870008721516, + 0.8879950900731314, + 0.8635687420830164, + 0.8339084307085887, + 0.8352800360324399, + 0.8496349456362947, + 0.8068069972624154, + 0.7980735879608324, + 0.7816033112642425, + 0.8087059126173602, + 0.7873138052692271, + 0.7429236325846394, + 0.764331152835165, + 0.758892194731483, + 0.7033168555439085, + 0.7241654184701425, + 0.7092442496356411, + 0.7153409176168969, + 0.7150673622185955, + 0.6791497236360412, + 0.7323348968742677, + 0.6911731550688223, + 0.682362933319248, + 0.6602693198462726, + 0.6639998868232699, + 0.6546217153776387, + 0.674560405984463, + 0.6629944779841443, + 0.7114034969512981, + 0.7390724016314859, + 0.6580640932881485, + 0.7471172376160868, + 0.6133396425926472, + 0.5956537397574527, + 0.5936472982396476, + 0.5982606996214318, + 0.5960627390019742, + 0.6114695393682568, + 0.6084616916638932, + 0.584933704039917, + 0.5957384700903543, + 0.591684959775297, + 0.5881133314691278, + 0.6462920677180479, + 0.5991029882891162, + 0.6017056980344946, + 0.5731385147764807, + 0.5780941740668561, + 0.5788595255405685, + 0.5745612237452568, + 0.5770774067583829, + 0.573234336442943, + 0.5739969844435675, + 0.5629333505577268, + 0.5681781014667889, + 0.5615879551240726, + 0.569055528697628, + 0.5694522336845022, + 0.5645923656254894, + 0.572506981195496, + 0.5681928967642401, + 0.5692618862991153, + 0.568623423776135, + 0.5682103747951208, + 0.5662436782509243, + 0.563510111870316, + 0.5683955962747158, + 0.5686949629540852, + 0.5668917175443898, + 0.566755638766767, + 0.5646861945021832, + 0.5661456734472625, + 0.566452941174017, + 0.5648880757946291, + 0.5644424875247882, + 0.5638586133079185, + 0.5639836723370014, + 0.5631460841669758, + 0.5636733531745283, + 0.5639281619954726, + 0.5645867110905506, + 0.5639237291572914, + 0.5634546966664876, + 0.5638112064356126, + 0.5636850399873725, + 0.5639688542870023, + 0.5636112123204786, + 0.5640249978010969, + 0.5636967983628246, + 0.5635701552085556, + 0.5634580569709935 + ], + "test_acc": [ + 56.89025358674361, + 60.37905659378474, + 63.60844936206258, + 64.61432579808354, + 66.40902112340515, + 68.08724654560856, + 67.22960453173805, + 69.05606437609191, + 70.25782201281169, + 69.82900100587644, + 71.7136958017894, + 72.66133728625125, + 73.6513314627561, + 73.7519191063582, + 73.2913335803907, + 74.6095611202287, + 74.80544232092753, + 75.33485097146487, + 74.66779607178782, + 75.34543914447562, + 76.39896235904494, + 75.52014399915295, + 76.52602043517392, + 77.79660119646356, + 77.45777966011964, + 77.84424797501191, + 77.4471914871089, + 77.96071787813013, + 79.05129969823706, + 77.1666049023241, + 78.77071311345227, + 78.64894912382869, + 79.3848271480756, + 79.305415850495, + 79.76070728995712, + 79.00894700619409, + 79.53306157022607, + 77.92365927259252, + 78.26777489544179, + 79.75011911694637, + 79.21012229339827, + 81.2006988194187, + 81.7830483350098, + 82.01598814124623, + 81.86775371909577, + 82.05834083328921, + 81.64540208587007, + 81.77775424850442, + 82.38657419662238, + 82.16951664990206, + 82.31245698554714, + 82.30716289904177, + 81.063052570279, + 82.22775160146116, + 82.21716342845042, + 82.93715919318122, + 82.96362962570808, + 82.83657154957912, + 83.09598178834243, + 82.88421832812747, + 83.07480544232094, + 83.10127587484779, + 83.31303933506274, + 83.29715707554661, + 83.41892106517021, + 83.22303986447139, + 83.30245116205198, + 83.40833289215945, + 83.25480438350363, + 83.41892106517021, + 83.34480385409498, + 83.36598020011647, + 83.45597967070782, + 83.53009688178304, + 83.54597914129917, + 83.47715601672931, + 83.37656837312721, + 83.5195087087723, + 83.48245010323468, + 83.47186193022395, + 83.45068558420245, + 83.49303827624543, + 83.50362644925619, + 83.52480279527768, + 83.50362644925619, + 83.50892053576156, + 83.53539096828843, + 83.53539096828843, + 83.5406850547938, + 83.53009688178304, + 83.5406850547938, + 83.54597914129917, + 83.56186140081529, + 83.53539096828843, + 83.55656731430992, + 83.54597914129917, + 83.55127322780454, + 83.5406850547938, + 83.55127322780454, + 83.55656731430992 + ] +} \ No newline at end of file diff --git a/backend/models/abjad_bisindo/label_encoder.json b/backend/models/abjad_bisindo/label_encoder.json new file mode 100644 index 0000000..f96dc84 --- /dev/null +++ b/backend/models/abjad_bisindo/label_encoder.json @@ -0,0 +1,58 @@ +{ + "label_to_idx": { + "a": 0, + "b": 1, + "c": 2, + "d": 3, + "e": 4, + "f": 5, + "g": 6, + "h": 7, + "i": 8, + "j": 9, + "k": 10, + "l": 11, + "m": 12, + "n": 13, + "o": 14, + "p": 15, + "q": 16, + "r": 17, + "s": 18, + "t": 19, + "u": 20, + "v": 21, + "w": 22, + "x": 23, + "y": 24, + "z": 25 + }, + "idx_to_label": { + "0": "a", + "1": "b", + "2": "c", + "3": "d", + "4": "e", + "5": "f", + "6": "g", + "7": "h", + "8": "i", + "9": "j", + "10": "k", + "11": "l", + "12": "m", + "13": "n", + "14": "o", + "15": "p", + "16": "q", + "17": "r", + "18": "s", + "19": "t", + "20": "u", + "21": "v", + "22": "w", + "23": "x", + "24": "y", + "25": "z" + } +} \ No newline at end of file diff --git a/backend/models/abjad_bisindo/model_abjad_bisindo.pt b/backend/models/abjad_bisindo/model_abjad_bisindo.pt new file mode 100644 index 0000000..b2b38ae Binary files /dev/null and b/backend/models/abjad_bisindo/model_abjad_bisindo.pt differ diff --git a/backend/models/abjad_bisindo_label_encoder.json b/backend/models/abjad_bisindo_label_encoder.json new file mode 100644 index 0000000..f96dc84 --- /dev/null +++ b/backend/models/abjad_bisindo_label_encoder.json @@ -0,0 +1,58 @@ +{ + "label_to_idx": { + "a": 0, + "b": 1, + "c": 2, + "d": 3, + "e": 4, + "f": 5, + "g": 6, + "h": 7, + "i": 8, + "j": 9, + "k": 10, + "l": 11, + "m": 12, + "n": 13, + "o": 14, + "p": 15, + "q": 16, + "r": 17, + "s": 18, + "t": 19, + "u": 20, + "v": 21, + "w": 22, + "x": 23, + "y": 24, + "z": 25 + }, + "idx_to_label": { + "0": "a", + "1": "b", + "2": "c", + "3": "d", + "4": "e", + "5": "f", + "6": "g", + "7": "h", + "8": "i", + "9": "j", + "10": "k", + "11": "l", + "12": "m", + "13": "n", + "14": "o", + "15": "p", + "16": "q", + "17": "r", + "18": "s", + "19": "t", + "20": "u", + "21": "v", + "22": "w", + "23": "x", + "24": "y", + "25": "z" + } +} \ No newline at end of file diff --git a/backend/models/abjad_sibi/history.json b/backend/models/abjad_sibi/history.json new file mode 100644 index 0000000..add0bd4 --- /dev/null +++ b/backend/models/abjad_sibi/history.json @@ -0,0 +1,410 @@ +{ + "train_loss": [ + 1.2834653825799505, + 0.9140516127684698, + 0.8145102460872228, + 0.7603161162615432, + 0.7060375585771135, + 0.6762364317996734, + 0.642818652468003, + 0.613622042627844, + 0.5831606312633881, + 0.5563305722478925, + 0.5314189669913023, + 0.5147748424536277, + 0.4956288671872567, + 0.4753852308282179, + 0.45346550292769566, + 0.4382230565785206, + 0.4277682321984995, + 0.4130221423192272, + 0.39662947527616643, + 0.38594889591146825, + 0.36729169809152334, + 0.3586809866147812, + 0.3474830513908145, + 0.33371530015460144, + 0.3293629377645559, + 0.3196520884967721, + 0.3080180528445025, + 0.29027106263149016, + 0.30095674189747657, + 0.2814731045311303, + 0.2739838833801328, + 0.26099389065636386, + 0.25981360811700577, + 0.26090800600077974, + 0.2433857079855349, + 0.24312437650943844, + 0.2374434623758695, + 0.22710524853760292, + 0.23101539726521458, + 0.21759296734029154, + 0.21945560141189407, + 0.20954379972267773, + 0.2045060028293839, + 0.19869253167029077, + 0.19871912105050962, + 0.1940560657047716, + 0.19955063250773353, + 0.17712399834927292, + 0.17836054943486795, + 0.18634969011599364, + 0.1720256170819889, + 0.17594308219124455, + 0.1790394185509018, + 0.16671430122801192, + 0.1721502415839379, + 0.16577501389608107, + 0.16282134572448706, + 0.17346351389906584, + 0.15412643652896876, + 0.15812593342648978, + 0.09728019668279275, + 0.08954804734891725, + 0.0874640556064856, + 0.08489817156755493, + 0.08434453350234158, + 0.08206935701368614, + 0.07853580465919398, + 0.0813095707379272, + 0.07851537666940617, + 0.0797919544945084, + 0.07318438762591085, + 0.0753526979297368, + 0.07573781120718924, + 0.07756245176179608, + 0.07213849885844105, + 0.0708513313949998, + 0.07195700137405911, + 0.05092342184011429, + 0.05018997639462439, + 0.04822785823998495, + 0.0479910907298479, + 0.047013260090945426, + 0.04603087534042384, + 0.04707458927420607, + 0.04653745897578682, + 0.04623564681166444, + 0.0446462921952158, + 0.04441322237381759, + 0.044111464996058634, + 0.04417448175968046, + 0.04277583615827224, + 0.03627636843706056, + 0.03684215013757042, + 0.03559506300687317, + 0.03478639612469493, + 0.03483614058751847, + 0.03396160188920175, + 0.034749298566674205, + 0.03420072415783699, + 0.03431222870153384 + ], + "train_acc": [ + 59.35614486074668, + 72.06009669814462, + 75.1581667310433, + 76.68589086162575, + 78.25357290320862, + 79.0713781483504, + 80.1129476950945, + 80.89745468106926, + 81.81915049481213, + 82.47978795668563, + 83.24697984789356, + 83.70516389403161, + 84.13670933283608, + 84.83996856644335, + 85.39538352934909, + 85.76432824091957, + 86.16657121165704, + 86.78192304106341, + 87.01367892486581, + 87.483850344304, + 88.00729897840941, + 88.27901277321222, + 88.75850770521717, + 89.13278013825438, + 89.30593108592284, + 89.56299364669215, + 89.90130395982898, + 90.53663474473555, + 90.2542655069993, + 90.86428961493893, + 91.05209179664088, + 91.4170407171113, + 91.52892286791247, + 91.5275909375458, + 91.87522476324938, + 92.06435887531799, + 92.30277441095379, + 92.58381171832336, + 92.51987906072272, + 92.86484902569295, + 92.78759706442547, + 93.19783161736305, + 93.28707095193063, + 93.53081420903315, + 93.47753699436593, + 93.68931392266813, + 93.6506879420344, + 94.19411553164001, + 94.12485515257262, + 93.95436806563752, + 94.44718230130928, + 94.43386299764248, + 94.33929594160817, + 94.58703498981073, + 94.51644268037667, + 94.81479508251309, + 94.63631641337791, + 94.67893818511169, + 94.99460568201495, + 94.86940422754698, + 96.77939237336672, + 96.92457278333488, + 96.97252227653537, + 97.09639180063667, + 97.16032445823733, + 97.23757641950479, + 97.20960588180449, + 97.1749756922708, + 97.34013505773919, + 97.38009296873959, + 97.49064318917408, + 97.47865581587395, + 97.4413617656069, + 97.42537860120673, + 97.57588673264162, + 97.51728179650769, + 97.54658426457465, + 98.26582666258207, + 98.26715859294876, + 98.30312071284914, + 98.32309966834933, + 98.32975932018275, + 98.35772985788303, + 98.34840634531626, + 98.34707441494959, + 98.36438950971643, + 98.43897761025053, + 98.41100707255025, + 98.42166251548369, + 98.44297340135057, + 98.46694814795083, + 98.6733973547863, + 98.69470824065318, + 98.72667456945351, + 98.75597703752048, + 98.72401070872014, + 98.78261564485409, + 98.77329213228732, + 98.774624062654, + 98.76530055008725 + ], + "test_loss": [ + 0.9495380779750505, + 0.8454916531781728, + 0.7500483741193654, + 0.724956223033559, + 0.7166072071499779, + 0.6382759664512321, + 0.6434632320848676, + 0.6054592402477382, + 0.5903961382793609, + 0.600178299373368, + 0.5179051167947031, + 0.5110781946632214, + 0.48389085544857513, + 0.4873954885636474, + 0.498782135348579, + 0.44703874263692894, + 0.41541416332474784, + 0.4165091624527494, + 0.44903350077968907, + 0.41326331598296595, + 0.3999581425509495, + 0.3956252216586857, + 0.3827793349291591, + 0.4147448368827716, + 0.3354098783748595, + 0.31587568986395875, + 0.3368951572174503, + 0.30424319425853524, + 0.3029211760043421, + 0.3027502619162896, + 0.31338621648486864, + 0.3034445780205569, + 0.29763352448119185, + 0.28067741289167353, + 0.30748832044286495, + 0.2549264556676373, + 0.28416098999598133, + 0.2660491675343777, + 0.25904787206955054, + 0.2754591647992146, + 0.24990849822545136, + 0.2798872224224069, + 0.2341508988150454, + 0.2969360208049199, + 0.2928750260280674, + 0.25958505257837133, + 0.23276228872041588, + 0.23833544743144058, + 0.2516250917057986, + 0.264014729979567, + 0.23070169198209947, + 0.3179965194512585, + 0.24619845728253276, + 0.20696377739980565, + 0.22004448862255258, + 0.2948828727835921, + 0.413959264370155, + 0.22903930274812634, + 0.2958962345582445, + 0.3675484016359643, + 0.18464745133432992, + 0.1846121348263772, + 0.19721820412342794, + 0.18460690221363213, + 0.18600233655061404, + 0.1822518373950427, + 0.17274777137555553, + 0.20084515367137892, + 0.19352545284037576, + 0.2025058166377032, + 0.16848128297594778, + 0.1713334670080564, + 0.18094804359735586, + 0.1693721191936986, + 0.16912922051165244, + 0.17397966236919968, + 0.19191828788326026, + 0.15026410247524855, + 0.14991817271107633, + 0.14216455162481573, + 0.14770843184435245, + 0.16265288196507252, + 0.16156027967543551, + 0.14418528882987672, + 0.14033630086845186, + 0.14298629250034717, + 0.1439859902518838, + 0.16138584974248135, + 0.16080916697324285, + 0.15558331950496618, + 0.1632984312742658, + 0.16373497906145257, + 0.1425799340574174, + 0.14460372966388518, + 0.1361196535330013, + 0.1367615892623591, + 0.14067662850196785, + 0.14056633463736432, + 0.13797360463408373, + 0.14427850871097872 + ], + "test_acc": [ + 70.85884670677812, + 74.37836110963207, + 77.43464139289708, + 77.02465257441031, + 77.49321122410947, + 80.4696235557212, + 79.7774346413929, + 80.83169160321602, + 81.3321974335765, + 80.93818220542038, + 83.65369256163143, + 83.98381342846494, + 84.50029284915607, + 84.37250412651083, + 83.86667376604015, + 85.87934614770246, + 86.53426335125926, + 86.6354294233534, + 85.52260263031788, + 86.74192002555773, + 87.2051541451467, + 87.34891645812257, + 87.55124860231085, + 86.47569352004686, + 88.87173206964485, + 89.61184175496513, + 89.21250199669879, + 89.97923433257014, + 90.16559288642777, + 90.33065331984453, + 89.87806826047601, + 90.18156647675842, + 90.51168734359193, + 90.96427240296043, + 90.13897023587668, + 91.68840849795006, + 91.00154411373195, + 91.49672541398222, + 91.83749534103616, + 91.31036686012459, + 91.82684628081572, + 91.07608753527501, + 92.38059741227836, + 90.95362334274, + 90.91102710185827, + 91.95463500346095, + 92.54565784569512, + 92.62020126723817, + 92.10372184654703, + 91.94931047335074, + 93.01421649539428, + 91.21452531814067, + 92.26878227996379, + 93.65848463873063, + 93.31239018156647, + 91.55529524519461, + 89.94728715190884, + 93.1100580373782, + 91.15063095681806, + 89.55327192375273, + 94.61157552845961, + 94.75533784143549, + 94.64884723923113, + 94.79260955220703, + 94.58495287790852, + 94.80858314253767, + 95.13870400937118, + 94.45183962515308, + 94.55833022735743, + 94.34002449283851, + 95.27714179223683, + 95.25584367179596, + 94.88845109419094, + 95.04818699749748, + 95.24519461157553, + 95.20259837069379, + 94.48378680581439, + 95.72440232149513, + 96.06517224854908, + 96.16101379053299, + 95.9214099355732, + 95.80427027314839, + 95.7776476225973, + 96.05452318832863, + 96.24620627229648, + 96.08647036898994, + 96.31010063361907, + 96.02257600766733, + 95.96400617645493, + 95.95868164634471, + 96.01725147755711, + 95.94270805601406, + 96.3473723443906, + 96.25153080240669, + 96.45918747670518, + 96.50178371758692, + 96.59762525957085, + 96.44321388637452, + 96.56567807890953, + 96.4964591874767 + ] +} \ No newline at end of file diff --git a/backend/models/abjad_sibi/label_encoder.json b/backend/models/abjad_sibi/label_encoder.json new file mode 100644 index 0000000..f96dc84 --- /dev/null +++ b/backend/models/abjad_sibi/label_encoder.json @@ -0,0 +1,58 @@ +{ + "label_to_idx": { + "a": 0, + "b": 1, + "c": 2, + "d": 3, + "e": 4, + "f": 5, + "g": 6, + "h": 7, + "i": 8, + "j": 9, + "k": 10, + "l": 11, + "m": 12, + "n": 13, + "o": 14, + "p": 15, + "q": 16, + "r": 17, + "s": 18, + "t": 19, + "u": 20, + "v": 21, + "w": 22, + "x": 23, + "y": 24, + "z": 25 + }, + "idx_to_label": { + "0": "a", + "1": "b", + "2": "c", + "3": "d", + "4": "e", + "5": "f", + "6": "g", + "7": "h", + "8": "i", + "9": "j", + "10": "k", + "11": "l", + "12": "m", + "13": "n", + "14": "o", + "15": "p", + "16": "q", + "17": "r", + "18": "s", + "19": "t", + "20": "u", + "21": "v", + "22": "w", + "23": "x", + "24": "y", + "25": "z" + } +} \ No newline at end of file diff --git a/backend/models/abjad_sibi/model_abjad_sibi.pt b/backend/models/abjad_sibi/model_abjad_sibi.pt new file mode 100644 index 0000000..02cd8f2 Binary files /dev/null and b/backend/models/abjad_sibi/model_abjad_sibi.pt differ diff --git a/backend/models/abjad_sibi_label_encoder.json b/backend/models/abjad_sibi_label_encoder.json new file mode 100644 index 0000000..f382804 --- /dev/null +++ b/backend/models/abjad_sibi_label_encoder.json @@ -0,0 +1,28 @@ +{ + "a": 0, + "b": 1, + "c": 2, + "d": 3, + "e": 4, + "f": 5, + "g": 6, + "h": 7, + "i": 8, + "j": 9, + "k": 10, + "l": 11, + "m": 12, + "n": 13, + "o": 14, + "p": 15, + "q": 16, + "r": 17, + "s": 18, + "t": 19, + "u": 20, + "v": 21, + "w": 22, + "x": 23, + "y": 24, + "z": 25 +} \ No newline at end of file diff --git a/backend/models/angka/history.json b/backend/models/angka/history.json new file mode 100644 index 0000000..3885a70 --- /dev/null +++ b/backend/models/angka/history.json @@ -0,0 +1,410 @@ +{ + "train_loss": [ + 0.7636361829090683, + 0.49721478224389054, + 0.43865926426937407, + 0.42299071711390157, + 0.3885281641525377, + 0.3887109880322758, + 0.3620548944648857, + 0.359657923542956, + 0.3413998733689734, + 0.335858025804607, + 0.33459905559070197, + 0.32327442109375193, + 0.3238065281210822, + 0.30815456408434083, + 0.30271901781252436, + 0.2925835411181087, + 0.29239921550803444, + 0.2766186786868463, + 0.28175233447960485, + 0.2727835036631045, + 0.2651435263875007, + 0.25801988361284517, + 0.2497332574393259, + 0.2492525341096629, + 0.2522363509648518, + 0.23641331622823564, + 0.2306328794321921, + 0.22645481675983048, + 0.22594264079777943, + 0.21785812866684023, + 0.20836614779640467, + 0.20718188542469967, + 0.20715493348475728, + 0.20089983985978016, + 0.19387944460012865, + 0.1927437309961124, + 0.1840197721724981, + 0.1904311918737988, + 0.18306771335695668, + 0.1773854478600291, + 0.16995718791308362, + 0.1727036345335898, + 0.17574299686183867, + 0.15623854223055034, + 0.16146486784599948, + 0.1666037972334102, + 0.1530704233943875, + 0.1520947290017212, + 0.1471134107068523, + 0.14932299774470204, + 0.152661097513967, + 0.1357908850771382, + 0.14138954336315648, + 0.1299473644079403, + 0.1433466609125366, + 0.13532013918420735, + 0.13080143330102403, + 0.13244204376281915, + 0.10318501227947798, + 0.09761429622711619, + 0.09962504350181273, + 0.09676554178414085, + 0.09561752629452885, + 0.09269482570379585, + 0.09397823078018477, + 0.09030663951243713, + 0.08146244039525388, + 0.08120535827201458, + 0.07930281748744994, + 0.08055342244517744, + 0.07869007187205761, + 0.0770269763980969, + 0.07802040743665192, + 0.07837402015450902, + 0.07340333785954453, + 0.07273701232877351, + 0.07209388269074912, + 0.07162818952564165, + 0.0718914760267008, + 0.07224314432372304, + 0.07007893693656278, + 0.06977489633303673, + 0.0697521451704201, + 0.0705585122904153, + 0.07002707278700832, + 0.06929085833410971, + 0.06890882859863512, + 0.06820005702064122, + 0.06856560288057682, + 0.06845003201065214, + 0.0682872959312614, + 0.06823094015891301, + 0.0675902266402977, + 0.06769187839202599, + 0.06788777009533856, + 0.06769364609748435, + 0.06774747690266689, + 0.06726075990988706, + 0.06766948739769194, + 0.06772732285841741 + ], + "train_acc": [ + 71.29626771216003, + 81.92202018954713, + 83.83292686691568, + 84.60778563269842, + 85.66356928966134, + 85.94758126755781, + 86.73170129348934, + 86.96014571049301, + 87.3769024171889, + 87.69795943568054, + 87.83996542462877, + 88.09001944864632, + 88.31846386564999, + 88.77535269965733, + 88.96983916278208, + 89.29707035470626, + 89.35572500231531, + 89.70147871453709, + 89.68913036767204, + 90.0348840798938, + 90.34359275152039, + 90.41768283271077, + 90.90852962059705, + 90.8930941870157, + 90.76652363164881, + 91.3283734140092, + 91.44259562251104, + 91.75130429413763, + 91.88404902293705, + 92.08470965949434, + 92.3224153366468, + 92.49529219275769, + 92.44589880529745, + 92.66816904886859, + 92.9645293736301, + 93.09418701571327, + 93.29793473898683, + 93.27632513197295, + 93.29176056555428, + 93.57885963016702, + 93.7702590065755, + 93.75482357299416, + 93.86595869477974, + 94.25184453431297, + 94.12527397894607, + 93.94931003611892, + 94.44633099743773, + 94.44324391072145, + 94.5018985583305, + 94.65933998086005, + 94.70255919488778, + 95.0452258203933, + 94.96187447905412, + 95.15944802889513, + 94.8816102244312, + 95.25206063038311, + 95.24588645695059, + 95.25823480381564, + 96.22758003272311, + 96.43750192942919, + 96.35415058809002, + 96.4282406692804, + 96.47145988330811, + 96.65977217300033, + 96.51467909733584, + 96.69373012687926, + 97.03330966566851, + 97.08579013984503, + 97.06726761954744, + 96.99626462507331, + 97.11666100700769, + 97.22162195536073, + 97.13518352730529, + 97.1876640014818, + 97.36980211774149, + 97.42536967863428, + 97.40684715833667, + 97.3605408575927, + 97.40993424505294, + 97.39449881147162, + 97.43463093878307, + 97.49328558639212, + 97.48711141295959, + 97.45315345908067, + 97.41919550520174, + 97.49945975982466, + 97.468588892662, + 97.52415645355478, + 97.56120149414997, + 97.55502732071744, + 97.51489519340598, + 97.58589818788009, + 97.58589818788009, + 97.58281110116384, + 97.53033062698732, + 97.58281110116384, + 97.53959188713611, + 97.58281110116384, + 97.5581144074337, + 97.6229432284753 + ], + "test_loss": [ + 0.4821583384897296, + 0.42387530508297167, + 0.38802325607123456, + 0.39216163870328524, + 0.36728777073866387, + 0.3586744003797199, + 0.338629928952831, + 0.3442136275684681, + 0.3302532174449421, + 0.3187703682535042, + 0.3197924793243232, + 0.31725302685485735, + 0.39475784122056556, + 0.3077779227485477, + 0.28969186663583624, + 0.31821264312639247, + 0.30931969059037234, + 0.2842195305775666, + 0.27954857257115984, + 0.2851857732606531, + 0.27895083383137664, + 0.26327753039872126, + 0.2684490170122874, + 0.32718702706446007, + 0.2567324720386507, + 0.2693244741118594, + 0.26179516532318114, + 0.2537716095863489, + 0.249747345171487, + 0.2502480407870674, + 0.246032616140594, + 0.24428655427157459, + 0.2641567068453374, + 0.24865152285049655, + 0.23179181965533072, + 0.25554003685393817, + 0.2355854851913838, + 0.22034702429507083, + 0.23895914520327047, + 0.21865103779606487, + 0.2355885252100591, + 0.21533341432825368, + 0.2309404674796126, + 0.2147533376273011, + 0.24825391772505437, + 0.2529776999616893, + 0.2170251631005789, + 0.21550008139997173, + 0.22493291983205088, + 0.20755464591537304, + 0.22185767910264803, + 0.19173403310776305, + 0.19222667776587757, + 0.20295112281280134, + 0.19615874595220384, + 0.21707751816723692, + 0.21166490677532154, + 0.22466862721663833, + 0.17595847195546038, + 0.17460707242682294, + 0.18756344360487442, + 0.18355660001977714, + 0.1802039628171469, + 0.18336527966973584, + 0.17777490489781475, + 0.20829978748218253, + 0.17520286799853388, + 0.1703460241550583, + 0.17851701445762017, + 0.18191781441810037, + 0.17623377645680646, + 0.18520181069044897, + 0.1891294972036312, + 0.1834536531727901, + 0.17363809159654678, + 0.17495350346779767, + 0.18082957921699408, + 0.17649341681195974, + 0.17346997457303742, + 0.1812001225930465, + 0.1776049373041867, + 0.17229601414642534, + 0.17856412863826507, + 0.17642245994300748, + 0.18379140403989785, + 0.17447235703339767, + 0.17951101223763455, + 0.18481418433074326, + 0.1812018331346744, + 0.18114464241051206, + 0.1808676687037405, + 0.1800637769462855, + 0.18226401238854836, + 0.1788910074610863, + 0.17988903109542764, + 0.18245814936428395, + 0.1829004843286316, + 0.1814681848123518, + 0.180023730451091, + 0.18243581857367128 + ], + "test_acc": [ + 82.5888450148075, + 84.6618953603159, + 86.11796643632775, + 85.5009871668312, + 87.0187561697927, + 86.88302073050346, + 87.93188548864758, + 87.59871668311945, + 88.16633761105626, + 88.61056268509378, + 88.38845014807502, + 88.52418558736427, + 86.22902270483712, + 88.72161895360315, + 89.53603158933859, + 88.79565646594274, + 88.96841066140178, + 90.00493583415597, + 90.17769002961501, + 90.19002961500495, + 90.14067127344522, + 90.65893385982231, + 90.9304047384008, + 88.75863770977294, + 91.14017769002962, + 90.60957551826259, + 90.96742349457058, + 91.59674234945706, + 91.48568608094769, + 91.94225074037512, + 91.44866732477789, + 91.51036525172755, + 91.36229022704838, + 91.52270483711747, + 92.2507403751234, + 91.42398815399802, + 91.97926949654492, + 92.79368213228035, + 92.18904244817374, + 92.32477788746299, + 92.27541954590326, + 92.88005923000988, + 92.32477788746299, + 93.13919052319842, + 92.2507403751234, + 92.0162882527147, + 93.26258637709773, + 93.0898321816387, + 93.33662388943732, + 93.44768015794669, + 93.27492596248767, + 94.07699901283317, + 94.28677196446199, + 93.97828232971372, + 93.74383020730504, + 93.43534057255675, + 93.47235932872655, + 93.3242843040474, + 94.68163869693979, + 94.68163869693979, + 94.38548864758144, + 94.55824284304047, + 94.84205330700888, + 94.91609081934847, + 94.92843040473839, + 94.68163869693979, + 95.4837117472853, + 95.42201382033564, + 95.29861796643632, + 95.27393879565646, + 95.70582428430404, + 95.26159921026654, + 95.47137216189536, + 95.31095755182626, + 95.55774925962488, + 95.63178677196447, + 95.60710760118461, + 95.63178677196447, + 95.79220138203355, + 95.63178677196447, + 95.74284304047383, + 95.81688055281342, + 95.66880552813426, + 95.69348469891412, + 95.61944718657453, + 95.75518262586377, + 95.71816386969397, + 95.66880552813426, + 95.80454096742349, + 95.71816386969397, + 95.81688055281342, + 95.80454096742349, + 95.74284304047383, + 95.79220138203355, + 95.76752221125369, + 95.70582428430404, + 95.74284304047383, + 95.76752221125369, + 95.75518262586377, + 95.74284304047383 + ] +} \ No newline at end of file diff --git a/backend/models/angka/label_encoder.json b/backend/models/angka/label_encoder.json new file mode 100644 index 0000000..cd98d47 --- /dev/null +++ b/backend/models/angka/label_encoder.json @@ -0,0 +1,28 @@ +{ + "label_to_idx": { + "0": 0, + "1": 1, + "10": 2, + "2": 3, + "3": 4, + "4": 5, + "5": 6, + "6": 7, + "7": 8, + "8": 9, + "9": 10 + }, + "idx_to_label": { + "0": "0", + "1": "1", + "2": "10", + "3": "2", + "4": "3", + "5": "4", + "6": "5", + "7": "6", + "8": "7", + "9": "8", + "10": "9" + } +} \ No newline at end of file diff --git a/backend/models/angka/model_angka.pt b/backend/models/angka/model_angka.pt new file mode 100644 index 0000000..cea2ad7 Binary files /dev/null and b/backend/models/angka/model_angka.pt differ diff --git a/backend/models/angka_label_encoder.json b/backend/models/angka_label_encoder.json new file mode 100644 index 0000000..4d6145a --- /dev/null +++ b/backend/models/angka_label_encoder.json @@ -0,0 +1,13 @@ +{ + "0": 0, + "1": 1, + "10": 2, + "2": 3, + "3": 4, + "4": 5, + "5": 6, + "6": 7, + "7": 8, + "8": 9, + "9": 10 +} \ No newline at end of file diff --git a/backend/models/kata/history.json b/backend/models/kata/history.json new file mode 100644 index 0000000..db4a923 --- /dev/null +++ b/backend/models/kata/history.json @@ -0,0 +1,410 @@ +{ + "train_loss": [ + 1.0322966694656799, + 0.7105154862487894, + 0.6494718791148386, + 0.5959148770868341, + 0.5930463688095235, + 0.535461616629896, + 0.5247100995150606, + 0.5050578966364812, + 0.4878458133878162, + 0.46512079871261, + 0.4432702473441355, + 0.43039216084211585, + 0.4068903884578294, + 0.39413704883289097, + 0.3818127369992271, + 0.36273485365243036, + 0.3589807510835484, + 0.3484799203226307, + 0.3286204333150439, + 0.31764261690779555, + 0.3066730431927896, + 0.28779330640681167, + 0.2903694096438466, + 0.26581503058101286, + 0.25992803741327664, + 0.243608061136697, + 0.24603306241989922, + 0.21886675011656534, + 0.2213409483588674, + 0.20950187207849233, + 0.20377055414037426, + 0.1864586104399548, + 0.17101035147451313, + 0.1814401149201014, + 0.1666612278016369, + 0.16214551685635062, + 0.16080468475432283, + 0.13924201583850818, + 0.15077892165834078, + 0.13351096243190028, + 0.12392057193776185, + 0.13271552131021677, + 0.12858448556532726, + 0.10870555548379253, + 0.11053386800317298, + 0.1067439715180371, + 0.09910364716317882, + 0.11920973066164992, + 0.11227167316263872, + 0.09588277388809224, + 0.0989105580089116, + 0.09679456899320726, + 0.09524081990650339, + 0.08395262895714048, + 0.08492385551154616, + 0.0898694942503444, + 0.09342785689525028, + 0.0911411639581528, + 0.07562122320115776, + 0.0784903593557241, + 0.07655741388618224, + 0.07660539848782962, + 0.08216755856782652, + 0.08703030421149509, + 0.05738548445038818, + 0.0694606915002865, + 0.08903009281186752, + 0.058793735381202736, + 0.0955236261358287, + 0.059719061562352134, + 0.03418785757130683, + 0.03135814258261019, + 0.03129677840422122, + 0.03881934009236275, + 0.03266810573757501, + 0.039393692762778286, + 0.038395770587875216, + 0.024290860875163814, + 0.023070882710930104, + 0.025508001386597874, + 0.024833115068692235, + 0.024738080995266552, + 0.02313491725673882, + 0.02274823644769461, + 0.02217860416711977, + 0.02188886170920564, + 0.021770902023427196, + 0.022878333028962097, + 0.02145262443747693, + 0.022225845755725972, + 0.021321946746626182, + 0.02147376138687061, + 0.020974892129910494, + 0.021125833119707723, + 0.020845347819439444, + 0.021486535927610566, + 0.021381682124178514, + 0.020807361430205323, + 0.020474237822601698, + 0.020568245971358944 + ], + "train_acc": [ + 61.88835286009648, + 73.60900528371238, + 75.79140822421319, + 77.24328049620951, + 77.59705949919595, + 79.00298644612911, + 79.72892258212727, + 80.45945325063175, + 81.03376981392142, + 81.82862393751435, + 82.5775327360441, + 83.11509303928325, + 83.864001837813, + 84.43372386859636, + 84.85182632667126, + 85.61451872271996, + 85.93613599816219, + 86.14748449345278, + 86.99747300712153, + 87.26855042499426, + 87.56260050539858, + 88.26096944635883, + 88.59177578681368, + 89.18906501263496, + 89.50608775557087, + 90.1171605789111, + 90.2779692166322, + 91.05444521019986, + 91.13714679531357, + 91.6425453710085, + 91.84470480128647, + 92.39604870204457, + 93.03928325292901, + 92.97495979784057, + 93.32873880082701, + 93.43900758097864, + 93.68251780381345, + 94.00413507925569, + 94.04548587181255, + 94.44521019986216, + 94.9827705031013, + 94.86331265793706, + 94.8495290604181, + 95.45600735125201, + 95.44681828623938, + 95.72249023661843, + 95.81897541925109, + 95.24006432345509, + 95.44222375373306, + 96.01194578451643, + 96.0303239145417, + 96.01194578451643, + 96.19572708476912, + 96.34275212497128, + 96.40248104755341, + 96.18194348725018, + 96.24626694233862, + 96.46220997013553, + 96.88031242821043, + 96.71031472547669, + 96.86193429818516, + 96.69653112795773, + 96.74247645302091, + 96.47139903514818, + 97.45922352400643, + 97.09625545600736, + 96.32896852745233, + 97.3213875488169, + 96.51274982770504, + 97.30300941879165, + 98.2908339076499, + 98.35056283023202, + 98.32299563519412, + 98.1070526073972, + 98.28164484263726, + 98.04732368481507, + 98.04732368481507, + 98.62164024810475, + 98.65380197564897, + 98.52974959797841, + 98.56191132552263, + 98.59866758557317, + 98.63082931311739, + 98.60326211807948, + 98.66299104066161, + 98.64920744314266, + 98.67218010567424, + 98.61704571559844, + 98.68136917068688, + 98.68136917068688, + 98.69974730071215, + 98.66299104066161, + 98.7135308982311, + 98.69515276820584, + 98.70893636572478, + 98.68596370319321, + 98.66758557316793, + 98.72271996324375, + 98.72731449575006, + 98.69515276820584 + ], + "test_loss": [ + 0.7520760576627408, + 0.6355705165375046, + 0.6696101296693087, + 0.5765327896203911, + 0.5228067354969027, + 0.5175599360672419, + 0.6234327146772578, + 0.4678873422457592, + 0.4702892160786545, + 0.4527146184697137, + 0.4859043585697505, + 0.6269138020338022, + 0.3975915463957173, + 0.4016990479999576, + 0.4216740512212858, + 0.36092982880569646, + 0.38606062279306136, + 0.383108207034165, + 0.32663644912776235, + 0.34108600920550963, + 0.3430349418975162, + 0.3461061647856227, + 0.30207430145048675, + 0.2937200436050655, + 0.31083015933308966, + 0.3161707732673974, + 0.2970403897819757, + 0.27057727033927054, + 0.26968532783201504, + 0.2634912455249465, + 0.23534685890741744, + 0.2724271023937291, + 0.23684936158045206, + 0.21410813914204824, + 0.19880588251038056, + 0.22482921394784336, + 0.19524523058377477, + 0.23337905390952815, + 0.23304167961443498, + 0.2880258113072686, + 0.17263241940824564, + 0.19763661361447868, + 0.22440637828365342, + 0.15131372145880204, + 0.15796489068114733, + 0.15606751370997843, + 0.14873634260470564, + 0.17925791447511893, + 0.1604248992360436, + 0.23385071945813252, + 0.2172099882788553, + 0.1487210725372013, + 0.3346225968056792, + 0.15416515896222124, + 0.17809210764470776, + 0.18131846792980016, + 0.1329088453875884, + 0.1662869308832645, + 0.13454099506453684, + 0.1269256775203465, + 0.1578913730987147, + 0.18675539850853703, + 0.24373668509218327, + 0.11364883727172617, + 0.12188592965177329, + 0.12958281927819226, + 0.15152587212716115, + 0.12993033672046828, + 0.15210475259967327, + 0.13715898440440394, + 0.08403506831626767, + 0.1088136758750741, + 0.16173632551040473, + 0.09483659155784198, + 0.14439252742524905, + 0.1088209686429963, + 0.0978264953964993, + 0.0849262292033366, + 0.09487659452352011, + 0.1018295276181546, + 0.10558917458442585, + 0.08747310334094545, + 0.08596765852508821, + 0.09507962710005836, + 0.0829415585904111, + 0.07998849220711264, + 0.08356946200927942, + 0.08201091521832725, + 0.08909611776220944, + 0.09686661362106874, + 0.0845091898077324, + 0.09440301364487923, + 0.08726219244497914, + 0.08242777447369433, + 0.08657222491303856, + 0.0891349096316345, + 0.08398969176887254, + 0.08218829033063883, + 0.08246097339637856, + 0.08516075031491974 + ], + "test_acc": [ + 71.8009913713971, + 76.09693409216082, + 75.19735634294106, + 78.24490545254268, + 80.08077841013403, + 80.7600514044428, + 76.13365155131265, + 82.08187993390858, + 81.86157517899761, + 82.87130530567285, + 81.95336882687718, + 76.99651184138058, + 84.30328621259409, + 84.06462272810722, + 84.48687350835323, + 85.47824490545254, + 84.81733064071966, + 84.68881953368826, + 87.36919405177161, + 86.59812740958326, + 85.93721314485038, + 86.81843216449423, + 87.60785753625849, + 87.71800991371397, + 88.21369561226363, + 87.64457499541032, + 88.34220671929502, + 89.09491463190747, + 89.829263814944, + 89.35193684597026, + 90.01285111070314, + 89.68239397833669, + 90.76555902331559, + 91.37139709932073, + 91.55498439507987, + 91.8670827978704, + 92.6932256287865, + 90.94914631907473, + 91.26124472186524, + 90.32494951349366, + 93.50100973012667, + 92.74830181751422, + 91.81200660914264, + 94.52909858637783, + 93.8131081329172, + 94.36387002019461, + 94.38222874977052, + 93.46429227097485, + 94.05177161740409, + 92.06902882320543, + 92.63814944005875, + 94.27207637231504, + 90.9307875894988, + 94.58417477510557, + 93.75803194418945, + 94.2170001835873, + 95.06150174407931, + 94.56581604552964, + 94.45566366807418, + 95.52046998347714, + 94.58417477510557, + 93.42757481182302, + 92.583073251331, + 95.92436203414724, + 95.61226363135671, + 95.336882687718, + 94.34551129061869, + 95.57554617220488, + 95.24508903983845, + 94.82283825959243, + 97.00752707912612, + 96.29153662566551, + 94.73104461171286, + 96.62199375803195, + 95.68569854966036, + 96.51184138057647, + 96.58527629888012, + 96.91573343124655, + 97.08096199742978, + 97.00752707912612, + 96.62199375803195, + 96.97080961997429, + 97.15439691573343, + 97.13603818615752, + 97.31962548191665, + 97.41141912979622, + 97.2645492931889, + 97.19111437488526, + 97.2645492931889, + 97.09932072700569, + 97.19111437488526, + 97.28290802276483, + 97.246190563613, + 97.33798421149257, + 97.2645492931889, + 97.19111437488526, + 97.35634294106848, + 97.3747016706444, + 97.33798421149257, + 97.41141912979622 + ] +} \ No newline at end of file diff --git a/backend/models/kata/label_encoder.json b/backend/models/kata/label_encoder.json new file mode 100644 index 0000000..5bd2104 --- /dev/null +++ b/backend/models/kata/label_encoder.json @@ -0,0 +1,26 @@ +{ + "label_to_idx": { + "halo": 0, + "maaf": 1, + "nama": 2, + "perkenalkan": 3, + "sama-sama": 4, + "saya": 5, + "terimakasih": 6, + "tidak": 7, + "tolong": 8, + "ya": 9 + }, + "idx_to_label": { + "0": "halo", + "1": "maaf", + "2": "nama", + "3": "perkenalkan", + "4": "sama-sama", + "5": "saya", + "6": "terimakasih", + "7": "tidak", + "8": "tolong", + "9": "ya" + } +} \ No newline at end of file diff --git a/backend/models/kata/model_kata.pt b/backend/models/kata/model_kata.pt new file mode 100644 index 0000000..3dffda1 Binary files /dev/null and b/backend/models/kata/model_kata.pt differ diff --git a/backend/models/kata_label_encoder.json b/backend/models/kata_label_encoder.json new file mode 100644 index 0000000..09e794c --- /dev/null +++ b/backend/models/kata_label_encoder.json @@ -0,0 +1,12 @@ +{ + "halo": 0, + "maaf": 1, + "nama": 2, + "perkenalkan": 3, + "sama-sama": 4, + "saya": 5, + "terimakasih": 6, + "tidak": 7, + "tolong": 8, + "ya": 9 +} \ No newline at end of file diff --git a/backend/realtime/realtime_abjad_bisindo.py b/backend/realtime/realtime_abjad_bisindo.py new file mode 100644 index 0000000..fa46bad --- /dev/null +++ b/backend/realtime/realtime_abjad_bisindo.py @@ -0,0 +1,95 @@ +""" +Realtime Detection untuk Abjad BISINDO dengan visualisasi MediaPipe landmarks. +""" + +import sys +import cv2 +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from utils.inference import GestureRecognizer +from utils.config import MODEL_DIR, WINDOW_SIZE + + +def run_realtime(category: str = "abjad_bisindo", camera_id: int = 0): + model_path = MODEL_DIR / "abjad_bisindo" / "model_abjad_bisindo.pt" + encoder_path = MODEL_DIR / "abjad_bisindo" / "label_encoder.json" + + if not model_path.exists(): + print(f"Error: Model tidak ditemukan di {model_path}") + return + + recognizer = GestureRecognizer( + model_path, + encoder_path, + window_size=WINDOW_SIZE, + draw_landmarks=True + ) + cap = cv2.VideoCapture(camera_id) + + if not cap.isOpened(): + print(f"Error: Tidak dapat membuka camera {camera_id}") + return + + print("\n" + "="*60) + print(f"REALTIME RECOGNITION - {category.upper()}") + print("="*60) + print("Tekan 'q' untuk quit, 'r' untuk reset, 'l' toggle landmarks") + print("="*60 + "\n") + + show_landmarks = True + + try: + while True: + ret, frame = cap.read() + if not ret: + break + + if show_landmarks: + frame = recognizer.draw_landmarks_on_frame(frame) + + label, confidence = recognizer.process_frame(frame) + + if label is not None: + text = f"{label} ({confidence*100:.1f}%)" + cv2.putText(frame, text, (10, 50), + cv2.FONT_HERSHEY_SIMPLEX, 1.5, (0, 255, 0), 3) + + buffer_len = len(recognizer.buffer) + cv2.putText(frame, f"Buffer: {buffer_len}/{recognizer.window_size}", + (10, 100), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (255, 255, 255), 2) + + landmark_status = "Landmarks: ON" if show_landmarks else "Landmarks: OFF" + cv2.putText(frame, landmark_status, (10, 130), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 255), 2) + + if label is None: + cv2.putText(frame, "Tunjukkan gestur isyarat...", (10, 160), + cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2) + + cv2.imshow(f"Realtime - {category}", frame) + key = cv2.waitKey(1) & 0xFF + if key == ord('q'): + break + elif key == ord('r'): + recognizer.reset() + elif key == ord('l'): + show_landmarks = not show_landmarks + + finally: + cap.release() + cv2.destroyAllWindows() + recognizer.close() + + +def main(): + import argparse + parser = argparse.ArgumentParser() + parser.add_argument("--camera", type=int, default=0) + args = parser.parse_args() + run_realtime("abjad_bisindo", args.camera) + + +if __name__ == "__main__": + main() diff --git a/backend/realtime/realtime_abjad_sibi.py b/backend/realtime/realtime_abjad_sibi.py new file mode 100644 index 0000000..7cc9789 --- /dev/null +++ b/backend/realtime/realtime_abjad_sibi.py @@ -0,0 +1,95 @@ +""" +Realtime Detection untuk Abjad SIBI dengan visualisasi MediaPipe landmarks. +""" + +import sys +import cv2 +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from utils.inference import GestureRecognizer +from utils.config import MODEL_DIR, WINDOW_SIZE + + +def run_realtime(category: str = "abjad_sibi", camera_id: int = 0): + model_path = MODEL_DIR / "abjad_sibi" / "model_abjad_sibi.pt" + encoder_path = MODEL_DIR / "abjad_sibi" / "label_encoder.json" + + if not model_path.exists(): + print(f"Error: Model tidak ditemukan di {model_path}") + return + + recognizer = GestureRecognizer( + model_path, + encoder_path, + window_size=WINDOW_SIZE, + draw_landmarks=True + ) + cap = cv2.VideoCapture(camera_id) + + if not cap.isOpened(): + print(f"Error: Tidak dapat membuka camera {camera_id}") + return + + print("\n" + "="*60) + print(f"REALTIME RECOGNITION - {category.upper()}") + print("="*60) + print("Tekan 'q' untuk quit, 'r' untuk reset, 'l' toggle landmarks") + print("="*60 + "\n") + + show_landmarks = True + + try: + while True: + ret, frame = cap.read() + if not ret: + break + + if show_landmarks: + frame = recognizer.draw_landmarks_on_frame(frame) + + label, confidence = recognizer.process_frame(frame) + + if label is not None: + text = f"{label} ({confidence*100:.1f}%)" + cv2.putText(frame, text, (10, 50), + cv2.FONT_HERSHEY_SIMPLEX, 1.5, (0, 255, 0), 3) + + buffer_len = len(recognizer.buffer) + cv2.putText(frame, f"Buffer: {buffer_len}/{recognizer.window_size}", + (10, 100), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (255, 255, 255), 2) + + landmark_status = "Landmarks: ON" if show_landmarks else "Landmarks: OFF" + cv2.putText(frame, landmark_status, (10, 130), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 255), 2) + + if label is None: + cv2.putText(frame, "Tunjukkan gestur isyarat...", (10, 160), + cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2) + + cv2.imshow(f"Realtime - {category}", frame) + key = cv2.waitKey(1) & 0xFF + if key == ord('q'): + break + elif key == ord('r'): + recognizer.reset() + elif key == ord('l'): + show_landmarks = not show_landmarks + + finally: + cap.release() + cv2.destroyAllWindows() + recognizer.close() + + +def main(): + import argparse + parser = argparse.ArgumentParser() + parser.add_argument("--camera", type=int, default=0) + args = parser.parse_args() + run_realtime("abjad_sibi", args.camera) + + +if __name__ == "__main__": + main() diff --git a/backend/realtime/realtime_angka.py b/backend/realtime/realtime_angka.py new file mode 100644 index 0000000..5e79467 --- /dev/null +++ b/backend/realtime/realtime_angka.py @@ -0,0 +1,95 @@ +""" +Realtime Detection untuk Angka dengan visualisasi MediaPipe landmarks. +""" + +import sys +import cv2 +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from utils.inference import GestureRecognizer +from utils.config import MODEL_DIR, WINDOW_SIZE + + +def run_realtime(category: str = "angka", camera_id: int = 0): + model_path = MODEL_DIR / "angka" / "model_angka.pt" + encoder_path = MODEL_DIR / "angka" / "label_encoder.json" + + if not model_path.exists(): + print(f"Error: Model tidak ditemukan di {model_path}") + return + + recognizer = GestureRecognizer( + model_path, + encoder_path, + window_size=WINDOW_SIZE, + draw_landmarks=True + ) + cap = cv2.VideoCapture(camera_id) + + if not cap.isOpened(): + print(f"Error: Tidak dapat membuka camera {camera_id}") + return + + print("\n" + "="*60) + print(f"REALTIME RECOGNITION - {category.upper()}") + print("="*60) + print("Tekan 'q' untuk quit, 'r' untuk reset, 'l' toggle landmarks") + print("="*60 + "\n") + + show_landmarks = True + + try: + while True: + ret, frame = cap.read() + if not ret: + break + + if show_landmarks: + frame = recognizer.draw_landmarks_on_frame(frame) + + label, confidence = recognizer.process_frame(frame) + + if label is not None: + text = f"{label} ({confidence*100:.1f}%)" + cv2.putText(frame, text, (10, 50), + cv2.FONT_HERSHEY_SIMPLEX, 1.5, (0, 255, 0), 3) + + buffer_len = len(recognizer.buffer) + cv2.putText(frame, f"Buffer: {buffer_len}/{recognizer.window_size}", + (10, 100), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (255, 255, 255), 2) + + landmark_status = "Landmarks: ON" if show_landmarks else "Landmarks: OFF" + cv2.putText(frame, landmark_status, (10, 130), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 255), 2) + + if label is None: + cv2.putText(frame, "Tunjukkan gestur isyarat...", (10, 160), + cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2) + + cv2.imshow(f"Realtime - {category}", frame) + key = cv2.waitKey(1) & 0xFF + if key == ord('q'): + break + elif key == ord('r'): + recognizer.reset() + elif key == ord('l'): + show_landmarks = not show_landmarks + + finally: + cap.release() + cv2.destroyAllWindows() + recognizer.close() + + +def main(): + import argparse + parser = argparse.ArgumentParser() + parser.add_argument("--camera", type=int, default=0) + args = parser.parse_args() + run_realtime("angka", args.camera) + + +if __name__ == "__main__": + main() diff --git a/backend/realtime/realtime_kata.py b/backend/realtime/realtime_kata.py new file mode 100644 index 0000000..8e0e192 --- /dev/null +++ b/backend/realtime/realtime_kata.py @@ -0,0 +1,132 @@ +""" +Realtime Detection untuk Kata dengan visualisasi MediaPipe landmarks. + +Proses sesuai README: +1. Webcam +2. MediaPipe +3. Landmark +4. Normalisasi +5. Window 30 Frame +6. Model TCN +7. Prediksi +8. Tampilkan hasil +""" + +import sys +import cv2 +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from utils.inference import GestureRecognizer +from utils.config import MODEL_DIR, WINDOW_SIZE + + +def run_realtime(category: str = "kata", camera_id: int = 0): + """ + Run real-time recognition dengan visualisasi landmarks. + + Args: + category: Nama kategori + camera_id: Camera index + """ + model_path = MODEL_DIR / "kata" / "model_kata.pt" + encoder_path = MODEL_DIR / "kata" / "label_encoder.json" + + if not model_path.exists(): + print(f"Error: Model tidak ditemukan di {model_path}") + print("Jalankan training terlebih dahulu!") + return + + # Initialize recognizer + recognizer = GestureRecognizer( + model_path, + encoder_path, + window_size=WINDOW_SIZE, + draw_landmarks=True + ) + + # Open camera + cap = cv2.VideoCapture(camera_id) + + if not cap.isOpened(): + print(f"Error: Tidak dapat membuka camera {camera_id}") + return + + print("\n" + "="*60) + print(f"REALTIME RECOGNITION - {category.upper()}") + print("="*60) + print("Tekan 'q' untuk quit") + print("Tekan 'r' untuk reset buffer") + print("Tekan 'l' untuk toggle landmark visibility") + print("="*60 + "\n") + + show_landmarks = True + + try: + while True: + ret, frame = cap.read() + + if not ret: + print("Error: Tidak dapat membaca frame") + break + + # Draw landmarks + if show_landmarks: + frame = recognizer.draw_landmarks_on_frame(frame) + + # Process frame + label, confidence = recognizer.process_frame(frame) + + # Draw prediction + if label is not None: + text = f"{label} ({confidence*100:.1f}%)" + cv2.putText(frame, text, (10, 50), + cv2.FONT_HERSHEY_SIMPLEX, 1.5, (0, 255, 0), 3) + + # Draw buffer status + buffer_len = len(recognizer.buffer) + status = f"Buffer: {buffer_len}/{recognizer.window_size}" + cv2.putText(frame, status, (10, 100), + cv2.FONT_HERSHEY_SIMPLEX, 0.8, (255, 255, 255), 2) + + # Draw landmark status + landmark_status = "Landmarks: ON" if show_landmarks else "Landmarks: OFF" + cv2.putText(frame, landmark_status, (10, 130), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 255), 2) + + # Draw detection status + if label is None: + cv2.putText(frame, "Tunjukkan gestur isyarat...", (10, 160), + cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2) + + # Show frame + cv2.imshow(f"Realtime Recognition - {category}", frame) + + # Check key + key = cv2.waitKey(1) & 0xFF + if key == ord('q'): + break + elif key == ord('r'): + recognizer.reset() + print("Buffer reset") + elif key == ord('l'): + show_landmarks = not show_landmarks + print(f"Landmarks: {'ON' if show_landmarks else 'OFF'}") + + finally: + cap.release() + cv2.destroyAllWindows() + recognizer.close() + + +def main(): + import argparse + parser = argparse.ArgumentParser(description="Realtime recognition kata") + parser.add_argument("--camera", type=int, default=0, help="Camera index") + args = parser.parse_args() + run_realtime("kata", args.camera) + + +if __name__ == "__main__": + main() diff --git a/backend/requirements.txt b/backend/requirements.txt new file mode 100644 index 0000000..ed7a845 --- /dev/null +++ b/backend/requirements.txt @@ -0,0 +1,31 @@ +# Requirements untuk Bahasa Isyarat Translator + +# Core dependencies +numpy>=1.21.0 +torch>=1.10.0 +torchvision>=0.11.0 + +# MediaPipe for hand and face landmark detection +# Upgrade ml_dtypes first to fix tensorflow conflict +ml_dtypes>=0.3.0 + +# MediaPipe +mediapipe>=0.10.0 + +# OpenCV for video processing +opencv-python>=4.8.0 +opencv-python-headless>=4.8.0 + +# Machine Learning utilities +scikit-learn>=0.24.0 + +# Visualization +matplotlib>=3.7.0 +seaborn>=0.12.0 + +# Progress bars +tqdm>=4.65.0 + +# Web framework for API +Flask>=2.3.0 +flask-cors>=4.0.0 diff --git a/backend/scripts/01_extract_landmark.py b/backend/scripts/01_extract_landmark.py new file mode 100644 index 0000000..d6b197a --- /dev/null +++ b/backend/scripts/01_extract_landmark.py @@ -0,0 +1,344 @@ +""" +Tahap 1: Extract Landmark + +Script ini mengekstrak landmark tangan dan wajah dari video +menggunakan MediaPipe dan menyimpannya dalam format .npy + +Sesuai README: +- Input: dataset/raw/ +- Output: dataset/landmark/ +- Proses: + - Membaca seluruh video + - Membaca frame menggunakan OpenCV + - Deteksi MediaPipe Hands (21 landmark tangan kiri, 21 landmark tangan kanan) + - Deteksi MediaPipe Face Mesh (60 landmark) + - Membentuk Sliding Window 30 Frame dan 5 Stride + - Menyimpan file .npy +""" + +import os +import sys +import cv2 +import numpy as np +from pathlib import Path +from tqdm import tqdm +from typing import List, Tuple, Optional +import argparse + +# Add parent directory to path +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from utils.mediapipe_detector import MediaPipeDetector +from utils.config import ( + RAW_DIR, + LANDMARK_DIR, + WINDOW_SIZE, + STRIDE, + VIDEO_EXTENSIONS, + FEATURE_SIZE +) + + +class LandmarkExtractor: + """ + Extractor untuk mengambil landmark dari video streams. + + Sesuai README: + - Sliding Window 30 Frame + - Stride 5 + - Output shape: (30, feature_size) + """ + + def __init__( + self, + window_size: int = WINDOW_SIZE, + stride: int = STRIDE, + detector: Optional[MediaPipeDetector] = None + ): + """ + Inisialisasi extractor. + + Args: + window_size: Jumlah frame per window (30) + stride: Step size untuk sliding window (5) + detector: MediaPipe detector instance + """ + self.window_size = window_size + self.stride = stride + self.detector = detector or MediaPipeDetector() + + def extract_from_video(self, video_path: Path) -> Optional[np.ndarray]: + """ + Extract landmark dari satu video dengan sliding window. + + Args: + video_path: Path ke video file + + Returns: + Array landmark sequence shape (num_windows, 30, 306) atau None jika gagal + """ + # Open video + cap = cv2.VideoCapture(str(video_path)) + + if not cap.isOpened(): + print(f"Error: Cannot open video {video_path}") + return None + + # Collect all frame landmarks + all_landmarks = [] + frame_count = 0 + + try: + while True: + ret, frame = cap.read() + + if not ret: + break + + # Extract landmarks dari frame + left_hand, right_hand, face = self.detector.detect(frame) + + # Flatten dan concatenate + features = np.concatenate([ + left_hand.flatten(), # 63 values + right_hand.flatten(), # 63 values + face.flatten() # 180 values + ]) + + all_landmarks.append(features) + frame_count += 1 + + finally: + cap.release() + + if len(all_landmarks) < self.window_size: + print(f"Warning: Video {video_path.name} hanya memiliki {len(all_landmarks)} frames") + return None + + # Create sliding windows + windows = [] + for i in range(0, len(all_landmarks) - self.window_size + 1, self.stride): + window = all_landmarks[i:i + self.window_size] + windows.append(np.array(window)) + + if not windows: + # Jika tidak ada window penuh, ambil dari akhir + windows.append(np.array(all_landmarks[-self.window_size:])) + + return np.array(windows) # Shape: (num_windows, 30, 306) + + def close(self): + """Cleanup resources.""" + self.detector.close() + + +def get_video_files(directory: Path) -> List[Path]: + """ + Get semua video files dalam directory dan subdirectories. + + Args: + directory: Root directory + + Returns: + List of video file paths + """ + video_files = [] + + for ext in VIDEO_EXTENSIONS: + video_files.extend(directory.rglob(f"*{ext}")) + + return sorted(video_files) + + +def extract_category( + raw_dir: Path, + landmark_dir: Path, + category: str, + window_size: int = WINDOW_SIZE, + stride: int = STRIDE +) -> Tuple[int, int]: + """ + Extract landmarks untuk satu kategori. + + Args: + raw_dir: Directory data mentah + landmark_dir: Directory output landmark + category: Nama kategori + window_size: Window size + stride: Stride + + Returns: + Tuple (success_count, fail_count) + """ + category_raw_dir = raw_dir / category + + if not category_raw_dir.exists(): + print(f"Warning: Category directory tidak ditemukan: {category_raw_dir}") + return 0, 0 + + # Get all video files + video_files = get_video_files(category_raw_dir) + + if not video_files: + print(f"Warning: Tidak ada video ditemukan untuk kategori {category}") + return 0, 0 + + print(f"\n{'='*60}") + print(f"Extracting landmarks untuk kategori: {category}") + print(f"Total video files: {len(video_files)}") + print(f"Window size: {window_size}, Stride: {stride}") + print(f"{'='*60}\n") + + # Initialize extractor + extractor = LandmarkExtractor(window_size=window_size, stride=stride) + + success_count = 0 + fail_count = 0 + + try: + for video_path in tqdm(video_files, desc=f"Extracting {category}"): + try: + # Get label dari folder structure + # Format: raw/{category}/{label}/video.mkv + relative_path = video_path.relative_to(category_raw_dir) + parts = relative_path.parts + + if len(parts) >= 2: + label = parts[0] # Folder label (huruf/angka/kata) + video_name = video_path.stem # Filename tanpa extension + + # Create output path (sama dengan struktur raw) + output_dir = landmark_dir / category / label + output_dir.mkdir(parents=True, exist_ok=True) + + # Extract landmarks + windows = extractor.extract_from_video(video_path) + + if windows is not None and len(windows) > 0: + # Save setiap window sebagai file terpisah + for i, window in enumerate(windows): + output_path = output_dir / f"{video_name}_w{i}.npy" + np.save(output_path, window) + success_count += 1 + else: + fail_count += 1 + + except Exception as e: + print(f"Error processing {video_path}: {e}") + fail_count += 1 + + finally: + extractor.close() + + return success_count, fail_count + + +def extract_all_categories( + raw_dir: Path = RAW_DIR, + landmark_dir: Path = LANDMARK_DIR, + categories: List[str] = None, + window_size: int = WINDOW_SIZE, + stride: int = STRIDE +): + """ + Extract landmarks untuk semua kategori. + + Args: + raw_dir: Directory data mentah + landmark_dir: Directory output landmark + categories: List kategori yang akan diproses + window_size: Window size (30) + stride: Stride (5) + """ + if categories is None: + from utils.config import CATEGORIES + categories = CATEGORIES + + print("="*60) + print("TAHAP 1: EXTRACT LANDMARK") + print("="*60) + print(f"Input: {raw_dir}") + print(f"Output: {landmark_dir}") + print(f"Categories: {categories}") + print(f"Window size: {window_size}, Stride: {stride}") + print("="*60) + + total_success = 0 + total_fail = 0 + + for category in categories: + success, fail = extract_category( + raw_dir=raw_dir, + landmark_dir=landmark_dir, + category=category, + window_size=window_size, + stride=stride + ) + total_success += success + total_fail += fail + + print("\n" + "="*60) + print("EXTRACTION COMPLETE") + print("="*60) + print(f"Total berhasil: {total_success}") + print(f"Total gagal: {total_fail}") + if total_success + total_fail > 0: + print(f"Success rate: {total_success/(total_success+total_fail)*100:.2f}%") + print("="*60) + + +def main(): + """Main entry point.""" + parser = argparse.ArgumentParser( + description="Extract landmarks dari video untuk gesture recognition" + ) + parser.add_argument( + "--input", + type=str, + default=None, + help="Directory data mentah (default: dataset/raw)" + ) + parser.add_argument( + "--output", + type=str, + default=None, + help="Directory output landmark (default: dataset/landmark)" + ) + parser.add_argument( + "--categories", + nargs="+", + choices=["abjad_bisindo", "abjad_sibi", "angka", "kata"], + default=None, + help="Kategori yang akan diproses" + ) + parser.add_argument( + "--window-size", + type=int, + default=WINDOW_SIZE, + help=f"Window size (default: {WINDOW_SIZE})" + ) + parser.add_argument( + "--stride", + type=int, + default=STRIDE, + help=f"Stride (default: {STRIDE})" + ) + + args = parser.parse_args() + + # Setup paths + raw_dir = Path(args.input) if args.input else RAW_DIR + landmark_dir = Path(args.output) if args.output else LANDMARK_DIR + + # Extract + extract_all_categories( + raw_dir=raw_dir, + landmark_dir=landmark_dir, + categories=args.categories, + window_size=args.window_size, + stride=args.stride + ) + + +if __name__ == "__main__": + main() diff --git a/backend/scripts/02_normalize_landmark.py b/backend/scripts/02_normalize_landmark.py new file mode 100644 index 0000000..3ef554c --- /dev/null +++ b/backend/scripts/02_normalize_landmark.py @@ -0,0 +1,262 @@ +""" +Tahap 2: Normalize Landmark + +Script ini menormalisasi landmark yang sudah diekstrak +dengan translation dan scaling untuk konsistensi. + +Sesuai README: +- Input: dataset/landmark/ +- Output: dataset/normalized/ +- Proses: + - Membaca seluruh landmark + - Menggeser koordinat ke titik referensi + - Scaling + - Menyimpan landmark hasil normalisasi + +Catatan: Struktur folder pada normalized/ akan dibuat identik dengan folder landmark/ dan raw/. +""" + +import sys +import numpy as np +from pathlib import Path +from tqdm import tqdm +from typing import List, Tuple, Optional +import argparse + +# Add parent directory to path +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from utils.normalization import LandmarkNormalizer +from utils.config import ( + LANDMARK_DIR, + NORMALIZED_DIR, + CATEGORIES, + WINDOW_SIZE, + FEATURE_SIZE +) + + +class LandmarkNormalizerPipeline: + """ + Pipeline untuk normalisasi landmark sequences. + """ + + def __init__(self, scale_factor: Optional[float] = None): + """ + Inisialisasi normalizer pipeline. + + Args: + scale_factor: Optional fixed scale factor + """ + self.normalizer = LandmarkNormalizer(scale_factor=scale_factor) + + def normalize_sequence(self, landmarks: np.ndarray) -> np.ndarray: + """ + Normalisasi satu sequence landmark. + + Args: + landmarks: Array shape (30, 306) + + Returns: + Normalized sequence + """ + if landmarks is None or len(landmarks) == 0: + return np.zeros((WINDOW_SIZE, FEATURE_SIZE)) + + normalized_sequence = [] + + for frame_features in landmarks: + # Split frame ke components + left_hand = frame_features[:63].reshape(21, 3) + right_hand = frame_features[63:126].reshape(21, 3) + face = frame_features[126:].reshape(60, 3) + + # Normalize + norm_frame = self.normalizer.normalize_frame(left_hand, right_hand, face) + normalized_sequence.append(norm_frame) + + return np.array(normalized_sequence) + + +def get_landmark_files(directory: Path) -> List[Path]: + """ + Get semua landmark .npy files dalam directory. + + Args: + directory: Root directory + + Returns: + List of .npy file paths + """ + if not directory.exists(): + return [] + + return sorted(directory.rglob("*.npy")) + + +def normalize_category( + landmark_dir: Path, + normalized_dir: Path, + category: str +) -> Tuple[int, int]: + """ + Normalisasi landmarks untuk satu kategori. + + Args: + landmark_dir: Directory landmark mentah + normalized_dir: Directory output normalized + category: Nama kategori + + Returns: + Tuple (success_count, fail_count) + """ + category_landmark_dir = landmark_dir / category + + if not category_landmark_dir.exists(): + print(f"Warning: Category directory tidak ditemukan: {category_landmark_dir}") + return 0, 0 + + # Get all landmark files + landmark_files = get_landmark_files(category_landmark_dir) + + if not landmark_files: + print(f"Warning: Tidak ada landmark file ditemukan untuk kategori {category}") + return 0, 0 + + print(f"\n{'='*60}") + print(f"Normalizing landmarks untuk kategori: {category}") + print(f"Total files: {len(landmark_files)}") + print(f"{'='*60}\n") + + # Initialize normalizer + normalizer = LandmarkNormalizerPipeline() + + success_count = 0 + fail_count = 0 + + for landmark_path in tqdm(landmark_files, desc=f"Normalizing {category}"): + try: + # Load landmark + sequence = np.load(landmark_path) + + # Get label dari path + # Format: landmark/{category}/{label}/file.npy + relative_path = landmark_path.relative_to(category_landmark_dir) + parts = relative_path.parts + + if len(parts) >= 2: + label = parts[0] # Folder label + + # Create output path (struktur identik) + output_dir = normalized_dir / category / label + output_dir.mkdir(parents=True, exist_ok=True) + + output_path = output_dir / landmark_path.name + + # Skip jika sudah ada + if output_path.exists(): + continue + + # Normalize + normalized_sequence = normalizer.normalize_sequence(sequence) + + # Save + np.save(output_path, normalized_sequence) + success_count += 1 + else: + fail_count += 1 + + except Exception as e: + print(f"Error processing {landmark_path}: {e}") + fail_count += 1 + + return success_count, fail_count + + +def normalize_all_categories( + landmark_dir: Path = LANDMARK_DIR, + normalized_dir: Path = NORMALIZED_DIR, + categories: List[str] = None +): + """ + Normalisasi landmarks untuk semua kategori. + + Args: + landmark_dir: Directory landmark mentah + normalized_dir: Directory output normalized + categories: List kategori yang akan diproses + """ + if categories is None: + categories = CATEGORIES + + print("="*60) + print("TAHAP 2: NORMALIZE LANDMARK") + print("="*60) + print(f"Input: {landmark_dir}") + print(f"Output: {normalized_dir}") + print(f"Categories: {categories}") + print("="*60) + + total_success = 0 + total_fail = 0 + + for category in categories: + success, fail = normalize_category( + landmark_dir=landmark_dir, + normalized_dir=normalized_dir, + category=category + ) + total_success += success + total_fail += fail + + print("\n" + "="*60) + print("NORMALIZATION COMPLETE") + print("="*60) + print(f"Total berhasil: {total_success}") + print(f"Total gagal: {total_fail}") + if total_success + total_fail > 0: + print(f"Success rate: {total_success/(total_success+total_fail)*100:.2f}%") + print("="*60) + + +def main(): + """Main entry point.""" + parser = argparse.ArgumentParser( + description="Normalisasi landmark untuk gesture recognition" + ) + parser.add_argument( + "--input", + type=str, + default=None, + help="Directory landmark (default: dataset/landmark)" + ) + parser.add_argument( + "--output", + type=str, + default=None, + help="Directory output normalized (default: dataset/normalized)" + ) + parser.add_argument( + "--categories", + nargs="+", + choices=["abjad_bisindo", "abjad_sibi", "angka", "kata"], + default=None, + help="Kategori yang akan diproses" + ) + + args = parser.parse_args() + + # Setup paths + landmark_dir = Path(args.input) if args.input else LANDMARK_DIR + normalized_dir = Path(args.output) if args.output else NORMALIZED_DIR + + # Normalize + normalize_all_categories( + landmark_dir=landmark_dir, + normalized_dir=normalized_dir, + categories=args.categories + ) + + +if __name__ == "__main__": + main() diff --git a/backend/scripts/03_split_dataset.py b/backend/scripts/03_split_dataset.py new file mode 100644 index 0000000..d0b3476 --- /dev/null +++ b/backend/scripts/03_split_dataset.py @@ -0,0 +1,387 @@ +""" +Tahap 3: Split Dataset + +Script ini membagi dataset menjadi 80% training dan 20% testing +secara stratified untuk setiap kelas. + +Sesuai README: +- Input: dataset/normalized/ +- Output: dataset/train/ dan dataset/test/ +- Pembagian dilakukan untuk setiap kelas +- Struktur folder dibuat sama persis dengan folder normalized/ + +Persentase: +- Train: 80% +- Test: 20% +""" + +import sys +import json +import shutil +import numpy as np +from pathlib import Path +from tqdm import tqdm +from typing import List, Tuple, Dict +import argparse +from collections import defaultdict, Counter + +# Add parent directory to path +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from utils.config import ( + NORMALIZED_DIR, + TRAIN_DIR, + TEST_DIR, + CATEGORIES, + TRAIN_RATIO +) + + +class DatasetSplitter: + """ + Splitter untuk membagi dataset menjadi train dan test sets. + Pembagian dilakukan untuk setiap kelas secara stratified. + """ + + def __init__(self, train_ratio: float = TRAIN_RATIO, random_seed: int = 42): + """ + Inisialisasi splitter. + + Args: + train_ratio: Rasio data untuk training (0.8 = 80%) + random_seed: Seed untuk random + """ + self.train_ratio = train_ratio + + # Set random seed + np.random.seed(random_seed) + + def split_samples( + self, + samples: List[Path], + labels: List[str] + ) -> Tuple[List[Path], List[str], List[Path], List[str]]: + """ + Split samples menjadi train dan test sets secara stratified. + + Args: + samples: List of sample paths + labels: List of corresponding labels + + Returns: + Tuple (train_samples, train_labels, test_samples, test_labels) + """ + # Group samples by label + label_to_samples = defaultdict(list) + for sample, label in zip(samples, labels): + label_to_samples[label].append(sample) + + train_samples = [] + train_labels = [] + test_samples = [] + test_labels = [] + + # Split setiap kelas secara proporsional + for label, class_samples in label_to_samples.items(): + # Shuffle samples + indices = np.arange(len(class_samples)) + np.random.shuffle(indices) + + # Calculate split point + n_train = int(len(class_samples) * self.train_ratio) + n_train = max(1, n_train) # Minimal 1 sample untuk train + + # Split + train_indices = indices[:n_train] + test_indices = indices[n_train:] + + # Jika test set kosong, ambil 1 dari train + if len(test_indices) == 0 and len(train_indices) > 1: + test_indices = [train_indices[-1]] + train_indices = train_indices[:-1] + + train_samples.extend([class_samples[i] for i in train_indices]) + train_labels.extend([label] * len(train_indices)) + + test_samples.extend([class_samples[i] for i in test_indices]) + test_labels.extend([label] * len(test_indices)) + + return train_samples, train_labels, test_samples, test_labels + + def copy_samples( + self, + samples: List[Path], + labels: List[str], + output_dir: Path, + category: str, + split: str + ) -> int: + """ + Copy samples ke output directory. + + Args: + samples: List of sample paths + labels: List of labels + output_dir: Root output directory + category: Category name + split: 'train' atau 'test' + + Returns: + Number of copied files + """ + copied_count = 0 + + for sample_path, label in tqdm( + zip(samples, labels), + total=len(samples), + desc=f"Copying {split}" + ): + try: + # Create output path (struktur sama persis dengan normalized/) + # Format: {train|test}/{category}/{label}/file.npy + # output_dir sudah berupa dataset/train atau dataset/test + output_path = output_dir / category / label / sample_path.name + + # Skip jika sudah ada + if output_path.exists(): + continue + + # Create directory + output_path.parent.mkdir(parents=True, exist_ok=True) + + # Copy file + shutil.copy2(sample_path, output_path) + copied_count += 1 + + except Exception as e: + print(f"Error copying {sample_path}: {e}") + + return copied_count + + +def get_normalized_files(normalized_dir: Path, category: str) -> Tuple[List[Path], List[str]]: + """ + Get semua file normalized dan labels-nya. + + Args: + normalized_dir: Root normalized directory + category: Category name + + Returns: + Tuple (list of files, list of labels) + """ + category_dir = normalized_dir / category + + if not category_dir.exists(): + return [], [] + + files = [] + labels = [] + + # Iterate semua subdirectories (labels) + for label_dir in category_dir.iterdir(): + if label_dir.is_dir(): + label = label_dir.name + + # Get semua .npy files + for npy_file in label_dir.glob("*.npy"): + files.append(npy_file) + labels.append(label) + + return files, labels + + +def split_category( + normalized_dir: Path, + train_dir: Path, + test_dir: Path, + category: str, + train_ratio: float = TRAIN_RATIO +) -> Dict: + """ + Split dataset untuk satu kategori. + + Args: + normalized_dir: Directory normalized data + train_dir: Directory untuk data train + test_dir: Directory untuk data test + category: Category name + train_ratio: Train ratio + + Returns: + Statistics dictionary + """ + print(f"\n{'='*60}") + print(f"Splitting dataset untuk kategori: {category}") + print(f"{'='*60}\n") + + # Get files dan labels + files, labels = get_normalized_files(normalized_dir, category) + + if not files: + print(f"Warning: Tidak ada file ditemukan untuk kategori {category}") + return {} + + print(f"Total samples: {len(files)}") + + # Count per class + class_counts = Counter(labels) + print(f"Classes: {len(class_counts)}") + for label, count in sorted(class_counts.items()): + print(f" - {label}: {count} samples") + + # Split + splitter = DatasetSplitter(train_ratio=train_ratio) + train_files, train_labels, test_files, test_labels = splitter.split_samples(files, labels) + + print(f"\nAfter split:") + print(f" Train: {len(train_files)} samples ({len(train_files)/len(files)*100:.1f}%)") + print(f" Test: {len(test_files)} samples ({len(test_files)/len(files)*100:.1f}%)") + + # Copy files + print("\nCopying files...") + train_copied = splitter.copy_samples( + train_files, train_labels, train_dir, category, "train" + ) + test_copied = splitter.copy_samples( + test_files, test_labels, test_dir, category, "test" + ) + print(f"Copied: {train_copied} train, {test_copied} test files") + + # Generate label encoder + unique_labels = sorted(set(labels)) + label_encoder = {label: idx for idx, label in enumerate(unique_labels)} + + # Save label encoder di folder models/ + model_dir = Path(__file__).parent.parent / "models" + encoder_path = model_dir / f"{category}_label_encoder.json" + encoder_path.parent.mkdir(parents=True, exist_ok=True) + with open(encoder_path, 'w') as f: + json.dump(label_encoder, f, indent=2) + print(f"\nSaved label encoder to: {encoder_path}") + + # Return statistics + return { + "category": category, + "train_ratio": train_ratio, + "total_samples": len(files), + "train_samples": len(train_files), + "test_samples": len(test_files), + "num_classes": len(unique_labels), + "classes": unique_labels + } + + +def split_all_categories( + normalized_dir: Path = NORMALIZED_DIR, + train_dir: Path = TRAIN_DIR, + test_dir: Path = TEST_DIR, + categories: List[str] = None, + train_ratio: float = TRAIN_RATIO +): + """ + Split semua kategori dataset. + + Args: + normalized_dir: Directory normalized data + train_dir: Directory untuk data train + test_dir: Directory untuk data test + categories: List kategori + train_ratio: Train ratio + """ + if categories is None: + categories = CATEGORIES + + print("="*60) + print("TAHAP 3: SPLIT DATASET") + print("="*60) + print(f"Input: {normalized_dir}") + print(f"Output:") + print(f" - Train: {train_dir}") + print(f" - Test: {test_dir}") + print(f"Categories: {categories}") + print(f"Train ratio: {train_ratio*100}%") + print(f"Test ratio: {(1-train_ratio)*100}%") + print("="*60) + + all_split_info = {} + + for category in categories: + split_info = split_category( + normalized_dir=normalized_dir, + train_dir=train_dir, + test_dir=test_dir, + category=category, + train_ratio=train_ratio + ) + if split_info: + all_split_info[category] = split_info + + # Print summary table + print("\n" + "="*60) + print("SPLIT SUMMARY") + print("="*60) + print(f"{'Kategori':<20} {'Total':<10} {'Train':<10} {'Test':<10} {'Kelas':<10}") + print("-"*60) + for cat, info in all_split_info.items(): + print(f"{cat:<20} {info['total_samples']:<10} {info['train_samples']:<10} " + f"{info['test_samples']:<10} {info['num_classes']:<10}") + print("-"*60) + + +def main(): + """Main entry point.""" + parser = argparse.ArgumentParser( + description="Split dataset untuk training dan testing" + ) + parser.add_argument( + "--input", + type=str, + default=None, + help="Directory normalized data (default: dataset/normalized)" + ) + parser.add_argument( + "--train-dir", + type=str, + default=None, + help="Directory untuk data train (default: dataset/train)" + ) + parser.add_argument( + "--test-dir", + type=str, + default=None, + help="Directory untuk data test (default: dataset/test)" + ) + parser.add_argument( + "--categories", + nargs="+", + choices=["abjad_bisindo", "abjad_sibi", "angka", "kata"], + default=None, + help="Kategori yang akan diproses" + ) + parser.add_argument( + "--train-ratio", + type=float, + default=TRAIN_RATIO, + help=f"Rasio training (default: {TRAIN_RATIO})" + ) + + args = parser.parse_args() + + # Setup paths + normalized_dir = Path(args.input) if args.input else NORMALIZED_DIR + train_dir = Path(args.train_dir) if args.train_dir else TRAIN_DIR + test_dir = Path(args.test_dir) if args.test_dir else TEST_DIR + + # Split + split_all_categories( + normalized_dir=normalized_dir, + train_dir=train_dir, + test_dir=test_dir, + categories=args.categories, + train_ratio=args.train_ratio + ) + + +if __name__ == "__main__": + main() diff --git a/backend/scripts/04_train_kata.py b/backend/scripts/04_train_kata.py new file mode 100644 index 0000000..4f9ab77 --- /dev/null +++ b/backend/scripts/04_train_kata.py @@ -0,0 +1,274 @@ +""" +Tahap 4: Training Kata + +Script ini melatih model TCN untuk mengenali kata dalam bahasa isyarat. + +Sesuai README: +- Input: dataset/train/kata/ +- Output: models/kata/model_kata.pt +- Output tambahan: Accuracy, Loss, Training History +""" + +import sys +import json +import numpy as np +from pathlib import Path +from typing import Tuple +import argparse + +import torch +import torch.nn as nn +import torch.optim as optim +from torch.utils.data import DataLoader, Dataset +from tqdm import tqdm +import matplotlib.pyplot as plt + +# Add parent directory to path +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from utils.tcn import TCNClassifier +from utils.config import ( + TRAIN_DIR, + MODEL_DIR, + FEATURE_SIZE, + BATCH_SIZE, + EPOCHS, + LEARNING_RATE, + ACCURACY_DIR, + LOSS_DIR +) + + +class KataDataset(Dataset): + """Dataset untuk kata.""" + + def __init__(self, data_dir: Path): + self.data_dir = data_dir + self.samples = [] + self.labels = [] + self.label_to_idx = {} + self._load_data() + + def _load_data(self): + if not self.data_dir.exists(): + return + + label_dirs = sorted([d for d in self.data_dir.iterdir() if d.is_dir()]) + self.label_to_idx = {d.name: idx for idx, d in enumerate(label_dirs)} + + for label_dir in label_dirs: + label_idx = self.label_to_idx[label_dir.name] + for npy_file in label_dir.glob("*.npy"): + try: + sequence = np.load(npy_file) + self.samples.append(sequence) + self.labels.append(label_idx) + except Exception as e: + print(f"Error loading {npy_file}: {e}") + + def __len__(self): + return len(self.samples) + + def __getitem__(self, idx) -> Tuple: + return torch.FloatTensor(self.samples[idx]), torch.LongTensor([self.labels[idx]]).squeeze() + + +def train_epoch(model, train_loader, criterion, optimizer, device): + model.train() + total_loss, correct, total = 0, 0, 0 + + for sequences, labels in tqdm(train_loader, desc="Training"): + sequences, labels = sequences.to(device), labels.to(device) + + optimizer.zero_grad() + outputs = model(sequences) + loss = criterion(outputs, labels) + loss.backward() + optimizer.step() + + total_loss += loss.item() + _, predicted = torch.max(outputs, 1) + total += labels.size(0) + correct += (predicted == labels).sum().item() + + return total_loss / len(train_loader), correct / total * 100 + + +def evaluate(model, eval_loader, criterion, device): + model.eval() + total_loss, correct, total = 0, 0, 0 + + with torch.no_grad(): + for sequences, labels in eval_loader: + sequences, labels = sequences.to(device), labels.to(device) + + outputs = model(sequences) + loss = criterion(outputs, labels) + + total_loss += loss.item() + _, predicted = torch.max(outputs, 1) + total += labels.size(0) + correct += (predicted == labels).sum().item() + + return total_loss / len(eval_loader), correct / total * 100 + + +def train_kata(data_dir: Path, model_dir: Path, epochs: int = EPOCHS, + batch_size: int = BATCH_SIZE, learning_rate: float = LEARNING_RATE): + """Training model untuk kata.""" + + # Setup device + device = 'cuda' if torch.cuda.is_available() else 'cpu' + print(f"Using device: {device}") + + # Create dataset + print(f"\nLoading dataset from: {data_dir}") + train_dataset = KataDataset(data_dir / "train" / "kata") + test_dataset = KataDataset(data_dir / "test" / "kata") + + print(f"Train samples: {len(train_dataset)}") + print(f"Test samples: {len(test_dataset)}") + + if len(train_dataset) == 0: + raise ValueError("No training samples found!") + + num_classes = len(train_dataset.label_to_idx) + print(f"Number of classes: {num_classes}") + print(f"Classes: {list(train_dataset.label_to_idx.keys())}") + + # Create data loaders + train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=0) + test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0) + + # Create model + model = TCNClassifier( + input_size=FEATURE_SIZE, + num_classes=num_classes, + num_channels=[64, 128, 256] + ).to(device) + + print(f"\nTotal parameters: {sum(p.numel() for p in model.parameters()):,}") + + # Loss and optimizer + criterion = nn.CrossEntropyLoss() + optimizer = optim.Adam(model.parameters(), lr=learning_rate) + scheduler = optim.lr_scheduler.ReduceLROnPlateau( + optimizer, mode='min', factor=0.5, patience=5 + ) + + # Training loop + history = { + "train_loss": [], "train_acc": [], + "test_loss": [], "test_acc": [] + } + + best_test_acc = 0 + best_model_state = None + + print("\n" + "="*60) + print("Training Kata Model") + print("="*60) + + for epoch in range(epochs): + print(f"\nEpoch {epoch+1}/{epochs}") + print("-" * 40) + + train_loss, train_acc = train_epoch(model, train_loader, criterion, optimizer, device) + test_loss, test_acc = evaluate(model, test_loader, criterion, device) + scheduler.step(test_loss) + + history["train_loss"].append(train_loss) + history["train_acc"].append(train_acc) + history["test_loss"].append(test_loss) + history["test_acc"].append(test_acc) + + print(f"Train Loss: {train_loss:.4f} | Train Acc: {train_acc:.2f}%") + print(f"Test Loss: {test_loss:.4f} | Test Acc: {test_acc:.2f}%") + + if test_acc > best_test_acc: + best_test_acc = test_acc + best_model_state = model.state_dict().copy() + print(f"*** New best model! Test Acc: {best_test_acc:.2f}% ***") + + # Load best model + if best_model_state is not None: + model.load_state_dict(best_model_state) + + # Save model (sesuai README: models/kata/model_kata.pt) + model_dir.mkdir(parents=True, exist_ok=True) + kata_model_dir = model_dir / "kata" + kata_model_dir.mkdir(exist_ok=True) + + model_path = kata_model_dir / "model_kata.pt" + torch.save(model, model_path) + print(f"\nModel saved to: {model_path}") + + # Save label encoder + idx_to_label = {v: k for k, v in train_dataset.label_to_idx.items()} + encoder_path = kata_model_dir / "label_encoder.json" + with open(encoder_path, 'w') as f: + json.dump({"label_to_idx": train_dataset.label_to_idx, "idx_to_label": idx_to_label}, f, indent=2) + + # Save history + history_path = kata_model_dir / "history.json" + with open(history_path, 'w') as f: + json.dump(history, f, indent=2) + + # Plot Accuracy + plt.figure(figsize=(10, 6)) + plt.plot(history['train_acc'], label='Train Accuracy') + plt.plot(history['test_acc'], label='Test (Validation) Accuracy') + plt.title('Kata Model Accuracy') + plt.xlabel('Epoch') + plt.ylabel('Accuracy (%)') + plt.legend() + plt.grid(True) + plt.savefig(ACCURACY_DIR / "kata_accuracy.png") + plt.close() + print(f"Accuracy plot saved to: {ACCURACY_DIR / 'kata_accuracy.png'}") + + # Plot Loss + plt.figure(figsize=(10, 6)) + plt.plot(history['train_loss'], label='Train Loss') + plt.plot(history['test_loss'], label='Test (Validation) Loss') + plt.title('Kata Model Loss') + plt.xlabel('Epoch') + plt.ylabel('Loss') + plt.legend() + plt.grid(True) + plt.savefig(LOSS_DIR / "kata_loss.png") + plt.close() + print(f"Loss plot saved to: {LOSS_DIR / 'kata_loss.png'}") + + print("\n" + "="*60) + print(f"Best Test Accuracy: {best_test_acc:.2f}%") + print("="*60) + + return history + + +def main(): + """Main entry point.""" + parser = argparse.ArgumentParser(description="Training model kata") + parser.add_argument("--data-dir", type=str, default=None, + help="Directory dataset (default: dataset/)") + parser.add_argument("--model-dir", type=str, default=None, + help="Directory model (default: models/)") + parser.add_argument("--epochs", type=int, default=EPOCHS) + parser.add_argument("--batch-size", type=int, default=BATCH_SIZE) + parser.add_argument("--lr", type=float, default=LEARNING_RATE) + + args = parser.parse_args() + + data_dir = Path(args.data_dir) if args.data_dir else TRAIN_DIR.parent + model_dir = Path(args.model_dir) if args.model_dir else MODEL_DIR + + print("="*60) + print("TAHAP 4: TRAINING KATA") + print("="*60) + + train_kata(data_dir, model_dir, args.epochs, args.batch_size, args.lr) + + +if __name__ == "__main__": + main() diff --git a/backend/scripts/05_test_kata.py b/backend/scripts/05_test_kata.py new file mode 100644 index 0000000..fb938c2 --- /dev/null +++ b/backend/scripts/05_test_kata.py @@ -0,0 +1,239 @@ +""" +Tahap 5: Testing Kata + +Script ini menguji model yang sudah dilatih untuk mengenali kata. + +Sesuai README: +- Input: dataset/test/kata/ +- Model: models/kata/model_kata.pt +- Output: + - Confusion Matrix + - Classification Report + - Accuracy + - Precision + - Recall + - F1 Score +""" + +import sys +import json +import numpy as np +from pathlib import Path +import argparse + +import torch +from sklearn.metrics import ( + classification_report, + confusion_matrix, + accuracy_score, + precision_recall_fscore_support +) +import matplotlib.pyplot as plt +import seaborn as sns + +# Add parent directory to path +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from utils.config import ( + TEST_DIR, + MODEL_DIR, + BATCH_SIZE, + OUTPUT_DIR, + CONFUSION_MATRIX_DIR, + CLASSIFICATION_REPORT_DIR +) + + +class KataDataset: + """Dataset untuk kata.""" + + def __init__(self, data_dir: Path): + self.data_dir = data_dir / "test" / "kata" + self.samples = [] + self.labels = [] + self.label_to_idx = {} + self._load_data() + + def _load_data(self): + if not self.data_dir.exists(): + return + + label_dirs = sorted([d for d in self.data_dir.iterdir() if d.is_dir()]) + self.label_to_idx = {d.name: idx for idx, d in enumerate(label_dirs)} + + for label_dir in label_dirs: + label_idx = self.label_to_idx[label_dir.name] + for npy_file in label_dir.glob("*.npy"): + try: + sequence = np.load(npy_file) + self.samples.append(sequence) + self.labels.append(label_idx) + except: + pass + + def __len__(self): + return len(self.samples) + + +def load_model(model_path: Path, device: str): + model = torch.load(model_path, map_location=device, weights_only=False) + model.eval() + return model + + +def plot_confusion_matrix(cm, labels, output_path, title): + plt.figure(figsize=(14, 12)) + sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', + xticklabels=labels, yticklabels=labels) + plt.title(title) + plt.ylabel('True Label') + plt.xlabel('Predicted Label') + plt.tight_layout() + plt.savefig(output_path, dpi=150) + plt.close() + print(f"Confusion matrix saved to: {output_path}") + + +def test_kata(data_dir: Path, model_dir: Path, output_dir: Path): + """Testing model kata.""" + + device = 'cuda' if torch.cuda.is_available() else 'cpu' + print(f"Using device: {device}") + + # Load model + model_path = model_dir / "kata" / "model_kata.pt" + encoder_path = model_dir / "kata" / "label_encoder.json" + + if not model_path.exists(): + print(f"Error: Model tidak ditemukan di {model_path}") + return + + if not encoder_path.exists(): + print(f"Error: Label encoder tidak ditemukan di {encoder_path}") + return + + print(f"\nLoading model from: {model_path}") + model = load_model(model_path, device) + + with open(encoder_path, 'r') as f: + encoder_data = json.load(f) + idx_to_label = encoder_data["idx_to_label"] + + # Load test dataset + print(f"Loading test dataset from: {data_dir}") + dataset = KataDataset(data_dir) + + print(f"Test samples: {len(dataset)}") + + if len(dataset) == 0: + print("No test samples found!") + return + + # Prepare data + sequences = torch.FloatTensor(np.array(dataset.samples)) + true_labels = np.array(dataset.labels) + + # Predict + print("\nGenerating predictions...") + model.eval() + predictions = [] + + with torch.no_grad(): + for i in range(0, len(sequences), BATCH_SIZE): + batch = sequences[i:i+BATCH_SIZE].to(device) + outputs = model(batch) + _, preds = torch.max(outputs, 1) + predictions.extend(preds.cpu().numpy().tolist()) + + # Calculate metrics + accuracy = accuracy_score(true_labels, predictions) + precision, recall, f1, _ = precision_recall_fscore_support( + true_labels, predictions, average='weighted', zero_division=0 + ) + + # Confusion matrix + cm = confusion_matrix(true_labels, predictions) + + # Classification report + label_names = [idx_to_label[str(i)] for i in range(len(idx_to_label))] + report = classification_report( + true_labels, predictions, + target_names=label_names, + zero_division=0 + ) + + # Print results + print("\n" + "="*60) + print("TESTING KATA MODEL") + print("="*60) + print(f"\nAccuracy: {accuracy*100:.2f}%") + print(f"Precision: {precision*100:.2f}%") + print(f"Recall: {recall*100:.2f}%") + print(f"F1-Score: {f1*100:.2f}%") + print("\n" + "-"*60) + print("Classification Report:") + print("-"*60) + print(report) + + # Save confusion matrix + cm_path = CONFUSION_MATRIX_DIR / "kata_confusion_matrix.png" + plot_confusion_matrix(cm, label_names, cm_path, "Kata Confusion Matrix") + + # Save classification report + report_path = CLASSIFICATION_REPORT_DIR / "kata_classification_report.txt" + with open(report_path, 'w') as f: + f.write("Classification Report - Kata\n") + f.write("="*60 + "\n\n") + f.write(f"Accuracy: {accuracy*100:.2f}%\n") + f.write(f"Precision: {precision*100:.2f}%\n") + f.write(f"Recall: {recall*100:.2f}%\n") + f.write(f"F1-Score: {f1*100:.2f}%\n\n") + f.write("-"*60 + "\n") + f.write(report) + print(f"Classification report saved to: {report_path}") + + # Save results JSON + results = { + "category": "kata", + "accuracy": float(accuracy), + "precision": float(precision), + "recall": float(recall), + "f1_score": float(f1), + "num_test_samples": len(dataset), + "num_classes": len(label_names) + } + + results_path = output_dir / "kata_test_results.json" + with open(results_path, 'w') as f: + json.dump(results, f, indent=2) + print(f"Results saved to: {results_path}") + + print("\n" + "="*60) + print("TESTING COMPLETE") + print("="*60) + + return results + + +def main(): + """Main entry point.""" + parser = argparse.ArgumentParser(description="Testing model kata") + parser.add_argument("--data-dir", type=str, default=None) + parser.add_argument("--model-dir", type=str, default=None) + parser.add_argument("--output-dir", type=str, default=None) + + args = parser.parse_args() + + data_dir = Path(args.data_dir) if args.data_dir else TEST_DIR.parent + model_dir = Path(args.model_dir) if args.model_dir else MODEL_DIR + output_dir = Path(args.output_dir) if args.output_dir else OUTPUT_DIR + + print("="*60) + print("TAHAP 5: TESTING KATA") + print("="*60) + + test_kata(data_dir, model_dir, output_dir) + + +if __name__ == "__main__": + main() diff --git a/backend/scripts/06_train_abjad_bisindo.py b/backend/scripts/06_train_abjad_bisindo.py new file mode 100644 index 0000000..d723c03 --- /dev/null +++ b/backend/scripts/06_train_abjad_bisindo.py @@ -0,0 +1,216 @@ +""" +Tahap 6: Training Abjad BISINDO + +Script ini melatih model TCN untuk mengenali abjad BISINDO. + +Sesuai README: +- Input: dataset/train/abjad_bisindo/ +- Output: models/abjad_bisindo/model_abjad_bisindo.pt +""" + +import sys +import json +import numpy as np +from pathlib import Path +from typing import Tuple +import argparse + +import torch +import torch.nn as nn +import torch.optim as optim +from torch.utils.data import DataLoader, Dataset +from tqdm import tqdm +import matplotlib.pyplot as plt + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from utils.tcn import TCNClassifier +from utils.config import TRAIN_DIR, MODEL_DIR, FEATURE_SIZE, BATCH_SIZE, EPOCHS, LEARNING_RATE, ACCURACY_DIR, LOSS_DIR + + +class AbjadDataset(Dataset): + """Dataset untuk abjad.""" + + def __init__(self, data_dir: Path): + self.data_dir = data_dir + self.samples = [] + self.labels = [] + self.label_to_idx = {} + self._load_data() + + def _load_data(self): + if not self.data_dir.exists(): + return + label_dirs = sorted([d for d in self.data_dir.iterdir() if d.is_dir()]) + self.label_to_idx = {d.name: idx for idx, d in enumerate(label_dirs)} + for label_dir in label_dirs: + label_idx = self.label_to_idx[label_dir.name] + for npy_file in label_dir.glob("*.npy"): + try: + self.samples.append(np.load(npy_file)) + self.labels.append(label_idx) + except Exception as e: + print(f"Error loading {npy_file}: {e}") + + def __len__(self): + return len(self.samples) + + def __getitem__(self, idx) -> Tuple: + return torch.FloatTensor(self.samples[idx]), torch.LongTensor([self.labels[idx]]).squeeze() + + +def train_epoch(model, train_loader, criterion, optimizer, device): + model.train() + total_loss, correct, total = 0, 0, 0 + for sequences, labels in tqdm(train_loader, desc="Training"): + sequences, labels = sequences.to(device), labels.to(device) + optimizer.zero_grad() + outputs = model(sequences) + loss = criterion(outputs, labels) + loss.backward() + optimizer.step() + total_loss += loss.item() + _, predicted = torch.max(outputs, 1) + total += labels.size(0) + correct += (predicted == labels).sum().item() + return total_loss / len(train_loader), correct / total * 100 + + +def evaluate(model, eval_loader, criterion, device): + model.eval() + total_loss, correct, total = 0, 0, 0 + with torch.no_grad(): + for sequences, labels in eval_loader: + sequences, labels = sequences.to(device), labels.to(device) + outputs = model(sequences) + loss = criterion(outputs, labels) + total_loss += loss.item() + _, predicted = torch.max(outputs, 1) + total += labels.size(0) + correct += (predicted == labels).sum().item() + return total_loss / len(eval_loader), correct / total * 100 + + +def train_abjad_bisindo(data_dir: Path, model_dir: Path, epochs: int = EPOCHS, + batch_size: int = BATCH_SIZE, lr: float = LEARNING_RATE): + device = 'cuda' if torch.cuda.is_available() else 'cpu' + print(f"Using device: {device}") + + print(f"\nLoading dataset from: {data_dir}") + train_dataset = AbjadDataset(data_dir / "train" / "abjad_bisindo") + test_dataset = AbjadDataset(data_dir / "test" / "abjad_bisindo") + + print(f"Train samples: {len(train_dataset)}, Test samples: {len(test_dataset)}") + + if len(train_dataset) == 0: + raise ValueError("No training samples found!") + + num_classes = len(train_dataset.label_to_idx) + print(f"Number of classes: {num_classes}") + print(f"Classes: {list(train_dataset.label_to_idx.keys())}") + + train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=0) + test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0) + + model = TCNClassifier(input_size=FEATURE_SIZE, num_classes=num_classes, num_channels=[64, 128, 256]).to(device) + + print(f"\nTotal parameters: {sum(p.numel() for p in model.parameters()):,}") + + criterion = nn.CrossEntropyLoss() + optimizer = optim.Adam(model.parameters(), lr=lr) + scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=5) + + history = {"train_loss": [], "train_acc": [], "test_loss": [], "test_acc": []} + best_acc = 0 + + print("\n" + "="*60) + print("Training Abjad BISINDO Model") + print("="*60) + + for epoch in range(epochs): + print(f"\nEpoch {epoch+1}/{epochs}") + train_loss, train_acc = train_epoch(model, train_loader, criterion, optimizer, device) + test_loss, test_acc = evaluate(model, test_loader, criterion, device) + scheduler.step(test_loss) + history["train_loss"].append(train_loss) + history["train_acc"].append(train_acc) + history["test_loss"].append(test_loss) + history["test_acc"].append(test_acc) + print(f"Train Loss: {train_loss:.4f} | Train Acc: {train_acc:.2f}%") + print(f"Test Loss: {test_loss:.4f} | Test Acc: {test_acc:.2f}%") + if test_acc > best_acc: + best_acc = test_acc + best_state = model.state_dict().copy() + print(f"*** Best model! Acc: {best_acc:.2f}% ***") + + if best_state: + model.load_state_dict(best_state) + + # Save model + model_dir.mkdir(parents=True, exist_ok=True) + bisindo_model_dir = model_dir / "abjad_bisindo" + bisindo_model_dir.mkdir(exist_ok=True) + model_path = bisindo_model_dir / "model_abjad_bisindo.pt" + torch.save(model, model_path) + print(f"\nModel saved to: {model_path}") + + # Save label encoder + idx_to_label = {v: k for k, v in train_dataset.label_to_idx.items()} + with open(bisindo_model_dir / "label_encoder.json", 'w') as f: + json.dump({"label_to_idx": train_dataset.label_to_idx, "idx_to_label": idx_to_label}, f, indent=2) + + # Save history + with open(bisindo_model_dir / "history.json", 'w') as f: + json.dump(history, f, indent=2) + + # Plot Accuracy + plt.figure(figsize=(10, 6)) + plt.plot(history['train_acc'], label='Train Accuracy') + plt.plot(history['test_acc'], label='Test (Validation) Accuracy') + plt.title('Abjad BISINDO Model Accuracy') + plt.xlabel('Epoch') + plt.ylabel('Accuracy (%)') + plt.legend() + plt.grid(True) + plt.savefig(ACCURACY_DIR / "abjad_bisindo_accuracy.png") + plt.close() + print(f"Accuracy plot saved to: {ACCURACY_DIR / 'abjad_bisindo_accuracy.png'}") + + # Plot Loss + plt.figure(figsize=(10, 6)) + plt.plot(history['train_loss'], label='Train Loss') + plt.plot(history['test_loss'], label='Test (Validation) Loss') + plt.title('Abjad BISINDO Model Loss') + plt.xlabel('Epoch') + plt.ylabel('Loss') + plt.legend() + plt.grid(True) + plt.savefig(LOSS_DIR / "abjad_bisindo_loss.png") + plt.close() + print(f"Loss plot saved to: {LOSS_DIR / 'abjad_bisindo_loss.png'}") + + print(f"\nBest Test Accuracy: {best_acc:.2f}%") + return history + + +def main(): + parser = argparse.ArgumentParser(description="Training model abjad BISINDO") + parser.add_argument("--data-dir", type=str, default=None) + parser.add_argument("--model-dir", type=str, default=None) + parser.add_argument("--epochs", type=int, default=EPOCHS) + parser.add_argument("--batch-size", type=int, default=BATCH_SIZE) + parser.add_argument("--lr", type=float, default=LEARNING_RATE) + args = parser.parse_args() + + data_dir = Path(args.data_dir) if args.data_dir else TRAIN_DIR.parent + model_dir = Path(args.model_dir) if args.model_dir else MODEL_DIR + + print("="*60) + print("TAHAP 6: TRAINING ABJAD BISINDO") + print("="*60) + + train_abjad_bisindo(data_dir, model_dir, args.epochs, args.batch_size, args.lr) + + +if __name__ == "__main__": + main() diff --git a/backend/scripts/07_test_abjad_bisindo.py b/backend/scripts/07_test_abjad_bisindo.py new file mode 100644 index 0000000..cc2e966 --- /dev/null +++ b/backend/scripts/07_test_abjad_bisindo.py @@ -0,0 +1,154 @@ +""" +Tahap 7: Testing Abjad BISINDO + +Sesuai README. +""" + +import sys +import json +import numpy as np +from pathlib import Path +import argparse + +import torch +from sklearn.metrics import classification_report, confusion_matrix, accuracy_score, precision_recall_fscore_support +import matplotlib.pyplot as plt +import seaborn as sns + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from utils.config import TEST_DIR, MODEL_DIR, BATCH_SIZE, OUTPUT_DIR, CONFUSION_MATRIX_DIR, CLASSIFICATION_REPORT_DIR + + +class AbjadDataset: + def __init__(self, data_dir: Path): + self.data_dir = data_dir + self.samples = [] + self.labels = [] + self.label_to_idx = {} + self._load_data() + + def _load_data(self): + if not self.data_dir.exists(): + return + label_dirs = sorted([d for d in self.data_dir.iterdir() if d.is_dir()]) + self.label_to_idx = {d.name: idx for idx, d in enumerate(label_dirs)} + for label_dir in label_dirs: + label_idx = self.label_to_idx[label_dir.name] + for npy_file in label_dir.glob("*.npy"): + try: + self.samples.append(np.load(npy_file)) + self.labels.append(label_idx) + except: + pass + + def __len__(self): + return len(self.samples) + + +def plot_confusion_matrix(cm, labels, output_path, title): + plt.figure(figsize=(14, 12)) + sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', xticklabels=labels, yticklabels=labels) + plt.title(title) + plt.ylabel('True Label') + plt.xlabel('Predicted Label') + plt.tight_layout() + plt.savefig(output_path, dpi=150) + plt.close() + print(f"Confusion matrix saved to: {output_path}") + + +def test_abjad_bisindo(data_dir: Path, model_dir: Path, output_dir: Path): + device = 'cuda' if torch.cuda.is_available() else 'cpu' + print(f"Using device: {device}") + + model_path = model_dir / "abjad_bisindo" / "model_abjad_bisindo.pt" + encoder_path = model_dir / "abjad_bisindo" / "label_encoder.json" + + if not model_path.exists(): + print(f"Error: Model tidak ditemukan di {model_path}") + return + + print(f"Loading model from: {model_path}") + model = torch.load(model_path, map_location=device, weights_only=False) + model.eval() + + with open(encoder_path, 'r') as f: + idx_to_label = json.load(f)["idx_to_label"] + + print(f"Loading test dataset...") + dataset = AbjadDataset(data_dir / "test" / "abjad_bisindo") + print(f"Test samples: {len(dataset)}") + + if len(dataset) == 0: + print("No test samples found!") + return + + sequences = torch.FloatTensor(np.array(dataset.samples)) + true_labels = np.array(dataset.labels) + + print("Generating predictions...") + predictions = [] + with torch.no_grad(): + for i in range(0, len(sequences), BATCH_SIZE): + batch = sequences[i:i+BATCH_SIZE].to(device) + outputs = model(batch) + _, preds = torch.max(outputs, 1) + predictions.extend(preds.cpu().numpy().tolist()) + + accuracy = accuracy_score(true_labels, predictions) + precision, recall, f1, _ = precision_recall_fscore_support(true_labels, predictions, average='weighted', zero_division=0) + cm = confusion_matrix(true_labels, predictions) + + label_names = [idx_to_label[str(i)] for i in range(len(idx_to_label))] + report = classification_report(true_labels, predictions, target_names=label_names, zero_division=0) + + print("\n" + "="*60) + print("TESTING ABJAD BISINDO MODEL") + print("="*60) + print(f"\nAccuracy: {accuracy*100:.2f}%") + print(f"Precision: {precision*100:.2f}%") + print(f"Recall: {recall*100:.2f}%") + print(f"F1-Score: {f1*100:.2f}%") + print("\n" + report) + + plot_confusion_matrix(cm, label_names, CONFUSION_MATRIX_DIR / "abjad_bisindo_confusion_matrix.png", "Abjad BISINDO Confusion Matrix") + + with open(CLASSIFICATION_REPORT_DIR / "abjad_bisindo_classification_report.txt", 'w') as f: + f.write("Classification Report - Abjad BISINDO\n") + f.write("="*60 + "\n\n") + f.write(f"Accuracy: {accuracy*100:.2f}%\n") + f.write(f"Precision: {precision*100:.2f}%\n") + f.write(f"Recall: {recall*100:.2f}%\n") + f.write(f"F1-Score: {f1*100:.2f}%\n\n") + f.write(report) + + results = {"accuracy": float(accuracy), "precision": float(precision), "recall": float(recall), "f1_score": float(f1)} + with open(output_dir / "abjad_bisindo_test_results.json", 'w') as f: + json.dump(results, f, indent=2) + + print("\n" + "="*60) + print("TESTING COMPLETE") + print("="*60) + + +def main(): + parser = argparse.ArgumentParser(description="Testing model abjad BISINDO") + parser.add_argument("--data-dir", type=str, default=None) + parser.add_argument("--model-dir", type=str, default=None) + parser.add_argument("--output-dir", type=str, default=None) + args = parser.parse_args() + + data_dir = Path(args.data_dir) if args.data_dir else TEST_DIR.parent + model_dir = Path(args.model_dir) if args.model_dir else MODEL_DIR + output_dir = Path(args.output_dir) if args.output_dir else OUTPUT_DIR + + print("="*60) + print("TAHAP 7: TESTING ABJAD BISINDO") + print("="*60) + + test_abjad_bisindo(data_dir, model_dir, output_dir) + + +if __name__ == "__main__": + main() diff --git a/backend/scripts/08_train_abjad_sibi.py b/backend/scripts/08_train_abjad_sibi.py new file mode 100644 index 0000000..47b6fb0 --- /dev/null +++ b/backend/scripts/08_train_abjad_sibi.py @@ -0,0 +1,216 @@ +""" +Tahap 8: Training Abjad SIBI + +Script ini melatih model TCN untuk mengenali abjad SIBI. + +Sesuai README: +- Input: dataset/train/abjad_sibi/ +- Output: models/abjad_sibi/model_abjad_sibi.pt +""" + +import sys +import json +import numpy as np +from pathlib import Path +from typing import Tuple +import argparse + +import torch +import torch.nn as nn +import torch.optim as optim +from torch.utils.data import DataLoader, Dataset +from tqdm import tqdm +import matplotlib.pyplot as plt + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from utils.tcn import TCNClassifier +from utils.config import TRAIN_DIR, MODEL_DIR, FEATURE_SIZE, BATCH_SIZE, EPOCHS, LEARNING_RATE, ACCURACY_DIR, LOSS_DIR + + +class AbjadDataset(Dataset): + """Dataset untuk abjad.""" + + def __init__(self, data_dir: Path): + self.data_dir = data_dir + self.samples = [] + self.labels = [] + self.label_to_idx = {} + self._load_data() + + def _load_data(self): + if not self.data_dir.exists(): + return + label_dirs = sorted([d for d in self.data_dir.iterdir() if d.is_dir()]) + self.label_to_idx = {d.name: idx for idx, d in enumerate(label_dirs)} + for label_dir in label_dirs: + label_idx = self.label_to_idx[label_dir.name] + for npy_file in label_dir.glob("*.npy"): + try: + self.samples.append(np.load(npy_file)) + self.labels.append(label_idx) + except Exception as e: + print(f"Error loading {npy_file}: {e}") + + def __len__(self): + return len(self.samples) + + def __getitem__(self, idx) -> Tuple: + return torch.FloatTensor(self.samples[idx]), torch.LongTensor([self.labels[idx]]).squeeze() + + +def train_epoch(model, train_loader, criterion, optimizer, device): + model.train() + total_loss, correct, total = 0, 0, 0 + for sequences, labels in tqdm(train_loader, desc="Training"): + sequences, labels = sequences.to(device), labels.to(device) + optimizer.zero_grad() + outputs = model(sequences) + loss = criterion(outputs, labels) + loss.backward() + optimizer.step() + total_loss += loss.item() + _, predicted = torch.max(outputs, 1) + total += labels.size(0) + correct += (predicted == labels).sum().item() + return total_loss / len(train_loader), correct / total * 100 + + +def evaluate(model, eval_loader, criterion, device): + model.eval() + total_loss, correct, total = 0, 0, 0 + with torch.no_grad(): + for sequences, labels in eval_loader: + sequences, labels = sequences.to(device), labels.to(device) + outputs = model(sequences) + loss = criterion(outputs, labels) + total_loss += loss.item() + _, predicted = torch.max(outputs, 1) + total += labels.size(0) + correct += (predicted == labels).sum().item() + return total_loss / len(eval_loader), correct / total * 100 + + +def train_abjad_sibi(data_dir: Path, model_dir: Path, epochs: int = EPOCHS, + batch_size: int = BATCH_SIZE, lr: float = LEARNING_RATE): + device = 'cuda' if torch.cuda.is_available() else 'cpu' + print(f"Using device: {device}") + + print(f"\nLoading dataset from: {data_dir}") + train_dataset = AbjadDataset(data_dir / "train" / "abjad_sibi") + test_dataset = AbjadDataset(data_dir / "test" / "abjad_sibi") + + print(f"Train samples: {len(train_dataset)}, Test samples: {len(test_dataset)}") + + if len(train_dataset) == 0: + raise ValueError("No training samples found!") + + num_classes = len(train_dataset.label_to_idx) + print(f"Number of classes: {num_classes}") + print(f"Classes: {list(train_dataset.label_to_idx.keys())}") + + train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=0) + test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0) + + model = TCNClassifier(input_size=FEATURE_SIZE, num_classes=num_classes, num_channels=[64, 128, 256]).to(device) + + print(f"\nTotal parameters: {sum(p.numel() for p in model.parameters()):,}") + + criterion = nn.CrossEntropyLoss() + optimizer = optim.Adam(model.parameters(), lr=lr) + scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=5) + + history = {"train_loss": [], "train_acc": [], "test_loss": [], "test_acc": []} + best_acc = 0 + + print("\n" + "="*60) + print("Training Abjad SIBI Model") + print("="*60) + + for epoch in range(epochs): + print(f"\nEpoch {epoch+1}/{epochs}") + train_loss, train_acc = train_epoch(model, train_loader, criterion, optimizer, device) + test_loss, test_acc = evaluate(model, test_loader, criterion, device) + scheduler.step(test_loss) + history["train_loss"].append(train_loss) + history["train_acc"].append(train_acc) + history["test_loss"].append(test_loss) + history["test_acc"].append(test_acc) + print(f"Train Loss: {train_loss:.4f} | Train Acc: {train_acc:.2f}%") + print(f"Test Loss: {test_loss:.4f} | Test Acc: {test_acc:.2f}%") + if test_acc > best_acc: + best_acc = test_acc + best_state = model.state_dict().copy() + print(f"*** Best model! Acc: {best_acc:.2f}% ***") + + if best_state: + model.load_state_dict(best_state) + + # Save model + model_dir.mkdir(parents=True, exist_ok=True) + sibi_model_dir = model_dir / "abjad_sibi" + sibi_model_dir.mkdir(exist_ok=True) + model_path = sibi_model_dir / "model_abjad_sibi.pt" + torch.save(model, model_path) + print(f"\nModel saved to: {model_path}") + + # Save label encoder + idx_to_label = {v: k for k, v in train_dataset.label_to_idx.items()} + with open(sibi_model_dir / "label_encoder.json", 'w') as f: + json.dump({"label_to_idx": train_dataset.label_to_idx, "idx_to_label": idx_to_label}, f, indent=2) + + # Save history + with open(sibi_model_dir / "history.json", 'w') as f: + json.dump(history, f, indent=2) + + # Plot Accuracy + plt.figure(figsize=(10, 6)) + plt.plot(history['train_acc'], label='Train Accuracy') + plt.plot(history['test_acc'], label='Test (Validation) Accuracy') + plt.title('Abjad SIBI Model Accuracy') + plt.xlabel('Epoch') + plt.ylabel('Accuracy (%)') + plt.legend() + plt.grid(True) + plt.savefig(ACCURACY_DIR / "abjad_sibi_accuracy.png") + plt.close() + print(f"Accuracy plot saved to: {ACCURACY_DIR / 'abjad_sibi_accuracy.png'}") + + # Plot Loss + plt.figure(figsize=(10, 6)) + plt.plot(history['train_loss'], label='Train Loss') + plt.plot(history['test_loss'], label='Test (Validation) Loss') + plt.title('Abjad SIBI Model Loss') + plt.xlabel('Epoch') + plt.ylabel('Loss') + plt.legend() + plt.grid(True) + plt.savefig(LOSS_DIR / "abjad_sibi_loss.png") + plt.close() + print(f"Loss plot saved to: {LOSS_DIR / 'abjad_sibi_loss.png'}") + + print(f"\nBest Test Accuracy: {best_acc:.2f}%") + return history + + +def main(): + parser = argparse.ArgumentParser(description="Training model abjad SIBI") + parser.add_argument("--data-dir", type=str, default=None) + parser.add_argument("--model-dir", type=str, default=None) + parser.add_argument("--epochs", type=int, default=EPOCHS) + parser.add_argument("--batch-size", type=int, default=BATCH_SIZE) + parser.add_argument("--lr", type=float, default=LEARNING_RATE) + args = parser.parse_args() + + data_dir = Path(args.data_dir) if args.data_dir else TRAIN_DIR.parent + model_dir = Path(args.model_dir) if args.model_dir else MODEL_DIR + + print("="*60) + print("TAHAP 8: TRAINING ABJAD SIBI") + print("="*60) + + train_abjad_sibi(data_dir, model_dir, args.epochs, args.batch_size, args.lr) + + +if __name__ == "__main__": + main() diff --git a/backend/scripts/09_test_abjad_sibi.py b/backend/scripts/09_test_abjad_sibi.py new file mode 100644 index 0000000..dcae98a --- /dev/null +++ b/backend/scripts/09_test_abjad_sibi.py @@ -0,0 +1,154 @@ +""" +Tahap 9: Testing Abjad SIBI + +Sesuai README. +""" + +import sys +import json +import numpy as np +from pathlib import Path +import argparse + +import torch +from sklearn.metrics import classification_report, confusion_matrix, accuracy_score, precision_recall_fscore_support +import matplotlib.pyplot as plt +import seaborn as sns + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from utils.config import TEST_DIR, MODEL_DIR, BATCH_SIZE, OUTPUT_DIR, CONFUSION_MATRIX_DIR, CLASSIFICATION_REPORT_DIR + + +class AbjadDataset: + def __init__(self, data_dir: Path): + self.data_dir = data_dir + self.samples = [] + self.labels = [] + self.label_to_idx = {} + self._load_data() + + def _load_data(self): + if not self.data_dir.exists(): + return + label_dirs = sorted([d for d in self.data_dir.iterdir() if d.is_dir()]) + self.label_to_idx = {d.name: idx for idx, d in enumerate(label_dirs)} + for label_dir in label_dirs: + label_idx = self.label_to_idx[label_dir.name] + for npy_file in label_dir.glob("*.npy"): + try: + self.samples.append(np.load(npy_file)) + self.labels.append(label_idx) + except: + pass + + def __len__(self): + return len(self.samples) + + +def plot_confusion_matrix(cm, labels, output_path, title): + plt.figure(figsize=(14, 12)) + sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', xticklabels=labels, yticklabels=labels) + plt.title(title) + plt.ylabel('True Label') + plt.xlabel('Predicted Label') + plt.tight_layout() + plt.savefig(output_path, dpi=150) + plt.close() + print(f"Confusion matrix saved to: {output_path}") + + +def test_abjad_sibi(data_dir: Path, model_dir: Path, output_dir: Path): + device = 'cuda' if torch.cuda.is_available() else 'cpu' + print(f"Using device: {device}") + + model_path = model_dir / "abjad_sibi" / "model_abjad_sibi.pt" + encoder_path = model_dir / "abjad_sibi" / "label_encoder.json" + + if not model_path.exists(): + print(f"Error: Model tidak ditemukan di {model_path}") + return + + print(f"Loading model from: {model_path}") + model = torch.load(model_path, map_location=device, weights_only=False) + model.eval() + + with open(encoder_path, 'r') as f: + idx_to_label = json.load(f)["idx_to_label"] + + print(f"Loading test dataset...") + dataset = AbjadDataset(data_dir / "test" / "abjad_sibi") + print(f"Test samples: {len(dataset)}") + + if len(dataset) == 0: + print("No test samples found!") + return + + sequences = torch.FloatTensor(np.array(dataset.samples)) + true_labels = np.array(dataset.labels) + + print("Generating predictions...") + predictions = [] + with torch.no_grad(): + for i in range(0, len(sequences), BATCH_SIZE): + batch = sequences[i:i+BATCH_SIZE].to(device) + outputs = model(batch) + _, preds = torch.max(outputs, 1) + predictions.extend(preds.cpu().numpy().tolist()) + + accuracy = accuracy_score(true_labels, predictions) + precision, recall, f1, _ = precision_recall_fscore_support(true_labels, predictions, average='weighted', zero_division=0) + cm = confusion_matrix(true_labels, predictions) + + label_names = [idx_to_label[str(i)] for i in range(len(idx_to_label))] + report = classification_report(true_labels, predictions, target_names=label_names, zero_division=0) + + print("\n" + "="*60) + print("TESTING ABJAD SIBI MODEL") + print("="*60) + print(f"\nAccuracy: {accuracy*100:.2f}%") + print(f"Precision: {precision*100:.2f}%") + print(f"Recall: {recall*100:.2f}%") + print(f"F1-Score: {f1*100:.2f}%") + print("\n" + report) + + plot_confusion_matrix(cm, label_names, CONFUSION_MATRIX_DIR / "abjad_sibi_confusion_matrix.png", "Abjad SIBI Confusion Matrix") + + with open(CLASSIFICATION_REPORT_DIR / "abjad_sibi_classification_report.txt", 'w') as f: + f.write("Classification Report - Abjad SIBI\n") + f.write("="*60 + "\n\n") + f.write(f"Accuracy: {accuracy*100:.2f}%\n") + f.write(f"Precision: {precision*100:.2f}%\n") + f.write(f"Recall: {recall*100:.2f}%\n") + f.write(f"F1-Score: {f1*100:.2f}%\n\n") + f.write(report) + + results = {"accuracy": float(accuracy), "precision": float(precision), "recall": float(recall), "f1_score": float(f1)} + with open(output_dir / "abjad_sibi_test_results.json", 'w') as f: + json.dump(results, f, indent=2) + + print("\n" + "="*60) + print("TESTING COMPLETE") + print("="*60) + + +def main(): + parser = argparse.ArgumentParser(description="Testing model abjad SIBI") + parser.add_argument("--data-dir", type=str, default=None) + parser.add_argument("--model-dir", type=str, default=None) + parser.add_argument("--output-dir", type=str, default=None) + args = parser.parse_args() + + data_dir = Path(args.data_dir) if args.data_dir else TEST_DIR.parent + model_dir = Path(args.model_dir) if args.model_dir else MODEL_DIR + output_dir = Path(args.output_dir) if args.output_dir else OUTPUT_DIR + + print("="*60) + print("TAHAP 9: TESTING ABJAD SIBI") + print("="*60) + + test_abjad_sibi(data_dir, model_dir, output_dir) + + +if __name__ == "__main__": + main() diff --git a/backend/scripts/10_train_angka.py b/backend/scripts/10_train_angka.py new file mode 100644 index 0000000..8aef720 --- /dev/null +++ b/backend/scripts/10_train_angka.py @@ -0,0 +1,216 @@ +""" +Tahap 10: Training Angka + +Script ini melatih model TCN untuk mengenali angka dalam bahasa isyarat. + +Sesuai README: +- Input: dataset/train/angka/ +- Output: models/angka/model_angka.pt +""" + +import sys +import json +import numpy as np +from pathlib import Path +from typing import Tuple +import argparse + +import torch +import torch.nn as nn +import torch.optim as optim +from torch.utils.data import DataLoader, Dataset +from tqdm import tqdm +import matplotlib.pyplot as plt + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from utils.tcn import TCNClassifier +from utils.config import TRAIN_DIR, MODEL_DIR, FEATURE_SIZE, BATCH_SIZE, EPOCHS, LEARNING_RATE, ACCURACY_DIR, LOSS_DIR + + +class AngkaDataset(Dataset): + """Dataset untuk angka.""" + + def __init__(self, data_dir: Path): + self.data_dir = data_dir + self.samples = [] + self.labels = [] + self.label_to_idx = {} + self._load_data() + + def _load_data(self): + if not self.data_dir.exists(): + return + label_dirs = sorted([d for d in self.data_dir.iterdir() if d.is_dir()]) + self.label_to_idx = {d.name: idx for idx, d in enumerate(label_dirs)} + for label_dir in label_dirs: + label_idx = self.label_to_idx[label_dir.name] + for npy_file in label_dir.glob("*.npy"): + try: + self.samples.append(np.load(npy_file)) + self.labels.append(label_idx) + except Exception as e: + print(f"Error loading {npy_file}: {e}") + + def __len__(self): + return len(self.samples) + + def __getitem__(self, idx) -> Tuple: + return torch.FloatTensor(self.samples[idx]), torch.LongTensor([self.labels[idx]]).squeeze() + + +def train_epoch(model, train_loader, criterion, optimizer, device): + model.train() + total_loss, correct, total = 0, 0, 0 + for sequences, labels in tqdm(train_loader, desc="Training"): + sequences, labels = sequences.to(device), labels.to(device) + optimizer.zero_grad() + outputs = model(sequences) + loss = criterion(outputs, labels) + loss.backward() + optimizer.step() + total_loss += loss.item() + _, predicted = torch.max(outputs, 1) + total += labels.size(0) + correct += (predicted == labels).sum().item() + return total_loss / len(train_loader), correct / total * 100 + + +def evaluate(model, eval_loader, criterion, device): + model.eval() + total_loss, correct, total = 0, 0, 0 + with torch.no_grad(): + for sequences, labels in eval_loader: + sequences, labels = sequences.to(device), labels.to(device) + outputs = model(sequences) + loss = criterion(outputs, labels) + total_loss += loss.item() + _, predicted = torch.max(outputs, 1) + total += labels.size(0) + correct += (predicted == labels).sum().item() + return total_loss / len(eval_loader), correct / total * 100 + + +def train_angka(data_dir: Path, model_dir: Path, epochs: int = EPOCHS, + batch_size: int = BATCH_SIZE, lr: float = LEARNING_RATE): + device = 'cuda' if torch.cuda.is_available() else 'cpu' + print(f"Using device: {device}") + + print(f"\nLoading dataset from: {data_dir}") + train_dataset = AngkaDataset(data_dir / "train" / "angka") + test_dataset = AngkaDataset(data_dir / "test" / "angka") + + print(f"Train samples: {len(train_dataset)}, Test samples: {len(test_dataset)}") + + if len(train_dataset) == 0: + raise ValueError("No training samples found!") + + num_classes = len(train_dataset.label_to_idx) + print(f"Number of classes: {num_classes}") + print(f"Classes: {list(train_dataset.label_to_idx.keys())}") + + train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=0) + test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0) + + model = TCNClassifier(input_size=FEATURE_SIZE, num_classes=num_classes, num_channels=[64, 128, 256]).to(device) + + print(f"\nTotal parameters: {sum(p.numel() for p in model.parameters()):,}") + + criterion = nn.CrossEntropyLoss() + optimizer = optim.Adam(model.parameters(), lr=lr) + scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=5) + + history = {"train_loss": [], "train_acc": [], "test_loss": [], "test_acc": []} + best_acc = 0 + + print("\n" + "="*60) + print("Training Angka Model") + print("="*60) + + for epoch in range(epochs): + print(f"\nEpoch {epoch+1}/{epochs}") + train_loss, train_acc = train_epoch(model, train_loader, criterion, optimizer, device) + test_loss, test_acc = evaluate(model, test_loader, criterion, device) + scheduler.step(test_loss) + history["train_loss"].append(train_loss) + history["train_acc"].append(train_acc) + history["test_loss"].append(test_loss) + history["test_acc"].append(test_acc) + print(f"Train Loss: {train_loss:.4f} | Train Acc: {train_acc:.2f}%") + print(f"Test Loss: {test_loss:.4f} | Test Acc: {test_acc:.2f}%") + if test_acc > best_acc: + best_acc = test_acc + best_state = model.state_dict().copy() + print(f"*** Best model! Acc: {best_acc:.2f}% ***") + + if best_state: + model.load_state_dict(best_state) + + # Save model + model_dir.mkdir(parents=True, exist_ok=True) + angka_model_dir = model_dir / "angka" + angka_model_dir.mkdir(exist_ok=True) + model_path = angka_model_dir / "model_angka.pt" + torch.save(model, model_path) + print(f"\nModel saved to: {model_path}") + + # Save label encoder + idx_to_label = {v: k for k, v in train_dataset.label_to_idx.items()} + with open(angka_model_dir / "label_encoder.json", 'w') as f: + json.dump({"label_to_idx": train_dataset.label_to_idx, "idx_to_label": idx_to_label}, f, indent=2) + + # Save history + with open(angka_model_dir / "history.json", 'w') as f: + json.dump(history, f, indent=2) + + # Plot Accuracy + plt.figure(figsize=(10, 6)) + plt.plot(history['train_acc'], label='Train Accuracy') + plt.plot(history['test_acc'], label='Test (Validation) Accuracy') + plt.title('Angka Model Accuracy') + plt.xlabel('Epoch') + plt.ylabel('Accuracy (%)') + plt.legend() + plt.grid(True) + plt.savefig(ACCURACY_DIR / "angka_accuracy.png") + plt.close() + print(f"Accuracy plot saved to: {ACCURACY_DIR / 'angka_accuracy.png'}") + + # Plot Loss + plt.figure(figsize=(10, 6)) + plt.plot(history['train_loss'], label='Train Loss') + plt.plot(history['test_loss'], label='Test (Validation) Loss') + plt.title('Angka Model Loss') + plt.xlabel('Epoch') + plt.ylabel('Loss') + plt.legend() + plt.grid(True) + plt.savefig(LOSS_DIR / "angka_loss.png") + plt.close() + print(f"Loss plot saved to: {LOSS_DIR / 'angka_loss.png'}") + + print(f"\nBest Test Accuracy: {best_acc:.2f}%") + return history + + +def main(): + parser = argparse.ArgumentParser(description="Training model angka") + parser.add_argument("--data-dir", type=str, default=None) + parser.add_argument("--model-dir", type=str, default=None) + parser.add_argument("--epochs", type=int, default=EPOCHS) + parser.add_argument("--batch-size", type=int, default=BATCH_SIZE) + parser.add_argument("--lr", type=float, default=LEARNING_RATE) + args = parser.parse_args() + + data_dir = Path(args.data_dir) if args.data_dir else TRAIN_DIR.parent + model_dir = Path(args.model_dir) if args.model_dir else MODEL_DIR + + print("="*60) + print("TAHAP 10: TRAINING ANGKA") + print("="*60) + + train_angka(data_dir, model_dir, args.epochs, args.batch_size, args.lr) + + +if __name__ == "__main__": + main() diff --git a/backend/scripts/11_test_angka.py b/backend/scripts/11_test_angka.py new file mode 100644 index 0000000..90d7bd9 --- /dev/null +++ b/backend/scripts/11_test_angka.py @@ -0,0 +1,160 @@ +""" +Tahap 11: Testing Angka + +Sesuai README: +- Output: + - Accuracy + - Precision + - Recall + - F1 Score + - Confusion Matrix +""" + +import sys +import json +import numpy as np +from pathlib import Path +import argparse + +import torch +from sklearn.metrics import classification_report, confusion_matrix, accuracy_score, precision_recall_fscore_support +import matplotlib.pyplot as plt +import seaborn as sns + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from utils.config import TEST_DIR, MODEL_DIR, BATCH_SIZE, OUTPUT_DIR, CONFUSION_MATRIX_DIR, CLASSIFICATION_REPORT_DIR + + +class AngkaDataset: + def __init__(self, data_dir: Path): + self.data_dir = data_dir + self.samples = [] + self.labels = [] + self.label_to_idx = {} + self._load_data() + + def _load_data(self): + if not self.data_dir.exists(): + return + label_dirs = sorted([d for d in self.data_dir.iterdir() if d.is_dir()]) + self.label_to_idx = {d.name: idx for idx, d in enumerate(label_dirs)} + for label_dir in label_dirs: + label_idx = self.label_to_idx[label_dir.name] + for npy_file in label_dir.glob("*.npy"): + try: + self.samples.append(np.load(npy_file)) + self.labels.append(label_idx) + except: + pass + + def __len__(self): + return len(self.samples) + + +def plot_confusion_matrix(cm, labels, output_path, title): + plt.figure(figsize=(12, 10)) + sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', xticklabels=labels, yticklabels=labels) + plt.title(title) + plt.ylabel('True Label') + plt.xlabel('Predicted Label') + plt.tight_layout() + plt.savefig(output_path, dpi=150) + plt.close() + print(f"Confusion matrix saved to: {output_path}") + + +def test_angka(data_dir: Path, model_dir: Path, output_dir: Path): + device = 'cuda' if torch.cuda.is_available() else 'cpu' + print(f"Using device: {device}") + + model_path = model_dir / "angka" / "model_angka.pt" + encoder_path = model_dir / "angka" / "label_encoder.json" + + if not model_path.exists(): + print(f"Error: Model tidak ditemukan di {model_path}") + return + + print(f"Loading model from: {model_path}") + model = torch.load(model_path, map_location=device, weights_only=False) + model.eval() + + with open(encoder_path, 'r') as f: + idx_to_label = json.load(f)["idx_to_label"] + + print(f"Loading test dataset...") + dataset = AngkaDataset(data_dir / "test" / "angka") + print(f"Test samples: {len(dataset)}") + + if len(dataset) == 0: + print("No test samples found!") + return + + sequences = torch.FloatTensor(np.array(dataset.samples)) + true_labels = np.array(dataset.labels) + + print("Generating predictions...") + predictions = [] + with torch.no_grad(): + for i in range(0, len(sequences), BATCH_SIZE): + batch = sequences[i:i+BATCH_SIZE].to(device) + outputs = model(batch) + _, preds = torch.max(outputs, 1) + predictions.extend(preds.cpu().numpy().tolist()) + + accuracy = accuracy_score(true_labels, predictions) + precision, recall, f1, _ = precision_recall_fscore_support(true_labels, predictions, average='weighted', zero_division=0) + cm = confusion_matrix(true_labels, predictions) + + label_names = [idx_to_label[str(i)] for i in range(len(idx_to_label))] + report = classification_report(true_labels, predictions, target_names=label_names, zero_division=0) + + print("\n" + "="*60) + print("TESTING ANGKA MODEL") + print("="*60) + print(f"\nAccuracy: {accuracy*100:.2f}%") + print(f"Precision: {precision*100:.2f}%") + print(f"Recall: {recall*100:.2f}%") + print(f"F1-Score: {f1*100:.2f}%") + print("\n" + report) + + plot_confusion_matrix(cm, label_names, CONFUSION_MATRIX_DIR / "angka_confusion_matrix.png", "Angka Confusion Matrix") + + with open(CLASSIFICATION_REPORT_DIR / "angka_classification_report.txt", 'w') as f: + f.write("Classification Report - Angka\n") + f.write("="*60 + "\n\n") + f.write(f"Accuracy: {accuracy*100:.2f}%\n") + f.write(f"Precision: {precision*100:.2f}%\n") + f.write(f"Recall: {recall*100:.2f}%\n") + f.write(f"F1-Score: {f1*100:.2f}%\n\n") + f.write(report) + + results = {"accuracy": float(accuracy), "precision": float(precision), "recall": float(recall), "f1_score": float(f1)} + with open(output_dir / "angka_test_results.json", 'w') as f: + json.dump(results, f, indent=2) + + print("\n" + "="*60) + print("TESTING COMPLETE") + print("="*60) + + +def main(): + parser = argparse.ArgumentParser(description="Testing model angka") + parser.add_argument("--data-dir", type=str, default=None) + parser.add_argument("--model-dir", type=str, default=None) + parser.add_argument("--output-dir", type=str, default=None) + args = parser.parse_args() + + data_dir = Path(args.data_dir) if args.data_dir else TEST_DIR.parent + model_dir = Path(args.model_dir) if args.model_dir else MODEL_DIR + output_dir = Path(args.output_dir) if args.output_dir else OUTPUT_DIR + + print("="*60) + print("TAHAP 11: TESTING ANGKA") + print("="*60) + + test_angka(data_dir, model_dir, output_dir) + + +if __name__ == "__main__": + main() diff --git a/backend/scripts/plot_history.py b/backend/scripts/plot_history.py new file mode 100644 index 0000000..8b2d4f3 --- /dev/null +++ b/backend/scripts/plot_history.py @@ -0,0 +1,54 @@ +import sys +import json +from pathlib import Path +import matplotlib.pyplot as plt + +sys.path.insert(0, str(Path(__file__).parent.parent)) +from utils.config import MODEL_DIR, ACCURACY_DIR, LOSS_DIR + +CATEGORIES = ["kata", "abjad_bisindo", "abjad_sibi", "angka"] + +def main(): + for category in CATEGORIES: + history_path = MODEL_DIR / category / "history.json" + + if not history_path.exists(): + print(f"History not found for {category} at {history_path}") + continue + + with open(history_path, 'r') as f: + history = json.load(f) + + # Plot Accuracy + plt.figure(figsize=(10, 6)) + plt.plot(history['train_acc'], label='Train Accuracy') + plt.plot(history['test_acc'], label='Test (Validation) Accuracy') + + title_name = category.replace("_", " ").title() + + plt.title(f'{title_name} Model Accuracy') + plt.xlabel('Epoch') + plt.ylabel('Accuracy (%)') + plt.legend() + plt.grid(True) + acc_path = ACCURACY_DIR / f"{category}_accuracy.png" + plt.savefig(acc_path) + plt.close() + print(f"[{category}] Accuracy plot saved to: {acc_path}") + + # Plot Loss + plt.figure(figsize=(10, 6)) + plt.plot(history['train_loss'], label='Train Loss') + plt.plot(history['test_loss'], label='Test (Validation) Loss') + plt.title(f'{title_name} Model Loss') + plt.xlabel('Epoch') + plt.ylabel('Loss') + plt.legend() + plt.grid(True) + loss_path = LOSS_DIR / f"{category}_loss.png" + plt.savefig(loss_path) + plt.close() + print(f"[{category}] Loss plot saved to: {loss_path}") + +if __name__ == "__main__": + main() diff --git a/backend/utils/config.py b/backend/utils/config.py new file mode 100644 index 0000000..0210c00 --- /dev/null +++ b/backend/utils/config.py @@ -0,0 +1,98 @@ +""" +Konfigurasi untuk project penerjemah bahasa isyarat Indonesia. + +Berisi konfigurasi project sesuai README.md. +""" + +import random +import numpy as np +import torch +from pathlib import Path + +# Random seed for reproducibility +RANDOM_SEED = 42 + +# Set random seeds +random.seed(RANDOM_SEED) +np.random.seed(RANDOM_SEED) +torch.manual_seed(RANDOM_SEED) +if torch.cuda.is_available(): + torch.cuda.manual_seed_all(RANDOM_SEED) + +# Base paths +BASE_DIR = Path(__file__).parent.parent +DATASET_DIR = BASE_DIR / "dataset" + +# Dataset paths +RAW_DIR = DATASET_DIR / "raw" +LANDMARK_DIR = DATASET_DIR / "landmark" +NORMALIZED_DIR = DATASET_DIR / "normalized" +TRAIN_DIR = DATASET_DIR / "train" +TEST_DIR = DATASET_DIR / "test" + +# Model paths (sesuai README: models/{category}/) +MODEL_DIR = BASE_DIR / "models" + +# Output paths +OUTPUT_DIR = BASE_DIR / "outputs" +CONFUSION_MATRIX_DIR = OUTPUT_DIR / "confusion_matrix" +CLASSIFICATION_REPORT_DIR = OUTPUT_DIR / "classification_report" +ACCURACY_DIR = OUTPUT_DIR / "accuracy" +LOSS_DIR = OUTPUT_DIR / "loss" + +# Create directories +for dir_path in [MODEL_DIR, CONFUSION_MATRIX_DIR, CLASSIFICATION_REPORT_DIR, ACCURACY_DIR, LOSS_DIR]: + dir_path.mkdir(parents=True, exist_ok=True) + +# Categories (urutan sesuai README) +CATEGORIES = ["abjad_bisindo", "abjad_sibi", "angka", "kata"] + +# Sliding window configuration +# Sesuai README: 30 Frame dan 5 Stride +WINDOW_SIZE = 30 +STRIDE = 5 + +# Landmark configuration +NUM_HAND_LANDMARKS = 21 # MediaPipe hand landmarks +NUM_FACE_LANDMARKS = 60 # Selected face mesh landmarks + +# Feature size calculation: +# - Left hand: 21 landmarks x 3 coordinates = 63 +# - Right hand: 21 landmarks x 3 coordinates = 63 +# - Face mesh: 60 landmarks x 3 coordinates = 180 +# Total: 63 + 63 + 180 = 306 +FEATURE_SIZE = 306 + +# Training configuration +BATCH_SIZE = 32 +EPOCHS = 100 +LEARNING_RATE = 0.001 + +# Train/test split ratio (sesuai README: 80% / 20%) +TRAIN_RATIO = 0.8 +TEST_RATIO = 0.2 + +# Video extensions (sesuai README) +VIDEO_EXTENSIONS = [".mp4"] + +# Face mesh landmark indices (60 landmarks dari dagu, mulut, bibir, hidung, mata, pipi) +FACE_LANDMARK_INDICES = [ + # Outer face outline (dagu) + 152, 234, 454, 358, 465, 452, 253, 389, + # Eyebrows + 336, 296, 334, 293, 300, 276, 283, 282, + # Eyes (mata) + 33, 160, 158, 133, 153, 144, 362, 385, 387, 263, 373, 249, + # Nose (hidung) + 1, 2, 98, 327, 4, 75, 221, 189, 417, 62, 305, 293, + # Mouth outer (mulut luar) + 13, 14, 17, 84, 185, 61, 291, 0, + # Mouth inner (bibir) + 78, 95, 88, 178, 87, 317, 402, 310, + # Cheeks (pipi) + 50, 187, 411, 93, 323, 234, 454, 36, + # Additional key points + 168, 6, 195, 197, 199, 175, 246, 123, + # Forehead + 10, 151, 9, 107 +] diff --git a/backend/utils/dataset.py b/backend/utils/dataset.py new file mode 100644 index 0000000..4b57511 --- /dev/null +++ b/backend/utils/dataset.py @@ -0,0 +1,127 @@ +""" +Dataset utilities untuk loading dan preprocessing. + +Membaca dataset .npy menjadi Dataset PyTorch sesuai README. +""" + +import numpy as np +import torch +from torch.utils.data import Dataset +from pathlib import Path +from typing import Optional, List, Tuple + + +class LandmarkDataset(Dataset): + """ + PyTorch Dataset untuk landmark sequences. + + Sesuai README: + - Input shape: (30, feature_size) = (30, 306) + """ + + def __init__( + self, + data_dir: Path, + category: str, + split: str = "train" + ): + """ + Inisialisasi dataset. + + Args: + data_dir: Root directory (dataset/train/ atau dataset/test/) + category: Nama kategori + split: 'train' atau 'test' + """ + # Path sesuai README: dataset/{train|test}/{category}/{label}/ + self.data_dir = data_dir / split / category + self.samples = [] + self.labels = [] + self.label_to_idx = {} + + self._load_data() + + def _load_data(self): + """Load semua data.""" + if not self.data_dir.exists(): + return + + # Build label mapping dari subdirectories + label_dirs = sorted([d for d in self.data_dir.iterdir() if d.is_dir()]) + self.label_to_idx = {d.name: idx for idx, d in enumerate(label_dirs)} + + # Load samples + for label_dir in label_dirs: + label_idx = self.label_to_idx[label_dir.name] + + for npy_file in label_dir.glob("*.npy"): + try: + sequence = np.load(npy_file) + self.samples.append(sequence) + self.labels.append(label_idx) + except Exception as e: + print(f"Error loading {npy_file}: {e}") + + def __len__(self) -> int: + return len(self.samples) + + def __getitem__(self, idx: int) -> Tuple[torch.Tensor, int]: + """ + Get satu sample. + + Returns: + Tuple (features, label) + """ + sequence = self.samples[idx] + label = self.labels[idx] + + return torch.FloatTensor(sequence), torch.LongTensor([label]).squeeze() + + +def load_landmark_sequence(sequence_path: Path) -> np.ndarray: + """ + Load landmark sequence dari .npy file. + + Args: + sequence_path: Path ke file .npy + + Returns: + Array shape (num_frames, feature_size) + """ + return np.load(sequence_path) + + +def get_category_labels(category: str, data_dir: Path, split: str = "train") -> List[str]: + """ + Get semua label untuk kategori. + + Args: + category: Nama kategori + data_dir: Root data directory + split: 'train' atau 'test' + + Returns: + List of label names + """ + split_dir = data_dir / split / category + + if not split_dir.exists(): + return [] + + labels = sorted([d.name for d in split_dir.iterdir() if d.is_dir()]) + return labels + + +def get_num_classes(category: str, data_dir: Path, split: str = "train") -> int: + """ + Get jumlah kelas untuk kategori. + + Args: + category: Nama kategori + data_dir: Root data directory + split: 'train' atau 'test' + + Returns: + Jumlah kelas + """ + return len(get_category_labels(category, data_dir, split)) diff --git a/backend/utils/inference.py b/backend/utils/inference.py new file mode 100644 index 0000000..7481ff2 --- /dev/null +++ b/backend/utils/inference.py @@ -0,0 +1,246 @@ +""" +Inference utilities untuk real-time gesture recognition. + +Melakukan prediksi menggunakan model TCN sesuai README. +""" + +import cv2 +import json +import mediapipe as mp +import numpy as np +import torch +from pathlib import Path +from typing import Optional, Tuple + +from utils.mediapipe_detector import MediaPipeDetector +from utils.normalization import LandmarkNormalizer +from utils.config import WINDOW_SIZE + + +# MediaPipe Drawing utils +mp_drawing = mp.solutions.drawing_utils +mp_hands = mp.solutions.hands +mp_face_mesh = mp.solutions.face_mesh + + +class GestureRecognizer: + """ + Real-time gesture recognizer untuk inferensi. + + Proses sesuai README: + 1. Webcam + 2. MediaPipe + 3. Landmark + 4. Normalisasi + 5. Window 30 Frame + 6. Model TCN + 7. Prediksi + 8. Tampilkan hasil + """ + + def __init__( + self, + model_path: Path, + label_encoder_path: Path, + window_size: int = WINDOW_SIZE, + smoothing_size: int = 5, + draw_landmarks: bool = True + ): + """ + Inisialisasi recognizer. + + Args: + model_path: Path ke model .pt + label_encoder_path: Path ke label encoder JSON + window_size: Window size untuk sequence (30) + smoothing_size: Buffer size untuk temporal smoothing + draw_landmarks: Apakah perlu menggambar landmarks + """ + self.window_size = window_size + self.smoothing_size = smoothing_size + self.draw_landmarks = draw_landmarks + + # Load model + self.device = 'cuda' if torch.cuda.is_available() else 'cpu' + self.model = torch.load(model_path, map_location=self.device, weights_only=False) + self.model.eval() + + # Load label encoder + with open(label_encoder_path, 'r') as f: + encoder_data = json.load(f) + self.idx_to_label = encoder_data["idx_to_label"] + + # Initialize detector dan normalizer + self.detector = MediaPipeDetector() + self.normalizer = LandmarkNormalizer() + + # Buffer untuk temporal smoothing + self.buffer = [] + self.prob_buffer = [] + + # MediaPipe drawing specs + self.hand_landmark_spec = mp_drawing.DrawingSpec(color=(0, 255, 0), thickness=2, circle_radius=3) + self.hand_connection_spec = mp_drawing.DrawingSpec(color=(0, 255, 0), thickness=2) + self.face_landmark_spec = mp_drawing.DrawingSpec(color=(255, 255, 0), thickness=1, circle_radius=1) + + def process_frame(self, frame) -> Tuple[Optional[str], float]: + """ + Process satu frame dan return prediction. + + Args: + frame: Frame gambar (BGR) + + Returns: + Tuple (predicted_label, confidence) + """ + # Extract landmarks + left_hand, right_hand, face = self.detector.detect(frame) + + # Check jika ada gerakan (minimal satu tangan terdeteksi) + left_detected = np.any(left_hand != 0) + right_detected = np.any(right_hand != 0) + + # Jika tidak ada tangan terdeteksi beberapa frame berturut-turut, reset buffer + if not (left_detected or right_detected): + self.no_hand_count = getattr(self, 'no_hand_count', 0) + 1 + if self.no_hand_count >= 5: + self.buffer = [] + self.prob_buffer = [] + return None, 0.0 + + self.no_hand_count = 0 + + # Normalize + features = self.normalizer.normalize_frame(left_hand, right_hand, face) + + # Add ke buffer + self.buffer.append(features) + + # Keep buffer size + if len(self.buffer) > self.window_size: + self.buffer.pop(0) + + # Jika buffer belum penuh, return None + if len(self.buffer) < self.window_size: + return None, 0.0 + + # Prepare input + sequence = np.array([self.buffer]) + sequence_tensor = torch.FloatTensor(sequence).to(self.device) + + # Predict + with torch.no_grad(): + outputs = self.model(sequence_tensor) + probabilities = torch.softmax(outputs, dim=1) + + # Store probabilities for smoothing + self.prob_buffer.append(probabilities.cpu().numpy()[0]) + if len(self.prob_buffer) > self.smoothing_size: + self.prob_buffer.pop(0) + + # Average probabilities + avg_probs = np.mean(self.prob_buffer, axis=0) + predicted_idx = np.argmax(avg_probs) + confidence = avg_probs[predicted_idx] + + # Map ke label + label = self.idx_to_label[str(predicted_idx)] + + return label, confidence + + def draw_landmarks_on_frame(self, frame): + """ + Gambar hand dan face landmarks pada frame. + + Args: + frame: Frame gambar (BGR) + + Returns: + Frame dengan landmark yang sudah digambar + """ + rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + + # Detect hands + hands_results = self.detector.hands.process(rgb_frame) + + # Detect face + face_results = self.detector.face_mesh.process(rgb_frame) + + # Draw hands + if hands_results.multi_hand_landmarks: + for hand_landmarks in hands_results.multi_hand_landmarks: + mp_drawing.draw_landmarks( + frame, + hand_landmarks, + mp_hands.HAND_CONNECTIONS, + self.hand_landmark_spec, + self.hand_connection_spec + ) + + # Draw face mesh + if face_results.multi_face_landmarks: + for face_landmarks in face_results.multi_face_landmarks: + mp_drawing.draw_landmarks( + frame, + face_landmarks, + mp_face_mesh.FACEMESH_TESSELATION, + self.face_landmark_spec, + self.face_landmark_spec + ) + + return frame + + def reset(self): + """Reset buffers.""" + self.buffer = [] + self.prob_buffer = [] + + def close(self): + """Cleanup resources.""" + self.detector.close() + + +def load_model(model_path: Path, device: str = None): + """ + Load model dari file. + + Args: + model_path: Path ke model file + device: Target device + + Returns: + Loaded model + """ + if device is None: + device = 'cuda' if torch.cuda.is_available() else 'cpu' + + model = torch.load(model_path, map_location=device, weights_only=False) + model.eval() + + return model + + +def load_label_encoder(json_path: Path) -> dict: + """ + Load label encoder dari JSON file. + + Args: + json_path: Path ke label encoder JSON + + Returns: + Label encoder dictionary + """ + with open(json_path, 'r') as f: + return json.load(f) + + +def save_label_encoder(label_encoder: dict, json_path: Path): + """ + Save label encoder ke JSON file. + + Args: + label_encoder: Label encoder dictionary + json_path: Output path + """ + with open(json_path, 'w') as f: + json.dump(label_encoder, f, indent=2) \ No newline at end of file diff --git a/backend/utils/mediapipe_detector.py b/backend/utils/mediapipe_detector.py new file mode 100644 index 0000000..ef17274 --- /dev/null +++ b/backend/utils/mediapipe_detector.py @@ -0,0 +1,153 @@ +""" +MediaPipe detector untuk mengambil landmark tangan dan wajah. + +Mengambil landmark tangan dan wajah menggunakan MediaPipe Hands dan Face Mesh. +Feature size: 21 (left hand) + 21 (right hand) + 60 (face) = 306 +""" + +import cv2 +import mediapipe as mp +import numpy as np +from typing import Tuple + + +class MediaPipeDetector: + """ + Kelas untuk mendeteksi landmark tangan dan wajah menggunakan MediaPipe. + + Feature breakdown: + - Left hand: 21 landmarks x 3 coords = 63 + - Right hand: 21 landmarks x 3 coords = 63 + - Face: 60 landmarks x 3 coords = 180 + - Total: 306 + """ + + def __init__( + self, + static_image_mode: bool = False, + max_num_hands: int = 2, + max_num_faces: int = 1, + min_detection_confidence: float = 0.5, + min_tracking_confidence: float = 0.5 + ): + """ + Inisialisasi MediaPipe detector. + """ + self.hands = mp.solutions.hands.Hands( + static_image_mode=static_image_mode, + max_num_hands=max_num_hands, + min_detection_confidence=min_detection_confidence, + min_tracking_confidence=min_tracking_confidence + ) + + self.face_mesh = mp.solutions.face_mesh.FaceMesh( + static_image_mode=static_image_mode, + max_num_faces=max_num_faces, + min_detection_confidence=min_detection_confidence, + min_tracking_confidence=min_tracking_confidence, + refine_landmarks=True + ) + + # 60 Face landmarks indices dari: dagu, mulut, bibir, hidung, mata, pipi + # Ini adalah indeks dari MediaPipe Face Mesh 468 landmarks + self.face_indices = [ + # Dagu (Chin) - 8 landmarks + 152, 234, 454, 358, 465, 452, 253, 389, + # Alis (Eyebrows) - 8 landmarks + 336, 296, 334, 293, 300, 276, 283, 282, + # Mata (Eyes) - 12 landmarks + 33, 160, 158, 133, 153, 144, 362, 385, 387, 263, 373, 249, + # Hidung (Nose) - 12 landmarks + 1, 2, 98, 327, 4, 75, 221, 189, 417, 62, 305, 293, + # Mulut luar (Mouth outer) - 8 landmarks + 13, 14, 17, 84, 185, 61, 291, 0, + # Bibir (Mouth inner/Lips) - 8 landmarks + 78, 95, 88, 178, 87, 317, 402, 310, + # Pipi (Cheeks) - 4 landmarks + 50, 187, 411, 93 + ] + + def detect_hands(self, frame: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: + """ + Deteksi landmark tangan kiri dan kanan. + + Args: + frame: Frame gambar (BGR) + + Returns: + Tuple dari (left_hand_landmarks, right_hand_landmarks) + Setiap landmark adalah array shape (21, 3) + """ + rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + results = self.hands.process(rgb_frame) + + left_hand = np.zeros((21, 3)) + right_hand = np.zeros((21, 3)) + + if results.multi_hand_landmarks: + for hand_landmarks, handedness in zip( + results.multi_hand_landmarks, + results.multi_handedness + ): + hand_type = handedness.classification[0].label + landmarks = np.array([ + [lm.x, lm.y, lm.z] + for lm in hand_landmarks.landmark + ]) + + if hand_type == "Left": + left_hand = landmarks + else: + right_hand = landmarks + + return left_hand, right_hand + + def detect_face(self, frame: np.ndarray) -> np.ndarray: + """ + Deteksi 60 landmark wajah dari dagu, mulut, bibir, hidung, mata, pipi. + + Args: + frame: Frame gambar (BGR) + + Returns: + Array landmark wajah shape (60, 3) + """ + rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + results = self.face_mesh.process(rgb_frame) + + # Default: zeros + face = np.zeros((60, 3)) + + if results.multi_face_landmarks: + face_landmarks = results.multi_face_landmarks[0] + face = np.array([ + [face_landmarks.landmark[idx].x, + face_landmarks.landmark[idx].y, + face_landmarks.landmark[idx].z] + for idx in self.face_indices + ]) + + return face + + def detect(self, frame: np.ndarray) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: + """ + Deteksi semua landmark (kedua tangan dan wajah). + + Args: + frame: Frame gambar (BGR) + + Returns: + Tuple (left_hand, right_hand, face) + - left_hand: array shape (21, 3) + - right_hand: array shape (21, 3) + - face: array shape (60, 3) + """ + left_hand, right_hand = self.detect_hands(frame) + face = self.detect_face(frame) + + return left_hand, right_hand, face + + def close(self): + """Tutup semua MediaPipe resources.""" + self.hands.close() + self.face_mesh.close() diff --git a/backend/utils/normalization.py b/backend/utils/normalization.py new file mode 100644 index 0000000..239fce2 --- /dev/null +++ b/backend/utils/normalization.py @@ -0,0 +1,140 @@ +""" +Normalisasi landmark untuk konsistensi across different positions dan scales. + +Melakukan normalisasi koordinat landmark sesuai README. +""" + +import numpy as np +from typing import Optional + + +class LandmarkNormalizer: + """ + Kelas untuk menormalisasi landmark tangan dan wajah. + + Proses normalisasi sesuai README: + 1. Menggeser koordinat ke titik referensi + 2. Scaling + """ + + def __init__(self, scale_factor: Optional[float] = None): + """ + Inisialisasi normalizer. + + Args: + scale_factor: Faktor skala tetap. Jika None, dihitung dari data. + """ + self.scale_factor = scale_factor + + def normalize_hand(self, landmarks: np.ndarray) -> np.ndarray: + """ + Normalisasi landmark satu tangan. + + Args: + landmarks: Array shape (21, 3) dari landmark tangan + + Returns: + Array landmark yang sudah dinormalisasi + """ + if landmarks is None or landmarks.shape != (21, 3): + return np.zeros((21, 3)) + + # Buat copy + normalized = landmarks.copy() + + # 1. Translation: Geser ke wrist (origin) - index 0 + wrist = normalized[0] + normalized -= wrist + + # 2. Scaling: Normalisasi berdasarkan distance dari wrist ke middle finger MCP (index 9) + middle_mcp = normalized[9] + distance = np.linalg.norm(middle_mcp) + + if distance > 1e-6: # Avoid division by zero + normalized = normalized / distance + + if self.scale_factor is not None: + normalized *= self.scale_factor + + return normalized + + def normalize_face(self, landmarks: np.ndarray) -> np.ndarray: + """ + Normalisasi landmark wajah. + + Args: + landmarks: Array shape (60, 3) dari landmark wajah + + Returns: + Array landmark yang sudah dinormalisasi + """ + if landmarks is None or len(landmarks) == 0: + return np.zeros((60, 3)) + + # Buat copy + normalized = landmarks.copy() + + # 1. Translation: Geser ke nose (center) + nose = normalized[0] + normalized -= nose + + # 2. Scaling: Normalisasi berdasarkan max distance from center + max_distance = np.max(np.linalg.norm(normalized, axis=1)) + + if max_distance > 1e-6: + normalized = normalized / max_distance + + if self.scale_factor is not None: + normalized *= self.scale_factor + + return normalized + + def normalize_frame( + self, + left_hand: np.ndarray, + right_hand: np.ndarray, + face: np.ndarray + ) -> np.ndarray: + """ + Normalisasi semua landmark dari satu frame. + + Args: + left_hand: Array (21, 3) landmark tangan kiri + right_hand: Array (21, 3) landmark tangan kanan + face: Array (60, 3) landmark wajah + + Returns: + Array 1D concatenation dari semua landmark yang sudah dinormalisasi + """ + norm_left = self.normalize_hand(left_hand) + norm_right = self.normalize_hand(right_hand) + norm_face = self.normalize_face(face) + + # Concatenate: 63 + 63 + 180 = 306 + return np.concatenate([ + norm_left.flatten(), # 63 values + norm_right.flatten(), # 63 values + norm_face.flatten() # 180 values + ]) + + +def normalize_landmarks( + left_hand: np.ndarray, + right_hand: np.ndarray, + face: np.ndarray, + scale_factor: Optional[float] = None +) -> np.ndarray: + """ + Fungsi convenience untuk normalisasi landmark. + + Args: + left_hand: Landmark tangan kiri + right_hand: Landmark tangan kanan + face: Landmark wajah + scale_factor: Optional scale factor + + Returns: + Array landmark yang sudah dinormalisasi + """ + normalizer = LandmarkNormalizer(scale_factor=scale_factor) + return normalizer.normalize_frame(left_hand, right_hand, face) diff --git a/backend/utils/tcn.py b/backend/utils/tcn.py new file mode 100644 index 0000000..be35fa3 --- /dev/null +++ b/backend/utils/tcn.py @@ -0,0 +1,187 @@ +""" +Temporal Convolutional Network (TCN) untuk sequence classification. + +Berisi implementasi model Temporal Convolutional Network sesuai README. + +""" + +import torch +import torch.nn as nn +from torch.nn.utils import weight_norm + + +class Chomp1d(nn.Module): + """ + Remove padding dari akhir sequence. + """ + def __init__(self, chomp_size): + super(Chomp1d, self).__init__() + self.chomp_size = chomp_size + + def forward(self, x): + return x[:, :, :-self.chomp_size].contiguous() + + +class TemporalBlock(nn.Module): + """ + Temporal block dari TCN. + Terdiri dari dua dilated conv layers dengan residual connection. + """ + def __init__( + self, + n_inputs, + n_outputs, + kernel_size, + stride, + dilation, + padding, + dropout=0.2 + ): + super(TemporalBlock, self).__init__() + + # First dilated convolution + self.conv1 = weight_norm(nn.Conv1d( + n_inputs, n_outputs, kernel_size, + stride=stride, padding=padding, dilation=dilation + )) + self.chomp1 = Chomp1d(padding) + self.relu1 = nn.ReLU() + self.dropout1 = nn.Dropout(dropout) + + # Second dilated convolution + self.conv2 = weight_norm(nn.Conv1d( + n_outputs, n_outputs, kernel_size, + stride=stride, padding=padding, dilation=dilation + )) + self.chomp2 = Chomp1d(padding) + self.relu2 = nn.ReLU() + self.dropout2 = nn.Dropout(dropout) + + # Residual connection + self.net = nn.Sequential( + self.conv1, self.chomp1, self.relu1, self.dropout1, + self.conv2, self.chomp2, self.relu2, self.dropout2 + ) + + # Downsample jika perlu + self.downsample = nn.Conv1d(n_inputs, n_outputs, 1) if n_inputs != n_outputs else None + self.relu = nn.ReLU() + + # Initialize weights + self.conv1.weight.data.normal_(0, 0.01) + self.conv2.weight.data.normal_(0, 0.01) + + def forward(self, x): + out = self.net(x) + res = x if self.downsample is None else self.downsample(x) + return self.relu(out + res) + + +class TemporalConvNet(nn.Module): + """ + Temporal Convolutional Network. + Stack dari temporal blocks dengan increasing dilation. + """ + def __init__( + self, + num_inputs, + num_channels, + kernel_size=3, + dropout=0.2 + ): + """ + Args: + num_inputs: Jumlah input channels + num_channels: List of hidden layer sizes + kernel_size: Kernel size untuk convolutions + dropout: Dropout rate + """ + super(TemporalConvNet, self).__init__() + + layers = [] + num_levels = len(num_channels) + + for i in range(num_levels): + dilation_size = 2 ** i + in_channels = num_inputs if i == 0 else num_channels[i - 1] + out_channels = num_channels[i] + + layers.append(TemporalBlock( + in_channels, + out_channels, + kernel_size, + stride=1, + dilation=dilation_size, + padding=(kernel_size - 1) * dilation_size, + dropout=dropout + )) + + self.network = nn.Sequential(*layers) + + def forward(self, x): + return self.network(x) + + +class TCNClassifier(nn.Module): + """ + TCN-based classifier untuk gesture recognition. + + Sesuai README: + - Input: Sequence dari landmark (30, 306) + - Output: Probability untuk setiap kelas + """ + def __init__( + self, + input_size: int, + num_classes: int, + num_channels: list = None, + kernel_size: int = 3, + dropout: float = 0.2, + fc_hidden_size: int = 512 + ): + """ + Args: + input_size: Ukuran input features (306) + num_classes: Jumlah kelas output + num_channels: Hidden layer sizes + kernel_size: Kernel size + dropout: Dropout rate + fc_hidden_size: Hidden size untuk fully connected layer + """ + super(TCNClassifier, self).__init__() + + if num_channels is None: + num_channels = [64, 128, 256] + + self.tcn = TemporalConvNet( + num_inputs=input_size, + num_channels=num_channels, + kernel_size=kernel_size, + dropout=dropout + ) + + tcn_output_size = num_channels[-1] + + # Fully connected layers + self.fc = nn.Sequential( + nn.Linear(tcn_output_size, fc_hidden_size), + nn.ReLU(), + nn.Dropout(dropout), + nn.Linear(fc_hidden_size, num_classes) + ) + + def forward(self, x): + # x shape: (batch, seq_len, features) + # Transpose ke (batch, features, seq_len) untuk Conv1d + x = x.transpose(1, 2) + + # TCN + tcn_out = self.tcn(x) + + # Global average pooling + pooled = tcn_out.mean(dim=2) + + # Classify + out = self.fc(pooled) + + return out diff --git a/frontend b/frontend new file mode 160000 index 0000000..a2b103a --- /dev/null +++ b/frontend @@ -0,0 +1 @@ +Subproject commit a2b103aeaea701601e92ce6afe6b9713bd5fd564