PLJ-TIF_E41253311/backend/app/routes/tfidf.py

417 lines
9.8 KiB
Python

from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy.orm import Session
from sqlalchemy import or_
import json
import ast
from ..database import get_db
from ..models import Komentar
from ..ml_model import tfidf_vectorizer
router = APIRouter(
prefix="/tfidf",
tags=["TF-IDF"]
)
# =========================================================
# FUNGSI MEMBACA DATA TF-IDF
# =========================================================
def parse_tfidf_data(tfidf_value):
"""
Membaca data TF-IDF dari database.
Mendukung beberapa format:
1. Format JSON object:
{
"habis": 0.357312,
"notifikasi": 0.214512
}
2. Format JSON/Python list:
[
{
"kata": "habis",
"nilai": 0.357312
}
]
3. Format lama Python list dengan single quote:
[
{'kata': 'habis', 'nilai': 0.357312}
]
Hasil akhir selalu dikembalikan sebagai object:
{
"habis": 0.357312,
"notifikasi": 0.214512
}
"""
# -----------------------------------------------------
# Jika tidak ada data
# -----------------------------------------------------
if not tfidf_value:
return {}
# -----------------------------------------------------
# Jika data sudah berupa dictionary
# -----------------------------------------------------
if isinstance(tfidf_value, dict):
return tfidf_value
# -----------------------------------------------------
# Jika data berupa list
# -----------------------------------------------------
if isinstance(tfidf_value, list):
hasil = {}
for item in tfidf_value:
if not isinstance(item, dict):
continue
kata = item.get("kata")
nilai = item.get("nilai")
if kata is None or nilai is None:
continue
try:
hasil[str(kata)] = float(nilai)
except (TypeError, ValueError):
continue
return hasil
# -----------------------------------------------------
# Pastikan data berupa string
# -----------------------------------------------------
if not isinstance(tfidf_value, str):
return {}
tfidf_value = tfidf_value.strip()
if not tfidf_value:
return {}
# -----------------------------------------------------
# COBA BACA SEBAGAI JSON
# -----------------------------------------------------
try:
parsed = json.loads(tfidf_value)
# ---------------------------------------------
# JSON object
# ---------------------------------------------
if isinstance(parsed, dict):
hasil = {}
for kata, nilai in parsed.items():
try:
hasil[str(kata)] = float(nilai)
except (TypeError, ValueError):
continue
return hasil
# ---------------------------------------------
# JSON list
# ---------------------------------------------
if isinstance(parsed, list):
hasil = {}
for item in parsed:
if not isinstance(item, dict):
continue
kata = item.get("kata")
nilai = item.get("nilai")
if kata is None or nilai is None:
continue
try:
hasil[str(kata)] = float(nilai)
except (TypeError, ValueError):
continue
return hasil
except (json.JSONDecodeError, TypeError, ValueError):
pass
# -----------------------------------------------------
# COBA FORMAT LAMA PYTHON
# Contoh:
#
# [{'kata': 'habis', 'nilai': 0.357312}]
# -----------------------------------------------------
try:
parsed = ast.literal_eval(tfidf_value)
# ---------------------------------------------
# Dictionary
# ---------------------------------------------
if isinstance(parsed, dict):
hasil = {}
for kata, nilai in parsed.items():
try:
hasil[str(kata)] = float(nilai)
except (TypeError, ValueError):
continue
return hasil
# ---------------------------------------------
# List
# ---------------------------------------------
if isinstance(parsed, list):
hasil = {}
for item in parsed:
if not isinstance(item, dict):
continue
kata = item.get("kata")
nilai = item.get("nilai")
if kata is None or nilai is None:
continue
try:
hasil[str(kata)] = float(nilai)
except (TypeError, ValueError):
continue
return hasil
except (ValueError, SyntaxError, TypeError):
pass
# -----------------------------------------------------
# Jika semua cara gagal
# -----------------------------------------------------
return {}
# =========================================================
# GET DATA TF-IDF
# =========================================================
@router.get("/")
def get_tfidf_data(
db: Session = Depends(get_db)
):
data = db.query(Komentar).all()
hasil = []
for item in data:
# ---------------------------------------------
# Baca TF-IDF dari database
# ---------------------------------------------
tfidf_data = parse_tfidf_data(
item.tfidf
)
hasil.append({
"id": item.id,
"komentar": item.komentar,
"text_preprocessed": item.text_preprocessed,
"tfidf": tfidf_data
})
return {
"jumlah_data": len(hasil),
"data": hasil
}
# =========================================================
# CALCULATE TF-IDF
# =========================================================
@router.post("/calculate")
def calculate_tfidf(
db: Session = Depends(get_db)
):
# -----------------------------------------------------
# -----------------------------------------------------
# AMBIL HANYA DATA BARU YANG:
# 1. SUDAH MELALUI PREPROCESSING
# 2. BELUM MEMILIKI TF-IDF
#
# TF-IDF lama TETAP DISIMPAN dan tidak dihitung ulang.
# -----------------------------------------------------
data = db.query(Komentar).filter(
Komentar.text_preprocessed.isnot(None),
Komentar.text_preprocessed != "",
or_(
Komentar.tfidf.is_(None),
Komentar.tfidf == ""
)
).all()
if not data:
return {
"message": "Tidak ada data baru yang perlu dihitung TF-IDF",
"jumlah_data": 0,
"jumlah_fitur": 0
}
# PASTIKAN VECTORIZER TERSEDIA
# -----------------------------------------------------
if tfidf_vectorizer is None:
raise HTTPException(
status_code=500,
detail="TF-IDF vectorizer belum tersedia."
)
# -----------------------------------------------------
# AMBIL TEXT HASIL PREPROCESSING
# -----------------------------------------------------
texts = [
item.text_preprocessed
for item in data
]
# -----------------------------------------------------
# TRANSFORM MENGGUNAKAN VECTORIZER
# YANG SUDAH FIT DARI MODEL
# -----------------------------------------------------
try:
tfidf_matrix = (
tfidf_vectorizer.transform(texts)
)
except Exception as e:
raise HTTPException(
status_code=500,
detail=(
f"Gagal menghitung TF-IDF: {str(e)}"
)
)
# -----------------------------------------------------
# AMBIL NAMA FITUR
# -----------------------------------------------------
feature_names = (
tfidf_vectorizer
.get_feature_names_out()
)
# -----------------------------------------------------
# SIMPAN TF-IDF SETIAP KOMENTAR
# -----------------------------------------------------
for index, item in enumerate(data):
row = tfidf_matrix[index]
tfidf_dict = {}
# ---------------------------------------------
# Ambil fitur yang memiliki nilai > 0
# ---------------------------------------------
for col_index, value in zip(
row.indices,
row.data
):
value = float(value)
if value <= 0:
continue
kata = feature_names[col_index]
tfidf_dict[str(kata)] = round(
value,
6
)
# ---------------------------------------------
# Simpan sebagai JSON
#
# Format:
#
# {
# "kata": nilai
# }
# ---------------------------------------------
item.tfidf = json.dumps(
tfidf_dict,
ensure_ascii=False
)
# -----------------------------------------------------
# SIMPAN KE DATABASE
# -----------------------------------------------------
try:
db.commit()
except Exception as e:
db.rollback()
raise HTTPException(
status_code=500,
detail=(
f"Gagal menyimpan TF-IDF: {str(e)}"
)
)
# -----------------------------------------------------
# RESPONSE
# -----------------------------------------------------
return {
"message": "TF-IDF data baru berhasil dihitung",
"jumlah_data": len(data),
"jumlah_fitur": len(feature_names)
}