417 lines
9.8 KiB
Python
417 lines
9.8 KiB
Python
from fastapi import APIRouter, Depends, HTTPException
|
|
from sqlalchemy.orm import Session
|
|
from sqlalchemy import or_
|
|
|
|
import json
|
|
import ast
|
|
|
|
from ..database import get_db
|
|
from ..models import Komentar
|
|
from ..ml_model import tfidf_vectorizer
|
|
|
|
|
|
router = APIRouter(
|
|
prefix="/tfidf",
|
|
tags=["TF-IDF"]
|
|
)
|
|
|
|
|
|
# =========================================================
|
|
# FUNGSI MEMBACA DATA TF-IDF
|
|
# =========================================================
|
|
def parse_tfidf_data(tfidf_value):
|
|
"""
|
|
Membaca data TF-IDF dari database.
|
|
|
|
Mendukung beberapa format:
|
|
|
|
1. Format JSON object:
|
|
{
|
|
"habis": 0.357312,
|
|
"notifikasi": 0.214512
|
|
}
|
|
|
|
2. Format JSON/Python list:
|
|
[
|
|
{
|
|
"kata": "habis",
|
|
"nilai": 0.357312
|
|
}
|
|
]
|
|
|
|
3. Format lama Python list dengan single quote:
|
|
[
|
|
{'kata': 'habis', 'nilai': 0.357312}
|
|
]
|
|
|
|
Hasil akhir selalu dikembalikan sebagai object:
|
|
{
|
|
"habis": 0.357312,
|
|
"notifikasi": 0.214512
|
|
}
|
|
"""
|
|
|
|
# -----------------------------------------------------
|
|
# Jika tidak ada data
|
|
# -----------------------------------------------------
|
|
if not tfidf_value:
|
|
return {}
|
|
|
|
|
|
# -----------------------------------------------------
|
|
# Jika data sudah berupa dictionary
|
|
# -----------------------------------------------------
|
|
if isinstance(tfidf_value, dict):
|
|
return tfidf_value
|
|
|
|
|
|
# -----------------------------------------------------
|
|
# Jika data berupa list
|
|
# -----------------------------------------------------
|
|
if isinstance(tfidf_value, list):
|
|
|
|
hasil = {}
|
|
|
|
for item in tfidf_value:
|
|
|
|
if not isinstance(item, dict):
|
|
continue
|
|
|
|
kata = item.get("kata")
|
|
nilai = item.get("nilai")
|
|
|
|
if kata is None or nilai is None:
|
|
continue
|
|
|
|
try:
|
|
hasil[str(kata)] = float(nilai)
|
|
|
|
except (TypeError, ValueError):
|
|
continue
|
|
|
|
return hasil
|
|
|
|
|
|
# -----------------------------------------------------
|
|
# Pastikan data berupa string
|
|
# -----------------------------------------------------
|
|
if not isinstance(tfidf_value, str):
|
|
return {}
|
|
|
|
|
|
tfidf_value = tfidf_value.strip()
|
|
|
|
if not tfidf_value:
|
|
return {}
|
|
|
|
|
|
# -----------------------------------------------------
|
|
# COBA BACA SEBAGAI JSON
|
|
# -----------------------------------------------------
|
|
try:
|
|
|
|
parsed = json.loads(tfidf_value)
|
|
|
|
# ---------------------------------------------
|
|
# JSON object
|
|
# ---------------------------------------------
|
|
if isinstance(parsed, dict):
|
|
|
|
hasil = {}
|
|
|
|
for kata, nilai in parsed.items():
|
|
|
|
try:
|
|
hasil[str(kata)] = float(nilai)
|
|
|
|
except (TypeError, ValueError):
|
|
continue
|
|
|
|
return hasil
|
|
|
|
|
|
# ---------------------------------------------
|
|
# JSON list
|
|
# ---------------------------------------------
|
|
if isinstance(parsed, list):
|
|
|
|
hasil = {}
|
|
|
|
for item in parsed:
|
|
|
|
if not isinstance(item, dict):
|
|
continue
|
|
|
|
kata = item.get("kata")
|
|
nilai = item.get("nilai")
|
|
|
|
if kata is None or nilai is None:
|
|
continue
|
|
|
|
try:
|
|
hasil[str(kata)] = float(nilai)
|
|
|
|
except (TypeError, ValueError):
|
|
continue
|
|
|
|
return hasil
|
|
|
|
except (json.JSONDecodeError, TypeError, ValueError):
|
|
pass
|
|
|
|
|
|
# -----------------------------------------------------
|
|
# COBA FORMAT LAMA PYTHON
|
|
# Contoh:
|
|
#
|
|
# [{'kata': 'habis', 'nilai': 0.357312}]
|
|
# -----------------------------------------------------
|
|
try:
|
|
|
|
parsed = ast.literal_eval(tfidf_value)
|
|
|
|
# ---------------------------------------------
|
|
# Dictionary
|
|
# ---------------------------------------------
|
|
if isinstance(parsed, dict):
|
|
|
|
hasil = {}
|
|
|
|
for kata, nilai in parsed.items():
|
|
|
|
try:
|
|
hasil[str(kata)] = float(nilai)
|
|
|
|
except (TypeError, ValueError):
|
|
continue
|
|
|
|
return hasil
|
|
|
|
|
|
# ---------------------------------------------
|
|
# List
|
|
# ---------------------------------------------
|
|
if isinstance(parsed, list):
|
|
|
|
hasil = {}
|
|
|
|
for item in parsed:
|
|
|
|
if not isinstance(item, dict):
|
|
continue
|
|
|
|
kata = item.get("kata")
|
|
nilai = item.get("nilai")
|
|
|
|
if kata is None or nilai is None:
|
|
continue
|
|
|
|
try:
|
|
hasil[str(kata)] = float(nilai)
|
|
|
|
except (TypeError, ValueError):
|
|
continue
|
|
|
|
return hasil
|
|
|
|
except (ValueError, SyntaxError, TypeError):
|
|
pass
|
|
|
|
|
|
# -----------------------------------------------------
|
|
# Jika semua cara gagal
|
|
# -----------------------------------------------------
|
|
return {}
|
|
|
|
|
|
# =========================================================
|
|
# GET DATA TF-IDF
|
|
# =========================================================
|
|
@router.get("/")
|
|
def get_tfidf_data(
|
|
db: Session = Depends(get_db)
|
|
):
|
|
|
|
data = db.query(Komentar).all()
|
|
|
|
hasil = []
|
|
|
|
for item in data:
|
|
|
|
# ---------------------------------------------
|
|
# Baca TF-IDF dari database
|
|
# ---------------------------------------------
|
|
tfidf_data = parse_tfidf_data(
|
|
item.tfidf
|
|
)
|
|
|
|
hasil.append({
|
|
"id": item.id,
|
|
"komentar": item.komentar,
|
|
"text_preprocessed": item.text_preprocessed,
|
|
"tfidf": tfidf_data
|
|
})
|
|
|
|
return {
|
|
"jumlah_data": len(hasil),
|
|
"data": hasil
|
|
}
|
|
|
|
|
|
# =========================================================
|
|
# CALCULATE TF-IDF
|
|
# =========================================================
|
|
@router.post("/calculate")
|
|
def calculate_tfidf(
|
|
db: Session = Depends(get_db)
|
|
):
|
|
|
|
# -----------------------------------------------------
|
|
# -----------------------------------------------------
|
|
# AMBIL HANYA DATA BARU YANG:
|
|
# 1. SUDAH MELALUI PREPROCESSING
|
|
# 2. BELUM MEMILIKI TF-IDF
|
|
#
|
|
# TF-IDF lama TETAP DISIMPAN dan tidak dihitung ulang.
|
|
# -----------------------------------------------------
|
|
data = db.query(Komentar).filter(
|
|
Komentar.text_preprocessed.isnot(None),
|
|
Komentar.text_preprocessed != "",
|
|
or_(
|
|
Komentar.tfidf.is_(None),
|
|
Komentar.tfidf == ""
|
|
)
|
|
).all()
|
|
|
|
if not data:
|
|
return {
|
|
"message": "Tidak ada data baru yang perlu dihitung TF-IDF",
|
|
"jumlah_data": 0,
|
|
"jumlah_fitur": 0
|
|
}
|
|
|
|
|
|
# PASTIKAN VECTORIZER TERSEDIA
|
|
# -----------------------------------------------------
|
|
if tfidf_vectorizer is None:
|
|
|
|
raise HTTPException(
|
|
status_code=500,
|
|
detail="TF-IDF vectorizer belum tersedia."
|
|
)
|
|
|
|
|
|
# -----------------------------------------------------
|
|
# AMBIL TEXT HASIL PREPROCESSING
|
|
# -----------------------------------------------------
|
|
texts = [
|
|
item.text_preprocessed
|
|
for item in data
|
|
]
|
|
|
|
|
|
# -----------------------------------------------------
|
|
# TRANSFORM MENGGUNAKAN VECTORIZER
|
|
# YANG SUDAH FIT DARI MODEL
|
|
# -----------------------------------------------------
|
|
try:
|
|
|
|
tfidf_matrix = (
|
|
tfidf_vectorizer.transform(texts)
|
|
)
|
|
|
|
except Exception as e:
|
|
|
|
raise HTTPException(
|
|
status_code=500,
|
|
detail=(
|
|
f"Gagal menghitung TF-IDF: {str(e)}"
|
|
)
|
|
)
|
|
|
|
|
|
# -----------------------------------------------------
|
|
# AMBIL NAMA FITUR
|
|
# -----------------------------------------------------
|
|
feature_names = (
|
|
tfidf_vectorizer
|
|
.get_feature_names_out()
|
|
)
|
|
|
|
|
|
# -----------------------------------------------------
|
|
# SIMPAN TF-IDF SETIAP KOMENTAR
|
|
# -----------------------------------------------------
|
|
for index, item in enumerate(data):
|
|
|
|
row = tfidf_matrix[index]
|
|
|
|
tfidf_dict = {}
|
|
|
|
|
|
# ---------------------------------------------
|
|
# Ambil fitur yang memiliki nilai > 0
|
|
# ---------------------------------------------
|
|
for col_index, value in zip(
|
|
row.indices,
|
|
row.data
|
|
):
|
|
|
|
value = float(value)
|
|
|
|
if value <= 0:
|
|
continue
|
|
|
|
|
|
kata = feature_names[col_index]
|
|
|
|
|
|
tfidf_dict[str(kata)] = round(
|
|
value,
|
|
6
|
|
)
|
|
|
|
|
|
# ---------------------------------------------
|
|
# Simpan sebagai JSON
|
|
#
|
|
# Format:
|
|
#
|
|
# {
|
|
# "kata": nilai
|
|
# }
|
|
# ---------------------------------------------
|
|
item.tfidf = json.dumps(
|
|
tfidf_dict,
|
|
ensure_ascii=False
|
|
)
|
|
|
|
|
|
# -----------------------------------------------------
|
|
# SIMPAN KE DATABASE
|
|
# -----------------------------------------------------
|
|
try:
|
|
|
|
db.commit()
|
|
|
|
except Exception as e:
|
|
|
|
db.rollback()
|
|
|
|
raise HTTPException(
|
|
status_code=500,
|
|
detail=(
|
|
f"Gagal menyimpan TF-IDF: {str(e)}"
|
|
)
|
|
)
|
|
|
|
|
|
# -----------------------------------------------------
|
|
# RESPONSE
|
|
# -----------------------------------------------------
|
|
return {
|
|
"message": "TF-IDF data baru berhasil dihitung",
|
|
"jumlah_data": len(data),
|
|
"jumlah_fitur": len(feature_names)
|
|
}
|