184 lines
4.3 KiB
Python
184 lines
4.3 KiB
Python
from fastapi import APIRouter, Depends
|
|
|
|
from sqlalchemy.orm import Session
|
|
|
|
from ..database import get_db
|
|
from ..models import Komentar
|
|
|
|
from ..preprocessing import (
|
|
preprocess_text
|
|
)
|
|
|
|
|
|
# =========================================================
|
|
# ROUTER
|
|
# =========================================================
|
|
|
|
router = APIRouter(
|
|
prefix="/preprocessing",
|
|
tags=["Pre-processing"]
|
|
)
|
|
|
|
|
|
# =========================================================
|
|
# RUN PREPROCESSING
|
|
# =========================================================
|
|
|
|
@router.post("/")
|
|
def run_preprocessing(
|
|
db: Session = Depends(get_db)
|
|
):
|
|
|
|
# =====================================================
|
|
# AMBIL HANYA DATA YANG BELUM PREPROCESSING
|
|
# =====================================================
|
|
|
|
data = (
|
|
db.query(Komentar)
|
|
.filter(
|
|
(Komentar.text_preprocessed.is_(None)) |
|
|
(Komentar.text_preprocessed == "")
|
|
)
|
|
.all()
|
|
)
|
|
|
|
|
|
# =====================================================
|
|
# JIKA TIDAK ADA DATA BARU
|
|
# =====================================================
|
|
|
|
if not data:
|
|
|
|
return {
|
|
|
|
"message":
|
|
"Tidak ada data baru yang perlu diproses",
|
|
|
|
"jumlah_data":
|
|
0
|
|
}
|
|
|
|
|
|
# =====================================================
|
|
# PROSES PREPROCESSING
|
|
# =====================================================
|
|
|
|
jumlah_data = 0
|
|
|
|
|
|
for item in data:
|
|
|
|
# -------------------------------------------------
|
|
# Jalankan seluruh tahapan preprocessing
|
|
# -------------------------------------------------
|
|
|
|
hasil = preprocess_text(
|
|
item.komentar
|
|
)
|
|
|
|
|
|
# -------------------------------------------------
|
|
# Simpan hasil Case Folding
|
|
# -------------------------------------------------
|
|
|
|
item.case_folding = (
|
|
hasil["case_folding"]
|
|
)
|
|
|
|
|
|
# -------------------------------------------------
|
|
# Simpan hasil Normalisasi
|
|
# -------------------------------------------------
|
|
|
|
item.normalisasi = (
|
|
hasil["normalisasi"]
|
|
)
|
|
|
|
|
|
# -------------------------------------------------
|
|
# Simpan hasil Cleaning
|
|
# -------------------------------------------------
|
|
|
|
item.cleaning = (
|
|
hasil["cleaning"]
|
|
)
|
|
|
|
|
|
# -------------------------------------------------
|
|
# Simpan hasil Tokenization
|
|
# -------------------------------------------------
|
|
|
|
item.tokenization = (
|
|
hasil["tokenization"]
|
|
)
|
|
|
|
|
|
# -------------------------------------------------
|
|
# Simpan hasil Stopword Removal
|
|
# -------------------------------------------------
|
|
|
|
item.stopword_removal = (
|
|
hasil["stopword_removal"]
|
|
)
|
|
|
|
|
|
# -------------------------------------------------
|
|
# Simpan hasil Stemming
|
|
# -------------------------------------------------
|
|
|
|
item.stemming = (
|
|
hasil["stemming"]
|
|
)
|
|
|
|
|
|
# -------------------------------------------------
|
|
# Simpan hasil akhir preprocessing
|
|
# -------------------------------------------------
|
|
|
|
item.text_preprocessed = (
|
|
hasil["text_preprocessed"]
|
|
)
|
|
|
|
|
|
# =================================================
|
|
# TIDAK MENGHAPUS TF-IDF LAMA
|
|
# =================================================
|
|
#
|
|
# Data yang diproses di sini adalah data baru,
|
|
# sehingga TF-IDF-nya memang belum ada.
|
|
#
|
|
# TF-IDF komentar lama TETAP dipertahankan.
|
|
#
|
|
|
|
|
|
# =================================================
|
|
# RESET KLASIFIKASI UNTUK DATA BARU
|
|
# =================================================
|
|
|
|
item.sentiment = None
|
|
|
|
item.probability = None
|
|
|
|
|
|
jumlah_data += 1
|
|
|
|
|
|
# =====================================================
|
|
# SIMPAN PERUBAHAN KE DATABASE
|
|
# =====================================================
|
|
|
|
db.commit()
|
|
|
|
|
|
# =====================================================
|
|
# RESPONSE
|
|
# =====================================================
|
|
|
|
return {
|
|
|
|
"message":
|
|
"Pre-processing berhasil",
|
|
|
|
"jumlah_data":
|
|
jumlah_data
|
|
} |