125 lines
3.2 KiB
Python
125 lines
3.2 KiB
Python
import re, json
|
|
from pathlib import Path
|
|
from nltk.tokenize import word_tokenize
|
|
from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
|
|
|
|
config = json.loads(
|
|
(Path(__file__).parent / 'preprocessing_config.json')
|
|
.read_text(encoding='utf-8')
|
|
)
|
|
|
|
kamus_normalisasi = config['kamus_normalisasi']
|
|
frasa_awal = config['frasa_awal']
|
|
frasa_akhir = config['frasa_akhir']
|
|
koreksi_validator = config['koreksi_validator']
|
|
list_stopwords = set(config['list_stopwords'])
|
|
kata_dilindungi_stemming = set(
|
|
config['kata_dilindungi_stemming']
|
|
)
|
|
|
|
stemmer = StemmerFactory().create_stemmer()
|
|
|
|
def cleaning(text):
|
|
text = str(text)
|
|
|
|
# 1. Menghapus URL/Link (tidak peka huruf besar/kecil)
|
|
text = re.sub(r"https?://\S+|www\.\S+", " ", text, flags=re.IGNORECASE)
|
|
|
|
# 2. Menghapus mention (@), pertahankan isi hashtag (#)
|
|
text = re.sub(r"@\w+", " ", text)
|
|
text = re.sub(r"#(\w+)", r"\1", text)
|
|
|
|
# 3. Menghapus angka
|
|
text = re.sub(r"\d+", " ", text)
|
|
|
|
# 4. Menghapus karakter selain huruf
|
|
text = re.sub(r"[^a-zA-Z\s]", " ", text)
|
|
|
|
# 5. Merapikan spasi berlebih
|
|
text = re.sub(r"\s+", " ", text).strip()
|
|
|
|
return text
|
|
|
|
|
|
def case_folding(text):
|
|
# Mengubah seluruh teks menjadi huruf kecil
|
|
return str(text).lower()
|
|
|
|
|
|
def ganti_frasa(text, kamus_frasa):
|
|
for pola, ganti in kamus_frasa.items():
|
|
text = re.sub(pola, ganti, text)
|
|
return text
|
|
|
|
|
|
def normalisasi(text):
|
|
text = ganti_frasa(str(text), frasa_awal)
|
|
text = re.sub(r"\b\w+\b", lambda m: kamus_normalisasi.get(m.group(), m.group()), text)
|
|
return ganti_frasa(text, frasa_akhir)
|
|
|
|
|
|
def koreksi_normalisasi_validator(text):
|
|
text = str(text)
|
|
text = re.sub(r"\bnge\s+lag\b", "lag", text) # koreksi frasa hasil validasi
|
|
text = re.sub(r"\bits\s+okey\b", "its okey", text) # "its okey" dipertahankan
|
|
return re.sub(r"\b\w+\b", lambda x: koreksi_validator.get(x.group(), x.group()), text)
|
|
|
|
|
|
def tokenisasi(text):
|
|
return word_tokenize(str(text))
|
|
|
|
|
|
def hapus_stopword(tokens):
|
|
return " ".join(
|
|
kata for kata in tokens
|
|
if kata not in list_stopwords and len(kata) > 1
|
|
)
|
|
|
|
|
|
def stemming(text):
|
|
hasil = []
|
|
|
|
for kata in str(text).split():
|
|
if kata in kata_dilindungi_stemming:
|
|
hasil.append(kata)
|
|
else:
|
|
hasil.append(stemmer.stem(kata))
|
|
|
|
return " ".join(hasil)
|
|
|
|
|
|
def preprocessing_kalimat_baru(text):
|
|
|
|
# 1. Cleaning
|
|
text = cleaning(text)
|
|
|
|
# 2. Case Folding
|
|
text = case_folding(text)
|
|
|
|
# 3. Normalisasi
|
|
text = normalisasi(text)
|
|
text = koreksi_normalisasi_validator(text)
|
|
|
|
# 4. Tokenisasi
|
|
tokens = tokenisasi(text)
|
|
|
|
# 5. Stopword Removal
|
|
text = hapus_stopword(tokens)
|
|
|
|
# 6. Stemming
|
|
text = stemming(text)
|
|
|
|
return text
|
|
|
|
|
|
def prediksi_sentimen(teks, model, vectorizer):
|
|
if not isinstance(teks, str) or not teks.strip():
|
|
raise ValueError("Masukkan komentar terlebih dahulu.")
|
|
hasil = preprocessing_kalimat_baru(teks)
|
|
if not hasil.strip():
|
|
raise ValueError("Komentar tidak memiliki kata yang dapat diproses.")
|
|
fitur = vectorizer.transform([hasil])
|
|
if fitur.nnz == 0:
|
|
raise ValueError("Kata dalam komentar belum dikenali model.")
|
|
return int(model.predict(fitur)[0])
|