import re, json from pathlib import Path from nltk.tokenize import word_tokenize from Sastrawi.Stemmer.StemmerFactory import StemmerFactory config = json.loads( (Path(__file__).parent / 'preprocessing_config.json') .read_text(encoding='utf-8') ) kamus_normalisasi = config['kamus_normalisasi'] frasa_awal = config['frasa_awal'] frasa_akhir = config['frasa_akhir'] koreksi_validator = config['koreksi_validator'] list_stopwords = set(config['list_stopwords']) kata_dilindungi_stemming = set( config['kata_dilindungi_stemming'] ) stemmer = StemmerFactory().create_stemmer() def cleaning(text): text = str(text) # 1. Menghapus URL/Link (tidak peka huruf besar/kecil) text = re.sub(r"https?://\S+|www\.\S+", " ", text, flags=re.IGNORECASE) # 2. Menghapus mention (@), pertahankan isi hashtag (#) text = re.sub(r"@\w+", " ", text) text = re.sub(r"#(\w+)", r"\1", text) # 3. Menghapus angka text = re.sub(r"\d+", " ", text) # 4. Menghapus karakter selain huruf text = re.sub(r"[^a-zA-Z\s]", " ", text) # 5. Merapikan spasi berlebih text = re.sub(r"\s+", " ", text).strip() return text def case_folding(text): # Mengubah seluruh teks menjadi huruf kecil return str(text).lower() def ganti_frasa(text, kamus_frasa): for pola, ganti in kamus_frasa.items(): text = re.sub(pola, ganti, text) return text def normalisasi(text): text = ganti_frasa(str(text), frasa_awal) text = re.sub(r"\b\w+\b", lambda m: kamus_normalisasi.get(m.group(), m.group()), text) return ganti_frasa(text, frasa_akhir) def koreksi_normalisasi_validator(text): text = str(text) text = re.sub(r"\bnge\s+lag\b", "lag", text) # koreksi frasa hasil validasi text = re.sub(r"\bits\s+okey\b", "its okey", text) # "its okey" dipertahankan return re.sub(r"\b\w+\b", lambda x: koreksi_validator.get(x.group(), x.group()), text) def tokenisasi(text): return word_tokenize(str(text)) def hapus_stopword(tokens): return " ".join( kata for kata in tokens if kata not in list_stopwords and len(kata) > 1 ) def stemming(text): hasil = [] for kata in str(text).split(): if kata in kata_dilindungi_stemming: hasil.append(kata) else: hasil.append(stemmer.stem(kata)) return " ".join(hasil) def preprocessing_kalimat_baru(text): # 1. Cleaning text = cleaning(text) # 2. Case Folding text = case_folding(text) # 3. Normalisasi text = normalisasi(text) text = koreksi_normalisasi_validator(text) # 4. Tokenisasi tokens = tokenisasi(text) # 5. Stopword Removal text = hapus_stopword(tokens) # 6. Stemming text = stemming(text) return text def prediksi_sentimen(teks, model, vectorizer): if not isinstance(teks, str) or not teks.strip(): raise ValueError("Masukkan komentar terlebih dahulu.") hasil = preprocessing_kalimat_baru(teks) if not hasil.strip(): raise ValueError("Komentar tidak memiliki kata yang dapat diproses.") fitur = vectorizer.transform([hasil]) if fitur.nnz == 0: raise ValueError("Kata dalam komentar belum dikenali model.") return int(model.predict(fitur)[0])