193 lines
4.7 KiB
Python
193 lines
4.7 KiB
Python
import re
|
|
from nltk.tokenize import word_tokenize
|
|
from nltk.corpus import stopwords
|
|
from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
|
|
|
|
# ============================================================
|
|
# STEMMER SASTRAWI
|
|
# ============================================================
|
|
|
|
stemmer_factory = StemmerFactory()
|
|
stemmer = stemmer_factory.create_stemmer()
|
|
|
|
# ============================================================
|
|
# STOPWORD
|
|
# ============================================================
|
|
|
|
partikel_tambahan = {
|
|
"sih",
|
|
"si",
|
|
"kan",
|
|
"ya",
|
|
"bang",
|
|
"dong",
|
|
"lah",
|
|
"deh",
|
|
"kah",
|
|
"kok"
|
|
}
|
|
|
|
kata_negasi = {
|
|
"tidak",
|
|
"bukan",
|
|
"belum",
|
|
"jangan",
|
|
"tanpa",
|
|
"kurang",
|
|
"malah"
|
|
}
|
|
|
|
list_stopwords = set(stopwords.words("indonesian"))
|
|
list_stopwords = list_stopwords | partikel_tambahan
|
|
|
|
# Kata negasi harus tetap dipertahankan
|
|
list_stopwords = list_stopwords - kata_negasi
|
|
|
|
# ============================================================
|
|
# KAMUS NORMALISASI
|
|
# ============================================================
|
|
#
|
|
# Ini baru struktur awal backend.
|
|
# Seluruh kamus final dari notebook penelitian nanti
|
|
# dimasukkan ke bagian ini agar hasil preprocessing web
|
|
# identik dengan penelitian.
|
|
# ============================================================
|
|
|
|
normalization_map = {
|
|
"download": "unduh",
|
|
"upload": "unggah",
|
|
"scan": "pindai",
|
|
"matkul": "mata kuliah",
|
|
"gabisa": "tidak bisa",
|
|
"gakbisa": "tidak bisa",
|
|
"nggakbisa": "tidak bisa",
|
|
"lemot": "lambat",
|
|
"submit": "unggah",
|
|
"userfriendly": "ramah pengguna",
|
|
"gak": "tidak",
|
|
"ga": "tidak",
|
|
"nggak": "tidak"
|
|
}
|
|
|
|
phrase_normalization = {
|
|
"di gunakan": "digunakan",
|
|
"didalam": "di dalam",
|
|
"diluar": "di luar"
|
|
}
|
|
|
|
# ============================================================
|
|
# 1. CASE FOLDING
|
|
# ============================================================
|
|
|
|
def case_folding(text):
|
|
return str(text).lower()
|
|
|
|
# ============================================================
|
|
# 2. CLEANING
|
|
# ============================================================
|
|
|
|
def cleaning(text):
|
|
text = str(text)
|
|
|
|
# Hapus URL
|
|
text = re.sub(r"https?://\S+|www\.\S+", " ", text)
|
|
|
|
# Hapus mention
|
|
text = re.sub(r"@\w+", " ", text)
|
|
|
|
# Hapus hashtag beserta simbol #
|
|
text = re.sub(r"#\w+", " ", text)
|
|
|
|
# Hapus angka
|
|
text = re.sub(r"\d+", " ", text)
|
|
|
|
# Hanya pertahankan huruf dan spasi
|
|
text = re.sub(r"[^a-zA-Z\s]", " ", text)
|
|
|
|
# Rapikan spasi
|
|
text = re.sub(r"\s+", " ", text).strip()
|
|
|
|
return text
|
|
|
|
# ============================================================
|
|
# 3. NORMALISASI
|
|
# ============================================================
|
|
|
|
def normalization(text):
|
|
text = str(text)
|
|
|
|
# Normalisasi bentuk/frasa tertentu terlebih dahulu
|
|
for source, target in phrase_normalization.items():
|
|
text = re.sub(
|
|
rf"\b{re.escape(source)}\b",
|
|
target,
|
|
text
|
|
)
|
|
|
|
words = text.split()
|
|
normalized_words = []
|
|
|
|
for word in words:
|
|
replacement = normalization_map.get(word, word)
|
|
|
|
# Replacement boleh menghasilkan lebih dari satu kata
|
|
normalized_words.extend(replacement.split())
|
|
|
|
return " ".join(normalized_words)
|
|
|
|
# ============================================================
|
|
# 4. TOKENIZING
|
|
# ============================================================
|
|
|
|
def tokenizing(text):
|
|
return word_tokenize(str(text))
|
|
|
|
# ============================================================
|
|
# 5. STOPWORD REMOVAL
|
|
# ============================================================
|
|
|
|
def stopword_removal(tokens):
|
|
return [
|
|
token
|
|
for token in tokens
|
|
if token not in list_stopwords
|
|
]
|
|
|
|
# ============================================================
|
|
# 6. STEMMING
|
|
# ============================================================
|
|
|
|
def stemming(tokens):
|
|
text = " ".join(tokens)
|
|
return stemmer.stem(text)
|
|
|
|
# ============================================================
|
|
# PIPELINE PREPROCESSING
|
|
# ============================================================
|
|
|
|
def preprocess_text(text):
|
|
original_text = str(text)
|
|
|
|
casefolded_text = case_folding(original_text)
|
|
|
|
cleaned_text = cleaning(casefolded_text)
|
|
|
|
normalized_text = normalization(cleaned_text)
|
|
|
|
tokenized_text = tokenizing(normalized_text)
|
|
|
|
stopword_tokens = stopword_removal(tokenized_text)
|
|
|
|
stopword_text = " ".join(stopword_tokens)
|
|
|
|
stemmed_text = stemming(stopword_tokens)
|
|
|
|
return {
|
|
"original_text": original_text,
|
|
"case_folding": casefolded_text,
|
|
"cleaning": cleaned_text,
|
|
"normalization": normalized_text,
|
|
"tokenizing": tokenized_text,
|
|
"stopword_removal": stopword_text,
|
|
"stemming": stemmed_text
|
|
} |