715 lines
14 KiB
Python
715 lines
14 KiB
Python
import re
|
|
import nltk
|
|
|
|
from nltk.tokenize import word_tokenize
|
|
|
|
from Sastrawi.Stemmer.StemmerFactory import (
|
|
StemmerFactory
|
|
)
|
|
|
|
from Sastrawi.StopWordRemover.StopWordRemoverFactory import (
|
|
StopWordRemoverFactory
|
|
)
|
|
|
|
|
|
# =========================================================
|
|
# NLTK
|
|
# =========================================================
|
|
|
|
nltk.download("punkt", quiet=True)
|
|
nltk.download("punkt_tab", quiet=True)
|
|
|
|
|
|
# =========================================================
|
|
# STEMMER
|
|
# =========================================================
|
|
|
|
factory_stemmer = StemmerFactory()
|
|
|
|
stemmer = (
|
|
factory_stemmer
|
|
.create_stemmer()
|
|
)
|
|
|
|
|
|
# =========================================================
|
|
# STOPWORD
|
|
# =========================================================
|
|
|
|
factory_stopword = StopWordRemoverFactory()
|
|
|
|
stopwords = set(
|
|
factory_stopword.get_stop_words()
|
|
)
|
|
|
|
|
|
# =========================================================
|
|
# KATA PENTING
|
|
# =========================================================
|
|
# Kata-kata ini dikeluarkan dari daftar stopword
|
|
# agar tetap dipertahankan karena memiliki konteks
|
|
# terhadap sentimen.
|
|
|
|
kata_penting = {
|
|
|
|
# -------------------------
|
|
# Negasi
|
|
# -------------------------
|
|
|
|
"tidak",
|
|
"bukan",
|
|
"belum",
|
|
"jangan",
|
|
"kurang",
|
|
|
|
|
|
# -------------------------
|
|
# Konteks waktu dan kondisi
|
|
# -------------------------
|
|
|
|
"masih",
|
|
"sudah",
|
|
"sering",
|
|
"jarang",
|
|
"kadang",
|
|
"lagi",
|
|
"sebelum",
|
|
"setelah",
|
|
"kembali",
|
|
|
|
|
|
# -------------------------
|
|
# Konjungsi yang membawa konteks
|
|
# -------------------------
|
|
|
|
"namun",
|
|
"tetapi",
|
|
"meskipun",
|
|
"walaupun",
|
|
|
|
|
|
# -------------------------
|
|
# Intensitas
|
|
# -------------------------
|
|
|
|
"sangat",
|
|
"cukup",
|
|
"terlalu",
|
|
|
|
|
|
# -------------------------
|
|
# Kondisi dan kemampuan
|
|
# -------------------------
|
|
|
|
"hanya",
|
|
"tanpa",
|
|
"harus",
|
|
"bisa",
|
|
"dapat",
|
|
"perlu",
|
|
"seharusnya",
|
|
|
|
|
|
# -------------------------
|
|
# Keinginan
|
|
# -------------------------
|
|
|
|
"ingin"
|
|
}
|
|
|
|
|
|
# Hapus kata penting dari stopword
|
|
stopwords = stopwords - kata_penting
|
|
|
|
# Kata tambahan yang tidak membawa informasi sentimen
|
|
stopwords.update({
|
|
"nya"
|
|
})
|
|
|
|
|
|
# =========================================================
|
|
# KAMUS NORMALISASI
|
|
# =========================================================
|
|
|
|
kamus_normalisasi = {
|
|
|
|
# -------------------------
|
|
# Singkatan
|
|
# -------------------------
|
|
"yg": "yang",
|
|
"yng": "yang",
|
|
"dgn": "dengan",
|
|
"utk": "untuk",
|
|
|
|
# -------------------------
|
|
# Negasi
|
|
# -------------------------
|
|
"tdk": "tidak",
|
|
"gak": "tidak",
|
|
"ga": "tidak",
|
|
"nggak": "tidak",
|
|
"ngga": "tidak",
|
|
"enggak": "tidak",
|
|
|
|
# -------------------------
|
|
# Kata umum
|
|
# -------------------------
|
|
"krn": "karena",
|
|
"karna": "karena",
|
|
"dr": "dari",
|
|
"jd": "jadi",
|
|
"jg": "juga",
|
|
"tp": "tetapi",
|
|
"sm": "sama",
|
|
"sdh": "sudah",
|
|
"udh": "sudah",
|
|
"udah": "sudah",
|
|
"blm": "belum",
|
|
"belom": "belum",
|
|
"bgt": "banget",
|
|
"aja": "saja",
|
|
"sbg": "sebagai",
|
|
"dpt": "dapat",
|
|
"bbrp": "beberapa",
|
|
|
|
# -------------------------
|
|
# Kata tanya
|
|
# -------------------------
|
|
"gmn": "bagaimana",
|
|
"gimana": "bagaimana",
|
|
|
|
# -------------------------
|
|
# Kata kondisi
|
|
# -------------------------
|
|
"klo": "kalau",
|
|
"kalo": "kalau",
|
|
"kalok": "kalau",
|
|
|
|
# -------------------------
|
|
# Kata informal
|
|
# -------------------------
|
|
"emg": "memang",
|
|
"emang": "memang",
|
|
"kayak": "seperti",
|
|
"kaya": "seperti",
|
|
|
|
# -------------------------
|
|
# Kata khusus penelitian
|
|
# -------------------------
|
|
"terkadang": "kadang"
|
|
}
|
|
|
|
|
|
# =========================================================
|
|
# CASE FOLDING
|
|
# =========================================================
|
|
|
|
def case_folding(text):
|
|
|
|
return str(text).lower()
|
|
|
|
|
|
# =========================================================
|
|
# NORMALISASI
|
|
# =========================================================
|
|
|
|
def normalisasi(text):
|
|
|
|
text = str(text).lower()
|
|
|
|
|
|
# =====================================================
|
|
# S-LEARN
|
|
# =====================================================
|
|
|
|
# S-Learn
|
|
# S Learn
|
|
# Slearn
|
|
# s-learn
|
|
|
|
text = re.sub(
|
|
r"\bs[\s-]*learn\b",
|
|
"slearn",
|
|
text
|
|
)
|
|
|
|
|
|
# =====================================================
|
|
# LOGIN
|
|
# =====================================================
|
|
|
|
# log in
|
|
# log-in
|
|
# login
|
|
|
|
text = re.sub(
|
|
r"\blog[\s-]*in\b",
|
|
"login",
|
|
text
|
|
)
|
|
|
|
|
|
# =====================================================
|
|
# LOGOUT
|
|
# =====================================================
|
|
|
|
# log out
|
|
# log-out
|
|
# logout
|
|
|
|
text = re.sub(
|
|
r"\blog[\s-]*out\b",
|
|
"logout",
|
|
text
|
|
)
|
|
|
|
|
|
# =====================================================
|
|
# CHECK IN
|
|
# =====================================================
|
|
|
|
# check in
|
|
# check-in
|
|
# checkin
|
|
|
|
text = re.sub(
|
|
r"\bcheck[\s-]+in\b",
|
|
"checkin",
|
|
text
|
|
)
|
|
|
|
|
|
# =====================================================
|
|
# HP-NYA
|
|
# =====================================================
|
|
|
|
text = re.sub(
|
|
r"\bhpnya\b",
|
|
"hp nya",
|
|
text
|
|
)
|
|
|
|
text = re.sub(
|
|
r"\bhp\s+nya\b",
|
|
"hp nya",
|
|
text
|
|
)
|
|
|
|
|
|
# =====================================================
|
|
# PENGULANGAN HURUF
|
|
# =====================================================
|
|
#
|
|
# bagusss -> bagus
|
|
# lamaaa -> lama
|
|
# lambattt -> lambat
|
|
#
|
|
# Hanya huruf yang berulang 3 kali atau lebih
|
|
# yang dikurangi.
|
|
|
|
text = re.sub(
|
|
r"([a-z])\1{2,}",
|
|
r"\1",
|
|
text
|
|
)
|
|
|
|
|
|
# =====================================================
|
|
# KAMUS NORMALISASI
|
|
# =====================================================
|
|
|
|
words = text.split()
|
|
|
|
words = [
|
|
kamus_normalisasi.get(
|
|
word,
|
|
word
|
|
)
|
|
for word in words
|
|
]
|
|
|
|
|
|
return " ".join(words)
|
|
|
|
|
|
# =========================================================
|
|
# CLEANING
|
|
# =========================================================
|
|
|
|
def cleaning(text):
|
|
|
|
text = str(text)
|
|
|
|
|
|
# =====================================================
|
|
# URL
|
|
# =====================================================
|
|
|
|
text = re.sub(
|
|
r"http\S+|www\S+",
|
|
" ",
|
|
text
|
|
)
|
|
|
|
|
|
# =====================================================
|
|
# MENTION
|
|
# =====================================================
|
|
|
|
text = re.sub(
|
|
r"@\w+",
|
|
" ",
|
|
text
|
|
)
|
|
|
|
|
|
# =====================================================
|
|
# HASHTAG
|
|
# =====================================================
|
|
|
|
text = re.sub(
|
|
r"#\w+",
|
|
" ",
|
|
text
|
|
)
|
|
|
|
|
|
# =====================================================
|
|
# ANGKA
|
|
# =====================================================
|
|
|
|
text = re.sub(
|
|
r"\d+",
|
|
" ",
|
|
text
|
|
)
|
|
|
|
|
|
# =====================================================
|
|
# SIMBOL DAN TANDA BACA
|
|
# =====================================================
|
|
|
|
text = re.sub(
|
|
r"[^a-zA-Z\s]",
|
|
" ",
|
|
text
|
|
)
|
|
|
|
|
|
# =====================================================
|
|
# SPASI BERLEBIH
|
|
# =====================================================
|
|
|
|
text = re.sub(
|
|
r"\s+",
|
|
" ",
|
|
text
|
|
).strip()
|
|
|
|
|
|
return text
|
|
|
|
|
|
# =========================================================
|
|
# TOKENIZATION
|
|
# =========================================================
|
|
|
|
def tokenisasi(text):
|
|
|
|
if not str(text).strip():
|
|
return []
|
|
|
|
return word_tokenize(
|
|
str(text)
|
|
)
|
|
|
|
|
|
# =========================================================
|
|
# STOPWORD REMOVAL
|
|
# =========================================================
|
|
|
|
def stopword_removal(tokens):
|
|
|
|
return [
|
|
word
|
|
for word in tokens
|
|
if word not in stopwords
|
|
]
|
|
|
|
|
|
# =========================================================
|
|
# ISTILAH TEKNIS
|
|
# =========================================================
|
|
# Istilah berikut tidak dilakukan stemming karena
|
|
# merupakan istilah teknis yang digunakan pada
|
|
# konteks aplikasi S-Learn.
|
|
|
|
istilah_teknis = {
|
|
|
|
"slearn",
|
|
|
|
"mobile",
|
|
"error",
|
|
"lag",
|
|
"bug",
|
|
"online",
|
|
"offline",
|
|
|
|
"deadline",
|
|
|
|
"server",
|
|
"loading",
|
|
"website",
|
|
|
|
"login",
|
|
"logout",
|
|
|
|
"ios",
|
|
"android",
|
|
|
|
"update",
|
|
|
|
"ui",
|
|
"ux",
|
|
|
|
"user",
|
|
"interface",
|
|
|
|
"upload",
|
|
"download",
|
|
|
|
"menu",
|
|
"button",
|
|
|
|
"file",
|
|
"link",
|
|
|
|
"feedback",
|
|
|
|
"friendly",
|
|
|
|
"dashboard",
|
|
|
|
"password",
|
|
"username",
|
|
|
|
"checkin",
|
|
|
|
"refresh",
|
|
|
|
"performance",
|
|
"responsive",
|
|
|
|
"submit",
|
|
|
|
"session",
|
|
"support"
|
|
}
|
|
|
|
|
|
# =========================================================
|
|
# KAMUS PENGECUALIAN STEMMING
|
|
# =========================================================
|
|
|
|
kamus_pengecualian_stemming = {
|
|
|
|
# =====================================================
|
|
# KATA YANG JANGAN DIUBAH
|
|
# KARENA STEMMING DAPAT MERUSAK MAKNA
|
|
# =====================================================
|
|
|
|
"terasa": "terasa",
|
|
"terdapat": "terdapat",
|
|
"pengalaman": "pengalaman",
|
|
"mengenai": "mengenai",
|
|
"pengumuman": "pengumuman",
|
|
"bersamaan": "bersamaan",
|
|
"berbagai": "berbagai",
|
|
"melalui": "melalui",
|
|
"sebelumnya": "sebelumnya",
|
|
"pembaruan": "pembaruan",
|
|
"kegiatan": "kegiatan",
|
|
"tersedia": "tersedia",
|
|
"tampilan": "tampilan",
|
|
"pengguna": "pengguna",
|
|
"ketergantungan": "ketergantungan",
|
|
"dibandingkan": "dibandingkan",
|
|
"menarik": "menarik",
|
|
"melakukan": "melakukan",
|
|
"setalah": "setelah",
|
|
"terjadi": "terjadi",
|
|
"digunakan": "digunakan",
|
|
"tampilannya": "tampilan",
|
|
"dikembangkan": "dikembangkan",
|
|
"melakukan": "melakukan",
|
|
"berikutnya": "berikutnya",
|
|
"diperbaiki": "diperbaiki",
|
|
"diperbarui": "diperbarui",
|
|
"dilakukan": "dilakukan",
|
|
"pengembangan": "pengembangan",
|
|
"sebagian": "sebagian",
|
|
"menggunakannya": "menggunakan",
|
|
"menggunakan": "menggunakan",
|
|
"semoga": "semoga",
|
|
"memadai": "memadai",
|
|
"lemot": "lemot",
|
|
|
|
# =====================================================
|
|
# KATA YANG MEMANG DIINGINKAN
|
|
# MENJADI BENTUK TERTENTU
|
|
# =====================================================
|
|
|
|
"berurutan": "urut",
|
|
"leluasa": "luas"
|
|
}
|
|
|
|
|
|
# =========================================================
|
|
# STEMMING
|
|
# =========================================================
|
|
|
|
def stemming(tokens):
|
|
|
|
hasil = []
|
|
|
|
|
|
for word in tokens:
|
|
|
|
# -------------------------------------------------
|
|
# 1. Istilah teknis Inggris
|
|
# -------------------------------------------------
|
|
|
|
if word in istilah_teknis:
|
|
|
|
hasil.append(
|
|
word
|
|
)
|
|
|
|
|
|
# -------------------------------------------------
|
|
# 2. Kata yang harus dipertahankan
|
|
# -------------------------------------------------
|
|
|
|
elif word in kamus_pengecualian_stemming:
|
|
|
|
hasil.append(
|
|
kamus_pengecualian_stemming[word]
|
|
)
|
|
|
|
|
|
# -------------------------------------------------
|
|
# 3. Stemming normal Sastrawi
|
|
# -------------------------------------------------
|
|
|
|
else:
|
|
|
|
hasil.append(
|
|
stemmer.stem(word)
|
|
)
|
|
|
|
|
|
return hasil
|
|
|
|
|
|
# =========================================================
|
|
# FULL PREPROCESSING
|
|
# =========================================================
|
|
|
|
def preprocess_text(text):
|
|
|
|
# =====================================================
|
|
# 1. CASE FOLDING
|
|
# =====================================================
|
|
|
|
hasil_case_folding = case_folding(
|
|
text
|
|
)
|
|
|
|
|
|
# =====================================================
|
|
# 2. NORMALISASI
|
|
# =====================================================
|
|
|
|
hasil_normalisasi = normalisasi(
|
|
hasil_case_folding
|
|
)
|
|
|
|
|
|
# =====================================================
|
|
# 3. CLEANING
|
|
# =====================================================
|
|
|
|
hasil_cleaning = cleaning(
|
|
hasil_normalisasi
|
|
)
|
|
|
|
|
|
# =====================================================
|
|
# 4. TOKENIZATION
|
|
# =====================================================
|
|
|
|
hasil_tokenization = tokenisasi(
|
|
hasil_cleaning
|
|
)
|
|
|
|
|
|
# =====================================================
|
|
# 5. STOPWORD REMOVAL
|
|
# =====================================================
|
|
|
|
hasil_stopword = stopword_removal(
|
|
hasil_tokenization
|
|
)
|
|
|
|
|
|
# =====================================================
|
|
# 6. STEMMING
|
|
# =====================================================
|
|
|
|
hasil_stemming = stemming(
|
|
hasil_stopword
|
|
)
|
|
|
|
|
|
# =====================================================
|
|
# 7. TEXT PREPROCESSED
|
|
# =====================================================
|
|
|
|
hasil_preprocessed = " ".join(
|
|
hasil_stemming
|
|
)
|
|
|
|
|
|
# =====================================================
|
|
# RETURN HASIL SETIAP TAHAP
|
|
# =====================================================
|
|
|
|
return {
|
|
|
|
"case_folding":
|
|
hasil_case_folding,
|
|
|
|
"normalisasi":
|
|
hasil_normalisasi,
|
|
|
|
"cleaning":
|
|
hasil_cleaning,
|
|
|
|
"tokenization":
|
|
" ".join(
|
|
hasil_tokenization
|
|
),
|
|
|
|
"stopword_removal":
|
|
" ".join(
|
|
hasil_stopword
|
|
),
|
|
|
|
"stemming":
|
|
" ".join(
|
|
hasil_stemming
|
|
),
|
|
|
|
"text_preprocessed":
|
|
hasil_preprocessed
|
|
} |