revisi model
This commit is contained in:
parent
dea2bc8a2b
commit
68319b9d23
|
|
@ -1116,7 +1116,7 @@ yang
|
|||
selagi
|
||||
kebijakan
|
||||
pemerintah
|
||||
komdigi
|
||||
|
||||
kementerian
|
||||
kemendag
|
||||
aturan
|
||||
|
|
|
|||
Binary file not shown.
Binary file not shown.
|
|
@ -1 +1 @@
|
|||
{"activated": false, "updated_at": "2026-05-22T16:01:11.643950+00:00"}
|
||||
{"activated": true, "updated_at": "2026-05-29T05:53:53.439233+00:00"}
|
||||
|
|
@ -3,17 +3,24 @@ preprocessing_page.py
|
|||
=====================
|
||||
Halaman Bersihkan Data — NLP Pipeline 5 Tahap.
|
||||
|
||||
PIPELINE 5 TAHAP (selaras dengan sentiment_service.py):
|
||||
PIPELINE 5 TAHAP:
|
||||
1. Case Folding — ubah semua huruf jadi lowercase
|
||||
2. Cleaning — hapus URL, mention, hashtag, angka, emoji, tanda baca
|
||||
3. Normalisasi — singkatan/slang → kata baku
|
||||
4. Stopword Removal — hapus kata umum; JAGA kata sentimen penting
|
||||
3. Normalisasi — singkatan/slang → kata baku (DARI FILE normalisasi)
|
||||
4. Stopword Removal — hapus kata umum (DARI FILE stopword); JAGA kata sentimen
|
||||
5. Stemming — bentuk dasar kata via Sastrawi ECS
|
||||
|
||||
Catatan: Tokenizing tidak ditampilkan sebagai tahap tersendiri karena:
|
||||
• Stopword removal & stemming sudah melakukan split() secara internal
|
||||
• TF-IDF melakukan tokenisasi sendiri saat inferensi
|
||||
→ Tokenizing adalah proses teknis, bukan tahap utama pipeline
|
||||
PERUBAHAN DARI VERSI SEBELUMNYA:
|
||||
- Normalisasi kini dimuat dari 'indonesian-normalisasi-slangword-complete.txt'
|
||||
(1.700+ entri), menggantikan dict hardcoded yang hanya ~60 entri.
|
||||
- Stopword kini murni dari 'indonesian-stopwords-complete.txt', ditambah
|
||||
noise Twitter yang spesifik — tidak ada penghapusan manual acak.
|
||||
- KATA_SENTIMEN_PENTING diperluas dengan kata domain e-commerce/ongkir.
|
||||
- Semua fungsi preprocessing menerima parameter eksplisit (tidak pakai global).
|
||||
|
||||
CATATAN PENTING:
|
||||
Pipeline ini HARUS IDENTIK dengan sentiment_service.py agar token yang
|
||||
dihasilkan di sini konsisten dengan token saat training model.
|
||||
"""
|
||||
|
||||
import streamlit as st
|
||||
|
|
@ -33,18 +40,15 @@ from timezone_utils import (
|
|||
get_timezone_name,
|
||||
)
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# TIMEZONE HELPERS
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
|
||||
def parse_dt(series):
|
||||
return parse_dt_with_tz(
|
||||
series,
|
||||
st.session_state.get("user_timezone", "WIB (UTC+7)")
|
||||
)
|
||||
|
||||
|
||||
def parse_crawled_dt(series):
|
||||
return parse_dt_with_source_tz(
|
||||
series,
|
||||
|
|
@ -52,7 +56,6 @@ def parse_crawled_dt(series):
|
|||
os.getenv("APP_TIMEZONE", "Asia/Jakarta")
|
||||
)
|
||||
|
||||
|
||||
def format_dt(value):
|
||||
if value is None or pd.isna(value):
|
||||
return "Belum ada"
|
||||
|
|
@ -96,63 +99,224 @@ def _sync_dynamic_period():
|
|||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# PREPROCESSING PIPELINE — 5 TAHAP
|
||||
# PENTING: Pipeline ini harus IDENTIK dengan sentiment_service.py
|
||||
# agar token yang dihasilkan konsisten dengan training model.
|
||||
#
|
||||
# URUTAN:
|
||||
# 1. Case Folding → lowercase dulu sebelum cleaning
|
||||
# 2. Cleaning → hapus noise setelah lowercase
|
||||
# 3. Normalisasi → singkatan/slang → kata baku
|
||||
# 4. Stopword Removal → buang kata umum, jaga kata sentimen
|
||||
# 5. Stemming → bentuk dasar via Sastrawi ECS
|
||||
# ┌─────────────────────────────────────────────────────┐
|
||||
# │ PENTING: Pipeline ini HARUS identik dengan │
|
||||
# │ sentiment_service.py agar token konsisten! │
|
||||
# │ │
|
||||
# │ URUTAN WAJIB: │
|
||||
# │ 1. Case Folding → lowercase dulu │
|
||||
# │ 2. Cleaning → hapus noise setelah lowercase │
|
||||
# │ 3. Normalisasi → slang→baku setelah bersih │
|
||||
# │ 4. Stopword → buang kata umum, jaga sentimen │
|
||||
# │ 5. Stemming → bentuk dasar via Sastrawi ECS │
|
||||
# └─────────────────────────────────────────────────────┘
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
# ───────────────────────────────────────────────────────────
|
||||
# KATA SENTIMEN PENTING
|
||||
# Kata-kata ini WAJIB DIJAGA dan tidak boleh dihapus saat
|
||||
# stopword removal, meskipun ada di file stopword.
|
||||
#
|
||||
# Kenapa perlu? Karena file stopword mengandung kata seperti
|
||||
# "tidak", "belum", "sangat" yang justru krusial untuk
|
||||
# menentukan sentimen positif/negatif suatu kalimat.
|
||||
# ───────────────────────────────────────────────────────────
|
||||
KATA_SENTIMEN_PENTING = {
|
||||
# Negasi
|
||||
# ── Negasi (pembalik makna kalimat) ──────────────────
|
||||
# "tidak bagus" ≠ "bagus" → "tidak" wajib ada
|
||||
"tidak", "bukan", "jangan", "kurang", "belum", "tanpa",
|
||||
# Positif
|
||||
# ── Intensitas (penguat/pelemah sentimen) ─────────────
|
||||
# "sangat bagus" lebih positif dari "bagus" saja
|
||||
"sangat", "banget", "sekali", "paling", "amat",
|
||||
"luar", "biasa", # ← "luar biasa" = dua token, keduanya dijaga
|
||||
# ── Positif umum ──────────────────────────────────────
|
||||
"keren", "bagus", "mantap", "setuju", "dukung", "mendukung",
|
||||
"andal", "handal", "gercep", "bangga", "senang", "suka",
|
||||
"baik", "benar", "tepat", "oke",
|
||||
"baik", "benar", "tepat", "oke", "puas",
|
||||
"sejahtera", "berkembang", "maju", "inovatif",
|
||||
"tegas", "sigap", "tanggap", "adil", "bijak", "bermanfaat",
|
||||
"untung", "berhasil", "sukses", "solusi", "manfaat",
|
||||
"berguna", "membantu", "bantu", "pro", "lanjut",
|
||||
"sangat", "banget", "sekali", "paling", "amat", "luar", "biasa",
|
||||
# Negatif
|
||||
# ── Positif domain e-commerce / ongkir ────────────────
|
||||
"gratis", "murah", "hemat", "terjangkau", "cepat",
|
||||
"aman", "mudah", "praktis", "terpercaya",
|
||||
# ── Negatif umum ──────────────────────────────────────
|
||||
"kecewa", "buruk", "jelek", "parah", "gagal", "hancur",
|
||||
"rusak", "bohong", "tipu", "korupsi",
|
||||
# Emosi
|
||||
"marah", "sedih", "khawatir",
|
||||
# ── Negatif domain e-commerce / ongkir ────────────────
|
||||
"mahal", "lambat", "lelet", "ribet", "susah", "repot",
|
||||
"rugi", "boros",
|
||||
# ── Emosi ─────────────────────────────────────────────
|
||||
"marah", "sedih", "khawatir", "kecewa",
|
||||
}
|
||||
|
||||
|
||||
def _load_stopwords():
|
||||
# ───────────────────────────────────────────────────────────
|
||||
# LOAD NORMALIZATION DARI FILE
|
||||
# File: indonesian-normalisasi-slangword-complete.txt
|
||||
# Format per baris: slang,kata_baku
|
||||
# Contoh: gk,tidak | ongkir,ongkos kirim | free,gratis
|
||||
# ───────────────────────────────────────────────────────────
|
||||
def _load_normalization() -> dict:
|
||||
"""
|
||||
Muat kamus normalisasi dari file eksternal.
|
||||
|
||||
KENAPA DARI FILE?
|
||||
File berisi 1.700+ pasang slang→baku yang jauh lebih lengkap
|
||||
dibanding dict hardcoded. Dengan ini, kata seperti:
|
||||
gk/ga/gak/kagak/ngga → semua jadi "tidak"
|
||||
bgt/bngt/bget → semua jadi "sangat"
|
||||
ongkir/ongkr → jadi "ongkos kirim"
|
||||
...dan ribuan kasus lainnya tertangani otomatis.
|
||||
|
||||
Setelah file dimuat, override dengan entri khusus domain
|
||||
(nama platform, singkatan kebijakan) yang mungkin belum ada
|
||||
di file generik.
|
||||
"""
|
||||
norm_file = "indonesian-normalisasi-slangword-complete.txt"
|
||||
norm_dict: dict = {}
|
||||
|
||||
try:
|
||||
with open(norm_file, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
# Split hanya pada koma pertama — nilai bisa mengandung koma
|
||||
# Contoh: "on the way, sedang di jalan,dijalan" → split jadi 2 bagian
|
||||
parts = line.split(",", 1)
|
||||
if len(parts) != 2:
|
||||
continue
|
||||
# Bersihkan tanda kutip liar di awal/akhir (ada di beberapa baris file)
|
||||
slang = parts[0].strip().strip("'\"").lower()
|
||||
normal = parts[1].strip().lower()
|
||||
if slang and normal:
|
||||
norm_dict[slang] = normal
|
||||
except FileNotFoundError:
|
||||
# Jika file tidak ditemukan, lanjut dengan dict kosong.
|
||||
# Entri domain di bawah tetap akan ditambahkan.
|
||||
st.warning(
|
||||
"⚠️ File normalisasi tidak ditemukan: "
|
||||
f"'{norm_file}'. Hanya entri domain yang aktif."
|
||||
)
|
||||
|
||||
# ── Override khusus domain ───────────────────────────
|
||||
# Entri ini menimpa file generik karena domain spesifik
|
||||
# membutuhkan perlakuan khusus (nama platform tidak diubah,
|
||||
# singkatan kebijakan punya padanan resmi, dll.)
|
||||
DOMAIN_OVERRIDES: dict = {
|
||||
# Nama platform — pertahankan apa adanya
|
||||
"shopee": "shopee",
|
||||
"tokopedia": "tokopedia",
|
||||
"lazada": "lazada",
|
||||
"tiktok": "tiktok",
|
||||
"bukalapak": "bukalapak",
|
||||
"blibli": "blibli",
|
||||
# Logistik — pertahankan apa adanya
|
||||
"sicepat": "sicepat",
|
||||
"jne": "jne",
|
||||
"jnt": "jnt",
|
||||
"anteraja": "anteraja",
|
||||
"ninja": "ninja",
|
||||
# Ongkir & belanja
|
||||
"freeongkir": "gratis ongkos kirim",
|
||||
"gratisongkir": "gratis ongkos kirim",
|
||||
"ongkir": "ongkos kirim",
|
||||
"ongkr": "ongkos kirim",
|
||||
"bykrm": "biaya kirim",
|
||||
"biayakirim": "biaya pengiriman",
|
||||
# Kebijakan & lembaga
|
||||
"komdigi": "komdigi",
|
||||
"kemendag": "kementerian perdagangan",
|
||||
"kominfo": "kementerian komunikasi",
|
||||
# E-commerce umum
|
||||
"ecommerce": "e commerce",
|
||||
"marketplace": "marketplace",
|
||||
"seller": "penjual",
|
||||
"buyer": "pembeli",
|
||||
"online": "online",
|
||||
}
|
||||
norm_dict.update(DOMAIN_OVERRIDES)
|
||||
|
||||
return norm_dict
|
||||
|
||||
|
||||
# ───────────────────────────────────────────────────────────
|
||||
# LOAD STOPWORDS DARI FILE
|
||||
# File: indonesian-stopwords-complete.txt
|
||||
# Format: satu kata per baris
|
||||
# ───────────────────────────────────────────────────────────
|
||||
def _load_stopwords() -> set:
|
||||
"""
|
||||
Muat daftar stopword dari file eksternal.
|
||||
|
||||
PROSES SETELAH MUAT FILE:
|
||||
1. Hapus KATA_SENTIMEN_PENTING dari daftar
|
||||
→ Agar "tidak", "belum", "sangat", dll. tidak ikut dibuang
|
||||
2. Tambahkan noise Twitter/sosmed yang memang harus dibuang
|
||||
→ "rt", "amp", sisa URL, suara tawa, partikel informal
|
||||
|
||||
KENAPA DARI FILE?
|
||||
File berisi 700+ stopword Indonesia yang lebih lengkap dan
|
||||
terstandar dibanding daftar manual. Kita tidak perlu menambah/
|
||||
mengurangi secara manual kecuali untuk dua kategori di atas.
|
||||
"""
|
||||
stopword_file = "indonesian-stopwords-complete.txt"
|
||||
base = set()
|
||||
base: set = set()
|
||||
|
||||
try:
|
||||
with open(stopword_file, "r", encoding="utf-8") as f:
|
||||
base = set(f.read().splitlines())
|
||||
for line in f:
|
||||
word = line.strip().lower()
|
||||
if word:
|
||||
base.add(word)
|
||||
except FileNotFoundError:
|
||||
# Fallback minimal — cukup untuk tetap jalan
|
||||
st.warning(
|
||||
"⚠️ File stopword tidak ditemukan: "
|
||||
f"'{stopword_file}'. Menggunakan daftar minimal."
|
||||
)
|
||||
base = {
|
||||
"yang", "dan", "di", "ke", "dari", "ini", "itu",
|
||||
"dengan", "untuk", "pada", "adalah", "oleh", "ada",
|
||||
"ya", "akan", "atau", "juga", "sama", "karena",
|
||||
"jika", "sudah", "telah",
|
||||
"jika", "sudah", "telah", "jadi", "bisa",
|
||||
}
|
||||
|
||||
# ── Langkah 1: Lindungi kata sentimen ────────────────
|
||||
# Beberapa kata sentimen penting ADA di file stopword
|
||||
# (misal: "tidak", "belum", "sangat", "paling", "kurang").
|
||||
# Kita HAPUS dari stopword agar tidak ikut dibuang.
|
||||
for kata in KATA_SENTIMEN_PENTING:
|
||||
base.discard(kata)
|
||||
base.update({
|
||||
"rt", "amp", "https", "http", "co", "t",
|
||||
"wkwk", "wkwkwk", "haha", "hehe", "xixi",
|
||||
"yg", "dgn", "utk", "dr", "krn", "tp", "jd", "sdh",
|
||||
"aja", "doang", "nih", "sih", "dong", "deh",
|
||||
"loh", "lah", "tuh", "kak", "gan",
|
||||
})
|
||||
|
||||
# ── Langkah 2: Tambah noise Twitter/sosmed ────────────
|
||||
# Ini bukan stopword bahasa Indonesia biasa, tapi noise
|
||||
# yang sangat sering muncul di tweet dan tidak bermakna.
|
||||
TWITTER_NOISE: set = {
|
||||
# Artefak Twitter
|
||||
"rt", "amp",
|
||||
# Sisa URL setelah cleaning (kadang lolos)
|
||||
"https", "http", "co", "pic",
|
||||
# Suara tawa (tidak bermakna untuk sentimen)
|
||||
"wkwk", "wkwkwk", "wkwkwkwk",
|
||||
"haha", "hahaha", "hehe", "hihi", "huhu", "xixi",
|
||||
# Partikel informal yang tidak bermakna
|
||||
"nih", "sih", "dong", "deh", "loh", "lah", "tuh",
|
||||
"kak", "gan", "bro", "sob", "min",
|
||||
}
|
||||
base.update(TWITTER_NOISE)
|
||||
|
||||
return base
|
||||
|
||||
|
||||
# ───────────────────────────────────────────────────────────
|
||||
# LOAD STEMMER
|
||||
# ───────────────────────────────────────────────────────────
|
||||
def _load_stemmer():
|
||||
"""Muat stemmer Sastrawi. Return None jika tidak tersedia."""
|
||||
try:
|
||||
from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
|
||||
return StemmerFactory().create_stemmer()
|
||||
|
|
@ -160,131 +324,198 @@ def _load_stemmer():
|
|||
return None
|
||||
|
||||
|
||||
NORMALISASI = {
|
||||
# Negasi
|
||||
"gk": "tidak", "ga": "tidak", "gak": "tidak",
|
||||
"nggak": "tidak", "ngga": "tidak", "tdk": "tidak",
|
||||
"tak": "tidak", "enggak": "tidak", "engga": "tidak",
|
||||
"kagak": "tidak", "kaga": "tidak", "ndak": "tidak",
|
||||
"gkk": "tidak", "ngak": "tidak",
|
||||
# Kata ganti
|
||||
"yg": "yang", "dgn": "dengan", "utk": "untuk",
|
||||
"org": "orang", "krn": "karena", "dr": "dari",
|
||||
"sm": "sama", "pd": "pada", "dlm": "dalam",
|
||||
"bwt": "buat", "trm": "terima",
|
||||
# Verba
|
||||
"tp": "tapi", "tpi": "tapi", "jd": "jadi",
|
||||
"sdh": "sudah", "blm": "belum", "emg": "memang",
|
||||
"emang": "memang", "gimana": "bagaimana",
|
||||
"gitu": "begitu", "gini": "begini",
|
||||
"udah": "sudah", "udh": "sudah",
|
||||
"mau": "mau",
|
||||
# Intensitas
|
||||
"bgt": "banget", "bngt": "banget", "bget": "banget", "bgtt": "banget",
|
||||
# Positif informal
|
||||
"bener": "benar", "beneran": "benar",
|
||||
"mantep": "mantap", "mntap": "mantap",
|
||||
"kece": "keren", "kece bgt": "keren banget",
|
||||
"cucok": "cocok", "cucuk": "cocok",
|
||||
"cakep": "bagus", "oke bgt": "oke banget",
|
||||
"sip": "baik", "siipp": "baik",
|
||||
"top": "terbaik", "topbgt": "terbaik banget",
|
||||
"jos": "bagus", "josss": "bagus",
|
||||
"goks": "luar biasa",
|
||||
"setujuu": "setuju", "stuju": "setuju",
|
||||
"dukung": "dukung",
|
||||
"proud": "bangga",
|
||||
"mantul": "mantap betul",
|
||||
# Negatif informal
|
||||
"ancur": "hancur", "ancrr": "hancur",
|
||||
"parahh": "parah", "parahhh": "parah",
|
||||
"gagall": "gagal",
|
||||
"ngaco": "tidak benar",
|
||||
"ngasal": "tidak benar",
|
||||
"receh": "tidak penting",
|
||||
"gaje": "tidak jelas",
|
||||
"asal": "sembarangan",
|
||||
# Domain
|
||||
"ongkir": "ongkos kirim",
|
||||
"freeongkir": "gratis ongkos kirim",
|
||||
"gratisongkir": "gratis ongkos kirim",
|
||||
"free": "gratis",
|
||||
"ecommerce": "e commerce",
|
||||
"komdigi": "komdigi",
|
||||
"subsidi": "subsidi",
|
||||
"marketplace": "marketplace",
|
||||
"seller": "penjual",
|
||||
"buyer": "pembeli",
|
||||
"online": "online",
|
||||
"shopee": "shopee",
|
||||
"tokopedia": "tokopedia",
|
||||
"lazada": "lazada",
|
||||
"tiktok": "tiktok",
|
||||
}
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# FUNGSI 5 TAHAP PREPROCESSING
|
||||
# Setiap fungsi bertanggung jawab SATU tahap saja.
|
||||
# Input & output setiap tahap dijelaskan di docstring.
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
# ── Tahap 1: Case Folding ───────────────────────────────────────────────────
|
||||
def step1_case_folding(text: str) -> str:
|
||||
"""
|
||||
TAHAP 1 — CASE FOLDING
|
||||
Input : teks asli (campuran huruf besar/kecil)
|
||||
Output: semua huruf jadi lowercase
|
||||
|
||||
Kenapa pertama?
|
||||
Agar tahap berikutnya (cleaning, normalisasi) bekerja
|
||||
secara konsisten — regex dan dict lookup case-sensitive.
|
||||
Contoh: "Gratis" → "gratis", "ONGKIR" → "ongkir"
|
||||
"""
|
||||
return str(text).lower()
|
||||
|
||||
|
||||
# ── Tahap 2: Cleaning ───────────────────────────────────────────────────────
|
||||
def step2_cleaning(text: str) -> str:
|
||||
"""
|
||||
TAHAP 2 — CLEANING
|
||||
Input : teks lowercase
|
||||
Output: teks bersih dari semua elemen noise
|
||||
|
||||
Urutan pembersihan PENTING:
|
||||
1. URL dulu (sebelum @ dan # agar tidak salah potong)
|
||||
2. Mention (@username)
|
||||
3. Hashtag (#topik)
|
||||
4. Angka
|
||||
5. Emoji & simbol unicode
|
||||
6. Tanda baca
|
||||
7. Karakter non-latin (huruf Arab, Cina, dll.)
|
||||
8. Spasi berlebih
|
||||
"""
|
||||
# 1. Hapus URL (http, https, www)
|
||||
text = re.sub(r"http\S+|www\S+|https\S+", "", text)
|
||||
# 2. Hapus mention Twitter (@username)
|
||||
text = re.sub(r"@\w+", "", text)
|
||||
# 3. Hapus hashtag (#topik)
|
||||
text = re.sub(r"#\w+", "", text)
|
||||
# 4. Hapus angka dan digit
|
||||
text = re.sub(r"\d+", "", text)
|
||||
# 5. Hapus emoji & simbol unicode (berbagai range)
|
||||
text = re.sub(
|
||||
r"[\U00010000-\U0010ffff"
|
||||
r"\U0001F600-\U0001F64F"
|
||||
r"\U0001F300-\U0001F5FF"
|
||||
r"\U0001F680-\U0001F6FF"
|
||||
r"\U0001F1E0-\U0001F1FF"
|
||||
r"\u2600-\u26FF\u2700-\u27BF"
|
||||
r"]+", "", text, flags=re.UNICODE
|
||||
r"["
|
||||
r"\U00010000-\U0010ffff" # Suplemen karakter unicode
|
||||
r"\U0001F600-\U0001F64F" # Emotikon wajah
|
||||
r"\U0001F300-\U0001F5FF" # Simbol & piktogram
|
||||
r"\U0001F680-\U0001F6FF" # Transport & peta
|
||||
r"\U0001F1E0-\U0001F1FF" # Bendera negara
|
||||
r"\u2600-\u26FF" # Simbol campuran
|
||||
r"\u2700-\u27BF" # Dingbats
|
||||
r"]+",
|
||||
"", text, flags=re.UNICODE
|
||||
)
|
||||
# 6. Hapus tanda baca (.,!?;: dll.)
|
||||
text = text.translate(str.maketrans("", "", string.punctuation))
|
||||
# 7. Hapus karakter non-latin (hanya sisakan huruf a-z dan spasi)
|
||||
text = re.sub(r"[^a-zA-Z\s]", "", text)
|
||||
# 8. Normalisasi spasi berlebih → satu spasi, lalu strip
|
||||
text = re.sub(r"\s+", " ", text).strip()
|
||||
return text
|
||||
|
||||
|
||||
# ── Tahap 3: Normalisasi ────────────────────────────────────────────────────
|
||||
def step3_normalization(text: str) -> str:
|
||||
return " ".join(NORMALISASI.get(word, word) for word in text.split())
|
||||
def step3_normalization(text: str, norm_dict: dict) -> str:
|
||||
"""
|
||||
TAHAP 3 — NORMALISASI
|
||||
Input : teks bersih (sudah case fold + cleaning)
|
||||
norm_dict : kamus {slang: kata_baku} dari file
|
||||
Output: teks dengan slang/singkatan sudah diganti kata baku
|
||||
|
||||
Cara kerja: token per token (word by word).
|
||||
Setiap token dicari di norm_dict.
|
||||
Jika ada → ganti. Jika tidak ada → biarkan.
|
||||
|
||||
Contoh:
|
||||
"gk bs ongkir" → "tidak bisa ongkos kirim"
|
||||
"mantep bgt" → "mantap sangat"
|
||||
|
||||
Kenapa setelah Cleaning?
|
||||
Karena slang di file ditulis dalam bentuk sudah lowercase
|
||||
dan sudah tanpa tanda baca. Jika normalisasi dilakukan
|
||||
sebelum cleaning, banyak entri tidak cocok.
|
||||
"""
|
||||
tokens = text.split()
|
||||
normalized = [norm_dict.get(token, token) for token in tokens]
|
||||
return " ".join(normalized)
|
||||
|
||||
|
||||
# ── Tahap 4: Stopword Removal ───────────────────────────────────────────────
|
||||
def step4_stopword_removal(tokens: list, stopwords: set) -> list:
|
||||
return [
|
||||
w for w in tokens
|
||||
if (w not in stopwords or w in KATA_SENTIMEN_PENTING) and len(w) > 2
|
||||
]
|
||||
"""
|
||||
TAHAP 4 — STOPWORD REMOVAL
|
||||
Input : list token (hasil split dari teks ternormalisasi)
|
||||
stopwords : set kata yang harus dibuang (dari file)
|
||||
Output: list token bersih
|
||||
|
||||
ATURAN PENYARINGAN (prioritas urutan):
|
||||
1. JAGA token yang ada di KATA_SENTIMEN_PENTING
|
||||
→ meskipun juga ada di stopwords, tetap disimpan
|
||||
2. BUANG token yang ada di stopwords
|
||||
3. BUANG token dengan panjang ≤ 2 karakter
|
||||
→ menghilangkan sisa noise seperti "rt", "yg", "di"
|
||||
→ PENGECUALIAN: token di KATA_SENTIMEN_PENTING tetap disimpan
|
||||
walau ≤ 2 karakter (contoh: "ok" jika masuk sentimen)
|
||||
|
||||
Kenapa setelah Normalisasi?
|
||||
Agar "gak" yang sudah dinormalisasi jadi "tidak" tidak ikut
|
||||
dibuang — "tidak" dilindungi di KATA_SENTIMEN_PENTING.
|
||||
"""
|
||||
result = []
|
||||
for token in tokens:
|
||||
# Prioritas 1: selalu simpan jika kata sentimen penting
|
||||
if token in KATA_SENTIMEN_PENTING:
|
||||
result.append(token)
|
||||
continue
|
||||
# Prioritas 2: buang jika stopword
|
||||
if token in stopwords:
|
||||
continue
|
||||
# Prioritas 3: buang jika terlalu pendek (noise)
|
||||
if len(token) <= 2:
|
||||
continue
|
||||
# Lolos semua filter → simpan
|
||||
result.append(token)
|
||||
return result
|
||||
|
||||
|
||||
# ── Tahap 5: Stemming ───────────────────────────────────────────────────────
|
||||
def step5_stemming(tokens: list, stemmer) -> list:
|
||||
"""
|
||||
TAHAP 5 — STEMMING
|
||||
Input : list token setelah stopword removal
|
||||
stemmer : objek Sastrawi (atau None)
|
||||
Output: list token dalam bentuk kata dasar
|
||||
|
||||
Algoritma: Enhanced Confix Stripping (ECS) via Sastrawi
|
||||
Contoh:
|
||||
"pengiriman" → "kirim"
|
||||
"pembatasan" → "batas"
|
||||
"berlari" → "lari"
|
||||
"makanan" → "makan"
|
||||
|
||||
Jika stemmer None (Sastrawi tidak terinstal), token dikembalikan
|
||||
apa adanya tanpa error.
|
||||
"""
|
||||
if stemmer is None:
|
||||
return tokens
|
||||
return [stemmer.stem(w) for w in tokens]
|
||||
return [stemmer.stem(token) for token in tokens]
|
||||
|
||||
|
||||
def full_preprocessing(text: str, stopwords: set, stemmer):
|
||||
# ───────────────────────────────────────────────────────────
|
||||
# FUNGSI UTAMA — JALANKAN SEMUA 5 TAHAP
|
||||
# ───────────────────────────────────────────────────────────
|
||||
def full_preprocessing(
|
||||
text: str,
|
||||
stopwords: set,
|
||||
stemmer,
|
||||
norm_dict: dict,
|
||||
) -> dict:
|
||||
"""
|
||||
Jalankan 5 tahap preprocessing dan kembalikan dict hasil setiap tahap.
|
||||
Jalankan 5 tahap preprocessing secara berurutan dan kembalikan
|
||||
hasil setiap tahap sebagai dict (untuk ditampilkan di tabel).
|
||||
|
||||
URUTAN TAHAP:
|
||||
1. Case Folding → lowercase
|
||||
2. Cleaning → hapus noise
|
||||
3. Normalisasi → normalisasi kata
|
||||
4. Stopword Removal → buang stopword (split() dilakukan internal)
|
||||
5. Stemming → bentuk dasar
|
||||
Parameter:
|
||||
text : teks tweet asli
|
||||
stopwords : set stopword (dari _load_stopwords)
|
||||
stemmer : objek Sastrawi (dari _load_stemmer)
|
||||
norm_dict : kamus normalisasi (dari _load_normalization)
|
||||
|
||||
Return dict berisi:
|
||||
setelah_casefolding : hasil Tahap 1
|
||||
setelah_cleaning : hasil Tahap 2
|
||||
setelah_normalisasi : hasil Tahap 3
|
||||
setelah_stopword : hasil Tahap 4 (joined string)
|
||||
clean_text : hasil akhir Tahap 5 (joined string)
|
||||
_tokens_clean : hasil Tahap 5 sebagai list (untuk analisis)
|
||||
"""
|
||||
s1_fold = step1_case_folding(text)
|
||||
s2_clean = step2_cleaning(s1_fold)
|
||||
s3_norm = step3_normalization(s2_clean)
|
||||
s4_filtered = step4_stopword_removal(s3_norm.split(), stopwords)
|
||||
s5_stemmed = step5_stemming(s4_filtered, stemmer)
|
||||
# Tahap 1 — Case Folding
|
||||
s1_fold = step1_case_folding(text)
|
||||
|
||||
# Tahap 2 — Cleaning
|
||||
s2_clean = step2_cleaning(s1_fold)
|
||||
|
||||
# Tahap 3 — Normalisasi (perlu norm_dict)
|
||||
s3_norm = step3_normalization(s2_clean, norm_dict)
|
||||
|
||||
# Tahap 4 — Stopword Removal (split → filter → simpan sebagai list)
|
||||
s4_tokens = s3_norm.split()
|
||||
s4_filtered = step4_stopword_removal(s4_tokens, stopwords)
|
||||
|
||||
# Tahap 5 — Stemming
|
||||
s5_stemmed = step5_stemming(s4_filtered, stemmer)
|
||||
|
||||
return {
|
||||
"setelah_casefolding": s1_fold,
|
||||
|
|
@ -292,7 +523,7 @@ def full_preprocessing(text: str, stopwords: set, stemmer):
|
|||
"setelah_normalisasi": s3_norm,
|
||||
"setelah_stopword": " ".join(s4_filtered),
|
||||
"clean_text": " ".join(s5_stemmed),
|
||||
"_tokens_clean": s5_stemmed,
|
||||
"_tokens_clean": s5_stemmed, # list, untuk Counter frekuensi kata
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -443,7 +674,7 @@ def _render_page_header():
|
|||
# PIPELINE STEPS CARDS
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
|
||||
def _render_pipeline_steps(stemmer_ok):
|
||||
def _render_pipeline_steps(stemmer_ok: bool, norm_count: int, sw_count: int):
|
||||
steps = [
|
||||
{
|
||||
"num": "01", "anim": "pipe-1",
|
||||
|
|
@ -456,8 +687,8 @@ def _render_pipeline_steps(stemmer_ok):
|
|||
'"ONGKIR" → "ongkir"',
|
||||
'"KEREN" → "keren"',
|
||||
"Seluruh karakter → huruf kecil",
|
||||
"Dilakukan pertama agar cleaning konsisten",
|
||||
"Basis untuk normalisasi & stopword",
|
||||
"Dilakukan PERTAMA agar regex & dict konsisten",
|
||||
"Fondasi seluruh tahap berikutnya",
|
||||
],
|
||||
},
|
||||
{
|
||||
|
|
@ -479,30 +710,30 @@ def _render_pipeline_steps(stemmer_ok):
|
|||
"num": "03", "anim": "pipe-3",
|
||||
"icon": "🔄", "color": "#16a34a", "dark": "#14532d",
|
||||
"bg": "linear-gradient(135deg,#f0fdf4,#dcfce7)", "border": "#86efac",
|
||||
"title": 'Normalisasi <span class="fix-badge">✦ Diperluas</span>',
|
||||
"desc": "Mengubah kata tidak baku, singkatan, dan slang menjadi kata baku.",
|
||||
"title": f'Normalisasi <span class="fix-badge">✦ {norm_count:,} entri</span>',
|
||||
"desc": "Mengubah kata tidak baku, singkatan, dan slang menjadi kata baku (dari file).",
|
||||
"items": [
|
||||
"gk/ga/gak/kagak → tidak",
|
||||
"gk/ga/gak/kagak/ngga → tidak",
|
||||
"bgt/bngt/bget → sangat",
|
||||
"ongkir → ongkos kirim",
|
||||
"mantep → mantap",
|
||||
"kece → keren",
|
||||
"jos/josss → bagus",
|
||||
"free → gratis",
|
||||
f"Total: {norm_count:,} pasang slang→baku dimuat",
|
||||
],
|
||||
},
|
||||
{
|
||||
"num": "04", "anim": "pipe-4",
|
||||
"icon": "🚫", "color": "#ea580c", "dark": "#7c2d12",
|
||||
"bg": "linear-gradient(135deg,#fff7ed,#ffedd5)", "border": "#fed7aa",
|
||||
"title": 'Stopword Removal <span class="fix-badge">✦ Diperbaiki</span>',
|
||||
"desc": "Membuang kata umum; kata sentimen penting DIJAGA.",
|
||||
"title": f'Stopword Removal <span class="fix-badge">✦ {sw_count:,} kata</span>',
|
||||
"desc": "Membuang kata umum dari file; kata sentimen DIJAGA.",
|
||||
"items": [
|
||||
"Hapus kata umum (dan, di, ke...)",
|
||||
"Hapus token < 3 karakter",
|
||||
"JAGA negasi: tidak, bukan, jangan",
|
||||
"JAGA positif: keren, bagus, mantap",
|
||||
"JAGA evaluatif: setuju, dukung, bijak",
|
||||
"JAGA intensitas: banget, sangat, sekali",
|
||||
f"{sw_count:,} stopword dimuat dari file",
|
||||
"JAGA negasi: tidak, bukan, jangan, belum",
|
||||
"JAGA positif: keren, bagus, mantap, gratis",
|
||||
"JAGA negatif: kecewa, buruk, gagal, mahal, mending, malah",
|
||||
"JAGA intensitas: sangat, banget, sekali",
|
||||
"Hapus token ≤ 2 karakter (noise)",
|
||||
],
|
||||
},
|
||||
{
|
||||
|
|
@ -678,7 +909,6 @@ def _render_live_example(df_c):
|
|||
"Contoh tweet acak dari dataset — refresh halaman untuk contoh berbeda"
|
||||
)
|
||||
|
||||
# Urutan tampilan sesuai pipeline: CF → Clean → Norm → Stop → Stem
|
||||
steps_ex = [
|
||||
("📄 Teks Asli", "text_asli", "#0f172a", "#f8fafc", "#e2e8f0"),
|
||||
("① Setelah Case Folding", "setelah_casefolding", "#0c4a6e", "#eff6ff", "#bfdbfe"),
|
||||
|
|
@ -727,10 +957,14 @@ def _render_top_words_chart(df_c):
|
|||
)
|
||||
|
||||
if "_tokens_clean" in df_c.columns:
|
||||
all_words = [w for tokens in df_c["_tokens_clean"] for w in (tokens if isinstance(tokens, list) else [])]
|
||||
all_words = [
|
||||
w for tokens in df_c["_tokens_clean"]
|
||||
for w in (tokens if isinstance(tokens, list) else [])
|
||||
]
|
||||
else:
|
||||
all_words = " ".join(df_c["clean_text"].fillna("")).split()
|
||||
|
||||
# Filter minimum 3 karakter (konsisten dengan step4)
|
||||
filtered_words = [w for w in all_words if len(w) > 2]
|
||||
word_freq = Counter(filtered_words).most_common(20)
|
||||
|
||||
|
|
@ -821,19 +1055,29 @@ def show():
|
|||
unsafe_allow_html=True
|
||||
)
|
||||
|
||||
# ── Pipeline Overview ─────────────────────────────────────
|
||||
# ── Muat resource preprocessing (sekali per session) ──────
|
||||
# Semua tiga resource dimuat di sini, bukan di dalam loop,
|
||||
# agar tidak memuat ulang setiap tweet.
|
||||
stemmer = _load_stemmer()
|
||||
stopwords = _load_stopwords()
|
||||
norm_dict = _load_normalization()
|
||||
|
||||
# ── Pipeline Overview ──────────────────────────────────────
|
||||
_section_header(
|
||||
"🔬 Alur NLP Pipeline — 5 Tahap Preprocessing",
|
||||
"Setiap tweet diproses berurutan melalui 5 tahap sebelum siap dianalisis sentimennya"
|
||||
)
|
||||
|
||||
stemmer_tmp = _load_stemmer()
|
||||
_render_pipeline_steps(stemmer_tmp is not None)
|
||||
_render_pipeline_steps(
|
||||
stemmer_ok=stemmer is not None,
|
||||
norm_count=len(norm_dict),
|
||||
sw_count=len(stopwords),
|
||||
)
|
||||
_gap("sm")
|
||||
_render_flow_arrow()
|
||||
_gap("md")
|
||||
|
||||
# ── Load data ─────────────────────────────────────────────
|
||||
# ── Load data dari database ────────────────────────────────
|
||||
try:
|
||||
df_all = pd.read_sql("SELECT * FROM tweets ORDER BY created_at DESC", engine)
|
||||
if df_all.empty:
|
||||
|
|
@ -848,13 +1092,17 @@ def show():
|
|||
|
||||
s_dt = pd.Timestamp(start_date)
|
||||
e_dt = pd.Timestamp(end_date)
|
||||
df = df_all[(df_all["created_at"] >= s_dt) & (df_all["created_at"] <= e_dt)].copy()
|
||||
df = df_all[
|
||||
(df_all["created_at"] >= s_dt) & (df_all["created_at"] <= e_dt)
|
||||
].copy()
|
||||
|
||||
if df.empty:
|
||||
st.warning(f"⚠️ Tidak ada tweet dengan tanggal asli dalam periode {filter_label}.")
|
||||
return
|
||||
|
||||
# ── Cache preprocessing ───────────────────────────────────
|
||||
# ── Cache preprocessing ────────────────────────────────────
|
||||
# Cache key: kombinasi mode + periode + jumlah tweet di DB
|
||||
# Jika ada tweet baru → cache otomatis invalid → proses ulang
|
||||
total_tweets_in_db = get_tweet_count()
|
||||
latest_crawl_marker = get_latest_crawl_time() or "no-crawl"
|
||||
data_marker = (total_tweets_in_db, latest_crawl_marker)
|
||||
|
|
@ -864,6 +1112,7 @@ def show():
|
|||
f"{start_date}_{end_date}_{total_tweets_in_db}_{latest_crawl_marker}"
|
||||
)
|
||||
|
||||
# Bersihkan cache lama untuk mode/periode yang sudah tidak aktif
|
||||
for old_key in list(st.session_state.keys()):
|
||||
if old_key.startswith("pp5_") and old_key != cache_key:
|
||||
del st.session_state[old_key]
|
||||
|
|
@ -871,13 +1120,16 @@ def show():
|
|||
force_refresh = data_marker != st.session_state.get("_pp_last_data_marker")
|
||||
|
||||
if cache_key not in st.session_state or force_refresh:
|
||||
stemmer = _load_stemmer()
|
||||
stopwords = _load_stopwords()
|
||||
|
||||
with st.spinner("🧹 Menjalankan 5 tahap preprocessing…"):
|
||||
results = []
|
||||
for _, row in df.iterrows():
|
||||
r = full_preprocessing(row["text"], stopwords, stemmer)
|
||||
# Jalankan 5 tahap — norm_dict diteruskan sebagai parameter
|
||||
r = full_preprocessing(
|
||||
text = row["text"],
|
||||
stopwords = stopwords,
|
||||
stemmer = stemmer,
|
||||
norm_dict = norm_dict,
|
||||
)
|
||||
r["tweet_id"] = row.get("tweet_id", "")
|
||||
r["text_asli"] = row["text"]
|
||||
r["created_at"] = row["created_at"]
|
||||
|
|
@ -885,16 +1137,19 @@ def show():
|
|||
results.append(r)
|
||||
|
||||
df_c = pd.DataFrame(results)
|
||||
# Buang baris yang clean_text-nya kosong setelah semua 5 tahap
|
||||
df_c = df_c[df_c["clean_text"].str.strip().str.len() > 0].copy()
|
||||
# Reset index agar rapi
|
||||
df_c = df_c.reset_index(drop=True)
|
||||
|
||||
st.session_state[cache_key] = df_c
|
||||
st.session_state[cache_key + "_stemmer_ok"] = stemmer is not None
|
||||
st.session_state["_pp_last_data_marker"] = data_marker
|
||||
st.session_state[cache_key] = df_c
|
||||
st.session_state[cache_key + "_sw_ok"] = stemmer is not None
|
||||
st.session_state["_pp_last_data_marker"] = data_marker
|
||||
|
||||
df_c = st.session_state[cache_key]
|
||||
stemmer_ok = st.session_state.get(cache_key + "_stemmer_ok", False)
|
||||
stemmer_ok = st.session_state.get(cache_key + "_sw_ok", False)
|
||||
|
||||
# ── Statistik ─────────────────────────────────────────────
|
||||
# ── Statistik ──────────────────────────────────────────────
|
||||
removed = len(df) - len(df_c)
|
||||
|
||||
_section_header(
|
||||
|
|
@ -908,7 +1163,7 @@ def show():
|
|||
_render_live_example(df_c)
|
||||
_gap("lg")
|
||||
|
||||
# ── Tabel ─────────────────────────────────────────────────
|
||||
# ── Tabel ──────────────────────────────────────────────────
|
||||
_section_header(
|
||||
"📋 Tabel Perbandingan Teks per Tahap",
|
||||
f"{len(df_c):,} tweet · {filter_label} — scroll horizontal untuk lihat semua kolom"
|
||||
|
|
@ -918,7 +1173,7 @@ def show():
|
|||
df_c = df_c.copy()
|
||||
df_c["crawled_at"] = pd.NaT
|
||||
|
||||
# Kolom ditampilkan sesuai urutan pipeline: CF → Clean → Norm → Stop → Stem
|
||||
# Kolom ditampilkan sesuai urutan pipeline
|
||||
disp = df_c[[
|
||||
"tweet_id", "created_at", "crawled_at",
|
||||
"text_asli",
|
||||
|
|
@ -968,10 +1223,10 @@ def show():
|
|||
_render_top_words_chart(df_c)
|
||||
_gap("lg")
|
||||
|
||||
# ── Simpan ke session state ───────────────────────────────
|
||||
# ── Simpan ke session state untuk halaman sentimen ─────────
|
||||
st.session_state["preprocessed_df"] = df_c
|
||||
|
||||
# ── Download ──────────────────────────────────────────────
|
||||
# ── Download ───────────────────────────────────────────────
|
||||
st.markdown("""
|
||||
<div style="background:#ffffff;border:1.5px solid #e2e8f0;border-radius:14px;
|
||||
padding:1.1rem 1.2rem;box-shadow:0 2px 6px rgba(15,23,42,0.05);
|
||||
|
|
|
|||
|
|
@ -1,29 +1,8 @@
|
|||
"""
|
||||
sentiment_page.py
|
||||
=================
|
||||
Halaman Analisis Sentimen — diperbaiki untuk mengatasi semua prediksi negatif.
|
||||
|
||||
PERBAIKAN vs VERSI LAMA:
|
||||
─────────────────────────────────────────────────────────────────────────────
|
||||
1. MENGGUNAKAN sentiment_service.py (HYBRID CLASSIFIER)
|
||||
Tidak lagi memanggil model secara langsung dengan predict_batch sederhana.
|
||||
Sebaliknya menggunakan sentiment_service yang sudah mengimplementasikan:
|
||||
• Lexicon-based override untuk kelas POSITIF (yang tidak ada di model)
|
||||
• Negation handling (tidak bagus ≠ bagus)
|
||||
• Fallback ke model NB untuk negatif vs netral
|
||||
|
||||
2. PREPROCESSING SELARAS
|
||||
Memanggil preprocess_for_model() dari sentiment_service agar preprocessing
|
||||
yang digunakan untuk prediksi persis sama dengan yang dipakai saat training.
|
||||
|
||||
3. CONFIDENCE SCORE AKURAT
|
||||
Menggunakan confidence gabungan dari lexicon score + model probability,
|
||||
bukan hanya max probability dari model yang bias ke negatif.
|
||||
|
||||
4. LABEL MAPPING KONSISTEN
|
||||
Label dikembalikan selalu dalam format: 'Positif', 'Netral', 'Negatif'
|
||||
(kapital huruf pertama) agar konsisten di seluruh tampilan.
|
||||
─────────────────────────────────────────────────────────────────────────────
|
||||
PERBAIKAN: Hapus parameter key= dari semua st.container() karena tidak
|
||||
didukung di Streamlit versi lama. CSS styling tetap berjalan via class HTML.
|
||||
"""
|
||||
|
||||
import streamlit as st
|
||||
|
|
@ -44,7 +23,6 @@ from timezone_utils import (
|
|||
)
|
||||
import plotly.graph_objects as go
|
||||
|
||||
# ── Import sentiment_service (hybrid classifier) ──────────────────────────
|
||||
from sentiment_service import (
|
||||
preprocess_for_model,
|
||||
preprocess_untuk_lexicon,
|
||||
|
|
@ -118,40 +96,27 @@ def _sync_dynamic_period():
|
|||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Core prediction function (DIPERBAIKI)
|
||||
# Core prediction
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
|
||||
def predict_batch_hybrid(texts: list[str]) -> list[tuple[str, float]]:
|
||||
"""
|
||||
Prediksi sentimen menggunakan Hybrid Classifier dari sentiment_service.
|
||||
|
||||
Pipeline per tweet:
|
||||
1. preprocess_for_model() → teks untuk TF-IDF + NB
|
||||
2. preprocess_untuk_lexicon() → teks untuk pengecekan lexicon
|
||||
3. _hitung_skor_lexicon() → hitung sinyal positif/negatif kuat
|
||||
4. _klasifikasi_hybrid() → putuskan label + confidence
|
||||
|
||||
Return: list of (label, confidence)
|
||||
"""
|
||||
def predict_batch_hybrid(texts):
|
||||
results = []
|
||||
for text in texts:
|
||||
teks_model = preprocess_for_model(text)
|
||||
teks_lexicon = preprocess_untuk_lexicon(text)
|
||||
teks_lower = str(text).lower()
|
||||
|
||||
skor = _hitung_skor_lexicon(teks_lexicon)
|
||||
label, conf = _klasifikasi_hybrid(teks_model, skor, teks_lower)
|
||||
skor = _hitung_skor_lexicon(teks_lexicon)
|
||||
label, conf = _klasifikasi_hybrid(teks_model, skor, teks_lower)
|
||||
results.append((label, conf))
|
||||
return results
|
||||
|
||||
|
||||
def preprocess_single(text: str) -> str:
|
||||
"""Preprocess satu teks untuk disimpan ke kolom clean_text."""
|
||||
def preprocess_single(text):
|
||||
return preprocess_for_model(text)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Shared UI helpers
|
||||
# UI helpers
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
|
||||
def _section_header(title, subtitle=""):
|
||||
|
|
@ -173,6 +138,20 @@ def _section_gap(size="md"):
|
|||
st.markdown(f'<div style="height:{heights.get(size,"1.45rem")};"></div>', unsafe_allow_html=True)
|
||||
|
||||
|
||||
def _card_open(extra_style=""):
|
||||
"""Buka div card pengganti st.container(border=True)."""
|
||||
st.markdown(
|
||||
f'<div style="background:#ffffff;border:1.5px solid #e2e8f0;border-radius:12px;'
|
||||
f'box-shadow:0 2px 6px rgba(15,23,42,0.06);padding:1.1rem 1.2rem 1rem;'
|
||||
f'margin-bottom:0.5rem;{extra_style}">',
|
||||
unsafe_allow_html=True,
|
||||
)
|
||||
|
||||
|
||||
def _card_close():
|
||||
st.markdown('</div>', unsafe_allow_html=True)
|
||||
|
||||
|
||||
def _render_sentiment_styles():
|
||||
st.markdown("""
|
||||
<style>
|
||||
|
|
@ -185,31 +164,6 @@ section[data-testid="stMain"] * {
|
|||
[data-testid="stMainBlockContainer"] { padding-top: 1rem !important; }
|
||||
header[data-testid="stHeader"] { height: 0 !important; min-height: 0 !important; }
|
||||
|
||||
.st-key-sentiment_keyword_panel,
|
||||
.st-key-sentiment_table_controls,
|
||||
.st-key-sentiment_download_panel,
|
||||
.st-key-sentiment_chart_panel,
|
||||
.st-key-sentiment_trend_panel,
|
||||
.st-key-sentiment_wordfreq_panel {
|
||||
background: #ffffff !important;
|
||||
border: 1.5px solid #e2e8f0 !important;
|
||||
border-radius: 12px !important;
|
||||
box-shadow: 0 2px 6px rgba(15,23,42,0.06) !important;
|
||||
padding: 1.1rem 1.2rem 1rem !important;
|
||||
}
|
||||
|
||||
.st-key-sentiment_keyword_panel [data-testid="stTextInput"] label p,
|
||||
.st-key-sentiment_table_controls [data-testid="stTextInput"] label p,
|
||||
.st-key-sentiment_table_controls [data-testid="stSelectbox"] label p {
|
||||
color: #334155 !important;
|
||||
font-size: 0.72rem !important;
|
||||
font-weight: 800 !important;
|
||||
letter-spacing: 0.04em !important;
|
||||
line-height: 1.2 !important;
|
||||
margin-bottom: 0.22rem !important;
|
||||
text-transform: uppercase !important;
|
||||
}
|
||||
|
||||
.stButton > button {
|
||||
border-radius: 12px !important;
|
||||
font-weight: 700 !important;
|
||||
|
|
@ -419,7 +373,7 @@ def _render_donut_chart(pos_n, neu_n, neg_n, total, filter_label):
|
|||
legend=dict(orientation="h", y=-0.1, x=0.5, xanchor="center", font=dict(size=11)),
|
||||
paper_bgcolor="rgba(0,0,0,0)",
|
||||
)
|
||||
st.plotly_chart(fig, width="stretch", config={"displayModeBar": False})
|
||||
st.plotly_chart(fig, use_container_width=True, config={"displayModeBar": False})
|
||||
return dominant
|
||||
|
||||
|
||||
|
|
@ -446,7 +400,7 @@ def _render_bar_chart(pos_n, neu_n, neg_n, total):
|
|||
yaxis=dict(showgrid=True, gridcolor="rgba(226,232,240,0.8)", griddash="dot",
|
||||
tickfont=dict(size=10, color="#94a3b8")),
|
||||
)
|
||||
st.plotly_chart(fig, width="stretch", config={"displayModeBar": False})
|
||||
st.plotly_chart(fig, use_container_width=True, config={"displayModeBar": False})
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
|
|
@ -496,49 +450,9 @@ def _render_trend_chart(fdf, filter_label, dt_start, dt_end):
|
|||
tickfont=dict(size=10, color="#94a3b8")),
|
||||
hovermode="x unified",
|
||||
)
|
||||
with st.container(border=True, key="sentiment_trend_panel"):
|
||||
st.plotly_chart(fig, width="stretch", config={"displayModeBar": False})
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Confidence histogram
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
|
||||
# def _render_confidence_chart(fdf):
|
||||
# _section_header(
|
||||
# "🎯 Distribusi Keyakinan Model",
|
||||
# "Seberapa yakin classifier dalam mengklasifikasikan setiap tweet"
|
||||
# )
|
||||
|
||||
# fig = go.Figure()
|
||||
# sent_cfg = [
|
||||
# ("Positif", "#16a34a", "rgba(22,163,74,0.7)"),
|
||||
# ("Netral", "#94a3b8", "rgba(148,163,184,0.7)"),
|
||||
# ("Negatif", "#ef4444", "rgba(239,68,68,0.7)"),
|
||||
# ]
|
||||
# for sent, color, fill_color in sent_cfg:
|
||||
# sub = fdf[fdf["sentiment"] == sent]["confidence"]
|
||||
# if sub.empty:
|
||||
# continue
|
||||
# fig.add_trace(go.Histogram(
|
||||
# x=sub, name=sent, nbinsx=20,
|
||||
# marker=dict(color=fill_color, line=dict(color=color, width=1)),
|
||||
# opacity=0.85,
|
||||
# hovertemplate=f"<b>{sent}</b><br>Keyakinan: %{{x:.0%}}<br>Jumlah: %{{y}}<extra></extra>",
|
||||
# ))
|
||||
# fig.update_layout(
|
||||
# height=260, margin=dict(l=0, r=0, t=10, b=0), barmode="overlay",
|
||||
# paper_bgcolor="rgba(0,0,0,0)", plot_bgcolor="rgba(0,0,0,0)",
|
||||
# legend=dict(orientation="h", y=1.1, x=0, font=dict(size=11)),
|
||||
# xaxis=dict(tickformat=".0%", title="Tingkat Keyakinan",
|
||||
# tickfont=dict(size=10, color="#94a3b8"),
|
||||
# showgrid=True, gridcolor="rgba(226,232,240,0.6)"),
|
||||
# yaxis=dict(title="Jumlah Tweet",
|
||||
# tickfont=dict(size=10, color="#94a3b8"),
|
||||
# showgrid=True, gridcolor="rgba(226,232,240,0.6)", griddash="dot"),
|
||||
# )
|
||||
# with st.container(border=True, key="sentiment_conf_panel"):
|
||||
# st.plotly_chart(fig, width="stretch", config={"displayModeBar": False})
|
||||
# ─── PERBAIKAN: hapus key= dari st.container() ───
|
||||
with st.container():
|
||||
st.plotly_chart(fig, use_container_width=True, config={"displayModeBar": False})
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
|
|
@ -609,7 +523,7 @@ def _render_word_freq_per_sentiment(fdf):
|
|||
tickfont=dict(size=8, color="#94a3b8"), fixedrange=True),
|
||||
yaxis=dict(showgrid=False, tickfont=dict(size=9, color="#334155"), fixedrange=True),
|
||||
)
|
||||
st.plotly_chart(fig, width="stretch", config={"displayModeBar": False})
|
||||
st.plotly_chart(fig, use_container_width=True, config={"displayModeBar": False})
|
||||
st.markdown('</div>', unsafe_allow_html=True)
|
||||
|
||||
|
||||
|
|
@ -671,25 +585,25 @@ def _render_tweet_table(fdf, filter_label):
|
|||
f"Total {len(fdf):,} tweet · {filter_label}"
|
||||
)
|
||||
|
||||
with st.container(border=True, key="sentiment_table_controls"):
|
||||
tf1, tf2, tf3 = st.columns([3, 1.6, 1.7], gap="medium", vertical_alignment="bottom")
|
||||
with tf1:
|
||||
search = st.text_input(
|
||||
"Search tweet", placeholder="Ketik kata kunci di isi tweet...",
|
||||
key="sentiment_table_search",
|
||||
)
|
||||
with tf2:
|
||||
sf = st.selectbox(
|
||||
"Filter sentimen",
|
||||
["Semua", "Positif 😊", "Netral 😐", "Negatif 😞"],
|
||||
key="sentiment_table_filter",
|
||||
)
|
||||
with tf3:
|
||||
sort_by = st.selectbox(
|
||||
"Urutkan",
|
||||
["Terbaru dulu", "Terlama dulu", "Keyakinan tertinggi"],
|
||||
key="sentiment_table_sort",
|
||||
)
|
||||
# ─── PERBAIKAN: ganti st.container(key=) dengan st.columns biasa ───
|
||||
tf1, tf2, tf3 = st.columns([3, 1.6, 1.7], gap="medium")
|
||||
with tf1:
|
||||
search = st.text_input(
|
||||
"🔍 Cari tweet", placeholder="Ketik kata kunci di isi tweet...",
|
||||
key="sentiment_table_search",
|
||||
)
|
||||
with tf2:
|
||||
sf = st.selectbox(
|
||||
"Filter sentimen",
|
||||
["Semua", "Positif 😊", "Netral 😐", "Negatif 😞"],
|
||||
key="sentiment_table_filter",
|
||||
)
|
||||
with tf3:
|
||||
sort_by = st.selectbox(
|
||||
"Urutkan",
|
||||
["Terbaru dulu", "Terlama dulu", "Keyakinan tertinggi"],
|
||||
key="sentiment_table_sort",
|
||||
)
|
||||
|
||||
_section_gap("sm")
|
||||
|
||||
|
|
@ -711,26 +625,23 @@ def _render_tweet_table(fdf, filter_label):
|
|||
else:
|
||||
tdf = tdf.sort_values("confidence", ascending=False)
|
||||
|
||||
# out = tdf[["created_at", "crawled_at", "text", "clean_text", "sentiment", "confidence"]].copy()
|
||||
out = tdf[["created_at", "crawled_at", "text", "clean_text", "sentiment"]].copy()
|
||||
out["created_at"] = out["created_at"].apply(format_dt)
|
||||
out["crawled_at"] = out["crawled_at"].apply(format_dt)
|
||||
# out["confidence"] = out["confidence"].apply(lambda x: f"{x:.0%}")
|
||||
out["sentiment"] = out["sentiment"].map({
|
||||
"Positif": "😊 Positif", "Netral": "😐 Netral", "Negatif": "😞 Negatif",
|
||||
}).fillna(out["sentiment"])
|
||||
# out.columns = ["Tanggal Tweet", "Masuk Database", "Tweet Asli", "Tweet Bersih", "Sentimen", "Keyakinan"]
|
||||
out.columns = ["Tanggal Tweet", "Masuk Database", "Tweet Asli", "Tweet Bersih", "Sentimen"]
|
||||
|
||||
render_standard_table(
|
||||
out, height=400, min_width=1220,
|
||||
badge_columns=["Sentimen"],
|
||||
nowrap=["Tanggal Tweet", "Masuk Database", "Sentimen", "Keyakinan"],
|
||||
nowrap=["Tanggal Tweet", "Masuk Database", "Sentimen"],
|
||||
wide_columns=["Tweet Asli", "Tweet Bersih"],
|
||||
column_widths={
|
||||
"Tanggal Tweet": "170px", "Masuk Database": "170px",
|
||||
"Tweet Asli": "360px", "Tweet Bersih": "360px",
|
||||
"Sentimen": "130px", "Keyakinan": "110px",
|
||||
"Sentimen": "130px",
|
||||
},
|
||||
)
|
||||
st.caption(f"Menampilkan {len(tdf):,} tweet")
|
||||
|
|
@ -927,11 +838,9 @@ def show():
|
|||
if force_refresh:
|
||||
st.info("🔄 Menyegarkan prediksi sentimen dengan data terbaru...")
|
||||
with st.spinner("🔍 Preprocessing & prediksi sentimen (Hybrid Classifier)..."):
|
||||
# Preprocess semua tweet
|
||||
df["clean_text"] = df["text"].apply(preprocess_single)
|
||||
dfc = df[df["clean_text"].str.strip().str.len() > 0].copy()
|
||||
|
||||
# Prediksi menggunakan hybrid classifier
|
||||
results = predict_batch_hybrid(dfc["text"].tolist())
|
||||
if results:
|
||||
sentiments, confidences = zip(*results)
|
||||
|
|
@ -949,7 +858,8 @@ def show():
|
|||
# ── Keyword filter ────────────────────────────────────────
|
||||
_section_header("🎯 Filter Kata Kunci", "Kosongkan untuk melihat semua tweet")
|
||||
|
||||
with st.container(border=True, key="sentiment_keyword_panel"):
|
||||
# ─── PERBAIKAN: hapus key= dari st.container() ───
|
||||
with st.container():
|
||||
kw = st.text_input(
|
||||
"Kata kunci", placeholder="Contoh: ongkir, kurir, komdigi...",
|
||||
key="sentiment_keyword_search",
|
||||
|
|
@ -1000,11 +910,12 @@ def show():
|
|||
col_left, col_right = st.columns(2, gap="medium")
|
||||
with col_left:
|
||||
_section_header("🔵 Sebaran Sentimen", f"Periode {filter_label}")
|
||||
with st.container(border=True, key="sentiment_chart_panel"):
|
||||
# ─── PERBAIKAN: hapus key= dari st.container() ───
|
||||
with st.container():
|
||||
dominant = _render_donut_chart(pos_n, neu_n, neg_n, total, filter_label)
|
||||
with col_right:
|
||||
_section_header("📊 Perbandingan Jumlah per Sentimen", f"Periode {filter_label}")
|
||||
with st.container(border=True, key="sentiment_bar_panel"):
|
||||
with st.container():
|
||||
_render_bar_chart(pos_n, neu_n, neg_n, total)
|
||||
_section_gap("lg")
|
||||
|
||||
|
|
@ -1012,10 +923,6 @@ def show():
|
|||
_render_trend_chart(fdf, filter_label, start_date, end_date)
|
||||
_section_gap("lg")
|
||||
|
||||
# ── Confidence ────────────────────────────────────────────
|
||||
# _render_confidence_chart(fdf)
|
||||
# _section_gap("lg")
|
||||
|
||||
# ── Word freq ─────────────────────────────────────────────
|
||||
_render_word_freq_per_sentiment(fdf)
|
||||
_section_gap("lg")
|
||||
|
|
@ -1035,15 +942,17 @@ def show():
|
|||
# ── Download ──────────────────────────────────────────────
|
||||
_section_header("📥 Unduh Hasil Analisis")
|
||||
|
||||
with st.container(border=True, key="sentiment_download_panel"):
|
||||
d1, d2, d3 = st.columns(3, gap="medium", vertical_alignment="bottom")
|
||||
# ─── PERBAIKAN: hapus key= dari st.container() ───
|
||||
with st.container():
|
||||
d1, d2, d3 = st.columns(3, gap="medium")
|
||||
|
||||
with d1:
|
||||
st.download_button(
|
||||
"📥 Semua Hasil Prediksi",
|
||||
fdf.to_csv(index=False).encode("utf-8"),
|
||||
f"hasil_prediksi_{datetime.now().strftime('%Y%m%d_%H%M')}.csv",
|
||||
"text/csv", width="stretch",
|
||||
"text/csv",
|
||||
use_container_width=True,
|
||||
)
|
||||
with d2:
|
||||
summary = pd.DataFrame({
|
||||
|
|
@ -1055,12 +964,14 @@ def show():
|
|||
"📈 Ringkasan Sentimen",
|
||||
summary.to_csv(index=False).encode("utf-8"),
|
||||
f"ringkasan_{datetime.now().strftime('%Y%m%d_%H%M')}.csv",
|
||||
"text/csv", width="stretch",
|
||||
"text/csv",
|
||||
use_container_width=True,
|
||||
)
|
||||
with d3:
|
||||
st.download_button(
|
||||
"🎯 Rekomendasi Tindakan",
|
||||
df_rek.to_csv(index=False).encode("utf-8"),
|
||||
f"rekomendasi_{datetime.now().strftime('%Y%m%d_%H%M')}.csv",
|
||||
"text/csv", width="stretch",
|
||||
"text/csv",
|
||||
use_container_width=True,
|
||||
)
|
||||
|
|
@ -1,25 +1,15 @@
|
|||
"""
|
||||
sentiment_service.py
|
||||
====================
|
||||
Hybrid Classifier untuk analisis sentimen tweet Bahasa Indonesia.
|
||||
|
||||
PIPELINE PREPROCESSING — 5 TAHAP (selaras dengan preprocessing_page.py):
|
||||
1. Case Folding → lowercase dulu sebelum cleaning
|
||||
2. Cleaning → hapus URL, mention, hashtag, angka, emoji, tanda baca
|
||||
3. Normalisasi → singkatan/slang → kata baku
|
||||
4. Stopword Removal → buang kata umum, jaga kata sentimen penting
|
||||
5. Stemming → bentuk dasar via Sastrawi ECS
|
||||
|
||||
Catatan: Tokenizing tidak menjadi tahap tersendiri karena:
|
||||
• Stopword removal & stemming sudah melakukan split() secara internal
|
||||
• TF-IDF melakukan tokenisasi sendiri saat inferensi
|
||||
|
||||
ARSITEKTUR HYBRID:
|
||||
Teks Asli
|
||||
├── preprocess_untuk_lexicon() ──→ _hitung_skor_lexicon() ──┐
|
||||
└── preprocess_for_model() ──→ TF-IDF → NB Model ───────┘
|
||||
↓
|
||||
_klasifikasi_hybrid() → Label + Confidence
|
||||
sentiment_service.py — Hybrid Classifier (versi perbaikan)
|
||||
=============================================================
|
||||
Perbaikan utama vs versi sebelumnya:
|
||||
1. LEXICON DIPERLUAS — kata positif & negatif yang tidak ada di vocab TF-IDF
|
||||
(bagus, mantap, setuju, puas, hemat, berhasil, dll.) kini tetap bisa
|
||||
terdeteksi melalui lexicon scoring.
|
||||
2. THRESHOLD DISESUAIKAN — Layer 1 diperlonggar (net >= 2 → Positif),
|
||||
Layer anti-bias diperketat agar prediksi lebih proporsional.
|
||||
3. NEGATION WINDOW DIPERLUAS — window 3 kata (sebelumnya 2) agar
|
||||
"tidak terlalu bagus" tetap terdeteksi negasinya.
|
||||
4. PREPROCESSING IDENTIK dengan preprocessing_page.py (5 tahap).
|
||||
"""
|
||||
|
||||
import re
|
||||
|
|
@ -28,93 +18,44 @@ import joblib
|
|||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# KATA SENTIMEN PENTING
|
||||
# Tidak boleh dihapus di tahap stopword removal
|
||||
# KATA SENTIMEN PENTING (dijaga dari stopword removal)
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
|
||||
KATA_SENTIMEN_PENTING = {
|
||||
# ── Negasi ──────────────────────────────────────────────
|
||||
# Negasi
|
||||
"tidak", "bukan", "jangan", "kurang", "belum", "tanpa",
|
||||
|
||||
# ── Sentimen POSITIF ────────────────────────────────────
|
||||
# Intensitas
|
||||
"sangat", "banget", "sekali", "paling", "amat", "luar", "biasa",
|
||||
# Positif umum
|
||||
"keren", "bagus", "mantap", "setuju", "dukung", "mendukung",
|
||||
"andal", "handal", "gercep", "bangga", "senang", "suka",
|
||||
"baik", "benar", "tepat", "oke",
|
||||
"baik", "benar", "tepat", "oke", "puas",
|
||||
"sejahtera", "berkembang", "maju", "inovatif",
|
||||
"tegas", "sigap", "tanggap", "adil", "bijak", "bermanfaat",
|
||||
"untung", "berhasil", "sukses", "solusi", "manfaat",
|
||||
"berguna", "membantu", "bantu", "pro", "lanjut",
|
||||
"sangat", "banget", "sekali", "paling", "amat", "luar", "biasa",
|
||||
|
||||
# ── Sentimen NEGATIF evaluatif ──────────────────────────
|
||||
"kecewa", "buruk", "jelek", "parah", "gagal", "hancur",
|
||||
"rusak", "bohong", "tipu", "korupsi",
|
||||
|
||||
# ── Emosi ───────────────────────────────────────────────
|
||||
# Positif domain e-commerce/ongkir
|
||||
"gratis", "murah", "hemat", "terjangkau", "cepat",
|
||||
"aman", "mudah", "praktis", "terpercaya",
|
||||
# Negatif umum
|
||||
"kecewa", "mending", "malah", "buruk", "jelek", "parah",
|
||||
"gagal", "hancur", "rusak", "bohong", "tipu", "korupsi",
|
||||
# Negatif domain e-commerce/ongkir
|
||||
"mahal", "lambat", "lelet", "ribet", "susah", "repot",
|
||||
"rugi", "boros",
|
||||
# Emosi
|
||||
"marah", "sedih", "khawatir",
|
||||
}
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# NORMALISASI (selaras dengan preprocessing_page.py)
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
|
||||
NORMALISASI = {
|
||||
# Negasi
|
||||
"gk": "tidak", "ga": "tidak", "gak": "tidak",
|
||||
"nggak": "tidak", "ngga": "tidak", "tdk": "tidak",
|
||||
"tak": "tidak", "enggak": "tidak", "engga": "tidak",
|
||||
"kagak": "tidak", "kaga": "tidak", "ndak": "tidak",
|
||||
"gkk": "tidak", "ngak": "tidak",
|
||||
# Kata ganti
|
||||
"yg": "yang", "dgn": "dengan", "utk": "untuk",
|
||||
"org": "orang", "krn": "karena", "dr": "dari",
|
||||
"sm": "sama", "pd": "pada", "dlm": "dalam",
|
||||
"bwt": "buat", "trm": "terima",
|
||||
# Verba
|
||||
"tp": "tapi", "tpi": "tapi", "jd": "jadi",
|
||||
"sdh": "sudah", "blm": "belum", "emg": "memang",
|
||||
"emang": "memang", "gimana": "bagaimana",
|
||||
"gitu": "begitu", "gini": "begini",
|
||||
"udah": "sudah", "udh": "sudah",
|
||||
# Intensitas
|
||||
"bgt": "banget", "bngt": "banget", "bget": "banget", "bgtt": "banget",
|
||||
# Positif informal
|
||||
"bener": "benar", "beneran": "benar",
|
||||
"mantep": "mantap", "mntap": "mantap",
|
||||
"kece": "keren",
|
||||
"cucok": "cocok", "cucuk": "cocok",
|
||||
"cakep": "bagus",
|
||||
"sip": "baik", "siipp": "baik",
|
||||
"top": "terbaik",
|
||||
"jos": "bagus", "josss": "bagus",
|
||||
"goks": "luar biasa",
|
||||
"setujuu": "setuju", "stuju": "setuju",
|
||||
"proud": "bangga",
|
||||
"mantul": "mantap betul",
|
||||
# Negatif informal
|
||||
"ancur": "hancur", "ancrr": "hancur",
|
||||
"parahh": "parah", "parahhh": "parah",
|
||||
"gagall": "gagal",
|
||||
"ngaco": "tidak benar",
|
||||
"ngasal": "tidak benar",
|
||||
"gaje": "tidak jelas",
|
||||
# Domain
|
||||
"ongkir": "ongkos kirim",
|
||||
"freeongkir": "gratis ongkos kirim",
|
||||
"gratisongkir": "gratis ongkos kirim",
|
||||
"free": "gratis",
|
||||
"ecommerce": "e commerce",
|
||||
"seller": "penjual",
|
||||
"buyer": "pembeli",
|
||||
}
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# LEXICON SENTIMEN
|
||||
# Diperluas agar kata yang tidak ada di vocab TF-IDF tetap
|
||||
# bisa berkontribusi melalui jalur lexicon scoring.
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
|
||||
LEXICON_POSITIF = {
|
||||
# Umum & informal
|
||||
"bagus", "baik", "keren", "mantap", "mantep", "hebat",
|
||||
"oke", "sip", "top", "jos", "goks", "kece", "mantul",
|
||||
"setuju", "dukung", "mendukung", "pro", "lanjut", "sepakat",
|
||||
|
|
@ -127,71 +68,147 @@ LEXICON_POSITIF = {
|
|||
"untung", "gratis", "murah", "hemat", "terjangkau",
|
||||
"terbaik", "luar biasa",
|
||||
"cakep", "cucok", "gaskeun", "kuy",
|
||||
"dukung", "bantu", "solusi", "manfaat",
|
||||
"solusi", "manfaat", "menguntungkan",
|
||||
"memuaskan", "membanggakan", "mengagumkan",
|
||||
"sehat", "fair", "wajar",
|
||||
"syukur", "alhamdulillah",
|
||||
"saing", "kompetitif",
|
||||
# Domain ongkir/ecommerce positif
|
||||
"terjangkau", "hemat", "efisien", "mudah", "praktis",
|
||||
"cepat", "aman", "terpercaya", "andalan",
|
||||
# Dukungan kebijakan
|
||||
"dukung", "setuju", "bagus", "tepat", "bijak",
|
||||
"perlu", "penting", "benar", "wajar", "adil",
|
||||
}
|
||||
|
||||
LEXICON_NEGATIF = {
|
||||
# Umum
|
||||
"buruk", "jelek", "parah", "rusak", "hancur", "ancur",
|
||||
"gagal", "gagall", "ambruk", "terpuruk", "bangkrut",
|
||||
"bohong", "tipu", "curang", "manipulasi", "korupsi", "penipuan",
|
||||
"kebohongan",
|
||||
"kecewa", "marah", "sedih", "khawatir", "takut", "benci",
|
||||
"jijik", "muak", "kesal", "frustrasi",
|
||||
"mahal", "rugi", "merugikan",
|
||||
"lambat", "lemot", "ribet", "susah", "sulit", "bermasalah",
|
||||
"gagal", "ambruk", "terpuruk", "bangkrut",
|
||||
"bohong", "tipu", "curang", "manipulasi", "korupsi",
|
||||
"kebohongan", "penipuan",
|
||||
"kecewa", "mending", "malah", "marah", "sedih",
|
||||
"khawatir", "takut", "benci", "jijik", "muak",
|
||||
"kesal", "frustrasi", "geram", "dongkol",
|
||||
"mahal", "rugi", "merugikan", "rugikan",
|
||||
"lambat", "lemot", "lelet", "ribet", "susah",
|
||||
"sulit", "bermasalah",
|
||||
"ngaco", "ngasal", "gaje", "receh",
|
||||
"tidak benar", "tidak jelas", "tidak adil", "tidak berguna",
|
||||
"tolol", "bodoh", "idiot", "goblok",
|
||||
"mengecewakan", "menyebalkan", "menyusahkan",
|
||||
"monopoli", "licik",
|
||||
# Domain ongkir negatif
|
||||
"boros", "memberatkan", "menyulitkan",
|
||||
"repot", "ribet", "ngeributin", "ribut",
|
||||
# Kritik kebijakan
|
||||
"salah", "keliru", "gegabah", "sembarangan",
|
||||
"tidak jelas", "ngawur", "asal",
|
||||
}
|
||||
|
||||
KATA_NEGASI = {
|
||||
"tidak", "bukan", "jangan", "belum", "tanpa", "kurang",
|
||||
"anti", "non",
|
||||
"anti", "non", "tak", "ga", "gak", "nggak", "ngga",
|
||||
"enggak", "engga",
|
||||
}
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# STOPWORDS
|
||||
# LOAD NORMALISASI DARI FILE
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
|
||||
def _load_normalization() -> dict:
|
||||
norm_file = "indonesian-normalisasi-slangword-complete.txt"
|
||||
norm_dict: dict = {}
|
||||
try:
|
||||
with open(norm_file, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
parts = line.split(",", 1)
|
||||
if len(parts) != 2:
|
||||
continue
|
||||
slang = parts[0].strip().strip("'\"").lower()
|
||||
normal = parts[1].strip().lower()
|
||||
if slang and normal:
|
||||
norm_dict[slang] = normal
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
DOMAIN_OVERRIDES = {
|
||||
"shopee": "shopee", "tokopedia": "tokopedia", "lazada": "lazada",
|
||||
"tiktok": "tiktok", "bukalapak": "bukalapak", "blibli": "blibli",
|
||||
"sicepat": "sicepat", "jne": "jne", "jnt": "jnt",
|
||||
"anteraja": "anteraja", "ninja": "ninja",
|
||||
"freeongkir": "gratis ongkos kirim",
|
||||
"gratisongkir": "gratis ongkos kirim",
|
||||
"ongkir": "ongkos kirim", "ongkr": "ongkos kirim",
|
||||
"bykrm": "biaya kirim", "biayakirim": "biaya pengiriman",
|
||||
"komdigi": "komdigi", "kemendag": "kementerian perdagangan",
|
||||
"kominfo": "kementerian komunikasi",
|
||||
"ecommerce": "e commerce", "marketplace": "marketplace",
|
||||
"seller": "penjual", "buyer": "pembeli", "online": "online",
|
||||
# Negasi tambahan
|
||||
"gk": "tidak", "ga": "tidak", "gak": "tidak",
|
||||
"nggak": "tidak", "ngga": "tidak", "tdk": "tidak",
|
||||
"tak": "tidak", "enggak": "tidak", "engga": "tidak",
|
||||
"kagak": "tidak", "kaga": "tidak", "ndak": "tidak",
|
||||
"ngak": "tidak",
|
||||
# Intensitas
|
||||
"bgt": "banget", "bngt": "banget", "bget": "banget",
|
||||
"bgtt": "banget",
|
||||
# Positif informal
|
||||
"mantep": "mantap", "mntap": "mantap",
|
||||
"bener": "benar", "beneran": "benar",
|
||||
"kece": "keren", "sip": "baik",
|
||||
# Negatif informal
|
||||
"ancur": "hancur", "parahh": "parah",
|
||||
}
|
||||
norm_dict.update(DOMAIN_OVERRIDES)
|
||||
return norm_dict
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# LOAD STOPWORDS DARI FILE
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
|
||||
def _load_stopwords() -> set:
|
||||
"""Load stopword dengan penjagaan kata sentimen penting."""
|
||||
stopword_file = "indonesian-stopwords-complete.txt"
|
||||
base = set()
|
||||
base: set = set()
|
||||
try:
|
||||
with open(stopword_file, "r", encoding="utf-8") as f:
|
||||
base = set(f.read().splitlines())
|
||||
for line in f:
|
||||
word = line.strip().lower()
|
||||
if word:
|
||||
base.add(word)
|
||||
except FileNotFoundError:
|
||||
base = {
|
||||
"yang", "dan", "di", "ke", "dari", "ini", "itu",
|
||||
"dengan", "untuk", "pada", "adalah", "oleh", "ada",
|
||||
"ya", "akan", "atau", "juga", "sama", "karena",
|
||||
"jika", "sudah", "telah", "saat", "agar", "maka",
|
||||
"lagi", "bila", "bisa", "pun", "nya",
|
||||
"jika", "sudah", "telah", "jadi", "bisa",
|
||||
}
|
||||
|
||||
# Jangan hapus kata sentimen penting
|
||||
# Lindungi kata sentimen penting
|
||||
for kata in KATA_SENTIMEN_PENTING:
|
||||
base.discard(kata)
|
||||
|
||||
# Tambahan stopword domain-spesifik
|
||||
# Tambah noise Twitter
|
||||
base.update({
|
||||
"rt", "amp", "https", "http", "co", "t",
|
||||
"wkwk", "wkwkwk", "haha", "hehe", "xixi", "hahaha",
|
||||
"yg", "dgn", "utk", "dr", "krn", "tp", "jd", "sdh",
|
||||
"aja", "doang", "nih", "sih", "dong", "deh",
|
||||
"loh", "lah", "tuh", "kak", "gan", "bro", "sis",
|
||||
"rt", "amp", "https", "http", "co", "pic",
|
||||
"wkwk", "wkwkwk", "wkwkwkwk",
|
||||
"haha", "hahaha", "hehe", "hihi", "huhu", "xixi",
|
||||
"nih", "sih", "dong", "deh", "loh", "lah", "tuh",
|
||||
"kak", "gan", "bro", "sob", "min",
|
||||
})
|
||||
return base
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# STEMMER
|
||||
# LOAD STEMMER
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
|
||||
def _load_stemmer():
|
||||
"""Load Sastrawi stemmer. Return None jika tidak terinstall."""
|
||||
try:
|
||||
from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
|
||||
return StemmerFactory().create_stemmer()
|
||||
|
|
@ -199,13 +216,14 @@ def _load_stemmer():
|
|||
return None
|
||||
|
||||
|
||||
# ── Inisialisasi global ─────────────────────────────────────────────────────
|
||||
# ── Inisialisasi global (dimuat sekali saat modul diimport) ─
|
||||
_STOPWORDS = _load_stopwords()
|
||||
_STEMMER = _load_stemmer()
|
||||
_NORM_DICT = _load_normalization()
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# MODEL LOADING (lazy)
|
||||
# MODEL LOADING (lazy, singleton)
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
|
||||
_model = None
|
||||
|
|
@ -213,10 +231,6 @@ _tfidf = None
|
|||
|
||||
|
||||
def _load_model():
|
||||
"""
|
||||
Lazy load NB model + TF-IDF vectorizer.
|
||||
Return: (model, tfidf) — keduanya bisa None.
|
||||
"""
|
||||
global _model, _tfidf
|
||||
if _model is None:
|
||||
try:
|
||||
|
|
@ -232,35 +246,29 @@ def _load_model():
|
|||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# PREPROCESSING PIPELINE — 5 TAHAP
|
||||
#
|
||||
# URUTAN (selaras dengan preprocessing_page.py):
|
||||
# 1. Case Folding → lowercase
|
||||
# 2. Cleaning → hapus noise
|
||||
# 3. Normalisasi → normalisasi kata
|
||||
# 4. Stopword Removal → buang stopword (split() dilakukan internal)
|
||||
# 5. Stemming → bentuk dasar
|
||||
# 5-TAHAP PREPROCESSING PIPELINE
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
|
||||
def _case_folding(text: str) -> str:
|
||||
"""Tahap 1: Lowercase seluruh teks."""
|
||||
def _step1_case_folding(text: str) -> str:
|
||||
return str(text).lower()
|
||||
|
||||
|
||||
def _cleaning(text: str) -> str:
|
||||
"""Tahap 2: Hapus noise (URL, mention, hashtag, angka, emoji, tanda baca)."""
|
||||
def _step2_cleaning(text: str) -> str:
|
||||
text = re.sub(r"http\S+|www\S+|https\S+", "", text)
|
||||
text = re.sub(r"@\w+", "", text)
|
||||
text = re.sub(r"#\w+", "", text)
|
||||
text = re.sub(r"\d+", "", text)
|
||||
text = re.sub(
|
||||
r"[\U00010000-\U0010ffff"
|
||||
r"["
|
||||
r"\U00010000-\U0010ffff"
|
||||
r"\U0001F600-\U0001F64F"
|
||||
r"\U0001F300-\U0001F5FF"
|
||||
r"\U0001F680-\U0001F6FF"
|
||||
r"\U0001F1E0-\U0001F1FF"
|
||||
r"\u2600-\u26FF\u2700-\u27BF"
|
||||
r"]+", "", text, flags=re.UNICODE,
|
||||
r"\u2600-\u26FF"
|
||||
r"\u2700-\u27BF"
|
||||
r"]+",
|
||||
"", text, flags=re.UNICODE,
|
||||
)
|
||||
text = text.translate(str.maketrans("", "", string.punctuation))
|
||||
text = re.sub(r"[^a-zA-Z\s]", "", text)
|
||||
|
|
@ -268,79 +276,72 @@ def _cleaning(text: str) -> str:
|
|||
return text
|
||||
|
||||
|
||||
def _normalisasi(text: str) -> str:
|
||||
"""Tahap 3: Normalisasi singkatan dan kata tidak baku."""
|
||||
return " ".join(NORMALISASI.get(word, word) for word in text.split())
|
||||
def _step3_normalization(text: str, norm_dict: dict) -> str:
|
||||
tokens = text.split()
|
||||
return " ".join(norm_dict.get(token, token) for token in tokens)
|
||||
|
||||
|
||||
def _remove_stopwords(tokens: list) -> list:
|
||||
"""Tahap 4: Stopword removal dengan penjagaan kata sentimen."""
|
||||
return [
|
||||
w for w in tokens
|
||||
if (w not in _STOPWORDS or w in KATA_SENTIMEN_PENTING) and len(w) > 2
|
||||
]
|
||||
def _step4_stopword_removal(tokens: list, stopwords: set) -> list:
|
||||
result = []
|
||||
for token in tokens:
|
||||
if token in KATA_SENTIMEN_PENTING:
|
||||
result.append(token)
|
||||
continue
|
||||
if token in stopwords:
|
||||
continue
|
||||
if len(token) <= 2:
|
||||
continue
|
||||
result.append(token)
|
||||
return result
|
||||
|
||||
|
||||
def _stemming(tokens: list) -> list:
|
||||
"""Tahap 5: Stemming ke bentuk dasar via Sastrawi ECS."""
|
||||
if _STEMMER is None:
|
||||
def _step5_stemming(tokens: list, stemmer) -> list:
|
||||
if stemmer is None:
|
||||
return tokens
|
||||
return [_STEMMER.stem(w) for w in tokens]
|
||||
return [stemmer.stem(token) for token in tokens]
|
||||
|
||||
|
||||
def preprocess_for_model(text: str) -> str:
|
||||
"""
|
||||
Full 5-tahap preprocessing → string teks bersih siap TF-IDF.
|
||||
|
||||
URUTAN: Case Folding → Cleaning → Normalisasi
|
||||
→ Stopword Removal → Stemming
|
||||
|
||||
Split kata dilakukan secara internal di tahap Stopword Removal.
|
||||
Pipeline HARUS sama persis dengan yang dipakai saat training model.
|
||||
"""
|
||||
s1 = _case_folding(text) # Tahap 1
|
||||
s2 = _cleaning(s1) # Tahap 2
|
||||
s3 = _normalisasi(s2) # Tahap 3
|
||||
s4 = _remove_stopwords(s3.split()) # Tahap 4 (split inline)
|
||||
s5 = _stemming(s4) # Tahap 5
|
||||
"""Full 5-tahap preprocessing → string teks bersih siap TF-IDF."""
|
||||
s1 = _step1_case_folding(text)
|
||||
s2 = _step2_cleaning(s1)
|
||||
s3 = _step3_normalization(s2, _NORM_DICT)
|
||||
s4 = _step4_stopword_removal(s3.split(), _STOPWORDS)
|
||||
s5 = _step5_stemming(s4, _STEMMER)
|
||||
return " ".join(s5)
|
||||
|
||||
|
||||
def preprocess_untuk_lexicon(text: str) -> str:
|
||||
"""
|
||||
Preprocessing RINGAN untuk lexicon matching.
|
||||
Tidak di-stem → kata asli bisa dicocokkan dengan lexicon.
|
||||
Pipeline: Case Folding → Cleaning → Normalisasi saja.
|
||||
"""
|
||||
s1 = _case_folding(text)
|
||||
s2 = _cleaning(s1)
|
||||
s3 = _normalisasi(s2)
|
||||
"""Preprocessing ringan untuk lexicon matching (tanpa stemming)."""
|
||||
s1 = _step1_case_folding(text)
|
||||
s2 = _step2_cleaning(s1)
|
||||
s3 = _step3_normalization(s2, _NORM_DICT)
|
||||
return s3
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# LEXICON SCORER
|
||||
# LEXICON SCORER (dengan negation window diperluas ke 3)
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
|
||||
def _hitung_skor_lexicon(teks_lexicon: str) -> dict:
|
||||
"""
|
||||
Hitung skor positif dan negatif dari teks via lexicon.
|
||||
Hitung skor sentimen via lexicon + negation handling.
|
||||
|
||||
NEGATION HANDLING:
|
||||
Kata negasi dalam window 2 kata sebelum kata sentimen → polaritas dibalik.
|
||||
Contoh: "tidak bagus" → ada "tidak" sebelum "bagus" (POSITIF)
|
||||
→ skor_neg += 1 (bukan skor_pos)
|
||||
Perubahan vs versi lama:
|
||||
- Window negasi diperluas menjadi 3 kata (sebelumnya 2)
|
||||
- Lexicon positif/negatif lebih luas (kata yang tidak ada di vocab TF-IDF)
|
||||
|
||||
Return: {"positif": int, "negatif": int, "net": int}
|
||||
"""
|
||||
tokens = teks_lexicon.split()
|
||||
tokens = teks_lexicon.split()
|
||||
skor_pos = 0
|
||||
skor_neg = 0
|
||||
|
||||
for i, token in enumerate(tokens):
|
||||
# Cek negasi dalam window 3 kata sebelumnya
|
||||
ada_negasi = any(
|
||||
tokens[i - j] in KATA_NEGASI
|
||||
for j in range(1, 3)
|
||||
for j in range(1, 4)
|
||||
if i - j >= 0
|
||||
)
|
||||
|
||||
|
|
@ -363,30 +364,32 @@ def _hitung_skor_lexicon(teks_lexicon: str) -> dict:
|
|||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# HYBRID CLASSIFIER
|
||||
# PREDIKSI MODEL NB
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
|
||||
# Normalisasi label dari berbagai format yang mungkin dipakai model
|
||||
_LABEL_MAP = {
|
||||
"positif": "Positif", "Positif": "Positif", "positive": "Positif", "pos": "Positif",
|
||||
"negatif": "Negatif", "Negatif": "Negatif", "negative": "Negatif", "neg": "Negatif",
|
||||
"netral": "Netral", "Netral": "Netral", "neutral": "Netral", "net": "Netral",
|
||||
"positif": "Positif", "Positif": "Positif",
|
||||
"positive": "Positif", "pos": "Positif",
|
||||
"negatif": "Negatif", "Negatif": "Negatif",
|
||||
"negative": "Negatif", "neg": "Negatif",
|
||||
"netral": "Netral", "Netral": "Netral",
|
||||
"neutral": "Netral", "net": "Netral",
|
||||
}
|
||||
|
||||
|
||||
def _prediksi_model(teks_model: str):
|
||||
"""
|
||||
Dapatkan prediksi dari NB model.
|
||||
Return: (label_norm, confidence, proba_dict) atau None.
|
||||
Prediksi dari model NB 3-kelas.
|
||||
Return: (label_norm, confidence, proba_dict) atau None jika gagal.
|
||||
"""
|
||||
model, tfidf = _load_model()
|
||||
if model is None or tfidf is None or not teks_model.strip():
|
||||
return None
|
||||
try:
|
||||
vec = tfidf.transform([teks_model])
|
||||
pred_raw = model.predict(vec)[0]
|
||||
proba = model.predict_proba(vec)[0]
|
||||
classes = list(model.classes_)
|
||||
vec = tfidf.transform([teks_model])
|
||||
pred_raw = model.predict(vec)[0]
|
||||
proba = model.predict_proba(vec)[0]
|
||||
classes = list(model.classes_)
|
||||
|
||||
label_norm = _LABEL_MAP.get(str(pred_raw), "Netral")
|
||||
pred_idx = classes.index(pred_raw) if pred_raw in classes else 0
|
||||
|
|
@ -402,79 +405,105 @@ def _prediksi_model(teks_model: str):
|
|||
return None
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# HYBRID CLASSIFIER (diperkuat)
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
|
||||
def _klasifikasi_hybrid(
|
||||
teks_model: str,
|
||||
skor: dict,
|
||||
teks_lower: str,
|
||||
) -> tuple:
|
||||
"""
|
||||
Klasifikasi hybrid: sinyal lexicon + model NB.
|
||||
|
||||
LOGIKA KEPUTUSAN (berurutan):
|
||||
Klasifikasi hybrid: sinyal lexicon + model NB 3-kelas.
|
||||
|
||||
PERUBAHAN vs versi lama:
|
||||
─────────────────────────────────────────────────────────
|
||||
Layer 1 — Override Positif KUAT (net >= 2):
|
||||
→ Positif, confidence 60–92%
|
||||
Confidence lebih tinggi karena lexicon lebih kuat.
|
||||
Range: 0.65–0.92
|
||||
|
||||
Layer 1b — Override Negatif KUAT (net <= -2):
|
||||
Simetris dengan positif.
|
||||
Range: 0.65–0.90
|
||||
|
||||
Layer 2 — Override Positif LEMAH (net == 1):
|
||||
→ Cek model; jika model < 65% yakin Negatif → Positif
|
||||
→ Jika model sangat yakin Negatif → tetap Positif (confidence rendah)
|
||||
Diperlonggar: sekarang berlaku jika model tidak sangat
|
||||
yakin negatif (threshold 0.70, sebelumnya 0.65).
|
||||
|
||||
Layer 3 — Override Negatif KUAT (net <= -2):
|
||||
→ Negatif, confidence 60–90%
|
||||
Layer 2b — Override Negatif LEMAH (net == -1):
|
||||
Simetris baru. Sebelumnya tidak ada.
|
||||
|
||||
Layer 4 — Fallback Model NB:
|
||||
→ Prediksi model dipakai, TAPI:
|
||||
• Jika model = Negatif AND net >= 0 AND confidence < 75%
|
||||
→ downgrade ke Netral (koreksi bias model)
|
||||
• Kasus lainnya → percaya model
|
||||
|
||||
Layer 5 — Ultimate Fallback (model tidak tersedia):
|
||||
→ Gunakan skor lexicon saja
|
||||
Layer 3 — Fallback Model NB:
|
||||
Anti-bias diperketat:
|
||||
- Model Negatif + lexicon bersih (net >= 0) + conf < 0.65
|
||||
→ turunkan ke Netral (threshold naik dari 0.70 ke 0.65)
|
||||
- Model Positif + lexicon negatif kuat (net <= -1) + conf < 0.65
|
||||
→ turunkan ke Netral
|
||||
|
||||
Layer 4 — Ultimate Fallback:
|
||||
Lexicon saja jika model tidak tersedia.
|
||||
─────────────────────────────────────────────────────────
|
||||
Return: (label: str, confidence: float)
|
||||
"""
|
||||
net = skor["net"]
|
||||
pos = skor["positif"]
|
||||
neg = skor["negatif"]
|
||||
|
||||
# ── Layer 1: Positif KUAT ─────────────────────────────────────────────────
|
||||
# ── Layer 1a: Positif KUAT ───────────────────────────────────────────────
|
||||
if net >= 2:
|
||||
conf = min(0.60 + (net * 0.07), 0.92)
|
||||
conf = min(0.65 + (net * 0.05), 0.92)
|
||||
return ("Positif", round(conf, 3))
|
||||
|
||||
# ── Layer 2: Positif LEMAH ────────────────────────────────────────────────
|
||||
# ── Layer 1b: Negatif KUAT ───────────────────────────────────────────────
|
||||
if net <= -2:
|
||||
conf = min(0.65 + (abs(net) * 0.05), 0.90)
|
||||
return ("Negatif", round(conf, 3))
|
||||
|
||||
# ── Layer 2a: Positif LEMAH ──────────────────────────────────────────────
|
||||
if net == 1 and pos >= 1:
|
||||
model_result = _prediksi_model(teks_model)
|
||||
if model_result is not None:
|
||||
_, _, proba_dict = model_result
|
||||
neg_prob = proba_dict.get("Negatif", 0.0)
|
||||
if neg_prob < 0.65:
|
||||
conf = round(0.55 + (0.65 - neg_prob) * 0.3, 3)
|
||||
return ("Positif", min(conf, 0.80))
|
||||
if neg_prob < 0.70: # diperlonggar dari 0.65
|
||||
conf = round(0.55 + (0.70 - neg_prob) * 0.30, 3)
|
||||
return ("Positif", min(conf, 0.82))
|
||||
else:
|
||||
return ("Positif", 0.55)
|
||||
else:
|
||||
return ("Positif", 0.58)
|
||||
return ("Positif", 0.58)
|
||||
|
||||
# ── Layer 3: Negatif KUAT ─────────────────────────────────────────────────
|
||||
if net <= -2:
|
||||
conf = min(0.60 + (abs(net) * 0.06), 0.90)
|
||||
return ("Negatif", round(conf, 3))
|
||||
# ── Layer 2b: Negatif LEMAH (baru) ───────────────────────────────────────
|
||||
if net == -1 and neg >= 1:
|
||||
model_result = _prediksi_model(teks_model)
|
||||
if model_result is not None:
|
||||
_, _, proba_dict = model_result
|
||||
pos_prob = proba_dict.get("Positif", 0.0)
|
||||
if pos_prob < 0.70:
|
||||
conf = round(0.55 + (0.70 - pos_prob) * 0.30, 3)
|
||||
return ("Negatif", min(conf, 0.82))
|
||||
else:
|
||||
return ("Negatif", 0.55)
|
||||
return ("Negatif", 0.58)
|
||||
|
||||
# ── Layer 4: Fallback Model NB ────────────────────────────────────────────
|
||||
# ── Layer 3: Fallback Model NB 3-kelas ───────────────────────────────────
|
||||
model_result = _prediksi_model(teks_model)
|
||||
if model_result is not None:
|
||||
label_norm, confidence, proba_dict = model_result
|
||||
|
||||
# Anti-bias correction:
|
||||
# Model prediksi Negatif tapi tidak ada sinyal negatif dari lexicon
|
||||
# dan confidence < 75% → kemungkinan bias → turunkan ke Netral
|
||||
if label_norm == "Negatif" and net >= 0 and confidence < 0.75:
|
||||
corrected_conf = round(0.50 + max(0, confidence - 0.50) * 0.2, 3)
|
||||
# Anti-bias 1: Model Negatif tapi lexicon bersih → Netral
|
||||
if label_norm == "Negatif" and net >= 0 and confidence < 0.65:
|
||||
corrected_conf = round(0.50 + max(0, confidence - 0.50) * 0.20, 3)
|
||||
return ("Netral", corrected_conf)
|
||||
|
||||
# Anti-bias 2: Model Positif tapi lexicon negatif → Netral
|
||||
if label_norm == "Positif" and net <= -1 and confidence < 0.65:
|
||||
corrected_conf = round(0.50 + max(0, confidence - 0.50) * 0.20, 3)
|
||||
return ("Netral", corrected_conf)
|
||||
|
||||
return (label_norm, round(confidence, 3))
|
||||
|
||||
# ── Layer 5: Ultimate Fallback ────────────────────────────────────────────
|
||||
# ── Layer 4: Ultimate Fallback ────────────────────────────────────────────
|
||||
if net > 0:
|
||||
return ("Positif", 0.55)
|
||||
elif net < 0:
|
||||
|
|
@ -483,27 +512,45 @@ def _klasifikasi_hybrid(
|
|||
return ("Netral", 0.50)
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# PUBLIC API
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
|
||||
def analisis_sentimen_single(text: str) -> tuple:
|
||||
"""
|
||||
Analisis sentimen satu teks.
|
||||
Return: (label, confidence)
|
||||
label: 'Positif' | 'Netral' | 'Negatif'
|
||||
"""
|
||||
teks_model = preprocess_for_model(text)
|
||||
teks_lexicon = preprocess_untuk_lexicon(text)
|
||||
teks_lower = str(text).lower()
|
||||
skor = _hitung_skor_lexicon(teks_lexicon)
|
||||
return _klasifikasi_hybrid(teks_model, skor, teks_lower)
|
||||
|
||||
|
||||
def analisis_sentimen_batch(texts: list) -> list:
|
||||
"""
|
||||
Analisis sentimen batch.
|
||||
Return: list of (label, confidence)
|
||||
"""
|
||||
return [analisis_sentimen_single(text) for text in texts]
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# BACKWARD COMPATIBILITY
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
|
||||
def bersihkan_teks(text: str) -> str:
|
||||
"""[LEGACY] Gunakan preprocess_for_model() untuk pipeline lengkap."""
|
||||
"""[LEGACY] Gunakan preprocess_for_model()."""
|
||||
return preprocess_for_model(text)
|
||||
|
||||
|
||||
def prediksi_sentimen(list_text: list):
|
||||
"""
|
||||
[LEGACY] Prediksi batch dengan hybrid classifier.
|
||||
[LEGACY] Prediksi batch.
|
||||
Return: (list clean_texts, list labels)
|
||||
"""
|
||||
clean_texts = [preprocess_for_model(t) for t in list_text]
|
||||
labels = []
|
||||
for text in list_text:
|
||||
teks_model = preprocess_for_model(text)
|
||||
teks_lexicon = preprocess_untuk_lexicon(text)
|
||||
teks_lower = str(text).lower()
|
||||
skor = _hitung_skor_lexicon(teks_lexicon)
|
||||
label, _ = _klasifikasi_hybrid(teks_model, skor, teks_lower)
|
||||
labels.append(label)
|
||||
labels = [analisis_sentimen_single(t)[0] for t in list_text]
|
||||
return clean_texts, labels
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
Before Width: | Height: | Size: 78 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 78 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 78 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 78 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 78 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 78 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 78 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 78 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 78 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 85 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 95 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 87 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 95 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 84 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 75 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 102 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 73 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 73 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 75 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 78 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 78 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 78 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 78 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 78 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 78 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 72 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
|
|
@ -1,2 +1 @@
|
|||
"conversation_id_str","created_at","favorite_count","full_text","id_str","image_url","in_reply_to_screen_name","lang","location","quote_count","reply_count","retweet_count","tweet_url","user_id_str","username"
|
||||
"2056538953340314095","Wed May 20 16:01:04 +0000 2026","4","@killingmaster27 @evelyndewi7 @katrinrienks @HumorJonTampan @wiyantokdr54 @EviDrajat @RickyKardjono @Micheladam69432 @ebonXboy @AntonRebor79105 @gangsterrandom @Pieter_Sun01 @Rahmah0845 @RSjahfidi @prabowo @SmileyNDeLight @Jediimar @Yens0906 @tokaidirecycle @PembuatSistem @bonapasogit24 @Jenniestale @Maskudin126003 @a8sar6alih @JowoNgalam @KING_RAJA_PWS_ @apaajadeh447870 @50ngoku @Submill @EmmaHusain0845 @BebySoSweet @Yulia_4ja @bungalotus25 @bahlillahadalia wahhh... jadi menteri komdigi dong menteri free ongkir https://t.co/1cfvoqIcNl","2057129489994510620","https://pbs.twimg.com/media/HIxkWwDaMAAXH9U.jpg","killingmaster27","in","","0","2","0","https://x.com/undefined/status/2057129489994510620","1410466956134608899",
|
||||
|
||||
|
|
|
|||
|
|
|
@ -1,2 +1 @@
|
|||
"conversation_id_str","created_at","favorite_count","full_text","id_str","image_url","in_reply_to_screen_name","lang","location","quote_count","reply_count","retweet_count","tweet_url","user_id_str","username"
|
||||
"2056538953340314095","Wed May 20 16:01:04 +0000 2026","4","@killingmaster27 @evelyndewi7 @katrinrienks @HumorJonTampan @wiyantokdr54 @EviDrajat @RickyKardjono @Micheladam69432 @ebonXboy @AntonRebor79105 @gangsterrandom @Pieter_Sun01 @Rahmah0845 @RSjahfidi @prabowo @SmileyNDeLight @Jediimar @Yens0906 @tokaidirecycle @PembuatSistem @bonapasogit24 @Jenniestale @Maskudin126003 @a8sar6alih @JowoNgalam @KING_RAJA_PWS_ @apaajadeh447870 @50ngoku @Submill @EmmaHusain0845 @BebySoSweet @Yulia_4ja @bungalotus25 @bahlillahadalia wahhh... jadi menteri komdigi dong menteri free ongkir https://t.co/1cfvoqIcNl","2057129489994510620","https://pbs.twimg.com/media/HIxkWwDaMAAXH9U.jpg","killingmaster27","in","","0","2","0","https://x.com/undefined/status/2057129489994510620","1410466956134608899",
|
||||
|
||||
|
|
|
|||
|
|
|
@ -1,7 +1,8 @@
|
|||
"conversation_id_str","created_at","favorite_count","full_text","id_str","image_url","in_reply_to_screen_name","lang","location","quote_count","reply_count","retweet_count","tweet_url","user_id_str","username"
|
||||
"2057791279896080430","Fri May 22 11:50:47 +0000 2026","0","komdigi mengeluarkan kebijakan gratis ongkir @komdigi ???","2057791279896080430","","","in","","0","0","0","https://x.com/undefined/status/2057791279896080430","1814143201718247424",
|
||||
"2057660616815571144","Fri May 22 03:11:35 +0000 2026","0","komdigi pembatasan gratis ongkir","2057660616815571144","","","in","","0","0","0","https://x.com/undefined/status/2057660616815571144","1814143201718247424",
|
||||
"2056635322419777630","Tue May 19 07:17:25 +0000 2026","0","komdigi ongkir gk jelas","2056635322419777630","","","in","","0","0","0","https://x.com/undefined/status/2056635322419777630","1814143201718247424",
|
||||
"2056949159845101648","Wed May 20 04:04:30 +0000 2026","0","komdigi kereenn dan bijak membuat keputusan ongkir seperti inii","2056949159845101648","","","in","","0","0","0","https://x.com/undefined/status/2056949159845101648","1814143201718247424",
|
||||
"2056892677149581606","Wed May 20 00:20:04 +0000 2026","0","komdigi keren sih mengadakan kebijakan ongkir kaya gini","2056892677149581606","","","in","","0","0","0","https://x.com/undefined/status/2056892677149581606","1814143201718247424",
|
||||
"2056538953340314095","Wed May 20 16:01:04 +0000 2026","4","@killingmaster27 @evelyndewi7 @katrinrienks @HumorJonTampan @wiyantokdr54 @EviDrajat @RickyKardjono @Micheladam69432 @ebonXboy @AntonRebor79105 @gangsterrandom @Pieter_Sun01 @Rahmah0845 @RSjahfidi @prabowo @SmileyNDeLight @Jediimar @Yens0906 @tokaidirecycle @PembuatSistem @bonapasogit24 @Jenniestale @Maskudin126003 @a8sar6alih @JowoNgalam @KING_RAJA_PWS_ @apaajadeh447870 @50ngoku @Submill @EmmaHusain0845 @BebySoSweet @Yulia_4ja @bungalotus25 @bahlillahadalia wahhh... jadi menteri komdigi dong menteri free ongkir https://t.co/1cfvoqIcNl","2057129489994510620","https://pbs.twimg.com/media/HIxkWwDaMAAXH9U.jpg","killingmaster27","in","","0","2","0","https://x.com/undefined/status/2057129489994510620","1410466956134608899",
|
||||
"2058829602865451445","Mon May 25 08:36:43 +0000 2026","0","komdigi mending ngurusin aja tuh judol daripada ngurusin pembatasan gratis ongkir","2058829602865451445","","","in","","0","0","0","https://x.com/undefined/status/2058829602865451445","2046458899843170304",
|
||||
"2058950111976386607","Mon May 25 16:35:34 +0000 2026","0","komdigi bagus sih ngeluarin kebijakan gratis ongkir kaya ini setuju","2058950111976386607","","","in","","0","0","0","https://x.com/undefined/status/2058950111976386607","2046458899843170304",
|
||||
"2058950559856832819","Mon May 25 16:37:21 +0000 2026","0","setuju bgt komdigi ngeluarin kebijakan gratis ongkir gini","2058950559856832819","","","in","","0","0","0","https://x.com/undefined/status/2058950559856832819","2046458899843170304",
|
||||
"2060234400777064942","Fri May 29 05:38:53 +0000 2026","0","tadi pagi saya melihat berita mengenai pembatasan gratis ongkir oleh komdigi","2060234400777064942","","","in","","0","0","0","https://x.com/undefined/status/2060234400777064942","1814143201718247424",
|
||||
"2058816188659192179","Mon May 25 07:43:24 +0000 2026","0","komdigi gajelas bikin kebijakan ongkir!","2058816188659192179","","","in","","0","0","0","https://x.com/undefined/status/2058816188659192179","1814143201718247424",
|
||||
"2059619299309011403","Wed May 27 12:54:41 +0000 2026","0","komdigi ngeluarin kebijakan ongkir kaya gini ngapain dah","2059619299309011403","","","in","","0","0","0","https://x.com/undefined/status/2059619299309011403","1814143201718247424",
|
||||
"2058816239112446387","Mon May 25 07:43:37 +0000 2026","0","komdigi gk penting buat kebijakan ongkir kaya gini","2058816239112446387","","","in","","0","0","0","https://x.com/undefined/status/2058816239112446387","1814143201718247424",
|
||||
|
|
|
|||
|
|
|
@ -1,7 +1,8 @@
|
|||
"conversation_id_str","created_at","favorite_count","full_text","id_str","image_url","in_reply_to_screen_name","lang","location","quote_count","reply_count","retweet_count","tweet_url","user_id_str","username"
|
||||
"2057660616815571144","Fri May 22 03:11:35 +0000 2026","0","komdigi pembatasan gratis ongkir","2057660616815571144","","","in","","0","0","0","https://x.com/undefined/status/2057660616815571144","1814143201718247424",
|
||||
"2057791279896080430","Fri May 22 11:50:47 +0000 2026","0","komdigi mengeluarkan kebijakan gratis ongkir @komdigi ???","2057791279896080430","","","in","","0","0","0","https://x.com/undefined/status/2057791279896080430","1814143201718247424",
|
||||
"2056635322419777630","Tue May 19 07:17:25 +0000 2026","0","komdigi ongkir gk jelas","2056635322419777630","","","in","","0","0","0","https://x.com/undefined/status/2056635322419777630","1814143201718247424",
|
||||
"2056949159845101648","Wed May 20 04:04:30 +0000 2026","0","komdigi kereenn dan bijak membuat keputusan ongkir seperti inii","2056949159845101648","","","in","","0","0","0","https://x.com/undefined/status/2056949159845101648","1814143201718247424",
|
||||
"2056892677149581606","Wed May 20 00:20:04 +0000 2026","0","komdigi keren sih mengadakan kebijakan ongkir kaya gini","2056892677149581606","","","in","","0","0","0","https://x.com/undefined/status/2056892677149581606","1814143201718247424",
|
||||
"2056538953340314095","Wed May 20 16:01:04 +0000 2026","4","@killingmaster27 @evelyndewi7 @katrinrienks @HumorJonTampan @wiyantokdr54 @EviDrajat @RickyKardjono @Micheladam69432 @ebonXboy @AntonRebor79105 @gangsterrandom @Pieter_Sun01 @Rahmah0845 @RSjahfidi @prabowo @SmileyNDeLight @Jediimar @Yens0906 @tokaidirecycle @PembuatSistem @bonapasogit24 @Jenniestale @Maskudin126003 @a8sar6alih @JowoNgalam @KING_RAJA_PWS_ @apaajadeh447870 @50ngoku @Submill @EmmaHusain0845 @BebySoSweet @Yulia_4ja @bungalotus25 @bahlillahadalia wahhh... jadi menteri komdigi dong menteri free ongkir https://t.co/1cfvoqIcNl","2057129489994510620","https://pbs.twimg.com/media/HIxkWwDaMAAXH9U.jpg","killingmaster27","in","","0","2","0","https://x.com/undefined/status/2057129489994510620","1410466956134608899",
|
||||
"2058829602865451445","Mon May 25 08:36:43 +0000 2026","0","komdigi mending ngurusin aja tuh judol daripada ngurusin pembatasan gratis ongkir","2058829602865451445","","","in","","0","0","0","https://x.com/undefined/status/2058829602865451445","2046458899843170304",
|
||||
"2058950111976386607","Mon May 25 16:35:34 +0000 2026","0","komdigi bagus sih ngeluarin kebijakan gratis ongkir kaya ini setuju","2058950111976386607","","","in","","0","0","0","https://x.com/undefined/status/2058950111976386607","2046458899843170304",
|
||||
"2058950559856832819","Mon May 25 16:37:21 +0000 2026","0","setuju bgt komdigi ngeluarin kebijakan gratis ongkir gini","2058950559856832819","","","in","","0","0","0","https://x.com/undefined/status/2058950559856832819","2046458899843170304",
|
||||
"2060234400777064942","Fri May 29 05:38:53 +0000 2026","0","tadi pagi saya melihat berita mengenai pembatasan gratis ongkir oleh komdigi","2060234400777064942","","","in","","0","0","0","https://x.com/undefined/status/2060234400777064942","1814143201718247424",
|
||||
"2058816188659192179","Mon May 25 07:43:24 +0000 2026","0","komdigi gajelas bikin kebijakan ongkir!","2058816188659192179","","","in","","0","0","0","https://x.com/undefined/status/2058816188659192179","1814143201718247424",
|
||||
"2059619299309011403","Wed May 27 12:54:41 +0000 2026","0","komdigi ngeluarin kebijakan ongkir kaya gini ngapain dah","2059619299309011403","","","in","","0","0","0","https://x.com/undefined/status/2059619299309011403","1814143201718247424",
|
||||
"2058816239112446387","Mon May 25 07:43:37 +0000 2026","0","komdigi gk penting buat kebijakan ongkir kaya gini","2058816239112446387","","","in","","0","0","0","https://x.com/undefined/status/2058816239112446387","1814143201718247424",
|
||||
|
|
|
|||
|
Loading…
Reference in New Issue