revisi model
This commit is contained in:
parent
dea2bc8a2b
commit
68319b9d23
|
|
@ -1116,7 +1116,7 @@ yang
|
||||||
selagi
|
selagi
|
||||||
kebijakan
|
kebijakan
|
||||||
pemerintah
|
pemerintah
|
||||||
komdigi
|
|
||||||
kementerian
|
kementerian
|
||||||
kemendag
|
kemendag
|
||||||
aturan
|
aturan
|
||||||
|
|
|
||||||
Binary file not shown.
Binary file not shown.
|
|
@ -1 +1 @@
|
||||||
{"activated": false, "updated_at": "2026-05-22T16:01:11.643950+00:00"}
|
{"activated": true, "updated_at": "2026-05-29T05:53:53.439233+00:00"}
|
||||||
|
|
@ -3,17 +3,24 @@ preprocessing_page.py
|
||||||
=====================
|
=====================
|
||||||
Halaman Bersihkan Data — NLP Pipeline 5 Tahap.
|
Halaman Bersihkan Data — NLP Pipeline 5 Tahap.
|
||||||
|
|
||||||
PIPELINE 5 TAHAP (selaras dengan sentiment_service.py):
|
PIPELINE 5 TAHAP:
|
||||||
1. Case Folding — ubah semua huruf jadi lowercase
|
1. Case Folding — ubah semua huruf jadi lowercase
|
||||||
2. Cleaning — hapus URL, mention, hashtag, angka, emoji, tanda baca
|
2. Cleaning — hapus URL, mention, hashtag, angka, emoji, tanda baca
|
||||||
3. Normalisasi — singkatan/slang → kata baku
|
3. Normalisasi — singkatan/slang → kata baku (DARI FILE normalisasi)
|
||||||
4. Stopword Removal — hapus kata umum; JAGA kata sentimen penting
|
4. Stopword Removal — hapus kata umum (DARI FILE stopword); JAGA kata sentimen
|
||||||
5. Stemming — bentuk dasar kata via Sastrawi ECS
|
5. Stemming — bentuk dasar kata via Sastrawi ECS
|
||||||
|
|
||||||
Catatan: Tokenizing tidak ditampilkan sebagai tahap tersendiri karena:
|
PERUBAHAN DARI VERSI SEBELUMNYA:
|
||||||
• Stopword removal & stemming sudah melakukan split() secara internal
|
- Normalisasi kini dimuat dari 'indonesian-normalisasi-slangword-complete.txt'
|
||||||
• TF-IDF melakukan tokenisasi sendiri saat inferensi
|
(1.700+ entri), menggantikan dict hardcoded yang hanya ~60 entri.
|
||||||
→ Tokenizing adalah proses teknis, bukan tahap utama pipeline
|
- Stopword kini murni dari 'indonesian-stopwords-complete.txt', ditambah
|
||||||
|
noise Twitter yang spesifik — tidak ada penghapusan manual acak.
|
||||||
|
- KATA_SENTIMEN_PENTING diperluas dengan kata domain e-commerce/ongkir.
|
||||||
|
- Semua fungsi preprocessing menerima parameter eksplisit (tidak pakai global).
|
||||||
|
|
||||||
|
CATATAN PENTING:
|
||||||
|
Pipeline ini HARUS IDENTIK dengan sentiment_service.py agar token yang
|
||||||
|
dihasilkan di sini konsisten dengan token saat training model.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import streamlit as st
|
import streamlit as st
|
||||||
|
|
@ -33,18 +40,15 @@ from timezone_utils import (
|
||||||
get_timezone_name,
|
get_timezone_name,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
# ═══════════════════════════════════════════════════════════
|
# ═══════════════════════════════════════════════════════════
|
||||||
# TIMEZONE HELPERS
|
# TIMEZONE HELPERS
|
||||||
# ═══════════════════════════════════════════════════════════
|
# ═══════════════════════════════════════════════════════════
|
||||||
|
|
||||||
def parse_dt(series):
|
def parse_dt(series):
|
||||||
return parse_dt_with_tz(
|
return parse_dt_with_tz(
|
||||||
series,
|
series,
|
||||||
st.session_state.get("user_timezone", "WIB (UTC+7)")
|
st.session_state.get("user_timezone", "WIB (UTC+7)")
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def parse_crawled_dt(series):
|
def parse_crawled_dt(series):
|
||||||
return parse_dt_with_source_tz(
|
return parse_dt_with_source_tz(
|
||||||
series,
|
series,
|
||||||
|
|
@ -52,7 +56,6 @@ def parse_crawled_dt(series):
|
||||||
os.getenv("APP_TIMEZONE", "Asia/Jakarta")
|
os.getenv("APP_TIMEZONE", "Asia/Jakarta")
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def format_dt(value):
|
def format_dt(value):
|
||||||
if value is None or pd.isna(value):
|
if value is None or pd.isna(value):
|
||||||
return "Belum ada"
|
return "Belum ada"
|
||||||
|
|
@ -96,63 +99,224 @@ def _sync_dynamic_period():
|
||||||
|
|
||||||
# ═══════════════════════════════════════════════════════════
|
# ═══════════════════════════════════════════════════════════
|
||||||
# PREPROCESSING PIPELINE — 5 TAHAP
|
# PREPROCESSING PIPELINE — 5 TAHAP
|
||||||
# PENTING: Pipeline ini harus IDENTIK dengan sentiment_service.py
|
|
||||||
# agar token yang dihasilkan konsisten dengan training model.
|
|
||||||
#
|
#
|
||||||
# URUTAN:
|
# ┌─────────────────────────────────────────────────────┐
|
||||||
# 1. Case Folding → lowercase dulu sebelum cleaning
|
# │ PENTING: Pipeline ini HARUS identik dengan │
|
||||||
# 2. Cleaning → hapus noise setelah lowercase
|
# │ sentiment_service.py agar token konsisten! │
|
||||||
# 3. Normalisasi → singkatan/slang → kata baku
|
# │ │
|
||||||
# 4. Stopword Removal → buang kata umum, jaga kata sentimen
|
# │ URUTAN WAJIB: │
|
||||||
# 5. Stemming → bentuk dasar via Sastrawi ECS
|
# │ 1. Case Folding → lowercase dulu │
|
||||||
|
# │ 2. Cleaning → hapus noise setelah lowercase │
|
||||||
|
# │ 3. Normalisasi → slang→baku setelah bersih │
|
||||||
|
# │ 4. Stopword → buang kata umum, jaga sentimen │
|
||||||
|
# │ 5. Stemming → bentuk dasar via Sastrawi ECS │
|
||||||
|
# └─────────────────────────────────────────────────────┘
|
||||||
# ═══════════════════════════════════════════════════════════
|
# ═══════════════════════════════════════════════════════════
|
||||||
|
|
||||||
|
|
||||||
|
# ───────────────────────────────────────────────────────────
|
||||||
|
# KATA SENTIMEN PENTING
|
||||||
|
# Kata-kata ini WAJIB DIJAGA dan tidak boleh dihapus saat
|
||||||
|
# stopword removal, meskipun ada di file stopword.
|
||||||
|
#
|
||||||
|
# Kenapa perlu? Karena file stopword mengandung kata seperti
|
||||||
|
# "tidak", "belum", "sangat" yang justru krusial untuk
|
||||||
|
# menentukan sentimen positif/negatif suatu kalimat.
|
||||||
|
# ───────────────────────────────────────────────────────────
|
||||||
KATA_SENTIMEN_PENTING = {
|
KATA_SENTIMEN_PENTING = {
|
||||||
# Negasi
|
# ── Negasi (pembalik makna kalimat) ──────────────────
|
||||||
|
# "tidak bagus" ≠ "bagus" → "tidak" wajib ada
|
||||||
"tidak", "bukan", "jangan", "kurang", "belum", "tanpa",
|
"tidak", "bukan", "jangan", "kurang", "belum", "tanpa",
|
||||||
# Positif
|
# ── Intensitas (penguat/pelemah sentimen) ─────────────
|
||||||
|
# "sangat bagus" lebih positif dari "bagus" saja
|
||||||
|
"sangat", "banget", "sekali", "paling", "amat",
|
||||||
|
"luar", "biasa", # ← "luar biasa" = dua token, keduanya dijaga
|
||||||
|
# ── Positif umum ──────────────────────────────────────
|
||||||
"keren", "bagus", "mantap", "setuju", "dukung", "mendukung",
|
"keren", "bagus", "mantap", "setuju", "dukung", "mendukung",
|
||||||
"andal", "handal", "gercep", "bangga", "senang", "suka",
|
"andal", "handal", "gercep", "bangga", "senang", "suka",
|
||||||
"baik", "benar", "tepat", "oke",
|
"baik", "benar", "tepat", "oke", "puas",
|
||||||
"sejahtera", "berkembang", "maju", "inovatif",
|
"sejahtera", "berkembang", "maju", "inovatif",
|
||||||
"tegas", "sigap", "tanggap", "adil", "bijak", "bermanfaat",
|
"tegas", "sigap", "tanggap", "adil", "bijak", "bermanfaat",
|
||||||
"untung", "berhasil", "sukses", "solusi", "manfaat",
|
"untung", "berhasil", "sukses", "solusi", "manfaat",
|
||||||
"berguna", "membantu", "bantu", "pro", "lanjut",
|
"berguna", "membantu", "bantu", "pro", "lanjut",
|
||||||
"sangat", "banget", "sekali", "paling", "amat", "luar", "biasa",
|
# ── Positif domain e-commerce / ongkir ────────────────
|
||||||
# Negatif
|
"gratis", "murah", "hemat", "terjangkau", "cepat",
|
||||||
|
"aman", "mudah", "praktis", "terpercaya",
|
||||||
|
# ── Negatif umum ──────────────────────────────────────
|
||||||
"kecewa", "buruk", "jelek", "parah", "gagal", "hancur",
|
"kecewa", "buruk", "jelek", "parah", "gagal", "hancur",
|
||||||
"rusak", "bohong", "tipu", "korupsi",
|
"rusak", "bohong", "tipu", "korupsi",
|
||||||
# Emosi
|
# ── Negatif domain e-commerce / ongkir ────────────────
|
||||||
"marah", "sedih", "khawatir",
|
"mahal", "lambat", "lelet", "ribet", "susah", "repot",
|
||||||
|
"rugi", "boros",
|
||||||
|
# ── Emosi ─────────────────────────────────────────────
|
||||||
|
"marah", "sedih", "khawatir", "kecewa",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def _load_stopwords():
|
# ───────────────────────────────────────────────────────────
|
||||||
|
# LOAD NORMALIZATION DARI FILE
|
||||||
|
# File: indonesian-normalisasi-slangword-complete.txt
|
||||||
|
# Format per baris: slang,kata_baku
|
||||||
|
# Contoh: gk,tidak | ongkir,ongkos kirim | free,gratis
|
||||||
|
# ───────────────────────────────────────────────────────────
|
||||||
|
def _load_normalization() -> dict:
|
||||||
|
"""
|
||||||
|
Muat kamus normalisasi dari file eksternal.
|
||||||
|
|
||||||
|
KENAPA DARI FILE?
|
||||||
|
File berisi 1.700+ pasang slang→baku yang jauh lebih lengkap
|
||||||
|
dibanding dict hardcoded. Dengan ini, kata seperti:
|
||||||
|
gk/ga/gak/kagak/ngga → semua jadi "tidak"
|
||||||
|
bgt/bngt/bget → semua jadi "sangat"
|
||||||
|
ongkir/ongkr → jadi "ongkos kirim"
|
||||||
|
...dan ribuan kasus lainnya tertangani otomatis.
|
||||||
|
|
||||||
|
Setelah file dimuat, override dengan entri khusus domain
|
||||||
|
(nama platform, singkatan kebijakan) yang mungkin belum ada
|
||||||
|
di file generik.
|
||||||
|
"""
|
||||||
|
norm_file = "indonesian-normalisasi-slangword-complete.txt"
|
||||||
|
norm_dict: dict = {}
|
||||||
|
|
||||||
|
try:
|
||||||
|
with open(norm_file, "r", encoding="utf-8") as f:
|
||||||
|
for line in f:
|
||||||
|
line = line.strip()
|
||||||
|
if not line:
|
||||||
|
continue
|
||||||
|
# Split hanya pada koma pertama — nilai bisa mengandung koma
|
||||||
|
# Contoh: "on the way, sedang di jalan,dijalan" → split jadi 2 bagian
|
||||||
|
parts = line.split(",", 1)
|
||||||
|
if len(parts) != 2:
|
||||||
|
continue
|
||||||
|
# Bersihkan tanda kutip liar di awal/akhir (ada di beberapa baris file)
|
||||||
|
slang = parts[0].strip().strip("'\"").lower()
|
||||||
|
normal = parts[1].strip().lower()
|
||||||
|
if slang and normal:
|
||||||
|
norm_dict[slang] = normal
|
||||||
|
except FileNotFoundError:
|
||||||
|
# Jika file tidak ditemukan, lanjut dengan dict kosong.
|
||||||
|
# Entri domain di bawah tetap akan ditambahkan.
|
||||||
|
st.warning(
|
||||||
|
"⚠️ File normalisasi tidak ditemukan: "
|
||||||
|
f"'{norm_file}'. Hanya entri domain yang aktif."
|
||||||
|
)
|
||||||
|
|
||||||
|
# ── Override khusus domain ───────────────────────────
|
||||||
|
# Entri ini menimpa file generik karena domain spesifik
|
||||||
|
# membutuhkan perlakuan khusus (nama platform tidak diubah,
|
||||||
|
# singkatan kebijakan punya padanan resmi, dll.)
|
||||||
|
DOMAIN_OVERRIDES: dict = {
|
||||||
|
# Nama platform — pertahankan apa adanya
|
||||||
|
"shopee": "shopee",
|
||||||
|
"tokopedia": "tokopedia",
|
||||||
|
"lazada": "lazada",
|
||||||
|
"tiktok": "tiktok",
|
||||||
|
"bukalapak": "bukalapak",
|
||||||
|
"blibli": "blibli",
|
||||||
|
# Logistik — pertahankan apa adanya
|
||||||
|
"sicepat": "sicepat",
|
||||||
|
"jne": "jne",
|
||||||
|
"jnt": "jnt",
|
||||||
|
"anteraja": "anteraja",
|
||||||
|
"ninja": "ninja",
|
||||||
|
# Ongkir & belanja
|
||||||
|
"freeongkir": "gratis ongkos kirim",
|
||||||
|
"gratisongkir": "gratis ongkos kirim",
|
||||||
|
"ongkir": "ongkos kirim",
|
||||||
|
"ongkr": "ongkos kirim",
|
||||||
|
"bykrm": "biaya kirim",
|
||||||
|
"biayakirim": "biaya pengiriman",
|
||||||
|
# Kebijakan & lembaga
|
||||||
|
"komdigi": "komdigi",
|
||||||
|
"kemendag": "kementerian perdagangan",
|
||||||
|
"kominfo": "kementerian komunikasi",
|
||||||
|
# E-commerce umum
|
||||||
|
"ecommerce": "e commerce",
|
||||||
|
"marketplace": "marketplace",
|
||||||
|
"seller": "penjual",
|
||||||
|
"buyer": "pembeli",
|
||||||
|
"online": "online",
|
||||||
|
}
|
||||||
|
norm_dict.update(DOMAIN_OVERRIDES)
|
||||||
|
|
||||||
|
return norm_dict
|
||||||
|
|
||||||
|
|
||||||
|
# ───────────────────────────────────────────────────────────
|
||||||
|
# LOAD STOPWORDS DARI FILE
|
||||||
|
# File: indonesian-stopwords-complete.txt
|
||||||
|
# Format: satu kata per baris
|
||||||
|
# ───────────────────────────────────────────────────────────
|
||||||
|
def _load_stopwords() -> set:
|
||||||
|
"""
|
||||||
|
Muat daftar stopword dari file eksternal.
|
||||||
|
|
||||||
|
PROSES SETELAH MUAT FILE:
|
||||||
|
1. Hapus KATA_SENTIMEN_PENTING dari daftar
|
||||||
|
→ Agar "tidak", "belum", "sangat", dll. tidak ikut dibuang
|
||||||
|
2. Tambahkan noise Twitter/sosmed yang memang harus dibuang
|
||||||
|
→ "rt", "amp", sisa URL, suara tawa, partikel informal
|
||||||
|
|
||||||
|
KENAPA DARI FILE?
|
||||||
|
File berisi 700+ stopword Indonesia yang lebih lengkap dan
|
||||||
|
terstandar dibanding daftar manual. Kita tidak perlu menambah/
|
||||||
|
mengurangi secara manual kecuali untuk dua kategori di atas.
|
||||||
|
"""
|
||||||
stopword_file = "indonesian-stopwords-complete.txt"
|
stopword_file = "indonesian-stopwords-complete.txt"
|
||||||
base = set()
|
base: set = set()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
with open(stopword_file, "r", encoding="utf-8") as f:
|
with open(stopword_file, "r", encoding="utf-8") as f:
|
||||||
base = set(f.read().splitlines())
|
for line in f:
|
||||||
|
word = line.strip().lower()
|
||||||
|
if word:
|
||||||
|
base.add(word)
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
|
# Fallback minimal — cukup untuk tetap jalan
|
||||||
|
st.warning(
|
||||||
|
"⚠️ File stopword tidak ditemukan: "
|
||||||
|
f"'{stopword_file}'. Menggunakan daftar minimal."
|
||||||
|
)
|
||||||
base = {
|
base = {
|
||||||
"yang", "dan", "di", "ke", "dari", "ini", "itu",
|
"yang", "dan", "di", "ke", "dari", "ini", "itu",
|
||||||
"dengan", "untuk", "pada", "adalah", "oleh", "ada",
|
"dengan", "untuk", "pada", "adalah", "oleh", "ada",
|
||||||
"ya", "akan", "atau", "juga", "sama", "karena",
|
"ya", "akan", "atau", "juga", "sama", "karena",
|
||||||
"jika", "sudah", "telah",
|
"jika", "sudah", "telah", "jadi", "bisa",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# ── Langkah 1: Lindungi kata sentimen ────────────────
|
||||||
|
# Beberapa kata sentimen penting ADA di file stopword
|
||||||
|
# (misal: "tidak", "belum", "sangat", "paling", "kurang").
|
||||||
|
# Kita HAPUS dari stopword agar tidak ikut dibuang.
|
||||||
for kata in KATA_SENTIMEN_PENTING:
|
for kata in KATA_SENTIMEN_PENTING:
|
||||||
base.discard(kata)
|
base.discard(kata)
|
||||||
base.update({
|
|
||||||
"rt", "amp", "https", "http", "co", "t",
|
# ── Langkah 2: Tambah noise Twitter/sosmed ────────────
|
||||||
"wkwk", "wkwkwk", "haha", "hehe", "xixi",
|
# Ini bukan stopword bahasa Indonesia biasa, tapi noise
|
||||||
"yg", "dgn", "utk", "dr", "krn", "tp", "jd", "sdh",
|
# yang sangat sering muncul di tweet dan tidak bermakna.
|
||||||
"aja", "doang", "nih", "sih", "dong", "deh",
|
TWITTER_NOISE: set = {
|
||||||
"loh", "lah", "tuh", "kak", "gan",
|
# Artefak Twitter
|
||||||
})
|
"rt", "amp",
|
||||||
|
# Sisa URL setelah cleaning (kadang lolos)
|
||||||
|
"https", "http", "co", "pic",
|
||||||
|
# Suara tawa (tidak bermakna untuk sentimen)
|
||||||
|
"wkwk", "wkwkwk", "wkwkwkwk",
|
||||||
|
"haha", "hahaha", "hehe", "hihi", "huhu", "xixi",
|
||||||
|
# Partikel informal yang tidak bermakna
|
||||||
|
"nih", "sih", "dong", "deh", "loh", "lah", "tuh",
|
||||||
|
"kak", "gan", "bro", "sob", "min",
|
||||||
|
}
|
||||||
|
base.update(TWITTER_NOISE)
|
||||||
|
|
||||||
return base
|
return base
|
||||||
|
|
||||||
|
|
||||||
|
# ───────────────────────────────────────────────────────────
|
||||||
|
# LOAD STEMMER
|
||||||
|
# ───────────────────────────────────────────────────────────
|
||||||
def _load_stemmer():
|
def _load_stemmer():
|
||||||
|
"""Muat stemmer Sastrawi. Return None jika tidak tersedia."""
|
||||||
try:
|
try:
|
||||||
from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
|
from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
|
||||||
return StemmerFactory().create_stemmer()
|
return StemmerFactory().create_stemmer()
|
||||||
|
|
@ -160,131 +324,198 @@ def _load_stemmer():
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
NORMALISASI = {
|
# ═══════════════════════════════════════════════════════════
|
||||||
# Negasi
|
# FUNGSI 5 TAHAP PREPROCESSING
|
||||||
"gk": "tidak", "ga": "tidak", "gak": "tidak",
|
# Setiap fungsi bertanggung jawab SATU tahap saja.
|
||||||
"nggak": "tidak", "ngga": "tidak", "tdk": "tidak",
|
# Input & output setiap tahap dijelaskan di docstring.
|
||||||
"tak": "tidak", "enggak": "tidak", "engga": "tidak",
|
# ═══════════════════════════════════════════════════════════
|
||||||
"kagak": "tidak", "kaga": "tidak", "ndak": "tidak",
|
|
||||||
"gkk": "tidak", "ngak": "tidak",
|
|
||||||
# Kata ganti
|
|
||||||
"yg": "yang", "dgn": "dengan", "utk": "untuk",
|
|
||||||
"org": "orang", "krn": "karena", "dr": "dari",
|
|
||||||
"sm": "sama", "pd": "pada", "dlm": "dalam",
|
|
||||||
"bwt": "buat", "trm": "terima",
|
|
||||||
# Verba
|
|
||||||
"tp": "tapi", "tpi": "tapi", "jd": "jadi",
|
|
||||||
"sdh": "sudah", "blm": "belum", "emg": "memang",
|
|
||||||
"emang": "memang", "gimana": "bagaimana",
|
|
||||||
"gitu": "begitu", "gini": "begini",
|
|
||||||
"udah": "sudah", "udh": "sudah",
|
|
||||||
"mau": "mau",
|
|
||||||
# Intensitas
|
|
||||||
"bgt": "banget", "bngt": "banget", "bget": "banget", "bgtt": "banget",
|
|
||||||
# Positif informal
|
|
||||||
"bener": "benar", "beneran": "benar",
|
|
||||||
"mantep": "mantap", "mntap": "mantap",
|
|
||||||
"kece": "keren", "kece bgt": "keren banget",
|
|
||||||
"cucok": "cocok", "cucuk": "cocok",
|
|
||||||
"cakep": "bagus", "oke bgt": "oke banget",
|
|
||||||
"sip": "baik", "siipp": "baik",
|
|
||||||
"top": "terbaik", "topbgt": "terbaik banget",
|
|
||||||
"jos": "bagus", "josss": "bagus",
|
|
||||||
"goks": "luar biasa",
|
|
||||||
"setujuu": "setuju", "stuju": "setuju",
|
|
||||||
"dukung": "dukung",
|
|
||||||
"proud": "bangga",
|
|
||||||
"mantul": "mantap betul",
|
|
||||||
# Negatif informal
|
|
||||||
"ancur": "hancur", "ancrr": "hancur",
|
|
||||||
"parahh": "parah", "parahhh": "parah",
|
|
||||||
"gagall": "gagal",
|
|
||||||
"ngaco": "tidak benar",
|
|
||||||
"ngasal": "tidak benar",
|
|
||||||
"receh": "tidak penting",
|
|
||||||
"gaje": "tidak jelas",
|
|
||||||
"asal": "sembarangan",
|
|
||||||
# Domain
|
|
||||||
"ongkir": "ongkos kirim",
|
|
||||||
"freeongkir": "gratis ongkos kirim",
|
|
||||||
"gratisongkir": "gratis ongkos kirim",
|
|
||||||
"free": "gratis",
|
|
||||||
"ecommerce": "e commerce",
|
|
||||||
"komdigi": "komdigi",
|
|
||||||
"subsidi": "subsidi",
|
|
||||||
"marketplace": "marketplace",
|
|
||||||
"seller": "penjual",
|
|
||||||
"buyer": "pembeli",
|
|
||||||
"online": "online",
|
|
||||||
"shopee": "shopee",
|
|
||||||
"tokopedia": "tokopedia",
|
|
||||||
"lazada": "lazada",
|
|
||||||
"tiktok": "tiktok",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# ── Tahap 1: Case Folding ───────────────────────────────────────────────────
|
|
||||||
def step1_case_folding(text: str) -> str:
|
def step1_case_folding(text: str) -> str:
|
||||||
|
"""
|
||||||
|
TAHAP 1 — CASE FOLDING
|
||||||
|
Input : teks asli (campuran huruf besar/kecil)
|
||||||
|
Output: semua huruf jadi lowercase
|
||||||
|
|
||||||
|
Kenapa pertama?
|
||||||
|
Agar tahap berikutnya (cleaning, normalisasi) bekerja
|
||||||
|
secara konsisten — regex dan dict lookup case-sensitive.
|
||||||
|
Contoh: "Gratis" → "gratis", "ONGKIR" → "ongkir"
|
||||||
|
"""
|
||||||
return str(text).lower()
|
return str(text).lower()
|
||||||
|
|
||||||
|
|
||||||
# ── Tahap 2: Cleaning ───────────────────────────────────────────────────────
|
|
||||||
def step2_cleaning(text: str) -> str:
|
def step2_cleaning(text: str) -> str:
|
||||||
|
"""
|
||||||
|
TAHAP 2 — CLEANING
|
||||||
|
Input : teks lowercase
|
||||||
|
Output: teks bersih dari semua elemen noise
|
||||||
|
|
||||||
|
Urutan pembersihan PENTING:
|
||||||
|
1. URL dulu (sebelum @ dan # agar tidak salah potong)
|
||||||
|
2. Mention (@username)
|
||||||
|
3. Hashtag (#topik)
|
||||||
|
4. Angka
|
||||||
|
5. Emoji & simbol unicode
|
||||||
|
6. Tanda baca
|
||||||
|
7. Karakter non-latin (huruf Arab, Cina, dll.)
|
||||||
|
8. Spasi berlebih
|
||||||
|
"""
|
||||||
|
# 1. Hapus URL (http, https, www)
|
||||||
text = re.sub(r"http\S+|www\S+|https\S+", "", text)
|
text = re.sub(r"http\S+|www\S+|https\S+", "", text)
|
||||||
|
# 2. Hapus mention Twitter (@username)
|
||||||
text = re.sub(r"@\w+", "", text)
|
text = re.sub(r"@\w+", "", text)
|
||||||
|
# 3. Hapus hashtag (#topik)
|
||||||
text = re.sub(r"#\w+", "", text)
|
text = re.sub(r"#\w+", "", text)
|
||||||
|
# 4. Hapus angka dan digit
|
||||||
text = re.sub(r"\d+", "", text)
|
text = re.sub(r"\d+", "", text)
|
||||||
|
# 5. Hapus emoji & simbol unicode (berbagai range)
|
||||||
text = re.sub(
|
text = re.sub(
|
||||||
r"[\U00010000-\U0010ffff"
|
r"["
|
||||||
r"\U0001F600-\U0001F64F"
|
r"\U00010000-\U0010ffff" # Suplemen karakter unicode
|
||||||
r"\U0001F300-\U0001F5FF"
|
r"\U0001F600-\U0001F64F" # Emotikon wajah
|
||||||
r"\U0001F680-\U0001F6FF"
|
r"\U0001F300-\U0001F5FF" # Simbol & piktogram
|
||||||
r"\U0001F1E0-\U0001F1FF"
|
r"\U0001F680-\U0001F6FF" # Transport & peta
|
||||||
r"\u2600-\u26FF\u2700-\u27BF"
|
r"\U0001F1E0-\U0001F1FF" # Bendera negara
|
||||||
r"]+", "", text, flags=re.UNICODE
|
r"\u2600-\u26FF" # Simbol campuran
|
||||||
|
r"\u2700-\u27BF" # Dingbats
|
||||||
|
r"]+",
|
||||||
|
"", text, flags=re.UNICODE
|
||||||
)
|
)
|
||||||
|
# 6. Hapus tanda baca (.,!?;: dll.)
|
||||||
text = text.translate(str.maketrans("", "", string.punctuation))
|
text = text.translate(str.maketrans("", "", string.punctuation))
|
||||||
|
# 7. Hapus karakter non-latin (hanya sisakan huruf a-z dan spasi)
|
||||||
text = re.sub(r"[^a-zA-Z\s]", "", text)
|
text = re.sub(r"[^a-zA-Z\s]", "", text)
|
||||||
|
# 8. Normalisasi spasi berlebih → satu spasi, lalu strip
|
||||||
text = re.sub(r"\s+", " ", text).strip()
|
text = re.sub(r"\s+", " ", text).strip()
|
||||||
return text
|
return text
|
||||||
|
|
||||||
|
|
||||||
# ── Tahap 3: Normalisasi ────────────────────────────────────────────────────
|
def step3_normalization(text: str, norm_dict: dict) -> str:
|
||||||
def step3_normalization(text: str) -> str:
|
"""
|
||||||
return " ".join(NORMALISASI.get(word, word) for word in text.split())
|
TAHAP 3 — NORMALISASI
|
||||||
|
Input : teks bersih (sudah case fold + cleaning)
|
||||||
|
norm_dict : kamus {slang: kata_baku} dari file
|
||||||
|
Output: teks dengan slang/singkatan sudah diganti kata baku
|
||||||
|
|
||||||
|
Cara kerja: token per token (word by word).
|
||||||
|
Setiap token dicari di norm_dict.
|
||||||
|
Jika ada → ganti. Jika tidak ada → biarkan.
|
||||||
|
|
||||||
|
Contoh:
|
||||||
|
"gk bs ongkir" → "tidak bisa ongkos kirim"
|
||||||
|
"mantep bgt" → "mantap sangat"
|
||||||
|
|
||||||
|
Kenapa setelah Cleaning?
|
||||||
|
Karena slang di file ditulis dalam bentuk sudah lowercase
|
||||||
|
dan sudah tanpa tanda baca. Jika normalisasi dilakukan
|
||||||
|
sebelum cleaning, banyak entri tidak cocok.
|
||||||
|
"""
|
||||||
|
tokens = text.split()
|
||||||
|
normalized = [norm_dict.get(token, token) for token in tokens]
|
||||||
|
return " ".join(normalized)
|
||||||
|
|
||||||
|
|
||||||
# ── Tahap 4: Stopword Removal ───────────────────────────────────────────────
|
|
||||||
def step4_stopword_removal(tokens: list, stopwords: set) -> list:
|
def step4_stopword_removal(tokens: list, stopwords: set) -> list:
|
||||||
return [
|
"""
|
||||||
w for w in tokens
|
TAHAP 4 — STOPWORD REMOVAL
|
||||||
if (w not in stopwords or w in KATA_SENTIMEN_PENTING) and len(w) > 2
|
Input : list token (hasil split dari teks ternormalisasi)
|
||||||
]
|
stopwords : set kata yang harus dibuang (dari file)
|
||||||
|
Output: list token bersih
|
||||||
|
|
||||||
|
ATURAN PENYARINGAN (prioritas urutan):
|
||||||
|
1. JAGA token yang ada di KATA_SENTIMEN_PENTING
|
||||||
|
→ meskipun juga ada di stopwords, tetap disimpan
|
||||||
|
2. BUANG token yang ada di stopwords
|
||||||
|
3. BUANG token dengan panjang ≤ 2 karakter
|
||||||
|
→ menghilangkan sisa noise seperti "rt", "yg", "di"
|
||||||
|
→ PENGECUALIAN: token di KATA_SENTIMEN_PENTING tetap disimpan
|
||||||
|
walau ≤ 2 karakter (contoh: "ok" jika masuk sentimen)
|
||||||
|
|
||||||
|
Kenapa setelah Normalisasi?
|
||||||
|
Agar "gak" yang sudah dinormalisasi jadi "tidak" tidak ikut
|
||||||
|
dibuang — "tidak" dilindungi di KATA_SENTIMEN_PENTING.
|
||||||
|
"""
|
||||||
|
result = []
|
||||||
|
for token in tokens:
|
||||||
|
# Prioritas 1: selalu simpan jika kata sentimen penting
|
||||||
|
if token in KATA_SENTIMEN_PENTING:
|
||||||
|
result.append(token)
|
||||||
|
continue
|
||||||
|
# Prioritas 2: buang jika stopword
|
||||||
|
if token in stopwords:
|
||||||
|
continue
|
||||||
|
# Prioritas 3: buang jika terlalu pendek (noise)
|
||||||
|
if len(token) <= 2:
|
||||||
|
continue
|
||||||
|
# Lolos semua filter → simpan
|
||||||
|
result.append(token)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
# ── Tahap 5: Stemming ───────────────────────────────────────────────────────
|
|
||||||
def step5_stemming(tokens: list, stemmer) -> list:
|
def step5_stemming(tokens: list, stemmer) -> list:
|
||||||
|
"""
|
||||||
|
TAHAP 5 — STEMMING
|
||||||
|
Input : list token setelah stopword removal
|
||||||
|
stemmer : objek Sastrawi (atau None)
|
||||||
|
Output: list token dalam bentuk kata dasar
|
||||||
|
|
||||||
|
Algoritma: Enhanced Confix Stripping (ECS) via Sastrawi
|
||||||
|
Contoh:
|
||||||
|
"pengiriman" → "kirim"
|
||||||
|
"pembatasan" → "batas"
|
||||||
|
"berlari" → "lari"
|
||||||
|
"makanan" → "makan"
|
||||||
|
|
||||||
|
Jika stemmer None (Sastrawi tidak terinstal), token dikembalikan
|
||||||
|
apa adanya tanpa error.
|
||||||
|
"""
|
||||||
if stemmer is None:
|
if stemmer is None:
|
||||||
return tokens
|
return tokens
|
||||||
return [stemmer.stem(w) for w in tokens]
|
return [stemmer.stem(token) for token in tokens]
|
||||||
|
|
||||||
|
|
||||||
def full_preprocessing(text: str, stopwords: set, stemmer):
|
# ───────────────────────────────────────────────────────────
|
||||||
|
# FUNGSI UTAMA — JALANKAN SEMUA 5 TAHAP
|
||||||
|
# ───────────────────────────────────────────────────────────
|
||||||
|
def full_preprocessing(
|
||||||
|
text: str,
|
||||||
|
stopwords: set,
|
||||||
|
stemmer,
|
||||||
|
norm_dict: dict,
|
||||||
|
) -> dict:
|
||||||
"""
|
"""
|
||||||
Jalankan 5 tahap preprocessing dan kembalikan dict hasil setiap tahap.
|
Jalankan 5 tahap preprocessing secara berurutan dan kembalikan
|
||||||
|
hasil setiap tahap sebagai dict (untuk ditampilkan di tabel).
|
||||||
|
|
||||||
URUTAN TAHAP:
|
Parameter:
|
||||||
1. Case Folding → lowercase
|
text : teks tweet asli
|
||||||
2. Cleaning → hapus noise
|
stopwords : set stopword (dari _load_stopwords)
|
||||||
3. Normalisasi → normalisasi kata
|
stemmer : objek Sastrawi (dari _load_stemmer)
|
||||||
4. Stopword Removal → buang stopword (split() dilakukan internal)
|
norm_dict : kamus normalisasi (dari _load_normalization)
|
||||||
5. Stemming → bentuk dasar
|
|
||||||
|
Return dict berisi:
|
||||||
|
setelah_casefolding : hasil Tahap 1
|
||||||
|
setelah_cleaning : hasil Tahap 2
|
||||||
|
setelah_normalisasi : hasil Tahap 3
|
||||||
|
setelah_stopword : hasil Tahap 4 (joined string)
|
||||||
|
clean_text : hasil akhir Tahap 5 (joined string)
|
||||||
|
_tokens_clean : hasil Tahap 5 sebagai list (untuk analisis)
|
||||||
"""
|
"""
|
||||||
s1_fold = step1_case_folding(text)
|
# Tahap 1 — Case Folding
|
||||||
s2_clean = step2_cleaning(s1_fold)
|
s1_fold = step1_case_folding(text)
|
||||||
s3_norm = step3_normalization(s2_clean)
|
|
||||||
s4_filtered = step4_stopword_removal(s3_norm.split(), stopwords)
|
# Tahap 2 — Cleaning
|
||||||
s5_stemmed = step5_stemming(s4_filtered, stemmer)
|
s2_clean = step2_cleaning(s1_fold)
|
||||||
|
|
||||||
|
# Tahap 3 — Normalisasi (perlu norm_dict)
|
||||||
|
s3_norm = step3_normalization(s2_clean, norm_dict)
|
||||||
|
|
||||||
|
# Tahap 4 — Stopword Removal (split → filter → simpan sebagai list)
|
||||||
|
s4_tokens = s3_norm.split()
|
||||||
|
s4_filtered = step4_stopword_removal(s4_tokens, stopwords)
|
||||||
|
|
||||||
|
# Tahap 5 — Stemming
|
||||||
|
s5_stemmed = step5_stemming(s4_filtered, stemmer)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"setelah_casefolding": s1_fold,
|
"setelah_casefolding": s1_fold,
|
||||||
|
|
@ -292,7 +523,7 @@ def full_preprocessing(text: str, stopwords: set, stemmer):
|
||||||
"setelah_normalisasi": s3_norm,
|
"setelah_normalisasi": s3_norm,
|
||||||
"setelah_stopword": " ".join(s4_filtered),
|
"setelah_stopword": " ".join(s4_filtered),
|
||||||
"clean_text": " ".join(s5_stemmed),
|
"clean_text": " ".join(s5_stemmed),
|
||||||
"_tokens_clean": s5_stemmed,
|
"_tokens_clean": s5_stemmed, # list, untuk Counter frekuensi kata
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -443,7 +674,7 @@ def _render_page_header():
|
||||||
# PIPELINE STEPS CARDS
|
# PIPELINE STEPS CARDS
|
||||||
# ═══════════════════════════════════════════════════════════
|
# ═══════════════════════════════════════════════════════════
|
||||||
|
|
||||||
def _render_pipeline_steps(stemmer_ok):
|
def _render_pipeline_steps(stemmer_ok: bool, norm_count: int, sw_count: int):
|
||||||
steps = [
|
steps = [
|
||||||
{
|
{
|
||||||
"num": "01", "anim": "pipe-1",
|
"num": "01", "anim": "pipe-1",
|
||||||
|
|
@ -456,8 +687,8 @@ def _render_pipeline_steps(stemmer_ok):
|
||||||
'"ONGKIR" → "ongkir"',
|
'"ONGKIR" → "ongkir"',
|
||||||
'"KEREN" → "keren"',
|
'"KEREN" → "keren"',
|
||||||
"Seluruh karakter → huruf kecil",
|
"Seluruh karakter → huruf kecil",
|
||||||
"Dilakukan pertama agar cleaning konsisten",
|
"Dilakukan PERTAMA agar regex & dict konsisten",
|
||||||
"Basis untuk normalisasi & stopword",
|
"Fondasi seluruh tahap berikutnya",
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|
@ -479,30 +710,30 @@ def _render_pipeline_steps(stemmer_ok):
|
||||||
"num": "03", "anim": "pipe-3",
|
"num": "03", "anim": "pipe-3",
|
||||||
"icon": "🔄", "color": "#16a34a", "dark": "#14532d",
|
"icon": "🔄", "color": "#16a34a", "dark": "#14532d",
|
||||||
"bg": "linear-gradient(135deg,#f0fdf4,#dcfce7)", "border": "#86efac",
|
"bg": "linear-gradient(135deg,#f0fdf4,#dcfce7)", "border": "#86efac",
|
||||||
"title": 'Normalisasi <span class="fix-badge">✦ Diperluas</span>',
|
"title": f'Normalisasi <span class="fix-badge">✦ {norm_count:,} entri</span>',
|
||||||
"desc": "Mengubah kata tidak baku, singkatan, dan slang menjadi kata baku.",
|
"desc": "Mengubah kata tidak baku, singkatan, dan slang menjadi kata baku (dari file).",
|
||||||
"items": [
|
"items": [
|
||||||
"gk/ga/gak/kagak → tidak",
|
"gk/ga/gak/kagak/ngga → tidak",
|
||||||
|
"bgt/bngt/bget → sangat",
|
||||||
"ongkir → ongkos kirim",
|
"ongkir → ongkos kirim",
|
||||||
"mantep → mantap",
|
"mantep → mantap",
|
||||||
"kece → keren",
|
|
||||||
"jos/josss → bagus",
|
|
||||||
"free → gratis",
|
"free → gratis",
|
||||||
|
f"Total: {norm_count:,} pasang slang→baku dimuat",
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"num": "04", "anim": "pipe-4",
|
"num": "04", "anim": "pipe-4",
|
||||||
"icon": "🚫", "color": "#ea580c", "dark": "#7c2d12",
|
"icon": "🚫", "color": "#ea580c", "dark": "#7c2d12",
|
||||||
"bg": "linear-gradient(135deg,#fff7ed,#ffedd5)", "border": "#fed7aa",
|
"bg": "linear-gradient(135deg,#fff7ed,#ffedd5)", "border": "#fed7aa",
|
||||||
"title": 'Stopword Removal <span class="fix-badge">✦ Diperbaiki</span>',
|
"title": f'Stopword Removal <span class="fix-badge">✦ {sw_count:,} kata</span>',
|
||||||
"desc": "Membuang kata umum; kata sentimen penting DIJAGA.",
|
"desc": "Membuang kata umum dari file; kata sentimen DIJAGA.",
|
||||||
"items": [
|
"items": [
|
||||||
"Hapus kata umum (dan, di, ke...)",
|
f"{sw_count:,} stopword dimuat dari file",
|
||||||
"Hapus token < 3 karakter",
|
"JAGA negasi: tidak, bukan, jangan, belum",
|
||||||
"JAGA negasi: tidak, bukan, jangan",
|
"JAGA positif: keren, bagus, mantap, gratis",
|
||||||
"JAGA positif: keren, bagus, mantap",
|
"JAGA negatif: kecewa, buruk, gagal, mahal, mending, malah",
|
||||||
"JAGA evaluatif: setuju, dukung, bijak",
|
"JAGA intensitas: sangat, banget, sekali",
|
||||||
"JAGA intensitas: banget, sangat, sekali",
|
"Hapus token ≤ 2 karakter (noise)",
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|
@ -678,7 +909,6 @@ def _render_live_example(df_c):
|
||||||
"Contoh tweet acak dari dataset — refresh halaman untuk contoh berbeda"
|
"Contoh tweet acak dari dataset — refresh halaman untuk contoh berbeda"
|
||||||
)
|
)
|
||||||
|
|
||||||
# Urutan tampilan sesuai pipeline: CF → Clean → Norm → Stop → Stem
|
|
||||||
steps_ex = [
|
steps_ex = [
|
||||||
("📄 Teks Asli", "text_asli", "#0f172a", "#f8fafc", "#e2e8f0"),
|
("📄 Teks Asli", "text_asli", "#0f172a", "#f8fafc", "#e2e8f0"),
|
||||||
("① Setelah Case Folding", "setelah_casefolding", "#0c4a6e", "#eff6ff", "#bfdbfe"),
|
("① Setelah Case Folding", "setelah_casefolding", "#0c4a6e", "#eff6ff", "#bfdbfe"),
|
||||||
|
|
@ -727,10 +957,14 @@ def _render_top_words_chart(df_c):
|
||||||
)
|
)
|
||||||
|
|
||||||
if "_tokens_clean" in df_c.columns:
|
if "_tokens_clean" in df_c.columns:
|
||||||
all_words = [w for tokens in df_c["_tokens_clean"] for w in (tokens if isinstance(tokens, list) else [])]
|
all_words = [
|
||||||
|
w for tokens in df_c["_tokens_clean"]
|
||||||
|
for w in (tokens if isinstance(tokens, list) else [])
|
||||||
|
]
|
||||||
else:
|
else:
|
||||||
all_words = " ".join(df_c["clean_text"].fillna("")).split()
|
all_words = " ".join(df_c["clean_text"].fillna("")).split()
|
||||||
|
|
||||||
|
# Filter minimum 3 karakter (konsisten dengan step4)
|
||||||
filtered_words = [w for w in all_words if len(w) > 2]
|
filtered_words = [w for w in all_words if len(w) > 2]
|
||||||
word_freq = Counter(filtered_words).most_common(20)
|
word_freq = Counter(filtered_words).most_common(20)
|
||||||
|
|
||||||
|
|
@ -821,19 +1055,29 @@ def show():
|
||||||
unsafe_allow_html=True
|
unsafe_allow_html=True
|
||||||
)
|
)
|
||||||
|
|
||||||
# ── Pipeline Overview ─────────────────────────────────────
|
# ── Muat resource preprocessing (sekali per session) ──────
|
||||||
|
# Semua tiga resource dimuat di sini, bukan di dalam loop,
|
||||||
|
# agar tidak memuat ulang setiap tweet.
|
||||||
|
stemmer = _load_stemmer()
|
||||||
|
stopwords = _load_stopwords()
|
||||||
|
norm_dict = _load_normalization()
|
||||||
|
|
||||||
|
# ── Pipeline Overview ──────────────────────────────────────
|
||||||
_section_header(
|
_section_header(
|
||||||
"🔬 Alur NLP Pipeline — 5 Tahap Preprocessing",
|
"🔬 Alur NLP Pipeline — 5 Tahap Preprocessing",
|
||||||
"Setiap tweet diproses berurutan melalui 5 tahap sebelum siap dianalisis sentimennya"
|
"Setiap tweet diproses berurutan melalui 5 tahap sebelum siap dianalisis sentimennya"
|
||||||
)
|
)
|
||||||
|
|
||||||
stemmer_tmp = _load_stemmer()
|
_render_pipeline_steps(
|
||||||
_render_pipeline_steps(stemmer_tmp is not None)
|
stemmer_ok=stemmer is not None,
|
||||||
|
norm_count=len(norm_dict),
|
||||||
|
sw_count=len(stopwords),
|
||||||
|
)
|
||||||
_gap("sm")
|
_gap("sm")
|
||||||
_render_flow_arrow()
|
_render_flow_arrow()
|
||||||
_gap("md")
|
_gap("md")
|
||||||
|
|
||||||
# ── Load data ─────────────────────────────────────────────
|
# ── Load data dari database ────────────────────────────────
|
||||||
try:
|
try:
|
||||||
df_all = pd.read_sql("SELECT * FROM tweets ORDER BY created_at DESC", engine)
|
df_all = pd.read_sql("SELECT * FROM tweets ORDER BY created_at DESC", engine)
|
||||||
if df_all.empty:
|
if df_all.empty:
|
||||||
|
|
@ -848,13 +1092,17 @@ def show():
|
||||||
|
|
||||||
s_dt = pd.Timestamp(start_date)
|
s_dt = pd.Timestamp(start_date)
|
||||||
e_dt = pd.Timestamp(end_date)
|
e_dt = pd.Timestamp(end_date)
|
||||||
df = df_all[(df_all["created_at"] >= s_dt) & (df_all["created_at"] <= e_dt)].copy()
|
df = df_all[
|
||||||
|
(df_all["created_at"] >= s_dt) & (df_all["created_at"] <= e_dt)
|
||||||
|
].copy()
|
||||||
|
|
||||||
if df.empty:
|
if df.empty:
|
||||||
st.warning(f"⚠️ Tidak ada tweet dengan tanggal asli dalam periode {filter_label}.")
|
st.warning(f"⚠️ Tidak ada tweet dengan tanggal asli dalam periode {filter_label}.")
|
||||||
return
|
return
|
||||||
|
|
||||||
# ── Cache preprocessing ───────────────────────────────────
|
# ── Cache preprocessing ────────────────────────────────────
|
||||||
|
# Cache key: kombinasi mode + periode + jumlah tweet di DB
|
||||||
|
# Jika ada tweet baru → cache otomatis invalid → proses ulang
|
||||||
total_tweets_in_db = get_tweet_count()
|
total_tweets_in_db = get_tweet_count()
|
||||||
latest_crawl_marker = get_latest_crawl_time() or "no-crawl"
|
latest_crawl_marker = get_latest_crawl_time() or "no-crawl"
|
||||||
data_marker = (total_tweets_in_db, latest_crawl_marker)
|
data_marker = (total_tweets_in_db, latest_crawl_marker)
|
||||||
|
|
@ -864,6 +1112,7 @@ def show():
|
||||||
f"{start_date}_{end_date}_{total_tweets_in_db}_{latest_crawl_marker}"
|
f"{start_date}_{end_date}_{total_tweets_in_db}_{latest_crawl_marker}"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Bersihkan cache lama untuk mode/periode yang sudah tidak aktif
|
||||||
for old_key in list(st.session_state.keys()):
|
for old_key in list(st.session_state.keys()):
|
||||||
if old_key.startswith("pp5_") and old_key != cache_key:
|
if old_key.startswith("pp5_") and old_key != cache_key:
|
||||||
del st.session_state[old_key]
|
del st.session_state[old_key]
|
||||||
|
|
@ -871,13 +1120,16 @@ def show():
|
||||||
force_refresh = data_marker != st.session_state.get("_pp_last_data_marker")
|
force_refresh = data_marker != st.session_state.get("_pp_last_data_marker")
|
||||||
|
|
||||||
if cache_key not in st.session_state or force_refresh:
|
if cache_key not in st.session_state or force_refresh:
|
||||||
stemmer = _load_stemmer()
|
|
||||||
stopwords = _load_stopwords()
|
|
||||||
|
|
||||||
with st.spinner("🧹 Menjalankan 5 tahap preprocessing…"):
|
with st.spinner("🧹 Menjalankan 5 tahap preprocessing…"):
|
||||||
results = []
|
results = []
|
||||||
for _, row in df.iterrows():
|
for _, row in df.iterrows():
|
||||||
r = full_preprocessing(row["text"], stopwords, stemmer)
|
# Jalankan 5 tahap — norm_dict diteruskan sebagai parameter
|
||||||
|
r = full_preprocessing(
|
||||||
|
text = row["text"],
|
||||||
|
stopwords = stopwords,
|
||||||
|
stemmer = stemmer,
|
||||||
|
norm_dict = norm_dict,
|
||||||
|
)
|
||||||
r["tweet_id"] = row.get("tweet_id", "")
|
r["tweet_id"] = row.get("tweet_id", "")
|
||||||
r["text_asli"] = row["text"]
|
r["text_asli"] = row["text"]
|
||||||
r["created_at"] = row["created_at"]
|
r["created_at"] = row["created_at"]
|
||||||
|
|
@ -885,16 +1137,19 @@ def show():
|
||||||
results.append(r)
|
results.append(r)
|
||||||
|
|
||||||
df_c = pd.DataFrame(results)
|
df_c = pd.DataFrame(results)
|
||||||
|
# Buang baris yang clean_text-nya kosong setelah semua 5 tahap
|
||||||
df_c = df_c[df_c["clean_text"].str.strip().str.len() > 0].copy()
|
df_c = df_c[df_c["clean_text"].str.strip().str.len() > 0].copy()
|
||||||
|
# Reset index agar rapi
|
||||||
|
df_c = df_c.reset_index(drop=True)
|
||||||
|
|
||||||
st.session_state[cache_key] = df_c
|
st.session_state[cache_key] = df_c
|
||||||
st.session_state[cache_key + "_stemmer_ok"] = stemmer is not None
|
st.session_state[cache_key + "_sw_ok"] = stemmer is not None
|
||||||
st.session_state["_pp_last_data_marker"] = data_marker
|
st.session_state["_pp_last_data_marker"] = data_marker
|
||||||
|
|
||||||
df_c = st.session_state[cache_key]
|
df_c = st.session_state[cache_key]
|
||||||
stemmer_ok = st.session_state.get(cache_key + "_stemmer_ok", False)
|
stemmer_ok = st.session_state.get(cache_key + "_sw_ok", False)
|
||||||
|
|
||||||
# ── Statistik ─────────────────────────────────────────────
|
# ── Statistik ──────────────────────────────────────────────
|
||||||
removed = len(df) - len(df_c)
|
removed = len(df) - len(df_c)
|
||||||
|
|
||||||
_section_header(
|
_section_header(
|
||||||
|
|
@ -908,7 +1163,7 @@ def show():
|
||||||
_render_live_example(df_c)
|
_render_live_example(df_c)
|
||||||
_gap("lg")
|
_gap("lg")
|
||||||
|
|
||||||
# ── Tabel ─────────────────────────────────────────────────
|
# ── Tabel ──────────────────────────────────────────────────
|
||||||
_section_header(
|
_section_header(
|
||||||
"📋 Tabel Perbandingan Teks per Tahap",
|
"📋 Tabel Perbandingan Teks per Tahap",
|
||||||
f"{len(df_c):,} tweet · {filter_label} — scroll horizontal untuk lihat semua kolom"
|
f"{len(df_c):,} tweet · {filter_label} — scroll horizontal untuk lihat semua kolom"
|
||||||
|
|
@ -918,7 +1173,7 @@ def show():
|
||||||
df_c = df_c.copy()
|
df_c = df_c.copy()
|
||||||
df_c["crawled_at"] = pd.NaT
|
df_c["crawled_at"] = pd.NaT
|
||||||
|
|
||||||
# Kolom ditampilkan sesuai urutan pipeline: CF → Clean → Norm → Stop → Stem
|
# Kolom ditampilkan sesuai urutan pipeline
|
||||||
disp = df_c[[
|
disp = df_c[[
|
||||||
"tweet_id", "created_at", "crawled_at",
|
"tweet_id", "created_at", "crawled_at",
|
||||||
"text_asli",
|
"text_asli",
|
||||||
|
|
@ -968,10 +1223,10 @@ def show():
|
||||||
_render_top_words_chart(df_c)
|
_render_top_words_chart(df_c)
|
||||||
_gap("lg")
|
_gap("lg")
|
||||||
|
|
||||||
# ── Simpan ke session state ───────────────────────────────
|
# ── Simpan ke session state untuk halaman sentimen ─────────
|
||||||
st.session_state["preprocessed_df"] = df_c
|
st.session_state["preprocessed_df"] = df_c
|
||||||
|
|
||||||
# ── Download ──────────────────────────────────────────────
|
# ── Download ───────────────────────────────────────────────
|
||||||
st.markdown("""
|
st.markdown("""
|
||||||
<div style="background:#ffffff;border:1.5px solid #e2e8f0;border-radius:14px;
|
<div style="background:#ffffff;border:1.5px solid #e2e8f0;border-radius:14px;
|
||||||
padding:1.1rem 1.2rem;box-shadow:0 2px 6px rgba(15,23,42,0.05);
|
padding:1.1rem 1.2rem;box-shadow:0 2px 6px rgba(15,23,42,0.05);
|
||||||
|
|
|
||||||
|
|
@ -1,29 +1,8 @@
|
||||||
"""
|
"""
|
||||||
sentiment_page.py
|
sentiment_page.py
|
||||||
=================
|
=================
|
||||||
Halaman Analisis Sentimen — diperbaiki untuk mengatasi semua prediksi negatif.
|
PERBAIKAN: Hapus parameter key= dari semua st.container() karena tidak
|
||||||
|
didukung di Streamlit versi lama. CSS styling tetap berjalan via class HTML.
|
||||||
PERBAIKAN vs VERSI LAMA:
|
|
||||||
─────────────────────────────────────────────────────────────────────────────
|
|
||||||
1. MENGGUNAKAN sentiment_service.py (HYBRID CLASSIFIER)
|
|
||||||
Tidak lagi memanggil model secara langsung dengan predict_batch sederhana.
|
|
||||||
Sebaliknya menggunakan sentiment_service yang sudah mengimplementasikan:
|
|
||||||
• Lexicon-based override untuk kelas POSITIF (yang tidak ada di model)
|
|
||||||
• Negation handling (tidak bagus ≠ bagus)
|
|
||||||
• Fallback ke model NB untuk negatif vs netral
|
|
||||||
|
|
||||||
2. PREPROCESSING SELARAS
|
|
||||||
Memanggil preprocess_for_model() dari sentiment_service agar preprocessing
|
|
||||||
yang digunakan untuk prediksi persis sama dengan yang dipakai saat training.
|
|
||||||
|
|
||||||
3. CONFIDENCE SCORE AKURAT
|
|
||||||
Menggunakan confidence gabungan dari lexicon score + model probability,
|
|
||||||
bukan hanya max probability dari model yang bias ke negatif.
|
|
||||||
|
|
||||||
4. LABEL MAPPING KONSISTEN
|
|
||||||
Label dikembalikan selalu dalam format: 'Positif', 'Netral', 'Negatif'
|
|
||||||
(kapital huruf pertama) agar konsisten di seluruh tampilan.
|
|
||||||
─────────────────────────────────────────────────────────────────────────────
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import streamlit as st
|
import streamlit as st
|
||||||
|
|
@ -44,7 +23,6 @@ from timezone_utils import (
|
||||||
)
|
)
|
||||||
import plotly.graph_objects as go
|
import plotly.graph_objects as go
|
||||||
|
|
||||||
# ── Import sentiment_service (hybrid classifier) ──────────────────────────
|
|
||||||
from sentiment_service import (
|
from sentiment_service import (
|
||||||
preprocess_for_model,
|
preprocess_for_model,
|
||||||
preprocess_untuk_lexicon,
|
preprocess_untuk_lexicon,
|
||||||
|
|
@ -118,40 +96,27 @@ def _sync_dynamic_period():
|
||||||
|
|
||||||
|
|
||||||
# ─────────────────────────────────────────────────────────────
|
# ─────────────────────────────────────────────────────────────
|
||||||
# Core prediction function (DIPERBAIKI)
|
# Core prediction
|
||||||
# ─────────────────────────────────────────────────────────────
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
def predict_batch_hybrid(texts: list[str]) -> list[tuple[str, float]]:
|
def predict_batch_hybrid(texts):
|
||||||
"""
|
|
||||||
Prediksi sentimen menggunakan Hybrid Classifier dari sentiment_service.
|
|
||||||
|
|
||||||
Pipeline per tweet:
|
|
||||||
1. preprocess_for_model() → teks untuk TF-IDF + NB
|
|
||||||
2. preprocess_untuk_lexicon() → teks untuk pengecekan lexicon
|
|
||||||
3. _hitung_skor_lexicon() → hitung sinyal positif/negatif kuat
|
|
||||||
4. _klasifikasi_hybrid() → putuskan label + confidence
|
|
||||||
|
|
||||||
Return: list of (label, confidence)
|
|
||||||
"""
|
|
||||||
results = []
|
results = []
|
||||||
for text in texts:
|
for text in texts:
|
||||||
teks_model = preprocess_for_model(text)
|
teks_model = preprocess_for_model(text)
|
||||||
teks_lexicon = preprocess_untuk_lexicon(text)
|
teks_lexicon = preprocess_untuk_lexicon(text)
|
||||||
teks_lower = str(text).lower()
|
teks_lower = str(text).lower()
|
||||||
|
skor = _hitung_skor_lexicon(teks_lexicon)
|
||||||
skor = _hitung_skor_lexicon(teks_lexicon)
|
label, conf = _klasifikasi_hybrid(teks_model, skor, teks_lower)
|
||||||
label, conf = _klasifikasi_hybrid(teks_model, skor, teks_lower)
|
|
||||||
results.append((label, conf))
|
results.append((label, conf))
|
||||||
return results
|
return results
|
||||||
|
|
||||||
|
|
||||||
def preprocess_single(text: str) -> str:
|
def preprocess_single(text):
|
||||||
"""Preprocess satu teks untuk disimpan ke kolom clean_text."""
|
|
||||||
return preprocess_for_model(text)
|
return preprocess_for_model(text)
|
||||||
|
|
||||||
|
|
||||||
# ─────────────────────────────────────────────────────────────
|
# ─────────────────────────────────────────────────────────────
|
||||||
# Shared UI helpers
|
# UI helpers
|
||||||
# ─────────────────────────────────────────────────────────────
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
def _section_header(title, subtitle=""):
|
def _section_header(title, subtitle=""):
|
||||||
|
|
@ -173,6 +138,20 @@ def _section_gap(size="md"):
|
||||||
st.markdown(f'<div style="height:{heights.get(size,"1.45rem")};"></div>', unsafe_allow_html=True)
|
st.markdown(f'<div style="height:{heights.get(size,"1.45rem")};"></div>', unsafe_allow_html=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _card_open(extra_style=""):
|
||||||
|
"""Buka div card pengganti st.container(border=True)."""
|
||||||
|
st.markdown(
|
||||||
|
f'<div style="background:#ffffff;border:1.5px solid #e2e8f0;border-radius:12px;'
|
||||||
|
f'box-shadow:0 2px 6px rgba(15,23,42,0.06);padding:1.1rem 1.2rem 1rem;'
|
||||||
|
f'margin-bottom:0.5rem;{extra_style}">',
|
||||||
|
unsafe_allow_html=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _card_close():
|
||||||
|
st.markdown('</div>', unsafe_allow_html=True)
|
||||||
|
|
||||||
|
|
||||||
def _render_sentiment_styles():
|
def _render_sentiment_styles():
|
||||||
st.markdown("""
|
st.markdown("""
|
||||||
<style>
|
<style>
|
||||||
|
|
@ -185,31 +164,6 @@ section[data-testid="stMain"] * {
|
||||||
[data-testid="stMainBlockContainer"] { padding-top: 1rem !important; }
|
[data-testid="stMainBlockContainer"] { padding-top: 1rem !important; }
|
||||||
header[data-testid="stHeader"] { height: 0 !important; min-height: 0 !important; }
|
header[data-testid="stHeader"] { height: 0 !important; min-height: 0 !important; }
|
||||||
|
|
||||||
.st-key-sentiment_keyword_panel,
|
|
||||||
.st-key-sentiment_table_controls,
|
|
||||||
.st-key-sentiment_download_panel,
|
|
||||||
.st-key-sentiment_chart_panel,
|
|
||||||
.st-key-sentiment_trend_panel,
|
|
||||||
.st-key-sentiment_wordfreq_panel {
|
|
||||||
background: #ffffff !important;
|
|
||||||
border: 1.5px solid #e2e8f0 !important;
|
|
||||||
border-radius: 12px !important;
|
|
||||||
box-shadow: 0 2px 6px rgba(15,23,42,0.06) !important;
|
|
||||||
padding: 1.1rem 1.2rem 1rem !important;
|
|
||||||
}
|
|
||||||
|
|
||||||
.st-key-sentiment_keyword_panel [data-testid="stTextInput"] label p,
|
|
||||||
.st-key-sentiment_table_controls [data-testid="stTextInput"] label p,
|
|
||||||
.st-key-sentiment_table_controls [data-testid="stSelectbox"] label p {
|
|
||||||
color: #334155 !important;
|
|
||||||
font-size: 0.72rem !important;
|
|
||||||
font-weight: 800 !important;
|
|
||||||
letter-spacing: 0.04em !important;
|
|
||||||
line-height: 1.2 !important;
|
|
||||||
margin-bottom: 0.22rem !important;
|
|
||||||
text-transform: uppercase !important;
|
|
||||||
}
|
|
||||||
|
|
||||||
.stButton > button {
|
.stButton > button {
|
||||||
border-radius: 12px !important;
|
border-radius: 12px !important;
|
||||||
font-weight: 700 !important;
|
font-weight: 700 !important;
|
||||||
|
|
@ -419,7 +373,7 @@ def _render_donut_chart(pos_n, neu_n, neg_n, total, filter_label):
|
||||||
legend=dict(orientation="h", y=-0.1, x=0.5, xanchor="center", font=dict(size=11)),
|
legend=dict(orientation="h", y=-0.1, x=0.5, xanchor="center", font=dict(size=11)),
|
||||||
paper_bgcolor="rgba(0,0,0,0)",
|
paper_bgcolor="rgba(0,0,0,0)",
|
||||||
)
|
)
|
||||||
st.plotly_chart(fig, width="stretch", config={"displayModeBar": False})
|
st.plotly_chart(fig, use_container_width=True, config={"displayModeBar": False})
|
||||||
return dominant
|
return dominant
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -446,7 +400,7 @@ def _render_bar_chart(pos_n, neu_n, neg_n, total):
|
||||||
yaxis=dict(showgrid=True, gridcolor="rgba(226,232,240,0.8)", griddash="dot",
|
yaxis=dict(showgrid=True, gridcolor="rgba(226,232,240,0.8)", griddash="dot",
|
||||||
tickfont=dict(size=10, color="#94a3b8")),
|
tickfont=dict(size=10, color="#94a3b8")),
|
||||||
)
|
)
|
||||||
st.plotly_chart(fig, width="stretch", config={"displayModeBar": False})
|
st.plotly_chart(fig, use_container_width=True, config={"displayModeBar": False})
|
||||||
|
|
||||||
|
|
||||||
# ─────────────────────────────────────────────────────────────
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
|
@ -496,49 +450,9 @@ def _render_trend_chart(fdf, filter_label, dt_start, dt_end):
|
||||||
tickfont=dict(size=10, color="#94a3b8")),
|
tickfont=dict(size=10, color="#94a3b8")),
|
||||||
hovermode="x unified",
|
hovermode="x unified",
|
||||||
)
|
)
|
||||||
with st.container(border=True, key="sentiment_trend_panel"):
|
# ─── PERBAIKAN: hapus key= dari st.container() ───
|
||||||
st.plotly_chart(fig, width="stretch", config={"displayModeBar": False})
|
with st.container():
|
||||||
|
st.plotly_chart(fig, use_container_width=True, config={"displayModeBar": False})
|
||||||
|
|
||||||
# ─────────────────────────────────────────────────────────────
|
|
||||||
# Confidence histogram
|
|
||||||
# ─────────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
# def _render_confidence_chart(fdf):
|
|
||||||
# _section_header(
|
|
||||||
# "🎯 Distribusi Keyakinan Model",
|
|
||||||
# "Seberapa yakin classifier dalam mengklasifikasikan setiap tweet"
|
|
||||||
# )
|
|
||||||
|
|
||||||
# fig = go.Figure()
|
|
||||||
# sent_cfg = [
|
|
||||||
# ("Positif", "#16a34a", "rgba(22,163,74,0.7)"),
|
|
||||||
# ("Netral", "#94a3b8", "rgba(148,163,184,0.7)"),
|
|
||||||
# ("Negatif", "#ef4444", "rgba(239,68,68,0.7)"),
|
|
||||||
# ]
|
|
||||||
# for sent, color, fill_color in sent_cfg:
|
|
||||||
# sub = fdf[fdf["sentiment"] == sent]["confidence"]
|
|
||||||
# if sub.empty:
|
|
||||||
# continue
|
|
||||||
# fig.add_trace(go.Histogram(
|
|
||||||
# x=sub, name=sent, nbinsx=20,
|
|
||||||
# marker=dict(color=fill_color, line=dict(color=color, width=1)),
|
|
||||||
# opacity=0.85,
|
|
||||||
# hovertemplate=f"<b>{sent}</b><br>Keyakinan: %{{x:.0%}}<br>Jumlah: %{{y}}<extra></extra>",
|
|
||||||
# ))
|
|
||||||
# fig.update_layout(
|
|
||||||
# height=260, margin=dict(l=0, r=0, t=10, b=0), barmode="overlay",
|
|
||||||
# paper_bgcolor="rgba(0,0,0,0)", plot_bgcolor="rgba(0,0,0,0)",
|
|
||||||
# legend=dict(orientation="h", y=1.1, x=0, font=dict(size=11)),
|
|
||||||
# xaxis=dict(tickformat=".0%", title="Tingkat Keyakinan",
|
|
||||||
# tickfont=dict(size=10, color="#94a3b8"),
|
|
||||||
# showgrid=True, gridcolor="rgba(226,232,240,0.6)"),
|
|
||||||
# yaxis=dict(title="Jumlah Tweet",
|
|
||||||
# tickfont=dict(size=10, color="#94a3b8"),
|
|
||||||
# showgrid=True, gridcolor="rgba(226,232,240,0.6)", griddash="dot"),
|
|
||||||
# )
|
|
||||||
# with st.container(border=True, key="sentiment_conf_panel"):
|
|
||||||
# st.plotly_chart(fig, width="stretch", config={"displayModeBar": False})
|
|
||||||
|
|
||||||
|
|
||||||
# ─────────────────────────────────────────────────────────────
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
|
@ -609,7 +523,7 @@ def _render_word_freq_per_sentiment(fdf):
|
||||||
tickfont=dict(size=8, color="#94a3b8"), fixedrange=True),
|
tickfont=dict(size=8, color="#94a3b8"), fixedrange=True),
|
||||||
yaxis=dict(showgrid=False, tickfont=dict(size=9, color="#334155"), fixedrange=True),
|
yaxis=dict(showgrid=False, tickfont=dict(size=9, color="#334155"), fixedrange=True),
|
||||||
)
|
)
|
||||||
st.plotly_chart(fig, width="stretch", config={"displayModeBar": False})
|
st.plotly_chart(fig, use_container_width=True, config={"displayModeBar": False})
|
||||||
st.markdown('</div>', unsafe_allow_html=True)
|
st.markdown('</div>', unsafe_allow_html=True)
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -671,25 +585,25 @@ def _render_tweet_table(fdf, filter_label):
|
||||||
f"Total {len(fdf):,} tweet · {filter_label}"
|
f"Total {len(fdf):,} tweet · {filter_label}"
|
||||||
)
|
)
|
||||||
|
|
||||||
with st.container(border=True, key="sentiment_table_controls"):
|
# ─── PERBAIKAN: ganti st.container(key=) dengan st.columns biasa ───
|
||||||
tf1, tf2, tf3 = st.columns([3, 1.6, 1.7], gap="medium", vertical_alignment="bottom")
|
tf1, tf2, tf3 = st.columns([3, 1.6, 1.7], gap="medium")
|
||||||
with tf1:
|
with tf1:
|
||||||
search = st.text_input(
|
search = st.text_input(
|
||||||
"Search tweet", placeholder="Ketik kata kunci di isi tweet...",
|
"🔍 Cari tweet", placeholder="Ketik kata kunci di isi tweet...",
|
||||||
key="sentiment_table_search",
|
key="sentiment_table_search",
|
||||||
)
|
)
|
||||||
with tf2:
|
with tf2:
|
||||||
sf = st.selectbox(
|
sf = st.selectbox(
|
||||||
"Filter sentimen",
|
"Filter sentimen",
|
||||||
["Semua", "Positif 😊", "Netral 😐", "Negatif 😞"],
|
["Semua", "Positif 😊", "Netral 😐", "Negatif 😞"],
|
||||||
key="sentiment_table_filter",
|
key="sentiment_table_filter",
|
||||||
)
|
)
|
||||||
with tf3:
|
with tf3:
|
||||||
sort_by = st.selectbox(
|
sort_by = st.selectbox(
|
||||||
"Urutkan",
|
"Urutkan",
|
||||||
["Terbaru dulu", "Terlama dulu", "Keyakinan tertinggi"],
|
["Terbaru dulu", "Terlama dulu", "Keyakinan tertinggi"],
|
||||||
key="sentiment_table_sort",
|
key="sentiment_table_sort",
|
||||||
)
|
)
|
||||||
|
|
||||||
_section_gap("sm")
|
_section_gap("sm")
|
||||||
|
|
||||||
|
|
@ -711,26 +625,23 @@ def _render_tweet_table(fdf, filter_label):
|
||||||
else:
|
else:
|
||||||
tdf = tdf.sort_values("confidence", ascending=False)
|
tdf = tdf.sort_values("confidence", ascending=False)
|
||||||
|
|
||||||
# out = tdf[["created_at", "crawled_at", "text", "clean_text", "sentiment", "confidence"]].copy()
|
|
||||||
out = tdf[["created_at", "crawled_at", "text", "clean_text", "sentiment"]].copy()
|
out = tdf[["created_at", "crawled_at", "text", "clean_text", "sentiment"]].copy()
|
||||||
out["created_at"] = out["created_at"].apply(format_dt)
|
out["created_at"] = out["created_at"].apply(format_dt)
|
||||||
out["crawled_at"] = out["crawled_at"].apply(format_dt)
|
out["crawled_at"] = out["crawled_at"].apply(format_dt)
|
||||||
# out["confidence"] = out["confidence"].apply(lambda x: f"{x:.0%}")
|
|
||||||
out["sentiment"] = out["sentiment"].map({
|
out["sentiment"] = out["sentiment"].map({
|
||||||
"Positif": "😊 Positif", "Netral": "😐 Netral", "Negatif": "😞 Negatif",
|
"Positif": "😊 Positif", "Netral": "😐 Netral", "Negatif": "😞 Negatif",
|
||||||
}).fillna(out["sentiment"])
|
}).fillna(out["sentiment"])
|
||||||
# out.columns = ["Tanggal Tweet", "Masuk Database", "Tweet Asli", "Tweet Bersih", "Sentimen", "Keyakinan"]
|
|
||||||
out.columns = ["Tanggal Tweet", "Masuk Database", "Tweet Asli", "Tweet Bersih", "Sentimen"]
|
out.columns = ["Tanggal Tweet", "Masuk Database", "Tweet Asli", "Tweet Bersih", "Sentimen"]
|
||||||
|
|
||||||
render_standard_table(
|
render_standard_table(
|
||||||
out, height=400, min_width=1220,
|
out, height=400, min_width=1220,
|
||||||
badge_columns=["Sentimen"],
|
badge_columns=["Sentimen"],
|
||||||
nowrap=["Tanggal Tweet", "Masuk Database", "Sentimen", "Keyakinan"],
|
nowrap=["Tanggal Tweet", "Masuk Database", "Sentimen"],
|
||||||
wide_columns=["Tweet Asli", "Tweet Bersih"],
|
wide_columns=["Tweet Asli", "Tweet Bersih"],
|
||||||
column_widths={
|
column_widths={
|
||||||
"Tanggal Tweet": "170px", "Masuk Database": "170px",
|
"Tanggal Tweet": "170px", "Masuk Database": "170px",
|
||||||
"Tweet Asli": "360px", "Tweet Bersih": "360px",
|
"Tweet Asli": "360px", "Tweet Bersih": "360px",
|
||||||
"Sentimen": "130px", "Keyakinan": "110px",
|
"Sentimen": "130px",
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
st.caption(f"Menampilkan {len(tdf):,} tweet")
|
st.caption(f"Menampilkan {len(tdf):,} tweet")
|
||||||
|
|
@ -927,11 +838,9 @@ def show():
|
||||||
if force_refresh:
|
if force_refresh:
|
||||||
st.info("🔄 Menyegarkan prediksi sentimen dengan data terbaru...")
|
st.info("🔄 Menyegarkan prediksi sentimen dengan data terbaru...")
|
||||||
with st.spinner("🔍 Preprocessing & prediksi sentimen (Hybrid Classifier)..."):
|
with st.spinner("🔍 Preprocessing & prediksi sentimen (Hybrid Classifier)..."):
|
||||||
# Preprocess semua tweet
|
|
||||||
df["clean_text"] = df["text"].apply(preprocess_single)
|
df["clean_text"] = df["text"].apply(preprocess_single)
|
||||||
dfc = df[df["clean_text"].str.strip().str.len() > 0].copy()
|
dfc = df[df["clean_text"].str.strip().str.len() > 0].copy()
|
||||||
|
|
||||||
# Prediksi menggunakan hybrid classifier
|
|
||||||
results = predict_batch_hybrid(dfc["text"].tolist())
|
results = predict_batch_hybrid(dfc["text"].tolist())
|
||||||
if results:
|
if results:
|
||||||
sentiments, confidences = zip(*results)
|
sentiments, confidences = zip(*results)
|
||||||
|
|
@ -949,7 +858,8 @@ def show():
|
||||||
# ── Keyword filter ────────────────────────────────────────
|
# ── Keyword filter ────────────────────────────────────────
|
||||||
_section_header("🎯 Filter Kata Kunci", "Kosongkan untuk melihat semua tweet")
|
_section_header("🎯 Filter Kata Kunci", "Kosongkan untuk melihat semua tweet")
|
||||||
|
|
||||||
with st.container(border=True, key="sentiment_keyword_panel"):
|
# ─── PERBAIKAN: hapus key= dari st.container() ───
|
||||||
|
with st.container():
|
||||||
kw = st.text_input(
|
kw = st.text_input(
|
||||||
"Kata kunci", placeholder="Contoh: ongkir, kurir, komdigi...",
|
"Kata kunci", placeholder="Contoh: ongkir, kurir, komdigi...",
|
||||||
key="sentiment_keyword_search",
|
key="sentiment_keyword_search",
|
||||||
|
|
@ -1000,11 +910,12 @@ def show():
|
||||||
col_left, col_right = st.columns(2, gap="medium")
|
col_left, col_right = st.columns(2, gap="medium")
|
||||||
with col_left:
|
with col_left:
|
||||||
_section_header("🔵 Sebaran Sentimen", f"Periode {filter_label}")
|
_section_header("🔵 Sebaran Sentimen", f"Periode {filter_label}")
|
||||||
with st.container(border=True, key="sentiment_chart_panel"):
|
# ─── PERBAIKAN: hapus key= dari st.container() ───
|
||||||
|
with st.container():
|
||||||
dominant = _render_donut_chart(pos_n, neu_n, neg_n, total, filter_label)
|
dominant = _render_donut_chart(pos_n, neu_n, neg_n, total, filter_label)
|
||||||
with col_right:
|
with col_right:
|
||||||
_section_header("📊 Perbandingan Jumlah per Sentimen", f"Periode {filter_label}")
|
_section_header("📊 Perbandingan Jumlah per Sentimen", f"Periode {filter_label}")
|
||||||
with st.container(border=True, key="sentiment_bar_panel"):
|
with st.container():
|
||||||
_render_bar_chart(pos_n, neu_n, neg_n, total)
|
_render_bar_chart(pos_n, neu_n, neg_n, total)
|
||||||
_section_gap("lg")
|
_section_gap("lg")
|
||||||
|
|
||||||
|
|
@ -1012,10 +923,6 @@ def show():
|
||||||
_render_trend_chart(fdf, filter_label, start_date, end_date)
|
_render_trend_chart(fdf, filter_label, start_date, end_date)
|
||||||
_section_gap("lg")
|
_section_gap("lg")
|
||||||
|
|
||||||
# ── Confidence ────────────────────────────────────────────
|
|
||||||
# _render_confidence_chart(fdf)
|
|
||||||
# _section_gap("lg")
|
|
||||||
|
|
||||||
# ── Word freq ─────────────────────────────────────────────
|
# ── Word freq ─────────────────────────────────────────────
|
||||||
_render_word_freq_per_sentiment(fdf)
|
_render_word_freq_per_sentiment(fdf)
|
||||||
_section_gap("lg")
|
_section_gap("lg")
|
||||||
|
|
@ -1035,15 +942,17 @@ def show():
|
||||||
# ── Download ──────────────────────────────────────────────
|
# ── Download ──────────────────────────────────────────────
|
||||||
_section_header("📥 Unduh Hasil Analisis")
|
_section_header("📥 Unduh Hasil Analisis")
|
||||||
|
|
||||||
with st.container(border=True, key="sentiment_download_panel"):
|
# ─── PERBAIKAN: hapus key= dari st.container() ───
|
||||||
d1, d2, d3 = st.columns(3, gap="medium", vertical_alignment="bottom")
|
with st.container():
|
||||||
|
d1, d2, d3 = st.columns(3, gap="medium")
|
||||||
|
|
||||||
with d1:
|
with d1:
|
||||||
st.download_button(
|
st.download_button(
|
||||||
"📥 Semua Hasil Prediksi",
|
"📥 Semua Hasil Prediksi",
|
||||||
fdf.to_csv(index=False).encode("utf-8"),
|
fdf.to_csv(index=False).encode("utf-8"),
|
||||||
f"hasil_prediksi_{datetime.now().strftime('%Y%m%d_%H%M')}.csv",
|
f"hasil_prediksi_{datetime.now().strftime('%Y%m%d_%H%M')}.csv",
|
||||||
"text/csv", width="stretch",
|
"text/csv",
|
||||||
|
use_container_width=True,
|
||||||
)
|
)
|
||||||
with d2:
|
with d2:
|
||||||
summary = pd.DataFrame({
|
summary = pd.DataFrame({
|
||||||
|
|
@ -1055,12 +964,14 @@ def show():
|
||||||
"📈 Ringkasan Sentimen",
|
"📈 Ringkasan Sentimen",
|
||||||
summary.to_csv(index=False).encode("utf-8"),
|
summary.to_csv(index=False).encode("utf-8"),
|
||||||
f"ringkasan_{datetime.now().strftime('%Y%m%d_%H%M')}.csv",
|
f"ringkasan_{datetime.now().strftime('%Y%m%d_%H%M')}.csv",
|
||||||
"text/csv", width="stretch",
|
"text/csv",
|
||||||
|
use_container_width=True,
|
||||||
)
|
)
|
||||||
with d3:
|
with d3:
|
||||||
st.download_button(
|
st.download_button(
|
||||||
"🎯 Rekomendasi Tindakan",
|
"🎯 Rekomendasi Tindakan",
|
||||||
df_rek.to_csv(index=False).encode("utf-8"),
|
df_rek.to_csv(index=False).encode("utf-8"),
|
||||||
f"rekomendasi_{datetime.now().strftime('%Y%m%d_%H%M')}.csv",
|
f"rekomendasi_{datetime.now().strftime('%Y%m%d_%H%M')}.csv",
|
||||||
"text/csv", width="stretch",
|
"text/csv",
|
||||||
|
use_container_width=True,
|
||||||
)
|
)
|
||||||
|
|
@ -1,25 +1,15 @@
|
||||||
"""
|
"""
|
||||||
sentiment_service.py
|
sentiment_service.py — Hybrid Classifier (versi perbaikan)
|
||||||
====================
|
=============================================================
|
||||||
Hybrid Classifier untuk analisis sentimen tweet Bahasa Indonesia.
|
Perbaikan utama vs versi sebelumnya:
|
||||||
|
1. LEXICON DIPERLUAS — kata positif & negatif yang tidak ada di vocab TF-IDF
|
||||||
PIPELINE PREPROCESSING — 5 TAHAP (selaras dengan preprocessing_page.py):
|
(bagus, mantap, setuju, puas, hemat, berhasil, dll.) kini tetap bisa
|
||||||
1. Case Folding → lowercase dulu sebelum cleaning
|
terdeteksi melalui lexicon scoring.
|
||||||
2. Cleaning → hapus URL, mention, hashtag, angka, emoji, tanda baca
|
2. THRESHOLD DISESUAIKAN — Layer 1 diperlonggar (net >= 2 → Positif),
|
||||||
3. Normalisasi → singkatan/slang → kata baku
|
Layer anti-bias diperketat agar prediksi lebih proporsional.
|
||||||
4. Stopword Removal → buang kata umum, jaga kata sentimen penting
|
3. NEGATION WINDOW DIPERLUAS — window 3 kata (sebelumnya 2) agar
|
||||||
5. Stemming → bentuk dasar via Sastrawi ECS
|
"tidak terlalu bagus" tetap terdeteksi negasinya.
|
||||||
|
4. PREPROCESSING IDENTIK dengan preprocessing_page.py (5 tahap).
|
||||||
Catatan: Tokenizing tidak menjadi tahap tersendiri karena:
|
|
||||||
• Stopword removal & stemming sudah melakukan split() secara internal
|
|
||||||
• TF-IDF melakukan tokenisasi sendiri saat inferensi
|
|
||||||
|
|
||||||
ARSITEKTUR HYBRID:
|
|
||||||
Teks Asli
|
|
||||||
├── preprocess_untuk_lexicon() ──→ _hitung_skor_lexicon() ──┐
|
|
||||||
└── preprocess_for_model() ──→ TF-IDF → NB Model ───────┘
|
|
||||||
↓
|
|
||||||
_klasifikasi_hybrid() → Label + Confidence
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import re
|
import re
|
||||||
|
|
@ -28,93 +18,44 @@ import joblib
|
||||||
|
|
||||||
|
|
||||||
# ═══════════════════════════════════════════════════════════
|
# ═══════════════════════════════════════════════════════════
|
||||||
# KATA SENTIMEN PENTING
|
# KATA SENTIMEN PENTING (dijaga dari stopword removal)
|
||||||
# Tidak boleh dihapus di tahap stopword removal
|
|
||||||
# ═══════════════════════════════════════════════════════════
|
# ═══════════════════════════════════════════════════════════
|
||||||
|
|
||||||
KATA_SENTIMEN_PENTING = {
|
KATA_SENTIMEN_PENTING = {
|
||||||
# ── Negasi ──────────────────────────────────────────────
|
# Negasi
|
||||||
"tidak", "bukan", "jangan", "kurang", "belum", "tanpa",
|
"tidak", "bukan", "jangan", "kurang", "belum", "tanpa",
|
||||||
|
# Intensitas
|
||||||
# ── Sentimen POSITIF ────────────────────────────────────
|
"sangat", "banget", "sekali", "paling", "amat", "luar", "biasa",
|
||||||
|
# Positif umum
|
||||||
"keren", "bagus", "mantap", "setuju", "dukung", "mendukung",
|
"keren", "bagus", "mantap", "setuju", "dukung", "mendukung",
|
||||||
"andal", "handal", "gercep", "bangga", "senang", "suka",
|
"andal", "handal", "gercep", "bangga", "senang", "suka",
|
||||||
"baik", "benar", "tepat", "oke",
|
"baik", "benar", "tepat", "oke", "puas",
|
||||||
"sejahtera", "berkembang", "maju", "inovatif",
|
"sejahtera", "berkembang", "maju", "inovatif",
|
||||||
"tegas", "sigap", "tanggap", "adil", "bijak", "bermanfaat",
|
"tegas", "sigap", "tanggap", "adil", "bijak", "bermanfaat",
|
||||||
"untung", "berhasil", "sukses", "solusi", "manfaat",
|
"untung", "berhasil", "sukses", "solusi", "manfaat",
|
||||||
"berguna", "membantu", "bantu", "pro", "lanjut",
|
"berguna", "membantu", "bantu", "pro", "lanjut",
|
||||||
"sangat", "banget", "sekali", "paling", "amat", "luar", "biasa",
|
# Positif domain e-commerce/ongkir
|
||||||
|
"gratis", "murah", "hemat", "terjangkau", "cepat",
|
||||||
# ── Sentimen NEGATIF evaluatif ──────────────────────────
|
"aman", "mudah", "praktis", "terpercaya",
|
||||||
"kecewa", "buruk", "jelek", "parah", "gagal", "hancur",
|
# Negatif umum
|
||||||
"rusak", "bohong", "tipu", "korupsi",
|
"kecewa", "mending", "malah", "buruk", "jelek", "parah",
|
||||||
|
"gagal", "hancur", "rusak", "bohong", "tipu", "korupsi",
|
||||||
# ── Emosi ───────────────────────────────────────────────
|
# Negatif domain e-commerce/ongkir
|
||||||
|
"mahal", "lambat", "lelet", "ribet", "susah", "repot",
|
||||||
|
"rugi", "boros",
|
||||||
|
# Emosi
|
||||||
"marah", "sedih", "khawatir",
|
"marah", "sedih", "khawatir",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
# ═══════════════════════════════════════════════════════════
|
|
||||||
# NORMALISASI (selaras dengan preprocessing_page.py)
|
|
||||||
# ═══════════════════════════════════════════════════════════
|
|
||||||
|
|
||||||
NORMALISASI = {
|
|
||||||
# Negasi
|
|
||||||
"gk": "tidak", "ga": "tidak", "gak": "tidak",
|
|
||||||
"nggak": "tidak", "ngga": "tidak", "tdk": "tidak",
|
|
||||||
"tak": "tidak", "enggak": "tidak", "engga": "tidak",
|
|
||||||
"kagak": "tidak", "kaga": "tidak", "ndak": "tidak",
|
|
||||||
"gkk": "tidak", "ngak": "tidak",
|
|
||||||
# Kata ganti
|
|
||||||
"yg": "yang", "dgn": "dengan", "utk": "untuk",
|
|
||||||
"org": "orang", "krn": "karena", "dr": "dari",
|
|
||||||
"sm": "sama", "pd": "pada", "dlm": "dalam",
|
|
||||||
"bwt": "buat", "trm": "terima",
|
|
||||||
# Verba
|
|
||||||
"tp": "tapi", "tpi": "tapi", "jd": "jadi",
|
|
||||||
"sdh": "sudah", "blm": "belum", "emg": "memang",
|
|
||||||
"emang": "memang", "gimana": "bagaimana",
|
|
||||||
"gitu": "begitu", "gini": "begini",
|
|
||||||
"udah": "sudah", "udh": "sudah",
|
|
||||||
# Intensitas
|
|
||||||
"bgt": "banget", "bngt": "banget", "bget": "banget", "bgtt": "banget",
|
|
||||||
# Positif informal
|
|
||||||
"bener": "benar", "beneran": "benar",
|
|
||||||
"mantep": "mantap", "mntap": "mantap",
|
|
||||||
"kece": "keren",
|
|
||||||
"cucok": "cocok", "cucuk": "cocok",
|
|
||||||
"cakep": "bagus",
|
|
||||||
"sip": "baik", "siipp": "baik",
|
|
||||||
"top": "terbaik",
|
|
||||||
"jos": "bagus", "josss": "bagus",
|
|
||||||
"goks": "luar biasa",
|
|
||||||
"setujuu": "setuju", "stuju": "setuju",
|
|
||||||
"proud": "bangga",
|
|
||||||
"mantul": "mantap betul",
|
|
||||||
# Negatif informal
|
|
||||||
"ancur": "hancur", "ancrr": "hancur",
|
|
||||||
"parahh": "parah", "parahhh": "parah",
|
|
||||||
"gagall": "gagal",
|
|
||||||
"ngaco": "tidak benar",
|
|
||||||
"ngasal": "tidak benar",
|
|
||||||
"gaje": "tidak jelas",
|
|
||||||
# Domain
|
|
||||||
"ongkir": "ongkos kirim",
|
|
||||||
"freeongkir": "gratis ongkos kirim",
|
|
||||||
"gratisongkir": "gratis ongkos kirim",
|
|
||||||
"free": "gratis",
|
|
||||||
"ecommerce": "e commerce",
|
|
||||||
"seller": "penjual",
|
|
||||||
"buyer": "pembeli",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# ═══════════════════════════════════════════════════════════
|
# ═══════════════════════════════════════════════════════════
|
||||||
# LEXICON SENTIMEN
|
# LEXICON SENTIMEN
|
||||||
|
# Diperluas agar kata yang tidak ada di vocab TF-IDF tetap
|
||||||
|
# bisa berkontribusi melalui jalur lexicon scoring.
|
||||||
# ═══════════════════════════════════════════════════════════
|
# ═══════════════════════════════════════════════════════════
|
||||||
|
|
||||||
LEXICON_POSITIF = {
|
LEXICON_POSITIF = {
|
||||||
|
# Umum & informal
|
||||||
"bagus", "baik", "keren", "mantap", "mantep", "hebat",
|
"bagus", "baik", "keren", "mantap", "mantep", "hebat",
|
||||||
"oke", "sip", "top", "jos", "goks", "kece", "mantul",
|
"oke", "sip", "top", "jos", "goks", "kece", "mantul",
|
||||||
"setuju", "dukung", "mendukung", "pro", "lanjut", "sepakat",
|
"setuju", "dukung", "mendukung", "pro", "lanjut", "sepakat",
|
||||||
|
|
@ -127,71 +68,147 @@ LEXICON_POSITIF = {
|
||||||
"untung", "gratis", "murah", "hemat", "terjangkau",
|
"untung", "gratis", "murah", "hemat", "terjangkau",
|
||||||
"terbaik", "luar biasa",
|
"terbaik", "luar biasa",
|
||||||
"cakep", "cucok", "gaskeun", "kuy",
|
"cakep", "cucok", "gaskeun", "kuy",
|
||||||
"dukung", "bantu", "solusi", "manfaat",
|
"solusi", "manfaat", "menguntungkan",
|
||||||
"memuaskan", "membanggakan", "mengagumkan",
|
"memuaskan", "membanggakan", "mengagumkan",
|
||||||
|
"sehat", "fair", "wajar",
|
||||||
|
"syukur", "alhamdulillah",
|
||||||
|
"saing", "kompetitif",
|
||||||
|
# Domain ongkir/ecommerce positif
|
||||||
|
"terjangkau", "hemat", "efisien", "mudah", "praktis",
|
||||||
|
"cepat", "aman", "terpercaya", "andalan",
|
||||||
|
# Dukungan kebijakan
|
||||||
|
"dukung", "setuju", "bagus", "tepat", "bijak",
|
||||||
|
"perlu", "penting", "benar", "wajar", "adil",
|
||||||
}
|
}
|
||||||
|
|
||||||
LEXICON_NEGATIF = {
|
LEXICON_NEGATIF = {
|
||||||
|
# Umum
|
||||||
"buruk", "jelek", "parah", "rusak", "hancur", "ancur",
|
"buruk", "jelek", "parah", "rusak", "hancur", "ancur",
|
||||||
"gagal", "gagall", "ambruk", "terpuruk", "bangkrut",
|
"gagal", "ambruk", "terpuruk", "bangkrut",
|
||||||
"bohong", "tipu", "curang", "manipulasi", "korupsi", "penipuan",
|
"bohong", "tipu", "curang", "manipulasi", "korupsi",
|
||||||
"kebohongan",
|
"kebohongan", "penipuan",
|
||||||
"kecewa", "marah", "sedih", "khawatir", "takut", "benci",
|
"kecewa", "mending", "malah", "marah", "sedih",
|
||||||
"jijik", "muak", "kesal", "frustrasi",
|
"khawatir", "takut", "benci", "jijik", "muak",
|
||||||
"mahal", "rugi", "merugikan",
|
"kesal", "frustrasi", "geram", "dongkol",
|
||||||
"lambat", "lemot", "ribet", "susah", "sulit", "bermasalah",
|
"mahal", "rugi", "merugikan", "rugikan",
|
||||||
|
"lambat", "lemot", "lelet", "ribet", "susah",
|
||||||
|
"sulit", "bermasalah",
|
||||||
"ngaco", "ngasal", "gaje", "receh",
|
"ngaco", "ngasal", "gaje", "receh",
|
||||||
"tidak benar", "tidak jelas", "tidak adil", "tidak berguna",
|
"tolol", "bodoh", "idiot", "goblok",
|
||||||
"mengecewakan", "menyebalkan", "menyusahkan",
|
"mengecewakan", "menyebalkan", "menyusahkan",
|
||||||
|
"monopoli", "licik",
|
||||||
|
# Domain ongkir negatif
|
||||||
|
"boros", "memberatkan", "menyulitkan",
|
||||||
|
"repot", "ribet", "ngeributin", "ribut",
|
||||||
|
# Kritik kebijakan
|
||||||
|
"salah", "keliru", "gegabah", "sembarangan",
|
||||||
|
"tidak jelas", "ngawur", "asal",
|
||||||
}
|
}
|
||||||
|
|
||||||
KATA_NEGASI = {
|
KATA_NEGASI = {
|
||||||
"tidak", "bukan", "jangan", "belum", "tanpa", "kurang",
|
"tidak", "bukan", "jangan", "belum", "tanpa", "kurang",
|
||||||
"anti", "non",
|
"anti", "non", "tak", "ga", "gak", "nggak", "ngga",
|
||||||
|
"enggak", "engga",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
# ═══════════════════════════════════════════════════════════
|
# ═══════════════════════════════════════════════════════════
|
||||||
# STOPWORDS
|
# LOAD NORMALISASI DARI FILE
|
||||||
|
# ═══════════════════════════════════════════════════════════
|
||||||
|
|
||||||
|
def _load_normalization() -> dict:
|
||||||
|
norm_file = "indonesian-normalisasi-slangword-complete.txt"
|
||||||
|
norm_dict: dict = {}
|
||||||
|
try:
|
||||||
|
with open(norm_file, "r", encoding="utf-8") as f:
|
||||||
|
for line in f:
|
||||||
|
line = line.strip()
|
||||||
|
if not line:
|
||||||
|
continue
|
||||||
|
parts = line.split(",", 1)
|
||||||
|
if len(parts) != 2:
|
||||||
|
continue
|
||||||
|
slang = parts[0].strip().strip("'\"").lower()
|
||||||
|
normal = parts[1].strip().lower()
|
||||||
|
if slang and normal:
|
||||||
|
norm_dict[slang] = normal
|
||||||
|
except FileNotFoundError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
DOMAIN_OVERRIDES = {
|
||||||
|
"shopee": "shopee", "tokopedia": "tokopedia", "lazada": "lazada",
|
||||||
|
"tiktok": "tiktok", "bukalapak": "bukalapak", "blibli": "blibli",
|
||||||
|
"sicepat": "sicepat", "jne": "jne", "jnt": "jnt",
|
||||||
|
"anteraja": "anteraja", "ninja": "ninja",
|
||||||
|
"freeongkir": "gratis ongkos kirim",
|
||||||
|
"gratisongkir": "gratis ongkos kirim",
|
||||||
|
"ongkir": "ongkos kirim", "ongkr": "ongkos kirim",
|
||||||
|
"bykrm": "biaya kirim", "biayakirim": "biaya pengiriman",
|
||||||
|
"komdigi": "komdigi", "kemendag": "kementerian perdagangan",
|
||||||
|
"kominfo": "kementerian komunikasi",
|
||||||
|
"ecommerce": "e commerce", "marketplace": "marketplace",
|
||||||
|
"seller": "penjual", "buyer": "pembeli", "online": "online",
|
||||||
|
# Negasi tambahan
|
||||||
|
"gk": "tidak", "ga": "tidak", "gak": "tidak",
|
||||||
|
"nggak": "tidak", "ngga": "tidak", "tdk": "tidak",
|
||||||
|
"tak": "tidak", "enggak": "tidak", "engga": "tidak",
|
||||||
|
"kagak": "tidak", "kaga": "tidak", "ndak": "tidak",
|
||||||
|
"ngak": "tidak",
|
||||||
|
# Intensitas
|
||||||
|
"bgt": "banget", "bngt": "banget", "bget": "banget",
|
||||||
|
"bgtt": "banget",
|
||||||
|
# Positif informal
|
||||||
|
"mantep": "mantap", "mntap": "mantap",
|
||||||
|
"bener": "benar", "beneran": "benar",
|
||||||
|
"kece": "keren", "sip": "baik",
|
||||||
|
# Negatif informal
|
||||||
|
"ancur": "hancur", "parahh": "parah",
|
||||||
|
}
|
||||||
|
norm_dict.update(DOMAIN_OVERRIDES)
|
||||||
|
return norm_dict
|
||||||
|
|
||||||
|
|
||||||
|
# ═══════════════════════════════════════════════════════════
|
||||||
|
# LOAD STOPWORDS DARI FILE
|
||||||
# ═══════════════════════════════════════════════════════════
|
# ═══════════════════════════════════════════════════════════
|
||||||
|
|
||||||
def _load_stopwords() -> set:
|
def _load_stopwords() -> set:
|
||||||
"""Load stopword dengan penjagaan kata sentimen penting."""
|
|
||||||
stopword_file = "indonesian-stopwords-complete.txt"
|
stopword_file = "indonesian-stopwords-complete.txt"
|
||||||
base = set()
|
base: set = set()
|
||||||
try:
|
try:
|
||||||
with open(stopword_file, "r", encoding="utf-8") as f:
|
with open(stopword_file, "r", encoding="utf-8") as f:
|
||||||
base = set(f.read().splitlines())
|
for line in f:
|
||||||
|
word = line.strip().lower()
|
||||||
|
if word:
|
||||||
|
base.add(word)
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
base = {
|
base = {
|
||||||
"yang", "dan", "di", "ke", "dari", "ini", "itu",
|
"yang", "dan", "di", "ke", "dari", "ini", "itu",
|
||||||
"dengan", "untuk", "pada", "adalah", "oleh", "ada",
|
"dengan", "untuk", "pada", "adalah", "oleh", "ada",
|
||||||
"ya", "akan", "atau", "juga", "sama", "karena",
|
"ya", "akan", "atau", "juga", "sama", "karena",
|
||||||
"jika", "sudah", "telah", "saat", "agar", "maka",
|
"jika", "sudah", "telah", "jadi", "bisa",
|
||||||
"lagi", "bila", "bisa", "pun", "nya",
|
|
||||||
}
|
}
|
||||||
|
|
||||||
# Jangan hapus kata sentimen penting
|
# Lindungi kata sentimen penting
|
||||||
for kata in KATA_SENTIMEN_PENTING:
|
for kata in KATA_SENTIMEN_PENTING:
|
||||||
base.discard(kata)
|
base.discard(kata)
|
||||||
|
|
||||||
# Tambahan stopword domain-spesifik
|
# Tambah noise Twitter
|
||||||
base.update({
|
base.update({
|
||||||
"rt", "amp", "https", "http", "co", "t",
|
"rt", "amp", "https", "http", "co", "pic",
|
||||||
"wkwk", "wkwkwk", "haha", "hehe", "xixi", "hahaha",
|
"wkwk", "wkwkwk", "wkwkwkwk",
|
||||||
"yg", "dgn", "utk", "dr", "krn", "tp", "jd", "sdh",
|
"haha", "hahaha", "hehe", "hihi", "huhu", "xixi",
|
||||||
"aja", "doang", "nih", "sih", "dong", "deh",
|
"nih", "sih", "dong", "deh", "loh", "lah", "tuh",
|
||||||
"loh", "lah", "tuh", "kak", "gan", "bro", "sis",
|
"kak", "gan", "bro", "sob", "min",
|
||||||
})
|
})
|
||||||
return base
|
return base
|
||||||
|
|
||||||
|
|
||||||
# ═══════════════════════════════════════════════════════════
|
# ═══════════════════════════════════════════════════════════
|
||||||
# STEMMER
|
# LOAD STEMMER
|
||||||
# ═══════════════════════════════════════════════════════════
|
# ═══════════════════════════════════════════════════════════
|
||||||
|
|
||||||
def _load_stemmer():
|
def _load_stemmer():
|
||||||
"""Load Sastrawi stemmer. Return None jika tidak terinstall."""
|
|
||||||
try:
|
try:
|
||||||
from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
|
from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
|
||||||
return StemmerFactory().create_stemmer()
|
return StemmerFactory().create_stemmer()
|
||||||
|
|
@ -199,13 +216,14 @@ def _load_stemmer():
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
# ── Inisialisasi global ─────────────────────────────────────────────────────
|
# ── Inisialisasi global (dimuat sekali saat modul diimport) ─
|
||||||
_STOPWORDS = _load_stopwords()
|
_STOPWORDS = _load_stopwords()
|
||||||
_STEMMER = _load_stemmer()
|
_STEMMER = _load_stemmer()
|
||||||
|
_NORM_DICT = _load_normalization()
|
||||||
|
|
||||||
|
|
||||||
# ═══════════════════════════════════════════════════════════
|
# ═══════════════════════════════════════════════════════════
|
||||||
# MODEL LOADING (lazy)
|
# MODEL LOADING (lazy, singleton)
|
||||||
# ═══════════════════════════════════════════════════════════
|
# ═══════════════════════════════════════════════════════════
|
||||||
|
|
||||||
_model = None
|
_model = None
|
||||||
|
|
@ -213,10 +231,6 @@ _tfidf = None
|
||||||
|
|
||||||
|
|
||||||
def _load_model():
|
def _load_model():
|
||||||
"""
|
|
||||||
Lazy load NB model + TF-IDF vectorizer.
|
|
||||||
Return: (model, tfidf) — keduanya bisa None.
|
|
||||||
"""
|
|
||||||
global _model, _tfidf
|
global _model, _tfidf
|
||||||
if _model is None:
|
if _model is None:
|
||||||
try:
|
try:
|
||||||
|
|
@ -232,35 +246,29 @@ def _load_model():
|
||||||
|
|
||||||
|
|
||||||
# ═══════════════════════════════════════════════════════════
|
# ═══════════════════════════════════════════════════════════
|
||||||
# PREPROCESSING PIPELINE — 5 TAHAP
|
# 5-TAHAP PREPROCESSING PIPELINE
|
||||||
#
|
|
||||||
# URUTAN (selaras dengan preprocessing_page.py):
|
|
||||||
# 1. Case Folding → lowercase
|
|
||||||
# 2. Cleaning → hapus noise
|
|
||||||
# 3. Normalisasi → normalisasi kata
|
|
||||||
# 4. Stopword Removal → buang stopword (split() dilakukan internal)
|
|
||||||
# 5. Stemming → bentuk dasar
|
|
||||||
# ═══════════════════════════════════════════════════════════
|
# ═══════════════════════════════════════════════════════════
|
||||||
|
|
||||||
def _case_folding(text: str) -> str:
|
def _step1_case_folding(text: str) -> str:
|
||||||
"""Tahap 1: Lowercase seluruh teks."""
|
|
||||||
return str(text).lower()
|
return str(text).lower()
|
||||||
|
|
||||||
|
|
||||||
def _cleaning(text: str) -> str:
|
def _step2_cleaning(text: str) -> str:
|
||||||
"""Tahap 2: Hapus noise (URL, mention, hashtag, angka, emoji, tanda baca)."""
|
|
||||||
text = re.sub(r"http\S+|www\S+|https\S+", "", text)
|
text = re.sub(r"http\S+|www\S+|https\S+", "", text)
|
||||||
text = re.sub(r"@\w+", "", text)
|
text = re.sub(r"@\w+", "", text)
|
||||||
text = re.sub(r"#\w+", "", text)
|
text = re.sub(r"#\w+", "", text)
|
||||||
text = re.sub(r"\d+", "", text)
|
text = re.sub(r"\d+", "", text)
|
||||||
text = re.sub(
|
text = re.sub(
|
||||||
r"[\U00010000-\U0010ffff"
|
r"["
|
||||||
|
r"\U00010000-\U0010ffff"
|
||||||
r"\U0001F600-\U0001F64F"
|
r"\U0001F600-\U0001F64F"
|
||||||
r"\U0001F300-\U0001F5FF"
|
r"\U0001F300-\U0001F5FF"
|
||||||
r"\U0001F680-\U0001F6FF"
|
r"\U0001F680-\U0001F6FF"
|
||||||
r"\U0001F1E0-\U0001F1FF"
|
r"\U0001F1E0-\U0001F1FF"
|
||||||
r"\u2600-\u26FF\u2700-\u27BF"
|
r"\u2600-\u26FF"
|
||||||
r"]+", "", text, flags=re.UNICODE,
|
r"\u2700-\u27BF"
|
||||||
|
r"]+",
|
||||||
|
"", text, flags=re.UNICODE,
|
||||||
)
|
)
|
||||||
text = text.translate(str.maketrans("", "", string.punctuation))
|
text = text.translate(str.maketrans("", "", string.punctuation))
|
||||||
text = re.sub(r"[^a-zA-Z\s]", "", text)
|
text = re.sub(r"[^a-zA-Z\s]", "", text)
|
||||||
|
|
@ -268,79 +276,72 @@ def _cleaning(text: str) -> str:
|
||||||
return text
|
return text
|
||||||
|
|
||||||
|
|
||||||
def _normalisasi(text: str) -> str:
|
def _step3_normalization(text: str, norm_dict: dict) -> str:
|
||||||
"""Tahap 3: Normalisasi singkatan dan kata tidak baku."""
|
tokens = text.split()
|
||||||
return " ".join(NORMALISASI.get(word, word) for word in text.split())
|
return " ".join(norm_dict.get(token, token) for token in tokens)
|
||||||
|
|
||||||
|
|
||||||
def _remove_stopwords(tokens: list) -> list:
|
def _step4_stopword_removal(tokens: list, stopwords: set) -> list:
|
||||||
"""Tahap 4: Stopword removal dengan penjagaan kata sentimen."""
|
result = []
|
||||||
return [
|
for token in tokens:
|
||||||
w for w in tokens
|
if token in KATA_SENTIMEN_PENTING:
|
||||||
if (w not in _STOPWORDS or w in KATA_SENTIMEN_PENTING) and len(w) > 2
|
result.append(token)
|
||||||
]
|
continue
|
||||||
|
if token in stopwords:
|
||||||
|
continue
|
||||||
|
if len(token) <= 2:
|
||||||
|
continue
|
||||||
|
result.append(token)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
def _stemming(tokens: list) -> list:
|
def _step5_stemming(tokens: list, stemmer) -> list:
|
||||||
"""Tahap 5: Stemming ke bentuk dasar via Sastrawi ECS."""
|
if stemmer is None:
|
||||||
if _STEMMER is None:
|
|
||||||
return tokens
|
return tokens
|
||||||
return [_STEMMER.stem(w) for w in tokens]
|
return [stemmer.stem(token) for token in tokens]
|
||||||
|
|
||||||
|
|
||||||
def preprocess_for_model(text: str) -> str:
|
def preprocess_for_model(text: str) -> str:
|
||||||
"""
|
"""Full 5-tahap preprocessing → string teks bersih siap TF-IDF."""
|
||||||
Full 5-tahap preprocessing → string teks bersih siap TF-IDF.
|
s1 = _step1_case_folding(text)
|
||||||
|
s2 = _step2_cleaning(s1)
|
||||||
URUTAN: Case Folding → Cleaning → Normalisasi
|
s3 = _step3_normalization(s2, _NORM_DICT)
|
||||||
→ Stopword Removal → Stemming
|
s4 = _step4_stopword_removal(s3.split(), _STOPWORDS)
|
||||||
|
s5 = _step5_stemming(s4, _STEMMER)
|
||||||
Split kata dilakukan secara internal di tahap Stopword Removal.
|
|
||||||
Pipeline HARUS sama persis dengan yang dipakai saat training model.
|
|
||||||
"""
|
|
||||||
s1 = _case_folding(text) # Tahap 1
|
|
||||||
s2 = _cleaning(s1) # Tahap 2
|
|
||||||
s3 = _normalisasi(s2) # Tahap 3
|
|
||||||
s4 = _remove_stopwords(s3.split()) # Tahap 4 (split inline)
|
|
||||||
s5 = _stemming(s4) # Tahap 5
|
|
||||||
return " ".join(s5)
|
return " ".join(s5)
|
||||||
|
|
||||||
|
|
||||||
def preprocess_untuk_lexicon(text: str) -> str:
|
def preprocess_untuk_lexicon(text: str) -> str:
|
||||||
"""
|
"""Preprocessing ringan untuk lexicon matching (tanpa stemming)."""
|
||||||
Preprocessing RINGAN untuk lexicon matching.
|
s1 = _step1_case_folding(text)
|
||||||
Tidak di-stem → kata asli bisa dicocokkan dengan lexicon.
|
s2 = _step2_cleaning(s1)
|
||||||
Pipeline: Case Folding → Cleaning → Normalisasi saja.
|
s3 = _step3_normalization(s2, _NORM_DICT)
|
||||||
"""
|
|
||||||
s1 = _case_folding(text)
|
|
||||||
s2 = _cleaning(s1)
|
|
||||||
s3 = _normalisasi(s2)
|
|
||||||
return s3
|
return s3
|
||||||
|
|
||||||
|
|
||||||
# ═══════════════════════════════════════════════════════════
|
# ═══════════════════════════════════════════════════════════
|
||||||
# LEXICON SCORER
|
# LEXICON SCORER (dengan negation window diperluas ke 3)
|
||||||
# ═══════════════════════════════════════════════════════════
|
# ═══════════════════════════════════════════════════════════
|
||||||
|
|
||||||
def _hitung_skor_lexicon(teks_lexicon: str) -> dict:
|
def _hitung_skor_lexicon(teks_lexicon: str) -> dict:
|
||||||
"""
|
"""
|
||||||
Hitung skor positif dan negatif dari teks via lexicon.
|
Hitung skor sentimen via lexicon + negation handling.
|
||||||
|
|
||||||
NEGATION HANDLING:
|
Perubahan vs versi lama:
|
||||||
Kata negasi dalam window 2 kata sebelum kata sentimen → polaritas dibalik.
|
- Window negasi diperluas menjadi 3 kata (sebelumnya 2)
|
||||||
Contoh: "tidak bagus" → ada "tidak" sebelum "bagus" (POSITIF)
|
- Lexicon positif/negatif lebih luas (kata yang tidak ada di vocab TF-IDF)
|
||||||
→ skor_neg += 1 (bukan skor_pos)
|
|
||||||
|
|
||||||
Return: {"positif": int, "negatif": int, "net": int}
|
Return: {"positif": int, "negatif": int, "net": int}
|
||||||
"""
|
"""
|
||||||
tokens = teks_lexicon.split()
|
tokens = teks_lexicon.split()
|
||||||
skor_pos = 0
|
skor_pos = 0
|
||||||
skor_neg = 0
|
skor_neg = 0
|
||||||
|
|
||||||
for i, token in enumerate(tokens):
|
for i, token in enumerate(tokens):
|
||||||
|
# Cek negasi dalam window 3 kata sebelumnya
|
||||||
ada_negasi = any(
|
ada_negasi = any(
|
||||||
tokens[i - j] in KATA_NEGASI
|
tokens[i - j] in KATA_NEGASI
|
||||||
for j in range(1, 3)
|
for j in range(1, 4)
|
||||||
if i - j >= 0
|
if i - j >= 0
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -363,30 +364,32 @@ def _hitung_skor_lexicon(teks_lexicon: str) -> dict:
|
||||||
|
|
||||||
|
|
||||||
# ═══════════════════════════════════════════════════════════
|
# ═══════════════════════════════════════════════════════════
|
||||||
# HYBRID CLASSIFIER
|
# PREDIKSI MODEL NB
|
||||||
# ═══════════════════════════════════════════════════════════
|
# ═══════════════════════════════════════════════════════════
|
||||||
|
|
||||||
# Normalisasi label dari berbagai format yang mungkin dipakai model
|
|
||||||
_LABEL_MAP = {
|
_LABEL_MAP = {
|
||||||
"positif": "Positif", "Positif": "Positif", "positive": "Positif", "pos": "Positif",
|
"positif": "Positif", "Positif": "Positif",
|
||||||
"negatif": "Negatif", "Negatif": "Negatif", "negative": "Negatif", "neg": "Negatif",
|
"positive": "Positif", "pos": "Positif",
|
||||||
"netral": "Netral", "Netral": "Netral", "neutral": "Netral", "net": "Netral",
|
"negatif": "Negatif", "Negatif": "Negatif",
|
||||||
|
"negative": "Negatif", "neg": "Negatif",
|
||||||
|
"netral": "Netral", "Netral": "Netral",
|
||||||
|
"neutral": "Netral", "net": "Netral",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def _prediksi_model(teks_model: str):
|
def _prediksi_model(teks_model: str):
|
||||||
"""
|
"""
|
||||||
Dapatkan prediksi dari NB model.
|
Prediksi dari model NB 3-kelas.
|
||||||
Return: (label_norm, confidence, proba_dict) atau None.
|
Return: (label_norm, confidence, proba_dict) atau None jika gagal.
|
||||||
"""
|
"""
|
||||||
model, tfidf = _load_model()
|
model, tfidf = _load_model()
|
||||||
if model is None or tfidf is None or not teks_model.strip():
|
if model is None or tfidf is None or not teks_model.strip():
|
||||||
return None
|
return None
|
||||||
try:
|
try:
|
||||||
vec = tfidf.transform([teks_model])
|
vec = tfidf.transform([teks_model])
|
||||||
pred_raw = model.predict(vec)[0]
|
pred_raw = model.predict(vec)[0]
|
||||||
proba = model.predict_proba(vec)[0]
|
proba = model.predict_proba(vec)[0]
|
||||||
classes = list(model.classes_)
|
classes = list(model.classes_)
|
||||||
|
|
||||||
label_norm = _LABEL_MAP.get(str(pred_raw), "Netral")
|
label_norm = _LABEL_MAP.get(str(pred_raw), "Netral")
|
||||||
pred_idx = classes.index(pred_raw) if pred_raw in classes else 0
|
pred_idx = classes.index(pred_raw) if pred_raw in classes else 0
|
||||||
|
|
@ -402,79 +405,105 @@ def _prediksi_model(teks_model: str):
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
# ═══════════════════════════════════════════════════════════
|
||||||
|
# HYBRID CLASSIFIER (diperkuat)
|
||||||
|
# ═══════════════════════════════════════════════════════════
|
||||||
|
|
||||||
def _klasifikasi_hybrid(
|
def _klasifikasi_hybrid(
|
||||||
teks_model: str,
|
teks_model: str,
|
||||||
skor: dict,
|
skor: dict,
|
||||||
teks_lower: str,
|
teks_lower: str,
|
||||||
) -> tuple:
|
) -> tuple:
|
||||||
"""
|
"""
|
||||||
Klasifikasi hybrid: sinyal lexicon + model NB.
|
Klasifikasi hybrid: sinyal lexicon + model NB 3-kelas.
|
||||||
|
|
||||||
LOGIKA KEPUTUSAN (berurutan):
|
|
||||||
|
|
||||||
|
PERUBAHAN vs versi lama:
|
||||||
|
─────────────────────────────────────────────────────────
|
||||||
Layer 1 — Override Positif KUAT (net >= 2):
|
Layer 1 — Override Positif KUAT (net >= 2):
|
||||||
→ Positif, confidence 60–92%
|
Confidence lebih tinggi karena lexicon lebih kuat.
|
||||||
|
Range: 0.65–0.92
|
||||||
|
|
||||||
|
Layer 1b — Override Negatif KUAT (net <= -2):
|
||||||
|
Simetris dengan positif.
|
||||||
|
Range: 0.65–0.90
|
||||||
|
|
||||||
Layer 2 — Override Positif LEMAH (net == 1):
|
Layer 2 — Override Positif LEMAH (net == 1):
|
||||||
→ Cek model; jika model < 65% yakin Negatif → Positif
|
Diperlonggar: sekarang berlaku jika model tidak sangat
|
||||||
→ Jika model sangat yakin Negatif → tetap Positif (confidence rendah)
|
yakin negatif (threshold 0.70, sebelumnya 0.65).
|
||||||
|
|
||||||
Layer 3 — Override Negatif KUAT (net <= -2):
|
Layer 2b — Override Negatif LEMAH (net == -1):
|
||||||
→ Negatif, confidence 60–90%
|
Simetris baru. Sebelumnya tidak ada.
|
||||||
|
|
||||||
Layer 4 — Fallback Model NB:
|
Layer 3 — Fallback Model NB:
|
||||||
→ Prediksi model dipakai, TAPI:
|
Anti-bias diperketat:
|
||||||
• Jika model = Negatif AND net >= 0 AND confidence < 75%
|
- Model Negatif + lexicon bersih (net >= 0) + conf < 0.65
|
||||||
→ downgrade ke Netral (koreksi bias model)
|
→ turunkan ke Netral (threshold naik dari 0.70 ke 0.65)
|
||||||
• Kasus lainnya → percaya model
|
- Model Positif + lexicon negatif kuat (net <= -1) + conf < 0.65
|
||||||
|
→ turunkan ke Netral
|
||||||
Layer 5 — Ultimate Fallback (model tidak tersedia):
|
|
||||||
→ Gunakan skor lexicon saja
|
|
||||||
|
|
||||||
|
Layer 4 — Ultimate Fallback:
|
||||||
|
Lexicon saja jika model tidak tersedia.
|
||||||
|
─────────────────────────────────────────────────────────
|
||||||
Return: (label: str, confidence: float)
|
Return: (label: str, confidence: float)
|
||||||
"""
|
"""
|
||||||
net = skor["net"]
|
net = skor["net"]
|
||||||
pos = skor["positif"]
|
pos = skor["positif"]
|
||||||
|
neg = skor["negatif"]
|
||||||
|
|
||||||
# ── Layer 1: Positif KUAT ─────────────────────────────────────────────────
|
# ── Layer 1a: Positif KUAT ───────────────────────────────────────────────
|
||||||
if net >= 2:
|
if net >= 2:
|
||||||
conf = min(0.60 + (net * 0.07), 0.92)
|
conf = min(0.65 + (net * 0.05), 0.92)
|
||||||
return ("Positif", round(conf, 3))
|
return ("Positif", round(conf, 3))
|
||||||
|
|
||||||
# ── Layer 2: Positif LEMAH ────────────────────────────────────────────────
|
# ── Layer 1b: Negatif KUAT ───────────────────────────────────────────────
|
||||||
|
if net <= -2:
|
||||||
|
conf = min(0.65 + (abs(net) * 0.05), 0.90)
|
||||||
|
return ("Negatif", round(conf, 3))
|
||||||
|
|
||||||
|
# ── Layer 2a: Positif LEMAH ──────────────────────────────────────────────
|
||||||
if net == 1 and pos >= 1:
|
if net == 1 and pos >= 1:
|
||||||
model_result = _prediksi_model(teks_model)
|
model_result = _prediksi_model(teks_model)
|
||||||
if model_result is not None:
|
if model_result is not None:
|
||||||
_, _, proba_dict = model_result
|
_, _, proba_dict = model_result
|
||||||
neg_prob = proba_dict.get("Negatif", 0.0)
|
neg_prob = proba_dict.get("Negatif", 0.0)
|
||||||
if neg_prob < 0.65:
|
if neg_prob < 0.70: # diperlonggar dari 0.65
|
||||||
conf = round(0.55 + (0.65 - neg_prob) * 0.3, 3)
|
conf = round(0.55 + (0.70 - neg_prob) * 0.30, 3)
|
||||||
return ("Positif", min(conf, 0.80))
|
return ("Positif", min(conf, 0.82))
|
||||||
else:
|
else:
|
||||||
return ("Positif", 0.55)
|
return ("Positif", 0.55)
|
||||||
else:
|
return ("Positif", 0.58)
|
||||||
return ("Positif", 0.58)
|
|
||||||
|
|
||||||
# ── Layer 3: Negatif KUAT ─────────────────────────────────────────────────
|
# ── Layer 2b: Negatif LEMAH (baru) ───────────────────────────────────────
|
||||||
if net <= -2:
|
if net == -1 and neg >= 1:
|
||||||
conf = min(0.60 + (abs(net) * 0.06), 0.90)
|
model_result = _prediksi_model(teks_model)
|
||||||
return ("Negatif", round(conf, 3))
|
if model_result is not None:
|
||||||
|
_, _, proba_dict = model_result
|
||||||
|
pos_prob = proba_dict.get("Positif", 0.0)
|
||||||
|
if pos_prob < 0.70:
|
||||||
|
conf = round(0.55 + (0.70 - pos_prob) * 0.30, 3)
|
||||||
|
return ("Negatif", min(conf, 0.82))
|
||||||
|
else:
|
||||||
|
return ("Negatif", 0.55)
|
||||||
|
return ("Negatif", 0.58)
|
||||||
|
|
||||||
# ── Layer 4: Fallback Model NB ────────────────────────────────────────────
|
# ── Layer 3: Fallback Model NB 3-kelas ───────────────────────────────────
|
||||||
model_result = _prediksi_model(teks_model)
|
model_result = _prediksi_model(teks_model)
|
||||||
if model_result is not None:
|
if model_result is not None:
|
||||||
label_norm, confidence, proba_dict = model_result
|
label_norm, confidence, proba_dict = model_result
|
||||||
|
|
||||||
# Anti-bias correction:
|
# Anti-bias 1: Model Negatif tapi lexicon bersih → Netral
|
||||||
# Model prediksi Negatif tapi tidak ada sinyal negatif dari lexicon
|
if label_norm == "Negatif" and net >= 0 and confidence < 0.65:
|
||||||
# dan confidence < 75% → kemungkinan bias → turunkan ke Netral
|
corrected_conf = round(0.50 + max(0, confidence - 0.50) * 0.20, 3)
|
||||||
if label_norm == "Negatif" and net >= 0 and confidence < 0.75:
|
return ("Netral", corrected_conf)
|
||||||
corrected_conf = round(0.50 + max(0, confidence - 0.50) * 0.2, 3)
|
|
||||||
|
# Anti-bias 2: Model Positif tapi lexicon negatif → Netral
|
||||||
|
if label_norm == "Positif" and net <= -1 and confidence < 0.65:
|
||||||
|
corrected_conf = round(0.50 + max(0, confidence - 0.50) * 0.20, 3)
|
||||||
return ("Netral", corrected_conf)
|
return ("Netral", corrected_conf)
|
||||||
|
|
||||||
return (label_norm, round(confidence, 3))
|
return (label_norm, round(confidence, 3))
|
||||||
|
|
||||||
# ── Layer 5: Ultimate Fallback ────────────────────────────────────────────
|
# ── Layer 4: Ultimate Fallback ────────────────────────────────────────────
|
||||||
if net > 0:
|
if net > 0:
|
||||||
return ("Positif", 0.55)
|
return ("Positif", 0.55)
|
||||||
elif net < 0:
|
elif net < 0:
|
||||||
|
|
@ -483,27 +512,45 @@ def _klasifikasi_hybrid(
|
||||||
return ("Netral", 0.50)
|
return ("Netral", 0.50)
|
||||||
|
|
||||||
|
|
||||||
|
# ═══════════════════════════════════════════════════════════
|
||||||
|
# PUBLIC API
|
||||||
|
# ═══════════════════════════════════════════════════════════
|
||||||
|
|
||||||
|
def analisis_sentimen_single(text: str) -> tuple:
|
||||||
|
"""
|
||||||
|
Analisis sentimen satu teks.
|
||||||
|
Return: (label, confidence)
|
||||||
|
label: 'Positif' | 'Netral' | 'Negatif'
|
||||||
|
"""
|
||||||
|
teks_model = preprocess_for_model(text)
|
||||||
|
teks_lexicon = preprocess_untuk_lexicon(text)
|
||||||
|
teks_lower = str(text).lower()
|
||||||
|
skor = _hitung_skor_lexicon(teks_lexicon)
|
||||||
|
return _klasifikasi_hybrid(teks_model, skor, teks_lower)
|
||||||
|
|
||||||
|
|
||||||
|
def analisis_sentimen_batch(texts: list) -> list:
|
||||||
|
"""
|
||||||
|
Analisis sentimen batch.
|
||||||
|
Return: list of (label, confidence)
|
||||||
|
"""
|
||||||
|
return [analisis_sentimen_single(text) for text in texts]
|
||||||
|
|
||||||
|
|
||||||
# ═══════════════════════════════════════════════════════════
|
# ═══════════════════════════════════════════════════════════
|
||||||
# BACKWARD COMPATIBILITY
|
# BACKWARD COMPATIBILITY
|
||||||
# ═══════════════════════════════════════════════════════════
|
# ═══════════════════════════════════════════════════════════
|
||||||
|
|
||||||
def bersihkan_teks(text: str) -> str:
|
def bersihkan_teks(text: str) -> str:
|
||||||
"""[LEGACY] Gunakan preprocess_for_model() untuk pipeline lengkap."""
|
"""[LEGACY] Gunakan preprocess_for_model()."""
|
||||||
return preprocess_for_model(text)
|
return preprocess_for_model(text)
|
||||||
|
|
||||||
|
|
||||||
def prediksi_sentimen(list_text: list):
|
def prediksi_sentimen(list_text: list):
|
||||||
"""
|
"""
|
||||||
[LEGACY] Prediksi batch dengan hybrid classifier.
|
[LEGACY] Prediksi batch.
|
||||||
Return: (list clean_texts, list labels)
|
Return: (list clean_texts, list labels)
|
||||||
"""
|
"""
|
||||||
clean_texts = [preprocess_for_model(t) for t in list_text]
|
clean_texts = [preprocess_for_model(t) for t in list_text]
|
||||||
labels = []
|
labels = [analisis_sentimen_single(t)[0] for t in list_text]
|
||||||
for text in list_text:
|
|
||||||
teks_model = preprocess_for_model(text)
|
|
||||||
teks_lexicon = preprocess_untuk_lexicon(text)
|
|
||||||
teks_lower = str(text).lower()
|
|
||||||
skor = _hitung_skor_lexicon(teks_lexicon)
|
|
||||||
label, _ = _klasifikasi_hybrid(teks_model, skor, teks_lower)
|
|
||||||
labels.append(label)
|
|
||||||
return clean_texts, labels
|
return clean_texts, labels
|
||||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
Before Width: | Height: | Size: 78 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 78 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 78 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 78 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 78 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 78 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 78 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 78 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 78 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 85 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 95 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 87 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 95 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 84 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 75 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 102 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 73 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 73 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 75 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 78 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 78 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 78 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 78 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 78 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 78 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 72 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
|
|
@ -1,2 +1 @@
|
||||||
"conversation_id_str","created_at","favorite_count","full_text","id_str","image_url","in_reply_to_screen_name","lang","location","quote_count","reply_count","retweet_count","tweet_url","user_id_str","username"
|
|
||||||
"2056538953340314095","Wed May 20 16:01:04 +0000 2026","4","@killingmaster27 @evelyndewi7 @katrinrienks @HumorJonTampan @wiyantokdr54 @EviDrajat @RickyKardjono @Micheladam69432 @ebonXboy @AntonRebor79105 @gangsterrandom @Pieter_Sun01 @Rahmah0845 @RSjahfidi @prabowo @SmileyNDeLight @Jediimar @Yens0906 @tokaidirecycle @PembuatSistem @bonapasogit24 @Jenniestale @Maskudin126003 @a8sar6alih @JowoNgalam @KING_RAJA_PWS_ @apaajadeh447870 @50ngoku @Submill @EmmaHusain0845 @BebySoSweet @Yulia_4ja @bungalotus25 @bahlillahadalia wahhh... jadi menteri komdigi dong menteri free ongkir https://t.co/1cfvoqIcNl","2057129489994510620","https://pbs.twimg.com/media/HIxkWwDaMAAXH9U.jpg","killingmaster27","in","","0","2","0","https://x.com/undefined/status/2057129489994510620","1410466956134608899",
|
|
||||||
|
|
|
||||||
|
|
|
@ -1,2 +1 @@
|
||||||
"conversation_id_str","created_at","favorite_count","full_text","id_str","image_url","in_reply_to_screen_name","lang","location","quote_count","reply_count","retweet_count","tweet_url","user_id_str","username"
|
|
||||||
"2056538953340314095","Wed May 20 16:01:04 +0000 2026","4","@killingmaster27 @evelyndewi7 @katrinrienks @HumorJonTampan @wiyantokdr54 @EviDrajat @RickyKardjono @Micheladam69432 @ebonXboy @AntonRebor79105 @gangsterrandom @Pieter_Sun01 @Rahmah0845 @RSjahfidi @prabowo @SmileyNDeLight @Jediimar @Yens0906 @tokaidirecycle @PembuatSistem @bonapasogit24 @Jenniestale @Maskudin126003 @a8sar6alih @JowoNgalam @KING_RAJA_PWS_ @apaajadeh447870 @50ngoku @Submill @EmmaHusain0845 @BebySoSweet @Yulia_4ja @bungalotus25 @bahlillahadalia wahhh... jadi menteri komdigi dong menteri free ongkir https://t.co/1cfvoqIcNl","2057129489994510620","https://pbs.twimg.com/media/HIxkWwDaMAAXH9U.jpg","killingmaster27","in","","0","2","0","https://x.com/undefined/status/2057129489994510620","1410466956134608899",
|
|
||||||
|
|
|
||||||
|
|
|
@ -1,7 +1,8 @@
|
||||||
"conversation_id_str","created_at","favorite_count","full_text","id_str","image_url","in_reply_to_screen_name","lang","location","quote_count","reply_count","retweet_count","tweet_url","user_id_str","username"
|
"conversation_id_str","created_at","favorite_count","full_text","id_str","image_url","in_reply_to_screen_name","lang","location","quote_count","reply_count","retweet_count","tweet_url","user_id_str","username"
|
||||||
"2057791279896080430","Fri May 22 11:50:47 +0000 2026","0","komdigi mengeluarkan kebijakan gratis ongkir @komdigi ???","2057791279896080430","","","in","","0","0","0","https://x.com/undefined/status/2057791279896080430","1814143201718247424",
|
"2058829602865451445","Mon May 25 08:36:43 +0000 2026","0","komdigi mending ngurusin aja tuh judol daripada ngurusin pembatasan gratis ongkir","2058829602865451445","","","in","","0","0","0","https://x.com/undefined/status/2058829602865451445","2046458899843170304",
|
||||||
"2057660616815571144","Fri May 22 03:11:35 +0000 2026","0","komdigi pembatasan gratis ongkir","2057660616815571144","","","in","","0","0","0","https://x.com/undefined/status/2057660616815571144","1814143201718247424",
|
"2058950111976386607","Mon May 25 16:35:34 +0000 2026","0","komdigi bagus sih ngeluarin kebijakan gratis ongkir kaya ini setuju","2058950111976386607","","","in","","0","0","0","https://x.com/undefined/status/2058950111976386607","2046458899843170304",
|
||||||
"2056635322419777630","Tue May 19 07:17:25 +0000 2026","0","komdigi ongkir gk jelas","2056635322419777630","","","in","","0","0","0","https://x.com/undefined/status/2056635322419777630","1814143201718247424",
|
"2058950559856832819","Mon May 25 16:37:21 +0000 2026","0","setuju bgt komdigi ngeluarin kebijakan gratis ongkir gini","2058950559856832819","","","in","","0","0","0","https://x.com/undefined/status/2058950559856832819","2046458899843170304",
|
||||||
"2056949159845101648","Wed May 20 04:04:30 +0000 2026","0","komdigi kereenn dan bijak membuat keputusan ongkir seperti inii","2056949159845101648","","","in","","0","0","0","https://x.com/undefined/status/2056949159845101648","1814143201718247424",
|
"2060234400777064942","Fri May 29 05:38:53 +0000 2026","0","tadi pagi saya melihat berita mengenai pembatasan gratis ongkir oleh komdigi","2060234400777064942","","","in","","0","0","0","https://x.com/undefined/status/2060234400777064942","1814143201718247424",
|
||||||
"2056892677149581606","Wed May 20 00:20:04 +0000 2026","0","komdigi keren sih mengadakan kebijakan ongkir kaya gini","2056892677149581606","","","in","","0","0","0","https://x.com/undefined/status/2056892677149581606","1814143201718247424",
|
"2058816188659192179","Mon May 25 07:43:24 +0000 2026","0","komdigi gajelas bikin kebijakan ongkir!","2058816188659192179","","","in","","0","0","0","https://x.com/undefined/status/2058816188659192179","1814143201718247424",
|
||||||
"2056538953340314095","Wed May 20 16:01:04 +0000 2026","4","@killingmaster27 @evelyndewi7 @katrinrienks @HumorJonTampan @wiyantokdr54 @EviDrajat @RickyKardjono @Micheladam69432 @ebonXboy @AntonRebor79105 @gangsterrandom @Pieter_Sun01 @Rahmah0845 @RSjahfidi @prabowo @SmileyNDeLight @Jediimar @Yens0906 @tokaidirecycle @PembuatSistem @bonapasogit24 @Jenniestale @Maskudin126003 @a8sar6alih @JowoNgalam @KING_RAJA_PWS_ @apaajadeh447870 @50ngoku @Submill @EmmaHusain0845 @BebySoSweet @Yulia_4ja @bungalotus25 @bahlillahadalia wahhh... jadi menteri komdigi dong menteri free ongkir https://t.co/1cfvoqIcNl","2057129489994510620","https://pbs.twimg.com/media/HIxkWwDaMAAXH9U.jpg","killingmaster27","in","","0","2","0","https://x.com/undefined/status/2057129489994510620","1410466956134608899",
|
"2059619299309011403","Wed May 27 12:54:41 +0000 2026","0","komdigi ngeluarin kebijakan ongkir kaya gini ngapain dah","2059619299309011403","","","in","","0","0","0","https://x.com/undefined/status/2059619299309011403","1814143201718247424",
|
||||||
|
"2058816239112446387","Mon May 25 07:43:37 +0000 2026","0","komdigi gk penting buat kebijakan ongkir kaya gini","2058816239112446387","","","in","","0","0","0","https://x.com/undefined/status/2058816239112446387","1814143201718247424",
|
||||||
|
|
|
||||||
|
|
|
@ -1,7 +1,8 @@
|
||||||
"conversation_id_str","created_at","favorite_count","full_text","id_str","image_url","in_reply_to_screen_name","lang","location","quote_count","reply_count","retweet_count","tweet_url","user_id_str","username"
|
"conversation_id_str","created_at","favorite_count","full_text","id_str","image_url","in_reply_to_screen_name","lang","location","quote_count","reply_count","retweet_count","tweet_url","user_id_str","username"
|
||||||
"2057660616815571144","Fri May 22 03:11:35 +0000 2026","0","komdigi pembatasan gratis ongkir","2057660616815571144","","","in","","0","0","0","https://x.com/undefined/status/2057660616815571144","1814143201718247424",
|
"2058829602865451445","Mon May 25 08:36:43 +0000 2026","0","komdigi mending ngurusin aja tuh judol daripada ngurusin pembatasan gratis ongkir","2058829602865451445","","","in","","0","0","0","https://x.com/undefined/status/2058829602865451445","2046458899843170304",
|
||||||
"2057791279896080430","Fri May 22 11:50:47 +0000 2026","0","komdigi mengeluarkan kebijakan gratis ongkir @komdigi ???","2057791279896080430","","","in","","0","0","0","https://x.com/undefined/status/2057791279896080430","1814143201718247424",
|
"2058950111976386607","Mon May 25 16:35:34 +0000 2026","0","komdigi bagus sih ngeluarin kebijakan gratis ongkir kaya ini setuju","2058950111976386607","","","in","","0","0","0","https://x.com/undefined/status/2058950111976386607","2046458899843170304",
|
||||||
"2056635322419777630","Tue May 19 07:17:25 +0000 2026","0","komdigi ongkir gk jelas","2056635322419777630","","","in","","0","0","0","https://x.com/undefined/status/2056635322419777630","1814143201718247424",
|
"2058950559856832819","Mon May 25 16:37:21 +0000 2026","0","setuju bgt komdigi ngeluarin kebijakan gratis ongkir gini","2058950559856832819","","","in","","0","0","0","https://x.com/undefined/status/2058950559856832819","2046458899843170304",
|
||||||
"2056949159845101648","Wed May 20 04:04:30 +0000 2026","0","komdigi kereenn dan bijak membuat keputusan ongkir seperti inii","2056949159845101648","","","in","","0","0","0","https://x.com/undefined/status/2056949159845101648","1814143201718247424",
|
"2060234400777064942","Fri May 29 05:38:53 +0000 2026","0","tadi pagi saya melihat berita mengenai pembatasan gratis ongkir oleh komdigi","2060234400777064942","","","in","","0","0","0","https://x.com/undefined/status/2060234400777064942","1814143201718247424",
|
||||||
"2056892677149581606","Wed May 20 00:20:04 +0000 2026","0","komdigi keren sih mengadakan kebijakan ongkir kaya gini","2056892677149581606","","","in","","0","0","0","https://x.com/undefined/status/2056892677149581606","1814143201718247424",
|
"2058816188659192179","Mon May 25 07:43:24 +0000 2026","0","komdigi gajelas bikin kebijakan ongkir!","2058816188659192179","","","in","","0","0","0","https://x.com/undefined/status/2058816188659192179","1814143201718247424",
|
||||||
"2056538953340314095","Wed May 20 16:01:04 +0000 2026","4","@killingmaster27 @evelyndewi7 @katrinrienks @HumorJonTampan @wiyantokdr54 @EviDrajat @RickyKardjono @Micheladam69432 @ebonXboy @AntonRebor79105 @gangsterrandom @Pieter_Sun01 @Rahmah0845 @RSjahfidi @prabowo @SmileyNDeLight @Jediimar @Yens0906 @tokaidirecycle @PembuatSistem @bonapasogit24 @Jenniestale @Maskudin126003 @a8sar6alih @JowoNgalam @KING_RAJA_PWS_ @apaajadeh447870 @50ngoku @Submill @EmmaHusain0845 @BebySoSweet @Yulia_4ja @bungalotus25 @bahlillahadalia wahhh... jadi menteri komdigi dong menteri free ongkir https://t.co/1cfvoqIcNl","2057129489994510620","https://pbs.twimg.com/media/HIxkWwDaMAAXH9U.jpg","killingmaster27","in","","0","2","0","https://x.com/undefined/status/2057129489994510620","1410466956134608899",
|
"2059619299309011403","Wed May 27 12:54:41 +0000 2026","0","komdigi ngeluarin kebijakan ongkir kaya gini ngapain dah","2059619299309011403","","","in","","0","0","0","https://x.com/undefined/status/2059619299309011403","1814143201718247424",
|
||||||
|
"2058816239112446387","Mon May 25 07:43:37 +0000 2026","0","komdigi gk penting buat kebijakan ongkir kaya gini","2058816239112446387","","","in","","0","0","0","https://x.com/undefined/status/2058816239112446387","1814143201718247424",
|
||||||
|
|
|
||||||
|
Loading…
Reference in New Issue