revisi model

indonesian-stopwords-complete.txt

@@ -1116,7 +1116,7 @@ yang
 selagi
 kebijakan
 pemerintah
-komdigi
+
 kementerian
 kemendag
 aturan

nrt_activation.json

@@ -1 +1 @@
-{"activated": false, "updated_at": "2026-05-22T16:01:11.643950+00:00"}
+{"activated": true, "updated_at": "2026-05-29T05:53:53.439233+00:00"}

page_modules/preprocessing_page.py

@@ -3,17 +3,24 @@ preprocessing_page.py
 =====================
 Halaman Bersihkan Data — NLP Pipeline 5 Tahap.
 
-PIPELINE 5 TAHAP (selaras dengan sentiment_service.py):
+PIPELINE 5 TAHAP:
   1. Case Folding     — ubah semua huruf jadi lowercase
   2. Cleaning         — hapus URL, mention, hashtag, angka, emoji, tanda baca
-  3. Normalisasi      — singkatan/slang → kata baku
+  3. Normalisasi      — singkatan/slang → kata baku (DARI FILE normalisasi)
-  4. Stopword Removal — hapus kata umum; JAGA kata sentimen penting
+  4. Stopword Removal — hapus kata umum (DARI FILE stopword); JAGA kata sentimen
   5. Stemming         — bentuk dasar kata via Sastrawi ECS
 
-Catatan: Tokenizing tidak ditampilkan sebagai tahap tersendiri karena:
+PERUBAHAN DARI VERSI SEBELUMNYA:
-  • Stopword removal & stemming sudah melakukan split() secara internal
+  - Normalisasi kini dimuat dari 'indonesian-normalisasi-slangword-complete.txt'
-  • TF-IDF melakukan tokenisasi sendiri saat inferensi
+    (1.700+ entri), menggantikan dict hardcoded yang hanya ~60 entri.
-  → Tokenizing adalah proses teknis, bukan tahap utama pipeline
+  - Stopword kini murni dari 'indonesian-stopwords-complete.txt', ditambah
+    noise Twitter yang spesifik — tidak ada penghapusan manual acak.
+  - KATA_SENTIMEN_PENTING diperluas dengan kata domain e-commerce/ongkir.
+  - Semua fungsi preprocessing menerima parameter eksplisit (tidak pakai global).
+
+CATATAN PENTING:
+  Pipeline ini HARUS IDENTIK dengan sentiment_service.py agar token yang
+  dihasilkan di sini konsisten dengan token saat training model.
 """
 
 import streamlit as st
@@ -33,18 +40,15 @@ from timezone_utils import (
     get_timezone_name,
 )
 
-
 # ═══════════════════════════════════════════════════════════
 #  TIMEZONE HELPERS
 # ═══════════════════════════════════════════════════════════
-
 def parse_dt(series):
     return parse_dt_with_tz(
         series,
         st.session_state.get("user_timezone", "WIB (UTC+7)")
     )
 
-
 def parse_crawled_dt(series):
     return parse_dt_with_source_tz(
         series,
@@ -52,7 +56,6 @@ def parse_crawled_dt(series):
         os.getenv("APP_TIMEZONE", "Asia/Jakarta")
     )
 
-
 def format_dt(value):
     if value is None or pd.isna(value):
         return "Belum ada"
@@ -96,63 +99,224 @@ def _sync_dynamic_period():
 
 # ═══════════════════════════════════════════════════════════
 #  PREPROCESSING PIPELINE — 5 TAHAP
-#  PENTING: Pipeline ini harus IDENTIK dengan sentiment_service.py
-#  agar token yang dihasilkan konsisten dengan training model.
 #
-#  URUTAN:
+#  ┌─────────────────────────────────────────────────────┐
-#    1. Case Folding     → lowercase dulu sebelum cleaning
+#  │  PENTING: Pipeline ini HARUS identik dengan         │
-#    2. Cleaning         → hapus noise setelah lowercase
+#  │  sentiment_service.py agar token konsisten!         │
-#    3. Normalisasi      → singkatan/slang → kata baku
+#  │                                                     │
-#    4. Stopword Removal → buang kata umum, jaga kata sentimen
+#  │  URUTAN WAJIB:                                      │
-#    5. Stemming         → bentuk dasar via Sastrawi ECS
+#  │  1. Case Folding  → lowercase dulu                  │
+#  │  2. Cleaning      → hapus noise setelah lowercase   │
+#  │  3. Normalisasi   → slang→baku setelah bersih       │
+#  │  4. Stopword      → buang kata umum, jaga sentimen  │
+#  │  5. Stemming      → bentuk dasar via Sastrawi ECS   │
+#  └─────────────────────────────────────────────────────┘
 # ═══════════════════════════════════════════════════════════
 
+
+# ───────────────────────────────────────────────────────────
+#  KATA SENTIMEN PENTING
+#  Kata-kata ini WAJIB DIJAGA dan tidak boleh dihapus saat
+#  stopword removal, meskipun ada di file stopword.
+#
+#  Kenapa perlu? Karena file stopword mengandung kata seperti
+#  "tidak", "belum", "sangat" yang justru krusial untuk
+#  menentukan sentimen positif/negatif suatu kalimat.
+# ───────────────────────────────────────────────────────────
 KATA_SENTIMEN_PENTING = {
-    # Negasi
+    # ── Negasi (pembalik makna kalimat) ──────────────────
+    # "tidak bagus" ≠ "bagus" → "tidak" wajib ada
     "tidak", "bukan", "jangan", "kurang", "belum", "tanpa",
-    # Positif
+    # ── Intensitas (penguat/pelemah sentimen) ─────────────
+    # "sangat bagus" lebih positif dari "bagus" saja
+    "sangat", "banget", "sekali", "paling", "amat",
+    "luar", "biasa",   # ← "luar biasa" = dua token, keduanya dijaga
+    # ── Positif umum ──────────────────────────────────────
     "keren", "bagus", "mantap", "setuju", "dukung", "mendukung",
     "andal", "handal", "gercep", "bangga", "senang", "suka",
-    "baik", "benar", "tepat", "oke",
+    "baik", "benar", "tepat", "oke", "puas",
     "sejahtera", "berkembang", "maju", "inovatif",
     "tegas", "sigap", "tanggap", "adil", "bijak", "bermanfaat",
     "untung", "berhasil", "sukses", "solusi", "manfaat",
     "berguna", "membantu", "bantu", "pro", "lanjut",
-    "sangat", "banget", "sekali", "paling", "amat", "luar", "biasa",
+    # ── Positif domain e-commerce / ongkir ────────────────
-    # Negatif
+    "gratis", "murah", "hemat", "terjangkau", "cepat",
+    "aman", "mudah", "praktis", "terpercaya",
+    # ── Negatif umum ──────────────────────────────────────
     "kecewa", "buruk", "jelek", "parah", "gagal", "hancur",
     "rusak", "bohong", "tipu", "korupsi",
-    # Emosi
+    # ── Negatif domain e-commerce / ongkir ────────────────
-    "marah", "sedih", "khawatir",
+    "mahal", "lambat", "lelet", "ribet", "susah", "repot",
+    "rugi", "boros",
+    # ── Emosi ─────────────────────────────────────────────
+    "marah", "sedih", "khawatir", "kecewa",
 }
 
 
-def _load_stopwords():
+# ───────────────────────────────────────────────────────────
+#  LOAD NORMALIZATION DARI FILE
+#  File: indonesian-normalisasi-slangword-complete.txt
+#  Format per baris: slang,kata_baku
+#  Contoh: gk,tidak | ongkir,ongkos kirim | free,gratis
+# ───────────────────────────────────────────────────────────
+def _load_normalization() -> dict:
+    """
+    Muat kamus normalisasi dari file eksternal.
+
+    KENAPA DARI FILE?
+    File berisi 1.700+ pasang slang→baku yang jauh lebih lengkap
+    dibanding dict hardcoded. Dengan ini, kata seperti:
+      gk/ga/gak/kagak/ngga → semua jadi "tidak"
+      bgt/bngt/bget        → semua jadi "sangat"
+      ongkir/ongkr         → jadi "ongkos kirim"
+    ...dan ribuan kasus lainnya tertangani otomatis.
+
+    Setelah file dimuat, override dengan entri khusus domain
+    (nama platform, singkatan kebijakan) yang mungkin belum ada
+    di file generik.
+    """
+    norm_file = "indonesian-normalisasi-slangword-complete.txt"
+    norm_dict: dict = {}
+
+    try:
+        with open(norm_file, "r", encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                # Split hanya pada koma pertama — nilai bisa mengandung koma
+                # Contoh: "on the way, sedang di jalan,dijalan" → split jadi 2 bagian
+                parts = line.split(",", 1)
+                if len(parts) != 2:
+                    continue
+                # Bersihkan tanda kutip liar di awal/akhir (ada di beberapa baris file)
+                slang  = parts[0].strip().strip("'\"").lower()
+                normal = parts[1].strip().lower()
+                if slang and normal:
+                    norm_dict[slang] = normal
+    except FileNotFoundError:
+        # Jika file tidak ditemukan, lanjut dengan dict kosong.
+        # Entri domain di bawah tetap akan ditambahkan.
+        st.warning(
+            "⚠️ File normalisasi tidak ditemukan: "
+            f"'{norm_file}'. Hanya entri domain yang aktif."
+        )
+
+    # ── Override khusus domain ───────────────────────────
+    # Entri ini menimpa file generik karena domain spesifik
+    # membutuhkan perlakuan khusus (nama platform tidak diubah,
+    # singkatan kebijakan punya padanan resmi, dll.)
+    DOMAIN_OVERRIDES: dict = {
+        # Nama platform — pertahankan apa adanya
+        "shopee":       "shopee",
+        "tokopedia":    "tokopedia",
+        "lazada":       "lazada",
+        "tiktok":       "tiktok",
+        "bukalapak":    "bukalapak",
+        "blibli":       "blibli",
+        # Logistik — pertahankan apa adanya
+        "sicepat":      "sicepat",
+        "jne":          "jne",
+        "jnt":          "jnt",
+        "anteraja":     "anteraja",
+        "ninja":        "ninja",
+        # Ongkir & belanja
+        "freeongkir":   "gratis ongkos kirim",
+        "gratisongkir": "gratis ongkos kirim",
+        "ongkir":       "ongkos kirim",
+        "ongkr":        "ongkos kirim",
+        "bykrm":        "biaya kirim",
+        "biayakirim":   "biaya pengiriman",
+        # Kebijakan & lembaga
+        "komdigi":      "komdigi",
+        "kemendag":     "kementerian perdagangan",
+        "kominfo":      "kementerian komunikasi",
+        # E-commerce umum
+        "ecommerce":    "e commerce",
+        "marketplace":  "marketplace",
+        "seller":       "penjual",
+        "buyer":        "pembeli",
+        "online":       "online",
+    }
+    norm_dict.update(DOMAIN_OVERRIDES)
+
+    return norm_dict
+
+
+# ───────────────────────────────────────────────────────────
+#  LOAD STOPWORDS DARI FILE
+#  File: indonesian-stopwords-complete.txt
+#  Format: satu kata per baris
+# ───────────────────────────────────────────────────────────
+def _load_stopwords() -> set:
+    """
+    Muat daftar stopword dari file eksternal.
+
+    PROSES SETELAH MUAT FILE:
+    1. Hapus KATA_SENTIMEN_PENTING dari daftar
+       → Agar "tidak", "belum", "sangat", dll. tidak ikut dibuang
+    2. Tambahkan noise Twitter/sosmed yang memang harus dibuang
+       → "rt", "amp", sisa URL, suara tawa, partikel informal
+
+    KENAPA DARI FILE?
+    File berisi 700+ stopword Indonesia yang lebih lengkap dan
+    terstandar dibanding daftar manual. Kita tidak perlu menambah/
+    mengurangi secara manual kecuali untuk dua kategori di atas.
+    """
     stopword_file = "indonesian-stopwords-complete.txt"
-    base = set()
+    base: set = set()
+
     try:
         with open(stopword_file, "r", encoding="utf-8") as f:
-            base = set(f.read().splitlines())
+            for line in f:
+                word = line.strip().lower()
+                if word:
+                    base.add(word)
     except FileNotFoundError:
+        # Fallback minimal — cukup untuk tetap jalan
+        st.warning(
+            "⚠️ File stopword tidak ditemukan: "
+            f"'{stopword_file}'. Menggunakan daftar minimal."
+        )
         base = {
             "yang", "dan", "di", "ke", "dari", "ini", "itu",
             "dengan", "untuk", "pada", "adalah", "oleh", "ada",
             "ya", "akan", "atau", "juga", "sama", "karena",
-            "jika", "sudah", "telah",
+            "jika", "sudah", "telah", "jadi", "bisa",
         }
+
+    # ── Langkah 1: Lindungi kata sentimen ────────────────
+    # Beberapa kata sentimen penting ADA di file stopword
+    # (misal: "tidak", "belum", "sangat", "paling", "kurang").
+    # Kita HAPUS dari stopword agar tidak ikut dibuang.
     for kata in KATA_SENTIMEN_PENTING:
         base.discard(kata)
-    base.update({
+
-        "rt", "amp", "https", "http", "co", "t",
+    # ── Langkah 2: Tambah noise Twitter/sosmed ────────────
-        "wkwk", "wkwkwk", "haha", "hehe", "xixi",
+    # Ini bukan stopword bahasa Indonesia biasa, tapi noise
-        "yg", "dgn", "utk", "dr", "krn", "tp", "jd", "sdh",
+    # yang sangat sering muncul di tweet dan tidak bermakna.
-        "aja", "doang", "nih", "sih", "dong", "deh",
+    TWITTER_NOISE: set = {
-        "loh", "lah", "tuh", "kak", "gan",
+        # Artefak Twitter
-    })
+        "rt", "amp",
+        # Sisa URL setelah cleaning (kadang lolos)
+        "https", "http", "co", "pic",
+        # Suara tawa (tidak bermakna untuk sentimen)
+        "wkwk", "wkwkwk", "wkwkwkwk",
+        "haha", "hahaha", "hehe", "hihi", "huhu", "xixi",
+        # Partikel informal yang tidak bermakna
+        "nih", "sih", "dong", "deh", "loh", "lah", "tuh",
+        "kak", "gan", "bro", "sob", "min",
+    }
+    base.update(TWITTER_NOISE)
+
     return base
 
 
+# ───────────────────────────────────────────────────────────
+#  LOAD STEMMER
+# ───────────────────────────────────────────────────────────
 def _load_stemmer():
+    """Muat stemmer Sastrawi. Return None jika tidak tersedia."""
     try:
         from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
         return StemmerFactory().create_stemmer()
@@ -160,130 +324,197 @@ def _load_stemmer():
         return None
 
 
-NORMALISASI = {
+# ═══════════════════════════════════════════════════════════
-    # Negasi
+#  FUNGSI 5 TAHAP PREPROCESSING
-    "gk": "tidak", "ga": "tidak", "gak": "tidak",
+#  Setiap fungsi bertanggung jawab SATU tahap saja.
-    "nggak": "tidak", "ngga": "tidak", "tdk": "tidak",
+#  Input & output setiap tahap dijelaskan di docstring.
-    "tak": "tidak", "enggak": "tidak", "engga": "tidak",
+# ═══════════════════════════════════════════════════════════
-    "kagak": "tidak", "kaga": "tidak", "ndak": "tidak",
-    "gkk": "tidak", "ngak": "tidak",
-    # Kata ganti
-    "yg": "yang", "dgn": "dengan", "utk": "untuk",
-    "org": "orang", "krn": "karena", "dr": "dari",
-    "sm": "sama", "pd": "pada", "dlm": "dalam",
-    "bwt": "buat", "trm": "terima",
-    # Verba
-    "tp": "tapi", "tpi": "tapi", "jd": "jadi",
-    "sdh": "sudah", "blm": "belum", "emg": "memang",
-    "emang": "memang", "gimana": "bagaimana",
-    "gitu": "begitu", "gini": "begini",
-    "udah": "sudah", "udh": "sudah",
-    "mau": "mau",
-    # Intensitas
-    "bgt": "banget", "bngt": "banget", "bget": "banget", "bgtt": "banget",
-    # Positif informal
-    "bener": "benar", "beneran": "benar",
-    "mantep": "mantap", "mntap": "mantap",
-    "kece": "keren", "kece bgt": "keren banget",
-    "cucok": "cocok", "cucuk": "cocok",
-    "cakep": "bagus", "oke bgt": "oke banget",
-    "sip": "baik", "siipp": "baik",
-    "top": "terbaik", "topbgt": "terbaik banget",
-    "jos": "bagus", "josss": "bagus",
-    "goks": "luar biasa",
-    "setujuu": "setuju", "stuju": "setuju",
-    "dukung": "dukung",
-    "proud": "bangga",
-    "mantul": "mantap betul",
-    # Negatif informal
-    "ancur": "hancur", "ancrr": "hancur",
-    "parahh": "parah", "parahhh": "parah",
-    "gagall": "gagal",
-    "ngaco": "tidak benar",
-    "ngasal": "tidak benar",
-    "receh": "tidak penting",
-    "gaje": "tidak jelas",
-    "asal": "sembarangan",
-    # Domain
-    "ongkir": "ongkos kirim",
-    "freeongkir": "gratis ongkos kirim",
-    "gratisongkir": "gratis ongkos kirim",
-    "free": "gratis",
-    "ecommerce": "e commerce",
-    "komdigi": "komdigi",
-    "subsidi": "subsidi",
-    "marketplace": "marketplace",
-    "seller": "penjual",
-    "buyer": "pembeli",
-    "online": "online",
-    "shopee": "shopee",
-    "tokopedia": "tokopedia",
-    "lazada": "lazada",
-    "tiktok": "tiktok",
-}
 
-
-# ── Tahap 1: Case Folding ───────────────────────────────────────────────────
 def step1_case_folding(text: str) -> str:
+    """
+    TAHAP 1 — CASE FOLDING
+    Input : teks asli (campuran huruf besar/kecil)
+    Output: semua huruf jadi lowercase
+
+    Kenapa pertama?
+    Agar tahap berikutnya (cleaning, normalisasi) bekerja
+    secara konsisten — regex dan dict lookup case-sensitive.
+    Contoh: "Gratis" → "gratis", "ONGKIR" → "ongkir"
+    """
     return str(text).lower()
 
 
-# ── Tahap 2: Cleaning ───────────────────────────────────────────────────────
 def step2_cleaning(text: str) -> str:
+    """
+    TAHAP 2 — CLEANING
+    Input : teks lowercase
+    Output: teks bersih dari semua elemen noise
+
+    Urutan pembersihan PENTING:
+    1. URL dulu (sebelum @ dan # agar tidak salah potong)
+    2. Mention (@username)
+    3. Hashtag (#topik)
+    4. Angka
+    5. Emoji & simbol unicode
+    6. Tanda baca
+    7. Karakter non-latin (huruf Arab, Cina, dll.)
+    8. Spasi berlebih
+    """
+    # 1. Hapus URL (http, https, www)
     text = re.sub(r"http\S+|www\S+|https\S+", "", text)
+    # 2. Hapus mention Twitter (@username)
     text = re.sub(r"@\w+", "", text)
+    # 3. Hapus hashtag (#topik)
     text = re.sub(r"#\w+", "", text)
+    # 4. Hapus angka dan digit
     text = re.sub(r"\d+", "", text)
+    # 5. Hapus emoji & simbol unicode (berbagai range)
     text = re.sub(
-        r"[\U00010000-\U0010ffff"
+        r"["
-        r"\U0001F600-\U0001F64F"
+        r"\U00010000-\U0010ffff"   # Suplemen karakter unicode
-        r"\U0001F300-\U0001F5FF"
+        r"\U0001F600-\U0001F64F"   # Emotikon wajah
-        r"\U0001F680-\U0001F6FF"
+        r"\U0001F300-\U0001F5FF"   # Simbol & piktogram
-        r"\U0001F1E0-\U0001F1FF"
+        r"\U0001F680-\U0001F6FF"   # Transport & peta
-        r"\u2600-\u26FF\u2700-\u27BF"
+        r"\U0001F1E0-\U0001F1FF"   # Bendera negara
-        r"]+", "", text, flags=re.UNICODE
+        r"\u2600-\u26FF"           # Simbol campuran
+        r"\u2700-\u27BF"           # Dingbats
+        r"]+",
+        "", text, flags=re.UNICODE
     )
+    # 6. Hapus tanda baca (.,!?;: dll.)
     text = text.translate(str.maketrans("", "", string.punctuation))
+    # 7. Hapus karakter non-latin (hanya sisakan huruf a-z dan spasi)
     text = re.sub(r"[^a-zA-Z\s]", "", text)
+    # 8. Normalisasi spasi berlebih → satu spasi, lalu strip
     text = re.sub(r"\s+", " ", text).strip()
     return text
 
 
-# ── Tahap 3: Normalisasi ────────────────────────────────────────────────────
+def step3_normalization(text: str, norm_dict: dict) -> str:
-def step3_normalization(text: str) -> str:
+    """
-    return " ".join(NORMALISASI.get(word, word) for word in text.split())
+    TAHAP 3 — NORMALISASI
+    Input : teks bersih (sudah case fold + cleaning)
+             norm_dict : kamus {slang: kata_baku} dari file
+    Output: teks dengan slang/singkatan sudah diganti kata baku
+
+    Cara kerja: token per token (word by word).
+    Setiap token dicari di norm_dict.
+    Jika ada → ganti. Jika tidak ada → biarkan.
+
+    Contoh:
+      "gk bs ongkir" → "tidak bisa ongkos kirim"
+      "mantep bgt"   → "mantap sangat"
+
+    Kenapa setelah Cleaning?
+    Karena slang di file ditulis dalam bentuk sudah lowercase
+    dan sudah tanpa tanda baca. Jika normalisasi dilakukan
+    sebelum cleaning, banyak entri tidak cocok.
+    """
+    tokens = text.split()
+    normalized = [norm_dict.get(token, token) for token in tokens]
+    return " ".join(normalized)
 
 
-# ── Tahap 4: Stopword Removal ───────────────────────────────────────────────
 def step4_stopword_removal(tokens: list, stopwords: set) -> list:
-    return [
+    """
-        w for w in tokens
+    TAHAP 4 — STOPWORD REMOVAL
-        if (w not in stopwords or w in KATA_SENTIMEN_PENTING) and len(w) > 2
+    Input : list token (hasil split dari teks ternormalisasi)
-    ]
+             stopwords : set kata yang harus dibuang (dari file)
+    Output: list token bersih
+
+    ATURAN PENYARINGAN (prioritas urutan):
+    1. JAGA token yang ada di KATA_SENTIMEN_PENTING
+       → meskipun juga ada di stopwords, tetap disimpan
+    2. BUANG token yang ada di stopwords
+    3. BUANG token dengan panjang ≤ 2 karakter
+       → menghilangkan sisa noise seperti "rt", "yg", "di"
+       → PENGECUALIAN: token di KATA_SENTIMEN_PENTING tetap disimpan
+         walau ≤ 2 karakter (contoh: "ok" jika masuk sentimen)
+
+    Kenapa setelah Normalisasi?
+    Agar "gak" yang sudah dinormalisasi jadi "tidak" tidak ikut
+    dibuang — "tidak" dilindungi di KATA_SENTIMEN_PENTING.
+    """
+    result = []
+    for token in tokens:
+        # Prioritas 1: selalu simpan jika kata sentimen penting
+        if token in KATA_SENTIMEN_PENTING:
+            result.append(token)
+            continue
+        # Prioritas 2: buang jika stopword
+        if token in stopwords:
+            continue
+        # Prioritas 3: buang jika terlalu pendek (noise)
+        if len(token) <= 2:
+            continue
+        # Lolos semua filter → simpan
+        result.append(token)
+    return result
 
 
-# ── Tahap 5: Stemming ───────────────────────────────────────────────────────
 def step5_stemming(tokens: list, stemmer) -> list:
+    """
+    TAHAP 5 — STEMMING
+    Input : list token setelah stopword removal
+             stemmer : objek Sastrawi (atau None)
+    Output: list token dalam bentuk kata dasar
+
+    Algoritma: Enhanced Confix Stripping (ECS) via Sastrawi
+    Contoh:
+      "pengiriman" → "kirim"
+      "pembatasan" → "batas"
+      "berlari"    → "lari"
+      "makanan"    → "makan"
+
+    Jika stemmer None (Sastrawi tidak terinstal), token dikembalikan
+    apa adanya tanpa error.
+    """
     if stemmer is None:
         return tokens
-    return [stemmer.stem(w) for w in tokens]
+    return [stemmer.stem(token) for token in tokens]
 
 
-def full_preprocessing(text: str, stopwords: set, stemmer):
+# ───────────────────────────────────────────────────────────
+#  FUNGSI UTAMA — JALANKAN SEMUA 5 TAHAP
+# ───────────────────────────────────────────────────────────
+def full_preprocessing(
+    text: str,
+    stopwords: set,
+    stemmer,
+    norm_dict: dict,
+) -> dict:
     """
-    Jalankan 5 tahap preprocessing dan kembalikan dict hasil setiap tahap.
+    Jalankan 5 tahap preprocessing secara berurutan dan kembalikan
+    hasil setiap tahap sebagai dict (untuk ditampilkan di tabel).
 
-    URUTAN TAHAP:
+    Parameter:
-      1. Case Folding     → lowercase
+      text      : teks tweet asli
-      2. Cleaning         → hapus noise
+      stopwords : set stopword (dari _load_stopwords)
-      3. Normalisasi      → normalisasi kata
+      stemmer   : objek Sastrawi (dari _load_stemmer)
-      4. Stopword Removal → buang stopword (split() dilakukan internal)
+      norm_dict : kamus normalisasi (dari _load_normalization)
-      5. Stemming         → bentuk dasar
+
+    Return dict berisi:
+      setelah_casefolding  : hasil Tahap 1
+      setelah_cleaning     : hasil Tahap 2
+      setelah_normalisasi  : hasil Tahap 3
+      setelah_stopword     : hasil Tahap 4 (joined string)
+      clean_text           : hasil akhir Tahap 5 (joined string)
+      _tokens_clean        : hasil Tahap 5 sebagai list (untuk analisis)
     """
+    # Tahap 1 — Case Folding
     s1_fold = step1_case_folding(text)
+
+    # Tahap 2 — Cleaning
     s2_clean = step2_cleaning(s1_fold)
-    s3_norm     = step3_normalization(s2_clean)
+
-    s4_filtered = step4_stopword_removal(s3_norm.split(), stopwords)
+    # Tahap 3 — Normalisasi (perlu norm_dict)
+    s3_norm = step3_normalization(s2_clean, norm_dict)
+
+    # Tahap 4 — Stopword Removal (split → filter → simpan sebagai list)
+    s4_tokens = s3_norm.split()
+    s4_filtered = step4_stopword_removal(s4_tokens, stopwords)
+
+    # Tahap 5 — Stemming
     s5_stemmed = step5_stemming(s4_filtered, stemmer)
 
     return {
@@ -292,7 +523,7 @@ def full_preprocessing(text: str, stopwords: set, stemmer):
         "setelah_normalisasi": s3_norm,
         "setelah_stopword":    " ".join(s4_filtered),
         "clean_text":          " ".join(s5_stemmed),
-        "_tokens_clean":       s5_stemmed,
+        "_tokens_clean":       s5_stemmed,   # list, untuk Counter frekuensi kata
     }
 
 
@@ -443,7 +674,7 @@ def _render_page_header():
 #  PIPELINE STEPS CARDS
 # ═══════════════════════════════════════════════════════════
 
-def _render_pipeline_steps(stemmer_ok):
+def _render_pipeline_steps(stemmer_ok: bool, norm_count: int, sw_count: int):
     steps = [
         {
             "num": "01", "anim": "pipe-1",
@@ -456,8 +687,8 @@ def _render_pipeline_steps(stemmer_ok):
                 '"ONGKIR" → "ongkir"',
                 '"KEREN" → "keren"',
                 "Seluruh karakter → huruf kecil",
-                "Dilakukan pertama agar cleaning konsisten",
+                "Dilakukan PERTAMA agar regex & dict konsisten",
-                "Basis untuk normalisasi & stopword",
+                "Fondasi seluruh tahap berikutnya",
             ],
         },
         {
@@ -479,30 +710,30 @@ def _render_pipeline_steps(stemmer_ok):
             "num": "03", "anim": "pipe-3",
             "icon": "🔄", "color": "#16a34a", "dark": "#14532d",
             "bg": "linear-gradient(135deg,#f0fdf4,#dcfce7)", "border": "#86efac",
-            "title": 'Normalisasi <span class="fix-badge">✦ Diperluas</span>',
+            "title": f'Normalisasi <span class="fix-badge">✦ {norm_count:,} entri</span>',
-            "desc": "Mengubah kata tidak baku, singkatan, dan slang menjadi kata baku.",
+            "desc": "Mengubah kata tidak baku, singkatan, dan slang menjadi kata baku (dari file).",
             "items": [
-                "gk/ga/gak/kagak → tidak",
+                "gk/ga/gak/kagak/ngga → tidak",
+                "bgt/bngt/bget → sangat",
                 "ongkir → ongkos kirim",
                 "mantep → mantap",
-                "kece → keren",
-                "jos/josss → bagus",
                 "free → gratis",
+                f"Total: {norm_count:,} pasang slang→baku dimuat",
             ],
         },
         {
             "num": "04", "anim": "pipe-4",
             "icon": "🚫", "color": "#ea580c", "dark": "#7c2d12",
             "bg": "linear-gradient(135deg,#fff7ed,#ffedd5)", "border": "#fed7aa",
-            "title": 'Stopword Removal <span class="fix-badge">✦ Diperbaiki</span>',
+            "title": f'Stopword Removal <span class="fix-badge">✦ {sw_count:,} kata</span>',
-            "desc": "Membuang kata umum; kata sentimen penting DIJAGA.",
+            "desc": "Membuang kata umum dari file; kata sentimen DIJAGA.",
             "items": [
-                "Hapus kata umum (dan, di, ke...)",
+                f"{sw_count:,} stopword dimuat dari file",
-                "Hapus token < 3 karakter",
+                "JAGA negasi: tidak, bukan, jangan, belum",
-                "JAGA negasi: tidak, bukan, jangan",
+                "JAGA positif: keren, bagus, mantap, gratis",
-                "JAGA positif: keren, bagus, mantap",
+                "JAGA negatif: kecewa, buruk, gagal, mahal, mending, malah",
-                "JAGA evaluatif: setuju, dukung, bijak",
+                "JAGA intensitas: sangat, banget, sekali",
-                "JAGA intensitas: banget, sangat, sekali",
+                "Hapus token ≤ 2 karakter (noise)",
             ],
         },
         {
@@ -678,7 +909,6 @@ def _render_live_example(df_c):
         "Contoh tweet acak dari dataset — refresh halaman untuk contoh berbeda"
     )
 
-    # Urutan tampilan sesuai pipeline: CF → Clean → Norm → Stop → Stem
     steps_ex = [
         ("📄 Teks Asli",            "text_asli",           "#0f172a", "#f8fafc",  "#e2e8f0"),
         ("① Setelah Case Folding",  "setelah_casefolding", "#0c4a6e", "#eff6ff",  "#bfdbfe"),
@@ -727,10 +957,14 @@ def _render_top_words_chart(df_c):
     )
 
     if "_tokens_clean" in df_c.columns:
-        all_words = [w for tokens in df_c["_tokens_clean"] for w in (tokens if isinstance(tokens, list) else [])]
+        all_words = [
+            w for tokens in df_c["_tokens_clean"]
+            for w in (tokens if isinstance(tokens, list) else [])
+        ]
     else:
         all_words = " ".join(df_c["clean_text"].fillna("")).split()
 
+    # Filter minimum 3 karakter (konsisten dengan step4)
     filtered_words = [w for w in all_words if len(w) > 2]
     word_freq = Counter(filtered_words).most_common(20)
 
@@ -821,19 +1055,29 @@ def show():
         unsafe_allow_html=True
     )
 
-    # ── Pipeline Overview ─────────────────────────────────────
+    # ── Muat resource preprocessing (sekali per session) ──────
+    # Semua tiga resource dimuat di sini, bukan di dalam loop,
+    # agar tidak memuat ulang setiap tweet.
+    stemmer   = _load_stemmer()
+    stopwords = _load_stopwords()
+    norm_dict = _load_normalization()
+
+    # ── Pipeline Overview ──────────────────────────────────────
     _section_header(
         "🔬 Alur NLP Pipeline — 5 Tahap Preprocessing",
         "Setiap tweet diproses berurutan melalui 5 tahap sebelum siap dianalisis sentimennya"
     )
 
-    stemmer_tmp = _load_stemmer()
+    _render_pipeline_steps(
-    _render_pipeline_steps(stemmer_tmp is not None)
+        stemmer_ok=stemmer is not None,
+        norm_count=len(norm_dict),
+        sw_count=len(stopwords),
+    )
     _gap("sm")
     _render_flow_arrow()
     _gap("md")
 
-    # ── Load data ─────────────────────────────────────────────
+    # ── Load data dari database ────────────────────────────────
     try:
         df_all = pd.read_sql("SELECT * FROM tweets ORDER BY created_at DESC", engine)
         if df_all.empty:
@@ -848,13 +1092,17 @@ def show():
 
     s_dt = pd.Timestamp(start_date)
     e_dt = pd.Timestamp(end_date)
-    df   = df_all[(df_all["created_at"] >= s_dt) & (df_all["created_at"] <= e_dt)].copy()
+    df   = df_all[
+        (df_all["created_at"] >= s_dt) & (df_all["created_at"] <= e_dt)
+    ].copy()
 
     if df.empty:
         st.warning(f"⚠️ Tidak ada tweet dengan tanggal asli dalam periode {filter_label}.")
         return
 
-    # ── Cache preprocessing ───────────────────────────────────
+    # ── Cache preprocessing ────────────────────────────────────
+    # Cache key: kombinasi mode + periode + jumlah tweet di DB
+    # Jika ada tweet baru → cache otomatis invalid → proses ulang
     total_tweets_in_db  = get_tweet_count()
     latest_crawl_marker = get_latest_crawl_time() or "no-crawl"
     data_marker         = (total_tweets_in_db, latest_crawl_marker)
@@ -864,6 +1112,7 @@ def show():
         f"{start_date}_{end_date}_{total_tweets_in_db}_{latest_crawl_marker}"
     )
 
+    # Bersihkan cache lama untuk mode/periode yang sudah tidak aktif
     for old_key in list(st.session_state.keys()):
         if old_key.startswith("pp5_") and old_key != cache_key:
             del st.session_state[old_key]
@@ -871,13 +1120,16 @@ def show():
     force_refresh = data_marker != st.session_state.get("_pp_last_data_marker")
 
     if cache_key not in st.session_state or force_refresh:
-        stemmer   = _load_stemmer()
-        stopwords = _load_stopwords()
-
         with st.spinner("🧹 Menjalankan 5 tahap preprocessing…"):
             results = []
             for _, row in df.iterrows():
-                r = full_preprocessing(row["text"], stopwords, stemmer)
+                # Jalankan 5 tahap — norm_dict diteruskan sebagai parameter
+                r = full_preprocessing(
+                    text      = row["text"],
+                    stopwords = stopwords,
+                    stemmer   = stemmer,
+                    norm_dict = norm_dict,
+                )
                 r["tweet_id"]   = row.get("tweet_id", "")
                 r["text_asli"]  = row["text"]
                 r["created_at"] = row["created_at"]
@@ -885,16 +1137,19 @@ def show():
                 results.append(r)
 
             df_c = pd.DataFrame(results)
+            # Buang baris yang clean_text-nya kosong setelah semua 5 tahap
             df_c = df_c[df_c["clean_text"].str.strip().str.len() > 0].copy()
+            # Reset index agar rapi
+            df_c = df_c.reset_index(drop=True)
 
             st.session_state[cache_key]              = df_c
-            st.session_state[cache_key + "_stemmer_ok"]  = stemmer is not None
+            st.session_state[cache_key + "_sw_ok"]   = stemmer is not None
             st.session_state["_pp_last_data_marker"] = data_marker
 
     df_c       = st.session_state[cache_key]
-    stemmer_ok = st.session_state.get(cache_key + "_stemmer_ok", False)
+    stemmer_ok = st.session_state.get(cache_key + "_sw_ok", False)
 
-    # ── Statistik ─────────────────────────────────────────────
+    # ── Statistik ──────────────────────────────────────────────
     removed = len(df) - len(df_c)
 
     _section_header(
@@ -908,7 +1163,7 @@ def show():
     _render_live_example(df_c)
     _gap("lg")
 
-    # ── Tabel ─────────────────────────────────────────────────
+    # ── Tabel ──────────────────────────────────────────────────
     _section_header(
         "📋 Tabel Perbandingan Teks per Tahap",
         f"{len(df_c):,} tweet · {filter_label} — scroll horizontal untuk lihat semua kolom"
@@ -918,7 +1173,7 @@ def show():
         df_c = df_c.copy()
         df_c["crawled_at"] = pd.NaT
 
-    # Kolom ditampilkan sesuai urutan pipeline: CF → Clean → Norm → Stop → Stem
+    # Kolom ditampilkan sesuai urutan pipeline
     disp = df_c[[
         "tweet_id", "created_at", "crawled_at",
         "text_asli",
@@ -968,10 +1223,10 @@ def show():
     _render_top_words_chart(df_c)
     _gap("lg")
 
-    # ── Simpan ke session state ───────────────────────────────
+    # ── Simpan ke session state untuk halaman sentimen ─────────
     st.session_state["preprocessed_df"] = df_c
 
-    # ── Download ──────────────────────────────────────────────
+    # ── Download ───────────────────────────────────────────────
     st.markdown("""
 <div style="background:#ffffff;border:1.5px solid #e2e8f0;border-radius:14px;
             padding:1.1rem 1.2rem;box-shadow:0 2px 6px rgba(15,23,42,0.05);

page_modules/sentiment_page.py

@@ -1,29 +1,8 @@
 """
 sentiment_page.py
 =================
-Halaman Analisis Sentimen — diperbaiki untuk mengatasi semua prediksi negatif.
+PERBAIKAN: Hapus parameter key= dari semua st.container() karena tidak
-
+didukung di Streamlit versi lama. CSS styling tetap berjalan via class HTML.
-PERBAIKAN vs VERSI LAMA:
-─────────────────────────────────────────────────────────────────────────────
-1. MENGGUNAKAN sentiment_service.py (HYBRID CLASSIFIER)
-   Tidak lagi memanggil model secara langsung dengan predict_batch sederhana.
-   Sebaliknya menggunakan sentiment_service yang sudah mengimplementasikan:
-   • Lexicon-based override untuk kelas POSITIF (yang tidak ada di model)
-   • Negation handling (tidak bagus ≠ bagus)
-   • Fallback ke model NB untuk negatif vs netral
-
-2. PREPROCESSING SELARAS
-   Memanggil preprocess_for_model() dari sentiment_service agar preprocessing
-   yang digunakan untuk prediksi persis sama dengan yang dipakai saat training.
-
-3. CONFIDENCE SCORE AKURAT
-   Menggunakan confidence gabungan dari lexicon score + model probability,
-   bukan hanya max probability dari model yang bias ke negatif.
-
-4. LABEL MAPPING KONSISTEN
-   Label dikembalikan selalu dalam format: 'Positif', 'Netral', 'Negatif'
-   (kapital huruf pertama) agar konsisten di seluruh tampilan.
-─────────────────────────────────────────────────────────────────────────────
 """
 
 import streamlit as st
@@ -44,7 +23,6 @@ from timezone_utils import (
 )
 import plotly.graph_objects as go
 
-# ── Import sentiment_service (hybrid classifier) ──────────────────────────
 from sentiment_service import (
     preprocess_for_model,
     preprocess_untuk_lexicon,
@@ -118,40 +96,27 @@ def _sync_dynamic_period():
 
 
 # ─────────────────────────────────────────────────────────────
-# Core prediction function (DIPERBAIKI)
+# Core prediction
 # ─────────────────────────────────────────────────────────────
 
-def predict_batch_hybrid(texts: list[str]) -> list[tuple[str, float]]:
+def predict_batch_hybrid(texts):
-    """
-    Prediksi sentimen menggunakan Hybrid Classifier dari sentiment_service.
-
-    Pipeline per tweet:
-    1. preprocess_for_model()   → teks untuk TF-IDF + NB
-    2. preprocess_untuk_lexicon() → teks untuk pengecekan lexicon
-    3. _hitung_skor_lexicon()   → hitung sinyal positif/negatif kuat
-    4. _klasifikasi_hybrid()    → putuskan label + confidence
-
-    Return: list of (label, confidence)
-    """
     results = []
     for text in texts:
         teks_model   = preprocess_for_model(text)
         teks_lexicon = preprocess_untuk_lexicon(text)
         teks_lower   = str(text).lower()
-
         skor         = _hitung_skor_lexicon(teks_lexicon)
         label, conf  = _klasifikasi_hybrid(teks_model, skor, teks_lower)
         results.append((label, conf))
     return results
 
 
-def preprocess_single(text: str) -> str:
+def preprocess_single(text):
-    """Preprocess satu teks untuk disimpan ke kolom clean_text."""
     return preprocess_for_model(text)
 
 
 # ─────────────────────────────────────────────────────────────
-# Shared UI helpers
+# UI helpers
 # ─────────────────────────────────────────────────────────────
 
 def _section_header(title, subtitle=""):
@@ -173,6 +138,20 @@ def _section_gap(size="md"):
     st.markdown(f'<div style="height:{heights.get(size,"1.45rem")};"></div>', unsafe_allow_html=True)
 
 
+def _card_open(extra_style=""):
+    """Buka div card pengganti st.container(border=True)."""
+    st.markdown(
+        f'<div style="background:#ffffff;border:1.5px solid #e2e8f0;border-radius:12px;'
+        f'box-shadow:0 2px 6px rgba(15,23,42,0.06);padding:1.1rem 1.2rem 1rem;'
+        f'margin-bottom:0.5rem;{extra_style}">',
+        unsafe_allow_html=True,
+    )
+
+
+def _card_close():
+    st.markdown('</div>', unsafe_allow_html=True)
+
+
 def _render_sentiment_styles():
     st.markdown("""
 <style>
@@ -185,31 +164,6 @@ section[data-testid="stMain"] * {
 [data-testid="stMainBlockContainer"] { padding-top: 1rem !important; }
 header[data-testid="stHeader"] { height: 0 !important; min-height: 0 !important; }
 
-.st-key-sentiment_keyword_panel,
-.st-key-sentiment_table_controls,
-.st-key-sentiment_download_panel,
-.st-key-sentiment_chart_panel,
-.st-key-sentiment_trend_panel,
-.st-key-sentiment_wordfreq_panel {
-    background: #ffffff !important;
-    border: 1.5px solid #e2e8f0 !important;
-    border-radius: 12px !important;
-    box-shadow: 0 2px 6px rgba(15,23,42,0.06) !important;
-    padding: 1.1rem 1.2rem 1rem !important;
-}
-
-.st-key-sentiment_keyword_panel [data-testid="stTextInput"] label p,
-.st-key-sentiment_table_controls [data-testid="stTextInput"] label p,
-.st-key-sentiment_table_controls [data-testid="stSelectbox"] label p {
-    color: #334155 !important;
-    font-size: 0.72rem !important;
-    font-weight: 800 !important;
-    letter-spacing: 0.04em !important;
-    line-height: 1.2 !important;
-    margin-bottom: 0.22rem !important;
-    text-transform: uppercase !important;
-}
-
 .stButton > button {
     border-radius: 12px !important;
     font-weight: 700 !important;
@@ -419,7 +373,7 @@ def _render_donut_chart(pos_n, neu_n, neg_n, total, filter_label):
         legend=dict(orientation="h", y=-0.1, x=0.5, xanchor="center", font=dict(size=11)),
         paper_bgcolor="rgba(0,0,0,0)",
     )
-    st.plotly_chart(fig, width="stretch", config={"displayModeBar": False})
+    st.plotly_chart(fig, use_container_width=True, config={"displayModeBar": False})
     return dominant
 
 
@@ -446,7 +400,7 @@ def _render_bar_chart(pos_n, neu_n, neg_n, total):
         yaxis=dict(showgrid=True, gridcolor="rgba(226,232,240,0.8)", griddash="dot",
                    tickfont=dict(size=10, color="#94a3b8")),
     )
-    st.plotly_chart(fig, width="stretch", config={"displayModeBar": False})
+    st.plotly_chart(fig, use_container_width=True, config={"displayModeBar": False})
 
 
 # ─────────────────────────────────────────────────────────────
@@ -496,49 +450,9 @@ def _render_trend_chart(fdf, filter_label, dt_start, dt_end):
                    tickfont=dict(size=10, color="#94a3b8")),
         hovermode="x unified",
     )
-    with st.container(border=True, key="sentiment_trend_panel"):
+    # ─── PERBAIKAN: hapus key= dari st.container() ───
-        st.plotly_chart(fig, width="stretch", config={"displayModeBar": False})
+    with st.container():
-
+        st.plotly_chart(fig, use_container_width=True, config={"displayModeBar": False})
-
-# ─────────────────────────────────────────────────────────────
-# Confidence histogram
-# ─────────────────────────────────────────────────────────────
-
-# def _render_confidence_chart(fdf):
-#     _section_header(
-#         "🎯 Distribusi Keyakinan Model",
-#         "Seberapa yakin classifier dalam mengklasifikasikan setiap tweet"
-#     )
-
-#     fig = go.Figure()
-#     sent_cfg = [
-#         ("Positif", "#16a34a", "rgba(22,163,74,0.7)"),
-#         ("Netral",  "#94a3b8", "rgba(148,163,184,0.7)"),
-#         ("Negatif", "#ef4444", "rgba(239,68,68,0.7)"),
-#     ]
-#     for sent, color, fill_color in sent_cfg:
-#         sub = fdf[fdf["sentiment"] == sent]["confidence"]
-#         if sub.empty:
-#             continue
-#         fig.add_trace(go.Histogram(
-#             x=sub, name=sent, nbinsx=20,
-#             marker=dict(color=fill_color, line=dict(color=color, width=1)),
-#             opacity=0.85,
-#             hovertemplate=f"<b>{sent}</b><br>Keyakinan: %{{x:.0%}}<br>Jumlah: %{{y}}<extra></extra>",
-#         ))
-#     fig.update_layout(
-#         height=260, margin=dict(l=0, r=0, t=10, b=0), barmode="overlay",
-#         paper_bgcolor="rgba(0,0,0,0)", plot_bgcolor="rgba(0,0,0,0)",
-#         legend=dict(orientation="h", y=1.1, x=0, font=dict(size=11)),
-#         xaxis=dict(tickformat=".0%", title="Tingkat Keyakinan",
-#                    tickfont=dict(size=10, color="#94a3b8"),
-#                    showgrid=True, gridcolor="rgba(226,232,240,0.6)"),
-#         yaxis=dict(title="Jumlah Tweet",
-#                    tickfont=dict(size=10, color="#94a3b8"),
-#                    showgrid=True, gridcolor="rgba(226,232,240,0.6)", griddash="dot"),
-#     )
-#     with st.container(border=True, key="sentiment_conf_panel"):
-#         st.plotly_chart(fig, width="stretch", config={"displayModeBar": False})
 
 
 # ─────────────────────────────────────────────────────────────
@@ -609,7 +523,7 @@ def _render_word_freq_per_sentiment(fdf):
                            tickfont=dict(size=8, color="#94a3b8"), fixedrange=True),
                 yaxis=dict(showgrid=False, tickfont=dict(size=9, color="#334155"), fixedrange=True),
             )
-            st.plotly_chart(fig, width="stretch", config={"displayModeBar": False})
+            st.plotly_chart(fig, use_container_width=True, config={"displayModeBar": False})
             st.markdown('</div>', unsafe_allow_html=True)
 
 
@@ -671,11 +585,11 @@ def _render_tweet_table(fdf, filter_label):
         f"Total {len(fdf):,} tweet · {filter_label}"
     )
 
-    with st.container(border=True, key="sentiment_table_controls"):
+    # ─── PERBAIKAN: ganti st.container(key=) dengan st.columns biasa ───
-        tf1, tf2, tf3 = st.columns([3, 1.6, 1.7], gap="medium", vertical_alignment="bottom")
+    tf1, tf2, tf3 = st.columns([3, 1.6, 1.7], gap="medium")
     with tf1:
         search = st.text_input(
-                "Search tweet", placeholder="Ketik kata kunci di isi tweet...",
+            "🔍 Cari tweet", placeholder="Ketik kata kunci di isi tweet...",
             key="sentiment_table_search",
         )
     with tf2:
@@ -711,26 +625,23 @@ def _render_tweet_table(fdf, filter_label):
     else:
         tdf = tdf.sort_values("confidence", ascending=False)
 
-    # out = tdf[["created_at", "crawled_at", "text", "clean_text", "sentiment", "confidence"]].copy()
     out = tdf[["created_at", "crawled_at", "text", "clean_text", "sentiment"]].copy()
     out["created_at"]  = out["created_at"].apply(format_dt)
     out["crawled_at"]  = out["crawled_at"].apply(format_dt)
-    # out["confidence"]  = out["confidence"].apply(lambda x: f"{x:.0%}")
     out["sentiment"]   = out["sentiment"].map({
         "Positif": "😊 Positif", "Netral": "😐 Netral", "Negatif": "😞 Negatif",
     }).fillna(out["sentiment"])
-    # out.columns = ["Tanggal Tweet", "Masuk Database", "Tweet Asli", "Tweet Bersih", "Sentimen", "Keyakinan"]
     out.columns = ["Tanggal Tweet", "Masuk Database", "Tweet Asli", "Tweet Bersih", "Sentimen"]
 
     render_standard_table(
         out, height=400, min_width=1220,
         badge_columns=["Sentimen"],
-        nowrap=["Tanggal Tweet", "Masuk Database", "Sentimen", "Keyakinan"],
+        nowrap=["Tanggal Tweet", "Masuk Database", "Sentimen"],
         wide_columns=["Tweet Asli", "Tweet Bersih"],
         column_widths={
             "Tanggal Tweet": "170px", "Masuk Database": "170px",
             "Tweet Asli": "360px",   "Tweet Bersih": "360px",
-            "Sentimen": "130px",     "Keyakinan": "110px",
+            "Sentimen": "130px",
         },
     )
     st.caption(f"Menampilkan {len(tdf):,} tweet")
@@ -927,11 +838,9 @@ def show():
         if force_refresh:
             st.info("🔄 Menyegarkan prediksi sentimen dengan data terbaru...")
         with st.spinner("🔍 Preprocessing & prediksi sentimen (Hybrid Classifier)..."):
-            # Preprocess semua tweet
             df["clean_text"] = df["text"].apply(preprocess_single)
             dfc = df[df["clean_text"].str.strip().str.len() > 0].copy()
 
-            # Prediksi menggunakan hybrid classifier
             results = predict_batch_hybrid(dfc["text"].tolist())
             if results:
                 sentiments, confidences = zip(*results)
@@ -949,7 +858,8 @@ def show():
     # ── Keyword filter ────────────────────────────────────────
     _section_header("🎯 Filter Kata Kunci", "Kosongkan untuk melihat semua tweet")
 
-    with st.container(border=True, key="sentiment_keyword_panel"):
+    # ─── PERBAIKAN: hapus key= dari st.container() ───
+    with st.container():
         kw = st.text_input(
             "Kata kunci", placeholder="Contoh: ongkir, kurir, komdigi...",
             key="sentiment_keyword_search",
@@ -1000,11 +910,12 @@ def show():
     col_left, col_right = st.columns(2, gap="medium")
     with col_left:
         _section_header("🔵 Sebaran Sentimen", f"Periode {filter_label}")
-        with st.container(border=True, key="sentiment_chart_panel"):
+        # ─── PERBAIKAN: hapus key= dari st.container() ───
+        with st.container():
             dominant = _render_donut_chart(pos_n, neu_n, neg_n, total, filter_label)
     with col_right:
         _section_header("📊 Perbandingan Jumlah per Sentimen", f"Periode {filter_label}")
-        with st.container(border=True, key="sentiment_bar_panel"):
+        with st.container():
             _render_bar_chart(pos_n, neu_n, neg_n, total)
     _section_gap("lg")
 
@@ -1012,10 +923,6 @@ def show():
     _render_trend_chart(fdf, filter_label, start_date, end_date)
     _section_gap("lg")
 
-    # ── Confidence ────────────────────────────────────────────
-    # _render_confidence_chart(fdf)
-    # _section_gap("lg")
-
     # ── Word freq ─────────────────────────────────────────────
     _render_word_freq_per_sentiment(fdf)
     _section_gap("lg")
@@ -1035,15 +942,17 @@ def show():
     # ── Download ──────────────────────────────────────────────
     _section_header("📥 Unduh Hasil Analisis")
 
-    with st.container(border=True, key="sentiment_download_panel"):
+    # ─── PERBAIKAN: hapus key= dari st.container() ───
-        d1, d2, d3 = st.columns(3, gap="medium", vertical_alignment="bottom")
+    with st.container():
+        d1, d2, d3 = st.columns(3, gap="medium")
 
         with d1:
             st.download_button(
                 "📥 Semua Hasil Prediksi",
                 fdf.to_csv(index=False).encode("utf-8"),
                 f"hasil_prediksi_{datetime.now().strftime('%Y%m%d_%H%M')}.csv",
-                "text/csv", width="stretch",
+                "text/csv",
+                use_container_width=True,
             )
         with d2:
             summary = pd.DataFrame({
@@ -1055,12 +964,14 @@ def show():
                 "📈 Ringkasan Sentimen",
                 summary.to_csv(index=False).encode("utf-8"),
                 f"ringkasan_{datetime.now().strftime('%Y%m%d_%H%M')}.csv",
-                "text/csv", width="stretch",
+                "text/csv",
+                use_container_width=True,
             )
         with d3:
             st.download_button(
                 "🎯 Rekomendasi Tindakan",
                 df_rek.to_csv(index=False).encode("utf-8"),
                 f"rekomendasi_{datetime.now().strftime('%Y%m%d_%H%M')}.csv",
-                "text/csv", width="stretch",
+                "text/csv",
+                use_container_width=True,
             )

sentiment_service.py

@@ -1,25 +1,15 @@
 """
-sentiment_service.py
+sentiment_service.py  —  Hybrid Classifier (versi perbaikan)
-====================
+=============================================================
-Hybrid Classifier untuk analisis sentimen tweet Bahasa Indonesia.
+Perbaikan utama vs versi sebelumnya:
-
+  1. LEXICON DIPERLUAS  — kata positif & negatif yang tidak ada di vocab TF-IDF
-PIPELINE PREPROCESSING — 5 TAHAP (selaras dengan preprocessing_page.py):
+     (bagus, mantap, setuju, puas, hemat, berhasil, dll.) kini tetap bisa
-  1. Case Folding     → lowercase dulu sebelum cleaning
+     terdeteksi melalui lexicon scoring.
-  2. Cleaning         → hapus URL, mention, hashtag, angka, emoji, tanda baca
+  2. THRESHOLD DISESUAIKAN — Layer 1 diperlonggar (net >= 2 → Positif),
-  3. Normalisasi      → singkatan/slang → kata baku
+     Layer anti-bias diperketat agar prediksi lebih proporsional.
-  4. Stopword Removal → buang kata umum, jaga kata sentimen penting
+  3. NEGATION WINDOW DIPERLUAS — window 3 kata (sebelumnya 2) agar
-  5. Stemming         → bentuk dasar via Sastrawi ECS
+     "tidak terlalu bagus" tetap terdeteksi negasinya.
-
+  4. PREPROCESSING IDENTIK dengan preprocessing_page.py (5 tahap).
-Catatan: Tokenizing tidak menjadi tahap tersendiri karena:
-  • Stopword removal & stemming sudah melakukan split() secara internal
-  • TF-IDF melakukan tokenisasi sendiri saat inferensi
-
-ARSITEKTUR HYBRID:
-  Teks Asli
-    ├── preprocess_untuk_lexicon() ──→ _hitung_skor_lexicon() ──┐
-    └── preprocess_for_model()     ──→ TF-IDF → NB Model ───────┘
-                                                                  ↓
-                                              _klasifikasi_hybrid() → Label + Confidence
 """
 
 import re
@@ -28,93 +18,44 @@ import joblib
 
 
 # ═══════════════════════════════════════════════════════════
-#  KATA SENTIMEN PENTING
+#  KATA SENTIMEN PENTING  (dijaga dari stopword removal)
-#  Tidak boleh dihapus di tahap stopword removal
 # ═══════════════════════════════════════════════════════════
 
 KATA_SENTIMEN_PENTING = {
-    # ── Negasi ──────────────────────────────────────────────
+    # Negasi
     "tidak", "bukan", "jangan", "kurang", "belum", "tanpa",
-
+    # Intensitas
-    # ── Sentimen POSITIF ────────────────────────────────────
+    "sangat", "banget", "sekali", "paling", "amat", "luar", "biasa",
+    # Positif umum
     "keren", "bagus", "mantap", "setuju", "dukung", "mendukung",
     "andal", "handal", "gercep", "bangga", "senang", "suka",
-    "baik", "benar", "tepat", "oke",
+    "baik", "benar", "tepat", "oke", "puas",
     "sejahtera", "berkembang", "maju", "inovatif",
     "tegas", "sigap", "tanggap", "adil", "bijak", "bermanfaat",
     "untung", "berhasil", "sukses", "solusi", "manfaat",
     "berguna", "membantu", "bantu", "pro", "lanjut",
-    "sangat", "banget", "sekali", "paling", "amat", "luar", "biasa",
+    # Positif domain e-commerce/ongkir
-
+    "gratis", "murah", "hemat", "terjangkau", "cepat",
-    # ── Sentimen NEGATIF evaluatif ──────────────────────────
+    "aman", "mudah", "praktis", "terpercaya",
-    "kecewa", "buruk", "jelek", "parah", "gagal", "hancur",
+    # Negatif umum
-    "rusak", "bohong", "tipu", "korupsi",
+    "kecewa", "mending", "malah", "buruk", "jelek", "parah",
-
+    "gagal", "hancur", "rusak", "bohong", "tipu", "korupsi",
-    # ── Emosi ───────────────────────────────────────────────
+    # Negatif domain e-commerce/ongkir
+    "mahal", "lambat", "lelet", "ribet", "susah", "repot",
+    "rugi", "boros",
+    # Emosi
     "marah", "sedih", "khawatir",
 }
 
 
-# ═══════════════════════════════════════════════════════════
-#  NORMALISASI (selaras dengan preprocessing_page.py)
-# ═══════════════════════════════════════════════════════════
-
-NORMALISASI = {
-    # Negasi
-    "gk": "tidak", "ga": "tidak", "gak": "tidak",
-    "nggak": "tidak", "ngga": "tidak", "tdk": "tidak",
-    "tak": "tidak", "enggak": "tidak", "engga": "tidak",
-    "kagak": "tidak", "kaga": "tidak", "ndak": "tidak",
-    "gkk": "tidak", "ngak": "tidak",
-    # Kata ganti
-    "yg": "yang", "dgn": "dengan", "utk": "untuk",
-    "org": "orang", "krn": "karena", "dr": "dari",
-    "sm": "sama", "pd": "pada", "dlm": "dalam",
-    "bwt": "buat", "trm": "terima",
-    # Verba
-    "tp": "tapi", "tpi": "tapi", "jd": "jadi",
-    "sdh": "sudah", "blm": "belum", "emg": "memang",
-    "emang": "memang", "gimana": "bagaimana",
-    "gitu": "begitu", "gini": "begini",
-    "udah": "sudah", "udh": "sudah",
-    # Intensitas
-    "bgt": "banget", "bngt": "banget", "bget": "banget", "bgtt": "banget",
-    # Positif informal
-    "bener": "benar", "beneran": "benar",
-    "mantep": "mantap", "mntap": "mantap",
-    "kece": "keren",
-    "cucok": "cocok", "cucuk": "cocok",
-    "cakep": "bagus",
-    "sip": "baik", "siipp": "baik",
-    "top": "terbaik",
-    "jos": "bagus", "josss": "bagus",
-    "goks": "luar biasa",
-    "setujuu": "setuju", "stuju": "setuju",
-    "proud": "bangga",
-    "mantul": "mantap betul",
-    # Negatif informal
-    "ancur": "hancur", "ancrr": "hancur",
-    "parahh": "parah", "parahhh": "parah",
-    "gagall": "gagal",
-    "ngaco": "tidak benar",
-    "ngasal": "tidak benar",
-    "gaje": "tidak jelas",
-    # Domain
-    "ongkir": "ongkos kirim",
-    "freeongkir": "gratis ongkos kirim",
-    "gratisongkir": "gratis ongkos kirim",
-    "free": "gratis",
-    "ecommerce": "e commerce",
-    "seller": "penjual",
-    "buyer": "pembeli",
-}
-
-
 # ═══════════════════════════════════════════════════════════
 #  LEXICON SENTIMEN
+#  Diperluas agar kata yang tidak ada di vocab TF-IDF tetap
+#  bisa berkontribusi melalui jalur lexicon scoring.
 # ═══════════════════════════════════════════════════════════
 
 LEXICON_POSITIF = {
+    # Umum & informal
     "bagus", "baik", "keren", "mantap", "mantep", "hebat",
     "oke", "sip", "top", "jos", "goks", "kece", "mantul",
     "setuju", "dukung", "mendukung", "pro", "lanjut", "sepakat",
@@ -127,71 +68,147 @@ LEXICON_POSITIF = {
     "untung", "gratis", "murah", "hemat", "terjangkau",
     "terbaik", "luar biasa",
     "cakep", "cucok", "gaskeun", "kuy",
-    "dukung", "bantu", "solusi", "manfaat",
+    "solusi", "manfaat", "menguntungkan",
     "memuaskan", "membanggakan", "mengagumkan",
+    "sehat", "fair", "wajar",
+    "syukur", "alhamdulillah",
+    "saing", "kompetitif",
+    # Domain ongkir/ecommerce positif
+    "terjangkau", "hemat", "efisien", "mudah", "praktis",
+    "cepat", "aman", "terpercaya", "andalan",
+    # Dukungan kebijakan
+    "dukung", "setuju", "bagus", "tepat", "bijak",
+    "perlu", "penting", "benar", "wajar", "adil",
 }
 
 LEXICON_NEGATIF = {
+    # Umum
     "buruk", "jelek", "parah", "rusak", "hancur", "ancur",
-    "gagal", "gagall", "ambruk", "terpuruk", "bangkrut",
+    "gagal", "ambruk", "terpuruk", "bangkrut",
-    "bohong", "tipu", "curang", "manipulasi", "korupsi", "penipuan",
+    "bohong", "tipu", "curang", "manipulasi", "korupsi",
-    "kebohongan",
+    "kebohongan", "penipuan",
-    "kecewa", "marah", "sedih", "khawatir", "takut", "benci",
+    "kecewa", "mending", "malah", "marah", "sedih",
-    "jijik", "muak", "kesal", "frustrasi",
+    "khawatir", "takut", "benci", "jijik", "muak",
-    "mahal", "rugi", "merugikan",
+    "kesal", "frustrasi", "geram", "dongkol",
-    "lambat", "lemot", "ribet", "susah", "sulit", "bermasalah",
+    "mahal", "rugi", "merugikan", "rugikan",
+    "lambat", "lemot", "lelet", "ribet", "susah",
+    "sulit", "bermasalah",
     "ngaco", "ngasal", "gaje", "receh",
-    "tidak benar", "tidak jelas", "tidak adil", "tidak berguna",
+    "tolol", "bodoh", "idiot", "goblok",
     "mengecewakan", "menyebalkan", "menyusahkan",
+    "monopoli", "licik",
+    # Domain ongkir negatif
+    "boros", "memberatkan", "menyulitkan",
+    "repot", "ribet", "ngeributin", "ribut",
+    # Kritik kebijakan
+    "salah", "keliru", "gegabah", "sembarangan",
+    "tidak jelas", "ngawur", "asal",
 }
 
 KATA_NEGASI = {
     "tidak", "bukan", "jangan", "belum", "tanpa", "kurang",
-    "anti", "non",
+    "anti", "non", "tak", "ga", "gak", "nggak", "ngga",
+    "enggak", "engga",
 }
 
 
 # ═══════════════════════════════════════════════════════════
-#  STOPWORDS
+#  LOAD NORMALISASI DARI FILE
+# ═══════════════════════════════════════════════════════════
+
+def _load_normalization() -> dict:
+    norm_file = "indonesian-normalisasi-slangword-complete.txt"
+    norm_dict: dict = {}
+    try:
+        with open(norm_file, "r", encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                parts = line.split(",", 1)
+                if len(parts) != 2:
+                    continue
+                slang  = parts[0].strip().strip("'\"").lower()
+                normal = parts[1].strip().lower()
+                if slang and normal:
+                    norm_dict[slang] = normal
+    except FileNotFoundError:
+        pass
+
+    DOMAIN_OVERRIDES = {
+        "shopee": "shopee", "tokopedia": "tokopedia", "lazada": "lazada",
+        "tiktok": "tiktok", "bukalapak": "bukalapak", "blibli": "blibli",
+        "sicepat": "sicepat", "jne": "jne", "jnt": "jnt",
+        "anteraja": "anteraja", "ninja": "ninja",
+        "freeongkir": "gratis ongkos kirim",
+        "gratisongkir": "gratis ongkos kirim",
+        "ongkir": "ongkos kirim", "ongkr": "ongkos kirim",
+        "bykrm": "biaya kirim", "biayakirim": "biaya pengiriman",
+        "komdigi": "komdigi", "kemendag": "kementerian perdagangan",
+        "kominfo": "kementerian komunikasi",
+        "ecommerce": "e commerce", "marketplace": "marketplace",
+        "seller": "penjual", "buyer": "pembeli", "online": "online",
+        # Negasi tambahan
+        "gk": "tidak", "ga": "tidak", "gak": "tidak",
+        "nggak": "tidak", "ngga": "tidak", "tdk": "tidak",
+        "tak": "tidak", "enggak": "tidak", "engga": "tidak",
+        "kagak": "tidak", "kaga": "tidak", "ndak": "tidak",
+        "ngak": "tidak",
+        # Intensitas
+        "bgt": "banget", "bngt": "banget", "bget": "banget",
+        "bgtt": "banget",
+        # Positif informal
+        "mantep": "mantap", "mntap": "mantap",
+        "bener": "benar", "beneran": "benar",
+        "kece": "keren", "sip": "baik",
+        # Negatif informal
+        "ancur": "hancur", "parahh": "parah",
+    }
+    norm_dict.update(DOMAIN_OVERRIDES)
+    return norm_dict
+
+
+# ═══════════════════════════════════════════════════════════
+#  LOAD STOPWORDS DARI FILE
 # ═══════════════════════════════════════════════════════════
 
 def _load_stopwords() -> set:
-    """Load stopword dengan penjagaan kata sentimen penting."""
     stopword_file = "indonesian-stopwords-complete.txt"
-    base = set()
+    base: set = set()
     try:
         with open(stopword_file, "r", encoding="utf-8") as f:
-            base = set(f.read().splitlines())
+            for line in f:
+                word = line.strip().lower()
+                if word:
+                    base.add(word)
     except FileNotFoundError:
         base = {
             "yang", "dan", "di", "ke", "dari", "ini", "itu",
             "dengan", "untuk", "pada", "adalah", "oleh", "ada",
             "ya", "akan", "atau", "juga", "sama", "karena",
-            "jika", "sudah", "telah", "saat", "agar", "maka",
+            "jika", "sudah", "telah", "jadi", "bisa",
-            "lagi", "bila", "bisa", "pun", "nya",
         }
 
-    # Jangan hapus kata sentimen penting
+    # Lindungi kata sentimen penting
     for kata in KATA_SENTIMEN_PENTING:
         base.discard(kata)
 
-    # Tambahan stopword domain-spesifik
+    # Tambah noise Twitter
     base.update({
-        "rt", "amp", "https", "http", "co", "t",
+        "rt", "amp", "https", "http", "co", "pic",
-        "wkwk", "wkwkwk", "haha", "hehe", "xixi", "hahaha",
+        "wkwk", "wkwkwk", "wkwkwkwk",
-        "yg", "dgn", "utk", "dr", "krn", "tp", "jd", "sdh",
+        "haha", "hahaha", "hehe", "hihi", "huhu", "xixi",
-        "aja", "doang", "nih", "sih", "dong", "deh",
+        "nih", "sih", "dong", "deh", "loh", "lah", "tuh",
-        "loh", "lah", "tuh", "kak", "gan", "bro", "sis",
+        "kak", "gan", "bro", "sob", "min",
     })
     return base
 
 
 # ═══════════════════════════════════════════════════════════
-#  STEMMER
+#  LOAD STEMMER
 # ═══════════════════════════════════════════════════════════
 
 def _load_stemmer():
-    """Load Sastrawi stemmer. Return None jika tidak terinstall."""
     try:
         from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
         return StemmerFactory().create_stemmer()
@@ -199,13 +216,14 @@ def _load_stemmer():
         return None
 
 
-# ── Inisialisasi global ─────────────────────────────────────────────────────
+# ── Inisialisasi global (dimuat sekali saat modul diimport) ─
 _STOPWORDS = _load_stopwords()
 _STEMMER   = _load_stemmer()
+_NORM_DICT = _load_normalization()
 
 
 # ═══════════════════════════════════════════════════════════
-#  MODEL LOADING (lazy)
+#  MODEL LOADING (lazy, singleton)
 # ═══════════════════════════════════════════════════════════
 
 _model = None
@@ -213,10 +231,6 @@ _tfidf = None
 
 
 def _load_model():
-    """
-    Lazy load NB model + TF-IDF vectorizer.
-    Return: (model, tfidf) — keduanya bisa None.
-    """
     global _model, _tfidf
     if _model is None:
         try:
@@ -232,35 +246,29 @@ def _load_model():
 
 
 # ═══════════════════════════════════════════════════════════
-#  PREPROCESSING PIPELINE — 5 TAHAP
+#  5-TAHAP PREPROCESSING PIPELINE
-#
-#  URUTAN (selaras dengan preprocessing_page.py):
-#    1. Case Folding     → lowercase
-#    2. Cleaning         → hapus noise
-#    3. Normalisasi      → normalisasi kata
-#    4. Stopword Removal → buang stopword (split() dilakukan internal)
-#    5. Stemming         → bentuk dasar
 # ═══════════════════════════════════════════════════════════
 
-def _case_folding(text: str) -> str:
+def _step1_case_folding(text: str) -> str:
-    """Tahap 1: Lowercase seluruh teks."""
     return str(text).lower()
 
 
-def _cleaning(text: str) -> str:
+def _step2_cleaning(text: str) -> str:
-    """Tahap 2: Hapus noise (URL, mention, hashtag, angka, emoji, tanda baca)."""
     text = re.sub(r"http\S+|www\S+|https\S+", "", text)
     text = re.sub(r"@\w+", "", text)
     text = re.sub(r"#\w+", "", text)
     text = re.sub(r"\d+", "", text)
     text = re.sub(
-        r"[\U00010000-\U0010ffff"
+        r"["
+        r"\U00010000-\U0010ffff"
         r"\U0001F600-\U0001F64F"
         r"\U0001F300-\U0001F5FF"
         r"\U0001F680-\U0001F6FF"
         r"\U0001F1E0-\U0001F1FF"
-        r"\u2600-\u26FF\u2700-\u27BF"
+        r"\u2600-\u26FF"
-        r"]+", "", text, flags=re.UNICODE,
+        r"\u2700-\u27BF"
+        r"]+",
+        "", text, flags=re.UNICODE,
     )
     text = text.translate(str.maketrans("", "", string.punctuation))
     text = re.sub(r"[^a-zA-Z\s]", "", text)
@@ -268,68 +276,60 @@ def _cleaning(text: str) -> str:
     return text
 
 
-def _normalisasi(text: str) -> str:
+def _step3_normalization(text: str, norm_dict: dict) -> str:
-    """Tahap 3: Normalisasi singkatan dan kata tidak baku."""
+    tokens = text.split()
-    return " ".join(NORMALISASI.get(word, word) for word in text.split())
+    return " ".join(norm_dict.get(token, token) for token in tokens)
 
 
-def _remove_stopwords(tokens: list) -> list:
+def _step4_stopword_removal(tokens: list, stopwords: set) -> list:
-    """Tahap 4: Stopword removal dengan penjagaan kata sentimen."""
+    result = []
-    return [
+    for token in tokens:
-        w for w in tokens
+        if token in KATA_SENTIMEN_PENTING:
-        if (w not in _STOPWORDS or w in KATA_SENTIMEN_PENTING) and len(w) > 2
+            result.append(token)
-    ]
+            continue
+        if token in stopwords:
+            continue
+        if len(token) <= 2:
+            continue
+        result.append(token)
+    return result
 
 
-def _stemming(tokens: list) -> list:
+def _step5_stemming(tokens: list, stemmer) -> list:
-    """Tahap 5: Stemming ke bentuk dasar via Sastrawi ECS."""
+    if stemmer is None:
-    if _STEMMER is None:
         return tokens
-    return [_STEMMER.stem(w) for w in tokens]
+    return [stemmer.stem(token) for token in tokens]
 
 
 def preprocess_for_model(text: str) -> str:
-    """
+    """Full 5-tahap preprocessing → string teks bersih siap TF-IDF."""
-    Full 5-tahap preprocessing → string teks bersih siap TF-IDF.
+    s1 = _step1_case_folding(text)
-
+    s2 = _step2_cleaning(s1)
-    URUTAN: Case Folding → Cleaning → Normalisasi
+    s3 = _step3_normalization(s2, _NORM_DICT)
-            → Stopword Removal → Stemming
+    s4 = _step4_stopword_removal(s3.split(), _STOPWORDS)
-
+    s5 = _step5_stemming(s4, _STEMMER)
-    Split kata dilakukan secara internal di tahap Stopword Removal.
-    Pipeline HARUS sama persis dengan yang dipakai saat training model.
-    """
-    s1 = _case_folding(text)          # Tahap 1
-    s2 = _cleaning(s1)                # Tahap 2
-    s3 = _normalisasi(s2)             # Tahap 3
-    s4 = _remove_stopwords(s3.split()) # Tahap 4 (split inline)
-    s5 = _stemming(s4)                # Tahap 5
     return " ".join(s5)
 
 
 def preprocess_untuk_lexicon(text: str) -> str:
-    """
+    """Preprocessing ringan untuk lexicon matching (tanpa stemming)."""
-    Preprocessing RINGAN untuk lexicon matching.
+    s1 = _step1_case_folding(text)
-    Tidak di-stem → kata asli bisa dicocokkan dengan lexicon.
+    s2 = _step2_cleaning(s1)
-    Pipeline: Case Folding → Cleaning → Normalisasi saja.
+    s3 = _step3_normalization(s2, _NORM_DICT)
-    """
-    s1 = _case_folding(text)
-    s2 = _cleaning(s1)
-    s3 = _normalisasi(s2)
     return s3
 
 
 # ═══════════════════════════════════════════════════════════
-#  LEXICON SCORER
+#  LEXICON SCORER  (dengan negation window diperluas ke 3)
 # ═══════════════════════════════════════════════════════════
 
 def _hitung_skor_lexicon(teks_lexicon: str) -> dict:
     """
-    Hitung skor positif dan negatif dari teks via lexicon.
+    Hitung skor sentimen via lexicon + negation handling.
 
-    NEGATION HANDLING:
+    Perubahan vs versi lama:
-    Kata negasi dalam window 2 kata sebelum kata sentimen → polaritas dibalik.
+    - Window negasi diperluas menjadi 3 kata (sebelumnya 2)
-    Contoh: "tidak bagus" → ada "tidak" sebelum "bagus" (POSITIF)
+    - Lexicon positif/negatif lebih luas (kata yang tidak ada di vocab TF-IDF)
-            → skor_neg += 1 (bukan skor_pos)
 
     Return: {"positif": int, "negatif": int, "net": int}
     """
@@ -338,9 +338,10 @@ def _hitung_skor_lexicon(teks_lexicon: str) -> dict:
     skor_neg = 0
 
     for i, token in enumerate(tokens):
+        # Cek negasi dalam window 3 kata sebelumnya
         ada_negasi = any(
             tokens[i - j] in KATA_NEGASI
-            for j in range(1, 3)
+            for j in range(1, 4)
             if i - j >= 0
         )
 
@@ -363,21 +364,23 @@ def _hitung_skor_lexicon(teks_lexicon: str) -> dict:
 
 
 # ═══════════════════════════════════════════════════════════
-#  HYBRID CLASSIFIER
+#  PREDIKSI MODEL NB
 # ═══════════════════════════════════════════════════════════
 
-# Normalisasi label dari berbagai format yang mungkin dipakai model
 _LABEL_MAP = {
-    "positif": "Positif", "Positif": "Positif", "positive": "Positif", "pos": "Positif",
+    "positif": "Positif", "Positif": "Positif",
-    "negatif": "Negatif", "Negatif": "Negatif", "negative": "Negatif", "neg": "Negatif",
+    "positive": "Positif", "pos": "Positif",
-    "netral":  "Netral",  "Netral":  "Netral",  "neutral":  "Netral",  "net": "Netral",
+    "negatif": "Negatif", "Negatif": "Negatif",
+    "negative": "Negatif", "neg": "Negatif",
+    "netral":  "Netral",  "Netral":  "Netral",
+    "neutral": "Netral",  "net": "Netral",
 }
 
 
 def _prediksi_model(teks_model: str):
     """
-    Dapatkan prediksi dari NB model.
+    Prediksi dari model NB 3-kelas.
-    Return: (label_norm, confidence, proba_dict) atau None.
+    Return: (label_norm, confidence, proba_dict) atau None jika gagal.
     """
     model, tfidf = _load_model()
     if model is None or tfidf is None or not teks_model.strip():
@@ -402,79 +405,105 @@ def _prediksi_model(teks_model: str):
         return None
 
 
+# ═══════════════════════════════════════════════════════════
+#  HYBRID CLASSIFIER  (diperkuat)
+# ═══════════════════════════════════════════════════════════
+
 def _klasifikasi_hybrid(
     teks_model: str,
     skor: dict,
     teks_lower: str,
 ) -> tuple:
     """
-    Klasifikasi hybrid: sinyal lexicon + model NB.
+    Klasifikasi hybrid: sinyal lexicon + model NB 3-kelas.
-
-    LOGIKA KEPUTUSAN (berurutan):
 
+    PERUBAHAN vs versi lama:
+    ─────────────────────────────────────────────────────────
     Layer 1 — Override Positif KUAT (net >= 2):
-      → Positif, confidence 60–92%
+      Confidence lebih tinggi karena lexicon lebih kuat.
+      Range: 0.65–0.92
+
+    Layer 1b — Override Negatif KUAT (net <= -2):
+      Simetris dengan positif.
+      Range: 0.65–0.90
 
     Layer 2 — Override Positif LEMAH (net == 1):
-      → Cek model; jika model < 65% yakin Negatif → Positif
+      Diperlonggar: sekarang berlaku jika model tidak sangat
-      → Jika model sangat yakin Negatif → tetap Positif (confidence rendah)
+      yakin negatif (threshold 0.70, sebelumnya 0.65).
 
-    Layer 3 — Override Negatif KUAT (net <= -2):
+    Layer 2b — Override Negatif LEMAH (net == -1):
-      → Negatif, confidence 60–90%
+      Simetris baru. Sebelumnya tidak ada.
 
-    Layer 4 — Fallback Model NB:
+    Layer 3 — Fallback Model NB:
-      → Prediksi model dipakai, TAPI:
+      Anti-bias diperketat:
-        • Jika model = Negatif AND net >= 0 AND confidence < 75%
+      - Model Negatif + lexicon bersih (net >= 0) + conf < 0.65
-          → downgrade ke Netral (koreksi bias model)
+        → turunkan ke Netral (threshold naik dari 0.70 ke 0.65)
-        • Kasus lainnya → percaya model
+      - Model Positif + lexicon negatif kuat (net <= -1) + conf < 0.65
-
+        → turunkan ke Netral
-    Layer 5 — Ultimate Fallback (model tidak tersedia):
-      → Gunakan skor lexicon saja
 
+    Layer 4 — Ultimate Fallback:
+      Lexicon saja jika model tidak tersedia.
+    ─────────────────────────────────────────────────────────
     Return: (label: str, confidence: float)
     """
     net = skor["net"]
     pos = skor["positif"]
+    neg = skor["negatif"]
 
-    # ── Layer 1: Positif KUAT ─────────────────────────────────────────────────
+    # ── Layer 1a: Positif KUAT ───────────────────────────────────────────────
     if net >= 2:
-        conf = min(0.60 + (net * 0.07), 0.92)
+        conf = min(0.65 + (net * 0.05), 0.92)
         return ("Positif", round(conf, 3))
 
-    # ── Layer 2: Positif LEMAH ────────────────────────────────────────────────
+    # ── Layer 1b: Negatif KUAT ───────────────────────────────────────────────
+    if net <= -2:
+        conf = min(0.65 + (abs(net) * 0.05), 0.90)
+        return ("Negatif", round(conf, 3))
+
+    # ── Layer 2a: Positif LEMAH ──────────────────────────────────────────────
     if net == 1 and pos >= 1:
         model_result = _prediksi_model(teks_model)
         if model_result is not None:
             _, _, proba_dict = model_result
             neg_prob = proba_dict.get("Negatif", 0.0)
-            if neg_prob < 0.65:
+            if neg_prob < 0.70:  # diperlonggar dari 0.65
-                conf = round(0.55 + (0.65 - neg_prob) * 0.3, 3)
+                conf = round(0.55 + (0.70 - neg_prob) * 0.30, 3)
-                return ("Positif", min(conf, 0.80))
+                return ("Positif", min(conf, 0.82))
             else:
                 return ("Positif", 0.55)
-        else:
         return ("Positif", 0.58)
 
-    # ── Layer 3: Negatif KUAT ─────────────────────────────────────────────────
+    # ── Layer 2b: Negatif LEMAH (baru) ───────────────────────────────────────
-    if net <= -2:
+    if net == -1 and neg >= 1:
-        conf = min(0.60 + (abs(net) * 0.06), 0.90)
+        model_result = _prediksi_model(teks_model)
-        return ("Negatif", round(conf, 3))
+        if model_result is not None:
+            _, _, proba_dict = model_result
+            pos_prob = proba_dict.get("Positif", 0.0)
+            if pos_prob < 0.70:
+                conf = round(0.55 + (0.70 - pos_prob) * 0.30, 3)
+                return ("Negatif", min(conf, 0.82))
+            else:
+                return ("Negatif", 0.55)
+        return ("Negatif", 0.58)
 
-    # ── Layer 4: Fallback Model NB ────────────────────────────────────────────
+    # ── Layer 3: Fallback Model NB 3-kelas ───────────────────────────────────
     model_result = _prediksi_model(teks_model)
     if model_result is not None:
         label_norm, confidence, proba_dict = model_result
 
-        # Anti-bias correction:
+        # Anti-bias 1: Model Negatif tapi lexicon bersih → Netral
-        # Model prediksi Negatif tapi tidak ada sinyal negatif dari lexicon
+        if label_norm == "Negatif" and net >= 0 and confidence < 0.65:
-        # dan confidence < 75% → kemungkinan bias → turunkan ke Netral
+            corrected_conf = round(0.50 + max(0, confidence - 0.50) * 0.20, 3)
-        if label_norm == "Negatif" and net >= 0 and confidence < 0.75:
+            return ("Netral", corrected_conf)
-            corrected_conf = round(0.50 + max(0, confidence - 0.50) * 0.2, 3)
+
+        # Anti-bias 2: Model Positif tapi lexicon negatif → Netral
+        if label_norm == "Positif" and net <= -1 and confidence < 0.65:
+            corrected_conf = round(0.50 + max(0, confidence - 0.50) * 0.20, 3)
             return ("Netral", corrected_conf)
 
         return (label_norm, round(confidence, 3))
 
-    # ── Layer 5: Ultimate Fallback ────────────────────────────────────────────
+    # ── Layer 4: Ultimate Fallback ────────────────────────────────────────────
     if net > 0:
         return ("Positif", 0.55)
     elif net < 0:
@@ -483,27 +512,45 @@ def _klasifikasi_hybrid(
         return ("Netral", 0.50)
 
 
+# ═══════════════════════════════════════════════════════════
+#  PUBLIC API
+# ═══════════════════════════════════════════════════════════
+
+def analisis_sentimen_single(text: str) -> tuple:
+    """
+    Analisis sentimen satu teks.
+    Return: (label, confidence)
+      label: 'Positif' | 'Netral' | 'Negatif'
+    """
+    teks_model   = preprocess_for_model(text)
+    teks_lexicon = preprocess_untuk_lexicon(text)
+    teks_lower   = str(text).lower()
+    skor         = _hitung_skor_lexicon(teks_lexicon)
+    return _klasifikasi_hybrid(teks_model, skor, teks_lower)
+
+
+def analisis_sentimen_batch(texts: list) -> list:
+    """
+    Analisis sentimen batch.
+    Return: list of (label, confidence)
+    """
+    return [analisis_sentimen_single(text) for text in texts]
+
+
 # ═══════════════════════════════════════════════════════════
 #  BACKWARD COMPATIBILITY
 # ═══════════════════════════════════════════════════════════
 
 def bersihkan_teks(text: str) -> str:
-    """[LEGACY] Gunakan preprocess_for_model() untuk pipeline lengkap."""
+    """[LEGACY] Gunakan preprocess_for_model()."""
     return preprocess_for_model(text)
 
 
 def prediksi_sentimen(list_text: list):
     """
-    [LEGACY] Prediksi batch dengan hybrid classifier.
+    [LEGACY] Prediksi batch.
     Return: (list clean_texts, list labels)
     """
     clean_texts = [preprocess_for_model(t) for t in list_text]
-    labels = []
+    labels = [analisis_sentimen_single(t)[0] for t in list_text]
-    for text in list_text:
-        teks_model   = preprocess_for_model(text)
-        teks_lexicon = preprocess_untuk_lexicon(text)
-        teks_lower   = str(text).lower()
-        skor         = _hitung_skor_lexicon(teks_lexicon)
-        label, _     = _klasifikasi_hybrid(teks_model, skor, teks_lower)
-        labels.append(label)
     return clean_texts, labels

tweets-data/hasil_latest.csv

@@ -1,2 +1 @@
-"conversation_id_str","created_at","favorite_count","full_text","id_str","image_url","in_reply_to_screen_name","lang","location","quote_count","reply_count","retweet_count","tweet_url","user_id_str","username"
+
-"2056538953340314095","Wed May 20 16:01:04 +0000 2026","4","@killingmaster27 @evelyndewi7 @katrinrienks @HumorJonTampan @wiyantokdr54 @EviDrajat @RickyKardjono @Micheladam69432 @ebonXboy @AntonRebor79105 @gangsterrandom @Pieter_Sun01 @Rahmah0845 @RSjahfidi @prabowo @SmileyNDeLight @Jediimar @Yens0906 @tokaidirecycle @PembuatSistem @bonapasogit24 @Jenniestale @Maskudin126003 @a8sar6alih @JowoNgalam @KING_RAJA_PWS_ @apaajadeh447870 @50ngoku @Submill @EmmaHusain0845 @BebySoSweet @Yulia_4ja @bungalotus25 @bahlillahadalia wahhh... jadi menteri komdigi dong menteri free ongkir https://t.co/1cfvoqIcNl","2057129489994510620","https://pbs.twimg.com/media/HIxkWwDaMAAXH9U.jpg","killingmaster27","in","","0","2","0","https://x.com/undefined/status/2057129489994510620","1410466956134608899",

tweets-data/hasil_latest.old.csv

@@ -1,2 +1 @@
-"conversation_id_str","created_at","favorite_count","full_text","id_str","image_url","in_reply_to_screen_name","lang","location","quote_count","reply_count","retweet_count","tweet_url","user_id_str","username"
+
-"2056538953340314095","Wed May 20 16:01:04 +0000 2026","4","@killingmaster27 @evelyndewi7 @katrinrienks @HumorJonTampan @wiyantokdr54 @EviDrajat @RickyKardjono @Micheladam69432 @ebonXboy @AntonRebor79105 @gangsterrandom @Pieter_Sun01 @Rahmah0845 @RSjahfidi @prabowo @SmileyNDeLight @Jediimar @Yens0906 @tokaidirecycle @PembuatSistem @bonapasogit24 @Jenniestale @Maskudin126003 @a8sar6alih @JowoNgalam @KING_RAJA_PWS_ @apaajadeh447870 @50ngoku @Submill @EmmaHusain0845 @BebySoSweet @Yulia_4ja @bungalotus25 @bahlillahadalia wahhh... jadi menteri komdigi dong menteri free ongkir https://t.co/1cfvoqIcNl","2057129489994510620","https://pbs.twimg.com/media/HIxkWwDaMAAXH9U.jpg","killingmaster27","in","","0","2","0","https://x.com/undefined/status/2057129489994510620","1410466956134608899",

tweets-data/hasil_top.csv

@@ -1,7 +1,8 @@
 "conversation_id_str","created_at","favorite_count","full_text","id_str","image_url","in_reply_to_screen_name","lang","location","quote_count","reply_count","retweet_count","tweet_url","user_id_str","username"
-"2057791279896080430","Fri May 22 11:50:47 +0000 2026","0","komdigi mengeluarkan kebijakan gratis ongkir @komdigi ???","2057791279896080430","","","in","","0","0","0","https://x.com/undefined/status/2057791279896080430","1814143201718247424",
+"2058829602865451445","Mon May 25 08:36:43 +0000 2026","0","komdigi mending ngurusin aja tuh judol daripada ngurusin pembatasan gratis ongkir","2058829602865451445","","","in","","0","0","0","https://x.com/undefined/status/2058829602865451445","2046458899843170304",
-"2057660616815571144","Fri May 22 03:11:35 +0000 2026","0","komdigi pembatasan gratis ongkir","2057660616815571144","","","in","","0","0","0","https://x.com/undefined/status/2057660616815571144","1814143201718247424",
+"2058950111976386607","Mon May 25 16:35:34 +0000 2026","0","komdigi bagus sih ngeluarin kebijakan gratis ongkir kaya ini setuju","2058950111976386607","","","in","","0","0","0","https://x.com/undefined/status/2058950111976386607","2046458899843170304",
-"2056635322419777630","Tue May 19 07:17:25 +0000 2026","0","komdigi ongkir gk jelas","2056635322419777630","","","in","","0","0","0","https://x.com/undefined/status/2056635322419777630","1814143201718247424",
+"2058950559856832819","Mon May 25 16:37:21 +0000 2026","0","setuju bgt komdigi ngeluarin kebijakan gratis ongkir gini","2058950559856832819","","","in","","0","0","0","https://x.com/undefined/status/2058950559856832819","2046458899843170304",
-"2056949159845101648","Wed May 20 04:04:30 +0000 2026","0","komdigi kereenn dan bijak membuat keputusan ongkir seperti inii","2056949159845101648","","","in","","0","0","0","https://x.com/undefined/status/2056949159845101648","1814143201718247424",
+"2060234400777064942","Fri May 29 05:38:53 +0000 2026","0","tadi pagi saya melihat berita mengenai pembatasan gratis ongkir oleh komdigi","2060234400777064942","","","in","","0","0","0","https://x.com/undefined/status/2060234400777064942","1814143201718247424",
-"2056892677149581606","Wed May 20 00:20:04 +0000 2026","0","komdigi keren sih mengadakan kebijakan ongkir kaya gini","2056892677149581606","","","in","","0","0","0","https://x.com/undefined/status/2056892677149581606","1814143201718247424",
+"2058816188659192179","Mon May 25 07:43:24 +0000 2026","0","komdigi gajelas bikin kebijakan ongkir!","2058816188659192179","","","in","","0","0","0","https://x.com/undefined/status/2058816188659192179","1814143201718247424",
-"2056538953340314095","Wed May 20 16:01:04 +0000 2026","4","@killingmaster27 @evelyndewi7 @katrinrienks @HumorJonTampan @wiyantokdr54 @EviDrajat @RickyKardjono @Micheladam69432 @ebonXboy @AntonRebor79105 @gangsterrandom @Pieter_Sun01 @Rahmah0845 @RSjahfidi @prabowo @SmileyNDeLight @Jediimar @Yens0906 @tokaidirecycle @PembuatSistem @bonapasogit24 @Jenniestale @Maskudin126003 @a8sar6alih @JowoNgalam @KING_RAJA_PWS_ @apaajadeh447870 @50ngoku @Submill @EmmaHusain0845 @BebySoSweet @Yulia_4ja @bungalotus25 @bahlillahadalia wahhh... jadi menteri komdigi dong menteri free ongkir https://t.co/1cfvoqIcNl","2057129489994510620","https://pbs.twimg.com/media/HIxkWwDaMAAXH9U.jpg","killingmaster27","in","","0","2","0","https://x.com/undefined/status/2057129489994510620","1410466956134608899",
+"2059619299309011403","Wed May 27 12:54:41 +0000 2026","0","komdigi ngeluarin kebijakan ongkir kaya gini ngapain dah","2059619299309011403","","","in","","0","0","0","https://x.com/undefined/status/2059619299309011403","1814143201718247424",
+"2058816239112446387","Mon May 25 07:43:37 +0000 2026","0","komdigi gk penting buat kebijakan ongkir kaya gini","2058816239112446387","","","in","","0","0","0","https://x.com/undefined/status/2058816239112446387","1814143201718247424",

tweets-data/hasil_top.old.csv

@@ -1,7 +1,8 @@
 "conversation_id_str","created_at","favorite_count","full_text","id_str","image_url","in_reply_to_screen_name","lang","location","quote_count","reply_count","retweet_count","tweet_url","user_id_str","username"
-"2057660616815571144","Fri May 22 03:11:35 +0000 2026","0","komdigi pembatasan gratis ongkir","2057660616815571144","","","in","","0","0","0","https://x.com/undefined/status/2057660616815571144","1814143201718247424",
+"2058829602865451445","Mon May 25 08:36:43 +0000 2026","0","komdigi mending ngurusin aja tuh judol daripada ngurusin pembatasan gratis ongkir","2058829602865451445","","","in","","0","0","0","https://x.com/undefined/status/2058829602865451445","2046458899843170304",
-"2057791279896080430","Fri May 22 11:50:47 +0000 2026","0","komdigi mengeluarkan kebijakan gratis ongkir @komdigi ???","2057791279896080430","","","in","","0","0","0","https://x.com/undefined/status/2057791279896080430","1814143201718247424",
+"2058950111976386607","Mon May 25 16:35:34 +0000 2026","0","komdigi bagus sih ngeluarin kebijakan gratis ongkir kaya ini setuju","2058950111976386607","","","in","","0","0","0","https://x.com/undefined/status/2058950111976386607","2046458899843170304",
-"2056635322419777630","Tue May 19 07:17:25 +0000 2026","0","komdigi ongkir gk jelas","2056635322419777630","","","in","","0","0","0","https://x.com/undefined/status/2056635322419777630","1814143201718247424",
+"2058950559856832819","Mon May 25 16:37:21 +0000 2026","0","setuju bgt komdigi ngeluarin kebijakan gratis ongkir gini","2058950559856832819","","","in","","0","0","0","https://x.com/undefined/status/2058950559856832819","2046458899843170304",
-"2056949159845101648","Wed May 20 04:04:30 +0000 2026","0","komdigi kereenn dan bijak membuat keputusan ongkir seperti inii","2056949159845101648","","","in","","0","0","0","https://x.com/undefined/status/2056949159845101648","1814143201718247424",
+"2060234400777064942","Fri May 29 05:38:53 +0000 2026","0","tadi pagi saya melihat berita mengenai pembatasan gratis ongkir oleh komdigi","2060234400777064942","","","in","","0","0","0","https://x.com/undefined/status/2060234400777064942","1814143201718247424",
-"2056892677149581606","Wed May 20 00:20:04 +0000 2026","0","komdigi keren sih mengadakan kebijakan ongkir kaya gini","2056892677149581606","","","in","","0","0","0","https://x.com/undefined/status/2056892677149581606","1814143201718247424",
+"2058816188659192179","Mon May 25 07:43:24 +0000 2026","0","komdigi gajelas bikin kebijakan ongkir!","2058816188659192179","","","in","","0","0","0","https://x.com/undefined/status/2058816188659192179","1814143201718247424",
-"2056538953340314095","Wed May 20 16:01:04 +0000 2026","4","@killingmaster27 @evelyndewi7 @katrinrienks @HumorJonTampan @wiyantokdr54 @EviDrajat @RickyKardjono @Micheladam69432 @ebonXboy @AntonRebor79105 @gangsterrandom @Pieter_Sun01 @Rahmah0845 @RSjahfidi @prabowo @SmileyNDeLight @Jediimar @Yens0906 @tokaidirecycle @PembuatSistem @bonapasogit24 @Jenniestale @Maskudin126003 @a8sar6alih @JowoNgalam @KING_RAJA_PWS_ @apaajadeh447870 @50ngoku @Submill @EmmaHusain0845 @BebySoSweet @Yulia_4ja @bungalotus25 @bahlillahadalia wahhh... jadi menteri komdigi dong menteri free ongkir https://t.co/1cfvoqIcNl","2057129489994510620","https://pbs.twimg.com/media/HIxkWwDaMAAXH9U.jpg","killingmaster27","in","","0","2","0","https://x.com/undefined/status/2057129489994510620","1410466956134608899",
+"2059619299309011403","Wed May 27 12:54:41 +0000 2026","0","komdigi ngeluarin kebijakan ongkir kaya gini ngapain dah","2059619299309011403","","","in","","0","0","0","https://x.com/undefined/status/2059619299309011403","1814143201718247424",
+"2058816239112446387","Mon May 25 07:43:37 +0000 2026","0","komdigi gk penting buat kebijakan ongkir kaya gini","2058816239112446387","","","in","","0","0","0","https://x.com/undefined/status/2058816239112446387","1814143201718247424",