""" preprocessing_page.py ===================== Halaman Bersihkan Data — NLP Pipeline 5 Tahap. PIPELINE 5 TAHAP: 1. Case Folding — ubah semua huruf jadi lowercase 2. Cleaning — hapus URL, mention, hashtag, angka, emoji, tanda baca 3. Normalisasi — singkatan/slang → kata baku (DARI FILE normalisasi) 4. Stopword Removal — hapus kata umum (DARI FILE stopword); JAGA kata sentimen 5. Stemming — bentuk dasar kata via Sastrawi ECS PERUBAHAN DARI VERSI SEBELUMNYA: - Normalisasi kini dimuat dari 'indonesian-normalisasi-slangword-complete.txt' (1.700+ entri), menggantikan dict hardcoded yang hanya ~60 entri. - Stopword kini murni dari 'indonesian-stopwords-complete.txt', ditambah noise Twitter yang spesifik — tidak ada penghapusan manual acak. - KATA_SENTIMEN_PENTING diperluas dengan kata domain e-commerce/ongkir. - Semua fungsi preprocessing menerima parameter eksplisit (tidak pakai global). CATATAN PENTING: Pipeline ini HARUS IDENTIK dengan sentiment_service.py agar token yang dihasilkan di sini konsisten dengan token saat training model. """ import streamlit as st import pandas as pd import os import re import string from collections import Counter import plotly.graph_objects as go from datetime import datetime, timedelta from database import engine, get_tweet_count, get_latest_crawl_time from page_modules.table_utils import render_standard_table from timezone_utils import ( parse_dt_with_tz, parse_dt_with_source_tz, get_timezone_label, get_timezone_name, ) # ═══════════════════════════════════════════════════════════ # TIMEZONE HELPERS # ═══════════════════════════════════════════════════════════ def parse_dt(series): return parse_dt_with_tz( series, st.session_state.get("user_timezone", "WIB (UTC+7)") ) def parse_crawled_dt(series): return parse_dt_with_source_tz( series, st.session_state.get("user_timezone", "WIB (UTC+7)"), os.getenv("APP_TIMEZONE", "Asia/Jakarta") ) def format_dt(value): if value is None or pd.isna(value): return "Belum ada" try: tz_label = get_timezone_label( st.session_state.get("user_timezone", "WIB (UTC+7)") ) return f"{value.strftime('%d/%m/%Y %H:%M')} {tz_label}" except Exception: return "Belum ada" def user_today(): timezone_choice = st.session_state.get("user_timezone", "WIB (UTC+7)") return pd.Timestamp.now(tz=get_timezone_name(timezone_choice)).date() def _sync_dynamic_period(): mode = st.session_state.get("analysis_mode") today = user_today() configs = { "realtime": (today - timedelta(days=6), today, "Tweet Terkini — 7 Hari Terakhir"), "30days": (today - timedelta(days=29), today, "30 Hari Terakhir"), "captured": (today, today, "Tweet Hari Ini"), } if mode not in configs: return start_day, end_day, mode_display = configs[mode] dt_start = datetime.combine(start_day, datetime.min.time()) dt_end = datetime.combine(end_day, datetime.max.time().replace(microsecond=0)) st.session_state.filter_start_date = dt_start st.session_state.filter_end_date = dt_end st.session_state.filter_label = f"{dt_start.strftime('%d/%m/%Y')} s/d {dt_end.strftime('%d/%m/%Y')}" st.session_state.mode_display = mode_display st.session_state.filter_date_column = "created_at" # ═══════════════════════════════════════════════════════════ # PREPROCESSING PIPELINE — 5 TAHAP # # ┌─────────────────────────────────────────────────────┐ # │ PENTING: Pipeline ini HARUS identik dengan │ # │ sentiment_service.py agar token konsisten! │ # │ │ # │ URUTAN WAJIB: │ # │ 1. Case Folding → lowercase dulu │ # │ 2. Cleaning → hapus noise setelah lowercase │ # │ 3. Normalisasi → slang→baku setelah bersih │ # │ 4. Stopword → buang kata umum, jaga sentimen │ # │ 5. Stemming → bentuk dasar via Sastrawi ECS │ # └─────────────────────────────────────────────────────┘ # ═══════════════════════════════════════════════════════════ # ─────────────────────────────────────────────────────────── # KATA SENTIMEN PENTING # Kata-kata ini WAJIB DIJAGA dan tidak boleh dihapus saat # stopword removal, meskipun ada di file stopword. # # Kenapa perlu? Karena file stopword mengandung kata seperti # "tidak", "belum", "sangat" yang justru krusial untuk # menentukan sentimen positif/negatif suatu kalimat. # ─────────────────────────────────────────────────────────── KATA_SENTIMEN_PENTING = { # ── Negasi (pembalik makna kalimat) ────────────────── # "tidak bagus" ≠ "bagus" → "tidak" wajib ada "tidak", "bukan", "jangan", "kurang", "belum", "tanpa", # ── Intensitas (penguat/pelemah sentimen) ───────────── # "sangat bagus" lebih positif dari "bagus" saja "sangat", "banget", "sekali", "paling", "amat", "luar", "biasa", # ← "luar biasa" = dua token, keduanya dijaga # ── Positif umum ────────────────────────────────────── "keren", "bagus", "mantap", "setuju", "dukung", "mendukung", "andal", "handal", "gercep", "bangga", "senang", "suka", "baik", "benar", "tepat", "oke", "puas", "sejahtera", "berkembang", "maju", "inovatif", "tegas", "sigap", "tanggap", "adil", "bijak", "bermanfaat", "untung", "berhasil", "sukses", "solusi", "manfaat", "berguna", "membantu", "bantu", "pro", "lanjut", # ── Positif domain e-commerce / ongkir ──────────────── "gratis", "murah", "hemat", "terjangkau", "cepat", "aman", "mudah", "praktis", "terpercaya", # ── Negatif umum ────────────────────────────────────── "kecewa", "buruk", "jelek", "parah", "gagal", "hancur", "rusak", "bohong", "tipu", "korupsi", # ── Negatif domain e-commerce / ongkir ──────────────── "mahal", "lambat", "lelet", "ribet", "susah", "repot", "rugi", "boros", # ── Emosi ───────────────────────────────────────────── "marah", "sedih", "khawatir", "kecewa", } # ─────────────────────────────────────────────────────────── # LOAD NORMALIZATION DARI FILE # File: indonesian-normalisasi-slangword-complete.txt # Format per baris: slang,kata_baku # Contoh: gk,tidak | ongkir,ongkos kirim | free,gratis # ─────────────────────────────────────────────────────────── def _load_normalization() -> dict: """ Muat kamus normalisasi dari file eksternal. KENAPA DARI FILE? File berisi 1.700+ pasang slang→baku yang jauh lebih lengkap dibanding dict hardcoded. Dengan ini, kata seperti: gk/ga/gak/kagak/ngga → semua jadi "tidak" bgt/bngt/bget → semua jadi "sangat" ongkir/ongkr → jadi "ongkos kirim" ...dan ribuan kasus lainnya tertangani otomatis. Setelah file dimuat, override dengan entri khusus domain (nama platform, singkatan kebijakan) yang mungkin belum ada di file generik. """ norm_file = "indonesian-normalisasi-slangword-complete.txt" norm_dict: dict = {} try: with open(norm_file, "r", encoding="utf-8") as f: for line in f: line = line.strip() if not line: continue # Split hanya pada koma pertama — nilai bisa mengandung koma # Contoh: "on the way, sedang di jalan,dijalan" → split jadi 2 bagian parts = line.split(",", 1) if len(parts) != 2: continue # Bersihkan tanda kutip liar di awal/akhir (ada di beberapa baris file) slang = parts[0].strip().strip("'\"").lower() normal = parts[1].strip().lower() if slang and normal: norm_dict[slang] = normal except FileNotFoundError: # Jika file tidak ditemukan, lanjut dengan dict kosong. # Entri domain di bawah tetap akan ditambahkan. st.warning( "⚠️ File normalisasi tidak ditemukan: " f"'{norm_file}'. Hanya entri domain yang aktif." ) # ── Override khusus domain ─────────────────────────── # Entri ini menimpa file generik karena domain spesifik # membutuhkan perlakuan khusus (nama platform tidak diubah, # singkatan kebijakan punya padanan resmi, dll.) DOMAIN_OVERRIDES: dict = { # Nama platform — pertahankan apa adanya "shopee": "shopee", "tokopedia": "tokopedia", "lazada": "lazada", "tiktok": "tiktok", "bukalapak": "bukalapak", "blibli": "blibli", # Logistik — pertahankan apa adanya "sicepat": "sicepat", "jne": "jne", "jnt": "jnt", "anteraja": "anteraja", "ninja": "ninja", # Ongkir & belanja "freeongkir": "gratis ongkos kirim", "gratisongkir": "gratis ongkos kirim", "ongkir": "ongkos kirim", "ongkr": "ongkos kirim", "bykrm": "biaya kirim", "biayakirim": "biaya pengiriman", # Kebijakan & lembaga "komdigi": "komdigi", "kemendag": "kementerian perdagangan", "kominfo": "kementerian komunikasi", # E-commerce umum "ecommerce": "e commerce", "marketplace": "marketplace", "seller": "penjual", "buyer": "pembeli", "online": "online", } norm_dict.update(DOMAIN_OVERRIDES) return norm_dict # ─────────────────────────────────────────────────────────── # LOAD STOPWORDS DARI FILE # File: indonesian-stopwords-complete.txt # Format: satu kata per baris # ─────────────────────────────────────────────────────────── def _load_stopwords() -> set: """ Muat daftar stopword dari file eksternal. PROSES SETELAH MUAT FILE: 1. Hapus KATA_SENTIMEN_PENTING dari daftar → Agar "tidak", "belum", "sangat", dll. tidak ikut dibuang 2. Tambahkan noise Twitter/sosmed yang memang harus dibuang → "rt", "amp", sisa URL, suara tawa, partikel informal KENAPA DARI FILE? File berisi 700+ stopword Indonesia yang lebih lengkap dan terstandar dibanding daftar manual. Kita tidak perlu menambah/ mengurangi secara manual kecuali untuk dua kategori di atas. """ stopword_file = "indonesian-stopwords-complete.txt" base: set = set() try: with open(stopword_file, "r", encoding="utf-8") as f: for line in f: word = line.strip().lower() if word: base.add(word) except FileNotFoundError: # Fallback minimal — cukup untuk tetap jalan st.warning( "⚠️ File stopword tidak ditemukan: " f"'{stopword_file}'. Menggunakan daftar minimal." ) base = { "yang", "dan", "di", "ke", "dari", "ini", "itu", "dengan", "untuk", "pada", "adalah", "oleh", "ada", "ya", "akan", "atau", "juga", "sama", "karena", "jika", "sudah", "telah", "jadi", "bisa", } # ── Langkah 1: Lindungi kata sentimen ──────────────── # Beberapa kata sentimen penting ADA di file stopword # (misal: "tidak", "belum", "sangat", "paling", "kurang"). # Kita HAPUS dari stopword agar tidak ikut dibuang. for kata in KATA_SENTIMEN_PENTING: base.discard(kata) # ── Langkah 2: Tambah noise Twitter/sosmed ──────────── # Ini bukan stopword bahasa Indonesia biasa, tapi noise # yang sangat sering muncul di tweet dan tidak bermakna. TWITTER_NOISE: set = { # Artefak Twitter "rt", "amp", # Sisa URL setelah cleaning (kadang lolos) "https", "http", "co", "pic", # Suara tawa (tidak bermakna untuk sentimen) "wkwk", "wkwkwk", "wkwkwkwk", "haha", "hahaha", "hehe", "hihi", "huhu", "xixi", # Partikel informal yang tidak bermakna "nih", "sih", "dong", "deh", "loh", "lah", "tuh", "kak", "gan", "bro", "sob", "min", } base.update(TWITTER_NOISE) return base # ─────────────────────────────────────────────────────────── # LOAD STEMMER # ─────────────────────────────────────────────────────────── def _load_stemmer(): """Muat stemmer Sastrawi. Return None jika tidak tersedia.""" try: from Sastrawi.Stemmer.StemmerFactory import StemmerFactory return StemmerFactory().create_stemmer() except Exception: return None # ═══════════════════════════════════════════════════════════ # FUNGSI 5 TAHAP PREPROCESSING # Setiap fungsi bertanggung jawab SATU tahap saja. # Input & output setiap tahap dijelaskan di docstring. # ═══════════════════════════════════════════════════════════ def step1_case_folding(text: str) -> str: """ TAHAP 1 — CASE FOLDING Input : teks asli (campuran huruf besar/kecil) Output: semua huruf jadi lowercase Kenapa pertama? Agar tahap berikutnya (cleaning, normalisasi) bekerja secara konsisten — regex dan dict lookup case-sensitive. Contoh: "Gratis" → "gratis", "ONGKIR" → "ongkir" """ return str(text).lower() def step2_cleaning(text: str) -> str: """ TAHAP 2 — CLEANING Input : teks lowercase Output: teks bersih dari semua elemen noise Urutan pembersihan PENTING: 1. URL dulu (sebelum @ dan # agar tidak salah potong) 2. Mention (@username) 3. Hashtag (#topik) 4. Angka 5. Emoji & simbol unicode 6. Tanda baca 7. Karakter non-latin (huruf Arab, Cina, dll.) 8. Spasi berlebih """ # 1. Hapus URL (http, https, www) text = re.sub(r"http\S+|www\S+|https\S+", "", text) # 2. Hapus mention Twitter (@username) text = re.sub(r"@\w+", "", text) # 3. Hapus hashtag (#topik) text = re.sub(r"#\w+", "", text) # 4. Hapus angka dan digit text = re.sub(r"\d+", "", text) # 5. Hapus emoji & simbol unicode (berbagai range) text = re.sub( r"[" r"\U00010000-\U0010ffff" # Suplemen karakter unicode r"\U0001F600-\U0001F64F" # Emotikon wajah r"\U0001F300-\U0001F5FF" # Simbol & piktogram r"\U0001F680-\U0001F6FF" # Transport & peta r"\U0001F1E0-\U0001F1FF" # Bendera negara r"\u2600-\u26FF" # Simbol campuran r"\u2700-\u27BF" # Dingbats r"]+", "", text, flags=re.UNICODE ) # 6. Hapus tanda baca (.,!?;: dll.) text = text.translate(str.maketrans("", "", string.punctuation)) # 7. Hapus karakter non-latin (hanya sisakan huruf a-z dan spasi) text = re.sub(r"[^a-zA-Z\s]", "", text) # 8. Normalisasi spasi berlebih → satu spasi, lalu strip text = re.sub(r"\s+", " ", text).strip() return text def step3_normalization(text: str, norm_dict: dict) -> str: """ TAHAP 3 — NORMALISASI Input : teks bersih (sudah case fold + cleaning) norm_dict : kamus {slang: kata_baku} dari file Output: teks dengan slang/singkatan sudah diganti kata baku Cara kerja: token per token (word by word). Setiap token dicari di norm_dict. Jika ada → ganti. Jika tidak ada → biarkan. Contoh: "gk bs ongkir" → "tidak bisa ongkos kirim" "mantep bgt" → "mantap sangat" Kenapa setelah Cleaning? Karena slang di file ditulis dalam bentuk sudah lowercase dan sudah tanpa tanda baca. Jika normalisasi dilakukan sebelum cleaning, banyak entri tidak cocok. """ tokens = text.split() normalized = [norm_dict.get(token, token) for token in tokens] return " ".join(normalized) def step4_stopword_removal(tokens: list, stopwords: set) -> list: """ TAHAP 4 — STOPWORD REMOVAL Input : list token (hasil split dari teks ternormalisasi) stopwords : set kata yang harus dibuang (dari file) Output: list token bersih ATURAN PENYARINGAN (prioritas urutan): 1. JAGA token yang ada di KATA_SENTIMEN_PENTING → meskipun juga ada di stopwords, tetap disimpan 2. BUANG token yang ada di stopwords 3. BUANG token dengan panjang ≤ 2 karakter → menghilangkan sisa noise seperti "rt", "yg", "di" → PENGECUALIAN: token di KATA_SENTIMEN_PENTING tetap disimpan walau ≤ 2 karakter (contoh: "ok" jika masuk sentimen) Kenapa setelah Normalisasi? Agar "gak" yang sudah dinormalisasi jadi "tidak" tidak ikut dibuang — "tidak" dilindungi di KATA_SENTIMEN_PENTING. """ result = [] for token in tokens: # Prioritas 1: selalu simpan jika kata sentimen penting if token in KATA_SENTIMEN_PENTING: result.append(token) continue # Prioritas 2: buang jika stopword if token in stopwords: continue # Prioritas 3: buang jika terlalu pendek (noise) if len(token) <= 2: continue # Lolos semua filter → simpan result.append(token) return result def step5_stemming(tokens: list, stemmer) -> list: """ TAHAP 5 — STEMMING Input : list token setelah stopword removal stemmer : objek Sastrawi (atau None) Output: list token dalam bentuk kata dasar Algoritma: Enhanced Confix Stripping (ECS) via Sastrawi Contoh: "pengiriman" → "kirim" "pembatasan" → "batas" "berlari" → "lari" "makanan" → "makan" Jika stemmer None (Sastrawi tidak terinstal), token dikembalikan apa adanya tanpa error. """ if stemmer is None: return tokens return [stemmer.stem(token) for token in tokens] # ─────────────────────────────────────────────────────────── # FUNGSI UTAMA — JALANKAN SEMUA 5 TAHAP # ─────────────────────────────────────────────────────────── def full_preprocessing( text: str, stopwords: set, stemmer, norm_dict: dict, ) -> dict: """ Jalankan 5 tahap preprocessing secara berurutan dan kembalikan hasil setiap tahap sebagai dict (untuk ditampilkan di tabel). Parameter: text : teks tweet asli stopwords : set stopword (dari _load_stopwords) stemmer : objek Sastrawi (dari _load_stemmer) norm_dict : kamus normalisasi (dari _load_normalization) Return dict berisi: setelah_casefolding : hasil Tahap 1 setelah_cleaning : hasil Tahap 2 setelah_normalisasi : hasil Tahap 3 setelah_stopword : hasil Tahap 4 (joined string) clean_text : hasil akhir Tahap 5 (joined string) _tokens_clean : hasil Tahap 5 sebagai list (untuk analisis) """ # Tahap 1 — Case Folding s1_fold = step1_case_folding(text) # Tahap 2 — Cleaning s2_clean = step2_cleaning(s1_fold) # Tahap 3 — Normalisasi (perlu norm_dict) s3_norm = step3_normalization(s2_clean, norm_dict) # Tahap 4 — Stopword Removal (split → filter → simpan sebagai list) s4_tokens = s3_norm.split() s4_filtered = step4_stopword_removal(s4_tokens, stopwords) # Tahap 5 — Stemming s5_stemmed = step5_stemming(s4_filtered, stemmer) return { "setelah_casefolding": s1_fold, "setelah_cleaning": s2_clean, "setelah_normalisasi": s3_norm, "setelah_stopword": " ".join(s4_filtered), "clean_text": " ".join(s5_stemmed), "_tokens_clean": s5_stemmed, # list, untuk Counter frekuensi kata } # ═══════════════════════════════════════════════════════════ # UI HELPERS # ═══════════════════════════════════════════════════════════ def _section_header(title, subtitle=""): sub_html = ( f'
{subtitle}
' if subtitle else "" ) st.markdown(f"""
{title}
{sub_html}
""", unsafe_allow_html=True) def _gap(size="md"): heights = {"xs": "0.6rem", "sm": "1rem", "md": "1.45rem", "lg": "2rem"} st.markdown(f'
', unsafe_allow_html=True) # ═══════════════════════════════════════════════════════════ # STYLES # ═══════════════════════════════════════════════════════════ def _render_preprocessing_styles(): st.markdown(""" """, unsafe_allow_html=True) # ═══════════════════════════════════════════════════════════ # PAGE HEADER # ═══════════════════════════════════════════════════════════ def _render_page_header(): st.markdown("""
🧹

Bersihkan Data

Preprocessing teks 5 tahap otomatis: Case Folding → Cleaning → Normalisasi → Stopword Removal → Stemming

✨ NLP Pipeline
""", unsafe_allow_html=True) # ═══════════════════════════════════════════════════════════ # PIPELINE STEPS CARDS # ═══════════════════════════════════════════════════════════ def _render_pipeline_steps(stemmer_ok: bool, norm_count: int, sw_count: int): steps = [ { "num": "01", "anim": "pipe-1", "icon": "🔡", "color": "#0284c7", "dark": "#0c4a6e", "bg": "linear-gradient(135deg,#eff6ff,#dbeafe)", "border": "#bfdbfe", "title": "Case Folding", "desc": "Menyeragamkan semua huruf menjadi lowercase sebelum proses lainnya.", "items": [ '"Gratis" → "gratis"', '"ONGKIR" → "ongkir"', '"KEREN" → "keren"', "Seluruh karakter → huruf kecil", "Dilakukan PERTAMA agar regex & dict konsisten", "Fondasi seluruh tahap berikutnya", ], }, { "num": "02", "anim": "pipe-2", "icon": "🧽", "color": "#3b6cf7", "dark": "#1e3a8a", "bg": "linear-gradient(135deg,#eef2ff,#e0e7ff)", "border": "#c7d2fe", "title": "Cleaning", "desc": "Menghapus semua elemen noise yang tidak bermakna dari teks.", "items": [ "Hapus URL (http, https, www)", "Hapus mention (@username)", "Hapus hashtag (#topik)", "Hapus angka & digit", "Hapus emoji & simbol unicode", "Hapus tanda baca & karakter non-latin", ], }, { "num": "03", "anim": "pipe-3", "icon": "🔄", "color": "#16a34a", "dark": "#14532d", "bg": "linear-gradient(135deg,#f0fdf4,#dcfce7)", "border": "#86efac", "title": f'Normalisasi ✦ {norm_count:,} entri', "desc": "Mengubah kata tidak baku, singkatan, dan slang menjadi kata baku (dari file).", "items": [ "gk/ga/gak/kagak/ngga → tidak", "bgt/bngt/bget → sangat", "ongkir → ongkos kirim", "mantep → mantap", "free → gratis", f"Total: {norm_count:,} pasang slang→baku dimuat", ], }, { "num": "04", "anim": "pipe-4", "icon": "🚫", "color": "#ea580c", "dark": "#7c2d12", "bg": "linear-gradient(135deg,#fff7ed,#ffedd5)", "border": "#fed7aa", "title": f'Stopword Removal ✦ {sw_count:,} kata', "desc": "Membuang kata umum dari file; kata sentimen DIJAGA.", "items": [ f"{sw_count:,} stopword dimuat dari file", "JAGA negasi: tidak, bukan, jangan, belum", "JAGA positif: keren, bagus, mantap, gratis", "JAGA negatif: kecewa, buruk, gagal, mahal, mending, malah", "JAGA intensitas: sangat, banget, sekali", "Hapus token ≤ 2 karakter (noise)", ], }, { "num": "05", "anim": "pipe-5", "icon": "🌱", "color": "#ca8a04", "dark": "#713f12", "bg": "linear-gradient(135deg,#fefce8,#fef9c3)", "border": "#fde68a", "title": "Stemming", "desc": "Mengubah kata ke bentuk dasar via ECS Sastrawi.", "items": [ "berlari → lari", "makanan → makan", "pembatasan → batas", "pengiriman → kirim", f"Status: {'✅ Sastrawi aktif' if stemmer_ok else '⚠️ Sastrawi tidak tersedia'}", "Algoritma: Enhanced Confix Stripping", ], }, ] # Baris 1: 3 kartu pertama row1 = st.columns(3, gap="medium") for col, step in zip(row1, steps[:3]): _render_step_card(col, step) _gap("sm") # Baris 2: 2 kartu terakhir (tengah agar simetris) _, col4, col5, _ = st.columns([0.5, 1, 1, 0.5], gap="medium") _render_step_card(col4, steps[3]) _render_step_card(col5, steps[4]) def _render_step_card(col, step): items_html = "".join( f'
' f'▶' f'{item}' f'
' for item in step["items"] ) with col: st.markdown(f"""
{step['icon']}
Tahap {step['num']}
{step['title']}
{step['desc']}
{items_html}
""", unsafe_allow_html=True) # ═══════════════════════════════════════════════════════════ # FLOW ARROW # ═══════════════════════════════════════════════════════════ def _render_flow_arrow(): nodes = [ ("📄 Teks Asli", "#94a3b8", "#f8fafc", "#e2e8f0"), ("① Case Folding", "#0284c7", "#eff6ff", "#bfdbfe"), ("② Cleaning", "#3b6cf7", "#eef2ff", "#c7d2fe"), ("③ Normalisasi", "#16a34a", "#f0fdf4", "#86efac"), ("④ Stopword", "#ea580c", "#fff7ed", "#fed7aa"), ("⑤ Stemming", "#ca8a04", "#fefce8", "#fde68a"), ("✅ Teks Bersih", "#0f172a", "#0f172a", "#334155"), ] parts = "" for i, (label, color, bg, border) in enumerate(nodes): text_c = "#f8fafc" if label == "✅ Teks Bersih" else color parts += ( f'
{label}
' ) if i < len(nodes) - 1: next_color = nodes[i + 1][1] parts += ( f'
' f'
' f'
' f'
' ) st.markdown(f"""
{parts}
""", unsafe_allow_html=True) # ═══════════════════════════════════════════════════════════ # STAT PILLS # ═══════════════════════════════════════════════════════════ def _render_stat_pills(total_raw, total_clean, removed): c1, c2, c3 = st.columns(3, gap="medium") cards = [ (c1, "📥", "linear-gradient(135deg,#eef2ff,#e0e7ff)", "#3b6cf7", "#1e3a8a", "#c7d2fe", "Tweet Diproses", f"{total_raw:,}", "Total tweet periode ini"), (c2, "✅", "linear-gradient(135deg,#f0fdf4,#dcfce7)", "#16a34a", "#14532d", "#86efac", "Tweet Siap Analisis", f"{total_clean:,}", "Lulus semua 5 tahap"), (c3, "🗑️" if removed > 0 else "✅", ("linear-gradient(135deg,#fff7ed,#ffedd5)" if removed > 0 else "linear-gradient(135deg,#f0fdf4,#dcfce7)"), ("#ea580c" if removed > 0 else "#16a34a"), ("#7c2d12" if removed > 0 else "#14532d"), ("#fed7aa" if removed > 0 else "#86efac"), "Tweet Dibuang", f"{removed:,}", ("Teks kosong setelah preprocessing" if removed > 0 else "Semua tweet lolos")), ] for col, icon, bg, color, dark, border, label, val, sub in cards: with col: fs = "1.1rem" if len(str(val)) > 8 else "1.55rem" st.markdown(f"""
{icon}
{label}
{val}
{sub}
""", unsafe_allow_html=True) # ═══════════════════════════════════════════════════════════ # LIVE EXAMPLE # ═══════════════════════════════════════════════════════════ def _render_live_example(df_c): if df_c.empty: return sample = df_c.sample(1).iloc[0] _section_header( "🔍 Contoh Hasil Preprocessing per Tahap", "Contoh tweet acak dari dataset — refresh halaman untuk contoh berbeda" ) steps_ex = [ ("📄 Teks Asli", "text_asli", "#0f172a", "#f8fafc", "#e2e8f0"), ("① Setelah Case Folding", "setelah_casefolding", "#0c4a6e", "#eff6ff", "#bfdbfe"), ("② Setelah Cleaning", "setelah_cleaning", "#1e3a8a", "#eef2ff", "#c7d2fe"), ("③ Setelah Normalisasi", "setelah_normalisasi", "#14532d", "#f0fdf4", "#86efac"), ("④ Setelah Stopword", "setelah_stopword", "#7c2d12", "#fff7ed", "#fed7aa"), ("⑤ Hasil Akhir (Stem)", "clean_text", "#713f12", "#fefce8", "#fde68a"), ] for label, col_key, text_color, bg, border in steps_ex: raw = sample.get(col_key, "-") text_display = str(raw) if raw and str(raw).strip() else "—" word_count = len(text_display.split()) if text_display != "—" else 0 char_count = len(text_display) if text_display != "—" else 0 char_info = f"{word_count} kata · {char_count} karakter" content_html = ( f'
{text_display}
' ) st.markdown( f'
' f'
' f'{label}' f'{char_info}' f'
' f'{content_html}' f'
', unsafe_allow_html=True ) # ═══════════════════════════════════════════════════════════ # TOP WORDS CHART # ═══════════════════════════════════════════════════════════ def _render_top_words_chart(df_c): _section_header( "📊 Kata-Kata Paling Sering Muncul", f"Dari {len(df_c):,} tweet yang sudah bersih (hasil akhir tahap 5) — Top 20 kata" ) if "_tokens_clean" in df_c.columns: all_words = [ w for tokens in df_c["_tokens_clean"] for w in (tokens if isinstance(tokens, list) else []) ] else: all_words = " ".join(df_c["clean_text"].fillna("")).split() # Filter minimum 3 karakter (konsisten dengan step4) filtered_words = [w for w in all_words if len(w) > 2] word_freq = Counter(filtered_words).most_common(20) if not word_freq: st.info("⚠️ Belum cukup kata untuk ditampilkan.") return words = [w[0] for w in word_freq] counts = [w[1] for w in word_freq] max_c = max(counts) if counts else 1 bar_colors = [f"rgba(59,108,247,{0.35 + 0.65*(c/max_c):.2f})" for c in counts[::-1]] fig = go.Figure(data=[ go.Bar( y=words[::-1], x=counts[::-1], orientation="h", marker=dict(color=bar_colors, line=dict(width=0), cornerradius=6), text=[str(c) for c in counts[::-1]], textposition="outside", textfont=dict(size=10, color="#475569"), hovertemplate="%{y}
Muncul %{x} kali", ) ]) fig.update_layout( height=540, margin=dict(l=0, r=60, t=8, b=8), paper_bgcolor="rgba(0,0,0,0)", plot_bgcolor="rgba(0,0,0,0)", xaxis=dict(showgrid=True, gridcolor="rgba(203,213,225,0.8)", tickfont=dict(size=10, color="#94a3b8"), zeroline=False, showline=False, fixedrange=True), yaxis=dict(showgrid=False, tickfont=dict(size=11, color="#334155"), fixedrange=True), showlegend=False, ) st.markdown("""
""", unsafe_allow_html=True) st.plotly_chart(fig, use_container_width=True, config={"displayModeBar": False}) st.markdown("
", unsafe_allow_html=True) # ═══════════════════════════════════════════════════════════ # MAIN SHOW # ═══════════════════════════════════════════════════════════ def show(): _render_preprocessing_styles() _render_page_header() if "analysis_mode" not in st.session_state: st.warning("⚠️ Silakan pilih mode tampilan di halaman Ambil Data Twitter terlebih dahulu.") return _sync_dynamic_period() start_date = st.session_state.get("filter_start_date") end_date = st.session_state.get("filter_end_date") filter_label = st.session_state.get("filter_label", "-") mode_display = st.session_state.get("mode_display", "-") if start_date is None or end_date is None: st.warning("⚠️ Silakan buka halaman Ambil Data Twitter terlebih dahulu.") return mode_meta = { "realtime": ("#16a34a", "📡"), "30days": ("#3b6cf7", "📅"), "captured": ("#0284c7", "📆"), "custom": ("#d97706", "🔍"), } mode_color, mode_icon = mode_meta.get( st.session_state.analysis_mode, ("#3b6cf7", "📊") ) st.markdown( f'
' f'{mode_icon}' f'
' f'
{mode_display}
' f'
' f'Periode: {filter_label}
' f'
', unsafe_allow_html=True ) # ── Muat resource preprocessing (sekali per session) ────── # Semua tiga resource dimuat di sini, bukan di dalam loop, # agar tidak memuat ulang setiap tweet. stemmer = _load_stemmer() stopwords = _load_stopwords() norm_dict = _load_normalization() # ── Pipeline Overview ────────────────────────────────────── _section_header( "🔬 Alur NLP Pipeline — 5 Tahap Preprocessing", "Setiap tweet diproses berurutan melalui 5 tahap sebelum siap dianalisis sentimennya" ) _render_pipeline_steps( stemmer_ok=stemmer is not None, norm_count=len(norm_dict), sw_count=len(stopwords), ) _gap("sm") _render_flow_arrow() _gap("md") # ── Load data dari database ──────────────────────────────── try: df_all = pd.read_sql("SELECT * FROM tweets ORDER BY created_at DESC", engine) if df_all.empty: st.warning("⚠️ Belum ada data. Kembali ke halaman Ambil Data Twitter.") return df_all["created_at"] = parse_dt(df_all["created_at"]) if "crawled_at" in df_all.columns: df_all["crawled_at"] = parse_crawled_dt(df_all["crawled_at"]) except Exception as e: st.error(f"❌ Gagal membaca database: {e}") return s_dt = pd.Timestamp(start_date) e_dt = pd.Timestamp(end_date) df = df_all[ (df_all["created_at"] >= s_dt) & (df_all["created_at"] <= e_dt) ].copy() if df.empty: st.warning(f"⚠️ Tidak ada tweet dengan tanggal asli dalam periode {filter_label}.") return # ── Cache preprocessing ──────────────────────────────────── # Cache key: kombinasi mode + periode + jumlah tweet di DB # Jika ada tweet baru → cache otomatis invalid → proses ulang total_tweets_in_db = get_tweet_count() latest_crawl_marker = get_latest_crawl_time() or "no-crawl" data_marker = (total_tweets_in_db, latest_crawl_marker) cache_key = ( f"pp5_{st.session_state.analysis_mode}_" f"{start_date}_{end_date}_{total_tweets_in_db}_{latest_crawl_marker}" ) # Bersihkan cache lama untuk mode/periode yang sudah tidak aktif for old_key in list(st.session_state.keys()): if old_key.startswith("pp5_") and old_key != cache_key: del st.session_state[old_key] force_refresh = data_marker != st.session_state.get("_pp_last_data_marker") if cache_key not in st.session_state or force_refresh: with st.spinner("🧹 Menjalankan 5 tahap preprocessing…"): results = [] for _, row in df.iterrows(): # Jalankan 5 tahap — norm_dict diteruskan sebagai parameter r = full_preprocessing( text = row["text"], stopwords = stopwords, stemmer = stemmer, norm_dict = norm_dict, ) r["tweet_id"] = row.get("tweet_id", "") r["text_asli"] = row["text"] r["created_at"] = row["created_at"] r["crawled_at"] = row.get("crawled_at") results.append(r) df_c = pd.DataFrame(results) # Buang baris yang clean_text-nya kosong setelah semua 5 tahap df_c = df_c[df_c["clean_text"].str.strip().str.len() > 0].copy() # Reset index agar rapi df_c = df_c.reset_index(drop=True) st.session_state[cache_key] = df_c st.session_state[cache_key + "_sw_ok"] = stemmer is not None st.session_state["_pp_last_data_marker"] = data_marker df_c = st.session_state[cache_key] stemmer_ok = st.session_state.get(cache_key + "_sw_ok", False) # ── Statistik ────────────────────────────────────────────── removed = len(df) - len(df_c) _section_header( "📌 Ringkasan Hasil Preprocessing", f"Berdasarkan tanggal asli tweet · {filter_label}" ) _gap("xs") _render_stat_pills(len(df), len(df_c), removed) _gap("lg") _render_live_example(df_c) _gap("lg") # ── Tabel ────────────────────────────────────────────────── _section_header( "📋 Tabel Perbandingan Teks per Tahap", f"{len(df_c):,} tweet · {filter_label} — scroll horizontal untuk lihat semua kolom" ) if "crawled_at" not in df_c.columns: df_c = df_c.copy() df_c["crawled_at"] = pd.NaT # Kolom ditampilkan sesuai urutan pipeline disp = df_c[[ "tweet_id", "created_at", "crawled_at", "text_asli", "setelah_casefolding", "setelah_cleaning", "setelah_normalisasi", "setelah_stopword", "clean_text", ]].copy() disp.columns = [ "ID Tweet", "Tanggal Tweet", "Masuk Database", "Teks Asli", "① Case Folding", "② Cleaning", "③ Normalisasi", "④ Stopword", "⑤ Hasil Akhir", ] disp["Tanggal Tweet"] = disp["Tanggal Tweet"].apply(format_dt) disp["Masuk Database"] = disp["Masuk Database"].apply(format_dt) render_standard_table( disp, height=360, min_width=1980, nowrap=["ID Tweet", "Tanggal Tweet", "Masuk Database"], wide_columns=[ "Teks Asli", "① Case Folding", "② Cleaning", "③ Normalisasi", "④ Stopword", "⑤ Hasil Akhir", ], column_widths={ "ID Tweet": "155px", "Tanggal Tweet": "165px", "Masuk Database": "165px", "Teks Asli": "280px", "① Case Folding": "240px", "② Cleaning": "240px", "③ Normalisasi": "240px", "④ Stopword": "230px", "⑤ Hasil Akhir": "230px", }, ) _gap("lg") _render_top_words_chart(df_c) _gap("lg") # ── Simpan ke session state untuk halaman sentimen ───────── st.session_state["preprocessed_df"] = df_c # ── Download ─────────────────────────────────────────────── st.markdown("""
""", unsafe_allow_html=True) d1, d2 = st.columns(2, gap="medium") with d1: st.download_button( "📥 Unduh Hasil Preprocessing Lengkap (semua kolom)", df_c.drop(columns=["_tokens_clean"], errors="ignore" ).to_csv(index=False).encode("utf-8"), f"preprocessing_lengkap_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv", "text/csv", use_container_width=True, ) with d2: out2 = df_c[["tweet_id", "text_asli", "clean_text"]].copy() out2.columns = ["tweet_id", "tweet", "clean_text"] st.download_button( "📥 Unduh Teks Bersih Saja (siap analisis sentimen)", out2.to_csv(index=False).encode("utf-8"), f"teks_bersih_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv", "text/csv", use_container_width=True, ) st.markdown("
", unsafe_allow_html=True) _gap("sm") if st.button( "📈 Lanjut ke Analisis Sentimen →", type="primary", use_container_width=True, ): st.session_state.current_page = "sentiment" st.rerun() _gap("sm")