import streamlit as st import pandas as pd import os import re import string from collections import Counter import plotly.graph_objects as go from datetime import datetime, timedelta from database import engine, get_tweet_count, get_latest_crawl_time from page_modules.table_utils import render_standard_table from timezone_utils import ( parse_dt_with_tz, parse_dt_with_source_tz, get_timezone_label, get_timezone_name, ) # ═══════════════════════════════════════════════════════════ # TIMEZONE HELPERS # ═══════════════════════════════════════════════════════════ def parse_dt(series): return parse_dt_with_tz( series, st.session_state.get("user_timezone", "WIB (UTC+7)") ) def parse_crawled_dt(series): return parse_dt_with_source_tz( series, st.session_state.get("user_timezone", "WIB (UTC+7)"), os.getenv("APP_TIMEZONE", "Asia/Jakarta") ) def format_dt(value): if value is None or pd.isna(value): return "Belum ada" try: tz_label = get_timezone_label( st.session_state.get("user_timezone", "WIB (UTC+7)") ) return f"{value.strftime('%d/%m/%Y %H:%M')} {tz_label}" except Exception: return "Belum ada" def user_today(): timezone_choice = st.session_state.get("user_timezone", "WIB (UTC+7)") return pd.Timestamp.now(tz=get_timezone_name(timezone_choice)).date() def _sync_dynamic_period(): mode = st.session_state.get("analysis_mode") today = user_today() configs = { "realtime": (today - timedelta(days=6), today, "Tweet Terkini — 7 Hari Terakhir"), "30days": (today - timedelta(days=29), today, "30 Hari Terakhir"), "captured": (today, today, "Tweet Hari Ini"), } if mode not in configs: return start_day, end_day, mode_display = configs[mode] dt_start = datetime.combine(start_day, datetime.min.time()) dt_end = datetime.combine(end_day, datetime.max.time().replace(microsecond=0)) st.session_state.filter_start_date = dt_start st.session_state.filter_end_date = dt_end st.session_state.filter_label = f"{dt_start.strftime('%d/%m/%Y')} s/d {dt_end.strftime('%d/%m/%Y')}" st.session_state.mode_display = mode_display st.session_state.filter_date_column = "created_at" # ═══════════════════════════════════════════════════════════ # PREPROCESSING PIPELINE — 6 TAHAP # # Urutan tahap: # 1. Cleaning — hapus noise (URL, mention, hashtag, angka, # tanda baca, emoji, karakter non-latin) # 2. Case Folding — ubah semua huruf jadi huruf kecil (lowercase) # 3. Normalisasi — ganti singkatan/slang → kata baku # 4. Tokenizing — pecah kalimat menjadi list token/kata # 5. Stopword Removal — buang kata umum yang tidak bermakna # 6. Stemming — bentuk dasar kata via Sastrawi # # Catatan desain: # • Case Folding dilakukan SETELAH cleaning awal supaya regex pendeteksi # emoji/karakter khusus tetap bekerja dengan baik, lalu baru dilanjut # ke langkah selanjutnya yang semuanya butuh lowercase. # • Tokenizing menghasilkan list; untuk keperluan tabel & analisis kata # hasil tokenizing dikembalikan sebagai string (join spasi) agar mudah # disimpan dan diproses step berikutnya. # • Normalisasi berjalan pada level token sehingga pencocokan kata tepat # (tidak partial match di tengah kata). # ═══════════════════════════════════════════════════════════ def _load_stopwords(): """ Load stopword dari file indonesian-stopwords-complete.txt. Kata-kata negasi penting (tidak, bukan, jangan, kurang, belum) dikeluarkan dari stopword agar tidak hilang saat filtering — keberadaannya krusial untuk analisis sentimen. """ stopword_file = "indonesian-stopwords-complete.txt" base = set() try: with open(stopword_file, "r", encoding="utf-8") as f: base = set(f.read().splitlines()) # Pertahankan kata negasi — penting untuk sentimen for kata in ["tidak", "bukan", "jangan", "kurang", "belum", "tanpa"]: base.discard(kata) except FileNotFoundError: # Fallback minimal jika file tidak ada base = { "yang", "dan", "di", "ke", "dari", "ini", "itu", "dengan", "untuk", "pada", "adalah", "oleh", "ada", "ya", "akan", "atau", "juga", "sama", "karena", "jika", "sudah", "telah", } # Tambahan stopword domain-spesifik & informal base.update({ "rt", "amp", "https", "http", "co", "t", "wkwk", "wkwkwk", "haha", "hehe", "xixi", "yg", "dgn", "utk", "dr", "krn", "tp", "jd", "sdh", "aja", "doang", "banget", "bgt", "nih", "sih", "dong", "deh", "loh", "lah", "tuh", "dong", "kak", "gan", }) return base def _load_stemmer(): """ Load stemmer Sastrawi jika tersedia. Sastrawi adalah library stemming Bahasa Indonesia berbasis algoritma Enhanced Confix Stripping (ECS) yang mampu menghapus awalan dan akhiran secara bertahap (berlari → lari, makanan → makan, dst). Mengembalikan None jika Sastrawi tidak terinstall — stemming di-skip. """ try: from Sastrawi.Stemmer.StemmerFactory import StemmerFactory return StemmerFactory().create_stemmer() except Exception: return None # Kamus normalisasi: kata informal/singkatan → kata baku # Mencakup: singkatan SMS/chat, kata gaul, singkatan domain crawling NORMALISASI = { # Negasi "gk": "tidak", "ga": "tidak", "gak": "tidak", "nggak": "tidak", "ngga": "tidak", "tdk": "tidak", "tak": "tidak", "enggak": "tidak", # Kata ganti & preposisi "yg": "yang", "dgn": "dengan", "utk": "untuk", "org": "orang", "krn": "karena", "dr": "dari", "sm": "sama", "pd": "pada", "dlm": "dalam", # Verba & adjektiva "tp": "tapi", "tpi": "tapi", "jd": "jadi", "sdh": "sudah", "blm": "belum", "emg": "memang", "emang": "memang", "gimana": "bagaimana", "gitu": "begitu", "gini": "begini", "bgt": "banget", "udah": "sudah", "udh": "sudah", "mau": "mau", "bngt": "banget", "bener": "benar", "beneran": "benar", "mantep": "mantap", "keren": "keren", # Domain e-commerce & topik crawling "ongkir": "ongkos kirim", "freeongkir": "gratis ongkos kirim", "free": "gratis", "ecommerce": "e commerce", "komdigi": "komdigi", "subsidi": "subsidi", "marketplace": "marketplace", "seller": "penjual", "buyer": "pembeli", "online": "online", "shopee": "shopee", "tokopedia": "tokopedia", "lazada": "lazada", "tiktok": "tiktok", } # ── Tahap 1: Cleaning ──────────────────────────────────────────────────────── # Menghapus semua elemen yang bukan teks bermakna: # URL, mention (@user), hashtag (#), angka, tanda baca standar, # emoji & simbol unicode, dan karakter non-latin (arab, kanji, dll). # Teks masih bisa uppercase/mixed-case di tahap ini. def step1_cleaning(text: str) -> str: text = str(text) text = re.sub(r"http\S+|www\S+|https\S+", "", text) # hapus URL text = re.sub(r"@\w+", "", text) # hapus mention text = re.sub(r"#\w+", "", text) # hapus hashtag (beserta kata) text = re.sub(r"\d+", "", text) # hapus angka # Hapus emoji & simbol unicode (range karakter di luar Basic Latin) text = re.sub( r"[\U00010000-\U0010ffff" # supplementary planes (emoji berwarna) r"\U0001F600-\U0001F64F" # emoticons r"\U0001F300-\U0001F5FF" # symbols & pictographs r"\U0001F680-\U0001F6FF" # transport & map r"\U0001F1E0-\U0001F1FF" # flags r"\u2600-\u26FF\u2700-\u27BF" # misc symbols & dingbats r"]+", "", text, flags=re.UNICODE ) text = text.translate(str.maketrans("", "", string.punctuation)) # hapus tanda baca text = re.sub(r"[^a-zA-Z\s]", "", text) # hapus karakter non-latin text = re.sub(r"\s+", " ", text).strip() return text # ── Tahap 2: Case Folding ──────────────────────────────────────────────────── # Mengubah seluruh teks menjadi huruf kecil (lowercase). # Dipisah dari cleaning agar jelas sebagai tahap tersendiri dalam pipeline. def step2_case_folding(text: str) -> str: return text.lower() # ── Tahap 3: Normalisasi ───────────────────────────────────────────────────── # Mengganti kata tidak baku / singkatan / slang dengan kata baku. # Pencocokan dilakukan per-token (bukan substring) agar "dr" di tengah kata # tidak ikut diganti. def step3_normalization(text: str) -> str: return " ".join(NORMALISASI.get(word, word) for word in text.split()) # ── Tahap 4: Tokenizing ────────────────────────────────────────────────────── # Memecah kalimat menjadi daftar token (kata). # Hasilnya dikembalikan sebagai list untuk proses berikutnya, # tapi juga disimpan sebagai string (join) untuk tampilan tabel. def step4_tokenizing(text: str) -> list: return text.split() # ── Tahap 5: Stopword Removal ──────────────────────────────────────────────── # Membuang kata-kata yang tidak bermakna dari daftar token. # Juga membuang token dengan panjang ≤ 2 karakter. # Kata negasi (tidak, bukan, jangan, dll) TIDAK dihapus. def step5_stopword_removal(tokens: list, stopwords: set) -> list: return [w for w in tokens if w not in stopwords and len(w) > 2] # ── Tahap 6: Stemming ──────────────────────────────────────────────────────── # Mengubah kata ke bentuk dasar menggunakan algoritma Enhanced Confix # Stripping (ECS) dari library Sastrawi. # Contoh: "berlari" → "lari", "makanan" → "makan", "pembatasan" → "batas" # Jika Sastrawi tidak tersedia, token dikembalikan apa adanya. def step6_stemming(tokens: list, stemmer) -> list: if stemmer is None: return tokens return [stemmer.stem(w) for w in tokens] def full_preprocessing(text: str, stopwords: set, stemmer): """ Menjalankan seluruh pipeline 6 tahap dan mengembalikan dict berisi hasil di setiap tahap untuk keperluan tampilan tabel & debugging. """ s1_clean = step1_cleaning(text) s2_fold = step2_case_folding(s1_clean) s3_norm = step3_normalization(s2_fold) s4_tokens = step4_tokenizing(s3_norm) s5_filtered = step5_stopword_removal(s4_tokens, stopwords) s6_stemmed = step6_stemming(s5_filtered, stemmer) return { "setelah_cleaning": s1_clean, "setelah_casefolding": s2_fold, "setelah_normalisasi": s3_norm, "setelah_tokenizing": " | ".join(s4_tokens), # tampilkan token dengan separator "setelah_stopword": " ".join(s5_filtered), "clean_text": " ".join(s6_stemmed), # hasil akhir untuk analisis sentimen "_tokens_raw": s4_tokens, # list mentah (untuk statistik) "_tokens_clean": s6_stemmed, # list bersih (untuk word freq) } # ═══════════════════════════════════════════════════════════ # UI HELPERS # ═══════════════════════════════════════════════════════════ def _section_header(title, subtitle=""): sub_html = ( f'
{subtitle}
' if subtitle else "" ) st.markdown(f"""
{title}
{sub_html}
""", unsafe_allow_html=True) def _gap(size="md"): heights = {"xs": "0.6rem", "sm": "1rem", "md": "1.45rem", "lg": "2rem"} st.markdown(f'
', unsafe_allow_html=True) def _card_wrap(content_fn, *args, **kwargs): """Wrapper card tanpa key — kompatibel semua versi Streamlit.""" st.markdown("""
""", unsafe_allow_html=True) content_fn(*args, **kwargs) # ═══════════════════════════════════════════════════════════ # STYLES # ═══════════════════════════════════════════════════════════ def _render_preprocessing_styles(): st.markdown(""" """, unsafe_allow_html=True) # ═══════════════════════════════════════════════════════════ # PAGE HEADER # ═══════════════════════════════════════════════════════════ def _render_page_header(): st.markdown("""
🧹

Bersihkan Data

Preprocessing teks 6 tahap otomatis: Cleaning → Case Folding → Normalisasi → Tokenizing → Stopword Removal → Stemming

✨ NLP Pipeline
""", unsafe_allow_html=True) # ═══════════════════════════════════════════════════════════ # PIPELINE STEPS CARDS — 6 TAHAP # ═══════════════════════════════════════════════════════════ def _render_pipeline_steps(stemmer_ok): """ Menampilkan 6 kartu tahap preprocessing dalam 2 baris × 3 kolom. Setiap kartu berisi nomor tahap, ikon, judul, deskripsi singkat, dan daftar aksi yang dilakukan. """ steps = [ { "num": "01", "anim": "pipe-1", "icon": "🧽", "color": "#3b6cf7", "dark": "#1e3a8a", "bg": "linear-gradient(135deg,#eef2ff,#e0e7ff)", "border": "#c7d2fe", "title": "Cleaning", "desc": "Menghapus semua elemen noise yang tidak bermakna dari teks mentah.", "items": [ "Hapus URL (http, https, www)", "Hapus mention (@username)", "Hapus hashtag (#topik)", "Hapus angka & digit", "Hapus emoji & simbol unicode", "Hapus tanda baca & karakter non-latin", ], }, { "num": "02", "anim": "pipe-2", "icon": "🔡", "color": "#0284c7", "dark": "#0c4a6e", "bg": "linear-gradient(135deg,#eff6ff,#dbeafe)", "border": "#bfdbfe", "title": "Case Folding", "desc": "Menyeragamkan huruf menjadi lowercase agar tidak ada duplikat akibat perbedaan kapitalisasi.", "items": [ "\"Gratis\" → \"gratis\"", "\"ONGKIR\" → \"ongkir\"", "\"Komdigi\" → \"komdigi\"", "Seluruh karakter → huruf kecil", ], }, { "num": "03", "anim": "pipe-3", "icon": "🔄", "color": "#16a34a", "dark": "#14532d", "bg": "linear-gradient(135deg,#f0fdf4,#dcfce7)", "border": "#86efac", "title": "Normalisasi", "desc": "Mengubah kata tidak baku, singkatan, dan slang menjadi kata baku standar.", "items": [ "gk / ga / gak → tidak", "ongkir → ongkos kirim", "bgt → banget", "gimana → bagaimana", "free → gratis", "seller → penjual", ], }, { "num": "04", "anim": "pipe-4", "icon": "✂️", "color": "#7c3aed", "dark": "#3b0764", "bg": "linear-gradient(135deg,#f5f3ff,#ede9fe)", "border": "#c4b5fd", "title": "Tokenizing", "desc": "Memecah kalimat menjadi unit kata (token) yang dapat diproses secara individual.", "items": [ "Pisahkan berdasarkan spasi", "\"gratis ongkos kirim\" → [gratis, ongkos, kirim]", "Setiap token diproses mandiri", "Hasil: daftar kata terpisah", ], }, { "num": "05", "anim": "pipe-5", "icon": "🚫", "color": "#ea580c", "dark": "#7c2d12", "bg": "linear-gradient(135deg,#fff7ed,#ffedd5)", "border": "#fed7aa", "title": "Stopword Removal", "desc": "Membuang kata-kata umum yang tidak berkontribusi pada makna atau sentimen teks.", "items": [ "Hapus kata umum (dan, di, ke, dari…)", "Hapus kata < 3 karakter", "PERTAHANKAN: tidak, bukan, jangan", "PERTAHANKAN: belum, kurang, tanpa", f"Sumber: indonesian-stopwords-complete.txt", "Kata negasi dijaga untuk sentimen", ], }, { "num": "06", "anim": "pipe-6", "icon": "🌱", "color": "#ca8a04", "dark": "#713f12", "bg": "linear-gradient(135deg,#fefce8,#fef9c3)", "border": "#fde68a", "title": "Stemming", "desc": "Mengubah kata ke bentuk dasarnya menggunakan algoritma Enhanced Confix Stripping (ECS) Sastrawi.", "items": [ "berlari → lari", "makanan → makan", "pembatasan → batas", "pengiriman → kirim", f"Status: {'✅ Sastrawi aktif' if stemmer_ok else '⚠️ Sastrawi tidak terinstall — skip'}", "Algoritma: Enhanced Confix Stripping", ], }, ] # Baris pertama: tahap 1–3 row1 = st.columns(3, gap="medium") for col, step in zip(row1, steps[:3]): _render_step_card(col, step) _gap("sm") # Baris kedua: tahap 4–6 row2 = st.columns(3, gap="medium") for col, step in zip(row2, steps[3:]): _render_step_card(col, step) def _render_step_card(col, step): items_html = "".join( f'
' f'▶' f'{item}' f'
' for item in step["items"] ) with col: st.markdown(f"""
{step['icon']}
Tahap {step['num']}
{step['title']}
{step['desc']}
{items_html}
""", unsafe_allow_html=True) # ═══════════════════════════════════════════════════════════ # FLOW ARROW — pipeline visual # ═══════════════════════════════════════════════════════════ def _render_flow_arrow(): nodes = [ ("📄 Teks Asli", "#94a3b8", "#f8fafc", "#e2e8f0"), ("① Cleaning", "#3b6cf7", "#eef2ff", "#c7d2fe"), ("② Case Folding", "#0284c7", "#eff6ff", "#bfdbfe"), ("③ Normalisasi", "#16a34a", "#f0fdf4", "#86efac"), ("④ Tokenizing", "#7c3aed", "#f5f3ff", "#c4b5fd"), ("⑤ Stopword", "#ea580c", "#fff7ed", "#fed7aa"), ("⑥ Stemming", "#ca8a04", "#fefce8", "#fde68a"), ("✅ Teks Bersih", "#0f172a", "#0f172a", "#334155"), ] parts = "" for i, (label, color, bg, border) in enumerate(nodes): text_c = "#f8fafc" if label == "✅ Teks Bersih" else color parts += ( f'
{label}
' ) if i < len(nodes) - 1: next_color = nodes[i + 1][1] parts += ( f'
' f'
' f'
' f'
' ) st.markdown(f"""
{parts}
""", unsafe_allow_html=True) # ═══════════════════════════════════════════════════════════ # STAT PILLS # ═══════════════════════════════════════════════════════════ def _render_stat_pills(total_raw, total_clean, avg_tokens_before, avg_tokens_after, removed): """ Menampilkan 4 kartu ringkasan hasil preprocessing: 1. Total tweet yang masuk pipeline 2. Tweet yang lolos semua tahap (clean_text tidak kosong) 3. Tweet yang dibuang (clean_text kosong setelah semua tahap) 4. Rata-rata jumlah token sebelum vs sesudah stopword removal """ c1, c2, c3, c4 = st.columns(4, gap="medium") cards = [ (c1, "📥", "linear-gradient(135deg,#eef2ff,#e0e7ff)", "#3b6cf7", "#1e3a8a", "#c7d2fe", "Tweet Diproses", f"{total_raw:,}", "Total tweet periode ini"), (c2, "✅", "linear-gradient(135deg,#f0fdf4,#dcfce7)", "#16a34a", "#14532d", "#86efac", "Tweet Siap Analisis", f"{total_clean:,}", "Lulus semua 6 tahap"), (c3, "🗑️" if removed > 0 else "✅", ("linear-gradient(135deg,#fff7ed,#ffedd5)" if removed > 0 else "linear-gradient(135deg,#f0fdf4,#dcfce7)"), ("#ea580c" if removed > 0 else "#16a34a"), ("#7c2d12" if removed > 0 else "#14532d"), ("#fed7aa" if removed > 0 else "#86efac"), "Tweet Dibuang", f"{removed:,}", ("Teks kosong setelah preprocessing" if removed > 0 else "Semua tweet lolos")), (c4, "🔤", "linear-gradient(135deg,#f5f3ff,#ede9fe)", "#7c3aed", "#3b0764", "#c4b5fd", "Rata-rata Token", f"{avg_tokens_before:.0f} → {avg_tokens_after:.0f}", "Sebelum → Sesudah stopword"), ] for col, icon, bg, color, dark, border, label, val, sub in cards: with col: fs = "1.1rem" if len(str(val)) > 8 else "1.55rem" st.markdown(f"""
{icon}
{label}
{val}
{sub}
""", unsafe_allow_html=True) # ═══════════════════════════════════════════════════════════ # LIVE EXAMPLE — contoh hasil tiap tahap # ═══════════════════════════════════════════════════════════ def _render_live_example(df_c): if df_c.empty: return sample = df_c.sample(1).iloc[0] _section_header( "🔍 Contoh Hasil Preprocessing per Tahap", "Contoh tweet acak dari dataset — refresh halaman untuk contoh berbeda" ) steps_ex = [ ("📄 Teks Asli", "text_asli", "#0f172a", "#f8fafc", "#e2e8f0", False), ("① Setelah Cleaning", "setelah_cleaning", "#1e3a8a", "#eef2ff", "#c7d2fe", False), ("② Setelah Case Folding","setelah_casefolding", "#0c4a6e", "#eff6ff", "#bfdbfe", False), ("③ Setelah Normalisasi", "setelah_normalisasi", "#14532d", "#f0fdf4", "#86efac", False), ("④ Setelah Tokenizing", "setelah_tokenizing", "#3b0764", "#f5f3ff", "#c4b5fd", True), # tampilkan sebagai chip ("⑤ Setelah Stopword", "setelah_stopword", "#7c2d12", "#fff7ed", "#fed7aa", False), ("⑥ Hasil Akhir (Stem)", "clean_text", "#713f12", "#fefce8", "#fde68a", False), ] for label, col_key, text_color, bg, border, is_token in steps_ex: raw = sample.get(col_key, "-") text_display = str(raw) if raw and str(raw).strip() else "—" # Untuk tokenizing, tampilkan sebagai chip/badge per kata if is_token and text_display != "—": words = text_display.split(" | ") chips = "".join( f'{w}' for w in words if w.strip() ) content_html = f'
{chips}
' word_count = len(words) char_info = f"{word_count} token" else: content_html = ( f'
{text_display}
' ) word_count = len(text_display.split()) if text_display != "—" else 0 char_count = len(text_display) if text_display != "—" else 0 char_info = f"{word_count} kata · {char_count} karakter" st.markdown( f'
' f'
' f'{label}' f'{char_info}' f'
' f'{content_html}' f'
', unsafe_allow_html=True ) # ═══════════════════════════════════════════════════════════ # LENGTH COMPARISON CHART # ═══════════════════════════════════════════════════════════ def _render_length_comparison_chart(df, df_c): """ Grafik batang: rata-rata jumlah karakter per tweet di setiap tahap. Catatan: jumlah karakter bisa naik di tahap Normalisasi karena singkatan diekspansi (ongkir 6 kar → ongkos kirim 12 kar) — ini NORMAL. Turun signifikan terjadi di tahap Stopword dan Stemming. """ _section_header( "📐 Perbandingan Panjang Teks per Tahap", "Rata-rata jumlah karakter per tweet di setiap tahap preprocessing · Naik di normalisasi adalah normal (ekspansi singkatan)" ) stages = [ "Teks Asli", "① Cleaning", "② Case Folding", "③ Normalisasi", "④ Tokenizing", "⑤ Stopword", "⑥ Stemming" ] avgs = [ df["text"].astype(str).str.len().mean(), df_c["setelah_cleaning"].astype(str).str.len().mean(), df_c["setelah_casefolding"].astype(str).str.len().mean(), df_c["setelah_normalisasi"].astype(str).str.len().mean(), # tokenizing: hitung panjang tanpa separator " | " df_c["setelah_tokenizing"].apply( lambda x: len(" ".join(str(x).split(" | "))) ).mean(), df_c["setelah_stopword"].astype(str).str.len().mean(), df_c["clean_text"].astype(str).str.len().mean(), ] avgs = [round(a, 1) for a in avgs] colors = ["#94a3b8", "#3b6cf7", "#0284c7", "#16a34a", "#7c3aed", "#ea580c", "#ca8a04"] fig = go.Figure(data=[ go.Bar( x=stages, y=avgs, marker=dict(color=colors, line=dict(width=0), opacity=0.9, cornerradius=8), text=[f"{v}" for v in avgs], textposition="outside", textfont=dict(size=10, color="#475569"), width=0.55, hovertemplate="%{x}
Rata-rata: %{y} karakter", ) ]) fig.update_layout( height=300, margin=dict(l=0, r=10, t=24, b=10), paper_bgcolor="rgba(0,0,0,0)", plot_bgcolor="rgba(0,0,0,0)", xaxis=dict(tickfont=dict(size=10, color="#64748b"), showgrid=False, zeroline=False, showline=False, fixedrange=True), yaxis=dict(tickfont=dict(size=9, color="#94a3b8"), showgrid=True, gridcolor="rgba(226,232,240,0.7)", griddash="dot", gridwidth=1, zeroline=False, showline=False, fixedrange=True), showlegend=False, hovermode="x unified", ) st.markdown("""
""", unsafe_allow_html=True) st.plotly_chart(fig, use_container_width=True, config={"displayModeBar": False}) st.markdown("
", unsafe_allow_html=True) # ═══════════════════════════════════════════════════════════ # TOP WORDS CHART # ═══════════════════════════════════════════════════════════ def _render_top_words_chart(df_c): _section_header( "📊 Kata-Kata Paling Sering Muncul", f"Dari {len(df_c):,} tweet yang sudah bersih (hasil akhir tahap 6) — Top 20 kata" ) # Gunakan _tokens_clean (list) jika tersedia, fallback ke clean_text string if "_tokens_clean" in df_c.columns: all_words = [w for tokens in df_c["_tokens_clean"] for w in (tokens if isinstance(tokens, list) else [])] else: all_words = " ".join(df_c["clean_text"].fillna("")).split() filtered_words = [w for w in all_words if len(w) > 2] word_freq = Counter(filtered_words).most_common(20) if not word_freq: st.info("⚠️ Belum cukup kata untuk ditampilkan.") return words = [w[0] for w in word_freq] counts = [w[1] for w in word_freq] max_c = max(counts) if counts else 1 bar_colors = [f"rgba(59,108,247,{0.35 + 0.65*(c/max_c):.2f})" for c in counts[::-1]] fig = go.Figure(data=[ go.Bar( y=words[::-1], x=counts[::-1], orientation="h", marker=dict(color=bar_colors, line=dict(width=0), cornerradius=6), text=[str(c) for c in counts[::-1]], textposition="outside", textfont=dict(size=10, color="#475569"), hovertemplate="%{y}
Muncul %{x} kali", ) ]) fig.update_layout( height=540, margin=dict(l=0, r=60, t=8, b=8), paper_bgcolor="rgba(0,0,0,0)", plot_bgcolor="rgba(0,0,0,0)", xaxis=dict(showgrid=True, gridcolor="rgba(203,213,225,0.8)", tickfont=dict(size=10, color="#94a3b8"), zeroline=False, showline=False, fixedrange=True), yaxis=dict(showgrid=False, tickfont=dict(size=11, color="#334155"), fixedrange=True), showlegend=False, ) # ── Tanpa key — kompatibel semua versi Streamlit ── st.markdown("""
""", unsafe_allow_html=True) st.plotly_chart(fig, use_container_width=True, config={"displayModeBar": False}) st.markdown("
", unsafe_allow_html=True) # ═══════════════════════════════════════════════════════════ # MAIN SHOW # ═══════════════════════════════════════════════════════════ def show(): _render_preprocessing_styles() _render_page_header() if "analysis_mode" not in st.session_state: st.warning("⚠️ Silakan pilih mode tampilan di halaman Ambil Data Twitter terlebih dahulu.") return _sync_dynamic_period() start_date = st.session_state.get("filter_start_date") end_date = st.session_state.get("filter_end_date") filter_label = st.session_state.get("filter_label", "-") mode_display = st.session_state.get("mode_display", "-") if start_date is None or end_date is None: st.warning("⚠️ Silakan buka halaman Ambil Data Twitter terlebih dahulu.") return mode_meta = { "realtime": ("#16a34a", "📡"), "30days": ("#3b6cf7", "📅"), "captured": ("#0284c7", "📆"), "custom": ("#d97706", "🔍"), } mode_color, mode_icon = mode_meta.get( st.session_state.analysis_mode, ("#3b6cf7", "📊") ) # ── Active Period Banner ────────────────────────────────── st.markdown( f'
' f'{mode_icon}' f'
' f'
{mode_display}
' f'
' f'Periode: {filter_label}
' f'
', unsafe_allow_html=True ) # ── Pipeline Overview ───────────────────────────────────── _section_header( "🔬 Alur NLP Pipeline — 6 Tahap Preprocessing", "Setiap tweet diproses berurutan melalui 6 tahap sebelum siap dianalisis sentimennya" ) stemmer_tmp = _load_stemmer() _render_pipeline_steps(stemmer_tmp is not None) _gap("sm") _render_flow_arrow() _gap("md") # ── Load data dari DB ───────────────────────────────────── try: df_all = pd.read_sql("SELECT * FROM tweets ORDER BY created_at DESC", engine) if df_all.empty: st.warning("⚠️ Belum ada data. Kembali ke halaman Ambil Data Twitter.") return df_all["created_at"] = parse_dt(df_all["created_at"]) if "crawled_at" in df_all.columns: df_all["crawled_at"] = parse_crawled_dt(df_all["crawled_at"]) except Exception as e: st.error(f"❌ Gagal membaca database: {e}") return s_dt = pd.Timestamp(start_date) e_dt = pd.Timestamp(end_date) df = df_all[(df_all["created_at"] >= s_dt) & (df_all["created_at"] <= e_dt)].copy() if df.empty: st.warning(f"⚠️ Tidak ada tweet dengan tanggal asli dalam periode {filter_label}.") return # ── Cache preprocessing ─────────────────────────────────── total_tweets_in_db = get_tweet_count() latest_crawl_marker = get_latest_crawl_time() or "no-crawl" data_marker = (total_tweets_in_db, latest_crawl_marker) cache_key = ( f"pp6_{st.session_state.analysis_mode}_" f"{start_date}_{end_date}_{total_tweets_in_db}_{latest_crawl_marker}" ) # Hapus cache lama dari session state for old_key in list(st.session_state.keys()): if old_key.startswith("pp6_") and old_key != cache_key: del st.session_state[old_key] force_refresh = data_marker != st.session_state.get("_pp_last_data_marker") if cache_key not in st.session_state or force_refresh: stemmer = _load_stemmer() stopwords = _load_stopwords() with st.spinner("🧹 Menjalankan 6 tahap preprocessing…"): results = [] for _, row in df.iterrows(): r = full_preprocessing(row["text"], stopwords, stemmer) r["tweet_id"] = row.get("tweet_id", "") r["text_asli"] = row["text"] r["created_at"] = row["created_at"] r["crawled_at"] = row.get("crawled_at") results.append(r) df_c = pd.DataFrame(results) # Buang tweet yang clean_text-nya kosong setelah semua tahap df_c = df_c[df_c["clean_text"].str.strip().str.len() > 0].copy() st.session_state[cache_key] = df_c st.session_state[cache_key + "_stemmer_ok"] = stemmer is not None st.session_state["_pp_last_data_marker"] = data_marker df_c = st.session_state[cache_key] stemmer_ok = st.session_state.get(cache_key + "_stemmer_ok", False) # ── Hitung statistik ringkasan ──────────────────────────── # Token sebelum stopword = jumlah kata di tahap tokenizing avg_tok_before = df_c["setelah_tokenizing"].apply( lambda x: len(str(x).split(" | ")) if str(x).strip() else 0 ).mean() # Token sesudah stopword = jumlah kata di clean_text avg_tok_after = df_c["clean_text"].apply( lambda x: len(str(x).split()) if str(x).strip() else 0 ).mean() removed = len(df) - len(df_c) # ── Ringkasan ───────────────────────────────────────────── _section_header( "📌 Ringkasan Hasil Preprocessing", f"Berdasarkan tanggal asli tweet · {filter_label}" ) _gap("xs") _render_stat_pills(len(df), len(df_c), avg_tok_before, avg_tok_after, removed) _gap("lg") # ── Live example ────────────────────────────────────────── _render_live_example(df_c) _gap("lg") # ── Tabel perbandingan tahap ────────────────────────────── _section_header( "📋 Tabel Perbandingan Teks per Tahap", f"{len(df_c):,} tweet · {filter_label} — scroll horizontal untuk lihat semua kolom" ) if "crawled_at" not in df_c.columns: df_c = df_c.copy() df_c["crawled_at"] = pd.NaT disp = df_c[[ "tweet_id", "created_at", "crawled_at", "text_asli", "setelah_cleaning", "setelah_casefolding", "setelah_normalisasi", "setelah_tokenizing", "setelah_stopword", "clean_text", ]].copy() disp.columns = [ "ID Tweet", "Tanggal Tweet", "Masuk Database", "Teks Asli", "① Cleaning", "② Case Folding", "③ Normalisasi", "④ Tokenizing", "⑤ Stopword", "⑥ Hasil Akhir", ] disp["Tanggal Tweet"] = disp["Tanggal Tweet"].apply(format_dt) disp["Masuk Database"] = disp["Masuk Database"].apply(format_dt) render_standard_table( disp, height=360, min_width=2200, nowrap=["ID Tweet", "Tanggal Tweet", "Masuk Database"], wide_columns=[ "Teks Asli", "① Cleaning", "② Case Folding", "③ Normalisasi", "④ Tokenizing", "⑤ Stopword", "⑥ Hasil Akhir", ], column_widths={ "ID Tweet": "155px", "Tanggal Tweet": "165px", "Masuk Database": "165px", "Teks Asli": "280px", "① Cleaning": "240px", "② Case Folding": "220px", "③ Normalisasi": "220px", "④ Tokenizing": "240px", "⑤ Stopword": "220px", "⑥ Hasil Akhir": "220px", }, ) _gap("lg") # ── Chart panjang teks ──────────────────────────────────── _render_length_comparison_chart(df, df_c) _gap("lg") # ── Top words ───────────────────────────────────────────── _render_top_words_chart(df_c) _gap("lg") # ── Simpan ke session state untuk halaman sentimen ──────── st.session_state["preprocessed_df"] = df_c # ── Download buttons ────────────────────────────────────── st.markdown("""
""", unsafe_allow_html=True) d1, d2 = st.columns(2, gap="medium") with d1: st.download_button( "📥 Unduh Hasil Preprocessing Lengkap (semua kolom)", df_c.drop(columns=["_tokens_raw", "_tokens_clean"], errors="ignore" ).to_csv(index=False).encode("utf-8"), f"preprocessing_lengkap_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv", "text/csv", use_container_width=True, ) with d2: out2 = df_c[["tweet_id", "text_asli", "clean_text"]].copy() out2.columns = ["tweet_id", "tweet", "clean_text"] st.download_button( "📥 Unduh Teks Bersih Saja (siap analisis sentimen)", out2.to_csv(index=False).encode("utf-8"), f"teks_bersih_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv", "text/csv", use_container_width=True, ) st.markdown("
", unsafe_allow_html=True) _gap("sm") # ── Navigasi ────────────────────────────────────────────── if st.button( "📈 Lanjut ke Analisis Sentimen →", type="primary", use_container_width=True, ): st.session_state.current_page = "sentiment" st.rerun() _gap("sm")