""" preprocessing_page.py ===================== Halaman Bersihkan Data — NLP Pipeline 5 Tahap. PIPELINE 5 TAHAP (selaras dengan sentiment_service.py): 1. Case Folding — ubah semua huruf jadi lowercase 2. Cleaning — hapus URL, mention, hashtag, angka, emoji, tanda baca 3. Normalisasi — singkatan/slang → kata baku 4. Stopword Removal — hapus kata umum; JAGA kata sentimen penting 5. Stemming — bentuk dasar kata via Sastrawi ECS Catatan: Tokenizing tidak ditampilkan sebagai tahap tersendiri karena: • Stopword removal & stemming sudah melakukan split() secara internal • TF-IDF melakukan tokenisasi sendiri saat inferensi → Tokenizing adalah proses teknis, bukan tahap utama pipeline """ import streamlit as st import pandas as pd import os import re import string from collections import Counter import plotly.graph_objects as go from datetime import datetime, timedelta from database import engine, get_tweet_count, get_latest_crawl_time from page_modules.table_utils import render_standard_table from timezone_utils import ( parse_dt_with_tz, parse_dt_with_source_tz, get_timezone_label, get_timezone_name, ) # ═══════════════════════════════════════════════════════════ # TIMEZONE HELPERS # ═══════════════════════════════════════════════════════════ def parse_dt(series): return parse_dt_with_tz( series, st.session_state.get("user_timezone", "WIB (UTC+7)") ) def parse_crawled_dt(series): return parse_dt_with_source_tz( series, st.session_state.get("user_timezone", "WIB (UTC+7)"), os.getenv("APP_TIMEZONE", "Asia/Jakarta") ) def format_dt(value): if value is None or pd.isna(value): return "Belum ada" try: tz_label = get_timezone_label( st.session_state.get("user_timezone", "WIB (UTC+7)") ) return f"{value.strftime('%d/%m/%Y %H:%M')} {tz_label}" except Exception: return "Belum ada" def user_today(): timezone_choice = st.session_state.get("user_timezone", "WIB (UTC+7)") return pd.Timestamp.now(tz=get_timezone_name(timezone_choice)).date() def _sync_dynamic_period(): mode = st.session_state.get("analysis_mode") today = user_today() configs = { "realtime": (today - timedelta(days=6), today, "Tweet Terkini — 7 Hari Terakhir"), "30days": (today - timedelta(days=29), today, "30 Hari Terakhir"), "captured": (today, today, "Tweet Hari Ini"), } if mode not in configs: return start_day, end_day, mode_display = configs[mode] dt_start = datetime.combine(start_day, datetime.min.time()) dt_end = datetime.combine(end_day, datetime.max.time().replace(microsecond=0)) st.session_state.filter_start_date = dt_start st.session_state.filter_end_date = dt_end st.session_state.filter_label = f"{dt_start.strftime('%d/%m/%Y')} s/d {dt_end.strftime('%d/%m/%Y')}" st.session_state.mode_display = mode_display st.session_state.filter_date_column = "created_at" # ═══════════════════════════════════════════════════════════ # PREPROCESSING PIPELINE — 5 TAHAP # PENTING: Pipeline ini harus IDENTIK dengan sentiment_service.py # agar token yang dihasilkan konsisten dengan training model. # # URUTAN: # 1. Case Folding → lowercase dulu sebelum cleaning # 2. Cleaning → hapus noise setelah lowercase # 3. Normalisasi → singkatan/slang → kata baku # 4. Stopword Removal → buang kata umum, jaga kata sentimen # 5. Stemming → bentuk dasar via Sastrawi ECS # ═══════════════════════════════════════════════════════════ KATA_SENTIMEN_PENTING = { # Negasi "tidak", "bukan", "jangan", "kurang", "belum", "tanpa", # Positif "keren", "bagus", "mantap", "setuju", "dukung", "mendukung", "andal", "handal", "gercep", "bangga", "senang", "suka", "baik", "benar", "tepat", "oke", "sejahtera", "berkembang", "maju", "inovatif", "tegas", "sigap", "tanggap", "adil", "bijak", "bermanfaat", "untung", "berhasil", "sukses", "solusi", "manfaat", "berguna", "membantu", "bantu", "pro", "lanjut", "sangat", "banget", "sekali", "paling", "amat", "luar", "biasa", # Negatif "kecewa", "buruk", "jelek", "parah", "gagal", "hancur", "rusak", "bohong", "tipu", "korupsi", # Emosi "marah", "sedih", "khawatir", } def _load_stopwords(): stopword_file = "indonesian-stopwords-complete.txt" base = set() try: with open(stopword_file, "r", encoding="utf-8") as f: base = set(f.read().splitlines()) except FileNotFoundError: base = { "yang", "dan", "di", "ke", "dari", "ini", "itu", "dengan", "untuk", "pada", "adalah", "oleh", "ada", "ya", "akan", "atau", "juga", "sama", "karena", "jika", "sudah", "telah", } for kata in KATA_SENTIMEN_PENTING: base.discard(kata) base.update({ "rt", "amp", "https", "http", "co", "t", "wkwk", "wkwkwk", "haha", "hehe", "xixi", "yg", "dgn", "utk", "dr", "krn", "tp", "jd", "sdh", "aja", "doang", "nih", "sih", "dong", "deh", "loh", "lah", "tuh", "kak", "gan", }) return base def _load_stemmer(): try: from Sastrawi.Stemmer.StemmerFactory import StemmerFactory return StemmerFactory().create_stemmer() except Exception: return None NORMALISASI = { # Negasi "gk": "tidak", "ga": "tidak", "gak": "tidak", "nggak": "tidak", "ngga": "tidak", "tdk": "tidak", "tak": "tidak", "enggak": "tidak", "engga": "tidak", "kagak": "tidak", "kaga": "tidak", "ndak": "tidak", "gkk": "tidak", "ngak": "tidak", # Kata ganti "yg": "yang", "dgn": "dengan", "utk": "untuk", "org": "orang", "krn": "karena", "dr": "dari", "sm": "sama", "pd": "pada", "dlm": "dalam", "bwt": "buat", "trm": "terima", # Verba "tp": "tapi", "tpi": "tapi", "jd": "jadi", "sdh": "sudah", "blm": "belum", "emg": "memang", "emang": "memang", "gimana": "bagaimana", "gitu": "begitu", "gini": "begini", "udah": "sudah", "udh": "sudah", "mau": "mau", # Intensitas "bgt": "banget", "bngt": "banget", "bget": "banget", "bgtt": "banget", # Positif informal "bener": "benar", "beneran": "benar", "mantep": "mantap", "mntap": "mantap", "kece": "keren", "kece bgt": "keren banget", "cucok": "cocok", "cucuk": "cocok", "cakep": "bagus", "oke bgt": "oke banget", "sip": "baik", "siipp": "baik", "top": "terbaik", "topbgt": "terbaik banget", "jos": "bagus", "josss": "bagus", "goks": "luar biasa", "setujuu": "setuju", "stuju": "setuju", "dukung": "dukung", "proud": "bangga", "mantul": "mantap betul", # Negatif informal "ancur": "hancur", "ancrr": "hancur", "parahh": "parah", "parahhh": "parah", "gagall": "gagal", "ngaco": "tidak benar", "ngasal": "tidak benar", "receh": "tidak penting", "gaje": "tidak jelas", "asal": "sembarangan", # Domain "ongkir": "ongkos kirim", "freeongkir": "gratis ongkos kirim", "gratisongkir": "gratis ongkos kirim", "free": "gratis", "ecommerce": "e commerce", "komdigi": "komdigi", "subsidi": "subsidi", "marketplace": "marketplace", "seller": "penjual", "buyer": "pembeli", "online": "online", "shopee": "shopee", "tokopedia": "tokopedia", "lazada": "lazada", "tiktok": "tiktok", } # ── Tahap 1: Case Folding ─────────────────────────────────────────────────── def step1_case_folding(text: str) -> str: return str(text).lower() # ── Tahap 2: Cleaning ─────────────────────────────────────────────────────── def step2_cleaning(text: str) -> str: text = re.sub(r"http\S+|www\S+|https\S+", "", text) text = re.sub(r"@\w+", "", text) text = re.sub(r"#\w+", "", text) text = re.sub(r"\d+", "", text) text = re.sub( r"[\U00010000-\U0010ffff" r"\U0001F600-\U0001F64F" r"\U0001F300-\U0001F5FF" r"\U0001F680-\U0001F6FF" r"\U0001F1E0-\U0001F1FF" r"\u2600-\u26FF\u2700-\u27BF" r"]+", "", text, flags=re.UNICODE ) text = text.translate(str.maketrans("", "", string.punctuation)) text = re.sub(r"[^a-zA-Z\s]", "", text) text = re.sub(r"\s+", " ", text).strip() return text # ── Tahap 3: Normalisasi ──────────────────────────────────────────────────── def step3_normalization(text: str) -> str: return " ".join(NORMALISASI.get(word, word) for word in text.split()) # ── Tahap 4: Stopword Removal ─────────────────────────────────────────────── def step4_stopword_removal(tokens: list, stopwords: set) -> list: return [ w for w in tokens if (w not in stopwords or w in KATA_SENTIMEN_PENTING) and len(w) > 2 ] # ── Tahap 5: Stemming ─────────────────────────────────────────────────────── def step5_stemming(tokens: list, stemmer) -> list: if stemmer is None: return tokens return [stemmer.stem(w) for w in tokens] def full_preprocessing(text: str, stopwords: set, stemmer): """ Jalankan 5 tahap preprocessing dan kembalikan dict hasil setiap tahap. URUTAN TAHAP: 1. Case Folding → lowercase 2. Cleaning → hapus noise 3. Normalisasi → normalisasi kata 4. Stopword Removal → buang stopword (split() dilakukan internal) 5. Stemming → bentuk dasar """ s1_fold = step1_case_folding(text) s2_clean = step2_cleaning(s1_fold) s3_norm = step3_normalization(s2_clean) s4_filtered = step4_stopword_removal(s3_norm.split(), stopwords) s5_stemmed = step5_stemming(s4_filtered, stemmer) return { "setelah_casefolding": s1_fold, "setelah_cleaning": s2_clean, "setelah_normalisasi": s3_norm, "setelah_stopword": " ".join(s4_filtered), "clean_text": " ".join(s5_stemmed), "_tokens_clean": s5_stemmed, } # ═══════════════════════════════════════════════════════════ # UI HELPERS # ═══════════════════════════════════════════════════════════ def _section_header(title, subtitle=""): sub_html = ( f'
{subtitle}
' if subtitle else "" ) st.markdown(f"""
{title}
{sub_html}
""", unsafe_allow_html=True) def _gap(size="md"): heights = {"xs": "0.6rem", "sm": "1rem", "md": "1.45rem", "lg": "2rem"} st.markdown(f'
', unsafe_allow_html=True) # ═══════════════════════════════════════════════════════════ # STYLES # ═══════════════════════════════════════════════════════════ def _render_preprocessing_styles(): st.markdown(""" """, unsafe_allow_html=True) # ═══════════════════════════════════════════════════════════ # PAGE HEADER # ═══════════════════════════════════════════════════════════ def _render_page_header(): st.markdown("""
🧹

Bersihkan Data

Preprocessing teks 5 tahap otomatis: Case Folding → Cleaning → Normalisasi → Stopword Removal → Stemming

✨ NLP Pipeline
""", unsafe_allow_html=True) # ═══════════════════════════════════════════════════════════ # PIPELINE STEPS CARDS # ═══════════════════════════════════════════════════════════ def _render_pipeline_steps(stemmer_ok): steps = [ { "num": "01", "anim": "pipe-1", "icon": "🔡", "color": "#0284c7", "dark": "#0c4a6e", "bg": "linear-gradient(135deg,#eff6ff,#dbeafe)", "border": "#bfdbfe", "title": "Case Folding", "desc": "Menyeragamkan semua huruf menjadi lowercase sebelum proses lainnya.", "items": [ '"Gratis" → "gratis"', '"ONGKIR" → "ongkir"', '"KEREN" → "keren"', "Seluruh karakter → huruf kecil", "Dilakukan pertama agar cleaning konsisten", "Basis untuk normalisasi & stopword", ], }, { "num": "02", "anim": "pipe-2", "icon": "🧽", "color": "#3b6cf7", "dark": "#1e3a8a", "bg": "linear-gradient(135deg,#eef2ff,#e0e7ff)", "border": "#c7d2fe", "title": "Cleaning", "desc": "Menghapus semua elemen noise yang tidak bermakna dari teks.", "items": [ "Hapus URL (http, https, www)", "Hapus mention (@username)", "Hapus hashtag (#topik)", "Hapus angka & digit", "Hapus emoji & simbol unicode", "Hapus tanda baca & karakter non-latin", ], }, { "num": "03", "anim": "pipe-3", "icon": "🔄", "color": "#16a34a", "dark": "#14532d", "bg": "linear-gradient(135deg,#f0fdf4,#dcfce7)", "border": "#86efac", "title": 'Normalisasi ✦ Diperluas', "desc": "Mengubah kata tidak baku, singkatan, dan slang menjadi kata baku.", "items": [ "gk/ga/gak/kagak → tidak", "ongkir → ongkos kirim", "mantep → mantap", "kece → keren", "jos/josss → bagus", "free → gratis", ], }, { "num": "04", "anim": "pipe-4", "icon": "🚫", "color": "#ea580c", "dark": "#7c2d12", "bg": "linear-gradient(135deg,#fff7ed,#ffedd5)", "border": "#fed7aa", "title": 'Stopword Removal ✦ Diperbaiki', "desc": "Membuang kata umum; kata sentimen penting DIJAGA.", "items": [ "Hapus kata umum (dan, di, ke...)", "Hapus token < 3 karakter", "JAGA negasi: tidak, bukan, jangan", "JAGA positif: keren, bagus, mantap", "JAGA evaluatif: setuju, dukung, bijak", "JAGA intensitas: banget, sangat, sekali", ], }, { "num": "05", "anim": "pipe-5", "icon": "🌱", "color": "#ca8a04", "dark": "#713f12", "bg": "linear-gradient(135deg,#fefce8,#fef9c3)", "border": "#fde68a", "title": "Stemming", "desc": "Mengubah kata ke bentuk dasar via ECS Sastrawi.", "items": [ "berlari → lari", "makanan → makan", "pembatasan → batas", "pengiriman → kirim", f"Status: {'✅ Sastrawi aktif' if stemmer_ok else '⚠️ Sastrawi tidak tersedia'}", "Algoritma: Enhanced Confix Stripping", ], }, ] # Baris 1: 3 kartu pertama row1 = st.columns(3, gap="medium") for col, step in zip(row1, steps[:3]): _render_step_card(col, step) _gap("sm") # Baris 2: 2 kartu terakhir (tengah agar simetris) _, col4, col5, _ = st.columns([0.5, 1, 1, 0.5], gap="medium") _render_step_card(col4, steps[3]) _render_step_card(col5, steps[4]) def _render_step_card(col, step): items_html = "".join( f'
' f'▶' f'{item}' f'
' for item in step["items"] ) with col: st.markdown(f"""
{step['icon']}
Tahap {step['num']}
{step['title']}
{step['desc']}
{items_html}
""", unsafe_allow_html=True) # ═══════════════════════════════════════════════════════════ # FLOW ARROW # ═══════════════════════════════════════════════════════════ def _render_flow_arrow(): nodes = [ ("📄 Teks Asli", "#94a3b8", "#f8fafc", "#e2e8f0"), ("① Case Folding", "#0284c7", "#eff6ff", "#bfdbfe"), ("② Cleaning", "#3b6cf7", "#eef2ff", "#c7d2fe"), ("③ Normalisasi", "#16a34a", "#f0fdf4", "#86efac"), ("④ Stopword", "#ea580c", "#fff7ed", "#fed7aa"), ("⑤ Stemming", "#ca8a04", "#fefce8", "#fde68a"), ("✅ Teks Bersih", "#0f172a", "#0f172a", "#334155"), ] parts = "" for i, (label, color, bg, border) in enumerate(nodes): text_c = "#f8fafc" if label == "✅ Teks Bersih" else color parts += ( f'
{label}
' ) if i < len(nodes) - 1: next_color = nodes[i + 1][1] parts += ( f'
' f'
' f'
' f'
' ) st.markdown(f"""
{parts}
""", unsafe_allow_html=True) # ═══════════════════════════════════════════════════════════ # STAT PILLS # ═══════════════════════════════════════════════════════════ def _render_stat_pills(total_raw, total_clean, removed): c1, c2, c3 = st.columns(3, gap="medium") cards = [ (c1, "📥", "linear-gradient(135deg,#eef2ff,#e0e7ff)", "#3b6cf7", "#1e3a8a", "#c7d2fe", "Tweet Diproses", f"{total_raw:,}", "Total tweet periode ini"), (c2, "✅", "linear-gradient(135deg,#f0fdf4,#dcfce7)", "#16a34a", "#14532d", "#86efac", "Tweet Siap Analisis", f"{total_clean:,}", "Lulus semua 5 tahap"), (c3, "🗑️" if removed > 0 else "✅", ("linear-gradient(135deg,#fff7ed,#ffedd5)" if removed > 0 else "linear-gradient(135deg,#f0fdf4,#dcfce7)"), ("#ea580c" if removed > 0 else "#16a34a"), ("#7c2d12" if removed > 0 else "#14532d"), ("#fed7aa" if removed > 0 else "#86efac"), "Tweet Dibuang", f"{removed:,}", ("Teks kosong setelah preprocessing" if removed > 0 else "Semua tweet lolos")), ] for col, icon, bg, color, dark, border, label, val, sub in cards: with col: fs = "1.1rem" if len(str(val)) > 8 else "1.55rem" st.markdown(f"""
{icon}
{label}
{val}
{sub}
""", unsafe_allow_html=True) # ═══════════════════════════════════════════════════════════ # LIVE EXAMPLE # ═══════════════════════════════════════════════════════════ def _render_live_example(df_c): if df_c.empty: return sample = df_c.sample(1).iloc[0] _section_header( "🔍 Contoh Hasil Preprocessing per Tahap", "Contoh tweet acak dari dataset — refresh halaman untuk contoh berbeda" ) # Urutan tampilan sesuai pipeline: CF → Clean → Norm → Stop → Stem steps_ex = [ ("📄 Teks Asli", "text_asli", "#0f172a", "#f8fafc", "#e2e8f0"), ("① Setelah Case Folding", "setelah_casefolding", "#0c4a6e", "#eff6ff", "#bfdbfe"), ("② Setelah Cleaning", "setelah_cleaning", "#1e3a8a", "#eef2ff", "#c7d2fe"), ("③ Setelah Normalisasi", "setelah_normalisasi", "#14532d", "#f0fdf4", "#86efac"), ("④ Setelah Stopword", "setelah_stopword", "#7c2d12", "#fff7ed", "#fed7aa"), ("⑤ Hasil Akhir (Stem)", "clean_text", "#713f12", "#fefce8", "#fde68a"), ] for label, col_key, text_color, bg, border in steps_ex: raw = sample.get(col_key, "-") text_display = str(raw) if raw and str(raw).strip() else "—" word_count = len(text_display.split()) if text_display != "—" else 0 char_count = len(text_display) if text_display != "—" else 0 char_info = f"{word_count} kata · {char_count} karakter" content_html = ( f'
{text_display}
' ) st.markdown( f'
' f'
' f'{label}' f'{char_info}' f'
' f'{content_html}' f'
', unsafe_allow_html=True ) # ═══════════════════════════════════════════════════════════ # TOP WORDS CHART # ═══════════════════════════════════════════════════════════ def _render_top_words_chart(df_c): _section_header( "📊 Kata-Kata Paling Sering Muncul", f"Dari {len(df_c):,} tweet yang sudah bersih (hasil akhir tahap 5) — Top 20 kata" ) if "_tokens_clean" in df_c.columns: all_words = [w for tokens in df_c["_tokens_clean"] for w in (tokens if isinstance(tokens, list) else [])] else: all_words = " ".join(df_c["clean_text"].fillna("")).split() filtered_words = [w for w in all_words if len(w) > 2] word_freq = Counter(filtered_words).most_common(20) if not word_freq: st.info("⚠️ Belum cukup kata untuk ditampilkan.") return words = [w[0] for w in word_freq] counts = [w[1] for w in word_freq] max_c = max(counts) if counts else 1 bar_colors = [f"rgba(59,108,247,{0.35 + 0.65*(c/max_c):.2f})" for c in counts[::-1]] fig = go.Figure(data=[ go.Bar( y=words[::-1], x=counts[::-1], orientation="h", marker=dict(color=bar_colors, line=dict(width=0), cornerradius=6), text=[str(c) for c in counts[::-1]], textposition="outside", textfont=dict(size=10, color="#475569"), hovertemplate="%{y}
Muncul %{x} kali", ) ]) fig.update_layout( height=540, margin=dict(l=0, r=60, t=8, b=8), paper_bgcolor="rgba(0,0,0,0)", plot_bgcolor="rgba(0,0,0,0)", xaxis=dict(showgrid=True, gridcolor="rgba(203,213,225,0.8)", tickfont=dict(size=10, color="#94a3b8"), zeroline=False, showline=False, fixedrange=True), yaxis=dict(showgrid=False, tickfont=dict(size=11, color="#334155"), fixedrange=True), showlegend=False, ) st.markdown("""
""", unsafe_allow_html=True) st.plotly_chart(fig, use_container_width=True, config={"displayModeBar": False}) st.markdown("
", unsafe_allow_html=True) # ═══════════════════════════════════════════════════════════ # MAIN SHOW # ═══════════════════════════════════════════════════════════ def show(): _render_preprocessing_styles() _render_page_header() if "analysis_mode" not in st.session_state: st.warning("⚠️ Silakan pilih mode tampilan di halaman Ambil Data Twitter terlebih dahulu.") return _sync_dynamic_period() start_date = st.session_state.get("filter_start_date") end_date = st.session_state.get("filter_end_date") filter_label = st.session_state.get("filter_label", "-") mode_display = st.session_state.get("mode_display", "-") if start_date is None or end_date is None: st.warning("⚠️ Silakan buka halaman Ambil Data Twitter terlebih dahulu.") return mode_meta = { "realtime": ("#16a34a", "📡"), "30days": ("#3b6cf7", "📅"), "captured": ("#0284c7", "📆"), "custom": ("#d97706", "🔍"), } mode_color, mode_icon = mode_meta.get( st.session_state.analysis_mode, ("#3b6cf7", "📊") ) st.markdown( f'
' f'{mode_icon}' f'
' f'
{mode_display}
' f'
' f'Periode: {filter_label}
' f'
', unsafe_allow_html=True ) # ── Pipeline Overview ───────────────────────────────────── _section_header( "🔬 Alur NLP Pipeline — 5 Tahap Preprocessing", "Setiap tweet diproses berurutan melalui 5 tahap sebelum siap dianalisis sentimennya" ) stemmer_tmp = _load_stemmer() _render_pipeline_steps(stemmer_tmp is not None) _gap("sm") _render_flow_arrow() _gap("md") # ── Load data ───────────────────────────────────────────── try: df_all = pd.read_sql("SELECT * FROM tweets ORDER BY created_at DESC", engine) if df_all.empty: st.warning("⚠️ Belum ada data. Kembali ke halaman Ambil Data Twitter.") return df_all["created_at"] = parse_dt(df_all["created_at"]) if "crawled_at" in df_all.columns: df_all["crawled_at"] = parse_crawled_dt(df_all["crawled_at"]) except Exception as e: st.error(f"❌ Gagal membaca database: {e}") return s_dt = pd.Timestamp(start_date) e_dt = pd.Timestamp(end_date) df = df_all[(df_all["created_at"] >= s_dt) & (df_all["created_at"] <= e_dt)].copy() if df.empty: st.warning(f"⚠️ Tidak ada tweet dengan tanggal asli dalam periode {filter_label}.") return # ── Cache preprocessing ─────────────────────────────────── total_tweets_in_db = get_tweet_count() latest_crawl_marker = get_latest_crawl_time() or "no-crawl" data_marker = (total_tweets_in_db, latest_crawl_marker) cache_key = ( f"pp5_{st.session_state.analysis_mode}_" f"{start_date}_{end_date}_{total_tweets_in_db}_{latest_crawl_marker}" ) for old_key in list(st.session_state.keys()): if old_key.startswith("pp5_") and old_key != cache_key: del st.session_state[old_key] force_refresh = data_marker != st.session_state.get("_pp_last_data_marker") if cache_key not in st.session_state or force_refresh: stemmer = _load_stemmer() stopwords = _load_stopwords() with st.spinner("🧹 Menjalankan 5 tahap preprocessing…"): results = [] for _, row in df.iterrows(): r = full_preprocessing(row["text"], stopwords, stemmer) r["tweet_id"] = row.get("tweet_id", "") r["text_asli"] = row["text"] r["created_at"] = row["created_at"] r["crawled_at"] = row.get("crawled_at") results.append(r) df_c = pd.DataFrame(results) df_c = df_c[df_c["clean_text"].str.strip().str.len() > 0].copy() st.session_state[cache_key] = df_c st.session_state[cache_key + "_stemmer_ok"] = stemmer is not None st.session_state["_pp_last_data_marker"] = data_marker df_c = st.session_state[cache_key] stemmer_ok = st.session_state.get(cache_key + "_stemmer_ok", False) # ── Statistik ───────────────────────────────────────────── removed = len(df) - len(df_c) _section_header( "📌 Ringkasan Hasil Preprocessing", f"Berdasarkan tanggal asli tweet · {filter_label}" ) _gap("xs") _render_stat_pills(len(df), len(df_c), removed) _gap("lg") _render_live_example(df_c) _gap("lg") # ── Tabel ───────────────────────────────────────────────── _section_header( "📋 Tabel Perbandingan Teks per Tahap", f"{len(df_c):,} tweet · {filter_label} — scroll horizontal untuk lihat semua kolom" ) if "crawled_at" not in df_c.columns: df_c = df_c.copy() df_c["crawled_at"] = pd.NaT # Kolom ditampilkan sesuai urutan pipeline: CF → Clean → Norm → Stop → Stem disp = df_c[[ "tweet_id", "created_at", "crawled_at", "text_asli", "setelah_casefolding", "setelah_cleaning", "setelah_normalisasi", "setelah_stopword", "clean_text", ]].copy() disp.columns = [ "ID Tweet", "Tanggal Tweet", "Masuk Database", "Teks Asli", "① Case Folding", "② Cleaning", "③ Normalisasi", "④ Stopword", "⑤ Hasil Akhir", ] disp["Tanggal Tweet"] = disp["Tanggal Tweet"].apply(format_dt) disp["Masuk Database"] = disp["Masuk Database"].apply(format_dt) render_standard_table( disp, height=360, min_width=1980, nowrap=["ID Tweet", "Tanggal Tweet", "Masuk Database"], wide_columns=[ "Teks Asli", "① Case Folding", "② Cleaning", "③ Normalisasi", "④ Stopword", "⑤ Hasil Akhir", ], column_widths={ "ID Tweet": "155px", "Tanggal Tweet": "165px", "Masuk Database": "165px", "Teks Asli": "280px", "① Case Folding": "240px", "② Cleaning": "240px", "③ Normalisasi": "240px", "④ Stopword": "230px", "⑤ Hasil Akhir": "230px", }, ) _gap("lg") _render_top_words_chart(df_c) _gap("lg") # ── Simpan ke session state ─────────────────────────────── st.session_state["preprocessed_df"] = df_c # ── Download ────────────────────────────────────────────── st.markdown("""
""", unsafe_allow_html=True) d1, d2 = st.columns(2, gap="medium") with d1: st.download_button( "📥 Unduh Hasil Preprocessing Lengkap (semua kolom)", df_c.drop(columns=["_tokens_clean"], errors="ignore" ).to_csv(index=False).encode("utf-8"), f"preprocessing_lengkap_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv", "text/csv", use_container_width=True, ) with d2: out2 = df_c[["tweet_id", "text_asli", "clean_text"]].copy() out2.columns = ["tweet_id", "tweet", "clean_text"] st.download_button( "📥 Unduh Teks Bersih Saja (siap analisis sentimen)", out2.to_csv(index=False).encode("utf-8"), f"teks_bersih_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv", "text/csv", use_container_width=True, ) st.markdown("
", unsafe_allow_html=True) _gap("sm") if st.button( "📈 Lanjut ke Analisis Sentimen →", type="primary", use_container_width=True, ): st.session_state.current_page = "sentiment" st.rerun() _gap("sm")