From c886a9891cf0a1c2817892a7ca861f7e991a980d Mon Sep 17 00:00:00 2001 From: adistya13 Date: Thu, 21 May 2026 21:53:09 +0700 Subject: [PATCH] perbaikan revisi per halaman dan informatif penyempurnaan per halaman dan tambahan tampilah lebih informatif --- app.py | 35 +- page_modules/crawling_page.py | 656 +++++++----- page_modules/preprocessing_page.py | 269 ++--- page_modules/sentiment_page.py | 1528 +++++++++++++++++++--------- sentiment_service.py | 510 +++++++++- tweets-data/hasil_latest.csv | 3 +- tweets-data/hasil_latest.old.csv | 3 +- tweets-data/hasil_top.csv | 12 - tweets-data/hasil_top.old.csv | 25 +- 9 files changed, 2028 insertions(+), 1013 deletions(-) delete mode 100644 tweets-data/hasil_top.csv diff --git a/app.py b/app.py index 75596fd..2be90a7 100644 --- a/app.py +++ b/app.py @@ -858,19 +858,42 @@ with st.sidebar: """, unsafe_allow_html=True) # ── QUERY ──────────────────────────────────────────────────── +# ── QUERY ──────────────────────────────────────────────────── crawler_query = os.getenv( "QUERY", - 'komdigi (ongkir OR "gratis ongkir" OR "free ongkir")' + 'komdigi (ongkir OR "gratis ongkir" OR "free ongkir") ' + 'OR "pembatasan gratis ongkir" OR "gratis ongkir dibatasi"' ) - query_html = escape(crawler_query) + + # Highlight keyword utama dengan warna biru + _QUERY_TERMS = [ + "komdigi", "ongkir", "gratis ongkir", "free ongkir", + "pembatasan gratis ongkir", "gratis ongkir dibatasi", + ] + query_highlighted = escape(crawler_query) + for term in _QUERY_TERMS: + escaped_term = escape(term) + query_highlighted = query_highlighted.replace( + escaped_term, + f'{escaped_term}', + 1, + ) st.markdown(f"""
-
- Query Aktif +
+ 🔍 Query Aktif
-
- {query_html} +
+ {query_highlighted} +
+
+
+ Topik: kebijakan ongkir gratis & Komdigi
+ Sumber: X (Twitter) via tweet-harvest +
""", unsafe_allow_html=True) diff --git a/page_modules/crawling_page.py b/page_modules/crawling_page.py index ce6b958..4671973 100644 --- a/page_modules/crawling_page.py +++ b/page_modules/crawling_page.py @@ -358,7 +358,7 @@ def _render_nrt_status_banner(sched_ok): f"{NRT_INTERVAL_MINUTES} menit. " f"Data mencakup 7 hari terakhir berdasarkan tanggal asli tweet." ) - badge = "● LIVE" + badge = "● LIVE" else: bg = "linear-gradient(135deg,#eff6ff,#dbeafe)" bdr = "#93c5fd" @@ -369,9 +369,9 @@ def _render_nrt_status_banner(sched_ok): title = "Crawler Siap Dijalankan" desc = ( f"Scheduler otomatis aktif setiap {NRT_INTERVAL_MINUTES} menit " - f"sejak aplikasi pertama dibuka — tidak perlu terminal terpisah." + f"sejak aplikasi pertama dibuka — tidak perlu terminal terpisah." ) - badge = "◎ STANDBY" + badge = "⚬ STANDBY" st.markdown(f"""
0 else "ℹ️ Belum ada tweet baru hari ini" + note_text = f"{tweet_today:,} tweet masuk hari ini" if tweet_today > 0 else "Belum ada tweet baru hari ini" note_bg = "#f0fdf4" if tweet_today > 0 else "#f8fafc" note_border = "#bbf7d0" if tweet_today > 0 else "#e8edf5" note_color = "#166534" if tweet_today > 0 else "#64748b" - _section_header("📡 Status Monitoring Sistem", "Pembaruan data near-realtime · interval otomatis") + _section_header("📡 Status Monitoring Sistem", "Pembaruan data near-realtime · interval otomatis") col_a, col_b = st.columns(2, gap="medium") @@ -467,59 +467,62 @@ def _render_monitoring_panel(last_log, last_dt, sched_ok): rows_html_a = "" for i, (label, value, vc) in enumerate(left_rows): border = "none" if i == len(left_rows) - 1 else "1px solid #f1f5f9" - rows_html_a += f""" -
- {label} - {value} -
""" + rows_html_a += ( + f'
' + f'{label}' + f'{value}' + f'
' + ) with col_a: - st.markdown(f""" -
-
- ⚙️ Sistem Crawling - {status_text} -
- {rows_html_a} -
-""", unsafe_allow_html=True) + st.markdown( + f'
' + f'
' + f'Sistem Crawling' + f'{status_text}' + f'
' + f'{rows_html_a}' + f'
', + unsafe_allow_html=True, + ) # ── Right card: ringkasan dataset ── right_rows = [ - ("Update terakhir", last_crawl_str, "#0f172a"), - ("Countdown berikutnya", countdown_str, "#16a34a"), - ("Tweet masuk hari ini", f"{tweet_today:,} tweet", "#3b6cf7"), - ("Total dataset", f"{total_db:,} tweet", "#0f172a"), + ("Update terakhir", last_crawl_str, "#0f172a"), + ("Countdown berikutnya", countdown_str, "#16a34a"), + ("Tweet masuk hari ini", f"{tweet_today:,} tweet", "#3b6cf7"), + ("Total dataset", f"{total_db:,} tweet", "#0f172a"), ] rows_html_b = "" for i, (label, value, vc) in enumerate(right_rows): border = "none" if i == len(right_rows) - 1 else "1px solid #f1f5f9" - rows_html_b += f""" -
- {label} - {value} -
""" + rows_html_b += ( + f'
' + f'{label}' + f'{value}' + f'
' + ) with col_b: - st.markdown(f""" -
-
- 📊 Ringkasan Dataset -
- {rows_html_b} -
{note_text}
-
-""", unsafe_allow_html=True) + st.markdown( + f'
' + f'
' + f'Ringkasan Dataset
' + f'{rows_html_b}' + f'
{note_text}
' + f'
', + unsafe_allow_html=True, + ) # ═══════════════════════════════════════════════════════════ @@ -527,25 +530,25 @@ def _render_monitoring_panel(last_log, last_dt, sched_ok): # ═══════════════════════════════════════════════════════════ def _render_metrics(total, days, earliest, latest, filter_label, basis_label): - st.markdown(f""" -
- 🔄 - - Data aktif: {filter_label} -  ·  {basis_label} - -
-""", unsafe_allow_html=True) + st.markdown( + f'
' + f'🔄' + f'' + f'Data aktif: {filter_label}' + f' ·  {basis_label}' + f'
', + unsafe_allow_html=True, + ) pills = [ - ("📊", "linear-gradient(135deg,#eef2ff,#e0e7ff)", "#3b6cf7", "#1e3a8a", "#dbeafe", + ("📊", "linear-gradient(135deg,#eef2ff,#e0e7ff)", "#3b6cf7", "#1e3a8a", "#dbeafe", "Total Tweet", f"{total:,}", "Terkumpul periode ini"), - ("📅", "linear-gradient(135deg,#f0fdf4,#dcfce7)", "#16a34a", "#14532d", "#bbf7d0", + ("📅", "linear-gradient(135deg,#f0fdf4,#dcfce7)", "#16a34a", "#14532d", "#bbf7d0", "Rentang Waktu", f"{days} Hari", basis_label), - ("🗓️", "linear-gradient(135deg,#fff7ed,#ffedd5)", "#ea580c", "#7c2d12", "#fed7aa", + ("📇", "linear-gradient(135deg,#fff7ed,#ffedd5)", "#ea580c", "#7c2d12", "#fed7aa", "Mulai Dari", earliest, "Tanggal awal"), - ("📆", "linear-gradient(135deg,#fefce8,#fef9c3)", "#ca8a04", "#713f12", "#fde68a", + ("📆", "linear-gradient(135deg,#fefce8,#fef9c3)", "#ca8a04", "#713f12", "#fde68a", "Sampai Dengan", latest, "Tanggal akhir"), ] @@ -553,20 +556,21 @@ def _render_metrics(total, days, earliest, latest, filter_label, basis_label): for col, (icon, bg, color, dark, border_c, label, val, sub) in zip([c1, c2, c3, c4], pills): with col: fs = "1.1rem" if len(str(val)) > 10 else "1.55rem" - st.markdown(f""" -
-
{icon}
-
{label}
-
{val}
-
{sub}
-
-""", unsafe_allow_html=True) + st.markdown( + f'
' + f'
{icon}
' + f'
{label}
' + f'
{val}
' + f'
{sub}
' + f'
', + unsafe_allow_html=True, + ) # ═══════════════════════════════════════════════════════════ @@ -667,6 +671,7 @@ def _render_charts(df, filter_label, date_col, chart_label, dt_start, dt_end): legend_items.insert(1, ("#a78bfa", "Terendah")) if any(c == 0 for c in counts_day): legend_items.append(("#cbd5e1", "Tidak ada data")) + legend_html = "".join([ f'' f'' @@ -683,7 +688,7 @@ def _render_charts(df, filter_label, date_col, chart_label, dt_start, dt_end): def _date_pills(days_list, color, max_show=4): if not days_list: - return '—' + return '—' pills = "" for d in days_list[:max_show]: pills += ( @@ -692,11 +697,14 @@ def _render_charts(df, filter_label, date_col, chart_label, dt_start, dt_end): f'font-weight:700;margin:2px 2px 0 0;">{d}' ) if len(days_list) > max_show: - pills += f'+{len(days_list)-max_show} lainnya' + pills += ( + f'' + f'+{len(days_list)-max_show} lainnya' + ) return pills if not busiest_list: - busy_main, busy_count, busy_pills = "—", "Belum ada data", "" + busy_main, busy_count, busy_pills = "—", "Belum ada data", "" elif len(busiest_list) == 1: busy_main, busy_count, busy_pills = busiest_list[0], f"{max_cnt:,} tweet", "" else: @@ -705,7 +713,7 @@ def _render_charts(df, filter_label, date_col, chart_label, dt_start, dt_end): busy_pills = _date_pills(busiest_list, "#ea580c") if not quietest_list: - quiet_main, quiet_count, quiet_pills = "—", "Belum ada data aktif", "" + quiet_main, quiet_count, quiet_pills = "—", "Belum ada data aktif", "" elif len(quietest_list) == 1: quiet_main, quiet_count, quiet_pills = quietest_list[0], f"{min_cnt:,} tweet", "" else: @@ -718,11 +726,11 @@ def _render_charts(df, filter_label, date_col, chart_label, dt_start, dt_end): aktif_main, aktif_count, aktif_pills = "0 Hari Aktif", f"Dari {total_days} hari, belum ada data", "" elif active_days == total_days: aktif_main = f"{active_days} / {total_days} Hari" - aktif_count = f"100% hari ada tweet · total {total_tweets:,} tweet" + aktif_count = f"100% hari ada tweet · total {total_tweets:,} tweet" aktif_pills = "" else: aktif_main = f"{active_days} / {total_days} Hari" - aktif_count = f"{pct_active}% hari aktif · {inactive_days} hari kosong" + aktif_count = f"{pct_active}% hari aktif · {inactive_days} hari kosong" aktif_pills = "" def _insight_card(col, icon, bg, color, dark, border_c, title, main, count_txt, pills_html): @@ -731,32 +739,33 @@ def _render_charts(df, filter_label, date_col, chart_label, dt_start, dt_end): f'
{pills_html}
' if pills_html else "" ) - st.markdown(f""" -
-
-
{icon}
- {title} -
-
{main}
-
{count_txt}
- {pills_block} -
-""", unsafe_allow_html=True) + st.markdown( + f'
' + f'
' + f'
{icon}
' + f'{title}' + f'
' + f'
{main}
' + f'
{count_txt}
' + f'{pills_block}' + f'
', + unsafe_allow_html=True, + ) - _insight_card(pc1, "🔥", "linear-gradient(135deg,#fff7ed,#ffedd5)", + _insight_card(pc1, "🔥", "linear-gradient(135deg,#fff7ed,#ffedd5)", "#ea580c", "#7c2d12", "#fed7aa", "Hari Paling Ramai", busy_main, busy_count, busy_pills) - _insight_card(pc2, "🌙", "linear-gradient(135deg,#f5f3ff,#ede9fe)", + _insight_card(pc2, "🌙", "linear-gradient(135deg,#f5f3ff,#ede9fe)", "#7c3aed", "#3b0764", "#ddd6fe", "Hari Paling Sepi", quiet_main, quiet_count, quiet_pills) - _insight_card(pc3, "📆", "linear-gradient(135deg,#f0fdf4,#dcfce7)", + _insight_card(pc3, "📆", "linear-gradient(135deg,#f0fdf4,#dcfce7)", "#16a34a", "#14532d", "#bbf7d0", "Hari Aktif", aktif_main, aktif_count, aktif_pills) @@ -771,9 +780,9 @@ def get_filtered_data(df_all): mode = st.session_state.analysis_mode if mode == "realtime": - mode_display = "Tweet Terkini — 7 Hari Terakhir" + mode_display = "Tweet Terkini — 7 Hari Terakhir" mode_color = "#16a34a" - mode_icon = "📡" + mode_icon = "📡" date_col = "created_at" basis_label = "Berdasarkan tanggal asli tweet" chart_label = "tanggal asli tweet" @@ -783,7 +792,7 @@ def get_filtered_data(df_all): elif mode == "30days": mode_display = "30 Hari Terakhir" mode_color = "#3b6cf7" - mode_icon = "📅" + mode_icon = "📅" date_col = "created_at" basis_label = "Berdasarkan tanggal asli tweet" chart_label = "tanggal asli tweet" @@ -793,7 +802,7 @@ def get_filtered_data(df_all): elif mode == "captured": mode_display = "Tweet Hari Ini" mode_color = "#0284c7" - mode_icon = "📆" + mode_icon = "📆" date_col = "created_at" basis_label = "Berdasarkan tanggal asli tweet" chart_label = "tanggal asli tweet" @@ -805,7 +814,7 @@ def get_filtered_data(df_all): end_date = st.session_state.get("custom_end_date", today) mode_display = "Periode Historis Pilihan" mode_color = "#d97706" - mode_icon = "🔍" + mode_icon = "🔍" date_col = "created_at" basis_label = "Berdasarkan tanggal asli tweet" chart_label = "tanggal asli tweet" @@ -834,6 +843,146 @@ def get_filtered_data(df_all): } +# ═══════════════════════════════════════════════════════════ +# PAGE HEADER (dipecah jadi beberapa st.markdown() kecil) +# ═══════════════════════════════════════════════════════════ + +def _render_page_header(): + # Bangun semua bagian dalam Python dulu, baru render SEKALI + chip_data = [ + ("🔄", "Interval crawling", f"Setiap {NRT_INTERVAL_MINUTES} menit"), + ("📅", "Jangkauan data", "7 hari terakhir"), + ("🗃", "Penyimpanan", "Database lokal"), + ("🤖", "Model prediksi", "Naive Bayes"), + ] + chips_html = "".join( + '
' + f'{ico}' + '
' + f'
{lbl}
' + f'
{val}
' + '
' + for ico, lbl, val in chip_data + ) + + PIPELINE_STEPS = [ + ("🐦", "Crawling tweet"), + ("🗃", "Simpan ke database"), + ("🧹", "Preprocessing teks"), + ("🤖", "Prediksi sentimen"), + ("📊", "Visualisasi hasil"), + ] + pipeline_steps_html = " ".join( + ( + f'' + f'{icon} {label}' + + ( + '→' + if i < len(PIPELINE_STEPS) - 1 else "" + ) + ) + for i, (icon, label) in enumerate(PIPELINE_STEPS) + ) + + html = ( + # ── outer card ── + '
' + + # ── row 1: icon + title + description ── + '
' + + # icon + '
🐦
' + + # title block + '
' + '
' + '

' + 'Crawling Data Twitter — Komdigi & Isu Ongkir' + '

' + '' + '' + 'Live Monitor' + '' + '
' + '

' + 'Dashboard analisis sentimen tweet seputar ' + 'kebijakan ongkos kirim gratis dan ' + 'Komdigi. ' + 'Sistem mengumpulkan tweet otomatis, menyimpan ke database lokal, ' + 'membersihkan teks Bahasa Indonesia, lalu memprediksi sentimen ' + '(positif / negatif / netral) via model ' + 'Naive Bayes ' + '— near-realtime tanpa intervensi manual.' + '

' + '
' + '
' # end row 1 + + # ── divider ── + '
' + + # ── row 2: chips ── + f'
' + f'{chips_html}' + '
' + + # ── divider ── + '
' + + # ── row 3: query box ── + '
' + '
' + '🔍 Query pencarian aktif' + '' + 'Twitter / X Search' + '' + '
' + '
' + 'komdigi ' + '(ongkir OR ' + '"gratis ongkir" OR ' + '"free ongkir") OR ' + '"pembatasan gratis ongkir" OR ' + '"gratis ongkir dibatasi"' + '
' + '
' + 'Mencakup variasi: ongkir, gratis ongkir, free ongkir' + ' — termasuk konteks pembatasan subsidi ongkir oleh Komdigi.' + '
' + '
' # end query box + + # ── row 4: pipeline ── + '
' + '' + '⚙ Alur kerja' + f'{pipeline_steps_html}' + '
' + + '
' # end outer card + ) + + st.markdown(html, unsafe_allow_html=True) + + # ═══════════════════════════════════════════════════════════ # MAIN PAGE # ═══════════════════════════════════════════════════════════ @@ -850,51 +999,7 @@ def show(): st.session_state.filter_end_date = today # ── Page Header ────────────────────────────────────────────── - st.markdown(f""" -
-
🐦
-
-

Crawling Data Twitter

-

- Sistem crawling berjalan otomatis setiap - {NRT_INTERVAL_MINUTES} menit - — mengambil data - 7 hari terakhir - secara near-realtime -

-
-
🔄 Live Monitor
-
-""", unsafe_allow_html=True) + _render_page_header() # ── Mode Selector ───────────────────────────────────────────── if "analysis_mode" not in st.session_state: @@ -904,29 +1009,29 @@ def show(): MODE_CFG = { "captured": { - "col": col1, "key": "btn_captured", "label": "📆 Hari Ini", - "title": "Tweet Hari Ini", "icon": "📆", + "col": col1, "key": "btn_captured", "label": "Hari Ini", + "title": "Tweet Hari Ini", "icon": "📆", "desc": "Tanggal asli hari ini", "gradient": "linear-gradient(135deg,#eff6ff,#dbeafe)", "border": "#bfdbfe", "color": "#1d4ed8", "dark": "#1e3a8a", }, "realtime": { - "col": col2, "key": "btn_realtime", "label": "📡 7 Hari", - "title": "Terkini (7 Hari)", "icon": "📡", + "col": col2, "key": "btn_realtime", "label": "7 Hari", + "title": "Terkini (7 Hari)", "icon": "📡", "desc": "7 hari terakhir", "gradient": "linear-gradient(135deg,#f0fdf4,#dcfce7)", "border": "#86efac", "color": "#16a34a", "dark": "#14532d", }, "30days": { - "col": col3, "key": "btn_30days", "label": "📅 30 Hari", - "title": "30 Hari Terakhir", "icon": "📅", + "col": col3, "key": "btn_30days", "label": "30 Hari", + "title": "30 Hari Terakhir", "icon": "📅", "desc": "30 hari terakhir", "gradient": "linear-gradient(135deg,#eef2ff,#e0e7ff)", "border": "#a5b4fc", "color": "#3b6cf7", "dark": "#1e3a8a", }, "custom": { - "col": col4, "key": "btn_custom", "label": "🔍 Pilih Tanggal", - "title": "Pilih Tanggal", "icon": "🔍", + "col": col4, "key": "btn_custom", "label": "Pilih Tanggal", + "title": "Pilih Tanggal", "icon": "🔍", "desc": "Historis custom", "gradient": "linear-gradient(135deg,#fefce8,#fef9c3)", "border": "#fde68a", "color": "#d97706", "dark": "#713f12", @@ -957,20 +1062,21 @@ def show(): if active else "" ) - st.markdown(f""" -
-
{cfg['icon']}
-
- {cfg['title']}
-
{cfg['desc']}
- {underline} -
-""", unsafe_allow_html=True) + st.markdown( + f'
' + f'
{cfg["icon"]}
' + f'
' + f'{cfg["title"]}
' + f'
{cfg["desc"]}
' + f'{underline}' + f'
', + unsafe_allow_html=True, + ) _gap("md") @@ -978,7 +1084,7 @@ def show(): try: df_all = pd.read_sql("SELECT * FROM tweets ORDER BY created_at DESC", engine) except Exception as e: - st.error(f"❌ Gagal membaca database: {str(e)}") + st.error(f"Gagal membaca database: {str(e)}") return if len(df_all) == 0: @@ -995,18 +1101,19 @@ def show(): min_db = df_check["created_at"].min().date() max_db = df_check["created_at"].max().date() - st.markdown(f""" -
-

- 📅 Pilih Rentang Tanggal Tweet Historis

-

- Data tersedia dari - {min_db.strftime('%d/%m/%Y')} - hingga - {max_db.strftime('%d/%m/%Y')} -

-
-""", unsafe_allow_html=True) + st.markdown( + '
' + '

' + '📅 Pilih Rentang Tanggal Tweet Historis

' + f'

' + f'Data tersedia dari ' + f'{min_db.strftime("%d/%m/%Y")}' + f' hingga ' + f'{max_db.strftime("%d/%m/%Y")}' + f'

' + '
', + unsafe_allow_html=True, + ) with st.container(): c1, c2, c3 = st.columns([2, 2, 1], gap="medium") @@ -1038,13 +1145,13 @@ def show(): key="custom_end_input") st.session_state.custom_end_date = ce with c3: - if st.button("✅ Terapkan", type="primary", use_container_width=True): - st.success("✅ Periode diterapkan!") + if st.button("Terapkan", type="primary", use_container_width=True): + st.success("Periode diterapkan!") time.sleep(0.8) st.rerun() # ═══════════════════════════════════════════════════════════ - # MODE: TWEET HARI INI (captured) — monitoring + log table + # MODE: TWEET HARI INI (captured) # ═══════════════════════════════════════════════════════════ if st.session_state.analysis_mode == "captured": st_autorefresh(interval=1000, key="crawler_monitor_refresh") @@ -1075,14 +1182,14 @@ def show(): crawl_time = log.get("timestamp", "-") or "-" if status == "success": - status_label = "✅ Berhasil" + status_label = "Berhasil" note = ( f"{total_saved:,} tweet baru tersimpan" if total_saved > 0 else "Crawling berhasil, tidak ada tweet baru" ) else: - status_label = "❌ Gagal" + status_label = "Gagal" note = error_msg or "Terjadi kesalahan saat crawling" rows.append({ @@ -1094,8 +1201,8 @@ def show(): }) _section_header( - "🕒 Riwayat Crawling Otomatis", - f"{len(rows)} aktivitas terbaru · interval {NRT_INTERVAL_MINUTES} menit" + "🕐 Riwayat Crawling Otomatis", + f"{len(rows)} aktivitas terbaru · interval {NRT_INTERVAL_MINUTES} menit" ) _gap("xs") render_standard_table( @@ -1111,17 +1218,18 @@ def show(): }, ) else: - st.markdown(f""" -
-
⏳
-
- Menunggu Crawl Pertama
-
- Riwayat akan muncul setelah crawl otomatis pertama selesai - (maks. {NRT_INTERVAL_MINUTES} menit sejak aplikasi dibuka) -
-
-""", unsafe_allow_html=True) + st.markdown( + '
' + '
⏳
' + '
' + 'Menunggu Crawl Pertama
' + f'
' + f'Riwayat akan muncul setelah crawl otomatis pertama selesai ' + f'(maks. {NRT_INTERVAL_MINUTES} menit sejak aplikasi dibuka)' + f'
' + '
', + unsafe_allow_html=True, + ) _gap("lg") @@ -1140,22 +1248,23 @@ def show(): ].sort_values("crawled_at", ascending=False) _section_header( - "🤖 Tweet yang Diambil Bot Hari Ini", + "🤖 Tweet yang Diambil Bot Hari Ini", "Semua tweet yang berhasil disimpan crawler hari ini" ) _gap("xs") if bot_df.empty: - st.markdown(f""" -
-
🐦
-
- Belum Ada Tweet Hari Ini
-
- Tweet baru akan muncul setelah crawl berikutnya selesai -
-
-""", unsafe_allow_html=True) + st.markdown( + '
' + '
🐦
' + '
' + 'Belum Ada Tweet Hari Ini
' + '
' + 'Tweet baru akan muncul setelah crawl berikutnya selesai' + '
' + '
', + unsafe_allow_html=True, + ) else: bot_disp = bot_df.head(100).copy() bot_disp["Tanggal Tweet"] = bot_disp["created_at"].apply(format_dt) @@ -1201,32 +1310,33 @@ def show(): # ── Active Filter Banner ────────────────────────────────── total = len(df) - st.markdown(f""" -
- {mode_icon} -
-

{mode_display}

-

- Periode aktif: {filter_label} -  ·  {basis_label} -

-
-
- {total:,} tweet -
-
-""", unsafe_allow_html=True) + st.markdown( + f'
' + f'{mode_icon}' + f'
' + f'

{mode_display}

' + f'

' + f'Periode aktif: {filter_label}' + f' ·  {basis_label}' + f'

' + f'
' + f'
' + f'{total:,} tweet' + f'
' + f'
', + unsafe_allow_html=True, + ) days = (pd.Timestamp(dt_end).date() - pd.Timestamp(dt_start).date()).days + 1 earliest = pd.Timestamp(dt_start).strftime("%d/%m/%Y") @@ -1235,42 +1345,44 @@ def show(): _render_metrics(total, days, earliest, latest, filter_label, basis_label) if total == 0: - _section_header("📋 Daftar Tweet yang Terkumpul", f"0 tweet · {filter_label}") - st.markdown(f""" -
-
🔍
-
- Tidak Ada Data
-
- Belum ada tweet dengan tanggal asli pada periode ini -
-
-""", unsafe_allow_html=True) + _section_header("📋 Daftar Tweet yang Terkumpul", f"0 tweet · {filter_label}") + st.markdown( + '
' + '
🔍
' + '
' + 'Tidak Ada Data
' + '
' + 'Belum ada tweet dengan tanggal asli pada periode ini' + '
' + '
', + unsafe_allow_html=True, + ) return # ── Chart ───────────────────────────────────────────────── - st.markdown(f""" -
-
-

- 📊 Distribusi Tweet Per Hari

- {filter_label} -
-

- Berdasarkan {chart_label} -

-
-""", unsafe_allow_html=True) + st.markdown( + f'
' + f'
' + f'

' + f'📊 Distribusi Tweet Per Hari

' + f'{filter_label}' + f'
' + f'

' + f'Berdasarkan {chart_label}' + f'

' + f'
', + unsafe_allow_html=True, + ) _render_charts(df, filter_label, date_col, chart_label, dt_start, dt_end) # ── Tabel ───────────────────────────────────────────────── display_limit = 100 _section_header( - "📋 Daftar Tweet yang Terkumpul", - f"Menampilkan hingga {display_limit} tweet terbaru · {filter_label}" + "📋 Daftar Tweet yang Terkumpul", + f"Menampilkan hingga {display_limit} tweet terbaru · {filter_label}" ) _gap("xs") @@ -1297,7 +1409,7 @@ def show(): c_dl, c_nav = st.columns(2) with c_dl: st.download_button( - f"📥 Unduh Semua Data ({total:,} tweet)", + f"Unduh Semua Data ({total:,} tweet)", df.to_csv(index=False).encode("utf-8"), f"data_twitter_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv", "text/csv", @@ -1305,7 +1417,7 @@ def show(): ) with c_nav: if st.button( - "🧹 Lanjut ke Bersihkan Data →", + "Lanjut ke Bersihkan Data", type="primary", use_container_width=True, key="btn_go_preprocessing" diff --git a/page_modules/preprocessing_page.py b/page_modules/preprocessing_page.py index f9a8ff8..017f36d 100644 --- a/page_modules/preprocessing_page.py +++ b/page_modules/preprocessing_page.py @@ -1,15 +1,19 @@ """ preprocessing_page.py ===================== -Halaman Bersihkan Data — NLP Pipeline 6 Tahap. +Halaman Bersihkan Data — NLP Pipeline 5 Tahap. -PIPELINE 6 TAHAP (selaras dengan sentiment_service.py): +PIPELINE 5 TAHAP (selaras dengan sentiment_service.py): 1. Case Folding — ubah semua huruf jadi lowercase 2. Cleaning — hapus URL, mention, hashtag, angka, emoji, tanda baca 3. Normalisasi — singkatan/slang → kata baku - 4. Tokenizing — pecah kalimat → list token - 5. Stopword Removal — hapus kata umum; JAGA kata sentimen penting - 6. Stemming — bentuk dasar kata via Sastrawi ECS + 4. Stopword Removal — hapus kata umum; JAGA kata sentimen penting + 5. Stemming — bentuk dasar kata via Sastrawi ECS + +Catatan: Tokenizing tidak ditampilkan sebagai tahap tersendiri karena: + • Stopword removal & stemming sudah melakukan split() secara internal + • TF-IDF melakukan tokenisasi sendiri saat inferensi + → Tokenizing adalah proses teknis, bukan tahap utama pipeline """ import streamlit as st @@ -91,7 +95,7 @@ def _sync_dynamic_period(): # ═══════════════════════════════════════════════════════════ -# PREPROCESSING PIPELINE — 6 TAHAP +# PREPROCESSING PIPELINE — 5 TAHAP # PENTING: Pipeline ini harus IDENTIK dengan sentiment_service.py # agar token yang dihasilkan konsisten dengan training model. # @@ -99,9 +103,8 @@ def _sync_dynamic_period(): # 1. Case Folding → lowercase dulu sebelum cleaning # 2. Cleaning → hapus noise setelah lowercase # 3. Normalisasi → singkatan/slang → kata baku -# 4. Tokenizing → split menjadi list token -# 5. Stopword Removal → buang kata umum, jaga kata sentimen -# 6. Stemming → bentuk dasar via Sastrawi ECS +# 4. Stopword Removal → buang kata umum, jaga kata sentimen +# 5. Stemming → bentuk dasar via Sastrawi ECS # ═══════════════════════════════════════════════════════════ KATA_SENTIMEN_PENTING = { @@ -251,21 +254,16 @@ def step3_normalization(text: str) -> str: return " ".join(NORMALISASI.get(word, word) for word in text.split()) -# ── Tahap 4: Tokenizing ───────────────────────────────────────────────────── -def step4_tokenizing(text: str) -> list: - return text.split() - - -# ── Tahap 5: Stopword Removal ─────────────────────────────────────────────── -def step5_stopword_removal(tokens: list, stopwords: set) -> list: +# ── Tahap 4: Stopword Removal ─────────────────────────────────────────────── +def step4_stopword_removal(tokens: list, stopwords: set) -> list: return [ w for w in tokens if (w not in stopwords or w in KATA_SENTIMEN_PENTING) and len(w) > 2 ] -# ── Tahap 6: Stemming ─────────────────────────────────────────────────────── -def step6_stemming(tokens: list, stemmer) -> list: +# ── Tahap 5: Stemming ─────────────────────────────────────────────────────── +def step5_stemming(tokens: list, stemmer) -> list: if stemmer is None: return tokens return [stemmer.stem(w) for w in tokens] @@ -273,32 +271,28 @@ def step6_stemming(tokens: list, stemmer) -> list: def full_preprocessing(text: str, stopwords: set, stemmer): """ - Jalankan 6 tahap preprocessing dan kembalikan dict hasil setiap tahap. + Jalankan 5 tahap preprocessing dan kembalikan dict hasil setiap tahap. URUTAN TAHAP: 1. Case Folding → lowercase 2. Cleaning → hapus noise 3. Normalisasi → normalisasi kata - 4. Tokenizing → split token - 5. Stopword Removal → buang stopword - 6. Stemming → bentuk dasar + 4. Stopword Removal → buang stopword (split() dilakukan internal) + 5. Stemming → bentuk dasar """ s1_fold = step1_case_folding(text) s2_clean = step2_cleaning(s1_fold) s3_norm = step3_normalization(s2_clean) - s4_tokens = step4_tokenizing(s3_norm) - s5_filtered = step5_stopword_removal(s4_tokens, stopwords) - s6_stemmed = step6_stemming(s5_filtered, stemmer) + s4_filtered = step4_stopword_removal(s3_norm.split(), stopwords) + s5_stemmed = step5_stemming(s4_filtered, stemmer) return { "setelah_casefolding": s1_fold, "setelah_cleaning": s2_clean, "setelah_normalisasi": s3_norm, - "setelah_tokenizing": " | ".join(s4_tokens), - "setelah_stopword": " ".join(s5_filtered), - "clean_text": " ".join(s6_stemmed), - "_tokens_raw": s4_tokens, - "_tokens_clean": s6_stemmed, + "setelah_stopword": " ".join(s4_filtered), + "clean_text": " ".join(s5_stemmed), + "_tokens_clean": s5_stemmed, } @@ -376,7 +370,6 @@ header[data-testid="stHeader"] { height: 0 !important; min-height: 0 !important; .pipe-3 { animation: fadeRight 0.35s 0.19s ease both; } .pipe-4 { animation: fadeRight 0.35s 0.26s ease both; } .pipe-5 { animation: fadeRight 0.35s 0.33s ease both; } -.pipe-6 { animation: fadeRight 0.35s 0.40s ease both; } .example-box { transition: all 0.18s ease; } .example-box:hover { @@ -384,19 +377,6 @@ header[data-testid="stHeader"] { height: 0 !important; min-height: 0 !important; box-shadow: 0 4px 14px rgba(59,108,247,0.10) !important; } -.token-chip { - display: inline-block; - background: #eef2ff; - color: #3b6cf7; - border: 1px solid #c7d2fe; - border-radius: 5px; - padding: 1px 7px; - font-size: 0.7rem; - font-weight: 600; - margin: 2px 2px 0 0; - line-height: 1.6; -} - .fix-badge { display: inline-block; background: #fef9c3; @@ -442,8 +422,8 @@ def _render_page_header(): margin:0 0 4px;letter-spacing:-0.01em;line-height:1.2;"> Bersihkan Data

- Preprocessing teks 6 tahap otomatis: - Case Folding → Cleaning → Normalisasi → Tokenizing → Stopword Removal → Stemming + Preprocessing teks 5 tahap otomatis: + Case Folding → Cleaning → Normalisasi → Stopword Removal → Stemming

✦ Diperbaiki', @@ -541,7 +506,7 @@ def _render_pipeline_steps(stemmer_ok): ], }, { - "num": "06", "anim": "pipe-6", + "num": "05", "anim": "pipe-5", "icon": "🌱", "color": "#ca8a04", "dark": "#713f12", "bg": "linear-gradient(135deg,#fefce8,#fef9c3)", "border": "#fde68a", "title": "Stemming", @@ -557,15 +522,17 @@ def _render_pipeline_steps(stemmer_ok): }, ] + # Baris 1: 3 kartu pertama row1 = st.columns(3, gap="medium") for col, step in zip(row1, steps[:3]): _render_step_card(col, step) _gap("sm") - row2 = st.columns(3, gap="medium") - for col, step in zip(row2, steps[3:]): - _render_step_card(col, step) + # Baris 2: 2 kartu terakhir (tengah agar simetris) + _, col4, col5, _ = st.columns([0.5, 1, 1, 0.5], gap="medium") + _render_step_card(col4, steps[3]) + _render_step_card(col5, steps[4]) def _render_step_card(col, step): @@ -618,14 +585,13 @@ def _render_step_card(col, step): def _render_flow_arrow(): nodes = [ - ("📄 Teks Asli", "#94a3b8", "#f8fafc", "#e2e8f0"), - ("① Case Folding", "#0284c7", "#eff6ff", "#bfdbfe"), - ("② Cleaning", "#3b6cf7", "#eef2ff", "#c7d2fe"), - ("③ Normalisasi", "#16a34a", "#f0fdf4", "#86efac"), - ("④ Tokenizing", "#7c3aed", "#f5f3ff", "#c4b5fd"), - ("⑤ Stopword", "#ea580c", "#fff7ed", "#fed7aa"), - ("⑥ Stemming", "#ca8a04", "#fefce8", "#fde68a"), - ("✅ Teks Bersih", "#0f172a", "#0f172a", "#334155"), + ("📄 Teks Asli", "#94a3b8", "#f8fafc", "#e2e8f0"), + ("① Case Folding", "#0284c7", "#eff6ff", "#bfdbfe"), + ("② Cleaning", "#3b6cf7", "#eef2ff", "#c7d2fe"), + ("③ Normalisasi", "#16a34a", "#f0fdf4", "#86efac"), + ("④ Stopword", "#ea580c", "#fff7ed", "#fed7aa"), + ("⑤ Stemming", "#ca8a04", "#fefce8", "#fde68a"), + ("✅ Teks Bersih", "#0f172a", "#0f172a", "#334155"), ] parts = "" @@ -660,14 +626,14 @@ def _render_flow_arrow(): # STAT PILLS # ═══════════════════════════════════════════════════════════ -def _render_stat_pills(total_raw, total_clean, avg_tokens_before, avg_tokens_after, removed): - c1, c2, c3, c4 = st.columns(4, gap="medium") +def _render_stat_pills(total_raw, total_clean, removed): + c1, c2, c3 = st.columns(3, gap="medium") cards = [ (c1, "📥", "linear-gradient(135deg,#eef2ff,#e0e7ff)", "#3b6cf7", "#1e3a8a", "#c7d2fe", "Tweet Diproses", f"{total_raw:,}", "Total tweet periode ini"), (c2, "✅", "linear-gradient(135deg,#f0fdf4,#dcfce7)", "#16a34a", "#14532d", "#86efac", - "Tweet Siap Analisis", f"{total_clean:,}", "Lulus semua 6 tahap"), + "Tweet Siap Analisis", f"{total_clean:,}", "Lulus semua 5 tahap"), (c3, "🗑️" if removed > 0 else "✅", ("linear-gradient(135deg,#fff7ed,#ffedd5)" if removed > 0 else "linear-gradient(135deg,#f0fdf4,#dcfce7)"), @@ -676,10 +642,6 @@ def _render_stat_pills(total_raw, total_clean, avg_tokens_before, avg_tokens_aft ("#fed7aa" if removed > 0 else "#86efac"), "Tweet Dibuang", f"{removed:,}", ("Teks kosong setelah preprocessing" if removed > 0 else "Semua tweet lolos")), - (c4, "🔤", "linear-gradient(135deg,#f5f3ff,#ede9fe)", "#7c3aed", "#3b0764", "#c4b5fd", - "Rata-rata Token", - f"{avg_tokens_before:.0f} → {avg_tokens_after:.0f}", - "Sebelum → Sesudah stopword"), ] for col, icon, bg, color, dark, border, label, val, sub in cards: @@ -716,37 +678,28 @@ def _render_live_example(df_c): "Contoh tweet acak dari dataset — refresh halaman untuk contoh berbeda" ) - # Urutan tampilan sesuai pipeline: CF → Clean → Norm → Token → Stop → Stem + # Urutan tampilan sesuai pipeline: CF → Clean → Norm → Stop → Stem steps_ex = [ - ("📄 Teks Asli", "text_asli", "#0f172a", "#f8fafc", "#e2e8f0", False), - ("① Setelah Case Folding", "setelah_casefolding", "#0c4a6e", "#eff6ff", "#bfdbfe", False), - ("② Setelah Cleaning", "setelah_cleaning", "#1e3a8a", "#eef2ff", "#c7d2fe", False), - ("③ Setelah Normalisasi", "setelah_normalisasi", "#14532d", "#f0fdf4", "#86efac", False), - ("④ Setelah Tokenizing", "setelah_tokenizing", "#3b0764", "#f5f3ff", "#c4b5fd", True), - ("⑤ Setelah Stopword", "setelah_stopword", "#7c2d12", "#fff7ed", "#fed7aa", False), - ("⑥ Hasil Akhir (Stem)", "clean_text", "#713f12", "#fefce8", "#fde68a", False), + ("📄 Teks Asli", "text_asli", "#0f172a", "#f8fafc", "#e2e8f0"), + ("① Setelah Case Folding", "setelah_casefolding", "#0c4a6e", "#eff6ff", "#bfdbfe"), + ("② Setelah Cleaning", "setelah_cleaning", "#1e3a8a", "#eef2ff", "#c7d2fe"), + ("③ Setelah Normalisasi", "setelah_normalisasi", "#14532d", "#f0fdf4", "#86efac"), + ("④ Setelah Stopword", "setelah_stopword", "#7c2d12", "#fff7ed", "#fed7aa"), + ("⑤ Hasil Akhir (Stem)", "clean_text", "#713f12", "#fefce8", "#fde68a"), ] - for label, col_key, text_color, bg, border, is_token in steps_ex: + for label, col_key, text_color, bg, border in steps_ex: raw = sample.get(col_key, "-") text_display = str(raw) if raw and str(raw).strip() else "—" - if is_token and text_display != "—": - words = text_display.split(" | ") - chips = "".join( - f'{w}' - for w in words if w.strip() - ) - content_html = f'
{chips}
' - char_info = f"{len(words)} token" - else: - content_html = ( - f'
{text_display}
' - ) - word_count = len(text_display.split()) if text_display != "—" else 0 - char_count = len(text_display) if text_display != "—" else 0 - char_info = f"{word_count} kata · {char_count} karakter" + word_count = len(text_display.split()) if text_display != "—" else 0 + char_count = len(text_display) if text_display != "—" else 0 + char_info = f"{word_count} kata · {char_count} karakter" + + content_html = ( + f'
{text_display}
' + ) st.markdown( f'
-""", unsafe_allow_html=True) - st.plotly_chart(fig, use_container_width=True, config={"displayModeBar": False}) - st.markdown("
", unsafe_allow_html=True) - - # ═══════════════════════════════════════════════════════════ # TOP WORDS CHART # ═══════════════════════════════════════════════════════════ @@ -830,7 +723,7 @@ def _render_length_comparison_chart(df, df_c): def _render_top_words_chart(df_c): _section_header( "📊 Kata-Kata Paling Sering Muncul", - f"Dari {len(df_c):,} tweet yang sudah bersih (hasil akhir tahap 6) — Top 20 kata" + f"Dari {len(df_c):,} tweet yang sudah bersih (hasil akhir tahap 5) — Top 20 kata" ) if "_tokens_clean" in df_c.columns: @@ -930,8 +823,8 @@ def show(): # ── Pipeline Overview ───────────────────────────────────── _section_header( - "🔬 Alur NLP Pipeline — 6 Tahap Preprocessing", - "Setiap tweet diproses berurutan melalui 6 tahap sebelum siap dianalisis sentimennya" + "🔬 Alur NLP Pipeline — 5 Tahap Preprocessing", + "Setiap tweet diproses berurutan melalui 5 tahap sebelum siap dianalisis sentimennya" ) stemmer_tmp = _load_stemmer() @@ -967,12 +860,12 @@ def show(): data_marker = (total_tweets_in_db, latest_crawl_marker) cache_key = ( - f"pp6_{st.session_state.analysis_mode}_" + f"pp5_{st.session_state.analysis_mode}_" f"{start_date}_{end_date}_{total_tweets_in_db}_{latest_crawl_marker}" ) for old_key in list(st.session_state.keys()): - if old_key.startswith("pp6_") and old_key != cache_key: + if old_key.startswith("pp5_") and old_key != cache_key: del st.session_state[old_key] force_refresh = data_marker != st.session_state.get("_pp_last_data_marker") @@ -981,7 +874,7 @@ def show(): stemmer = _load_stemmer() stopwords = _load_stopwords() - with st.spinner("🧹 Menjalankan 6 tahap preprocessing…"): + with st.spinner("🧹 Menjalankan 5 tahap preprocessing…"): results = [] for _, row in df.iterrows(): r = full_preprocessing(row["text"], stopwords, stemmer) @@ -1002,12 +895,6 @@ def show(): stemmer_ok = st.session_state.get(cache_key + "_stemmer_ok", False) # ── Statistik ───────────────────────────────────────────── - avg_tok_before = df_c["setelah_tokenizing"].apply( - lambda x: len(str(x).split(" | ")) if str(x).strip() else 0 - ).mean() - avg_tok_after = df_c["clean_text"].apply( - lambda x: len(str(x).split()) if str(x).strip() else 0 - ).mean() removed = len(df) - len(df_c) _section_header( @@ -1015,7 +902,7 @@ def show(): f"Berdasarkan tanggal asli tweet · {filter_label}" ) _gap("xs") - _render_stat_pills(len(df), len(df_c), avg_tok_before, avg_tok_after, removed) + _render_stat_pills(len(df), len(df_c), removed) _gap("lg") _render_live_example(df_c) @@ -1031,14 +918,13 @@ def show(): df_c = df_c.copy() df_c["crawled_at"] = pd.NaT - # Kolom ditampilkan sesuai urutan pipeline: CF → Clean → Norm → Token → Stop → Stem + # Kolom ditampilkan sesuai urutan pipeline: CF → Clean → Norm → Stop → Stem disp = df_c[[ "tweet_id", "created_at", "crawled_at", "text_asli", "setelah_casefolding", "setelah_cleaning", "setelah_normalisasi", - "setelah_tokenizing", "setelah_stopword", "clean_text", ]].copy() @@ -1049,9 +935,8 @@ def show(): "① Case Folding", "② Cleaning", "③ Normalisasi", - "④ Tokenizing", - "⑤ Stopword", - "⑥ Hasil Akhir", + "④ Stopword", + "⑤ Hasil Akhir", ] disp["Tanggal Tweet"] = disp["Tanggal Tweet"].apply(format_dt) @@ -1060,30 +945,26 @@ def show(): render_standard_table( disp, height=360, - min_width=2200, + min_width=1980, nowrap=["ID Tweet", "Tanggal Tweet", "Masuk Database"], wide_columns=[ "Teks Asli", "① Case Folding", "② Cleaning", - "③ Normalisasi", "④ Tokenizing", "⑤ Stopword", "⑥ Hasil Akhir", + "③ Normalisasi", "④ Stopword", "⑤ Hasil Akhir", ], column_widths={ "ID Tweet": "155px", "Tanggal Tweet": "165px", "Masuk Database": "165px", "Teks Asli": "280px", - "① Case Folding": "220px", + "① Case Folding": "240px", "② Cleaning": "240px", - "③ Normalisasi": "220px", - "④ Tokenizing": "240px", - "⑤ Stopword": "220px", - "⑥ Hasil Akhir": "220px", + "③ Normalisasi": "240px", + "④ Stopword": "230px", + "⑤ Hasil Akhir": "230px", }, ) _gap("lg") - _render_length_comparison_chart(df, df_c) - _gap("lg") - _render_top_words_chart(df_c) _gap("lg") @@ -1102,7 +983,7 @@ def show(): with d1: st.download_button( "📥 Unduh Hasil Preprocessing Lengkap (semua kolom)", - df_c.drop(columns=["_tokens_raw", "_tokens_clean"], errors="ignore" + df_c.drop(columns=["_tokens_clean"], errors="ignore" ).to_csv(index=False).encode("utf-8"), f"preprocessing_lengkap_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv", "text/csv", diff --git a/page_modules/sentiment_page.py b/page_modules/sentiment_page.py index c6af26d..b785c46 100644 --- a/page_modules/sentiment_page.py +++ b/page_modules/sentiment_page.py @@ -1,512 +1,1066 @@ """ -sentiment_service.py -==================== -Hybrid Classifier untuk analisis sentimen tweet Bahasa Indonesia. +sentiment_page.py +================= +Halaman Analisis Sentimen — diperbaiki untuk mengatasi semua prediksi negatif. -PIPELINE PREPROCESSING — 6 TAHAP (selaras dengan preprocessing_page.py): - 1. Case Folding → lowercase dulu sebelum cleaning - 2. Cleaning → hapus URL, mention, hashtag, angka, emoji, tanda baca - 3. Normalisasi → singkatan/slang → kata baku - 4. Tokenizing → split menjadi list token - 5. Stopword Removal → buang kata umum, jaga kata sentimen penting - 6. Stemming → bentuk dasar via Sastrawi ECS +PERBAIKAN vs VERSI LAMA: +───────────────────────────────────────────────────────────────────────────── +1. MENGGUNAKAN sentiment_service.py (HYBRID CLASSIFIER) + Tidak lagi memanggil model secara langsung dengan predict_batch sederhana. + Sebaliknya menggunakan sentiment_service yang sudah mengimplementasikan: + • Lexicon-based override untuk kelas POSITIF (yang tidak ada di model) + • Negation handling (tidak bagus ≠ bagus) + • Fallback ke model NB untuk negatif vs netral -ARSITEKTUR HYBRID: - Teks Asli - ├── preprocess_untuk_lexicon() ──→ _hitung_skor_lexicon() ──┐ - └── preprocess_for_model() ──→ TF-IDF → NB Model ───────┘ - ↓ - _klasifikasi_hybrid() → Label + Confidence +2. PREPROCESSING SELARAS + Memanggil preprocess_for_model() dari sentiment_service agar preprocessing + yang digunakan untuk prediksi persis sama dengan yang dipakai saat training. + +3. CONFIDENCE SCORE AKURAT + Menggunakan confidence gabungan dari lexicon score + model probability, + bukan hanya max probability dari model yang bias ke negatif. + +4. LABEL MAPPING KONSISTEN + Label dikembalikan selalu dalam format: 'Positif', 'Netral', 'Negatif' + (kapital huruf pertama) agar konsisten di seluruh tampilan. +───────────────────────────────────────────────────────────────────────────── """ +import streamlit as st +import pandas as pd +import numpy as np +import os import re import string -import joblib +from collections import Counter +from datetime import datetime, timedelta +from html import escape +from database import engine, get_tweet_count, get_latest_crawl_time +from timezone_utils import ( + parse_dt_with_tz, + parse_dt_with_source_tz, + get_timezone_label, + get_timezone_name, +) +import plotly.graph_objects as go + +# ── Import sentiment_service (hybrid classifier) ────────────────────────── +from sentiment_service import ( + preprocess_for_model, + preprocess_untuk_lexicon, + _hitung_skor_lexicon, + _klasifikasi_hybrid, + _load_stopwords, + _load_stemmer, + _STOPWORDS, + _STEMMER, +) + +from page_modules.table_utils import render_standard_table -# ═══════════════════════════════════════════════════════════ -# KATA SENTIMEN PENTING -# Tidak boleh dihapus di tahap stopword removal -# ═══════════════════════════════════════════════════════════ +# ───────────────────────────────────────────────────────────── +# Timezone & formatting helpers +# ───────────────────────────────────────────────────────────── -KATA_SENTIMEN_PENTING = { - # ── Negasi ────────────────────────────────────────────── - "tidak", "bukan", "jangan", "kurang", "belum", "tanpa", - - # ── Sentimen POSITIF ──────────────────────────────────── - "keren", "bagus", "mantap", "setuju", "dukung", "mendukung", - "andal", "handal", "gercep", "bangga", "senang", "suka", - "baik", "benar", "tepat", "oke", - "sejahtera", "berkembang", "maju", "inovatif", - "tegas", "sigap", "tanggap", "adil", "bijak", "bermanfaat", - "untung", "berhasil", "sukses", "solusi", "manfaat", - "berguna", "membantu", "bantu", "pro", "lanjut", - "sangat", "banget", "sekali", "paling", "amat", "luar", "biasa", - - # ── Sentimen NEGATIF evaluatif ────────────────────────── - "kecewa", "buruk", "jelek", "parah", "gagal", "hancur", - "rusak", "bohong", "tipu", "korupsi", - - # ── Emosi ─────────────────────────────────────────────── - "marah", "sedih", "khawatir", -} +def parse_dt(series): + return parse_dt_with_tz(series, st.session_state.get("user_timezone", "WIB (UTC+7)")) -# ═══════════════════════════════════════════════════════════ -# NORMALISASI (selaras dengan preprocessing_page.py) -# ═══════════════════════════════════════════════════════════ - -NORMALISASI = { - # Negasi - "gk": "tidak", "ga": "tidak", "gak": "tidak", - "nggak": "tidak", "ngga": "tidak", "tdk": "tidak", - "tak": "tidak", "enggak": "tidak", "engga": "tidak", - "kagak": "tidak", "kaga": "tidak", "ndak": "tidak", - "gkk": "tidak", "ngak": "tidak", - # Kata ganti - "yg": "yang", "dgn": "dengan", "utk": "untuk", - "org": "orang", "krn": "karena", "dr": "dari", - "sm": "sama", "pd": "pada", "dlm": "dalam", - "bwt": "buat", "trm": "terima", - # Verba - "tp": "tapi", "tpi": "tapi", "jd": "jadi", - "sdh": "sudah", "blm": "belum", "emg": "memang", - "emang": "memang", "gimana": "bagaimana", - "gitu": "begitu", "gini": "begini", - "udah": "sudah", "udh": "sudah", - # Intensitas - "bgt": "banget", "bngt": "banget", "bget": "banget", "bgtt": "banget", - # Positif informal - "bener": "benar", "beneran": "benar", - "mantep": "mantap", "mntap": "mantap", - "kece": "keren", - "cucok": "cocok", "cucuk": "cocok", - "cakep": "bagus", - "sip": "baik", "siipp": "baik", - "top": "terbaik", - "jos": "bagus", "josss": "bagus", - "goks": "luar biasa", - "setujuu": "setuju", "stuju": "setuju", - "proud": "bangga", - "mantul": "mantap betul", - # Negatif informal - "ancur": "hancur", "ancrr": "hancur", - "parahh": "parah", "parahhh": "parah", - "gagall": "gagal", - "ngaco": "tidak benar", - "ngasal": "tidak benar", - "gaje": "tidak jelas", - # Domain - "ongkir": "ongkos kirim", - "freeongkir": "gratis ongkos kirim", - "gratisongkir": "gratis ongkos kirim", - "free": "gratis", - "ecommerce": "e commerce", - "seller": "penjual", - "buyer": "pembeli", -} - - -# ═══════════════════════════════════════════════════════════ -# LEXICON SENTIMEN -# ═══════════════════════════════════════════════════════════ - -LEXICON_POSITIF = { - "bagus", "baik", "keren", "mantap", "mantep", "hebat", - "oke", "sip", "top", "jos", "goks", "kece", "mantul", - "setuju", "dukung", "mendukung", "pro", "lanjut", "sepakat", - "bangga", "senang", "suka", "puas", "gembira", "bahagia", - "berhasil", "sukses", "berjaya", "prestasi", "pencapaian", - "andal", "handal", "gercep", "sigap", "tanggap", "tegas", - "adil", "bijak", "bermanfaat", "berguna", "membantu", - "inovatif", "maju", "berkembang", "sejahtera", - "benar", "tepat", "jelas", "transparan", "amanah", "terpercaya", - "untung", "gratis", "murah", "hemat", "terjangkau", - "terbaik", "luar biasa", - "cakep", "cucok", "gaskeun", "kuy", - "dukung", "bantu", "solusi", "manfaat", - "memuaskan", "membanggakan", "mengagumkan", -} - -LEXICON_NEGATIF = { - "buruk", "jelek", "parah", "rusak", "hancur", "ancur", - "gagal", "gagall", "ambruk", "terpuruk", "bangkrut", - "bohong", "tipu", "curang", "manipulasi", "korupsi", "penipuan", - "kebohongan", - "kecewa", "marah", "sedih", "khawatir", "takut", "benci", - "jijik", "muak", "kesal", "frustrasi", - "mahal", "rugi", "merugikan", - "lambat", "lemot", "ribet", "susah", "sulit", "bermasalah", - "ngaco", "ngasal", "gaje", "receh", - "tidak benar", "tidak jelas", "tidak adil", "tidak berguna", - "mengecewakan", "menyebalkan", "menyusahkan", -} - -KATA_NEGASI = { - "tidak", "bukan", "jangan", "belum", "tanpa", "kurang", - "anti", "non", -} - - -# ═══════════════════════════════════════════════════════════ -# STOPWORDS -# ═══════════════════════════════════════════════════════════ - -def _load_stopwords() -> set: - """Load stopword dengan penjagaan kata sentimen penting.""" - stopword_file = "indonesian-stopwords-complete.txt" - base = set() - try: - with open(stopword_file, "r", encoding="utf-8") as f: - base = set(f.read().splitlines()) - except FileNotFoundError: - base = { - "yang", "dan", "di", "ke", "dari", "ini", "itu", - "dengan", "untuk", "pada", "adalah", "oleh", "ada", - "ya", "akan", "atau", "juga", "sama", "karena", - "jika", "sudah", "telah", "saat", "agar", "maka", - "lagi", "bila", "bisa", "pun", "nya", - } - - # Jangan hapus kata sentimen penting - for kata in KATA_SENTIMEN_PENTING: - base.discard(kata) - - # Tambahan stopword domain-spesifik - base.update({ - "rt", "amp", "https", "http", "co", "t", - "wkwk", "wkwkwk", "haha", "hehe", "xixi", "hahaha", - "yg", "dgn", "utk", "dr", "krn", "tp", "jd", "sdh", - "aja", "doang", "nih", "sih", "dong", "deh", - "loh", "lah", "tuh", "kak", "gan", "bro", "sis", - }) - return base - - -# ═══════════════════════════════════════════════════════════ -# STEMMER -# ═══════════════════════════════════════════════════════════ - -def _load_stemmer(): - """Load Sastrawi stemmer. Return None jika tidak terinstall.""" - try: - from Sastrawi.Stemmer.StemmerFactory import StemmerFactory - return StemmerFactory().create_stemmer() - except Exception: - return None - - -# ── Inisialisasi global ───────────────────────────────────────────────────── -_STOPWORDS = _load_stopwords() -_STEMMER = _load_stemmer() - - -# ═══════════════════════════════════════════════════════════ -# MODEL LOADING (lazy) -# ═══════════════════════════════════════════════════════════ - -_model = None -_tfidf = None - - -def _load_model(): - """ - Lazy load NB model + TF-IDF vectorizer. - Return: (model, tfidf) — keduanya bisa None. - """ - global _model, _tfidf - if _model is None: - try: - _model = joblib.load("model_naive_bayes.pkl") - except Exception as e: - print(f"[WARNING] Gagal load model_naive_bayes.pkl: {e}") - if _tfidf is None: - try: - _tfidf = joblib.load("tfidf_vectorizer.pkl") - except Exception as e: - print(f"[WARNING] Gagal load tfidf_vectorizer.pkl: {e}") - return _model, _tfidf - - -# ═══════════════════════════════════════════════════════════ -# PREPROCESSING PIPELINE — 6 TAHAP -# -# URUTAN (selaras dengan preprocessing_page.py): -# 1. Case Folding → lowercase -# 2. Cleaning → hapus noise -# 3. Normalisasi → normalisasi kata -# 4. Tokenizing → split token -# 5. Stopword Removal → buang stopword -# 6. Stemming → bentuk dasar -# ═══════════════════════════════════════════════════════════ - -def _case_folding(text: str) -> str: - """Tahap 1: Lowercase seluruh teks.""" - return str(text).lower() - - -def _cleaning(text: str) -> str: - """Tahap 2: Hapus noise (URL, mention, hashtag, angka, emoji, tanda baca).""" - text = re.sub(r"http\S+|www\S+|https\S+", "", text) - text = re.sub(r"@\w+", "", text) - text = re.sub(r"#\w+", "", text) - text = re.sub(r"\d+", "", text) - text = re.sub( - r"[\U00010000-\U0010ffff" - r"\U0001F600-\U0001F64F" - r"\U0001F300-\U0001F5FF" - r"\U0001F680-\U0001F6FF" - r"\U0001F1E0-\U0001F1FF" - r"\u2600-\u26FF\u2700-\u27BF" - r"]+", "", text, flags=re.UNICODE, +def parse_crawled_dt(series): + return parse_dt_with_source_tz( + series, + st.session_state.get("user_timezone", "WIB (UTC+7)"), + os.getenv("APP_TIMEZONE", "Asia/Makassar"), ) - text = text.translate(str.maketrans("", "", string.punctuation)) - text = re.sub(r"[^a-zA-Z\s]", "", text) - text = re.sub(r"\s+", " ", text).strip() - return text -def _normalisasi(text: str) -> str: - """Tahap 3: Normalisasi singkatan dan kata tidak baku.""" - return " ".join(NORMALISASI.get(word, word) for word in text.split()) - - -def _tokenize(text: str) -> list: - """Tahap 4: Tokenizing — split ke list token.""" - return text.split() - - -def _remove_stopwords(tokens: list) -> list: - """Tahap 5: Stopword removal dengan penjagaan kata sentimen.""" - return [ - w for w in tokens - if (w not in _STOPWORDS or w in KATA_SENTIMEN_PENTING) and len(w) > 2 - ] - - -def _stemming(tokens: list) -> list: - """Tahap 6: Stemming ke bentuk dasar via Sastrawi ECS.""" - if _STEMMER is None: - return tokens - return [_STEMMER.stem(w) for w in tokens] - - -def preprocess_for_model(text: str) -> str: - """ - Full 6-tahap preprocessing → string teks bersih siap TF-IDF. - - URUTAN: Case Folding → Cleaning → Normalisasi → Tokenizing - → Stopword Removal → Stemming - - Pipeline HARUS sama persis dengan yang dipakai saat training model. - """ - s1 = _case_folding(text) # Tahap 1 - s2 = _cleaning(s1) # Tahap 2 - s3 = _normalisasi(s2) # Tahap 3 - s4 = _tokenize(s3) # Tahap 4 - s5 = _remove_stopwords(s4) # Tahap 5 - s6 = _stemming(s5) # Tahap 6 - return " ".join(s6) - - -def preprocess_untuk_lexicon(text: str) -> str: - """ - Preprocessing RINGAN untuk lexicon matching. - Tidak di-stem → kata asli bisa dicocokkan dengan lexicon. - Pipeline: Case Folding → Cleaning → Normalisasi saja. - """ - s1 = _case_folding(text) - s2 = _cleaning(s1) - s3 = _normalisasi(s2) - return s3 - - -# ═══════════════════════════════════════════════════════════ -# LEXICON SCORER -# ═══════════════════════════════════════════════════════════ - -def _hitung_skor_lexicon(teks_lexicon: str) -> dict: - """ - Hitung skor positif dan negatif dari teks via lexicon. - - NEGATION HANDLING: - Kata negasi dalam window 2 kata sebelum kata sentimen → polaritas dibalik. - Contoh: "tidak bagus" → ada "tidak" sebelum "bagus" (POSITIF) - → skor_neg += 1 (bukan skor_pos) - - Return: {"positif": int, "negatif": int, "net": int} - """ - tokens = teks_lexicon.split() - skor_pos = 0 - skor_neg = 0 - - for i, token in enumerate(tokens): - ada_negasi = any( - tokens[i - j] in KATA_NEGASI - for j in range(1, 3) - if i - j >= 0 - ) - - if token in LEXICON_POSITIF: - if ada_negasi: - skor_neg += 1 - else: - skor_pos += 1 - elif token in LEXICON_NEGATIF: - if ada_negasi: - skor_pos += 1 - else: - skor_neg += 1 - - return { - "positif": skor_pos, - "negatif": skor_neg, - "net": skor_pos - skor_neg, - } - - -# ═══════════════════════════════════════════════════════════ -# HYBRID CLASSIFIER -# ═══════════════════════════════════════════════════════════ - -# Normalisasi label dari berbagai format yang mungkin dipakai model -_LABEL_MAP = { - "positif": "Positif", "Positif": "Positif", "positive": "Positif", "pos": "Positif", - "negatif": "Negatif", "Negatif": "Negatif", "negative": "Negatif", "neg": "Negatif", - "netral": "Netral", "Netral": "Netral", "neutral": "Netral", "net": "Netral", -} - - -def _prediksi_model(teks_model: str): - """ - Dapatkan prediksi dari NB model. - Return: (label_norm, confidence, proba_dict) atau None. - """ - model, tfidf = _load_model() - if model is None or tfidf is None or not teks_model.strip(): - return None +def format_dt(value): + if value is None or pd.isna(value): + return "Belum ada" try: - vec = tfidf.transform([teks_model]) - pred_raw = model.predict(vec)[0] - proba = model.predict_proba(vec)[0] - classes = list(model.classes_) - - label_norm = _LABEL_MAP.get(str(pred_raw), "Netral") - pred_idx = classes.index(pred_raw) if pred_raw in classes else 0 - confidence = float(proba[pred_idx]) - - proba_dict = { - _LABEL_MAP.get(str(cls), str(cls)): float(p) - for cls, p in zip(classes, proba) - } - return label_norm, confidence, proba_dict - except Exception as e: - print(f"[WARNING] Prediksi model gagal: {e}") - return None + tz_label = get_timezone_label(st.session_state.get("user_timezone", "WIB (UTC+7)")) + return f"{value.strftime('%d/%m/%Y %H:%M')} {tz_label}" + except Exception: + return "Belum ada" -def _klasifikasi_hybrid( - teks_model: str, - skor: dict, - teks_lower: str, -) -> tuple: +def format_now(): + now = parse_dt(pd.Series([datetime.utcnow().isoformat()])).iloc[0] + return format_dt(now) + + +def user_today(): + timezone_choice = st.session_state.get("user_timezone", "WIB (UTC+7)") + return pd.Timestamp.now(tz=get_timezone_name(timezone_choice)).date() + + +def _sync_dynamic_period(): + mode = st.session_state.get("analysis_mode") + today = user_today() + configs = { + "realtime": (today - timedelta(days=6), today, "Tweet Terkini — 7 Hari Terakhir"), + "30days": (today - timedelta(days=29), today, "30 Hari Terakhir"), + "captured": (today, today, "Tweet Hari Ini"), + } + if mode not in configs: + return + start_day, end_day, mode_display = configs[mode] + start_day = pd.Timestamp(start_day).date() + end_day = pd.Timestamp(end_day).date() + dt_start = datetime.combine(start_day, datetime.min.time()) + dt_end = datetime.combine(end_day + timedelta(days=1), datetime.min.time()) + st.session_state.filter_start_date = dt_start + st.session_state.filter_end_date = dt_end + st.session_state.filter_label = f"{dt_start.strftime('%d/%m/%Y')} s/d {end_day.strftime('%d/%m/%Y')}" + st.session_state.mode_display = mode_display + st.session_state.filter_date_column = "created_at" + + +# ───────────────────────────────────────────────────────────── +# Core prediction function (DIPERBAIKI) +# ───────────────────────────────────────────────────────────── + +def predict_batch_hybrid(texts: list[str]) -> list[tuple[str, float]]: """ - Klasifikasi hybrid: sinyal lexicon + model NB. + Prediksi sentimen menggunakan Hybrid Classifier dari sentiment_service. - LOGIKA KEPUTUSAN (berurutan): + Pipeline per tweet: + 1. preprocess_for_model() → teks untuk TF-IDF + NB + 2. preprocess_untuk_lexicon() → teks untuk pengecekan lexicon + 3. _hitung_skor_lexicon() → hitung sinyal positif/negatif kuat + 4. _klasifikasi_hybrid() → putuskan label + confidence - Layer 1 — Override Positif KUAT (net >= 2): - → Positif, confidence 60–92% - - Layer 2 — Override Positif LEMAH (net == 1): - → Cek model; jika model < 65% yakin Negatif → Positif - → Jika model sangat yakin Negatif → tetap Positif (confidence rendah) - - Layer 3 — Override Negatif KUAT (net <= -2): - → Negatif, confidence 60–90% - - Layer 4 — Fallback Model NB: - → Prediksi model dipakai, TAPI: - • Jika model = Negatif AND net >= 0 AND confidence < 75% - → downgrade ke Netral (koreksi bias model) - • Kasus lainnya → percaya model - - Layer 5 — Ultimate Fallback (model tidak tersedia): - → Gunakan skor lexicon saja - - Return: (label: str, confidence: float) + Return: list of (label, confidence) """ - net = skor["net"] - pos = skor["positif"] - - # ── Layer 1: Positif KUAT ───────────────────────────────────────────────── - if net >= 2: - conf = min(0.60 + (net * 0.07), 0.92) - return ("Positif", round(conf, 3)) - - # ── Layer 2: Positif LEMAH ──────────────────────────────────────────────── - if net == 1 and pos >= 1: - model_result = _prediksi_model(teks_model) - if model_result is not None: - _, _, proba_dict = model_result - neg_prob = proba_dict.get("Negatif", 0.0) - if neg_prob < 0.65: - conf = round(0.55 + (0.65 - neg_prob) * 0.3, 3) - return ("Positif", min(conf, 0.80)) - else: - return ("Positif", 0.55) - else: - return ("Positif", 0.58) - - # ── Layer 3: Negatif KUAT ───────────────────────────────────────────────── - if net <= -2: - conf = min(0.60 + (abs(net) * 0.06), 0.90) - return ("Negatif", round(conf, 3)) - - # ── Layer 4: Fallback Model NB ──────────────────────────────────────────── - model_result = _prediksi_model(teks_model) - if model_result is not None: - label_norm, confidence, proba_dict = model_result - - # Anti-bias correction: - # Model prediksi Negatif tapi tidak ada sinyal negatif dari lexicon - # dan confidence < 75% → kemungkinan bias → turunkan ke Netral - if label_norm == "Negatif" and net >= 0 and confidence < 0.75: - corrected_conf = round(0.50 + max(0, confidence - 0.50) * 0.2, 3) - return ("Netral", corrected_conf) - - return (label_norm, round(confidence, 3)) - - # ── Layer 5: Ultimate Fallback ──────────────────────────────────────────── - if net > 0: - return ("Positif", 0.55) - elif net < 0: - return ("Negatif", 0.55) - else: - return ("Netral", 0.50) - - -# ═══════════════════════════════════════════════════════════ -# BACKWARD COMPATIBILITY -# ═══════════════════════════════════════════════════════════ - -def bersihkan_teks(text: str) -> str: - """[LEGACY] Gunakan preprocess_for_model() untuk pipeline lengkap.""" - return preprocess_for_model(text) - - -def prediksi_sentimen(list_text: list): - """ - [LEGACY] Prediksi batch dengan hybrid classifier. - Return: (list clean_texts, list labels) - """ - clean_texts = [preprocess_for_model(t) for t in list_text] - labels = [] - for text in list_text: + results = [] + for text in texts: teks_model = preprocess_for_model(text) teks_lexicon = preprocess_untuk_lexicon(text) teks_lower = str(text).lower() - skor = _hitung_skor_lexicon(teks_lexicon) - label, _ = _klasifikasi_hybrid(teks_model, skor, teks_lower) - labels.append(label) - return clean_texts, labels \ No newline at end of file + + skor = _hitung_skor_lexicon(teks_lexicon) + label, conf = _klasifikasi_hybrid(teks_model, skor, teks_lower) + results.append((label, conf)) + return results + + +def preprocess_single(text: str) -> str: + """Preprocess satu teks untuk disimpan ke kolom clean_text.""" + return preprocess_for_model(text) + + +# ───────────────────────────────────────────────────────────── +# Shared UI helpers +# ───────────────────────────────────────────────────────────── + +def _section_header(title, subtitle=""): + sub_html = ( + f'
{subtitle}
' + if subtitle else "" + ) + st.markdown( + f'
' + f'
{title}
' + f'{sub_html}
', + unsafe_allow_html=True, + ) + + +def _section_gap(size="md"): + heights = {"sm": "1rem", "md": "1.45rem", "lg": "1.9rem"} + st.markdown(f'
', unsafe_allow_html=True) + + +def _render_sentiment_styles(): + st.markdown(""" + +""", unsafe_allow_html=True) + + +# ───────────────────────────────────────────────────────────── +# Page Header +# ───────────────────────────────────────────────────────────── + +def _render_page_header(): + st.markdown(""" +
+
📈
+
+

+ Analisis Sentimen

+

+ Klasifikasi otomatis tweet menggunakan Hybrid Classifier — Positif · Netral · Negatif

+
+
🤖 Hybrid Classifier
+
+""", unsafe_allow_html=True) + + +# ───────────────────────────────────────────────────────────── +# Summary Pills +# ───────────────────────────────────────────────────────────── + +def _render_summary_pills(total, pos_n, neu_n, neg_n, pos_p, neu_p, neg_p, filter_label): + pills = [ + ("stat-1", "📊", "linear-gradient(135deg,#eef2ff,#e0e7ff)", "#3b6cf7", "#1e3a8a", "#c7d2fe", + "Total Dianalisis", f"{total:,}", f"Periode {filter_label}"), + ("stat-2", "😊", "linear-gradient(135deg,#f0fdf4,#dcfce7)", "#16a34a", "#14532d", "#86efac", + "Positif", f"{pos_n:,}", f"{pos_p:.1f}% dari total"), + ("stat-3", "😐", "linear-gradient(135deg,#f8fafc,#f1f5f9)", "#64748b", "#334155", "#cbd5e1", + "Netral", f"{neu_n:,}", f"{neu_p:.1f}% dari total"), + ("stat-4", "😞", "linear-gradient(135deg,#fef2f2,#fee2e2)", "#ef4444", "#7f1d1d", "#fca5a5", + "Negatif", f"{neg_n:,}", f"{neg_p:.1f}% dari total"), + ] + + cols = st.columns(4, gap="medium") + for col, (anim, icon, bg, color, dark, border, label, val, sub) in zip(cols, pills): + fs = "1.6rem" if len(str(val)) <= 6 else "1.2rem" + with col: + st.markdown( + f'
' + f'
{icon}
' + f'
{label}
' + f'
{val}
' + f'
{sub}
' + f'
', + unsafe_allow_html=True, + ) + + +# ───────────────────────────────────────────────────────────── +# Proportion Bar +# ───────────────────────────────────────────────────────────── + +def _render_proportion_bar(pos_p, neu_p, neg_p): + pos_p = round(pos_p, 1) + neu_p = round(neu_p, 1) + neg_p = round(neg_p, 1) + + st.markdown( + f'
' + f'
Proporsi Sentimen Keseluruhan
' + f'
' + f'
' + f'{"😊 " + str(pos_p) + "%" if pos_p >= 8 else ""}
' + f'
' + f'{"😐 " + str(neu_p) + "%" if neu_p >= 8 else ""}
' + f'
' + f'{"😞 " + str(neg_p) + "%" if neg_p >= 8 else ""}
' + f'
' + f'
' + f'● Positif {pos_p}%' + f'● Netral {neu_p}%' + f'● Negatif {neg_p}%' + f'
', + unsafe_allow_html=True, + ) + + +# ───────────────────────────────────────────────────────────── +# Donut + Bar chart +# ───────────────────────────────────────────────────────────── + +def _render_donut_chart(pos_n, neu_n, neg_n, total, filter_label): + labels, values, colors = [], [], [] + color_map = {"Positif": "#16a34a", "Netral": "#94a3b8", "Negatif": "#ef4444"} + for label, val in [("Positif", pos_n), ("Netral", neu_n), ("Negatif", neg_n)]: + if val > 0: + labels.append(label) + values.append(val) + colors.append(color_map[label]) + + dominant = max([("Positif", pos_n), ("Netral", neu_n), ("Negatif", neg_n)], key=lambda x: x[1]) + dom_pct = round(dominant[1] / total * 100) if total > 0 else 0 + dom_emoji = {"Positif": "😊", "Netral": "😐", "Negatif": "😞"}.get(dominant[0], "📊") + + fig = go.Figure(data=[go.Pie( + labels=labels, values=values, hole=0.62, + marker=dict(colors=colors, line=dict(color="white", width=4)), + textinfo="label+percent", textfont=dict(size=12), + hovertemplate="%{label}
%{value:,} tweet — %{percent}", + direction="clockwise", sort=False, + )]) + fig.add_annotation( + text=f"{dom_pct}%
{dom_emoji} {dominant[0]}", + x=0.5, y=0.5, showarrow=False, align="center", font=dict(size=18, color="#0f172a"), + ) + fig.update_layout( + height=290, margin=dict(l=10, r=10, t=10, b=10), + showlegend=True, + legend=dict(orientation="h", y=-0.1, x=0.5, xanchor="center", font=dict(size=11)), + paper_bgcolor="rgba(0,0,0,0)", + ) + st.plotly_chart(fig, width="stretch", config={"displayModeBar": False}) + return dominant + + +def _render_bar_chart(pos_n, neu_n, neg_n, total): + categories = ["Positif 😊", "Netral 😐", "Negatif 😞"] + values = [pos_n, neu_n, neg_n] + colors = ["#16a34a", "#94a3b8", "#ef4444"] + percentages = [(v / total * 100) if total > 0 else 0 for v in values] + + fig = go.Figure() + for cat, val, color, pct in zip(categories, values, colors, percentages): + fig.add_trace(go.Bar( + x=[cat], y=[val], + marker=dict(color=color, opacity=0.88, cornerradius=8), + text=[f"{val:,}"], textposition="outside", + textfont=dict(size=13, color="#0f172a"), width=0.5, + hovertemplate=f"{cat}
{val:,} tweet ({pct:.1f}%)", + )) + fig.update_layout( + height=290, margin=dict(l=0, r=0, t=30, b=0), + paper_bgcolor="rgba(0,0,0,0)", plot_bgcolor="rgba(0,0,0,0)", + showlegend=False, + xaxis=dict(showgrid=False, tickfont=dict(size=11, color="#64748b")), + yaxis=dict(showgrid=True, gridcolor="rgba(226,232,240,0.8)", griddash="dot", + tickfont=dict(size=10, color="#94a3b8")), + ) + st.plotly_chart(fig, width="stretch", config={"displayModeBar": False}) + + +# ───────────────────────────────────────────────────────────── +# Trend chart +# ───────────────────────────────────────────────────────────── + +def _render_trend_chart(fdf, filter_label, dt_start, dt_end): + _section_header( + "📅 Tren Sentimen dari Hari ke Hari", + f"Periode {filter_label} · Diupdate: {format_now()}" + ) + + df_tl = fdf.copy() + df_tl["date"] = pd.to_datetime(df_tl["created_at"], errors="coerce").dt.date + actual_tl = df_tl.groupby(["date", "sentiment"]).size().reset_index(name="count") + + date_range = pd.date_range(pd.Timestamp(dt_start).date(), pd.Timestamp(dt_end).date(), freq="D") + base_dates = pd.DataFrame({"date": date_range.date}) + + fig = go.Figure() + config_lines = [ + ("Positif", "#16a34a", "rgba(22,163,74,0.08)"), + ("Netral", "#94a3b8", "rgba(148,163,184,0.06)"), + ("Negatif", "#ef4444", "rgba(239,68,68,0.08)"), + ] + for sent, color, fill in config_lines: + data = base_dates.merge( + actual_tl[actual_tl["sentiment"] == sent][["date", "count"]], + on="date", how="left", + ) + data["count"] = data["count"].fillna(0).astype(int) + fig.add_trace(go.Scatter( + x=data["date"], y=data["count"], name=sent, + mode="lines+markers", + line=dict(color=color, width=2.5, shape="spline", smoothing=0.8), + marker=dict(size=6, color="white", line=dict(color=color, width=2.5)), + fill="tozeroy", fillcolor=fill, + hovertemplate=f"{sent}
%{{x}}: %{{y}} tweet", + )) + fig.update_layout( + height=290, margin=dict(l=0, r=0, t=10, b=0), + paper_bgcolor="rgba(0,0,0,0)", plot_bgcolor="rgba(0,0,0,0)", + legend=dict(orientation="h", y=1.1, x=0, font=dict(size=11)), + xaxis=dict(showgrid=True, gridcolor="rgba(226,232,240,0.6)", + tickfont=dict(size=10, color="#94a3b8")), + yaxis=dict(showgrid=True, gridcolor="rgba(226,232,240,0.6)", griddash="dot", + tickfont=dict(size=10, color="#94a3b8")), + hovermode="x unified", + ) + with st.container(border=True, key="sentiment_trend_panel"): + st.plotly_chart(fig, width="stretch", config={"displayModeBar": False}) + + +# ───────────────────────────────────────────────────────────── +# Confidence histogram +# ───────────────────────────────────────────────────────────── + +# def _render_confidence_chart(fdf): +# _section_header( +# "🎯 Distribusi Keyakinan Model", +# "Seberapa yakin classifier dalam mengklasifikasikan setiap tweet" +# ) + +# fig = go.Figure() +# sent_cfg = [ +# ("Positif", "#16a34a", "rgba(22,163,74,0.7)"), +# ("Netral", "#94a3b8", "rgba(148,163,184,0.7)"), +# ("Negatif", "#ef4444", "rgba(239,68,68,0.7)"), +# ] +# for sent, color, fill_color in sent_cfg: +# sub = fdf[fdf["sentiment"] == sent]["confidence"] +# if sub.empty: +# continue +# fig.add_trace(go.Histogram( +# x=sub, name=sent, nbinsx=20, +# marker=dict(color=fill_color, line=dict(color=color, width=1)), +# opacity=0.85, +# hovertemplate=f"{sent}
Keyakinan: %{{x:.0%}}
Jumlah: %{{y}}", +# )) +# fig.update_layout( +# height=260, margin=dict(l=0, r=0, t=10, b=0), barmode="overlay", +# paper_bgcolor="rgba(0,0,0,0)", plot_bgcolor="rgba(0,0,0,0)", +# legend=dict(orientation="h", y=1.1, x=0, font=dict(size=11)), +# xaxis=dict(tickformat=".0%", title="Tingkat Keyakinan", +# tickfont=dict(size=10, color="#94a3b8"), +# showgrid=True, gridcolor="rgba(226,232,240,0.6)"), +# yaxis=dict(title="Jumlah Tweet", +# tickfont=dict(size=10, color="#94a3b8"), +# showgrid=True, gridcolor="rgba(226,232,240,0.6)", griddash="dot"), +# ) +# with st.container(border=True, key="sentiment_conf_panel"): +# st.plotly_chart(fig, width="stretch", config={"displayModeBar": False}) + + +# ───────────────────────────────────────────────────────────── +# Word frequency per sentimen +# ───────────────────────────────────────────────────────────── + +def _render_word_freq_per_sentiment(fdf): + _section_header( + "📊 Kata Dominan per Sentimen", + "Top 15 kata paling sering muncul di masing-masing kelompok sentimen" + ) + + stop_extra = {"ongkos", "kirim", "gratis", "komdigi", "ongkir"} + sent_cfg = [ + ("Positif", "#16a34a", "#1e3a8a"), + ("Netral", "#64748b", "#334155"), + ("Negatif", "#ef4444", "#7f1d1d"), + ] + + cols = st.columns(3, gap="medium") + for col, (sent, color, dark) in zip(cols, sent_cfg): + with col: + sub = fdf[fdf["sentiment"] == sent] + all_words = " ".join(sub["clean_text"].fillna("")).split() + filtered = [w for w in all_words if len(w) > 2 and w not in stop_extra] + wf = Counter(filtered).most_common(15) + + st.markdown( + f'
' + f'
' + f'
' + f'{"😊" if sent=="Positif" else "😐" if sent=="Netral" else "😞"}
' + f'
{sent}
' + f'
' + f'{len(sub):,} tweet
' + f'
', + unsafe_allow_html=True, + ) + + if not wf: + st.info("Belum ada data") + st.markdown('
', unsafe_allow_html=True) + continue + + words_list = [w[0] for w in wf] + counts_list = [w[1] for w in wf] + max_c = max(counts_list) if counts_list else 1 + + fig = go.Figure(data=[go.Bar( + y=words_list[::-1], x=counts_list[::-1], orientation="h", + marker=dict( + color=[f"rgba({int(color[1:3],16)},{int(color[3:5],16)},{int(color[5:7],16)},{0.35+0.65*(c/max_c):.2f})" + for c in counts_list[::-1]], + line=dict(width=0), cornerradius=4, + ), + text=[str(c) for c in counts_list[::-1]], + textposition="outside", + textfont=dict(size=9, color="#475569"), + hovertemplate="%{y}
%{x} kali", + )]) + fig.update_layout( + height=360, margin=dict(l=0, r=40, t=4, b=4), + paper_bgcolor="rgba(0,0,0,0)", plot_bgcolor="rgba(0,0,0,0)", + showlegend=False, + xaxis=dict(showgrid=True, gridcolor="rgba(203,213,225,0.6)", + tickfont=dict(size=8, color="#94a3b8"), fixedrange=True), + yaxis=dict(showgrid=False, tickfont=dict(size=9, color="#334155"), fixedrange=True), + ) + st.plotly_chart(fig, width="stretch", config={"displayModeBar": False}) + st.markdown('
', unsafe_allow_html=True) + + +# ───────────────────────────────────────────────────────────── +# Word cloud +# ───────────────────────────────────────────────────────────── + +def _render_wordcloud(fdf): + _section_header( + "☁️ Word Cloud per Sentimen", + "Kata-kata populer dari tweet yang sudah melalui preprocessing" + ) + try: + from wordcloud import WordCloud + import matplotlib.pyplot as plt + + stop_wc = {"ongkos", "kirim", "gratis", "komdigi", "ongkir"} + wc1, wc2, wc3 = st.columns(3, gap="medium") + cfg = [ + (wc1, "Positif", "Greens", "😊 Positif", "#f0fdf4", "#14532d"), + (wc2, "Netral", "Blues", "😐 Netral", "#f8fafc", "#334155"), + (wc3, "Negatif", "Reds", "😞 Negatif", "#fef2f2", "#7f1d1d"), + ] + for col, sent, cmap, title, bg, tc in cfg: + with col: + st.markdown( + f'
' + f'{title}
', + unsafe_allow_html=True, + ) + sub = fdf[fdf["sentiment"] == sent] + words = [w for w in " ".join(sub["clean_text"].fillna("")).split() + if len(w) > 2 and w not in stop_wc] + if words: + wc = WordCloud( + width=420, height=260, background_color="white", + colormap=cmap, max_words=60, + relative_scaling=0.5, collocations=False, + ).generate(" ".join(words)) + fig, ax = plt.subplots(figsize=(5, 3.1)) + ax.imshow(wc, interpolation="bilinear") + ax.axis("off") + plt.tight_layout(pad=0) + st.pyplot(fig, clear_figure=True) + else: + st.info("Data kata tidak cukup") + except ImportError: + st.warning("Install `wordcloud` dan `matplotlib` terlebih dahulu.") + + +# ───────────────────────────────────────────────────────────── +# Tweet table +# ───────────────────────────────────────────────────────────── + +def _render_tweet_table(fdf, filter_label): + _section_header( + "📋 Tabel Analisis Sentimen", + f"Total {len(fdf):,} tweet · {filter_label}" + ) + + with st.container(border=True, key="sentiment_table_controls"): + tf1, tf2, tf3 = st.columns([3, 1.6, 1.7], gap="medium", vertical_alignment="bottom") + with tf1: + search = st.text_input( + "Search tweet", placeholder="Ketik kata kunci di isi tweet...", + key="sentiment_table_search", + ) + with tf2: + sf = st.selectbox( + "Filter sentimen", + ["Semua", "Positif 😊", "Netral 😐", "Negatif 😞"], + key="sentiment_table_filter", + ) + with tf3: + sort_by = st.selectbox( + "Urutkan", + ["Terbaru dulu", "Terlama dulu", "Keyakinan tertinggi"], + key="sentiment_table_sort", + ) + + _section_gap("sm") + + tdf = fdf.copy() + if "crawled_at" not in tdf.columns: + tdf["crawled_at"] = pd.NaT + + if search: + tdf = tdf[tdf["text"].str.contains(search, case=False, na=False)] + + sf_map = {"Positif 😊": "Positif", "Netral 😐": "Netral", "Negatif 😞": "Negatif"} + if sf != "Semua": + tdf = tdf[tdf["sentiment"] == sf_map.get(sf, sf)] + + if sort_by == "Terbaru dulu": + tdf = tdf.sort_values("created_at", ascending=False) + elif sort_by == "Terlama dulu": + tdf = tdf.sort_values("created_at", ascending=True) + else: + tdf = tdf.sort_values("confidence", ascending=False) + + # out = tdf[["created_at", "crawled_at", "text", "clean_text", "sentiment", "confidence"]].copy() + out = tdf[["created_at", "crawled_at", "text", "clean_text", "sentiment"]].copy() + out["created_at"] = out["created_at"].apply(format_dt) + out["crawled_at"] = out["crawled_at"].apply(format_dt) + # out["confidence"] = out["confidence"].apply(lambda x: f"{x:.0%}") + out["sentiment"] = out["sentiment"].map({ + "Positif": "😊 Positif", "Netral": "😐 Netral", "Negatif": "😞 Negatif", + }).fillna(out["sentiment"]) + # out.columns = ["Tanggal Tweet", "Masuk Database", "Tweet Asli", "Tweet Bersih", "Sentimen", "Keyakinan"] + out.columns = ["Tanggal Tweet", "Masuk Database", "Tweet Asli", "Tweet Bersih", "Sentimen"] + + render_standard_table( + out, height=400, min_width=1220, + badge_columns=["Sentimen"], + nowrap=["Tanggal Tweet", "Masuk Database", "Sentimen", "Keyakinan"], + wide_columns=["Tweet Asli", "Tweet Bersih"], + column_widths={ + "Tanggal Tweet": "170px", "Masuk Database": "170px", + "Tweet Asli": "360px", "Tweet Bersih": "360px", + "Sentimen": "130px", "Keyakinan": "110px", + }, + ) + st.caption(f"Menampilkan {len(tdf):,} tweet") + return tdf + + +# ───────────────────────────────────────────────────────────── +# Insight & Recommendation +# ───────────────────────────────────────────────────────────── + +def _render_insight_panel(dominant, pos_n, neu_n, neg_n, total, filter_label): + dom_name = dominant[0] + dom_pct = dominant[1] / total * 100 if total else 0 + neg_pct = neg_n / total * 100 if total else 0 + pos_pct = pos_n / total * 100 if total else 0 + neu_pct = neu_n / total * 100 if total else 0 + + _section_header( + "💡 Insight & Rekomendasi Tindakan", + f"Berdasarkan analisis {total:,} tweet · {filter_label}" + ) + + dom_color = {"Positif": "#16a34a", "Netral": "#64748b", "Negatif": "#ef4444"}.get(dom_name, "#3b6cf7") + dom_bg = {"Positif": "#f0fdf4", "Netral": "#f8fafc", "Negatif": "#fef2f2"}.get(dom_name, "#eef2ff") + dom_emoji = {"Positif": "😊", "Netral": "😐", "Negatif": "😞"}.get(dom_name, "📊") + + st.markdown( + f'
' + f'
' + f'
{dom_emoji}
' + f'
' + f'
Sentimen Dominan
' + f'
' + f'{dom_name} · {dom_pct:.1f}%
' + f'
' + f'
' + f'Positif: {pos_pct:.1f}% · ' + f'Netral: {neu_pct:.1f}% · ' + f'Negatif: {neg_pct:.1f}%' + f'
', + unsafe_allow_html=True, + ) + + rows = [] + if neg_pct >= 40: + rows.append(("🔴 URGENT", "#fef2f2", "#7f1d1d", "#ef4444", + "Tanggapi Keluhan Publik", + "Sentimen negatif tinggi — ketidakpuasan signifikan", + "Buat klarifikasi resmi dan buka ruang dialog publik", + "Humas / Tim Kebijakan")) + if neg_pct >= 20: + rows.append(("🟠 TINGGI", "#fff7ed", "#7c2d12", "#ea580c", + "Tinjau Ulang Kebijakan", + f"Sentimen negatif mencapai {neg_pct:.1f}%", + "Evaluasi poin kebijakan yang paling banyak dikeluhkan", + "Tim Kebijakan")) + if neu_pct >= 30: + rows.append(("🟡 SEDANG", "#fefce8", "#713f12", "#ca8a04", + "Tingkatkan Sosialisasi", + f"Sentimen netral {neu_pct:.1f}% — publik belum berpihak", + "Perbanyak konten edukatif dan FAQ resmi", + "Tim Komunikasi")) + if pos_pct >= 20: + rows.append(("🟢 INFO", "#f0fdf4", "#14532d", "#16a34a", + "Pertahankan Momentum Positif", + f"Sentimen positif {pos_pct:.1f}%", + "Perkuat narasi positif via kanal resmi secara konsisten", + "Tim Media Sosial")) + rows.append(("🔵 RUTIN", "#eff6ff", "#1e3a8a", "#3b6cf7", + "Pemantauan Berkelanjutan", + "Opini publik dapat berubah sewaktu-waktu", + "Pantau sentimen harian dan buat laporan berkala", + "Tim Analis Data")) + + df_rek = pd.DataFrame(rows, columns=["Prioritas", "bg", "tc", "bc", + "Tindakan", "Dasar Analisis", + "Rekomendasi", "Penanggung Jawab"]) + + for _, row in df_rek.iterrows(): + st.markdown( + f'
' + f'
' + f'
' + f'{row.Prioritas}
' + f'
' + f'
' + f'{row.Tindakan}
' + f'
' + f'📌 {row["Dasar Analisis"]}
' + f'
' + f'✅ {row.Rekomendasi}
' + f'
' + f'
' + f'' + f'👤 {row["Penanggung Jawab"]}
' + f'
', + unsafe_allow_html=True, + ) + + export_df = df_rek[["Prioritas", "Tindakan", "Dasar Analisis", "Rekomendasi", "Penanggung Jawab"]] + return export_df + + +# ───────────────────────────────────────────────────────────── +# Main show() +# ───────────────────────────────────────────────────────────── + +def show(): + _render_sentiment_styles() + _render_page_header() + + if "analysis_mode" not in st.session_state: + st.warning("⚠️ Silakan pilih mode tampilan di halaman Ambil Data Twitter terlebih dahulu.") + return + + _sync_dynamic_period() + + start_date = st.session_state.get("filter_start_date") + end_date = st.session_state.get("filter_end_date") + filter_label = st.session_state.get("filter_label", "-") + mode_display = st.session_state.get("mode_display", "-") + + if start_date is None or end_date is None: + st.warning("⚠️ Silakan buka halaman Ambil Data Twitter terlebih dahulu.") + return + + mode_meta = { + "realtime": ("#16a34a", "📡"), + "30days": ("#3b6cf7", "📅"), + "captured": ("#0284c7", "📆"), + "custom": ("#d97706", "🔍"), + } + mode_color, mode_icon = mode_meta.get(st.session_state.analysis_mode, ("#3b6cf7", "📊")) + + st.markdown( + f'
' + f'{mode_icon}' + f'
' + f'
{mode_display}
' + f'
' + f'Periode: {filter_label}
' + f'
', + unsafe_allow_html=True, + ) + + # ── Load data ───────────────────────────────────────────── + try: + df_all = pd.read_sql("SELECT * FROM tweets ORDER BY created_at DESC", engine) + if df_all.empty: + st.warning("⚠️ Belum ada data tweet.") + return + df_all["created_at"] = parse_dt(df_all["created_at"]) + if "crawled_at" in df_all.columns: + df_all["crawled_at"] = parse_crawled_dt(df_all["crawled_at"]) + except Exception as e: + st.error(f"❌ Gagal membaca database: {e}") + return + + s_dt = pd.Timestamp(start_date) + e_dt = pd.Timestamp(end_date) + df = df_all[(df_all["created_at"] >= s_dt) & (df_all["created_at"] < e_dt)].copy() + + if df.empty: + st.warning(f"⚠️ Tidak ada tweet dalam periode {filter_label}.") + return + + # ── Caching ─────────────────────────────────────────────── + total_tweets_in_db = get_tweet_count() + latest_crawl_marker = get_latest_crawl_time() or "no-crawl" + data_marker = (total_tweets_in_db, latest_crawl_marker) + cache_key = ( + f"sent_{st.session_state.analysis_mode}_" + f"{start_date}_{end_date}_{total_tweets_in_db}_{latest_crawl_marker}" + ) + + for old_key in list(st.session_state.keys()): + if old_key.startswith("sent_") and old_key != cache_key: + del st.session_state[old_key] + + force_refresh = data_marker != st.session_state.get("_last_sentiment_data_marker") + + if cache_key not in st.session_state or force_refresh: + if force_refresh: + st.info("🔄 Menyegarkan prediksi sentimen dengan data terbaru...") + with st.spinner("🔍 Preprocessing & prediksi sentimen (Hybrid Classifier)..."): + # Preprocess semua tweet + df["clean_text"] = df["text"].apply(preprocess_single) + dfc = df[df["clean_text"].str.strip().str.len() > 0].copy() + + # Prediksi menggunakan hybrid classifier + results = predict_batch_hybrid(dfc["text"].tolist()) + if results: + sentiments, confidences = zip(*results) + else: + sentiments, confidences = [], [] + + dfc["sentiment"] = list(sentiments) + dfc["confidence"] = list(confidences) + + st.session_state[cache_key] = dfc + st.session_state["_last_sentiment_data_marker"] = data_marker + + df_s = st.session_state[cache_key] + + # ── Keyword filter ──────────────────────────────────────── + _section_header("🎯 Filter Kata Kunci", "Kosongkan untuk melihat semua tweet") + + with st.container(border=True, key="sentiment_keyword_panel"): + kw = st.text_input( + "Kata kunci", placeholder="Contoh: ongkir, kurir, komdigi...", + key="sentiment_keyword_search", + ) + st.markdown( + f'
' + f'' + f'Mode {escape(str(mode_display))}' + f'' + f'Periode {escape(str(filter_label))}' + f'
', + unsafe_allow_html=True, + ) + + _section_gap("sm") + + fdf = ( + df_s[df_s["text"].str.contains(kw, case=False, na=False)].copy() + if kw else df_s.copy() + ) + + if fdf.empty: + st.warning("⚠️ Tidak ada tweet yang cocok dengan kata kunci tersebut.") + return + + sc = fdf["sentiment"].value_counts() + total = len(fdf) + pos_n = int(sc.get("Positif", 0)) + neu_n = int(sc.get("Netral", 0)) + neg_n = int(sc.get("Negatif", 0)) + pos_p = pos_n / total * 100 + neu_p = neu_n / total * 100 + neg_p = neg_n / total * 100 + + # ── Ringkasan ───────────────────────────────────────────── + _section_header( + "📌 Ringkasan Sentimen", + f"Berdasarkan tanggal asli tweet · {filter_label}" + ) + _section_gap("sm") + _render_summary_pills(total, pos_n, neu_n, neg_n, pos_p, neu_p, neg_p, filter_label) + _section_gap("sm") + _render_proportion_bar(pos_p, neu_p, neg_p) + _section_gap("lg") + + # ── Donut + Bar ─────────────────────────────────────────── + col_left, col_right = st.columns(2, gap="medium") + with col_left: + _section_header("🔵 Sebaran Sentimen", f"Periode {filter_label}") + with st.container(border=True, key="sentiment_chart_panel"): + dominant = _render_donut_chart(pos_n, neu_n, neg_n, total, filter_label) + with col_right: + _section_header("📊 Perbandingan Jumlah per Sentimen", f"Periode {filter_label}") + with st.container(border=True, key="sentiment_bar_panel"): + _render_bar_chart(pos_n, neu_n, neg_n, total) + _section_gap("lg") + + # ── Trend ───────────────────────────────────────────────── + _render_trend_chart(fdf, filter_label, start_date, end_date) + _section_gap("lg") + + # ── Confidence ──────────────────────────────────────────── + # _render_confidence_chart(fdf) + # _section_gap("lg") + + # ── Word freq ───────────────────────────────────────────── + _render_word_freq_per_sentiment(fdf) + _section_gap("lg") + + # ── Word cloud ──────────────────────────────────────────── + _render_wordcloud(fdf) + _section_gap("lg") + + # ── Tweet table ─────────────────────────────────────────── + tdf = _render_tweet_table(fdf, filter_label) + _section_gap("lg") + + # ── Insight ─────────────────────────────────────────────── + df_rek = _render_insight_panel(dominant, pos_n, neu_n, neg_n, total, filter_label) + _section_gap("lg") + + # ── Download ────────────────────────────────────────────── + _section_header("📥 Unduh Hasil Analisis") + + with st.container(border=True, key="sentiment_download_panel"): + d1, d2, d3 = st.columns(3, gap="medium", vertical_alignment="bottom") + + with d1: + st.download_button( + "📥 Semua Hasil Prediksi", + fdf.to_csv(index=False).encode("utf-8"), + f"hasil_prediksi_{datetime.now().strftime('%Y%m%d_%H%M')}.csv", + "text/csv", width="stretch", + ) + with d2: + summary = pd.DataFrame({ + "Sentimen": ["Positif", "Netral", "Negatif"], + "Jumlah": [pos_n, neu_n, neg_n], + "Persen": [f"{pos_p:.2f}%", f"{neu_p:.2f}%", f"{neg_p:.2f}%"], + }) + st.download_button( + "📈 Ringkasan Sentimen", + summary.to_csv(index=False).encode("utf-8"), + f"ringkasan_{datetime.now().strftime('%Y%m%d_%H%M')}.csv", + "text/csv", width="stretch", + ) + with d3: + st.download_button( + "🎯 Rekomendasi Tindakan", + df_rek.to_csv(index=False).encode("utf-8"), + f"rekomendasi_{datetime.now().strftime('%Y%m%d_%H%M')}.csv", + "text/csv", width="stretch", + ) \ No newline at end of file diff --git a/sentiment_service.py b/sentiment_service.py index 3e776af..889422b 100644 --- a/sentiment_service.py +++ b/sentiment_service.py @@ -1,39 +1,509 @@ +""" +sentiment_service.py +==================== +Hybrid Classifier untuk analisis sentimen tweet Bahasa Indonesia. + +PIPELINE PREPROCESSING — 5 TAHAP (selaras dengan preprocessing_page.py): + 1. Case Folding → lowercase dulu sebelum cleaning + 2. Cleaning → hapus URL, mention, hashtag, angka, emoji, tanda baca + 3. Normalisasi → singkatan/slang → kata baku + 4. Stopword Removal → buang kata umum, jaga kata sentimen penting + 5. Stemming → bentuk dasar via Sastrawi ECS + +Catatan: Tokenizing tidak menjadi tahap tersendiri karena: + • Stopword removal & stemming sudah melakukan split() secara internal + • TF-IDF melakukan tokenisasi sendiri saat inferensi + +ARSITEKTUR HYBRID: + Teks Asli + ├── preprocess_untuk_lexicon() ──→ _hitung_skor_lexicon() ──┐ + └── preprocess_for_model() ──→ TF-IDF → NB Model ───────┘ + ↓ + _klasifikasi_hybrid() → Label + Confidence +""" + import re import string import joblib -model = joblib.load("model_naive_bayes.pkl") -tfidf = joblib.load("tfidf_vectorizer.pkl") + +# ═══════════════════════════════════════════════════════════ +# KATA SENTIMEN PENTING +# Tidak boleh dihapus di tahap stopword removal +# ═══════════════════════════════════════════════════════════ + +KATA_SENTIMEN_PENTING = { + # ── Negasi ────────────────────────────────────────────── + "tidak", "bukan", "jangan", "kurang", "belum", "tanpa", + + # ── Sentimen POSITIF ──────────────────────────────────── + "keren", "bagus", "mantap", "setuju", "dukung", "mendukung", + "andal", "handal", "gercep", "bangga", "senang", "suka", + "baik", "benar", "tepat", "oke", + "sejahtera", "berkembang", "maju", "inovatif", + "tegas", "sigap", "tanggap", "adil", "bijak", "bermanfaat", + "untung", "berhasil", "sukses", "solusi", "manfaat", + "berguna", "membantu", "bantu", "pro", "lanjut", + "sangat", "banget", "sekali", "paling", "amat", "luar", "biasa", + + # ── Sentimen NEGATIF evaluatif ────────────────────────── + "kecewa", "buruk", "jelek", "parah", "gagal", "hancur", + "rusak", "bohong", "tipu", "korupsi", + + # ── Emosi ─────────────────────────────────────────────── + "marah", "sedih", "khawatir", +} -def bersihkan_teks(text): +# ═══════════════════════════════════════════════════════════ +# NORMALISASI (selaras dengan preprocessing_page.py) +# ═══════════════════════════════════════════════════════════ - text = str(text).lower() +NORMALISASI = { + # Negasi + "gk": "tidak", "ga": "tidak", "gak": "tidak", + "nggak": "tidak", "ngga": "tidak", "tdk": "tidak", + "tak": "tidak", "enggak": "tidak", "engga": "tidak", + "kagak": "tidak", "kaga": "tidak", "ndak": "tidak", + "gkk": "tidak", "ngak": "tidak", + # Kata ganti + "yg": "yang", "dgn": "dengan", "utk": "untuk", + "org": "orang", "krn": "karena", "dr": "dari", + "sm": "sama", "pd": "pada", "dlm": "dalam", + "bwt": "buat", "trm": "terima", + # Verba + "tp": "tapi", "tpi": "tapi", "jd": "jadi", + "sdh": "sudah", "blm": "belum", "emg": "memang", + "emang": "memang", "gimana": "bagaimana", + "gitu": "begitu", "gini": "begini", + "udah": "sudah", "udh": "sudah", + # Intensitas + "bgt": "banget", "bngt": "banget", "bget": "banget", "bgtt": "banget", + # Positif informal + "bener": "benar", "beneran": "benar", + "mantep": "mantap", "mntap": "mantap", + "kece": "keren", + "cucok": "cocok", "cucuk": "cocok", + "cakep": "bagus", + "sip": "baik", "siipp": "baik", + "top": "terbaik", + "jos": "bagus", "josss": "bagus", + "goks": "luar biasa", + "setujuu": "setuju", "stuju": "setuju", + "proud": "bangga", + "mantul": "mantap betul", + # Negatif informal + "ancur": "hancur", "ancrr": "hancur", + "parahh": "parah", "parahhh": "parah", + "gagall": "gagal", + "ngaco": "tidak benar", + "ngasal": "tidak benar", + "gaje": "tidak jelas", + # Domain + "ongkir": "ongkos kirim", + "freeongkir": "gratis ongkos kirim", + "gratisongkir": "gratis ongkos kirim", + "free": "gratis", + "ecommerce": "e commerce", + "seller": "penjual", + "buyer": "pembeli", +} + +# ═══════════════════════════════════════════════════════════ +# LEXICON SENTIMEN +# ═══════════════════════════════════════════════════════════ + +LEXICON_POSITIF = { + "bagus", "baik", "keren", "mantap", "mantep", "hebat", + "oke", "sip", "top", "jos", "goks", "kece", "mantul", + "setuju", "dukung", "mendukung", "pro", "lanjut", "sepakat", + "bangga", "senang", "suka", "puas", "gembira", "bahagia", + "berhasil", "sukses", "berjaya", "prestasi", "pencapaian", + "andal", "handal", "gercep", "sigap", "tanggap", "tegas", + "adil", "bijak", "bermanfaat", "berguna", "membantu", + "inovatif", "maju", "berkembang", "sejahtera", + "benar", "tepat", "jelas", "transparan", "amanah", "terpercaya", + "untung", "gratis", "murah", "hemat", "terjangkau", + "terbaik", "luar biasa", + "cakep", "cucok", "gaskeun", "kuy", + "dukung", "bantu", "solusi", "manfaat", + "memuaskan", "membanggakan", "mengagumkan", +} + +LEXICON_NEGATIF = { + "buruk", "jelek", "parah", "rusak", "hancur", "ancur", + "gagal", "gagall", "ambruk", "terpuruk", "bangkrut", + "bohong", "tipu", "curang", "manipulasi", "korupsi", "penipuan", + "kebohongan", + "kecewa", "marah", "sedih", "khawatir", "takut", "benci", + "jijik", "muak", "kesal", "frustrasi", + "mahal", "rugi", "merugikan", + "lambat", "lemot", "ribet", "susah", "sulit", "bermasalah", + "ngaco", "ngasal", "gaje", "receh", + "tidak benar", "tidak jelas", "tidak adil", "tidak berguna", + "mengecewakan", "menyebalkan", "menyusahkan", +} + +KATA_NEGASI = { + "tidak", "bukan", "jangan", "belum", "tanpa", "kurang", + "anti", "non", +} + + +# ═══════════════════════════════════════════════════════════ +# STOPWORDS +# ═══════════════════════════════════════════════════════════ + +def _load_stopwords() -> set: + """Load stopword dengan penjagaan kata sentimen penting.""" + stopword_file = "indonesian-stopwords-complete.txt" + base = set() + try: + with open(stopword_file, "r", encoding="utf-8") as f: + base = set(f.read().splitlines()) + except FileNotFoundError: + base = { + "yang", "dan", "di", "ke", "dari", "ini", "itu", + "dengan", "untuk", "pada", "adalah", "oleh", "ada", + "ya", "akan", "atau", "juga", "sama", "karena", + "jika", "sudah", "telah", "saat", "agar", "maka", + "lagi", "bila", "bisa", "pun", "nya", + } + + # Jangan hapus kata sentimen penting + for kata in KATA_SENTIMEN_PENTING: + base.discard(kata) + + # Tambahan stopword domain-spesifik + base.update({ + "rt", "amp", "https", "http", "co", "t", + "wkwk", "wkwkwk", "haha", "hehe", "xixi", "hahaha", + "yg", "dgn", "utk", "dr", "krn", "tp", "jd", "sdh", + "aja", "doang", "nih", "sih", "dong", "deh", + "loh", "lah", "tuh", "kak", "gan", "bro", "sis", + }) + return base + + +# ═══════════════════════════════════════════════════════════ +# STEMMER +# ═══════════════════════════════════════════════════════════ + +def _load_stemmer(): + """Load Sastrawi stemmer. Return None jika tidak terinstall.""" + try: + from Sastrawi.Stemmer.StemmerFactory import StemmerFactory + return StemmerFactory().create_stemmer() + except Exception: + return None + + +# ── Inisialisasi global ───────────────────────────────────────────────────── +_STOPWORDS = _load_stopwords() +_STEMMER = _load_stemmer() + + +# ═══════════════════════════════════════════════════════════ +# MODEL LOADING (lazy) +# ═══════════════════════════════════════════════════════════ + +_model = None +_tfidf = None + + +def _load_model(): + """ + Lazy load NB model + TF-IDF vectorizer. + Return: (model, tfidf) — keduanya bisa None. + """ + global _model, _tfidf + if _model is None: + try: + _model = joblib.load("model_naive_bayes.pkl") + except Exception as e: + print(f"[WARNING] Gagal load model_naive_bayes.pkl: {e}") + if _tfidf is None: + try: + _tfidf = joblib.load("tfidf_vectorizer.pkl") + except Exception as e: + print(f"[WARNING] Gagal load tfidf_vectorizer.pkl: {e}") + return _model, _tfidf + + +# ═══════════════════════════════════════════════════════════ +# PREPROCESSING PIPELINE — 5 TAHAP +# +# URUTAN (selaras dengan preprocessing_page.py): +# 1. Case Folding → lowercase +# 2. Cleaning → hapus noise +# 3. Normalisasi → normalisasi kata +# 4. Stopword Removal → buang stopword (split() dilakukan internal) +# 5. Stemming → bentuk dasar +# ═══════════════════════════════════════════════════════════ + +def _case_folding(text: str) -> str: + """Tahap 1: Lowercase seluruh teks.""" + return str(text).lower() + + +def _cleaning(text: str) -> str: + """Tahap 2: Hapus noise (URL, mention, hashtag, angka, emoji, tanda baca).""" + text = re.sub(r"http\S+|www\S+|https\S+", "", text) + text = re.sub(r"@\w+", "", text) + text = re.sub(r"#\w+", "", text) + text = re.sub(r"\d+", "", text) text = re.sub( - r"http\S+|www\S+|@\w+|#|\d+", - "", - text + r"[\U00010000-\U0010ffff" + r"\U0001F600-\U0001F64F" + r"\U0001F300-\U0001F5FF" + r"\U0001F680-\U0001F6FF" + r"\U0001F1E0-\U0001F1FF" + r"\u2600-\u26FF\u2700-\u27BF" + r"]+", "", text, flags=re.UNICODE, ) - - text = text.translate( - str.maketrans("", "", string.punctuation) - ) - + text = text.translate(str.maketrans("", "", string.punctuation)) + text = re.sub(r"[^a-zA-Z\s]", "", text) text = re.sub(r"\s+", " ", text).strip() - return text -def prediksi_sentimen(list_text): +def _normalisasi(text: str) -> str: + """Tahap 3: Normalisasi singkatan dan kata tidak baku.""" + return " ".join(NORMALISASI.get(word, word) for word in text.split()) - clean_texts = [ - bersihkan_teks(text) - for text in list_text + +def _remove_stopwords(tokens: list) -> list: + """Tahap 4: Stopword removal dengan penjagaan kata sentimen.""" + return [ + w for w in tokens + if (w not in _STOPWORDS or w in KATA_SENTIMEN_PENTING) and len(w) > 2 ] - vectors = tfidf.transform(clean_texts) - predictions = model.predict(vectors) +def _stemming(tokens: list) -> list: + """Tahap 5: Stemming ke bentuk dasar via Sastrawi ECS.""" + if _STEMMER is None: + return tokens + return [_STEMMER.stem(w) for w in tokens] - return clean_texts, predictions \ No newline at end of file + +def preprocess_for_model(text: str) -> str: + """ + Full 5-tahap preprocessing → string teks bersih siap TF-IDF. + + URUTAN: Case Folding → Cleaning → Normalisasi + → Stopword Removal → Stemming + + Split kata dilakukan secara internal di tahap Stopword Removal. + Pipeline HARUS sama persis dengan yang dipakai saat training model. + """ + s1 = _case_folding(text) # Tahap 1 + s2 = _cleaning(s1) # Tahap 2 + s3 = _normalisasi(s2) # Tahap 3 + s4 = _remove_stopwords(s3.split()) # Tahap 4 (split inline) + s5 = _stemming(s4) # Tahap 5 + return " ".join(s5) + + +def preprocess_untuk_lexicon(text: str) -> str: + """ + Preprocessing RINGAN untuk lexicon matching. + Tidak di-stem → kata asli bisa dicocokkan dengan lexicon. + Pipeline: Case Folding → Cleaning → Normalisasi saja. + """ + s1 = _case_folding(text) + s2 = _cleaning(s1) + s3 = _normalisasi(s2) + return s3 + + +# ═══════════════════════════════════════════════════════════ +# LEXICON SCORER +# ═══════════════════════════════════════════════════════════ + +def _hitung_skor_lexicon(teks_lexicon: str) -> dict: + """ + Hitung skor positif dan negatif dari teks via lexicon. + + NEGATION HANDLING: + Kata negasi dalam window 2 kata sebelum kata sentimen → polaritas dibalik. + Contoh: "tidak bagus" → ada "tidak" sebelum "bagus" (POSITIF) + → skor_neg += 1 (bukan skor_pos) + + Return: {"positif": int, "negatif": int, "net": int} + """ + tokens = teks_lexicon.split() + skor_pos = 0 + skor_neg = 0 + + for i, token in enumerate(tokens): + ada_negasi = any( + tokens[i - j] in KATA_NEGASI + for j in range(1, 3) + if i - j >= 0 + ) + + if token in LEXICON_POSITIF: + if ada_negasi: + skor_neg += 1 + else: + skor_pos += 1 + elif token in LEXICON_NEGATIF: + if ada_negasi: + skor_pos += 1 + else: + skor_neg += 1 + + return { + "positif": skor_pos, + "negatif": skor_neg, + "net": skor_pos - skor_neg, + } + + +# ═══════════════════════════════════════════════════════════ +# HYBRID CLASSIFIER +# ═══════════════════════════════════════════════════════════ + +# Normalisasi label dari berbagai format yang mungkin dipakai model +_LABEL_MAP = { + "positif": "Positif", "Positif": "Positif", "positive": "Positif", "pos": "Positif", + "negatif": "Negatif", "Negatif": "Negatif", "negative": "Negatif", "neg": "Negatif", + "netral": "Netral", "Netral": "Netral", "neutral": "Netral", "net": "Netral", +} + + +def _prediksi_model(teks_model: str): + """ + Dapatkan prediksi dari NB model. + Return: (label_norm, confidence, proba_dict) atau None. + """ + model, tfidf = _load_model() + if model is None or tfidf is None or not teks_model.strip(): + return None + try: + vec = tfidf.transform([teks_model]) + pred_raw = model.predict(vec)[0] + proba = model.predict_proba(vec)[0] + classes = list(model.classes_) + + label_norm = _LABEL_MAP.get(str(pred_raw), "Netral") + pred_idx = classes.index(pred_raw) if pred_raw in classes else 0 + confidence = float(proba[pred_idx]) + + proba_dict = { + _LABEL_MAP.get(str(cls), str(cls)): float(p) + for cls, p in zip(classes, proba) + } + return label_norm, confidence, proba_dict + except Exception as e: + print(f"[WARNING] Prediksi model gagal: {e}") + return None + + +def _klasifikasi_hybrid( + teks_model: str, + skor: dict, + teks_lower: str, +) -> tuple: + """ + Klasifikasi hybrid: sinyal lexicon + model NB. + + LOGIKA KEPUTUSAN (berurutan): + + Layer 1 — Override Positif KUAT (net >= 2): + → Positif, confidence 60–92% + + Layer 2 — Override Positif LEMAH (net == 1): + → Cek model; jika model < 65% yakin Negatif → Positif + → Jika model sangat yakin Negatif → tetap Positif (confidence rendah) + + Layer 3 — Override Negatif KUAT (net <= -2): + → Negatif, confidence 60–90% + + Layer 4 — Fallback Model NB: + → Prediksi model dipakai, TAPI: + • Jika model = Negatif AND net >= 0 AND confidence < 75% + → downgrade ke Netral (koreksi bias model) + • Kasus lainnya → percaya model + + Layer 5 — Ultimate Fallback (model tidak tersedia): + → Gunakan skor lexicon saja + + Return: (label: str, confidence: float) + """ + net = skor["net"] + pos = skor["positif"] + + # ── Layer 1: Positif KUAT ───────────────────────────────────────────────── + if net >= 2: + conf = min(0.60 + (net * 0.07), 0.92) + return ("Positif", round(conf, 3)) + + # ── Layer 2: Positif LEMAH ──────────────────────────────────────────────── + if net == 1 and pos >= 1: + model_result = _prediksi_model(teks_model) + if model_result is not None: + _, _, proba_dict = model_result + neg_prob = proba_dict.get("Negatif", 0.0) + if neg_prob < 0.65: + conf = round(0.55 + (0.65 - neg_prob) * 0.3, 3) + return ("Positif", min(conf, 0.80)) + else: + return ("Positif", 0.55) + else: + return ("Positif", 0.58) + + # ── Layer 3: Negatif KUAT ───────────────────────────────────────────────── + if net <= -2: + conf = min(0.60 + (abs(net) * 0.06), 0.90) + return ("Negatif", round(conf, 3)) + + # ── Layer 4: Fallback Model NB ──────────────────────────────────────────── + model_result = _prediksi_model(teks_model) + if model_result is not None: + label_norm, confidence, proba_dict = model_result + + # Anti-bias correction: + # Model prediksi Negatif tapi tidak ada sinyal negatif dari lexicon + # dan confidence < 75% → kemungkinan bias → turunkan ke Netral + if label_norm == "Negatif" and net >= 0 and confidence < 0.75: + corrected_conf = round(0.50 + max(0, confidence - 0.50) * 0.2, 3) + return ("Netral", corrected_conf) + + return (label_norm, round(confidence, 3)) + + # ── Layer 5: Ultimate Fallback ──────────────────────────────────────────── + if net > 0: + return ("Positif", 0.55) + elif net < 0: + return ("Negatif", 0.55) + else: + return ("Netral", 0.50) + + +# ═══════════════════════════════════════════════════════════ +# BACKWARD COMPATIBILITY +# ═══════════════════════════════════════════════════════════ + +def bersihkan_teks(text: str) -> str: + """[LEGACY] Gunakan preprocess_for_model() untuk pipeline lengkap.""" + return preprocess_for_model(text) + + +def prediksi_sentimen(list_text: list): + """ + [LEGACY] Prediksi batch dengan hybrid classifier. + Return: (list clean_texts, list labels) + """ + clean_texts = [preprocess_for_model(t) for t in list_text] + labels = [] + for text in list_text: + teks_model = preprocess_for_model(text) + teks_lexicon = preprocess_untuk_lexicon(text) + teks_lower = str(text).lower() + skor = _hitung_skor_lexicon(teks_lexicon) + label, _ = _klasifikasi_hybrid(teks_model, skor, teks_lower) + labels.append(label) + return clean_texts, labels \ No newline at end of file diff --git a/tweets-data/hasil_latest.csv b/tweets-data/hasil_latest.csv index 8b13789..7cb42a9 100644 --- a/tweets-data/hasil_latest.csv +++ b/tweets-data/hasil_latest.csv @@ -1 +1,2 @@ - +"conversation_id_str","created_at","favorite_count","full_text","id_str","image_url","in_reply_to_screen_name","lang","location","quote_count","reply_count","retweet_count","tweet_url","user_id_str","username" +"2056538953340314095","Wed May 20 16:01:04 +0000 2026","5","@killingmaster27 @evelyndewi7 @katrinrienks @HumorJonTampan @wiyantokdr54 @EviDrajat @RickyKardjono @Micheladam69432 @ebonXboy @AntonRebor79105 @gangsterrandom @Pieter_Sun01 @Rahmah0845 @RSjahfidi @prabowo @SmileyNDeLight @Jediimar @Yens0906 @tokaidirecycle @PembuatSistem @bonapasogit24 @Jenniestale @Maskudin126003 @a8sar6alih @JowoNgalam @KING_RAJA_PWS_ @apaajadeh447870 @50ngoku @Submill @EmmaHusain0845 @BebySoSweet @Yulia_4ja @bungalotus25 @bahlillahadalia wahhh... jadi menteri komdigi dong menteri free ongkir https://t.co/1cfvoqIcNl","2057129489994510620","https://pbs.twimg.com/media/HIxkWwDaMAAXH9U.jpg","killingmaster27","in","","0","2","0","https://x.com/undefined/status/2057129489994510620","1410466956134608899", diff --git a/tweets-data/hasil_latest.old.csv b/tweets-data/hasil_latest.old.csv index d3f5a12..60f198f 100644 --- a/tweets-data/hasil_latest.old.csv +++ b/tweets-data/hasil_latest.old.csv @@ -1 +1,2 @@ - +"conversation_id_str","created_at","favorite_count","full_text","id_str","image_url","in_reply_to_screen_name","lang","location","quote_count","reply_count","retweet_count","tweet_url","user_id_str","username" +"2056538953340314095","Wed May 20 16:01:04 +0000 2026","5","@killingmaster27 @evelyndewi7 @katrinrienks @HumorJonTampan @wiyantokdr54 @EviDrajat @RickyKardjono @Micheladam69432 @ebonXboy @AntonRebor79105 @gangsterrandom @Pieter_Sun01 @Rahmah0845 @RSjahfidi @prabowo @SmileyNDeLight @Jediimar @Yens0906 @tokaidirecycle @PembuatSistem @bonapasogit24 @Jenniestale @Maskudin126003 @a8sar6alih @JowoNgalam @KING_RAJA_PWS_ @apaajadeh447870 @50ngoku @Submill @EmmaHusain0845 @BebySoSweet @Yulia_4ja @bungalotus25 @bahlillahadalia wahhh... jadi menteri komdigi dong menteri free ongkir https://t.co/1cfvoqIcNl","2057129489994510620","https://pbs.twimg.com/media/HIxkWwDaMAAXH9U.jpg","killingmaster27","in","","0","2","0","https://x.com/undefined/status/2057129489994510620","1410466956134608899", diff --git a/tweets-data/hasil_top.csv b/tweets-data/hasil_top.csv deleted file mode 100644 index d82e1ec..0000000 --- a/tweets-data/hasil_top.csv +++ /dev/null @@ -1,12 +0,0 @@ -"conversation_id_str","created_at","favorite_count","full_text","id_str","image_url","in_reply_to_screen_name","lang","location","quote_count","reply_count","retweet_count","tweet_url","user_id_str","username" -"2054502530504364380","Wed May 13 10:02:28 +0000 2026","0","GRATIS ONGKIR DIBATASI??? PARAH ","2054502530504364380","","","in","","0","0","0","https://x.com/undefined/status/2054502530504364380","1814143201718247424", -"2054816635131994490","Thu May 14 06:50:37 +0000 2026","0","keren sih ada kebijakan dari komdigi ongkir jadi berasa lebih merataa","2054816635131994490","","","in","","0","0","0","https://x.com/undefined/status/2054816635131994490","1814143201718247424", -"2054839912529350807","Thu May 14 08:23:06 +0000 2026","0","komdigi mending ngurusin lainnya deh dsripada ngurusin ongkir giniii","2054839912529350807","","","in","","0","0","0","https://x.com/undefined/status/2054839912529350807","1814143201718247424", -"2054502757869199625","Wed May 13 10:03:22 +0000 2026","0","Komdigi mengeluarkan kebijakan terkait gratis ongkir.","2054502757869199625","","","in","","0","0","0","https://x.com/undefined/status/2054502757869199625","1814143201718247424", -"2054996678391091685","Thu May 14 18:46:02 +0000 2026","0","komdigi ongkir mending sampingkan dulu","2054996678391091685","","","in","","0","0","0","https://x.com/undefined/status/2054996678391091685","1814143201718247424", -"2055002136027767200","Thu May 14 19:07:43 +0000 2026","0","komdigi ongkir apa gunanyaa??222","2055002136027767200","","","in","","0","0","0","https://x.com/undefined/status/2055002136027767200","2043979880086339584", -"2054998005359751518","Thu May 14 18:51:19 +0000 2026","0","komdigi ongkir apa gunanyaa??","2054998005359751518","","","in","","0","0","0","https://x.com/undefined/status/2054998005359751518","2043979880086339584", -"2055002941288001599","Thu May 14 19:10:55 +0000 2026","0","yakin komdigi bikin kebijakan pembatasan gratis ongkir ini bakal bisa ngimbangin pemerataan umkm??","2055002941288001599","","","in","","0","0","0","https://x.com/undefined/status/2055002941288001599","2043979880086339584", -"2055269612334555468","Fri May 15 12:50:35 +0000 2026","0","komdigi + ongkir = chaos wkwkwk","2055269612334555468","","","in","","0","0","0","https://x.com/undefined/status/2055269612334555468","2043979880086339584", -"2055001478004367487","Thu May 14 19:05:07 +0000 2026","0","maksud dari komdigi bikin kebijakan gratis ongkir ini apa yaaa?","2055001478004367487","","","in","","0","0","0","https://x.com/undefined/status/2055001478004367487","2043979880086339584", -"2055013912555475243","Thu May 14 19:54:31 +0000 2026","0","komdigi haloo?? masa iya ngurusin ongkir dulu dibanding permasalahan sebelah??","2055013912555475243","","","in","","0","0","0","https://x.com/undefined/status/2055013912555475243","2043979880086339584", diff --git a/tweets-data/hasil_top.old.csv b/tweets-data/hasil_top.old.csv index f8b749f..fea545b 100644 --- a/tweets-data/hasil_top.old.csv +++ b/tweets-data/hasil_top.old.csv @@ -1,21 +1,6 @@ "conversation_id_str","created_at","favorite_count","full_text","id_str","image_url","in_reply_to_screen_name","lang","location","quote_count","reply_count","retweet_count","tweet_url","user_id_str","username" -"2054502269576728775","Wed May 13 10:01:26 +0000 2026","0","Kebijakan komdigi soal ongkir sangat merugikan konsumen.","2054502269576728775","","","in","","0","0","0","https://x.com/undefined/status/2054502269576728775","1814143201718247424", -"2054823109291852155","Thu May 14 07:16:20 +0000 2026","0","komdigi bikin aturan ongkir baru = langkah yang cukup bijak","2054823109291852155","","","in","","0","0","0","https://x.com/undefined/status/2054823109291852155","1814143201718247424", -"2054502892468662645","Wed May 13 10:03:55 +0000 2026","0","Komdigi dan gratis ongkir sedang ramai dibahas.","2054502892468662645","","","in","","0","0","0","https://x.com/undefined/status/2054502892468662645","1814143201718247424", -"2054501918895202348","Wed May 13 10:00:02 +0000 2026","0","Mantap sih @komdigi akhirnya free ongkir diatur juga ","2054501918895202348","","","in","","0","0","0","https://x.com/undefined/status/2054501918895202348","1814143201718247424", -"2054861268750901484","Thu May 14 09:47:58 +0000 2026","0","komdigi ongkir?","2054861268750901484","","","in","","0","0","0","https://x.com/undefined/status/2054861268750901484","1814143201718247424", -"2054502131672236531","Wed May 13 10:00:53 +0000 2026","0","kebijakan komdigi tentang ongkir patut diapresiasi.","2054502131672236531","","","in","","0","0","0","https://x.com/undefined/status/2054502131672236531","1814143201718247424", -"2054478880933855563","Wed May 13 08:28:30 +0000 2026","0","KOMDIGI KNP NGURUSIN ONGKIR SIH??","2054478880933855563","","","in","","0","0","0","https://x.com/undefined/status/2054478880933855563","1814143201718247424", -"2054502618484134392","Wed May 13 10:02:49 +0000 2026","0","@komdigi kenapa sih harus ngurusin ongkir segala???","2054502618484134392","","","in","","0","0","0","https://x.com/undefined/status/2054502618484134392","1814143201718247424", -"2054816992662790146","Thu May 14 06:52:02 +0000 2026","0","komdigi??? ongkir??? marketplace???","2054816992662790146","","","in","","0","0","0","https://x.com/undefined/status/2054816992662790146","1814143201718247424", -"2054502958315020394","Wed May 13 10:04:10 +0000 2026","0","Komdigi disebut dalam diskusi mengenai free ongkir.","2054502958315020394","","","in","","0","0","0","https://x.com/undefined/status/2054502958315020394","1814143201718247424", -"2054501981369352445","Wed May 13 10:00:17 +0000 2026","0","Gratis ongkir dibatasi malah bikin marketplace lebih fair.","2054501981369352445","","","in","","0","0","0","https://x.com/undefined/status/2054501981369352445","1814143201718247424", -"2054502181446078852","Wed May 13 10:01:05 +0000 2026","0","Menurut saya pembatasan gratis ongkir adalah keputusan tepat","2054502181446078852","","","in","","0","0","0","https://x.com/undefined/status/2054502181446078852","1814143201718247424", -"2054816880318427497","Thu May 14 06:51:35 +0000 2026","0","Ada diskusi publik mengenai kebijakan ongkir dari Komdigi.","2054816880318427497","","","in","","0","0","0","https://x.com/undefined/status/2054816880318427497","1814143201718247424", -"2054816818741862819","Thu May 14 06:51:20 +0000 2026","0","Saya melihat berita tentang komdigi dan ongkir tadi pagi.","2054816818741862819","","","in","","0","0","0","https://x.com/undefined/status/2054816818741862819","1814143201718247424", -"2054501862066524547","Wed May 13 09:59:49 +0000 2026","0","Saya setuju kebijakan Komdigi soal gratis ongkir dibatasi biar persaingan lebih sehat.","2054501862066524547","","","in","","0","0","0","https://x.com/undefined/status/2054501862066524547","1814143201718247424", -"2054502430956769299","Wed May 13 10:02:05 +0000 2026","0","Pembatasan gratis ongkir sangat mengecewakan.","2054502430956769299","","","in","","0","0","0","https://x.com/undefined/status/2054502430956769299","1814143201718247424", -"2054502817910706448","Wed May 13 10:03:37 +0000 2026","0","Saya membaca berita tentang gratis ongkir dibatasi.","2054502817910706448","","","in","","0","0","0","https://x.com/undefined/status/2054502817910706448","1814143201718247424", -"2054791169775964388","Thu May 14 05:09:25 +0000 2026","0","kebijakan komdigi gratis ongkir kerenn","2054791169775964388","","","in","","0","0","0","https://x.com/undefined/status/2054791169775964388","1814143201718247424", -"2054502657201733861","Wed May 13 10:02:58 +0000 2026","0","Gratis ongkir dibatasi? keputusan paling aneh tahun ini.","2054502657201733861","","","in","","0","0","0","https://x.com/undefined/status/2054502657201733861","1814143201718247424", -"2054502713959014566","Wed May 13 10:03:12 +0000 2026","0","Gratis ongkir dibatasi = belanja online makin mahal ","2054502713959014566","","","in","","0","0","0","https://x.com/undefined/status/2054502713959014566","1814143201718247424", +"2056635322419777630","Tue May 19 07:17:25 +0000 2026","0","komdigi ongkir gk jelas","2056635322419777630","","","in","","0","0","0","https://x.com/undefined/status/2056635322419777630","1814143201718247424", +"2056892677149581606","Wed May 20 00:20:04 +0000 2026","0","komdigi keren sih mengadakan kebijakan ongkir kaya gini","2056892677149581606","","","in","","0","0","0","https://x.com/undefined/status/2056892677149581606","1814143201718247424", +"2056949159845101648","Wed May 20 04:04:30 +0000 2026","0","komdigi kereenn dan bijak membuat keputusan ongkir seperti inii","2056949159845101648","","","in","","0","0","0","https://x.com/undefined/status/2056949159845101648","1814143201718247424", +"2056538953340314095","Wed May 20 16:01:04 +0000 2026","5","@killingmaster27 @evelyndewi7 @katrinrienks @HumorJonTampan @wiyantokdr54 @EviDrajat @RickyKardjono @Micheladam69432 @ebonXboy @AntonRebor79105 @gangsterrandom @Pieter_Sun01 @Rahmah0845 @RSjahfidi @prabowo @SmileyNDeLight @Jediimar @Yens0906 @tokaidirecycle @PembuatSistem @bonapasogit24 @Jenniestale @Maskudin126003 @a8sar6alih @JowoNgalam @KING_RAJA_PWS_ @apaajadeh447870 @50ngoku @Submill @EmmaHusain0845 @BebySoSweet @Yulia_4ja @bungalotus25 @bahlillahadalia wahhh... jadi menteri komdigi dong menteri free ongkir https://t.co/1cfvoqIcNl","2057129489994510620","https://pbs.twimg.com/media/HIxkWwDaMAAXH9U.jpg","killingmaster27","in","","0","2","0","https://x.com/undefined/status/2057129489994510620","1410466956134608899", +"2055269612334555468","Fri May 15 12:50:35 +0000 2026","0","komdigi + ongkir = chaos wkwkwk","2055269612334555468","","","in","","0","0","0","https://x.com/undefined/status/2055269612334555468","2043979880086339584",