From 8830accff5f451a7f31dbf3d928d4f063cf3dcbd Mon Sep 17 00:00:00 2001 From: adistya13 Date: Fri, 22 May 2026 20:42:18 +0700 Subject: [PATCH] penambahan button crawling penambahan button crawling dan pebaikan status crawling service --- app.py | 204 +++++++- crawler.py | 154 +++++- nrt_activation.json | 1 + page_modules/crawling_page.py | 853 +++++++++++++++++++------------ tweets-data/hasil_latest.csv | 2 +- tweets-data/hasil_latest.old.csv | 2 +- tweets-data/hasil_top.csv | 6 - tweets-data/hasil_top.old.csv | 14 +- 8 files changed, 843 insertions(+), 393 deletions(-) create mode 100644 nrt_activation.json delete mode 100644 tweets-data/hasil_top.csv diff --git a/app.py b/app.py index 2be90a7..d538e08 100644 --- a/app.py +++ b/app.py @@ -26,14 +26,10 @@ except Exception: init_db() -# ── Auto-start background crawler scheduler ────────────────────────────────── -# Dipanggil sekali saat app pertama dijalankan. -# Scheduler berjalan sebagai daemon thread — tidak perlu terminal terpisah. -try: - from crawler import ensure_scheduler_running - ensure_scheduler_running() -except Exception as _e: - print(f"[app] Gagal memulai scheduler: {_e}") +# ── Auto-start background crawler scheduler (disabled) ─────────────────── +# Scheduler NRT hanya boleh berjalan setelah user menekan tombol aktivasi +# pada mode "Hari Ini" (captured) agar sesuai kebutuhan (NRT dimulai manual). + st.set_page_config( page_title="Dashboard Sentimen Twitter", @@ -164,8 +160,15 @@ def _check_crawler_alive(): and now - heartbeat_at <= threshold ) +# Hanya update _scheduler_started dari database jika user TIDAK sedang nonaktifkan eksplisit +_raw_alive = _check_crawler_alive() +explicitly_off = st.session_state.get("nrt_explicitly_deactivated", False) -st.session_state._scheduler_started = _check_crawler_alive() +if explicitly_off: + # User sudah klik nonaktifkan — override database state, paksa False + st.session_state._scheduler_started = False +else: + st.session_state._scheduler_started = _raw_alive try: from crawler import NRT_INTERVAL_MINUTES @@ -190,8 +193,7 @@ st.markdown(""" --bg: #f8fafc; --card: #ffffff; } - -* { +html, body, [class*="css"], .stApp { font-family: 'Plus Jakarta Sans', sans-serif !important; box-sizing: border-box; } @@ -226,7 +228,7 @@ html, body, .stApp, [data-testid="stAppViewContainer"] { } .main > div { - padding: 1.5rem 2rem !important; + padding: 4.3rem 2rem 1.5rem 2rem !important; } [data-testid="stSidebar"] { @@ -683,27 +685,192 @@ span:not([style]) { box-shadow: 0 2px 6px rgba(15,23,42,0.05); } -#MainMenu, footer, header { visibility: hidden !important; } +/* ===================================================== + CLEAN STREAMLIT HEADER + SIDEBAR +===================================================== */ -[data-testid="stHeader"] { background: transparent !important; } +/* Hide bawaan Streamlit */ +#MainMenu { + visibility: hidden !important; +} + +footer { + visibility: hidden !important; +} + + + +/* ===================================================== + HEADER +===================================================== */ + +[data-testid="stHeader"] { + background: rgba(248,250,252,0.96) !important; + border-bottom: 1px solid #e2e8f0 !important; + backdrop-filter: blur(8px) !important; +} + +/* ===================================================== + SIDEBAR +===================================================== */ + +[data-testid="stSidebar"] { + + background: linear-gradient( + 180deg, + #ffffff 0%, + #f8fafc 100% + ) !important; + + border-right: 1px solid #e2e8f0 !important; +} + +/* ===================================================== + MAIN CONTENT +===================================================== */ + +.main > div { + + padding-top: 4.4rem !important; +} + +/* ===================================================== + SAFE SIDEBAR TOGGLE +===================================================== */ +/* ===================================================== + SAFE SIDEBAR TOGGLE +===================================================== */ + +[data-testid="stHeader"], +header[data-testid="stHeader"], +header[data-testid="stHeader"] > div { + overflow: visible !important; +} + +[data-testid="collapsedControl"] { + position: fixed !important; + top: 10px !important; + left: 10px !important; + z-index: 2147483647 !important; + display: flex !important; + align-items: center !important; + justify-content: center !important; + visibility: visible !important; + opacity: 1 !important; + pointer-events: all !important; + width: 44px !important; + height: 44px !important; + overflow: visible !important; + clip: unset !important; + clip-path: none !important; +} + +[data-testid="collapsedControl"] button { + width: 44px !important; + height: 44px !important; + border-radius: 50% !important; + background: linear-gradient(135deg, #3b6cf7, #5b7cfa) !important; + border: 2px solid #2d59d1 !important; + box-shadow: 0 4px 16px rgba(59,108,247,0.5) !important; + display: flex !important; + align-items: center !important; + justify-content: center !important; + cursor: pointer !important; + transition: all 0.2s ease !important; + padding: 0 !important; + overflow: visible !important; +} + +[data-testid="collapsedControl"] button svg { + width: 20px !important; + height: 20px !important; + fill: #ffffff !important; + color: #ffffff !important; + stroke: #ffffff !important; + flex-shrink: 0 !important; +} + +[data-testid="collapsedControl"] button svg * { + fill: #ffffff !important; + stroke: #ffffff !important; +} + +[data-testid="collapsedControl"] button:hover { + background: #2d59d1 !important; + transform: scale(1.1) !important; + box-shadow: 0 6px 22px rgba(59,108,247,0.6) !important; +} + +/* Tombol COLLAPSE di dalam sidebar */ +section[data-testid="stSidebar"] button[kind="header"] { + background: #ffffff !important; + border: 1px solid #e2e8f0 !important; + color: #0f172a !important; + border-radius: 10px !important; + box-shadow: 0 2px 6px rgba(15,23,42,0.08) !important; +} + +section[data-testid="stSidebar"] button[kind="header"]:hover { + background: #eef2ff !important; + border-color: #3b6cf7 !important; +} +/* ===================================================== + DOWNLOAD BUTTON +===================================================== */ [data-testid="stDownloadButton"] button { + background: var(--card) !important; + color: var(--blue-primary) !important; + border: 1.5px solid var(--border) !important; + border-radius: 10px !important; + font-weight: 600 !important; + transition: all 0.2s ease !important; } [data-testid="stDownloadButton"] button:hover { + background: var(--blue-light) !important; + border-color: var(--blue-primary) !important; } + +/* ===================================================== + SIDEBAR SAFE FIX +===================================================== */ + +[data-testid="stSidebar"] { + background: linear-gradient( + 180deg, + #ffffff 0%, + #f8fafc 100% + ) !important; + + border-right: 1px solid #e2e8f0 !important; + + z-index: 999999 !important; +} + +button[kind="header"] { + background: white !important; + border-radius: 10px !important; + border: 1px solid #dbe2ea !important; + box-shadow: 0 2px 6px rgba(15,23,42,0.08) !important; +} + +button[kind="header"]:hover { + background: #eef2ff !important; + border-color: #3b6cf7 !important; +} + """, unsafe_allow_html=True) - if "page" not in st.session_state: st.session_state.page = "crawling" @@ -750,12 +917,15 @@ with st.sidebar: # ── STATUS CRAWLER ────────────────────────────────────────── sched_ok = st.session_state.get("_scheduler_started", False) + explicitly_off = st.session_state.get("nrt_explicitly_deactivated", False) sched_interval = st.session_state.get("_scheduler_interval", 0) - sched_color = "#16a34a" if sched_ok else "#3b6cf7" + sched_color = "#16a34a" if sched_ok else "#64748b" sched_label = ( f"Aktif — diperbarui tiap {sched_interval} menit" if sched_ok - else "Standby — akan crawl otomatis setiap 5 menit" + else "Nonaktif — crawling dihentikan oleh pengguna" + if explicitly_off + else "Standby — aktifkan melalui menu Hari Ini" ) st.markdown(f""" diff --git a/crawler.py b/crawler.py index 5533663..ba55eaf 100644 --- a/crawler.py +++ b/crawler.py @@ -9,15 +9,24 @@ from scraper_worker import scrape_once AUTO_CRAWL_INTERVAL_HOURS = 0.083 NRT_INTERVAL_MINUTES = 5 -LOG_FILE = "auto_crawl_log.json" +LOG_FILE = "auto_crawl_log.json" STATE_FILE = "crawler_state.json" -# ── Internal lock agar tidak ada dua crawl berjalan bersamaan ── +# ── Lock agar tidak ada dua crawl berjalan bersamaan ────────────── _crawl_lock = threading.Lock() -# ── Flag untuk scheduler background thread ── +# ── Flag scheduler ──────────────────────────────────────────────── _scheduler_started = False -_scheduler_lock = threading.Lock() +_scheduler_lock = threading.Lock() + +# ── Flag aktivasi NRT — harus True sebelum scheduler boleh jalan ── +# PENTING: Nilai awal selalu False. Tidak ada kode module-level +# yang boleh mengubah ini menjadi True secara otomatis. +_nrt_enabled = False +_nrt_enable_lock = threading.Lock() + +# ── Event untuk menghentikan scheduler secara bersih ────────────── +_scheduler_stop_event = threading.Event() # ═══════════════════════════════════════════════════════════ @@ -47,10 +56,10 @@ def get_auto_crawl_logs(limit=10): def save_auto_crawl_log(status="success", total_saved=0, error=None): logs = get_auto_crawl_logs(50) logs.insert(0, { - "timestamp": datetime.now(timezone.utc).isoformat(), - "status": status, + "timestamp": datetime.now(timezone.utc).isoformat(), + "status": status, "total_saved": total_saved, - "error": error, + "error": error, }) try: with open(LOG_FILE, "w") as f: @@ -67,11 +76,11 @@ def set_crawler_state(is_running=False, service_active=None): service_active = previous_state.get("service_active", False) state = { - "is_running": bool(is_running), - "service_active": bool(service_active), - "updated_at": now, - "heartbeat_at": now if service_active else previous_state.get("heartbeat_at"), - "last_job_started_at": previous_state.get("last_job_started_at"), + "is_running": bool(is_running), + "service_active": bool(service_active), + "updated_at": now, + "heartbeat_at": now if service_active else previous_state.get("heartbeat_at"), + "last_job_started_at": previous_state.get("last_job_started_at"), "last_job_finished_at": previous_state.get("last_job_finished_at"), } @@ -86,7 +95,32 @@ def set_crawler_state(is_running=False, service_active=None): except Exception as e: print(f"[crawler] Gagal menulis state: {e}") +# ═══════════════════════════════════════════════════════════ +# NRT ACTIVATION PERSISTENCE +# ═══════════════════════════════════════════════════════════ +NRT_ACTIVATION_FILE = "nrt_activation.json" + +def save_nrt_activation(activated: bool): + """Simpan status aktivasi NRT ke file agar persist setelah refresh.""" + try: + with open(NRT_ACTIVATION_FILE, "w") as f: + json.dump({"activated": activated, "updated_at": datetime.now(timezone.utc).isoformat()}, f) + except Exception as e: + print(f"[crawler] Gagal menyimpan NRT activation: {e}") + + +def load_nrt_activation() -> bool: + """Baca status aktivasi NRT dari file.""" + if not os.path.exists(NRT_ACTIVATION_FILE): + return False + try: + with open(NRT_ACTIVATION_FILE, "r") as f: + data = json.load(f) + return bool(data.get("activated", False)) + except Exception: + return False + # ═══════════════════════════════════════════════════════════ # CORE JOB # ═══════════════════════════════════════════════════════════ @@ -95,7 +129,6 @@ def auto_crawl_job(): """ Jalankan satu siklus crawling. Thread-safe: hanya satu crawl yang boleh berjalan pada satu waktu. - Bisa dipanggil dari tombol UI maupun scheduler otomatis. """ if not _crawl_lock.acquire(blocking=False): print("[crawler] Crawl sedang berjalan, skip.") @@ -129,34 +162,38 @@ def auto_crawl_job(): # ═══════════════════════════════════════════════════════════ -# BACKGROUND SCHEDULER (dipakai Streamlit, tanpa terminal) +# BACKGROUND SCHEDULER # ═══════════════════════════════════════════════════════════ def _scheduler_loop(): """ - Loop yang berjalan di daemon thread. - Crawling pertama langsung dijalankan, lalu setiap NRT_INTERVAL_MINUTES menit. + Loop daemon thread. Crawl pertama langsung, lalu tiap NRT_INTERVAL_MINUTES. + Berhenti bersih saat _scheduler_stop_event di-set. """ print(f"[crawler] Scheduler dimulai — interval {NRT_INTERVAL_MINUTES} menit.") set_crawler_state(False, service_active=True) - while True: + while not _scheduler_stop_event.is_set(): auto_crawl_job() - # Tulis heartbeat setiap 30 detik selagi menunggu interval berikutnya + # Tunggu interval berikutnya sambil tulis heartbeat tiap 10 detik wait_until = time.time() + (NRT_INTERVAL_MINUTES * 60) - while time.time() < wait_until: - # Heartbeat agar is_crawler_service_active() tetap True + while time.time() < wait_until and not _scheduler_stop_event.is_set(): _write_heartbeat() - time.sleep(min(30, max(1, wait_until - time.time()))) + time.sleep(min(10, max(1, wait_until - time.time()))) - print(f"[crawler] Interval berikutnya dalam {NRT_INTERVAL_MINUTES} menit...") + if not _scheduler_stop_event.is_set(): + print(f"[crawler] Mulai crawl berikutnya...") + + # Bersihkan state saat loop selesai + set_crawler_state(False, service_active=False) + print("[crawler] Scheduler dihentikan.") def _write_heartbeat(): """Perbarui heartbeat_at tanpa mengubah field lain.""" state = get_crawler_state() - state["heartbeat_at"] = datetime.now(timezone.utc).isoformat() + state["heartbeat_at"] = datetime.now(timezone.utc).isoformat() state["service_active"] = True try: with open(STATE_FILE, "w") as f: @@ -165,31 +202,92 @@ def _write_heartbeat(): pass +# ═══════════════════════════════════════════════════════════ +# ACTIVATION API ← dipakai oleh crawling_page.py +# ═══════════════════════════════════════════════════════════ + +def is_nrt_enabled() -> bool: + """Kembalikan True jika NRT sudah diaktifkan user.""" + return _nrt_enabled + + +def activate_nrt_scheduler(): + """ + Aktifkan NRT scheduler. + Harus dipanggil HANYA dari tombol UI — tidak dari module-level manapun. + Idempotent: aman dipanggil berkali-kali. + """ + global _nrt_enabled + + with _nrt_enable_lock: + _nrt_enabled = True + + save_nrt_activation(True) # ← TAMBAH INI + _scheduler_stop_event.clear() # pastikan event tidak dalam kondisi set + ensure_scheduler_running() + print("[crawler] NRT diaktifkan oleh user.") + + +def deactivate_nrt_scheduler(): + """ + Hentikan NRT scheduler. + Thread yang berjalan akan berhenti bersih pada iterasi berikutnya. + """ + global _nrt_enabled, _scheduler_started + + with _nrt_enable_lock: + _nrt_enabled = False + + save_nrt_activation(False) + _scheduler_stop_event.set() # sinyal ke thread agar berhenti + + with _scheduler_lock: + _scheduler_started = False # izinkan start ulang di masa depan + + set_crawler_state(False, service_active=False) + print("[crawler] NRT dinonaktifkan oleh user.") + + def ensure_scheduler_running(): """ Pastikan background scheduler sudah berjalan. + HANYA berjalan jika _nrt_enabled = True (sudah diaktifkan user). Aman dipanggil berkali-kali (idempotent). - Dipanggil dari crawling_page.py saat halaman dirender. + + JANGAN panggil fungsi ini dari module-level atau saat app startup. + Panggil hanya setelah activate_nrt_scheduler() dipicu dari UI. """ global _scheduler_started + if not _nrt_enabled: + # Belum diaktifkan user — jangan mulai thread apapun + return + with _scheduler_lock: if _scheduler_started: return - t = threading.Thread(target=_scheduler_loop, name="CrawlerScheduler", daemon=True) + _scheduler_stop_event.clear() + t = threading.Thread( + target=_scheduler_loop, + name="CrawlerScheduler", + daemon=True, + ) t.start() _scheduler_started = True print("[crawler] Background scheduler thread dimulai.") # ═══════════════════════════════════════════════════════════ -# ENTRY POINT (tetap bisa dipakai via terminal jika mau) +# ENTRY POINT (via terminal, opsional) # ═══════════════════════════════════════════════════════════ def main(): - """Jalankan scheduler via terminal (opsional, tidak wajib).""" - print("Crawler realtime aktif.") + """Jalankan scheduler via terminal (run_scraper_loop.py).""" + global _nrt_enabled + _nrt_enabled = True # mode terminal: langsung aktif + + print("Crawler realtime aktif (mode terminal).") print(f"Interval: {NRT_INTERVAL_MINUTES} menit") set_crawler_state(False, service_active=True) diff --git a/nrt_activation.json b/nrt_activation.json new file mode 100644 index 0000000..c7d3c78 --- /dev/null +++ b/nrt_activation.json @@ -0,0 +1 @@ +{"activated": false, "updated_at": "2026-05-22T13:17:14.073811+00:00"} \ No newline at end of file diff --git a/page_modules/crawling_page.py b/page_modules/crawling_page.py index 4671973..7d51a88 100644 --- a/page_modules/crawling_page.py +++ b/page_modules/crawling_page.py @@ -21,16 +21,13 @@ from crawler import ( AUTO_CRAWL_INTERVAL_HOURS, NRT_INTERVAL_MINUTES, auto_crawl_job, + activate_nrt_scheduler, + deactivate_nrt_scheduler, + is_nrt_enabled, ensure_scheduler_running, ) -# ═══════════════════════════════════════════════════════════ -# AUTO-START SCHEDULER -# ═══════════════════════════════════════════════════════════ -ensure_scheduler_running() - - # ═══════════════════════════════════════════════════════════ # TIMEZONE & DATETIME HELPERS # ═══════════════════════════════════════════════════════════ @@ -181,39 +178,53 @@ def _render_crawling_styles(): """, unsafe_allow_html=True) @@ -321,25 +434,193 @@ def _gap(size="md"): st.markdown(f'
', unsafe_allow_html=True) +def _divider(): + st.markdown('
', unsafe_allow_html=True) + + def _section_header(title, subtitle=""): sub_html = ( f'

{subtitle}

' if subtitle else "" ) st.markdown(f""" -
-

{title}

+
+

{title}

{sub_html}
""", unsafe_allow_html=True) -def _divider(): - st.markdown('
', unsafe_allow_html=True) +# ═══════════════════════════════════════════════════════════ +# PAGE HEADER +# ═══════════════════════════════════════════════════════════ + +def _render_page_header(): + nrt = str(NRT_INTERVAL_MINUTES) + + st.markdown(f""" +
+
+
🐦
+
+
+Crawling Data Twitter — Komdigi & Isu Ongkir +Live Monitor +
+

Sistem ini memantau opini masyarakat Twitter/X mengenai kebijakan pembatasan gratis ongkir oleh Komdigi. Tweet dikumpulkan otomatis, dibersihkan melalui preprocessing Bahasa Indonesia, lalu dianalisis menggunakan model Naive Bayes untuk menentukan sentimen (positif, negatif, atau netral) secara near-realtime.

+
+📡 Auto Crawling +🤖 Naive Bayes +⚡ Near Realtime +📊 Dashboard Monitoring +
+
+
+
+""", unsafe_allow_html=True) + + st.markdown(f""" +
+
+
🔄
Interval Crawling
Setiap {nrt} menit
+
📅
Jangkauan Data
7 hari terakhir
+
🗃
Penyimpanan
Database lokal
+
🤖
Model Prediksi
Naive Bayes
+
+
+""", unsafe_allow_html=True) + + st.markdown(""" +
+
+
+🔍 Query +komdigi +gratis ongkir +free ongkir +pembatasan ongkir +
+
+⚙ Alur kerja +🐦 Crawling tweet +→ +🗃 Simpan ke database +→ +✁ Preprocessing teks +→ +🤖 Prediksi sentimen +→ +📊 Visualisasi hasil +
+
+
+""", unsafe_allow_html=True) +# ═══════════════════════════════════════════════════════════ +# NRT ACTIVATION GATE +# ═══════════════════════════════════════════════════════════ +def _render_activation_gate(): + items = [ + f'Scheduler berjalan setiap {NRT_INTERVAL_MINUTES} menit di background', + 'Mencakup tweet dalam 7 hari terakhir', + 'Tweet baru otomatis tersimpan ke database lokal', + 'Crawl pertama dimulai segera setelah tombol diklik', + ] + + grid_rows = "" + for item in items: + grid_rows += ( + '
' + '\u2713' + f'{item}' + '
' + ) + + html = ( + '
' + '
' + '
\u26a1
' + '
' + '
Pengambilan Data Otomatis Belum Aktif
' + f'
Aktifkan untuk mulai crawling tweet secara near-realtime setiap {NRT_INTERVAL_MINUTES} menit.
' + '
' + '
' + '
' + '
Yang akan terjadi setelah aktivasi
' + '
' + + grid_rows + + '
' + '
' + '
' + '' + '\U0001f4a1 Catatan: Crawling aktif selama sesi aplikasi berjalan. ' + 'Data historis tetap tersimpan dan bisa dilihat di mode 7 Hari atau 30 Hari tanpa perlu mengaktifkan crawling.' + '' + '
' + '
' + ) + st.markdown(html, unsafe_allow_html=True) + + col_l, col_btn, col_r = st.columns([1, 2, 1]) + with col_btn: + if st.button( + "\u26a1 Aktifkan Pengambilan Data Secara Otomatis", + type="primary", + use_container_width=True, + key="btn_activate_nrt", + ): + st.session_state.nrt_activated = True + st.session_state.nrt_explicitly_deactivated = False # ← RESET FLAG + st.session_state.crawl_history_started_at = datetime.now(tz=timezone.utc) + activate_nrt_scheduler() + st.success("\u2705 Crawling otomatis berhasil diaktifkan! Crawl pertama akan segera dimulai.") + time.sleep(1.0) + st.rerun() + + _gap("md") + + +# ═══════════════════════════════════════════════════════════ +# ACTIVE STRIP (setelah diaktifkan) +# ═══════════════════════════════════════════════════════════ +def _render_active_strip(): + col_info, col_btn = st.columns([3, 1], gap="medium") + + with col_info: + st.markdown(f""" +
+
+
+ + Crawling Otomatis Aktif + + + — berjalan setiap {NRT_INTERVAL_MINUTES} menit di background + +
+
+""", unsafe_allow_html=True) + + with col_btn: + _gap("xs") + if st.button( + "⏹ Nonaktifkan", + type="secondary", + use_container_width=True, + key="btn_deactivate_nrt", + ): + # Set semua flag nonaktif + st.session_state.nrt_activated = False + st.session_state.nrt_explicitly_deactivated = True # ← FLAG BARU + st.session_state.pop("crawl_history_started_at", None) + deactivate_nrt_scheduler() + st.warning("⏹ Crawling otomatis dihentikan untuk sesi ini.") + time.sleep(0.9) + st.rerun() # ═══════════════════════════════════════════════════════════ # NRT STATUS BANNER # ═══════════════════════════════════════════════════════════ @@ -369,7 +650,7 @@ def _render_nrt_status_banner(sched_ok): title = "Crawler Siap Dijalankan" desc = ( f"Scheduler otomatis aktif setiap {NRT_INTERVAL_MINUTES} menit " - f"sejak aplikasi pertama dibuka — tidak perlu terminal terpisah." + f"sejak diaktifkan — tidak perlu terminal terpisah." ) badge = "⚬ STANDBY" @@ -378,7 +659,8 @@ def _render_nrt_status_banner(sched_ok): padding:1rem 1.3rem;margin-bottom:1rem; display:flex;align-items:center;gap:1rem; box-shadow:0 2px 8px rgba(15,23,42,0.04);"> -
@@ -412,11 +694,11 @@ def _render_monitoring_panel(last_log, last_dt, sched_ok): ) latest_crawled = df_stats["crawled_at"].max() if not df_stats.empty else None except Exception: - total_db = 0 - tweet_today = 0 + total_db = 0 + tweet_today = 0 latest_crawled = None - last_log_dt = last_dt + last_log_dt = last_dt if last_log_dt is None and last_log: last_log_dt = parse_log_time(last_log.get("timestamp")) @@ -424,9 +706,9 @@ def _render_monitoring_panel(last_log, last_dt, sched_ok): if final_last_dt is None: final_last_dt = to_user_local_datetime(latest_crawled) - last_crawl_str = "Belum pernah" - next_crawl_str = "Belum terjadwal" - countdown_str = "Menunggu crawl pertama..." + last_crawl_str = "Belum pernah" + next_crawl_str = "Belum terjadwal" + countdown_str = "Menunggu crawl pertama..." monitoring_time_str = format_dt(now) if final_last_dt is not None: @@ -454,7 +736,6 @@ def _render_monitoring_panel(last_log, last_dt, sched_ok): col_a, col_b = st.columns(2, gap="medium") - # ── Left card: sistem crawling ── left_rows = [ ("Interval crawling", f"{NRT_INTERVAL_MINUTES} menit", "#3b6cf7"), ("Jangkauan data", "7 hari terakhir", "#3b6cf7"), @@ -490,12 +771,11 @@ def _render_monitoring_panel(last_log, last_dt, sched_ok): unsafe_allow_html=True, ) - # ── Right card: ringkasan dataset ── right_rows = [ - ("Update terakhir", last_crawl_str, "#0f172a"), - ("Countdown berikutnya", countdown_str, "#16a34a"), - ("Tweet masuk hari ini", f"{tweet_today:,} tweet", "#3b6cf7"), - ("Total dataset", f"{total_db:,} tweet", "#0f172a"), + ("Update terakhir", last_crawl_str, "#0f172a"), + ("Countdown berikutnya", countdown_str, "#16a34a"), + ("Tweet masuk hari ini", f"{tweet_today:,} tweet", "#3b6cf7"), + ("Total dataset", f"{total_db:,} tweet", "#0f172a"), ] rows_html_b = "" @@ -555,7 +835,7 @@ def _render_metrics(total, days, earliest, latest, filter_label, basis_label): c1, c2, c3, c4 = st.columns(4) for col, (icon, bg, color, dark, border_c, label, val, sub) in zip([c1, c2, c3, c4], pills): with col: - fs = "1.1rem" if len(str(val)) > 10 else "1.55rem" + fs = "1.45rem" if len(str(val)) > 10 else "2rem" st.markdown( f'
' @@ -665,7 +945,6 @@ def _render_charts(df, filter_label, date_col, chart_label, dt_start, dt_end): ) st.plotly_chart(fig_bar, use_container_width=True, config={"displayModeBar": False}) - # ── Legend ── legend_items = [("#1d4ed8", "Tertinggi"), ("#3b6cf7", "Normal")] if min_cnt > 0 and min_cnt != max_cnt: legend_items.insert(1, ("#a78bfa", "Terendah")) @@ -683,7 +962,6 @@ def _render_charts(df, filter_label, date_col, chart_label, dt_start, dt_end): unsafe_allow_html=True ) - # ── Insight cards ── pc1, pc2, pc3 = st.columns(3, gap="small") def _date_pills(days_list, color, max_show=4): @@ -830,159 +1108,19 @@ def get_filtered_data(df_all): filter_label = f"{dt_start.strftime('%d/%m/%Y')} s/d {dt_end.strftime('%d/%m/%Y')}" return { - "df": df, - "date_col": date_col, + "df": df, + "date_col": date_col, "mode_display": mode_display, - "mode_color": mode_color, - "mode_icon": mode_icon, + "mode_color": mode_color, + "mode_icon": mode_icon, "filter_label": filter_label, - "dt_start": dt_start, - "dt_end": dt_end, - "basis_label": basis_label, - "chart_label": chart_label, + "dt_start": dt_start, + "dt_end": dt_end, + "basis_label": basis_label, + "chart_label": chart_label, } -# ═══════════════════════════════════════════════════════════ -# PAGE HEADER (dipecah jadi beberapa st.markdown() kecil) -# ═══════════════════════════════════════════════════════════ - -def _render_page_header(): - # Bangun semua bagian dalam Python dulu, baru render SEKALI - chip_data = [ - ("🔄", "Interval crawling", f"Setiap {NRT_INTERVAL_MINUTES} menit"), - ("📅", "Jangkauan data", "7 hari terakhir"), - ("🗃", "Penyimpanan", "Database lokal"), - ("🤖", "Model prediksi", "Naive Bayes"), - ] - chips_html = "".join( - '
' - f'{ico}' - '
' - f'
{lbl}
' - f'
{val}
' - '
' - for ico, lbl, val in chip_data - ) - - PIPELINE_STEPS = [ - ("🐦", "Crawling tweet"), - ("🗃", "Simpan ke database"), - ("🧹", "Preprocessing teks"), - ("🤖", "Prediksi sentimen"), - ("📊", "Visualisasi hasil"), - ] - pipeline_steps_html = " ".join( - ( - f'' - f'{icon} {label}' - + ( - '→' - if i < len(PIPELINE_STEPS) - 1 else "" - ) - ) - for i, (icon, label) in enumerate(PIPELINE_STEPS) - ) - - html = ( - # ── outer card ── - '
' - - # ── row 1: icon + title + description ── - '
' - - # icon - '
🐦
' - - # title block - '
' - '
' - '

' - 'Crawling Data Twitter — Komdigi & Isu Ongkir' - '

' - '' - '' - 'Live Monitor' - '' - '
' - '

' - 'Dashboard analisis sentimen tweet seputar ' - 'kebijakan ongkos kirim gratis dan ' - 'Komdigi. ' - 'Sistem mengumpulkan tweet otomatis, menyimpan ke database lokal, ' - 'membersihkan teks Bahasa Indonesia, lalu memprediksi sentimen ' - '(positif / negatif / netral) via model ' - 'Naive Bayes ' - '— near-realtime tanpa intervensi manual.' - '

' - '
' - '
' # end row 1 - - # ── divider ── - '
' - - # ── row 2: chips ── - f'
' - f'{chips_html}' - '
' - - # ── divider ── - '
' - - # ── row 3: query box ── - '
' - '
' - '🔍 Query pencarian aktif' - '' - 'Twitter / X Search' - '' - '
' - '
' - 'komdigi ' - '(ongkir OR ' - '"gratis ongkir" OR ' - '"free ongkir") OR ' - '"pembatasan gratis ongkir" OR ' - '"gratis ongkir dibatasi"' - '
' - '
' - 'Mencakup variasi: ongkir, gratis ongkir, free ongkir' - ' — termasuk konteks pembatasan subsidi ongkir oleh Komdigi.' - '
' - '
' # end query box - - # ── row 4: pipeline ── - '
' - '' - '⚙ Alur kerja' - f'{pipeline_steps_html}' - '
' - - '
' # end outer card - ) - - st.markdown(html, unsafe_allow_html=True) - - # ═══════════════════════════════════════════════════════════ # MAIN PAGE # ═══════════════════════════════════════════════════════════ @@ -998,7 +1136,33 @@ def show(): if "filter_end_date" not in st.session_state: st.session_state.filter_end_date = today - # ── Page Header ────────────────────────────────────────────── + # ── State aktivasi NRT ───────────────────────────────────────────────── + from crawler import load_nrt_activation, ensure_scheduler_running as _ensure_sched + + # Inisialisasi pertama kali sesi dimulai + if "nrt_activated" not in st.session_state: + was_activated = load_nrt_activation() + # Jika file persist bilang aktif tapi user sudah eksplisit nonaktifkan → ikut flag + explicitly_off_init = st.session_state.get("nrt_explicitly_deactivated", False) + if was_activated and not explicitly_off_init: + st.session_state.nrt_activated = True + from crawler import activate_nrt_scheduler as _activate + _activate() + else: + st.session_state.nrt_activated = False + + # Sinkronisasi: _scheduler_started sudah dijaga oleh app.py (False jika explicitly_off) + # Cukup mirror nilainya — tidak perlu logika tambahan + explicitly_off = st.session_state.get("nrt_explicitly_deactivated", False) + if explicitly_off: + st.session_state.nrt_activated = False + elif st.session_state.get("_scheduler_started", False): + if not st.session_state.get("nrt_activated", False): + st.session_state.nrt_activated = True + if "crawl_history_started_at" not in st.session_state: + st.session_state.crawl_history_started_at = datetime.now(tz=timezone.utc) + + # ── Page Header ─────────────────────────────────────────────── _render_page_header() # ── Mode Selector ───────────────────────────────────────────── @@ -1049,10 +1213,10 @@ def show(): st.session_state.analysis_mode = mode_key st.rerun() - bg = cfg["gradient"] if active else "#ffffff" - color = cfg["color"] if active else "#94a3b8" - dark = cfg["dark"] if active else "#475569" - shadow = f"0 4px 16px {cfg['border']}55" if active else "0 1px 4px rgba(15,23,42,0.04)" + bg = cfg["gradient"] if active else "#ffffff" + color = cfg["color"] if active else "#94a3b8" + dark = cfg["dark"] if active else "#475569" + shadow = f"0 4px 16px {cfg['border']}55" if active else "0 1px 4px rgba(15,23,42,0.04)" border_style = f"2px solid {cfg['color']}" if active else "1.5px solid #e8edf5" icon_shadow = f"0 4px 10px {cfg['border']}88" if active else "none" icon_bg = "white" if active else "#f1f5f9" @@ -1080,7 +1244,7 @@ def show(): _gap("md") - # ── Load data ───────────────────────────────────────────────── + # ── Load data ────────────────────────────────────────────────── try: df_all = pd.read_sql("SELECT * FROM tweets ORDER BY created_at DESC", engine) except Exception as e: @@ -1094,7 +1258,7 @@ def show(): df_all["created_at"] = parse_dt(df_all["created_at"]) df_all["crawled_at"] = parse_crawled_dt(df_all["crawled_at"]) - # ── Custom Date Picker ──────────────────────────────────────── + # ── Custom Date Picker ───────────────────────────────────────── if st.session_state.analysis_mode == "custom": df_check = df_all.dropna(subset=["created_at"]).copy() if not df_check.empty: @@ -1151,139 +1315,158 @@ def show(): st.rerun() # ═══════════════════════════════════════════════════════════ - # MODE: TWEET HARI INI (captured) + # MODE: HARI INI (captured) # ═══════════════════════════════════════════════════════════ if st.session_state.analysis_mode == "captured": - st_autorefresh(interval=1000, key="crawler_monitor_refresh") - logs = get_auto_crawl_logs(5) - last_log = logs[0] if logs else None - sched_ok = is_crawler_service_active(logs) - last_dt = parse_log_time(last_log.get("timestamp")) if last_log else None + nrt_activated = st.session_state.get("nrt_activated", False) - _render_nrt_status_banner(sched_ok) - _gap("sm") - _render_monitoring_panel(last_log, last_dt, sched_ok) - _gap("md") + if not nrt_activated: + # ── Belum diaktifkan: tampilkan activation gate ─────── + _render_activation_gate() - # ── Crawl History ────────────────────────────────────── - visible_logs = get_visible_crawl_logs(logs) - if visible_logs: - rows = [] - for idx, log in enumerate(visible_logs, start=1): - status = log.get("status", "") - total_saved = int(log.get("total_saved") or 0) - error_msg = log.get("error") - try: - crawl_time = format_dt( - parse_dt(pd.Series([parse_log_time(log.get("timestamp"))])).iloc[0] - ) - except Exception: - crawl_time = log.get("timestamp", "-") or "-" + else: + # ── Sudah diaktifkan: pastikan thread scheduler hidup ─ + if is_nrt_enabled(): + ensure_scheduler_running() - if status == "success": - status_label = "Berhasil" - note = ( - f"{total_saved:,} tweet baru tersimpan" - if total_saved > 0 - else "Crawling berhasil, tidak ada tweet baru" - ) - else: - status_label = "Gagal" - note = error_msg or "Terjadi kesalahan saat crawling" + # Auto-refresh setiap 1 detik untuk countdown live + st_autorefresh(interval=1000, key="crawler_monitor_refresh") - rows.append({ - "No": idx, - "Waktu Crawl": crawl_time, - "Status": status_label, - "Tweet Baru": total_saved, - "Keterangan": note, - }) + logs = get_auto_crawl_logs(5) + last_log = logs[0] if logs else None + sched_ok = is_crawler_service_active(logs) + last_dt = parse_log_time(last_log.get("timestamp")) if last_log else None + + # Strip status aktif + tombol nonaktifkan + _render_active_strip() + + # Banner status crawler + _render_nrt_status_banner(sched_ok) + _gap("sm") + + # Panel monitoring + _render_monitoring_panel(last_log, last_dt, sched_ok) + _gap("md") + + # Riwayat crawl + visible_logs = get_visible_crawl_logs(logs) + if visible_logs: + rows = [] + for idx, log in enumerate(visible_logs, start=1): + status = log.get("status", "") + total_saved = int(log.get("total_saved") or 0) + error_msg = log.get("error") + try: + crawl_time = format_dt( + parse_dt(pd.Series([parse_log_time(log.get("timestamp"))])).iloc[0] + ) + except Exception: + crawl_time = log.get("timestamp", "-") or "-" + + if status == "success": + status_label = "Berhasil" + note = ( + f"{total_saved:,} tweet baru tersimpan" + if total_saved > 0 + else "Crawling berhasil, tidak ada tweet baru" + ) + else: + status_label = "Gagal" + note = error_msg or "Terjadi kesalahan saat crawling" + + rows.append({ + "No": idx, + "Waktu Crawl": crawl_time, + "Status": status_label, + "Tweet Baru": total_saved, + "Keterangan": note, + }) + + _section_header( + "🕐 Riwayat Crawling Otomatis", + f"{len(rows)} aktivitas terbaru · interval {NRT_INTERVAL_MINUTES} menit" + ) + _gap("xs") + render_standard_table( + pd.DataFrame(rows), + height=300, min_width=760, + right_align=["Tweet Baru"], + badge_columns=["Status"], + nowrap=["No", "Waktu Crawl", "Status"], + wide_columns=["Keterangan"], + column_widths={ + "No": "56px", "Waktu Crawl": "160px", + "Status": "138px", "Tweet Baru": "118px", "Keterangan": "320px", + }, + ) + else: + st.markdown( + '
' + '
⏳
' + '
' + 'Menunggu Crawl Pertama
' + f'
' + f'Riwayat akan muncul setelah crawl otomatis pertama selesai ' + f'(maks. {NRT_INTERVAL_MINUTES} menit sejak aktivasi)' + f'
' + '
', + unsafe_allow_html=True, + ) + + _gap("lg") + + # Tweet yang diambil bot hari ini + today_start = datetime.combine(today, datetime.min.time()) + today_end = datetime.combine(today, datetime.max.time().replace(microsecond=0)) + + bot_df = df_all.copy() + if "crawl_type" in bot_df.columns: + bot_df = bot_df[bot_df["crawl_type"].fillna("").str.lower().eq("realtime")].copy() + + bot_df = bot_df.dropna(subset=["crawled_at"]) + bot_df = bot_df[ + (bot_df["crawled_at"] >= pd.Timestamp(today_start)) & + (bot_df["crawled_at"] <= pd.Timestamp(today_end)) + ].sort_values("crawled_at", ascending=False) _section_header( - "🕐 Riwayat Crawling Otomatis", - f"{len(rows)} aktivitas terbaru · interval {NRT_INTERVAL_MINUTES} menit" + "🤖 Tweet yang Diambil Bot Hari Ini", + "Semua tweet yang berhasil disimpan crawler hari ini" ) _gap("xs") - render_standard_table( - pd.DataFrame(rows), - height=300, min_width=760, - right_align=["Tweet Baru"], - badge_columns=["Status"], - nowrap=["No", "Waktu Crawl", "Status"], - wide_columns=["Keterangan"], - column_widths={ - "No": "56px", "Waktu Crawl": "160px", - "Status": "138px", "Tweet Baru": "118px", "Keterangan": "320px", - }, - ) - else: - st.markdown( - '
' - '
⏳
' - '
' - 'Menunggu Crawl Pertama
' - f'
' - f'Riwayat akan muncul setelah crawl otomatis pertama selesai ' - f'(maks. {NRT_INTERVAL_MINUTES} menit sejak aplikasi dibuka)' - f'
' - '
', - unsafe_allow_html=True, - ) - _gap("lg") - - # ── Tweet yang dicrawl bot hari ini ─────────────────── - today_start = datetime.combine(today, datetime.min.time()) - today_end = datetime.combine(today, datetime.max.time().replace(microsecond=0)) - - bot_df = df_all.copy() - if "crawl_type" in bot_df.columns: - bot_df = bot_df[bot_df["crawl_type"].fillna("").str.lower().eq("realtime")].copy() - - bot_df = bot_df.dropna(subset=["crawled_at"]) - bot_df = bot_df[ - (bot_df["crawled_at"] >= pd.Timestamp(today_start)) & - (bot_df["crawled_at"] <= pd.Timestamp(today_end)) - ].sort_values("crawled_at", ascending=False) - - _section_header( - "🤖 Tweet yang Diambil Bot Hari Ini", - "Semua tweet yang berhasil disimpan crawler hari ini" - ) - _gap("xs") - - if bot_df.empty: - st.markdown( - '
' - '
🐦
' - '
' - 'Belum Ada Tweet Hari Ini
' - '
' - 'Tweet baru akan muncul setelah crawl berikutnya selesai' - '
' - '
', - unsafe_allow_html=True, - ) - else: - bot_disp = bot_df.head(100).copy() - bot_disp["Tanggal Tweet"] = bot_disp["created_at"].apply(format_dt) - bot_disp["Waktu Bot Ambil"] = bot_disp["crawled_at"].apply(format_dt) - render_standard_table( - bot_disp[["tweet_id", "text", "Tanggal Tweet", "Waktu Bot Ambil"]].rename( - columns={"tweet_id": "ID Tweet", "text": "Isi Tweet"} - ), - height=340, min_width=920, - nowrap=["ID Tweet", "Tanggal Tweet", "Waktu Bot Ambil"], - wide_columns=["Isi Tweet"], - column_widths={ - "ID Tweet": "170px", "Isi Tweet": "430px", - "Tanggal Tweet": "150px", "Waktu Bot Ambil": "160px", - }, - ) - st.caption( - f"Menampilkan {len(bot_disp):,} dari {len(bot_df):,} tweet yang disimpan bot hari ini" - ) + if bot_df.empty: + st.markdown( + '
' + '
🐦
' + '
' + 'Belum Ada Tweet Hari Ini
' + '
' + 'Tweet baru akan muncul setelah crawl berikutnya selesai' + '
' + '
', + unsafe_allow_html=True, + ) + else: + bot_disp = bot_df.head(100).copy() + bot_disp["Tanggal Tweet"] = bot_disp["created_at"].apply(format_dt) + bot_disp["Waktu Bot Ambil"] = bot_disp["crawled_at"].apply(format_dt) + render_standard_table( + bot_disp[["tweet_id", "text", "Tanggal Tweet", "Waktu Bot Ambil"]].rename( + columns={"tweet_id": "ID Tweet", "text": "Isi Tweet"} + ), + height=340, min_width=920, + nowrap=["ID Tweet", "Tanggal Tweet", "Waktu Bot Ambil"], + wide_columns=["Isi Tweet"], + column_widths={ + "ID Tweet": "170px", "Isi Tweet": "430px", + "Tanggal Tweet": "150px", "Waktu Bot Ambil": "160px", + }, + ) + st.caption( + f"Menampilkan {len(bot_disp):,} dari {len(bot_df):,} tweet yang disimpan bot hari ini" + ) # ═══════════════════════════════════════════════════════════ # FILTERED DATA — tampil di semua mode @@ -1308,7 +1491,6 @@ def show(): _divider() - # ── Active Filter Banner ────────────────────────────────── total = len(df) st.markdown( f'
' f'
' @@ -1378,7 +1559,6 @@ def show(): _render_charts(df, filter_label, date_col, chart_label, dt_start, dt_end) - # ── Tabel ───────────────────────────────────────────────── display_limit = 100 _section_header( "📋 Daftar Tweet yang Terkumpul", @@ -1405,7 +1585,6 @@ def show(): st.caption(f"Menampilkan {len(df_disp):,} dari {total:,} tweet") _gap("sm") - # ── Actions ─────────────────────────────────────────────── c_dl, c_nav = st.columns(2) with c_dl: st.download_button( diff --git a/tweets-data/hasil_latest.csv b/tweets-data/hasil_latest.csv index 7cb42a9..9d60e49 100644 --- a/tweets-data/hasil_latest.csv +++ b/tweets-data/hasil_latest.csv @@ -1,2 +1,2 @@ "conversation_id_str","created_at","favorite_count","full_text","id_str","image_url","in_reply_to_screen_name","lang","location","quote_count","reply_count","retweet_count","tweet_url","user_id_str","username" -"2056538953340314095","Wed May 20 16:01:04 +0000 2026","5","@killingmaster27 @evelyndewi7 @katrinrienks @HumorJonTampan @wiyantokdr54 @EviDrajat @RickyKardjono @Micheladam69432 @ebonXboy @AntonRebor79105 @gangsterrandom @Pieter_Sun01 @Rahmah0845 @RSjahfidi @prabowo @SmileyNDeLight @Jediimar @Yens0906 @tokaidirecycle @PembuatSistem @bonapasogit24 @Jenniestale @Maskudin126003 @a8sar6alih @JowoNgalam @KING_RAJA_PWS_ @apaajadeh447870 @50ngoku @Submill @EmmaHusain0845 @BebySoSweet @Yulia_4ja @bungalotus25 @bahlillahadalia wahhh... jadi menteri komdigi dong menteri free ongkir https://t.co/1cfvoqIcNl","2057129489994510620","https://pbs.twimg.com/media/HIxkWwDaMAAXH9U.jpg","killingmaster27","in","","0","2","0","https://x.com/undefined/status/2057129489994510620","1410466956134608899", +"2056538953340314095","Wed May 20 16:01:04 +0000 2026","4","@killingmaster27 @evelyndewi7 @katrinrienks @HumorJonTampan @wiyantokdr54 @EviDrajat @RickyKardjono @Micheladam69432 @ebonXboy @AntonRebor79105 @gangsterrandom @Pieter_Sun01 @Rahmah0845 @RSjahfidi @prabowo @SmileyNDeLight @Jediimar @Yens0906 @tokaidirecycle @PembuatSistem @bonapasogit24 @Jenniestale @Maskudin126003 @a8sar6alih @JowoNgalam @KING_RAJA_PWS_ @apaajadeh447870 @50ngoku @Submill @EmmaHusain0845 @BebySoSweet @Yulia_4ja @bungalotus25 @bahlillahadalia wahhh... jadi menteri komdigi dong menteri free ongkir https://t.co/1cfvoqIcNl","2057129489994510620","https://pbs.twimg.com/media/HIxkWwDaMAAXH9U.jpg","killingmaster27","in","","0","2","0","https://x.com/undefined/status/2057129489994510620","1410466956134608899", diff --git a/tweets-data/hasil_latest.old.csv b/tweets-data/hasil_latest.old.csv index 60f198f..28257ee 100644 --- a/tweets-data/hasil_latest.old.csv +++ b/tweets-data/hasil_latest.old.csv @@ -1,2 +1,2 @@ "conversation_id_str","created_at","favorite_count","full_text","id_str","image_url","in_reply_to_screen_name","lang","location","quote_count","reply_count","retweet_count","tweet_url","user_id_str","username" -"2056538953340314095","Wed May 20 16:01:04 +0000 2026","5","@killingmaster27 @evelyndewi7 @katrinrienks @HumorJonTampan @wiyantokdr54 @EviDrajat @RickyKardjono @Micheladam69432 @ebonXboy @AntonRebor79105 @gangsterrandom @Pieter_Sun01 @Rahmah0845 @RSjahfidi @prabowo @SmileyNDeLight @Jediimar @Yens0906 @tokaidirecycle @PembuatSistem @bonapasogit24 @Jenniestale @Maskudin126003 @a8sar6alih @JowoNgalam @KING_RAJA_PWS_ @apaajadeh447870 @50ngoku @Submill @EmmaHusain0845 @BebySoSweet @Yulia_4ja @bungalotus25 @bahlillahadalia wahhh... jadi menteri komdigi dong menteri free ongkir https://t.co/1cfvoqIcNl","2057129489994510620","https://pbs.twimg.com/media/HIxkWwDaMAAXH9U.jpg","killingmaster27","in","","0","2","0","https://x.com/undefined/status/2057129489994510620","1410466956134608899", +"2056538953340314095","Wed May 20 16:01:04 +0000 2026","4","@killingmaster27 @evelyndewi7 @katrinrienks @HumorJonTampan @wiyantokdr54 @EviDrajat @RickyKardjono @Micheladam69432 @ebonXboy @AntonRebor79105 @gangsterrandom @Pieter_Sun01 @Rahmah0845 @RSjahfidi @prabowo @SmileyNDeLight @Jediimar @Yens0906 @tokaidirecycle @PembuatSistem @bonapasogit24 @Jenniestale @Maskudin126003 @a8sar6alih @JowoNgalam @KING_RAJA_PWS_ @apaajadeh447870 @50ngoku @Submill @EmmaHusain0845 @BebySoSweet @Yulia_4ja @bungalotus25 @bahlillahadalia wahhh... jadi menteri komdigi dong menteri free ongkir https://t.co/1cfvoqIcNl","2057129489994510620","https://pbs.twimg.com/media/HIxkWwDaMAAXH9U.jpg","killingmaster27","in","","0","2","0","https://x.com/undefined/status/2057129489994510620","1410466956134608899", diff --git a/tweets-data/hasil_top.csv b/tweets-data/hasil_top.csv deleted file mode 100644 index 0253d5c..0000000 --- a/tweets-data/hasil_top.csv +++ /dev/null @@ -1,6 +0,0 @@ -"conversation_id_str","created_at","favorite_count","full_text","id_str","image_url","in_reply_to_screen_name","lang","location","quote_count","reply_count","retweet_count","tweet_url","user_id_str","username" -"2056892677149581606","Wed May 20 00:20:04 +0000 2026","0","komdigi keren sih mengadakan kebijakan ongkir kaya gini","2056892677149581606","","","in","","0","0","0","https://x.com/undefined/status/2056892677149581606","1814143201718247424", -"2056949159845101648","Wed May 20 04:04:30 +0000 2026","0","komdigi kereenn dan bijak membuat keputusan ongkir seperti inii","2056949159845101648","","","in","","0","0","0","https://x.com/undefined/status/2056949159845101648","1814143201718247424", -"2056635322419777630","Tue May 19 07:17:25 +0000 2026","0","komdigi ongkir gk jelas","2056635322419777630","","","in","","0","0","0","https://x.com/undefined/status/2056635322419777630","1814143201718247424", -"2055269612334555468","Fri May 15 12:50:35 +0000 2026","0","komdigi + ongkir = chaos wkwkwk","2055269612334555468","","","in","","0","0","0","https://x.com/undefined/status/2055269612334555468","2043979880086339584", -"2056538953340314095","Wed May 20 16:01:04 +0000 2026","5","@killingmaster27 @evelyndewi7 @katrinrienks @HumorJonTampan @wiyantokdr54 @EviDrajat @RickyKardjono @Micheladam69432 @ebonXboy @AntonRebor79105 @gangsterrandom @Pieter_Sun01 @Rahmah0845 @RSjahfidi @prabowo @SmileyNDeLight @Jediimar @Yens0906 @tokaidirecycle @PembuatSistem @bonapasogit24 @Jenniestale @Maskudin126003 @a8sar6alih @JowoNgalam @KING_RAJA_PWS_ @apaajadeh447870 @50ngoku @Submill @EmmaHusain0845 @BebySoSweet @Yulia_4ja @bungalotus25 @bahlillahadalia wahhh... jadi menteri komdigi dong menteri free ongkir https://t.co/1cfvoqIcNl","2057129489994510620","https://pbs.twimg.com/media/HIxkWwDaMAAXH9U.jpg","killingmaster27","in","","0","2","0","https://x.com/undefined/status/2057129489994510620","1410466956134608899", diff --git a/tweets-data/hasil_top.old.csv b/tweets-data/hasil_top.old.csv index fea545b..37847d0 100644 --- a/tweets-data/hasil_top.old.csv +++ b/tweets-data/hasil_top.old.csv @@ -1,6 +1,14 @@ "conversation_id_str","created_at","favorite_count","full_text","id_str","image_url","in_reply_to_screen_name","lang","location","quote_count","reply_count","retweet_count","tweet_url","user_id_str","username" +"2057791279896080430","Fri May 22 11:50:47 +0000 2026","0","komdigi mengeluarkan kebijakan gratis ongkir @komdigi ???","2057791279896080430","","","in","","0","0","0","https://x.com/undefined/status/2057791279896080430","1814143201718247424", +"2057660616815571144","Fri May 22 03:11:35 +0000 2026","0","komdigi pembatasan gratis ongkir","2057660616815571144","","","in","","0","0","0","https://x.com/undefined/status/2057660616815571144","1814143201718247424", +"2056538953340314095","Wed May 20 16:01:04 +0000 2026","4","@killingmaster27 @evelyndewi7 @katrinrienks @HumorJonTampan @wiyantokdr54 @EviDrajat @RickyKardjono @Micheladam69432 @ebonXboy @AntonRebor79105 @gangsterrandom @Pieter_Sun01 @Rahmah0845 @RSjahfidi @prabowo @SmileyNDeLight @Jediimar @Yens0906 @tokaidirecycle @PembuatSistem @bonapasogit24 @Jenniestale @Maskudin126003 @a8sar6alih @JowoNgalam @KING_RAJA_PWS_ @apaajadeh447870 @50ngoku @Submill @EmmaHusain0845 @BebySoSweet @Yulia_4ja @bungalotus25 @bahlillahadalia wahhh... jadi menteri komdigi dong menteri free ongkir https://t.co/1cfvoqIcNl","2057129489994510620","https://pbs.twimg.com/media/HIxkWwDaMAAXH9U.jpg","killingmaster27","in","","0","2","0","https://x.com/undefined/status/2057129489994510620","1410466956134608899", "2056635322419777630","Tue May 19 07:17:25 +0000 2026","0","komdigi ongkir gk jelas","2056635322419777630","","","in","","0","0","0","https://x.com/undefined/status/2056635322419777630","1814143201718247424", -"2056892677149581606","Wed May 20 00:20:04 +0000 2026","0","komdigi keren sih mengadakan kebijakan ongkir kaya gini","2056892677149581606","","","in","","0","0","0","https://x.com/undefined/status/2056892677149581606","1814143201718247424", "2056949159845101648","Wed May 20 04:04:30 +0000 2026","0","komdigi kereenn dan bijak membuat keputusan ongkir seperti inii","2056949159845101648","","","in","","0","0","0","https://x.com/undefined/status/2056949159845101648","1814143201718247424", -"2056538953340314095","Wed May 20 16:01:04 +0000 2026","5","@killingmaster27 @evelyndewi7 @katrinrienks @HumorJonTampan @wiyantokdr54 @EviDrajat @RickyKardjono @Micheladam69432 @ebonXboy @AntonRebor79105 @gangsterrandom @Pieter_Sun01 @Rahmah0845 @RSjahfidi @prabowo @SmileyNDeLight @Jediimar @Yens0906 @tokaidirecycle @PembuatSistem @bonapasogit24 @Jenniestale @Maskudin126003 @a8sar6alih @JowoNgalam @KING_RAJA_PWS_ @apaajadeh447870 @50ngoku @Submill @EmmaHusain0845 @BebySoSweet @Yulia_4ja @bungalotus25 @bahlillahadalia wahhh... jadi menteri komdigi dong menteri free ongkir https://t.co/1cfvoqIcNl","2057129489994510620","https://pbs.twimg.com/media/HIxkWwDaMAAXH9U.jpg","killingmaster27","in","","0","2","0","https://x.com/undefined/status/2057129489994510620","1410466956134608899", -"2055269612334555468","Fri May 15 12:50:35 +0000 2026","0","komdigi + ongkir = chaos wkwkwk","2055269612334555468","","","in","","0","0","0","https://x.com/undefined/status/2055269612334555468","2043979880086339584", +"2056892677149581606","Wed May 20 00:20:04 +0000 2026","0","komdigi keren sih mengadakan kebijakan ongkir kaya gini","2056892677149581606","","","in","","0","0","0","https://x.com/undefined/status/2056892677149581606","1814143201718247424", +"conversation_id_str","created_at","favorite_count","full_text","id_str","image_url","in_reply_to_screen_name","lang","location","quote_count","reply_count","retweet_count","tweet_url","user_id_str","username" +"2057791279896080430","Fri May 22 11:50:47 +0000 2026","0","komdigi mengeluarkan kebijakan gratis ongkir @komdigi ???","2057791279896080430","","","in","","0","0","0","https://x.com/undefined/status/2057791279896080430","1814143201718247424", +"2057660616815571144","Fri May 22 03:11:35 +0000 2026","0","komdigi pembatasan gratis ongkir","2057660616815571144","","","in","","0","0","0","https://x.com/undefined/status/2057660616815571144","1814143201718247424", +"2056635322419777630","Tue May 19 07:17:25 +0000 2026","0","komdigi ongkir gk jelas","2056635322419777630","","","in","","0","0","0","https://x.com/undefined/status/2056635322419777630","1814143201718247424", +"2056538953340314095","Wed May 20 16:01:04 +0000 2026","4","@killingmaster27 @evelyndewi7 @katrinrienks @HumorJonTampan @wiyantokdr54 @EviDrajat @RickyKardjono @Micheladam69432 @ebonXboy @AntonRebor79105 @gangsterrandom @Pieter_Sun01 @Rahmah0845 @RSjahfidi @prabowo @SmileyNDeLight @Jediimar @Yens0906 @tokaidirecycle @PembuatSistem @bonapasogit24 @Jenniestale @Maskudin126003 @a8sar6alih @JowoNgalam @KING_RAJA_PWS_ @apaajadeh447870 @50ngoku @Submill @EmmaHusain0845 @BebySoSweet @Yulia_4ja @bungalotus25 @bahlillahadalia wahhh... jadi menteri komdigi dong menteri free ongkir https://t.co/1cfvoqIcNl","2057129489994510620","https://pbs.twimg.com/media/HIxkWwDaMAAXH9U.jpg","killingmaster27","in","","0","2","0","https://x.com/undefined/status/2057129489994510620","1410466956134608899", +"2056949159845101648","Wed May 20 04:04:30 +0000 2026","0","komdigi kereenn dan bijak membuat keputusan ongkir seperti inii","2056949159845101648","","","in","","0","0","0","https://x.com/undefined/status/2056949159845101648","1814143201718247424", +"2056892677149581606","Wed May 20 00:20:04 +0000 2026","0","komdigi keren sih mengadakan kebijakan ongkir kaya gini","2056892677149581606","","","in","","0","0","0","https://x.com/undefined/status/2056892677149581606","1814143201718247424",