revisi NRT

nrt jalan sisa preprocessing dan sentiment

app.py

@@ -26,6 +26,15 @@ except Exception:
 
 init_db()
 
+# ── Auto-start background crawler scheduler ──────────────────────────────────
+# Dipanggil sekali saat app pertama dijalankan.
+# Scheduler berjalan sebagai daemon thread — tidak perlu terminal terpisah.
+try:
+    from crawler import ensure_scheduler_running
+    ensure_scheduler_running()
+except Exception as _e:
+    print(f"[app] Gagal memulai scheduler: {_e}")
+
 st.set_page_config(
     page_title="Dashboard Sentimen Twitter",
     layout="wide",
@@ -46,10 +55,8 @@ def _get_query_param(name):
         value = st.query_params.get(name)
     except Exception:
         value = None
-
     if isinstance(value, list):
         return value[0] if value else None
-
     return value
 
 
@@ -65,13 +72,9 @@ def _get_context_offset():
         offset = st.context.timezone_offset
     except Exception:
         return None
-
     if offset is None:
         return None
-
     try:
-        # st.context.timezone_offset follows JavaScript getTimezoneOffset:
-        # UTC+8 is -480. The app mapping uses UTC offset minutes: UTC+8 is 480.
         return str(-int(offset))
     except (TypeError, ValueError):
         return None
@@ -79,27 +82,25 @@ def _get_context_offset():
 
 def _sync_timezone_from_browser():
     browser_timezone = _get_context_timezone() or _get_query_param("browser_tz")
-    browser_offset = _get_context_offset() or _get_query_param("browser_offset")
-    mapped_timezone = (
+    browser_offset   = _get_context_offset()   or _get_query_param("browser_offset")
+    mapped_timezone  = (
         browser_timezone_to_choice(browser_timezone)
         or browser_offset_to_choice(browser_offset)
     )
 
     st.session_state.browser_timezone = browser_timezone
-    st.session_state.browser_offset = browser_offset
+    st.session_state.browser_offset   = browser_offset
 
     if st.session_state.get("follow_device_timezone", True):
         timezone_choice = mapped_timezone or get_default_timezone()
-
         if st.session_state.user_timezone != timezone_choice:
             st.session_state.user_timezone = timezone_choice
 
 
 _sync_timezone_from_browser()
 
-# Determine refresh interval berdasarkan aktivitas crawler
+
 def get_refresh_interval():
-    """Refresh lebih cepat jika ada data baru dari crawler"""
     try:
         latest_crawl = get_latest_crawl_time()
         if latest_crawl:
@@ -107,16 +108,16 @@ def get_refresh_interval():
             latest_time = parse_dt_with_source_tz(
                 [latest_crawl],
                 timezone_choice,
-                os.getenv("APP_TIMEZONE", "Asia/Makassar")
+                os.getenv("APP_TIMEZONE", "Asia/Jakarta")
             ).iloc[0]
             now = pd.Timestamp.now(tz=get_timezone_name(timezone_choice)).tz_localize(None)
             time_diff = (now - latest_time).total_seconds()
-            # Jika data baru < 5 menit, refresh setiap 30 detik, jika tidak setiap 60 detik
             return 30000 if time_diff < 300 else 60000
     except Exception:
         pass
     return 60000
 
+
 refresh_interval = get_refresh_interval()
 
 st_autorefresh(
@@ -126,7 +127,7 @@ st_autorefresh(
 
 
 def _check_crawler_alive():
-    from datetime import datetime, timedelta, timezone
+    from datetime import timedelta
     from crawler import NRT_INTERVAL_MINUTES, get_crawler_state
 
     threshold = timedelta(minutes=max((NRT_INTERVAL_MINUTES * 2) + 1, 3))
@@ -134,7 +135,6 @@ def _check_crawler_alive():
     def parse_dt(value):
         if not value:
             return None
-
         try:
             parsed = datetime.fromisoformat(value)
             if parsed.tzinfo is None:
@@ -147,9 +147,9 @@ def _check_crawler_alive():
     if not state:
         return False
 
-    updated_at = parse_dt(state.get("updated_at"))
+    updated_at   = parse_dt(state.get("updated_at"))
     heartbeat_at = parse_dt(state.get("heartbeat_at"))
-    now = datetime.now(timezone.utc)
+    now          = datetime.now(timezone.utc)
 
     if (
         state.get("is_running", False)
@@ -191,10 +191,6 @@ st.markdown("""
     --card: #ffffff;
 }
 
-/* =========================
-   GLOBAL STYLING
-========================= */
-
 * {
     font-family: 'Plus Jakarta Sans', sans-serif !important;
     box-sizing: border-box;
@@ -233,26 +229,75 @@ html, body, .stApp, [data-testid="stAppViewContainer"] {
     padding: 1.5rem 2rem !important;
 }
 
-/* =========================
-   SIDEBAR
-========================= */
-
 [data-testid="stSidebar"] {
-    background: var(--card) !important;
-    box-shadow: 2px 0 8px rgba(15,23,42,0.08) !important;
+    background: linear-gradient(180deg, #ffffff 0%, #f8fafc 100%) !important;
+    border-right: 1px solid #e2e8f0 !important;
 }
 
 [data-testid="stSidebar"] > div:first-child {
-    padding: 1.5rem 1.25rem !important;
+    padding: 1.1rem 0.9rem !important;
 }
 
 [data-testid="stSidebar"] * {
     color: var(--text) !important;
 }
 
-/* =========================
-   BUTTONS & NAVIGATION
-========================= */
+.sidebar-section-title {
+    font-size: 0.63rem;
+    font-weight: 800;
+    color: #94a3b8;
+    letter-spacing: 0.08em;
+    text-transform: uppercase;
+    margin: 1rem 0 0.5rem;
+    padding-left: 0.35rem;
+}
+
+.sidebar-card {
+    background: rgba(255,255,255,0.8);
+    backdrop-filter: blur(10px);
+    border: 1px solid #e2e8f0;
+    border-radius: 14px;
+    padding: 0.85rem 0.95rem;
+    margin-bottom: 0.9rem;
+    box-shadow: 0 2px 10px rgba(15,23,42,0.04);
+}
+
+[data-testid="stSidebar"] .stButton > button {
+    height: 46px !important;
+    border-radius: 14px !important;
+    font-size: 0.87rem !important;
+    font-weight: 700 !important;
+    border: 1px solid transparent !important;
+    background: transparent !important;
+    color: #475569 !important;
+    transition: all 0.18s ease !important;
+    padding-left: 1rem !important;
+    width: 100% !important;
+}
+
+[data-testid="stSidebar"] .stButton > button:hover {
+    background: #eef2ff !important;
+    color: #3b6cf7 !important;
+    border-color: #c7d2fe !important;
+    transform: translateX(2px);
+}
+
+[data-testid="stSidebar"] .stButton > button[kind="primary"] {
+    background: linear-gradient(135deg,#3b6cf7,#5b7cfa) !important;
+    color: white !important;
+    border: none !important;
+    box-shadow: 0 6px 16px rgba(59,108,247,0.22) !important;
+}
+
+[data-testid="stSidebar"] .stButton > button[kind="primary"]:hover {
+    transform: translateX(2px);
+}
+
+[data-testid="stSidebar"] .stCaption {
+    font-size: 0.72rem !important;
+    line-height: 1.5 !important;
+    color: #64748b !important;
+}
 
 .stButton > button {
     background: var(--card) !important;
@@ -314,10 +359,6 @@ html, body, .stApp, [data-testid="stAppViewContainer"] {
     opacity: 1 !important;
 }
 
-/* =========================
-   HEADERS & TITLES
-========================= */
-
 .top-header {
     background: var(--card);
     padding: 1.5rem;
@@ -329,9 +370,7 @@ html, body, .stApp, [data-testid="stAppViewContainer"] {
     gap: 1rem;
 }
 
-.top-header * {
-    color: var(--text) !important;
-}
+.top-header * { color: var(--text) !important; }
 
 .page-title {
     font-size: 1.875rem !important;
@@ -341,13 +380,7 @@ html, body, .stApp, [data-testid="stAppViewContainer"] {
     line-height: 1.2;
 }
 
-h1, h2, h3, h4, h5, h6 {
-    color: var(--text) !important;
-}
-
-/* =========================
-   CARDS & CONTAINERS
-========================= */
+h1, h2, h3, h4, h5, h6 { color: var(--text) !important; }
 
 .card {
     background: var(--card);
@@ -379,10 +412,6 @@ h1, h2, h3, h4, h5, h6 {
     margin-top: 3px;
 }
 
-/* =========================
-   METRICS & PILLS
-========================= */
-
 [data-testid="stMetric"] {
     background: var(--card) !important;
     border: 1.5px solid var(--border) !important;
@@ -406,13 +435,7 @@ h1, h2, h3, h4, h5, h6 {
     font-size: 1.75rem !important;
 }
 
-/* =========================
-   TEXT & CONTENT
-========================= */
-
-.stMarkdown {
-    color: var(--text) !important;
-}
+.stMarkdown { color: var(--text) !important; }
 
 p:not([style]), li:not([style]) {
     color: var(--text-secondary) !important;
@@ -425,13 +448,8 @@ span:not([style]) {
 }
 
 .stMarkdown p:not([style]),
-.stMarkdown li:not([style]) {
-    color: var(--text-secondary) !important;
-}
-
-.stMarkdown span:not([style]) {
-    color: inherit !important;
-}
+.stMarkdown li:not([style]) { color: var(--text-secondary) !important; }
+.stMarkdown span:not([style]) { color: inherit !important; }
 
 .stButton > button p,
 .stButton > button span,
@@ -441,10 +459,6 @@ span:not([style]) {
     line-height: 1.25 !important;
 }
 
-/* =========================
-   EXPANDERS
-========================= */
-
 [data-testid="stExpander"] {
     background: var(--card) !important;
     border: 1.5px solid var(--border) !important;
@@ -453,9 +467,7 @@ span:not([style]) {
     overflow: hidden !important;
 }
 
-[data-testid="stExpander"] details {
-    border: 0 !important;
-}
+[data-testid="stExpander"] details { border: 0 !important; }
 
 [data-testid="stExpander"] summary {
     background: var(--card) !important;
@@ -463,37 +475,23 @@ span:not([style]) {
     padding: 0.75rem 1rem !important;
 }
 
-[data-testid="stExpander"] summary:hover {
-    background: #f8fafc !important;
-}
+[data-testid="stExpander"] summary:hover { background: #f8fafc !important; }
 
 [data-testid="stExpander"] summary *,
-[data-testid="stExpander"] [data-testid="stIconMaterial"] {
-    color: var(--text) !important;
-}
+[data-testid="stExpander"] [data-testid="stIconMaterial"] { color: var(--text) !important; }
 
 .stCaption, .stCaptionContainer, .stCaptionContainer * {
     color: var(--text-tertiary) !important;
     font-size: 0.8125rem !important;
 }
 
-/* =========================
-   DATA & TABLES
-========================= */
-
 [data-testid="stDataFrame"] {
     border-radius: 12px !important;
     border: 1px solid var(--border) !important;
     overflow: hidden !important;
 }
 
-[data-testid="stDataFrame"] * {
-    color: var(--text) !important;
-}
-
-/* =========================
-   ALERTS & WARNINGS
-========================= */
+[data-testid="stDataFrame"] * { color: var(--text) !important; }
 
 .stAlert {
     border-radius: 12px !important;
@@ -524,10 +522,6 @@ span:not([style]) {
     color: #0c4a6e !important;
 }
 
-/* =========================
-   FORM ELEMENTS
-========================= */
-
 .stTextInput input,
 .stDateInput input,
 .stSelectbox select,
@@ -563,9 +557,7 @@ span:not([style]) {
 
 [data-baseweb="select"] span,
 [data-baseweb="select"] div,
-[data-baseweb="input"] div {
-    color: var(--text) !important;
-}
+[data-baseweb="input"] div { color: var(--text) !important; }
 
 .stTextInput input:focus,
 .stSelectbox select:focus,
@@ -588,9 +580,7 @@ span:not([style]) {
 }
 
 [data-baseweb="menu"] *,
-[data-baseweb="calendar"] * {
-    color: var(--text) !important;
-}
+[data-baseweb="calendar"] * { color: var(--text) !important; }
 
 [data-baseweb="popover"] * {
     background-color: #ffffff !important;
@@ -675,10 +665,6 @@ span:not([style]) {
     -webkit-text-fill-color: #ffffff !important;
 }
 
-/* =========================
-   PILLS & CUSTOM ELEMENTS
-========================= */
-
 .pill-container {
     display: flex;
     gap: 1rem;
@@ -697,17 +683,9 @@ span:not([style]) {
     box-shadow: 0 2px 6px rgba(15,23,42,0.05);
 }
 
-/* =========================
-   LAYOUT FIX
-========================= */
+#MainMenu, footer, header { visibility: hidden !important; }
 
-#MainMenu, footer, header {
-    visibility: hidden !important;
-}
-
-[data-testid="stHeader"] {
-    background: transparent !important;
-}
+[data-testid="stHeader"] { background: transparent !important; }
 
 [data-testid="stDownloadButton"] button {
     background: var(--card) !important;
@@ -722,7 +700,6 @@ span:not([style]) {
     background: var(--blue-light) !important;
     border-color: var(--blue-primary) !important;
 }
-
 </style>
 """, unsafe_allow_html=True)
 
@@ -745,88 +722,109 @@ if (
 
 with st.sidebar:
 
+    # ── HEADER ──────────────────────────────────────────────────
     st.markdown("""
-    <div style="display:flex;align-items:center;gap:0.75rem;padding:0.5rem 0.5rem 1.5rem;">
-        <div style="width:38px;height:38px;background:linear-gradient(135deg,#3b6cf7,#6389f8);
-                    border-radius:10px;display:flex;align-items:center;justify-content:center;
-                    font-size:1.1rem;flex-shrink:0;color:white;">📊</div>
+    <div style="
+        display:flex;align-items:center;gap:0.85rem;
+        padding:0.4rem 0.4rem 1.2rem;
+    ">
+        <div style="
+            width:42px;height:42px;
+            background:linear-gradient(135deg,#3b6cf7,#6d8cff);
+            border-radius:14px;
+            display:flex;align-items:center;justify-content:center;
+            font-size:1.15rem;
+            box-shadow:0 8px 20px rgba(59,108,247,0.25);
+            flex-shrink:0;color:white;
+        ">📊</div>
         <div>
-            <div style="font-size:1rem;font-weight:800;color:#0f172a;line-height:1.2;">SentimenX</div>
-            <div style="font-size:0.7rem;color:#94a3b8;font-weight:500;">Twitter Analytics</div>
+            <div style="font-size:1.02rem;font-weight:800;color:#0f172a;line-height:1.1;">
+                SentimenX
+            </div>
+            <div style="font-size:0.72rem;color:#64748b;font-weight:600;margin-top:2px;">
+                Twitter Analytics Dashboard
+            </div>
         </div>
     </div>
     """, unsafe_allow_html=True)
 
-    sched_ok = st.session_state.get("_scheduler_started", False)
+    # ── STATUS CRAWLER ──────────────────────────────────────────
+    sched_ok       = st.session_state.get("_scheduler_started", False)
     sched_interval = st.session_state.get("_scheduler_interval", 0)
-
-    sched_color = "#16a34a" if sched_ok else "#f59e0b"
-
-    sched_label = (
+    sched_color    = "#16a34a" if sched_ok else "#3b6cf7"
+    sched_label    = (
         f"Aktif — diperbarui tiap {sched_interval} menit"
         if sched_ok
-        else "Nonaktif — jalankan: python3 crawler.py"
+        else "Standby — akan crawl otomatis setiap 5 menit"
     )
 
-    sched_dot = "●" if sched_ok else "○"
-
     st.markdown(f"""
-    <div style="background:#f8fafc;border:1.5px solid #e2e8f0;border-radius:10px;
-                padding:0.625rem 0.75rem;margin-bottom:1.25rem;">
-        <div style="display:flex;align-items:center;gap:0.5rem;margin-bottom:2px;">
-            <span style="color:{sched_color};font-size:0.9rem;">{sched_dot}</span>
-            <span style="font-size:0.75rem;font-weight:700;color:#334155;">Crawler Service</span>
+    <div class="sidebar-card">
+        <div style="display:flex;justify-content:space-between;align-items:center;">
+            <div>
+                <div style="font-size:0.72rem;font-weight:800;color:#0f172a;">
+                    Crawler Service
+                </div>
+                <div style="font-size:0.72rem;color:#64748b;margin-top:2px;">
+                    {sched_label}
+                </div>
+            </div>
+            <div style="
+                width:10px;height:10px;border-radius:50%;
+                background:{sched_color};
+                box-shadow:0 0 10px {sched_color};
+            "></div>
         </div>
-        <div style="font-size:0.7rem;color:#64748b;line-height:1.5;">{sched_label}</div>
     </div>
     """, unsafe_allow_html=True)
 
-    # Tampilkan waktu update terakhir data
+    # ── UPDATE TERBARU ──────────────────────────────────────────
     try:
         latest_crawl = get_latest_crawl_time()
         if latest_crawl:
             crawl_dt = parse_dt_with_source_tz(
                 [latest_crawl],
                 st.session_state.user_timezone,
-                os.getenv("APP_TIMEZONE", "Asia/Makassar")
+                os.getenv("APP_TIMEZONE", "Asia/Jakarta")
             ).iloc[0]
-            tz_label = get_timezone_label(st.session_state.user_timezone)
+
+            tz_label    = get_timezone_label(st.session_state.user_timezone)
             update_date = crawl_dt.strftime("%d/%m/%Y")
             update_time = f"{crawl_dt.strftime('%H:%M:%S')} {tz_label}"
+
             st.markdown(f"""
-            <div style="background:#eef2ff;border:1.5px solid #c7d2fe;border-radius:10px;
-                        padding:0.625rem 0.75rem;margin-bottom:1.25rem;">
-                <div style="font-size:0.7rem;font-weight:700;color:#1e3a8a;margin-bottom:2px;">
-                    🔄 Update Terbaru</div>
-                <div style="font-size:0.7rem;color:#1e40af;line-height:1.5;">
-                    {update_date}<br/>{update_time}</div>
+            <div class="sidebar-card">
+                <div style="font-size:0.72rem;font-weight:800;color:#0f172a;">
+                    Update Terbaru
+                </div>
+                <div style="margin-top:0.35rem;font-size:0.76rem;color:#475569;line-height:1.7;">
+                    {update_date}<br/>
+                    <span style="font-weight:700;color:#3b6cf7;">{update_time}</span>
+                </div>
             </div>
             """, unsafe_allow_html=True)
     except Exception:
         pass
 
-    st.markdown("""
-    <div style="font-size:0.65rem;font-weight:700;color:#94a3b8;
-                letter-spacing:0.08em;text-transform:uppercase;
-                padding:0 0.5rem;margin-bottom:0.5rem;">MENU UTAMA</div>
-    """, unsafe_allow_html=True)
+    # ── MENU UTAMA ──────────────────────────────────────────────
+    st.markdown('<div class="sidebar-section-title">MENU UTAMA</div>', unsafe_allow_html=True)
 
     current_page = st.session_state.get("page", "crawling")
 
     nav_items = [
-        ("crawling", "🔄  Ambil Data Twitter"),
-        ("preprocessing", "🧹  Bersihkan Data"),
-        ("sentiment", "📈  Analisis Sentimen"),
+        ("crawling",      "🔄 Ambil Data Twitter"),
+        ("preprocessing", "🧹 Bersihkan Data"),
+        ("sentiment",     "📈 Analisis Sentimen"),
     ]
 
     for page_key, label in nav_items:
         is_active = current_page == page_key
-
+        # Tidak memakai width="stretch" agar kompatibel dengan semua versi Streamlit
         if st.button(
             label,
-            width="stretch",
             type="primary" if is_active else "secondary",
-            key=f"nav_{page_key}"
+            key=f"nav_{page_key}",
+            use_container_width=True,
         ):
             st.session_state.page = page_key
             st.session_state.scroll_to_top = True
@@ -834,12 +832,8 @@ with st.sidebar:
 
     st.markdown("<br>", unsafe_allow_html=True)
 
-    # Timezone selector
-    st.markdown("""
-    <div style="font-size:0.65rem;font-weight:700;color:#94a3b8;
-                letter-spacing:0.08em;text-transform:uppercase;
-                padding:0 0.5rem;margin-bottom:0.5rem;">⏰ ZONA WAKTU</div>
-    """, unsafe_allow_html=True)
+    # ── TIMEZONE ────────────────────────────────────────────────
+    st.markdown('<div class="sidebar-section-title">⏰ ZONA WAKTU</div>', unsafe_allow_html=True)
 
     follow_device = st.toggle(
         "Ikuti timezone device",
@@ -848,128 +842,53 @@ with st.sidebar:
         help="Jika aktif, dashboard membaca zona waktu dari browser/device."
     )
 
-    if follow_device != st.session_state.follow_device_timezone:
-        st.session_state.follow_device_timezone = follow_device
-
-        if follow_device:
-            mapped_timezone = (
-                browser_timezone_to_choice(
-                    st.session_state.get("browser_timezone")
-                )
-                or browser_offset_to_choice(st.session_state.get("browser_offset"))
-            )
-            st.session_state.user_timezone = mapped_timezone or get_default_timezone()
-
-        st.rerun()
-
-    timezone_options = list(INDONESIA_TIMEZONES.keys())
-
-    if (
-        not st.session_state.follow_device_timezone
-        and st.session_state.user_timezone not in timezone_options
-    ):
-        st.session_state.user_timezone = get_default_timezone()
-
-    if st.session_state.follow_device_timezone:
-        raw_detected_tz = st.session_state.get("browser_timezone")
-        detected_offset = st.session_state.get("browser_offset")
-        detected_tz = raw_detected_tz or (
-            "Timezone dari offset browser" if detected_offset else "Belum terdeteksi"
-        )
-        mapped_timezone = (
-            browser_timezone_to_choice(raw_detected_tz)
-            or browser_offset_to_choice(detected_offset)
-        )
-        active_label = mapped_timezone or get_default_timezone()
-        st.session_state.user_timezone = active_label
-        try:
-            offset_minutes = int(detected_offset)
-            sign = "+" if offset_minutes >= 0 else "-"
-            abs_minutes = abs(offset_minutes)
-            offset_label = f"UTC{sign}{abs_minutes // 60:02d}:{abs_minutes % 60:02d}"
-        except (TypeError, ValueError):
-            offset_label = "offset belum terdeteksi"
-        st.caption(f"Device: {detected_tz} · {offset_label} · Aktif: {active_label}")
-    else:
-        selected_tz = st.selectbox(
-            "Pilih zona waktu Indonesia",
-            timezone_options,
-            index=timezone_options.index(st.session_state.user_timezone),
-            label_visibility="collapsed",
-            key="tz_selector"
-        )
-
-        if selected_tz != st.session_state.user_timezone:
-            st.session_state.user_timezone = selected_tz
-            st.rerun()
-
     active_tz_name = get_timezone_name(st.session_state.user_timezone)
-    active_now = pd.Timestamp.now(tz=active_tz_name)
-    active_clock = active_now.strftime("%d/%m/%Y %H:%M:%S")
-    st.markdown(
-        f"""
-        <div style="background:#fff;border:1.5px solid #e2e8f0;border-radius:10px;
-                    padding:0.625rem 0.75rem;margin:0.6rem 0 1.25rem;
-                    font-family:'Plus Jakarta Sans',sans-serif;">
-            <div style="font-size:0.7rem;font-weight:800;color:#334155;margin-bottom:2px;">
-                Jam Aktif
-            </div>
-            <div style="font-size:0.82rem;font-weight:800;color:#0f172a;line-height:1.5;">
-                {active_clock}
-            </div>
-        </div>
-        """,
-        unsafe_allow_html=True,
-    )
-
-    if (
-        st.session_state.follow_device_timezone
-        and (
-            browser_timezone_to_choice(st.session_state.get("browser_timezone"))
-            or browser_offset_to_choice(st.session_state.get("browser_offset"))
-        ) is None
-        and st.session_state.get("browser_timezone")
-    ):
-        st.caption("Timezone device belum valid dari browser, memakai default aplikasi.")
-
-    if not st.session_state.follow_device_timezone and selected_tz != st.session_state.user_timezone:
-        st.session_state.user_timezone = selected_tz
-        st.rerun()
-
-    st.markdown("<br>", unsafe_allow_html=True)
-    crawler_query = os.getenv(
-        "QUERY",
-        'komdigi (ongkir OR "gratis ongkir" OR "free ongkir") OR "pembatasan gratis ongkir" OR "gratis ongkir dibatasi"'
-    )
-
-    query_terms = [
-        term.strip().strip('"')
-        for term in crawler_query.split(" OR ")
-        if term.strip()
-    ]
-
-    query_html = "<br/>".join(
-        f"• {escape(term)}" for term in query_terms
-    ) or f"• {escape(crawler_query)}"
+    active_now     = pd.Timestamp.now(tz=active_tz_name)
+    active_clock   = active_now.strftime("%d/%m/%Y %H:%M:%S")
 
     st.markdown(f"""
-    <div style="background:#f0f4fb;border-radius:12px;padding:1rem;margin-top:auto;">
-        <div style="font-size:0.7rem;font-weight:700;color:#0f172a;text-transform:uppercase;
-                    letter-spacing:0.06em;margin-bottom:0.625rem;">🔍 Query Crawler Aktif</div>
-        <div style="font-size:0.78rem;color:#475569;line-height:1.9;">
+    <div class="sidebar-card">
+        <div style="font-size:0.72rem;font-weight:800;color:#0f172a;margin-bottom:0.35rem;">
+            Jam Aktif
+        </div>
+        <div style="font-size:0.88rem;font-weight:800;color:#3b6cf7;line-height:1.5;">
+            {active_clock}
+        </div>
+    </div>
+    """, unsafe_allow_html=True)
+
+    # ── QUERY ────────────────────────────────────────────────────
+    crawler_query = os.getenv(
+        "QUERY",
+        'komdigi (ongkir OR "gratis ongkir" OR "free ongkir")'
+    )
+    query_html = escape(crawler_query)
+
+    st.markdown(f"""
+    <div class="sidebar-card">
+        <div style="font-size:0.72rem;font-weight:800;color:#0f172a;margin-bottom:0.55rem;">
+            Query Aktif
+        </div>
+        <div style="font-size:0.74rem;color:#475569;line-height:1.8;word-break:break-word;">
             {query_html}
         </div>
     </div>
     """, unsafe_allow_html=True)
 
+    # ── FOOTER ───────────────────────────────────────────────────
     st.markdown("""
-    <div style="font-size:0.68rem;color:#94a3b8;text-align:center;margin-top:1.5rem;
-                padding-top:1rem;border-top:1px solid #e2e8f0;line-height:1.6;">
-        Data bersumber dari X (Twitter)<br/>via tweet-harvest
+    <div style="
+        font-size:0.68rem;color:#94a3b8;text-align:center;
+        margin-top:1.5rem;padding-top:1rem;
+        border-top:1px solid #e2e8f0;line-height:1.6;
+    ">
+        Data bersumber dari X (Twitter)<br/>
+        via tweet-harvest
     </div>
     """, unsafe_allow_html=True)
 
 
+# ── PAGE ROUTING ─────────────────────────────────────────────────────────────
 from page_modules import crawling_page, preprocessing_page, sentiment_page
 
 if st.session_state.page == "crawling":

crawler.py

@@ -1,5 +1,6 @@
 import json
 import os
+import threading
 import time
 from datetime import datetime, timezone
 
@@ -11,11 +12,21 @@ NRT_INTERVAL_MINUTES = 5
 LOG_FILE = "auto_crawl_log.json"
 STATE_FILE = "crawler_state.json"
 
+# ── Internal lock agar tidak ada dua crawl berjalan bersamaan ──
+_crawl_lock = threading.Lock()
+
+# ── Flag untuk scheduler background thread ──
+_scheduler_started = False
+_scheduler_lock = threading.Lock()
+
+
+# ═══════════════════════════════════════════════════════════
+#  STATE & LOG HELPERS
+# ═══════════════════════════════════════════════════════════
 
 def get_crawler_state():
     if not os.path.exists(STATE_FILE):
         return {}
-
     try:
         with open(STATE_FILE, "r") as f:
             return json.load(f)
@@ -35,16 +46,17 @@ def get_auto_crawl_logs(limit=10):
 
 def save_auto_crawl_log(status="success", total_saved=0, error=None):
     logs = get_auto_crawl_logs(50)
-
     logs.insert(0, {
         "timestamp": datetime.now(timezone.utc).isoformat(),
         "status": status,
         "total_saved": total_saved,
         "error": error,
     })
-
-    with open(LOG_FILE, "w") as f:
-        json.dump(logs[:50], f, indent=2)
+    try:
+        with open(LOG_FILE, "w") as f:
+            json.dump(logs[:50], f, indent=2)
+    except Exception as e:
+        print(f"[crawler] Gagal menulis log: {e}")
 
 
 def set_crawler_state(is_running=False, service_active=None):
@@ -58,7 +70,7 @@ def set_crawler_state(is_running=False, service_active=None):
         "is_running": bool(is_running),
         "service_active": bool(service_active),
         "updated_at": now,
-        "heartbeat_at": now if service_active else None,
+        "heartbeat_at": now if service_active else previous_state.get("heartbeat_at"),
         "last_job_started_at": previous_state.get("last_job_started_at"),
         "last_job_finished_at": previous_state.get("last_job_finished_at"),
     }
@@ -68,11 +80,27 @@ def set_crawler_state(is_running=False, service_active=None):
     elif previous_state.get("is_running"):
         state["last_job_finished_at"] = now
 
-    with open(STATE_FILE, "w") as f:
-        json.dump(state, f, indent=2)
+    try:
+        with open(STATE_FILE, "w") as f:
+            json.dump(state, f, indent=2)
+    except Exception as e:
+        print(f"[crawler] Gagal menulis state: {e}")
 
 
+# ═══════════════════════════════════════════════════════════
+#  CORE JOB
+# ═══════════════════════════════════════════════════════════
+
 def auto_crawl_job():
+    """
+    Jalankan satu siklus crawling.
+    Thread-safe: hanya satu crawl yang boleh berjalan pada satu waktu.
+    Bisa dipanggil dari tombol UI maupun scheduler otomatis.
+    """
+    if not _crawl_lock.acquire(blocking=False):
+        print("[crawler] Crawl sedang berjalan, skip.")
+        return 0
+
     print("=" * 50)
     print("CRAWLING MULAI:", datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
     print("=" * 50)
@@ -80,55 +108,95 @@ def auto_crawl_job():
     service_active = get_crawler_state().get("service_active", False)
     set_crawler_state(True, service_active=service_active)
 
+    total = 0
     try:
-        total = scrape_once(
-            limit=int(os.getenv("SCRAPE_LIMIT", "50"))
-        )
-
+        total = scrape_once(limit=int(os.getenv("SCRAPE_LIMIT", "50")))
         if total is None:
             total = 0
 
-        save_auto_crawl_log(
-            status="success",
-            total_saved=total,
-            error=None
-        )
-
-        print(f"Selesai. {total} tweet baru tersimpan.")
+        save_auto_crawl_log(status="success", total_saved=total, error=None)
+        print(f"[crawler] Selesai. {total} tweet baru tersimpan.")
 
     except Exception as e:
-        save_auto_crawl_log(
-            status="error",
-            total_saved=0,
-            error=str(e)
-        )
-
-        print("Error crawler:", e)
+        save_auto_crawl_log(status="error", total_saved=0, error=str(e))
+        print(f"[crawler] Error: {e}")
 
     finally:
         set_crawler_state(False, service_active=service_active)
+        _crawl_lock.release()
 
+    return total
+
+
+# ═══════════════════════════════════════════════════════════
+#  BACKGROUND SCHEDULER (dipakai Streamlit, tanpa terminal)
+# ═══════════════════════════════════════════════════════════
+
+def _scheduler_loop():
+    """
+    Loop yang berjalan di daemon thread.
+    Crawling pertama langsung dijalankan, lalu setiap NRT_INTERVAL_MINUTES menit.
+    """
+    print(f"[crawler] Scheduler dimulai — interval {NRT_INTERVAL_MINUTES} menit.")
+    set_crawler_state(False, service_active=True)
+
+    while True:
+        auto_crawl_job()
+
+        # Tulis heartbeat setiap 30 detik selagi menunggu interval berikutnya
+        wait_until = time.time() + (NRT_INTERVAL_MINUTES * 60)
+        while time.time() < wait_until:
+            # Heartbeat agar is_crawler_service_active() tetap True
+            _write_heartbeat()
+            time.sleep(min(30, max(1, wait_until - time.time())))
+
+        print(f"[crawler] Interval berikutnya dalam {NRT_INTERVAL_MINUTES} menit...")
+
+
+def _write_heartbeat():
+    """Perbarui heartbeat_at tanpa mengubah field lain."""
+    state = get_crawler_state()
+    state["heartbeat_at"] = datetime.now(timezone.utc).isoformat()
+    state["service_active"] = True
+    try:
+        with open(STATE_FILE, "w") as f:
+            json.dump(state, f, indent=2)
+    except Exception:
+        pass
+
+
+def ensure_scheduler_running():
+    """
+    Pastikan background scheduler sudah berjalan.
+    Aman dipanggil berkali-kali (idempotent).
+    Dipanggil dari crawling_page.py saat halaman dirender.
+    """
+    global _scheduler_started
+
+    with _scheduler_lock:
+        if _scheduler_started:
+            return
+
+        t = threading.Thread(target=_scheduler_loop, name="CrawlerScheduler", daemon=True)
+        t.start()
+        _scheduler_started = True
+        print("[crawler] Background scheduler thread dimulai.")
+
+
+# ═══════════════════════════════════════════════════════════
+#  ENTRY POINT (tetap bisa dipakai via terminal jika mau)
+# ═══════════════════════════════════════════════════════════
 
 def main():
+    """Jalankan scheduler via terminal (opsional, tidak wajib)."""
     print("Crawler realtime aktif.")
     print(f"Interval: {NRT_INTERVAL_MINUTES} menit")
-
     set_crawler_state(False, service_active=True)
 
     try:
-        while True:
-            auto_crawl_job()
-            print(f"Menunggu {NRT_INTERVAL_MINUTES} menit...")
-
-            wait_until = time.time() + (NRT_INTERVAL_MINUTES * 60)
-
-            while time.time() < wait_until:
-                set_crawler_state(False, service_active=True)
-                time.sleep(min(30, max(1, wait_until - time.time())))
-
+        _scheduler_loop()
     except KeyboardInterrupt:
         print("Crawler dihentikan.")
-
     finally:
         set_crawler_state(False, service_active=False)
 

database.py

@@ -1,97 +1,100 @@
 import os
-import sqlite3
+import pymysql
 import pandas as pd
 from dotenv import load_dotenv
-from sqlalchemy import create_engine
+from sqlalchemy import create_engine, text
 
 load_dotenv()
 
-DB_PATH = os.getenv("DB_PATH", "data_sentimen_komdigi.db")
+DB_PATH = os.getenv("DB_PATH", "mysql+pymysql://root:@localhost:3306/sentiment_komdigi")
 
-engine = create_engine(f"sqlite:///{DB_PATH}", echo=False)
+# Daftarkan dialect pymysql agar SQLAlchemy bisa menggunakannya
+pymysql.install_as_MySQLdb()
+
+engine = create_engine(
+    DB_PATH,
+    echo=False,
+    pool_recycle=3600,       # Reconnect otomatis setiap 1 jam (cegah timeout)
+    pool_pre_ping=True,      # Cek koneksi sebelum pakai (penting untuk realtime)
+)
 
 
 def init_db():
-    conn = sqlite3.connect(DB_PATH)
-    cursor = conn.cursor()
-
-    cursor.execute("""
-        CREATE TABLE IF NOT EXISTS tweets (
-            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            tweet_id TEXT UNIQUE,
-            text TEXT,
-            created_at TEXT,
-            crawled_at TEXT,
-            crawl_type TEXT DEFAULT 'realtime'
-        )
-    """)
-
-    conn.commit()
-    conn.close()
+    """Buat tabel tweets jika belum ada (kompatibel MySQL)."""
+    with engine.connect() as conn:
+        conn.execute(text("""
+            CREATE TABLE IF NOT EXISTS tweets (
+                id         INT AUTO_INCREMENT PRIMARY KEY,
+                tweet_id   VARCHAR(255) UNIQUE,
+                text       LONGTEXT,
+                created_at VARCHAR(50),
+                crawled_at VARCHAR(50),
+                crawl_type VARCHAR(50) DEFAULT 'realtime'
+            ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
+        """))
+        conn.commit()
 
 
 def save_tweets(data, crawl_type="realtime"):
+    """Simpan daftar tweet ke MySQL. Lewati duplikat berdasarkan tweet_id."""
     init_db()
 
-    conn = sqlite3.connect(DB_PATH)
-    cursor = conn.cursor()
-
     saved = 0
 
-    for tweet in data:
-        try:
-            cursor.execute("""
-                INSERT OR IGNORE INTO tweets
-                (tweet_id, text, created_at, crawled_at, crawl_type)
-                VALUES (?, ?, ?, ?, ?)
-            """, (
-                str(tweet.get("tweet_id", "")),
-                str(tweet.get("text", "")),
-                str(tweet.get("created_at", "")),
-                str(tweet.get("crawled_at", "")),
-                crawl_type,
-            ))
+    with engine.connect() as conn:
+        for tweet in data:
+            try:
+                result = conn.execute(
+                    text("""
+                        INSERT IGNORE INTO tweets
+                            (tweet_id, text, created_at, crawled_at, crawl_type)
+                        VALUES
+                            (:tweet_id, :text, :created_at, :crawled_at, :crawl_type)
+                    """),
+                    {
+                        "tweet_id":   str(tweet.get("tweet_id", "")),
+                        "text":       str(tweet.get("text", "")),
+                        "created_at": str(tweet.get("created_at", "")),
+                        "crawled_at": str(tweet.get("crawled_at", "")),
+                        "crawl_type": crawl_type,
+                    }
+                )
+                if result.rowcount > 0:
+                    saved += 1
 
-            if cursor.rowcount > 0:
-                saved += 1
+            except Exception as e:
+                print("Gagal simpan tweet:", e)
 
-        except Exception as e:
-            print("Gagal simpan tweet:", e)
-
-    conn.commit()
-    conn.close()
+        conn.commit()
 
     return saved
 
 
 def load_tweets():
+    """Baca semua tweet dari MySQL, urut dari yang terbaru."""
     init_db()
 
-    conn = sqlite3.connect(DB_PATH)
-
-    df = pd.read_sql_query("""
-        SELECT *
-        FROM tweets
-        ORDER BY created_at DESC
-    """, conn)
-
-    conn.close()
+    df = pd.read_sql_query(
+        "SELECT * FROM tweets ORDER BY created_at DESC",
+        engine
+    )
 
     return df
 
 
 def insert_tweets(df):
+    """Masukkan DataFrame ke database (dipakai saat import data historis CSV)."""
     data = []
 
     for _, row in df.iterrows():
         full_text = row.get("full_text", row.get("text", ""))
 
         data.append({
-            "tweet_id": row.get(
+            "tweet_id":   row.get(
                 "tweet_id",
                 f"{row.get('username', '')}_{hash(full_text)}"
             ),
-            "text": full_text,
+            "text":       full_text,
             "created_at": row.get("created_at", ""),
             "crawled_at": row.get("crawled_at", ""),
         })
@@ -100,58 +103,57 @@ def insert_tweets(df):
 
 
 def get_tweet_count():
-    """Dapatkan jumlah tweet terbaru dari database untuk cache invalidation"""
+    """Dapatkan jumlah tweet untuk cache invalidation."""
     init_db()
-    conn = sqlite3.connect(DB_PATH)
-    cursor = conn.cursor()
-    cursor.execute("SELECT COUNT(*) FROM tweets")
-    count = cursor.fetchone()[0]
-    conn.close()
-    return count
+
+    with engine.connect() as conn:
+        result = conn.execute(text("SELECT COUNT(*) FROM tweets"))
+        return result.scalar()
 
 
 def get_existing_tweet_ids(tweet_ids):
-    """Ambil tweet_id yang sudah ada agar crawler bisa melewati duplikat."""
-    ids = [str(tweet_id) for tweet_id in tweet_ids if str(tweet_id).strip()]
+    """Kembalikan set tweet_id yang sudah ada agar crawler bisa lewati duplikat."""
+    ids = [str(tid) for tid in tweet_ids if str(tid).strip()]
 
     if not ids:
         return set()
 
     init_db()
-    conn = sqlite3.connect(DB_PATH)
-    cursor = conn.cursor()
     existing = set()
 
-    for start in range(0, len(ids), 500):
-        chunk = ids[start:start + 500]
-        placeholders = ",".join(["?"] * len(chunk))
+    with engine.connect() as conn:
+        for start in range(0, len(ids), 500):
+            chunk = ids[start:start + 500]
+            placeholders = ", ".join([f":id{i}" for i in range(len(chunk))])
+            params = {f"id{i}": v for i, v in enumerate(chunk)}
 
-        cursor.execute(
-            f"SELECT tweet_id FROM tweets WHERE tweet_id IN ({placeholders})",
-            chunk
-        )
+            rows = conn.execute(
+                text(f"SELECT tweet_id FROM tweets WHERE tweet_id IN ({placeholders})"),
+                params
+            ).fetchall()
 
-        existing.update(str(row[0]) for row in cursor.fetchall())
+            existing.update(str(row[0]) for row in rows)
 
-    conn.close()
     return existing
 
 
 def get_latest_crawl_time():
-    """Dapatkan waktu crawl terakhir untuk mendeteksi data baru"""
+    """Dapatkan waktu crawl terakhir untuk deteksi data baru."""
     init_db()
-    conn = sqlite3.connect(DB_PATH)
-    df = pd.read_sql_query("""
-        SELECT MAX(crawled_at) as latest_crawl
-        FROM tweets
-    """, conn)
-    conn.close()
 
-    if df.empty or df['latest_crawl'].isna().all():
+    with engine.connect() as conn:
+        result = conn.execute(
+            text("SELECT MAX(crawled_at) AS latest_crawl FROM tweets")
+        )
+        row = result.fetchone()
+
+    if row is None or row[0] is None:
         return None
-    return df['latest_crawl'].iloc[0]
+
+    return row[0]
 
 
 if __name__ == "__main__":
     init_db()
-    print("Database berhasil dibuat.")
+    print("Database MySQL berhasil diinisialisasi.")
+    print("Total tweet:", get_tweet_count())

scraper_worker.py

@@ -21,15 +21,12 @@ QUERY = os.getenv(
     'komdigi (ongkir OR "gratis ongkir" OR "free ongkir") OR "pembatasan gratis ongkir" OR "gratis ongkir dibatasi"'
 )
 
-SCRAPE_LIMIT = int(
-    os.getenv("SCRAPE_LIMIT", "50")
-)
+SCRAPE_LIMIT = int(os.getenv("SCRAPE_LIMIT", "50"))
 
-RECENT_DAYS = int(
-    os.getenv("RECENT_DAYS", "2")
-)
+# Default 7 hari — sesuai .env RECENT_DAYS=7
+RECENT_DAYS = int(os.getenv("RECENT_DAYS", "7"))
 
-APP_TIMEZONE = os.getenv("APP_TIMEZONE", "Asia/Makassar")
+APP_TIMEZONE = os.getenv("APP_TIMEZONE", "Asia/Jakarta")
 
 SCRAPE_TABS = [
     tab.strip().upper()
@@ -40,39 +37,32 @@ SCRAPE_TABS = [
 
 def get_recent_window():
     today = datetime.now(ZoneInfo(APP_TIMEZONE)).date()
+    # Mundur (RECENT_DAYS - 1) hari agar jangkauan = 7 hari penuh
     since_day = today - timedelta(days=max(RECENT_DAYS - 1, 0))
     until_day = today + timedelta(days=1)
-
     return since_day, until_day
 
 
 def build_recent_query():
     query_lower = f" {QUERY.lower()}"
-
     if " since:" in query_lower or " until:" in query_lower:
         return QUERY
 
     since_day, until_day = get_recent_window()
-
     return f"({QUERY}) since:{since_day.isoformat()} until:{until_day.isoformat()}"
 
 
 def normalize_tweet_date(value):
     parsed = pd.to_datetime(value, errors="coerce", utc=True)
-
     if pd.isna(parsed):
-        # Selalu gunakan UTC timezone untuk konsistensi
         return datetime.now(timezone.utc).isoformat()
-
     return parsed.isoformat()
 
 
 def is_in_recent_window(value):
     parsed = pd.to_datetime(value, errors="coerce", utc=True)
-
     if pd.isna(parsed):
         return False
-
     try:
         local_dt = parsed.tz_convert(APP_TIMEZONE)
     except Exception:
@@ -80,26 +70,20 @@ def is_in_recent_window(value):
 
     since_day, until_day = get_recent_window()
     tweet_day = local_dt.date()
-
     return since_day <= tweet_day < until_day
 
 
 def stable_fallback_id(row, full_text):
     username = str(row.get("username", "") or "unknown").strip() or "unknown"
-    digest = hashlib.sha1(
-        full_text.encode("utf-8", errors="ignore")
-    ).hexdigest()[:16]
-
+    digest = hashlib.sha1(full_text.encode("utf-8", errors="ignore")).hexdigest()[:16]
     return f"{username}_{digest}"
 
 
 def extract_tweet_id(row, full_text):
     for column in ("id_str", "tweet_id", "id"):
         value = row.get(column)
-
         if pd.notna(value) and str(value).strip():
             return str(value).strip()
-
     return stable_fallback_id(row, full_text)
 
 
@@ -107,16 +91,10 @@ def is_header_like_row(row, tweet_id, full_text):
     text = str(full_text or "").strip().lower()
     tweet_id_text = str(tweet_id or "").strip().lower()
     created_at = str(row.get("created_at", "") or "").strip().lower()
-
     header_values = {
-        "full_text",
-        "text",
-        "id_str",
-        "tweet_id",
-        "created_at",
-        "conversation_id_str",
+        "full_text", "text", "id_str", "tweet_id",
+        "created_at", "conversation_id_str",
     }
-
     return (
         text in header_values
         or tweet_id_text in header_values
@@ -129,10 +107,7 @@ def ambil_file_csv_terbaru(min_mtime=None, output_name=None):
         output_path = os.path.join("tweets-data", output_name)
         root, ext = os.path.splitext(output_path)
         files = [
-            path for path in (
-                output_path,
-                f"{root}.old{ext}",
-            )
+            path for path in (output_path, f"{root}.old{ext}")
             if os.path.exists(path)
         ]
     else:
@@ -142,26 +117,16 @@ def ambil_file_csv_terbaru(min_mtime=None, output_name=None):
         return None
 
     if min_mtime is not None:
-        files = [
-            path for path in files
-            if os.path.getmtime(path) >= min_mtime
-        ]
+        files = [p for p in files if os.path.getmtime(p) >= min_mtime]
 
     if not files:
         return None
 
-    non_empty_files = [
-        path for path in files
-        if os.path.getsize(path) > 2
-    ]
+    non_empty = [p for p in files if os.path.getsize(p) > 2]
+    if non_empty:
+        files = non_empty
 
-    if non_empty_files:
-        files = non_empty_files
-
-    return max(
-        files,
-        key=os.path.getmtime
-    )
+    return max(files, key=os.path.getmtime)
 
 
 def scrape_tab(tab, effective_query, limit):
@@ -169,69 +134,58 @@ def scrape_tab(tab, effective_query, limit):
     started_at = time.time() - 1
 
     command = [
-        "npx",
-        "--yes",
-        "tweet-harvest@2.6.1",
-        "-o",
-        output_name,
-        "-s",
-        effective_query,
-        "--tab",
-        tab,
-        "-l",
-        str(limit),
-        "--token",
-        AUTH_TOKEN,
+        "npx", "--yes", "tweet-harvest@2.6.1",
+        "-o", output_name,
+        "-s", effective_query,
+        "--tab", tab,
+        "-l", str(limit),
+        "--token", AUTH_TOKEN,
     ]
 
-    print(f"Scraping tab {tab}...")
+    print(f"[scraper] Scraping tab {tab}...")
     try:
-        result = subprocess.run(command, capture_output=True, text=True, timeout=180)
+        result = subprocess.run(
+            command, capture_output=True, text=True, timeout=180, shell=True
+        )
     except subprocess.TimeoutExpired:
-        print(f"tweet-harvest tab {tab} timed out after 180 seconds")
         raise RuntimeError(
-            f"tweet-harvest tab {tab} timed out (likely no results atau network issue)"
+            f"tweet-harvest tab {tab} timeout (180 detik) — mungkin tidak ada hasil atau jaringan bermasalah"
         )
 
     if result.returncode != 0:
         error_msg = result.stderr if result.stderr else result.stdout
-        print(f"tweet-harvest stderr: {error_msg}")
         raise RuntimeError(
-            f"tweet-harvest tab {tab} gagal dengan exit code {result.returncode}: {error_msg}"
+            f"tweet-harvest tab {tab} gagal (exit {result.returncode}): {error_msg}"
         )
 
     latest_file = ambil_file_csv_terbaru(started_at, output_name)
-
     if latest_file is None:
-        print(f"CSV baru untuk tab {tab} tidak ditemukan")
+        print(f"[scraper] CSV baru tab {tab} tidak ditemukan")
         return pd.DataFrame()
 
-    # Cek ukuran file sebelum membaca
     try:
         file_size = os.path.getsize(latest_file)
         if file_size <= 2:
-            print(f"CSV tab {tab} kosong atau tidak valid (ukuran: {file_size} bytes)")
+            print(f"[scraper] CSV tab {tab} kosong ({file_size} bytes)")
             return pd.DataFrame()
     except Exception as e:
-        print(f"Gagal memeriksa ukuran file {latest_file}: {e}")
+        print(f"[scraper] Gagal cek ukuran file: {e}")
         return pd.DataFrame()
 
     try:
         df = pd.read_csv(latest_file)
-    except EmptyDataError as e:
-        print(f"CSV tab {tab} kosong atau tidak valid: {e}")
+    except EmptyDataError:
+        print(f"[scraper] CSV tab {tab} kosong")
         return pd.DataFrame()
     except Exception as e:
-        print(f"Gagal membaca CSV tab {tab}: {e}")
+        print(f"[scraper] Gagal baca CSV tab {tab}: {e}")
         return pd.DataFrame()
 
     if df.empty:
-        print(f"Data tab {tab} kosong")
         return pd.DataFrame()
 
     if "full_text" not in df.columns:
-        print(f"Kolom full_text tidak ditemukan pada tab {tab}")
-        print(df.columns.tolist())
+        print(f"[scraper] Kolom full_text tidak ada pada tab {tab}: {df.columns.tolist()}")
         return pd.DataFrame()
 
     df["_source_tab"] = tab
@@ -242,16 +196,15 @@ def scrape_once(limit=SCRAPE_LIMIT):
     init_db()
 
     if not AUTH_TOKEN:
-        raise ValueError(
-            "TWITTER_AUTH_TOKEN belum diisi di file .env"
-        )
+        raise ValueError("TWITTER_AUTH_TOKEN belum diisi di .env")
 
     effective_query = build_recent_query()
     tabs = SCRAPE_TABS or ["LATEST"]
 
-    print("Scraping tweet realtime...")
-    print("Query:", effective_query)
-    print("Tab:", ", ".join(tabs))
+    print("[scraper] Mulai scraping tweet...")
+    print(f"[scraper] Query : {effective_query}")
+    print(f"[scraper] Tab   : {', '.join(tabs)}")
+    print(f"[scraper] Window: {RECENT_DAYS} hari terakhir")
 
     frames = []
     errors = []
@@ -261,23 +214,21 @@ def scrape_once(limit=SCRAPE_LIMIT):
             tab_df = scrape_tab(tab, effective_query, limit)
         except Exception as e:
             errors.append(f"{tab}: {e}")
-            print(f"Tab {tab} gagal:", e)
+            print(f"[scraper] Tab {tab} gagal: {e}")
             continue
 
         if not tab_df.empty:
             frames.append(tab_df)
 
     if not frames:
-        # Jika tidak ada data baru dari scraper, kembalikan 0 daripada error
-        # Ini normal jika tidak ada tweet baru yang sesuai kriteria
         if errors:
-            print("Warning - Scrape failed with errors:", " | ".join(errors))
+            print("[scraper] Warning:", " | ".join(errors))
         else:
-            print("No new data found in twitter search")
+            print("[scraper] Tidak ada tweet baru yang ditemukan")
         return 0
 
     df = pd.concat(frames, ignore_index=True)
-    print(f"Total hasil mentah dari crawler: {len(df)} tweet")
+    print(f"[scraper] Total hasil mentah: {len(df)} tweet")
 
     df["_full_text_clean"] = df["full_text"].fillna("").astype(str)
     df = df[df["_full_text_clean"].str.strip().ne("")].copy()
@@ -308,55 +259,36 @@ def scrape_once(limit=SCRAPE_LIMIT):
         })
 
     if outside_window:
-        print(
-            f"{outside_window} tweet dilewati karena di luar rentang realtime"
-        )
+        print(f"[scraper] {outside_window} tweet dilewati (di luar {RECENT_DAYS} hari terakhir)")
 
+    # Deduplikasi antar tab
     before_dedupe = len(data)
     deduped = {}
-
     for tweet in data:
         deduped.setdefault(tweet["tweet_id"], tweet)
-
     data = list(deduped.values())
-    duplicate_between_tabs = before_dedupe - len(data)
 
-    if duplicate_between_tabs:
-        print(
-            f"{duplicate_between_tabs} tweet duplikat dari LATEST/TOP dilewati"
-        )
+    dup_count = before_dedupe - len(data)
+    if dup_count:
+        print(f"[scraper] {dup_count} tweet duplikat antar tab dilewati")
 
     if not data:
-        print("Tidak ada tweet hari ini yang bisa disimpan")
+        print("[scraper] Tidak ada tweet dalam window yang bisa disimpan")
         return 0
 
-    existing_ids = get_existing_tweet_ids(
-        [tweet["tweet_id"] for tweet in data]
-    )
-
-    new_data = [
-        tweet for tweet in data
-        if tweet["tweet_id"] not in existing_ids
-    ]
-
+    existing_ids = get_existing_tweet_ids([t["tweet_id"] for t in data])
+    new_data = [t for t in data if t["tweet_id"] not in existing_ids]
     skipped = len(data) - len(new_data)
 
     if skipped:
-        print(f"{skipped} tweet dilewati karena sudah ada di database")
+        print(f"[scraper] {skipped} tweet sudah ada di database, dilewati")
 
     if not new_data:
-        print("Tidak ada tweet baru untuk disimpan")
+        print("[scraper] Tidak ada tweet baru untuk disimpan")
         return 0
 
-    saved = save_tweets(
-        new_data,
-        "realtime"
-    )
-
-    print(
-        f"{saved} tweet baru berhasil disimpan"
-    )
-
+    saved = save_tweets(new_data, "realtime")
+    print(f"[scraper] {saved} tweet baru berhasil disimpan")
     return saved
 
 

tweets-data/hasil_latest.csv

@@ -1 +0,0 @@
-

tweets-data/hasil_latest.old.csv

@@ -1 +1,4 @@
 
+
+
+

tweets-data/hasil_top.old.csv

@@ -1,8 +1,42 @@
 "conversation_id_str","created_at","favorite_count","full_text","id_str","image_url","in_reply_to_screen_name","lang","location","quote_count","reply_count","retweet_count","tweet_url","user_id_str","username"
-"2053330583707914695","Sun May 10 04:25:34 +0000 2026","0","komdigi gaada kerjaan ngurus ongkir","2053330583707914695","","","in","","0","0","0","https://x.com/undefined/status/2053330583707914695","1916470659917758464",
-"2053330483665375476","Sun May 10 04:25:10 +0000 2026","0","komdigi ngurusin gratis ongkir doang","2053330483665375476","","","in","","0","0","0","https://x.com/undefined/status/2053330483665375476","1916470659917758464",
-"2053067823627518260","Sat May 09 11:01:27 +0000 2026","0","komdigi gratis ongkir ya","2053067823627518260","","","in","","0","0","0","https://x.com/undefined/status/2053067823627518260","1916470659917758464",
-"2052921293289496669","Sat May 09 01:19:12 +0000 2026","0","komdigi gratis ongkir","2052921293289496669","","","in","","0","0","0","https://x.com/undefined/status/2052921293289496669","1916470659917758464",
-"2052921254785786307","Sat May 09 01:19:03 +0000 2026","0","komdigi ongkir","2052921254785786307","","","in","","0","0","0","https://x.com/undefined/status/2052921254785786307","1916470659917758464",
-"2053046421184827846","Sat May 09 09:36:25 +0000 2026","0","komdigi gratis ongkir ga jelas","2053046421184827846","","","in","","0","0","0","https://x.com/undefined/status/2053046421184827846","1916470659917758464",
-"2053046378579050583","Sat May 09 09:36:15 +0000 2026","0","komdigi gratis ongkir tai","2053046378579050583","","","in","","0","0","0","https://x.com/undefined/status/2053046378579050583","1916470659917758464",
+"2054791169775964388","Thu May 14 05:09:25 +0000 2026","0","kebijakan komdigi gratis ongkir kerenn","2054791169775964388","","","in","","0","0","0","https://x.com/undefined/status/2054791169775964388","1814143201718247424",
+"2055002136027767200","Thu May 14 19:07:43 +0000 2026","0","komdigi ongkir apa gunanyaa??222","2055002136027767200","","","in","","0","0","0","https://x.com/undefined/status/2055002136027767200","2043979880086339584",
+"2054861268750901484","Thu May 14 09:47:58 +0000 2026","0","komdigi ongkir?","2054861268750901484","","","in","","0","0","0","https://x.com/undefined/status/2054861268750901484","1814143201718247424",
+"2054816992662790146","Thu May 14 06:52:02 +0000 2026","0","komdigi??? ongkir??? marketplace???","2054816992662790146","","","in","","0","0","0","https://x.com/undefined/status/2054816992662790146","1814143201718247424",
+"2054823109291852155","Thu May 14 07:16:20 +0000 2026","0","komdigi bikin aturan ongkir baru = langkah yang cukup bijak","2054823109291852155","","","in","","0","0","0","https://x.com/undefined/status/2054823109291852155","1814143201718247424",
+"2055269612334555468","Fri May 15 12:50:35 +0000 2026","0","komdigi + ongkir = chaos wkwkwk","2055269612334555468","","","in","","0","0","0","https://x.com/undefined/status/2055269612334555468","2043979880086339584",
+"2054998005359751518","Thu May 14 18:51:19 +0000 2026","0","komdigi ongkir apa gunanyaa??","2054998005359751518","","","in","","0","0","0","https://x.com/undefined/status/2054998005359751518","2043979880086339584",
+"2054816818741862819","Thu May 14 06:51:20 +0000 2026","0","Saya melihat berita tentang komdigi dan ongkir tadi pagi.","2054816818741862819","","","in","","0","0","0","https://x.com/undefined/status/2054816818741862819","1814143201718247424",
+"2054502430956769299","Wed May 13 10:02:05 +0000 2026","0","Pembatasan gratis ongkir sangat mengecewakan.","2054502430956769299","","","in","","0","0","0","https://x.com/undefined/status/2054502430956769299","1814143201718247424",
+"2054816635131994490","Thu May 14 06:50:37 +0000 2026","0","keren sih ada kebijakan dari komdigi ongkir jadi berasa lebih merataa","2054816635131994490","","","in","","0","0","0","https://x.com/undefined/status/2054816635131994490","1814143201718247424",
+"2054502269576728775","Wed May 13 10:01:26 +0000 2026","0","Kebijakan komdigi soal ongkir sangat merugikan konsumen.","2054502269576728775","","","in","","0","0","0","https://x.com/undefined/status/2054502269576728775","1814143201718247424",
+"2055001478004367487","Thu May 14 19:05:07 +0000 2026","0","maksud dari komdigi bikin kebijakan gratis ongkir ini apa yaaa?","2055001478004367487","","","in","","0","0","0","https://x.com/undefined/status/2055001478004367487","2043979880086339584",
+"2054502958315020394","Wed May 13 10:04:10 +0000 2026","0","Komdigi disebut dalam diskusi mengenai free ongkir.","2054502958315020394","","","in","","0","0","0","https://x.com/undefined/status/2054502958315020394","1814143201718247424",
+"2054502618484134392","Wed May 13 10:02:49 +0000 2026","0","@komdigi kenapa sih harus ngurusin ongkir segala???","2054502618484134392","","","in","","0","0","0","https://x.com/undefined/status/2054502618484134392","1814143201718247424",
+"2054996678391091685","Thu May 14 18:46:02 +0000 2026","0","komdigi ongkir mending sampingkan dulu","2054996678391091685","","","in","","0","0","0","https://x.com/undefined/status/2054996678391091685","1814143201718247424",
+"2054502657201733861","Wed May 13 10:02:58 +0000 2026","0","Gratis ongkir dibatasi? keputusan paling aneh tahun ini.","2054502657201733861","","","in","","0","0","0","https://x.com/undefined/status/2054502657201733861","1814143201718247424",
+"2054501981369352445","Wed May 13 10:00:17 +0000 2026","0","Gratis ongkir dibatasi malah bikin marketplace lebih fair.","2054501981369352445","","","in","","0","0","0","https://x.com/undefined/status/2054501981369352445","1814143201718247424",
+"2054839912529350807","Thu May 14 08:23:06 +0000 2026","0","komdigi mending ngurusin lainnya deh dsripada ngurusin ongkir giniii","2054839912529350807","","","in","","0","0","0","https://x.com/undefined/status/2054839912529350807","1814143201718247424",
+"2054502817910706448","Wed May 13 10:03:37 +0000 2026","0","Saya membaca berita tentang gratis ongkir dibatasi.","2054502817910706448","","","in","","0","0","0","https://x.com/undefined/status/2054502817910706448","1814143201718247424",
+"2054502131672236531","Wed May 13 10:00:53 +0000 2026","0","kebijakan komdigi tentang ongkir patut diapresiasi.","2054502131672236531","","","in","","0","0","0","https://x.com/undefined/status/2054502131672236531","1814143201718247424",
+"conversation_id_str","created_at","favorite_count","full_text","id_str","image_url","in_reply_to_screen_name","lang","location","quote_count","reply_count","retweet_count","tweet_url","user_id_str","username"
+"2054791169775964388","Thu May 14 05:09:25 +0000 2026","0","kebijakan komdigi gratis ongkir kerenn","2054791169775964388","","","in","","0","0","0","https://x.com/undefined/status/2054791169775964388","1814143201718247424",
+"2055002136027767200","Thu May 14 19:07:43 +0000 2026","0","komdigi ongkir apa gunanyaa??222","2055002136027767200","","","in","","0","0","0","https://x.com/undefined/status/2055002136027767200","2043979880086339584",
+"2054861268750901484","Thu May 14 09:47:58 +0000 2026","0","komdigi ongkir?","2054861268750901484","","","in","","0","0","0","https://x.com/undefined/status/2054861268750901484","1814143201718247424",
+"2054816992662790146","Thu May 14 06:52:02 +0000 2026","0","komdigi??? ongkir??? marketplace???","2054816992662790146","","","in","","0","0","0","https://x.com/undefined/status/2054816992662790146","1814143201718247424",
+"2054823109291852155","Thu May 14 07:16:20 +0000 2026","0","komdigi bikin aturan ongkir baru = langkah yang cukup bijak","2054823109291852155","","","in","","0","0","0","https://x.com/undefined/status/2054823109291852155","1814143201718247424",
+"2055269612334555468","Fri May 15 12:50:35 +0000 2026","0","komdigi + ongkir = chaos wkwkwk","2055269612334555468","","","in","","0","0","0","https://x.com/undefined/status/2055269612334555468","2043979880086339584",
+"2054998005359751518","Thu May 14 18:51:19 +0000 2026","0","komdigi ongkir apa gunanyaa??","2054998005359751518","","","in","","0","0","0","https://x.com/undefined/status/2054998005359751518","2043979880086339584",
+"2054816818741862819","Thu May 14 06:51:20 +0000 2026","0","Saya melihat berita tentang komdigi dan ongkir tadi pagi.","2054816818741862819","","","in","","0","0","0","https://x.com/undefined/status/2054816818741862819","1814143201718247424",
+"2054502430956769299","Wed May 13 10:02:05 +0000 2026","0","Pembatasan gratis ongkir sangat mengecewakan.","2054502430956769299","","","in","","0","0","0","https://x.com/undefined/status/2054502430956769299","1814143201718247424",
+"2054816635131994490","Thu May 14 06:50:37 +0000 2026","0","keren sih ada kebijakan dari komdigi ongkir jadi berasa lebih merataa","2054816635131994490","","","in","","0","0","0","https://x.com/undefined/status/2054816635131994490","1814143201718247424",
+"2054502269576728775","Wed May 13 10:01:26 +0000 2026","0","Kebijakan komdigi soal ongkir sangat merugikan konsumen.","2054502269576728775","","","in","","0","0","0","https://x.com/undefined/status/2054502269576728775","1814143201718247424",
+"2055001478004367487","Thu May 14 19:05:07 +0000 2026","0","maksud dari komdigi bikin kebijakan gratis ongkir ini apa yaaa?","2055001478004367487","","","in","","0","0","0","https://x.com/undefined/status/2055001478004367487","2043979880086339584",
+"2054502958315020394","Wed May 13 10:04:10 +0000 2026","0","Komdigi disebut dalam diskusi mengenai free ongkir.","2054502958315020394","","","in","","0","0","0","https://x.com/undefined/status/2054502958315020394","1814143201718247424",
+"2054502618484134392","Wed May 13 10:02:49 +0000 2026","0","@komdigi kenapa sih harus ngurusin ongkir segala???","2054502618484134392","","","in","","0","0","0","https://x.com/undefined/status/2054502618484134392","1814143201718247424",
+"2054996678391091685","Thu May 14 18:46:02 +0000 2026","0","komdigi ongkir mending sampingkan dulu","2054996678391091685","","","in","","0","0","0","https://x.com/undefined/status/2054996678391091685","1814143201718247424",
+"2054502657201733861","Wed May 13 10:02:58 +0000 2026","0","Gratis ongkir dibatasi? keputusan paling aneh tahun ini.","2054502657201733861","","","in","","0","0","0","https://x.com/undefined/status/2054502657201733861","1814143201718247424",
+"2054501981369352445","Wed May 13 10:00:17 +0000 2026","0","Gratis ongkir dibatasi malah bikin marketplace lebih fair.","2054501981369352445","","","in","","0","0","0","https://x.com/undefined/status/2054501981369352445","1814143201718247424",
+"2054839912529350807","Thu May 14 08:23:06 +0000 2026","0","komdigi mending ngurusin lainnya deh dsripada ngurusin ongkir giniii","2054839912529350807","","","in","","0","0","0","https://x.com/undefined/status/2054839912529350807","1814143201718247424",
+"2054502817910706448","Wed May 13 10:03:37 +0000 2026","0","Saya membaca berita tentang gratis ongkir dibatasi.","2054502817910706448","","","in","","0","0","0","https://x.com/undefined/status/2054502817910706448","1814143201718247424",
+"2054502131672236531","Wed May 13 10:00:53 +0000 2026","0","kebijakan komdigi tentang ongkir patut diapresiasi.","2054502131672236531","","","in","","0","0","0","https://x.com/undefined/status/2054502131672236531","1814143201718247424",