from datetime import datetime, timezone
from html import escape
import os
import time
import pandas as pd
import streamlit as st
from streamlit_autorefresh import st_autorefresh
from database import init_db, get_latest_crawl_time
from timezone_utils import (
INDONESIA_TIMEZONES,
browser_offset_to_choice,
browser_timezone_to_choice,
get_default_timezone,
get_timezone_label,
get_timezone_name,
parse_dt_with_source_tz,
)
try:
from dotenv import load_dotenv
load_dotenv()
except Exception:
pass
init_db()
# ── Auto-start background crawler scheduler (disabled) ───────────────────
# Scheduler NRT hanya boleh berjalan setelah user menekan tombol aktivasi
# pada mode "Hari Ini" (captured) agar sesuai kebutuhan (NRT dimulai manual).
st.set_page_config(
page_title="Dashboard Sentimen Twitter",
layout="wide",
page_icon="📊",
initial_sidebar_state="expanded"
)
# Initialize timezone selection in session state
if "user_timezone" not in st.session_state:
st.session_state.user_timezone = get_default_timezone()
if "follow_device_timezone" not in st.session_state:
st.session_state.follow_device_timezone = True
def _get_query_param(name):
try:
value = st.query_params.get(name)
except Exception:
value = None
if isinstance(value, list):
return value[0] if value else None
return value
def _get_context_timezone():
try:
return st.context.timezone
except Exception:
return None
def _get_context_offset():
try:
offset = st.context.timezone_offset
except Exception:
return None
if offset is None:
return None
try:
return str(-int(offset))
except (TypeError, ValueError):
return None
def _sync_timezone_from_browser():
browser_timezone = _get_context_timezone() or _get_query_param("browser_tz")
browser_offset = _get_context_offset() or _get_query_param("browser_offset")
mapped_timezone = (
browser_timezone_to_choice(browser_timezone)
or browser_offset_to_choice(browser_offset)
)
st.session_state.browser_timezone = browser_timezone
st.session_state.browser_offset = browser_offset
if st.session_state.get("follow_device_timezone", True):
timezone_choice = mapped_timezone or get_default_timezone()
if st.session_state.user_timezone != timezone_choice:
st.session_state.user_timezone = timezone_choice
_sync_timezone_from_browser()
def get_refresh_interval():
try:
latest_crawl = get_latest_crawl_time()
if latest_crawl:
timezone_choice = st.session_state.get("user_timezone", get_default_timezone())
latest_time = parse_dt_with_source_tz(
[latest_crawl],
timezone_choice,
os.getenv("APP_TIMEZONE", "Asia/Jakarta")
).iloc[0]
now = pd.Timestamp.now(tz=get_timezone_name(timezone_choice)).tz_localize(None)
time_diff = (now - latest_time).total_seconds()
return 30000 if time_diff < 300 else 60000
except Exception:
pass
return 60000
refresh_interval = get_refresh_interval()
st_autorefresh(
interval=refresh_interval,
key="auto_refresh_dashboard"
)
def _check_crawler_alive():
from datetime import timedelta
from crawler import NRT_INTERVAL_MINUTES, get_crawler_state
threshold = timedelta(minutes=max((NRT_INTERVAL_MINUTES * 2) + 1, 3))
def parse_dt(value):
if not value:
return None
try:
parsed = datetime.fromisoformat(value)
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=timezone.utc)
return parsed.astimezone(timezone.utc)
except Exception:
return None
state = get_crawler_state()
if not state:
return False
updated_at = parse_dt(state.get("updated_at"))
heartbeat_at = parse_dt(state.get("heartbeat_at"))
now = datetime.now(timezone.utc)
if (
state.get("is_running", False)
and updated_at is not None
and now - updated_at <= threshold
):
return True
return (
state.get("service_active", False)
and heartbeat_at is not None
and now - heartbeat_at <= threshold
)
# Hanya update _scheduler_started dari database jika user TIDAK sedang nonaktifkan eksplisit
_raw_alive = _check_crawler_alive()
explicitly_off = st.session_state.get("nrt_explicitly_deactivated", False)
if explicitly_off:
# User sudah klik nonaktifkan — override database state, paksa False
st.session_state._scheduler_started = False
else:
st.session_state._scheduler_started = _raw_alive
try:
from crawler import NRT_INTERVAL_MINUTES
st.session_state._scheduler_interval = NRT_INTERVAL_MINUTES
except Exception:
st.session_state._scheduler_interval = 0
st.markdown("""
""", unsafe_allow_html=True)
if "page" not in st.session_state:
st.session_state.page = "crawling"
if st.session_state.get("current_page"):
st.session_state.page = st.session_state.current_page
st.session_state.scroll_to_top = True
del st.session_state["current_page"]
if (
st.session_state.get("scroll_to_top")
or st.session_state.get("_last_rendered_page") != st.session_state.page
):
st.session_state.scroll_to_top = False
st.session_state._last_rendered_page = st.session_state.page
with st.sidebar:
# ── HEADER ──────────────────────────────────────────────────
st.markdown("""
📊
SentimenX
Twitter Analytics Dashboard
""", unsafe_allow_html=True)
# ── STATUS CRAWLER ──────────────────────────────────────────
sched_ok = st.session_state.get("_scheduler_started", False)
explicitly_off = st.session_state.get("nrt_explicitly_deactivated", False)
sched_interval = st.session_state.get("_scheduler_interval", 0)
sched_color = "#16a34a" if sched_ok else "#64748b"
sched_label = (
f"Aktif — diperbarui tiap {sched_interval} menit"
if sched_ok
else "Nonaktif — crawling dihentikan oleh pengguna"
if explicitly_off
else "Standby — aktifkan melalui menu Hari Ini"
)
st.markdown(f"""
""", unsafe_allow_html=True)
# ── UPDATE TERBARU ──────────────────────────────────────────
try:
latest_crawl = get_latest_crawl_time()
if latest_crawl:
crawl_dt = parse_dt_with_source_tz(
[latest_crawl],
st.session_state.user_timezone,
os.getenv("APP_TIMEZONE", "Asia/Jakarta")
).iloc[0]
tz_label = get_timezone_label(st.session_state.user_timezone)
update_date = crawl_dt.strftime("%d/%m/%Y")
update_time = f"{crawl_dt.strftime('%H:%M:%S')} {tz_label}"
st.markdown(f"""
""", unsafe_allow_html=True)
except Exception:
pass
# ── MENU UTAMA ──────────────────────────────────────────────
st.markdown('', unsafe_allow_html=True)
current_page = st.session_state.get("page", "crawling")
nav_items = [
("crawling", "🔄 Ambil Data Twitter"),
("preprocessing", "🧹 Bersihkan Data"),
("sentiment", "📈 Analisis Sentimen"),
]
for page_key, label in nav_items:
is_active = current_page == page_key
# Tidak memakai width="stretch" agar kompatibel dengan semua versi Streamlit
if st.button(
label,
type="primary" if is_active else "secondary",
key=f"nav_{page_key}",
use_container_width=True,
):
st.session_state.page = page_key
st.session_state.scroll_to_top = True
st.rerun()
st.markdown("
", unsafe_allow_html=True)
# ── TIMEZONE ────────────────────────────────────────────────
st.markdown('', unsafe_allow_html=True)
follow_device = st.toggle(
"Ikuti timezone device",
value=st.session_state.get("follow_device_timezone", True),
key="follow_device_timezone_toggle",
help="Jika aktif, dashboard membaca zona waktu dari browser/device."
)
active_tz_name = get_timezone_name(st.session_state.user_timezone)
active_now = pd.Timestamp.now(tz=active_tz_name)
active_clock = active_now.strftime("%d/%m/%Y %H:%M:%S")
st.markdown(f"""
""", unsafe_allow_html=True)
# ── QUERY ────────────────────────────────────────────────────
# ── QUERY ────────────────────────────────────────────────────
crawler_query = os.getenv(
"QUERY",
'komdigi (ongkir OR "gratis ongkir" OR "free ongkir") '
'OR "pembatasan gratis ongkir" OR "gratis ongkir dibatasi"'
)
# Highlight keyword utama dengan warna biru
_QUERY_TERMS = [
"komdigi", "ongkir", "gratis ongkir", "free ongkir",
"pembatasan gratis ongkir", "gratis ongkir dibatasi",
]
query_highlighted = escape(crawler_query)
for term in _QUERY_TERMS:
escaped_term = escape(term)
query_highlighted = query_highlighted.replace(
escaped_term,
f'{escaped_term}',
1,
)
st.markdown(f"""
""", unsafe_allow_html=True)
# ── FOOTER ───────────────────────────────────────────────────
st.markdown("""
Data bersumber dari X (Twitter)
via tweet-harvest
""", unsafe_allow_html=True)
# ── PAGE ROUTING ─────────────────────────────────────────────────────────────
from page_modules import crawling_page, preprocessing_page, sentiment_page
if st.session_state.page == "crawling":
crawling_page.show()
elif st.session_state.page == "preprocessing":
preprocessing_page.show()
elif st.session_state.page == "sentiment":
sentiment_page.show()