from datetime import datetime, timezone
from html import escape
import os
import time
import pandas as pd
import streamlit as st
from streamlit_autorefresh import st_autorefresh
from database import init_db, get_latest_crawl_time
from timezone_utils import (
INDONESIA_TIMEZONES,
browser_offset_to_choice,
browser_timezone_to_choice,
get_default_timezone,
get_timezone_label,
get_timezone_name,
parse_dt_with_source_tz,
)
try:
from dotenv import load_dotenv
load_dotenv()
except Exception:
pass
init_db()
st.set_page_config(
page_title="Dashboard Sentimen Twitter",
layout="wide",
page_icon="๐",
initial_sidebar_state="expanded"
)
# Initialize timezone selection in session state
if "user_timezone" not in st.session_state:
st.session_state.user_timezone = get_default_timezone()
if "follow_device_timezone" not in st.session_state:
st.session_state.follow_device_timezone = True
def _get_query_param(name):
try:
value = st.query_params.get(name)
except Exception:
value = None
if isinstance(value, list):
return value[0] if value else None
return value
def _get_context_timezone():
try:
return st.context.timezone
except Exception:
return None
def _get_context_offset():
try:
offset = st.context.timezone_offset
except Exception:
return None
if offset is None:
return None
try:
# st.context.timezone_offset follows JavaScript getTimezoneOffset:
# UTC+8 is -480. The app mapping uses UTC offset minutes: UTC+8 is 480.
return str(-int(offset))
except (TypeError, ValueError):
return None
def _sync_timezone_from_browser():
browser_timezone = _get_context_timezone() or _get_query_param("browser_tz")
browser_offset = _get_context_offset() or _get_query_param("browser_offset")
mapped_timezone = (
browser_timezone_to_choice(browser_timezone)
or browser_offset_to_choice(browser_offset)
)
st.session_state.browser_timezone = browser_timezone
st.session_state.browser_offset = browser_offset
if st.session_state.get("follow_device_timezone", True):
timezone_choice = mapped_timezone or get_default_timezone()
if st.session_state.user_timezone != timezone_choice:
st.session_state.user_timezone = timezone_choice
_sync_timezone_from_browser()
# Determine refresh interval berdasarkan aktivitas crawler
def get_refresh_interval():
"""Refresh lebih cepat jika ada data baru dari crawler"""
try:
latest_crawl = get_latest_crawl_time()
if latest_crawl:
timezone_choice = st.session_state.get("user_timezone", get_default_timezone())
latest_time = parse_dt_with_source_tz(
[latest_crawl],
timezone_choice,
os.getenv("APP_TIMEZONE", "Asia/Makassar")
).iloc[0]
now = pd.Timestamp.now(tz=get_timezone_name(timezone_choice)).tz_localize(None)
time_diff = (now - latest_time).total_seconds()
# Jika data baru < 5 menit, refresh setiap 30 detik, jika tidak setiap 60 detik
return 30000 if time_diff < 300 else 60000
except Exception:
pass
return 60000
refresh_interval = get_refresh_interval()
st_autorefresh(
interval=refresh_interval,
key="auto_refresh_dashboard"
)
def _check_crawler_alive():
from datetime import datetime, timedelta, timezone
from crawler import NRT_INTERVAL_MINUTES, get_crawler_state
threshold = timedelta(minutes=max((NRT_INTERVAL_MINUTES * 2) + 1, 3))
def parse_dt(value):
if not value:
return None
try:
parsed = datetime.fromisoformat(value)
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=timezone.utc)
return parsed.astimezone(timezone.utc)
except Exception:
return None
state = get_crawler_state()
if not state:
return False
updated_at = parse_dt(state.get("updated_at"))
heartbeat_at = parse_dt(state.get("heartbeat_at"))
now = datetime.now(timezone.utc)
if (
state.get("is_running", False)
and updated_at is not None
and now - updated_at <= threshold
):
return True
return (
state.get("service_active", False)
and heartbeat_at is not None
and now - heartbeat_at <= threshold
)
st.session_state._scheduler_started = _check_crawler_alive()
try:
from crawler import NRT_INTERVAL_MINUTES
st.session_state._scheduler_interval = NRT_INTERVAL_MINUTES
except Exception:
st.session_state._scheduler_interval = 0
st.markdown("""
""", unsafe_allow_html=True)
if "page" not in st.session_state:
st.session_state.page = "crawling"
if st.session_state.get("current_page"):
st.session_state.page = st.session_state.current_page
st.session_state.scroll_to_top = True
del st.session_state["current_page"]
if (
st.session_state.get("scroll_to_top")
or st.session_state.get("_last_rendered_page") != st.session_state.page
):
st.session_state.scroll_to_top = False
st.session_state._last_rendered_page = st.session_state.page
with st.sidebar:
st.markdown("""
๐
SentimenX
Twitter Analytics
""", unsafe_allow_html=True)
sched_ok = st.session_state.get("_scheduler_started", False)
sched_interval = st.session_state.get("_scheduler_interval", 0)
sched_color = "#16a34a" if sched_ok else "#f59e0b"
sched_label = (
f"Aktif โ diperbarui tiap {sched_interval} menit"
if sched_ok
else "Nonaktif โ jalankan: python3 crawler.py"
)
sched_dot = "โ" if sched_ok else "โ"
st.markdown(f"""
{sched_dot}
Crawler Service
{sched_label}
""", unsafe_allow_html=True)
# Tampilkan waktu update terakhir data
try:
latest_crawl = get_latest_crawl_time()
if latest_crawl:
crawl_dt = parse_dt_with_source_tz(
[latest_crawl],
st.session_state.user_timezone,
os.getenv("APP_TIMEZONE", "Asia/Makassar")
).iloc[0]
tz_label = get_timezone_label(st.session_state.user_timezone)
update_date = crawl_dt.strftime("%d/%m/%Y")
update_time = f"{crawl_dt.strftime('%H:%M:%S')} {tz_label}"
st.markdown(f"""
๐ Update Terbaru
{update_date}
{update_time}
""", unsafe_allow_html=True)
except Exception:
pass
st.markdown("""
MENU UTAMA
""", unsafe_allow_html=True)
current_page = st.session_state.get("page", "crawling")
nav_items = [
("crawling", "๐ Ambil Data Twitter"),
("preprocessing", "๐งน Bersihkan Data"),
("sentiment", "๐ Analisis Sentimen"),
]
for page_key, label in nav_items:
is_active = current_page == page_key
if st.button(
label,
width="stretch",
type="primary" if is_active else "secondary",
key=f"nav_{page_key}"
):
st.session_state.page = page_key
st.session_state.scroll_to_top = True
st.rerun()
st.markdown("
", unsafe_allow_html=True)
# Timezone selector
st.markdown("""
โฐ ZONA WAKTU
""", unsafe_allow_html=True)
follow_device = st.toggle(
"Ikuti timezone device",
value=st.session_state.get("follow_device_timezone", True),
key="follow_device_timezone_toggle",
help="Jika aktif, dashboard membaca zona waktu dari browser/device."
)
if follow_device != st.session_state.follow_device_timezone:
st.session_state.follow_device_timezone = follow_device
if follow_device:
mapped_timezone = (
browser_timezone_to_choice(
st.session_state.get("browser_timezone")
)
or browser_offset_to_choice(st.session_state.get("browser_offset"))
)
st.session_state.user_timezone = mapped_timezone or get_default_timezone()
st.rerun()
timezone_options = list(INDONESIA_TIMEZONES.keys())
if (
not st.session_state.follow_device_timezone
and st.session_state.user_timezone not in timezone_options
):
st.session_state.user_timezone = get_default_timezone()
if st.session_state.follow_device_timezone:
raw_detected_tz = st.session_state.get("browser_timezone")
detected_offset = st.session_state.get("browser_offset")
detected_tz = raw_detected_tz or (
"Timezone dari offset browser" if detected_offset else "Belum terdeteksi"
)
mapped_timezone = (
browser_timezone_to_choice(raw_detected_tz)
or browser_offset_to_choice(detected_offset)
)
active_label = mapped_timezone or get_default_timezone()
st.session_state.user_timezone = active_label
try:
offset_minutes = int(detected_offset)
sign = "+" if offset_minutes >= 0 else "-"
abs_minutes = abs(offset_minutes)
offset_label = f"UTC{sign}{abs_minutes // 60:02d}:{abs_minutes % 60:02d}"
except (TypeError, ValueError):
offset_label = "offset belum terdeteksi"
st.caption(f"Device: {detected_tz} ยท {offset_label} ยท Aktif: {active_label}")
else:
selected_tz = st.selectbox(
"Pilih zona waktu Indonesia",
timezone_options,
index=timezone_options.index(st.session_state.user_timezone),
label_visibility="collapsed",
key="tz_selector"
)
if selected_tz != st.session_state.user_timezone:
st.session_state.user_timezone = selected_tz
st.rerun()
active_tz_name = get_timezone_name(st.session_state.user_timezone)
active_now = pd.Timestamp.now(tz=active_tz_name)
active_clock = active_now.strftime("%d/%m/%Y %H:%M:%S")
st.markdown(
f"""
""",
unsafe_allow_html=True,
)
if (
st.session_state.follow_device_timezone
and (
browser_timezone_to_choice(st.session_state.get("browser_timezone"))
or browser_offset_to_choice(st.session_state.get("browser_offset"))
) is None
and st.session_state.get("browser_timezone")
):
st.caption("Timezone device belum valid dari browser, memakai default aplikasi.")
if not st.session_state.follow_device_timezone and selected_tz != st.session_state.user_timezone:
st.session_state.user_timezone = selected_tz
st.rerun()
st.markdown("
", unsafe_allow_html=True)
crawler_query = os.getenv(
"QUERY",
'komdigi (ongkir OR "gratis ongkir" OR "free ongkir") OR "pembatasan gratis ongkir" OR "gratis ongkir dibatasi"'
)
query_terms = [
term.strip().strip('"')
for term in crawler_query.split(" OR ")
if term.strip()
]
query_html = "
".join(
f"โข {escape(term)}" for term in query_terms
) or f"โข {escape(crawler_query)}"
st.markdown(f"""
๐ Query Crawler Aktif
{query_html}
""", unsafe_allow_html=True)
st.markdown("""
Data bersumber dari X (Twitter)
via tweet-harvest
""", unsafe_allow_html=True)
from page_modules import crawling_page, preprocessing_page, sentiment_page
if st.session_state.page == "crawling":
crawling_page.show()
elif st.session_state.page == "preprocessing":
preprocessing_page.show()
elif st.session_state.page == "sentiment":
sentiment_page.show()