import os
import time
from datetime import datetime, timedelta, timezone
import pandas as pd
import plotly.graph_objects as go
import streamlit as st
from streamlit_autorefresh import st_autorefresh
from page_modules.table_utils import render_standard_table
from database import engine
from timezone_utils import (
parse_dt_with_tz,
parse_dt_with_source_tz,
get_timezone_label,
get_timezone_name,
)
from crawler import (
get_auto_crawl_logs,
get_crawler_state,
AUTO_CRAWL_INTERVAL_HOURS,
NRT_INTERVAL_MINUTES,
auto_crawl_job,
ensure_scheduler_running,
)
# ═══════════════════════════════════════════════════════════
# AUTO-START SCHEDULER
# ═══════════════════════════════════════════════════════════
ensure_scheduler_running()
# ═══════════════════════════════════════════════════════════
# TIMEZONE & DATETIME HELPERS
# ═══════════════════════════════════════════════════════════
def parse_dt(series):
return parse_dt_with_tz(
series,
st.session_state.get("user_timezone", "WIB (UTC+7)")
)
def parse_crawled_dt(series):
return parse_dt_with_source_tz(
series,
st.session_state.get("user_timezone", "WIB (UTC+7)"),
os.getenv("APP_TIMEZONE", "Asia/Jakarta")
)
def format_dt(value):
if value is None or pd.isna(value):
return "Belum ada"
try:
if not hasattr(value, "strftime"):
value = parse_dt(pd.Series([value])).iloc[0]
if value is None or pd.isna(value):
return "Belum ada"
tz_label = get_timezone_label(st.session_state.get("user_timezone", "WIB (UTC+7)"))
return f"{value.strftime('%d/%m/%Y %H:%M')} {tz_label}"
except Exception:
return "Belum ada"
def user_now():
timezone_choice = st.session_state.get("user_timezone", "WIB (UTC+7)")
return pd.Timestamp.now(tz=get_timezone_name(timezone_choice)).tz_localize(None)
def user_today():
return user_now().date()
def format_countdown(delta):
total_seconds = max(0, int(delta.total_seconds()))
minutes, seconds = divmod(total_seconds, 60)
return f"{minutes:02d}:{seconds:02d}"
def parse_log_time(value):
try:
parsed = pd.to_datetime(value, errors="coerce")
except Exception:
return None
if pd.isna(parsed):
return None
if parsed.tzinfo is None:
parsed = parsed.tz_localize(os.getenv("APP_TIMEZONE", "Asia/Jakarta"))
return parsed.tz_convert("UTC").to_pydatetime()
def to_user_local_datetime(value):
if value is None or pd.isna(value):
return None
try:
parsed = pd.to_datetime(value, errors="coerce", format="mixed")
except Exception:
return None
if pd.isna(parsed):
return None
if parsed.tzinfo is None:
return parsed.to_pydatetime() if hasattr(parsed, "to_pydatetime") else parsed
converted = parse_dt(pd.Series([parsed])).iloc[0]
if converted is None or pd.isna(converted):
return None
return converted.to_pydatetime() if hasattr(converted, "to_pydatetime") else converted
def get_visible_crawl_logs(logs):
started_at = st.session_state.get("crawl_history_started_at")
if started_at is None:
return logs
started_at = parse_log_time(started_at.isoformat())
visible = []
for log in logs:
log_dt = parse_log_time(log.get("timestamp"))
if log_dt is not None and log_dt >= started_at:
visible.append(log)
return visible
def is_crawler_running():
state = get_crawler_state()
updated_at = _parse_state_datetime(state.get("updated_at"))
if updated_at is None:
return False
return (
state.get("is_running", False)
and datetime.now(timezone.utc) - updated_at <= _crawler_state_threshold()
)
def _parse_state_datetime(value):
if not value:
return None
try:
parsed = datetime.fromisoformat(value)
except Exception:
return None
if parsed.tzinfo is None:
parsed = pd.Timestamp(parsed).tz_localize(
os.getenv("APP_TIMEZONE", "Asia/Jakarta")
).to_pydatetime()
return parsed.astimezone(timezone.utc)
def _crawler_state_threshold():
return timedelta(minutes=max((NRT_INTERVAL_MINUTES * 2) + 1, 3))
def is_crawler_service_active(logs=None):
state = get_crawler_state()
threshold = _crawler_state_threshold()
now = datetime.now(timezone.utc)
if not state:
return False
if is_crawler_running():
return True
heartbeat_at = _parse_state_datetime(state.get("heartbeat_at"))
if (
state.get("service_active", False)
and heartbeat_at is not None
and now - heartbeat_at <= threshold
):
return True
return False
# ═══════════════════════════════════════════════════════════
# UI HELPERS
# ═══════════════════════════════════════════════════════════
def _render_crawling_styles():
st.markdown("""
""", unsafe_allow_html=True)
def _gap(size="md"):
heights = {"xs": 6, "sm": 12, "md": 20, "lg": 32}
h = heights.get(size, 20)
st.markdown(f'', unsafe_allow_html=True)
def _section_header(title, subtitle=""):
sub_html = (
f'{subtitle}
'
if subtitle else ""
)
st.markdown(f"""
""", unsafe_allow_html=True)
def _divider():
st.markdown('', unsafe_allow_html=True)
# ═══════════════════════════════════════════════════════════
# NRT STATUS BANNER
# ═══════════════════════════════════════════════════════════
def _render_nrt_status_banner(sched_ok):
if sched_ok:
bg = "linear-gradient(135deg,#f0fdf4,#dcfce7)"
bdr = "#86efac"
dot_cls = "pulse-dot-green"
dot_color = "#16a34a"
badge_bg = "#dcfce7"
badge_c = "#15803d"
title = "Crawler Service Aktif"
desc = (
f"Sistem berjalan otomatis di background setiap "
f"{NRT_INTERVAL_MINUTES} menit. "
f"Data mencakup 7 hari terakhir berdasarkan tanggal asli tweet."
)
badge = "● LIVE"
else:
bg = "linear-gradient(135deg,#eff6ff,#dbeafe)"
bdr = "#93c5fd"
dot_cls = "pulse-dot-blue"
dot_color = "#3b6cf7"
badge_bg = "#dbeafe"
badge_c = "#1d4ed8"
title = "Crawler Siap Dijalankan"
desc = (
f"Scheduler otomatis aktif setiap {NRT_INTERVAL_MINUTES} menit "
f"sejak aplikasi pertama dibuka — tidak perlu terminal terpisah."
)
badge = "◎ STANDBY"
st.markdown(f"""
""", unsafe_allow_html=True)
# ═══════════════════════════════════════════════════════════
# MONITORING PANEL
# ═══════════════════════════════════════════════════════════
def _render_monitoring_panel(last_log, last_dt, sched_ok):
now = user_now()
try:
df_stats = pd.read_sql("SELECT crawled_at FROM tweets", engine)
if not df_stats.empty:
df_stats["crawled_at"] = parse_crawled_dt(df_stats["crawled_at"])
total_db = len(df_stats)
today_start = pd.Timestamp(now.date())
tweet_today = (
len(df_stats[df_stats["crawled_at"] >= today_start])
if not df_stats.empty else 0
)
latest_crawled = df_stats["crawled_at"].max() if not df_stats.empty else None
except Exception:
total_db = 0
tweet_today = 0
latest_crawled = None
last_log_dt = last_dt
if last_log_dt is None and last_log:
last_log_dt = parse_log_time(last_log.get("timestamp"))
final_last_dt = to_user_local_datetime(last_log_dt)
if final_last_dt is None:
final_last_dt = to_user_local_datetime(latest_crawled)
last_crawl_str = "Belum pernah"
next_crawl_str = "Belum terjadwal"
countdown_str = "Menunggu crawl pertama..."
monitoring_time_str = format_dt(now)
if final_last_dt is not None:
next_dt = final_last_dt + timedelta(minutes=NRT_INTERVAL_MINUTES)
remaining = next_dt - now
last_crawl_str = format_dt(final_last_dt)
next_crawl_str = format_dt(next_dt)
if remaining.total_seconds() > 0:
countdown_str = format_countdown(remaining)
elif is_crawler_running():
countdown_str = "⏳ Sedang crawling..."
else:
countdown_str = "🔄 Segera diperbarui..."
status_text = "Aktif Otomatis" if sched_ok else "Standby"
status_color = "#16a34a" if sched_ok else "#3b6cf7"
status_bg = "#dcfce7" if sched_ok else "#dbeafe"
note_text = f"✅ {tweet_today:,} tweet masuk hari ini" if tweet_today > 0 else "ℹ️ Belum ada tweet baru hari ini"
note_bg = "#f0fdf4" if tweet_today > 0 else "#f8fafc"
note_border = "#bbf7d0" if tweet_today > 0 else "#e8edf5"
note_color = "#166534" if tweet_today > 0 else "#64748b"
_section_header("📡 Status Monitoring Sistem", "Pembaruan data near-realtime · interval otomatis")
col_a, col_b = st.columns(2, gap="medium")
# ── Left card: sistem crawling ──
left_rows = [
("Interval crawling", f"{NRT_INTERVAL_MINUTES} menit", "#3b6cf7"),
("Jangkauan data", "7 hari terakhir", "#3b6cf7"),
("Waktu monitoring", monitoring_time_str, "#64748b"),
("Last crawl", last_crawl_str, "#0f172a"),
("Next crawl", next_crawl_str, "#0f172a"),
("Countdown", countdown_str, "#16a34a"),
]
rows_html_a = ""
for i, (label, value, vc) in enumerate(left_rows):
border = "none" if i == len(left_rows) - 1 else "1px solid #f1f5f9"
rows_html_a += f"""
{label}
{value}
"""
with col_a:
st.markdown(f"""
⚙️ Sistem Crawling
{status_text}
{rows_html_a}
""", unsafe_allow_html=True)
# ── Right card: ringkasan dataset ──
right_rows = [
("Update terakhir", last_crawl_str, "#0f172a"),
("Countdown berikutnya", countdown_str, "#16a34a"),
("Tweet masuk hari ini", f"{tweet_today:,} tweet", "#3b6cf7"),
("Total dataset", f"{total_db:,} tweet", "#0f172a"),
]
rows_html_b = ""
for i, (label, value, vc) in enumerate(right_rows):
border = "none" if i == len(right_rows) - 1 else "1px solid #f1f5f9"
rows_html_b += f"""
{label}
{value}
"""
with col_b:
st.markdown(f"""
📊 Ringkasan Dataset
{rows_html_b}
{note_text}
""", unsafe_allow_html=True)
# ═══════════════════════════════════════════════════════════
# METRICS ROW
# ═══════════════════════════════════════════════════════════
def _render_metrics(total, days, earliest, latest, filter_label, basis_label):
st.markdown(f"""
🔄
Data aktif: {filter_label}
· {basis_label}
""", unsafe_allow_html=True)
pills = [
("📊", "linear-gradient(135deg,#eef2ff,#e0e7ff)", "#3b6cf7", "#1e3a8a", "#dbeafe",
"Total Tweet", f"{total:,}", "Terkumpul periode ini"),
("📅", "linear-gradient(135deg,#f0fdf4,#dcfce7)", "#16a34a", "#14532d", "#bbf7d0",
"Rentang Waktu", f"{days} Hari", basis_label),
("🗓️", "linear-gradient(135deg,#fff7ed,#ffedd5)", "#ea580c", "#7c2d12", "#fed7aa",
"Mulai Dari", earliest, "Tanggal awal"),
("📆", "linear-gradient(135deg,#fefce8,#fef9c3)", "#ca8a04", "#713f12", "#fde68a",
"Sampai Dengan", latest, "Tanggal akhir"),
]
c1, c2, c3, c4 = st.columns(4)
for col, (icon, bg, color, dark, border_c, label, val, sub) in zip([c1, c2, c3, c4], pills):
with col:
fs = "1.1rem" if len(str(val)) > 10 else "1.55rem"
st.markdown(f"""
{icon}
{label}
{val}
{sub}
""", unsafe_allow_html=True)
# ═══════════════════════════════════════════════════════════
# CHARTS
# ═══════════════════════════════════════════════════════════
def _render_charts(df, filter_label, date_col, chart_label, dt_start, dt_end):
df_tl = df.copy().dropna(subset=[date_col])
if df_tl.empty:
st.info("Belum ada data valid untuk grafik.")
return
df_tl["date_key"] = df_tl[date_col].dt.strftime("%Y-%m-%d")
actual_daily = (
df_tl.groupby("date_key").size().reset_index(name="count").sort_values("date_key")
)
date_range = pd.date_range(
pd.Timestamp(dt_start).date(),
pd.Timestamp(dt_end).date(),
freq="D"
)
daily = pd.DataFrame({
"date_key": date_range.strftime("%Y-%m-%d"),
"date_str": date_range.strftime("%d %b"),
}).merge(actual_daily, on="date_key", how="left")
daily["count"] = daily["count"].fillna(0).astype(int)
ordered = daily["date_str"].tolist()
counts_day = daily["count"].tolist()
active_days = int((daily["count"] > 0).sum())
inactive_days = len(daily) - active_days
total_days = len(daily)
total_tweets = int(daily["count"].sum())
max_cnt = int(daily["count"].max()) if daily["count"].max() > 0 else 0
nonzero = daily[daily["count"] > 0]
min_cnt = int(nonzero["count"].min()) if not nonzero.empty else 0
busiest_list = daily[daily["count"] == max_cnt]["date_str"].tolist() if max_cnt > 0 else []
quietest_list = nonzero[nonzero["count"] == min_cnt]["date_str"].tolist() if min_cnt > 0 else []
bar_colors = []
for c in counts_day:
if c == 0:
bar_colors.append("#e2e8f0")
elif max_cnt > 0 and c == max_cnt:
bar_colors.append("#1d4ed8")
elif min_cnt > 0 and c == min_cnt and min_cnt != max_cnt:
bar_colors.append("#a78bfa")
else:
bar_colors.append("#3b6cf7")
max_y = max(counts_day) if any(c > 0 for c in counts_day) else 1
y_range_top = max_y * 1.3
fig_bar = go.Figure()
fig_bar.add_trace(go.Bar(
x=ordered, y=counts_day,
marker=dict(color=bar_colors, opacity=0.10, line=dict(width=0), cornerradius=8),
width=0.68, showlegend=False, hoverinfo="skip",
))
fig_bar.add_trace(go.Bar(
x=ordered, y=counts_day,
marker=dict(color=bar_colors, opacity=0.93, line=dict(width=0), cornerradius=8),
width=0.46,
hovertemplate="%{x}
%{y:,} tweet",
text=[f"{c}" if c > 0 else "" for c in counts_day],
textposition="outside",
textfont=dict(size=10, color="#475569"),
cliponaxis=False, showlegend=False,
))
fig_bar.update_layout(
height=280, barmode="overlay",
margin=dict(l=0, r=4, t=8, b=4),
paper_bgcolor="rgba(0,0,0,0)", plot_bgcolor="rgba(0,0,0,0)",
xaxis=dict(
type="category", categoryorder="array", categoryarray=ordered,
tickfont=dict(size=10, color="#94a3b8"),
showgrid=False, zeroline=False, showline=False,
fixedrange=True, tickangle=-35 if total_days > 14 else 0,
),
yaxis=dict(
tickfont=dict(size=9, color="#cbd5e1"),
showgrid=True, gridcolor="rgba(226,232,240,0.45)",
griddash="dot", gridwidth=1,
zeroline=False, showline=False,
fixedrange=True, range=[0, y_range_top],
),
showlegend=False, hovermode="x unified",
hoverlabel=dict(bgcolor="#1e293b", font=dict(color="white", size=12), bordercolor="#334155"),
)
st.plotly_chart(fig_bar, use_container_width=True, config={"displayModeBar": False})
# ── Legend ──
legend_items = [("#1d4ed8", "Tertinggi"), ("#3b6cf7", "Normal")]
if min_cnt > 0 and min_cnt != max_cnt:
legend_items.insert(1, ("#a78bfa", "Terendah"))
if any(c == 0 for c in counts_day):
legend_items.append(("#cbd5e1", "Tidak ada data"))
legend_html = "".join([
f''
f''
f'{lbl}'
for clr, lbl in legend_items
])
st.markdown(
f'{legend_html}
',
unsafe_allow_html=True
)
# ── Insight cards ──
pc1, pc2, pc3 = st.columns(3, gap="small")
def _date_pills(days_list, color, max_show=4):
if not days_list:
return '—'
pills = ""
for d in days_list[:max_show]:
pills += (
f'{d}'
)
if len(days_list) > max_show:
pills += f'+{len(days_list)-max_show} lainnya'
return pills
if not busiest_list:
busy_main, busy_count, busy_pills = "—", "Belum ada data", ""
elif len(busiest_list) == 1:
busy_main, busy_count, busy_pills = busiest_list[0], f"{max_cnt:,} tweet", ""
else:
busy_main = f"{len(busiest_list)} hari tertinggi"
busy_count = f"masing-masing {max_cnt:,} tweet"
busy_pills = _date_pills(busiest_list, "#ea580c")
if not quietest_list:
quiet_main, quiet_count, quiet_pills = "—", "Belum ada data aktif", ""
elif len(quietest_list) == 1:
quiet_main, quiet_count, quiet_pills = quietest_list[0], f"{min_cnt:,} tweet", ""
else:
quiet_main = f"{len(quietest_list)} hari terendah"
quiet_count = f"masing-masing {min_cnt:,} tweet"
quiet_pills = _date_pills(quietest_list, "#7c3aed")
pct_active = round(active_days / total_days * 100) if total_days > 0 else 0
if active_days == 0:
aktif_main, aktif_count, aktif_pills = "0 Hari Aktif", f"Dari {total_days} hari, belum ada data", ""
elif active_days == total_days:
aktif_main = f"{active_days} / {total_days} Hari"
aktif_count = f"100% hari ada tweet · total {total_tweets:,} tweet"
aktif_pills = ""
else:
aktif_main = f"{active_days} / {total_days} Hari"
aktif_count = f"{pct_active}% hari aktif · {inactive_days} hari kosong"
aktif_pills = ""
def _insight_card(col, icon, bg, color, dark, border_c, title, main, count_txt, pills_html):
with col:
pills_block = (
f'{pills_html}
'
if pills_html else ""
)
st.markdown(f"""
{main}
{count_txt}
{pills_block}
""", unsafe_allow_html=True)
_insight_card(pc1, "🔥", "linear-gradient(135deg,#fff7ed,#ffedd5)",
"#ea580c", "#7c2d12", "#fed7aa",
"Hari Paling Ramai", busy_main, busy_count, busy_pills)
_insight_card(pc2, "🌙", "linear-gradient(135deg,#f5f3ff,#ede9fe)",
"#7c3aed", "#3b0764", "#ddd6fe",
"Hari Paling Sepi", quiet_main, quiet_count, quiet_pills)
_insight_card(pc3, "📆", "linear-gradient(135deg,#f0fdf4,#dcfce7)",
"#16a34a", "#14532d", "#bbf7d0",
"Hari Aktif", aktif_main, aktif_count, aktif_pills)
# ═══════════════════════════════════════════════════════════
# FILTER DATA
# ═══════════════════════════════════════════════════════════
def get_filtered_data(df_all):
now = user_now()
today = now.date()
mode = st.session_state.analysis_mode
if mode == "realtime":
mode_display = "Tweet Terkini — 7 Hari Terakhir"
mode_color = "#16a34a"
mode_icon = "📡"
date_col = "created_at"
basis_label = "Berdasarkan tanggal asli tweet"
chart_label = "tanggal asli tweet"
dt_start = datetime.combine(today - timedelta(days=6), datetime.min.time())
dt_end = datetime.combine(today, datetime.max.time().replace(microsecond=0))
elif mode == "30days":
mode_display = "30 Hari Terakhir"
mode_color = "#3b6cf7"
mode_icon = "📅"
date_col = "created_at"
basis_label = "Berdasarkan tanggal asli tweet"
chart_label = "tanggal asli tweet"
dt_start = datetime.combine(today - timedelta(days=29), datetime.min.time())
dt_end = datetime.combine(today, datetime.max.time().replace(microsecond=0))
elif mode == "captured":
mode_display = "Tweet Hari Ini"
mode_color = "#0284c7"
mode_icon = "📆"
date_col = "created_at"
basis_label = "Berdasarkan tanggal asli tweet"
chart_label = "tanggal asli tweet"
dt_start = datetime.combine(today, datetime.min.time())
dt_end = datetime.combine(today, datetime.max.time().replace(microsecond=0))
else:
start_date = st.session_state.get("custom_start_date", today)
end_date = st.session_state.get("custom_end_date", today)
mode_display = "Periode Historis Pilihan"
mode_color = "#d97706"
mode_icon = "🔍"
date_col = "created_at"
basis_label = "Berdasarkan tanggal asli tweet"
chart_label = "tanggal asli tweet"
dt_start = datetime.combine(start_date, datetime.min.time())
dt_end = datetime.combine(end_date, datetime.max.time().replace(microsecond=0))
df_source = df_all.dropna(subset=[date_col]).copy()
df = df_source[
(df_source[date_col] >= pd.Timestamp(dt_start)) &
(df_source[date_col] <= pd.Timestamp(dt_end))
].copy()
filter_label = f"{dt_start.strftime('%d/%m/%Y')} s/d {dt_end.strftime('%d/%m/%Y')}"
return {
"df": df,
"date_col": date_col,
"mode_display": mode_display,
"mode_color": mode_color,
"mode_icon": mode_icon,
"filter_label": filter_label,
"dt_start": dt_start,
"dt_end": dt_end,
"basis_label": basis_label,
"chart_label": chart_label,
}
# ═══════════════════════════════════════════════════════════
# MAIN PAGE
# ═══════════════════════════════════════════════════════════
def show():
today = user_today()
last_7_days = today - timedelta(days=6)
_render_crawling_styles()
if "filter_start_date" not in st.session_state:
st.session_state.filter_start_date = last_7_days
if "filter_end_date" not in st.session_state:
st.session_state.filter_end_date = today
# ── Page Header ──────────────────────────────────────────────
st.markdown(f"""
""", unsafe_allow_html=True)
# ── Mode Selector ─────────────────────────────────────────────
if "analysis_mode" not in st.session_state:
st.session_state.analysis_mode = "realtime"
col1, col2, col3, col4 = st.columns(4, gap="small")
MODE_CFG = {
"captured": {
"col": col1, "key": "btn_captured", "label": "📆 Hari Ini",
"title": "Tweet Hari Ini", "icon": "📆",
"desc": "Tanggal asli hari ini",
"gradient": "linear-gradient(135deg,#eff6ff,#dbeafe)",
"border": "#bfdbfe", "color": "#1d4ed8", "dark": "#1e3a8a",
},
"realtime": {
"col": col2, "key": "btn_realtime", "label": "📡 7 Hari",
"title": "Terkini (7 Hari)", "icon": "📡",
"desc": "7 hari terakhir",
"gradient": "linear-gradient(135deg,#f0fdf4,#dcfce7)",
"border": "#86efac", "color": "#16a34a", "dark": "#14532d",
},
"30days": {
"col": col3, "key": "btn_30days", "label": "📅 30 Hari",
"title": "30 Hari Terakhir", "icon": "📅",
"desc": "30 hari terakhir",
"gradient": "linear-gradient(135deg,#eef2ff,#e0e7ff)",
"border": "#a5b4fc", "color": "#3b6cf7", "dark": "#1e3a8a",
},
"custom": {
"col": col4, "key": "btn_custom", "label": "🔍 Pilih Tanggal",
"title": "Pilih Tanggal", "icon": "🔍",
"desc": "Historis custom",
"gradient": "linear-gradient(135deg,#fefce8,#fef9c3)",
"border": "#fde68a", "color": "#d97706", "dark": "#713f12",
},
}
for mode_key, cfg in MODE_CFG.items():
active = st.session_state.analysis_mode == mode_key
with cfg["col"]:
if st.button(
cfg["label"], use_container_width=True,
type="primary" if active else "secondary",
key=cfg["key"]
):
st.session_state.analysis_mode = mode_key
st.rerun()
bg = cfg["gradient"] if active else "#ffffff"
color = cfg["color"] if active else "#94a3b8"
dark = cfg["dark"] if active else "#475569"
shadow = f"0 4px 16px {cfg['border']}55" if active else "0 1px 4px rgba(15,23,42,0.04)"
border_style = f"2px solid {cfg['color']}" if active else "1.5px solid #e8edf5"
icon_shadow = f"0 4px 10px {cfg['border']}88" if active else "none"
icon_bg = "white" if active else "#f1f5f9"
underline = (
f''
if active else ""
)
st.markdown(f"""
{cfg['icon']}
{cfg['title']}
{cfg['desc']}
{underline}
""", unsafe_allow_html=True)
_gap("md")
# ── Load data ─────────────────────────────────────────────────
try:
df_all = pd.read_sql("SELECT * FROM tweets ORDER BY created_at DESC", engine)
except Exception as e:
st.error(f"❌ Gagal membaca database: {str(e)}")
return
if len(df_all) == 0:
st.warning("Belum ada data di database.")
return
df_all["created_at"] = parse_dt(df_all["created_at"])
df_all["crawled_at"] = parse_crawled_dt(df_all["crawled_at"])
# ── Custom Date Picker ────────────────────────────────────────
if st.session_state.analysis_mode == "custom":
df_check = df_all.dropna(subset=["created_at"]).copy()
if not df_check.empty:
min_db = df_check["created_at"].min().date()
max_db = df_check["created_at"].max().date()
st.markdown(f"""
📅 Pilih Rentang Tanggal Tweet Historis
Data tersedia dari
{min_db.strftime('%d/%m/%Y')}
hingga
{max_db.strftime('%d/%m/%Y')}
""", unsafe_allow_html=True)
with st.container():
c1, c2, c3 = st.columns([2, 2, 1], gap="medium")
try:
start_value = pd.to_datetime(st.session_state.get("custom_start_date", min_db)).date()
except Exception:
start_value = min_db
try:
end_value = pd.to_datetime(st.session_state.get("custom_end_date", max_db)).date()
except Exception:
end_value = max_db
if start_value < min_db or start_value > max_db:
start_value = min_db
if end_value < min_db or end_value > max_db:
end_value = max_db
if end_value < start_value:
end_value = start_value
with c1:
cs = st.date_input("Dari Tanggal", value=start_value,
min_value=min_db, max_value=max_db,
key="custom_start_input")
st.session_state.custom_start_date = cs
with c2:
ce = st.date_input("Sampai Tanggal", value=end_value,
min_value=min_db, max_value=max_db,
key="custom_end_input")
st.session_state.custom_end_date = ce
with c3:
if st.button("✅ Terapkan", type="primary", use_container_width=True):
st.success("✅ Periode diterapkan!")
time.sleep(0.8)
st.rerun()
# ═══════════════════════════════════════════════════════════
# MODE: TWEET HARI INI (captured) — monitoring + log table
# ═══════════════════════════════════════════════════════════
if st.session_state.analysis_mode == "captured":
st_autorefresh(interval=1000, key="crawler_monitor_refresh")
logs = get_auto_crawl_logs(5)
last_log = logs[0] if logs else None
sched_ok = is_crawler_service_active(logs)
last_dt = parse_log_time(last_log.get("timestamp")) if last_log else None
_render_nrt_status_banner(sched_ok)
_gap("sm")
_render_monitoring_panel(last_log, last_dt, sched_ok)
_gap("md")
# ── Crawl History ──────────────────────────────────────
visible_logs = get_visible_crawl_logs(logs)
if visible_logs:
rows = []
for idx, log in enumerate(visible_logs, start=1):
status = log.get("status", "")
total_saved = int(log.get("total_saved") or 0)
error_msg = log.get("error")
try:
crawl_time = format_dt(
parse_dt(pd.Series([parse_log_time(log.get("timestamp"))])).iloc[0]
)
except Exception:
crawl_time = log.get("timestamp", "-") or "-"
if status == "success":
status_label = "✅ Berhasil"
note = (
f"{total_saved:,} tweet baru tersimpan"
if total_saved > 0
else "Crawling berhasil, tidak ada tweet baru"
)
else:
status_label = "❌ Gagal"
note = error_msg or "Terjadi kesalahan saat crawling"
rows.append({
"No": idx,
"Waktu Crawl": crawl_time,
"Status": status_label,
"Tweet Baru": total_saved,
"Keterangan": note,
})
_section_header(
"🕒 Riwayat Crawling Otomatis",
f"{len(rows)} aktivitas terbaru · interval {NRT_INTERVAL_MINUTES} menit"
)
_gap("xs")
render_standard_table(
pd.DataFrame(rows),
height=300, min_width=760,
right_align=["Tweet Baru"],
badge_columns=["Status"],
nowrap=["No", "Waktu Crawl", "Status"],
wide_columns=["Keterangan"],
column_widths={
"No": "56px", "Waktu Crawl": "160px",
"Status": "138px", "Tweet Baru": "118px", "Keterangan": "320px",
},
)
else:
st.markdown(f"""
⏳
Menunggu Crawl Pertama
Riwayat akan muncul setelah crawl otomatis pertama selesai
(maks. {NRT_INTERVAL_MINUTES} menit sejak aplikasi dibuka)
""", unsafe_allow_html=True)
_gap("lg")
# ── Tweet yang dicrawl bot hari ini ───────────────────
today_start = datetime.combine(today, datetime.min.time())
today_end = datetime.combine(today, datetime.max.time().replace(microsecond=0))
bot_df = df_all.copy()
if "crawl_type" in bot_df.columns:
bot_df = bot_df[bot_df["crawl_type"].fillna("").str.lower().eq("realtime")].copy()
bot_df = bot_df.dropna(subset=["crawled_at"])
bot_df = bot_df[
(bot_df["crawled_at"] >= pd.Timestamp(today_start)) &
(bot_df["crawled_at"] <= pd.Timestamp(today_end))
].sort_values("crawled_at", ascending=False)
_section_header(
"🤖 Tweet yang Diambil Bot Hari Ini",
"Semua tweet yang berhasil disimpan crawler hari ini"
)
_gap("xs")
if bot_df.empty:
st.markdown(f"""
🐦
Belum Ada Tweet Hari Ini
Tweet baru akan muncul setelah crawl berikutnya selesai
""", unsafe_allow_html=True)
else:
bot_disp = bot_df.head(100).copy()
bot_disp["Tanggal Tweet"] = bot_disp["created_at"].apply(format_dt)
bot_disp["Waktu Bot Ambil"] = bot_disp["crawled_at"].apply(format_dt)
render_standard_table(
bot_disp[["tweet_id", "text", "Tanggal Tweet", "Waktu Bot Ambil"]].rename(
columns={"tweet_id": "ID Tweet", "text": "Isi Tweet"}
),
height=340, min_width=920,
nowrap=["ID Tweet", "Tanggal Tweet", "Waktu Bot Ambil"],
wide_columns=["Isi Tweet"],
column_widths={
"ID Tweet": "170px", "Isi Tweet": "430px",
"Tanggal Tweet": "150px", "Waktu Bot Ambil": "160px",
},
)
st.caption(
f"Menampilkan {len(bot_disp):,} dari {len(bot_df):,} tweet yang disimpan bot hari ini"
)
# ═══════════════════════════════════════════════════════════
# FILTERED DATA — tampil di semua mode
# ═══════════════════════════════════════════════════════════
result = get_filtered_data(df_all)
df = result["df"]
date_col = result["date_col"]
mode_display = result["mode_display"]
mode_color = result["mode_color"]
mode_icon = result["mode_icon"]
filter_label = result["filter_label"]
dt_start = result["dt_start"]
dt_end = result["dt_end"]
basis_label = result["basis_label"]
chart_label = result["chart_label"]
st.session_state.filter_start_date = dt_start
st.session_state.filter_end_date = dt_end
st.session_state.filter_label = filter_label
st.session_state.mode_display = mode_display
st.session_state.filter_date_column = date_col
_divider()
# ── Active Filter Banner ──────────────────────────────────
total = len(df)
st.markdown(f"""
{mode_icon}
{mode_display}
Periode aktif: {filter_label}
· {basis_label}
{total:,} tweet
""", unsafe_allow_html=True)
days = (pd.Timestamp(dt_end).date() - pd.Timestamp(dt_start).date()).days + 1
earliest = pd.Timestamp(dt_start).strftime("%d/%m/%Y")
latest = pd.Timestamp(dt_end).strftime("%d/%m/%Y")
_render_metrics(total, days, earliest, latest, filter_label, basis_label)
if total == 0:
_section_header("📋 Daftar Tweet yang Terkumpul", f"0 tweet · {filter_label}")
st.markdown(f"""
🔍
Tidak Ada Data
Belum ada tweet dengan tanggal asli pada periode ini
""", unsafe_allow_html=True)
return
# ── Chart ─────────────────────────────────────────────────
st.markdown(f"""
📊 Distribusi Tweet Per Hari
{filter_label}
Berdasarkan {chart_label}
""", unsafe_allow_html=True)
_render_charts(df, filter_label, date_col, chart_label, dt_start, dt_end)
# ── Tabel ─────────────────────────────────────────────────
display_limit = 100
_section_header(
"📋 Daftar Tweet yang Terkumpul",
f"Menampilkan hingga {display_limit} tweet terbaru · {filter_label}"
)
_gap("xs")
df_disp = df.sort_values(date_col, ascending=False).head(display_limit).copy()
df_disp["Tanggal Tweet"] = df_disp["created_at"].apply(format_dt)
df_disp["Masuk Database"] = df_disp["crawled_at"].apply(format_dt)
render_standard_table(
df_disp[["tweet_id", "text", "Tanggal Tweet", "Masuk Database"]].rename(
columns={"tweet_id": "ID Tweet", "text": "Isi Tweet"}
),
height=380, min_width=840,
nowrap=["ID Tweet", "Tanggal Tweet", "Masuk Database"],
wide_columns=["Isi Tweet"],
column_widths={
"ID Tweet": "180px", "Isi Tweet": "420px",
"Tanggal Tweet": "150px", "Masuk Database": "155px",
},
)
st.caption(f"Menampilkan {len(df_disp):,} dari {total:,} tweet")
_gap("sm")
# ── Actions ───────────────────────────────────────────────
c_dl, c_nav = st.columns(2)
with c_dl:
st.download_button(
f"📥 Unduh Semua Data ({total:,} tweet)",
df.to_csv(index=False).encode("utf-8"),
f"data_twitter_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv",
"text/csv",
use_container_width=True
)
with c_nav:
if st.button(
"🧹 Lanjut ke Bersihkan Data →",
type="primary",
use_container_width=True,
key="btn_go_preprocessing"
):
st.session_state.page = "preprocessing"
st.rerun()
_gap("sm")