import os
import time
from datetime import datetime, timedelta, timezone
import pandas as pd
import plotly.graph_objects as go
import streamlit as st
from streamlit_autorefresh import st_autorefresh
from page_modules.table_utils import render_standard_table
from database import engine
from timezone_utils import (
parse_dt_with_tz,
parse_dt_with_source_tz,
get_timezone_label,
get_timezone_name,
)
from crawler import (
get_auto_crawl_logs,
get_crawler_state,
AUTO_CRAWL_INTERVAL_HOURS,
NRT_INTERVAL_MINUTES,
auto_crawl_job,
)
def parse_dt(series):
return parse_dt_with_tz(
series,
st.session_state.get("user_timezone", "WIB (UTC+7)")
)
def parse_crawled_dt(series):
return parse_dt_with_source_tz(
series,
st.session_state.get("user_timezone", "WIB (UTC+7)"),
os.getenv("APP_TIMEZONE", "Asia/Makassar")
)
def format_dt(value):
if value is None or pd.isna(value):
return "Belum ada"
try:
if not hasattr(value, "strftime"):
value = parse_dt(pd.Series([value])).iloc[0]
if value is None or pd.isna(value):
return "Belum ada"
tz_label = get_timezone_label(
st.session_state.get("user_timezone", "WIB (UTC+7)")
)
return f"{value.strftime('%d/%m/%Y %H:%M')} {tz_label}"
except Exception:
return "Belum ada"
def user_now():
timezone_choice = st.session_state.get("user_timezone", "WIB (UTC+7)")
return pd.Timestamp.now(tz=get_timezone_name(timezone_choice)).tz_localize(None)
def user_today():
return user_now().date()
def format_countdown(delta):
total_seconds = max(0, int(delta.total_seconds()))
minutes, seconds = divmod(total_seconds, 60)
return f"{minutes:02d}:{seconds:02d}"
def parse_log_time(value):
try:
parsed = pd.to_datetime(value, errors="coerce")
except Exception:
return None
if pd.isna(parsed):
return None
if parsed.tzinfo is None:
parsed = parsed.tz_localize(os.getenv("APP_TIMEZONE", "Asia/Makassar"))
return parsed.tz_convert("UTC").to_pydatetime()
def to_user_local_datetime(value):
if value is None or pd.isna(value):
return None
try:
parsed = pd.to_datetime(value, errors="coerce", format="mixed")
except Exception:
return None
if pd.isna(parsed):
return None
if parsed.tzinfo is None:
return parsed.to_pydatetime() if hasattr(parsed, "to_pydatetime") else parsed
converted = parse_dt(pd.Series([parsed])).iloc[0]
if converted is None or pd.isna(converted):
return None
return converted.to_pydatetime() if hasattr(converted, "to_pydatetime") else converted
def get_visible_crawl_logs(logs):
started_at = st.session_state.get("crawl_history_started_at")
if started_at is None:
return logs
started_at = parse_log_time(started_at.isoformat())
visible = []
for log in logs:
log_dt = parse_log_time(log.get("timestamp"))
if log_dt is not None and log_dt >= started_at:
visible.append(log)
return visible
def is_crawler_running():
state = get_crawler_state()
updated_at = _parse_state_datetime(state.get("updated_at"))
if updated_at is None:
return False
return (
state.get ("is_running", False)
and datetime.now(timezone.utc) - updated_at <= _crawler_state_threshold()
)
def _parse_state_datetime(value):
if not value:
return None
try:
parsed = datetime.fromisoformat(value)
except Exception:
return None
if parsed.tzinfo is None:
parsed = pd.Timestamp(parsed).tz_localize(
os.getenv("APP_TIMEZONE", "Asia/Makassar")
).to_pydatetime()
return parsed.astimezone(timezone.utc)
def _crawler_state_threshold():
return timedelta(minutes=max((NRT_INTERVAL_MINUTES * 2) + 1, 3))
def is_crawler_service_active(logs=None):
state = get_crawler_state()
threshold = _crawler_state_threshold()
now = datetime.now(timezone.utc)
if not state:
return False
if is_crawler_running():
return True
heartbeat_at = _parse_state_datetime(state.get("heartbeat_at"))
if (
state.get("service_active", False)
and heartbeat_at is not None
and now - heartbeat_at <= threshold
):
return True
return False
def _section_header(title, subtitle=""):
"""Render consistent section header card"""
sub_html = (
f'{subtitle}
'
if subtitle else ""
)
st.markdown(f"""
""", unsafe_allow_html=True)
def _captured_section_gap(size="md"):
heights = {
"sm": 12,
"md": 20,
"lg": 28,
}
height = heights.get(size, heights["md"])
st.markdown(
f'',
unsafe_allow_html=True
)
def _render_crawling_styles():
st.markdown("""
""", unsafe_allow_html=True)
def _render_metrics(total, days, earliest, latest, filter_label, basis_label):
st.markdown(f"""
๐ Data aktif: {filter_label} ยท {basis_label}
""", unsafe_allow_html=True)
c1, c2, c3, c4 = st.columns(4)
pills = [
(
c1,
"๐",
"#eef2ff",
"#3b6cf7",
"#1e3a8a",
"Total Tweet",
f"{total:,}",
"Terkumpul dalam periode ini"
),
(
c2,
"๐
",
"#f0fdf4",
"#16a34a",
"#14532d",
"Rentang Waktu Data",
f"{days} Hari",
basis_label
),
(
c3,
"๐๏ธ",
"#fff7ed",
"#ea580c",
"#7c2d12",
"Mulai Dari",
earliest,
"Tanggal awal periode"
),
(
c4,
"๐",
"#fefce8",
"#ca8a04",
"#713f12",
"Sampai Dengan",
latest,
"Tanggal akhir periode"
),
]
for col, icon, bg, color, dark, label, val, sub in pills:
with col:
fs = "1.05rem" if len(str(val)) > 10 else "1.55rem"
st.markdown(f"""
{icon}
{label}
{val}
{sub}
""", unsafe_allow_html=True)
def _render_charts(df, filter_label, date_col, chart_label, dt_start, dt_end):
df_tl = df.copy()
df_tl = df_tl.dropna(subset=[date_col])
if df_tl.empty:
st.info("Belum ada data valid untuk grafik.")
return
is_realtime_basis = date_col == "crawled_at"
chart_title = (
"๐
Jumlah Data Masuk Per Hari"
if is_realtime_basis else
"๐
Jumlah Tweet Per Hari"
)
count_label = "data masuk" if is_realtime_basis else "tweet"
avg_label = (
"Rata-rata Data Masuk per Hari"
if is_realtime_basis else
"Rata-rata Tweet per Hari"
)
df_tl["date_key"] = df_tl[date_col].dt.strftime("%Y-%m-%d")
actual_daily = (
df_tl
.groupby("date_key")
.size()
.reset_index(name="count")
.sort_values("date_key")
)
date_range = pd.date_range(
pd.Timestamp(dt_start).date(),
pd.Timestamp(dt_end).date(),
freq="D"
)
daily = pd.DataFrame({
"date_key": date_range.strftime("%Y-%m-%d"),
"date_str": date_range.strftime("%d %b"),
}).merge(
actual_daily,
on="date_key",
how="left"
)
daily["count"] = daily["count"].fillna(0).astype(int)
ordered = daily["date_str"].tolist()
counts_day = daily["count"].tolist()
avg_per_day = round(len(df_tl) / len(daily), 1) if len(daily) > 0 else 0
busiest_row = daily.loc[daily["count"].idxmax()] if len(daily) > 0 and daily["count"].max() > 0 else None
busiest_day = busiest_row["date_str"] if busiest_row is not None else "-"
busiest_cnt = int(busiest_row["count"]) if busiest_row is not None else 0
_section_header(
chart_title,
f"Dihitung berdasarkan {chart_label} ยท {filter_label}"
)
bar_clrs = [
"#1d4ed8" if c == busiest_cnt else "#3b6cf7"
for c in counts_day
]
fig_bar = go.Figure(data=[
go.Bar(
x=ordered,
y=counts_day,
marker=dict(color=bar_clrs, line=dict(width=0), opacity=0.88),
width=0.5,
hovertemplate=f"%{{x}}
%{{y:,}} {count_label}",
text=[str(c) for c in counts_day],
textposition="outside",
textfont=dict(size=10, color="#475569"),
cliponaxis=False,
)
])
fig_bar.update_layout(
height=340,
margin=dict(l=0, r=10, t=30, b=10),
paper_bgcolor="rgba(0,0,0,0)",
plot_bgcolor="rgba(0,0,0,0)",
xaxis=dict(
type="category",
categoryorder="array",
categoryarray=ordered,
tickfont=dict(size=11, color="#334155"),
showgrid=False,
zeroline=False,
showline=False,
fixedrange=True
),
yaxis=dict(
tickfont=dict(size=10, color="#475569"),
showgrid=True,
gridcolor="rgba(203,213,225,0.8)",
griddash="dot",
gridwidth=1,
zeroline=False,
showline=False,
fixedrange=True
),
showlegend=False,
hovermode="x unified",
)
st.plotly_chart(
fig_bar,
width="stretch",
config={"displayModeBar": False}
)
pc1, pc2 = st.columns(2)
with pc1:
st.markdown(f"""
๐
{avg_label}
{avg_per_day} {count_label}
""", unsafe_allow_html=True)
with pc2:
st.markdown(f"""
๐ฅ
Hari {count_label.title()} Terbanyak
{busiest_day}
({busiest_cnt:,} {count_label})
""", unsafe_allow_html=True)
def _render_nrt_status_banner(sched_ok):
if sched_ok:
bg = "linear-gradient(135deg,#f0fdf4,#dcfce7)"
bdr = "#86efac"
dot = "#16a34a"
title = "๐ข Crawler Service Berjalan sebagai Background Job"
desc = (
f"Sistem crawler service berjalan di background secara otomatis "
f"setiap {NRT_INTERVAL_MINUTES} menit. "
f"Panel monitoring membaca update realtime berdasarkan crawled_at."
)
else:
bg = "linear-gradient(135deg,#fef9c3,#fef08a)"
bdr = "#fcd34d"
dot = "#f59e0b"
title = "โ ๏ธ Crawler Service Belum Aktif"
desc = (
f"Jalankan python3 crawler.py di terminal terpisah "
f"untuk mengaktifkan crawler otomatis setiap "
f"{NRT_INTERVAL_MINUTES} menit."
)
st.markdown(f"""
""", unsafe_allow_html=True)
def _render_monitoring_panel(last_log, last_dt, sched_ok):
now = user_now()
try:
df_stats = pd.read_sql(
"SELECT crawled_at FROM tweets",
engine
)
if not df_stats.empty:
df_stats["crawled_at"] = parse_crawled_dt(df_stats["crawled_at"])
total_db = len(df_stats)
interval_start = now - timedelta(minutes=NRT_INTERVAL_MINUTES)
tweet_interval = (
len(df_stats[df_stats["crawled_at"] >= pd.Timestamp(interval_start)])
if not df_stats.empty else 0
)
latest_crawled = (
df_stats["crawled_at"].max()
if not df_stats.empty else None
)
except Exception:
total_db = 0
tweet_interval = 0
latest_crawled = None
last_log_dt = last_dt
if last_log_dt is None and last_log:
last_log_dt = parse_log_time(last_log.get("timestamp"))
final_last_dt = to_user_local_datetime(last_log_dt)
if final_last_dt is None:
final_last_dt = to_user_local_datetime(latest_crawled)
last_crawl_str = "Belum pernah"
next_crawl_str = "Belum terjadwal"
monitoring_time_str = format_dt(now)
countdown_str = "Belum terjadwal"
if final_last_dt is not None:
next_dt = final_last_dt + timedelta(minutes=NRT_INTERVAL_MINUTES)
remaining = next_dt - now
last_crawl_str = format_dt(final_last_dt)
next_crawl_str = format_dt(next_dt)
if remaining.total_seconds() > 0:
countdown_str = format_countdown(remaining)
elif is_crawler_running():
countdown_str = "Sedang crawling..."
else:
countdown_str = "Menunggu hasil crawl terbaru..."
status_text = "Active" if sched_ok else "Nonaktif"
status_color = "#16a34a" if sched_ok else "#ca8a04"
note_text = (
"โ
Data berhasil diperbarui"
if tweet_interval > 0 else
"โน๏ธ Tidak ada tweet baru pada periode ini"
)
note_bg = "#f0fdf4" if tweet_interval > 0 else "#f8fafc"
note_border = "#bbf7d0" if tweet_interval > 0 else "#e2e8f0"
note_color = "#166534" if tweet_interval > 0 else "#475569"
_section_header(
"๐ก Status Monitoring Sistem",
"Transparansi proses pengambilan data secara near real-time"
)
col_a, col_b = st.columns(2)
with col_a:
st.markdown(f"""
โ๏ธ STATUS SISTEM CRAWLING
Crawler Status
{status_text}
Interval Crawling
{NRT_INTERVAL_MINUTES} menit
Last Crawling
{last_crawl_str}
Waktu Monitoring
{monitoring_time_str}
Next Crawling
โฐ {next_crawl_str}
Countdown
{countdown_str}
""", unsafe_allow_html=True)
with col_b:
st.markdown(f"""
๐ RINGKASAN UPDATE TERAKHIR
Update terakhir
{last_crawl_str}
Waktu Monitoring
{monitoring_time_str}
Menuju Crawl Berikutnya
{countdown_str}
Tweet baru dalam {NRT_INTERVAL_MINUTES} menit terakhir
{tweet_interval:,} tweet baru
Total dataset terkumpul
{total_db:,} tweet
{note_text}
""", unsafe_allow_html=True)
def get_filtered_data(df_all):
now = user_now()
today = now.date()
mode = st.session_state.analysis_mode
if mode == "realtime":
mode_display = "Tweet Terkini โ 7 Hari Terakhir"
mode_color = "#16a34a"
mode_icon = "๐ก"
date_col = "created_at"
basis_label = "Berdasarkan tanggal asli tweet"
chart_label = "tanggal asli tweet"
dt_start = datetime.combine(
today - timedelta(days=6),
datetime.min.time()
)
dt_end = datetime.combine(
today,
datetime.max.time().replace(microsecond=0)
)
elif mode == "30days":
mode_display = "30 Hari Terakhir"
mode_color = "#3b6cf7"
mode_icon = "๐
"
date_col = "created_at"
basis_label = "Berdasarkan tanggal asli tweet"
chart_label = "tanggal asli tweet"
dt_start = datetime.combine(
today - timedelta(days=29),
datetime.min.time()
)
dt_end = datetime.combine(
today,
datetime.max.time().replace(microsecond=0)
)
elif mode == "captured":
mode_display = "Tweet Hari Ini"
mode_color = "#0284c7"
mode_icon = "๐"
date_col = "created_at"
basis_label = "Berdasarkan tanggal asli tweet"
chart_label = "tanggal asli tweet"
dt_start = datetime.combine(today, datetime.min.time())
dt_end = datetime.combine(now.date(), datetime.max.time().replace(microsecond=0))
else:
start_date = st.session_state.get("custom_start_date", now.date())
end_date = st.session_state.get("custom_end_date", now.date())
mode_display = "Periode Historis Pilihan"
mode_color = "#d97706"
mode_icon = "๐"
date_col = "created_at"
basis_label = "Berdasarkan tanggal asli tweet"
chart_label = "tanggal asli tweet"
dt_start = datetime.combine(start_date, datetime.min.time())
dt_end = datetime.combine(end_date, datetime.max.time().replace(microsecond=0))
df_source = df_all.dropna(subset=[date_col]).copy()
df = df_source[
(df_source[date_col] >= pd.Timestamp(dt_start)) &
(df_source[date_col] <= pd.Timestamp(dt_end))
].copy()
filter_label = f"{dt_start.strftime('%d/%m/%Y')} s/d {dt_end.strftime('%d/%m/%Y')}"
return {
"df": df,
"date_col": date_col,
"mode_display": mode_display,
"mode_color": mode_color,
"mode_icon": mode_icon,
"filter_label": filter_label,
"dt_start": dt_start,
"dt_end": dt_end,
"basis_label": basis_label,
"chart_label": chart_label,
}
def show():
today = user_today()
last_7_days = today - timedelta(days=6)
_render_crawling_styles()
if "filter_start_date" not in st.session_state:
st.session_state.filter_start_date = last_7_days
if "filter_end_date" not in st.session_state:
st.session_state.filter_end_date = today
st.markdown("""
""", unsafe_allow_html=True)
st.markdown(f"""
๐ Apa yang dilakukan halaman ini?
Sistem menggunakan database historis sebagai data awal, lalu crawler menambahkan data baru
secara berkala setiap {NRT_INTERVAL_MINUTES} menit.
Semua filter tampilan membaca created_at atau tanggal asli tweet;
panel monitoring crawler tetap memakai crawled_at.
""", unsafe_allow_html=True)
if "analysis_mode" not in st.session_state:
st.session_state.analysis_mode = "realtime"
col1, col2, col3, col4 = st.columns(4)
MODE_CFG = {
"realtime": {
"col": col1,
"key": "btn_realtime",
"label": "๐ก Terkini (7 Hari)",
"title": "Tweet Terkini",
"icon": "๐ก",
"desc": "Tweet bertanggal asli dalam 7 hari terakhir",
"active_bg": "linear-gradient(135deg,#dcfce7,#bbf7d0)",
"active_border": "#22c55e",
"active_tc": "#14532d",
"icon_bg": "#16a34a",
},
"30days": {
"col": col2,
"key": "btn_30days",
"label": "๐
30 Hari Terakhir",
"title": "30 Hari Terakhir",
"icon": "๐
",
"desc": "Tweet bertanggal asli dalam 30 hari terakhir",
"active_bg": "linear-gradient(135deg,#eef2ff,#e0e7ff)",
"active_border": "#3b6cf7",
"active_tc": "#1e3a8a",
"icon_bg": "#3b6cf7",
},
"captured": {
"col": col3,
"key": "btn_captured",
"label": "๐ Tweet Hari Ini",
"title": "Tweet Hari Ini",
"icon": "๐",
"desc": "Tweet bertanggal asli hari ini",
"active_bg": "linear-gradient(135deg,#f0f9ff,#dbeafe)",
"active_border": "#0ea5e9",
"active_tc": "#0c4a6e",
"icon_bg": "#0284c7",
},
"custom": {
"col": col4,
"key": "btn_custom",
"label": "๐ Pilih Tanggal",
"title": "Pilih Tanggal",
"icon": "๐",
"desc": "Pilih data historis berdasarkan tanggal asli tweet",
"active_bg": "linear-gradient(135deg,#fef9c3,#fef08a)",
"active_border": "#f59e0b",
"active_tc": "#713f12",
"icon_bg": "#d97706",
},
}
for mode_key, cfg in MODE_CFG.items():
active = st.session_state.analysis_mode == mode_key
with cfg["col"]:
if st.button(
cfg["label"],
width="stretch",
type="primary" if active else "secondary",
key=cfg["key"]
):
st.session_state.analysis_mode = mode_key
st.rerun()
bg = cfg["active_bg"] if active else "#f8fafc"
bdr = cfg["active_border"] if active else "#e2e8f0"
tc = cfg["active_tc"] if active else "#475569"
ic_bg = cfg["icon_bg"] if active else "#e2e8f0"
ic_c = "white" if active else "#64748b"
title_c = cfg["active_tc"] if active else "#0f172a"
bdr_w = "2px" if active else "1.5px"
st.markdown(f"""
{cfg["icon"]}
{cfg["title"]}
{cfg["desc"]}
""", unsafe_allow_html=True)
st.markdown("
", unsafe_allow_html=True)
try:
df_all = pd.read_sql(
"SELECT * FROM tweets ORDER BY created_at DESC",
engine
)
except Exception as e:
st.error(f"โ Gagal membaca database: {str(e)}")
return
if len(df_all) == 0:
st.warning("Belum ada data di database.")
return
df_all["created_at"] = parse_dt(df_all["created_at"])
df_all["crawled_at"] = parse_crawled_dt(df_all["crawled_at"])
if st.session_state.analysis_mode == "custom":
df_check = df_all.dropna(subset=["created_at"]).copy()
if not df_check.empty:
min_db = df_check["created_at"].min().date()
max_db = df_check["created_at"].max().date()
st.markdown(f"""
๐
Pilih Rentang Tanggal Tweet Historis
Data tersedia dari {min_db.strftime('%d/%m/%Y')}
hingga {max_db.strftime('%d/%m/%Y')}
""", unsafe_allow_html=True)
with st.container(border=True, key="custom_date_panel"):
c1, c2, c3 = st.columns(
[2, 2, 1],
gap="medium",
vertical_alignment="bottom"
)
try:
start_value = pd.to_datetime(
st.session_state.get("custom_start_date", min_db)
).date()
except Exception:
start_value = min_db
try:
end_value = pd.to_datetime(
st.session_state.get("custom_end_date", max_db)
).date()
except Exception:
end_value = max_db
if start_value < min_db or start_value > max_db:
start_value = min_db
if end_value < min_db or end_value > max_db:
end_value = max_db
if end_value < start_value:
end_value = start_value
with c1:
cs = st.date_input(
"Dari Tanggal",
value=start_value,
min_value=min_db,
max_value=max_db,
key="custom_start_input"
)
st.session_state.custom_start_date = cs
with c2:
ce = st.date_input(
"Sampai Tanggal",
value=end_value,
min_value=min_db,
max_value=max_db,
key="custom_end_input"
)
st.session_state.custom_end_date = ce
with c3:
if st.button("โ
Terapkan", type="primary", width="stretch"):
st.success("โ
Periode diterapkan!")
time.sleep(0.8)
st.rerun()
if st.session_state.analysis_mode == "captured":
st_autorefresh(
interval=1000,
key="crawler_monitor_refresh"
)
logs = get_auto_crawl_logs(5)
last_log = logs[0] if logs else None
sched_ok = is_crawler_service_active(logs)
last_dt = None
if last_log:
last_dt = parse_log_time(last_log.get("timestamp"))
_section_header(
"๐ค Ambil Data dengan Bot",
"Panel ini khusus mode Tweet Hari Ini agar status crawler tidak memenuhi tampilan Terkini"
)
_render_nrt_status_banner(sched_ok)
_captured_section_gap("sm")
_render_monitoring_panel(last_log, last_dt, sched_ok)
crawler_running = is_crawler_running()
with st.container(border=True, key="bot_control_panel"):
col_status, col_refresh, col_action = st.columns(
[2.6, 1.1, 1.5],
gap="medium",
vertical_alignment="center"
)
with col_status:
if crawler_running:
st.info("โณ Crawler sedang berjalan โ tombol ambil data dinonaktifkan sementara")
elif sched_ok:
st.success("โ
Crawler aktif โ data diperbarui otomatis")
else:
st.info("โน๏ธ Jalankan `python3 crawler.py` untuk mengaktifkan crawler")
with col_refresh:
previous_auto_refresh = st.session_state.get("auto_refresh_ui", False)
auto_refresh = st.toggle(
"๐ Refresh Otomatis",
value=previous_auto_refresh,
key="ar_toggle",
help=f"Halaman diperbarui otomatis setiap {NRT_INTERVAL_MINUTES} menit"
)
st.session_state.auto_refresh_ui = auto_refresh
if auto_refresh and not previous_auto_refresh:
st.session_state.crawl_history_started_at = datetime.now(timezone.utc)
st.rerun()
with col_action:
if st.button(
"โณ Crawler Berjalan" if crawler_running else "๐ฅ Ambil Data",
type="secondary" if crawler_running else "primary",
disabled=crawler_running,
width="stretch",
key="btn_manual_crawl_now"
):
st.session_state.crawl_history_started_at = datetime.now(timezone.utc)
with st.spinner("โณ Mengambil tweet terbaru..."):
auto_crawl_job()
st.success("โ
Data berhasil diambil!")
st.rerun()
_captured_section_gap("md")
if st.session_state.get("auto_refresh_ui"):
st.caption(
"๐ Refresh otomatis aktif. Riwayat akan ikut bertambah saat crawler menulis log baru."
)
_captured_section_gap("sm")
visible_logs = get_visible_crawl_logs(logs)
if visible_logs:
rows = []
for idx, log in enumerate(visible_logs, start=1):
status = log.get("status", "")
total_saved = int(log.get("total_saved") or 0)
error_msg = log.get("error")
try:
crawl_time = format_dt(
parse_dt(pd.Series([parse_log_time(log.get("timestamp"))])).iloc[0]
)
except Exception:
crawl_time = log.get("timestamp", "-") or "-"
if status == "success":
status_label = "โ
Berhasil"
note = (
f"{total_saved:,} tweet baru tersimpan"
if total_saved > 0 else
"Crawling berhasil, tidak ada tweet realtime yang cocok"
)
else:
status_label = "โ Gagal"
note = error_msg or "Terjadi kesalahan saat crawling"
rows.append({
"No": idx,
"Waktu Crawl": crawl_time,
"Status": status_label,
"Tweet Baru": total_saved,
"Keterangan": note,
})
_section_header(
"Riwayat Pengambilan Data",
f"Menampilkan {len(rows)} aktivitas crawler terbaru ยท {NRT_INTERVAL_MINUTES} menit/interval"
)
_captured_section_gap("sm")
render_standard_table(
pd.DataFrame(rows),
height=300,
min_width=760,
right_align=["Tweet Baru"],
badge_columns=["Status"],
nowrap=["No", "Waktu Crawl", "Status"],
wide_columns=["Keterangan"],
column_widths={
"No": "56px",
"Waktu Crawl": "160px",
"Status": "138px",
"Tweet Baru": "118px",
"Keterangan": "320px",
},
)
else:
st.info("Riwayat akan tampil setelah Anda menekan Ambil Data atau mengaktifkan Refresh Otomatis.")
_captured_section_gap("lg")
today_start = datetime.combine(today, datetime.min.time())
today_end = datetime.combine(
today,
datetime.max.time().replace(microsecond=0)
)
scrape_tabs = os.getenv("SCRAPE_TABS", "LATEST,TOP")
scrape_tabs_label = " + ".join(
tab.strip().upper()
for tab in scrape_tabs.split(",")
if tab.strip()
) or "LATEST"
bot_df = df_all.copy()
if "crawl_type" in bot_df.columns:
bot_df = bot_df[
bot_df["crawl_type"].fillna("").str.lower().eq("realtime")
].copy()
bot_df = bot_df.dropna(subset=["created_at", "crawled_at"])
bot_df = bot_df[
(bot_df["crawled_at"] >= pd.Timestamp(today_start)) &
(bot_df["crawled_at"] <= pd.Timestamp(today_end)) &
(bot_df["created_at"] >= pd.Timestamp(today_start)) &
(bot_df["created_at"] <= pd.Timestamp(today_end))
].sort_values("crawled_at", ascending=False)
bot_date_note = f"tanggal tweet hari ini ({today_start.strftime('%d/%m/%Y')})"
_section_header(
"๐ค Tweet yang Berhasil Diambil Bot",
f"{len(bot_df):,} tweet baru ยท sumber {scrape_tabs_label} ยท duplikat dilewati ยท {bot_date_note}"
)
_captured_section_gap("sm")
if bot_df.empty:
st.info("Belum ada tweet baru yang berhasil disimpan bot hari ini.")
else:
bot_disp = bot_df.head(100).copy()
bot_disp["Tanggal Tweet"] = bot_disp["created_at"].apply(format_dt)
bot_disp["Waktu Bot Ambil"] = bot_disp["crawled_at"].apply(format_dt)
render_standard_table(
bot_disp[
[
"tweet_id",
"text",
"Tanggal Tweet",
"Waktu Bot Ambil",
]
].rename(
columns={
"tweet_id": "ID Tweet",
"text": "Isi Tweet",
}
),
height=340,
min_width=920,
nowrap=["ID Tweet", "Tanggal Tweet", "Waktu Bot Ambil"],
wide_columns=["Isi Tweet"],
column_widths={
"ID Tweet": "170px",
"Isi Tweet": "430px",
"Tanggal Tweet": "150px",
"Waktu Bot Ambil": "160px",
},
)
st.caption(
f"Menampilkan {len(bot_disp):,} dari {len(bot_df):,} tweet yang disimpan bot hari ini"
)
result = get_filtered_data(df_all)
df = result["df"]
date_col = result["date_col"]
mode_display = result["mode_display"]
mode_color = result["mode_color"]
mode_icon = result["mode_icon"]
filter_label = result["filter_label"]
dt_start = result["dt_start"]
dt_end = result["dt_end"]
basis_label = result["basis_label"]
chart_label = result["chart_label"]
st.session_state.filter_start_date = dt_start
st.session_state.filter_end_date = dt_end
st.session_state.filter_label = filter_label
st.session_state.mode_display = mode_display
st.session_state.filter_date_column = date_col
st.markdown(f"""
{mode_icon}
{mode_display}
Periode:
{filter_label}
""", unsafe_allow_html=True)
total = len(df)
days = (pd.Timestamp(dt_end).date() - pd.Timestamp(dt_start).date()).days + 1
earliest = pd.Timestamp(dt_start).strftime("%d/%m/%Y")
latest = pd.Timestamp(dt_end).strftime("%d/%m/%Y")
_render_metrics(total, days, earliest, latest, filter_label, basis_label)
_captured_section_gap("sm")
if len(df) == 0:
_section_header(
"๐ Daftar Tweet yang Terkumpul",
f"0 tweet ยท {filter_label}"
)
st.info("Belum ada tweet dengan tanggal asli pada periode ini.")
return
st.markdown("
", unsafe_allow_html=True)
_render_charts(df, filter_label, date_col, chart_label, dt_start, dt_end)
st.markdown("
", unsafe_allow_html=True)
display_limit = 100
_section_header(
"๐ Daftar Tweet yang Terkumpul",
f"Data dari filter aktif ยท {filter_label}"
)
df_disp = df.sort_values(date_col, ascending=False).head(display_limit).copy()
df_disp["Tanggal Tweet"] = df_disp["created_at"].apply(format_dt)
df_disp["Masuk Database"] = df_disp["crawled_at"].apply(format_dt)
render_standard_table(
df_disp[
[
"tweet_id",
"text",
"Tanggal Tweet",
"Masuk Database"
]
].rename(
columns={
"tweet_id": "ID Tweet",
"text": "Isi Tweet"
}
),
height=380,
min_width=840,
nowrap=["ID Tweet", "Tanggal Tweet", "Masuk Database"],
wide_columns=["Isi Tweet"],
column_widths={
"ID Tweet": "180px",
"Isi Tweet": "420px",
"Tanggal Tweet": "150px",
"Masuk Database": "155px",
},
)
st.caption(f"Menampilkan {len(df_disp):,} dari {total:,} tweet")
st.markdown("
", unsafe_allow_html=True)
c_dl, c_nav = st.columns(2)
with c_dl:
st.download_button(
f"๐ฅ Unduh Semua Data ({total:,} tweet)",
df.to_csv(index=False).encode("utf-8"),
f"data_twitter_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv",
"text/csv",
width="stretch"
)
with c_nav:
if st.button(
"๐งน Lanjut ke Bersihkan Data โ",
type="primary",
width="stretch",
key="btn_go_preprocessing"
):
st.session_state.current_page = "preprocessing"
st.rerun()