penambahan button crawling
penambahan button crawling dan pebaikan status crawling service
This commit is contained in:
parent
cc8342c6cd
commit
8830accff5
204
app.py
204
app.py
|
|
@ -26,14 +26,10 @@ except Exception:
|
||||||
|
|
||||||
init_db()
|
init_db()
|
||||||
|
|
||||||
# ── Auto-start background crawler scheduler ──────────────────────────────────
|
# ── Auto-start background crawler scheduler (disabled) ───────────────────
|
||||||
# Dipanggil sekali saat app pertama dijalankan.
|
# Scheduler NRT hanya boleh berjalan setelah user menekan tombol aktivasi
|
||||||
# Scheduler berjalan sebagai daemon thread — tidak perlu terminal terpisah.
|
# pada mode "Hari Ini" (captured) agar sesuai kebutuhan (NRT dimulai manual).
|
||||||
try:
|
|
||||||
from crawler import ensure_scheduler_running
|
|
||||||
ensure_scheduler_running()
|
|
||||||
except Exception as _e:
|
|
||||||
print(f"[app] Gagal memulai scheduler: {_e}")
|
|
||||||
|
|
||||||
st.set_page_config(
|
st.set_page_config(
|
||||||
page_title="Dashboard Sentimen Twitter",
|
page_title="Dashboard Sentimen Twitter",
|
||||||
|
|
@ -164,8 +160,15 @@ def _check_crawler_alive():
|
||||||
and now - heartbeat_at <= threshold
|
and now - heartbeat_at <= threshold
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Hanya update _scheduler_started dari database jika user TIDAK sedang nonaktifkan eksplisit
|
||||||
|
_raw_alive = _check_crawler_alive()
|
||||||
|
explicitly_off = st.session_state.get("nrt_explicitly_deactivated", False)
|
||||||
|
|
||||||
st.session_state._scheduler_started = _check_crawler_alive()
|
if explicitly_off:
|
||||||
|
# User sudah klik nonaktifkan — override database state, paksa False
|
||||||
|
st.session_state._scheduler_started = False
|
||||||
|
else:
|
||||||
|
st.session_state._scheduler_started = _raw_alive
|
||||||
|
|
||||||
try:
|
try:
|
||||||
from crawler import NRT_INTERVAL_MINUTES
|
from crawler import NRT_INTERVAL_MINUTES
|
||||||
|
|
@ -190,8 +193,7 @@ st.markdown("""
|
||||||
--bg: #f8fafc;
|
--bg: #f8fafc;
|
||||||
--card: #ffffff;
|
--card: #ffffff;
|
||||||
}
|
}
|
||||||
|
html, body, [class*="css"], .stApp {
|
||||||
* {
|
|
||||||
font-family: 'Plus Jakarta Sans', sans-serif !important;
|
font-family: 'Plus Jakarta Sans', sans-serif !important;
|
||||||
box-sizing: border-box;
|
box-sizing: border-box;
|
||||||
}
|
}
|
||||||
|
|
@ -226,7 +228,7 @@ html, body, .stApp, [data-testid="stAppViewContainer"] {
|
||||||
}
|
}
|
||||||
|
|
||||||
.main > div {
|
.main > div {
|
||||||
padding: 1.5rem 2rem !important;
|
padding: 4.3rem 2rem 1.5rem 2rem !important;
|
||||||
}
|
}
|
||||||
|
|
||||||
[data-testid="stSidebar"] {
|
[data-testid="stSidebar"] {
|
||||||
|
|
@ -683,27 +685,192 @@ span:not([style]) {
|
||||||
box-shadow: 0 2px 6px rgba(15,23,42,0.05);
|
box-shadow: 0 2px 6px rgba(15,23,42,0.05);
|
||||||
}
|
}
|
||||||
|
|
||||||
#MainMenu, footer, header { visibility: hidden !important; }
|
/* =====================================================
|
||||||
|
CLEAN STREAMLIT HEADER + SIDEBAR
|
||||||
|
===================================================== */
|
||||||
|
|
||||||
[data-testid="stHeader"] { background: transparent !important; }
|
/* Hide bawaan Streamlit */
|
||||||
|
#MainMenu {
|
||||||
|
visibility: hidden !important;
|
||||||
|
}
|
||||||
|
|
||||||
|
footer {
|
||||||
|
visibility: hidden !important;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/* =====================================================
|
||||||
|
HEADER
|
||||||
|
===================================================== */
|
||||||
|
|
||||||
|
[data-testid="stHeader"] {
|
||||||
|
background: rgba(248,250,252,0.96) !important;
|
||||||
|
border-bottom: 1px solid #e2e8f0 !important;
|
||||||
|
backdrop-filter: blur(8px) !important;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* =====================================================
|
||||||
|
SIDEBAR
|
||||||
|
===================================================== */
|
||||||
|
|
||||||
|
[data-testid="stSidebar"] {
|
||||||
|
|
||||||
|
background: linear-gradient(
|
||||||
|
180deg,
|
||||||
|
#ffffff 0%,
|
||||||
|
#f8fafc 100%
|
||||||
|
) !important;
|
||||||
|
|
||||||
|
border-right: 1px solid #e2e8f0 !important;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* =====================================================
|
||||||
|
MAIN CONTENT
|
||||||
|
===================================================== */
|
||||||
|
|
||||||
|
.main > div {
|
||||||
|
|
||||||
|
padding-top: 4.4rem !important;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* =====================================================
|
||||||
|
SAFE SIDEBAR TOGGLE
|
||||||
|
===================================================== */
|
||||||
|
/* =====================================================
|
||||||
|
SAFE SIDEBAR TOGGLE
|
||||||
|
===================================================== */
|
||||||
|
|
||||||
|
[data-testid="stHeader"],
|
||||||
|
header[data-testid="stHeader"],
|
||||||
|
header[data-testid="stHeader"] > div {
|
||||||
|
overflow: visible !important;
|
||||||
|
}
|
||||||
|
|
||||||
|
[data-testid="collapsedControl"] {
|
||||||
|
position: fixed !important;
|
||||||
|
top: 10px !important;
|
||||||
|
left: 10px !important;
|
||||||
|
z-index: 2147483647 !important;
|
||||||
|
display: flex !important;
|
||||||
|
align-items: center !important;
|
||||||
|
justify-content: center !important;
|
||||||
|
visibility: visible !important;
|
||||||
|
opacity: 1 !important;
|
||||||
|
pointer-events: all !important;
|
||||||
|
width: 44px !important;
|
||||||
|
height: 44px !important;
|
||||||
|
overflow: visible !important;
|
||||||
|
clip: unset !important;
|
||||||
|
clip-path: none !important;
|
||||||
|
}
|
||||||
|
|
||||||
|
[data-testid="collapsedControl"] button {
|
||||||
|
width: 44px !important;
|
||||||
|
height: 44px !important;
|
||||||
|
border-radius: 50% !important;
|
||||||
|
background: linear-gradient(135deg, #3b6cf7, #5b7cfa) !important;
|
||||||
|
border: 2px solid #2d59d1 !important;
|
||||||
|
box-shadow: 0 4px 16px rgba(59,108,247,0.5) !important;
|
||||||
|
display: flex !important;
|
||||||
|
align-items: center !important;
|
||||||
|
justify-content: center !important;
|
||||||
|
cursor: pointer !important;
|
||||||
|
transition: all 0.2s ease !important;
|
||||||
|
padding: 0 !important;
|
||||||
|
overflow: visible !important;
|
||||||
|
}
|
||||||
|
|
||||||
|
[data-testid="collapsedControl"] button svg {
|
||||||
|
width: 20px !important;
|
||||||
|
height: 20px !important;
|
||||||
|
fill: #ffffff !important;
|
||||||
|
color: #ffffff !important;
|
||||||
|
stroke: #ffffff !important;
|
||||||
|
flex-shrink: 0 !important;
|
||||||
|
}
|
||||||
|
|
||||||
|
[data-testid="collapsedControl"] button svg * {
|
||||||
|
fill: #ffffff !important;
|
||||||
|
stroke: #ffffff !important;
|
||||||
|
}
|
||||||
|
|
||||||
|
[data-testid="collapsedControl"] button:hover {
|
||||||
|
background: #2d59d1 !important;
|
||||||
|
transform: scale(1.1) !important;
|
||||||
|
box-shadow: 0 6px 22px rgba(59,108,247,0.6) !important;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Tombol COLLAPSE di dalam sidebar */
|
||||||
|
section[data-testid="stSidebar"] button[kind="header"] {
|
||||||
|
background: #ffffff !important;
|
||||||
|
border: 1px solid #e2e8f0 !important;
|
||||||
|
color: #0f172a !important;
|
||||||
|
border-radius: 10px !important;
|
||||||
|
box-shadow: 0 2px 6px rgba(15,23,42,0.08) !important;
|
||||||
|
}
|
||||||
|
|
||||||
|
section[data-testid="stSidebar"] button[kind="header"]:hover {
|
||||||
|
background: #eef2ff !important;
|
||||||
|
border-color: #3b6cf7 !important;
|
||||||
|
}
|
||||||
|
/* =====================================================
|
||||||
|
DOWNLOAD BUTTON
|
||||||
|
===================================================== */
|
||||||
|
|
||||||
[data-testid="stDownloadButton"] button {
|
[data-testid="stDownloadButton"] button {
|
||||||
|
|
||||||
background: var(--card) !important;
|
background: var(--card) !important;
|
||||||
|
|
||||||
color: var(--blue-primary) !important;
|
color: var(--blue-primary) !important;
|
||||||
|
|
||||||
border: 1.5px solid var(--border) !important;
|
border: 1.5px solid var(--border) !important;
|
||||||
|
|
||||||
border-radius: 10px !important;
|
border-radius: 10px !important;
|
||||||
|
|
||||||
font-weight: 600 !important;
|
font-weight: 600 !important;
|
||||||
|
|
||||||
transition: all 0.2s ease !important;
|
transition: all 0.2s ease !important;
|
||||||
}
|
}
|
||||||
|
|
||||||
[data-testid="stDownloadButton"] button:hover {
|
[data-testid="stDownloadButton"] button:hover {
|
||||||
|
|
||||||
background: var(--blue-light) !important;
|
background: var(--blue-light) !important;
|
||||||
|
|
||||||
border-color: var(--blue-primary) !important;
|
border-color: var(--blue-primary) !important;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* =====================================================
|
||||||
|
SIDEBAR SAFE FIX
|
||||||
|
===================================================== */
|
||||||
|
|
||||||
|
[data-testid="stSidebar"] {
|
||||||
|
background: linear-gradient(
|
||||||
|
180deg,
|
||||||
|
#ffffff 0%,
|
||||||
|
#f8fafc 100%
|
||||||
|
) !important;
|
||||||
|
|
||||||
|
border-right: 1px solid #e2e8f0 !important;
|
||||||
|
|
||||||
|
z-index: 999999 !important;
|
||||||
|
}
|
||||||
|
|
||||||
|
button[kind="header"] {
|
||||||
|
background: white !important;
|
||||||
|
border-radius: 10px !important;
|
||||||
|
border: 1px solid #dbe2ea !important;
|
||||||
|
box-shadow: 0 2px 6px rgba(15,23,42,0.08) !important;
|
||||||
|
}
|
||||||
|
|
||||||
|
button[kind="header"]:hover {
|
||||||
|
background: #eef2ff !important;
|
||||||
|
border-color: #3b6cf7 !important;
|
||||||
|
}
|
||||||
|
|
||||||
</style>
|
</style>
|
||||||
""", unsafe_allow_html=True)
|
""", unsafe_allow_html=True)
|
||||||
|
|
||||||
|
|
||||||
if "page" not in st.session_state:
|
if "page" not in st.session_state:
|
||||||
st.session_state.page = "crawling"
|
st.session_state.page = "crawling"
|
||||||
|
|
||||||
|
|
@ -750,12 +917,15 @@ with st.sidebar:
|
||||||
|
|
||||||
# ── STATUS CRAWLER ──────────────────────────────────────────
|
# ── STATUS CRAWLER ──────────────────────────────────────────
|
||||||
sched_ok = st.session_state.get("_scheduler_started", False)
|
sched_ok = st.session_state.get("_scheduler_started", False)
|
||||||
|
explicitly_off = st.session_state.get("nrt_explicitly_deactivated", False)
|
||||||
sched_interval = st.session_state.get("_scheduler_interval", 0)
|
sched_interval = st.session_state.get("_scheduler_interval", 0)
|
||||||
sched_color = "#16a34a" if sched_ok else "#3b6cf7"
|
sched_color = "#16a34a" if sched_ok else "#64748b"
|
||||||
sched_label = (
|
sched_label = (
|
||||||
f"Aktif — diperbarui tiap {sched_interval} menit"
|
f"Aktif — diperbarui tiap {sched_interval} menit"
|
||||||
if sched_ok
|
if sched_ok
|
||||||
else "Standby — akan crawl otomatis setiap 5 menit"
|
else "Nonaktif — crawling dihentikan oleh pengguna"
|
||||||
|
if explicitly_off
|
||||||
|
else "Standby — aktifkan melalui menu Hari Ini"
|
||||||
)
|
)
|
||||||
|
|
||||||
st.markdown(f"""
|
st.markdown(f"""
|
||||||
|
|
|
||||||
154
crawler.py
154
crawler.py
|
|
@ -9,15 +9,24 @@ from scraper_worker import scrape_once
|
||||||
AUTO_CRAWL_INTERVAL_HOURS = 0.083
|
AUTO_CRAWL_INTERVAL_HOURS = 0.083
|
||||||
NRT_INTERVAL_MINUTES = 5
|
NRT_INTERVAL_MINUTES = 5
|
||||||
|
|
||||||
LOG_FILE = "auto_crawl_log.json"
|
LOG_FILE = "auto_crawl_log.json"
|
||||||
STATE_FILE = "crawler_state.json"
|
STATE_FILE = "crawler_state.json"
|
||||||
|
|
||||||
# ── Internal lock agar tidak ada dua crawl berjalan bersamaan ──
|
# ── Lock agar tidak ada dua crawl berjalan bersamaan ──────────────
|
||||||
_crawl_lock = threading.Lock()
|
_crawl_lock = threading.Lock()
|
||||||
|
|
||||||
# ── Flag untuk scheduler background thread ──
|
# ── Flag scheduler ────────────────────────────────────────────────
|
||||||
_scheduler_started = False
|
_scheduler_started = False
|
||||||
_scheduler_lock = threading.Lock()
|
_scheduler_lock = threading.Lock()
|
||||||
|
|
||||||
|
# ── Flag aktivasi NRT — harus True sebelum scheduler boleh jalan ──
|
||||||
|
# PENTING: Nilai awal selalu False. Tidak ada kode module-level
|
||||||
|
# yang boleh mengubah ini menjadi True secara otomatis.
|
||||||
|
_nrt_enabled = False
|
||||||
|
_nrt_enable_lock = threading.Lock()
|
||||||
|
|
||||||
|
# ── Event untuk menghentikan scheduler secara bersih ──────────────
|
||||||
|
_scheduler_stop_event = threading.Event()
|
||||||
|
|
||||||
|
|
||||||
# ═══════════════════════════════════════════════════════════
|
# ═══════════════════════════════════════════════════════════
|
||||||
|
|
@ -47,10 +56,10 @@ def get_auto_crawl_logs(limit=10):
|
||||||
def save_auto_crawl_log(status="success", total_saved=0, error=None):
|
def save_auto_crawl_log(status="success", total_saved=0, error=None):
|
||||||
logs = get_auto_crawl_logs(50)
|
logs = get_auto_crawl_logs(50)
|
||||||
logs.insert(0, {
|
logs.insert(0, {
|
||||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||||
"status": status,
|
"status": status,
|
||||||
"total_saved": total_saved,
|
"total_saved": total_saved,
|
||||||
"error": error,
|
"error": error,
|
||||||
})
|
})
|
||||||
try:
|
try:
|
||||||
with open(LOG_FILE, "w") as f:
|
with open(LOG_FILE, "w") as f:
|
||||||
|
|
@ -67,11 +76,11 @@ def set_crawler_state(is_running=False, service_active=None):
|
||||||
service_active = previous_state.get("service_active", False)
|
service_active = previous_state.get("service_active", False)
|
||||||
|
|
||||||
state = {
|
state = {
|
||||||
"is_running": bool(is_running),
|
"is_running": bool(is_running),
|
||||||
"service_active": bool(service_active),
|
"service_active": bool(service_active),
|
||||||
"updated_at": now,
|
"updated_at": now,
|
||||||
"heartbeat_at": now if service_active else previous_state.get("heartbeat_at"),
|
"heartbeat_at": now if service_active else previous_state.get("heartbeat_at"),
|
||||||
"last_job_started_at": previous_state.get("last_job_started_at"),
|
"last_job_started_at": previous_state.get("last_job_started_at"),
|
||||||
"last_job_finished_at": previous_state.get("last_job_finished_at"),
|
"last_job_finished_at": previous_state.get("last_job_finished_at"),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -86,6 +95,31 @@ def set_crawler_state(is_running=False, service_active=None):
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"[crawler] Gagal menulis state: {e}")
|
print(f"[crawler] Gagal menulis state: {e}")
|
||||||
|
|
||||||
|
# ═══════════════════════════════════════════════════════════
|
||||||
|
# NRT ACTIVATION PERSISTENCE
|
||||||
|
# ═══════════════════════════════════════════════════════════
|
||||||
|
|
||||||
|
NRT_ACTIVATION_FILE = "nrt_activation.json"
|
||||||
|
|
||||||
|
def save_nrt_activation(activated: bool):
|
||||||
|
"""Simpan status aktivasi NRT ke file agar persist setelah refresh."""
|
||||||
|
try:
|
||||||
|
with open(NRT_ACTIVATION_FILE, "w") as f:
|
||||||
|
json.dump({"activated": activated, "updated_at": datetime.now(timezone.utc).isoformat()}, f)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[crawler] Gagal menyimpan NRT activation: {e}")
|
||||||
|
|
||||||
|
|
||||||
|
def load_nrt_activation() -> bool:
|
||||||
|
"""Baca status aktivasi NRT dari file."""
|
||||||
|
if not os.path.exists(NRT_ACTIVATION_FILE):
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
with open(NRT_ACTIVATION_FILE, "r") as f:
|
||||||
|
data = json.load(f)
|
||||||
|
return bool(data.get("activated", False))
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
|
||||||
# ═══════════════════════════════════════════════════════════
|
# ═══════════════════════════════════════════════════════════
|
||||||
# CORE JOB
|
# CORE JOB
|
||||||
|
|
@ -95,7 +129,6 @@ def auto_crawl_job():
|
||||||
"""
|
"""
|
||||||
Jalankan satu siklus crawling.
|
Jalankan satu siklus crawling.
|
||||||
Thread-safe: hanya satu crawl yang boleh berjalan pada satu waktu.
|
Thread-safe: hanya satu crawl yang boleh berjalan pada satu waktu.
|
||||||
Bisa dipanggil dari tombol UI maupun scheduler otomatis.
|
|
||||||
"""
|
"""
|
||||||
if not _crawl_lock.acquire(blocking=False):
|
if not _crawl_lock.acquire(blocking=False):
|
||||||
print("[crawler] Crawl sedang berjalan, skip.")
|
print("[crawler] Crawl sedang berjalan, skip.")
|
||||||
|
|
@ -129,34 +162,38 @@ def auto_crawl_job():
|
||||||
|
|
||||||
|
|
||||||
# ═══════════════════════════════════════════════════════════
|
# ═══════════════════════════════════════════════════════════
|
||||||
# BACKGROUND SCHEDULER (dipakai Streamlit, tanpa terminal)
|
# BACKGROUND SCHEDULER
|
||||||
# ═══════════════════════════════════════════════════════════
|
# ═══════════════════════════════════════════════════════════
|
||||||
|
|
||||||
def _scheduler_loop():
|
def _scheduler_loop():
|
||||||
"""
|
"""
|
||||||
Loop yang berjalan di daemon thread.
|
Loop daemon thread. Crawl pertama langsung, lalu tiap NRT_INTERVAL_MINUTES.
|
||||||
Crawling pertama langsung dijalankan, lalu setiap NRT_INTERVAL_MINUTES menit.
|
Berhenti bersih saat _scheduler_stop_event di-set.
|
||||||
"""
|
"""
|
||||||
print(f"[crawler] Scheduler dimulai — interval {NRT_INTERVAL_MINUTES} menit.")
|
print(f"[crawler] Scheduler dimulai — interval {NRT_INTERVAL_MINUTES} menit.")
|
||||||
set_crawler_state(False, service_active=True)
|
set_crawler_state(False, service_active=True)
|
||||||
|
|
||||||
while True:
|
while not _scheduler_stop_event.is_set():
|
||||||
auto_crawl_job()
|
auto_crawl_job()
|
||||||
|
|
||||||
# Tulis heartbeat setiap 30 detik selagi menunggu interval berikutnya
|
# Tunggu interval berikutnya sambil tulis heartbeat tiap 10 detik
|
||||||
wait_until = time.time() + (NRT_INTERVAL_MINUTES * 60)
|
wait_until = time.time() + (NRT_INTERVAL_MINUTES * 60)
|
||||||
while time.time() < wait_until:
|
while time.time() < wait_until and not _scheduler_stop_event.is_set():
|
||||||
# Heartbeat agar is_crawler_service_active() tetap True
|
|
||||||
_write_heartbeat()
|
_write_heartbeat()
|
||||||
time.sleep(min(30, max(1, wait_until - time.time())))
|
time.sleep(min(10, max(1, wait_until - time.time())))
|
||||||
|
|
||||||
print(f"[crawler] Interval berikutnya dalam {NRT_INTERVAL_MINUTES} menit...")
|
if not _scheduler_stop_event.is_set():
|
||||||
|
print(f"[crawler] Mulai crawl berikutnya...")
|
||||||
|
|
||||||
|
# Bersihkan state saat loop selesai
|
||||||
|
set_crawler_state(False, service_active=False)
|
||||||
|
print("[crawler] Scheduler dihentikan.")
|
||||||
|
|
||||||
|
|
||||||
def _write_heartbeat():
|
def _write_heartbeat():
|
||||||
"""Perbarui heartbeat_at tanpa mengubah field lain."""
|
"""Perbarui heartbeat_at tanpa mengubah field lain."""
|
||||||
state = get_crawler_state()
|
state = get_crawler_state()
|
||||||
state["heartbeat_at"] = datetime.now(timezone.utc).isoformat()
|
state["heartbeat_at"] = datetime.now(timezone.utc).isoformat()
|
||||||
state["service_active"] = True
|
state["service_active"] = True
|
||||||
try:
|
try:
|
||||||
with open(STATE_FILE, "w") as f:
|
with open(STATE_FILE, "w") as f:
|
||||||
|
|
@ -165,31 +202,92 @@ def _write_heartbeat():
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
# ═══════════════════════════════════════════════════════════
|
||||||
|
# ACTIVATION API ← dipakai oleh crawling_page.py
|
||||||
|
# ═══════════════════════════════════════════════════════════
|
||||||
|
|
||||||
|
def is_nrt_enabled() -> bool:
|
||||||
|
"""Kembalikan True jika NRT sudah diaktifkan user."""
|
||||||
|
return _nrt_enabled
|
||||||
|
|
||||||
|
|
||||||
|
def activate_nrt_scheduler():
|
||||||
|
"""
|
||||||
|
Aktifkan NRT scheduler.
|
||||||
|
Harus dipanggil HANYA dari tombol UI — tidak dari module-level manapun.
|
||||||
|
Idempotent: aman dipanggil berkali-kali.
|
||||||
|
"""
|
||||||
|
global _nrt_enabled
|
||||||
|
|
||||||
|
with _nrt_enable_lock:
|
||||||
|
_nrt_enabled = True
|
||||||
|
|
||||||
|
save_nrt_activation(True) # ← TAMBAH INI
|
||||||
|
_scheduler_stop_event.clear() # pastikan event tidak dalam kondisi set
|
||||||
|
ensure_scheduler_running()
|
||||||
|
print("[crawler] NRT diaktifkan oleh user.")
|
||||||
|
|
||||||
|
|
||||||
|
def deactivate_nrt_scheduler():
|
||||||
|
"""
|
||||||
|
Hentikan NRT scheduler.
|
||||||
|
Thread yang berjalan akan berhenti bersih pada iterasi berikutnya.
|
||||||
|
"""
|
||||||
|
global _nrt_enabled, _scheduler_started
|
||||||
|
|
||||||
|
with _nrt_enable_lock:
|
||||||
|
_nrt_enabled = False
|
||||||
|
|
||||||
|
save_nrt_activation(False)
|
||||||
|
_scheduler_stop_event.set() # sinyal ke thread agar berhenti
|
||||||
|
|
||||||
|
with _scheduler_lock:
|
||||||
|
_scheduler_started = False # izinkan start ulang di masa depan
|
||||||
|
|
||||||
|
set_crawler_state(False, service_active=False)
|
||||||
|
print("[crawler] NRT dinonaktifkan oleh user.")
|
||||||
|
|
||||||
|
|
||||||
def ensure_scheduler_running():
|
def ensure_scheduler_running():
|
||||||
"""
|
"""
|
||||||
Pastikan background scheduler sudah berjalan.
|
Pastikan background scheduler sudah berjalan.
|
||||||
|
HANYA berjalan jika _nrt_enabled = True (sudah diaktifkan user).
|
||||||
Aman dipanggil berkali-kali (idempotent).
|
Aman dipanggil berkali-kali (idempotent).
|
||||||
Dipanggil dari crawling_page.py saat halaman dirender.
|
|
||||||
|
JANGAN panggil fungsi ini dari module-level atau saat app startup.
|
||||||
|
Panggil hanya setelah activate_nrt_scheduler() dipicu dari UI.
|
||||||
"""
|
"""
|
||||||
global _scheduler_started
|
global _scheduler_started
|
||||||
|
|
||||||
|
if not _nrt_enabled:
|
||||||
|
# Belum diaktifkan user — jangan mulai thread apapun
|
||||||
|
return
|
||||||
|
|
||||||
with _scheduler_lock:
|
with _scheduler_lock:
|
||||||
if _scheduler_started:
|
if _scheduler_started:
|
||||||
return
|
return
|
||||||
|
|
||||||
t = threading.Thread(target=_scheduler_loop, name="CrawlerScheduler", daemon=True)
|
_scheduler_stop_event.clear()
|
||||||
|
t = threading.Thread(
|
||||||
|
target=_scheduler_loop,
|
||||||
|
name="CrawlerScheduler",
|
||||||
|
daemon=True,
|
||||||
|
)
|
||||||
t.start()
|
t.start()
|
||||||
_scheduler_started = True
|
_scheduler_started = True
|
||||||
print("[crawler] Background scheduler thread dimulai.")
|
print("[crawler] Background scheduler thread dimulai.")
|
||||||
|
|
||||||
|
|
||||||
# ═══════════════════════════════════════════════════════════
|
# ═══════════════════════════════════════════════════════════
|
||||||
# ENTRY POINT (tetap bisa dipakai via terminal jika mau)
|
# ENTRY POINT (via terminal, opsional)
|
||||||
# ═══════════════════════════════════════════════════════════
|
# ═══════════════════════════════════════════════════════════
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
"""Jalankan scheduler via terminal (opsional, tidak wajib)."""
|
"""Jalankan scheduler via terminal (run_scraper_loop.py)."""
|
||||||
print("Crawler realtime aktif.")
|
global _nrt_enabled
|
||||||
|
_nrt_enabled = True # mode terminal: langsung aktif
|
||||||
|
|
||||||
|
print("Crawler realtime aktif (mode terminal).")
|
||||||
print(f"Interval: {NRT_INTERVAL_MINUTES} menit")
|
print(f"Interval: {NRT_INTERVAL_MINUTES} menit")
|
||||||
set_crawler_state(False, service_active=True)
|
set_crawler_state(False, service_active=True)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
{"activated": false, "updated_at": "2026-05-22T13:17:14.073811+00:00"}
|
||||||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,2 +1,2 @@
|
||||||
"conversation_id_str","created_at","favorite_count","full_text","id_str","image_url","in_reply_to_screen_name","lang","location","quote_count","reply_count","retweet_count","tweet_url","user_id_str","username"
|
"conversation_id_str","created_at","favorite_count","full_text","id_str","image_url","in_reply_to_screen_name","lang","location","quote_count","reply_count","retweet_count","tweet_url","user_id_str","username"
|
||||||
"2056538953340314095","Wed May 20 16:01:04 +0000 2026","5","@killingmaster27 @evelyndewi7 @katrinrienks @HumorJonTampan @wiyantokdr54 @EviDrajat @RickyKardjono @Micheladam69432 @ebonXboy @AntonRebor79105 @gangsterrandom @Pieter_Sun01 @Rahmah0845 @RSjahfidi @prabowo @SmileyNDeLight @Jediimar @Yens0906 @tokaidirecycle @PembuatSistem @bonapasogit24 @Jenniestale @Maskudin126003 @a8sar6alih @JowoNgalam @KING_RAJA_PWS_ @apaajadeh447870 @50ngoku @Submill @EmmaHusain0845 @BebySoSweet @Yulia_4ja @bungalotus25 @bahlillahadalia wahhh... jadi menteri komdigi dong menteri free ongkir https://t.co/1cfvoqIcNl","2057129489994510620","https://pbs.twimg.com/media/HIxkWwDaMAAXH9U.jpg","killingmaster27","in","","0","2","0","https://x.com/undefined/status/2057129489994510620","1410466956134608899",
|
"2056538953340314095","Wed May 20 16:01:04 +0000 2026","4","@killingmaster27 @evelyndewi7 @katrinrienks @HumorJonTampan @wiyantokdr54 @EviDrajat @RickyKardjono @Micheladam69432 @ebonXboy @AntonRebor79105 @gangsterrandom @Pieter_Sun01 @Rahmah0845 @RSjahfidi @prabowo @SmileyNDeLight @Jediimar @Yens0906 @tokaidirecycle @PembuatSistem @bonapasogit24 @Jenniestale @Maskudin126003 @a8sar6alih @JowoNgalam @KING_RAJA_PWS_ @apaajadeh447870 @50ngoku @Submill @EmmaHusain0845 @BebySoSweet @Yulia_4ja @bungalotus25 @bahlillahadalia wahhh... jadi menteri komdigi dong menteri free ongkir https://t.co/1cfvoqIcNl","2057129489994510620","https://pbs.twimg.com/media/HIxkWwDaMAAXH9U.jpg","killingmaster27","in","","0","2","0","https://x.com/undefined/status/2057129489994510620","1410466956134608899",
|
||||||
|
|
|
||||||
|
|
|
@ -1,2 +1,2 @@
|
||||||
"conversation_id_str","created_at","favorite_count","full_text","id_str","image_url","in_reply_to_screen_name","lang","location","quote_count","reply_count","retweet_count","tweet_url","user_id_str","username"
|
"conversation_id_str","created_at","favorite_count","full_text","id_str","image_url","in_reply_to_screen_name","lang","location","quote_count","reply_count","retweet_count","tweet_url","user_id_str","username"
|
||||||
"2056538953340314095","Wed May 20 16:01:04 +0000 2026","5","@killingmaster27 @evelyndewi7 @katrinrienks @HumorJonTampan @wiyantokdr54 @EviDrajat @RickyKardjono @Micheladam69432 @ebonXboy @AntonRebor79105 @gangsterrandom @Pieter_Sun01 @Rahmah0845 @RSjahfidi @prabowo @SmileyNDeLight @Jediimar @Yens0906 @tokaidirecycle @PembuatSistem @bonapasogit24 @Jenniestale @Maskudin126003 @a8sar6alih @JowoNgalam @KING_RAJA_PWS_ @apaajadeh447870 @50ngoku @Submill @EmmaHusain0845 @BebySoSweet @Yulia_4ja @bungalotus25 @bahlillahadalia wahhh... jadi menteri komdigi dong menteri free ongkir https://t.co/1cfvoqIcNl","2057129489994510620","https://pbs.twimg.com/media/HIxkWwDaMAAXH9U.jpg","killingmaster27","in","","0","2","0","https://x.com/undefined/status/2057129489994510620","1410466956134608899",
|
"2056538953340314095","Wed May 20 16:01:04 +0000 2026","4","@killingmaster27 @evelyndewi7 @katrinrienks @HumorJonTampan @wiyantokdr54 @EviDrajat @RickyKardjono @Micheladam69432 @ebonXboy @AntonRebor79105 @gangsterrandom @Pieter_Sun01 @Rahmah0845 @RSjahfidi @prabowo @SmileyNDeLight @Jediimar @Yens0906 @tokaidirecycle @PembuatSistem @bonapasogit24 @Jenniestale @Maskudin126003 @a8sar6alih @JowoNgalam @KING_RAJA_PWS_ @apaajadeh447870 @50ngoku @Submill @EmmaHusain0845 @BebySoSweet @Yulia_4ja @bungalotus25 @bahlillahadalia wahhh... jadi menteri komdigi dong menteri free ongkir https://t.co/1cfvoqIcNl","2057129489994510620","https://pbs.twimg.com/media/HIxkWwDaMAAXH9U.jpg","killingmaster27","in","","0","2","0","https://x.com/undefined/status/2057129489994510620","1410466956134608899",
|
||||||
|
|
|
||||||
|
|
|
@ -1,6 +0,0 @@
|
||||||
"conversation_id_str","created_at","favorite_count","full_text","id_str","image_url","in_reply_to_screen_name","lang","location","quote_count","reply_count","retweet_count","tweet_url","user_id_str","username"
|
|
||||||
"2056892677149581606","Wed May 20 00:20:04 +0000 2026","0","komdigi keren sih mengadakan kebijakan ongkir kaya gini","2056892677149581606","","","in","","0","0","0","https://x.com/undefined/status/2056892677149581606","1814143201718247424",
|
|
||||||
"2056949159845101648","Wed May 20 04:04:30 +0000 2026","0","komdigi kereenn dan bijak membuat keputusan ongkir seperti inii","2056949159845101648","","","in","","0","0","0","https://x.com/undefined/status/2056949159845101648","1814143201718247424",
|
|
||||||
"2056635322419777630","Tue May 19 07:17:25 +0000 2026","0","komdigi ongkir gk jelas","2056635322419777630","","","in","","0","0","0","https://x.com/undefined/status/2056635322419777630","1814143201718247424",
|
|
||||||
"2055269612334555468","Fri May 15 12:50:35 +0000 2026","0","komdigi + ongkir = chaos wkwkwk","2055269612334555468","","","in","","0","0","0","https://x.com/undefined/status/2055269612334555468","2043979880086339584",
|
|
||||||
"2056538953340314095","Wed May 20 16:01:04 +0000 2026","5","@killingmaster27 @evelyndewi7 @katrinrienks @HumorJonTampan @wiyantokdr54 @EviDrajat @RickyKardjono @Micheladam69432 @ebonXboy @AntonRebor79105 @gangsterrandom @Pieter_Sun01 @Rahmah0845 @RSjahfidi @prabowo @SmileyNDeLight @Jediimar @Yens0906 @tokaidirecycle @PembuatSistem @bonapasogit24 @Jenniestale @Maskudin126003 @a8sar6alih @JowoNgalam @KING_RAJA_PWS_ @apaajadeh447870 @50ngoku @Submill @EmmaHusain0845 @BebySoSweet @Yulia_4ja @bungalotus25 @bahlillahadalia wahhh... jadi menteri komdigi dong menteri free ongkir https://t.co/1cfvoqIcNl","2057129489994510620","https://pbs.twimg.com/media/HIxkWwDaMAAXH9U.jpg","killingmaster27","in","","0","2","0","https://x.com/undefined/status/2057129489994510620","1410466956134608899",
|
|
||||||
|
|
|
@ -1,6 +1,14 @@
|
||||||
"conversation_id_str","created_at","favorite_count","full_text","id_str","image_url","in_reply_to_screen_name","lang","location","quote_count","reply_count","retweet_count","tweet_url","user_id_str","username"
|
"conversation_id_str","created_at","favorite_count","full_text","id_str","image_url","in_reply_to_screen_name","lang","location","quote_count","reply_count","retweet_count","tweet_url","user_id_str","username"
|
||||||
|
"2057791279896080430","Fri May 22 11:50:47 +0000 2026","0","komdigi mengeluarkan kebijakan gratis ongkir @komdigi ???","2057791279896080430","","","in","","0","0","0","https://x.com/undefined/status/2057791279896080430","1814143201718247424",
|
||||||
|
"2057660616815571144","Fri May 22 03:11:35 +0000 2026","0","komdigi pembatasan gratis ongkir","2057660616815571144","","","in","","0","0","0","https://x.com/undefined/status/2057660616815571144","1814143201718247424",
|
||||||
|
"2056538953340314095","Wed May 20 16:01:04 +0000 2026","4","@killingmaster27 @evelyndewi7 @katrinrienks @HumorJonTampan @wiyantokdr54 @EviDrajat @RickyKardjono @Micheladam69432 @ebonXboy @AntonRebor79105 @gangsterrandom @Pieter_Sun01 @Rahmah0845 @RSjahfidi @prabowo @SmileyNDeLight @Jediimar @Yens0906 @tokaidirecycle @PembuatSistem @bonapasogit24 @Jenniestale @Maskudin126003 @a8sar6alih @JowoNgalam @KING_RAJA_PWS_ @apaajadeh447870 @50ngoku @Submill @EmmaHusain0845 @BebySoSweet @Yulia_4ja @bungalotus25 @bahlillahadalia wahhh... jadi menteri komdigi dong menteri free ongkir https://t.co/1cfvoqIcNl","2057129489994510620","https://pbs.twimg.com/media/HIxkWwDaMAAXH9U.jpg","killingmaster27","in","","0","2","0","https://x.com/undefined/status/2057129489994510620","1410466956134608899",
|
||||||
"2056635322419777630","Tue May 19 07:17:25 +0000 2026","0","komdigi ongkir gk jelas","2056635322419777630","","","in","","0","0","0","https://x.com/undefined/status/2056635322419777630","1814143201718247424",
|
"2056635322419777630","Tue May 19 07:17:25 +0000 2026","0","komdigi ongkir gk jelas","2056635322419777630","","","in","","0","0","0","https://x.com/undefined/status/2056635322419777630","1814143201718247424",
|
||||||
"2056892677149581606","Wed May 20 00:20:04 +0000 2026","0","komdigi keren sih mengadakan kebijakan ongkir kaya gini","2056892677149581606","","","in","","0","0","0","https://x.com/undefined/status/2056892677149581606","1814143201718247424",
|
|
||||||
"2056949159845101648","Wed May 20 04:04:30 +0000 2026","0","komdigi kereenn dan bijak membuat keputusan ongkir seperti inii","2056949159845101648","","","in","","0","0","0","https://x.com/undefined/status/2056949159845101648","1814143201718247424",
|
"2056949159845101648","Wed May 20 04:04:30 +0000 2026","0","komdigi kereenn dan bijak membuat keputusan ongkir seperti inii","2056949159845101648","","","in","","0","0","0","https://x.com/undefined/status/2056949159845101648","1814143201718247424",
|
||||||
"2056538953340314095","Wed May 20 16:01:04 +0000 2026","5","@killingmaster27 @evelyndewi7 @katrinrienks @HumorJonTampan @wiyantokdr54 @EviDrajat @RickyKardjono @Micheladam69432 @ebonXboy @AntonRebor79105 @gangsterrandom @Pieter_Sun01 @Rahmah0845 @RSjahfidi @prabowo @SmileyNDeLight @Jediimar @Yens0906 @tokaidirecycle @PembuatSistem @bonapasogit24 @Jenniestale @Maskudin126003 @a8sar6alih @JowoNgalam @KING_RAJA_PWS_ @apaajadeh447870 @50ngoku @Submill @EmmaHusain0845 @BebySoSweet @Yulia_4ja @bungalotus25 @bahlillahadalia wahhh... jadi menteri komdigi dong menteri free ongkir https://t.co/1cfvoqIcNl","2057129489994510620","https://pbs.twimg.com/media/HIxkWwDaMAAXH9U.jpg","killingmaster27","in","","0","2","0","https://x.com/undefined/status/2057129489994510620","1410466956134608899",
|
"2056892677149581606","Wed May 20 00:20:04 +0000 2026","0","komdigi keren sih mengadakan kebijakan ongkir kaya gini","2056892677149581606","","","in","","0","0","0","https://x.com/undefined/status/2056892677149581606","1814143201718247424",
|
||||||
"2055269612334555468","Fri May 15 12:50:35 +0000 2026","0","komdigi + ongkir = chaos wkwkwk","2055269612334555468","","","in","","0","0","0","https://x.com/undefined/status/2055269612334555468","2043979880086339584",
|
"conversation_id_str","created_at","favorite_count","full_text","id_str","image_url","in_reply_to_screen_name","lang","location","quote_count","reply_count","retweet_count","tweet_url","user_id_str","username"
|
||||||
|
"2057791279896080430","Fri May 22 11:50:47 +0000 2026","0","komdigi mengeluarkan kebijakan gratis ongkir @komdigi ???","2057791279896080430","","","in","","0","0","0","https://x.com/undefined/status/2057791279896080430","1814143201718247424",
|
||||||
|
"2057660616815571144","Fri May 22 03:11:35 +0000 2026","0","komdigi pembatasan gratis ongkir","2057660616815571144","","","in","","0","0","0","https://x.com/undefined/status/2057660616815571144","1814143201718247424",
|
||||||
|
"2056635322419777630","Tue May 19 07:17:25 +0000 2026","0","komdigi ongkir gk jelas","2056635322419777630","","","in","","0","0","0","https://x.com/undefined/status/2056635322419777630","1814143201718247424",
|
||||||
|
"2056538953340314095","Wed May 20 16:01:04 +0000 2026","4","@killingmaster27 @evelyndewi7 @katrinrienks @HumorJonTampan @wiyantokdr54 @EviDrajat @RickyKardjono @Micheladam69432 @ebonXboy @AntonRebor79105 @gangsterrandom @Pieter_Sun01 @Rahmah0845 @RSjahfidi @prabowo @SmileyNDeLight @Jediimar @Yens0906 @tokaidirecycle @PembuatSistem @bonapasogit24 @Jenniestale @Maskudin126003 @a8sar6alih @JowoNgalam @KING_RAJA_PWS_ @apaajadeh447870 @50ngoku @Submill @EmmaHusain0845 @BebySoSweet @Yulia_4ja @bungalotus25 @bahlillahadalia wahhh... jadi menteri komdigi dong menteri free ongkir https://t.co/1cfvoqIcNl","2057129489994510620","https://pbs.twimg.com/media/HIxkWwDaMAAXH9U.jpg","killingmaster27","in","","0","2","0","https://x.com/undefined/status/2057129489994510620","1410466956134608899",
|
||||||
|
"2056949159845101648","Wed May 20 04:04:30 +0000 2026","0","komdigi kereenn dan bijak membuat keputusan ongkir seperti inii","2056949159845101648","","","in","","0","0","0","https://x.com/undefined/status/2056949159845101648","1814143201718247424",
|
||||||
|
"2056892677149581606","Wed May 20 00:20:04 +0000 2026","0","komdigi keren sih mengadakan kebijakan ongkir kaya gini","2056892677149581606","","","in","","0","0","0","https://x.com/undefined/status/2056892677149581606","1814143201718247424",
|
||||||
|
|
|
||||||
|
Loading…
Reference in New Issue