rekomendasi

app/Http/Controllers/RekomendasiController.php

@@ -23,23 +23,35 @@ class RekomendasiController extends Controller
             ],
         ],
         'Aksesibilitas' => [
-            'keywords' => ['parkir', 'lahan parkir', 'tempat parkir', 'parkiran', 'motor', 'mobil'],
+            // FIX: diperluas — sebelumnya hanya mencakup parkir,
+            // sekarang mencakup akses jalan, transportasi, dan kemudahan mencapai lokasi
+            'keywords' => [
+                'parkir', 'lahan parkir', 'tempat parkir', 'parkiran', 'motor', 'mobil',
+                'jalan', 'akses', 'transportasi', 'jalur', 'tanjakan',
+                'turunan', 'sempit', 'jauh', 'susah', 'sulit', 'capek',
+                'angkutan', 'ojek', 'kendaraan', 'macet',
+            ],
             'color'    => 'orange',
             'icon'     => '🚗',
             'saran'    => [
-                'actions' => ['Penataan area parkir', 'Tarif transparan', 'Petugas lebih ramah'],
+                'actions' => ['Penataan area parkir', 'Perbaikan akses jalan', 'Penambahan transportasi umum'],
-                'dampak'  => 'Mengurangi keluhan',
+                'dampak'  => 'Mengurangi keluhan aksesibilitas',
-                'tip'     => 'Sistem parkir lebih rapi dan transparan.',
+                'tip'     => 'Sistem parkir lebih rapi dan akses jalan diperbaiki.',
             ],
         ],
-        'Harga / Tiket' => [
+        // FIX: nama diganti dari 'Harga / Tiket' → 'HTM'
-            'keywords' => ['mahal', 'harga', 'tiket', 'bayar', 'biaya', 'tarif', 'murah', 'terjangkau'],
+        // agar konsisten dengan parameter TA (Harga Tiket Masuk)
+        'HTM' => [
+            'keywords' => [
+                'mahal', 'harga', 'tiket', 'bayar', 'biaya', 'tarif',
+                'murah', 'terjangkau', 'htm', 'retribusi', 'pungli',
+            ],
             'color'    => 'yellow',
             'icon'     => '🎟️',
             'saran'    => [
-                'actions' => ['Evaluasi harga tiket', 'Promo wisata', 'Diskon tertentu'],
+                'actions' => ['Evaluasi harga tiket masuk', 'Buat paket promo wisata', 'Transparansi retribusi'],
-                'dampak'  => 'Daya tarik meningkat',
+                'dampak'  => 'Daya tarik wisatawan meningkat',
-                'tip'     => 'Penyesuaian harga tiket agar lebih terjangkau.',
+                'tip'     => 'Penyesuaian HTM agar lebih terjangkau dan transparan.',
             ],
         ],
         'Fasilitas' => [
@@ -52,7 +64,6 @@ class RekomendasiController extends Controller
                 'tip'     => 'Fasilitas lengkap & terawat meningkatkan kepuasan.',
             ],
         ],
-        
     ];
 
     public function index(Request $request)
@@ -88,9 +99,12 @@ public function index(Request $request)
         };
 
         // --- 4. Ambil ulasan negatif & bersihkan teks ---
+        // FIX: fallback ke kolom 'ulasan' jika 'ulasan_bersih' tidak tersedia
         $ulasanNegatif = (clone $queryFilter)
             ->where('sentimen', 'negatif')
-            ->pluck('ulasan_bersih')   // gunakan kolom yang sudah dipreproses
+            ->get(['ulasan_bersih', 'ulasan'])
+            ->map(fn($r) => $r->ulasan_bersih ?? $r->ulasan ?? '')
+            ->filter()
             ->toArray();
 
         $text  = strtolower(implode(' ', $ulasanNegatif));
@@ -134,8 +148,40 @@ public function index(Request $request)
         }
 
         // --- 7. Isu dominan (ranking 1) ---
-        $isuDominan     = $isuUtama[0]['nama']   ?? 'Belum ada isu';
+        // FIX: guard jika tidak ada ulasan negatif sama sekali
-        $isuDominanPersen = $isuUtama[0]['persen'] ?? 0;
+        if (empty($isuUtama) || $isuUtama[0]['skor'] === 0) {
+            $isuDominan       = 'Tidak ada keluhan';
+            $isuDominanPersen = 0;
+            $prioritas        = [];
+            $saranPerbaikan   = [];
+        } else {
+            $isuDominan       = $isuUtama[0]['nama'];
+            $isuDominanPersen = $isuUtama[0]['persen'];
+
+            // --- 9. Prioritas rekomendasi (top 3 isu) ---
+            $prioritas = [];
+            $rank       = 1;
+            foreach (array_slice($issueSkor, 0, 3, true) as $nama => $skor) {
+                $prioritas[] = [
+                    'rank'    => $rank++,
+                    'nama'    => $nama,
+                    'icon'    => $this->issueRules[$nama]['icon'],
+                    'actions' => $this->issueRules[$nama]['saran']['actions'],
+                    'dampak'  => $this->issueRules[$nama]['saran']['dampak'],
+                    'color'   => $this->issueRules[$nama]['color'],
+                ];
+            }
+
+            // --- 10. Saran perbaikan (top 3) ---
+            $saranPerbaikan = [];
+            foreach (array_slice($issueSkor, 0, 3, true) as $nama => $skor) {
+                $saranPerbaikan[] = [
+                    'nama' => 'Perbaikan ' . $nama,
+                    'tip'  => $this->issueRules[$nama]['saran']['tip'],
+                    'icon' => $this->issueRules[$nama]['icon'],
+                ];
+            }
+        }
 
         // --- 8. Kata kunci dominan (global, top 10, filter stopword) ---
         $stopwords = [
@@ -158,30 +204,6 @@ public function index(Request $request)
         arsort($freq);
         $kataDominan = array_slice($freq, 0, 10, true);
 
-        // --- 9. Prioritas rekomendasi (top 3 isu) ---
-        $prioritas = [];
-        $rank       = 1;
-        foreach (array_slice($issueSkor, 0, 3, true) as $nama => $skor) {
-            $prioritas[] = [
-                'rank'    => $rank++,
-                'nama'    => $nama,
-                'icon'    => $this->issueRules[$nama]['icon'],
-                'actions' => $this->issueRules[$nama]['saran']['actions'],
-                'dampak'  => $this->issueRules[$nama]['saran']['dampak'],
-                'color'   => $this->issueRules[$nama]['color'],
-            ];
-        }
-
-        // --- 10. Saran perbaikan (top 3) ---
-        $saranPerbaikan = [];
-        foreach (array_slice($issueSkor, 0, 3, true) as $nama => $skor) {
-            $saranPerbaikan[] = [
-                'nama' => 'Perbaikan ' . $nama,
-                'tip'  => $this->issueRules[$nama]['saran']['tip'],
-                'icon' => $this->issueRules[$nama]['icon'],
-            ];
-        }
-
         // --- 11. Filter destinasi yang sedang aktif ---
         $destinasiAktif = $request->input('destinasi', 'Semua Destinasi');
 

artefak/model_metadata.json

@@ -0,0 +1,26 @@
+{
+    "model": "ComplementNB (Naive Bayes)",
+    "best_params": {
+        "clf__alpha": 0.1,
+        "clf__norm": true,
+        "tfidf__min_df": 3,
+        "tfidf__ngram_range": [
+            1,
+            1
+        ],
+        "tfidf__sublinear_tf": false
+    },
+    "cv_macro_f1": {
+        "mean": 0.3964,
+        "std": 0.0414
+    },
+    "evaluasi_test": {
+        "accuracy": 0.7143,
+        "macro_f1": 0.6746
+    },
+    "distribusi_label": {
+        "positif": 19,
+        "netral": 11,
+        "negatif": 4
+    }
+}

scraper/analisis.py

@@ -1,51 +1,342 @@
 # -*- coding: utf-8 -*-
-import argparse
+"""
+analisis.py — SENTARA
+=====================
+Sistem Analisis Sentimen Ulasan Wisatawan
+Metode   : Naive Bayes (ComplementNB) + TF-IDF
+Label    : Dari teks ulasan (bukan rating)
+"""
+
+import json
 import os
 import re
-import sys
 import sqlite3
+import sys
+import warnings
 from pathlib import Path
 from urllib.parse import quote_plus
 
+import joblib
+import numpy as np
 import pandas as pd
 import pymysql
 from sqlalchemy import create_engine
 
 from sklearn.feature_extraction.text import TfidfVectorizer
-from sklearn.metrics import accuracy_score, classification_report
+from sklearn.metrics import (
-from sklearn.model_selection import train_test_split
+    accuracy_score,
+    classification_report,
+    f1_score,
+)
+from sklearn.model_selection import (
+    GridSearchCV,
+    StratifiedKFold,
+    cross_val_score,
+    train_test_split,
+)
 from sklearn.naive_bayes import ComplementNB
+from sklearn.pipeline import Pipeline
 
 from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
 from Sastrawi.StopWordRemover.StopWordRemoverFactory import StopWordRemoverFactory
 
+warnings.filterwarnings("ignore")
 sys.stdout.reconfigure(encoding="utf-8")
 sys.stderr.reconfigure(encoding="utf-8")
 
-BASE_DIR = Path(__file__).resolve().parents[1]
+# ═══════════════════════════════════════════════════════════════
+# KONSTANTA & PATH
+# ═══════════════════════════════════════════════════════════════
 
+SEED     = 42
+BASE_DIR = Path(__file__).resolve().parents[1]
+ARTEFAK  = BASE_DIR / "artefak"
+ARTEFAK.mkdir(exist_ok=True)
+
+MODEL_PATH    = ARTEFAK / "sentimen_naive_bayes.pkl"
+METADATA_PATH = ARTEFAK / "model_metadata.json"
+
+# ═══════════════════════════════════════════════════════════════
+# LOGGER
+# ═══════════════════════════════════════════════════════════════
 
 def log(level, message):
     print(f"[{level}] {message}", flush=True)
 
-
 def fail(message, code=1):
     log("ERROR", message)
     sys.exit(code)
 
+# ═══════════════════════════════════════════════════════════════
+# KAMUS NORMALISASI LENGKAP (dari dataset.ipynb)
+# ═══════════════════════════════════════════════════════════════
 
-def parse_args():
+NORMALISASI = {
-    parser = argparse.ArgumentParser(description="Analisis sentimen ulasan untuk satu periode.")
+    # Negasi — JANGAN hapus
-    parser.add_argument("--periode-id", type=int, help="ID periode yang dianalisis. Default: periode terbaru.")
+    r"\b(ga|gak|gk|nggak|ngga|ngak|engga|enggak)\b": "tidak",
-    return parser.parse_args()
+    r"\b(blm|blom|blum)\b":   "belum",
+    r"\b(bkn|bukn)\b":        "bukan",
+    r"\b(jgn|jangan|jgan)\b": "jangan",
+    # Kata ganti
+    r"\b(ak|aq|gw|gue|gua)\b":          "aku",
+    r"\b(km|loe|lu|elo|lo)\b":           "kamu",
+    r"\b(sy|sya)\b":                      "saya",
+    # Kata kerja
+    r"\b(bs|bsa)\b":           "bisa",
+    r"\b(liat|lht)\b":         "lihat",
+    r"\b(mkn|mkan|maem)\b":    "makan",
+    r"\b(pake|pk|pke)\b":      "pakai",
+    r"\b(tau|taw|tw)\b":       "tahu",
+    r"\b(dtg)\b":              "datang",
+    r"\b(lgsg|lgsung)\b":      "langsung",
+    # Kata sifat
+    r"\b(bgs|bgus|nice|good|top)\b":  "bagus",
+    r"\b(josss|joss|sipp|sip)\b":     "mantap",
+    r"\b(byk|bnyk)\b":                "banyak",
+    r"\b(lbh|lbih)\b":                "lebih",
+    r"\b(bener|bnr)\b":               "benar",
+    # Kata sambung & keterangan
+    r"\b(aja|sja|ae)\b":       "saja",
+    r"\b(bgt|bangettt|bangett)\b": "banget",
+    r"\b(br|bru)\b":           "baru",
+    r"\b(cmn|cuma|cuman)\b":   "cuma",
+    r"\b(dgn|dngn|dg)\b":      "dengan",
+    r"\b(dl|dlu)\b":           "dulu",
+    r"\b(dlm|dalem)\b":        "dalam",
+    r"\b(dr|dri)\b":           "dari",
+    r"\b(emg|emang)\b":        "memang",
+    r"\b(gt|gitu|bgitu)\b":    "begitu",
+    r"\b(hbs|hbis)\b":         "habis",
+    r"\b(hrs|hrus)\b":         "harus",
+    r"\b(jd|jdi)\b":           "jadi",
+    r"\b(jg|jga)\b":           "juga",
+    r"\b(kdg|kdang)\b":        "kadang",
+    r"\b(klo|kalo|kl)\b":      "kalau",
+    r"\b(krn|karna)\b":        "karena",
+    r"\b(kyk|kek|kya)\b":      "seperti",
+    r"\b(lg|lgi)\b":           "lagi",
+    r"\b(mgkn|mngkin)\b":      "mungkin",
+    r"\b(msi|msh|msih)\b":     "masih",
+    r"\b(pd|pda)\b":           "pada",
+    r"\b(sdh|udh|udah|uda)\b": "sudah",
+    r"\b(skrg|skrng)\b":       "sekarang",
+    r"\b(sllu|slalu)\b":       "selalu",
+    r"\b(sm|ama)\b":           "sama",
+    r"\b(smpai|ampe|smpe)\b":  "sampai",
+    r"\b(smua)\b":             "semua",
+    r"\b(srg|sring)\b":        "sering",
+    r"\b(tp|tpi)\b":           "tapi",
+    r"\b(trs|trus)\b":         "lalu",
+    r"\b(ttp|ttep)\b":         "tetap",
+    r"\b(utk|untk)\b":         "untuk",
+    r"\b(yg|yng)\b":           "yang",
+    r"\b(pdhl|pdhal)\b":       "padahal",
+    r"\b(tmpt|tempt)\b":       "tempat",
+    r"\b(tmn|temen)\b":        "teman",
+    r"\b(org|orng)\b":         "orang",
+    # Konteks pariwisata
+    r"\b(recommended|recomended)\b": "rekomendasi",
+    r"\b(healing)\b":                "rekreasi",
+    r"\b(htm)\b":                    "harga tiket masuk",
+    r"\b(overall)\b":                "secara keseluruhan",
+    r"\b(spot foto)\b":              "lokasi foto",
+    # Hapus tawa & makian
+    r"\b(wkwk+|haha+|hehe+)\b": "",
+    r"\b(bjir|anjay|anjir)\b":  "",
+}
 
+# ═══════════════════════════════════════════════════════════════
+# KAMUS SENTIMEN (pseudo-label fallback)
+# ═══════════════════════════════════════════════════════════════
+
+POSITIF_WORDS = {
+    "bagus","indah","cantik","keren","mantap","asri","bersih","nyaman","rapi",
+    "adem","sejuk","segar","menarik","spektakuler","eksotis","unik","istimewa",
+    "menakjubkan","memukau","asyik","asik","senang","puas","suka","happy",
+    "enjoy","bahagia","menyenangkan","seru","recommended","rekomendasi","wajib",
+    "worth","memuaskan","healing","josss","joss","sip","lengkap","terawat",
+    "baik","oke","ramah","murah","terjangkau","luas","teduh","view","sunset",
+    "sunrise","jernih","bening","enak","lezat","amazing","beautiful","great",
+    "nice","good","perfect","best","lovely","wonderful","fantastic","awesome",
+}
+
+NEGATIF_WORDS = {
+    "kotor","jorok","jelek","buruk","rusak","kumuh","sempit","parah","payah",
+    "berantakan","mengecewakan","kecewa","nyesel","menyesal","bocor","mati",
+    "gelap","bau","busuk","pengap","mahal","kemahalan","lambat","antri","macet",
+    "sesak","penuh","berebut","kasar","jutek","cuek","berbahaya","bahaya",
+    "licin","curam","sampah","tidak puas","kapok","ogah","zonk","tipu","pungli",
+}
+
+NEGASI = {"tidak","bukan","jangan","belum","tanpa","kurang","ga","gak","nggak"}
+
+# ═══════════════════════════════════════════════════════════════
+# INISIALISASI NLP
+# ═══════════════════════════════════════════════════════════════
+
+log("INFO", "Memuat stemmer dan stopword Sastrawi...")
+_stemmer = StemmerFactory().create_stemmer()
+_sw_raw  = set(StopWordRemoverFactory().get_stop_words())
+# Pertahankan kata negasi — kritis untuk sentimen
+STOPWORDS = _sw_raw - NEGASI
+log("INFO", f"Stopword: {len(_sw_raw)} kata | Negasi dipertahankan: {NEGASI}")
+
+# ═══════════════════════════════════════════════════════════════
+# PREPROCESSING
+# ═══════════════════════════════════════════════════════════════
+
+def normalisasi_teks(text):
+    for pattern, replacement in NORMALISASI.items():
+        text = re.sub(pattern, replacement, text)
+    return text
+
+def preprocess_text(text):
+    """Pipeline preprocessing 6 tahap."""
+    text = str(text).lower()
+    text = re.sub(r"https?://\S+|www\.\S+", " ", text)   # hapus URL
+    text = re.sub(r"@\w+|#\w+", " ", text)               # hapus mention/hashtag
+    text = re.sub(r"[^a-z\s]", " ", text)                # hapus non-alfabet
+    text = re.sub(r"\s+", " ", text).strip()
+    text = normalisasi_teks(text)                         # normalisasi kamus
+    tokens = [t for t in text.split() if t not in STOPWORDS and len(t) > 2]
+    return _stemmer.stem(" ".join(tokens)).strip()
+
+# ═══════════════════════════════════════════════════════════════
+# PSEUDO-LABELING (fallback jika tidak ada model tersimpan)
+# ═══════════════════════════════════════════════════════════════
+
+def label_by_keyword(clean_text):
+    tokens = clean_text.split()
+    pos = neg = 0
+    i = 0
+    while i < len(tokens):
+        sebelum_negasi = (i > 0 and tokens[i-1] in NEGASI)
+        w = tokens[i]
+        if w in POSITIF_WORDS:
+            neg += 1 if sebelum_negasi else 0
+            pos += 0 if sebelum_negasi else 1
+        elif w in NEGATIF_WORDS:
+            pos += 1 if sebelum_negasi else 0
+            neg += 0 if sebelum_negasi else 1
+        i += 1
+
+    if pos > neg:   return "positif"
+    if neg > pos:   return "negatif"
+    return "netral"
+
+def make_pseudo_label(row):
+    return label_by_keyword(row["ulasan_bersih"])
+
+# ═══════════════════════════════════════════════════════════════
+# LOAD / TRAIN MODEL
+# ═══════════════════════════════════════════════════════════════
+
+def load_saved_model():
+    """Muat model tersimpan jika ada."""
+    if MODEL_PATH.exists():
+        log("INFO", f"Model tersimpan ditemukan → dimuat dari {MODEL_PATH}")
+        return joblib.load(MODEL_PATH)
+    return None
+
+def train_and_save_model(df):
+    """
+    Latih ComplementNB dengan:
+    - 5-Fold Stratified Cross Validation
+    - GridSearchCV hyperparameter tuning
+    - Evaluasi Macro F1 + classification report
+    """
+    log("INFO", "=" * 55)
+    log("INFO", "TRAINING MODEL NAIVE BAYES (ComplementNB)")
+    log("INFO", "=" * 55)
+
+    X = df["ulasan_bersih"].values
+    y = df["label"].values
+
+    label_counts = pd.Series(y).value_counts()
+    log("INFO", "Distribusi label: " + str(label_counts.to_dict()))
+
+    # Minimal 2 kelas dan tiap kelas >= 5
+    if label_counts.size < 2 or label_counts.min() < 2:
+        log("WARNING", "Data tidak cukup untuk training — pakai pseudo-label langsung.")
+        return None, 0, 0
+
+    # Split 80/20
+    can_stratify = label_counts.min() >= 2
+    X_train, X_test, y_train, y_test = train_test_split(
+        X, y, test_size=0.2, random_state=SEED,
+        stratify=y if can_stratify else None
+    )
+    log("INFO", f"Split — Train: {len(X_train)} | Test: {len(X_test)}")
+
+    kfold = StratifiedKFold(n_splits=min(5, label_counts.min()), shuffle=True, random_state=SEED)
+
+    # ── 5-Fold Cross Validation ──────────────────────────────
+    log("INFO", "Menjalankan 5-Fold Cross Validation...")
+    pipe_cv = Pipeline([
+        ("tfidf", TfidfVectorizer(ngram_range=(1,2), min_df=1, sublinear_tf=True)),
+        ("clf",   ComplementNB()),
+    ])
+    cv_scores = cross_val_score(pipe_cv, X_train, y_train, cv=kfold, scoring="f1_macro", n_jobs=-1)
+    log("INFO", f"CV Macro F1 per fold : {[round(s,4) for s in cv_scores]}")
+    log("INFO", f"CV Macro F1 rata-rata: {cv_scores.mean():.4f} ± {cv_scores.std():.4f}")
+
+    # ── GridSearchCV Tuning ───────────────────────────────────
+    log("INFO", "GridSearchCV tuning hiperparameter...")
+    param_grid = {
+        "tfidf__ngram_range":  [(1,1),(1,2)],
+        "tfidf__min_df":       [1, 2, 3],
+        "tfidf__sublinear_tf": [True, False],
+        "clf__alpha":          [0.1, 0.5, 1.0, 2.0],
+        "clf__norm":           [True, False],
+    }
+    pipe_gs = Pipeline([
+        ("tfidf", TfidfVectorizer()),
+        ("clf",   ComplementNB()),
+    ])
+    gs = GridSearchCV(pipe_gs, param_grid, cv=kfold, scoring="f1_macro", n_jobs=-1, refit=True)
+    gs.fit(X_train, y_train)
+
+    best = gs.best_estimator_
+    log("INFO", f"Parameter terbaik : {gs.best_params_}")
+    log("INFO", f"Best CV Macro F1  : {gs.best_score_:.4f}")
+
+    # ── Evaluasi Final ────────────────────────────────────────
+    y_pred   = best.predict(X_test)
+    accuracy = accuracy_score(y_test, y_pred)
+    macro_f1 = f1_score(y_test, y_pred, average="macro", zero_division=0)
+    report   = classification_report(y_test, y_pred, zero_division=0)
+    log("INFO", f"Akurasi       : {accuracy:.4f} ({accuracy*100:.2f}%)")
+    log("INFO", f"Macro F1-Score: {macro_f1:.4f}")
+    log("INFO", f"\nClassification Report:\n{report}")
+
+    # ── Simpan model & metadata ───────────────────────────────
+    joblib.dump(best, MODEL_PATH)
+    report_dict = classification_report(y_test, y_pred, output_dict=True, zero_division=0)
+    metadata = {
+        "model": "ComplementNB (Naive Bayes)",
+        "best_params": gs.best_params_,
+        "cv_macro_f1": {"mean": round(cv_scores.mean(),4), "std": round(cv_scores.std(),4)},
+        "evaluasi_test": {
+            "accuracy": round(accuracy,4),
+            "macro_f1": round(macro_f1,4),
+        },
+        "distribusi_label": pd.Series(y).value_counts().to_dict(),
+    }
+    with open(METADATA_PATH, "w", encoding="utf-8") as f:
+        json.dump(metadata, f, indent=4, ensure_ascii=False)
+
+    log("INFO", f"Model disimpan → {MODEL_PATH}")
+    return best, accuracy, macro_f1
+
+# ═══════════════════════════════════════════════════════════════
+# DATABASE
+# ═══════════════════════════════════════════════════════════════
 
 def read_laravel_env():
     env = {}
     env_file = BASE_DIR / ".env"
     if not env_file.exists():
         return env
-
     for line in env_file.read_text(encoding="utf-8").splitlines():
         line = line.strip()
         if not line or line.startswith("#") or "=" not in line:
@@ -53,281 +344,124 @@ def read_laravel_env():
         key, value = line.split("=", 1)
         value = value.strip().strip('"').strip("'")
         env.setdefault(key.strip(), value)
-
     return env
 
-
 def env_value(env, key, default=""):
     value = os.getenv(key, env.get(key, default))
     if value in {None, "", "null", "None"}:
         return default
     return value
 
-
 def db_config():
-    env = read_laravel_env()
+    env  = read_laravel_env()
-    connection = env_value(env, "DB_CONNECTION", "mysql")
+    conn = env_value(env, "DB_CONNECTION", "mysql")
-
+    if conn == "sqlite":
-    if connection == "sqlite":
+        db = env_value(env, "DB_DATABASE", str(BASE_DIR / "database" / "database.sqlite"))
-        database = env_value(env, "DB_DATABASE", str(BASE_DIR / "database" / "database.sqlite"))
+        db_path = Path(db)
-        database_path = Path(database)
+        if not db_path.is_absolute():
-        if not database_path.is_absolute():
+            db_path = BASE_DIR / db
-            database_path = BASE_DIR / database
+        return {"connection": conn, "database": str(db_path)}
-
+    if conn not in {"mysql", "mariadb"}:
-        return {
+        fail(f"DB_CONNECTION={conn} belum didukung.")
-            "connection": connection,
-            "database": str(database_path),
-        }
-
-    if connection not in {"mysql", "mariadb"}:
-        fail(f"DB_CONNECTION={connection} belum didukung oleh analisis.py. Gunakan sqlite/mysql/mariadb.")
-
     return {
-        "connection": connection,
+        "connection": conn,
-        "host": env_value(env, "DB_HOST", "127.0.0.1"),
+        "host":       env_value(env, "DB_HOST", "127.0.0.1"),
-        "port": int(env_value(env, "DB_PORT", "3306")),
+        "port":       int(env_value(env, "DB_PORT", "3306")),
-        "database": env_value(env, "DB_DATABASE", "sistem_analisis"),
+        "database":   env_value(env, "DB_DATABASE", "sistem_analisis"),
-        "user": env_value(env, "DB_USERNAME", "root"),
+        "user":       env_value(env, "DB_USERNAME", "root"),
-        "password": env_value(env, "DB_PASSWORD", ""),
+        "password":   env_value(env, "DB_PASSWORD", ""),
     }
 
-
 def make_connections(config):
     if config["connection"] == "sqlite":
         engine = create_engine(f"sqlite:///{config['database']}")
-        conn = sqlite3.connect(config["database"])
+        conn   = sqlite3.connect(config["database"])
         conn.row_factory = sqlite3.Row
         return engine, conn
-
+    url = (
-    engine_url = (
         "mysql+pymysql://"
         f"{quote_plus(config['user'])}:{quote_plus(config['password'])}"
         f"@{config['host']}:{config['port']}/{config['database']}?charset=utf8mb4"
     )
-    engine = create_engine(engine_url)
+    engine = create_engine(url)
-    conn = pymysql.connect(
+    conn   = pymysql.connect(
-        host=config["host"],
+        host=config["host"], port=config["port"],
-        port=config["port"],
+        user=config["user"], password=config["password"],
-        user=config["user"],
+        database=config["database"], charset="utf8mb4",
-        password=config["password"],
-        database=config["database"],
-        charset="utf8mb4",
         cursorclass=pymysql.cursors.DictCursor,
     )
     return engine, conn
 
-
+def is_sqlite(conn):
-def is_sqlite_connection(conn):
     return isinstance(conn, sqlite3.Connection)
 
-
 def prepare_sql(conn, sql):
-    if is_sqlite_connection(conn):
+    if is_sqlite(conn):
         return sql.replace("%s", "?").replace("NOW()", "CURRENT_TIMESTAMP")
     return sql
 
-
 def execute(cursor, conn, sql, params=()):
     cursor.execute(prepare_sql(conn, sql), params)
 
-
 def table_columns(cursor, conn, table):
-    if is_sqlite_connection(conn):
+    if is_sqlite(conn):
         cursor.execute(f"PRAGMA table_info({table})")
         return {row["name"] for row in cursor.fetchall()}
-
     cursor.execute(f"SHOW COLUMNS FROM {table}")
     return {row["Field"] for row in cursor.fetchall()}
 
-
+# ═══════════════════════════════════════════════════════════════
-SLANG_MAP = {
+# MAIN
-    "ga": "tidak",
+# ═══════════════════════════════════════════════════════════════
-    "gak": "tidak",
-    "gk": "tidak",
-    "nggak": "tidak",
-    "ngga": "tidak",
-    "ngak": "tidak",
-    "bgt": "banget",
-    "yg": "yang",
-    "tp": "tapi",
-}
-
-POSITIF_WORDS = {
-    "bagus", "indah", "mantap", "keren", "cantik", "menarik", "nyaman",
-    "bersih", "recommended", "rekomendasi", "suka", "senang", "puas",
-    "murah", "asyik", "ramah", "worth", "spektakuler", "memukau",
-    "sejuk", "kece", "amazing", "beautiful", "good", "nice", "best",
-    "great", "perfect", "recommend", "memuaskan", "menyenangkan", "view",
-    "seru", "enak", "adem",
-}
-
-NEGATIF_WORDS = {
-    "tidak", "buruk", "mahal", "jelek", "kotor", "kecewa", "rusak",
-    "sempit", "panas", "bau", "berbahaya", "sepi", "bosan",
-    "mengecewakan", "payah", "parah", "jorok", "macet", "antri",
-    "penuh", "sampah", "sayang", "kurang", "susah", "sulit", "jauh",
-    "capek", "lelah",
-}
-
-
-stemmer = StemmerFactory().create_stemmer()
-stopwords = set(StopWordRemoverFactory().get_stop_words())
-stopwords.discard("tidak")
-stopwords.discard("bukan")
-stopwords.discard("jangan")
-
-
-def normalize_rating(value):
-    if pd.isna(value):
-        return None
-    match = re.search(r"([1-5])", str(value))
-    return int(match.group(1)) if match else None
-
-
-def preprocess_text(text):
-    text = str(text).lower()
-    text = re.sub(r"https?://\S+|www\.\S+", " ", text)
-    text = re.sub(r"[^a-z\s]", " ", text)
-    text = re.sub(r"\s+", " ", text).strip()
-
-    words = [SLANG_MAP.get(word, word) for word in text.split()]
-    words = [word for word in words if word not in stopwords and len(word) > 2]
-
-    return stemmer.stem(" ".join(words)).strip()
-
-
-# def label_by_keyword(clean_text):
-#     words = set(clean_text.split())
-#     positive_score = len(words & POSITIF_WORDS)
-#     negative_score = len(words & NEGATIF_WORDS)
-
-#     if positive_score > negative_score:
-#         return "positif"
-#     if negative_score > positive_score:
-#         return "negatif"
-#     return "netral"
-
-
-def label_by_keyword(clean_text):
-    words = clean_text.split()
-
-    positive_score = sum(1 for word in words if word in POSITIF_WORDS)
-    negative_score = sum(1 for word in words if word in NEGATIF_WORDS)
-
-    if positive_score > negative_score:
-        return "positif"
-
-    elif negative_score > positive_score:
-        return "negatif"
-
-    return "netral"
-
-
-# def make_pseudo_label(row):
-#     rating = normalize_rating(row.get("rating"))
-#     if rating is not None:
-#         if rating >= 4:
-#             return "positif"
-#         if rating == 3:
-#             return "netral"
-#         return "negatif"
-
-#     return label_by_keyword(row["ulasan_bersih"])
-
-def make_pseudo_label(row):
-    return label_by_keyword(row["ulasan_bersih"])
-
-
-def rating_confidence(value):
-    rating = normalize_rating(value)
-    if rating is None:
-        return None
-    if rating in {1, 5}:
-        return 1.0
-    if rating in {2, 4}:
-        return 0.85
-    return 0.7
-
-
-def apply_rating_priority(row, model_classes=None):
-    sentiment_result = row["sentimen"]
-    probability = float(row["probabilitas"])
-
-    # Jika probabilitas model sangat rendah (di bawah 0.5), baru gunakan rating
-    if probability < 0.5:
-        rating_label = make_pseudo_label(row)
-        return rating_label, 0.5
-    
-    return sentiment_result, probability
-
-    # rating_label = make_pseudo_label(row)
-    # rating = normalize_rating(row.get("rating"))
-    # if rating is None:
-    #     return row["sentimen"], float(row["probabilitas"])
-
-    if row["sentimen"] != rating_label:
-        log(
-            "INFO",
-            f"Override sentimen berdasarkan rating {rating}: model={row['sentimen']} -> final={rating_label}",
-        )
-
-    probability = rating_confidence(rating)
-    if model_classes is not None and rating_label in model_classes:
-        try:
-            class_index = list(model_classes).index(rating_label)
-            probability = max(float(row["probabilitas_by_class"][class_index]), probability)
-        except Exception:
-            pass
-
-    return rating_label, probability
-
 
 def main():
-    args = parse_args()
+    import argparse
-    config = db_config()
+    parser = argparse.ArgumentParser(description="SENTARA — Analisis Sentimen")
-    if config["connection"] == "sqlite":
+    parser.add_argument("--periode-id", type=int, default=None)
-        log("INFO", f"Menggunakan database SQLite {config['database']}")
+    args = parser.parse_args()
-    else:
-        log("INFO", f"Menggunakan database {config['database']} di {config['host']}:{config['port']}")
 
+    log("INFO", "=" * 55)
+    log("INFO", "SENTARA — Sistem Analisis Sentimen Wisata Jember")
+    log("INFO", "=" * 55)
+
+    config         = db_config()
     engine, raw_conn = make_connections(config)
-    cursor = raw_conn.cursor()
+    cursor         = raw_conn.cursor()
 
     try:
+        # ── Cari periode ──────────────────────────────────────
         if args.periode_id:
-            execute(cursor, raw_conn, "SELECT id, nama FROM periode_analisis WHERE id = %s LIMIT 1", (args.periode_id,))
+            execute(cursor, raw_conn,
+                "SELECT id, nama FROM periode_analisis WHERE id = %s LIMIT 1",
+                (args.periode_id,))
         else:
             execute(cursor, raw_conn, """
-                SELECT p.id, p.nama
+                SELECT p.id, p.nama FROM periode_analisis p
-                FROM periode_analisis p
+                WHERE EXISTS (SELECT 1 FROM ulasan u WHERE u.periode_id = p.id)
-                WHERE EXISTS (
+                ORDER BY p.id DESC LIMIT 1
-                    SELECT 1 FROM ulasan u WHERE u.periode_id = p.id
-                )
-                ORDER BY p.id DESC
-                LIMIT 1
             """)
         periode = cursor.fetchone()
         if not periode:
-            fail("Belum ada periode yang memiliki ulasan. Jalankan Ambil Data terlebih dahulu.")
+            fail("Belum ada periode dengan ulasan.")
 
-        periode_id = periode["id"]
+        periode_id   = periode["id"]
         periode_nama = periode["nama"]
-        log("INFO", f"Analisis periode terbaru: {periode_nama} (periode_id={periode_id})")
+        log("INFO", f"Periode: {periode_nama} (id={periode_id})")
 
+        # ── Ambil ulasan baru (belum dianalisis) ──────────────
         df = pd.read_sql(
-            prepare_sql(
+            prepare_sql(raw_conn,
-                raw_conn,
                 "SELECT u.id, u.wisata, u.reviewer, u.rating, u.ulasan, u.tanggal, u.periode_id "
-                 "FROM ulasan u WHERE u.periode_id = %s AND NOT EXISTS ( SELECT 1 FROM hasil_analisis h WHERE h.ulasan_id = u.id  )"
+                "FROM ulasan u WHERE u.periode_id = %s "
+                "AND NOT EXISTS (SELECT 1 FROM hasil_analisis h WHERE h.ulasan_id = u.id)"
             ),
-            
+            engine, params=(periode_id,),
-            engine,
-            params=(periode_id,),
         )
 
         if df.empty:
-            fail(f"Tidak ada data ulasan untuk periode_id={periode_id}.")
+            fail(f"Tidak ada ulasan baru untuk periode_id={periode_id}.")
 
+        # ── Validasi & bersihkan ──────────────────────────────
         df = df.dropna(subset=["ulasan"]).copy()
         df["ulasan"] = df["ulasan"].astype(str)
         df = df[df["ulasan"].str.strip().ne("")]
@@ -336,97 +470,82 @@ def main():
         df = df[df["ulasan"].str.len() > 5]
 
         if df.empty:
-            fail("Data ulasan kosong setelah validasi teks.")
+            fail("Data kosong setelah validasi.")
 
+        # ── Preprocessing ─────────────────────────────────────
+        log("INFO", "Preprocessing teks (6 tahap)...")
         df["ulasan_bersih"] = df["ulasan"].apply(preprocess_text)
         df = df[df["ulasan_bersih"].str.strip().ne("")].copy()
+        log("INFO", f"Data valid setelah preprocessing: {len(df)} baris")
 
-        if df.empty:
+        # ── Pseudo-label ──────────────────────────────────────
-            fail("Data kosong setelah preprocessing. Tidak ada teks yang bisa dianalisis.")
-
         df["label"] = df.apply(make_pseudo_label, axis=1)
         label_counts = df["label"].value_counts()
-        log("INFO", "Distribusi pseudo-label: " + ", ".join(f"{k}={v}" for k, v in label_counts.items()))
+        log("INFO", "Distribusi pseudo-label: " + str(label_counts.to_dict()))
 
-        use_model = True
+        # ── Load atau Train model ─────────────────────────────
-        if label_counts.size < 2:
+        saved_pipeline = load_saved_model()
-            use_model = False
-            log("WARNING", "Jumlah kelas kurang dari 2. Prediksi memakai pseudo-label langsung tanpa training model.")
 
-        can_stratify = label_counts.min() >= 2
+        if saved_pipeline is not None:
-        if not can_stratify:
+            # Pakai model tersimpan (dari training manual sebelumnya)
-            log("WARNING", "Ada kelas dengan jumlah data kurang dari 2. Split evaluasi dibuat tanpa stratify.")
+            log("INFO", "Menggunakan model tersimpan untuk prediksi.")
-
+            best_pipeline = saved_pipeline
-        report = {"weighted avg": {"precision": 0, "recall": 0, "f1-score": 0}}
+            # Baca metadata akurasi
-        accuracy = 0
+            accuracy = macro_f1 = 0.0
-
+            if METADATA_PATH.exists():
-        if use_model:
+                with open(METADATA_PATH, encoding="utf-8") as f:
-            X = df["ulasan_bersih"]
+                    meta = json.load(f)
-            y = df["label"]
+                accuracy = meta.get("evaluasi_test", {}).get("accuracy", 0)
-
+                macro_f1 = meta.get("evaluasi_test", {}).get("macro_f1", 0)
-            if len(df) >= 5:
+                log("INFO", f"Akurasi model tersimpan : {accuracy:.4f}")
-                X_train, X_test, y_train, y_test = train_test_split(
+                log("INFO", f"Macro F1 model tersimpan: {macro_f1:.4f}")
-                    X,
+            report_dict = {"weighted avg": {"precision": accuracy, "recall": accuracy, "f1-score": macro_f1}}
-                    y,
-                    test_size=0.2,
-                    random_state=42,
-                    stratify=y if can_stratify else None,
-                )
-            else:
-                log("WARNING", "Data kurang dari 5 baris. Evaluasi memakai data latih yang sama.")
-                X_train, X_test, y_train, y_test = X, X, y, y
-
-            vectorizer = TfidfVectorizer(max_features=5000, ngram_range=(1, 2))
-            X_train_vec = vectorizer.fit_transform(X_train)
-            X_test_vec = vectorizer.transform(X_test)
-
-            model = ComplementNB()
-            model.fit(X_train_vec, y_train)
-
-            y_pred = model.predict(X_test_vec)
-            report = classification_report(y_test, y_pred, output_dict=True, zero_division=0)
-            accuracy = accuracy_score(y_test, y_pred)
-
-            X_all_vec = vectorizer.transform(df["ulasan_bersih"])
-            df["sentimen"] = model.predict(X_all_vec)
-            probability_matrix = model.predict_proba(X_all_vec)
-            df["probabilitas"] = probability_matrix.max(axis=1)
-            df["probabilitas"] = df["probabilitas"].clip(upper=0.99)
-            df["probabilitas_by_class"] = list(probability_matrix)
-            final_results = df.apply(lambda row: apply_rating_priority(row, model.classes_), axis=1)
-            df["sentimen"] = [result[0] for result in final_results]
-            df["probabilitas"] = [result[1] for result in final_results]
-            df = df.drop(columns=["probabilitas_by_class"])
         else:
-            df["sentimen"] = df["label"]
+            # Tidak ada model → train dari pseudo-label
-            df["probabilitas"] = df["rating"].apply(lambda rating: rating_confidence(rating) or 0.7)
+            log("WARNING",
+                "Model belum tersimpan. Training dari pseudo-label.\n"
+                "Untuk akurasi lebih baik, latih dengan dataset berlabel manual:\n"
+                "python analisis.py --mode train --dataset data_berlabel.csv")
+            best_pipeline, accuracy, macro_f1 = train_and_save_model(df)
 
-        log("INFO", "Evaluasi memakai pseudo-label dari rating/rule otomatis, bukan label manual.")
+            if best_pipeline is None:
+                # Fallback: langsung pakai pseudo-label
+                df["sentimen"]    = df["label"]
+                df["probabilitas"] = 0.60
+                accuracy = macro_f1 = 0.0
+                report_dict = {"weighted avg": {"precision": 0, "recall": 0, "f1-score": 0}}
+            else:
+                report_dict = {"weighted avg": {"precision": accuracy, "recall": accuracy, "f1-score": macro_f1}}
 
-        # execute(cursor, raw_conn, "DELETE FROM hasil_analisis WHERE periode_id = %s", (periode_id,))
+        # ── Prediksi ──────────────────────────────────────────
-        # execute(cursor, raw_conn, "DELETE FROM evaluasi_model WHERE periode_id = %s", (periode_id,))
+        if best_pipeline is not None:
+            log("INFO", "Memprediksi sentimen...")
+            df["sentimen"]    = best_pipeline.predict(df["ulasan_bersih"])
+            prob_matrix       = best_pipeline.predict_proba(df["ulasan_bersih"])
+            df["probabilitas"] = prob_matrix.max(axis=1).clip(max=0.99)
 
+            # Post-processing: probabilitas rendah → fallback kamus
+            mask_low = df["probabilitas"] < 0.45
+            if mask_low.sum() > 0:
+                log("INFO", f"{mask_low.sum()} ulasan probabilitas rendah → fallback ke kamus")
+                df.loc[mask_low, "sentimen"]    = df.loc[mask_low, "ulasan_bersih"].apply(label_by_keyword)
+                df.loc[mask_low, "probabilitas"] = 0.50
+
+        dist = df["sentimen"].value_counts().to_dict()
+        log("INFO", f"Distribusi sentimen hasil: {dist}")
+
+        # ── Simpan ke hasil_analisis ──────────────────────────
         hasil_columns = table_columns(cursor, raw_conn, "hasil_analisis")
         insert_columns = [
-            "ulasan_id",
+            "ulasan_id","wisata","ulasan_asli","ulasan_bersih",
-            "wisata",
+            "hasil_preprocessing","sentimen","probabilitas",
-            "ulasan_asli",
+            "periode_id","created_at","updated_at",
-            "ulasan_bersih",
-            "hasil_preprocessing",
-            "sentimen",
-            "probabilitas",
-            "periode_id",
-            "created_at",
-            "updated_at",
         ]
         if "ulasan_terolah" in hasil_columns:
             insert_columns.insert(3, "ulasan_terolah")
 
         placeholders = ", ".join(["%s"] * (len(insert_columns) - 2) + ["NOW()", "NOW()"])
-        insert_hasil = f"""
+        insert_sql   = f"INSERT INTO hasil_analisis ({', '.join(insert_columns)}) VALUES ({placeholders})"
-            INSERT INTO hasil_analisis ({", ".join(insert_columns)})
-            VALUES ({placeholders})
-        """
 
         for _, row in df.fillna("").iterrows():
             values = [
@@ -441,33 +560,24 @@ def main():
             ]
             if "ulasan_terolah" in hasil_columns:
                 values.insert(3, str(row["ulasan_bersih"]))
+            execute(cursor, raw_conn, insert_sql, tuple(values))
 
-            execute(cursor, raw_conn, insert_hasil, tuple(values))
+        # ── Simpan ke evaluasi_model ──────────────────────────
-
+        weighted = report_dict.get("weighted avg", {})
-        weighted = report.get("weighted avg", {})
+        execute(cursor, raw_conn, """
-        execute(
-            cursor,
-            raw_conn,
-            """
             INSERT INTO evaluasi_model
             (`precision`, `recall`, f1_score, accuracy, tp, tn, fp, fn, periode_id, created_at, updated_at)
             VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, NOW(), NOW())
-            """,
+        """, (
-            (
+            float(weighted.get("precision", 0)),
-                float(weighted.get("precision", 0)),
+            float(weighted.get("recall",    0)),
-                float(weighted.get("recall", 0)),
+            float(weighted.get("f1-score",  macro_f1)),
-                float(weighted.get("f1-score", 0)),
+            float(accuracy),
-                float(accuracy),
+            0, 0, 0, 0, periode_id,
-                0,
+        ))
-                0,
-                0,
-                0,
-                periode_id,
-            ),
-        )
 
         raw_conn.commit()
-        log("OK", f"{len(df)} hasil analisis disimpan untuk periode {periode_nama}.")
+        log("OK", f"{len(df)} hasil analisis disimpan untuk periode '{periode_nama}'.")
         log("OK", "Analisis selesai.")
 
     except SystemExit:

scraper/analisis_backup.py

@@ -0,0 +1,485 @@
+# -*- coding: utf-8 -*-
+import argparse
+import os
+import re
+import sys
+import sqlite3
+from pathlib import Path
+from urllib.parse import quote_plus
+
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics import accuracy_score, classification_report
+from sklearn.model_selection import train_test_split
+from sklearn.naive_bayes import ComplementNB
+
+from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
+from Sastrawi.StopWordRemover.StopWordRemoverFactory import StopWordRemoverFactory
+
+sys.stdout.reconfigure(encoding="utf-8")
+sys.stderr.reconfigure(encoding="utf-8")
+
+BASE_DIR = Path(__file__).resolve().parents[1]
+
+
+def log(level, message):
+    print(f"[{level}] {message}", flush=True)
+
+
+def fail(message, code=1):
+    log("ERROR", message)
+    sys.exit(code)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Analisis sentimen ulasan untuk satu periode.")
+    parser.add_argument("--periode-id", type=int, help="ID periode yang dianalisis. Default: periode terbaru.")
+    return parser.parse_args()
+
+
+def read_laravel_env():
+    env = {}
+    env_file = BASE_DIR / ".env"
+    if not env_file.exists():
+        return env
+
+    for line in env_file.read_text(encoding="utf-8").splitlines():
+        line = line.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        key, value = line.split("=", 1)
+        value = value.strip().strip('"').strip("'")
+        env.setdefault(key.strip(), value)
+
+    return env
+
+
+def env_value(env, key, default=""):
+    value = os.getenv(key, env.get(key, default))
+    if value in {None, "", "null", "None"}:
+        return default
+    return value
+
+
+def db_config():
+    env = read_laravel_env()
+    connection = env_value(env, "DB_CONNECTION", "mysql")
+
+    if connection == "sqlite":
+        database = env_value(env, "DB_DATABASE", str(BASE_DIR / "database" / "database.sqlite"))
+        database_path = Path(database)
+        if not database_path.is_absolute():
+            database_path = BASE_DIR / database
+
+        return {
+            "connection": connection,
+            "database": str(database_path),
+        }
+
+    if connection not in {"mysql", "mariadb"}:
+        fail(f"DB_CONNECTION={connection} belum didukung oleh analisis.py. Gunakan sqlite/mysql/mariadb.")
+
+    return {
+        "connection": connection,
+        "host": env_value(env, "DB_HOST", "127.0.0.1"),
+        "port": int(env_value(env, "DB_PORT", "3306")),
+        "database": env_value(env, "DB_DATABASE", "sistem_analisis"),
+        "user": env_value(env, "DB_USERNAME", "root"),
+        "password": env_value(env, "DB_PASSWORD", ""),
+    }
+
+
+def make_connections(config):
+    if config["connection"] == "sqlite":
+        engine = create_engine(f"sqlite:///{config['database']}")
+        conn = sqlite3.connect(config["database"])
+        conn.row_factory = sqlite3.Row
+        return engine, conn
+
+    engine_url = (
+        "mysql+pymysql://"
+        f"{quote_plus(config['user'])}:{quote_plus(config['password'])}"
+        f"@{config['host']}:{config['port']}/{config['database']}?charset=utf8mb4"
+    )
+    engine = create_engine(engine_url)
+    conn = pymysql.connect(
+        host=config["host"],
+        port=config["port"],
+        user=config["user"],
+        password=config["password"],
+        database=config["database"],
+        charset="utf8mb4",
+        cursorclass=pymysql.cursors.DictCursor,
+    )
+    return engine, conn
+
+
+def is_sqlite_connection(conn):
+    return isinstance(conn, sqlite3.Connection)
+
+
+def prepare_sql(conn, sql):
+    if is_sqlite_connection(conn):
+        return sql.replace("%s", "?").replace("NOW()", "CURRENT_TIMESTAMP")
+    return sql
+
+
+def execute(cursor, conn, sql, params=()):
+    cursor.execute(prepare_sql(conn, sql), params)
+
+
+def table_columns(cursor, conn, table):
+    if is_sqlite_connection(conn):
+        cursor.execute(f"PRAGMA table_info({table})")
+        return {row["name"] for row in cursor.fetchall()}
+
+    cursor.execute(f"SHOW COLUMNS FROM {table}")
+    return {row["Field"] for row in cursor.fetchall()}
+
+
+SLANG_MAP = {
+    "ga": "tidak",
+    "gak": "tidak",
+    "gk": "tidak",
+    "nggak": "tidak",
+    "ngga": "tidak",
+    "ngak": "tidak",
+    "bgt": "banget",
+    "yg": "yang",
+    "tp": "tapi",
+}
+
+POSITIF_WORDS = {
+    "bagus", "indah", "mantap", "keren", "cantik", "menarik", "nyaman",
+    "bersih", "recommended", "rekomendasi", "suka", "senang", "puas",
+    "murah", "asyik", "ramah", "worth", "spektakuler", "memukau",
+    "sejuk", "kece", "amazing", "beautiful", "good", "nice", "best",
+    "great", "perfect", "recommend", "memuaskan", "menyenangkan", "view",
+    "seru", "enak", "adem",
+}
+
+NEGATIF_WORDS = {
+    "tidak", "buruk", "mahal", "jelek", "kotor", "kecewa", "rusak",
+    "sempit", "panas", "bau", "berbahaya", "sepi", "bosan",
+    "mengecewakan", "payah", "parah", "jorok", "macet", "antri",
+    "penuh", "sampah", "sayang", "kurang", "susah", "sulit", "jauh",
+    "capek", "lelah",
+}
+
+
+stemmer = StemmerFactory().create_stemmer()
+stopwords = set(StopWordRemoverFactory().get_stop_words())
+stopwords.discard("tidak")
+stopwords.discard("bukan")
+stopwords.discard("jangan")
+
+
+def normalize_rating(value):
+    if pd.isna(value):
+        return None
+    match = re.search(r"([1-5])", str(value))
+    return int(match.group(1)) if match else None
+
+
+def preprocess_text(text):
+    text = str(text).lower()
+    text = re.sub(r"https?://\S+|www\.\S+", " ", text)
+    text = re.sub(r"[^a-z\s]", " ", text)
+    text = re.sub(r"\s+", " ", text).strip()
+
+    words = [SLANG_MAP.get(word, word) for word in text.split()]
+    words = [word for word in words if word not in stopwords and len(word) > 2]
+
+    return stemmer.stem(" ".join(words)).strip()
+
+
+# def label_by_keyword(clean_text):
+#     words = set(clean_text.split())
+#     positive_score = len(words & POSITIF_WORDS)
+#     negative_score = len(words & NEGATIF_WORDS)
+
+#     if positive_score > negative_score:
+#         return "positif"
+#     if negative_score > positive_score:
+#         return "negatif"
+#     return "netral"
+
+
+def label_by_keyword(clean_text):
+    words = clean_text.split()
+
+    positive_score = sum(1 for word in words if word in POSITIF_WORDS)
+    negative_score = sum(1 for word in words if word in NEGATIF_WORDS)
+
+    if positive_score > negative_score:
+        return "positif"
+
+    elif negative_score > positive_score:
+        return "negatif"
+
+    return "netral"
+
+
+# def make_pseudo_label(row):
+#     rating = normalize_rating(row.get("rating"))
+#     if rating is not None:
+#         if rating >= 4:
+#             return "positif"
+#         if rating == 3:
+#             return "netral"
+#         return "negatif"
+
+#     return label_by_keyword(row["ulasan_bersih"])
+
+def make_pseudo_label(row):
+    return label_by_keyword(row["ulasan_bersih"])
+
+
+def rating_confidence(value):
+    rating = normalize_rating(value)
+    if rating is None:
+        return None
+    if rating in {1, 5}:
+        return 1.0
+    if rating in {2, 4}:
+        return 0.85
+    return 0.7
+
+
+def apply_rating_priority(row, model_classes=None):
+    sentiment_result = row["sentimen"]
+    probability = float(row["probabilitas"])
+
+    # Jika probabilitas model sangat rendah (di bawah 0.5), baru gunakan rating
+    if probability < 0.5:
+        rating_label = make_pseudo_label(row)
+        return rating_label, 0.5
+    
+    return sentiment_result, probability
+
+    # rating_label = make_pseudo_label(row)
+    # rating = normalize_rating(row.get("rating"))
+    # if rating is None:
+    #     return row["sentimen"], float(row["probabilitas"])
+
+    if row["sentimen"] != rating_label:
+        log(
+            "INFO",
+            f"Override sentimen berdasarkan rating {rating}: model={row['sentimen']} -> final={rating_label}",
+        )
+
+    probability = rating_confidence(rating)
+    if model_classes is not None and rating_label in model_classes:
+        try:
+            class_index = list(model_classes).index(rating_label)
+            probability = max(float(row["probabilitas_by_class"][class_index]), probability)
+        except Exception:
+            pass
+
+    return rating_label, probability
+
+
+def main():
+    args = parse_args()
+    config = db_config()
+    if config["connection"] == "sqlite":
+        log("INFO", f"Menggunakan database SQLite {config['database']}")
+    else:
+        log("INFO", f"Menggunakan database {config['database']} di {config['host']}:{config['port']}")
+
+    engine, raw_conn = make_connections(config)
+    cursor = raw_conn.cursor()
+
+    try:
+        if args.periode_id:
+            execute(cursor, raw_conn, "SELECT id, nama FROM periode_analisis WHERE id = %s LIMIT 1", (args.periode_id,))
+        else:
+            execute(cursor, raw_conn, """
+                SELECT p.id, p.nama
+                FROM periode_analisis p
+                WHERE EXISTS (
+                    SELECT 1 FROM ulasan u WHERE u.periode_id = p.id
+                )
+                ORDER BY p.id DESC
+                LIMIT 1
+            """)
+        periode = cursor.fetchone()
+        if not periode:
+            fail("Belum ada periode yang memiliki ulasan. Jalankan Ambil Data terlebih dahulu.")
+
+        periode_id = periode["id"]
+        periode_nama = periode["nama"]
+        log("INFO", f"Analisis periode terbaru: {periode_nama} (periode_id={periode_id})")
+
+        df = pd.read_sql(
+            prepare_sql(
+                raw_conn,
+                "SELECT u.id, u.wisata, u.reviewer, u.rating, u.ulasan, u.tanggal, u.periode_id "
+                 "FROM ulasan u WHERE u.periode_id = %s AND NOT EXISTS ( SELECT 1 FROM hasil_analisis h WHERE h.ulasan_id = u.id  )"
+            ),
+            
+            engine,
+            params=(periode_id,),
+        )
+
+        if df.empty:
+            fail(f"Tidak ada data ulasan untuk periode_id={periode_id}.")
+
+        df = df.dropna(subset=["ulasan"]).copy()
+        df["ulasan"] = df["ulasan"].astype(str)
+        df = df[df["ulasan"].str.strip().ne("")]
+        df = df[df["ulasan"].str.strip().ne("0")]
+        df = df[~df["ulasan"].str.contains(r"\[Tanpa teks\]", na=False)]
+        df = df[df["ulasan"].str.len() > 5]
+
+        if df.empty:
+            fail("Data ulasan kosong setelah validasi teks.")
+
+        df["ulasan_bersih"] = df["ulasan"].apply(preprocess_text)
+        df = df[df["ulasan_bersih"].str.strip().ne("")].copy()
+
+        if df.empty:
+            fail("Data kosong setelah preprocessing. Tidak ada teks yang bisa dianalisis.")
+
+        df["label"] = df.apply(make_pseudo_label, axis=1)
+        label_counts = df["label"].value_counts()
+        log("INFO", "Distribusi pseudo-label: " + ", ".join(f"{k}={v}" for k, v in label_counts.items()))
+
+        use_model = True
+        if label_counts.size < 2:
+            use_model = False
+            log("WARNING", "Jumlah kelas kurang dari 2. Prediksi memakai pseudo-label langsung tanpa training model.")
+
+        can_stratify = label_counts.min() >= 2
+        if not can_stratify:
+            log("WARNING", "Ada kelas dengan jumlah data kurang dari 2. Split evaluasi dibuat tanpa stratify.")
+
+        report = {"weighted avg": {"precision": 0, "recall": 0, "f1-score": 0}}
+        accuracy = 0
+
+        if use_model:
+            X = df["ulasan_bersih"]
+            y = df["label"]
+
+            if len(df) >= 5:
+                X_train, X_test, y_train, y_test = train_test_split(
+                    X,
+                    y,
+                    test_size=0.2,
+                    random_state=42,
+                    stratify=y if can_stratify else None,
+                )
+            else:
+                log("WARNING", "Data kurang dari 5 baris. Evaluasi memakai data latih yang sama.")
+                X_train, X_test, y_train, y_test = X, X, y, y
+
+            vectorizer = TfidfVectorizer(max_features=5000, ngram_range=(1, 2))
+            X_train_vec = vectorizer.fit_transform(X_train)
+            X_test_vec = vectorizer.transform(X_test)
+
+            model = ComplementNB()
+            model.fit(X_train_vec, y_train)
+
+            y_pred = model.predict(X_test_vec)
+            report = classification_report(y_test, y_pred, output_dict=True, zero_division=0)
+            accuracy = accuracy_score(y_test, y_pred)
+
+            X_all_vec = vectorizer.transform(df["ulasan_bersih"])
+            df["sentimen"] = model.predict(X_all_vec)
+            probability_matrix = model.predict_proba(X_all_vec)
+            df["probabilitas"] = probability_matrix.max(axis=1)
+            df["probabilitas"] = df["probabilitas"].clip(upper=0.99)
+            df["probabilitas_by_class"] = list(probability_matrix)
+            final_results = df.apply(lambda row: apply_rating_priority(row, model.classes_), axis=1)
+            df["sentimen"] = [result[0] for result in final_results]
+            df["probabilitas"] = [result[1] for result in final_results]
+            df = df.drop(columns=["probabilitas_by_class"])
+        else:
+            df["sentimen"] = df["label"]
+            df["probabilitas"] = df["rating"].apply(lambda rating: rating_confidence(rating) or 0.7)
+
+        log("INFO", "Evaluasi memakai pseudo-label dari rating/rule otomatis, bukan label manual.")
+
+        # execute(cursor, raw_conn, "DELETE FROM hasil_analisis WHERE periode_id = %s", (periode_id,))
+        # execute(cursor, raw_conn, "DELETE FROM evaluasi_model WHERE periode_id = %s", (periode_id,))
+
+        hasil_columns = table_columns(cursor, raw_conn, "hasil_analisis")
+        insert_columns = [
+            "ulasan_id",
+            "wisata",
+            "ulasan_asli",
+            "ulasan_bersih",
+            "hasil_preprocessing",
+            "sentimen",
+            "probabilitas",
+            "periode_id",
+            "created_at",
+            "updated_at",
+        ]
+        if "ulasan_terolah" in hasil_columns:
+            insert_columns.insert(3, "ulasan_terolah")
+
+        placeholders = ", ".join(["%s"] * (len(insert_columns) - 2) + ["NOW()", "NOW()"])
+        insert_hasil = f"""
+            INSERT INTO hasil_analisis ({", ".join(insert_columns)})
+            VALUES ({placeholders})
+        """
+
+        for _, row in df.fillna("").iterrows():
+            values = [
+                int(row["id"]),
+                str(row["wisata"]),
+                str(row["ulasan"]),
+                str(row["ulasan_bersih"]),
+                str(row["ulasan_bersih"]),
+                str(row["sentimen"]).lower(),
+                float(row["probabilitas"]),
+                periode_id,
+            ]
+            if "ulasan_terolah" in hasil_columns:
+                values.insert(3, str(row["ulasan_bersih"]))
+
+            execute(cursor, raw_conn, insert_hasil, tuple(values))
+
+        weighted = report.get("weighted avg", {})
+        execute(
+            cursor,
+            raw_conn,
+            """
+            INSERT INTO evaluasi_model
+            (`precision`, `recall`, f1_score, accuracy, tp, tn, fp, fn, periode_id, created_at, updated_at)
+            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, NOW(), NOW())
+            """,
+            (
+                float(weighted.get("precision", 0)),
+                float(weighted.get("recall", 0)),
+                float(weighted.get("f1-score", 0)),
+                float(accuracy),
+                0,
+                0,
+                0,
+                0,
+                periode_id,
+            ),
+        )
+
+        raw_conn.commit()
+        log("OK", f"{len(df)} hasil analisis disimpan untuk periode {periode_nama}.")
+        log("OK", "Analisis selesai.")
+
+    except SystemExit:
+        raw_conn.rollback()
+        raise
+    except Exception as exc:
+        raw_conn.rollback()
+        fail(f"Analisis gagal: {exc}")
+    finally:
+        raw_conn.close()
+        engine.dispose()
+
+
+if __name__ == "__main__":
+    main()