# ============================================================
# EXPORT DATA PENELITIAN TF-IDF UNTUK WEBSITE S-LEARN
# VERSI FINAL — TF-IDF TANPA NORMALISASI L2 (norm=None)
# ============================================================

from pathlib import Path
from collections import Counter
from datetime import datetime
import json
import shutil

import joblib
import numpy as np
import pandas as pd

from sklearn.model_selection import train_test_split


# ============================================================
# 1. PATH PROJECT
# ============================================================

BASE_DIR = Path(__file__).resolve().parent
PROJECT_DIR = BASE_DIR.parent

ARTIFACT_DIR = BASE_DIR / "artifacts_slearn"

VECTORIZER_PATH = (
    ARTIFACT_DIR / "tfidf_vectorizer.joblib"
)

PUBLIC_DATA_DIR = (
    PROJECT_DIR
    / "web_slearn"
    / "public"
    / "data"
)

CSV_PATH = (
    PUBLIC_DATA_DIR
    / "hasil_preprocessing_slearn.csv"
)

OUTPUT_PATH = (
    PUBLIC_DATA_DIR
    / "tfidf_research.json"
)


# ============================================================
# 2. CEK FILE YANG DIBUTUHKAN
# ============================================================

print("=" * 80)
print("EXPORT DATA TF-IDF PENELITIAN S-LEARN")
print("=" * 80)

if not VECTORIZER_PATH.exists():
    raise FileNotFoundError(
        f"Vectorizer tidak ditemukan:\n{VECTORIZER_PATH}"
    )

if not CSV_PATH.exists():
    raise FileNotFoundError(
        f"CSV preprocessing tidak ditemukan:\n{CSV_PATH}"
    )

PUBLIC_DATA_DIR.mkdir(
    parents=True,
    exist_ok=True
)


# ============================================================
# 3. MEMBACA DATA PREPROCESSING
# ============================================================

print("\nMembaca data preprocessing...")

df = pd.read_csv(
    CSV_PATH,
    sep=None,
    engine="python",
    encoding="utf-8-sig"
)

kolom_wajib = {
    "komentar",
    "label",
    "stemmed_text"
}

kolom_hilang = kolom_wajib - set(df.columns)

if kolom_hilang:
    raise ValueError(
        "Kolom berikut tidak ditemukan pada CSV: "
        + ", ".join(sorted(kolom_hilang))
    )

df["komentar"] = (
    df["komentar"]
    .fillna("")
    .astype(str)
)

df["stemmed_text"] = (
    df["stemmed_text"]
    .fillna("")
    .astype(str)
)

df["label"] = pd.to_numeric(
    df["label"],
    errors="raise"
).astype(int)

if not df["label"].isin([0, 1]).all():
    raise ValueError(
        "Label harus hanya berisi 0 atau 1."
    )

if len(df) != 571:
    raise ValueError(
        f"Jumlah data seharusnya 571, "
        f"tetapi ditemukan {len(df)}."
    )

print(f"✓ Jumlah data: {len(df)}")


# ============================================================
# 4. MEMUAT TF-IDF VECTORIZER FINAL
# ============================================================

print("\nMemuat TF-IDF vectorizer final...")

tfidf = joblib.load(
    VECTORIZER_PATH
)

norm_tfidf = tfidf.get_params().get("norm")

print(
    f"Norm TF-IDF: {norm_tfidf}"
)

if norm_tfidf is not None:
    raise ValueError(
        "Vectorizer yang digunakan masih memakai "
        f"normalisasi TF-IDF: {norm_tfidf}. "
        "Vectorizer final seharusnya norm=None."
    )

kata_unik = tfidf.get_feature_names_out()

print(
    f"✓ Jumlah fitur TF-IDF: "
    f"{len(kata_unik)}"
)

if len(kata_unik) != 675:
    raise ValueError(
        f"Jumlah fitur seharusnya 675, "
        f"tetapi ditemukan {len(kata_unik)}."
    )


# ============================================================
# 5. MEMBENTUK ULANG SPLIT 80:20
#    SESUAI PENELITIAN
# ============================================================

print("\nMembentuk ulang split penelitian...")

semua_index = df.index.to_numpy()

train_index, test_index = train_test_split(
    semua_index,
    test_size=0.2,
    random_state=42,
    stratify=df["label"].to_numpy()
)

print(
    f"✓ Data training: {len(train_index)}"
)

print(
    f"✓ Data testing : {len(test_index)}"
)

if len(train_index) != 456:
    raise ValueError(
        "Jumlah training tidak sama dengan 456."
    )

if len(test_index) != 115:
    raise ValueError(
        "Jumlah testing tidak sama dengan 115."
    )


# ============================================================
# 6. TRANSFORM TF-IDF
#    TIDAK MELAKUKAN FIT ULANG
# ============================================================

print(
    "\nMengubah teks menjadi bobot TF-IDF "
    "menggunakan vectorizer final..."
)

X_train_text = (
    df.loc[train_index, "stemmed_text"]
    .astype(str)
)

X_test_text = (
    df.loc[test_index, "stemmed_text"]
    .astype(str)
)

# HANYA TRANSFORM.
# Vectorizer sudah difit pada penelitian final.
X_train = tfidf.transform(
    X_train_text
)

X_test = tfidf.transform(
    X_test_text
)

print(
    f"✓ Matriks training: {X_train.shape}"
)

print(
    f"✓ Matriks testing : {X_test.shape}"
)

if X_train.shape != (456, 675):
    raise ValueError(
        "Ukuran matriks training tidak sesuai."
    )

if X_test.shape != (115, 675):
    raise ValueError(
        "Ukuran matriks testing tidak sesuai."
    )


# ============================================================
# 7. MENGHITUNG DF DARI DATA TRAINING
# ============================================================

print(
    "\nMenghitung Document Frequency (DF) "
    "dari data training..."
)

df_per_fitur = np.asarray(
    (X_train > 0).sum(axis=0)
).ravel().astype(int)

idf_per_fitur = np.asarray(
    tfidf.idf_
)

if len(df_per_fitur) != len(kata_unik):
    raise ValueError(
        "Jumlah DF tidak sama dengan jumlah fitur."
    )

if len(idf_per_fitur) != len(kata_unik):
    raise ValueError(
        "Jumlah IDF tidak sama dengan jumlah fitur."
    )

print("✓ DF dan IDF berhasil disiapkan.")


# ============================================================
# 8. MEMBUAT POSISI TRAINING DAN TESTING
# ============================================================

posisi_training = {
    int(index): urutan
    for urutan, index
    in enumerate(
        train_index,
        start=1
    )
}

posisi_testing = {
    int(index): urutan
    for urutan, index
    in enumerate(
        test_index,
        start=1
    )
}


# ============================================================
# 9. MEMBENTUK DETAIL TF-IDF PER DATA
# ============================================================

def buat_data_tfidf(
    index_array,
    matrix,
    jenis_data
):
    hasil = []

    for posisi_matrix, index_asli in enumerate(
        index_array
    ):
        index_asli = int(index_asli)

        baris_matrix = matrix.getrow(
            posisi_matrix
        )

        stemmed_text = str(
            df.at[
                index_asli,
                "stemmed_text"
            ]
        ).strip()

        hitung_kata = Counter(
            stemmed_text.split()
        )

        bobot_tfidf = []

        for kolom, bobot in zip(
            baris_matrix.indices,
            baris_matrix.data
        ):
            kata = str(
                kata_unik[kolom]
            )

            tf = int(
                hitung_kata.get(
                    kata,
                    0
                )
            )

            df_kata = int(
                df_per_fitur[kolom]
            )

            idf = float(
                idf_per_fitur[kolom]
            )

            bobot = float(
                bobot
            )

            bobot_tfidf.append({
                "kata": kata,
                "tf": tf,
                "df": df_kata,
                "idf": idf,
                "bobot": bobot
            })

        # Bobot terbesar ditampilkan lebih dulu.
        bobot_tfidf.sort(
            key=lambda item: (
                -item["bobot"],
                item["kata"]
            )
        )

        if jenis_data == "Training":
            urutan_split = posisi_training[
                index_asli
            ]
        else:
            urutan_split = posisi_testing[
                index_asli
            ]

        hasil.append({
            "no_data": index_asli + 1,
            "ulasan_asli": str(
                df.at[
                    index_asli,
                    "komentar"
                ]
            ),
            "stemmed_text": stemmed_text,
            "label": int(
                df.at[
                    index_asli,
                    "label"
                ]
            ),
            "jenis_data": jenis_data,
            "urutan_split": int(
                urutan_split
            ),
            "jumlah_fitur_nonzero": int(
                baris_matrix.nnz
            ),
            "bobot_tfidf": bobot_tfidf
        })

    return hasil


print(
    "\nMenyusun detail TF-IDF "
    "seluruh 571 data..."
)

data_training = buat_data_tfidf(
    train_index,
    X_train,
    "Training"
)

data_testing = buat_data_tfidf(
    test_index,
    X_test,
    "Testing"
)

data_semua = (
    data_training
    + data_testing
)

# Agar website menampilkan
# Data ke-1 sampai Data ke-571.
data_semua.sort(
    key=lambda item: item["no_data"]
)

if len(data_semua) != 571:
    raise ValueError(
        "Jumlah hasil TF-IDF bukan 571 data."
    )

print(
    "✓ Detail TF-IDF 571 data berhasil dibuat."
)


# ============================================================
# 10. MEMBENTUK JSON FINAL
# ============================================================

hasil_json = {
    "summary": {
        "jumlah_data": 571,
        "jumlah_fitur": int(
            len(kata_unik)
        ),
        "data_training": int(
            len(train_index)
        ),
        "data_testing": int(
            len(test_index)
        ),
        "ukuran_matriks_training": [
            int(X_train.shape[0]),
            int(X_train.shape[1])
        ],
        "ukuran_matriks_testing": [
            int(X_test.shape[0]),
            int(X_test.shape[1])
        ],
        "test_size": 0.2,
        "random_state": 42,
        "stratify": "label",
        "norm": None,
        "smooth_idf": True,
        "use_idf": True,
        "sublinear_tf": False,
        "keterangan": (
            "TF-IDF dihitung tanpa normalisasi L2 "
            "(norm=None). Bobot akhir merupakan "
            "hasil TF × IDF."
        )
    },
    "data": data_semua
}


# ============================================================
# 11. BACKUP JSON LAMA
# ============================================================

if OUTPUT_PATH.exists():
    timestamp = datetime.now().strftime(
        "%Y%m%d_%H%M%S"
    )

    backup_path = (
        PUBLIC_DATA_DIR
        / f"tfidf_research_backup_{timestamp}.json"
    )

    shutil.copy2(
        OUTPUT_PATH,
        backup_path
    )

    print(
        "\n✓ JSON lama dibackup:"
    )

    print(
        f"  {backup_path.name}"
    )


# ============================================================
# 12. MENYIMPAN JSON BARU
# ============================================================

with open(
    OUTPUT_PATH,
    "w",
    encoding="utf-8"
) as file:
    json.dump(
        hasil_json,
        file,
        ensure_ascii=False,
        indent=2
    )

print(
    "\n✓ JSON TF-IDF baru berhasil disimpan:"
)

print(
    f"  {OUTPUT_PATH}"
)


# ============================================================
# 13. VERIFIKASI FILE HASIL
# ============================================================

with open(
    OUTPUT_PATH,
    "r",
    encoding="utf-8"
) as file:
    verifikasi = json.load(
        file
    )

if len(
    verifikasi.get(
        "data",
        []
    )
) != 571:
    raise ValueError(
        "Verifikasi gagal: jumlah data JSON bukan 571."
    )

summary_verifikasi = verifikasi.get(
    "summary",
    {}
)

if summary_verifikasi.get(
    "jumlah_fitur"
) != 675:
    raise ValueError(
        "Verifikasi gagal: jumlah fitur bukan 675."
    )

if summary_verifikasi.get(
    "norm",
    "SALAH"
) is not None:
    raise ValueError(
        "Verifikasi gagal: norm bukan None."
    )


# ============================================================
# 14. MENCARI BOBOT TF-IDF TERBESAR
# ============================================================

bobot_terbesar = 0.0
kata_bobot_terbesar = "-"
data_bobot_terbesar = None

for item in data_semua:
    for fitur in item["bobot_tfidf"]:
        if fitur["bobot"] > bobot_terbesar:
            bobot_terbesar = fitur["bobot"]
            kata_bobot_terbesar = fitur["kata"]
            data_bobot_terbesar = item["no_data"]


# ============================================================
# 15. OUTPUT FINAL
# ============================================================

print("\n" + "=" * 80)
print("EXPORT TF-IDF PENELITIAN BERHASIL")
print("=" * 80)

print(
    f"Jumlah data             : "
    f"{len(data_semua)}"
)

print(
    f"Data training           : "
    f"{len(train_index)}"
)

print(
    f"Data testing            : "
    f"{len(test_index)}"
)

print(
    f"Jumlah fitur            : "
    f"{len(kata_unik)}"
)

print(
    f"Ukuran matriks training : "
    f"{X_train.shape}"
)

print(
    f"Ukuran matriks testing  : "
    f"{X_test.shape}"
)

print(
    f"Normalisasi TF-IDF      : "
    f"{norm_tfidf}"
)

print(
    "Bobot akhir            : "
    "TF × IDF"
)

print(
    f"Bobot terbesar          : "
    f"{bobot_terbesar:.6f}"
)

print(
    f"Kata bobot terbesar     : "
    f"{kata_bobot_terbesar}"
)

print(
    f"Berada pada Data ke     : "
    f"{data_bobot_terbesar}"
)

print(
    f"\nFile output:\n{OUTPUT_PATH}"
)

print(
    "\n✓ TF-IDF website sekarang menggunakan "
    "hasil penelitian TANPA normalisasi L2."
)