681 lines
13 KiB
Python
681 lines
13 KiB
Python
# ============================================================
|
||
# EXPORT DATA PENELITIAN TF-IDF UNTUK WEBSITE S-LEARN
|
||
# VERSI FINAL — TF-IDF TANPA NORMALISASI L2 (norm=None)
|
||
# ============================================================
|
||
|
||
from pathlib import Path
|
||
from collections import Counter
|
||
from datetime import datetime
|
||
import json
|
||
import shutil
|
||
|
||
import joblib
|
||
import numpy as np
|
||
import pandas as pd
|
||
|
||
from sklearn.model_selection import train_test_split
|
||
|
||
|
||
# ============================================================
|
||
# 1. PATH PROJECT
|
||
# ============================================================
|
||
|
||
BASE_DIR = Path(__file__).resolve().parent
|
||
PROJECT_DIR = BASE_DIR.parent
|
||
|
||
ARTIFACT_DIR = BASE_DIR / "artifacts_slearn"
|
||
|
||
VECTORIZER_PATH = (
|
||
ARTIFACT_DIR / "tfidf_vectorizer.joblib"
|
||
)
|
||
|
||
PUBLIC_DATA_DIR = (
|
||
PROJECT_DIR
|
||
/ "web_slearn"
|
||
/ "public"
|
||
/ "data"
|
||
)
|
||
|
||
CSV_PATH = (
|
||
PUBLIC_DATA_DIR
|
||
/ "hasil_preprocessing_slearn.csv"
|
||
)
|
||
|
||
OUTPUT_PATH = (
|
||
PUBLIC_DATA_DIR
|
||
/ "tfidf_research.json"
|
||
)
|
||
|
||
|
||
# ============================================================
|
||
# 2. CEK FILE YANG DIBUTUHKAN
|
||
# ============================================================
|
||
|
||
print("=" * 80)
|
||
print("EXPORT DATA TF-IDF PENELITIAN S-LEARN")
|
||
print("=" * 80)
|
||
|
||
if not VECTORIZER_PATH.exists():
|
||
raise FileNotFoundError(
|
||
f"Vectorizer tidak ditemukan:\n{VECTORIZER_PATH}"
|
||
)
|
||
|
||
if not CSV_PATH.exists():
|
||
raise FileNotFoundError(
|
||
f"CSV preprocessing tidak ditemukan:\n{CSV_PATH}"
|
||
)
|
||
|
||
PUBLIC_DATA_DIR.mkdir(
|
||
parents=True,
|
||
exist_ok=True
|
||
)
|
||
|
||
|
||
# ============================================================
|
||
# 3. MEMBACA DATA PREPROCESSING
|
||
# ============================================================
|
||
|
||
print("\nMembaca data preprocessing...")
|
||
|
||
df = pd.read_csv(
|
||
CSV_PATH,
|
||
sep=None,
|
||
engine="python",
|
||
encoding="utf-8-sig"
|
||
)
|
||
|
||
kolom_wajib = {
|
||
"komentar",
|
||
"label",
|
||
"stemmed_text"
|
||
}
|
||
|
||
kolom_hilang = kolom_wajib - set(df.columns)
|
||
|
||
if kolom_hilang:
|
||
raise ValueError(
|
||
"Kolom berikut tidak ditemukan pada CSV: "
|
||
+ ", ".join(sorted(kolom_hilang))
|
||
)
|
||
|
||
df["komentar"] = (
|
||
df["komentar"]
|
||
.fillna("")
|
||
.astype(str)
|
||
)
|
||
|
||
df["stemmed_text"] = (
|
||
df["stemmed_text"]
|
||
.fillna("")
|
||
.astype(str)
|
||
)
|
||
|
||
df["label"] = pd.to_numeric(
|
||
df["label"],
|
||
errors="raise"
|
||
).astype(int)
|
||
|
||
if not df["label"].isin([0, 1]).all():
|
||
raise ValueError(
|
||
"Label harus hanya berisi 0 atau 1."
|
||
)
|
||
|
||
if len(df) != 571:
|
||
raise ValueError(
|
||
f"Jumlah data seharusnya 571, "
|
||
f"tetapi ditemukan {len(df)}."
|
||
)
|
||
|
||
print(f"✓ Jumlah data: {len(df)}")
|
||
|
||
|
||
# ============================================================
|
||
# 4. MEMUAT TF-IDF VECTORIZER FINAL
|
||
# ============================================================
|
||
|
||
print("\nMemuat TF-IDF vectorizer final...")
|
||
|
||
tfidf = joblib.load(
|
||
VECTORIZER_PATH
|
||
)
|
||
|
||
norm_tfidf = tfidf.get_params().get("norm")
|
||
|
||
print(
|
||
f"Norm TF-IDF: {norm_tfidf}"
|
||
)
|
||
|
||
if norm_tfidf is not None:
|
||
raise ValueError(
|
||
"Vectorizer yang digunakan masih memakai "
|
||
f"normalisasi TF-IDF: {norm_tfidf}. "
|
||
"Vectorizer final seharusnya norm=None."
|
||
)
|
||
|
||
kata_unik = tfidf.get_feature_names_out()
|
||
|
||
print(
|
||
f"✓ Jumlah fitur TF-IDF: "
|
||
f"{len(kata_unik)}"
|
||
)
|
||
|
||
if len(kata_unik) != 675:
|
||
raise ValueError(
|
||
f"Jumlah fitur seharusnya 675, "
|
||
f"tetapi ditemukan {len(kata_unik)}."
|
||
)
|
||
|
||
|
||
# ============================================================
|
||
# 5. MEMBENTUK ULANG SPLIT 80:20
|
||
# SESUAI PENELITIAN
|
||
# ============================================================
|
||
|
||
print("\nMembentuk ulang split penelitian...")
|
||
|
||
semua_index = df.index.to_numpy()
|
||
|
||
train_index, test_index = train_test_split(
|
||
semua_index,
|
||
test_size=0.2,
|
||
random_state=42,
|
||
stratify=df["label"].to_numpy()
|
||
)
|
||
|
||
print(
|
||
f"✓ Data training: {len(train_index)}"
|
||
)
|
||
|
||
print(
|
||
f"✓ Data testing : {len(test_index)}"
|
||
)
|
||
|
||
if len(train_index) != 456:
|
||
raise ValueError(
|
||
"Jumlah training tidak sama dengan 456."
|
||
)
|
||
|
||
if len(test_index) != 115:
|
||
raise ValueError(
|
||
"Jumlah testing tidak sama dengan 115."
|
||
)
|
||
|
||
|
||
# ============================================================
|
||
# 6. TRANSFORM TF-IDF
|
||
# TIDAK MELAKUKAN FIT ULANG
|
||
# ============================================================
|
||
|
||
print(
|
||
"\nMengubah teks menjadi bobot TF-IDF "
|
||
"menggunakan vectorizer final..."
|
||
)
|
||
|
||
X_train_text = (
|
||
df.loc[train_index, "stemmed_text"]
|
||
.astype(str)
|
||
)
|
||
|
||
X_test_text = (
|
||
df.loc[test_index, "stemmed_text"]
|
||
.astype(str)
|
||
)
|
||
|
||
# HANYA TRANSFORM.
|
||
# Vectorizer sudah difit pada penelitian final.
|
||
X_train = tfidf.transform(
|
||
X_train_text
|
||
)
|
||
|
||
X_test = tfidf.transform(
|
||
X_test_text
|
||
)
|
||
|
||
print(
|
||
f"✓ Matriks training: {X_train.shape}"
|
||
)
|
||
|
||
print(
|
||
f"✓ Matriks testing : {X_test.shape}"
|
||
)
|
||
|
||
if X_train.shape != (456, 675):
|
||
raise ValueError(
|
||
"Ukuran matriks training tidak sesuai."
|
||
)
|
||
|
||
if X_test.shape != (115, 675):
|
||
raise ValueError(
|
||
"Ukuran matriks testing tidak sesuai."
|
||
)
|
||
|
||
|
||
# ============================================================
|
||
# 7. MENGHITUNG DF DARI DATA TRAINING
|
||
# ============================================================
|
||
|
||
print(
|
||
"\nMenghitung Document Frequency (DF) "
|
||
"dari data training..."
|
||
)
|
||
|
||
df_per_fitur = np.asarray(
|
||
(X_train > 0).sum(axis=0)
|
||
).ravel().astype(int)
|
||
|
||
idf_per_fitur = np.asarray(
|
||
tfidf.idf_
|
||
)
|
||
|
||
if len(df_per_fitur) != len(kata_unik):
|
||
raise ValueError(
|
||
"Jumlah DF tidak sama dengan jumlah fitur."
|
||
)
|
||
|
||
if len(idf_per_fitur) != len(kata_unik):
|
||
raise ValueError(
|
||
"Jumlah IDF tidak sama dengan jumlah fitur."
|
||
)
|
||
|
||
print("✓ DF dan IDF berhasil disiapkan.")
|
||
|
||
|
||
# ============================================================
|
||
# 8. MEMBUAT POSISI TRAINING DAN TESTING
|
||
# ============================================================
|
||
|
||
posisi_training = {
|
||
int(index): urutan
|
||
for urutan, index
|
||
in enumerate(
|
||
train_index,
|
||
start=1
|
||
)
|
||
}
|
||
|
||
posisi_testing = {
|
||
int(index): urutan
|
||
for urutan, index
|
||
in enumerate(
|
||
test_index,
|
||
start=1
|
||
)
|
||
}
|
||
|
||
|
||
# ============================================================
|
||
# 9. MEMBENTUK DETAIL TF-IDF PER DATA
|
||
# ============================================================
|
||
|
||
def buat_data_tfidf(
|
||
index_array,
|
||
matrix,
|
||
jenis_data
|
||
):
|
||
hasil = []
|
||
|
||
for posisi_matrix, index_asli in enumerate(
|
||
index_array
|
||
):
|
||
index_asli = int(index_asli)
|
||
|
||
baris_matrix = matrix.getrow(
|
||
posisi_matrix
|
||
)
|
||
|
||
stemmed_text = str(
|
||
df.at[
|
||
index_asli,
|
||
"stemmed_text"
|
||
]
|
||
).strip()
|
||
|
||
hitung_kata = Counter(
|
||
stemmed_text.split()
|
||
)
|
||
|
||
bobot_tfidf = []
|
||
|
||
for kolom, bobot in zip(
|
||
baris_matrix.indices,
|
||
baris_matrix.data
|
||
):
|
||
kata = str(
|
||
kata_unik[kolom]
|
||
)
|
||
|
||
tf = int(
|
||
hitung_kata.get(
|
||
kata,
|
||
0
|
||
)
|
||
)
|
||
|
||
df_kata = int(
|
||
df_per_fitur[kolom]
|
||
)
|
||
|
||
idf = float(
|
||
idf_per_fitur[kolom]
|
||
)
|
||
|
||
bobot = float(
|
||
bobot
|
||
)
|
||
|
||
bobot_tfidf.append({
|
||
"kata": kata,
|
||
"tf": tf,
|
||
"df": df_kata,
|
||
"idf": idf,
|
||
"bobot": bobot
|
||
})
|
||
|
||
# Bobot terbesar ditampilkan lebih dulu.
|
||
bobot_tfidf.sort(
|
||
key=lambda item: (
|
||
-item["bobot"],
|
||
item["kata"]
|
||
)
|
||
)
|
||
|
||
if jenis_data == "Training":
|
||
urutan_split = posisi_training[
|
||
index_asli
|
||
]
|
||
else:
|
||
urutan_split = posisi_testing[
|
||
index_asli
|
||
]
|
||
|
||
hasil.append({
|
||
"no_data": index_asli + 1,
|
||
"ulasan_asli": str(
|
||
df.at[
|
||
index_asli,
|
||
"komentar"
|
||
]
|
||
),
|
||
"stemmed_text": stemmed_text,
|
||
"label": int(
|
||
df.at[
|
||
index_asli,
|
||
"label"
|
||
]
|
||
),
|
||
"jenis_data": jenis_data,
|
||
"urutan_split": int(
|
||
urutan_split
|
||
),
|
||
"jumlah_fitur_nonzero": int(
|
||
baris_matrix.nnz
|
||
),
|
||
"bobot_tfidf": bobot_tfidf
|
||
})
|
||
|
||
return hasil
|
||
|
||
|
||
print(
|
||
"\nMenyusun detail TF-IDF "
|
||
"seluruh 571 data..."
|
||
)
|
||
|
||
data_training = buat_data_tfidf(
|
||
train_index,
|
||
X_train,
|
||
"Training"
|
||
)
|
||
|
||
data_testing = buat_data_tfidf(
|
||
test_index,
|
||
X_test,
|
||
"Testing"
|
||
)
|
||
|
||
data_semua = (
|
||
data_training
|
||
+ data_testing
|
||
)
|
||
|
||
# Agar website menampilkan
|
||
# Data ke-1 sampai Data ke-571.
|
||
data_semua.sort(
|
||
key=lambda item: item["no_data"]
|
||
)
|
||
|
||
if len(data_semua) != 571:
|
||
raise ValueError(
|
||
"Jumlah hasil TF-IDF bukan 571 data."
|
||
)
|
||
|
||
print(
|
||
"✓ Detail TF-IDF 571 data berhasil dibuat."
|
||
)
|
||
|
||
|
||
# ============================================================
|
||
# 10. MEMBENTUK JSON FINAL
|
||
# ============================================================
|
||
|
||
hasil_json = {
|
||
"summary": {
|
||
"jumlah_data": 571,
|
||
"jumlah_fitur": int(
|
||
len(kata_unik)
|
||
),
|
||
"data_training": int(
|
||
len(train_index)
|
||
),
|
||
"data_testing": int(
|
||
len(test_index)
|
||
),
|
||
"ukuran_matriks_training": [
|
||
int(X_train.shape[0]),
|
||
int(X_train.shape[1])
|
||
],
|
||
"ukuran_matriks_testing": [
|
||
int(X_test.shape[0]),
|
||
int(X_test.shape[1])
|
||
],
|
||
"test_size": 0.2,
|
||
"random_state": 42,
|
||
"stratify": "label",
|
||
"norm": None,
|
||
"smooth_idf": True,
|
||
"use_idf": True,
|
||
"sublinear_tf": False,
|
||
"keterangan": (
|
||
"TF-IDF dihitung tanpa normalisasi L2 "
|
||
"(norm=None). Bobot akhir merupakan "
|
||
"hasil TF × IDF."
|
||
)
|
||
},
|
||
"data": data_semua
|
||
}
|
||
|
||
|
||
# ============================================================
|
||
# 11. BACKUP JSON LAMA
|
||
# ============================================================
|
||
|
||
if OUTPUT_PATH.exists():
|
||
timestamp = datetime.now().strftime(
|
||
"%Y%m%d_%H%M%S"
|
||
)
|
||
|
||
backup_path = (
|
||
PUBLIC_DATA_DIR
|
||
/ f"tfidf_research_backup_{timestamp}.json"
|
||
)
|
||
|
||
shutil.copy2(
|
||
OUTPUT_PATH,
|
||
backup_path
|
||
)
|
||
|
||
print(
|
||
"\n✓ JSON lama dibackup:"
|
||
)
|
||
|
||
print(
|
||
f" {backup_path.name}"
|
||
)
|
||
|
||
|
||
# ============================================================
|
||
# 12. MENYIMPAN JSON BARU
|
||
# ============================================================
|
||
|
||
with open(
|
||
OUTPUT_PATH,
|
||
"w",
|
||
encoding="utf-8"
|
||
) as file:
|
||
json.dump(
|
||
hasil_json,
|
||
file,
|
||
ensure_ascii=False,
|
||
indent=2
|
||
)
|
||
|
||
print(
|
||
"\n✓ JSON TF-IDF baru berhasil disimpan:"
|
||
)
|
||
|
||
print(
|
||
f" {OUTPUT_PATH}"
|
||
)
|
||
|
||
|
||
# ============================================================
|
||
# 13. VERIFIKASI FILE HASIL
|
||
# ============================================================
|
||
|
||
with open(
|
||
OUTPUT_PATH,
|
||
"r",
|
||
encoding="utf-8"
|
||
) as file:
|
||
verifikasi = json.load(
|
||
file
|
||
)
|
||
|
||
if len(
|
||
verifikasi.get(
|
||
"data",
|
||
[]
|
||
)
|
||
) != 571:
|
||
raise ValueError(
|
||
"Verifikasi gagal: jumlah data JSON bukan 571."
|
||
)
|
||
|
||
summary_verifikasi = verifikasi.get(
|
||
"summary",
|
||
{}
|
||
)
|
||
|
||
if summary_verifikasi.get(
|
||
"jumlah_fitur"
|
||
) != 675:
|
||
raise ValueError(
|
||
"Verifikasi gagal: jumlah fitur bukan 675."
|
||
)
|
||
|
||
if summary_verifikasi.get(
|
||
"norm",
|
||
"SALAH"
|
||
) is not None:
|
||
raise ValueError(
|
||
"Verifikasi gagal: norm bukan None."
|
||
)
|
||
|
||
|
||
# ============================================================
|
||
# 14. MENCARI BOBOT TF-IDF TERBESAR
|
||
# ============================================================
|
||
|
||
bobot_terbesar = 0.0
|
||
kata_bobot_terbesar = "-"
|
||
data_bobot_terbesar = None
|
||
|
||
for item in data_semua:
|
||
for fitur in item["bobot_tfidf"]:
|
||
if fitur["bobot"] > bobot_terbesar:
|
||
bobot_terbesar = fitur["bobot"]
|
||
kata_bobot_terbesar = fitur["kata"]
|
||
data_bobot_terbesar = item["no_data"]
|
||
|
||
|
||
# ============================================================
|
||
# 15. OUTPUT FINAL
|
||
# ============================================================
|
||
|
||
print("\n" + "=" * 80)
|
||
print("EXPORT TF-IDF PENELITIAN BERHASIL")
|
||
print("=" * 80)
|
||
|
||
print(
|
||
f"Jumlah data : "
|
||
f"{len(data_semua)}"
|
||
)
|
||
|
||
print(
|
||
f"Data training : "
|
||
f"{len(train_index)}"
|
||
)
|
||
|
||
print(
|
||
f"Data testing : "
|
||
f"{len(test_index)}"
|
||
)
|
||
|
||
print(
|
||
f"Jumlah fitur : "
|
||
f"{len(kata_unik)}"
|
||
)
|
||
|
||
print(
|
||
f"Ukuran matriks training : "
|
||
f"{X_train.shape}"
|
||
)
|
||
|
||
print(
|
||
f"Ukuran matriks testing : "
|
||
f"{X_test.shape}"
|
||
)
|
||
|
||
print(
|
||
f"Normalisasi TF-IDF : "
|
||
f"{norm_tfidf}"
|
||
)
|
||
|
||
print(
|
||
"Bobot akhir : "
|
||
"TF × IDF"
|
||
)
|
||
|
||
print(
|
||
f"Bobot terbesar : "
|
||
f"{bobot_terbesar:.6f}"
|
||
)
|
||
|
||
print(
|
||
f"Kata bobot terbesar : "
|
||
f"{kata_bobot_terbesar}"
|
||
)
|
||
|
||
print(
|
||
f"Berada pada Data ke : "
|
||
f"{data_bobot_terbesar}"
|
||
)
|
||
|
||
print(
|
||
f"\nFile output:\n{OUTPUT_PATH}"
|
||
)
|
||
|
||
print(
|
||
"\n✓ TF-IDF website sekarang menggunakan "
|
||
"hasil penelitian TANPA normalisasi L2."
|
||
) |