keluarkan file copy dari repository

This commit is contained in:
E41222120_syamaidzar adani syah 2026-07-28 09:03:02 +07:00
parent 07282b085c
commit b589c56e0c
4 changed files with 3 additions and 442 deletions

View File

@ -13,6 +13,9 @@ __pycache__/
*.pyc
*.pyo
# File duplikat/cadangan, cukup disimpan lokal
*_copy.py
# Sistem
.env
.DS_Store

View File

@ -1,210 +0,0 @@
# 3 evaluasi hasil model
import joblib
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.metrics import accuracy_score, classification_report, confusion_matrix
import os
print("📊 [TAHAP 3] Evaluasi Hasil Model...")
# --- PENYESUAIAN FOLDER ---
FOLDER_OUTPUT = 'virtualEnvironment/output'
FOLDER_MODELS = 'virtualEnvironment/models'
FOLDER_IMAGES = 'virtualEnvironment/images'
# 0. CEK KEAMANAN
file_xtest = f'{FOLDER_OUTPUT}/X_test.pkl'
file_svm = f'{FOLDER_MODELS}/model_svm.pkl'
if not os.path.exists(file_xtest):
raise FileNotFoundError(f"❌ File '{file_xtest}' tidak ditemukan! Pastikan Tahap 1 sudah dijalankan.")
if not os.path.exists(file_svm):
raise FileNotFoundError(f"❌ File '{file_svm}' tidak ditemukan! Pastikan Tahap 2 sudah selesai 100%.")
# Bikin folder images otomatis di dalam virtualEnvironment
os.makedirs(FOLDER_IMAGES, exist_ok=True)
# 1. Ambil Data Uji
print(" - Memuat data uji...")
X_test = joblib.load(file_xtest)
y_test = joblib.load(f'{FOLDER_OUTPUT}/y_test.pkl')
# 2. Daftar Model
print(" - Memuat model-model AI...")
daftar_model = {
"SVM": joblib.load(file_svm),
"KNN": joblib.load(f'{FOLDER_MODELS}/model_knn.pkl'),
"Ensemble": joblib.load(f'{FOLDER_MODELS}/model_ensemble.pkl')
}
# 3. Loop Evaluasi
for nama, model in daftar_model.items():
print(f"\n==========================================")
print(f"--- Evaluasi Model: {nama} ---")
print(f"==========================================")
# Lakukan Prediksi
y_pred = model.predict(X_test)
# Hitung Akurasi
acc = accuracy_score(y_test, y_pred)
print(f"🎯 Akurasi {nama}: {acc*100:.2f}%\n")
# Laporan Lengkap (Precision, Recall, F1-Score)
print("📋 Laporan Klasifikasi:")
print(classification_report(y_test, y_pred))
# Bikin Grafik Confusion Matrix
cm = confusion_matrix(y_test, y_pred)
plt.figure(figsize=(7, 5))
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues',
annot_kws={"size": 14}) # Angka di dalam kotak diperbesar
# Hiasan Grafik (Lebih rapi untuk masuk ke buku Skripsi)
plt.title(f'Confusion Matrix - {nama}\n(Akurasi: {acc*100:.2f}%)', fontsize=14, fontweight='bold', pad=15)
plt.xlabel('Prediksi Mesin', fontsize=12, fontweight='bold')
plt.ylabel('Label Asli', fontsize=12, fontweight='bold')
# Tambahan Keterangan Sumbu X dan Y biar dosen mudah baca
plt.xticks(ticks=[0.5, 1.5, 2.5], labels=['0 (Netral)', '1 (rasional negatif)', '2 (cacimaki/intoleransi)'])
plt.yticks(ticks=[0.5, 1.5, 2.5], labels=['0 (Netral)', '1 (rasional negatif)', '2 (cacimaki/intoleransi)'], rotation=0)
# Simpan Gambar dengan resolusi tinggi (dpi=300)
nama_file = f'{FOLDER_IMAGES}/cm_{nama}.png'
plt.savefig(nama_file, bbox_inches='tight', dpi=300)
print(f"🖼️ Grafik Confusion Matrix tersimpan: {nama_file}")
# Tutup plot supaya tidak numpuk di memori
plt.close()
print("\n🎉 SEMUA TAHAPAN SELESAI!")
print(f"Cek folder '{FOLDER_IMAGES}' untuk melihat gambar Confusion Matrix-nya ya.")
# 3 evaluasi hasil model
import joblib
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.metrics import accuracy_score, classification_report, confusion_matrix
import os
print("📊 [TAHAP 3] Evaluasi Hasil Model...")
# --- PENYESUAIAN FOLDER ---
FOLDER_OUTPUT = 'virtualEnvironment/output'
FOLDER_MODELS = 'virtualEnvironment/models'
FOLDER_IMAGES = 'virtualEnvironment/images'
# 0. CEK KEAMANAN
file_xtest = f'{FOLDER_OUTPUT}/X_test.pkl'
file_svm = f'{FOLDER_MODELS}/model_svm.pkl'
if not os.path.exists(file_xtest):
raise FileNotFoundError(f"❌ File '{file_xtest}' tidak ditemukan! Pastikan Tahap 1 sudah dijalankan.")
if not os.path.exists(file_svm):
raise FileNotFoundError(f"❌ File '{file_svm}' tidak ditemukan! Pastikan Tahap 2 sudah selesai 100%.")
# Bikin folder images otomatis di dalam virtualEnvironment
os.makedirs(FOLDER_IMAGES, exist_ok=True)
# 1. Ambil Data Uji
print(" - Memuat data uji...")
X_test = joblib.load(file_xtest)
y_test = joblib.load(f'{FOLDER_OUTPUT}/y_test.pkl')
# 2. Daftar Model
print(" - Memuat model-model AI...")
daftar_model = {
"SVM": joblib.load(file_svm),
"KNN": joblib.load(f'{FOLDER_MODELS}/model_knn.pkl'),
"Ensemble": joblib.load(f'{FOLDER_MODELS}/model_ensemble.pkl')
}
# 3. Loop Evaluasi
for nama, model in daftar_model.items():
print(f"\n==========================================")
print(f"--- Evaluasi Model: {nama} ---")
print(f"==========================================")
# Lakukan Prediksi
y_pred = model.predict(X_test)
# Hitung Akurasi
acc = accuracy_score(y_test, y_pred)
print(f"🎯 Akurasi {nama}: {acc*100:.2f}%\n")
# Laporan Lengkap (Precision, Recall, F1-Score)
print("📋 Laporan Klasifikasi:")
print(classification_report(y_test, y_pred))
# Bikin Grafik Confusion Matrix
cm = confusion_matrix(y_test, y_pred)
plt.figure(figsize=(7, 5))
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues',
annot_kws={"size": 14}) # Angka di dalam kotak diperbesar
# Hiasan Grafik (Lebih rapi untuk masuk ke buku Skripsi)
plt.title(f'Confusion Matrix - {nama}\n(Akurasi: {acc*100:.2f}%)', fontsize=14, fontweight='bold', pad=15)
plt.xlabel('Prediksi Mesin', fontsize=12, fontweight='bold')
plt.ylabel('Label Asli', fontsize=12, fontweight='bold')
# Tambahan Keterangan Sumbu X dan Y biar dosen mudah baca
plt.xticks(ticks=[0.5, 1.5, 2.5], labels=['0 (Netral)', '1 (rasional negatif)', '2 (cacimaki/intoleransi)'])
plt.yticks(ticks=[0.5, 1.5, 2.5], labels=['0 (Netral)', '1 (rasional negatif)', '2 (cacimaki/intoleransi)'], rotation=0)
# Simpan Gambar dengan resolusi tinggi (dpi=300)
nama_file = f'{FOLDER_IMAGES}/cm_{nama}.png'
plt.savefig(nama_file, bbox_inches='tight', dpi=300)
print(f"🖼️ Grafik Confusion Matrix tersimpan: {nama_file}")
# Tutup plot supaya tidak numpuk di memori
plt.close()
print("\n🎉 SEMUA TAHAPAN SELESAI!")
print(f"Cek folder '{FOLDER_IMAGES}' untuk melihat gambar Confusion Matrix-nya ya.")
# ============================================
# TAMBAHAN: Ekstraksi Pola Kesalahan SVM
# ============================================
print("\n" + "="*60)
print("📋 EKSTRAKSI POLA KESALAHAN KLASIFIKASI (SVM)")
print("="*60)
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
# Load ulang data asli untuk dapat teks tweet
FILE_DATA = r'D:\project skripsi machine learning intoleransi\virtualEnvironment\dataset\dataYangDiPakai\data_label_3_kategori_v2 - Copy.csv'
df_asli = pd.read_csv(FILE_DATA, sep=';')
df_asli = df_asli.dropna(subset=['clean_text', 'label']).reset_index(drop=True)
# Replikasi split yang sama dengan Tahap 1 (random_state=42)
indeks_semua = np.arange(len(df_asli))
_, indeks_test = train_test_split(indeks_semua, test_size=0.2, random_state=42)
# Prediksi pakai SVM
y_pred_svm = daftar_model['SVM'].predict(X_test)
y_test_arr = np.array(y_test)
# Filter yang salah
salah_mask = y_pred_svm != y_test_arr
indeks_salah = indeks_test[salah_mask]
# Bikin dataframe kesalahan
label_map = {0: 'Netral', 1: 'Rasional Negatif', 2: 'Cacimaki/Intoleransi'}
df_salah = df_asli.loc[indeks_salah].copy()
df_salah['label_asli'] = [label_map[x] for x in y_test_arr[salah_mask]]
df_salah['label_prediksi'] = [label_map[x] for x in y_pred_svm[salah_mask]]
df_salah['pola_kesalahan'] = df_salah['label_asli'] + '' + df_salah['label_prediksi']
# Simpan ke CSV
nama_file_salah = f'{FOLDER_OUTPUT}/kesalahan_svm.csv'
df_salah[['clean_text', 'label_asli', 'label_prediksi', 'pola_kesalahan']].to_csv(
nama_file_salah, index=False, sep=';'
)
# Statistik pola kesalahan
print(f"\n📊 Total kesalahan SVM: {len(df_salah)} dari {len(y_test_arr)} data uji")
print(f"\n📊 Distribusi Pola Kesalahan:")
print(df_salah['pola_kesalahan'].value_counts().to_string())
print(f"\n💾 Detail tersimpan di: {nama_file_salah}")

View File

@ -1,92 +0,0 @@
# 1 extraction features TF-IDF (VERSI MODIFIKASI - STRATIFIED + BIGRAM)
import pandas as pd
import joblib
import os
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split
print("🔄 [TAHAP 1] Memulai Preprocessing & TF-IDF...")
# --- CONFIG ---
FILE_DATA = r'D:\project skripsi machine learning intoleransi\virtualEnvironment\dataset\dataYangDiPakai\data_label_3_kategori_v2 - Copy.csv'
FOLDER_OUTPUT = 'virtualEnvironment/output'
FOLDER_MODELS = 'virtualEnvironment/models'
# 0. CEK KEAMANAN FILE
if not os.path.exists(FILE_DATA):
raise FileNotFoundError(f"❌ File tidak ditemukan di jalur:\n{FILE_DATA}")
os.makedirs(FOLDER_OUTPUT, exist_ok=True)
os.makedirs(FOLDER_MODELS, exist_ok=True)
# 1. LOAD DATA
print(" - Membaca dataset...")
df = pd.read_csv(FILE_DATA, sep=';')
if 'clean_text' not in df.columns or 'label' not in df.columns:
print(f"Daftar kolom: {df.columns.tolist()}")
raise KeyError("❌ Kolom 'clean_text' atau 'label' tidak ada!")
df = df.dropna(subset=['clean_text', 'label'])
print(f" - Total data bersih: {len(df)} baris")
# 2. TF-IDF (MODIFIKASI: Tambah bigram, min_df, max_df)
print(" - Melakukan ekstraksi fitur TF-IDF (Unigram + Bigram)...")
vectorizer = TfidfVectorizer(
max_features=10000, # naik dari 5000
ngram_range=(1, 2), # tambah bigram
min_df=2, # buang kata yang muncul cuma 1x
max_df=0.95 # buang kata yang muncul di >95% dokumen
)
X = vectorizer.fit_transform(df['clean_text'].astype(str))
y = df['label'].astype(int)
print(f" - Jumlah fitur TF-IDF aktual: {X.shape[1]}")
# 3. SIMPAN VECTORIZER
print(f" - Menyimpan vectorizer ke '{FOLDER_MODELS}'...")
joblib.dump(vectorizer, f'{FOLDER_MODELS}/vectorizer_tfidf.pkl')
# 4. SPLIT DATA dengan STRATIFIED (80% Latih, 20% Uji)
print(" - Memecah data dengan Stratified Split (menjaga distribusi label)...")
X_train, X_test, y_train, y_test = train_test_split(
X, y,
test_size=0.2,
random_state=42,
stratify=y # ← PERUBAHAN UTAMA: stratify
)
# 4.1 TAMPILKAN HASIL DISTRIBUSI LABEL
print("\n" + "="*55)
print("📊 HASIL STRATIFIED SPLIT")
print("="*55)
print(f"\n📈 Total Data Awal : {len(y)} baris")
print(f"📈 Total Data Latih : {len(y_train)} baris ({len(y_train)/len(y)*100:.1f}%)")
print(f"📈 Total Data Uji : {len(y_test)} baris ({len(y_test)/len(y)*100:.1f}%)")
print(f"\n📊 Distribusi Label Dataset Awal:")
for label, jumlah in y.value_counts().sort_index().items():
persen = jumlah / len(y) * 100
print(f" - Label {label}: {jumlah} data ({persen:.2f}%)")
print(f"\n📊 Distribusi Label Data Latih:")
for label, jumlah in y_train.value_counts().sort_index().items():
persen = jumlah / len(y_train) * 100
print(f" - Label {label}: {jumlah} data ({persen:.2f}%)")
print(f"\n📊 Distribusi Label Data Uji:")
for label, jumlah in y_test.value_counts().sort_index().items():
persen = jumlah / len(y_test) * 100
print(f" - Label {label}: {jumlah} data ({persen:.2f}%)")
print("="*55 + "\n")
# 5. SIMPAN DATA MATANG
print(f" - Menyimpan data matang ke '{FOLDER_OUTPUT}'...")
joblib.dump(X_train, f'{FOLDER_OUTPUT}/X_train.pkl')
joblib.dump(X_test, f'{FOLDER_OUTPUT}/X_test.pkl')
joblib.dump(y_train, f'{FOLDER_OUTPUT}/y_train.pkl')
joblib.dump(y_test, f'{FOLDER_OUTPUT}/y_test.pkl')
print("✅ SELESAI TAHAP 1. Lanjut jalankan 'training2.py'")

View File

@ -1,140 +0,0 @@
# 2 pelatihan model SVM, KNN dan ensemble
import joblib
import pandas as pd
import os
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import VotingClassifier
from sklearn.model_selection import GridSearchCV, cross_val_score
print("🏋️ [TAHAP 2] Training: DATA ASLI (Mode TURBO Aktif 🚀)...")
# --- PENYESUAIAN FOLDER ---
FOLDER_OUTPUT = 'virtualEnvironment/output'
FOLDER_MODELS = 'virtualEnvironment/models'
# 0. CEK KEAMANAN: Pastikan folder dan file Tahap 1 sudah ada
file_xtrain = f'{FOLDER_OUTPUT}/X_train.pkl'
if not os.path.exists(file_xtrain):
raise FileNotFoundError(f"❌ File '{file_xtrain}' tidak ditemukan! Pastikan kamu sudah menjalankan '1_preprocessing.py' terlebih dahulu.")
# Jaga-jaga kalau folder models belum ada, otomatis dibikin
os.makedirs(FOLDER_MODELS, exist_ok=True)
# 1. AMBIL DATA DARI TAHAP 1
print(" - Memuat data latih...")
X_train = joblib.load(f'{FOLDER_OUTPUT}/X_train.pkl')
y_train = joblib.load(f'{FOLDER_OUTPUT}/y_train.pkl')
print(f" - Jumlah Data Latih Asli: {len(y_train)} baris")
print(f" - Komposisi Label: {y_train.value_counts().to_dict()}")
# ---------------------------------------------------------
# 2. LATIH SVM (MODEL UTAMA)
# ---------------------------------------------------------
print("\n🚀 Melatih SVM (Mencari Settingan Terbaik)...")
print(" (Menggunakan seluruh inti CPU laptop...)")
param_svm = {
'C': [0.01, 0.1, 1, 10, 100],
'kernel': ['linear'],
'gamma': ['scale']
}
svm_grid = GridSearchCV(
SVC(probability=True, random_state=42),
param_svm,
cv=5,
scoring='f1_macro',
verbose=1,
n_jobs=-1
)
svm_grid.fit(X_train, y_train)
best_svm = svm_grid.best_estimator_
joblib.dump(best_svm, f'{FOLDER_MODELS}/model_svm.pkl')
# ⭐ TAMBAHAN: Print akurasi validasi & parameter terbaik SVM
svm_acc_val = svm_grid.best_score_ * 100
print(f" ✅ SVM Selesai (F1-Macro Validasi: {svm_acc_val:.2f}%)")
print(f" 📌 Parameter Terbaik SVM: {svm_grid.best_params_}")
# ---------------------------------------------------------
# 3. LATIH KNN (METRIC COSINE)
# ---------------------------------------------------------
print("\n🚀 Melatih KNN (Wajib Cosine)...")
param_knn = {
'n_neighbors': [3, 5, 7, 9, 11, 15, 21, 25],
'metric': ['cosine'],
'weights': ['uniform', 'distance']
}
knn_grid = GridSearchCV(
KNeighborsClassifier(),
param_knn,
cv=5,
scoring='f1_macro',
verbose=1,
n_jobs=-1
)
knn_grid.fit(X_train, y_train)
best_knn = knn_grid.best_estimator_
joblib.dump(best_knn, f'{FOLDER_MODELS}/model_knn.pkl')
# ⭐ TAMBAHAN: Print akurasi validasi & parameter terbaik KNN
knn_acc_val = knn_grid.best_score_ * 100
print(f" ✅ KNN Selesai (Best K: {knn_grid.best_params_['n_neighbors']})")
print(f" 📌 Parameter Terbaik KNN: {knn_grid.best_params_}")
print(f" 📌 F1-Macro Validasi KNN: {knn_acc_val:.2f}%")
# ---------------------------------------------------------
# 4. LATIH ENSEMBLE (SVM + KNN)
# ---------------------------------------------------------
print("\n🚀 Melatih ENSEMBLE (Voting SVM + KNN)...")
# Gabungkan dua model terbaik
ensemble_model = VotingClassifier(
estimators=[
('svm', best_svm),
('knn', best_knn)
],
voting='soft',
weights=[2, 1] # SVM kita beri bobot suara lebih tinggi
)
ensemble_model.fit(X_train, y_train)
joblib.dump(ensemble_model, f'{FOLDER_MODELS}/model_ensemble.pkl')
print(" ✅ Ensemble Selesai.")
# ⭐ TAMBAHAN: Hitung akurasi validasi Ensemble pakai cross_val_score
print(" 📌 Menghitung Akurasi Validasi Ensemble (3-fold CV)...")
print(" (Sabar ya, ini agak lama karena ensemble = SVM + KNN x 3 fold)")
ensemble_scores = cross_val_score(ensemble_model, X_train, y_train, cv=3, n_jobs=-1)
ensemble_acc_val = ensemble_scores.mean() * 100
print(f" 📌 Akurasi Validasi Ensemble: {ensemble_acc_val:.2f}%")
# ---------------------------------------------------------
# 5. RINGKASAN AKHIR
# ---------------------------------------------------------
print("\n" + "="*60)
print("📊 RINGKASAN HASIL TRAINING")
print("="*60)
print(f"\n🔹 SVM")
print(f" Parameter : {svm_grid.best_params_}")
print(f" Akurasi Validasi : {svm_acc_val:.2f}%")
print(f"\n🔹 KNN")
print(f" Parameter : {knn_grid.best_params_}")
print(f" Akurasi Validasi : {knn_acc_val:.2f}%")
print(f"\n🔹 Ensemble (SVM + KNN)")
print(f" Voting : soft, weights=[2, 1]")
print(f" Akurasi Validasi : {ensemble_acc_val:.2f}%")
print("\n" + "="*60)
print("🎉 TRAINING DATA MURNI SELESAI!")
print("👉 Silakan jalankan '3_evaluasi.py' untuk melihat hasil akhirnya.")
print("="*60)