180 lines
7.1 KiB
Python
180 lines
7.1 KiB
Python
# -*- coding: utf-8 -*-
|
|
import sys, io
|
|
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
|
|
"""
|
|
Evaluasi Efektivitas Sistem Rekomendasi Gemini AI
|
|
Aplikasi Muning Assistant - Amor Coffee
|
|
Menggunakan: scikit-learn, pandas, matplotlib
|
|
"""
|
|
|
|
import pandas as pd
|
|
import numpy as np
|
|
from sklearn.metrics import (
|
|
confusion_matrix,
|
|
classification_report,
|
|
accuracy_score,
|
|
precision_score,
|
|
recall_score,
|
|
f1_score,
|
|
ConfusionMatrixDisplay
|
|
)
|
|
import matplotlib
|
|
matplotlib.use('Agg') # non-interactive, tidak buka window GUI
|
|
import matplotlib.pyplot as plt
|
|
matplotlib.rcParams['font.family'] = 'DejaVu Sans'
|
|
|
|
# ─────────────────────────────────────────────
|
|
# BAGIAN 1: DATA HASIL PENGUJIAN (30 SKENARIO)
|
|
# ─────────────────────────────────────────────
|
|
# Keterangan kegagalan yang tercatat saat pengujian:
|
|
# S07 - Kategori: AI merekomendasikan bundling, padahal diminta minuman saja
|
|
# S22 - Kategori: AI merekomendasikan makanan, padahal diminta minuman
|
|
# S14 - Budget : Harga rekomendasi melebihi budget Rp15.000
|
|
# S29 - Budget : Harga rekomendasi melebihi budget Rp20.000
|
|
# S08 - Alergen : Rekomendasi mengandung susu (pengguna alergi susu)
|
|
# Selebihnya (25 skenario) → semua kriteria terpenuhi ✅
|
|
|
|
data = {
|
|
"skenario": [f"S{i:02d}" for i in range(1, 31)],
|
|
|
|
# Kriteria 1: Kesesuaian Kategori (minuman/makanan/bundling)
|
|
"kategori_expected": [1]*30,
|
|
"kategori_actual": [
|
|
1,1,1,1,1, 1,0,1,1,1, # S07 = 0
|
|
1,1,1,1,1, 1,1,1,1,1,
|
|
1,1,0,1,1, 1,1,1,1,1, # S23 = 0
|
|
],
|
|
|
|
# Kriteria 2: Kepatuhan Batas Budget
|
|
"budget_expected": [1]*30,
|
|
"budget_actual": [
|
|
1,1,1,1,1, 1,1,1,1,1,
|
|
1,1,1,0,1, 1,1,1,1,1, # S14 = 0
|
|
1,1,1,1,1, 1,1,1,0,1, # S29 = 0
|
|
],
|
|
|
|
# Kriteria 3: Bebas Alergen
|
|
"alergen_expected": [1]*30,
|
|
"alergen_actual": [
|
|
1,1,1,1,1, 1,1,0,1,1, # S08 = 0
|
|
1,1,1,1,1, 1,1,1,1,1,
|
|
1,1,1,1,1, 1,1,1,1,1,
|
|
],
|
|
|
|
# Kriteria 4: Kevalidan Keseluruhan (semua kriteria terpenuhi)
|
|
"overall_expected": [1]*30,
|
|
"overall_actual": [
|
|
1,1,1,1,1, 1,0,0,1,1, # S07,S08 = 0
|
|
1,1,1,0,1, 1,1,1,1,1, # S14 = 0
|
|
1,1,0,1,1, 1,1,1,0,1, # S23,S29 = 0
|
|
],
|
|
}
|
|
|
|
df = pd.DataFrame(data)
|
|
|
|
# ─────────────────────────────────────────────
|
|
# BAGIAN 2: FUNGSI EVALUASI PER KRITERIA
|
|
# ─────────────────────────────────────────────
|
|
|
|
def evaluasi_kriteria(y_true, y_pred, nama_kriteria):
|
|
print(f"\n{'='*55}")
|
|
print(f" EVALUASI: {nama_kriteria.upper()}")
|
|
print(f"{'='*55}")
|
|
|
|
acc = accuracy_score(y_true, y_pred)
|
|
prec = precision_score(y_true, y_pred, zero_division=0)
|
|
rec = recall_score(y_true, y_pred, zero_division=0)
|
|
f1 = f1_score(y_true, y_pred, zero_division=0)
|
|
|
|
cm = confusion_matrix(y_true, y_pred)
|
|
tn, fp, fn, tp = cm.ravel() if cm.size == 4 else (0, 0, 0, sum(y_true))
|
|
|
|
print(f"\n Confusion Matrix:")
|
|
print(f" +-----------------------------+")
|
|
print(f" | Prediksi AI |")
|
|
print(f" | Valid | Tidak Valid |")
|
|
print(f" +----------+------------------+")
|
|
print(f" | Aktual Valid | TP={tp:3d} FN={fn:3d} |")
|
|
print(f" | Aktual Tdk V | FP={fp:3d} TN={tn:3d} |")
|
|
print(f" +-----------------------------+")
|
|
print(f"\n Hasil Metrik:")
|
|
print(f" Accuracy : {acc:.4f} ({acc*100:.1f}%)")
|
|
print(f" Precision : {prec:.4f} ({prec*100:.1f}%)")
|
|
print(f" Recall : {rec:.4f} ({rec*100:.1f}%)")
|
|
print(f" F1-Score : {f1:.4f} ({f1*100:.1f}%)")
|
|
|
|
return {"kriteria": nama_kriteria, "TP": tp, "FP": fp, "FN": fn, "TN": tn,
|
|
"Accuracy": acc, "Precision": prec, "Recall": rec, "F1": f1}
|
|
|
|
# ─────────────────────────────────────────────
|
|
# BAGIAN 3: JALANKAN EVALUASI SEMUA KRITERIA
|
|
# ─────────────────────────────────────────────
|
|
|
|
hasil = []
|
|
kriteria_list = [
|
|
("kategori", "Kesesuaian Kategori Rekomendasi"),
|
|
("budget", "Kepatuhan Batas Budget"),
|
|
("alergen", "Bebas Alergen Pengguna"),
|
|
("overall", "Kevalidan Keseluruhan"),
|
|
]
|
|
|
|
for key, nama in kriteria_list:
|
|
r = evaluasi_kriteria(
|
|
df[f"{key}_expected"].tolist(),
|
|
df[f"{key}_actual"].tolist(),
|
|
nama
|
|
)
|
|
hasil.append(r)
|
|
|
|
# ─────────────────────────────────────────────
|
|
# BAGIAN 4: TABEL REKAP AKHIR
|
|
# ─────────────────────────────────────────────
|
|
|
|
df_hasil = pd.DataFrame(hasil)
|
|
df_hasil = df_hasil.set_index("kriteria")
|
|
df_hasil[["Accuracy","Precision","Recall","F1"]] = \
|
|
df_hasil[["Accuracy","Precision","Recall","F1"]].map(lambda x: f"{x*100:.1f}%")
|
|
|
|
print("\n" + "="*55)
|
|
print(" REKAP METRIK EVALUASI GEMINI AI")
|
|
print("="*55)
|
|
print(df_hasil[["TP","FP","FN","TN","Accuracy","Precision","Recall","F1"]].to_string())
|
|
|
|
# ─────────────────────────────────────────────
|
|
# BAGIAN 5: VISUALISASI CONFUSION MATRIX
|
|
# ─────────────────────────────────────────────
|
|
|
|
fig, axes = plt.subplots(1, 4, figsize=(20, 4))
|
|
fig.suptitle("Confusion Matrix - Evaluasi Efektivitas Gemini AI\nAplikasi Muning Assistant (Amor Coffee)",
|
|
fontsize=13, fontweight='bold', y=1.02)
|
|
|
|
labels = ["Tidak Valid", "Valid"]
|
|
colors = ["Blues", "Greens", "Oranges", "Purples"]
|
|
|
|
for i, (key, nama) in enumerate(kriteria_list):
|
|
y_true = df[f"{key}_expected"].tolist()
|
|
y_pred = df[f"{key}_actual"].tolist()
|
|
cm = confusion_matrix(y_true, y_pred)
|
|
disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=labels)
|
|
disp.plot(ax=axes[i], colorbar=False, cmap=colors[i])
|
|
axes[i].set_title(nama, fontsize=9, fontweight='bold')
|
|
axes[i].set_xlabel("Prediksi AI", fontsize=8)
|
|
axes[i].set_ylabel("Ground Truth", fontsize=8)
|
|
|
|
plt.tight_layout()
|
|
plt.savefig("confusion_matrix_gemini.png", dpi=150, bbox_inches='tight')
|
|
print("\n[OK] Gambar disimpan: confusion_matrix_gemini.png")
|
|
|
|
# ─────────────────────────────────────────────
|
|
# BAGIAN 6: CLASSIFICATION REPORT LENGKAP
|
|
# ─────────────────────────────────────────────
|
|
|
|
print("\n" + "="*55)
|
|
print(" CLASSIFICATION REPORT - OVERALL VALIDITY")
|
|
print("="*55)
|
|
print(classification_report(
|
|
df["overall_expected"],
|
|
df["overall_actual"],
|
|
target_names=["Tidak Valid", "Valid"]
|
|
))
|