{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9d179189",
   "metadata": {},
   "source": [
    "# Pelatihan Model Klasifikasi — SISTA MIF"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f932c049",
   "metadata": {},
   "source": [
    "## 1. Import Library & Konfigurasi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "cf9c85ad",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Konfigurasi selesai.\n",
      "   Training file : d:\\Project\\sista_mif_ta\\data\\processed\\training_corpus.csv\n",
      "   Test file     : d:\\Project\\sista_mif_ta\\data\\processed\\test_set.csv\n",
      "   Model output  : d:\\Project\\sista_mif_ta\\fastapi\\ml_assets\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import joblib\n",
    "import json\n",
    "import warnings\n",
    "from pathlib import Path\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.metrics import classification_report, confusion_matrix, ConfusionMatrixDisplay\n",
    "from sklearn.pipeline import Pipeline\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.ticker as mticker\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "BASE_DIR   = Path.cwd()\n",
    "TRAIN_FILE = BASE_DIR / \"processed\" / \"training_corpus.csv\"\n",
    "TEST_FILE  = BASE_DIR / \"processed\" / \"test_set.csv\"\n",
    "ML_DIR     = BASE_DIR.parent / \"fastapi\" / \"ml_assets\"\n",
    "ML_DIR.mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "TARGET_CLASSES       = [\"Programmer\", \"Data Analyst\", \"Wirausaha Informatika\", \"Non-IT\"]\n",
    "CONFIDENCE_THRESHOLD = 0.50\n",
    "COLORS               = [\"#4C72B0\", \"#DD8452\", \"#55A868\", \"#C44E52\"]\n",
    "\n",
    "STOPWORDS = {\n",
    "    \"yang\", \"di\", \"ke\", \"dari\", \"dan\", \"atau\", \"dengan\", \"untuk\", \"pada\", \"dalam\",\n",
    "    \"adalah\", \"ini\", \"itu\", \"tidak\", \"juga\", \"sudah\", \"akan\", \"bisa\", \"ada\", \"oleh\",\n",
    "    \"karena\", \"secara\", \"serta\", \"sebagai\", \"bagi\", \"telah\", \"maka\", \"namun\", \"sehingga\",\n",
    "    \"jika\", \"agar\", \"ketika\", \"saat\", \"sebelum\", \"sesudah\", \"hingga\", \"sampai\", \"antara\",\n",
    "    \"sekitar\", \"hanya\", \"saja\", \"belum\", \"masih\", \"lagi\", \"pun\", \"justru\", \"walaupun\",\n",
    "    \"meskipun\", \"bahkan\", \"cukup\", \"sangat\", \"paling\", \"lebih\", \"kurang\", \"lain\",\n",
    "    \"macam\", \"cara\", \"hal\", \"tentang\", \"mengenai\", \"terhadap\", \"kepada\", \"menuju\",\n",
    "    \"kecuali\", \"selain\", \"tanpa\", \"demi\", \"guna\", \"khususnya\", \"umumnya\", \"kebanyakan\",\n",
    "    \"sebagian\", \"beberapa\", \"semua\", \"setiap\", \"tiap\", \"satu\", \"dua\", \"tiga\", \"empat\",\n",
    "    \"lima\", \"enam\", \"tujuh\", \"delapan\", \"sembilan\", \"sepuluh\", \"ratus\", \"ribu\", \"juta\"\n",
    "}\n",
    "\n",
    "print(\"Konfigurasi selesai.\")\n",
    "print(f\"   Training file : {TRAIN_FILE}\")\n",
    "print(f\"   Test file     : {TEST_FILE}\")\n",
    "print(f\"   Model output  : {ML_DIR}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "665600e3",
   "metadata": {},
   "source": [
    "## 2. Definisi Fungsi & Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "87e60ec2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pipeline TF-IDF + Logistic Regression siap.\n",
      "   - TF-IDF : max_features=3000, ngram=(1,2), sublinear_tf=True, min_df=2\n",
      "   - LR     : max_iter=2000, class_weight='balanced', solver='lbfgs'\n"
     ]
    }
   ],
   "source": [
    "def preprocess_text(text):\n",
    "    \"\"\"Pembersihan dasar — tanpa stemming ulang karena korpus sudah di-stem.\"\"\"\n",
    "    if pd.isna(text) or not isinstance(text, str): return \"\"\n",
    "    text = text.lower().strip()\n",
    "    if text in {\"nan\", \"none\", \"null\", \"-\", \"0\", \"\", \"tidak diisi\"}: return \"\"\n",
    "    text = text.replace('-', ' ').replace('/', ' ').replace('_', ' ').replace(',', ' ')\n",
    "    text = re.sub(r'[^\\w\\s]', '', text)\n",
    "    text = re.sub(r'\\d+', '', text)\n",
    "    words = [w for w in text.split() if w not in STOPWORDS and len(w) > 2]\n",
    "    return \" \".join(words)\n",
    "\n",
    "def get_pipeline():\n",
    "    \"\"\"Fungsi helper untuk memastikan konsistensi pipeline antara K-Fold dan Final Model.\"\"\"\n",
    "    return Pipeline([\n",
    "        ('tfidf', TfidfVectorizer(\n",
    "            max_features=3000,\n",
    "            ngram_range=(1, 2),\n",
    "            sublinear_tf=True,\n",
    "            min_df=2,\n",
    "            norm='l2'\n",
    "        )),\n",
    "        ('clf', LogisticRegression(\n",
    "            max_iter=2000,\n",
    "            class_weight='balanced',\n",
    "            solver='lbfgs',\n",
    "            random_state=42\n",
    "        ))\n",
    "    ])\n",
    "\n",
    "print(\"Pipeline TF-IDF + Logistic Regression siap.\")\n",
    "print(\"   - TF-IDF : max_features=3000, ngram=(1,2), sublinear_tf=True, min_df=2\")\n",
    "print(\"   - LR     : max_iter=2000, class_weight='balanced', solver='lbfgs'\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0888b6bd",
   "metadata": {},
   "source": [
    "## 3. Memuat & Memproses Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "285eb81b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sedang memproses teks...\n",
      "\n",
      "Data siap:\n",
      "   Jumlah data latih : 250 baris\n",
      "   Jumlah data uji   : 107 baris\n",
      "\n",
      "Distribusi label training:\n",
      "label\n",
      "Wirausaha Informatika    89\n",
      "Non-IT                   87\n",
      "Programmer               68\n",
      "Data Analyst              6\n"
     ]
    }
   ],
   "source": [
    "assert TRAIN_FILE.exists(), \"File training_corpus.csv tidak ditemukan. Jalankan prepare_corpus.ipynb terlebih dahulu.\"\n",
    "\n",
    "df_train = pd.read_csv(TRAIN_FILE, sep=\";\", dtype=str).dropna(subset=[\"job_text_raw\", \"label\"])\n",
    "df_test  = pd.read_csv(TEST_FILE,  sep=\";\", dtype=str).dropna(subset=[\"job_text_raw\", \"label\"]) if TEST_FILE.exists() else None\n",
    "\n",
    "print(\"Sedang memproses teks...\")\n",
    "X_train = df_train[\"job_text_raw\"].apply(preprocess_text)\n",
    "y_train = df_train[\"label\"]\n",
    "mask = X_train.str.len() > 0\n",
    "X_train, y_train = X_train[mask], y_train[mask]\n",
    "\n",
    "X_test, y_test = pd.Series(dtype=str), pd.Series(dtype=str)\n",
    "if df_test is not None:\n",
    "    X_test = df_test[\"job_text_raw\"].apply(preprocess_text)\n",
    "    y_test = df_test[\"label\"]\n",
    "    mask_t = X_test.str.len() > 0\n",
    "    X_test, y_test = X_test[mask_t], y_test[mask_t]\n",
    "\n",
    "print(f\"\\nData siap:\")\n",
    "print(f\"   Jumlah data latih : {len(X_train)} baris\")\n",
    "print(f\"   Jumlah data uji   : {len(X_test)} baris\")\n",
    "print(\"\\nDistribusi label training:\")\n",
    "print(y_train.value_counts().to_string())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2cb77033",
   "metadata": {},
   "source": [
    "## 4. K-Fold Cross Validation (5 Lipatan)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9cdd6602",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Melakukan pengujian K-Fold (5 putaran) pada data latih...\n",
      "  Fold 1/5  ->  Akurasi: 0.8200\n",
      "  Fold 2/5  ->  Akurasi: 0.7400\n",
      "  Fold 3/5  ->  Akurasi: 0.7200\n",
      "  Fold 4/5  ->  Akurasi: 0.7200\n",
      "  Fold 5/5  ->  Akurasi: 0.5600\n",
      "\n",
      "RATA-RATA PENGUJIAN K-FOLD:\n",
      "   Akurasi Keseluruhan : 0.7120 +/- 0.0845\n",
      "\n",
      "   Programmer                | P: 0.978 | R: 0.398 | F1: 0.549\n",
      "   Data Analyst              | P: 0.132 | R: 0.500 | F1: 0.207\n",
      "   Wirausaha Informatika     | P: 0.788 | R: 0.737 | F1: 0.746\n",
      "   Non-IT                    | P: 0.706 | R: 0.942 | F1: 0.805\n"
     ]
    }
   ],
   "source": [
    "print(\"Melakukan pengujian K-Fold (5 putaran) pada data latih...\")\n",
    "skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n",
    "fold_metrics = []\n",
    "\n",
    "for i, (tr_idx, te_idx) in enumerate(skf.split(X_train, y_train)):\n",
    "    model = get_pipeline()\n",
    "    model.fit(X_train.iloc[tr_idx], y_train.iloc[tr_idx])\n",
    "    y_pred = model.predict(X_train.iloc[te_idx])\n",
    "    rep = classification_report(y_train.iloc[te_idx], y_pred, output_dict=True, zero_division=0)\n",
    "    fold_metrics.append(rep)\n",
    "    print(f\"  Fold {i+1}/5  ->  Akurasi: {rep['accuracy']:.4f}\")\n",
    "\n",
    "avg_acc = np.mean([f['accuracy'] for f in fold_metrics])\n",
    "std_acc = np.std([f['accuracy'] for f in fold_metrics])\n",
    "\n",
    "print(f\"\\nRATA-RATA PENGUJIAN K-FOLD:\")\n",
    "print(f\"   Akurasi Keseluruhan : {avg_acc:.4f} +/- {std_acc:.4f}\")\n",
    "print()\n",
    "for cls in TARGET_CLASSES:\n",
    "    p  = np.mean([f.get(cls, {}).get('precision', 0) for f in fold_metrics])\n",
    "    r  = np.mean([f.get(cls, {}).get('recall', 0) for f in fold_metrics])\n",
    "    f1 = np.mean([f.get(cls, {}).get('f1-score', 0) for f in fold_metrics])\n",
    "    print(f\"   {cls:25} | P: {p:.3f} | R: {r:.3f} | F1: {f1:.3f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c1828f40",
   "metadata": {},
   "source": [
    "### Visualisasi Akurasi per Fold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "11067a1b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 30156 characters]",
      "text/plain": [
       "<Figure size 800x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fold_accs = [f['accuracy'] for f in fold_metrics]\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(8, 4))\n",
    "bars = ax.bar([f\"Fold {i+1}\" for i in range(5)], fold_accs,\n",
    "              color=COLORS[0], edgecolor=\"white\", linewidth=0.8, alpha=0.9)\n",
    "ax.axhline(avg_acc, color=\"red\", linestyle=\"--\", linewidth=1.5, label=f\"Rata-rata: {avg_acc:.4f}\")\n",
    "ax.fill_between(range(5), avg_acc - std_acc, avg_acc + std_acc,\n",
    "                alpha=0.15, color=\"red\", label=f\"± Std Dev: {std_acc:.4f}\")\n",
    "for bar, val in zip(bars, fold_accs):\n",
    "    ax.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.002,\n",
    "            f\"{val:.4f}\", ha=\"center\", va=\"bottom\", fontsize=9)\n",
    "ax.set_ylim(min(fold_accs) * 0.95, 1.0)\n",
    "ax.set_xlabel(\"Fold\"); ax.set_ylabel(\"Akurasi\")\n",
    "ax.set_title(\"Akurasi K-Fold Cross Validation (5 Lipatan)\", fontsize=12, fontweight=\"bold\")\n",
    "ax.legend(); ax.spines[[\"top\", \"right\"]].set_visible(False)\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "913f7707",
   "metadata": {},
   "source": [
    "### Visualisasi F1-Score per Kelas (Rata-rata K-Fold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "59984ccb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 40972 characters]",
      "text/plain": [
       "<Figure size 1100x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "metrics_summary = {\n",
    "    cls: {\n",
    "        \"precision\": np.mean([f.get(cls, {}).get('precision', 0) for f in fold_metrics]),\n",
    "        \"recall\":    np.mean([f.get(cls, {}).get('recall', 0) for f in fold_metrics]),\n",
    "        \"f1\":        np.mean([f.get(cls, {}).get('f1-score', 0) for f in fold_metrics]),\n",
    "    }\n",
    "    for cls in TARGET_CLASSES\n",
    "}\n",
    "\n",
    "x = np.arange(len(TARGET_CLASSES))\n",
    "width = 0.25\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(11, 5))\n",
    "for j, (metric, color) in enumerate([(\"precision\", \"#4C72B0\"), (\"recall\", \"#DD8452\"), (\"f1\", \"#55A868\")]):\n",
    "    vals = [metrics_summary[cls][metric] for cls in TARGET_CLASSES]\n",
    "    bars = ax.bar(x + j * width, vals, width, label=metric.capitalize(), color=color, edgecolor=\"white\", alpha=0.9)\n",
    "    for bar, val in zip(bars, vals):\n",
    "        ax.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.005,\n",
    "                f\"{val:.2f}\", ha=\"center\", va=\"bottom\", fontsize=8)\n",
    "\n",
    "ax.set_xticks(x + width)\n",
    "ax.set_xticklabels(TARGET_CLASSES, rotation=10)\n",
    "ax.set_ylim(0, 1.1); ax.set_ylabel(\"Skor\")\n",
    "ax.set_title(\"Precision / Recall / F1-Score per Kelas (Rata-rata K-Fold)\", fontsize=12, fontweight=\"bold\")\n",
    "ax.legend(); ax.spines[[\"top\", \"right\"]].set_visible(False)\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a737c575",
   "metadata": {},
   "source": [
    "## 5. Pelatihan Model Final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "63615b5f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Membuat model final dari seluruh data latih yang tersedia...\n",
      "Model final berhasil dilatih.\n"
     ]
    }
   ],
   "source": [
    "print(\"Membuat model final dari seluruh data latih yang tersedia...\")\n",
    "final_model = get_pipeline()\n",
    "final_model.fit(X_train, y_train)\n",
    "print(\"Model final berhasil dilatih.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04c06078",
   "metadata": {},
   "source": [
    "## 6. Evaluasi pada Hold-Out Test Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2b517946",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HASIL PENGUJIAN PADA DATA TEST:\n",
      "                       precision    recall  f1-score   support\n",
      "\n",
      "         Data Analyst       0.00      0.00      0.00         2\n",
      "               Non-IT       0.77      0.94      0.85        36\n",
      "           Programmer       0.95      0.63      0.76        30\n",
      "Wirausaha Informatika       0.86      0.92      0.89        39\n",
      "\n",
      "             accuracy                           0.83       107\n",
      "            macro avg       0.64      0.63      0.62       107\n",
      "         weighted avg       0.84      0.83      0.82       107\n",
      "\n",
      "Catatan: Kelas 'Data Analyst' hanya punya 2.0 data uji — skor F1-nya mungkin kurang akurat.\n"
     ]
    }
   ],
   "source": [
    "metrics_test = None\n",
    "if len(X_test) > 0:\n",
    "    y_pred = final_model.predict(X_test)\n",
    "    rep_test = classification_report(y_test, y_pred, output_dict=True, zero_division=0)\n",
    "    metrics_test = rep_test\n",
    "\n",
    "    print(\"HASIL PENGUJIAN PADA DATA TEST:\")\n",
    "    print(classification_report(y_test, y_pred, zero_division=0))\n",
    "\n",
    "    for cls in TARGET_CLASSES:\n",
    "        sup = rep_test[cls]['support'] if cls in rep_test else 0\n",
    "        if sup < 5:\n",
    "            print(f\"Catatan: Kelas '{cls}' hanya punya {sup} data uji — skor F1-nya mungkin kurang akurat.\")\n",
    "else:\n",
    "    print(\"Tidak ada data test. Lewati evaluasi.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fb152996",
   "metadata": {},
   "source": [
    "### 6a. Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "756c106b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 44988 characters]",
      "text/plain": [
       "<Figure size 700x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Disimpan ke: d:\\Project\\sista_mif_ta\\fastapi\\ml_assets\\confusion_matrix_test.png\n"
     ]
    }
   ],
   "source": [
    "if len(X_test) > 0:\n",
    "    cm = confusion_matrix(y_test, y_pred, labels=TARGET_CLASSES)\n",
    "    disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=TARGET_CLASSES)\n",
    "\n",
    "    fig, ax = plt.subplots(figsize=(7, 6))\n",
    "    disp.plot(cmap='Blues', values_format='d', xticks_rotation=30, colorbar=True, ax=ax)\n",
    "    ax.set_title('Confusion Matrix — Hold-Out Test Set', fontsize=12, fontweight='bold', pad=15)\n",
    "    plt.tight_layout()\n",
    "    plt.savefig(ML_DIR / 'confusion_matrix_test.png', dpi=300, bbox_inches='tight')\n",
    "    plt.show()\n",
    "    print(f\"   Disimpan ke: {ML_DIR / 'confusion_matrix_test.png'}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c3c98bcc",
   "metadata": {},
   "source": [
    "### 6b. Distribusi Confidence Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "41588b9e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 34928 characters]",
      "text/plain": [
       "<Figure size 900x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "   46 dari 107 prediksi (43.0%) di bawah threshold 0.5.\n",
      "   Disimpan ke: d:\\Project\\sista_mif_ta\\fastapi\\ml_assets\\confidence_distribution.png\n"
     ]
    }
   ],
   "source": [
    "if len(X_test) > 0:\n",
    "    proba = final_model.predict_proba(X_test)\n",
    "    max_p = np.max(proba, axis=1)\n",
    "    below = np.sum(max_p < CONFIDENCE_THRESHOLD)\n",
    "\n",
    "    fig, ax = plt.subplots(figsize=(9, 4))\n",
    "    ax.hist(max_p, bins=15, edgecolor='black', alpha=0.75, color='teal', label='Distribusi confidence')\n",
    "    ax.axvline(x=CONFIDENCE_THRESHOLD, color='red', linestyle='--', linewidth=2,\n",
    "               label=f'Threshold ({CONFIDENCE_THRESHOLD})')\n",
    "    ax.axvspan(0, CONFIDENCE_THRESHOLD, alpha=0.07, color='red', label=f'Di bawah threshold: {below} ({below/len(max_p)*100:.1f}%)')\n",
    "    ax.set_xlabel('Confidence Score', fontsize=11)\n",
    "    ax.set_ylabel('Jumlah Prediksi', fontsize=11)\n",
    "    ax.set_title('Distribusi Confidence Score Model', fontsize=12, fontweight='bold')\n",
    "    ax.legend(); ax.grid(axis='y', alpha=0.3)\n",
    "    ax.spines[[\"top\", \"right\"]].set_visible(False)\n",
    "    plt.tight_layout()\n",
    "    plt.savefig(ML_DIR / 'confidence_distribution.png', dpi=300, bbox_inches='tight')\n",
    "    plt.show()\n",
    "\n",
    "    print(f\"\\n   {below} dari {len(max_p)} prediksi ({below/len(max_p)*100:.1f}%) di bawah threshold {CONFIDENCE_THRESHOLD}.\")\n",
    "    print(f\"   Disimpan ke: {ML_DIR / 'confidence_distribution.png'}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "03ebc2a4",
   "metadata": {},
   "source": [
    "### 6c. Feature Importance (Kata Paling Berpengaruh)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "5fab1fc4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 148380 characters]",
      "text/plain": [
       "<Figure size 1400x1000 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "3 KATA PALING BERPENGARUH (POSITIF & NEGATIF) TIAP KELAS:\n",
      "  [Data Analyst]\n",
      "    Mendorong (+) : analyst(2.84), admin(1.81), system analyst(1.36)\n",
      "    Menghambat (-): staff(-1.11), founder(-0.83), programmer(-0.25)\n",
      "  [Non-IT]\n",
      "    Mendorong (+) : staff(0.81), staf(0.61), teller(0.54)\n",
      "    Menghambat (-): founder(-1.61), analyst(-1.17), freelance(-1.01)\n",
      "  [Programmer]\n",
      "    Mendorong (+) : teknisi(1.31), staff teknisi(1.31), programmer(1.24)\n",
      "    Menghambat (-): analyst(-1.02), admin(-0.88), system analyst(-0.51)\n",
      "  [Wirausaha Informatika]\n",
      "    Mendorong (+) : founder(2.14), kerja lepas(0.61), kerja(0.61)\n",
      "    Menghambat (-): analyst(-0.66), admin(-0.52), system(-0.34)\n"
     ]
    }
   ],
   "source": [
    "if len(X_test) > 0:\n",
    "    vec   = final_model.named_steps['tfidf']\n",
    "    clf   = final_model.named_steps['clf']\n",
    "    feats = vec.get_feature_names_out()\n",
    "    coeffs = clf.coef_\n",
    "\n",
    "    TOP_N = 10\n",
    "    fig, axes = plt.subplots(2, 2, figsize=(14, 10))\n",
    "    fig.suptitle(f\"Top {TOP_N} Kata Berpengaruh per Kelas (Koefisien Logistic Regression)\",\n",
    "                 fontsize=13, fontweight=\"bold\")\n",
    "\n",
    "    for ax, (cls_idx, cls) in zip(axes.flat, enumerate(clf.classes_)):\n",
    "        coef = coeffs[cls_idx]\n",
    "        top_pos_idx = coef.argsort()[-TOP_N:][::-1]\n",
    "        top_neg_idx = coef.argsort()[:TOP_N]\n",
    "        combined_idx = np.concatenate([top_pos_idx, top_neg_idx])\n",
    "        combined_vals = coef[combined_idx]\n",
    "        combined_words = feats[combined_idx]\n",
    "        colors_bar = [\"#4C72B0\" if v > 0 else \"#C44E52\" for v in combined_vals]\n",
    "\n",
    "        sorted_order = np.argsort(combined_vals)\n",
    "        ax.barh(combined_words[sorted_order], combined_vals[sorted_order],\n",
    "                color=[colors_bar[i] for i in sorted_order], edgecolor='white')\n",
    "        ax.axvline(0, color='black', linewidth=0.8)\n",
    "        ax.set_title(f\"[{cls}]\", fontsize=11, fontweight=\"bold\")\n",
    "        ax.set_xlabel(\"Koefisien\")\n",
    "        ax.spines[[\"top\", \"right\"]].set_visible(False)\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.show()\n",
    "\n",
    "    print(\"\\n3 KATA PALING BERPENGARUH (POSITIF & NEGATIF) TIAP KELAS:\")\n",
    "    for i, cls in enumerate(clf.classes_):\n",
    "        pos_idx = coeffs[i].argsort()[-3:][::-1]\n",
    "        neg_idx = coeffs[i].argsort()[:3]\n",
    "        top_pos = [(feats[j], coeffs[i][j]) for j in pos_idx if coeffs[i][j] > 0]\n",
    "        top_neg = [(feats[j], coeffs[i][j]) for j in neg_idx if coeffs[i][j] < 0]\n",
    "        pos_str = \", \".join([f\"{w}({c:.2f})\" for w, c in top_pos]) if top_pos else \"Tidak ada\"\n",
    "        neg_str = \", \".join([f\"{w}({c:.2f})\" for w, c in top_neg]) if top_neg else \"Tidak ada\"\n",
    "        print(f\"  [{cls}]\")\n",
    "        print(f\"    Mendorong (+) : {pos_str}\")\n",
    "        print(f\"    Menghambat (-): {neg_str}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a90948c7",
   "metadata": {},
   "source": [
    "### 6d. Prediksi yang Meleset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "62ff108f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18 prediksi meleset dari 107 total data uji:\n",
      "\n"
     ]
    }
   ],
   "source": [
    "if len(X_test) > 0:\n",
    "    mis = np.where(y_test.values != y_pred)[0]\n",
    "    if len(mis) > 0:\n",
    "        print(f\"{len(mis)} prediksi meleset dari {len(y_test)} total data uji:\\n\")\n",
    "        rows = []\n",
    "        for idx in mis[:min(10, len(mis))]:\n",
    "            rows.append({\n",
    "                \"Teks (cuplikan)\": df_test.iloc[idx]['job_text_raw'][:60] + \"...\",\n",
    "                \"Label Seharusnya\": y_test.iloc[idx],\n",
    "                \"Prediksi Model\": y_pred[idx],\n",
    "            })\n",
    "        pd.DataFrame(rows)\n",
    "    else:\n",
    "        print(\"Semua prediksi pada test set benar!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "14917bc5",
   "metadata": {},
   "source": [
    "## 7. Simpan Model & Metrik"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "2f972209",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model berhasil disimpan di : d:\\Project\\sista_mif_ta\\fastapi\\ml_assets\\ml_pipeline_internal.pkl\n",
      "Metrik disimpan di         : d:\\Project\\sista_mif_ta\\fastapi\\ml_assets\\metrics_internal_only.json\n",
      "\n",
      "   Akurasi rata-rata K-Fold  : 0.7120 +/- 0.0845\n",
      "   Weighted F1 (test set)    : 0.8231\n"
     ]
    }
   ],
   "source": [
    "model_path = ML_DIR / \"ml_pipeline_internal.pkl\"\n",
    "joblib.dump(final_model, model_path)\n",
    "\n",
    "metrics = {\n",
    "    \"methodology\": \"Internal MIF only\",\n",
    "    \"k_fold\": {\n",
    "        \"accuracy_mean\": float(avg_acc),\n",
    "        \"accuracy_std\":  float(np.std([f['accuracy'] for f in fold_metrics])),\n",
    "        \"folds\": fold_metrics\n",
    "    },\n",
    "    \"threshold_config\": CONFIDENCE_THRESHOLD\n",
    "}\n",
    "if metrics_test:\n",
    "    metrics[\"hold_out_test\"] = metrics_test\n",
    "\n",
    "metrics_file = ML_DIR / \"metrics_internal_only.json\"\n",
    "with open(metrics_file, 'w', encoding='utf-8') as f:\n",
    "    json.dump(metrics, f, indent=2, ensure_ascii=False)\n",
    "\n",
    "print(f\"Model berhasil disimpan di : {model_path}\")\n",
    "print(f\"Metrik disimpan di         : {metrics_file}\")\n",
    "print(f\"\\n   Akurasi rata-rata K-Fold  : {avg_acc:.4f} +/- {np.std([f['accuracy'] for f in fold_metrics]):.4f}\")\n",
    "if metrics_test:\n",
    "    print(f\"   Weighted F1 (test set)    : {metrics_test.get('weighted avg', {}).get('f1-score', 0):.4f}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
