{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8354ba29-0c6f-41d8-86ab-76f296355a6a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/user/TA/confivoice3/ml/features.py:70: UserWarning: PySoundFile failed. Trying audioread instead.\n",
      "  y, sr = librosa.load(file_path, sr=sample_rate, mono=True)\n",
      "/Users/user/TA/confivoice3/cv_app/.venv/lib/python3.10/site-packages/librosa/core/audio.py:184: FutureWarning: librosa.core.audio.__audioread_load\n",
      "\tDeprecated as of librosa version 0.10.0.\n",
      "\tIt will be removed in librosa version 1.0.\n",
      "  y, sr_native = __audioread_load(path, offset, duration, dtype)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "=== Distribusi Label Dataset ===\n",
      "PD : 43\n",
      "TPD: 43\n",
      "Total data: 86\n",
      "Distribusi label: {'PD': 43, 'TPD': 43}\n",
      "Jumlah fitur: 78\n",
      "Fitting 5 folds for each of 16 candidates, totalling 80 fits\n",
      "Best params: {'svm__C': 10, 'svm__gamma': 'scale', 'svm__kernel': 'rbf'}\n",
      "Best CV f1_macro: 0.6554293822792274\n",
      "Urutan kelas: ['PD', 'TPD']\n",
      "Urutan label: ['PD', 'TPD']\n",
      "[[29 14]\n",
      " [15 28]]\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 24044 characters]",
      "text/plain": [
       "<Figure size 600x500 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from collections import Counter\n",
    "\n",
    "from train_model import DATA_DIR, build_cv, build_grid_search\n",
    "from features import load_dataset\n",
    "from sklearn.model_selection import cross_val_predict\n",
    "from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# 1. Load dataset\n",
    "X, y, files = load_dataset(DATA_DIR)\n",
    "\n",
    "print(\"Total data:\", len(files))\n",
    "print(\"Distribusi label:\", dict(Counter(y)))\n",
    "print(\"Jumlah fitur:\", X.shape[1])\n",
    "\n",
    "# 2. Buat cross-validation dan model\n",
    "cv = build_cv(y)\n",
    "grid_search = build_grid_search(cv)\n",
    "\n",
    "# 3. Training GridSearch\n",
    "grid_search.fit(X, y)\n",
    "\n",
    "best_model = grid_search.best_estimator_\n",
    "\n",
    "print(\"Best params:\", grid_search.best_params_)\n",
    "print(\"Best CV f1_macro:\", grid_search.best_score_)\n",
    "print(\"Urutan kelas:\", list(best_model.classes_))\n",
    "\n",
    "# 4. Prediksi cross-validation\n",
    "y_pred = cross_val_predict(best_model, X, y, cv=cv, n_jobs=1)\n",
    "\n",
    "# 5. Confusion matrix\n",
    "labels = [\"PD\", \"TPD\"]\n",
    "cm = confusion_matrix(y, y_pred, labels=labels)\n",
    "\n",
    "print(\"Urutan label:\", labels)\n",
    "print(cm)\n",
    "\n",
    "# 6. Tampilkan gambar confusion matrix\n",
    "disp = ConfusionMatrixDisplay(\n",
    "    confusion_matrix=cm,\n",
    "    display_labels=[\"PD\", \"TPD\"]\n",
    ")\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(6, 5))\n",
    "disp.plot(cmap=\"Blues\", values_format=\"d\", ax=ax, colorbar=True)\n",
    "\n",
    "ax.set_title(\"Confusion Matrix Model SVM\")\n",
    "ax.set_xlabel(\"Prediksi\")\n",
    "ax.set_ylabel(\"Label Asli\")\n",
    "ax.set_xticklabels([\"Prediksi PD\", \"Prediksi TPD\"])\n",
    "ax.set_yticklabels([\"Asli PD\", \"Asli TPD\"])\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d509eed5-2719-4c11-98cd-67c157034f07",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score, balanced_accuracy_score, precision_score, recall_score, f1_score, classification_report\n",
    "\n",
    "print(\"Accuracy          :\", accuracy_score(y, y_pred))\n",
    "print(\"Balanced Accuracy :\", balanced_accuracy_score(y, y_pred))\n",
    "print(\"Precision Macro   :\", precision_score(y, y_pred, average=\"macro\", zero_division=0))\n",
    "print(\"Recall Macro      :\", recall_score(y, y_pred, average=\"macro\", zero_division=0))\n",
    "print(\"F1 Macro          :\", f1_score(y, y_pred, average=\"macro\", zero_division=0))\n",
    "\n",
    "print(classification_report(\n",
    "    y,\n",
    "    y_pred,\n",
    "    labels=labels,\n",
    "    target_names=[\"PD - Percaya Diri\", \"TPD - Tidak Percaya Diri\"],\n",
    "    zero_division=0\n",
    "))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6d7215c2-6d9b-4233-921b-d79e262912a1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 16 candidates, totalling 80 fits\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {\n",
       "  /* Definition of color scheme common for light and dark mode */\n",
       "  --sklearn-color-text: black;\n",
       "  --sklearn-color-line: gray;\n",
       "  /* Definition of color scheme for unfitted estimators */\n",
       "  --sklearn-color-unfitted-level-0: #fff5e6;\n",
       "  --sklearn-color-unfitted-level-1: #f6e4d2;\n",
       "  --sklearn-color-unfitted-level-2: #ffe0b3;\n",
       "  --sklearn-color-unfitted-level-3: chocolate;\n",
       "  /* Definition of color scheme for fitted estimators */\n",
       "  --sklearn-color-fitted-level-0: #f0f8ff;\n",
       "  --sklearn-color-fitted-level-1: #d4ebff;\n",
       "  --sklearn-color-fitted-level-2: #b3dbfd;\n",
       "  --sklearn-color-fitted-level-3: cornflowerblue;\n",
       "\n",
       "  /* Specific color for light theme */\n",
       "  --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
       "  --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, white)));\n",
       "  --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
       "  --sklearn-color-icon: #696969;\n",
       "\n",
       "  @media (prefers-color-scheme: dark) {\n",
       "    /* Redefinition of color scheme for dark theme */\n",
       "    --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
       "    --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, #111)));\n",
       "    --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
       "    --sklearn-color-icon: #878787;\n",
       "  }\n",
       "}\n",
       "\n",
       "#sk-container-id-1 {\n",
       "  color: var(--sklearn-color-text);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 pre {\n",
       "  padding: 0;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 input.sk-hidden--visually {\n",
       "  border: 0;\n",
       "  clip: rect(1px 1px 1px 1px);\n",
       "  clip: rect(1px, 1px, 1px, 1px);\n",
       "  height: 1px;\n",
       "  margin: -1px;\n",
       "  overflow: hidden;\n",
       "  padding: 0;\n",
       "  position: absolute;\n",
       "  width: 1px;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-dashed-wrapped {\n",
       "  border: 1px dashed var(--sklearn-color-line);\n",
       "  margin: 0 0.4em 0.5em 0.4em;\n",
       "  box-sizing: border-box;\n",
       "  padding-bottom: 0.4em;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-container {\n",
       "  /* jupyter's `normalize.less` sets `[hidden] { display: none; }`\n",
       "     but bootstrap.min.css set `[hidden] { display: none !important; }`\n",
       "     so we also need the `!important` here to be able to override the\n",
       "     default hidden behavior on the sphinx rendered scikit-learn.org.\n",
       "     See: https://github.com/scikit-learn/scikit-learn/issues/21755 */\n",
       "  display: inline-block !important;\n",
       "  position: relative;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-text-repr-fallback {\n",
       "  display: none;\n",
       "}\n",
       "\n",
       "div.sk-parallel-item,\n",
       "div.sk-serial,\n",
       "div.sk-item {\n",
       "  /* draw centered vertical line to link estimators */\n",
       "  background-image: linear-gradient(var(--sklearn-color-text-on-default-background), var(--sklearn-color-text-on-default-background));\n",
       "  background-size: 2px 100%;\n",
       "  background-repeat: no-repeat;\n",
       "  background-position: center center;\n",
       "}\n",
       "\n",
       "/* Parallel-specific style estimator block */\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item::after {\n",
       "  content: \"\";\n",
       "  width: 100%;\n",
       "  border-bottom: 2px solid var(--sklearn-color-text-on-default-background);\n",
       "  flex-grow: 1;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel {\n",
       "  display: flex;\n",
       "  align-items: stretch;\n",
       "  justify-content: center;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  position: relative;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item:first-child::after {\n",
       "  align-self: flex-end;\n",
       "  width: 50%;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item:last-child::after {\n",
       "  align-self: flex-start;\n",
       "  width: 50%;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item:only-child::after {\n",
       "  width: 0;\n",
       "}\n",
       "\n",
       "/* Serial-specific style estimator block */\n",
       "\n",
       "#sk-container-id-1 div.sk-serial {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "  align-items: center;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  padding-right: 1em;\n",
       "  padding-left: 1em;\n",
       "}\n",
       "\n",
       "\n",
       "/* Toggleable style: style used for estimator/Pipeline/ColumnTransformer box that is\n",
       "clickable and can be expanded/collapsed.\n",
       "- Pipeline and ColumnTransformer use this feature and define the default style\n",
       "- Estimators will overwrite some part of the style using the `sk-estimator` class\n",
       "*/\n",
       "\n",
       "/* Pipeline and ColumnTransformer style (default) */\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable {\n",
       "  /* Default theme specific background. It is overwritten whether we have a\n",
       "  specific estimator or a Pipeline/ColumnTransformer */\n",
       "  background-color: var(--sklearn-color-background);\n",
       "}\n",
       "\n",
       "/* Toggleable label */\n",
       "#sk-container-id-1 label.sk-toggleable__label {\n",
       "  cursor: pointer;\n",
       "  display: block;\n",
       "  width: 100%;\n",
       "  margin-bottom: 0;\n",
       "  padding: 0.5em;\n",
       "  box-sizing: border-box;\n",
       "  text-align: center;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 label.sk-toggleable__label-arrow:before {\n",
       "  /* Arrow on the left of the label */\n",
       "  content: \"▸\";\n",
       "  float: left;\n",
       "  margin-right: 0.25em;\n",
       "  color: var(--sklearn-color-icon);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {\n",
       "  color: var(--sklearn-color-text);\n",
       "}\n",
       "\n",
       "/* Toggleable content - dropdown */\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable__content {\n",
       "  max-height: 0;\n",
       "  max-width: 0;\n",
       "  overflow: hidden;\n",
       "  text-align: left;\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable__content.fitted {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable__content pre {\n",
       "  margin: 0.2em;\n",
       "  border-radius: 0.25em;\n",
       "  color: var(--sklearn-color-text);\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable__content.fitted pre {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {\n",
       "  /* Expand drop-down */\n",
       "  max-height: 200px;\n",
       "  max-width: 100%;\n",
       "  overflow: auto;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {\n",
       "  content: \"▾\";\n",
       "}\n",
       "\n",
       "/* Pipeline/ColumnTransformer-specific style */\n",
       "\n",
       "#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-label.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Estimator-specific style */\n",
       "\n",
       "/* Colorize estimator box */\n",
       "#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-estimator.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-label label.sk-toggleable__label,\n",
       "#sk-container-id-1 div.sk-label label {\n",
       "  /* The background is the default theme color */\n",
       "  color: var(--sklearn-color-text-on-default-background);\n",
       "}\n",
       "\n",
       "/* On hover, darken the color of the background */\n",
       "#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "/* Label box, darken color on hover, fitted */\n",
       "#sk-container-id-1 div.sk-label.fitted:hover label.sk-toggleable__label.fitted {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Estimator label */\n",
       "\n",
       "#sk-container-id-1 div.sk-label label {\n",
       "  font-family: monospace;\n",
       "  font-weight: bold;\n",
       "  display: inline-block;\n",
       "  line-height: 1.2em;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-label-container {\n",
       "  text-align: center;\n",
       "}\n",
       "\n",
       "/* Estimator-specific */\n",
       "#sk-container-id-1 div.sk-estimator {\n",
       "  font-family: monospace;\n",
       "  border: 1px dotted var(--sklearn-color-border-box);\n",
       "  border-radius: 0.25em;\n",
       "  box-sizing: border-box;\n",
       "  margin-bottom: 0.5em;\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-estimator.fitted {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "/* on hover */\n",
       "#sk-container-id-1 div.sk-estimator:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-estimator.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Specification for estimator info (e.g. \"i\" and \"?\") */\n",
       "\n",
       "/* Common style for \"i\" and \"?\" */\n",
       "\n",
       ".sk-estimator-doc-link,\n",
       "a:link.sk-estimator-doc-link,\n",
       "a:visited.sk-estimator-doc-link {\n",
       "  float: right;\n",
       "  font-size: smaller;\n",
       "  line-height: 1em;\n",
       "  font-family: monospace;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  border-radius: 1em;\n",
       "  height: 1em;\n",
       "  width: 1em;\n",
       "  text-decoration: none !important;\n",
       "  margin-left: 1ex;\n",
       "  /* unfitted */\n",
       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-unfitted-level-1);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link.fitted,\n",
       "a:link.sk-estimator-doc-link.fitted,\n",
       "a:visited.sk-estimator-doc-link.fitted {\n",
       "  /* fitted */\n",
       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-fitted-level-1);\n",
       "}\n",
       "\n",
       "/* On hover */\n",
       "div.sk-estimator:hover .sk-estimator-doc-link:hover,\n",
       ".sk-estimator-doc-link:hover,\n",
       "div.sk-label-container:hover .sk-estimator-doc-link:hover,\n",
       ".sk-estimator-doc-link:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "div.sk-estimator.fitted:hover .sk-estimator-doc-link.fitted:hover,\n",
       ".sk-estimator-doc-link.fitted:hover,\n",
       "div.sk-label-container:hover .sk-estimator-doc-link.fitted:hover,\n",
       ".sk-estimator-doc-link.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "/* Span, style for the box shown on hovering the info icon */\n",
       ".sk-estimator-doc-link span {\n",
       "  display: none;\n",
       "  z-index: 9999;\n",
       "  position: relative;\n",
       "  font-weight: normal;\n",
       "  right: .2ex;\n",
       "  padding: .5ex;\n",
       "  margin: .5ex;\n",
       "  width: min-content;\n",
       "  min-width: 20ex;\n",
       "  max-width: 50ex;\n",
       "  color: var(--sklearn-color-text);\n",
       "  box-shadow: 2pt 2pt 4pt #999;\n",
       "  /* unfitted */\n",
       "  background: var(--sklearn-color-unfitted-level-0);\n",
       "  border: .5pt solid var(--sklearn-color-unfitted-level-3);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link.fitted span {\n",
       "  /* fitted */\n",
       "  background: var(--sklearn-color-fitted-level-0);\n",
       "  border: var(--sklearn-color-fitted-level-3);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link:hover span {\n",
       "  display: block;\n",
       "}\n",
       "\n",
       "/* \"?\"-specific style due to the `<a>` HTML tag */\n",
       "\n",
       "#sk-container-id-1 a.estimator_doc_link {\n",
       "  float: right;\n",
       "  font-size: 1rem;\n",
       "  line-height: 1em;\n",
       "  font-family: monospace;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  border-radius: 1rem;\n",
       "  height: 1rem;\n",
       "  width: 1rem;\n",
       "  text-decoration: none;\n",
       "  /* unfitted */\n",
       "  color: var(--sklearn-color-unfitted-level-1);\n",
       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 a.estimator_doc_link.fitted {\n",
       "  /* fitted */\n",
       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-fitted-level-1);\n",
       "}\n",
       "\n",
       "/* On hover */\n",
       "#sk-container-id-1 a.estimator_doc_link:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 a.estimator_doc_link.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-3);\n",
       "}\n",
       "</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=42, shuffle=True),\n",
       "             estimator=Pipeline(steps=[(&#x27;scaler&#x27;, StandardScaler()),\n",
       "                                       (&#x27;svm&#x27;,\n",
       "                                        SVC(class_weight=&#x27;balanced&#x27;,\n",
       "                                            probability=True,\n",
       "                                            random_state=42))]),\n",
       "             n_jobs=1,\n",
       "             param_grid={&#x27;svm__C&#x27;: [0.1, 1, 10, 100],\n",
       "                         &#x27;svm__gamma&#x27;: [&#x27;scale&#x27;, 0.01, 0.001, 0.0001],\n",
       "                         &#x27;svm__kernel&#x27;: [&#x27;rbf&#x27;]},\n",
       "             scoring=&#x27;f1_macro&#x27;, verbose=1)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label fitted sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" ><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label fitted sk-toggleable__label-arrow fitted\">&nbsp;&nbsp;GridSearchCV<a class=\"sk-estimator-doc-link fitted\" rel=\"noreferrer\" target=\"_blank\" href=\"https://scikit-learn.org/1.5/modules/generated/sklearn.model_selection.GridSearchCV.html\">?<span>Documentation for GridSearchCV</span></a><span class=\"sk-estimator-doc-link fitted\">i<span>Fitted</span></span></label><div class=\"sk-toggleable__content fitted\"><pre>GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=42, shuffle=True),\n",
       "             estimator=Pipeline(steps=[(&#x27;scaler&#x27;, StandardScaler()),\n",
       "                                       (&#x27;svm&#x27;,\n",
       "                                        SVC(class_weight=&#x27;balanced&#x27;,\n",
       "                                            probability=True,\n",
       "                                            random_state=42))]),\n",
       "             n_jobs=1,\n",
       "             param_grid={&#x27;svm__C&#x27;: [0.1, 1, 10, 100],\n",
       "                         &#x27;svm__gamma&#x27;: [&#x27;scale&#x27;, 0.01, 0.001, 0.0001],\n",
       "                         &#x27;svm__kernel&#x27;: [&#x27;rbf&#x27;]},\n",
       "             scoring=&#x27;f1_macro&#x27;, verbose=1)</pre></div> </div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label fitted sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" ><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label fitted sk-toggleable__label-arrow fitted\">best_estimator_: Pipeline</label><div class=\"sk-toggleable__content fitted\"><pre>Pipeline(steps=[(&#x27;scaler&#x27;, StandardScaler()),\n",
       "                (&#x27;svm&#x27;,\n",
       "                 SVC(C=10, class_weight=&#x27;balanced&#x27;, probability=True,\n",
       "                     random_state=42))])</pre></div> </div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator fitted sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" ><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label fitted sk-toggleable__label-arrow fitted\">&nbsp;StandardScaler<a class=\"sk-estimator-doc-link fitted\" rel=\"noreferrer\" target=\"_blank\" href=\"https://scikit-learn.org/1.5/modules/generated/sklearn.preprocessing.StandardScaler.html\">?<span>Documentation for StandardScaler</span></a></label><div class=\"sk-toggleable__content fitted\"><pre>StandardScaler()</pre></div> </div></div><div class=\"sk-item\"><div class=\"sk-estimator fitted sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-4\" type=\"checkbox\" ><label for=\"sk-estimator-id-4\" class=\"sk-toggleable__label fitted sk-toggleable__label-arrow fitted\">&nbsp;SVC<a class=\"sk-estimator-doc-link fitted\" rel=\"noreferrer\" target=\"_blank\" href=\"https://scikit-learn.org/1.5/modules/generated/sklearn.svm.SVC.html\">?<span>Documentation for SVC</span></a></label><div class=\"sk-toggleable__content fitted\"><pre>SVC(C=10, class_weight=&#x27;balanced&#x27;, probability=True, random_state=42)</pre></div> </div></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=42, shuffle=True),\n",
       "             estimator=Pipeline(steps=[('scaler', StandardScaler()),\n",
       "                                       ('svm',\n",
       "                                        SVC(class_weight='balanced',\n",
       "                                            probability=True,\n",
       "                                            random_state=42))]),\n",
       "             n_jobs=1,\n",
       "             param_grid={'svm__C': [0.1, 1, 10, 100],\n",
       "                         'svm__gamma': ['scale', 0.01, 0.001, 0.0001],\n",
       "                         'svm__kernel': ['rbf']},\n",
       "             scoring='f1_macro', verbose=1)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv = build_cv(y)\n",
    "grid_search = build_grid_search(cv)\n",
    "\n",
    "grid_search.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "83e533a4-bb17-477f-9566-d765fbbf7cc7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best params: {'svm__C': 10, 'svm__gamma': 'scale', 'svm__kernel': 'rbf'}\n",
      "Best CV f1_macro: 0.6554293822792274\n",
      "Urutan kelas: ['PD', 'TPD']\n"
     ]
    }
   ],
   "source": [
    "best_model = grid_search.best_estimator_\n",
    "\n",
    "print(\"Best params:\", grid_search.best_params_)\n",
    "print(\"Best CV f1_macro:\", grid_search.best_score_)\n",
    "print(\"Urutan kelas:\", list(best_model.classes_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e2279016-05d5-4a05-b199-d99c3ad2cf21",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "=== Evaluasi Cross-Validation ===\n",
      "Accuracy          : 0.6628\n",
      "Balanced Accuracy : 0.6628\n",
      "Precision Macro   : 0.6629\n",
      "Recall Macro      : 0.6628\n",
      "F1 Macro          : 0.6627\n",
      "\n",
      "=== Classification Report ===\n",
      "                          precision    recall  f1-score   support\n",
      "\n",
      "       PD - Percaya Diri       0.66      0.67      0.67        43\n",
      "TPD - Tidak Percaya Diri       0.67      0.65      0.66        43\n",
      "\n",
      "                accuracy                           0.66        86\n",
      "               macro avg       0.66      0.66      0.66        86\n",
      "            weighted avg       0.66      0.66      0.66        86\n",
      "\n",
      "=== Confusion Matrix ===\n",
      "Urutan label: ['PD', 'TPD']\n",
      "[[29 14]\n",
      " [15 28]]\n",
      "\n",
      "=== Ringkasan Benar/Salah per Kelas ===\n",
      "PD: benar=29, salah=14, total=43\n",
      "TPD: benar=28, salah=15, total=43\n",
      "\n",
      "=== File yang Salah Prediksi ===\n",
      "c_pd.wav | PD | TPD\n",
      "c_tpd.wav | TPD | PD\n",
      "f_tpd.wav | TPD | PD\n",
      "h_tpd.wav | TPD | PD\n",
      "j_pd.wav | PD | TPD\n",
      "j_tpd.wav | TPD | PD\n",
      "k_tpd.wav | TPD | PD\n",
      "m_pd.wav | PD | TPD\n",
      "p10_pd.wav | PD | TPD\n",
      "p10_tpd.wav | TPD | PD\n",
      "p2_pd.wav | PD | TPD\n",
      "p3_pd.wav | PD | TPD\n",
      "p4_pd.wav | PD | TPD\n",
      "p5_tpd.wav | TPD | PD\n",
      "p6_tpd.wav | TPD | PD\n",
      "p8_pd.wav | PD | TPD\n",
      "p_pd.wav | PD | TPD\n",
      "q_pd.wav | PD | TPD\n",
      "s_tpd.wav | TPD | PD\n",
      "w3_pd.wav | PD | TPD\n",
      "w3_tpd.wav | TPD | PD\n",
      "w4_pd.wav | PD | TPD\n",
      "w5_pd.wav | PD | TPD\n",
      "w5_tpd.wav | TPD | PD\n",
      "w6_tpd.wav | TPD | PD\n",
      "w7_tpd.wav | TPD | PD\n",
      "w8_pd.wav | PD | TPD\n",
      "w8_tpd.wav | TPD | PD\n",
      "w_tpd.wav | TPD | PD\n"
     ]
    }
   ],
   "source": [
    "evaluate_model(best_model, X, y, files, cv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "03747dfc-acbf-42aa-9814-efa02e1d1d5f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.model_selection import cross_val_predict\n",
    "from features import LABEL_PD, LABEL_TPD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a4e120f6-e45e-4a57-ab29-3cf80ee91cf5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/user/TA/confivoice3/ml/features.py:70: UserWarning: PySoundFile failed. Trying audioread instead.\n",
      "  y, sr = librosa.load(file_path, sr=sample_rate, mono=True)\n",
      "/Users/user/TA/confivoice3/cv_app/.venv/lib/python3.10/site-packages/librosa/core/audio.py:184: FutureWarning: librosa.core.audio.__audioread_load\n",
      "\tDeprecated as of librosa version 0.10.0.\n",
      "\tIt will be removed in librosa version 1.0.\n",
      "  y, sr_native = __audioread_load(path, offset, duration, dtype)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "=== Distribusi Label Dataset ===\n",
      "PD : 43\n",
      "TPD: 43\n",
      "Total data: 86\n",
      "Distribusi label: {'PD': 43, 'TPD': 43}\n",
      "Jumlah fitur: 78\n",
      "Fitting 5 folds for each of 16 candidates, totalling 80 fits\n",
      "Best params: {'svm__C': 10, 'svm__gamma': 'scale', 'svm__kernel': 'rbf'}\n",
      "Best CV f1_macro: 0.6554293822792274\n",
      "Urutan kelas: ['PD', 'TPD']\n"
     ]
    }
   ],
   "source": [
    "from collections import Counter\n",
    "\n",
    "from train_model import DATA_DIR, build_cv, build_grid_search\n",
    "from features import load_dataset\n",
    "\n",
    "X, y, files = load_dataset(DATA_DIR)\n",
    "\n",
    "print(\"Total data:\", len(files))\n",
    "print(\"Distribusi label:\", dict(Counter(y)))\n",
    "print(\"Jumlah fitur:\", X.shape[1])\n",
    "\n",
    "cv = build_cv(y)\n",
    "grid_search = build_grid_search(cv)\n",
    "\n",
    "grid_search.fit(X, y)\n",
    "\n",
    "best_model = grid_search.best_estimator_\n",
    "\n",
    "print(\"Best params:\", grid_search.best_params_)\n",
    "print(\"Best CV f1_macro:\", grid_search.best_score_)\n",
    "print(\"Urutan kelas:\", list(best_model.classes_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6b93e357-b6b5-474a-a020-fc7c925d172a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Urutan label: ['PD', 'TPD']\n",
      "[[29 14]\n",
      " [15 28]]\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 24044 characters]",
      "text/plain": [
       "<Figure size 600x500 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_predict\n",
    "from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "labels = [\"PD\", \"TPD\"]\n",
    "\n",
    "y_pred = cross_val_predict(best_model, X, y, cv=cv, n_jobs=1)\n",
    "\n",
    "cm = confusion_matrix(y, y_pred, labels=labels)\n",
    "\n",
    "print(\"Urutan label:\", labels)\n",
    "print(cm)\n",
    "\n",
    "disp = ConfusionMatrixDisplay(\n",
    "    confusion_matrix=cm,\n",
    "    display_labels=[\"PD\", \"TPD\"]\n",
    ")\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(6, 5))\n",
    "disp.plot(cmap=\"Blues\", values_format=\"d\", ax=ax, colorbar=True)\n",
    "\n",
    "ax.set_title(\"Confusion Matrix Model SVM\")\n",
    "ax.set_xlabel(\"Prediksi\")\n",
    "ax.set_ylabel(\"Label Asli\")\n",
    "ax.set_xticklabels([\"Prediksi PD\", \"Prediksi TPD\"])\n",
    "ax.set_yticklabels([\"Asli PD\", \"Asli TPD\"])\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db2c5b18-fb32-426a-b704-d21b0aae65cf",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (ConfiVoice)",
   "language": "python",
   "name": "confivoice"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.20"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
