{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "73b2b3ae",
   "metadata": {},
   "source": [
    "# Persiapan Corpus — SISTA MIF"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd472692",
   "metadata": {},
   "source": [
    "## 1. Import Library & Konfigurasi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ed4261a0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Konfigurasi selesai.\n",
      "   File input : d:\\Project\\sista_mif_ta\\data\\ts_internal_mif.xlsx\n",
      "   Output dir : d:\\Project\\sista_mif_ta\\data\\processed\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "from pathlib import Path\n",
    "from sklearn.model_selection import train_test_split\n",
    "from Sastrawi.Stemmer.StemmerFactory import StemmerFactory\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.ticker as mticker\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "factory = StemmerFactory()\n",
    "stemmer = factory.create_stemmer()\n",
    "\n",
    "BASE_DIR   = Path.cwd()\n",
    "DATA_DIR   = BASE_DIR\n",
    "OUTPUT_DIR = DATA_DIR / \"processed\"\n",
    "OUTPUT_DIR.mkdir(exist_ok=True)\n",
    "\n",
    "FILE_INTERNAL = DATA_DIR / \"ts_internal_mif.xlsx\"\n",
    "SEPARATOR     = \";\"\n",
    "ENCODING      = \"utf-8-sig\"\n",
    "TARGET_CLASSES = [\"Programmer\", \"Data Analyst\", \"Wirausaha Informatika\", \"Non-IT\"]\n",
    "\n",
    "KLASIFIKASI_MAP = {\n",
    "    \"Programmer\": \"Programmer\",\n",
    "    \"Data Analyst\": \"Data Analyst\",\n",
    "    \"Wirausaha IT\": \"Wirausaha Informatika\",\n",
    "    \"Wirausaha\": \"Wirausaha Informatika\",\n",
    "    \"Non-IT\": \"Non-IT\",\n",
    "    \"Infokom\": None, \"Pelajar\": None, \"Tidak Bekerja\": None, \"TIdak diketahui\": None\n",
    "}\n",
    "KEYWORD_RULES = {\n",
    "    \"Programmer\": [\"programmer\", \"developer\", \"engineer\", \"fullstack\", \"backend\", \"frontend\", \"mobile\", \"android\", \"ios\", \"software\", \"web dev\", \"coding\", \"it staff\", \"teknisi\", \"sistem informasi\", \"application\", \"network\", \"devops\", \"qa\", \"tester\", \"ui\", \"ux\", \"swe\"],\n",
    "    \"Data Analyst\": [\"data analyst\", \"analis data\", \"data science\", \"business analyst\", \"research\", \"statistik\", \"bi analyst\", \"reporting\", \"database\", \"sql\", \"etl\", \"data engineer\", \"big data\", \"analyst\", \"data mining\", \"machine learning\", \"data visual\", \"power bi\", \"tableau\", \"looker\", \"business intelligence\", \"bi developer\", \"data warehouse\"],\n",
    "    \"Wirausaha Informatika\": [\"founder\", \"owner\", \"ceo\", \"wiraswasta\", \"startup\", \"freelance\", \"freelancer\", \"wirausaha\", \"bisnis\", \"usaha mandiri\", \"konsultan\", \"co founder\", \"entrepreneur\", \"self employed\", \"owner toko\", \"usaha\", \"dagang online\", \"tokopedia\", \"shopee\", \"dropship\", \"reseller\"]\n",
    "}\n",
    "COMPANY_STOPWORDS = {\n",
    "    \"pt\", \"cv\", \"ud\", \"tbk\", \"persero\", \"corp\", \"inc\", \"ltd\", \"koperasi\", \"bumn\", \"bumd\",\n",
    "    \"dinas\", \"kantor\", \"pemkab\", \"pemprov\", \"politeknik\", \"universitas\", \"sekolah\", \"sma\", \"smk\", \"sd\",\n",
    "    \"bank\", \"bpr\", \"rs\", \"rumah sakit\", \"klinik\", \"apotek\", \"hotel\", \"restoran\", \"cafe\", \"toko\", \"konter\",\n",
    "    \"foundation\", \"yayasan\", \"perkumpulan\", \"organisasi\", \"agency\", \"studio\", \"consulting\", \"group\", \"holding\"\n",
    "}\n",
    "\n",
    "print(\"Konfigurasi selesai.\")\n",
    "print(f\"   File input : {FILE_INTERNAL}\")\n",
    "print(f\"   Output dir : {OUTPUT_DIR}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "79c9c79a",
   "metadata": {},
   "source": [
    "## 2. Definisi Fungsi Bantu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f880576a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Semua fungsi bantu berhasil didefinisikan.\n"
     ]
    }
   ],
   "source": [
    "def load_data_file(file_path: str) -> pd.DataFrame:\n",
    "    ext = Path(file_path).suffix.lower()\n",
    "    if ext == '.csv':\n",
    "        try:\n",
    "            return pd.read_csv(file_path, sep=SEPARATOR, encoding=ENCODING, dtype=str, on_bad_lines='skip', engine='python')\n",
    "        except UnicodeDecodeError:\n",
    "            return pd.read_csv(file_path, sep=SEPARATOR, encoding='latin1', dtype=str, on_bad_lines='skip', engine='python')\n",
    "    elif ext == '.xlsx':\n",
    "        return pd.read_excel(file_path, dtype=str)\n",
    "    else:\n",
    "        raise ValueError(f\"Format tidak didukung: {ext}\")\n",
    "\n",
    "def find_col(df: pd.DataFrame, keywords: list) -> str | None:\n",
    "    for col in df.columns:\n",
    "        col_clean = str(col).strip().lower()\n",
    "        if any(k.strip().lower() in col_clean for k in keywords):\n",
    "            return col\n",
    "    return None\n",
    "\n",
    "def clean_text(text: str) -> str:\n",
    "    if pd.isna(text) or str(text).strip().lower() in [\"nan\", \"none\", \"null\", \"-\", \"0\", \"\", \"tidak diisi\", \"tidak diketahui\"]:\n",
    "        return \"\"\n",
    "    text = str(text).strip().lower()\n",
    "    text = re.sub(r'^\\d+\\s*[-:/]\\s*', '', text)\n",
    "    text = text.replace('-', ' ').replace('/', ' ').replace('_', ' ').replace(',', ' ')\n",
    "    text = re.sub(r'[^\\w\\s]', '', text)\n",
    "    text = re.sub(r'\\d+', '', text)\n",
    "    tokens = [w for w in text.split() if w not in COMPANY_STOPWORDS and len(w) >= 3]\n",
    "    return \" \".join([stemmer.stem(w) for w in tokens])\n",
    "\n",
    "def is_likely_name(text: str, full_name: str) -> bool:\n",
    "    if not text or not full_name: return False\n",
    "    text_clean = re.sub(r'[^\\w\\s]', '', text.lower())\n",
    "    name_clean = re.sub(r'[^\\w\\s]', '', str(full_name).lower())\n",
    "    text_parts = set(text_clean.split())\n",
    "    name_parts = set(name_clean.split())\n",
    "    if len(text_parts) == 0: return False\n",
    "    if len(text_parts) <= 3 and text_parts.issubset(name_parts):\n",
    "        return True\n",
    "    return False\n",
    "\n",
    "def classify_rule_based(text: str) -> str:\n",
    "    if not text or len(text) < 3: return \"Non-IT\"\n",
    "    for profile in [\"Programmer\", \"Data Analyst\", \"Wirausaha Informatika\"]:\n",
    "        if any(kw in text for kw in KEYWORD_RULES[profile]): return profile\n",
    "    return \"Non-IT\"\n",
    "\n",
    "print(\"Semua fungsi bantu berhasil didefinisikan.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ebf96089",
   "metadata": {},
   "source": [
    "## 3. Memuat & Membersihkan Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "fa0d1e7f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1/4] Memuat & Membersihkan Data...\n",
      "   Kolom NIM         : nim\n",
      "   Kolom Nama        : nama_lengkap\n",
      "   Kolom Jabatan     : jabatan\n",
      "   Kolom Jabatan Baru: jabatan_terupdate\n",
      "   Kolom Klasifikasi : klasifikasi_pekerjaan\n",
      "   Kolom Status      : status_pekerjaan\n",
      "\n",
      "   Total baris dimuat: 488\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nama_lengkap</th>\n",
       "      <th>nim</th>\n",
       "      <th>email</th>\n",
       "      <th>no_telepon</th>\n",
       "      <th>alamat_domisili</th>\n",
       "      <th>jurusan</th>\n",
       "      <th>program_studi</th>\n",
       "      <th>tahun_masuk</th>\n",
       "      <th>tahun_lulus</th>\n",
       "      <th>status_pekerjaan</th>\n",
       "      <th>...</th>\n",
       "      <th>jumlah_undangan_wawancara</th>\n",
       "      <th>masa_tunggu_pra_lulus</th>\n",
       "      <th>masa_tunggu_pasca_lulus</th>\n",
       "      <th>total_masa_tunggu</th>\n",
       "      <th>instansi_terupdate</th>\n",
       "      <th>jabatan_terupdate</th>\n",
       "      <th>tahun_bekerja</th>\n",
       "      <th>status_pekerjaan_terupdate</th>\n",
       "      <th>linkedin_profile</th>\n",
       "      <th>sosmed_ig</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ABRAHAM FIRDAUS FATHURROSI</td>\n",
       "      <td>E31170249</td>\n",
       "      <td>abrahamfirdaus86@gmail.com</td>\n",
       "      <td>081217394021</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Teknologi Informasi</td>\n",
       "      <td>Manajemen Informatika</td>\n",
       "      <td>2016</td>\n",
       "      <td>2019</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.linkedin.com/in/abraham-firdaus-fa...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ACHMAD APRILIANDI ALALLAH</td>\n",
       "      <td>E31170384</td>\n",
       "      <td>andypunk64@gmail.com</td>\n",
       "      <td>082247877665</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Teknologi Informasi</td>\n",
       "      <td>Manajemen Informatika</td>\n",
       "      <td>2016</td>\n",
       "      <td>2019</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.linkedin.com/in/achmad-apriliandi-...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ADHITYA PUTRA WARDHANA</td>\n",
       "      <td>E31151908</td>\n",
       "      <td>adhityaputra1997@gmail.com</td>\n",
       "      <td>081333682804</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Teknologi Informasi</td>\n",
       "      <td>Manajemen Informatika</td>\n",
       "      <td>2016</td>\n",
       "      <td>2019</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.linkedin.com/in/adhitya-putra-ward...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 nama_lengkap        nim                       email  \\\n",
       "0  ABRAHAM FIRDAUS FATHURROSI  E31170249  abrahamfirdaus86@gmail.com   \n",
       "1   ACHMAD APRILIANDI ALALLAH  E31170384        andypunk64@gmail.com   \n",
       "2      ADHITYA PUTRA WARDHANA  E31151908  adhityaputra1997@gmail.com   \n",
       "\n",
       "     no_telepon alamat_domisili              jurusan          program_studi  \\\n",
       "0  081217394021             NaN  Teknologi Informasi  Manajemen Informatika   \n",
       "1  082247877665             NaN  Teknologi Informasi  Manajemen Informatika   \n",
       "2  081333682804             NaN  Teknologi Informasi  Manajemen Informatika   \n",
       "\n",
       "  tahun_masuk tahun_lulus status_pekerjaan  ... jumlah_undangan_wawancara  \\\n",
       "0        2016        2019              NaN  ...                       NaN   \n",
       "1        2016        2019              NaN  ...                       NaN   \n",
       "2        2016        2019              NaN  ...                       NaN   \n",
       "\n",
       "  masa_tunggu_pra_lulus masa_tunggu_pasca_lulus total_masa_tunggu  \\\n",
       "0                   NaN                     NaN               NaN   \n",
       "1                   NaN                     NaN               NaN   \n",
       "2                   NaN                     NaN               NaN   \n",
       "\n",
       "  instansi_terupdate jabatan_terupdate tahun_bekerja  \\\n",
       "0                NaN               NaN           NaN   \n",
       "1                NaN               NaN           NaN   \n",
       "2                NaN               NaN           NaN   \n",
       "\n",
       "  status_pekerjaan_terupdate  \\\n",
       "0                        NaN   \n",
       "1                        NaN   \n",
       "2                        NaN   \n",
       "\n",
       "                                    linkedin_profile sosmed_ig  \n",
       "0  https://www.linkedin.com/in/abraham-firdaus-fa...       NaN  \n",
       "1  https://www.linkedin.com/in/achmad-apriliandi-...       NaN  \n",
       "2  https://www.linkedin.com/in/adhitya-putra-ward...       NaN  \n",
       "\n",
       "[3 rows x 31 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "assert FILE_INTERNAL.exists(), f\"File tidak ditemukan: {FILE_INTERNAL}\"\n",
    "\n",
    "print(\"[1/4] Memuat & Membersihkan Data...\")\n",
    "df = load_data_file(str(FILE_INTERNAL))\n",
    "df.columns = df.columns.str.strip()\n",
    "\n",
    "col_nim         = find_col(df, [\"nim\"])\n",
    "col_nama        = find_col(df, [\"nama\", \"lengkap\"])\n",
    "col_jab_lama    = find_col(df, [\"jabatan\"])\n",
    "col_jab_baru    = find_col(df, [\"jabatan_terupdate\"])\n",
    "col_klasifikasi = find_col(df, [\"klasifikasi\"])\n",
    "col_status      = find_col(df, [\"status\", \"kerja\"])\n",
    "\n",
    "print(f\"   Kolom NIM         : {col_nim}\")\n",
    "print(f\"   Kolom Nama        : {col_nama}\")\n",
    "print(f\"   Kolom Jabatan     : {col_jab_lama}\")\n",
    "print(f\"   Kolom Jabatan Baru: {col_jab_baru}\")\n",
    "print(f\"   Kolom Klasifikasi : {col_klasifikasi}\")\n",
    "print(f\"   Kolom Status      : {col_status}\")\n",
    "print(f\"\\n   Total baris dimuat: {len(df)}\")\n",
    "\n",
    "assert all([col_nim, col_nama, col_jab_lama]), \"Kolom esensial tidak ditemukan!\"\n",
    "\n",
    "df.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6d4d385d",
   "metadata": {},
   "source": [
    "## 4. Penggabungan Kolom Jabatan & Filter Status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "8c1c8ef4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Kolom jabatan digabung: 448 baris fallback ke jabatan lama\n",
      "   Filter status kerja: 13 baris dihapus -> tersisa 475 baris\n"
     ]
    }
   ],
   "source": [
    "if col_jab_baru:\n",
    "    mask_empty = df[col_jab_baru].isna() | df[col_jab_baru].astype(str).str.strip().str.lower().isin(\n",
    "        [\"\", \"-\", \"0\", \"nan\", \"none\", \"null\", \"tidak diisi\", \"tidak diketahui\"]\n",
    "    )\n",
    "    df[\"jabatan_final\"] = df[col_jab_baru].where(~mask_empty, df[col_jab_lama])\n",
    "    print(f\"   Kolom jabatan digabung: {mask_empty.sum()} baris fallback ke jabatan lama\")\n",
    "else:\n",
    "    df[\"jabatan_final\"] = df[col_jab_lama]\n",
    "    print(\"   Menggunakan kolom jabatan lama (jabatan_terupdate tidak ditemukan)\")\n",
    "\n",
    "col_jabatan = \"jabatan_final\"\n",
    "\n",
    "before_filter = len(df)\n",
    "if col_status:\n",
    "    blacklist = [\"tidah diketahui\", \"tidak bekerja\", \"pelajar\", \"melanjutkan pendidikan\", \"nan\", \"\"]\n",
    "    mask = ~df[col_status].str.lower().str.strip().isin(blacklist)\n",
    "    df = df[mask].copy()\n",
    "    print(f\"   Filter status kerja: {before_filter - len(df)} baris dihapus -> tersisa {len(df)} baris\")\n",
    "else:\n",
    "    print(\"   Kolom status tidak ditemukan, tidak ada filter status.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "65797b5c",
   "metadata": {},
   "source": [
    "## 5. Pembersihan Teks (Stemming & Stopword Removal)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "bfc2abf6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2/4] Menjalankan pembersihan teks (ini bisa memakan waktu beberapa menit)...\n",
      "   Selesai. Contoh hasil cleaning:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Jabatan Asli</th>\n",
       "      <th>Setelah Cleaning</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CUSTOMER SERVICE</td>\n",
       "      <td>customer service</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Staff Administrasi Produksi</td>\n",
       "      <td>staff administrasi produksi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Customer Service Staff</td>\n",
       "      <td>customer service staff</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Web Developer</td>\n",
       "      <td>web developer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Human Resources Development Officer</td>\n",
       "      <td>human resources development officer</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           Jabatan Asli                     Setelah Cleaning\n",
       "1                      CUSTOMER SERVICE                     customer service\n",
       "5           Staff Administrasi Produksi          staff administrasi produksi\n",
       "8                Customer Service Staff               customer service staff\n",
       "9                         Web Developer                        web developer\n",
       "13  Human Resources Development Officer  human resources development officer"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"[2/4] Menjalankan pembersihan teks (ini bisa memakan waktu beberapa menit)...\")\n",
    "df[\"job_text_raw\"] = df[col_jabatan].apply(clean_text)\n",
    "print(f\"   Selesai. Contoh hasil cleaning:\")\n",
    "\n",
    "sample = df[[col_jabatan, \"job_text_raw\"]].dropna().head(5)\n",
    "sample.columns = [\"Jabatan Asli\", \"Setelah Cleaning\"]\n",
    "sample"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "377a080f",
   "metadata": {},
   "source": [
    "## 6. Fallback Teks & Deteksi Kebocoran Nama"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "7ffe70c0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Fallback teks dari kolom klasifikasi: 146 baris\n",
      "   Mendeteksi kebocoran nama pribadi...\n",
      "   Terdeteksi 0 baris dengan indikasi nama pribadi -> dihapus.\n"
     ]
    }
   ],
   "source": [
    "if col_klasifikasi:\n",
    "    fallback_map = {\n",
    "        \"Programmer\": \"programmer developer\", \"Data Analyst\": \"data analyst\",\n",
    "        \"Wirausaha IT\": \"wirausaha founder\", \"Wirausaha\": \"wirausaha founder\",\n",
    "        \"Non IT\": \"staff admin\", \"Infokom\": \"it staff teknisi\",\n",
    "        \"TIdah diketahui\": \"\", \"Pelajar\": \"\", \"Tidak Bekerja\": \"\"\n",
    "    }\n",
    "    empty_mask = df[\"job_text_raw\"] == \"\"\n",
    "    if empty_mask.any():\n",
    "        df.loc[empty_mask, \"job_text_raw\"] = df.loc[empty_mask, col_klasifikasi].map(fallback_map).fillna(\"\")\n",
    "        print(f\"   Fallback teks dari kolom klasifikasi: {empty_mask.sum()} baris\")\n",
    "    else:\n",
    "        print(\"   Tidak ada baris dengan teks kosong setelah cleaning.\")\n",
    "\n",
    "if col_nama:\n",
    "    print(\"   Mendeteksi kebocoran nama pribadi...\")\n",
    "    name_leak_mask = df.apply(lambda row: is_likely_name(row[\"job_text_raw\"], row[col_nama]), axis=1)\n",
    "    leaked_count = name_leak_mask.sum()\n",
    "    print(f\"   Terdeteksi {leaked_count} baris dengan indikasi nama pribadi -> dihapus.\")\n",
    "    df = df[~name_leak_mask].copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f565470",
   "metadata": {},
   "source": [
    "## 7. Pelabelan Kelas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "e8b6bbdf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3/4] Pelabelan kelas...\n",
      "   350 baris dilabeli dengan rule-based fallback.\n",
      "\n",
      "Total data valid: 358 baris\n",
      "\n",
      "Distribusi label keseluruhan:\n",
      "label\n",
      "Wirausaha Informatika    128\n",
      "Non-IT                   124\n",
      "Programmer                98\n",
      "Data Analyst               8\n"
     ]
    }
   ],
   "source": [
    "print(\"[3/4] Pelabelan kelas...\")\n",
    "\n",
    "if col_klasifikasi:\n",
    "    df[\"label\"] = df[col_klasifikasi].str.strip().map(KLASIFIKASI_MAP)\n",
    "    missing = df[\"label\"].isna()\n",
    "    if missing.any():\n",
    "        df.loc[missing, \"label\"] = df.loc[missing, \"job_text_raw\"].apply(classify_rule_based)\n",
    "        print(f\"   {missing.sum()} baris dilabeli dengan rule-based fallback.\")\n",
    "else:\n",
    "    df[\"label\"] = df[\"job_text_raw\"].apply(classify_rule_based)\n",
    "    print(\"   Semua baris dilabeli dengan rule-based (tidak ada kolom klasifikasi).\")\n",
    "\n",
    "df = df[df[\"job_text_raw\"].str.len() >= 3].copy()\n",
    "result = (\n",
    "    df[[col_nim, col_jabatan, \"job_text_raw\", \"label\"]]\n",
    "    .rename(columns={col_nim: \"nim\"})\n",
    "    .dropna(subset=[\"nim\", \"label\"])\n",
    "    .drop_duplicates(subset=[\"nim\"], keep=\"first\")\n",
    ")\n",
    "\n",
    "print(f\"\\nTotal data valid: {len(result)} baris\")\n",
    "print(\"\\nDistribusi label keseluruhan:\")\n",
    "print(result[\"label\"].value_counts().to_string())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0c803ba9",
   "metadata": {},
   "source": [
    "## 8. Train/Test Split & Simpan Output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "5849e195",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[4/4] Train/Test Split (70:30, stratified)...\n",
      "   Training set : 250 baris -> d:\\Project\\sista_mif_ta\\data\\processed\\training_corpus.csv\n",
      "   Test set     : 108 baris -> d:\\Project\\sista_mif_ta\\data\\processed\\test_set.csv\n",
      "\n",
      "DATA PENGUJIAN (TEST SET):\n",
      "  Programmer                : 30\n",
      "  Data Analyst              : 2 < 5\n",
      "  Wirausaha Informatika     : 39\n",
      "  Non-IT                    : 37\n"
     ]
    }
   ],
   "source": [
    "print(\"[4/4] Train/Test Split (70:30, stratified)...\")\n",
    "\n",
    "min_count = result[\"label\"].value_counts().min()\n",
    "if min_count < 2:\n",
    "    train_df, test_df = train_test_split(result, test_size=0.30, random_state=42)\n",
    "else:\n",
    "    train_df, test_df = train_test_split(result, test_size=0.30, stratify=result[\"label\"], random_state=42)\n",
    "\n",
    "cols_out = [\"nim\", \"job_text_raw\", \"label\"]\n",
    "train_df[cols_out].to_csv(OUTPUT_DIR / \"training_corpus.csv\", index=False, sep=\";\", encoding=ENCODING)\n",
    "test_df[cols_out].to_csv(OUTPUT_DIR / \"test_set.csv\", index=False, sep=\";\", encoding=ENCODING)\n",
    "\n",
    "print(f\"   Training set : {len(train_df)} baris -> {OUTPUT_DIR / 'training_corpus.csv'}\")\n",
    "print(f\"   Test set     : {len(test_df)} baris -> {OUTPUT_DIR / 'test_set.csv'}\")\n",
    "\n",
    "print(\"\\nDATA PENGUJIAN (TEST SET):\")\n",
    "for cls in TARGET_CLASSES:\n",
    "    c = test_df[\"label\"].value_counts().get(cls, 0)\n",
    "    flag = \" < 5\" if c < 5 else \"\"\n",
    "    print(f\"  {cls:25} : {c}{flag}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ea3f4559",
   "metadata": {},
   "source": [
    "## 9. Visualisasi Distribusi Kelas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "17be8acb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 61048 characters]",
      "text/plain": [
       "<Figure size 1600x500 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "COLORS = [\"#4C72B0\", \"#DD8452\", \"#55A868\", \"#C44E52\"]\n",
    "\n",
    "fig, axes = plt.subplots(1, 3, figsize=(16, 5))\n",
    "fig.suptitle(\"Distribusi Kelas Profil Karir Alumni MIF\", fontsize=14, fontweight=\"bold\", y=1.02)\n",
    "\n",
    "for ax, (df_plot, title) in zip(axes, [\n",
    "    (result, \"Keseluruhan\"),\n",
    "    (train_df, \"Training Set (70%)\"),\n",
    "    (test_df, \"Test Set (30%)\"),\n",
    "]):\n",
    "    counts = df_plot[\"label\"].value_counts().reindex(TARGET_CLASSES, fill_value=0)\n",
    "    bars = ax.bar(counts.index, counts.values, color=COLORS, edgecolor=\"white\", linewidth=0.8)\n",
    "    ax.set_title(title, fontsize=12, fontweight=\"bold\")\n",
    "    ax.set_ylabel(\"Jumlah Data\")\n",
    "    ax.set_ylim(0, counts.max() * 1.2)\n",
    "    ax.tick_params(axis=\"x\", rotation=15)\n",
    "    ax.yaxis.set_major_locator(mticker.MaxNLocator(integer=True))\n",
    "    for bar, val in zip(bars, counts.values):\n",
    "        ax.text(bar.get_x() + bar.get_width() / 2, bar.get_height() + 0.5,\n",
    "                str(val), ha=\"center\", va=\"bottom\", fontsize=10, fontweight=\"bold\")\n",
    "    ax.spines[[\"top\", \"right\"]].set_visible(False)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "cb8fd260",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 73804 characters]",
      "text/plain": [
       "<Figure size 1200x500 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Corpus berhasil disiapkan dan disimpan.\n"
     ]
    }
   ],
   "source": [
    "fig, ax = plt.subplots(1, 2, figsize=(12, 5))\n",
    "fig.suptitle(\"Proporsi Kelas (Pie Chart)\", fontsize=13, fontweight=\"bold\")\n",
    "\n",
    "for i, (df_plot, title) in enumerate([(train_df, \"Training Set\"), (test_df, \"Test Set\")]):\n",
    "    counts = df_plot[\"label\"].value_counts().reindex(TARGET_CLASSES, fill_value=0)\n",
    "    ax[i].pie(\n",
    "        counts.values,\n",
    "        labels=counts.index,\n",
    "        autopct=\"%1.1f%%\",\n",
    "        colors=COLORS,\n",
    "        startangle=140,\n",
    "        wedgeprops={\"edgecolor\": \"white\", \"linewidth\": 1.5}\n",
    "    )\n",
    "    ax[i].set_title(title, fontsize=11, fontweight=\"bold\")\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()\n",
    "\n",
    "print(\"\\nCorpus berhasil disiapkan dan disimpan.\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
