{ "cells": [ { "cell_type": "markdown", "id": "73b2b3ae", "metadata": {}, "source": [ "# Persiapan Corpus — SISTA MIF" ] }, { "cell_type": "markdown", "id": "fd472692", "metadata": {}, "source": [ "## 1. Import Library & Konfigurasi" ] }, { "cell_type": "code", "execution_count": 4, "id": "ed4261a0", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Konfigurasi selesai.\n", " File input : d:\\Project\\sista_mif_ta\\data\\ts_internal_mif.xlsx\n", " Output dir : d:\\Project\\sista_mif_ta\\data\\processed\n" ] } ], "source": [ "import pandas as pd\n", "import numpy as np\n", "import re\n", "from pathlib import Path\n", "from sklearn.model_selection import train_test_split\n", "from Sastrawi.Stemmer.StemmerFactory import StemmerFactory\n", "import matplotlib.pyplot as plt\n", "import matplotlib.ticker as mticker\n", "\n", "%matplotlib inline\n", "\n", "factory = StemmerFactory()\n", "stemmer = factory.create_stemmer()\n", "\n", "BASE_DIR = Path.cwd()\n", "DATA_DIR = BASE_DIR\n", "OUTPUT_DIR = DATA_DIR / \"processed\"\n", "OUTPUT_DIR.mkdir(exist_ok=True)\n", "\n", "FILE_INTERNAL = DATA_DIR / \"ts_internal_mif.xlsx\"\n", "SEPARATOR = \";\"\n", "ENCODING = \"utf-8-sig\"\n", "TARGET_CLASSES = [\"Programmer\", \"Data Analyst\", \"Wirausaha Informatika\", \"Non-IT\"]\n", "\n", "KLASIFIKASI_MAP = {\n", " \"Programmer\": \"Programmer\",\n", " \"Data Analyst\": \"Data Analyst\",\n", " \"Wirausaha IT\": \"Wirausaha Informatika\",\n", " \"Wirausaha\": \"Wirausaha Informatika\",\n", " \"Non-IT\": \"Non-IT\",\n", " \"Infokom\": None, \"Pelajar\": None, \"Tidak Bekerja\": None, \"TIdak diketahui\": None\n", "}\n", "KEYWORD_RULES = {\n", " \"Programmer\": [\"programmer\", \"developer\", \"engineer\", \"fullstack\", \"backend\", \"frontend\", \"mobile\", \"android\", \"ios\", \"software\", \"web dev\", \"coding\", \"it staff\", \"teknisi\", \"sistem informasi\", \"application\", \"network\", \"devops\", \"qa\", \"tester\", \"ui\", \"ux\", \"swe\"],\n", " \"Data Analyst\": [\"data analyst\", \"analis data\", \"data science\", \"business analyst\", \"research\", \"statistik\", \"bi analyst\", \"reporting\", \"database\", \"sql\", \"etl\", \"data engineer\", \"big data\", \"analyst\", \"data mining\", \"machine learning\", \"data visual\", \"power bi\", \"tableau\", \"looker\", \"business intelligence\", \"bi developer\", \"data warehouse\"],\n", " \"Wirausaha Informatika\": [\"founder\", \"owner\", \"ceo\", \"wiraswasta\", \"startup\", \"freelance\", \"freelancer\", \"wirausaha\", \"bisnis\", \"usaha mandiri\", \"konsultan\", \"co founder\", \"entrepreneur\", \"self employed\", \"owner toko\", \"usaha\", \"dagang online\", \"tokopedia\", \"shopee\", \"dropship\", \"reseller\"]\n", "}\n", "COMPANY_STOPWORDS = {\n", " \"pt\", \"cv\", \"ud\", \"tbk\", \"persero\", \"corp\", \"inc\", \"ltd\", \"koperasi\", \"bumn\", \"bumd\",\n", " \"dinas\", \"kantor\", \"pemkab\", \"pemprov\", \"politeknik\", \"universitas\", \"sekolah\", \"sma\", \"smk\", \"sd\",\n", " \"bank\", \"bpr\", \"rs\", \"rumah sakit\", \"klinik\", \"apotek\", \"hotel\", \"restoran\", \"cafe\", \"toko\", \"konter\",\n", " \"foundation\", \"yayasan\", \"perkumpulan\", \"organisasi\", \"agency\", \"studio\", \"consulting\", \"group\", \"holding\"\n", "}\n", "\n", "print(\"Konfigurasi selesai.\")\n", "print(f\" File input : {FILE_INTERNAL}\")\n", "print(f\" Output dir : {OUTPUT_DIR}\")" ] }, { "cell_type": "markdown", "id": "79c9c79a", "metadata": {}, "source": [ "## 2. Definisi Fungsi Bantu" ] }, { "cell_type": "code", "execution_count": 5, "id": "f880576a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Semua fungsi bantu berhasil didefinisikan.\n" ] } ], "source": [ "def load_data_file(file_path: str) -> pd.DataFrame:\n", " ext = Path(file_path).suffix.lower()\n", " if ext == '.csv':\n", " try:\n", " return pd.read_csv(file_path, sep=SEPARATOR, encoding=ENCODING, dtype=str, on_bad_lines='skip', engine='python')\n", " except UnicodeDecodeError:\n", " return pd.read_csv(file_path, sep=SEPARATOR, encoding='latin1', dtype=str, on_bad_lines='skip', engine='python')\n", " elif ext == '.xlsx':\n", " return pd.read_excel(file_path, dtype=str)\n", " else:\n", " raise ValueError(f\"Format tidak didukung: {ext}\")\n", "\n", "def find_col(df: pd.DataFrame, keywords: list) -> str | None:\n", " for col in df.columns:\n", " col_clean = str(col).strip().lower()\n", " if any(k.strip().lower() in col_clean for k in keywords):\n", " return col\n", " return None\n", "\n", "def clean_text(text: str) -> str:\n", " if pd.isna(text) or str(text).strip().lower() in [\"nan\", \"none\", \"null\", \"-\", \"0\", \"\", \"tidak diisi\", \"tidak diketahui\"]:\n", " return \"\"\n", " text = str(text).strip().lower()\n", " text = re.sub(r'^\\d+\\s*[-:/]\\s*', '', text)\n", " text = text.replace('-', ' ').replace('/', ' ').replace('_', ' ').replace(',', ' ')\n", " text = re.sub(r'[^\\w\\s]', '', text)\n", " text = re.sub(r'\\d+', '', text)\n", " tokens = [w for w in text.split() if w not in COMPANY_STOPWORDS and len(w) >= 3]\n", " return \" \".join([stemmer.stem(w) for w in tokens])\n", "\n", "def is_likely_name(text: str, full_name: str) -> bool:\n", " if not text or not full_name: return False\n", " text_clean = re.sub(r'[^\\w\\s]', '', text.lower())\n", " name_clean = re.sub(r'[^\\w\\s]', '', str(full_name).lower())\n", " text_parts = set(text_clean.split())\n", " name_parts = set(name_clean.split())\n", " if len(text_parts) == 0: return False\n", " if len(text_parts) <= 3 and text_parts.issubset(name_parts):\n", " return True\n", " return False\n", "\n", "def classify_rule_based(text: str) -> str:\n", " if not text or len(text) < 3: return \"Non-IT\"\n", " for profile in [\"Programmer\", \"Data Analyst\", \"Wirausaha Informatika\"]:\n", " if any(kw in text for kw in KEYWORD_RULES[profile]): return profile\n", " return \"Non-IT\"\n", "\n", "print(\"Semua fungsi bantu berhasil didefinisikan.\")" ] }, { "cell_type": "markdown", "id": "ebf96089", "metadata": {}, "source": [ "## 3. Memuat & Membersihkan Data" ] }, { "cell_type": "code", "execution_count": 7, "id": "fa0d1e7f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[1/4] Memuat & Membersihkan Data...\n", " Kolom NIM : nim\n", " Kolom Nama : nama_lengkap\n", " Kolom Jabatan : jabatan\n", " Kolom Jabatan Baru: jabatan_terupdate\n", " Kolom Klasifikasi : klasifikasi_pekerjaan\n", " Kolom Status : status_pekerjaan\n", "\n", " Total baris dimuat: 488\n" ] }, { "data": { "text/html": [ "
| \n", " | nama_lengkap | \n", "nim | \n", "no_telepon | \n", "alamat_domisili | \n", "jurusan | \n", "program_studi | \n", "tahun_masuk | \n", "tahun_lulus | \n", "status_pekerjaan | \n", "... | \n", "jumlah_undangan_wawancara | \n", "masa_tunggu_pra_lulus | \n", "masa_tunggu_pasca_lulus | \n", "total_masa_tunggu | \n", "instansi_terupdate | \n", "jabatan_terupdate | \n", "tahun_bekerja | \n", "status_pekerjaan_terupdate | \n", "linkedin_profile | \n", "sosmed_ig | \n", "|
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | \n", "ABRAHAM FIRDAUS FATHURROSI | \n", "E31170249 | \n", "abrahamfirdaus86@gmail.com | \n", "081217394021 | \n", "NaN | \n", "Teknologi Informasi | \n", "Manajemen Informatika | \n", "2016 | \n", "2019 | \n", "NaN | \n", "... | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "https://www.linkedin.com/in/abraham-firdaus-fa... | \n", "NaN | \n", "
| 1 | \n", "ACHMAD APRILIANDI ALALLAH | \n", "E31170384 | \n", "andypunk64@gmail.com | \n", "082247877665 | \n", "NaN | \n", "Teknologi Informasi | \n", "Manajemen Informatika | \n", "2016 | \n", "2019 | \n", "NaN | \n", "... | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "https://www.linkedin.com/in/achmad-apriliandi-... | \n", "NaN | \n", "
| 2 | \n", "ADHITYA PUTRA WARDHANA | \n", "E31151908 | \n", "adhityaputra1997@gmail.com | \n", "081333682804 | \n", "NaN | \n", "Teknologi Informasi | \n", "Manajemen Informatika | \n", "2016 | \n", "2019 | \n", "NaN | \n", "... | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "https://www.linkedin.com/in/adhitya-putra-ward... | \n", "NaN | \n", "
3 rows × 31 columns
\n", "| \n", " | Jabatan Asli | \n", "Setelah Cleaning | \n", "
|---|---|---|
| 1 | \n", "CUSTOMER SERVICE | \n", "customer service | \n", "
| 5 | \n", "Staff Administrasi Produksi | \n", "staff administrasi produksi | \n", "
| 8 | \n", "Customer Service Staff | \n", "customer service staff | \n", "
| 9 | \n", "Web Developer | \n", "web developer | \n", "
| 13 | \n", "Human Resources Development Officer | \n", "human resources development officer | \n", "