469 lines
17 KiB
Python
469 lines
17 KiB
Python
# ============================================================
|
|
# PIPELINE PREPROCESSING LENGKAP
|
|
# ============================================================
|
|
# Fungsi: Pipeline preprocessing data tracer study alumni
|
|
# dari import Excel sampai siap untuk ML
|
|
# ============================================================
|
|
|
|
import pandas as pd
|
|
import numpy as np
|
|
import re
|
|
from typing import List, Dict, Optional, Tuple
|
|
from pathlib import Path
|
|
|
|
# Import modul standarisasi
|
|
from .standarisasi_kolom import (
|
|
KolomStandarisasi,
|
|
gabung_data,
|
|
bersihkan_data,
|
|
COLUMN_MAPPING
|
|
)
|
|
|
|
# Import text preprocessing
|
|
from .text_preprocessing import preprocess_text
|
|
|
|
|
|
# ============================================================
|
|
# KELAS: DataPipeline
|
|
# ============================================================
|
|
# Pipeline lengkap untuk memproses data tracer study
|
|
# dari berbagai tahun menjadi data siap ML
|
|
# ============================================================
|
|
|
|
class DataPipeline:
|
|
"""
|
|
Pipeline lengkap untuk preprocessing data tracer study.
|
|
|
|
ALUR KERJA:
|
|
============
|
|
|
|
┌─────────────────────────────────────────────────────────┐
|
|
│ INPUT: File Excel dari berbagai tahun (2019-2025) │
|
|
└─────────────────────────────────────────────────────────┘
|
|
│
|
|
▼
|
|
┌─────────────────────────────────────────────────────────┐
|
|
│ STEP 1: STANDARISASI KOLOM │
|
|
│ - Ubah nama kolom berbeda → kolom standar │
|
|
│ - Kolom standar: nama, perusahaan, jabatan, nim │
|
|
└─────────────────────────────────────────────────────────┘
|
|
│
|
|
▼
|
|
┌─────────────────────────────────────────────────────────┐
|
|
│ STEP 2: GABUNG DATA │
|
|
│ - Gabungkan data dari berbagai tahun │
|
|
│ - Tambahkan kolom 'tahun' │
|
|
└─────────────────────────────────────────────────────────┘
|
|
│
|
|
▼
|
|
┌─────────────────────────────────────────────────────────┐
|
|
│ STEP 3: DATA CLEANING │
|
|
│ - Hapus duplikat │
|
|
│ - Hapus data kosong │
|
|
│ - Handle missing values │
|
|
└─────────────────────────────────────────────────────────┘
|
|
│
|
|
▼
|
|
┌─────────────────────────────────────────────────────────┐
|
|
│ STEP 4: TEXT PREPROCESSING │
|
|
│ - Case folding (lowercase) │
|
|
│ - Tokenizing │
|
|
│ - Stopword removal │
|
|
│ - Stemming │
|
|
└─────────────────────────────────────────────────────────┘
|
|
│
|
|
▼
|
|
┌─────────────────────────────────────────────────────────┐
|
|
│ OUTPUT: Data siap untuk TF-IDF & Clustering │
|
|
└─────────────────────────────────────────────────────────┘
|
|
|
|
CONTOH PENGGUNAAN:
|
|
==================
|
|
>>> pipeline = DataPipeline()
|
|
>>> pipeline.tambah_data(df_2019, "2019")
|
|
>>> pipeline.tambah_data(df_2024, "2024")
|
|
>>> result = pipeline.jalankan()
|
|
>>> print(result.head())
|
|
"""
|
|
|
|
def __init__(self):
|
|
"""Inisialisasi pipeline dengan komponen-komponen yang dibutuhkan."""
|
|
# Simpan data mentah yang belum diproses
|
|
self.data_list: List[Tuple[pd.DataFrame, str]] = []
|
|
|
|
# Komponen standarisasi
|
|
self.standarisasi = KolomStandarisasi()
|
|
|
|
# Status pipeline
|
|
self._sudah_standarisasi = False
|
|
self._sudah_gabung = False
|
|
self._sudah_bersih = False
|
|
self._sudah_preprocess = False
|
|
|
|
# Data hasil processing (disimpan untuk debugging)
|
|
self._df_standar = None
|
|
self._df_gabungan = None
|
|
self._df_bersih = None
|
|
self._df_final = None
|
|
|
|
print("🚀 DataPipeline initialized")
|
|
|
|
# ========================================================
|
|
# METODE: tambah_data
|
|
# ========================================================
|
|
def tambah_data(self, df: pd.DataFrame, tahun: str,
|
|
nama_file: str = None) -> 'DataPipeline':
|
|
"""
|
|
Tambahkan data dari satu tahun.
|
|
|
|
Args:
|
|
df: DataFrame dari file Excel
|
|
tahun: Tahun data (2019-2025)
|
|
nama_file: Nama file (untuk auto-detect)
|
|
|
|
Returns:
|
|
self (untuk method chaining)
|
|
|
|
Contoh:
|
|
=======
|
|
>>> pipeline = DataPipeline()
|
|
>>> pipeline.tambah_data(df_2019, "2019")
|
|
>>> pipeline.tambah_data(df_2024, "2024")
|
|
"""
|
|
# Validasi tahun
|
|
tahun_tersedia = self.standarisasi.get_tahun()
|
|
if tahun not in tahun_tersedia:
|
|
print(f"⚠️ Warning: Tahun {tahun} belum ada di mapping")
|
|
print(f" Mapping akan ditambahkan otomatis")
|
|
|
|
self.data_list.append((df, tahun))
|
|
print(f" ✅ Data {tahun} ditambahkan ({len(df)} records)")
|
|
|
|
return self
|
|
|
|
# ========================================================
|
|
# METODE: jalankan
|
|
# ========================================================
|
|
def jalankan(self, bersihkan: bool = True,
|
|
text_preprocess: bool = True) -> pd.DataFrame:
|
|
"""
|
|
Jalankan seluruh pipeline.
|
|
|
|
Args:
|
|
bersihkan: Apakah akan melakukan data cleaning
|
|
text_preprocess: Apakah akan melakukan text preprocessing
|
|
|
|
Returns:
|
|
DataFrame yang sudah diproses
|
|
|
|
ALUR:
|
|
=====
|
|
1. Standarisasi setiap data
|
|
2. Gabungkan semua data
|
|
3. Bersihkan data
|
|
4. Text preprocessing
|
|
"""
|
|
print("\n" + "=" * 60)
|
|
print("🚀 MEMULAI PIPELINE")
|
|
print("=" * 60)
|
|
|
|
# ================================================
|
|
# STEP 1: Standarisasi
|
|
# ================================================
|
|
print("\n📋 STEP 1: Standarisasi Kolom")
|
|
print("-" * 40)
|
|
|
|
df_standar_list = []
|
|
|
|
for df, tahun in self.data_list:
|
|
print(f"\n Memproses tahun {tahun}...")
|
|
try:
|
|
df_std = self.standarisasi.standarisasi(df, tahun=tahun)
|
|
df_standar_list.append(df_std)
|
|
print(f" ✅ {tahun}: {len(df_std)} records")
|
|
except Exception as e:
|
|
print(f" ❌ {tahun}: Error - {e}")
|
|
|
|
if not df_standar_list:
|
|
raise ValueError("Tidak ada data yang berhasil diproses")
|
|
|
|
self._df_standar = df_standar_list
|
|
self._sudah_standarisasi = True
|
|
|
|
# ================================================
|
|
# STEP 2: Gabung Data
|
|
# ================================================
|
|
print("\n📋 STEP 2: Gabung Data")
|
|
print("-" * 40)
|
|
|
|
tahun_list = [tahun for _, tahun in self.data_list]
|
|
self._df_gabungan = gabung_data(df_standar_list, tahun_list)
|
|
self._sudah_gabung = True
|
|
|
|
print(f" Total records setelah digabung: {len(self._df_gabungan)}")
|
|
|
|
# ================================================
|
|
# STEP 3: Cleaning
|
|
# ================================================
|
|
if bersihkan:
|
|
print("\n📋 STEP 3: Data Cleaning")
|
|
print("-" * 40)
|
|
|
|
self._df_bersih = bersihkan_data(self._df_gabungan)
|
|
self._sudah_bersih = True
|
|
else:
|
|
self._df_bersih = self._df_gabungan
|
|
|
|
# ================================================
|
|
# STEP 4: Text Preprocessing
|
|
# ================================================
|
|
if text_preprocess:
|
|
print("\n📋 STEP 4: Text Preprocessing")
|
|
print("-" * 40)
|
|
|
|
self._df_final = self._proses_teks(self._df_bersih)
|
|
self._sudah_preprocess = True
|
|
else:
|
|
self._df_final = self._df_bersih
|
|
|
|
# ================================================
|
|
# SELESAI
|
|
# ================================================
|
|
print("\n" + "=" * 60)
|
|
print("✅ PIPELINE SELESAI")
|
|
print("=" * 60)
|
|
print(f" Total records final: {len(self._df_final)}")
|
|
print(f" Kolom: {self._df_final.columns.tolist()}")
|
|
|
|
return self._df_final
|
|
|
|
# ========================================================
|
|
# METODE: _proses_teks
|
|
# ========================================================
|
|
def _proses_teks(self, df: pd.DataFrame) -> pd.DataFrame:
|
|
"""
|
|
Proses kolom teks untuk ML.
|
|
|
|
MELAKUKAN:
|
|
==========
|
|
1. Gabungkan kolom perusahaan + jabatan menjadi satu teks
|
|
2. Apply text preprocessing (lowercase, tokenize, dll)
|
|
3. Simpan hasil di kolom 'teks_bersih'
|
|
"""
|
|
print(" Memproses teks...")
|
|
|
|
df = df.copy()
|
|
|
|
# ================================================
|
|
# LANGKAH 1: Gabungkan teks
|
|
# ================================================
|
|
# Gabungkan "perusahaan" dan "jabatan" menjadi satu teks
|
|
# Ini akan menjadi input untuk TF-IDF
|
|
|
|
# Isi nilai kosong dengan string kosong
|
|
df['perusahaan'] = df['perusahaan'].fillna('')
|
|
df['jabatan'] = df['jabatan'].fillna('')
|
|
|
|
# Gabungkan dengan spasi
|
|
df['teks_gabungan'] = df['perusahaan'] + ' ' + df['jabatan']
|
|
|
|
# ================================================
|
|
# LANGKAH 2: Text Preprocessing
|
|
# ================================================
|
|
# Proses setiap teks dengan fungsi dari text_preprocessing.py
|
|
|
|
def proses(teks):
|
|
if not teks or teks.strip() == '':
|
|
return ''
|
|
return preprocess_text(teks)
|
|
|
|
df['teks_bersih'] = df['teks_gabungan'].apply(proses)
|
|
|
|
# ================================================
|
|
# LANGKAH 3: Hapus kolom bantu
|
|
# ================================================
|
|
df = df.drop(columns=['teks_gabungan'])
|
|
|
|
print(f" ✅ Teks berhasil diproses")
|
|
|
|
return df
|
|
|
|
# ========================================================
|
|
# METODE: get_statistik
|
|
# ========================================================
|
|
def get_statistik(self) -> Dict:
|
|
"""
|
|
Dapatkan statistik dari pipeline.
|
|
|
|
Returns:
|
|
Dictionary dengan statistik
|
|
"""
|
|
stats = {
|
|
"jumlah_tahun": len(self.data_list),
|
|
"tahun": [tahun for _, tahun in self.data_list],
|
|
"total_records": 0,
|
|
"records_per_tahun": {},
|
|
"kolom_standar": []
|
|
}
|
|
|
|
if self._df_gabungan is not None:
|
|
stats["total_records"] = len(self._df_gabungan)
|
|
|
|
# Hitung per tahun
|
|
if 'tahun' in self._df_gabungan.columns:
|
|
stats["records_per_tahun"] = (
|
|
self._df_gabungan['tahun']
|
|
.value_counts()
|
|
.to_dict()
|
|
)
|
|
|
|
if self._df_final is not None:
|
|
stats["kolom_standar"] = self._df_final.columns.tolist()
|
|
|
|
return stats
|
|
|
|
# ========================================================
|
|
# METODE: reset
|
|
# ========================================================
|
|
def reset(self) -> 'DataPipeline':
|
|
"""
|
|
Reset pipeline ke kondisi awal.
|
|
|
|
Returns:
|
|
self (untuk method chaining)
|
|
"""
|
|
self.data_list = []
|
|
self._sudah_standarisasi = False
|
|
self._sudah_gabung = False
|
|
self._sudah_bersih = False
|
|
self._sudah_preprocess = False
|
|
self._df_standar = None
|
|
self._df_gabungan = None
|
|
self._df_bersih = None
|
|
self._df_final = None
|
|
|
|
print("🔄 Pipeline reset")
|
|
|
|
return self
|
|
|
|
|
|
# ============================================================
|
|
# FUNGSI: proses_file_excel
|
|
# ============================================================
|
|
def proses_file_excel(file_paths: List[str],
|
|
tahun_list: List[str]) -> pd.DataFrame:
|
|
"""
|
|
Proses multiple file Excel sekaligus.
|
|
|
|
Args:
|
|
file_paths: List path ke file Excel
|
|
tahun_list: List tahun untuk setiap file
|
|
|
|
Returns:
|
|
DataFrame yang sudah diproses
|
|
|
|
Contoh:
|
|
=======
|
|
>>> files = ["data_2019.xlsx", "data_2024.xlsx"]
|
|
>>> tahun = ["2019", "2024"]
|
|
>>> result = proses_file_excel(files, tahun)
|
|
"""
|
|
# Validasi
|
|
if len(file_paths) != len(tahun_list):
|
|
raise ValueError(
|
|
f"Jumlah file ({len(file_paths)}) "
|
|
f"tidak sama dengan jumlah tahun ({len(tahun_list)})"
|
|
)
|
|
|
|
print("=" * 60)
|
|
print("MEMPROSES FILE EXCEL")
|
|
print("=" * 60)
|
|
|
|
# Inisialisasi pipeline
|
|
pipeline = DataPipeline()
|
|
|
|
# Load dan tambah setiap file
|
|
for file_path, tahun in zip(file_paths, tahun_list):
|
|
print(f"\n📂 Membaca: {file_path}")
|
|
df = pd.read_excel(file_path)
|
|
pipeline.tambah_data(df, tahun)
|
|
|
|
# Jalankan pipeline
|
|
result = pipeline.jalankan()
|
|
|
|
# Tampilkan statistik
|
|
stats = pipeline.get_statistik()
|
|
print("\n" + "=" * 60)
|
|
print("STATISTIK")
|
|
print("=" * 60)
|
|
print(f"Tahun yang diproses: {stats['tahun']}")
|
|
print(f"Total records: {stats['total_records']}")
|
|
print(f"Records per tahun: {stats['records_per_tahun']}")
|
|
|
|
return result
|
|
|
|
|
|
# ============================================================
|
|
# CONTOH PENGGUNAAN LENGKAP
|
|
# ============================================================
|
|
if __name__ == "__main__":
|
|
print("=" * 60)
|
|
print("CONTOH PENGGUNAAN DATA PIPELINE")
|
|
print("=" * 60)
|
|
|
|
# ================================================
|
|
# Buat data dummy untuk setiap tahun
|
|
# ================================================
|
|
|
|
# Data 2019
|
|
df_2019 = pd.DataFrame({
|
|
# Mapping 2019 - GANTI dengan nama kolom sebenarnya
|
|
"Nama Lengkap": ["Andi Wijaya", "Budi Cahya"],
|
|
"Perusahaan": ["Google", "Microsoft"],
|
|
"Jabatan": ["Engineer", "Manager"],
|
|
"NIM": ["E41190001", "E41190002"]
|
|
})
|
|
|
|
# Data 2024 (sudah diketahui)
|
|
df_2024 = pd.DataFrame({
|
|
"4. Nama Lengkap / nmmhsmsmh (Gunakan Huruf Besar)": [
|
|
"Citra Dewi", "Dewi Lestari"
|
|
],
|
|
"9. Apa nama perusahaan/kantor tempat Anda bekerja? f5b": [
|
|
"Tokopedia", "Gojek"
|
|
],
|
|
"10. Bila Anda berwiraswasta, apa posisi/jabatan Anda saat ini ? f5c": [
|
|
"Data Analyst", "Product Manager"
|
|
],
|
|
"3. NIM / nimhsmsmh (Gunakan Huruf Besar) ": [
|
|
"E41240001", "E41240002"
|
|
]
|
|
})
|
|
|
|
# ================================================
|
|
# Jalankan Pipeline
|
|
# ================================================
|
|
print("\n📊 Data yang akan diproses:")
|
|
print(f" - 2019: {len(df_2019)} records")
|
|
print(f" - 2024: {len(df_2024)} records")
|
|
|
|
# Inisialisasi pipeline
|
|
pipeline = DataPipeline()
|
|
|
|
# Tambah data
|
|
pipeline.tambah_data(df_2019, "2019")
|
|
pipeline.tambah_data(df_2024, "2024")
|
|
|
|
# Jalankan
|
|
result = pipeline.jalankan()
|
|
|
|
# Tampilkan hasil
|
|
print("\n📋 HASIL AKHIR:")
|
|
print(result)
|
|
|
|
# Tampilkan statistik
|
|
stats = pipeline.get_statistik()
|
|
print("\n📈 STATISTIK:")
|
|
for key, value in stats.items():
|
|
print(f" {key}: {value}")
|