220 lines
6.8 KiB
Python
220 lines
6.8 KiB
Python
"""
|
|
Main ML Pipeline
|
|
Pipeline utama untuk clustering alumni
|
|
"""
|
|
|
|
import pandas as pd
|
|
import numpy as np
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from .preprocessing.text_preprocessing import preprocess_text
|
|
from .feature_engineering.tfidf import TfidfExtractor
|
|
from .clustering.kmeans import KMeansClustering, find_optimal_k, get_cluster_summary
|
|
|
|
|
|
class AlumniClusteringPipeline:
|
|
"""
|
|
Pipeline utama untuk clustering alumni
|
|
|
|
Input: DataFrame dengan kolom ['nama', 'perusahaan', 'jabatan']
|
|
Output: DataFrame dengan kolom tambahan 'cluster'
|
|
"""
|
|
|
|
def __init__(self, n_clusters: int = 3, auto_k: bool = False):
|
|
"""
|
|
Args:
|
|
n_clusters: Jumlah cluster (diabaikan jika auto_k=True)
|
|
auto_k: Jika True, cari k optimal secara otomatis
|
|
"""
|
|
self.n_clusters = n_clusters
|
|
self.auto_k = auto_k
|
|
self.tfidf = None
|
|
self.kmeans = None
|
|
self.feature_names = None
|
|
|
|
def _combine_text(self, df: pd.DataFrame) -> list:
|
|
"""Menggabungkan kolom teks menjadi satu"""
|
|
combined = df['perusahaan'].fillna('') + ' ' + df['jabatan'].fillna('')
|
|
return combined.tolist()
|
|
|
|
def _preprocess_all(self, texts: list) -> list:
|
|
"""Preprocess semua teks"""
|
|
return [preprocess_text(t) for t in texts]
|
|
|
|
def fit(self, df: pd.DataFrame) -> 'AlumniClusteringPipeline':
|
|
"""
|
|
Fit pipeline ke data training
|
|
|
|
Args:
|
|
df: DataFrame dengan kolom ['nama', 'perusahaan', 'jabatan']
|
|
|
|
Returns:
|
|
self
|
|
"""
|
|
# Combine text
|
|
combined_texts = self._combine_text(df)
|
|
|
|
# Preprocess
|
|
processed_texts = self._preprocess_all(combined_texts)
|
|
|
|
# TF-IDF
|
|
self.tfidf = TfidfExtractor(max_features=500)
|
|
feature_matrix = self.tfidf.fit_transform(processed_texts)
|
|
self.feature_names = self.tfidf.get_feature_names()
|
|
|
|
# K-Means
|
|
n_clusters = self.n_clusters
|
|
|
|
if self.auto_k:
|
|
# Find optimal k
|
|
results = find_optimal_k(feature_matrix)
|
|
# Ambil k dengan silhouette score tertinggi
|
|
best_idx = np.argmax(results['silhouette'])
|
|
n_clusters = results['k'][best_idx]
|
|
print(f"Optimal K selected: {n_clusters}")
|
|
|
|
self.kmeans = KMeansClustering(n_clusters=n_clusters)
|
|
self.kmeans.fit(feature_matrix)
|
|
|
|
return self
|
|
|
|
def transform(self, df: pd.DataFrame) -> pd.DataFrame:
|
|
"""
|
|
Transform data dengan model yang sudah di-fit
|
|
|
|
Args:
|
|
df: DataFrame dengan kolom ['nama', 'perusahaan', 'jabatan']
|
|
|
|
Returns:
|
|
DataFrame dengan kolom 'cluster' dan 'cluster_label'
|
|
"""
|
|
if self.kmeans is None:
|
|
raise ValueError("Model belum di-fit. Jalankan fit() terlebih dahulu.")
|
|
|
|
# Combine text
|
|
combined_texts = self._combine_text(df)
|
|
|
|
# Preprocess
|
|
processed_texts = self._preprocess_all(combined_texts)
|
|
|
|
# TF-IDF transform
|
|
feature_matrix = self.tfidf.transform(processed_texts)
|
|
|
|
# Predict
|
|
labels = self.kmeans.predict(feature_matrix)
|
|
|
|
# Add to DataFrame
|
|
result = df.copy()
|
|
result['cluster'] = labels
|
|
result['cluster_label'] = [f"Cluster_{i}" for i in labels]
|
|
|
|
return result
|
|
|
|
def fit_transform(self, df: pd.DataFrame) -> pd.DataFrame:
|
|
"""Fit dan transform sekaligus"""
|
|
return self.fit(df).transform(df)
|
|
|
|
def get_cluster_summary(self, df: pd.DataFrame) -> dict:
|
|
"""Mendapatkan ringkasan cluster"""
|
|
if 'cluster' not in df.columns:
|
|
raise ValueError("Data belum di-cluster. Jalankan fit_transform() terlebih dahulu.")
|
|
return get_cluster_summary(df, df['cluster'].values)
|
|
|
|
def get_top_features(self, n: int = 10) -> dict:
|
|
"""Mendapatkan fitur teratas untuk setiap cluster"""
|
|
if self.kmeans is None or self.tfidf is None:
|
|
raise ValueError("Model belum di-fit.")
|
|
|
|
cluster_centers = self.kmeans.model.cluster_centers_
|
|
feature_names = self.feature_names
|
|
|
|
top_features = {}
|
|
for i, center in enumerate(cluster_centers):
|
|
# Get indices of top features
|
|
top_indices = center.argsort()[-n:][::-1]
|
|
top_features[f"Cluster_{i}"] = [
|
|
{"term": feature_names[idx], "weight": float(center[idx])}
|
|
for idx in top_indices
|
|
]
|
|
|
|
return top_features
|
|
|
|
def save_model(self, filepath: str):
|
|
"""Simpan model (placeholder - bisa di-extend dengan joblib)"""
|
|
model_data = {
|
|
'n_clusters': self.n_clusters,
|
|
'feature_names': self.feature_names
|
|
}
|
|
with open(filepath, 'w') as f:
|
|
json.dump(model_data, f)
|
|
|
|
@staticmethod
|
|
def load_model(filepath: str):
|
|
"""Load model (placeholder)"""
|
|
with open(filepath, 'r') as f:
|
|
model_data = json.load(f)
|
|
|
|
pipeline = AlumniClusteringPipeline(n_clusters=model_data['n_clusters'])
|
|
return pipeline
|
|
|
|
|
|
def run_pipeline(data_path: str = None, n_clusters: int = 3, output_path: str = None):
|
|
"""
|
|
Jalankan pipeline dari command line
|
|
|
|
Args:
|
|
data_path: Path ke file CSV input
|
|
n_clusters: Jumlah cluster
|
|
output_path: Path untuk menyimpan hasil
|
|
"""
|
|
# Load data
|
|
if data_path:
|
|
df = pd.read_csv(data_path)
|
|
else:
|
|
# Sample data untuk testing
|
|
df = pd.DataFrame({
|
|
'nama': ['Ali', 'Budi', 'Citra', 'Dewi', 'Eko'],
|
|
'perusahaan': ['Google', 'Tokopedia', 'Gojek', 'Traveloka', 'Bukalapak'],
|
|
'jabatan': ['Software Engineer', 'Data Analyst', 'Product Manager', 'UX Designer', 'DevOps']
|
|
})
|
|
|
|
print(f"Data loaded: {len(df)} records")
|
|
|
|
# Run pipeline
|
|
pipeline = AlumniClusteringPipeline(n_clusters=n_clusters)
|
|
result = pipeline.fit_transform(df)
|
|
|
|
print(f"\nClustering complete!")
|
|
print(f"\nCluster Distribution:")
|
|
print(result['cluster_label'].value_counts())
|
|
|
|
print(f"\nCluster Summary:")
|
|
summary = pipeline.get_cluster_summary(result)
|
|
for cluster, info in summary.items():
|
|
print(f" {cluster}: {info['count']} alumni ({info['percentage']}%)")
|
|
|
|
print(f"\nTop Features per Cluster:")
|
|
top_features = pipeline.get_top_features()
|
|
for cluster, features in top_features.items():
|
|
print(f" {cluster}: {[f['term'] for f in features[:5]]}")
|
|
|
|
# Save result
|
|
if output_path:
|
|
result.to_csv(output_path, index=False)
|
|
print(f"\nResult saved to: {output_path}")
|
|
|
|
return result
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import sys
|
|
|
|
if len(sys.argv) > 1:
|
|
data_path = sys.argv[1]
|
|
n_clusters = int(sys.argv[2]) if len(sys.argv) > 2 else 3
|
|
output_path = sys.argv[3] if len(sys.argv) > 3 else None
|
|
run_pipeline(data_path, n_clusters, output_path)
|
|
else:
|
|
run_pipeline()
|