""" Main ML Pipeline Pipeline utama untuk clustering alumni """ import pandas as pd import numpy as np import json from pathlib import Path from .preprocessing.text_preprocessing import preprocess_text from .feature_engineering.tfidf import TfidfExtractor from .clustering.kmeans import KMeansClustering, find_optimal_k, get_cluster_summary class AlumniClusteringPipeline: """ Pipeline utama untuk clustering alumni Input: DataFrame dengan kolom ['nama', 'perusahaan', 'jabatan'] Output: DataFrame dengan kolom tambahan 'cluster' """ def __init__(self, n_clusters: int = 3, auto_k: bool = False): """ Args: n_clusters: Jumlah cluster (diabaikan jika auto_k=True) auto_k: Jika True, cari k optimal secara otomatis """ self.n_clusters = n_clusters self.auto_k = auto_k self.tfidf = None self.kmeans = None self.feature_names = None def _combine_text(self, df: pd.DataFrame) -> list: """Menggabungkan kolom teks menjadi satu""" combined = df['perusahaan'].fillna('') + ' ' + df['jabatan'].fillna('') return combined.tolist() def _preprocess_all(self, texts: list) -> list: """Preprocess semua teks""" return [preprocess_text(t) for t in texts] def fit(self, df: pd.DataFrame) -> 'AlumniClusteringPipeline': """ Fit pipeline ke data training Args: df: DataFrame dengan kolom ['nama', 'perusahaan', 'jabatan'] Returns: self """ # Combine text combined_texts = self._combine_text(df) # Preprocess processed_texts = self._preprocess_all(combined_texts) # TF-IDF self.tfidf = TfidfExtractor(max_features=500) feature_matrix = self.tfidf.fit_transform(processed_texts) self.feature_names = self.tfidf.get_feature_names() # K-Means n_clusters = self.n_clusters if self.auto_k: # Find optimal k results = find_optimal_k(feature_matrix) # Ambil k dengan silhouette score tertinggi best_idx = np.argmax(results['silhouette']) n_clusters = results['k'][best_idx] print(f"Optimal K selected: {n_clusters}") self.kmeans = KMeansClustering(n_clusters=n_clusters) self.kmeans.fit(feature_matrix) return self def transform(self, df: pd.DataFrame) -> pd.DataFrame: """ Transform data dengan model yang sudah di-fit Args: df: DataFrame dengan kolom ['nama', 'perusahaan', 'jabatan'] Returns: DataFrame dengan kolom 'cluster' dan 'cluster_label' """ if self.kmeans is None: raise ValueError("Model belum di-fit. Jalankan fit() terlebih dahulu.") # Combine text combined_texts = self._combine_text(df) # Preprocess processed_texts = self._preprocess_all(combined_texts) # TF-IDF transform feature_matrix = self.tfidf.transform(processed_texts) # Predict labels = self.kmeans.predict(feature_matrix) # Add to DataFrame result = df.copy() result['cluster'] = labels result['cluster_label'] = [f"Cluster_{i}" for i in labels] return result def fit_transform(self, df: pd.DataFrame) -> pd.DataFrame: """Fit dan transform sekaligus""" return self.fit(df).transform(df) def get_cluster_summary(self, df: pd.DataFrame) -> dict: """Mendapatkan ringkasan cluster""" if 'cluster' not in df.columns: raise ValueError("Data belum di-cluster. Jalankan fit_transform() terlebih dahulu.") return get_cluster_summary(df, df['cluster'].values) def get_top_features(self, n: int = 10) -> dict: """Mendapatkan fitur teratas untuk setiap cluster""" if self.kmeans is None or self.tfidf is None: raise ValueError("Model belum di-fit.") cluster_centers = self.kmeans.model.cluster_centers_ feature_names = self.feature_names top_features = {} for i, center in enumerate(cluster_centers): # Get indices of top features top_indices = center.argsort()[-n:][::-1] top_features[f"Cluster_{i}"] = [ {"term": feature_names[idx], "weight": float(center[idx])} for idx in top_indices ] return top_features def save_model(self, filepath: str): """Simpan model (placeholder - bisa di-extend dengan joblib)""" model_data = { 'n_clusters': self.n_clusters, 'feature_names': self.feature_names } with open(filepath, 'w') as f: json.dump(model_data, f) @staticmethod def load_model(filepath: str): """Load model (placeholder)""" with open(filepath, 'r') as f: model_data = json.load(f) pipeline = AlumniClusteringPipeline(n_clusters=model_data['n_clusters']) return pipeline def run_pipeline(data_path: str = None, n_clusters: int = 3, output_path: str = None): """ Jalankan pipeline dari command line Args: data_path: Path ke file CSV input n_clusters: Jumlah cluster output_path: Path untuk menyimpan hasil """ # Load data if data_path: df = pd.read_csv(data_path) else: # Sample data untuk testing df = pd.DataFrame({ 'nama': ['Ali', 'Budi', 'Citra', 'Dewi', 'Eko'], 'perusahaan': ['Google', 'Tokopedia', 'Gojek', 'Traveloka', 'Bukalapak'], 'jabatan': ['Software Engineer', 'Data Analyst', 'Product Manager', 'UX Designer', 'DevOps'] }) print(f"Data loaded: {len(df)} records") # Run pipeline pipeline = AlumniClusteringPipeline(n_clusters=n_clusters) result = pipeline.fit_transform(df) print(f"\nClustering complete!") print(f"\nCluster Distribution:") print(result['cluster_label'].value_counts()) print(f"\nCluster Summary:") summary = pipeline.get_cluster_summary(result) for cluster, info in summary.items(): print(f" {cluster}: {info['count']} alumni ({info['percentage']}%)") print(f"\nTop Features per Cluster:") top_features = pipeline.get_top_features() for cluster, features in top_features.items(): print(f" {cluster}: {[f['term'] for f in features[:5]]}") # Save result if output_path: result.to_csv(output_path, index=False) print(f"\nResult saved to: {output_path}") return result if __name__ == "__main__": import sys if len(sys.argv) > 1: data_path = sys.argv[1] n_clusters = int(sys.argv[2]) if len(sys.argv) > 2 else 3 output_path = sys.argv[3] if len(sys.argv) > 3 else None run_pipeline(data_path, n_clusters, output_path) else: run_pipeline()