TIF_E41253310/Backend/app/routers/clustering.py

"""
Router untuk Clustering Endpoint
"""

import math
from datetime import datetime
from fastapi import APIRouter, UploadFile, File, HTTPException, Depends, Query, Body
from fastapi.responses import JSONResponse, StreamingResponse
from typing import List, Optional, Dict, Any
from sqlalchemy.orm import Session
from sqlalchemy import or_
import pandas as pd
import io

from app.config.database import get_db
from app.models.models import ClusteringResult, ModelEvaluation
from app.models.alumni import Alumni
from app.schemas.clustering import (
    ClusteringRequest,
    ClusteringResponse,
    ClusterResult,
    AlumniInput,
    StandardizeRequest,
    StandardizeResponse
)
from app.services.ml_service import get_ml_service
from app.utils.standardization import ColumnStandardizer, merge_datasets, clean_data

router = APIRouter(tags=["Clustering ML"])


@router.post("/process", summary="Jalankan Pipeline Clustering dari Database MySQL")
@router.post("/run-db", summary="Alias Jalankan Pipeline Clustering dari Database")
def process_clustering_db(
    tahun_lulus: Optional[int] = Query(None, description="Filter tahun kelulusan / angkatan (opsional)"),
    n_clusters: int = Query(4, ge=2, le=10, description="Jumlah cluster K-Means"),
    db: Session = Depends(get_db)
):
    """
    Menjalankan proses Machine Learning Clustering (TF-IDF + K-Means K=4 + Interpretasi C0-C3)
    langsung dari data alumni & pekerjaan yang ada di database MySQL.
    Hasil otomatis disimpan ke tabel clustering_result dan model_evaluation.
    """
    ml_service = get_ml_service()
    result = ml_service.process_clustering_from_db(
        db=db,
        tahun_lulus=tahun_lulus,
        n_clusters=n_clusters
    )

    if result.get("status") == "error":
        raise HTTPException(status_code=400, detail=result.get("message"))

    return result


@router.get("/results", summary="Ambil Hasil Clustering dengan Filter & Pagination")
def get_clustering_results(
    skip: int = Query(0, ge=0),
    limit: int = Query(10, ge=1, le=500),
    search: Optional[str] = Query(None, description="Cari nama atau NIM"),
    kategori: Optional[str] = Query(None, description="Filter kategori C0, C1, C2, C3"),
    kode_prodi: Optional[str] = Query(None, description="Filter kode prodi"),
    tahun_lulus: Optional[int] = Query(None, description="Filter tahun lulus"),
    db: Session = Depends(get_db)
):
    """
    Mengambil data hasil clustering alumni dari database lengkap dengan filter kategori C0-C3,
    prodi, tahun, dan pencarian nama/NIM.
    """
    from app.models.models import Pekerjaan

    query = db.query(
        ClusteringResult,
        Alumni.nim,
        Alumni.nama,
        Alumni.kode_prodi,
        Alumni.tahun_lulus,
        Alumni.pekerjaan,
        Alumni.instansi,
        Alumni.status_kerja,
        Pekerjaan.status_pekerjaan,
        Pekerjaan.jenis_perusahaan,
        Pekerjaan.nama_perusahaan,
        Pekerjaan.jabatan,
        Pekerjaan.teks_pekerjaan
    ).join(Alumni, ClusteringResult.alumni_id == Alumni.id)\
     .outerjoin(Pekerjaan, Alumni.id == Pekerjaan.alumni_id)

    if search:
        s = f"%{search.strip()}%"
        query = query.filter(or_(Alumni.nama.ilike(s), Alumni.nim.ilike(s), Alumni.pekerjaan.ilike(s), Pekerjaan.teks_pekerjaan.ilike(s)))

    if kategori:
        query = query.filter(ClusteringResult.kategori == kategori.strip().upper())

    if kode_prodi:
        query = query.filter(Alumni.kode_prodi == str(kode_prodi).strip())

    if tahun_lulus:
        query = query.filter(Alumni.tahun_lulus == int(tahun_lulus))

    total = query.count()
    rows = query.order_by(ClusteringResult.id.desc()).offset(skip).limit(limit).all()

    items = []
    for r in rows:
        cr = r[0]
        pek_nama = r[10] if (r[10] and r[10].strip() not in ['', 'nan', 'None']) else (r[5] or "-")
        pek_jab = r[11] if (r[11] and r[11].strip() not in ['', 'nan', 'None']) else (r[6] or "-")
        pek_status = r[8] if (r[8] and r[8].strip() not in ['', 'nan', 'None']) else (r[7] or "-")
        pek_jenis = r[9] if (r[9] and r[9].strip() not in ['', 'nan', 'None']) else "-"
        
        pek_teks = r[12] if (r[12] and r[12].strip() not in ['', 'nan', 'None']) else f"{pek_status} {pek_jenis} {pek_nama} {pek_jab}".replace('-', '').strip()
        if not pek_teks:
            pek_teks = "-"

        items.append({
            "id": cr.id,
            "alumni_id": cr.alumni_id,
            "nim": r[1],
            "nama": r[2],
            "kode_prodi": r[3],
            "tahun_lulus": r[4],
            "status_pekerjaan": pek_status,
            "jenis_perusahaan": pek_jenis,
            "nama_perusahaan": pek_nama,
            "jabatan": pek_jab,
            "teks_pekerjaan": pek_teks,
            "cluster": cr.cluster,
            "kategori": cr.kategori,
            "profile_match": cr.profile_match,
            "matched_keyword": cr.matched_keyword or "",
            "model_version": cr.model_version,
            "created_at": cr.created_at
        })

    page = (skip // limit) + 1 if limit > 0 else 1
    total_pages = math.ceil(total / limit) if limit > 0 else 1

    return {
        "success": True,
        "data": {
            "items": items,
            "total": total,
            "page": page,
            "limit": limit,
            "total_pages": total_pages
        }
    }


@router.get("/summary", summary="Ringkasan Statistik Hasil Clustering Terkini")
def get_clustering_summary(
    tahun_lulus: Optional[int] = Query(None, description="Filter tahun kelulusan / angkatan (opsional)"),
    kode_prodi: Optional[str] = Query(None, description="Filter kode prodi (opsional)"),
    db: Session = Depends(get_db)
):
    """Mengambil ringkasan statistik distribusi C0-C3 dan evaluasi model terkini."""
    query = db.query(ClusteringResult).join(Alumni, Alumni.id == ClusteringResult.alumni_id)
    if tahun_lulus:
        query = query.filter(Alumni.tahun_lulus == int(tahun_lulus))
    if kode_prodi:
        query = query.filter(Alumni.kode_prodi == str(kode_prodi).strip())

    from sqlalchemy import func

    # Eksekusi 1 single query agregasi GROUP BY kategori
    cat_counts = query.with_entities(
        ClusteringResult.kategori,
        func.count(ClusteringResult.id)
    ).group_by(ClusteringResult.kategori).all()

    dist = {'C0': 0, 'C1': 0, 'C2': 0, 'C3': 0}
    total = 0
    for kat, cnt in cat_counts:
        k_str = str(kat).upper().strip() if kat else 'C3'
        if k_str in dist:
            dist[k_str] = int(cnt)
        total += int(cnt)

    latest_eval = db.query(ModelEvaluation).order_by(ModelEvaluation.created_at.desc()).first()

    return {
        "total_clustered": total,
        "distribution": dist,
        "silhouette_score": latest_eval.silhouette_score if latest_eval else None,
        "model_version": latest_eval.model_version if latest_eval else None,
        "k": latest_eval.k if latest_eval else 4,
        "tahun_lulus": tahun_lulus,
        "kode_prodi": kode_prodi
    }


@router.get("/iteration-details", summary="Ambil Detail Riwayat Iterasi & Perhitungan Matematis K-Means")
def get_iteration_details(
    tahun_lulus: Optional[int] = Query(None, description="Filter tahun kelulusan / angkatan (opsional)"),
    alumni_id: Optional[int] = Query(None, description="ID alumni spesifik untuk inspeksi perhitungan detail"),
    db: Session = Depends(get_db)
):
    """
    Mengambil log lengkap iterasi perulangan K-Means, karakteristik centroid,
    dan alur transformasi 5 tahap (Kalimat -> Preprocessing -> TF-IDF -> K-Means -> Klaster)
    berdasarkan dataset tahun kelulusan yang dipilih.
    """
    ml_service = get_ml_service()
    details = ml_service.get_latest_iteration_details(db=db, tahun_lulus=tahun_lulus, target_alumni_id=alumni_id)
    return {
        "success": True,
        "data": details
    }


def _build_export_dataframe(
    db: Session,
    search: Optional[str] = None,
    kategori: Optional[str] = None,
    kode_prodi: Optional[str] = None,
    tahun_lulus: Optional[int] = None
) -> pd.DataFrame:
    from app.models.models import Pekerjaan

    query = db.query(
        ClusteringResult,
        Alumni.nim,
        Alumni.nama,
        Alumni.kode_prodi,
        Alumni.tahun_lulus,
        Alumni.pekerjaan,
        Alumni.instansi,
        Alumni.status_kerja,
        Pekerjaan.status_pekerjaan,
        Pekerjaan.jenis_perusahaan,
        Pekerjaan.nama_perusahaan,
        Pekerjaan.jabatan,
        Pekerjaan.teks_pekerjaan
    ).join(Alumni, ClusteringResult.alumni_id == Alumni.id)\
     .outerjoin(Pekerjaan, Alumni.id == Pekerjaan.alumni_id)

    if search:
        s = f"%{search.strip()}%"
        query = query.filter(or_(
            Alumni.nama.ilike(s),
            Alumni.nim.ilike(s),
            Alumni.pekerjaan.ilike(s),
            Pekerjaan.teks_pekerjaan.ilike(s)
        ))

    if kategori:
        query = query.filter(ClusteringResult.kategori == kategori.strip().upper())

    if kode_prodi:
        query = query.filter(Alumni.kode_prodi == str(kode_prodi).strip())

    if tahun_lulus:
        query = query.filter(Alumni.tahun_lulus == int(tahun_lulus))

    rows = query.order_by(ClusteringResult.id.asc()).all()

    prodi_map = {
        '55301': 'D4 Teknik Informatika',
        '57401': 'D3 Manajemen Informatika',
        '56401': 'D3 Teknik Komputer',
        '55302': 'D4 TIF Kampus Nganjuk',
        '55304': 'D4 TIF Kampus Sidoarjo',
    }

    kategori_labels = {
        'C0': 'Sesuai Profil Lulusan',
        'C1': 'Bekerja, Bidang Diketahui Formal',
        'C2': 'Bekerja, Kurang Teridentifikasi',
        'C3': 'Belum / Tidak Bekerja'
    }

    export_records = []
    for idx, r in enumerate(rows, start=1):
        cr = r[0]
        kp = str(r[3] or '').strip()
        prodi_name = prodi_map.get(kp, kp)
        kat = cr.kategori or '-'
        kat_label = kategori_labels.get(kat, '-')

        pek_nama = r[10] if (r[10] and r[10].strip() not in ['', 'nan', 'None']) else (r[5] or "-")
        pek_jab = r[11] if (r[11] and r[11].strip() not in ['', 'nan', 'None']) else (r[6] or "-")
        pek_status = r[8] if (r[8] and r[8].strip() not in ['', 'nan', 'None']) else (r[7] or "-")
        pek_jenis = r[9] if (r[9] and r[9].strip() not in ['', 'nan', 'None']) else "-"
        pek_teks = r[12] if (r[12] and r[12].strip() not in ['', 'nan', 'None']) else f"{pek_status} {pek_jenis} {pek_nama} {pek_jab}".replace('-', '').strip()

        export_records.append({
            "No": idx,
            "NIM": r[1],
            "Nama Alumni": r[2],
            "Program Studi": prodi_name,
            "Tahun Lulus": r[4],
            "Status Kerja": pek_status,
            "Nama Perusahaan / Instansi": pek_nama,
            "Jabatan / Posisi": pek_jab,
            "Jenis Perusahaan": pek_jenis,
            "Teks Pekerjaan": pek_teks if pek_teks else "-",
            "Klaster": f"Cluster {cr.cluster}",
            "Kategori": kat,
            "Interpretasi Relevansi": kat_label,
            "Kata Kunci Cocok": cr.matched_keyword or "-"
        })

    return pd.DataFrame(export_records)


@router.get("/export/excel", summary="Unduh Hasil Clustering ke File Excel (.xlsx)")
def export_clustering_excel(
    search: Optional[str] = Query(None, description="Cari nama atau NIM"),
    kategori: Optional[str] = Query(None, description="Filter kategori C0, C1, C2, C3"),
    kode_prodi: Optional[str] = Query(None, description="Filter kode prodi"),
    tahun_lulus: Optional[int] = Query(None, description="Filter tahun lulus"),
    db: Session = Depends(get_db)
):
    """
    Ekspor hasil clustering alumni yang tersimpan di MySQL ke berkas spreadsheet Microsoft Excel (.xlsx).
    Mendukung filter tahun kelulusan, program studi, kategori relevansi C0-C3, dan kata kunci pencarian.
    """
    df = _build_export_dataframe(db, search, kategori, kode_prodi, tahun_lulus)
    output = io.BytesIO()
    with pd.ExcelWriter(output, engine='openpyxl') as writer:
        df.to_excel(writer, index=False, sheet_name='Hasil Klastering')
    output.seek(0)
    
    filename = f"Hasil_Clustering_Alumni_{datetime.now().strftime('%Y%m%d_%H%M%S')}.xlsx"
    headers = {
        'Content-Disposition': f'attachment; filename="{filename}"'
    }
    return StreamingResponse(
        output,
        media_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
        headers=headers
    )


@router.get("/export/csv", summary="Unduh Hasil Clustering ke File CSV (.csv)")
def export_clustering_csv(
    search: Optional[str] = Query(None, description="Cari nama atau NIM"),
    kategori: Optional[str] = Query(None, description="Filter kategori C0, C1, C2, C3"),
    kode_prodi: Optional[str] = Query(None, description="Filter kode prodi"),
    tahun_lulus: Optional[int] = Query(None, description="Filter tahun lulus"),
    db: Session = Depends(get_db)
):
    """
    Ekspor hasil clustering alumni yang tersimpan di MySQL ke berkas CSV dengan encoding UTF-8.
    """
    df = _build_export_dataframe(db, search, kategori, kode_prodi, tahun_lulus)
    output = io.BytesIO()
    df.to_csv(output, index=False, encoding='utf-8-sig')
    output.seek(0)

    filename = f"Hasil_Clustering_Alumni_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv"
    headers = {
        'Content-Disposition': f'attachment; filename="{filename}"'
    }
    return StreamingResponse(
        output,
        media_type='text/csv; charset=utf-8',
        headers=headers
    )