28_05_2026
This commit is contained in:
parent
eb52aa7abf
commit
420604ef2d
|
|
@ -35,4 +35,4 @@ Thumbs.db
|
|||
# Model / Data (Jika di-generate/download lokal)
|
||||
fastapi/ml_assets/*.pkl
|
||||
fastapi/ml_assets/*.h5
|
||||
fastapi/ml_assets/*.pt
|
||||
fastapi/ml_assets/*.pt
|
||||
|
|
@ -0,0 +1,165 @@
|
|||
import pandas as pd
|
||||
import numpy as np
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from sklearn.model_selection import train_test_split
|
||||
from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
|
||||
|
||||
factory = StemmerFactory()
|
||||
stemmer = factory.create_stemmer()
|
||||
|
||||
BASE_DIR = Path(__file__).parent
|
||||
DATA_DIR = BASE_DIR
|
||||
OUTPUT_DIR = DATA_DIR / "processed"
|
||||
OUTPUT_DIR.mkdir(exist_ok=True)
|
||||
FILE_INTERNAL = DATA_DIR / "ts_internal_mif.xlsx"
|
||||
SEPARATOR = ";"
|
||||
ENCODING = "utf-8-sig"
|
||||
TARGET_CLASSES = ["Programmer", "Data Analyst", "Wirausaha Informatika", "Non-IT"]
|
||||
KLASIFIKASI_MAP = {
|
||||
"Programmer": "Programmer",
|
||||
"Data Analyst": "Data Analyst",
|
||||
"Wirausaha IT": "Wirausaha Informatika",
|
||||
"Wirausaha": "Wirausaha Informatika",
|
||||
"Non-IT": "Non-IT",
|
||||
"Infokom": None, "Pelajar": None, "Tidak Bekerja": None, "TIdak diketahui": None
|
||||
}
|
||||
KEYWORD_RULES = {
|
||||
"Programmer": ["programmer", "developer", "engineer", "fullstack", "backend", "frontend", "mobile", "android", "ios", "software", "web dev", "coding", "it staff", "teknisi", "sistem informasi", "application", "network", "devops", "qa", "tester", "ui", "ux", "swe"],
|
||||
"Data Analyst": ["data analyst", "analis data", "data science", "business analyst", "research", "statistik", "bi analyst", "reporting", "database", "sql", "etl", "data engineer", "big data", "analyst", "data mining", "machine learning", "data visual", "power bi", "tableau", "looker", "business intelligence", "bi developer", "data warehouse"],
|
||||
"Wirausaha Informatika": ["founder", "owner", "ceo", "wiraswasta", "startup", "freelance", "freelancer", "wirausaha", "bisnis", "usaha mandiri", "konsultan", "co founder", "entrepreneur", "self employed", "owner toko", "usaha", "dagang online", "tokopedia", "shopee", "dropship", "reseller"]
|
||||
}
|
||||
COMPANY_STOPWORDS = {
|
||||
"pt", "cv", "ud", "tbk", "persero", "corp", "inc", "ltd", "koperasi", "bumn", "bumd",
|
||||
"dinas", "kantor", "pemkab", "pemprov", "politeknik", "universitas", "sekolah", "sma", "smk", "sd",
|
||||
"bank", "bpr", "rs", "rumah sakit", "klinik", "apotek", "hotel", "restoran", "cafe", "toko", "konter",
|
||||
"foundation", "yayasan", "perkumpulan", "organisasi", "agency", "studio", "consulting", "group", "holding"
|
||||
}
|
||||
|
||||
def load_data_file(file_path: str) -> pd.DataFrame:
|
||||
ext = Path(file_path).suffix.lower()
|
||||
if ext == '.csv':
|
||||
try: return pd.read_csv(file_path, sep=SEPARATOR, encoding=ENCODING, dtype=str, on_bad_lines='skip', engine='python')
|
||||
except UnicodeDecodeError: return pd.read_csv(file_path, sep=SEPARATOR, encoding='latin1', dtype=str, on_bad_lines='skip', engine='python')
|
||||
elif ext == '.xlsx':
|
||||
return pd.read_excel(file_path, dtype=str)
|
||||
else:
|
||||
raise ValueError(f"Format tidak didukung: {ext}")
|
||||
|
||||
def find_col(df: pd.DataFrame, keywords: list) -> str | None:
|
||||
for col in df.columns:
|
||||
col_clean = str(col).strip().lower()
|
||||
if any(k.strip().lower() in col_clean for k in keywords): return col
|
||||
return None
|
||||
|
||||
def clean_text(text: str) -> str:
|
||||
if pd.isna(text) or str(text).strip().lower() in ["nan", "none", "null", "-", "0", "", "tidak diisi", "tidak diketahui"]: return ""
|
||||
text = str(text).strip().lower()
|
||||
text = re.sub(r'^\d+\s*[-:/]\s*', '', text)
|
||||
text = text.replace('-', ' ').replace('/', ' ').replace('_', ' ').replace(',', ' ')
|
||||
text = re.sub(r'[^\w\s]', '', text)
|
||||
text = re.sub(r'\d+', '', text)
|
||||
tokens = [w for w in text.split() if w not in COMPANY_STOPWORDS and len(w) >= 3]
|
||||
return " ".join([stemmer.stem(w) for w in tokens])
|
||||
|
||||
# [FIX v4] Logika lebih ketat: hanya flag jika token teks sepenuhnya subset dari nama
|
||||
# (max 3 token) — bukan sekadar rasio overlap 50% seperti v3 yang rawan false positive.
|
||||
def is_likely_name(text: str, full_name: str) -> bool:
|
||||
if not text or not full_name: return False
|
||||
text_clean = re.sub(r'[^\w\s]', '', text.lower())
|
||||
name_clean = re.sub(r'[^\w\s]', '', str(full_name).lower())
|
||||
text_parts = set(text_clean.split())
|
||||
name_parts = set(name_clean.split())
|
||||
if len(text_parts) == 0: return False
|
||||
if len(text_parts) <= 3 and text_parts.issubset(name_parts):
|
||||
return True
|
||||
return False
|
||||
|
||||
def classify_rule_based(text: str) -> str:
|
||||
if not text or len(text) < 3: return "Non-IT"
|
||||
for profile in ["Programmer", "Data Analyst", "Wirausaha Informatika"]:
|
||||
if any(kw in text for kw in KEYWORD_RULES[profile]): return profile
|
||||
return "Non-IT"
|
||||
|
||||
def main():
|
||||
if not FILE_INTERNAL.exists():
|
||||
print(f"Berkas tidak ditemukan: {FILE_INTERNAL}"); sys.exit(1)
|
||||
print("[1/4] Memuat & Membersihkan Data (Strict Cleaning)...")
|
||||
df = load_data_file(str(FILE_INTERNAL))
|
||||
df.columns = df.columns.str.strip()
|
||||
col_nim = find_col(df, ["nim"])
|
||||
col_nama = find_col(df, ["nama", "lengkap"])
|
||||
col_jab_lama = find_col(df, ["jabatan"])
|
||||
col_jab_baru = find_col(df, ["jabatan_terupdate"])
|
||||
col_klasifikasi = find_col(df, ["klasifikasi"])
|
||||
col_status = find_col(df, ["status", "kerja"])
|
||||
|
||||
if not all([col_nim, col_nama, col_jab_lama]):
|
||||
print(" Kolom esensial tidak ditemukan"); print(df.columns.tolist()[:10]); sys.exit(1)
|
||||
|
||||
if col_jab_baru:
|
||||
mask_empty = df[col_jab_baru].isna() | df[col_jab_baru].astype(str).str.strip().str.lower().isin(["", "-", "0", "nan", "none", "null", "tidak diisi", "tidak diketahui"])
|
||||
df["jabatan_final"] = df[col_jab_baru].where(~mask_empty, df[col_jab_lama])
|
||||
else:
|
||||
df["jabatan_final"] = df[col_jab_lama]
|
||||
|
||||
col_jabatan = "jabatan_final"
|
||||
if col_status:
|
||||
blacklist = ["tidah diketahui", "tidak bekerja", "pelajar", "melanjutkan pendidikan", "nan", ""]
|
||||
mask = ~df[col_status].str.lower().str.strip().isin(blacklist)
|
||||
df = df[mask].copy()
|
||||
|
||||
# Stemming berat terjadi HANYA di sini — satu kali saat corpus preparation
|
||||
df["job_text_raw"] = df[col_jabatan].apply(clean_text)
|
||||
|
||||
if col_klasifikasi:
|
||||
fallback_map = {"Programmer": "programmer developer", "Data Analyst": "data analyst", "Wirausaha IT": "wirausaha founder", "Wirausaha": "wirausaha founder", "Non IT": "staff admin", "Infokom": "it staff teknisi", "TIdah diketahui": "", "Pelajar": "", "Tidak Bekerja": ""}
|
||||
empty_mask = df["job_text_raw"] == ""
|
||||
if empty_mask.any(): df.loc[empty_mask, "job_text_raw"] = df.loc[empty_mask, col_klasifikasi].map(fallback_map).fillna("")
|
||||
|
||||
if col_nama:
|
||||
name_leak_mask = df.apply(lambda row: is_likely_name(row["job_text_raw"], row[col_nama]), axis=1)
|
||||
leaked_count = name_leak_mask.sum()
|
||||
if leaked_count > 0: print(f" Mengabaikan {leaked_count} baris yang terindikasi mengandung nama pribadi.")
|
||||
df = df[~name_leak_mask].copy()
|
||||
|
||||
if col_klasifikasi:
|
||||
df["label"] = df[col_klasifikasi].str.strip().map(KLASIFIKASI_MAP)
|
||||
missing = df["label"].isna()
|
||||
if missing.any(): df.loc[missing, "label"] = df.loc[missing, "job_text_raw"].apply(classify_rule_based)
|
||||
else:
|
||||
df["label"] = df["job_text_raw"].apply(classify_rule_based)
|
||||
|
||||
df = df[df["job_text_raw"].str.len() >= 3].copy()
|
||||
result = (
|
||||
df[[col_nim, col_jabatan, "job_text_raw", "label"]]
|
||||
.rename(columns={col_nim: "nim"})
|
||||
.dropna(subset=["nim", "label"])
|
||||
.drop_duplicates(subset=["nim"], keep="first")
|
||||
)
|
||||
print(f"Memuat {len(result)} data valid (berdasarkan teks pekerjaan)\n")
|
||||
|
||||
min_count = result["label"].value_counts().min()
|
||||
if min_count < 2:
|
||||
train_df, test_df = train_test_split(result, test_size=0.30, random_state=42)
|
||||
else:
|
||||
train_df, test_df = train_test_split(result, test_size=0.30, stratify=result["label"], random_state=42)
|
||||
|
||||
cols_out = ["nim", "job_text_raw", "label"]
|
||||
train_df[cols_out].to_csv(OUTPUT_DIR / "training_corpus.csv", index=False, sep=";", encoding=ENCODING)
|
||||
test_df[cols_out].to_csv(OUTPUT_DIR / "test_set.csv", index=False, sep=";", encoding=ENCODING)
|
||||
|
||||
print("DATA PELATIHAN (TRAINING SET):")
|
||||
print(train_df["label"].value_counts().to_string())
|
||||
|
||||
# [KEEP v3] Distribusi per kelas di test set — berguna untuk deteksi class imbalance
|
||||
print("\nDATA PENGUJIAN (TEST SET):")
|
||||
for cls in TARGET_CLASSES:
|
||||
c = test_df["label"].value_counts().get(cls, 0)
|
||||
print(f" {cls:25} : {c}{'<5' if c < 5 else ''}")
|
||||
|
||||
print(f"\nData berhasil disimpan ke {OUTPUT_DIR}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -0,0 +1,109 @@
|
|||
nim;job_text_raw;label
|
||||
E31190416;freelance kerja lepas;Wirausaha Informatika
|
||||
E31160787;software programmer;Programmer
|
||||
E31192134;freelance kerja lepas;Wirausaha Informatika
|
||||
E31201687;staff graphic designer;Non-IT
|
||||
E31200364;freelance kerja lepas;Data Analyst
|
||||
E31180838;wirausaha founder;Wirausaha Informatika
|
||||
E31181942;supervisor;Programmer
|
||||
E31190408;freelance kerja lepas;Wirausaha Informatika
|
||||
E31200061;marketing officer;Non-IT
|
||||
E31180644;software programmer;Programmer
|
||||
E31191447;freelance kerja lepas;Wirausaha Informatika
|
||||
E31190785;freelance kerja lepas;Wirausaha Informatika
|
||||
E31180926;mathematics tutor;Non-IT
|
||||
E31201932;freelance kerja lepas;Wirausaha Informatika
|
||||
E31181204;founder;Wirausaha Informatika
|
||||
E31181295;creative team lead;Programmer
|
||||
E31192292;staff;Non-IT
|
||||
E31180949;research and development engineer;Programmer
|
||||
E31192123;customer engineer;Programmer
|
||||
E31191441;freelance kerja lepas;Wirausaha Informatika
|
||||
E31180295;sales;Wirausaha Informatika
|
||||
E31160901;logistik;Non-IT
|
||||
E31182030;founder;Programmer
|
||||
E31191547;staf sdm parmas;Non-IT
|
||||
E31201950;marketing officer;Non-IT
|
||||
E31192213;freelance kerja lepas;Programmer
|
||||
E31171849;programmer developer;Programmer
|
||||
E31201829;staff;Non-IT
|
||||
E31192403;freelance kerja lepas;Wirausaha Informatika
|
||||
E31192060;freelance kerja lepas;Wirausaha Informatika
|
||||
E31171992;it staff teknisi;Programmer
|
||||
E31192166;founder;Wirausaha Informatika
|
||||
E31200117;freelance kerja lepas;Wirausaha Informatika
|
||||
E31181709;web developer supervisor;Programmer
|
||||
E31200973;programmer;Programmer
|
||||
E31201951;staff;Non-IT
|
||||
E31201372;founder;Wirausaha Informatika
|
||||
E31180509;freelance kerja lepas;Wirausaha Informatika
|
||||
E31180724;lain lain;Non-IT
|
||||
E31170050;it staff teknisi;Programmer
|
||||
E31180707;freelance kerja lepas;Programmer
|
||||
E31181971;digital marketing strategist;Non-IT
|
||||
E31192175;tugas barang bukti;Non-IT
|
||||
E31190980;staff;Programmer
|
||||
E31192116;freelance kerja lepas;Wirausaha Informatika
|
||||
E31192029;freelance kerja lepas;Programmer
|
||||
E31190341;staff;Programmer
|
||||
E31202314;staf laksana;Non-IT
|
||||
E31202562;staff;Non-IT
|
||||
E31191800;checker angkut barang;Non-IT
|
||||
E31172055;maintenance and technical support retail;Non-IT
|
||||
E31170468;teknisi support;Programmer
|
||||
E31202038;freelance kerja lepas;Wirausaha Informatika
|
||||
E31201422;freelance kerja lepas;Wirausaha Informatika
|
||||
E31181032;freelance kerja lepas;Wirausaha Informatika
|
||||
E31202278;founder;Wirausaha Informatika
|
||||
E31171733;staff divisi suhsi;Non-IT
|
||||
E31200561;freelance kerja lepas;Data Analyst
|
||||
E31171396;programmer developer;Programmer
|
||||
E31192024;teknisi laboratorium;Programmer
|
||||
E31181929;administrasi bmn;Non-IT
|
||||
E31190987;staff administrasi;Non-IT
|
||||
E31201009;staff;Non-IT
|
||||
E31191792;admin and finance staff;Non-IT
|
||||
E31170253;admin;Non-IT
|
||||
E31171904;freelance illustrator;Wirausaha Informatika
|
||||
E31180575;designer;Programmer
|
||||
E31160852;interviewer;Non-IT
|
||||
E31191848;fullstack developer;Programmer
|
||||
E31200909;founder;Wirausaha Informatika
|
||||
E31191660;founder;Wirausaha Informatika
|
||||
E31171317;web developer;Programmer
|
||||
E31202451;staff;Non-IT
|
||||
E31170570;software engineer;Programmer
|
||||
E31192159;staff data;Non-IT
|
||||
E31171106;phl;Non-IT
|
||||
E31202523;freelance kerja lepas;Wirausaha Informatika
|
||||
E31191793;founder;Wirausaha Informatika
|
||||
E31192043;freelance kerja lepas;Wirausaha Informatika
|
||||
E31201706;freelance kerja lepas;Wirausaha Informatika
|
||||
E31180303;staff;Non-IT
|
||||
E31192416;freelance kerja lepas;Wirausaha Informatika
|
||||
E31190286;staff;Non-IT
|
||||
E31190800;staff;Non-IT
|
||||
E31181976;teller;Non-IT
|
||||
E31201954;freelance kerja lepas;Wirausaha Informatika
|
||||
E31161636;web developer;Programmer
|
||||
E3117228;programmer developer;Programmer
|
||||
E31192264;founder;Wirausaha Informatika
|
||||
E31201573;staff;Non-IT
|
||||
E31160170;pranata teknologi informasi komputer;Programmer
|
||||
E31180686;founder;Wirausaha Informatika
|
||||
E31201020;staff;Non-IT
|
||||
E31190302;teknisi laboratorium;Programmer
|
||||
E31201300;freelance kerja lepas;Wirausaha Informatika
|
||||
E31192047;founder;Wirausaha Informatika
|
||||
E31140976;staff admin;Non-IT
|
||||
E31191621;staff;Non-IT
|
||||
E31180504;founder;Wirausaha Informatika
|
||||
E31180735;programer;Programmer
|
||||
E31172173;administrator;Non-IT
|
||||
E31180207;founder;Wirausaha Informatika
|
||||
E31160707;staf programmer;Programmer
|
||||
E31200700;staff;Wirausaha Informatika
|
||||
E31191272;staff;Wirausaha Informatika
|
||||
E31190068;staff administrasi produksi;Non-IT
|
||||
E31191006;freelance kerja lepas;Wirausaha Informatika
|
||||
E31180678;asisten tenaga latih;Non-IT
|
||||
|
|
|
@ -1,109 +0,0 @@
|
|||
nim;job_text_raw;label
|
||||
E31200637;freelance kerja lepas;Wirausaha Informatika
|
||||
E31200880;freelance kerja lepas;Wirausaha Informatika
|
||||
E31160787;software programmer;Programmer
|
||||
E31171733;staff divisi suhsi;Non-IT
|
||||
E31170445;it staff teknisi;Programmer
|
||||
E31200364;freelance kerja lepas;Data Analyst
|
||||
E31191677;freelance kerja lepas;Wirausaha Informatika
|
||||
E31201951;staff;Non-IT
|
||||
E31200912;staff;Non-IT
|
||||
E31191460;freelance kerja lepas;Wirausaha Informatika
|
||||
E31170468;teknisi support;Programmer
|
||||
E31181709;staff;Programmer
|
||||
E31201634;staff;Non-IT
|
||||
E31201095;freelance kerja lepas;Wirausaha Informatika
|
||||
E31180926;freelance kerja lepas;Wirausaha Informatika
|
||||
E31192259;owner;Wirausaha Informatika
|
||||
E31192120;freelance kerja lepas;Wirausaha Informatika
|
||||
E31191394;founder;Wirausaha Informatika
|
||||
E31170772;engineer product;Programmer
|
||||
E31140976;staff admin;Non-IT
|
||||
E31180222;freelance kerja lepas;Wirausaha Informatika
|
||||
E31190851;founder;Wirausaha Informatika
|
||||
E31190430;freelance kerja lepas;Wirausaha Informatika
|
||||
E31190801;staff;Non-IT
|
||||
E31200917;freelance kerja lepas;Wirausaha Informatika
|
||||
E31161636;web developer;Programmer
|
||||
E31171992;it staff teknisi;Programmer
|
||||
E31200680;founder;Wirausaha Informatika
|
||||
E31191309;freelance kerja lepas;Programmer
|
||||
E31180276;staff;Wirausaha Informatika
|
||||
E31171904;freelance illustrator;Wirausaha Informatika
|
||||
E31200909;founder;Wirausaha Informatika
|
||||
E31181885;staff admin;Non-IT
|
||||
E31181091;management informatika;Programmer
|
||||
E31190197;founder;Wirausaha Informatika
|
||||
E31160901;logistik;Non-IT
|
||||
E31200549;staff;Non-IT
|
||||
E31172055;maintenance and technical support retail;Non-IT
|
||||
E31201372;founder;Wirausaha Informatika
|
||||
E31191982;staff;Non-IT
|
||||
E31190993;staff;Non-IT
|
||||
E31192391;staff;Programmer
|
||||
E31191195;founder;Wirausaha Informatika
|
||||
E31181295;staf;Programmer
|
||||
E31200561;freelance kerja lepas;Data Analyst
|
||||
E31192079;staff;Programmer
|
||||
E31180299;it staff teknisi;Programmer
|
||||
E31192264;founder;Wirausaha Informatika
|
||||
E31180735;programer;Programmer
|
||||
E31202451;staff;Non-IT
|
||||
E31170253;admin;Non-IT
|
||||
E31190397;freelance kerja lepas;Wirausaha Informatika
|
||||
E31201177;freelance kerja lepas;Wirausaha Informatika
|
||||
E31192347;freelance kerja lepas;Wirausaha Informatika
|
||||
E31181452;staff admin;Non-IT
|
||||
E31190731;founder;Programmer
|
||||
E31191821;founder;Wirausaha Informatika
|
||||
E31160852;interviewer;Non-IT
|
||||
E31190596;staff;Non-IT
|
||||
E31180696;founder;Wirausaha Informatika
|
||||
E31201459;freelance kerja lepas;Wirausaha Informatika
|
||||
E31181942;freelance kerja lepas;Programmer
|
||||
E31171849;programmer developer;Programmer
|
||||
E31171103;it staff teknisi;Programmer
|
||||
E31190658;staff;Non-IT
|
||||
E31200910;staff;Non-IT
|
||||
E31200592;guru;Non-IT
|
||||
E31200649;human resource;Non-IT
|
||||
E31170050;it staff teknisi;Programmer
|
||||
E31180365;founder;Wirausaha Informatika
|
||||
E31171799;web programmer;Programmer
|
||||
E31200254;sekretaris kurikulum;Non-IT
|
||||
E31181597;it staff teknisi;Programmer
|
||||
E31170570;software engineer;Programmer
|
||||
E31201690;staff;Non-IT
|
||||
E31202490;freelance kerja lepas;Wirausaha Informatika
|
||||
E31171333;it staff teknisi;Programmer
|
||||
E31190068;staff administrasi produksi;Non-IT
|
||||
E31180112;it staff teknisi;Programmer
|
||||
E31181032;freelance kerja lepas;Wirausaha Informatika
|
||||
E31201228;founder;Wirausaha Informatika
|
||||
E31180837;freelance kerja lepas;Programmer
|
||||
E31200061;marketing officer;Non-IT
|
||||
E31160170;pranata teknologi informasi komputer;Programmer
|
||||
E31192072;staff;Non-IT
|
||||
E31171106;phl;Non-IT
|
||||
E31181567;founder;Wirausaha Informatika
|
||||
E31171281;direktur utama;Wirausaha Informatika
|
||||
E31180260;staff;Non-IT
|
||||
E31180509;freelance kerja lepas;Wirausaha Informatika
|
||||
E31190270;staff;Non-IT
|
||||
E31191250;founder;Wirausaha Informatika
|
||||
E31180464;staff;Wirausaha Informatika
|
||||
E31191708;desainer grafis;Non-IT
|
||||
E31190449;staff;Non-IT
|
||||
E31191272;staff;Wirausaha Informatika
|
||||
E31190679;staff;Non-IT
|
||||
E31200001;freelance kerja lepas;Wirausaha Informatika
|
||||
E31171396;programmer developer;Programmer
|
||||
E31160707;staf programmer;Programmer
|
||||
E31180838;wirausaha founder;Wirausaha Informatika
|
||||
E31180989;information technology services;Programmer
|
||||
E31181280;tenaga;Programmer
|
||||
E31172173;administrator;Non-IT
|
||||
E31202463;staff;Non-IT
|
||||
E31190672;freelance kerja lepas;Programmer
|
||||
E31192116;freelance kerja lepas;Wirausaha Informatika
|
||||
E31192221;freelance kerja lepas;Wirausaha Informatika
|
||||
|
|
|
@ -1,250 +1,251 @@
|
|||
nim;job_text_raw;label
|
||||
E31200864;freelance kerja lepas;Wirausaha Informatika
|
||||
E31202562;staff;Non-IT
|
||||
E31202523;freelance kerja lepas;Wirausaha Informatika
|
||||
E31201845;founder;Wirausaha Informatika
|
||||
E31201199;founder;Wirausaha Informatika
|
||||
E31171680;quality control specialist;Non-IT
|
||||
E31192166;founder;Wirausaha Informatika
|
||||
E31201422;freelance kerja lepas;Wirausaha Informatika
|
||||
E31190648;founder;Programmer
|
||||
E31190939;founder;Wirausaha Informatika
|
||||
E31201950;marketing officer;Non-IT
|
||||
E31190800;staff;Non-IT
|
||||
E31192123;customer engineer;Programmer
|
||||
E31201237;staff;Non-IT
|
||||
E31171791;pranata komputer;Non-IT
|
||||
E31192213;freelance kerja lepas;Programmer
|
||||
E31200102;freelance kerja lepas;Wirausaha Informatika
|
||||
E31170389;field coll;Non-IT
|
||||
E31190785;freelance kerja lepas;Wirausaha Informatika
|
||||
E31192403;freelance kerja lepas;Wirausaha Informatika
|
||||
E31201365;founder;Programmer
|
||||
E31162061;web developer;Programmer
|
||||
E31192304;founder;Wirausaha Informatika
|
||||
E31180306;staff;Data Analyst
|
||||
E31190980;staff;Programmer
|
||||
E31181553;staff admin;Non-IT
|
||||
E31181158;freelance kerja lepas;Programmer
|
||||
E31171868;information technology staff;Non-IT
|
||||
E31200700;staff;Wirausaha Informatika
|
||||
E31192175;petugas barang bukti;Non-IT
|
||||
E31201555;freelance kerja lepas;Wirausaha Informatika
|
||||
E31200251;ppic;Non-IT
|
||||
E31171935;machine operator;Non-IT
|
||||
E31170763;document control specialist;Non-IT
|
||||
E31160622;programmer;Programmer
|
||||
E31201573;staff;Non-IT
|
||||
E31201020;staff;Non-IT
|
||||
E31192043;freelance kerja lepas;Wirausaha Informatika
|
||||
E31192202;admin produksi;Non-IT
|
||||
E31171444;wirausaha founder;Wirausaha Informatika
|
||||
E31180373;staff;Non-IT
|
||||
E31171427;pranata komputer;Non-IT
|
||||
E31182030;freelance kerja lepas;Programmer
|
||||
E31171243;it staff teknisi;Programmer
|
||||
E31201687;staff graphic designer;Non-IT
|
||||
E31190631;staff;Non-IT
|
||||
E31180332;founder;Wirausaha Informatika
|
||||
E31202038;freelance kerja lepas;Wirausaha Informatika
|
||||
E31170217;human resources development officer;Non-IT
|
||||
E31191547;staf sdm parmas;Non-IT
|
||||
E31171076;support specialist;Programmer
|
||||
E31172262;it staff teknisi;Programmer
|
||||
E31170087;design development;Non-IT
|
||||
E31191300;staff;Non-IT
|
||||
E31200238;staff kontrak;Non-IT
|
||||
E31201300;freelance kerja lepas;Wirausaha Informatika
|
||||
E31171431;guru tik;Non-IT
|
||||
E31181991;wirausaha founder;Wirausaha Informatika
|
||||
E31171777;programmer developer;Programmer
|
||||
E31201932;freelance kerja lepas;Wirausaha Informatika
|
||||
E31201954;freelance kerja lepas;Wirausaha Informatika
|
||||
E31191441;freelance kerja lepas;Wirausaha Informatika
|
||||
E31192311;founder;Wirausaha Informatika
|
||||
E31180878;freelance kerja lepas;Programmer
|
||||
E31200293;staff;Non-IT
|
||||
E31180456;it staff teknisi;Programmer
|
||||
E31192292;staff;Non-IT
|
||||
E31171367;office administrator;Non-IT
|
||||
E31180748;freelance kerja lepas;Wirausaha Informatika
|
||||
E31201939;freelance kerja lepas;Programmer
|
||||
E31171151;sales executive;Non-IT
|
||||
E31182101;founder;Wirausaha Informatika
|
||||
E31170734;administrative assistant;Non-IT
|
||||
E31191621;staff;Non-IT
|
||||
E31192416;freelance kerja lepas;Wirausaha Informatika
|
||||
E31192353;founder;Wirausaha Informatika
|
||||
E31171236;it staff teknisi;Programmer
|
||||
E31201647;founder;Wirausaha Informatika
|
||||
E31190527;founder;Wirausaha Informatika
|
||||
E31190801;staff;Non-IT
|
||||
E31190672;freelance kerja lepas;Programmer
|
||||
E31192382;staff;Non-IT
|
||||
E31200245;staff;Non-IT
|
||||
E31191124;founder;Wirausaha Informatika
|
||||
E31160690;graphic designer;Non-IT
|
||||
E31202345;founder;Wirausaha Informatika
|
||||
E31181929;administrasi bmn;Non-IT
|
||||
E31190865;founder;Programmer
|
||||
E31190698;staff;Non-IT
|
||||
E31181100;freelance kerja lepas;Wirausaha Informatika
|
||||
E31201431;freelance kerja lepas;Data Analyst
|
||||
E31171296;cutting machine operator;Non-IT
|
||||
E31191159;founder;Wirausaha Informatika
|
||||
E31191793;founder;Wirausaha Informatika
|
||||
E31200456;data analyst;Data Analyst
|
||||
E31180678;freelance kerja lepas;Wirausaha Informatika
|
||||
E31181204;founder;Wirausaha Informatika
|
||||
E31190408;freelance kerja lepas;Wirausaha Informatika
|
||||
E31191228;it staff teknisi;Programmer
|
||||
E31202278;founder;Wirausaha Informatika
|
||||
E31201773;freelance kerja lepas;Programmer
|
||||
E31191006;freelance kerja lepas;Wirausaha Informatika
|
||||
E31192064;freelance kerja lepas;Wirausaha Informatika
|
||||
E31180207;founder;Wirausaha Informatika
|
||||
E31180513;it staff teknisi;Programmer
|
||||
E31202314;staf pelaksana;Non-IT
|
||||
E31201546;freelance kerja lepas;Wirausaha Informatika
|
||||
E31200888;freelance kerja lepas;Wirausaha Informatika
|
||||
E31191849;founder;Programmer
|
||||
E31200284;staff;Non-IT
|
||||
E31192029;freelance kerja lepas;Programmer
|
||||
E31180638;founder;Wirausaha Informatika
|
||||
E31180539;staff;Non-IT
|
||||
E31181496;programmer developer;Programmer
|
||||
E31192293;freelance kerja lepas;Wirausaha Informatika
|
||||
E31180504;founder;Wirausaha Informatika
|
||||
E31170392;it staff teknisi;Programmer
|
||||
E31201537;freelance kerja lepas;Wirausaha Informatika
|
||||
E31200837;founder;Wirausaha Informatika
|
||||
E31192060;freelance kerja lepas;Wirausaha Informatika
|
||||
E31180256;freelance kerja lepas;Wirausaha Informatika
|
||||
E31171971;it staff teknisi;Programmer
|
||||
E31181474;founder;Wirausaha Informatika
|
||||
E31181955;freelance kerja lepas;Programmer
|
||||
E31191879;founder;Wirausaha Informatika
|
||||
E31170933;tester;Programmer
|
||||
E31201634;staff;Non-IT
|
||||
E31172208;fullstack developer web app;Programmer
|
||||
E31182109;freelance kerja lepas;Wirausaha Informatika
|
||||
E31160524;kepala urusan perencanaan;Non-IT
|
||||
E31191234;staff;Non-IT
|
||||
E31201056;freelance kerja lepas;Wirausaha Informatika
|
||||
E31170906;wirausaha founder;Wirausaha Informatika
|
||||
E31171618;customer service staff;Non-IT
|
||||
E31181049;freelance kerja lepas;Programmer
|
||||
E31160306;admin stnk bpkb;Non-IT
|
||||
E31171470;staf;Non-IT
|
||||
E31190445;staff;Non-IT
|
||||
E31171869;programmer developer;Programmer
|
||||
E31200822;staff;Wirausaha Informatika
|
||||
E31191120;freelance kerja lepas;Programmer
|
||||
E31161169;customer service;Non-IT
|
||||
E31190279;founder;Programmer
|
||||
E31191894;fullstack developer;Programmer
|
||||
E31180703;it staff teknisi;Programmer
|
||||
E31201829;staff;Non-IT
|
||||
E31182077;data analyst;Data Analyst
|
||||
E31200356;founder;Programmer
|
||||
E31180644;freelance kerja lepas;Programmer
|
||||
E31200973;programmer;Programmer
|
||||
E31171313;it staff teknisi;Programmer
|
||||
E31192472;freelance kerja lepas;Wirausaha Informatika
|
||||
E31202319;freelance kerja lepas;Wirausaha Informatika
|
||||
E31191800;checker angkutan barang;Non-IT
|
||||
E31170805;senior account manager;Non-IT
|
||||
E31192024;teknisi laboratorium;Programmer
|
||||
E31192285;founder;Programmer
|
||||
E31191660;founder;Wirausaha Informatika
|
||||
E31191433;founder;Wirausaha Informatika
|
||||
E31160524;kepala urus rencana;Non-IT
|
||||
E31191401;staff;Programmer
|
||||
E31190679;staff;Non-IT
|
||||
E31181567;founder;Wirausaha Informatika
|
||||
E31191677;freelance kerja lepas;Wirausaha Informatika
|
||||
E31161776;web developer;Programmer
|
||||
E31200254;sekretaris kurikulum;Non-IT
|
||||
E31201237;staff;Non-IT
|
||||
E31170087;design development;Non-IT
|
||||
E31192064;freelance kerja lepas;Wirausaha Informatika
|
||||
E31160622;programmer;Programmer
|
||||
E31191394;founder;Wirausaha Informatika
|
||||
E31191234;staff;Non-IT
|
||||
E31201365;founder;Programmer
|
||||
E31160880;teller;Non-IT
|
||||
E31201706;freelance kerja lepas;Wirausaha Informatika
|
||||
E31190341;staff;Programmer
|
||||
E31202279;freelance kerja lepas;Programmer
|
||||
E31190883;staff;Programmer
|
||||
E31200997;freelance kerja lepas;Wirausaha Informatika
|
||||
E31182133;freelance kerja lepas;Programmer
|
||||
E31180707;freelance kerja lepas;Programmer
|
||||
E31180295;freelance kerja lepas;Wirausaha Informatika
|
||||
E31200117;freelance kerja lepas;Wirausaha Informatika
|
||||
E31190134;freelance kerja lepas;Wirausaha Informatika
|
||||
E31180724;lain lain;Non-IT
|
||||
E31190286;staff;Non-IT
|
||||
E31180686;founder;Wirausaha Informatika
|
||||
E31171096;customer service officer;Non-IT
|
||||
E31202345;founder;Wirausaha Informatika
|
||||
E31180299;teller;Non-IT
|
||||
E31170763;document control specialist;Non-IT
|
||||
E31180464;staff;Wirausaha Informatika
|
||||
E31180112;it staff teknisi;Programmer
|
||||
E31200888;freelance kerja lepas;Wirausaha Informatika
|
||||
E31180260;staff;Non-IT
|
||||
E31190070;staff administrator;Non-IT
|
||||
E31191124;founder;Wirausaha Informatika
|
||||
E31201690;staff;Non-IT
|
||||
E31200910;staff;Non-IT
|
||||
E31190957;freelance kerja lepas;Wirausaha Informatika
|
||||
E31170715;administrasi;Non-IT
|
||||
E31200284;staff;Non-IT
|
||||
E31200880;freelance kerja lepas;Wirausaha Informatika
|
||||
E31181885;staff admin;Non-IT
|
||||
E31200549;staff;Non-IT
|
||||
E31180703;it staff teknisi;Programmer
|
||||
E31171367;office administrator;Non-IT
|
||||
E31160910;administration assistant;Wirausaha Informatika
|
||||
E31201459;freelance kerja lepas;Wirausaha Informatika
|
||||
E31201095;freelance kerja lepas;Wirausaha Informatika
|
||||
E31160241;web developer;Programmer
|
||||
E31191460;freelance kerja lepas;Wirausaha Informatika
|
||||
E31200245;staff;Non-IT
|
||||
E31190851;founder;Wirausaha Informatika
|
||||
E31200238;staff kontrak;Non-IT
|
||||
E31190055;staff usaha labuh;Wirausaha Informatika
|
||||
E31200293;staff;Non-IT
|
||||
E31200356;founder;Programmer
|
||||
E31191894;fullstack developer;Programmer
|
||||
E31170392;it staff teknisi;Programmer
|
||||
E31171791;pranata komputer;Non-IT
|
||||
E31201177;freelance kerja lepas;Wirausaha Informatika
|
||||
E31191286;staff;Non-IT
|
||||
E31171680;quality control specialist;Non-IT
|
||||
E31171700;administration;Non-IT
|
||||
E31171618;customer service staff;Non-IT
|
||||
E31180306;fulfillment admin;Data Analyst
|
||||
E31171935;machine operator;Non-IT
|
||||
E31191159;founder;Wirausaha Informatika
|
||||
E31190939;founder;Wirausaha Informatika
|
||||
E31190197;founder;Wirausaha Informatika
|
||||
E31180878;management information system staf;Programmer
|
||||
E31191888;staf;Non-IT
|
||||
E31202319;freelance kerja lepas;Wirausaha Informatika
|
||||
E31181597;cashier;Non-IT
|
||||
E31181991;digital marketing specialist;Wirausaha Informatika
|
||||
E31180513;contact center agent;Non-IT
|
||||
E31171243;it staff teknisi;Programmer
|
||||
E31192304;founder;Wirausaha Informatika
|
||||
E31201431;freelance kerja lepas;Data Analyst
|
||||
E31200001;freelance kerja lepas;Wirausaha Informatika
|
||||
E31192311;founder;Wirausaha Informatika
|
||||
E31200456;data analyst;Data Analyst
|
||||
E31170933;tester;Programmer
|
||||
E31190883;staff;Programmer
|
||||
E31201773;freelance kerja lepas;Programmer
|
||||
E31181100;freelance kerja lepas;Wirausaha Informatika
|
||||
E31200592;guru;Non-IT
|
||||
E31200680;founder;Wirausaha Informatika
|
||||
E31190397;freelance kerja lepas;Wirausaha Informatika
|
||||
E31192391;staff;Programmer
|
||||
E31170372;administrative assistant pdpt upt tik;Non-IT
|
||||
E31180837;designer;Programmer
|
||||
E31190631;staff;Non-IT
|
||||
E31190430;freelance kerja lepas;Wirausaha Informatika
|
||||
E31171296;cutting machine operator;Non-IT
|
||||
E31192353;founder;Wirausaha Informatika
|
||||
E31201537;freelance kerja lepas;Wirausaha Informatika
|
||||
E31191309;freelance kerja lepas;Programmer
|
||||
E31181436;tenaga ajar informatika;Programmer
|
||||
E31171061;back end developer;Programmer
|
||||
E31191228;it staff teknisi;Programmer
|
||||
E31192463;founder;Wirausaha Informatika
|
||||
E31180365;founder;Wirausaha Informatika
|
||||
E31181496;game programmer;Programmer
|
||||
E31191879;founder;Wirausaha Informatika
|
||||
E31191434;staff;Programmer
|
||||
E31171868;information technology staff;Non-IT
|
||||
E31200649;human resource;Non-IT
|
||||
E31191551;freelance kerja lepas;Wirausaha Informatika
|
||||
E31171444;wirausaha founder;Wirausaha Informatika
|
||||
E31192259;owner;Wirausaha Informatika
|
||||
E31170906;wirausaha founder;Wirausaha Informatika
|
||||
E31191250;founder;Wirausaha Informatika
|
||||
E31191116;freelance kerja lepas;Wirausaha Informatika
|
||||
E31201246;digital marketing;Non-IT
|
||||
E31180370;mitra bps;Wirausaha Informatika
|
||||
E31200102;freelance kerja lepas;Wirausaha Informatika
|
||||
E31202394;staff;Non-IT
|
||||
E31190449;staff;Non-IT
|
||||
E31172013;wirausaha founder;Wirausaha Informatika
|
||||
E31190658;staff;Non-IT
|
||||
E31171333;it staff teknisi;Programmer
|
||||
E31181452;management trainer;Non-IT
|
||||
E31170389;field coll;Non-IT
|
||||
E31201939;freelance kerja lepas;Programmer
|
||||
E31181091;management informatika;Programmer
|
||||
E31182235;design director;Programmer
|
||||
E31191433;founder;Wirausaha Informatika
|
||||
E31180373;back office assistant;Non-IT
|
||||
E31190445;staff;Non-IT
|
||||
E31170772;engineer product;Programmer
|
||||
E31171103;it staff teknisi;Programmer
|
||||
E31192079;staff;Programmer
|
||||
E31192281;mitra rider operational;Non-IT
|
||||
E31182077;system analyst;Data Analyst
|
||||
E31162061;web developer;Programmer
|
||||
E31192214;staff;Programmer
|
||||
E31171777;programmer developer;Programmer
|
||||
E31180696;founder;Wirausaha Informatika
|
||||
E31160690;graphic designer;Non-IT
|
||||
E31171257;admin dan digital marketing;Non-IT
|
||||
E31201546;freelance kerja lepas;Wirausaha Informatika
|
||||
E31181373;sosial media and website admin;Non-IT
|
||||
E31202490;freelance kerja lepas;Wirausaha Informatika
|
||||
E31191195;founder;Wirausaha Informatika
|
||||
E31190698;staff;Non-IT
|
||||
E31200997;freelance kerja lepas;Wirausaha Informatika
|
||||
E31171236;it staff teknisi;Programmer
|
||||
E31182299;staff noc;Programmer
|
||||
E31201536;freelance kerja lepas;Programmer
|
||||
E31202536;freelance kerja lepas;Data Analyst
|
||||
E31171061;back end developer;Programmer
|
||||
E31181131;founder;Data Analyst
|
||||
E31200844;staff;Programmer
|
||||
E31192134;freelance kerja lepas;Wirausaha Informatika
|
||||
E31201009;staff;Non-IT
|
||||
E31180370;freelance kerja lepas;Wirausaha Informatika
|
||||
E31181436;staff;Programmer
|
||||
E31180691;founder;Wirausaha Informatika
|
||||
E31170372;administrative assistant pdpt upt tik;Non-IT
|
||||
E31190070;staff administrator;Non-IT
|
||||
E31190416;freelance kerja lepas;Wirausaha Informatika
|
||||
E31191551;freelance kerja lepas;Wirausaha Informatika
|
||||
E31170715;administrasi;Non-IT
|
||||
E31190957;freelance kerja lepas;Wirausaha Informatika
|
||||
E31191919;freelance kerja lepas;Wirausaha Informatika
|
||||
E31180638;founder;Wirausaha Informatika
|
||||
E31200822;staff;Wirausaha Informatika
|
||||
E31200837;founder;Wirausaha Informatika
|
||||
E31190134;freelance kerja lepas;Wirausaha Informatika
|
||||
E31180539;staff;Non-IT
|
||||
E31190893;staff;Non-IT
|
||||
E31180748;freelance kerja lepas;Wirausaha Informatika
|
||||
E31181263;staff;Non-IT
|
||||
E31171971;it staff teknisi;Programmer
|
||||
E31201146;freelance kerja lepas;Wirausaha Informatika
|
||||
E31171350;programmer developer;Programmer
|
||||
E31190302;teknisi laboratorium;Programmer
|
||||
E31180369;freelance kerja lepas;Wirausaha Informatika
|
||||
E31191888;staf;Non-IT
|
||||
E31181373;freelance kerja lepas;Wirausaha Informatika
|
||||
E31191447;freelance kerja lepas;Wirausaha Informatika
|
||||
E31171996;beauty advisor you;Non-IT
|
||||
E31160910;founder;Wirausaha Informatika
|
||||
E31191116;freelance kerja lepas;Wirausaha Informatika
|
||||
E31201794;founder;Wirausaha Informatika
|
||||
E31191849;founder;Programmer
|
||||
E31192293;freelance kerja lepas;Wirausaha Informatika
|
||||
E31181955;support technician;Programmer
|
||||
E31181131;system analyst;Data Analyst
|
||||
E31171076;support specialist;Programmer
|
||||
E31192120;freelance kerja lepas;Wirausaha Informatika
|
||||
E31171096;customer service officer;Non-IT
|
||||
E31180276;staff;Wirausaha Informatika
|
||||
E31171869;programmer developer;Programmer
|
||||
E31201199;founder;Wirausaha Informatika
|
||||
E31191300;staff;Non-IT
|
||||
E31170805;senior account manager;Non-IT
|
||||
E31200912;staff;Non-IT
|
||||
E31192472;freelance kerja lepas;Wirausaha Informatika
|
||||
E31181297;staff;Wirausaha Informatika
|
||||
E31191310;founder;Wirausaha Informatika
|
||||
E31200844;staff;Programmer
|
||||
E31190731;founder;Programmer
|
||||
E31190596;staff;Non-IT
|
||||
E31200864;freelance kerja lepas;Wirausaha Informatika
|
||||
E31190993;staff;Non-IT
|
||||
E31180369;admin;Wirausaha Informatika
|
||||
E31192117;freelance kerja lepas;Wirausaha Informatika
|
||||
E31180297;adminsitrasi marketing;Non-IT
|
||||
E31182140;programer;Programmer
|
||||
E31181553;staff admin;Non-IT
|
||||
E31180884;staff;Programmer
|
||||
E31181158;web developer;Programmer
|
||||
E31170445;it staff teknisi;Programmer
|
||||
E31172262;it staff teknisi;Programmer
|
||||
E31181391;staff;Non-IT
|
||||
E31180256;admin zoom;Non-IT
|
||||
E31170217;human resources development officer;Non-IT
|
||||
E31180222;freelance kerja lepas;Wirausaha Informatika
|
||||
E31170384;customer service;Non-IT
|
||||
E31202463;staff;Non-IT
|
||||
E31180989;information technology services;Programmer
|
||||
E31191708;desainer grafis;Non-IT
|
||||
E31190270;staff;Non-IT
|
||||
E31190279;founder;Programmer
|
||||
E31171313;it staff teknisi;Programmer
|
||||
E31171151;sales executive;Non-IT
|
||||
E31181474;founder;Wirausaha Informatika
|
||||
E31191982;staff;Non-IT
|
||||
E31200917;freelance kerja lepas;Wirausaha Informatika
|
||||
E31181097;call center officer;Non-IT
|
||||
E31191821;founder;Wirausaha Informatika
|
||||
E31192072;staff;Non-IT
|
||||
E31202279;freelance kerja lepas;Programmer
|
||||
E31201845;founder;Wirausaha Informatika
|
||||
E31201056;freelance kerja lepas;Wirausaha Informatika
|
||||
E31192202;admin produksi;Non-IT
|
||||
E31182133;freelance kerja lepas;Programmer
|
||||
E31180456;it staff teknisi;Programmer
|
||||
E31191120;freelance kerja lepas;Programmer
|
||||
E31202339;freelance kerja lepas;Wirausaha Informatika
|
||||
E31190648;founder;Programmer
|
||||
E31190933;staff;Non-IT
|
||||
E31161776;web developer;Programmer
|
||||
E31182235;programmer developer;Programmer
|
||||
E31192221;freelance kerja lepas;Wirausaha Informatika
|
||||
E31180825;staff;Non-IT
|
||||
E31192285;founder;Programmer
|
||||
E31180691;sales executive;Wirausaha Informatika
|
||||
E31200637;freelance kerja lepas;Wirausaha Informatika
|
||||
E31181049;programmer;Programmer
|
||||
E31201555;freelance kerja lepas;Wirausaha Informatika
|
||||
E31171996;beauty advisor you;Non-IT
|
||||
E31161169;customer service;Non-IT
|
||||
E31200918;freelance kerja lepas;Wirausaha Informatika
|
||||
E31191709;staf;Non-IT
|
||||
E31171470;staf;Non-IT
|
||||
E31192033;staff;Programmer
|
||||
E31171431;guru tik;Non-IT
|
||||
E31171350;programmer developer;Programmer
|
||||
E31201647;founder;Wirausaha Informatika
|
||||
E31181280;admin;Programmer
|
||||
E31201794;founder;Wirausaha Informatika
|
||||
E31192347;freelance kerja lepas;Wirausaha Informatika
|
||||
E31160548;senior backend engineer;Programmer
|
||||
E31201228;founder;Wirausaha Informatika
|
||||
E31201158;freelance kerja lepas;Wirausaha Informatika
|
||||
E31191919;freelance kerja lepas;Wirausaha Informatika
|
||||
E31190865;founder;Programmer
|
||||
E31180332;founder;Wirausaha Informatika
|
||||
E31171281;direktur utama;Wirausaha Informatika
|
||||
E31190320;staff;Wirausaha Informatika
|
||||
E31200251;ppic;Non-IT
|
||||
E31190237;staff;Non-IT
|
||||
E31171427;pranata komputer;Non-IT
|
||||
E31182101;founder;Wirausaha Informatika
|
||||
E31192254;freelance kerja lepas;Programmer
|
||||
E31201887;freelance kerja lepas;Programmer
|
||||
E31191310;founder;Wirausaha Informatika
|
||||
E31180575;staff;Programmer
|
||||
E31160548;senior backend engineer;Programmer
|
||||
E31181097;call center officer;Non-IT
|
||||
E31180825;staff;Non-IT
|
||||
E31202394;staff;Non-IT
|
||||
E31201158;freelance kerja lepas;Wirausaha Informatika
|
||||
E31201246;digital marketing;Non-IT
|
||||
E31181391;staff;Non-IT
|
||||
E31172208;fullstack developer web app;Programmer
|
||||
E31171700;administration;Non-IT
|
||||
E31191286;staff;Non-IT
|
||||
E31180949;programmer;Programmer
|
||||
E3117228;programmer developer;Programmer
|
||||
E31190237;staff;Non-IT
|
||||
E31182140;programer;Programmer
|
||||
E31170384;customer service;Non-IT
|
||||
E31192463;founder;Wirausaha Informatika
|
||||
E31181297;staff;Wirausaha Informatika
|
||||
E31191792;admin and finance staff;Non-IT
|
||||
E31172013;wirausaha founder;Wirausaha Informatika
|
||||
E31191709;staf;Non-IT
|
||||
E31191434;staff;Programmer
|
||||
E31200918;freelance kerja lepas;Wirausaha Informatika
|
||||
E31190055;staff usaha pelabuhan;Wirausaha Informatika
|
||||
E31190987;staff administrasi;Non-IT
|
||||
E31192117;freelance kerja lepas;Wirausaha Informatika
|
||||
E31180884;staff;Programmer
|
||||
E31192033;staff;Programmer
|
||||
E31191401;staff;Programmer
|
||||
E31181971;freelance kerja lepas;Wirausaha Informatika
|
||||
E31171317;web developer;Programmer
|
||||
E31192159;staff data;Non-IT
|
||||
E31180297;founder;Wirausaha Informatika
|
||||
E31181263;staff;Non-IT
|
||||
E31192214;staff;Programmer
|
||||
E31190893;staff;Non-IT
|
||||
E31192047;founder;Wirausaha Informatika
|
||||
E31191848;fullstack developer;Programmer
|
||||
E31192281;mitra rider operational;Non-IT
|
||||
E31180303;staff;Non-IT
|
||||
E31171257;admin dan digital marketing;Non-IT
|
||||
E31190320;staff;Wirausaha Informatika
|
||||
E31202339;freelance kerja lepas;Wirausaha Informatika
|
||||
E31202536;freelance kerja lepas;Data Analyst
|
||||
E31171799;web programmer;Programmer
|
||||
E31170734;administrative assistant;Non-IT
|
||||
|
|
|
@ -0,0 +1,177 @@
|
|||
import pandas as pd
|
||||
import numpy as np
|
||||
import re
|
||||
import joblib
|
||||
import json
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
from sklearn.feature_extraction.text import TfidfVectorizer
|
||||
from sklearn.linear_model import LogisticRegression
|
||||
from sklearn.model_selection import StratifiedKFold
|
||||
from sklearn.metrics import classification_report, confusion_matrix, ConfusionMatrixDisplay
|
||||
from sklearn.pipeline import Pipeline
|
||||
import matplotlib
|
||||
matplotlib.use('Agg')
|
||||
import matplotlib.pyplot as plt
|
||||
warnings.filterwarnings('ignore')
|
||||
|
||||
BASE_DIR = Path(__file__).parent
|
||||
TRAIN_FILE = BASE_DIR / "processed" / "training_corpus.csv"
|
||||
TEST_FILE = BASE_DIR / "processed" / "test_set.csv"
|
||||
ML_DIR = BASE_DIR.parent / "fastapi" / "ml_assets"
|
||||
ML_DIR.mkdir(parents=True, exist_ok=True)
|
||||
TARGET_CLASSES = ["Programmer", "Data Analyst", "Wirausaha Informatika", "Non-IT"]
|
||||
CONFIDENCE_THRESHOLD = 0.50
|
||||
|
||||
# [v4] Tidak import Sastrawi — corpus sudah di-stem oleh prepare_corpus.py
|
||||
# Stemming dua kali pada data yang sama dapat mendegradasi teks.
|
||||
STOPWORDS = {
|
||||
"yang", "di", "ke", "dari", "dan", "atau", "dengan", "untuk", "pada", "dalam",
|
||||
"adalah", "ini", "itu", "tidak", "juga", "sudah", "akan", "bisa", "ada", "oleh",
|
||||
"karena", "secara", "serta", "sebagai", "bagi", "telah", "maka", "namun", "sehingga",
|
||||
"jika", "agar", "ketika", "saat", "sebelum", "sesudah", "hingga", "sampai", "antara",
|
||||
"sekitar", "hanya", "saja", "belum", "masih", "lagi", "pun", "justru", "walaupun",
|
||||
"meskipun", "bahkan", "cukup", "sangat", "paling", "lebih", "kurang", "lain",
|
||||
"macam", "cara", "hal", "tentang", "mengenai", "terhadap", "kepada", "menuju",
|
||||
"kecuali", "selain", "tanpa", "demi", "guna", "khususnya", "umumnya", "kebanyakan",
|
||||
"sebagian", "beberapa", "semua", "setiap", "tiap", "satu", "dua", "tiga", "empat",
|
||||
"lima", "enam", "tujuh", "delapan", "sembilan", "sepuluh", "ratus", "ribu", "juta"
|
||||
}
|
||||
|
||||
def preprocess_text(text):
|
||||
"""Pembersihan dasar — tanpa stemming ulang karena korpus sudah di-stem."""
|
||||
if pd.isna(text) or not isinstance(text, str): return ""
|
||||
text = text.lower().strip()
|
||||
if text in {"nan", "none", "null", "-", "0", "", "tidak diisi"}: return ""
|
||||
text = text.replace('-', ' ').replace('/', ' ').replace('_', ' ').replace(',', ' ')
|
||||
text = re.sub(r'[^\w\s]', '', text)
|
||||
text = re.sub(r'\d+', '', text)
|
||||
words = [w for w in text.split() if w not in STOPWORDS and len(w) > 2]
|
||||
return " ".join(words)
|
||||
|
||||
def main():
|
||||
if not TRAIN_FILE.exists():
|
||||
print("File training_corpus.csv belum ada. Jalankan prepare_corpus.py terlebih dahulu."); return
|
||||
|
||||
df_train = pd.read_csv(TRAIN_FILE, sep=";", dtype=str).dropna(subset=["job_text_raw", "label"])
|
||||
df_test = pd.read_csv(TEST_FILE, sep=";", dtype=str).dropna(subset=["job_text_raw", "label"]) if TEST_FILE.exists() else None
|
||||
|
||||
print("Sedang memproses teks (cleaning dasar tanpa stemming redundan)...")
|
||||
X_train = df_train["job_text_raw"].apply(preprocess_text)
|
||||
y_train = df_train["label"]
|
||||
mask = X_train.str.len() > 0
|
||||
X_train, y_train = X_train[mask], y_train[mask]
|
||||
|
||||
X_test, y_test = pd.Series(dtype=str), pd.Series(dtype=str)
|
||||
if df_test is not None:
|
||||
X_test = df_test["job_text_raw"].apply(preprocess_text)
|
||||
y_test = df_test["label"]
|
||||
mask_t = X_test.str.len() > 0
|
||||
X_test, y_test = X_test[mask_t], y_test[mask_t]
|
||||
|
||||
print(f"Jumlah data latih: {len(X_train)} baris | Data uji: {len(X_test)} baris\n")
|
||||
|
||||
# [KEEP v3] K-Fold cross validation — estimasi variance performa model
|
||||
print("Melakukan pengujian K-Fold (5 putaran) pada data latih...")
|
||||
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
|
||||
fold_metrics = []
|
||||
for i, (tr_idx, te_idx) in enumerate(skf.split(X_train, y_train)):
|
||||
model = Pipeline([
|
||||
('tfidf', TfidfVectorizer(max_features=3000, ngram_range=(1,2), sublinear_tf=True, min_df=1)),
|
||||
('clf', LogisticRegression(max_iter=1000, class_weight='balanced', solver='lbfgs'))
|
||||
])
|
||||
model.fit(X_train.iloc[tr_idx], y_train.iloc[tr_idx])
|
||||
y_pred = model.predict(X_train.iloc[te_idx])
|
||||
rep = classification_report(y_train.iloc[te_idx], y_pred, output_dict=True, zero_division=0)
|
||||
fold_metrics.append(rep)
|
||||
print(f" Akurasi putaran ke-{i+1}: {rep['accuracy']:.4f}")
|
||||
|
||||
avg_acc = np.mean([f['accuracy'] for f in fold_metrics])
|
||||
print(f"\n RATA-RATA PENGUJIAN K-FOLD:")
|
||||
print(f" Akurasi Keseluruhan : {avg_acc:.4f} ± {np.std([f['accuracy'] for f in fold_metrics]):.4f}")
|
||||
for cls in TARGET_CLASSES:
|
||||
p = np.mean([f.get(cls, {}).get('precision', 0) for f in fold_metrics])
|
||||
r = np.mean([f.get(cls, {}).get('recall', 0) for f in fold_metrics])
|
||||
f1 = np.mean([f.get(cls, {}).get('f1-score', 0) for f in fold_metrics])
|
||||
print(f" {cls:25} | P: {p:.3f} | R: {r:.3f} | F1: {f1:.3f}")
|
||||
|
||||
print("\n Membuat model final dari seluruh data latih yang tersedia...")
|
||||
final_model = Pipeline([
|
||||
('tfidf', TfidfVectorizer(max_features=3000, ngram_range=(1,2), sublinear_tf=True, min_df=1)),
|
||||
('clf', LogisticRegression(max_iter=1000, class_weight='balanced', solver='lbfgs'))
|
||||
])
|
||||
final_model.fit(X_train, y_train)
|
||||
|
||||
metrics_test = None
|
||||
if len(X_test) > 0:
|
||||
y_pred = final_model.predict(X_test)
|
||||
rep_test = classification_report(y_test, y_pred, output_dict=True, zero_division=0)
|
||||
metrics_test = rep_test
|
||||
|
||||
print("\n HASIL PENGUJIAN PADA DATA TEST:")
|
||||
print(classification_report(y_test, y_pred, zero_division=0))
|
||||
|
||||
for cls in TARGET_CLASSES:
|
||||
sup = rep_test[cls]['support'] if cls in rep_test else 0
|
||||
if sup < 5: print(f"Catatan: Kelas '{cls}' cuma punya {sup} data uji — skor F1-nya mungkin kurang akurat.")
|
||||
|
||||
# [KEEP v3] Confusion matrix
|
||||
cm = confusion_matrix(y_test, y_pred, labels=TARGET_CLASSES)
|
||||
disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=TARGET_CLASSES)
|
||||
disp.plot(cmap='Blues', values_format='d', xticks_rotation=45, colorbar=False)
|
||||
plt.title('Confusion Matrix - Hold-Out Test'); plt.tight_layout()
|
||||
plt.savefig(ML_DIR / 'confusion_matrix_test.png', dpi=300); plt.close()
|
||||
print("Grafik confusion matrix berhasil disimpan ke confusion_matrix_test.png")
|
||||
|
||||
# [BUGFIX] Feature importance per kelas
|
||||
# clf.coef_ diurutkan mengikuti clf.classes_ (alfabetikal sklearn),
|
||||
# BUKAN urutan TARGET_CLASSES yang hardcoded. Pakai clf.classes_ sebagai
|
||||
# iterator agar index i selalu selaras dengan baris koefisien yang benar.
|
||||
vec, clf = final_model.named_steps['tfidf'], final_model.named_steps['clf']
|
||||
feats, coeffs = vec.get_feature_names_out(), clf.coef_
|
||||
print("\n5 KATA PALING BERPENGARUH UNTUK TIAP KELAS:")
|
||||
for i, cls in enumerate(clf.classes_): # <-- clf.classes_, bukan TARGET_CLASSES
|
||||
top = [(feats[j], coeffs[i][j]) for j in coeffs[i].argsort()[-5:][::-1] if coeffs[i][j] > 0]
|
||||
print(f" [{cls}] " + ", ".join([f"{w}({c:.2f})" for w, c in top]))
|
||||
|
||||
# [KEEP v3] Confidence distribution
|
||||
proba = final_model.predict_proba(X_test)
|
||||
max_p = np.max(proba, axis=1)
|
||||
plt.hist(max_p, bins=15, edgecolor='black', alpha=0.7, color='teal')
|
||||
plt.axvline(x=CONFIDENCE_THRESHOLD, color='red', linestyle='--', linewidth=2, label=f'Threshold {CONFIDENCE_THRESHOLD}')
|
||||
plt.xlabel('Confidence Score'); plt.ylabel('Count'); plt.title('Distribusi Confidence Score')
|
||||
plt.legend(); plt.grid(axis='y', alpha=0.3); plt.tight_layout()
|
||||
plt.savefig(ML_DIR / 'confidence_distribution.png', dpi=300); plt.close()
|
||||
below = np.sum(max_p < CONFIDENCE_THRESHOLD)
|
||||
print(f"\nPengecekan Keyakinan Model: {below} dari {len(max_p)} prediksi ({below/len(max_p)*100:.1f}%) di bawah threshold {CONFIDENCE_THRESHOLD}.")
|
||||
|
||||
# [KEEP v3] Daftar prediksi yang meleset
|
||||
mis = np.where(y_test.values != y_pred)[0]
|
||||
if len(mis) > 0:
|
||||
print("\n DAFTAR PREDIKSI YANG MELESET:")
|
||||
for idx in mis[:min(6, len(mis))]:
|
||||
txt = df_test.iloc[idx]['job_text_raw'][:80]
|
||||
print(f" • Seharusnya: {y_test.iloc[idx]:20} | Ditebak: {y_pred[idx]:20} | Teks: '{txt}...'")
|
||||
else:
|
||||
print("\n Semua prediksi pada test set benar.")
|
||||
|
||||
model_path = ML_DIR / "ml_pipeline_internal.pkl"
|
||||
joblib.dump(final_model, model_path)
|
||||
|
||||
metrics = {
|
||||
"methodology": "Internal MIF only",
|
||||
"k_fold": {
|
||||
"accuracy_mean": float(avg_acc),
|
||||
"accuracy_std": float(np.std([f['accuracy'] for f in fold_metrics])),
|
||||
"folds": fold_metrics
|
||||
},
|
||||
"threshold_config": CONFIDENCE_THRESHOLD
|
||||
}
|
||||
if metrics_test: metrics["hold_out_test"] = metrics_test
|
||||
with open(ML_DIR / "metrics_internal_only.json", 'w') as f: json.dump(metrics, f, indent=2)
|
||||
|
||||
print(f"\n Model berhasil disimpan di: {model_path}")
|
||||
print(f"Laporan metrik disimpan di: {ML_DIR / 'metrics_internal_only.json'}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -0,0 +1,384 @@
|
|||
from fastapi import FastAPI, UploadFile, File, HTTPException, BackgroundTasks, Depends, Security
|
||||
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
|
||||
from fastapi.responses import JSONResponse
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import joblib
|
||||
import re
|
||||
import io
|
||||
import json
|
||||
import uuid
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import os
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Any
|
||||
from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
|
||||
|
||||
|
||||
# KONFIGURASI & KONSTANTA
|
||||
|
||||
BASE_DIR = Path(__file__).parent.parent
|
||||
ML_DIR = BASE_DIR / "ml_assets"
|
||||
PIPELINE_PATH = ML_DIR / "ml_pipeline_internal.pkl"
|
||||
STATUS_PATH = ML_DIR / "retrain_status.json"
|
||||
RETRAIN_WORKER = Path(__file__).parent / "retrain_worker.py"
|
||||
TEMP_DIR = ML_DIR / "tmp"
|
||||
TEMP_DIR.mkdir(parents=True, exist_ok=True)
|
||||
CONFIDENCE_THRESHOLD = 0.50
|
||||
TARGET_CLASSES = ["Programmer", "Data Analyst", "Wirausaha Informatika", "Non-IT"]
|
||||
|
||||
KEYWORD_RULES = {
|
||||
"Programmer": ["programmer", "developer", "engineer", "fullstack", "backend", "frontend", "mobile", "android", "ios", "software", "web dev", "coding", "it staff", "teknisi", "sistem informasi", "application", "network", "devops", "qa", "tester", "ui", "ux", "swe"],
|
||||
"Data Analyst": ["data analyst", "analis data", "data science", "business analyst", "research", "statistik", "bi analyst", "reporting", "database", "sql", "etl", "data engineer", "big data", "analyst", "data mining", "machine learning", "data visual", "power bi", "tableau", "looker", "business intelligence", "bi developer", "data warehouse"],
|
||||
"Wirausaha Informatika": ["founder", "owner", "ceo", "wiraswasta", "startup", "freelance", "freelancer", "wirausaha", "bisnis", "usaha mandiri", "konsultan", "co founder", "entrepreneur", "self employed", "owner toko", "usaha", "dagang online", "tokopedia", "shopee", "dropship", "reseller"]
|
||||
}
|
||||
|
||||
F5C_MAP = {"1": "founder owner wirausaha startup", "2": "co-founder partner wirausaha", "3": "staff karyawan pegawai", "4": "freelance kerja lepas lepasan"}
|
||||
F1101_MAP = {"1": "instansi pemerintah dinas kementerian", "2": "non-profit lsm yayasan", "3": "perusahaan swasta corporate", "4": "wiraswasta usaha mandiri", "6": "bumn bumd pemerintah", "7": "multilateral internasional"}
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s | %(levelname)s | %(message)s")
|
||||
logger = logging.getLogger("tracer_worker")
|
||||
stemmer = StemmerFactory().create_stemmer()
|
||||
|
||||
STOPWORDS = {
|
||||
"yang", "di", "ke", "dari", "dan", "atau", "dengan", "untuk", "pada", "dalam",
|
||||
"adalah", "ini", "itu", "tidak", "juga", "sudah", "akan", "bisa", "ada", "oleh",
|
||||
"karena", "secara", "serta", "sebagai", "bagi", "telah", "maka", "namun", "sehingga",
|
||||
"jika", "agar", "ketika", "saat", "sebelum", "sesudah", "hingga", "sampai", "antara",
|
||||
"sekitar", "hanya", "saja", "belum", "masih", "lagi", "pun", "justru", "walaupun",
|
||||
"meskipun", "bahkan", "cukup", "sangat", "paling", "lebih", "kurang", "lain",
|
||||
"macam", "cara", "hal", "tentang", "mengenai", "terhadap", "kepada", "menuju",
|
||||
"kecuali", "selain", "tanpa", "demi", "guna", "khususnya", "umumnya", "kebanyakan",
|
||||
"sebagian", "beberapa", "semua", "setiap", "tiap", "satu", "dua", "tiga", "empat",
|
||||
"lima", "enam", "tujuh", "delapan", "sembilan", "sepuluh", "ratus", "ribu", "juta"
|
||||
}
|
||||
|
||||
|
||||
# MODEL LOADING
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
try:
|
||||
app.state.pipeline = joblib.load(PIPELINE_PATH)
|
||||
logger.info(f" ML Pipeline loaded successfully from {PIPELINE_PATH}")
|
||||
except Exception as e:
|
||||
logger.error(f" Failed to load ML pipeline: {e}")
|
||||
app.state.pipeline = None
|
||||
yield
|
||||
|
||||
app = FastAPI(title="Tracer Study Classification Worker", lifespan=lifespan)
|
||||
|
||||
# --- SECURITY SETUP ---
|
||||
security = HTTPBearer()
|
||||
EXPECTED_TOKEN = os.getenv("FASTAPI_SECRET_KEY", "")
|
||||
|
||||
def verify_token(credentials: HTTPAuthorizationCredentials = Security(security)):
|
||||
if not EXPECTED_TOKEN:
|
||||
return credentials.credentials
|
||||
if credentials.credentials != EXPECTED_TOKEN:
|
||||
raise HTTPException(status_code=401, detail="Unauthorized - Invalid Token")
|
||||
return credentials.credentials
|
||||
# ----------------------
|
||||
|
||||
def clean_text(text: str) -> str:
|
||||
if pd.isna(text) or not isinstance(text, str):
|
||||
return ""
|
||||
text = text.lower().strip()
|
||||
if text in {"nan", "none", "null", "-", "0", "", "tidak diisi"}:
|
||||
return ""
|
||||
text = text.replace('-', ' ').replace('/', ' ').replace('_', ' ').replace(',', ' ')
|
||||
text = re.sub(r'[^\w\s]', '', text)
|
||||
text = re.sub(r'\d+', '', text)
|
||||
words = [w for w in text.split() if w not in STOPWORDS and len(w) > 2]
|
||||
return " ".join([stemmer.stem(w) for w in words])
|
||||
|
||||
def find_column_by_code(df_columns: list, code: str) -> str | None:
|
||||
for col in df_columns:
|
||||
if code.lower() in col.lower():
|
||||
return col
|
||||
return None
|
||||
|
||||
def safe_get(row: pd.Series, df_columns: list, code: str, default: str = "") -> str:
|
||||
actual_col = find_column_by_code(df_columns, code)
|
||||
if actual_col and actual_col in row:
|
||||
val = row[actual_col]
|
||||
if pd.isna(val) or val is None:
|
||||
return default
|
||||
return str(val).strip()
|
||||
return default
|
||||
|
||||
|
||||
# CLASSIFICATION LOGIC
|
||||
|
||||
def classify_rule(job_text: str) -> dict:
|
||||
clean = clean_text(job_text)
|
||||
if not clean or len(clean) < 3:
|
||||
return {"profile": "Tidak Diketahui", "confidence": 0.65, "method": "rule_based_fallback"}
|
||||
|
||||
for profile, keywords in KEYWORD_RULES.items():
|
||||
if any(kw in clean for kw in keywords):
|
||||
return {"profile": profile, "confidence": 0.65, "method": "rule_based"}
|
||||
return None
|
||||
|
||||
def classify_ml(job_text: str, pipeline) -> dict:
|
||||
clean = clean_text(job_text)
|
||||
if not clean:
|
||||
return {"profile": "Tidak Diketahui", "confidence": 0.0, "method": "ml_empty_input"}
|
||||
try:
|
||||
proba = pipeline.predict_proba([clean])[0]
|
||||
max_conf = float(np.max(proba))
|
||||
pred_class = pipeline.classes_[np.argmax(proba)]
|
||||
method = "ml_fallback" if max_conf >= CONFIDENCE_THRESHOLD else "manual_review"
|
||||
return {"profile": pred_class, "confidence": round(max_conf, 4), "method": method}
|
||||
except Exception as e:
|
||||
logger.error(f"ML_INFER_ERROR | job_text='{job_text[:50]}' | err={e}")
|
||||
raise HTTPException(status_code=500, detail=f"ML inference failed: {str(e)}")
|
||||
|
||||
def extract_kemendik_text(row: pd.Series, df_columns: list) -> str:
|
||||
f5b_col = find_column_by_code(df_columns, "f5b")
|
||||
f5c_col = find_column_by_code(df_columns, "f5c")
|
||||
f1101_col = find_column_by_code(df_columns, "f1101")
|
||||
f1102_col = find_column_by_code(df_columns, "f1102")
|
||||
f5b = clean_text(row.get(f5b_col, "") if f5b_col else "")
|
||||
f1102 = clean_text(row.get(f1102_col, "") if f1102_col else "")
|
||||
f5c_raw = str(row.get(f5c_col, "")).strip() if f5c_col else ""
|
||||
f5c_code = re.match(r'^(\d+)', f5c_raw)
|
||||
f5c_text = F5C_MAP.get(f5c_code.group(1), "") if f5c_code else ""
|
||||
f1101_raw = str(row.get(f1101_col, "")).strip() if f1101_col else ""
|
||||
f1101_code = re.match(r'^(\d+)', f1101_raw)
|
||||
f1101_text = F1101_MAP.get(f1101_code.group(1), "") if f1101_code else ""
|
||||
|
||||
return " ".join(p for p in [f5b, f5c_text, f1101_text, f1102] if p).strip()
|
||||
|
||||
|
||||
# SOURCE DETECTION
|
||||
|
||||
def detect_source(df: pd.DataFrame) -> str:
|
||||
|
||||
cols_lower = [c.lower().strip() for c in df.columns]
|
||||
kemendik_codes = ["f5b", "f5c", "f8", "f1101", "nimhsmsmh", "nmmhsmsmh"]
|
||||
for col in cols_lower:
|
||||
if any(code in col for code in kemendik_codes):
|
||||
return "kemendik"
|
||||
if any("jabatan" in c for c in cols_lower):
|
||||
return "internal_mif"
|
||||
|
||||
return "unknown"
|
||||
|
||||
|
||||
# ROUTES
|
||||
@app.get("/health")
|
||||
def health_check():
|
||||
return {"status": "healthy", "pipeline_loaded": app.state.pipeline is not None}
|
||||
|
||||
|
||||
|
||||
# RE-TRAINING ENDPOINTS
|
||||
@app.post("/api/v1/retrain", dependencies=[Depends(verify_token)])
|
||||
async def trigger_retrain(file: UploadFile = File(None)):
|
||||
if STATUS_PATH.exists():
|
||||
try:
|
||||
current = json.loads(STATUS_PATH.read_text())
|
||||
if current.get("stage") in ["started", "backup", "loading_data",
|
||||
"preprocessing", "training", "evaluating", "promoting"]:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Re-training sedang berjalan. Tunggu hingga selesai."
|
||||
)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception:
|
||||
pass
|
||||
extra_csv_path = ""
|
||||
if file and file.filename:
|
||||
job_id = str(uuid.uuid4())[:8]
|
||||
extra_csv_path = str(TEMP_DIR / f"extra_{job_id}.csv")
|
||||
contents = await file.read()
|
||||
Path(extra_csv_path).write_bytes(contents)
|
||||
logger.info(f"Extra CSV disimpan: {extra_csv_path}")
|
||||
|
||||
STATUS_PATH.write_text(json.dumps({
|
||||
"stage": "started",
|
||||
"message": "Memulai proses re-training...",
|
||||
"timestamp": __import__('datetime').datetime.now().isoformat(),
|
||||
}, ensure_ascii=False))
|
||||
|
||||
cmd = [sys.executable, str(RETRAIN_WORKER)]
|
||||
if extra_csv_path:
|
||||
cmd.append(extra_csv_path)
|
||||
|
||||
try:
|
||||
subprocess.Popen(
|
||||
cmd,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
start_new_session=True
|
||||
)
|
||||
logger.info(f"Retrain subprocess spawned: {' '.join(cmd)}")
|
||||
except Exception as e:
|
||||
STATUS_PATH.write_text(json.dumps({
|
||||
"stage": "failed",
|
||||
"message": f"Gagal spawn subprocess: {str(e)}",
|
||||
}, ensure_ascii=False))
|
||||
raise HTTPException(status_code=500, detail=f"Gagal memulai training: {str(e)}")
|
||||
|
||||
return JSONResponse(content={"status": "started", "message": "Re-training dimulai di background."})
|
||||
|
||||
|
||||
@app.get("/api/v1/retrain/status", dependencies=[Depends(verify_token)])
|
||||
def retrain_status():
|
||||
if not STATUS_PATH.exists():
|
||||
return JSONResponse(content={"stage": "idle", "message": "Belum ada proses re-training."})
|
||||
|
||||
try:
|
||||
data = json.loads(STATUS_PATH.read_text())
|
||||
except Exception:
|
||||
return JSONResponse(content={"stage": "unknown", "message": "Status tidak terbaca."})
|
||||
|
||||
terminal_stages = {"promoted", "rolled_back", "failed"}
|
||||
if data.get("stage") in terminal_stages and not data.get("_reloaded"):
|
||||
try:
|
||||
app.state.pipeline = joblib.load(PIPELINE_PATH)
|
||||
data["_reloaded"] = True
|
||||
STATUS_PATH.write_text(json.dumps(data, indent=2, ensure_ascii=False))
|
||||
logger.info(f"Pipeline di-reload setelah retrain (stage={data['stage']})")
|
||||
except Exception as e:
|
||||
logger.error(f"Gagal reload pipeline: {e}")
|
||||
|
||||
return JSONResponse(content=data)
|
||||
|
||||
|
||||
@app.post("/api/v1/retrain/reload", dependencies=[Depends(verify_token)])
|
||||
def reload_model():
|
||||
try:
|
||||
app.state.pipeline = joblib.load(PIPELINE_PATH)
|
||||
logger.info("Pipeline di-reload secara manual.")
|
||||
return JSONResponse(content={"status": "ok", "message": "Model berhasil di-reload."})
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Gagal reload model: {str(e)}")
|
||||
|
||||
@app.post("/api/v1/classify", dependencies=[Depends(verify_token)])
|
||||
async def classify_tracer(file: UploadFile = File(...)):
|
||||
if not file.filename:
|
||||
raise HTTPException(status_code=400, detail="No file provided")
|
||||
|
||||
try:
|
||||
contents = await file.read()
|
||||
|
||||
if file.filename.endswith(".xlsx") or file.filename.endswith(".xls"):
|
||||
try:
|
||||
df = pd.read_excel(io.BytesIO(contents), dtype=str, engine="openpyxl")
|
||||
except ImportError:
|
||||
logger.error("openpyxl not installed")
|
||||
raise HTTPException(status_code=500, detail="Server misconfiguration: openpyxl missing")
|
||||
except Exception as excel_err:
|
||||
logger.error(f"Excel parsing error: {excel_err}")
|
||||
try:
|
||||
df = pd.read_csv(io.BytesIO(contents), sep=";", encoding="utf-8-sig", dtype=str)
|
||||
except:
|
||||
df = pd.read_csv(io.BytesIO(contents), sep=";", encoding="latin-1", dtype=str)
|
||||
else:
|
||||
try:
|
||||
df = pd.read_csv(io.BytesIO(contents), sep=";", encoding="utf-8-sig", dtype=str)
|
||||
except UnicodeDecodeError:
|
||||
df = pd.read_csv(io.BytesIO(contents), sep=";", encoding="latin-1", dtype=str)
|
||||
|
||||
df.columns = df.columns.str.strip()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"File parsing error: {type(e).__name__}: {str(e)}")
|
||||
raise HTTPException(status_code=400, detail=f"Failed to parse file: {type(e).__name__}: {str(e)[:200]}")
|
||||
|
||||
source_type = detect_source(df)
|
||||
if source_type == "unknown":
|
||||
raise HTTPException(status_code=400, detail="Unrecognized file format.")
|
||||
|
||||
if source_type == "internal_mif" and app.state.pipeline is None:
|
||||
logger.error("ML pipeline not loaded")
|
||||
raise HTTPException(status_code=503, detail="ML model not available.")
|
||||
|
||||
results = []
|
||||
pipeline = app.state.pipeline
|
||||
logger.info(f"Processing {len(df)} rows | Source: {source_type}")
|
||||
|
||||
find_col_cached = lambda p: next((c for c in df.columns if p.lower() in c.lower()), None)
|
||||
col_f5a1 = find_col_cached("f5a1")
|
||||
col_f5c = find_col_cached("f5c")
|
||||
col_f1101 = find_col_cached("f1101")
|
||||
col_f5b = find_col_cached("f5b")
|
||||
col_f1102 = find_col_cached("f1102")
|
||||
col_nim_kem = find_col_cached("nimhsmsmh")
|
||||
col_nama_kem = find_col_cached("nmmhsmsmh")
|
||||
col_tahun_kem = find_col_cached("tahun_lulus")
|
||||
col_nim_int = find_col_cached("nim")
|
||||
col_nama_int = find_col_cached("nama_lengkap")
|
||||
col_tahun_int = find_col_cached("tahun_lulus")
|
||||
|
||||
for _, row in df.iterrows():
|
||||
try:
|
||||
def safe_str(val):
|
||||
if pd.isna(val) or val is None: return ""
|
||||
return str(val).strip()
|
||||
|
||||
if source_type == "kemendik":
|
||||
nim = safe_str(row.get(col_nim_kem))
|
||||
nama = safe_str(row.get(col_nama_kem))
|
||||
tahun = safe_str(row.get(col_tahun_kem))
|
||||
else:
|
||||
nim = safe_str(row.get(col_nim_int))
|
||||
nama = safe_str(row.get(col_nama_int))
|
||||
tahun = safe_str(row.get(col_tahun_int))
|
||||
|
||||
if source_type == "kemendik":
|
||||
job_text = extract_kemendik_text(row, df.columns.tolist())
|
||||
raw_data_payload = {str(k): safe_str(v) for k, v in row.to_dict().items()}
|
||||
raw_data_payload.update({
|
||||
"F5a1": safe_str(row.get(col_f5a1)),
|
||||
"F5c": safe_str(row.get(col_f5c)),
|
||||
"F1101": safe_str(row.get(col_f1101)),
|
||||
"F5b": safe_str(row.get(col_f5b)),
|
||||
"F1102": safe_str(row.get(col_f1102))
|
||||
})
|
||||
results.append({
|
||||
"nim": nim, "nama": nama, "tahun_lulus": tahun,
|
||||
"job_text_raw": job_text, "source_type": "kemendik",
|
||||
"predicted_profile": None, "confidence_score": None,
|
||||
"classification_method": "dashboard_only", "status": "processed",
|
||||
"raw_data": raw_data_payload
|
||||
})
|
||||
else:
|
||||
col_jabatan = find_col_cached("jabatan")
|
||||
col_perusahaan = find_col_cached("perusahaan")
|
||||
col_deskripsi = find_col_cached("deskripsi")
|
||||
|
||||
job_text = clean_text(safe_str(row.get(col_jabatan))) if col_jabatan else ""
|
||||
if not job_text and col_perusahaan and col_deskripsi:
|
||||
job_text = clean_text(f"{safe_str(row.get(col_perusahaan))} {safe_str(row.get(col_deskripsi))}".strip())
|
||||
elif not job_text and col_perusahaan:
|
||||
job_text = clean_text(safe_str(row.get(col_perusahaan)))
|
||||
|
||||
rule_res = classify_rule(job_text)
|
||||
res = rule_res or (classify_ml(job_text, pipeline) if pipeline else {"profile": "Non-IT", "confidence": 0.0, "method": "ml_unavailable"})
|
||||
status = "auto_classified" if res["method"] in ["rule_based", "ml_fallback"] or res["profile"] == "Tidak Diketahui" else "needs_review"
|
||||
|
||||
raw_data_payload = {str(k): safe_str(v) for k, v in row.to_dict().items()}
|
||||
|
||||
results.append({
|
||||
"nim": nim, "nama": nama, "tahun_lulus": tahun,
|
||||
"job_text_raw": job_text, "source_type": "internal_mif",
|
||||
"predicted_profile": res["profile"], "confidence_score": res["confidence"],
|
||||
"classification_method": res["method"], "status": status,
|
||||
"raw_data": raw_data_payload
|
||||
})
|
||||
except Exception as row_err:
|
||||
logger.warning(f"Row error: {row_err}")
|
||||
results.append({"nim": nim if 'nim' in locals() else "", "status": "failed", "error_detail": str(row_err)[:100]})
|
||||
|
||||
return JSONResponse(content={"status": "success", "total_rows": len(df), "processed_rows": len(results), "source_type": source_type, "results": results})
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
uvicorn.run("app.main:app", host="127.0.0.1", port=8000, reload=True)
|
||||
|
|
@ -0,0 +1,310 @@
|
|||
from fastapi import FastAPI, UploadFile, File, HTTPException, BackgroundTasks, Depends, Security
|
||||
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
|
||||
from fastapi.responses import JSONResponse
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import joblib
|
||||
import re
|
||||
import io
|
||||
import json
|
||||
import uuid
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import os
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Any
|
||||
from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
|
||||
from filelock import FileLock, Timeout # PENAMBAHAN KUNCI MUTEX
|
||||
|
||||
# KONFIGURASI & KONSTANTA
|
||||
BASE_DIR = Path(__file__).parent.parent
|
||||
ML_DIR = BASE_DIR / "ml_assets"
|
||||
PIPELINE_PATH = ML_DIR / "ml_pipeline_internal.pkl"
|
||||
STATUS_PATH = ML_DIR / "retrain_status.json"
|
||||
LOCK_PATH = ML_DIR / "retrain_status.lock"
|
||||
RETRAIN_WORKER = Path(__file__).parent / "retrain_worker.py"
|
||||
TEMP_DIR = ML_DIR / "tmp"
|
||||
TEMP_DIR.mkdir(parents=True, exist_ok=True)
|
||||
CONFIDENCE_THRESHOLD = 0.50
|
||||
TARGET_CLASSES = ["Programmer", "Data Analyst", "Wirausaha Informatika", "Non-IT"]
|
||||
|
||||
KEYWORD_RULES = {
|
||||
"Programmer": ["programmer", "developer", "engineer", "fullstack", "backend", "frontend", "mobile", "android", "ios", "software", "web dev", "coding", "it staff", "teknisi", "sistem informasi", "application", "network", "devops", "qa", "tester", "ui", "ux", "swe"],
|
||||
"Data Analyst": ["data analyst", "analis data", "data science", "business analyst", "research", "statistik", "bi analyst", "reporting", "database", "sql", "etl", "data engineer", "big data", "analyst", "data mining", "machine learning", "data visual", "power bi", "tableau", "looker", "business intelligence", "bi developer", "data warehouse"],
|
||||
"Wirausaha Informatika": ["founder", "owner", "ceo", "wiraswasta", "startup", "freelance", "freelancer", "wirausaha", "bisnis", "usaha mandiri", "konsultan", "co founder", "entrepreneur", "self employed", "owner toko", "usaha", "dagang online", "tokopedia", "shopee", "dropship", "reseller"]
|
||||
}
|
||||
|
||||
F5C_MAP = {"1": "founder owner wirausaha startup", "2": "co-founder partner wirausaha", "3": "staff karyawan pegawai", "4": "freelance kerja lepas lepasan"}
|
||||
F1101_MAP = {"1": "instansi pemerintah dinas kementerian", "2": "non-profit lsm yayasan", "3": "perusahaan swasta corporate", "4": "wiraswasta usaha mandiri", "6": "bumn bumd pemerintah", "7": "multilateral internasional"}
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s | %(levelname)s | %(message)s")
|
||||
logger = logging.getLogger("tracer_worker")
|
||||
stemmer = StemmerFactory().create_stemmer()
|
||||
|
||||
STOPWORDS = {
|
||||
"yang", "di", "ke", "dari", "dan", "atau", "dengan", "untuk", "pada", "dalam",
|
||||
"adalah", "ini", "itu", "tidak", "juga", "sudah", "akan", "bisa", "ada", "oleh",
|
||||
"karena", "secara", "serta", "sebagai", "bagi", "telah", "maka", "namun", "sehingga",
|
||||
"jika", "agar", "ketika", "saat", "sebelum", "sesudah", "hingga", "sampai", "antara",
|
||||
"sekitar", "hanya", "saja", "belum", "masih", "lagi", "pun", "justru", "walaupun",
|
||||
"meskipun", "bahkan", "cukup", "sangat", "paling", "lebih", "kurang", "lain",
|
||||
"macam", "cara", "hal", "tentang", "mengenai", "terhadap", "kepada", "menuju",
|
||||
"kecuali", "selain", "tanpa", "demi", "guna", "khususnya", "umumnya", "kebanyakan",
|
||||
"sebagian", "beberapa", "semua", "setiap", "tiap", "satu", "dua", "tiga", "empat",
|
||||
"lima", "enam", "tujuh", "delapan", "sembilan", "sepuluh", "ratus", "ribu", "juta"
|
||||
}
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
try:
|
||||
app.state.pipeline = joblib.load(PIPELINE_PATH)
|
||||
logger.info(f" ML Pipeline loaded successfully from {PIPELINE_PATH}")
|
||||
except Exception as e:
|
||||
logger.error(f" Failed to load ML pipeline: {e}")
|
||||
app.state.pipeline = None
|
||||
yield
|
||||
|
||||
app = FastAPI(title="Tracer Study Classification Worker", lifespan=lifespan)
|
||||
|
||||
security = HTTPBearer()
|
||||
EXPECTED_TOKEN = os.getenv("FASTAPI_SECRET_KEY", "")
|
||||
|
||||
def verify_token(credentials: HTTPAuthorizationCredentials = Security(security)):
|
||||
if not EXPECTED_TOKEN: return credentials.credentials
|
||||
if credentials.credentials != EXPECTED_TOKEN:
|
||||
raise HTTPException(status_code=401, detail="Unauthorized - Invalid Token")
|
||||
return credentials.credentials
|
||||
|
||||
def clean_text(text: str) -> str:
|
||||
# Stemmer HANYA berjalan pada saat ada permintaan inferensi baru
|
||||
if pd.isna(text) or not isinstance(text, str): return ""
|
||||
text = text.lower().strip()
|
||||
if text in {"nan", "none", "null", "-", "0", "", "tidak diisi"}: return ""
|
||||
text = text.replace('-', ' ').replace('/', ' ').replace('_', ' ').replace(',', ' ')
|
||||
text = re.sub(r'[^\w\s]', '', text)
|
||||
text = re.sub(r'\d+', '', text)
|
||||
words = [w for w in text.split() if w not in STOPWORDS and len(w) > 2]
|
||||
return " ".join([stemmer.stem(w) for w in words])
|
||||
|
||||
def classify_rule(job_text: str) -> dict:
|
||||
clean = clean_text(job_text)
|
||||
if not clean or len(clean) < 3:
|
||||
return {"profile": "Tidak Diketahui", "confidence": 0.65, "method": "rule_based_fallback"}
|
||||
for profile, keywords in KEYWORD_RULES.items():
|
||||
if any(kw in clean for kw in keywords):
|
||||
return {"profile": profile, "confidence": 0.65, "method": "rule_based"}
|
||||
return None
|
||||
|
||||
def classify_ml(job_text: str, pipeline) -> dict:
|
||||
clean = clean_text(job_text)
|
||||
if not clean: return {"profile": "Tidak Diketahui", "confidence": 0.0, "method": "ml_empty_input"}
|
||||
try:
|
||||
proba = pipeline.predict_proba([clean])[0]
|
||||
max_conf = float(np.max(proba))
|
||||
pred_class = pipeline.classes_[np.argmax(proba)]
|
||||
method = "ml_fallback" if max_conf >= CONFIDENCE_THRESHOLD else "manual_review"
|
||||
return {"profile": pred_class, "confidence": round(max_conf, 4), "method": method}
|
||||
except Exception as e:
|
||||
logger.error(f"ML_INFER_ERROR | job_text='{job_text[:50]}' | err={e}")
|
||||
raise HTTPException(status_code=500, detail=f"ML inference failed: {str(e)}")
|
||||
|
||||
def detect_source(df: pd.DataFrame) -> str:
|
||||
cols_lower = [c.lower().strip() for c in df.columns]
|
||||
kemendik_codes = ["f5b", "f5c", "f8", "f1101", "nimhsmsmh", "nmmhsmsmh"]
|
||||
for col in cols_lower:
|
||||
if any(code in col for code in kemendik_codes): return "kemendik"
|
||||
if any("jabatan" in c for c in cols_lower): return "internal_mif"
|
||||
return "unknown"
|
||||
|
||||
@app.get("/health")
|
||||
def health_check():
|
||||
return {"status": "healthy", "pipeline_loaded": app.state.pipeline is not None}
|
||||
|
||||
@app.post("/api/v1/retrain", dependencies=[Depends(verify_token)])
|
||||
async def trigger_retrain(file: UploadFile = File(None)):
|
||||
lock = FileLock(LOCK_PATH, timeout=2) # Timeout 2 detik jika bentrok
|
||||
try:
|
||||
with lock:
|
||||
if STATUS_PATH.exists():
|
||||
try:
|
||||
current = json.loads(STATUS_PATH.read_text())
|
||||
if current.get("stage") in ["started", "backup", "loading_data", "preprocessing", "training", "evaluating", "promoting"]:
|
||||
raise HTTPException(status_code=409, detail="Re-training sedang berjalan. Tunggu hingga selesai.")
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
extra_csv_path = ""
|
||||
if file and file.filename:
|
||||
job_id = str(uuid.uuid4())[:8]
|
||||
extra_csv_path = str(TEMP_DIR / f"extra_{job_id}.csv")
|
||||
contents = await file.read()
|
||||
Path(extra_csv_path).write_bytes(contents)
|
||||
logger.info(f"Extra CSV disimpan: {extra_csv_path}")
|
||||
|
||||
STATUS_PATH.write_text(json.dumps({
|
||||
"stage": "started",
|
||||
"message": "Memulai proses re-training...",
|
||||
"timestamp": __import__('datetime').datetime.now().isoformat(),
|
||||
}, ensure_ascii=False))
|
||||
|
||||
cmd = [sys.executable, str(RETRAIN_WORKER)]
|
||||
if extra_csv_path: cmd.append(extra_csv_path)
|
||||
|
||||
subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True)
|
||||
logger.info(f"Retrain subprocess spawned: {' '.join(cmd)}")
|
||||
return JSONResponse(content={"status": "started", "message": "Re-training dimulai di background."})
|
||||
|
||||
except Timeout:
|
||||
raise HTTPException(status_code=429, detail="Sistem sibuk. Permintaan retraining lain sedang diproses.")
|
||||
except Exception as e:
|
||||
if not isinstance(e, HTTPException):
|
||||
raise HTTPException(status_code=500, detail=f"Gagal memulai training: {str(e)}")
|
||||
raise e
|
||||
|
||||
@app.get("/api/v1/retrain/status", dependencies=[Depends(verify_token)])
|
||||
def retrain_status():
|
||||
if not STATUS_PATH.exists(): return JSONResponse(content={"stage": "idle", "message": "Belum ada proses re-training."})
|
||||
try:
|
||||
lock = FileLock(LOCK_PATH, timeout=2)
|
||||
with lock:
|
||||
data = json.loads(STATUS_PATH.read_text())
|
||||
terminal_stages = {"promoted", "rolled_back", "failed"}
|
||||
if data.get("stage") in terminal_stages and not data.get("_reloaded"):
|
||||
try:
|
||||
app.state.pipeline = joblib.load(PIPELINE_PATH)
|
||||
data["_reloaded"] = True
|
||||
STATUS_PATH.write_text(json.dumps(data, indent=2, ensure_ascii=False))
|
||||
logger.info("Pipeline di-reload setelah retrain selesai.")
|
||||
except Exception as e:
|
||||
logger.error(f"Gagal reload pipeline: {e}")
|
||||
return JSONResponse(content=data)
|
||||
except Timeout:
|
||||
return JSONResponse(status_code=429, content={"stage": "locked", "message": "Status sedang diupdate sistem."})
|
||||
except Exception:
|
||||
return JSONResponse(content={"stage": "unknown", "message": "Status tidak terbaca."})
|
||||
|
||||
@app.post("/api/v1/classify", dependencies=[Depends(verify_token)])
|
||||
async def classify_tracer(file: UploadFile = File(...)):
|
||||
if not file.filename:
|
||||
raise HTTPException(status_code=400, detail="No file provided")
|
||||
|
||||
try:
|
||||
contents = await file.read()
|
||||
|
||||
if file.filename.endswith(".xlsx") or file.filename.endswith(".xls"):
|
||||
try:
|
||||
df = pd.read_excel(io.BytesIO(contents), dtype=str, engine="openpyxl")
|
||||
except ImportError:
|
||||
logger.error("openpyxl not installed")
|
||||
raise HTTPException(status_code=500, detail="Server misconfiguration: openpyxl missing")
|
||||
except Exception as excel_err:
|
||||
logger.error(f"Excel parsing error: {excel_err}")
|
||||
try:
|
||||
df = pd.read_csv(io.BytesIO(contents), sep=";", encoding="utf-8-sig", dtype=str)
|
||||
except:
|
||||
df = pd.read_csv(io.BytesIO(contents), sep=";", encoding="latin-1", dtype=str)
|
||||
else:
|
||||
try:
|
||||
df = pd.read_csv(io.BytesIO(contents), sep=";", encoding="utf-8-sig", dtype=str)
|
||||
except UnicodeDecodeError:
|
||||
df = pd.read_csv(io.BytesIO(contents), sep=";", encoding="latin-1", dtype=str)
|
||||
|
||||
df.columns = df.columns.str.strip()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"File parsing error: {type(e).__name__}: {str(e)}")
|
||||
raise HTTPException(status_code=400, detail=f"Failed to parse file: {type(e).__name__}: {str(e)[:200]}")
|
||||
|
||||
source_type = detect_source(df)
|
||||
if source_type == "unknown":
|
||||
raise HTTPException(status_code=400, detail="Unrecognized file format.")
|
||||
|
||||
if source_type == "internal_mif" and app.state.pipeline is None:
|
||||
logger.error("ML pipeline not loaded")
|
||||
raise HTTPException(status_code=503, detail="ML model not available.")
|
||||
|
||||
results = []
|
||||
pipeline = app.state.pipeline
|
||||
logger.info(f"Processing {len(df)} rows | Source: {source_type}")
|
||||
|
||||
find_col_cached = lambda p: next((c for c in df.columns if p.lower() in c.lower()), None)
|
||||
col_f5a1 = find_col_cached("f5a1")
|
||||
col_f5c = find_col_cached("f5c")
|
||||
col_f1101 = find_col_cached("f1101")
|
||||
col_f5b = find_col_cached("f5b")
|
||||
col_f1102 = find_col_cached("f1102")
|
||||
col_nim_kem = find_col_cached("nimhsmsmh")
|
||||
col_nama_kem = find_col_cached("nmmhsmsmh")
|
||||
col_tahun_kem = find_col_cached("tahun_lulus")
|
||||
col_nim_int = find_col_cached("nim")
|
||||
col_nama_int = find_col_cached("nama_lengkap")
|
||||
col_tahun_int = find_col_cached("tahun_lulus")
|
||||
|
||||
for _, row in df.iterrows():
|
||||
try:
|
||||
def safe_str(val):
|
||||
if pd.isna(val) or val is None: return ""
|
||||
return str(val).strip()
|
||||
|
||||
if source_type == "kemendik":
|
||||
nim = safe_str(row.get(col_nim_kem))
|
||||
nama = safe_str(row.get(col_nama_kem))
|
||||
tahun = safe_str(row.get(col_tahun_kem))
|
||||
else:
|
||||
nim = safe_str(row.get(col_nim_int))
|
||||
nama = safe_str(row.get(col_nama_int))
|
||||
tahun = safe_str(row.get(col_tahun_int))
|
||||
|
||||
if source_type == "kemendik":
|
||||
job_text = extract_kemendik_text(row, df.columns.tolist())
|
||||
raw_data_payload = {str(k): safe_str(v) for k, v in row.to_dict().items()}
|
||||
raw_data_payload.update({
|
||||
"F5a1": safe_str(row.get(col_f5a1)),
|
||||
"F5c": safe_str(row.get(col_f5c)),
|
||||
"F1101": safe_str(row.get(col_f1101)),
|
||||
"F5b": safe_str(row.get(col_f5b)),
|
||||
"F1102": safe_str(row.get(col_f1102))
|
||||
})
|
||||
results.append({
|
||||
"nim": nim, "nama": nama, "tahun_lulus": tahun,
|
||||
"job_text_raw": job_text, "source_type": "kemendik",
|
||||
"predicted_profile": None, "confidence_score": None,
|
||||
"classification_method": "dashboard_only", "status": "processed",
|
||||
"raw_data": raw_data_payload
|
||||
})
|
||||
else:
|
||||
col_jabatan = find_col_cached("jabatan")
|
||||
col_perusahaan = find_col_cached("perusahaan")
|
||||
col_deskripsi = find_col_cached("deskripsi")
|
||||
|
||||
job_text = clean_text(safe_str(row.get(col_jabatan))) if col_jabatan else ""
|
||||
if not job_text and col_perusahaan and col_deskripsi:
|
||||
job_text = clean_text(f"{safe_str(row.get(col_perusahaan))} {safe_str(row.get(col_deskripsi))}".strip())
|
||||
elif not job_text and col_perusahaan:
|
||||
job_text = clean_text(safe_str(row.get(col_perusahaan)))
|
||||
|
||||
rule_res = classify_rule(job_text)
|
||||
res = rule_res or (classify_ml(job_text, pipeline) if pipeline else {"profile": "Non-IT", "confidence": 0.0, "method": "ml_unavailable"})
|
||||
status = "auto_classified" if res["method"] in ["rule_based", "ml_fallback"] or res["profile"] == "Tidak Diketahui" else "needs_review"
|
||||
|
||||
raw_data_payload = {str(k): safe_str(v) for k, v in row.to_dict().items()}
|
||||
|
||||
results.append({
|
||||
"nim": nim, "nama": nama, "tahun_lulus": tahun,
|
||||
"job_text_raw": job_text, "source_type": "internal_mif",
|
||||
"predicted_profile": res["profile"], "confidence_score": res["confidence"],
|
||||
"classification_method": res["method"], "status": status,
|
||||
"raw_data": raw_data_payload
|
||||
})
|
||||
except Exception as row_err:
|
||||
logger.warning(f"Row error: {row_err}")
|
||||
results.append({"nim": nim if 'nim' in locals() else "", "status": "failed", "error_detail": str(row_err)[:100]})
|
||||
|
||||
return JSONResponse(content={"status": "success", "total_rows": len(df), "processed_rows": len(results), "source_type": source_type, "results": results})
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
uvicorn.run("app.main:app", host="127.0.0.1", port=8000, reload=True)
|
||||
|
|
@ -4,6 +4,10 @@ import re
|
|||
import sys
|
||||
from pathlib import Path
|
||||
from sklearn.model_selection import train_test_split
|
||||
from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
|
||||
|
||||
factory = StemmerFactory()
|
||||
stemmer = factory.create_stemmer()
|
||||
|
||||
BASE_DIR = Path(__file__).parent
|
||||
DATA_DIR = BASE_DIR
|
||||
|
|
@ -57,7 +61,7 @@ def clean_text(text: str) -> str:
|
|||
text = re.sub(r'[^\w\s]', '', text)
|
||||
text = re.sub(r'\d+', '', text)
|
||||
tokens = [w for w in text.split() if w not in COMPANY_STOPWORDS and len(w) >= 3]
|
||||
return " ".join(tokens)
|
||||
return " ".join([stemmer.stem(w) for w in tokens])
|
||||
|
||||
def is_likely_name(text: str, full_name: str) -> bool:
|
||||
if not text or not full_name: return False
|
||||
|
|
@ -79,11 +83,21 @@ def main():
|
|||
df.columns = df.columns.str.strip()
|
||||
col_nim = find_col(df, ["nim"])
|
||||
col_nama = find_col(df, ["nama", "lengkap"])
|
||||
col_jabatan = find_col(df, ["jabatan"])
|
||||
col_jab_lama = find_col(df, ["jabatan"])
|
||||
col_jab_baru = find_col(df, ["jabatan_terupdate"])
|
||||
col_klasifikasi = find_col(df, ["klasifikasi"])
|
||||
col_status = find_col(df, ["status", "kerja"])
|
||||
if not all([col_nim, col_nama, col_jabatan]):
|
||||
|
||||
if not all([col_nim, col_nama, col_jab_lama]):
|
||||
print(" Kolom esensial tidak ditemukan"); print(df.columns.tolist()[:10]); sys.exit(1)
|
||||
|
||||
if col_jab_baru:
|
||||
mask_empty = df[col_jab_baru].isna() | df[col_jab_baru].astype(str).str.strip().str.lower().isin(["", "-", "0", "nan", "none", "null", "tidak diisi", "tidak diketahui"])
|
||||
df["jabatan_final"] = df[col_jab_baru].where(~mask_empty, df[col_jab_lama])
|
||||
else:
|
||||
df["jabatan_final"] = df[col_jab_lama]
|
||||
|
||||
col_jabatan = "jabatan_final"
|
||||
if col_status:
|
||||
blacklist = ["tidah diketahui", "tidak bekerja", "pelajar", "melanjutkan pendidikan", "nan", ""]
|
||||
mask = ~df[col_status].str.lower().str.strip().isin(blacklist)
|
||||
|
|
@ -0,0 +1,152 @@
|
|||
import pandas as pd
|
||||
import numpy as np
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from sklearn.model_selection import train_test_split
|
||||
from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
|
||||
|
||||
factory = StemmerFactory()
|
||||
stemmer = factory.create_stemmer()
|
||||
|
||||
BASE_DIR = Path(__file__).parent
|
||||
DATA_DIR = BASE_DIR
|
||||
OUTPUT_DIR = DATA_DIR / "processed"
|
||||
OUTPUT_DIR.mkdir(exist_ok=True)
|
||||
FILE_INTERNAL = DATA_DIR / "ts_internal_mif.xlsx"
|
||||
SEPARATOR = ";"
|
||||
ENCODING = "utf-8-sig"
|
||||
TARGET_CLASSES = ["Programmer", "Data Analyst", "Wirausaha Informatika", "Non-IT"]
|
||||
KLASIFIKASI_MAP = {
|
||||
"Programmer": "Programmer",
|
||||
"Data Analyst": "Data Analyst",
|
||||
"Wirausaha IT": "Wirausaha Informatika",
|
||||
"Wirausaha": "Wirausaha Informatika",
|
||||
"Non-IT": "Non-IT",
|
||||
"Infokom": None, "Pelajar": None, "Tidak Bekerja": None, "TIdak diketahui": None
|
||||
}
|
||||
KEYWORD_RULES = {
|
||||
"Programmer": ["programmer", "developer", "engineer", "fullstack", "backend", "frontend", "mobile", "android", "ios", "software", "web dev", "coding", "it staff", "teknisi", "sistem informasi", "application", "network", "devops", "qa", "tester", "ui", "ux", "swe"],
|
||||
"Data Analyst": ["data analyst", "analis data", "data science", "business analyst", "research", "statistik", "bi analyst", "reporting", "database", "sql", "etl", "data engineer", "big data", "analyst", "data mining", "machine learning", "data visual", "power bi", "tableau", "looker", "business intelligence", "bi developer", "data warehouse"],
|
||||
"Wirausaha Informatika": ["founder", "owner", "ceo", "wiraswasta", "startup", "freelance", "freelancer", "wirausaha", "bisnis", "usaha mandiri", "konsultan", "co founder", "entrepreneur", "self employed", "owner toko", "usaha", "dagang online", "tokopedia", "shopee", "dropship", "reseller"]
|
||||
}
|
||||
COMPANY_STOPWORDS = {
|
||||
"pt", "cv", "ud", "tbk", "persero", "corp", "inc", "ltd", "koperasi", "bumn", "bumd",
|
||||
"dinas", "kantor", "pemkab", "pemprov", "politeknik", "universitas", "sekolah", "sma", "smk", "sd",
|
||||
"bank", "bpr", "rs", "rumah sakit", "klinik", "apotek", "hotel", "restoran", "cafe", "toko", "konter",
|
||||
"foundation", "yayasan", "perkumpulan", "organisasi", "agency", "studio", "consulting", "group", "holding"
|
||||
}
|
||||
|
||||
def load_data_file(file_path: str) -> pd.DataFrame:
|
||||
ext = Path(file_path).suffix.lower()
|
||||
if ext == '.csv':
|
||||
try: return pd.read_csv(file_path, sep=SEPARATOR, encoding=ENCODING, dtype=str, on_bad_lines='skip', engine='python')
|
||||
except UnicodeDecodeError: return pd.read_csv(file_path, sep=SEPARATOR, encoding='latin1', dtype=str, on_bad_lines='skip', engine='python')
|
||||
elif ext == '.xlsx':
|
||||
return pd.read_excel(file_path, dtype=str)
|
||||
else:
|
||||
raise ValueError(f"Format tidak didukung: {ext}")
|
||||
|
||||
def find_col(df: pd.DataFrame, keywords: list) -> str | None:
|
||||
for col in df.columns:
|
||||
col_clean = str(col).strip().lower()
|
||||
if any(k.strip().lower() in col_clean for k in keywords): return col
|
||||
return None
|
||||
|
||||
def clean_text(text: str) -> str:
|
||||
if pd.isna(text) or str(text).strip().lower() in ["nan", "none", "null", "-", "0", "", "tidak diisi", "tidak diketahui"]: return ""
|
||||
text = str(text).strip().lower()
|
||||
text = re.sub(r'^\d+\s*[-:/]\s*', '', text)
|
||||
text = text.replace('-', ' ').replace('/', ' ').replace('_', ' ').replace(',', ' ')
|
||||
text = re.sub(r'[^\w\s]', '', text)
|
||||
text = re.sub(r'\d+', '', text)
|
||||
tokens = [w for w in text.split() if w not in COMPANY_STOPWORDS and len(w) >= 3]
|
||||
return " ".join([stemmer.stem(w) for w in tokens])
|
||||
|
||||
def is_likely_name(text: str, full_name: str) -> bool:
|
||||
if not text or not full_name: return False
|
||||
text_clean = re.sub(r'[^\w\s]', '', text.lower())
|
||||
name_clean = re.sub(r'[^\w\s]', '', str(full_name).lower())
|
||||
text_parts = set(text_clean.split())
|
||||
name_parts = set(name_clean.split())
|
||||
|
||||
if len(text_parts) == 0: return False
|
||||
# Logika yang lebih ketat: Cek apakah input hanya berisi komponen nama (indikasi bocor nama)
|
||||
if len(text_parts) <= 3 and text_parts.issubset(name_parts):
|
||||
return True
|
||||
return False
|
||||
|
||||
def classify_rule_based(text: str) -> str:
|
||||
if not text or len(text) < 3: return "Non-IT"
|
||||
for profile in ["Programmer", "Data Analyst", "Wirausaha Informatika"]:
|
||||
if any(kw in text for kw in KEYWORD_RULES[profile]): return profile
|
||||
return "Non-IT"
|
||||
|
||||
def main():
|
||||
if not FILE_INTERNAL.exists():
|
||||
print(f"Berkas tidak ditemukan: {FILE_INTERNAL}"); sys.exit(1)
|
||||
print("[1/4] Memuat & Membersihkan Data (Strict Cleaning)...")
|
||||
df = load_data_file(str(FILE_INTERNAL))
|
||||
df.columns = df.columns.str.strip()
|
||||
col_nim = find_col(df, ["nim"])
|
||||
col_nama = find_col(df, ["nama", "lengkap"])
|
||||
col_jab_lama = find_col(df, ["jabatan"])
|
||||
col_jab_baru = find_col(df, ["jabatan_terupdate"])
|
||||
col_klasifikasi = find_col(df, ["klasifikasi"])
|
||||
col_status = find_col(df, ["status", "kerja"])
|
||||
|
||||
if not all([col_nim, col_nama, col_jab_lama]):
|
||||
print(" Kolom esensial tidak ditemukan"); print(df.columns.tolist()[:10]); sys.exit(1)
|
||||
|
||||
if col_jab_baru:
|
||||
mask_empty = df[col_jab_baru].isna() | df[col_jab_baru].astype(str).str.strip().str.lower().isin(["", "-", "0", "nan", "none", "null", "tidak diisi", "tidak diketahui"])
|
||||
df["jabatan_final"] = df[col_jab_baru].where(~mask_empty, df[col_jab_lama])
|
||||
else:
|
||||
df["jabatan_final"] = df[col_jab_lama]
|
||||
|
||||
col_jabatan = "jabatan_final"
|
||||
if col_status:
|
||||
blacklist = ["tidah diketahui", "tidak bekerja", "pelajar", "melanjutkan pendidikan", "nan", ""]
|
||||
mask = ~df[col_status].str.lower().str.strip().isin(blacklist)
|
||||
df = df[mask].copy()
|
||||
|
||||
# Stemming berat terjadi HANYA di sini
|
||||
df["job_text_raw"] = df[col_jabatan].apply(clean_text)
|
||||
|
||||
if col_klasifikasi:
|
||||
fallback_map = {"Programmer": "programmer developer", "Data Analyst": "data analyst", "Wirausaha IT": "wirausaha founder", "Wirausaha": "wirausaha founder", "Non IT": "staff admin", "Infokom": "it staff teknisi", "TIdah diketahui": "", "Pelajar": "", "Tidak Bekerja": ""}
|
||||
empty_mask = df["job_text_raw"] == ""
|
||||
if empty_mask.any(): df.loc[empty_mask, "job_text_raw"] = df.loc[empty_mask, col_klasifikasi].map(fallback_map).fillna("")
|
||||
if col_nama:
|
||||
name_leak_mask = df.apply(lambda row: is_likely_name(row["job_text_raw"], row[col_nama]), axis=1)
|
||||
leaked_count = name_leak_mask.sum()
|
||||
if leaked_count > 0: print(f" Mengabaikan {leaked_count} baris yang terindikasi mengandung nama pribadi.")
|
||||
df = df[~name_leak_mask].copy()
|
||||
if col_klasifikasi:
|
||||
df["label"] = df[col_klasifikasi].str.strip().map(KLASIFIKASI_MAP)
|
||||
missing = df["label"].isna()
|
||||
if missing.any(): df.loc[missing, "label"] = df.loc[missing, "job_text_raw"].apply(classify_rule_based)
|
||||
else: df["label"] = df["job_text_raw"].apply(classify_rule_based)
|
||||
|
||||
df = df[df["job_text_raw"].str.len() >= 3].copy()
|
||||
result = df[[col_nim, col_jabatan, "job_text_raw", "label"]].rename(columns={col_nim: "nim"}).dropna(subset=["nim", "label"]).drop_duplicates(subset=["nim"], keep="first")
|
||||
|
||||
print(f"Memuat {len(result)} data valid (berdasarkan teks pekerjaan)\n")
|
||||
|
||||
cols_out = ["nim", "job_text_raw", "label"]
|
||||
|
||||
# Kita tetap simpan data uji ke CSV (jika butuh untuk sanity check statis), tapi tidak akan digunakan
|
||||
# oleh retrain_worker secara buta lagi.
|
||||
min_count = result["label"].value_counts().min()
|
||||
if min_count < 2: train_df, test_df = train_test_split(result, test_size=0.30, random_state=42)
|
||||
else: train_df, test_df = train_test_split(result, test_size=0.30, stratify=result["label"], random_state=42)
|
||||
|
||||
train_df[cols_out].to_csv(OUTPUT_DIR / "training_corpus_4.csv", index=False, sep=";", encoding=ENCODING)
|
||||
test_df[cols_out].to_csv(OUTPUT_DIR / "test_set_4.csv", index=False, sep=";", encoding=ENCODING)
|
||||
|
||||
print("DATA PELATIHAN (TRAINING SET):")
|
||||
print(train_df["label"].value_counts().to_string())
|
||||
print(f"\nData berhasil disimpan ke {OUTPUT_DIR}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -0,0 +1,394 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
TRACER STUDY - RE-TRAINING WORKER (SUBPROCESS)
|
||||
Dipanggil oleh FastAPI sebagai background subprocess.
|
||||
Alur:
|
||||
1. Backup pkl lama → pkl.bak
|
||||
2. Merge corpus asli + data manual_override baru
|
||||
3. Train model baru (candidate)
|
||||
4. Evaluasi: bandingkan weighted F1-score baru vs lama
|
||||
5. Promote jika lebih baik; rollback jika tidak
|
||||
6. Update retrain_status.json di setiap tahap
|
||||
"""
|
||||
|
||||
import sys
|
||||
import json
|
||||
import shutil
|
||||
import logging
|
||||
import warnings
|
||||
import re
|
||||
import os
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import joblib
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
|
||||
from sklearn.feature_extraction.text import TfidfVectorizer
|
||||
from sklearn.linear_model import LogisticRegression
|
||||
from sklearn.model_selection import StratifiedKFold, train_test_split
|
||||
from sklearn.metrics import classification_report
|
||||
from sklearn.pipeline import Pipeline
|
||||
from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
|
||||
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# KONFIGURASI
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
BASE_DIR = Path(__file__).parent.parent
|
||||
ML_DIR = BASE_DIR / "ml_assets"
|
||||
DATA_DIR = BASE_DIR.parent / "data" / "processed"
|
||||
|
||||
PIPELINE_PATH = ML_DIR / "ml_pipeline_internal.pkl"
|
||||
PIPELINE_BAK_PATH = ML_DIR / "ml_pipeline_internal.pkl.bak"
|
||||
CANDIDATE_PATH = ML_DIR / "ml_pipeline_candidate.pkl"
|
||||
METRICS_PATH = ML_DIR / "metrics_internal_only.json"
|
||||
STATUS_PATH = ML_DIR / "retrain_status.json"
|
||||
CORPUS_PATH = DATA_DIR / "training_corpus_3.csv"
|
||||
TEST_PATH = DATA_DIR / "test_set_3.csv"
|
||||
|
||||
TARGET_CLASSES = ["Programmer", "Data Analyst", "Wirausaha Informatika", "Non-IT"]
|
||||
|
||||
# Threshold: model baru harus lebih baik minimal MIN_IMPROVEMENT dari model lama
|
||||
MIN_IMPROVEMENT_THRESHOLD = 0.01 # 1% weighted F1
|
||||
MAX_REGRESSION_ALLOWED = 0.02 # Toleransi: model baru boleh lebih buruk max 2% (di luar ini = rollback keras)
|
||||
|
||||
STOPWORDS = {
|
||||
"yang", "di", "ke", "dari", "dan", "atau", "dengan", "untuk", "pada", "dalam",
|
||||
"adalah", "ini", "itu", "tidak", "juga", "sudah", "akan", "bisa", "ada", "oleh",
|
||||
"karena", "secara", "serta", "sebagai", "bagi", "telah", "maka", "namun", "sehingga",
|
||||
"jika", "agar", "ketika", "saat", "sebelum", "sesudah", "hingga", "sampai", "antara",
|
||||
"sekitar", "hanya", "saja", "belum", "masih", "lagi", "pun", "justru", "walaupun",
|
||||
"meskipun", "bahkan", "cukup", "sangat", "paling", "lebih", "kurang", "lain",
|
||||
"macam", "cara", "hal", "tentang", "mengenai", "terhadap", "kepada", "menuju",
|
||||
"kecuali", "selain", "tanpa", "demi", "guna", "khususnya", "umumnya", "kebanyakan",
|
||||
"sebagian", "beberapa", "semua", "setiap", "tiap", "satu", "dua", "tiga", "empat",
|
||||
"lima", "enam", "tujuh", "delapan", "sembilan", "sepuluh", "ratus", "ribu", "juta"
|
||||
}
|
||||
|
||||
stemmer = StemmerFactory().create_stemmer()
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# LOGGING
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s | %(levelname)s | %(message)s",
|
||||
handlers=[logging.StreamHandler(sys.stdout)]
|
||||
)
|
||||
logger = logging.getLogger("retrain_worker")
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# STATUS WRITER
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
def write_status(stage: str, message: str, extra: dict = None):
|
||||
payload = {
|
||||
"stage": stage,
|
||||
"message": message,
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
}
|
||||
if extra:
|
||||
payload.update(extra)
|
||||
STATUS_PATH.write_text(json.dumps(payload, indent=2, ensure_ascii=False))
|
||||
logger.info(f"[{stage}] {message}")
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# PREPROCESSING — IDENTIK dengan main.py
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
def preprocess_text(text: str) -> str:
|
||||
if pd.isna(text) or not isinstance(text, str):
|
||||
return ""
|
||||
text = text.lower().strip()
|
||||
if text in {"nan", "none", "null", "-", "0", "", "tidak diisi"}:
|
||||
return ""
|
||||
text = text.replace('-', ' ').replace('/', ' ').replace('_', ' ').replace(',', ' ')
|
||||
text = re.sub(r'[^\w\s]', '', text)
|
||||
text = re.sub(r'\d+', '', text)
|
||||
words = [w for w in text.split() if w not in STOPWORDS and len(w) > 2]
|
||||
return " ".join([stemmer.stem(w) for w in words])
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# EVALUASI MODEL — weighted F1 pada hold-out test
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
def evaluate_model(model, X_test: pd.Series, y_test: pd.Series) -> dict:
|
||||
"""
|
||||
Evaluasi model pada hold-out test set.
|
||||
Return: dict berisi weighted F1, accuracy, dan per-class metrics.
|
||||
"""
|
||||
if len(X_test) == 0:
|
||||
return {"weighted_f1": 0.0, "accuracy": 0.0, "per_class": {}}
|
||||
|
||||
y_pred = model.predict(X_test)
|
||||
report = classification_report(
|
||||
y_test, y_pred,
|
||||
output_dict=True,
|
||||
zero_division=0,
|
||||
labels=TARGET_CLASSES
|
||||
)
|
||||
return {
|
||||
"weighted_f1": round(report.get("weighted avg", {}).get("f1-score", 0.0), 4),
|
||||
"accuracy": round(report.get("accuracy", 0.0), 4),
|
||||
"per_class": {
|
||||
cls: {
|
||||
"precision": round(report.get(cls, {}).get("precision", 0.0), 3),
|
||||
"recall": round(report.get(cls, {}).get("recall", 0.0), 3),
|
||||
"f1": round(report.get(cls, {}).get("f1-score", 0.0), 3),
|
||||
"support": int(report.get(cls, {}).get("support", 0)),
|
||||
}
|
||||
for cls in TARGET_CLASSES
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# AMBIL BASELINE DARI METRICS JSON
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
def get_baseline_f1() -> float:
|
||||
"""
|
||||
Baca weighted F1 model lama dari metrics_internal_only.json.
|
||||
Fallback ke 0 jika file tidak ada.
|
||||
"""
|
||||
try:
|
||||
if METRICS_PATH.exists():
|
||||
m = json.loads(METRICS_PATH.read_text())
|
||||
return float(m.get("hold_out_test", {}).get("weighted avg", {}).get("f1-score", 0.0))
|
||||
except Exception as e:
|
||||
logger.warning(f"Gagal baca baseline metrics: {e}")
|
||||
return 0.0
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# ROLLBACK
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
def do_rollback(reason: str, old_f1: float, new_f1: float):
|
||||
"""Kembalikan pkl aktif ke backup."""
|
||||
# Bersihkan candidate jika ada
|
||||
if CANDIDATE_PATH.exists():
|
||||
CANDIDATE_PATH.unlink()
|
||||
|
||||
# Restore dari backup
|
||||
if PIPELINE_BAK_PATH.exists():
|
||||
shutil.copy2(PIPELINE_BAK_PATH, PIPELINE_PATH)
|
||||
logger.info(f"Rollback berhasil: pkl lama dipulihkan dari .bak")
|
||||
else:
|
||||
logger.warning("File .bak tidak ditemukan, pkl aktif dibiarkan.")
|
||||
|
||||
write_status(
|
||||
stage="rolled_back",
|
||||
message=f"Model lama dipertahankan. {reason}",
|
||||
extra={
|
||||
"result": "rolled_back",
|
||||
"reason": reason,
|
||||
"old_f1": old_f1,
|
||||
"new_f1": new_f1,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# MAIN
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
def main(extra_csv_path: str = None):
|
||||
write_status("started", "Worker dimulai")
|
||||
|
||||
# ── STEP 1: BACKUP ──────────────────────────────────────
|
||||
write_status("backup", "Membuat backup model lama...")
|
||||
if PIPELINE_PATH.exists():
|
||||
shutil.copy2(PIPELINE_PATH, PIPELINE_BAK_PATH)
|
||||
logger.info(f"Backup tersimpan: {PIPELINE_BAK_PATH}")
|
||||
else:
|
||||
write_status("failed", "File pkl aktif tidak ditemukan, tidak bisa backup.")
|
||||
sys.exit(1)
|
||||
|
||||
baseline_f1 = get_baseline_f1()
|
||||
logger.info(f"Baseline weighted F1 (model lama): {baseline_f1:.4f}")
|
||||
|
||||
try:
|
||||
# ── STEP 2: LOAD & MERGE DATA ────────────────────────
|
||||
write_status("loading_data", "Memuat dan menggabungkan data training...")
|
||||
|
||||
if not CORPUS_PATH.exists():
|
||||
raise FileNotFoundError(f"Corpus asli tidak ditemukan: {CORPUS_PATH}")
|
||||
|
||||
df_corpus = pd.read_csv(CORPUS_PATH, sep=";", dtype=str).dropna(subset=["job_text_raw", "label"])
|
||||
logger.info(f"Corpus asli: {len(df_corpus)} baris")
|
||||
|
||||
df_extra = pd.DataFrame()
|
||||
if extra_csv_path and Path(extra_csv_path).exists():
|
||||
df_extra = pd.read_csv(extra_csv_path, sep=";", dtype=str).dropna(subset=["job_text_raw", "label"])
|
||||
# Validasi label
|
||||
valid_labels = set(TARGET_CLASSES)
|
||||
df_extra = df_extra[df_extra["label"].isin(valid_labels)]
|
||||
logger.info(f"Data manual_override: {len(df_extra)} baris valid")
|
||||
|
||||
if len(df_extra) > 0:
|
||||
df_all = pd.concat([df_corpus, df_extra], ignore_index=True)
|
||||
# Deduplicate: preferensikan data extra (manual override) jika job_text_raw sama
|
||||
df_all = df_all.drop_duplicates(subset=["job_text_raw"], keep="last")
|
||||
else:
|
||||
df_all = df_corpus.copy()
|
||||
logger.info("Tidak ada data tambahan — menggunakan corpus asli saja")
|
||||
|
||||
logger.info(f"Total setelah merge & deduplicate: {len(df_all)} baris")
|
||||
|
||||
if len(df_all) < 20:
|
||||
raise ValueError(f"Data terlalu sedikit untuk training: {len(df_all)} baris (minimum 20)")
|
||||
|
||||
# ── STEP 3: PREPROCESSING ────────────────────────────
|
||||
write_status("preprocessing", "Preprocessing teks...")
|
||||
X_all = df_all["job_text_raw"].apply(preprocess_text)
|
||||
y_all = df_all["label"]
|
||||
mask = X_all.str.len() > 0
|
||||
X_all, y_all = X_all[mask], y_all[mask]
|
||||
logger.info(f"Setelah filter kosong: {len(X_all)} baris")
|
||||
|
||||
# ── STEP 4: SPLIT TRAIN / TEST ───────────────────────
|
||||
# Cek apakah test_set_3.csv ada untuk hold-out
|
||||
if TEST_PATH.exists():
|
||||
df_test = pd.read_csv(TEST_PATH, sep=";", dtype=str).dropna(subset=["job_text_raw", "label"])
|
||||
X_test = df_test["job_text_raw"].apply(preprocess_text)
|
||||
y_test = df_test["label"]
|
||||
mask_t = X_test.str.len() > 0
|
||||
X_test, y_test = X_test[mask_t], y_test[mask_t]
|
||||
X_train, y_train = X_all, y_all
|
||||
logger.info(f"Hold-out test dari file: {len(X_test)} baris")
|
||||
else:
|
||||
# Fallback: split 70/30 dari data gabungan
|
||||
X_train, X_test, y_train, y_test = train_test_split(
|
||||
X_all, y_all, test_size=0.3, random_state=42, stratify=y_all
|
||||
)
|
||||
logger.info(f"Hold-out test dari split 30%: {len(X_test)} baris")
|
||||
|
||||
# ── STEP 5: TRAINING ─────────────────────────────────
|
||||
write_status("training", f"Training model baru... ({len(X_train)} sampel training)")
|
||||
|
||||
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
|
||||
fold_metrics = []
|
||||
for i, (tr_idx, te_idx) in enumerate(skf.split(X_train, y_train)):
|
||||
m = Pipeline([
|
||||
('tfidf', TfidfVectorizer(max_features=3000, ngram_range=(1, 2), sublinear_tf=True, min_df=1)),
|
||||
('clf', LogisticRegression(max_iter=1000, class_weight='balanced', solver='lbfgs'))
|
||||
])
|
||||
m.fit(X_train.iloc[tr_idx], y_train.iloc[tr_idx])
|
||||
y_pred = m.predict(X_train.iloc[te_idx])
|
||||
rep = classification_report(y_train.iloc[te_idx], y_pred, output_dict=True, zero_division=0)
|
||||
fold_metrics.append(rep)
|
||||
write_status("training", f"K-Fold selesai: fold {i+1}/5 | acc={rep['accuracy']:.4f}")
|
||||
|
||||
# Train final model on full training set
|
||||
write_status("training", "Training model final pada seluruh data training...")
|
||||
candidate_model = Pipeline([
|
||||
('tfidf', TfidfVectorizer(max_features=3000, ngram_range=(1, 2), sublinear_tf=True, min_df=1)),
|
||||
('clf', LogisticRegression(max_iter=1000, class_weight='balanced', solver='lbfgs'))
|
||||
])
|
||||
candidate_model.fit(X_train, y_train)
|
||||
|
||||
# Simpan sebagai candidate (belum overwrite aktif)
|
||||
joblib.dump(candidate_model, CANDIDATE_PATH)
|
||||
logger.info(f"Candidate model tersimpan: {CANDIDATE_PATH}")
|
||||
|
||||
# ── STEP 6: EVALUASI & KEPUTUSAN ─────────────────────
|
||||
write_status("evaluating", "Mengevaluasi model baru vs model lama...")
|
||||
|
||||
new_metrics = evaluate_model(candidate_model, X_test, y_test)
|
||||
new_f1 = new_metrics["weighted_f1"]
|
||||
delta = new_f1 - baseline_f1
|
||||
|
||||
logger.info(f"Baseline F1 : {baseline_f1:.4f}")
|
||||
logger.info(f"Candidate F1: {new_f1:.4f} (delta: {delta:+.4f})")
|
||||
|
||||
# ── KEPUTUSAN ────────────────────────────────────────
|
||||
if baseline_f1 == 0.0:
|
||||
# Tidak ada baseline (model lama belum pernah dievaluasi) → langsung promote
|
||||
reason_promote = "Tidak ada baseline metrics → model baru dipromote"
|
||||
should_promote = True
|
||||
elif delta >= MIN_IMPROVEMENT_THRESHOLD:
|
||||
should_promote = True
|
||||
reason_promote = f"Model baru lebih baik (+{delta*100:.2f}% weighted F1)"
|
||||
elif delta < -MAX_REGRESSION_ALLOWED:
|
||||
should_promote = False
|
||||
reason_rollback = f"Model baru lebih buruk secara signifikan ({delta*100:.2f}% weighted F1)"
|
||||
else:
|
||||
should_promote = False
|
||||
reason_rollback = (
|
||||
f"Peningkatan tidak signifikan ({delta*100:.2f}% weighted F1, "
|
||||
f"minimum dibutuhkan: +{MIN_IMPROVEMENT_THRESHOLD*100:.1f}%)"
|
||||
)
|
||||
|
||||
if not should_promote:
|
||||
do_rollback(reason_rollback, baseline_f1, new_f1)
|
||||
return
|
||||
|
||||
# ── STEP 7: PROMOTE ──────────────────────────────────
|
||||
write_status("promoting", "Model baru lebih baik — mempromote model baru...")
|
||||
|
||||
# Atomic rename: candidate → aktif
|
||||
shutil.move(str(CANDIDATE_PATH), str(PIPELINE_PATH))
|
||||
logger.info(f"Model baru dipromote: {PIPELINE_PATH}")
|
||||
|
||||
# Update metrics JSON
|
||||
avg_acc = float(np.mean([f["accuracy"] for f in fold_metrics]))
|
||||
y_pred_final = candidate_model.predict(X_test)
|
||||
new_metrics_full = {
|
||||
"methodology": "Internal MIF + manual_override",
|
||||
"retrained_at": datetime.now().isoformat(),
|
||||
"extra_samples_added": len(df_extra),
|
||||
"total_training_samples": len(X_train),
|
||||
"k_fold": {
|
||||
"accuracy_mean": avg_acc,
|
||||
"accuracy_std": float(np.std([f["accuracy"] for f in fold_metrics])),
|
||||
"folds": fold_metrics,
|
||||
},
|
||||
"threshold_config": 0.50,
|
||||
"hold_out_test": classification_report(
|
||||
y_test,
|
||||
y_pred_final,
|
||||
output_dict=True,
|
||||
zero_division=0
|
||||
),
|
||||
"comparison": {
|
||||
"old_weighted_f1": baseline_f1,
|
||||
"new_weighted_f1": new_f1,
|
||||
"delta": round(delta, 4),
|
||||
}
|
||||
}
|
||||
METRICS_PATH.write_text(json.dumps(new_metrics_full, indent=2, ensure_ascii=False))
|
||||
|
||||
|
||||
write_status(
|
||||
stage="promoted",
|
||||
message=f"Model baru berhasil dipromote. {reason_promote}",
|
||||
extra={
|
||||
"result": "promoted",
|
||||
"old_f1": baseline_f1,
|
||||
"new_f1": new_f1,
|
||||
"delta": round(delta, 4),
|
||||
"new_accuracy": new_metrics["accuracy"],
|
||||
"per_class": new_metrics["per_class"],
|
||||
"extra_samples_added": len(df_extra),
|
||||
"total_training_samples": len(X_train),
|
||||
}
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"ERROR saat training: {type(e).__name__}: {e}", exc_info=True)
|
||||
do_rollback(
|
||||
reason=f"Training gagal karena error: {type(e).__name__}: {str(e)[:200]}",
|
||||
old_f1=baseline_f1,
|
||||
new_f1=0.0
|
||||
)
|
||||
# Override stage ke 'failed' agar UI tahu ini bukan rollback biasa
|
||||
status = json.loads(STATUS_PATH.read_text())
|
||||
status["stage"] = "failed"
|
||||
STATUS_PATH.write_text(json.dumps(status, indent=2, ensure_ascii=False))
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Argumen: [extra_csv_path]
|
||||
extra_csv = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
main(extra_csv)
|
||||
|
|
@ -0,0 +1,178 @@
|
|||
#!/usr/bin/env python3
|
||||
import sys
|
||||
import json
|
||||
import shutil
|
||||
import logging
|
||||
import warnings
|
||||
import re
|
||||
import os
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import joblib
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from filelock import FileLock # PENAMBAHAN KUNCI MUTEX
|
||||
|
||||
from sklearn.feature_extraction.text import TfidfVectorizer
|
||||
from sklearn.linear_model import LogisticRegression
|
||||
from sklearn.model_selection import StratifiedKFold, train_test_split
|
||||
from sklearn.metrics import classification_report
|
||||
from sklearn.pipeline import Pipeline
|
||||
from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
|
||||
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
BASE_DIR = Path(__file__).parent.parent
|
||||
ML_DIR = BASE_DIR / "ml_assets"
|
||||
DATA_DIR = BASE_DIR.parent / "data" / "processed"
|
||||
|
||||
PIPELINE_PATH = ML_DIR / "ml_pipeline_internal.pkl"
|
||||
PIPELINE_BAK_PATH = ML_DIR / "ml_pipeline_internal.pkl.bak"
|
||||
CANDIDATE_PATH = ML_DIR / "ml_pipeline_candidate.pkl"
|
||||
METRICS_PATH = ML_DIR / "metrics_internal_only.json"
|
||||
STATUS_PATH = ML_DIR / "retrain_status.json"
|
||||
LOCK_PATH = ML_DIR / "retrain_status.lock"
|
||||
CORPUS_PATH = DATA_DIR / "training_corpus_3.csv"
|
||||
|
||||
TARGET_CLASSES = ["Programmer", "Data Analyst", "Wirausaha Informatika", "Non-IT"]
|
||||
MIN_IMPROVEMENT_THRESHOLD = 0.01
|
||||
MAX_REGRESSION_ALLOWED = 0.02
|
||||
|
||||
STOPWORDS = {
|
||||
"yang", "di", "ke", "dari", "dan", "atau", "dengan", "untuk", "pada", "dalam",
|
||||
"adalah", "ini", "itu", "tidak", "juga", "sudah", "akan", "bisa", "ada", "oleh"
|
||||
}
|
||||
stemmer = StemmerFactory().create_stemmer()
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s | %(levelname)s | %(message)s", handlers=[logging.StreamHandler(sys.stdout)])
|
||||
logger = logging.getLogger("retrain_worker")
|
||||
|
||||
def write_status(stage: str, message: str, extra: dict = None):
|
||||
# MENULIS STATUS SECARA ATOMIK
|
||||
lock = FileLock(LOCK_PATH, timeout=10)
|
||||
with lock:
|
||||
payload = {"stage": stage, "message": message, "timestamp": datetime.now().isoformat()}
|
||||
if extra: payload.update(extra)
|
||||
STATUS_PATH.write_text(json.dumps(payload, indent=2, ensure_ascii=False))
|
||||
logger.info(f"[{stage}] {message}")
|
||||
|
||||
def preprocess_raw_text(text: str) -> str:
|
||||
# Stemming HANYA untuk data manual override baru
|
||||
if pd.isna(text) or not isinstance(text, str): return ""
|
||||
text = text.lower().strip()
|
||||
text = re.sub(r'[^\w\s]', '', text.replace('-', ' ').replace('/', ' '))
|
||||
text = re.sub(r'\d+', '', text)
|
||||
words = [w for w in text.split() if w not in STOPWORDS and len(w) > 2]
|
||||
return " ".join([stemmer.stem(w) for w in words])
|
||||
|
||||
def preprocess_stemmed_text(text: str) -> str:
|
||||
# Pembersihan dasar tanpa memanggil Sastrawi untuk korpus lama
|
||||
if pd.isna(text) or not isinstance(text, str): return ""
|
||||
text = text.lower().strip()
|
||||
text = re.sub(r'[^\w\s]', '', text.replace('-', ' ').replace('/', ' '))
|
||||
words = [w for w in text.split() if w not in STOPWORDS and len(w) > 2]
|
||||
return " ".join(words)
|
||||
|
||||
def evaluate_model(model, X_test: pd.Series, y_test: pd.Series) -> dict:
|
||||
if len(X_test) == 0: return {"weighted_f1": 0.0, "accuracy": 0.0, "per_class": {}}
|
||||
y_pred = model.predict(X_test)
|
||||
report = classification_report(y_test, y_pred, output_dict=True, zero_division=0, labels=TARGET_CLASSES)
|
||||
return {
|
||||
"weighted_f1": round(report.get("weighted avg", {}).get("f1-score", 0.0), 4),
|
||||
"accuracy": round(report.get("accuracy", 0.0), 4),
|
||||
"per_class": {cls: {"f1": round(report.get(cls, {}).get("f1-score", 0.0), 3)} for cls in TARGET_CLASSES}
|
||||
}
|
||||
|
||||
def get_baseline_f1() -> float:
|
||||
try:
|
||||
if METRICS_PATH.exists():
|
||||
return float(json.loads(METRICS_PATH.read_text()).get("hold_out_test", {}).get("weighted avg", {}).get("f1-score", 0.0))
|
||||
except Exception: pass
|
||||
return 0.0
|
||||
|
||||
def do_rollback(reason: str, old_f1: float, new_f1: float):
|
||||
if CANDIDATE_PATH.exists(): CANDIDATE_PATH.unlink()
|
||||
if PIPELINE_BAK_PATH.exists():
|
||||
shutil.copy2(PIPELINE_BAK_PATH, PIPELINE_PATH)
|
||||
write_status("rolled_back", f"Model lama dipertahankan. {reason}", {"result": "rolled_back", "old_f1": old_f1, "new_f1": new_f1})
|
||||
|
||||
def main(extra_csv_path: str = None):
|
||||
write_status("started", "Worker dimulai")
|
||||
write_status("backup", "Membuat backup model lama...")
|
||||
if PIPELINE_PATH.exists(): shutil.copy2(PIPELINE_PATH, PIPELINE_BAK_PATH)
|
||||
|
||||
baseline_f1 = get_baseline_f1()
|
||||
|
||||
try:
|
||||
write_status("loading_data", "Memuat dan menggabungkan data training...")
|
||||
if not CORPUS_PATH.exists(): raise FileNotFoundError(f"Corpus asli tidak ditemukan: {CORPUS_PATH}")
|
||||
|
||||
df_corpus = pd.read_csv(CORPUS_PATH, sep=";", dtype=str).dropna(subset=["job_text_raw", "label"])
|
||||
df_corpus["features"] = df_corpus["job_text_raw"].apply(preprocess_stemmed_text)
|
||||
|
||||
df_extra = pd.DataFrame()
|
||||
if extra_csv_path and Path(extra_csv_path).exists():
|
||||
df_extra = pd.read_csv(extra_csv_path, sep=";", dtype=str).dropna(subset=["job_text_raw", "label"])
|
||||
df_extra = df_extra[df_extra["label"].isin(TARGET_CLASSES)]
|
||||
df_extra["features"] = df_extra["job_text_raw"].apply(preprocess_raw_text)
|
||||
|
||||
# Gabungkan tanpa drop_duplicates kasar. Model butuh tahu frekuensi/probabilitas kelas untuk teks yang bias.
|
||||
df_all = pd.concat([df_corpus, df_extra], ignore_index=True)
|
||||
mask = df_all["features"].str.len() > 0
|
||||
df_all = df_all[mask]
|
||||
|
||||
if len(df_all) < 20: raise ValueError("Data terlalu sedikit (minimum 20 baris).")
|
||||
|
||||
X_all = df_all["features"]
|
||||
y_all = df_all["label"]
|
||||
|
||||
# SPLIT DINAMIS BARU: Tidak ada lagi static test-set.
|
||||
X_train, X_test, y_train, y_test = train_test_split(X_all, y_all, test_size=0.3, random_state=42, stratify=y_all)
|
||||
|
||||
write_status("training", "Training model final pada data gabungan dinamis...")
|
||||
candidate_model = Pipeline([
|
||||
('tfidf', TfidfVectorizer(max_features=3000, ngram_range=(1, 2), sublinear_tf=True, min_df=1)),
|
||||
('clf', LogisticRegression(max_iter=1000, class_weight='balanced', solver='lbfgs'))
|
||||
])
|
||||
candidate_model.fit(X_train, y_train)
|
||||
joblib.dump(candidate_model, CANDIDATE_PATH)
|
||||
|
||||
write_status("evaluating", "Mengevaluasi model baru vs model lama...")
|
||||
new_metrics = evaluate_model(candidate_model, X_test, y_test)
|
||||
new_f1 = new_metrics["weighted_f1"]
|
||||
delta = new_f1 - baseline_f1
|
||||
|
||||
if baseline_f1 == 0.0 or delta >= MIN_IMPROVEMENT_THRESHOLD:
|
||||
should_promote, reason = True, "Model baru dipromote (Peningkatan Signifikan)"
|
||||
elif delta < -MAX_REGRESSION_ALLOWED:
|
||||
should_promote, reason = False, "Regresi skor terlalu tinggi. Rollback."
|
||||
else:
|
||||
should_promote, reason = False, "Peningkatan tidak memenuhi threshold minimal. Rollback."
|
||||
|
||||
if not should_promote:
|
||||
do_rollback(reason, baseline_f1, new_f1)
|
||||
return
|
||||
|
||||
write_status("promoting", "Mem-promote model baru...")
|
||||
shutil.move(str(CANDIDATE_PATH), str(PIPELINE_PATH))
|
||||
|
||||
new_metrics_full = {
|
||||
"methodology": "Dynamic Split Retrain",
|
||||
"comparison": {"old_weighted_f1": baseline_f1, "new_weighted_f1": new_f1, "delta": round(delta, 4)},
|
||||
"hold_out_test": classification_report(y_test, candidate_model.predict(X_test), output_dict=True, zero_division=0)
|
||||
}
|
||||
METRICS_PATH.write_text(json.dumps(new_metrics_full, indent=2, ensure_ascii=False))
|
||||
write_status("promoted", f"Selesai. {reason}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"ERROR: {e}", exc_info=True)
|
||||
do_rollback(f"Error sistem: {str(e)[:100]}", baseline_f1, 0.0)
|
||||
lock = FileLock(LOCK_PATH, timeout=10)
|
||||
with lock:
|
||||
status = json.loads(STATUS_PATH.read_text())
|
||||
status["stage"] = "failed"
|
||||
STATUS_PATH.write_text(json.dumps(status, indent=2, ensure_ascii=False))
|
||||
sys.exit(1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(sys.argv[1] if len(sys.argv) > 1 else None)
|
||||
|
|
@ -0,0 +1,114 @@
|
|||
import pandas as pd
|
||||
import numpy as np
|
||||
import re
|
||||
import joblib
|
||||
import json
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
from sklearn.feature_extraction.text import TfidfVectorizer
|
||||
from sklearn.linear_model import LogisticRegression
|
||||
from sklearn.model_selection import StratifiedKFold
|
||||
from sklearn.metrics import classification_report, confusion_matrix, ConfusionMatrixDisplay
|
||||
from sklearn.pipeline import Pipeline
|
||||
import matplotlib
|
||||
matplotlib.use('Agg')
|
||||
import matplotlib.pyplot as plt
|
||||
warnings.filterwarnings('ignore')
|
||||
|
||||
BASE_DIR = Path(__file__).parent
|
||||
TRAIN_FILE = BASE_DIR / "processed" / "training_corpus_4.csv"
|
||||
TEST_FILE = BASE_DIR / "processed" / "test_set_4.csv"
|
||||
ML_DIR = BASE_DIR.parent / "fastapi" / "ml_assets"
|
||||
ML_DIR.mkdir(parents=True, exist_ok=True)
|
||||
TARGET_CLASSES = ["Programmer", "Data Analyst", "Wirausaha Informatika", "Non-IT"]
|
||||
CONFIDENCE_THRESHOLD = 0.50
|
||||
|
||||
STOPWORDS = {
|
||||
"yang", "di", "ke", "dari", "dan", "atau", "dengan", "untuk", "pada", "dalam",
|
||||
"adalah", "ini", "itu", "tidak", "juga", "sudah", "akan", "bisa", "ada", "oleh",
|
||||
"karena", "secara", "serta", "sebagai", "bagi", "telah", "maka", "namun", "sehingga",
|
||||
"jika", "agar", "ketika", "saat", "sebelum", "sesudah", "hingga", "sampai", "antara",
|
||||
"sekitar", "hanya", "saja", "belum", "masih", "lagi", "pun", "justru", "walaupun",
|
||||
"meskipun", "bahkan", "cukup", "sangat", "paling", "lebih", "kurang", "lain",
|
||||
"macam", "cara", "hal", "tentang", "mengenai", "terhadap", "kepada", "menuju",
|
||||
"kecuali", "selain", "tanpa", "demi", "guna", "khususnya", "umumnya", "kebanyakan",
|
||||
"sebagian", "beberapa", "semua", "setiap", "tiap", "satu", "dua", "tiga", "empat",
|
||||
"lima", "enam", "tujuh", "delapan", "sembilan", "sepuluh", "ratus", "ribu", "juta"
|
||||
}
|
||||
|
||||
def preprocess_text(text):
|
||||
# Tidak ada Sastrawi! Teks korpus SUDAH di-stem.
|
||||
if pd.isna(text) or not isinstance(text, str): return ""
|
||||
text = text.lower().strip()
|
||||
if text in {"nan", "none", "null", "-", "0", "", "tidak diisi"}: return ""
|
||||
text = text.replace('-', ' ').replace('/', ' ').replace('_', ' ').replace(',', ' ')
|
||||
text = re.sub(r'[^\w\s]', '', text)
|
||||
text = re.sub(r'\d+', '', text)
|
||||
words = [w for w in text.split() if w not in STOPWORDS and len(w) > 2]
|
||||
return " ".join(words)
|
||||
|
||||
def main():
|
||||
if not TRAIN_FILE.exists():
|
||||
print("File training_corpus_4.csv belum ada"); return
|
||||
df_train = pd.read_csv(TRAIN_FILE, sep=";", dtype=str).dropna(subset=["job_text_raw", "label"])
|
||||
df_test = pd.read_csv(TEST_FILE, sep=";", dtype=str).dropna(subset=["job_text_raw", "label"]) if TEST_FILE.exists() else None
|
||||
|
||||
print("Sedang memproses teks (cleaning dasar tanpa stemming redundan)...")
|
||||
X_train = df_train["job_text_raw"].apply(preprocess_text)
|
||||
y_train = df_train["label"]
|
||||
mask = X_train.str.len() > 0
|
||||
X_train, y_train = X_train[mask], y_train[mask]
|
||||
|
||||
X_test, y_test = pd.Series(dtype=str), pd.Series(dtype=str)
|
||||
if df_test is not None:
|
||||
X_test = df_test["job_text_raw"].apply(preprocess_text)
|
||||
y_test = df_test["label"]
|
||||
mask_t = X_test.str.len() > 0
|
||||
X_test, y_test = X_test[mask_t], y_test[mask_t]
|
||||
|
||||
print(f"Jumlah data latih: {len(X_train)} baris | Data uji: {len(X_test)} baris\n")
|
||||
|
||||
print("Melakukan pengujian K-Fold (5 putaran) pada data latih...")
|
||||
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
|
||||
fold_metrics = []
|
||||
for i, (tr_idx, te_idx) in enumerate(skf.split(X_train, y_train)):
|
||||
model = Pipeline([
|
||||
('tfidf', TfidfVectorizer(max_features=3000, ngram_range=(1,2), sublinear_tf=True, min_df=1)),
|
||||
('clf', LogisticRegression(max_iter=1000, class_weight='balanced', solver='lbfgs'))
|
||||
])
|
||||
model.fit(X_train.iloc[tr_idx], y_train.iloc[tr_idx])
|
||||
y_pred = model.predict(X_train.iloc[te_idx])
|
||||
rep = classification_report(y_train.iloc[te_idx], y_pred, output_dict=True, zero_division=0)
|
||||
fold_metrics.append(rep)
|
||||
print(f" Akurasi putaran ke-{i+1}: {rep['accuracy']:.4f}")
|
||||
|
||||
avg_acc = np.mean([f['accuracy'] for f in fold_metrics])
|
||||
print(f"\n RATA-RATA PENGUJIAN K-FOLD:")
|
||||
print(f" Akurasi Keseluruhan : {avg_acc:.4f} ± {np.std([f['accuracy'] for f in fold_metrics]):.4f}")
|
||||
|
||||
print("\n Membuat model final dari seluruh data latih yang tersedia...")
|
||||
final_model = Pipeline([
|
||||
('tfidf', TfidfVectorizer(max_features=3000, ngram_range=(1,2), sublinear_tf=True, min_df=1)),
|
||||
('clf', LogisticRegression(max_iter=1000, class_weight='balanced', solver='lbfgs'))
|
||||
])
|
||||
final_model.fit(X_train, y_train)
|
||||
|
||||
metrics_test = None
|
||||
if len(X_test) > 0:
|
||||
y_pred = final_model.predict(X_test)
|
||||
rep_test = classification_report(y_test, y_pred, output_dict=True, zero_division=0)
|
||||
metrics_test = rep_test
|
||||
print("\n HASIL PENGUJIAN PADA DATA TEST:")
|
||||
print(classification_report(y_test, y_pred, zero_division=0))
|
||||
|
||||
model_path = ML_DIR / "ml_pipeline_internal.pkl"
|
||||
joblib.dump(final_model, model_path)
|
||||
|
||||
metrics = {"methodology": "Internal MIF only", "k_fold": {"accuracy_mean": float(avg_acc), "accuracy_std": float(np.std([f['accuracy'] for f in fold_metrics])), "folds": fold_metrics}, "threshold_config": CONFIDENCE_THRESHOLD}
|
||||
if metrics_test: metrics["hold_out_test"] = metrics_test
|
||||
|
||||
with open(ML_DIR / "metrics_internal_only.json", 'w') as f: json.dump(metrics, f, indent=2)
|
||||
print(f"\n Model berhasil disimpan di: {model_path}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
from fastapi import FastAPI, UploadFile, File, HTTPException, BackgroundTasks, Depends, Security
|
||||
from fastapi import FastAPI, UploadFile, File, HTTPException, Depends, Security, BackgroundTasks
|
||||
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
|
||||
from fastapi.responses import JSONResponse
|
||||
import pandas as pd
|
||||
|
|
@ -8,7 +8,6 @@ import re
|
|||
import io
|
||||
import json
|
||||
import uuid
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import os
|
||||
|
|
@ -17,17 +16,21 @@ from contextlib import asynccontextmanager
|
|||
from pathlib import Path
|
||||
from typing import List, Dict, Any
|
||||
from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
|
||||
from filelock import FileLock, Timeout
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# KONFIGURASI & KONSTANTA
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
BASE_DIR = Path(__file__).parent.parent
|
||||
ML_DIR = BASE_DIR / "ml_assets"
|
||||
PIPELINE_PATH = ML_DIR / "ml_pipeline_internal.pkl"
|
||||
STATUS_PATH = ML_DIR / "retrain_status.json"
|
||||
RETRAIN_WORKER = Path(__file__).parent / "retrain_worker.py"
|
||||
TEMP_DIR = ML_DIR / "tmp"
|
||||
PIPELINE_PATH = ML_DIR / "ml_pipeline_internal.pkl"
|
||||
STATUS_PATH = ML_DIR / "retrain_status.json"
|
||||
LOCK_PATH = ML_DIR / "retrain_status.lock"
|
||||
RETRAIN_WORKER = Path(__file__).parent / "retrain_worker.py"
|
||||
TEMP_DIR = ML_DIR / "tmp"
|
||||
TEMP_DIR.mkdir(parents=True, exist_ok=True)
|
||||
JOBS_DIR = ML_DIR / "jobs"
|
||||
JOBS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
CONFIDENCE_THRESHOLD = 0.50
|
||||
TARGET_CLASSES = ["Programmer", "Data Analyst", "Wirausaha Informatika", "Non-IT"]
|
||||
|
||||
|
|
@ -36,8 +39,7 @@ KEYWORD_RULES = {
|
|||
"Data Analyst": ["data analyst", "analis data", "data science", "business analyst", "research", "statistik", "bi analyst", "reporting", "database", "sql", "etl", "data engineer", "big data", "analyst", "data mining", "machine learning", "data visual", "power bi", "tableau", "looker", "business intelligence", "bi developer", "data warehouse"],
|
||||
"Wirausaha Informatika": ["founder", "owner", "ceo", "wiraswasta", "startup", "freelance", "freelancer", "wirausaha", "bisnis", "usaha mandiri", "konsultan", "co founder", "entrepreneur", "self employed", "owner toko", "usaha", "dagang online", "tokopedia", "shopee", "dropship", "reseller"]
|
||||
}
|
||||
|
||||
F5C_MAP = {"1": "founder owner wirausaha startup", "2": "co-founder partner wirausaha", "3": "staff karyawan pegawai", "4": "freelance kerja lepas lepasan"}
|
||||
F5C_MAP = {"1": "founder owner wirausaha startup", "2": "co-founder partner wirausaha", "3": "staff karyawan pegawai", "4": "freelance kerja lepas lepasan"}
|
||||
F1101_MAP = {"1": "instansi pemerintah dinas kementerian", "2": "non-profit lsm yayasan", "3": "perusahaan swasta corporate", "4": "wiraswasta usaha mandiri", "6": "bumn bumd pemerintah", "7": "multilateral internasional"}
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s | %(levelname)s | %(message)s")
|
||||
|
|
@ -57,8 +59,9 @@ STOPWORDS = {
|
|||
"lima", "enam", "tujuh", "delapan", "sembilan", "sepuluh", "ratus", "ribu", "juta"
|
||||
}
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# MODEL LOADING
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
try:
|
||||
|
|
@ -71,24 +74,25 @@ async def lifespan(app: FastAPI):
|
|||
|
||||
app = FastAPI(title="Tracer Study Classification Worker", lifespan=lifespan)
|
||||
|
||||
# --- SECURITY SETUP ---
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# SECURITY
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
security = HTTPBearer()
|
||||
EXPECTED_TOKEN = os.getenv("FASTAPI_SECRET_KEY", "")
|
||||
|
||||
def verify_token(credentials: HTTPAuthorizationCredentials = Security(security)):
|
||||
if not EXPECTED_TOKEN:
|
||||
return credentials.credentials
|
||||
if not EXPECTED_TOKEN: return credentials.credentials
|
||||
if credentials.credentials != EXPECTED_TOKEN:
|
||||
raise HTTPException(status_code=401, detail="Unauthorized - Invalid Token")
|
||||
return credentials.credentials
|
||||
# ----------------------
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# TEXT PROCESSING
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
def clean_text(text: str) -> str:
|
||||
if pd.isna(text) or not isinstance(text, str):
|
||||
return ""
|
||||
if pd.isna(text) or not isinstance(text, str): return ""
|
||||
text = text.lower().strip()
|
||||
if text in {"nan", "none", "null", "-", "0", "", "tidak diisi"}:
|
||||
return ""
|
||||
if text in {"nan", "none", "null", "-", "0", "", "tidak diisi"}: return ""
|
||||
text = text.replace('-', ' ').replace('/', ' ').replace('_', ' ').replace(',', ' ')
|
||||
text = re.sub(r'[^\w\s]', '', text)
|
||||
text = re.sub(r'\d+', '', text)
|
||||
|
|
@ -97,27 +101,24 @@ def clean_text(text: str) -> str:
|
|||
|
||||
def find_column_by_code(df_columns: list, code: str) -> str | None:
|
||||
for col in df_columns:
|
||||
if code.lower() in col.lower():
|
||||
return col
|
||||
if code.lower() in col.lower(): return col
|
||||
return None
|
||||
|
||||
def safe_get(row: pd.Series, df_columns: list, code: str, default: str = "") -> str:
|
||||
actual_col = find_column_by_code(df_columns, code)
|
||||
if actual_col and actual_col in row:
|
||||
val = row[actual_col]
|
||||
if pd.isna(val) or val is None:
|
||||
return default
|
||||
if pd.isna(val) or val is None: return default
|
||||
return str(val).strip()
|
||||
return default
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# CLASSIFICATION LOGIC
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
def classify_rule(job_text: str) -> dict:
|
||||
clean = clean_text(job_text)
|
||||
if not clean or len(clean) < 3:
|
||||
return {"profile": "Tidak Diketahui", "confidence": 0.65, "method": "rule_based_fallback"}
|
||||
|
||||
for profile, keywords in KEYWORD_RULES.items():
|
||||
if any(kw in clean for kw in keywords):
|
||||
return {"profile": profile, "confidence": 0.65, "method": "rule_based"}
|
||||
|
|
@ -137,120 +138,110 @@ def classify_ml(job_text: str, pipeline) -> dict:
|
|||
logger.error(f"ML_INFER_ERROR | job_text='{job_text[:50]}' | err={e}")
|
||||
raise HTTPException(status_code=500, detail=f"ML inference failed: {str(e)}")
|
||||
|
||||
# [KEEP v3] Ekstraksi teks dari format Kemendikbud
|
||||
def extract_kemendik_text(row: pd.Series, df_columns: list) -> str:
|
||||
f5b_col = find_column_by_code(df_columns, "f5b")
|
||||
f5c_col = find_column_by_code(df_columns, "f5c")
|
||||
f5b_col = find_column_by_code(df_columns, "f5b")
|
||||
f5c_col = find_column_by_code(df_columns, "f5c")
|
||||
f1101_col = find_column_by_code(df_columns, "f1101")
|
||||
f1102_col = find_column_by_code(df_columns, "f1102")
|
||||
f5b = clean_text(row.get(f5b_col, "") if f5b_col else "")
|
||||
f5b = clean_text(row.get(f5b_col, "") if f5b_col else "")
|
||||
f1102 = clean_text(row.get(f1102_col, "") if f1102_col else "")
|
||||
f5c_raw = str(row.get(f5c_col, "")).strip() if f5c_col else ""
|
||||
f5c_code = re.match(r'^(\d+)', f5c_raw)
|
||||
f5c_text = F5C_MAP.get(f5c_code.group(1), "") if f5c_code else ""
|
||||
f5c_raw = str(row.get(f5c_col, "")).strip() if f5c_col else ""
|
||||
f5c_code = re.match(r'^(\d+)', f5c_raw)
|
||||
f5c_text = F5C_MAP.get(f5c_code.group(1), "") if f5c_code else ""
|
||||
f1101_raw = str(row.get(f1101_col, "")).strip() if f1101_col else ""
|
||||
f1101_code = re.match(r'^(\d+)', f1101_raw)
|
||||
f1101_text = F1101_MAP.get(f1101_code.group(1), "") if f1101_code else ""
|
||||
|
||||
return " ".join(p for p in [f5b, f5c_text, f1101_text, f1102] if p).strip()
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# SOURCE DETECTION
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
def detect_source(df: pd.DataFrame) -> str:
|
||||
|
||||
cols_lower = [c.lower().strip() for c in df.columns]
|
||||
kemendik_codes = ["f5b", "f5c", "f8", "f1101", "nimhsmsmh", "nmmhsmsmh"]
|
||||
for col in cols_lower:
|
||||
if any(code in col for code in kemendik_codes):
|
||||
return "kemendik"
|
||||
if any("jabatan" in c for c in cols_lower):
|
||||
return "internal_mif"
|
||||
|
||||
if any(code in col for code in kemendik_codes): return "kemendik"
|
||||
if any("jabatan" in c for c in cols_lower): return "internal_mif"
|
||||
return "unknown"
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# ROUTES
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
@app.get("/health")
|
||||
def health_check():
|
||||
return {"status": "healthy", "pipeline_loaded": app.state.pipeline is not None}
|
||||
|
||||
|
||||
|
||||
# RE-TRAINING ENDPOINTS
|
||||
@app.post("/api/v1/retrain", dependencies=[Depends(verify_token)])
|
||||
async def trigger_retrain(file: UploadFile = File(None)):
|
||||
if STATUS_PATH.exists():
|
||||
try:
|
||||
current = json.loads(STATUS_PATH.read_text())
|
||||
if current.get("stage") in ["started", "backup", "loading_data",
|
||||
"preprocessing", "training", "evaluating", "promoting"]:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Re-training sedang berjalan. Tunggu hingga selesai."
|
||||
)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception:
|
||||
pass
|
||||
extra_csv_path = ""
|
||||
if file and file.filename:
|
||||
job_id = str(uuid.uuid4())[:8]
|
||||
extra_csv_path = str(TEMP_DIR / f"extra_{job_id}.csv")
|
||||
contents = await file.read()
|
||||
Path(extra_csv_path).write_bytes(contents)
|
||||
logger.info(f"Extra CSV disimpan: {extra_csv_path}")
|
||||
|
||||
STATUS_PATH.write_text(json.dumps({
|
||||
"stage": "started",
|
||||
"message": "Memulai proses re-training...",
|
||||
"timestamp": __import__('datetime').datetime.now().isoformat(),
|
||||
}, ensure_ascii=False))
|
||||
|
||||
cmd = [sys.executable, str(RETRAIN_WORKER)]
|
||||
if extra_csv_path:
|
||||
cmd.append(extra_csv_path)
|
||||
|
||||
# [v4] FileLock mencegah race condition dari concurrent requests
|
||||
lock = FileLock(LOCK_PATH, timeout=2)
|
||||
try:
|
||||
subprocess.Popen(
|
||||
cmd,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
start_new_session=True
|
||||
)
|
||||
logger.info(f"Retrain subprocess spawned: {' '.join(cmd)}")
|
||||
except Exception as e:
|
||||
STATUS_PATH.write_text(json.dumps({
|
||||
"stage": "failed",
|
||||
"message": f"Gagal spawn subprocess: {str(e)}",
|
||||
}, ensure_ascii=False))
|
||||
raise HTTPException(status_code=500, detail=f"Gagal memulai training: {str(e)}")
|
||||
with lock:
|
||||
if STATUS_PATH.exists():
|
||||
try:
|
||||
current = json.loads(STATUS_PATH.read_text())
|
||||
if current.get("stage") in ["started", "backup", "loading_data", "preprocessing", "training", "evaluating", "promoting"]:
|
||||
raise HTTPException(status_code=409, detail="Re-training sedang berjalan. Tunggu hingga selesai.")
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
return JSONResponse(content={"status": "started", "message": "Re-training dimulai di background."})
|
||||
extra_csv_path = ""
|
||||
if file and file.filename:
|
||||
job_id = str(uuid.uuid4())[:8]
|
||||
extra_csv_path = str(TEMP_DIR / f"extra_{job_id}.csv")
|
||||
contents = await file.read()
|
||||
Path(extra_csv_path).write_bytes(contents)
|
||||
logger.info(f"Extra CSV disimpan: {extra_csv_path}")
|
||||
|
||||
STATUS_PATH.write_text(json.dumps({
|
||||
"stage": "started",
|
||||
"message": "Memulai proses re-training...",
|
||||
"timestamp": __import__('datetime').datetime.now().isoformat(),
|
||||
}, ensure_ascii=False))
|
||||
|
||||
cmd = [sys.executable, str(RETRAIN_WORKER)]
|
||||
if extra_csv_path: cmd.append(extra_csv_path)
|
||||
|
||||
subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True)
|
||||
logger.info(f"Retrain subprocess spawned: {' '.join(cmd)}")
|
||||
return JSONResponse(content={"status": "started", "message": "Re-training dimulai di background."})
|
||||
|
||||
except Timeout:
|
||||
raise HTTPException(status_code=429, detail="Sistem sibuk. Permintaan retraining lain sedang diproses.")
|
||||
except Exception as e:
|
||||
if not isinstance(e, HTTPException):
|
||||
raise HTTPException(status_code=500, detail=f"Gagal memulai training: {str(e)}")
|
||||
raise e
|
||||
|
||||
|
||||
@app.get("/api/v1/retrain/status", dependencies=[Depends(verify_token)])
|
||||
def retrain_status():
|
||||
if not STATUS_PATH.exists():
|
||||
return JSONResponse(content={"stage": "idle", "message": "Belum ada proses re-training."})
|
||||
|
||||
try:
|
||||
data = json.loads(STATUS_PATH.read_text())
|
||||
# [v4] Lock saat baca untuk menghindari partial write dari worker
|
||||
lock = FileLock(LOCK_PATH, timeout=2)
|
||||
with lock:
|
||||
data = json.loads(STATUS_PATH.read_text())
|
||||
terminal_stages = {"promoted", "rolled_back", "failed"}
|
||||
if data.get("stage") in terminal_stages and not data.get("_reloaded"):
|
||||
try:
|
||||
app.state.pipeline = joblib.load(PIPELINE_PATH)
|
||||
data["_reloaded"] = True
|
||||
STATUS_PATH.write_text(json.dumps(data, indent=2, ensure_ascii=False))
|
||||
logger.info(f"Pipeline di-reload setelah retrain (stage={data['stage']})")
|
||||
except Exception as e:
|
||||
logger.error(f"Gagal reload pipeline: {e}")
|
||||
return JSONResponse(content=data)
|
||||
except Timeout:
|
||||
return JSONResponse(status_code=429, content={"stage": "locked", "message": "Status sedang diupdate sistem."})
|
||||
except Exception:
|
||||
return JSONResponse(content={"stage": "unknown", "message": "Status tidak terbaca."})
|
||||
|
||||
terminal_stages = {"promoted", "rolled_back", "failed"}
|
||||
if data.get("stage") in terminal_stages and not data.get("_reloaded"):
|
||||
try:
|
||||
app.state.pipeline = joblib.load(PIPELINE_PATH)
|
||||
data["_reloaded"] = True
|
||||
STATUS_PATH.write_text(json.dumps(data, indent=2, ensure_ascii=False))
|
||||
logger.info(f"Pipeline di-reload setelah retrain (stage={data['stage']})")
|
||||
except Exception as e:
|
||||
logger.error(f"Gagal reload pipeline: {e}")
|
||||
|
||||
return JSONResponse(content=data)
|
||||
|
||||
|
||||
# [KEEP v3] Endpoint reload manual — berguna jika pipeline perlu di-reload tanpa restart server
|
||||
@app.post("/api/v1/retrain/reload", dependencies=[Depends(verify_token)])
|
||||
def reload_model():
|
||||
try:
|
||||
|
|
@ -260,125 +251,171 @@ def reload_model():
|
|||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Gagal reload model: {str(e)}")
|
||||
|
||||
@app.post("/api/v1/classify", dependencies=[Depends(verify_token)])
|
||||
async def classify_tracer(file: UploadFile = File(...)):
|
||||
if not file.filename:
|
||||
raise HTTPException(status_code=400, detail="No file provided")
|
||||
|
||||
|
||||
def process_classification_bg(job_id: str, file_path: Path, filename: str, pipeline):
|
||||
status_file = JOBS_DIR / f"{job_id}.json"
|
||||
def update_status(status, message, results=None, processed_rows=0, total_rows=0, source_type="unknown"):
|
||||
data = {
|
||||
"status": status, "message": message,
|
||||
"processed_rows": processed_rows, "total_rows": total_rows,
|
||||
"source_type": source_type
|
||||
}
|
||||
if results is not None:
|
||||
data["results"] = results
|
||||
status_file.write_text(json.dumps(data, ensure_ascii=False))
|
||||
|
||||
try:
|
||||
contents = await file.read()
|
||||
|
||||
if file.filename.endswith(".xlsx") or file.filename.endswith(".xls"):
|
||||
update_status("processing", "Parsing file excel/csv...")
|
||||
contents = file_path.read_bytes()
|
||||
if filename.endswith(".xlsx") or filename.endswith(".xls"):
|
||||
try:
|
||||
df = pd.read_excel(io.BytesIO(contents), dtype=str, engine="openpyxl")
|
||||
except ImportError:
|
||||
logger.error("openpyxl not installed")
|
||||
raise HTTPException(status_code=500, detail="Server misconfiguration: openpyxl missing")
|
||||
update_status("error", "Server misconfiguration: openpyxl missing")
|
||||
return
|
||||
except Exception as excel_err:
|
||||
logger.error(f"Excel parsing error: {excel_err}")
|
||||
try:
|
||||
df = pd.read_csv(io.BytesIO(contents), sep=";", encoding="utf-8-sig", dtype=str)
|
||||
except:
|
||||
except Exception:
|
||||
df = pd.read_csv(io.BytesIO(contents), sep=";", encoding="latin-1", dtype=str)
|
||||
else:
|
||||
try:
|
||||
df = pd.read_csv(io.BytesIO(contents), sep=";", encoding="utf-8-sig", dtype=str)
|
||||
except UnicodeDecodeError:
|
||||
df = pd.read_csv(io.BytesIO(contents), sep=";", encoding="latin-1", dtype=str)
|
||||
|
||||
|
||||
df.columns = df.columns.str.strip()
|
||||
source_type = detect_source(df)
|
||||
if source_type == "unknown":
|
||||
update_status("error", "Unrecognized file format.")
|
||||
return
|
||||
if source_type == "internal_mif" and pipeline is None:
|
||||
update_status("error", "ML model not available.")
|
||||
return
|
||||
|
||||
update_status("processing", f"Processing {len(df)} rows...", total_rows=len(df), source_type=source_type)
|
||||
|
||||
results = []
|
||||
logger.info(f"Processing {len(df)} rows | Source: {source_type}")
|
||||
|
||||
find_col_cached = lambda p: next((c for c in df.columns if p.lower() in c.lower()), None)
|
||||
col_f5a1 = find_col_cached("f5a1")
|
||||
col_f5c = find_col_cached("f5c")
|
||||
col_f1101 = find_col_cached("f1101")
|
||||
col_f5b = find_col_cached("f5b")
|
||||
col_f1102 = find_col_cached("f1102")
|
||||
col_nim_kem = find_col_cached("nimhsmsmh")
|
||||
col_nama_kem = find_col_cached("nmmhsmsmh")
|
||||
col_tahun_kem = find_col_cached("tahun_lulus")
|
||||
col_nim_int = find_col_cached("nim")
|
||||
col_nama_int = find_col_cached("nama_lengkap")
|
||||
col_tahun_int = find_col_cached("tahun_lulus")
|
||||
|
||||
for _, row in df.iterrows():
|
||||
nim = ""
|
||||
try:
|
||||
def safe_str(val):
|
||||
if pd.isna(val) or val is None: return ""
|
||||
return str(val).strip()
|
||||
|
||||
if source_type == "kemendik":
|
||||
nim = safe_str(row.get(col_nim_kem))
|
||||
nama = safe_str(row.get(col_nama_kem))
|
||||
tahun = safe_str(row.get(col_tahun_kem))
|
||||
else:
|
||||
nim = safe_str(row.get(col_nim_int))
|
||||
nama = safe_str(row.get(col_nama_int))
|
||||
tahun = safe_str(row.get(col_tahun_int))
|
||||
|
||||
if source_type == "kemendik":
|
||||
job_text = extract_kemendik_text(row, df.columns.tolist())
|
||||
raw_data_payload = {str(k): safe_str(v) for k, v in row.to_dict().items()}
|
||||
raw_data_payload.update({
|
||||
"F5a1": safe_str(row.get(col_f5a1)),
|
||||
"F5c": safe_str(row.get(col_f5c)),
|
||||
"F1101": safe_str(row.get(col_f1101)),
|
||||
"F5b": safe_str(row.get(col_f5b)),
|
||||
"F1102": safe_str(row.get(col_f1102))
|
||||
})
|
||||
results.append({
|
||||
"nim": nim, "nama": nama, "tahun_lulus": tahun,
|
||||
"job_text_raw": job_text, "source_type": "kemendik",
|
||||
"predicted_profile": None, "confidence_score": None,
|
||||
"classification_method": "dashboard_only", "status": "processed",
|
||||
"raw_data": raw_data_payload
|
||||
})
|
||||
else:
|
||||
col_jabatan = find_col_cached("jabatan")
|
||||
col_perusahaan = find_col_cached("perusahaan")
|
||||
col_deskripsi = find_col_cached("deskripsi")
|
||||
|
||||
job_text = clean_text(safe_str(row.get(col_jabatan))) if col_jabatan else ""
|
||||
if not job_text and col_perusahaan and col_deskripsi:
|
||||
job_text = clean_text(f"{safe_str(row.get(col_perusahaan))} {safe_str(row.get(col_deskripsi))}".strip())
|
||||
elif not job_text and col_perusahaan:
|
||||
job_text = clean_text(safe_str(row.get(col_perusahaan)))
|
||||
|
||||
rule_res = classify_rule(job_text)
|
||||
res = rule_res or (classify_ml(job_text, pipeline) if pipeline else {"profile": "Non-IT", "confidence": 0.0, "method": "ml_unavailable"})
|
||||
status = "auto_classified" if res["method"] in ["rule_based", "ml_fallback"] or res["profile"] == "Tidak Diketahui" else "needs_review"
|
||||
raw_data_payload = {str(k): safe_str(v) for k, v in row.to_dict().items()}
|
||||
|
||||
results.append({
|
||||
"nim": nim, "nama": nama, "tahun_lulus": tahun,
|
||||
"job_text_raw": job_text, "source_type": "internal_mif",
|
||||
"predicted_profile": res["profile"], "confidence_score": res["confidence"],
|
||||
"classification_method": res["method"], "status": status,
|
||||
"raw_data": raw_data_payload
|
||||
})
|
||||
except Exception as row_err:
|
||||
logger.warning(f"Row error: {row_err}")
|
||||
results.append({"nim": nim, "status": "failed", "error_detail": str(row_err)[:100]})
|
||||
|
||||
update_status("success", "Selesai", results=results, processed_rows=len(results), total_rows=len(df), source_type=source_type)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"File parsing error: {type(e).__name__}: {str(e)}")
|
||||
raise HTTPException(status_code=400, detail=f"Failed to parse file: {type(e).__name__}: {str(e)[:200]}")
|
||||
logger.error(f"Job {job_id} error: {e}")
|
||||
update_status("error", f"Processing failed: {str(e)}")
|
||||
finally:
|
||||
if file_path.exists():
|
||||
try:
|
||||
file_path.unlink()
|
||||
except Exception as unlink_err:
|
||||
logger.warning(f"Failed to delete temp file {file_path}: {unlink_err}")
|
||||
|
||||
|
||||
@app.post("/api/v1/classify", dependencies=[Depends(verify_token)])
|
||||
async def classify_tracer(background_tasks: BackgroundTasks, file: UploadFile = File(...)):
|
||||
if not file.filename:
|
||||
raise HTTPException(status_code=400, detail="No file provided")
|
||||
|
||||
job_id = str(uuid.uuid4())
|
||||
temp_file = JOBS_DIR / f"raw_{job_id}_{file.filename}"
|
||||
|
||||
source_type = detect_source(df)
|
||||
if source_type == "unknown":
|
||||
raise HTTPException(status_code=400, detail="Unrecognized file format.")
|
||||
contents = await file.read()
|
||||
temp_file.write_bytes(contents)
|
||||
|
||||
if source_type == "internal_mif" and app.state.pipeline is None:
|
||||
logger.error("ML pipeline not loaded")
|
||||
raise HTTPException(status_code=503, detail="ML model not available.")
|
||||
status_file = JOBS_DIR / f"{job_id}.json"
|
||||
status_file.write_text(json.dumps({"status": "pending", "message": "Job masuk antrean"}))
|
||||
|
||||
results = []
|
||||
pipeline = app.state.pipeline
|
||||
logger.info(f"Processing {len(df)} rows | Source: {source_type}")
|
||||
background_tasks.add_task(process_classification_bg, job_id, temp_file, file.filename, app.state.pipeline)
|
||||
|
||||
return JSONResponse(content={"job_id": job_id, "status": "pending", "message": "Classification started in background."})
|
||||
|
||||
find_col_cached = lambda p: next((c for c in df.columns if p.lower() in c.lower()), None)
|
||||
col_f5a1 = find_col_cached("f5a1")
|
||||
col_f5c = find_col_cached("f5c")
|
||||
col_f1101 = find_col_cached("f1101")
|
||||
col_f5b = find_col_cached("f5b")
|
||||
col_f1102 = find_col_cached("f1102")
|
||||
col_nim_kem = find_col_cached("nimhsmsmh")
|
||||
col_nama_kem = find_col_cached("nmmhsmsmh")
|
||||
col_tahun_kem = find_col_cached("tahun_lulus")
|
||||
col_nim_int = find_col_cached("nim")
|
||||
col_nama_int = find_col_cached("nama_lengkap")
|
||||
col_tahun_int = find_col_cached("tahun_lulus")
|
||||
@app.get("/api/v1/classify/status/{job_id}", dependencies=[Depends(verify_token)])
|
||||
def classify_status(job_id: str):
|
||||
status_file = JOBS_DIR / f"{job_id}.json"
|
||||
if not status_file.exists():
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
|
||||
try:
|
||||
data = json.loads(status_file.read_text())
|
||||
return JSONResponse(content=data)
|
||||
except json.JSONDecodeError:
|
||||
return JSONResponse(content={"status": "processing", "message": "Updating status..."})
|
||||
|
||||
for _, row in df.iterrows():
|
||||
try:
|
||||
def safe_str(val):
|
||||
if pd.isna(val) or val is None: return ""
|
||||
return str(val).strip()
|
||||
|
||||
if source_type == "kemendik":
|
||||
nim = safe_str(row.get(col_nim_kem))
|
||||
nama = safe_str(row.get(col_nama_kem))
|
||||
tahun = safe_str(row.get(col_tahun_kem))
|
||||
else:
|
||||
nim = safe_str(row.get(col_nim_int))
|
||||
nama = safe_str(row.get(col_nama_int))
|
||||
tahun = safe_str(row.get(col_tahun_int))
|
||||
|
||||
if source_type == "kemendik":
|
||||
job_text = extract_kemendik_text(row, df.columns.tolist())
|
||||
raw_data_payload = {str(k): safe_str(v) for k, v in row.to_dict().items()}
|
||||
raw_data_payload.update({
|
||||
"F5a1": safe_str(row.get(col_f5a1)),
|
||||
"F5c": safe_str(row.get(col_f5c)),
|
||||
"F1101": safe_str(row.get(col_f1101)),
|
||||
"F5b": safe_str(row.get(col_f5b)),
|
||||
"F1102": safe_str(row.get(col_f1102))
|
||||
})
|
||||
results.append({
|
||||
"nim": nim, "nama": nama, "tahun_lulus": tahun,
|
||||
"job_text_raw": job_text, "source_type": "kemendik",
|
||||
"predicted_profile": None, "confidence_score": None,
|
||||
"classification_method": "dashboard_only", "status": "processed",
|
||||
"raw_data": raw_data_payload
|
||||
})
|
||||
else:
|
||||
col_jabatan = find_col_cached("jabatan")
|
||||
col_perusahaan = find_col_cached("perusahaan")
|
||||
col_deskripsi = find_col_cached("deskripsi")
|
||||
|
||||
job_text = clean_text(safe_str(row.get(col_jabatan))) if col_jabatan else ""
|
||||
if not job_text and col_perusahaan and col_deskripsi:
|
||||
job_text = clean_text(f"{safe_str(row.get(col_perusahaan))} {safe_str(row.get(col_deskripsi))}".strip())
|
||||
elif not job_text and col_perusahaan:
|
||||
job_text = clean_text(safe_str(row.get(col_perusahaan)))
|
||||
|
||||
rule_res = classify_rule(job_text)
|
||||
res = rule_res or (classify_ml(job_text, pipeline) if pipeline else {"profile": "Non-IT", "confidence": 0.0, "method": "ml_unavailable"})
|
||||
status = "auto_classified" if res["method"] in ["rule_based", "ml_fallback"] or res["profile"] == "Tidak Diketahui" else "needs_review"
|
||||
|
||||
raw_data_payload = {str(k): safe_str(v) for k, v in row.to_dict().items()}
|
||||
|
||||
results.append({
|
||||
"nim": nim, "nama": nama, "tahun_lulus": tahun,
|
||||
"job_text_raw": job_text, "source_type": "internal_mif",
|
||||
"predicted_profile": res["profile"], "confidence_score": res["confidence"],
|
||||
"classification_method": res["method"], "status": status,
|
||||
"raw_data": raw_data_payload
|
||||
})
|
||||
except Exception as row_err:
|
||||
logger.warning(f"Row error: {row_err}")
|
||||
results.append({"nim": nim if 'nim' in locals() else "", "status": "failed", "error_detail": str(row_err)[:100]})
|
||||
|
||||
return JSONResponse(content={"status": "success", "total_rows": len(df), "processed_rows": len(results), "source_type": source_type, "results": results})
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
uvicorn.run("app.main:app", host="127.0.0.1", port=8000, reload=True)
|
||||
uvicorn.run("app.main:app", host="127.0.0.1", port=8000, reload=True)
|
||||
|
|
|
|||
|
|
@ -1,14 +1,18 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
TRACER STUDY - RE-TRAINING WORKER (SUBPROCESS)
|
||||
Dipanggil oleh FastAPI sebagai background subprocess.
|
||||
TRACER STUDY - RE-TRAINING WORKER FINAL (SUBPROCESS)
|
||||
Dipanggil oleh FastAPI (main3.py) sebagai background subprocess.
|
||||
|
||||
Alur:
|
||||
1. Backup pkl lama → pkl.bak
|
||||
2. Merge corpus asli + data manual_override baru
|
||||
3. Train model baru (candidate)
|
||||
4. Evaluasi: bandingkan weighted F1-score baru vs lama
|
||||
5. Promote jika lebih baik; rollback jika tidak
|
||||
6. Update retrain_status.json di setiap tahap
|
||||
2. Merge corpus + manual_override (TANPA drop_duplicates)
|
||||
Override diberi sample_weight lebih tinggi, bukan menghapus data historis.
|
||||
3. Dynamic train/test split dari data gabungan (selalu fresh)
|
||||
4. Evaluasi MODEL LAMA pada test set yang sama (fair comparison)
|
||||
5. K-Fold + train model final (candidate) dengan sample_weight
|
||||
6. Evaluasi candidate pada test set yang sama → delta terhadap old model
|
||||
7. Promote jika lebih baik; rollback jika tidak
|
||||
8. Setiap write status dilindungi FileLock (atomic)
|
||||
"""
|
||||
|
||||
import sys
|
||||
|
|
@ -17,12 +21,13 @@ import shutil
|
|||
import logging
|
||||
import warnings
|
||||
import re
|
||||
import os
|
||||
import math
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import joblib
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from filelock import FileLock
|
||||
|
||||
from sklearn.feature_extraction.text import TfidfVectorizer
|
||||
from sklearn.linear_model import LogisticRegression
|
||||
|
|
@ -37,23 +42,58 @@ warnings.filterwarnings("ignore")
|
|||
# KONFIGURASI
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
BASE_DIR = Path(__file__).parent.parent
|
||||
ML_DIR = BASE_DIR / "ml_assets"
|
||||
ML_DIR = BASE_DIR / "ml_assets"
|
||||
DATA_DIR = BASE_DIR.parent / "data" / "processed"
|
||||
|
||||
PIPELINE_PATH = ML_DIR / "ml_pipeline_internal.pkl"
|
||||
PIPELINE_BAK_PATH = ML_DIR / "ml_pipeline_internal.pkl.bak"
|
||||
CANDIDATE_PATH = ML_DIR / "ml_pipeline_candidate.pkl"
|
||||
METRICS_PATH = ML_DIR / "metrics_internal_only.json"
|
||||
STATUS_PATH = ML_DIR / "retrain_status.json"
|
||||
CORPUS_PATH = DATA_DIR / "training_corpus_3.csv"
|
||||
TEST_PATH = DATA_DIR / "test_set_3.csv"
|
||||
PIPELINE_PATH = ML_DIR / "ml_pipeline_internal.pkl"
|
||||
PIPELINE_BAK_PATH = ML_DIR / "ml_pipeline_internal.pkl.bak"
|
||||
CANDIDATE_PATH = ML_DIR / "ml_pipeline_candidate.pkl"
|
||||
METRICS_PATH = ML_DIR / "metrics_internal_only.json"
|
||||
STATUS_PATH = ML_DIR / "retrain_status.json"
|
||||
LOCK_PATH = ML_DIR / "retrain_status.lock"
|
||||
CORPUS_PATH = DATA_DIR / "training_corpus.csv"
|
||||
TEST_PATH = DATA_DIR / "test_set.csv"
|
||||
|
||||
TARGET_CLASSES = ["Programmer", "Data Analyst", "Wirausaha Informatika", "Non-IT"]
|
||||
MIN_IMPROVEMENT_THRESHOLD = 0.01 # Model baru harus lebih baik minimal +1% weighted F1
|
||||
MAX_REGRESSION_ALLOWED = 0.02 # Toleransi degradasi maksimal 2% sebelum rollback keras
|
||||
|
||||
# Threshold: model baru harus lebih baik minimal MIN_IMPROVEMENT dari model lama
|
||||
MIN_IMPROVEMENT_THRESHOLD = 0.01 # 1% weighted F1
|
||||
MAX_REGRESSION_ALLOWED = 0.02 # Toleransi: model baru boleh lebih buruk max 2% (di luar ini = rollback keras)
|
||||
# ── KONFIGURASI BOBOT OVERRIDE ──────────────────────────────────────────────
|
||||
#
|
||||
# Masalah yang diselesaikan:
|
||||
# Pada skala besar (N_corpus >> N_override), bobot statis kehilangan daya
|
||||
# akibat dilusi. Formula proporsi murni w = t*N / (M*(1-t)) menyelesaikan
|
||||
# dilusi, tapi menghasilkan w=42.85 pada skenario 10k/100 — yang berisiko
|
||||
# overfitting ekstrem pada noise override.
|
||||
#
|
||||
# Solusi: Logarithmic damping — tanpa tembok statis.
|
||||
# w_raw = formula proporsi (jaminan 30% jika tidak di-damp)
|
||||
# w = MIN + ln(1 + max(0, w_raw - MIN))
|
||||
#
|
||||
# Fase linear (w_raw rendah): w ≈ w_raw → proporsi terpenuhi
|
||||
# Fase log (w_raw tinggi): w tumbuh tapi melambat → damp alami
|
||||
#
|
||||
# Implikasi jujur:
|
||||
# Target 30% TIDAK dipertahankan di skala ekstrem. Ini trade-off yang
|
||||
# disengaja: degradasi gradual lebih aman daripada overfitting ke 10
|
||||
# baris override berbobot 42x. Tanpa tembok statis, redaman terjadi
|
||||
# secara natural mengikuti kurva logaritmik, bukan menabrak batas arbitrer.
|
||||
#
|
||||
# Perilaku nyata (MIN=2.0, TARGET=0.30):
|
||||
# w_raw │ w_log │ influence aktual
|
||||
# ──────────────────────────────────
|
||||
# 2.00 │ 2.000 │ formula <= MIN, pakai floor
|
||||
# 4.29 │ 3.178 │ (1k corpus, 100 override) → ~24%
|
||||
# 8.57 │ 3.999 │ (100 korpus, 5 override) → ~17%
|
||||
# 42.86 │ 5.723 │ (10k corpus, 100 override)→ ~5.4%
|
||||
# 428.60 │ 8.063 │ (10k corpus, 10 override) → ~0.8%
|
||||
#
|
||||
# Jika angka influence aktual dianggap terlalu kecil → naikkan TARGET.
|
||||
# Jika model terlalu sensitif ke override → naikkan MIN agar floor lebih tinggi.
|
||||
OVERRIDE_INFLUENCE_TARGET = 0.30 # Titik acuan proporsi (valid di skala normal)
|
||||
OVERRIDE_MIN_WEIGHT = 2.0 # Lantai: override selalu minimal 2× korpus
|
||||
|
||||
# [FIX v4] STOPWORDS lengkap — versi retrain_worker2 hanya punya 10 kata (bug terpotong)
|
||||
STOPWORDS = {
|
||||
"yang", "di", "ke", "dari", "dan", "atau", "dengan", "untuk", "pada", "dalam",
|
||||
"adalah", "ini", "itu", "tidak", "juga", "sudah", "akan", "bisa", "ada", "oleh",
|
||||
|
|
@ -69,6 +109,39 @@ STOPWORDS = {
|
|||
|
||||
stemmer = StemmerFactory().create_stemmer()
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# DYNAMIC WEIGHT CALCULATOR
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
def compute_override_weight(n_corpus: int, n_override: int) -> float:
|
||||
"""
|
||||
Hitung bobot override dengan logarithmic damping.
|
||||
|
||||
TIDAK ada OVERRIDE_MAX_WEIGHT — tembok statis menghancurkan jaminan
|
||||
pengaruh tepat saat skala besar membutuhkannya. Sebagai gantinya,
|
||||
fungsi ln(1+x) menyediakan redaman alami:
|
||||
|
||||
w_raw = (t * n_corpus) / (n_override * (1 - t)) # proporsi murni
|
||||
w = MIN + ln(1 + max(0, w_raw - MIN)) # log damping
|
||||
|
||||
Jaminan matematis:
|
||||
- w selalu >= OVERRIDE_MIN_WEIGHT (floor tetap ada)
|
||||
- w tidak pernah meledak ke infinity karena ln tumbuh O(log n)
|
||||
- Tidak ada tembok statis yang membuat influence kolaps tiba-tiba
|
||||
|
||||
Catatan interaksi:
|
||||
LogisticRegression dipanggil dengan class_weight='balanced' DAN
|
||||
sample_weight. Keduanya dikalikan oleh sklearn secara internal.
|
||||
Artinya sampel override dari kelas minoritas mendapat boost ganda.
|
||||
Pantau per-class F1 di metrics JSON untuk mendeteksi efek ini.
|
||||
"""
|
||||
if n_override == 0:
|
||||
return 1.0
|
||||
t = OVERRIDE_INFLUENCE_TARGET
|
||||
w_raw = (t * n_corpus) / (n_override * (1.0 - t))
|
||||
# ln damping: linear di zona rendah, melambat secara alami di skala besar
|
||||
w_log = OVERRIDE_MIN_WEIGHT + math.log1p(max(0.0, w_raw - OVERRIDE_MIN_WEIGHT))
|
||||
return round(w_log, 4)
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# LOGGING
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
|
|
@ -79,116 +152,104 @@ logging.basicConfig(
|
|||
)
|
||||
logger = logging.getLogger("retrain_worker")
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# STATUS WRITER
|
||||
# STATUS WRITER — atomic dengan FileLock (v4)
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
def write_status(stage: str, message: str, extra: dict = None):
|
||||
payload = {
|
||||
"stage": stage,
|
||||
"message": message,
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
}
|
||||
if extra:
|
||||
payload.update(extra)
|
||||
STATUS_PATH.write_text(json.dumps(payload, indent=2, ensure_ascii=False))
|
||||
lock = FileLock(LOCK_PATH, timeout=10)
|
||||
with lock:
|
||||
payload = {"stage": stage, "message": message, "timestamp": datetime.now().isoformat()}
|
||||
if extra: payload.update(extra)
|
||||
STATUS_PATH.write_text(json.dumps(payload, indent=2, ensure_ascii=False))
|
||||
logger.info(f"[{stage}] {message}")
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# PREPROCESSING — IDENTIK dengan main.py
|
||||
# PREPROCESSING — dua fungsi terpisah (v4)
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
def preprocess_text(text: str) -> str:
|
||||
if pd.isna(text) or not isinstance(text, str):
|
||||
return ""
|
||||
def preprocess_stemmed(text: str) -> str:
|
||||
"""Untuk data korpus yang SUDAH di-stem oleh prepare_corpus — tidak re-stem."""
|
||||
if pd.isna(text) or not isinstance(text, str): return ""
|
||||
text = text.lower().strip()
|
||||
if text in {"nan", "none", "null", "-", "0", "", "tidak diisi"}:
|
||||
return ""
|
||||
if text in {"nan", "none", "null", "-", "0", "", "tidak diisi"}: return ""
|
||||
text = text.replace('-', ' ').replace('/', ' ').replace('_', ' ').replace(',', ' ')
|
||||
text = re.sub(r'[^\w\s]', '', text)
|
||||
text = re.sub(r'\d+', '', text)
|
||||
words = [w for w in text.split() if w not in STOPWORDS and len(w) > 2]
|
||||
return " ".join(words)
|
||||
|
||||
def preprocess_raw(text: str) -> str:
|
||||
"""Untuk data manual_override yang BELUM di-stem — jalankan Sastrawi."""
|
||||
if pd.isna(text) or not isinstance(text, str): return ""
|
||||
text = text.lower().strip()
|
||||
if text in {"nan", "none", "null", "-", "0", "", "tidak diisi"}: return ""
|
||||
text = text.replace('-', ' ').replace('/', ' ').replace('_', ' ').replace(',', ' ')
|
||||
text = re.sub(r'[^\w\s]', '', text)
|
||||
text = re.sub(r'\d+', '', text)
|
||||
words = [w for w in text.split() if w not in STOPWORDS and len(w) > 2]
|
||||
return " ".join([stemmer.stem(w) for w in words])
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# EVALUASI MODEL — weighted F1 pada hold-out test
|
||||
# EVALUASI MODEL
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
def evaluate_model(model, X_test: pd.Series, y_test: pd.Series) -> dict:
|
||||
"""
|
||||
Evaluasi model pada hold-out test set.
|
||||
Return: dict berisi weighted F1, accuracy, dan per-class metrics.
|
||||
"""
|
||||
if len(X_test) == 0:
|
||||
return {"weighted_f1": 0.0, "accuracy": 0.0, "per_class": {}}
|
||||
|
||||
y_pred = model.predict(X_test)
|
||||
report = classification_report(
|
||||
y_test, y_pred,
|
||||
output_dict=True,
|
||||
zero_division=0,
|
||||
labels=TARGET_CLASSES
|
||||
)
|
||||
report = classification_report(y_test, y_pred, output_dict=True, zero_division=0, labels=TARGET_CLASSES)
|
||||
return {
|
||||
"weighted_f1": round(report.get("weighted avg", {}).get("f1-score", 0.0), 4),
|
||||
"accuracy": round(report.get("accuracy", 0.0), 4),
|
||||
"per_class": {
|
||||
cls: {
|
||||
"precision": round(report.get(cls, {}).get("precision", 0.0), 3),
|
||||
"recall": round(report.get(cls, {}).get("recall", 0.0), 3),
|
||||
"f1": round(report.get(cls, {}).get("f1-score", 0.0), 3),
|
||||
"support": int(report.get(cls, {}).get("support", 0)),
|
||||
"recall": round(report.get(cls, {}).get("recall", 0.0), 3),
|
||||
"f1": round(report.get(cls, {}).get("f1-score", 0.0), 3),
|
||||
"support": int(report.get(cls, {}).get("support", 0)),
|
||||
}
|
||||
for cls in TARGET_CLASSES
|
||||
}
|
||||
}
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# BASELINE F1 — re-evaluasi model lama pada test set yang sama
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
def get_baseline_f1_on_testset(X_test: pd.Series, y_test: pd.Series) -> float:
|
||||
"""
|
||||
Evaluasi model lama (backup) pada X_test/y_test yang SAMA dengan yang
|
||||
akan digunakan mengevaluasi candidate. Ini satu-satunya cara yang adil
|
||||
(apples-to-apples) karena distribusi test set berubah setiap retrain.
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# AMBIL BASELINE DARI METRICS JSON
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
def get_baseline_f1() -> float:
|
||||
"""
|
||||
Baca weighted F1 model lama dari metrics_internal_only.json.
|
||||
Fallback ke 0 jika file tidak ada.
|
||||
Membandingkan F1 baru dengan angka JSON lama (dihitung di distribusi berbeda)
|
||||
adalah perbandingan apel vs jeruk — tidak valid untuk keputusan promote/rollback.
|
||||
"""
|
||||
if not PIPELINE_BAK_PATH.exists():
|
||||
logger.warning("Backup .bak tidak ditemukan — baseline F1 diasumsikan 0.0")
|
||||
return 0.0
|
||||
try:
|
||||
if METRICS_PATH.exists():
|
||||
m = json.loads(METRICS_PATH.read_text())
|
||||
return float(m.get("hold_out_test", {}).get("weighted avg", {}).get("f1-score", 0.0))
|
||||
old_model = joblib.load(PIPELINE_BAK_PATH)
|
||||
result = evaluate_model(old_model, X_test, y_test)
|
||||
logger.info(f"Baseline F1 (old model, same test set): {result['weighted_f1']:.4f}")
|
||||
return result["weighted_f1"]
|
||||
except Exception as e:
|
||||
logger.warning(f"Gagal baca baseline metrics: {e}")
|
||||
return 0.0
|
||||
|
||||
logger.warning(f"Gagal load/evaluasi model lama untuk baseline: {e}")
|
||||
return 0.0
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# ROLLBACK
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
def do_rollback(reason: str, old_f1: float, new_f1: float):
|
||||
"""Kembalikan pkl aktif ke backup."""
|
||||
# Bersihkan candidate jika ada
|
||||
if CANDIDATE_PATH.exists():
|
||||
CANDIDATE_PATH.unlink()
|
||||
|
||||
# Restore dari backup
|
||||
if CANDIDATE_PATH.exists(): CANDIDATE_PATH.unlink()
|
||||
if PIPELINE_BAK_PATH.exists():
|
||||
shutil.copy2(PIPELINE_BAK_PATH, PIPELINE_PATH)
|
||||
logger.info(f"Rollback berhasil: pkl lama dipulihkan dari .bak")
|
||||
logger.info("Rollback berhasil: pkl lama dipulihkan dari .bak")
|
||||
else:
|
||||
logger.warning("File .bak tidak ditemukan, pkl aktif dibiarkan.")
|
||||
|
||||
logger.warning("File .bak tidak ditemukan — pkl aktif dibiarkan.")
|
||||
write_status(
|
||||
stage="rolled_back",
|
||||
message=f"Model lama dipertahankan. {reason}",
|
||||
extra={
|
||||
"result": "rolled_back",
|
||||
"reason": reason,
|
||||
"old_f1": old_f1,
|
||||
"new_f1": new_f1,
|
||||
}
|
||||
extra={"result": "rolled_back", "reason": reason, "old_f1": old_f1, "new_f1": new_f1}
|
||||
)
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# MAIN
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
|
|
@ -197,15 +258,15 @@ def main(extra_csv_path: str = None):
|
|||
|
||||
# ── STEP 1: BACKUP ──────────────────────────────────────
|
||||
write_status("backup", "Membuat backup model lama...")
|
||||
if PIPELINE_PATH.exists():
|
||||
shutil.copy2(PIPELINE_PATH, PIPELINE_BAK_PATH)
|
||||
logger.info(f"Backup tersimpan: {PIPELINE_BAK_PATH}")
|
||||
else:
|
||||
if not PIPELINE_PATH.exists():
|
||||
write_status("failed", "File pkl aktif tidak ditemukan, tidak bisa backup.")
|
||||
sys.exit(1)
|
||||
shutil.copy2(PIPELINE_PATH, PIPELINE_BAK_PATH)
|
||||
logger.info(f"Backup tersimpan: {PIPELINE_BAK_PATH}")
|
||||
|
||||
baseline_f1 = get_baseline_f1()
|
||||
logger.info(f"Baseline weighted F1 (model lama): {baseline_f1:.4f}")
|
||||
# baseline_f1 akan dihitung setelah split dinamis terbentuk
|
||||
# (evaluasi model lama pada test set yang sama dengan candidate)
|
||||
baseline_f1 = 0.0
|
||||
|
||||
try:
|
||||
# ── STEP 2: LOAD & MERGE DATA ────────────────────────
|
||||
|
|
@ -215,57 +276,70 @@ def main(extra_csv_path: str = None):
|
|||
raise FileNotFoundError(f"Corpus asli tidak ditemukan: {CORPUS_PATH}")
|
||||
|
||||
df_corpus = pd.read_csv(CORPUS_PATH, sep=";", dtype=str).dropna(subset=["job_text_raw", "label"])
|
||||
df_corpus["features"] = df_corpus["job_text_raw"].apply(preprocess_stemmed)
|
||||
df_corpus["_weight"] = 1.0 # Bobot normal untuk data historis
|
||||
logger.info(f"Corpus asli: {len(df_corpus)} baris")
|
||||
|
||||
df_extra = pd.DataFrame()
|
||||
if extra_csv_path and Path(extra_csv_path).exists():
|
||||
df_extra = pd.read_csv(extra_csv_path, sep=";", dtype=str).dropna(subset=["job_text_raw", "label"])
|
||||
# Validasi label
|
||||
valid_labels = set(TARGET_CLASSES)
|
||||
df_extra = df_extra[df_extra["label"].isin(valid_labels)]
|
||||
logger.info(f"Data manual_override: {len(df_extra)} baris valid")
|
||||
df_extra = df_extra[df_extra["label"].isin(set(TARGET_CLASSES))]
|
||||
df_extra["features"] = df_extra["job_text_raw"].apply(preprocess_raw)
|
||||
override_w = compute_override_weight(len(df_corpus), len(df_extra))
|
||||
df_extra["_weight"] = override_w
|
||||
|
||||
# Pre-compute total bobot sekali — dipakai di logging & metrics JSON.
|
||||
# .sum() lebih benar daripada len() * .iloc[0] karena tidak mengasumsikan
|
||||
# homogenitas bobot di seluruh baris (future-proof jika bobot per-baris ditambahkan).
|
||||
total_corpus_weight = df_corpus["_weight"].sum()
|
||||
total_override_weight = df_extra["_weight"].sum() if len(df_extra) > 0 else 0.0
|
||||
actual_influence_pct = (
|
||||
total_override_weight / (total_corpus_weight + total_override_weight) * 100
|
||||
if total_override_weight > 0 else 0.0
|
||||
)
|
||||
|
||||
if len(df_extra) > 0:
|
||||
df_all = pd.concat([df_corpus, df_extra], ignore_index=True)
|
||||
# Deduplicate: preferensikan data extra (manual override) jika job_text_raw sama
|
||||
df_all = df_all.drop_duplicates(subset=["job_text_raw"], keep="last")
|
||||
logger.info(
|
||||
f"Data manual_override: {len(df_extra)} baris | "
|
||||
f"weight={override_w:.4f}x (log-damped) | "
|
||||
f"influence aktual={actual_influence_pct:.1f}% "
|
||||
f"(target nominal {OVERRIDE_INFLUENCE_TARGET*100:.0f}%)"
|
||||
)
|
||||
else:
|
||||
df_all = df_corpus.copy()
|
||||
logger.info("Tidak ada data tambahan — menggunakan corpus asli saja")
|
||||
|
||||
logger.info(f"Total setelah merge & deduplicate: {len(df_all)} baris")
|
||||
# Concat tanpa deduplication — semua frekuensi historis dipertahankan
|
||||
df_all = pd.concat([df_corpus, df_extra], ignore_index=True) if len(df_extra) > 0 else df_corpus.copy()
|
||||
|
||||
mask = df_all["features"].str.len() > 0
|
||||
df_all = df_all[mask]
|
||||
logger.info(f"Total setelah merge (tanpa deduplicate): {len(df_all)} baris")
|
||||
|
||||
if len(df_all) < 20:
|
||||
raise ValueError(f"Data terlalu sedikit untuk training: {len(df_all)} baris (minimum 20)")
|
||||
|
||||
# ── STEP 3: PREPROCESSING ────────────────────────────
|
||||
write_status("preprocessing", "Preprocessing teks...")
|
||||
X_all = df_all["job_text_raw"].apply(preprocess_text)
|
||||
X_all = df_all["features"]
|
||||
y_all = df_all["label"]
|
||||
mask = X_all.str.len() > 0
|
||||
X_all, y_all = X_all[mask], y_all[mask]
|
||||
logger.info(f"Setelah filter kosong: {len(X_all)} baris")
|
||||
w_all = df_all["_weight"]
|
||||
|
||||
# ── STEP 4: SPLIT TRAIN / TEST ───────────────────────
|
||||
# Cek apakah test_set_3.csv ada untuk hold-out
|
||||
if TEST_PATH.exists():
|
||||
df_test = pd.read_csv(TEST_PATH, sep=";", dtype=str).dropna(subset=["job_text_raw", "label"])
|
||||
X_test = df_test["job_text_raw"].apply(preprocess_text)
|
||||
y_test = df_test["label"]
|
||||
mask_t = X_test.str.len() > 0
|
||||
X_test, y_test = X_test[mask_t], y_test[mask_t]
|
||||
X_train, y_train = X_all, y_all
|
||||
logger.info(f"Hold-out test dari file: {len(X_test)} baris")
|
||||
else:
|
||||
# Fallback: split 70/30 dari data gabungan
|
||||
X_train, X_test, y_train, y_test = train_test_split(
|
||||
X_all, y_all, test_size=0.3, random_state=42, stratify=y_all
|
||||
)
|
||||
logger.info(f"Hold-out test dari split 30%: {len(X_test)} baris")
|
||||
# ── STEP 3: DYNAMIC SPLIT — selalu dari data gabungan terkini ────
|
||||
# Test set statis (test_set.csv) tidak digunakan karena:
|
||||
# (a) tidak mencerminkan pola baru dari data override
|
||||
# (b) baseline_f1 dari JSON dihitung pada distribusi berbeda →
|
||||
# perbandingan apel vs jeruk, tidak valid untuk keputusan promote.
|
||||
# Solusi: split dinamis, lalu evaluasi model LAMA pada test set YANG SAMA.
|
||||
X_train, X_test, y_train, y_test, w_train, _ = train_test_split(
|
||||
X_all, y_all, w_all, test_size=0.3, random_state=42, stratify=y_all
|
||||
)
|
||||
logger.info(f"Dynamic split: {len(X_train)} train | {len(X_test)} test")
|
||||
|
||||
# ── STEP 5: TRAINING ─────────────────────────────────
|
||||
write_status("training", f"Training model baru... ({len(X_train)} sampel training)")
|
||||
# ── STEP 4: BASELINE — evaluasi model LAMA pada test set yang sama ──
|
||||
# Ini satu-satunya cara perbandingan yang jujur (apples-to-apples).
|
||||
write_status("evaluating", "Mengevaluasi model lama pada test set baru...")
|
||||
baseline_f1 = get_baseline_f1_on_testset(X_test, y_test)
|
||||
|
||||
# ── STEP 5: K-FOLD VALIDATION ────────────────────────
|
||||
write_status("training", f"K-Fold validation & training... ({len(X_train)} sampel)")
|
||||
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
|
||||
fold_metrics = []
|
||||
for i, (tr_idx, te_idx) in enumerate(skf.split(X_train, y_train)):
|
||||
|
|
@ -273,70 +347,68 @@ def main(extra_csv_path: str = None):
|
|||
('tfidf', TfidfVectorizer(max_features=3000, ngram_range=(1, 2), sublinear_tf=True, min_df=1)),
|
||||
('clf', LogisticRegression(max_iter=1000, class_weight='balanced', solver='lbfgs'))
|
||||
])
|
||||
m.fit(X_train.iloc[tr_idx], y_train.iloc[tr_idx])
|
||||
# sample_weight diteruskan ke step 'clf' via Pipeline naming convention
|
||||
m.fit(
|
||||
X_train.iloc[tr_idx], y_train.iloc[tr_idx],
|
||||
clf__sample_weight=w_train.iloc[tr_idx].values
|
||||
)
|
||||
y_pred = m.predict(X_train.iloc[te_idx])
|
||||
rep = classification_report(y_train.iloc[te_idx], y_pred, output_dict=True, zero_division=0)
|
||||
fold_metrics.append(rep)
|
||||
write_status("training", f"K-Fold selesai: fold {i+1}/5 | acc={rep['accuracy']:.4f}")
|
||||
|
||||
# Train final model on full training set
|
||||
# ── STEP 6: TRAINING MODEL FINAL ─────────────────────
|
||||
write_status("training", "Training model final pada seluruh data training...")
|
||||
candidate_model = Pipeline([
|
||||
('tfidf', TfidfVectorizer(max_features=3000, ngram_range=(1, 2), sublinear_tf=True, min_df=1)),
|
||||
('clf', LogisticRegression(max_iter=1000, class_weight='balanced', solver='lbfgs'))
|
||||
])
|
||||
candidate_model.fit(X_train, y_train)
|
||||
|
||||
# Simpan sebagai candidate (belum overwrite aktif)
|
||||
candidate_model.fit(X_train, y_train, clf__sample_weight=w_train.values)
|
||||
joblib.dump(candidate_model, CANDIDATE_PATH)
|
||||
logger.info(f"Candidate model tersimpan: {CANDIDATE_PATH}")
|
||||
|
||||
# ── STEP 6: EVALUASI & KEPUTUSAN ─────────────────────
|
||||
write_status("evaluating", "Mengevaluasi model baru vs model lama...")
|
||||
|
||||
# ── STEP 7: EVALUASI & KEPUTUSAN ─────────────────────
|
||||
write_status("evaluating", "Mengevaluasi model baru vs model lama (test set sama)...")
|
||||
new_metrics = evaluate_model(candidate_model, X_test, y_test)
|
||||
new_f1 = new_metrics["weighted_f1"]
|
||||
delta = new_f1 - baseline_f1
|
||||
new_f1 = new_metrics["weighted_f1"]
|
||||
delta = new_f1 - baseline_f1
|
||||
|
||||
logger.info(f"Baseline F1 : {baseline_f1:.4f}")
|
||||
logger.info(f"Candidate F1: {new_f1:.4f} (delta: {delta:+.4f})")
|
||||
logger.info(f"Baseline F1 (old model, same test) : {baseline_f1:.4f}")
|
||||
logger.info(f"Candidate F1 (new model, same test): {new_f1:.4f} (delta: {delta:+.4f})")
|
||||
|
||||
# ── KEPUTUSAN ────────────────────────────────────────
|
||||
if baseline_f1 == 0.0:
|
||||
# Tidak ada baseline (model lama belum pernah dievaluasi) → langsung promote
|
||||
reason_promote = "Tidak ada baseline metrics → model baru dipromote"
|
||||
should_promote = True
|
||||
reason = "Model lama tidak bisa dievaluasi (backup tidak ada) → model baru dipromote"
|
||||
elif delta >= MIN_IMPROVEMENT_THRESHOLD:
|
||||
should_promote = True
|
||||
reason_promote = f"Model baru lebih baik (+{delta*100:.2f}% weighted F1)"
|
||||
reason = f"Model baru lebih baik (+{delta*100:.2f}% weighted F1)"
|
||||
elif delta < -MAX_REGRESSION_ALLOWED:
|
||||
should_promote = False
|
||||
reason_rollback = f"Model baru lebih buruk secara signifikan ({delta*100:.2f}% weighted F1)"
|
||||
reason = f"Model baru lebih buruk secara signifikan ({delta*100:.2f}% weighted F1)"
|
||||
else:
|
||||
should_promote = False
|
||||
reason_rollback = (
|
||||
reason = (
|
||||
f"Peningkatan tidak signifikan ({delta*100:.2f}% weighted F1, "
|
||||
f"minimum dibutuhkan: +{MIN_IMPROVEMENT_THRESHOLD*100:.1f}%)"
|
||||
)
|
||||
|
||||
if not should_promote:
|
||||
do_rollback(reason_rollback, baseline_f1, new_f1)
|
||||
do_rollback(reason, baseline_f1, new_f1)
|
||||
return
|
||||
|
||||
# ── STEP 7: PROMOTE ──────────────────────────────────
|
||||
write_status("promoting", "Model baru lebih baik — mempromote model baru...")
|
||||
|
||||
# Atomic rename: candidate → aktif
|
||||
shutil.move(str(CANDIDATE_PATH), str(PIPELINE_PATH))
|
||||
logger.info(f"Model baru dipromote: {PIPELINE_PATH}")
|
||||
|
||||
# Update metrics JSON
|
||||
avg_acc = float(np.mean([f["accuracy"] for f in fold_metrics]))
|
||||
y_pred_final = candidate_model.predict(X_test)
|
||||
new_metrics_full = {
|
||||
"methodology": "Internal MIF + manual_override",
|
||||
"methodology": "Internal MIF + manual_override (sample_weight, dynamic split)",
|
||||
"retrained_at": datetime.now().isoformat(),
|
||||
"extra_samples_added": len(df_extra),
|
||||
"override_weight_used": round(df_extra["_weight"].iloc[0], 4) if len(df_extra) > 0 else 1.0,
|
||||
"override_influence_pct_actual": round(actual_influence_pct, 2),
|
||||
"override_influence_target_pct": OVERRIDE_INFLUENCE_TARGET * 100,
|
||||
"total_training_samples": len(X_train),
|
||||
"k_fold": {
|
||||
"accuracy_mean": avg_acc,
|
||||
|
|
@ -345,12 +417,12 @@ def main(extra_csv_path: str = None):
|
|||
},
|
||||
"threshold_config": 0.50,
|
||||
"hold_out_test": classification_report(
|
||||
y_test,
|
||||
y_pred_final,
|
||||
output_dict=True,
|
||||
zero_division=0
|
||||
y_test, candidate_model.predict(X_test),
|
||||
output_dict=True, zero_division=0
|
||||
),
|
||||
# Perbandingan fair: kedua model dievaluasi pada test set yang SAMA
|
||||
"comparison": {
|
||||
"note": "Both models evaluated on identical dynamic test set",
|
||||
"old_weighted_f1": baseline_f1,
|
||||
"new_weighted_f1": new_f1,
|
||||
"delta": round(delta, 4),
|
||||
|
|
@ -358,10 +430,9 @@ def main(extra_csv_path: str = None):
|
|||
}
|
||||
METRICS_PATH.write_text(json.dumps(new_metrics_full, indent=2, ensure_ascii=False))
|
||||
|
||||
|
||||
write_status(
|
||||
stage="promoted",
|
||||
message=f"Model baru berhasil dipromote. {reason_promote}",
|
||||
message=f"Model baru berhasil dipromote. {reason}",
|
||||
extra={
|
||||
"result": "promoted",
|
||||
"old_f1": baseline_f1,
|
||||
|
|
@ -382,13 +453,13 @@ def main(extra_csv_path: str = None):
|
|||
new_f1=0.0
|
||||
)
|
||||
# Override stage ke 'failed' agar UI tahu ini bukan rollback biasa
|
||||
status = json.loads(STATUS_PATH.read_text())
|
||||
status["stage"] = "failed"
|
||||
STATUS_PATH.write_text(json.dumps(status, indent=2, ensure_ascii=False))
|
||||
lock = FileLock(LOCK_PATH, timeout=10)
|
||||
with lock:
|
||||
status = json.loads(STATUS_PATH.read_text())
|
||||
status["stage"] = "failed"
|
||||
STATUS_PATH.write_text(json.dumps(status, indent=2, ensure_ascii=False))
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Argumen: [extra_csv_path]
|
||||
extra_csv = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
main(extra_csv)
|
||||
main(sys.argv[1] if len(sys.argv) > 1 else None)
|
||||
|
|
|
|||
Binary file not shown.
|
Before Width: | Height: | Size: 60 KiB After Width: | Height: | Size: 62 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 113 KiB After Width: | Height: | Size: 114 KiB |
|
|
@ -1,8 +1,8 @@
|
|||
{
|
||||
"methodology": "Internal MIF only",
|
||||
"k_fold": {
|
||||
"accuracy_mean": 0.7902040816326531,
|
||||
"accuracy_std": 0.046079445366646855,
|
||||
"accuracy_mean": 0.728,
|
||||
"accuracy_std": 0.08634813257969161,
|
||||
"folds": [
|
||||
{
|
||||
"Data Analyst": {
|
||||
|
|
@ -12,34 +12,34 @@
|
|||
"support": 1.0
|
||||
},
|
||||
"Non-IT": {
|
||||
"precision": 0.7619047619047619,
|
||||
"recall": 1.0,
|
||||
"f1-score": 0.8648648648648649,
|
||||
"support": 16.0
|
||||
"precision": 0.8,
|
||||
"recall": 0.8888888888888888,
|
||||
"f1-score": 0.8421052631578947,
|
||||
"support": 18.0
|
||||
},
|
||||
"Programmer": {
|
||||
"precision": 1.0,
|
||||
"recall": 0.5714285714285714,
|
||||
"f1-score": 0.7272727272727273,
|
||||
"support": 14.0
|
||||
"precision": 0.8,
|
||||
"recall": 0.6153846153846154,
|
||||
"f1-score": 0.6956521739130435,
|
||||
"support": 13.0
|
||||
},
|
||||
"Wirausaha Informatika": {
|
||||
"precision": 0.8571428571428571,
|
||||
"recall": 0.9473684210526315,
|
||||
"f1-score": 0.9,
|
||||
"support": 19.0
|
||||
"precision": 0.8,
|
||||
"recall": 0.8888888888888888,
|
||||
"f1-score": 0.8421052631578947,
|
||||
"support": 18.0
|
||||
},
|
||||
"accuracy": 0.84,
|
||||
"accuracy": 0.8,
|
||||
"macro avg": {
|
||||
"precision": 0.6547619047619048,
|
||||
"recall": 0.6296992481203008,
|
||||
"f1-score": 0.6230343980343981,
|
||||
"precision": 0.6000000000000001,
|
||||
"recall": 0.5982905982905983,
|
||||
"f1-score": 0.5949656750572082,
|
||||
"support": 50.0
|
||||
},
|
||||
"weighted avg": {
|
||||
"precision": 0.8495238095238095,
|
||||
"recall": 0.84,
|
||||
"f1-score": 0.8223931203931204,
|
||||
"precision": 0.784,
|
||||
"recall": 0.8,
|
||||
"f1-score": 0.7871853546910755,
|
||||
"support": 50.0
|
||||
}
|
||||
},
|
||||
|
|
@ -51,10 +51,10 @@
|
|||
"support": 1.0
|
||||
},
|
||||
"Non-IT": {
|
||||
"precision": 0.8,
|
||||
"precision": 0.7727272727272727,
|
||||
"recall": 1.0,
|
||||
"f1-score": 0.8888888888888888,
|
||||
"support": 16.0
|
||||
"f1-score": 0.8717948717948718,
|
||||
"support": 17.0
|
||||
},
|
||||
"Programmer": {
|
||||
"precision": 1.0,
|
||||
|
|
@ -63,37 +63,37 @@
|
|||
"support": 14.0
|
||||
},
|
||||
"Wirausaha Informatika": {
|
||||
"precision": 0.7083333333333334,
|
||||
"recall": 0.8947368421052632,
|
||||
"f1-score": 0.7906976744186046,
|
||||
"support": 19.0
|
||||
"precision": 0.6818181818181818,
|
||||
"recall": 0.8333333333333334,
|
||||
"f1-score": 0.75,
|
||||
"support": 18.0
|
||||
},
|
||||
"accuracy": 0.78,
|
||||
"accuracy": 0.76,
|
||||
"macro avg": {
|
||||
"precision": 0.6270833333333333,
|
||||
"recall": 0.5808270676691729,
|
||||
"f1-score": 0.5698966408268734,
|
||||
"precision": 0.6136363636363636,
|
||||
"recall": 0.5654761904761905,
|
||||
"f1-score": 0.555448717948718,
|
||||
"support": 50.0
|
||||
},
|
||||
"weighted avg": {
|
||||
"precision": 0.8051666666666667,
|
||||
"recall": 0.78,
|
||||
"f1-score": 0.7529095607235142,
|
||||
"precision": 0.7881818181818181,
|
||||
"recall": 0.76,
|
||||
"f1-score": 0.7344102564102565,
|
||||
"support": 50.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"Data Analyst": {
|
||||
"precision": 0.0,
|
||||
"recall": 0.0,
|
||||
"f1-score": 0.0,
|
||||
"support": 2.0
|
||||
"precision": 1.0,
|
||||
"recall": 1.0,
|
||||
"f1-score": 1.0,
|
||||
"support": 1.0
|
||||
},
|
||||
"Non-IT": {
|
||||
"precision": 0.7142857142857143,
|
||||
"precision": 0.6296296296296297,
|
||||
"recall": 1.0,
|
||||
"f1-score": 0.8333333333333334,
|
||||
"support": 15.0
|
||||
"f1-score": 0.7727272727272727,
|
||||
"support": 17.0
|
||||
},
|
||||
"Programmer": {
|
||||
"precision": 1.0,
|
||||
|
|
@ -102,101 +102,101 @@
|
|||
"support": 14.0
|
||||
},
|
||||
"Wirausaha Informatika": {
|
||||
"precision": 0.7916666666666666,
|
||||
"recall": 1.0,
|
||||
"f1-score": 0.8837209302325582,
|
||||
"support": 19.0
|
||||
"precision": 0.9411764705882353,
|
||||
"recall": 0.8888888888888888,
|
||||
"f1-score": 0.9142857142857143,
|
||||
"support": 18.0
|
||||
},
|
||||
"accuracy": 0.78,
|
||||
"macro avg": {
|
||||
"precision": 0.6264880952380952,
|
||||
"recall": 0.5892857142857143,
|
||||
"f1-score": 0.560842513259894,
|
||||
"precision": 0.8927015250544663,
|
||||
"recall": 0.8115079365079365,
|
||||
"f1-score": 0.8033321941216678,
|
||||
"support": 50.0
|
||||
},
|
||||
"weighted avg": {
|
||||
"precision": 0.7951190476190476,
|
||||
"precision": 0.8528976034858389,
|
||||
"recall": 0.78,
|
||||
"f1-score": 0.7331823745410037,
|
||||
"f1-score": 0.7592385509227614,
|
||||
"support": 50.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"Data Analyst": {
|
||||
"precision": 0.0,
|
||||
"recall": 0.0,
|
||||
"f1-score": 0.0,
|
||||
"precision": 1.0,
|
||||
"recall": 1.0,
|
||||
"f1-score": 1.0,
|
||||
"support": 1.0
|
||||
},
|
||||
"Non-IT": {
|
||||
"precision": 0.7272727272727273,
|
||||
"precision": 0.6538461538461539,
|
||||
"recall": 1.0,
|
||||
"f1-score": 0.8421052631578947,
|
||||
"support": 16.0
|
||||
"f1-score": 0.7906976744186046,
|
||||
"support": 17.0
|
||||
},
|
||||
"Programmer": {
|
||||
"precision": 1.0,
|
||||
"recall": 0.5,
|
||||
"f1-score": 0.6666666666666666,
|
||||
"recall": 0.42857142857142855,
|
||||
"f1-score": 0.6,
|
||||
"support": 14.0
|
||||
},
|
||||
"Wirausaha Informatika": {
|
||||
"precision": 0.9,
|
||||
"recall": 1.0,
|
||||
"f1-score": 0.9473684210526315,
|
||||
"precision": 0.7647058823529411,
|
||||
"recall": 0.7222222222222222,
|
||||
"f1-score": 0.7428571428571429,
|
||||
"support": 18.0
|
||||
},
|
||||
"accuracy": 0.8367346938775511,
|
||||
"accuracy": 0.74,
|
||||
"macro avg": {
|
||||
"precision": 0.6568181818181819,
|
||||
"recall": 0.625,
|
||||
"f1-score": 0.6140350877192982,
|
||||
"support": 49.0
|
||||
"precision": 0.8546380090497737,
|
||||
"recall": 0.7876984126984127,
|
||||
"f1-score": 0.7833887043189369,
|
||||
"support": 50.0
|
||||
},
|
||||
"weighted avg": {
|
||||
"precision": 0.8538033395176252,
|
||||
"recall": 0.8367346938775511,
|
||||
"f1-score": 0.8134622269960615,
|
||||
"support": 49.0
|
||||
"precision": 0.797601809954751,
|
||||
"recall": 0.74,
|
||||
"f1-score": 0.724265780730897,
|
||||
"support": 50.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"Data Analyst": {
|
||||
"precision": 0.0,
|
||||
"recall": 0.0,
|
||||
"f1-score": 0.0,
|
||||
"support": 1.0
|
||||
"precision": 0.07692307692307693,
|
||||
"recall": 0.5,
|
||||
"f1-score": 0.13333333333333333,
|
||||
"support": 2.0
|
||||
},
|
||||
"Non-IT": {
|
||||
"precision": 0.8,
|
||||
"precision": 0.6923076923076923,
|
||||
"recall": 1.0,
|
||||
"f1-score": 0.8888888888888888,
|
||||
"support": 16.0
|
||||
"f1-score": 0.8181818181818182,
|
||||
"support": 18.0
|
||||
},
|
||||
"Programmer": {
|
||||
"precision": 1.0,
|
||||
"recall": 0.21428571428571427,
|
||||
"f1-score": 0.35294117647058826,
|
||||
"support": 14.0
|
||||
"recall": 0.23076923076923078,
|
||||
"f1-score": 0.375,
|
||||
"support": 13.0
|
||||
},
|
||||
"Wirausaha Informatika": {
|
||||
"precision": 0.6153846153846154,
|
||||
"recall": 0.8888888888888888,
|
||||
"f1-score": 0.7272727272727273,
|
||||
"support": 18.0
|
||||
"precision": 0.75,
|
||||
"recall": 0.35294117647058826,
|
||||
"f1-score": 0.48,
|
||||
"support": 17.0
|
||||
},
|
||||
"accuracy": 0.7142857142857143,
|
||||
"accuracy": 0.56,
|
||||
"macro avg": {
|
||||
"precision": 0.6038461538461539,
|
||||
"recall": 0.5257936507936507,
|
||||
"f1-score": 0.4922756981580511,
|
||||
"support": 49.0
|
||||
"precision": 0.6298076923076923,
|
||||
"recall": 0.5209276018099548,
|
||||
"f1-score": 0.4516287878787879,
|
||||
"support": 50.0
|
||||
},
|
||||
"weighted avg": {
|
||||
"precision": 0.7729984301412873,
|
||||
"recall": 0.7142857142857143,
|
||||
"f1-score": 0.6582511792595825,
|
||||
"support": 49.0
|
||||
"precision": 0.7673076923076922,
|
||||
"recall": 0.56,
|
||||
"f1-score": 0.5605787878787879,
|
||||
"support": 50.0
|
||||
}
|
||||
}
|
||||
]
|
||||
|
|
@ -210,35 +210,35 @@
|
|||
"support": 2.0
|
||||
},
|
||||
"Non-IT": {
|
||||
"precision": 0.7083333333333334,
|
||||
"recall": 1.0,
|
||||
"f1-score": 0.8292682926829268,
|
||||
"support": 34.0
|
||||
"precision": 0.7777777777777778,
|
||||
"recall": 0.9722222222222222,
|
||||
"f1-score": 0.8641975308641975,
|
||||
"support": 36.0
|
||||
},
|
||||
"Programmer": {
|
||||
"precision": 0.9444444444444444,
|
||||
"recall": 0.5483870967741935,
|
||||
"f1-score": 0.6938775510204082,
|
||||
"support": 31.0
|
||||
"precision": 0.95,
|
||||
"recall": 0.6333333333333333,
|
||||
"f1-score": 0.76,
|
||||
"support": 30.0
|
||||
},
|
||||
"Wirausaha Informatika": {
|
||||
"precision": 0.8333333333333334,
|
||||
"recall": 0.8536585365853658,
|
||||
"f1-score": 0.8433734939759037,
|
||||
"support": 41.0
|
||||
"precision": 0.8571428571428571,
|
||||
"recall": 0.9230769230769231,
|
||||
"f1-score": 0.8888888888888888,
|
||||
"support": 39.0
|
||||
},
|
||||
"accuracy": 0.7962962962962963,
|
||||
"accuracy": 0.8411214953271028,
|
||||
"macro avg": {
|
||||
"precision": 0.6215277777777778,
|
||||
"recall": 0.6005114083398898,
|
||||
"f1-score": 0.5916298344198097,
|
||||
"support": 108.0
|
||||
"precision": 0.6462301587301588,
|
||||
"recall": 0.6321581196581196,
|
||||
"f1-score": 0.6282716049382716,
|
||||
"support": 107.0
|
||||
},
|
||||
"weighted avg": {
|
||||
"precision": 0.8104423868312758,
|
||||
"recall": 0.7962962962962963,
|
||||
"f1-score": 0.7804040674617057,
|
||||
"support": 108.0
|
||||
"precision": 0.8404539385847796,
|
||||
"recall": 0.8411214953271028,
|
||||
"f1-score": 0.8278296988577363,
|
||||
"support": 107.0
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,10 +1,10 @@
|
|||
{
|
||||
"stage": "rolled_back",
|
||||
"message": "Model lama dipertahankan. Model baru lebih buruk secara signifikan (-12.92% weighted F1)",
|
||||
"timestamp": "2026-05-25T06:51:07.653672",
|
||||
"message": "Model lama dipertahankan. Model baru lebih buruk secara signifikan (-22.52% weighted F1)",
|
||||
"timestamp": "2026-05-28T16:37:10.275000",
|
||||
"result": "rolled_back",
|
||||
"reason": "Model baru lebih buruk secara signifikan (-12.92% weighted F1)",
|
||||
"old_f1": 0.7804040674617057,
|
||||
"new_f1": 0.6512,
|
||||
"reason": "Model baru lebih buruk secara signifikan (-22.52% weighted F1)",
|
||||
"old_f1": 0.8788,
|
||||
"new_f1": 0.6536,
|
||||
"_reloaded": true
|
||||
}
|
||||
Binary file not shown.
|
|
@ -18,118 +18,145 @@ public function __construct(protected FastApiWorkerService $fastApi) {}
|
|||
public function upload(Request $request) {
|
||||
$request->validate(['file' => 'required|file|mimes:csv,xlsx,xls|max:20480']);
|
||||
|
||||
$result = $this->fastApi->classifyFile($request->file('file')->getRealPath(), $request->file('file')->getClientOriginalName());
|
||||
$file = $request->file('file');
|
||||
$result = $this->fastApi->classifyFile($file->getRealPath(), $file->getClientOriginalName());
|
||||
|
||||
if (!isset($result['results']) && isset($result['error'])) {
|
||||
if (isset($result['error'])) {
|
||||
if ($request->wantsJson()) return response()->json(['success' => false, 'message' => $result['error']], 400);
|
||||
return back()->with('error', $result['error']);
|
||||
}
|
||||
|
||||
if (empty($result['results'])) {
|
||||
return back()->with('success', "Selesai. 0 data berhasil diproses.");
|
||||
if ($request->wantsJson()) {
|
||||
return response()->json([
|
||||
'success' => true,
|
||||
'job_id' => $result['job_id'] ?? null,
|
||||
'message' => $result['message'] ?? 'Proses dimulai...'
|
||||
]);
|
||||
}
|
||||
return back()->with('success', 'File sedang diproses. Silakan refresh beberapa saat lagi.');
|
||||
}
|
||||
|
||||
public function checkStatus(Request $request, string $jobId)
|
||||
{
|
||||
$status = $this->fastApi->getClassifyStatus($jobId);
|
||||
|
||||
if (!isset($status['status']) || in_array($status['status'], ['pending', 'processing'])) {
|
||||
return response()->json($status);
|
||||
}
|
||||
|
||||
DB::transaction(function () use ($result) {
|
||||
$kemendikUpsertData = [];
|
||||
$internalResultsUpsertData = [];
|
||||
$internalRawUpsertData = [];
|
||||
if ($status['status'] === 'error') {
|
||||
return response()->json($status);
|
||||
}
|
||||
|
||||
foreach ($result['results'] as $row) {
|
||||
if ($row['source_type'] === 'kemendik') {
|
||||
$raw = $row['raw_data'] ?? [];
|
||||
$kemendikUpsertData[] = [
|
||||
'nimhsmsmh' => $row['nim'],
|
||||
'source_type' => 'kemendik',
|
||||
'nmmhsmsmh' => $row['nama'],
|
||||
'tahun_lulus' => $row['tahun_lulus'] ?? null,
|
||||
'f5c_jabatan_kode' => $raw['F5c'] ?? null,
|
||||
'f5c_jabatan_text' => $this->mapF5c($raw['F5c'] ?? null),
|
||||
'f1101_jenis_instansi_kode' => $raw['F1101'] ?? null,
|
||||
'f1101_jenis_instansi_text' => $this->mapF1101($raw['F1101'] ?? null),
|
||||
'f5a1_provinsi' => $raw['F5a1'] ?? null,
|
||||
'raw_data' => json_encode($raw)
|
||||
];
|
||||
} else {
|
||||
$internalResultsUpsertData[] = [
|
||||
'nim' => $row['nim'],
|
||||
'source_type' => 'internal_mif',
|
||||
'nama' => $row['nama'],
|
||||
'tahun_lulus' => $row['tahun_lulus'] ?? null,
|
||||
'job_text_raw' => $row['job_text_raw'] ?? '',
|
||||
'predicted_profile' => $row['predicted_profile'],
|
||||
'confidence_score' => $row['confidence_score'] ?? 0,
|
||||
'classification_method' => $row['classification_method'],
|
||||
'status' => $row['status'],
|
||||
'error_detail' => $row['error_detail'] ?? null
|
||||
];
|
||||
|
||||
if (isset($row['raw_data'])) {
|
||||
$mapped = ['nim' => $row['nim']];
|
||||
$cols = [
|
||||
'nama_lengkap', 'email', 'no_telepon', 'alamat_domisili', 'jurusan',
|
||||
'program_studi', 'tahun_masuk', 'tahun_lulus', 'status_pekerjaan',
|
||||
'klasifikasi_pekerjaan', 'nama_perusahaan', 'jenis_perusahaan', 'jabatan',
|
||||
'lokasi_perusahaan', 'alamat_perusahaan', 'deskripsi_pekerjaan',
|
||||
'tahun_mulai_kerja', 'masa_kerja_bulan', 'jumlah_lamaran_dikirim',
|
||||
'jumlah_respons_lamaran', 'jumlah_undangan_wawancara', 'masa_tunggu_pra_lulus',
|
||||
'masa_tunggu_pasca_lulus', 'total_masa_tunggu', 'instansi_terupdate',
|
||||
'jabatan_terupdate', 'tahun_bekerja', 'status_pekerjaan_terupdate',
|
||||
'linkedin_profile', 'sosmed_ig'
|
||||
];
|
||||
foreach ($cols as $col) {
|
||||
$val = $row['raw_data'][$col] ?? null;
|
||||
$mapped[$col] = $val === '' ? null : $val;
|
||||
if ($status['status'] === 'success' && isset($status['results'])) {
|
||||
try {
|
||||
DB::transaction(function () use ($status) {
|
||||
$chunks = array_chunk($status['results'], 500);
|
||||
foreach ($chunks as $chunk) {
|
||||
$kemendikUpsertData = [];
|
||||
$internalResultsUpsertData = [];
|
||||
$internalRawUpsertData = [];
|
||||
|
||||
foreach ($chunk as $row) {
|
||||
if ($row['source_type'] === 'kemendik') {
|
||||
$raw = $row['raw_data'] ?? [];
|
||||
$kemendikUpsertData[] = [
|
||||
'nimhsmsmh' => $row['nim'],
|
||||
'source_type' => 'kemendik',
|
||||
'nmmhsmsmh' => $row['nama'],
|
||||
'tahun_lulus' => $row['tahun_lulus'] ?? null,
|
||||
'f5c_jabatan_kode' => $raw['F5c'] ?? null,
|
||||
'f5c_jabatan_text' => $this->mapF5c($raw['F5c'] ?? null),
|
||||
'f1101_jenis_instansi_kode' => $raw['F1101'] ?? null,
|
||||
'f1101_jenis_instansi_text' => $this->mapF1101($raw['F1101'] ?? null),
|
||||
'f5a1_provinsi' => $raw['F5a1'] ?? null,
|
||||
'raw_data' => json_encode($raw)
|
||||
];
|
||||
} else {
|
||||
$internalResultsUpsertData[] = [
|
||||
'nim' => $row['nim'],
|
||||
'source_type' => 'internal_mif',
|
||||
'job_text_raw' => $row['job_text_raw'] ?? '',
|
||||
'predicted_profile' => $row['predicted_profile'],
|
||||
'confidence_score' => $row['confidence_score'] ?? 0,
|
||||
'classification_method' => $row['classification_method'],
|
||||
'status' => $row['status'],
|
||||
'error_detail' => $row['error_detail'] ?? null
|
||||
];
|
||||
|
||||
if (isset($row['raw_data'])) {
|
||||
$mapped = ['nim' => $row['nim']];
|
||||
$cols = [
|
||||
'nama_lengkap', 'email', 'no_telepon', 'alamat_domisili', 'jurusan',
|
||||
'program_studi', 'tahun_masuk', 'tahun_lulus', 'status_pekerjaan',
|
||||
'klasifikasi_pekerjaan', 'nama_perusahaan', 'jenis_perusahaan', 'jabatan',
|
||||
'lokasi_perusahaan', 'alamat_perusahaan', 'deskripsi_pekerjaan',
|
||||
'tahun_mulai_kerja', 'masa_kerja_bulan', 'jumlah_lamaran_dikirim',
|
||||
'jumlah_respons_lamaran', 'jumlah_undangan_wawancara', 'masa_tunggu_pra_lulus',
|
||||
'masa_tunggu_pasca_lulus', 'total_masa_tunggu', 'instansi_terupdate',
|
||||
'jabatan_terupdate', 'tahun_bekerja', 'status_pekerjaan_terupdate',
|
||||
'linkedin_profile', 'sosmed_ig'
|
||||
];
|
||||
foreach ($cols as $col) {
|
||||
$val = $row['raw_data'][$col] ?? null;
|
||||
$mapped[$col] = $val === '' ? null : $val;
|
||||
}
|
||||
$internalRawUpsertData[] = $mapped;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!empty($kemendikUpsertData)) {
|
||||
KemendikRawData::upsert(
|
||||
$kemendikUpsertData,
|
||||
['nimhsmsmh', 'source_type'],
|
||||
['nmmhsmsmh', 'tahun_lulus', 'f5c_jabatan_kode', 'f5c_jabatan_text', 'f1101_jenis_instansi_kode', 'f1101_jenis_instansi_text', 'f5a1_provinsi', 'raw_data']
|
||||
);
|
||||
}
|
||||
|
||||
if (!empty($internalResultsUpsertData)) {
|
||||
ClassificationResult::upsert(
|
||||
$internalResultsUpsertData,
|
||||
['nim', 'source_type'],
|
||||
['job_text_raw', 'predicted_profile', 'confidence_score', 'classification_method', 'status', 'error_detail']
|
||||
);
|
||||
}
|
||||
|
||||
if (!empty($internalRawUpsertData)) {
|
||||
$cols = [
|
||||
'nama_lengkap', 'email', 'no_telepon', 'alamat_domisili', 'jurusan',
|
||||
'program_studi', 'tahun_masuk', 'tahun_lulus', 'status_pekerjaan',
|
||||
'klasifikasi_pekerjaan', 'nama_perusahaan', 'jenis_perusahaan', 'jabatan',
|
||||
'lokasi_perusahaan', 'alamat_perusahaan', 'deskripsi_pekerjaan',
|
||||
'tahun_mulai_kerja', 'masa_kerja_bulan', 'jumlah_lamaran_dikirim',
|
||||
'jumlah_respons_lamaran', 'jumlah_undangan_wawancara', 'masa_tunggu_pra_lulus',
|
||||
'masa_tunggu_pasca_lulus', 'total_masa_tunggu', 'instansi_terupdate',
|
||||
'jabatan_terupdate', 'tahun_bekerja', 'status_pekerjaan_terupdate',
|
||||
'linkedin_profile', 'sosmed_ig'
|
||||
];
|
||||
InternalRawData::upsert($internalRawUpsertData, ['nim'], $cols);
|
||||
}
|
||||
$internalRawUpsertData[] = $mapped;
|
||||
}
|
||||
});
|
||||
|
||||
$source_type = $status['source_type'] ?? 'unknown';
|
||||
if ($source_type === 'kemendik') {
|
||||
$count = collect($status['results'])->where('source_type','kemendik')->count();
|
||||
$msg = "Selesai. {$status['processed_rows']} baris diproses. {$count} data Kemendik masuk.";
|
||||
} else {
|
||||
$classified = collect($status['results'])->where('source_type','internal_mif')->where('status','auto_classified')->count();
|
||||
$msg = "Selesai. {$status['processed_rows']} data diproses. {$classified} otomatis (MIF).";
|
||||
}
|
||||
|
||||
// Hapus data results dari file status agar tidak berat saat diambil UI selanjutnya? Tidak perlu.
|
||||
return response()->json(['status' => 'completed', 'message' => $msg]);
|
||||
} catch (\Exception $e) {
|
||||
Log::error("DB Upsert failed: " . $e->getMessage());
|
||||
return response()->json(['status' => 'error', 'message' => "Gagal menyimpan database: " . $e->getMessage()]);
|
||||
}
|
||||
|
||||
if (!empty($kemendikUpsertData)) {
|
||||
foreach (array_chunk($kemendikUpsertData, 500) as $chunk) {
|
||||
KemendikRawData::upsert(
|
||||
$chunk,
|
||||
['nimhsmsmh', 'source_type'],
|
||||
['nmmhsmsmh', 'tahun_lulus', 'f5c_jabatan_kode', 'f5c_jabatan_text', 'f1101_jenis_instansi_kode', 'f1101_jenis_instansi_text', 'f5a1_provinsi', 'raw_data']
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (!empty($internalResultsUpsertData)) {
|
||||
foreach (array_chunk($internalResultsUpsertData, 500) as $chunk) {
|
||||
ClassificationResult::upsert(
|
||||
$chunk,
|
||||
['nim', 'source_type'],
|
||||
['nama', 'tahun_lulus', 'job_text_raw', 'predicted_profile', 'confidence_score', 'classification_method', 'status', 'error_detail']
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (!empty($internalRawUpsertData)) {
|
||||
$cols = [
|
||||
'nama_lengkap', 'email', 'no_telepon', 'alamat_domisili', 'jurusan',
|
||||
'program_studi', 'tahun_masuk', 'tahun_lulus', 'status_pekerjaan',
|
||||
'klasifikasi_pekerjaan', 'nama_perusahaan', 'jenis_perusahaan', 'jabatan',
|
||||
'lokasi_perusahaan', 'alamat_perusahaan', 'deskripsi_pekerjaan',
|
||||
'tahun_mulai_kerja', 'masa_kerja_bulan', 'jumlah_lamaran_dikirim',
|
||||
'jumlah_respons_lamaran', 'jumlah_undangan_wawancara', 'masa_tunggu_pra_lulus',
|
||||
'masa_tunggu_pasca_lulus', 'total_masa_tunggu', 'instansi_terupdate',
|
||||
'jabatan_terupdate', 'tahun_bekerja', 'status_pekerjaan_terupdate',
|
||||
'linkedin_profile', 'sosmed_ig'
|
||||
];
|
||||
foreach (array_chunk($internalRawUpsertData, 500) as $chunk) {
|
||||
InternalRawData::upsert($chunk, ['nim'], $cols);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
$isKemendik = $result['results'][0]['source_type'] === 'kemendik';
|
||||
if ($isKemendik) {
|
||||
$count = collect($result['results'])->where('source_type','kemendik')->count();
|
||||
return back()->with('success', "Selesai. {$result['processed_rows']} baris diproses. {$count} data Kemendik berhasil masuk.");
|
||||
} else {
|
||||
$classified = collect($result['results'])->where('source_type','internal_mif')->where('status','auto_classified')->count();
|
||||
return back()->with('success', "Selesai. {$result['processed_rows']} data diproses. {$classified} diklasifikasi otomatis (Internal MIF).");
|
||||
}
|
||||
|
||||
return response()->json(['status' => 'error', 'message' => 'Status tidak dikenali.']);
|
||||
}
|
||||
|
||||
private function mapF5c(?string $c): ?string { return ['1'=>'Founder','2'=>'Co-Founder','3'=>'Staff','4'=>'Freelance'][$c] ?? null; }
|
||||
|
|
@ -141,16 +168,17 @@ public function retrain(Request $request)
|
|||
->where('status', 'manual_override')
|
||||
->whereNotNull('job_text_raw')
|
||||
->whereNotNull('predicted_profile')
|
||||
->get(['job_text_raw', 'predicted_profile']);
|
||||
->get(['nim', 'job_text_raw', 'predicted_profile']);
|
||||
|
||||
$csvPath = null;
|
||||
|
||||
if ($overrideData->count() > 0) {
|
||||
$csvLines = ['job_text_raw;label'];
|
||||
$csvLines = ['nim;job_text_raw;label'];
|
||||
foreach ($overrideData as $row) {
|
||||
$nim = $row->nim;
|
||||
$text = str_replace([';', "\n", "\r"], [',', ' ', ' '], $row->job_text_raw);
|
||||
$label = $row->predicted_profile;
|
||||
$csvLines[] = "{$text};{$label}";
|
||||
$csvLines[] = "{$nim};{$text};{$label}";
|
||||
}
|
||||
|
||||
$tmpDir = storage_path('app/temp');
|
||||
|
|
|
|||
|
|
@ -21,10 +21,24 @@ public function index(Request $request)
|
|||
|
||||
$query->orderByRaw("FIELD(status, 'needs_review', 'manual_override', 'auto_classified', 'failed')");
|
||||
|
||||
if ($request->filled('search')) {
|
||||
$query->where(function($q) use ($request) {
|
||||
$q->where('job_text_raw', 'LIKE', '%' . $request->search . '%')
|
||||
->orWhere('nim', 'LIKE', '%' . $request->search . '%')
|
||||
->orWhereHas('internalRaw', function($q) use ($request) {
|
||||
$q->where('nama_lengkap', 'LIKE', '%' . $request->search . '%');
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
if ($request->has('sort')) {
|
||||
$allowedSorts = ['nim', 'nama', 'job_text_raw', 'predicted_profile', 'status'];
|
||||
if (in_array($request->sort, $allowedSorts)) {
|
||||
$query->orderBy($request->sort, $request->direction === 'desc' ? 'desc' : 'asc');
|
||||
$allowedSorts = ['nim', 'job_text_raw', 'predicted_profile', 'status'];
|
||||
if ($request->sort === 'nama') {
|
||||
$query->join('internal_raw_data', 'classification_results.nim', '=', 'internal_raw_data.nim')
|
||||
->orderBy('internal_raw_data.nama_lengkap', $request->direction === 'desc' ? 'desc' : 'asc')
|
||||
->select('classification_results.*');
|
||||
} elseif (in_array($request->sort, $allowedSorts)) {
|
||||
$query->orderBy('classification_results.'.$request->sort, $request->direction === 'desc' ? 'desc' : 'asc');
|
||||
}
|
||||
} else {
|
||||
$query->latest();
|
||||
|
|
@ -119,7 +133,7 @@ public function update(Request $request, $id)
|
|||
'status' => 'manual_override'
|
||||
]);
|
||||
|
||||
return back()->with('success', 'Data berhasil diperbarui!');
|
||||
return redirect(url()->previous() . '#tabel-data')->with('success', 'Data berhasil diperbarui!');
|
||||
}
|
||||
|
||||
public function destroy($id)
|
||||
|
|
@ -127,7 +141,7 @@ public function destroy($id)
|
|||
$record = ClassificationResult::findOrFail($id);
|
||||
$record->delete();
|
||||
|
||||
return back()->with('success', 'Data berhasil dihapus!');
|
||||
return redirect(url()->previous() . '#tabel-data')->with('success', 'Data berhasil dihapus!');
|
||||
}
|
||||
|
||||
public function bulkDestroy(Request $request)
|
||||
|
|
@ -139,7 +153,7 @@ public function bulkDestroy(Request $request)
|
|||
|
||||
ClassificationResult::whereIn('id', $request->ids)->delete();
|
||||
|
||||
return back()->with('success', count($request->ids) . ' data berhasil dihapus!');
|
||||
return redirect(url()->previous() . '#tabel-data')->with('success', count($request->ids) . ' data berhasil dihapus!');
|
||||
}
|
||||
|
||||
public function bulkUpdate(Request $request)
|
||||
|
|
@ -155,7 +169,7 @@ public function bulkUpdate(Request $request)
|
|||
'status' => 'manual_override'
|
||||
]);
|
||||
|
||||
return back()->with('success', count($request->ids) . ' data berhasil diubah profilnya!');
|
||||
return redirect(url()->previous() . '#tabel-data')->with('success', count($request->ids) . ' data berhasil diubah profilnya!');
|
||||
}
|
||||
|
||||
public function exportPdf(Request $request)
|
||||
|
|
@ -164,9 +178,13 @@ public function exportPdf(Request $request)
|
|||
$query->orderByRaw("FIELD(status, 'needs_review', 'manual_override', 'auto_classified', 'failed')");
|
||||
|
||||
if ($request->has('sort')) {
|
||||
$allowedSorts = ['nim', 'nama', 'job_text_raw', 'predicted_profile', 'status'];
|
||||
if (in_array($request->sort, $allowedSorts)) {
|
||||
$query->orderBy($request->sort, $request->direction === 'desc' ? 'desc' : 'asc');
|
||||
$allowedSorts = ['nim', 'job_text_raw', 'predicted_profile', 'status'];
|
||||
if ($request->sort === 'nama') {
|
||||
$query->join('internal_raw_data', 'classification_results.nim', '=', 'internal_raw_data.nim')
|
||||
->orderBy('internal_raw_data.nama_lengkap', $request->direction === 'desc' ? 'desc' : 'asc')
|
||||
->select('classification_results.*');
|
||||
} elseif (in_array($request->sort, $allowedSorts)) {
|
||||
$query->orderBy('classification_results.'.$request->sort, $request->direction === 'desc' ? 'desc' : 'asc');
|
||||
}
|
||||
} else {
|
||||
$query->latest();
|
||||
|
|
|
|||
|
|
@ -10,5 +10,15 @@ class ClassificationResult extends Model
|
|||
protected $casts = ['confidence_score' => 'float'];
|
||||
public function scopeInternal($q) { return $q->where('source_type', 'internal_mif'); }
|
||||
public function scopeNeedsReview($q) { return $q->where('status', 'needs_review'); }
|
||||
|
||||
public function internalRaw()
|
||||
{
|
||||
return $this->belongsTo(InternalRawData::class, 'nim', 'nim');
|
||||
}
|
||||
|
||||
public function kemendikRaw()
|
||||
{
|
||||
return $this->belongsTo(KemendikRawData::class, 'nim', 'nimhsmsmh');
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ public function classifyFile(string $filePath, string $filename): array
|
|||
throw new \Exception("Tidak dapat membaca file: {$filePath}");
|
||||
}
|
||||
|
||||
$response = $this->getClient(config('services.fastapi.timeout', 120))
|
||||
$response = $this->getClient(15)
|
||||
->attach('file', $fileStream, $filename)
|
||||
->post($this->baseUrl . '/api/v1/classify');
|
||||
|
||||
|
|
@ -61,6 +61,24 @@ public function classifyFile(string $filePath, string $filename): array
|
|||
}
|
||||
}
|
||||
|
||||
public function getClassifyStatus(string $jobId): array
|
||||
{
|
||||
try {
|
||||
$response = $this->getClient(10)->get($this->baseUrl . "/api/v1/classify/status/{$jobId}");
|
||||
if ($response->failed()) {
|
||||
if ($response->status() === 404) {
|
||||
return ['status' => 'error', 'message' => 'Job ID tidak ditemukan di server.'];
|
||||
}
|
||||
return ['status' => 'error', 'message' => "Gagal menghubungi server: HTTP {$response->status()}"];
|
||||
}
|
||||
return $response->json();
|
||||
} catch (ConnectionException $e) {
|
||||
return ['status' => 'error', 'message' => 'Koneksi ke server AI terputus.'];
|
||||
} catch (\Exception $e) {
|
||||
return ['status' => 'error', 'message' => "Kesalahan sistem: {$e->getMessage()}"];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Kirim CSV data manual_override ke FastAPI untuk memulai re-training.
|
||||
* CSV bersifat opsional — training tetap berjalan dengan corpus asli jika tidak ada.
|
||||
|
|
@ -85,6 +103,10 @@ public function triggerRetrain(?string $csvPath = null): array
|
|||
return ['error' => 'Re-training sedang berjalan. Tunggu hingga selesai.'];
|
||||
}
|
||||
|
||||
if ($response->status() === 429) {
|
||||
return ['error' => 'Sistem sibuk. Permintaan retraining lain sedang diproses.'];
|
||||
}
|
||||
|
||||
if ($response->failed()) {
|
||||
Log::error("FastAPI Retrain Error: {$response->status()} | {$response->body()}");
|
||||
return ['error' => "Gagal memulai re-training: HTTP {$response->status()}"];
|
||||
|
|
@ -113,6 +135,9 @@ public function getRetrainStatus(): array
|
|||
$response = $this->getClient(10)->get($this->baseUrl . '/api/v1/retrain/status');
|
||||
|
||||
if ($response->failed()) {
|
||||
if ($response->status() === 429) {
|
||||
return $response->json(); // Return stage locked dari FastAPI
|
||||
}
|
||||
return ['stage' => 'unknown', 'message' => 'Tidak dapat membaca status dari FastAPI.'];
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@
|
|||
],
|
||||
|
||||
'fastapi' => [
|
||||
'worker_url' => env('FASTAPI_WORKER_URL', 'http://127.0.0.1:8001'),
|
||||
'worker_url' => env('FASTAPI_WORKER_URL', ''),
|
||||
'secret_key' => env('FASTAPI_SECRET_KEY'),
|
||||
'timeout' => env('FASTAPI_TIMEOUT', 120),
|
||||
],
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@
|
|||
public function up(): void
|
||||
{
|
||||
Schema::table('users', function (Blueprint $table) {
|
||||
$table->enum('role', ['admin', 'cdc'])->default('cdc')->after('email');
|
||||
$table->string('role')->default('cdc');
|
||||
$table->boolean('is_active')->default(true)->after('role');
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,27 @@
|
|||
<?php
|
||||
|
||||
use Illuminate\Database\Migrations\Migration;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
use Illuminate\Support\Facades\Schema;
|
||||
|
||||
return new class extends Migration
|
||||
{
|
||||
public function up(): void
|
||||
{
|
||||
Schema::table('classification_results', function (Blueprint $table) {
|
||||
$table->dropColumn(['nama', 'tahun_lulus']);
|
||||
$table->index(['source_type', 'status']);
|
||||
$table->index('predicted_profile');
|
||||
});
|
||||
}
|
||||
|
||||
public function down(): void
|
||||
{
|
||||
Schema::table('classification_results', function (Blueprint $table) {
|
||||
$table->string('nama')->nullable();
|
||||
$table->string('tahun_lulus')->nullable();
|
||||
$table->dropIndex(['source_type', 'status']);
|
||||
$table->dropIndex(['predicted_profile']);
|
||||
});
|
||||
}
|
||||
};
|
||||
|
|
@ -103,20 +103,9 @@
|
|||
<p class="text-sm text-slate-500 mt-1">Unggah file Excel/CSV untuk memproses klasifikasi otomatis menggunakan Machine Learning.</p>
|
||||
</div>
|
||||
|
||||
@if(session('success'))
|
||||
<div class="bg-emerald-50 border border-emerald-200 text-emerald-700 px-4 py-3 rounded-xl mb-6 flex items-center space-x-3">
|
||||
<svg class="w-5 h-5" fill="currentColor" viewBox="0 0 20 20"><path fill-rule="evenodd" d="M10 18a8 8 0 100-16 8 8 0 000 16zm3.707-9.293a1 1 0 00-1.414-1.414L9 10.586 7.707 9.293a1 1 0 00-1.414 1.414l2 2a1 1 0 001.414 0l4-4z" clip-rule="evenodd"></path></svg>
|
||||
<span>{{ session('success') }}</span>
|
||||
</div>
|
||||
@endif
|
||||
@if(session('error'))
|
||||
<div class="bg-rose-50 border border-rose-200 text-rose-700 px-4 py-3 rounded-xl mb-6 flex items-center space-x-3">
|
||||
<svg class="w-5 h-5" fill="currentColor" viewBox="0 0 20 20"><path fill-rule="evenodd" d="M10 18a8 8 0 100-16 8 8 0 000 16zM8.707 7.293a1 1 0 00-1.414 1.414L8.586 10l-1.293 1.293a1 1 0 101.414 1.414L10 11.414l1.293 1.293a1 1 0 001.414-1.414L11.414 10l1.293-1.293a1 1 0 00-1.414-1.414L10 8.586 8.707 7.293z" clip-rule="evenodd"></path></svg>
|
||||
<span>{{ session('error') }}</span>
|
||||
</div>
|
||||
@endif
|
||||
|
||||
|
||||
<form action="{{ route('upload.process') }}" method="POST" enctype="multipart/form-data" class="flex flex-col md:flex-row gap-6 items-end">
|
||||
<form id="uploadForm" action="{{ route('upload.process') }}" method="POST" enctype="multipart/form-data" class="flex flex-col md:flex-row gap-6 items-end">
|
||||
@csrf
|
||||
<input type="hidden" name="source_type" value="internal_mif">
|
||||
|
||||
|
|
@ -339,13 +328,24 @@ class="inline-flex items-center gap-2 px-6 py-3 rounded-xl text-sm font-semibold
|
|||
</div>
|
||||
|
||||
<!-- Data Table -->
|
||||
<div class="bg-white rounded-2xl shadow-xl shadow-slate-200/50 border border-slate-100 overflow-hidden">
|
||||
<div id="tabel-data" class="bg-white rounded-2xl shadow-xl shadow-slate-200/50 border border-slate-100 overflow-hidden">
|
||||
<div class="px-6 py-5 border-b border-slate-100 bg-slate-50/50 flex justify-between items-center">
|
||||
<h3 class="text-lg font-bold text-slate-800">Tabel Hasil Klasifikasi</h3>
|
||||
<div class="flex items-center space-x-3">
|
||||
<div class="flex items-center space-x-3 w-full md:w-auto mt-4 md:mt-0">
|
||||
<form action="{{ route('dashboard') }}" method="GET" class="relative flex items-center">
|
||||
@if(request('sort')) <input type="hidden" name="sort" value="{{ request('sort') }}"> @endif
|
||||
@if(request('direction')) <input type="hidden" name="direction" value="{{ request('direction') }}"> @endif
|
||||
|
||||
<input type="text" name="search" value="{{ request('search') }}" placeholder="Cari job / nim / nama..." class="rounded-l-lg border-slate-300 text-sm focus:ring-blue-500 focus:border-blue-500 pl-4 py-2 w-full md:w-56 lg:w-64">
|
||||
<button type="submit" class="bg-blue-600 hover:bg-blue-700 text-white px-4 py-2 rounded-r-lg text-sm font-semibold border border-blue-600 transition-colors">Cari</button>
|
||||
|
||||
@if(request('search'))
|
||||
<a href="{{ route('dashboard', request()->except('search', 'page')) }}" class="ml-2 text-rose-500 hover:text-rose-700 text-sm font-medium whitespace-nowrap">Reset</a>
|
||||
@endif
|
||||
</form>
|
||||
<a href="{{ route('internal.download_pdf', request()->query()) }}" class="inline-flex items-center px-4 py-2 bg-emerald-600 text-white text-sm font-semibold rounded-lg hover:bg-emerald-700 transition-colors shadow-sm">
|
||||
<svg class="w-4 h-4 mr-2" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M12 10v6m0 0l-3-3m3 3l3-3m2 8H7a2 2 0 01-2-2V5a2 2 0 012-2h5.586a1 1 0 01.707.293l5.414 5.414a1 1 0 01.293.707V19a2 2 0 01-2 2z"></path></svg>
|
||||
Download PDF
|
||||
PDF
|
||||
</a>
|
||||
<div x-show="selectedIds.length > 0" style="display: none;" class="flex items-center space-x-2">
|
||||
<span class="text-sm text-slate-600 font-medium mr-2"><span x-text="selectedIds.length"></span> dipilih</span>
|
||||
|
|
@ -355,7 +355,7 @@ class="inline-flex items-center gap-2 px-6 py-3 rounded-xl text-sm font-semibold
|
|||
Edit
|
||||
</button>
|
||||
|
||||
<form action="{{ route('internal.bulk_destroy') }}" method="POST" onsubmit="return confirm('Yakin ingin menghapus data yang dipilih?');" class="inline-block">
|
||||
<form action="{{ route('internal.bulk_destroy') }}" method="POST" class="inline-block bulk-delete-form">
|
||||
@csrf
|
||||
<template x-for="id in selectedIds" :key="id">
|
||||
<input type="hidden" name="ids[]" :value="id">
|
||||
|
|
@ -410,7 +410,7 @@ class="inline-flex items-center gap-2 px-6 py-3 rounded-xl text-sm font-semibold
|
|||
</td>
|
||||
<td class="px-6 py-4 whitespace-nowrap">
|
||||
<div class="text-sm font-semibold text-slate-800">{{ $row->nim }}</div>
|
||||
<div class="text-sm text-slate-500">{{ $row->nama }}</div>
|
||||
<div class="text-sm text-slate-500">{{ $row->internalRaw->nama_lengkap ?? '-' }}</div>
|
||||
</td>
|
||||
<td class="px-6 py-4 text-sm text-slate-600 max-w-xs">
|
||||
<div class="line-clamp-2" title="{{ $row->job_text_raw }}">
|
||||
|
|
@ -454,8 +454,8 @@ class="inline-flex items-center gap-2 px-6 py-3 rounded-xl text-sm font-semibold
|
|||
@endif
|
||||
</td>
|
||||
<td class="px-6 py-4 whitespace-nowrap text-right text-sm font-medium space-x-2">
|
||||
<button @click="openEdit({{ $row->id }}, '{{ $row->predicted_profile }}', '{{ addslashes($row->nama) }}')" class="text-blue-600 hover:text-blue-900 transition-colors">Edit</button>
|
||||
<form action="{{ route('internal.destroy', $row->id) }}" method="POST" class="inline-block" onsubmit="return confirm('Yakin ingin menghapus data ini?');">
|
||||
<button @click="openEdit({{ $row->id }}, '{{ $row->predicted_profile }}', '{{ addslashes($row->internalRaw->nama_lengkap ?? '-') }}')" class="text-blue-600 hover:text-blue-900 transition-colors">Edit</button>
|
||||
<form action="{{ route('internal.destroy', $row->id) }}" method="POST" class="inline-block delete-form">
|
||||
@csrf
|
||||
@method('DELETE')
|
||||
<button type="submit" class="text-rose-600 hover:text-rose-900 transition-colors">Hapus</button>
|
||||
|
|
@ -472,7 +472,7 @@ class="inline-flex items-center gap-2 px-6 py-3 rounded-xl text-sm font-semibold
|
|||
<div class="bg-white px-4 pt-5 pb-4 sm:p-6 sm:pb-4 border-b border-slate-100">
|
||||
<div class="flex justify-between items-center mb-4">
|
||||
<h3 class="text-lg leading-6 font-bold text-slate-800" id="modal-title">
|
||||
Detail Data Internal - {{ $row->nama }}
|
||||
Detail Data Internal - {{ $row->internalRaw->nama_lengkap ?? '-' }}
|
||||
</h3>
|
||||
<button @click="openDetail = false" class="text-slate-400 hover:text-slate-600 focus:outline-none">
|
||||
<svg class="h-6 w-6" fill="none" viewBox="0 0 24 24" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M6 18L18 6M6 6l12 12" /></svg>
|
||||
|
|
@ -850,4 +850,150 @@ function retrainPanel() {
|
|||
};
|
||||
}
|
||||
</script>
|
||||
<script>
|
||||
document.addEventListener('DOMContentLoaded', () => {
|
||||
const tableContainer = document.getElementById('tabel-data');
|
||||
if (tableContainer) {
|
||||
tableContainer.addEventListener('click', e => {
|
||||
const link = e.target.closest('nav[role="navigation"] a');
|
||||
if (link && link.href && !link.closest('thead')) {
|
||||
e.preventDefault();
|
||||
const url = link.href;
|
||||
|
||||
tableContainer.classList.add('opacity-50', 'pointer-events-none', 'transition-opacity');
|
||||
|
||||
fetch(url, { headers: { 'X-Requested-With': 'XMLHttpRequest' } })
|
||||
.then(res => res.text())
|
||||
.then(html => {
|
||||
const doc = new DOMParser().parseFromString(html, 'text/html');
|
||||
const newTable = doc.getElementById('tabel-data');
|
||||
if (newTable) {
|
||||
tableContainer.innerHTML = newTable.innerHTML;
|
||||
window.history.pushState({ path: url }, '', url);
|
||||
}
|
||||
})
|
||||
.catch(err => console.error('Gagal memuat halaman tabel:', err))
|
||||
.finally(() => tableContainer.classList.remove('opacity-50', 'pointer-events-none'));
|
||||
}
|
||||
});
|
||||
window.addEventListener('popstate', () => window.location.reload());
|
||||
}
|
||||
|
||||
// SweetAlert Confirmations
|
||||
document.querySelectorAll('.delete-form').forEach(form => {
|
||||
form.addEventListener('submit', function(e) {
|
||||
e.preventDefault();
|
||||
Swal.fire({
|
||||
title: 'Hapus Data?',
|
||||
text: "Data yang dihapus tidak dapat dikembalikan!",
|
||||
icon: 'warning',
|
||||
showCancelButton: true,
|
||||
confirmButtonColor: '#ef4444',
|
||||
cancelButtonColor: '#64748b',
|
||||
confirmButtonText: 'Ya, Hapus!',
|
||||
cancelButtonText: 'Batal'
|
||||
}).then((result) => {
|
||||
if (result.isConfirmed) form.submit();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
document.querySelectorAll('.bulk-delete-form').forEach(form => {
|
||||
form.addEventListener('submit', function(e) {
|
||||
e.preventDefault();
|
||||
Swal.fire({
|
||||
title: 'Hapus Data Massal?',
|
||||
text: "Semua data yang dipilih akan dihapus permanen!",
|
||||
icon: 'warning',
|
||||
showCancelButton: true,
|
||||
confirmButtonColor: '#ef4444',
|
||||
cancelButtonColor: '#64748b',
|
||||
confirmButtonText: 'Ya, Hapus Semua!',
|
||||
cancelButtonText: 'Batal'
|
||||
}).then((result) => {
|
||||
if (result.isConfirmed) form.submit();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// AJAX Upload & Polling
|
||||
const uploadForm = document.getElementById('uploadForm');
|
||||
if (uploadForm) {
|
||||
uploadForm.addEventListener('submit', async function(e) {
|
||||
e.preventDefault();
|
||||
const formData = new FormData(this);
|
||||
const submitBtn = this.querySelector('button[type="submit"]');
|
||||
const originalBtnText = submitBtn.innerHTML;
|
||||
submitBtn.disabled = true;
|
||||
submitBtn.innerHTML = '<svg class="animate-spin h-5 w-5 mr-3" viewBox="0 0 24 24"><circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4" fill="none"></circle><path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path></svg> Mengirim...';
|
||||
|
||||
try {
|
||||
const response = await fetch(this.action, {
|
||||
method: 'POST',
|
||||
body: formData,
|
||||
headers: { 'Accept': 'application/json' }
|
||||
});
|
||||
const result = await response.json();
|
||||
|
||||
if (!result.success) {
|
||||
Swal.fire('Error', result.message, 'error');
|
||||
submitBtn.disabled = false;
|
||||
submitBtn.innerHTML = originalBtnText;
|
||||
return;
|
||||
}
|
||||
|
||||
Swal.fire({
|
||||
title: 'Memproses Data...',
|
||||
html: result.message || 'Mohon tunggu, sistem sedang memproses file Anda.',
|
||||
allowOutsideClick: false,
|
||||
didOpen: () => {
|
||||
Swal.showLoading();
|
||||
}
|
||||
});
|
||||
|
||||
pollStatus(result.job_id, submitBtn, originalBtnText);
|
||||
|
||||
} catch (err) {
|
||||
Swal.fire('Error', 'Gagal mengirim file.', 'error');
|
||||
submitBtn.disabled = false;
|
||||
submitBtn.innerHTML = originalBtnText;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function pollStatus(jobId, submitBtn, originalBtnText) {
|
||||
const interval = setInterval(async () => {
|
||||
try {
|
||||
const res = await fetch(`/upload/status/${jobId}`);
|
||||
const data = await res.json();
|
||||
|
||||
if (data.status === 'completed') {
|
||||
clearInterval(interval);
|
||||
Swal.fire('Selesai!', data.message, 'success').then(() => {
|
||||
window.location.reload();
|
||||
});
|
||||
} else if (data.status === 'error') {
|
||||
clearInterval(interval);
|
||||
Swal.fire('Gagal', data.message, 'error');
|
||||
if(submitBtn) {
|
||||
submitBtn.disabled = false;
|
||||
submitBtn.innerHTML = originalBtnText;
|
||||
}
|
||||
} else {
|
||||
if (data.message) {
|
||||
Swal.getHtmlContainer().textContent = data.message;
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
clearInterval(interval);
|
||||
Swal.fire('Error', 'Koneksi ke server terputus saat mengecek status.', 'error');
|
||||
if(submitBtn) {
|
||||
submitBtn.disabled = false;
|
||||
submitBtn.innerHTML = originalBtnText;
|
||||
}
|
||||
}
|
||||
}, 3000);
|
||||
}
|
||||
});
|
||||
</script>
|
||||
</x-app-layout>
|
||||
|
|
@ -29,20 +29,9 @@
|
|||
<p class="text-sm text-slate-500 mt-1">Unggah file Excel/CSV data Kemendik untuk agregasi dan mapping (tanpa proses NLP klasifikasi).</p>
|
||||
</div>
|
||||
|
||||
@if(session('success'))
|
||||
<div class="bg-emerald-50 border border-emerald-200 text-emerald-700 px-4 py-3 rounded-xl mb-6 flex items-center space-x-3">
|
||||
<svg class="w-5 h-5" fill="currentColor" viewBox="0 0 20 20"><path fill-rule="evenodd" d="M10 18a8 8 0 100-16 8 8 0 000 16zm3.707-9.293a1 1 0 00-1.414-1.414L9 10.586 7.707 9.293a1 1 0 00-1.414 1.414l2 2a1 1 0 001.414 0l4-4z" clip-rule="evenodd"></path></svg>
|
||||
<span>{{ session('success') }}</span>
|
||||
</div>
|
||||
@endif
|
||||
@if(session('error'))
|
||||
<div class="bg-rose-50 border border-rose-200 text-rose-700 px-4 py-3 rounded-xl mb-6 flex items-center space-x-3">
|
||||
<svg class="w-5 h-5" fill="currentColor" viewBox="0 0 20 20"><path fill-rule="evenodd" d="M10 18a8 8 0 100-16 8 8 0 000 16zM8.707 7.293a1 1 0 00-1.414 1.414L8.586 10l-1.293 1.293a1 1 0 101.414 1.414L10 11.414l1.293 1.293a1 1 0 001.414-1.414L11.414 10l1.293-1.293a1 1 0 00-1.414-1.414L10 8.586 8.707 7.293z" clip-rule="evenodd"></path></svg>
|
||||
<span>{{ session('error') }}</span>
|
||||
</div>
|
||||
@endif
|
||||
|
||||
|
||||
<form action="{{ route('upload.process') }}" method="POST" enctype="multipart/form-data" class="flex flex-col md:flex-row gap-6 items-end">
|
||||
<form id="uploadForm" action="{{ route('upload.process') }}" method="POST" enctype="multipart/form-data" class="flex flex-col md:flex-row gap-6 items-end">
|
||||
@csrf
|
||||
<input type="hidden" name="source_type" value="kemendik">
|
||||
|
||||
|
|
@ -121,7 +110,7 @@
|
|||
@endif
|
||||
|
||||
<!-- Data Table -->
|
||||
<div class="bg-white rounded-2xl shadow-xl shadow-purple-100/50 border border-purple-50 overflow-hidden">
|
||||
<div id="tabel-data" class="bg-white rounded-2xl shadow-xl shadow-purple-100/50 border border-purple-50 overflow-hidden">
|
||||
<div class="px-6 py-5 border-b border-purple-50 bg-purple-50/20 flex justify-between items-center">
|
||||
<h3 class="text-lg font-bold text-slate-800">Tabel Data Kemendiktisaintek</h3>
|
||||
</div>
|
||||
|
|
@ -324,4 +313,113 @@
|
|||
});
|
||||
</script>
|
||||
@endif
|
||||
<script>
|
||||
document.addEventListener('DOMContentLoaded', () => {
|
||||
const tableContainer = document.getElementById('tabel-data');
|
||||
if (tableContainer) {
|
||||
tableContainer.addEventListener('click', e => {
|
||||
const link = e.target.closest('nav[role="navigation"] a');
|
||||
if (link && link.href && !link.closest('thead')) {
|
||||
e.preventDefault();
|
||||
const url = link.href;
|
||||
|
||||
tableContainer.classList.add('opacity-50', 'pointer-events-none', 'transition-opacity');
|
||||
|
||||
fetch(url, { headers: { 'X-Requested-With': 'XMLHttpRequest' } })
|
||||
.then(res => res.text())
|
||||
.then(html => {
|
||||
const doc = new DOMParser().parseFromString(html, 'text/html');
|
||||
const newTable = doc.getElementById('tabel-data');
|
||||
if (newTable) {
|
||||
tableContainer.innerHTML = newTable.innerHTML;
|
||||
window.history.pushState({ path: url }, '', url);
|
||||
}
|
||||
})
|
||||
.catch(err => console.error('Gagal memuat halaman tabel:', err))
|
||||
.finally(() => tableContainer.classList.remove('opacity-50', 'pointer-events-none'));
|
||||
}
|
||||
});
|
||||
window.addEventListener('popstate', () => window.location.reload());
|
||||
}
|
||||
|
||||
// AJAX Upload & Polling
|
||||
const uploadForm = document.getElementById('uploadForm');
|
||||
if (uploadForm) {
|
||||
uploadForm.addEventListener('submit', async function(e) {
|
||||
e.preventDefault();
|
||||
const formData = new FormData(this);
|
||||
const submitBtn = this.querySelector('button[type="submit"]');
|
||||
const originalBtnText = submitBtn.innerHTML;
|
||||
submitBtn.disabled = true;
|
||||
submitBtn.innerHTML = '<svg class="animate-spin h-5 w-5 mr-3" viewBox="0 0 24 24"><circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4" fill="none"></circle><path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path></svg> Mengirim...';
|
||||
|
||||
try {
|
||||
const response = await fetch(this.action, {
|
||||
method: 'POST',
|
||||
body: formData,
|
||||
headers: { 'Accept': 'application/json' }
|
||||
});
|
||||
const result = await response.json();
|
||||
|
||||
if (!result.success) {
|
||||
Swal.fire('Error', result.message, 'error');
|
||||
submitBtn.disabled = false;
|
||||
submitBtn.innerHTML = originalBtnText;
|
||||
return;
|
||||
}
|
||||
|
||||
Swal.fire({
|
||||
title: 'Memproses Data...',
|
||||
html: result.message || 'Mohon tunggu, sistem sedang memproses file Anda.',
|
||||
allowOutsideClick: false,
|
||||
didOpen: () => {
|
||||
Swal.showLoading();
|
||||
}
|
||||
});
|
||||
|
||||
pollStatus(result.job_id, submitBtn, originalBtnText);
|
||||
|
||||
} catch (err) {
|
||||
Swal.fire('Error', 'Gagal mengirim file.', 'error');
|
||||
submitBtn.disabled = false;
|
||||
submitBtn.innerHTML = originalBtnText;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function pollStatus(jobId, submitBtn, originalBtnText) {
|
||||
const interval = setInterval(async () => {
|
||||
try {
|
||||
const res = await fetch(`/upload/status/${jobId}`);
|
||||
const data = await res.json();
|
||||
|
||||
if (data.status === 'completed') {
|
||||
clearInterval(interval);
|
||||
Swal.fire('Selesai!', data.message, 'success').then(() => {
|
||||
window.location.reload();
|
||||
});
|
||||
} else if (data.status === 'error') {
|
||||
clearInterval(interval);
|
||||
Swal.fire('Gagal', data.message, 'error');
|
||||
if(submitBtn) {
|
||||
submitBtn.disabled = false;
|
||||
submitBtn.innerHTML = originalBtnText;
|
||||
}
|
||||
} else {
|
||||
if (data.message) {
|
||||
Swal.getHtmlContainer().textContent = data.message;
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
clearInterval(interval);
|
||||
Swal.fire('Error', 'Koneksi ke server terputus saat mengecek status.', 'error');
|
||||
if(submitBtn) {
|
||||
submitBtn.disabled = false;
|
||||
submitBtn.innerHTML = originalBtnText;
|
||||
}
|
||||
}
|
||||
}, 3000);
|
||||
}
|
||||
});
|
||||
</script>
|
||||
</x-app-layout>
|
||||
|
|
|
|||
|
|
@ -32,5 +32,36 @@
|
|||
{{ $slot }}
|
||||
</main>
|
||||
</div>
|
||||
<!-- SweetAlert2 -->
|
||||
<script src="https://cdn.jsdelivr.net/npm/sweetalert2@11"></script>
|
||||
<script>
|
||||
document.addEventListener("DOMContentLoaded", () => {
|
||||
const Toast = Swal.mixin({
|
||||
toast: true,
|
||||
position: 'top-end',
|
||||
showConfirmButton: false,
|
||||
timer: 3000,
|
||||
timerProgressBar: true,
|
||||
didOpen: (toast) => {
|
||||
toast.addEventListener('mouseenter', Swal.stopTimer)
|
||||
toast.addEventListener('mouseleave', Swal.resumeTimer)
|
||||
}
|
||||
});
|
||||
|
||||
@if(session('success'))
|
||||
Toast.fire({
|
||||
icon: 'success',
|
||||
title: "{!! session('success') !!}"
|
||||
});
|
||||
@endif
|
||||
|
||||
@if(session('error'))
|
||||
Toast.fire({
|
||||
icon: 'error',
|
||||
title: "{!! session('error') !!}"
|
||||
});
|
||||
@endif
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@
|
|||
<tr>
|
||||
<td>{{ $index + 1 }}</td>
|
||||
<td>{{ $row->nim }}</td>
|
||||
<td>{{ $row->nama }}</td>
|
||||
<td>{{ $row->internalRaw->nama_lengkap ?? '-' }}</td>
|
||||
<td>{{ $row->job_text_raw }}</td>
|
||||
<td>
|
||||
{{ $row->predicted_profile ?? '-' }}
|
||||
|
|
|
|||
|
|
@ -11,21 +11,18 @@
|
|||
Route::middleware(['auth'])->group(function () {
|
||||
Route::get('/dashboard', [InternalDataController::class, 'index'])->name('dashboard');
|
||||
Route::get('/kemendik', [KemendikDataController::class, 'index'])->name('kemendik');
|
||||
|
||||
// Import Data
|
||||
Route::post('/upload', [ClassificationTaskController::class, 'upload'])->name('upload.process');
|
||||
Route::get('/upload/status/{job_id}', [ClassificationTaskController::class, 'checkStatus'])->name('upload.status');
|
||||
|
||||
// Internal Data Management
|
||||
Route::put('/internal/{id}', [InternalDataController::class, 'update'])->name('internal.update');
|
||||
Route::delete('/internal/{id}', [InternalDataController::class, 'destroy'])->name('internal.destroy');
|
||||
Route::post('/internal/bulk-delete', [InternalDataController::class, 'bulkDestroy'])->name('internal.bulk_destroy');
|
||||
Route::post('/internal/bulk-update', [InternalDataController::class, 'bulkUpdate'])->name('internal.bulk_update');
|
||||
Route::get('/internal/download-pdf', [InternalDataController::class, 'exportPdf'])->name('internal.download_pdf');
|
||||
|
||||
// ML Task
|
||||
Route::post('/retrain', [ClassificationTaskController::class, 'retrain'])->name('retrain.trigger');
|
||||
Route::get('/retrain/status', [ClassificationTaskController::class, 'retrainStatus'])->name('retrain.status');
|
||||
|
||||
Route::middleware('admin')->group(function () {
|
||||
Route::put('/internal/{id}', [InternalDataController::class, 'update'])->name('internal.update');
|
||||
Route::delete('/internal/{id}', [InternalDataController::class, 'destroy'])->name('internal.destroy');
|
||||
Route::post('/internal/bulk-delete', [InternalDataController::class, 'bulkDestroy'])->name('internal.bulk_destroy');
|
||||
Route::post('/internal/bulk-update', [InternalDataController::class, 'bulkUpdate'])->name('internal.bulk_update');
|
||||
Route::get('/internal/download-pdf', [InternalDataController::class, 'exportPdf'])->name('internal.download_pdf');
|
||||
Route::post('/retrain', [ClassificationTaskController::class, 'retrain'])->name('retrain.trigger');
|
||||
Route::get('/retrain/status', [ClassificationTaskController::class, 'retrainStatus'])->name('retrain.status');
|
||||
});
|
||||
|
||||
Route::prefix('profile')->group(function () {
|
||||
Route::get('/', [ProfileController::class, 'edit'])->name('profile.edit');
|
||||
|
|
|
|||
Loading…
Reference in New Issue