mhilya
/
sista_mif_ta
mirror of https://github.com/mhilya/sista_mif_ta.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

28_05_2026

This commit is contained in:
mhilya 2026-05-28 19:03:14 +07:00
parent eb52aa7abf
commit 420604ef2d
34 changed files with 3368 additions and 991 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

165
data/prepare_corpus.py Normal file
View File

@ -0,0 +1,165 @@
import pandas as pd
import numpy as np
import re
import sys
from pathlib import Path
from sklearn.model_selection import train_test_split
from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
factory = StemmerFactory()
stemmer = factory.create_stemmer()
BASE_DIR = Path(__file__).parent
DATA_DIR = BASE_DIR
OUTPUT_DIR = DATA_DIR / "processed"
OUTPUT_DIR.mkdir(exist_ok=True)
FILE_INTERNAL = DATA_DIR / "ts_internal_mif.xlsx"
SEPARATOR = ";"
ENCODING = "utf-8-sig"
TARGET_CLASSES = ["Programmer", "Data Analyst", "Wirausaha Informatika", "Non-IT"]
KLASIFIKASI_MAP = {
"Programmer": "Programmer",
"Data Analyst": "Data Analyst",
"Wirausaha IT": "Wirausaha Informatika",
"Wirausaha": "Wirausaha Informatika",
"Non-IT": "Non-IT",
"Infokom": None, "Pelajar": None, "Tidak Bekerja": None, "TIdak diketahui": None
}
KEYWORD_RULES = {
"Programmer": ["programmer", "developer", "engineer", "fullstack", "backend", "frontend", "mobile", "android", "ios", "software", "web dev", "coding", "it staff", "teknisi", "sistem informasi", "application", "network", "devops", "qa", "tester", "ui", "ux", "swe"],
"Data Analyst": ["data analyst", "analis data", "data science", "business analyst", "research", "statistik", "bi analyst", "reporting", "database", "sql", "etl", "data engineer", "big data", "analyst", "data mining", "machine learning", "data visual", "power bi", "tableau", "looker", "business intelligence", "bi developer", "data warehouse"],
"Wirausaha Informatika": ["founder", "owner", "ceo", "wiraswasta", "startup", "freelance", "freelancer", "wirausaha", "bisnis", "usaha mandiri", "konsultan", "co founder", "entrepreneur", "self employed", "owner toko", "usaha", "dagang online", "tokopedia", "shopee", "dropship", "reseller"]
}
COMPANY_STOPWORDS = {
"pt", "cv", "ud", "tbk", "persero", "corp", "inc", "ltd", "koperasi", "bumn", "bumd",
"dinas", "kantor", "pemkab", "pemprov", "politeknik", "universitas", "sekolah", "sma", "smk", "sd",
"bank", "bpr", "rs", "rumah sakit", "klinik", "apotek", "hotel", "restoran", "cafe", "toko", "konter",
"foundation", "yayasan", "perkumpulan", "organisasi", "agency", "studio", "consulting", "group", "holding"
}
def load_data_file(file_path: str) -> pd.DataFrame:
ext = Path(file_path).suffix.lower()
if ext == '.csv':
try: return pd.read_csv(file_path, sep=SEPARATOR, encoding=ENCODING, dtype=str, on_bad_lines='skip', engine='python')
except UnicodeDecodeError: return pd.read_csv(file_path, sep=SEPARATOR, encoding='latin1', dtype=str, on_bad_lines='skip', engine='python')
elif ext == '.xlsx':
return pd.read_excel(file_path, dtype=str)
else:
raise ValueError(f"Format tidak didukung: {ext}")
def find_col(df: pd.DataFrame, keywords: list) -> str | None:
for col in df.columns:
col_clean = str(col).strip().lower()
if any(k.strip().lower() in col_clean for k in keywords): return col
return None
def clean_text(text: str) -> str:
if pd.isna(text) or str(text).strip().lower() in ["nan", "none", "null", "-", "0", "", "tidak diisi", "tidak diketahui"]: return ""
text = str(text).strip().lower()
text = re.sub(r'^\d+\s*[-:/]\s*', '', text)
text = text.replace('-', ' ').replace('/', ' ').replace('_', ' ').replace(',', ' ')
text = re.sub(r'[^\w\s]', '', text)
text = re.sub(r'\d+', '', text)
tokens = [w for w in text.split() if w not in COMPANY_STOPWORDS and len(w) >= 3]
return " ".join([stemmer.stem(w) for w in tokens])
# [FIX v4] Logika lebih ketat: hanya flag jika token teks sepenuhnya subset dari nama
# (max 3 token) — bukan sekadar rasio overlap 50% seperti v3 yang rawan false positive.
def is_likely_name(text: str, full_name: str) -> bool:
if not text or not full_name: return False
text_clean = re.sub(r'[^\w\s]', '', text.lower())
name_clean = re.sub(r'[^\w\s]', '', str(full_name).lower())
text_parts = set(text_clean.split())
name_parts = set(name_clean.split())
if len(text_parts) == 0: return False
if len(text_parts) <= 3 and text_parts.issubset(name_parts):
return True
return False
def classify_rule_based(text: str) -> str:
if not text or len(text) < 3: return "Non-IT"
for profile in ["Programmer", "Data Analyst", "Wirausaha Informatika"]:
if any(kw in text for kw in KEYWORD_RULES[profile]): return profile
return "Non-IT"
def main():
if not FILE_INTERNAL.exists():
print(f"Berkas tidak ditemukan: {FILE_INTERNAL}"); sys.exit(1)
print("[1/4] Memuat & Membersihkan Data (Strict Cleaning)...")
df = load_data_file(str(FILE_INTERNAL))
df.columns = df.columns.str.strip()
col_nim = find_col(df, ["nim"])
col_nama = find_col(df, ["nama", "lengkap"])
col_jab_lama = find_col(df, ["jabatan"])
col_jab_baru = find_col(df, ["jabatan_terupdate"])
col_klasifikasi = find_col(df, ["klasifikasi"])
col_status = find_col(df, ["status", "kerja"])
if not all([col_nim, col_nama, col_jab_lama]):
print(" Kolom esensial tidak ditemukan"); print(df.columns.tolist()[:10]); sys.exit(1)
if col_jab_baru:
mask_empty = df[col_jab_baru].isna() | df[col_jab_baru].astype(str).str.strip().str.lower().isin(["", "-", "0", "nan", "none", "null", "tidak diisi", "tidak diketahui"])
df["jabatan_final"] = df[col_jab_baru].where(~mask_empty, df[col_jab_lama])
else:
df["jabatan_final"] = df[col_jab_lama]
col_jabatan = "jabatan_final"
if col_status:
blacklist = ["tidah diketahui", "tidak bekerja", "pelajar", "melanjutkan pendidikan", "nan", ""]
mask = ~df[col_status].str.lower().str.strip().isin(blacklist)
df = df[mask].copy()
# Stemming berat terjadi HANYA di sini — satu kali saat corpus preparation
df["job_text_raw"] = df[col_jabatan].apply(clean_text)
if col_klasifikasi:
fallback_map = {"Programmer": "programmer developer", "Data Analyst": "data analyst", "Wirausaha IT": "wirausaha founder", "Wirausaha": "wirausaha founder", "Non IT": "staff admin", "Infokom": "it staff teknisi", "TIdah diketahui": "", "Pelajar": "", "Tidak Bekerja": ""}
empty_mask = df["job_text_raw"] == ""
if empty_mask.any(): df.loc[empty_mask, "job_text_raw"] = df.loc[empty_mask, col_klasifikasi].map(fallback_map).fillna("")
if col_nama:
name_leak_mask = df.apply(lambda row: is_likely_name(row["job_text_raw"], row[col_nama]), axis=1)
leaked_count = name_leak_mask.sum()
if leaked_count > 0: print(f" Mengabaikan {leaked_count} baris yang terindikasi mengandung nama pribadi.")
df = df[~name_leak_mask].copy()
if col_klasifikasi:
df["label"] = df[col_klasifikasi].str.strip().map(KLASIFIKASI_MAP)
missing = df["label"].isna()
if missing.any(): df.loc[missing, "label"] = df.loc[missing, "job_text_raw"].apply(classify_rule_based)
else:
df["label"] = df["job_text_raw"].apply(classify_rule_based)
df = df[df["job_text_raw"].str.len() >= 3].copy()
result = (
df[[col_nim, col_jabatan, "job_text_raw", "label"]]
.rename(columns={col_nim: "nim"})
.dropna(subset=["nim", "label"])
.drop_duplicates(subset=["nim"], keep="first")
)
print(f"Memuat {len(result)} data valid (berdasarkan teks pekerjaan)\n")
min_count = result["label"].value_counts().min()
if min_count < 2:
train_df, test_df = train_test_split(result, test_size=0.30, random_state=42)
else:
train_df, test_df = train_test_split(result, test_size=0.30, stratify=result["label"], random_state=42)
cols_out = ["nim", "job_text_raw", "label"]
train_df[cols_out].to_csv(OUTPUT_DIR / "training_corpus.csv", index=False, sep=";", encoding=ENCODING)
test_df[cols_out].to_csv(OUTPUT_DIR / "test_set.csv", index=False, sep=";", encoding=ENCODING)
print("DATA PELATIHAN (TRAINING SET):")
print(train_df["label"].value_counts().to_string())
# [KEEP v3] Distribusi per kelas di test set — berguna untuk deteksi class imbalance
print("\nDATA PENGUJIAN (TEST SET):")
for cls in TARGET_CLASSES:
c = test_df["label"].value_counts().get(cls, 0)
print(f" {cls:25} : {c}{'<5' if c < 5 else ''}")
print(f"\nData berhasil disimpan ke {OUTPUT_DIR}")
if __name__ == "__main__":
main()

109
data/processed/test_set.csv Normal file
View File

@ -0,0 +1,109 @@
nim;job_text_raw;label
E31190416;freelance kerja lepas;Wirausaha Informatika
E31160787;software programmer;Programmer
E31192134;freelance kerja lepas;Wirausaha Informatika
E31201687;staff graphic designer;Non-IT
E31200364;freelance kerja lepas;Data Analyst
E31180838;wirausaha founder;Wirausaha Informatika
E31181942;supervisor;Programmer
E31190408;freelance kerja lepas;Wirausaha Informatika
E31200061;marketing officer;Non-IT
E31180644;software programmer;Programmer
E31191447;freelance kerja lepas;Wirausaha Informatika
E31190785;freelance kerja lepas;Wirausaha Informatika
E31180926;mathematics tutor;Non-IT
E31201932;freelance kerja lepas;Wirausaha Informatika
E31181204;founder;Wirausaha Informatika
E31181295;creative team lead;Programmer
E31192292;staff;Non-IT
E31180949;research and development engineer;Programmer
E31192123;customer engineer;Programmer
E31191441;freelance kerja lepas;Wirausaha Informatika
E31180295;sales;Wirausaha Informatika
E31160901;logistik;Non-IT
E31182030;founder;Programmer
E31191547;staf sdm parmas;Non-IT
E31201950;marketing officer;Non-IT
E31192213;freelance kerja lepas;Programmer
E31171849;programmer developer;Programmer
E31201829;staff;Non-IT
E31192403;freelance kerja lepas;Wirausaha Informatika
E31192060;freelance kerja lepas;Wirausaha Informatika
E31171992;it staff teknisi;Programmer
E31192166;founder;Wirausaha Informatika
E31200117;freelance kerja lepas;Wirausaha Informatika
E31181709;web developer supervisor;Programmer
E31200973;programmer;Programmer
E31201951;staff;Non-IT
E31201372;founder;Wirausaha Informatika
E31180509;freelance kerja lepas;Wirausaha Informatika
E31180724;lain lain;Non-IT
E31170050;it staff teknisi;Programmer
E31180707;freelance kerja lepas;Programmer
E31181971;digital marketing strategist;Non-IT
E31192175;tugas barang bukti;Non-IT
E31190980;staff;Programmer
E31192116;freelance kerja lepas;Wirausaha Informatika
E31192029;freelance kerja lepas;Programmer
E31190341;staff;Programmer
E31202314;staf laksana;Non-IT
E31202562;staff;Non-IT
E31191800;checker angkut barang;Non-IT
E31172055;maintenance and technical support retail;Non-IT
E31170468;teknisi support;Programmer
E31202038;freelance kerja lepas;Wirausaha Informatika
E31201422;freelance kerja lepas;Wirausaha Informatika
E31181032;freelance kerja lepas;Wirausaha Informatika
E31202278;founder;Wirausaha Informatika
E31171733;staff divisi suhsi;Non-IT
E31200561;freelance kerja lepas;Data Analyst
E31171396;programmer developer;Programmer
E31192024;teknisi laboratorium;Programmer
E31181929;administrasi bmn;Non-IT
E31190987;staff administrasi;Non-IT
E31201009;staff;Non-IT
E31191792;admin and finance staff;Non-IT
E31170253;admin;Non-IT
E31171904;freelance illustrator;Wirausaha Informatika
E31180575;designer;Programmer
E31160852;interviewer;Non-IT
E31191848;fullstack developer;Programmer
E31200909;founder;Wirausaha Informatika
E31191660;founder;Wirausaha Informatika
E31171317;web developer;Programmer
E31202451;staff;Non-IT
E31170570;software engineer;Programmer
E31192159;staff data;Non-IT
E31171106;phl;Non-IT
E31202523;freelance kerja lepas;Wirausaha Informatika
E31191793;founder;Wirausaha Informatika
E31192043;freelance kerja lepas;Wirausaha Informatika
E31201706;freelance kerja lepas;Wirausaha Informatika
E31180303;staff;Non-IT
E31192416;freelance kerja lepas;Wirausaha Informatika
E31190286;staff;Non-IT
E31190800;staff;Non-IT
E31181976;teller;Non-IT
E31201954;freelance kerja lepas;Wirausaha Informatika
E31161636;web developer;Programmer
E3117228;programmer developer;Programmer
E31192264;founder;Wirausaha Informatika
E31201573;staff;Non-IT
E31160170;pranata teknologi informasi komputer;Programmer
E31180686;founder;Wirausaha Informatika
E31201020;staff;Non-IT
E31190302;teknisi laboratorium;Programmer
E31201300;freelance kerja lepas;Wirausaha Informatika
E31192047;founder;Wirausaha Informatika
E31140976;staff admin;Non-IT
E31191621;staff;Non-IT
E31180504;founder;Wirausaha Informatika
E31180735;programer;Programmer
E31172173;administrator;Non-IT
E31180207;founder;Wirausaha Informatika
E31160707;staf programmer;Programmer
E31200700;staff;Wirausaha Informatika
E31191272;staff;Wirausaha Informatika
E31190068;staff administrasi produksi;Non-IT
E31191006;freelance kerja lepas;Wirausaha Informatika
E31180678;asisten tenaga latih;Non-IT
1 nim job_text_raw label
2 E31190416 freelance kerja lepas Wirausaha Informatika
3 E31160787 software programmer Programmer
4 E31192134 freelance kerja lepas Wirausaha Informatika
5 E31201687 staff graphic designer Non-IT
6 E31200364 freelance kerja lepas Data Analyst
7 E31180838 wirausaha founder Wirausaha Informatika
8 E31181942 supervisor Programmer
9 E31190408 freelance kerja lepas Wirausaha Informatika
10 E31200061 marketing officer Non-IT
11 E31180644 software programmer Programmer
12 E31191447 freelance kerja lepas Wirausaha Informatika
13 E31190785 freelance kerja lepas Wirausaha Informatika
14 E31180926 mathematics tutor Non-IT
15 E31201932 freelance kerja lepas Wirausaha Informatika
16 E31181204 founder Wirausaha Informatika
17 E31181295 creative team lead Programmer
18 E31192292 staff Non-IT
19 E31180949 research and development engineer Programmer
20 E31192123 customer engineer Programmer
21 E31191441 freelance kerja lepas Wirausaha Informatika
22 E31180295 sales Wirausaha Informatika
23 E31160901 logistik Non-IT
24 E31182030 founder Programmer
25 E31191547 staf sdm parmas Non-IT
26 E31201950 marketing officer Non-IT
27 E31192213 freelance kerja lepas Programmer
28 E31171849 programmer developer Programmer
29 E31201829 staff Non-IT
30 E31192403 freelance kerja lepas Wirausaha Informatika
31 E31192060 freelance kerja lepas Wirausaha Informatika
32 E31171992 it staff teknisi Programmer
33 E31192166 founder Wirausaha Informatika
34 E31200117 freelance kerja lepas Wirausaha Informatika
35 E31181709 web developer supervisor Programmer
36 E31200973 programmer Programmer
37 E31201951 staff Non-IT
38 E31201372 founder Wirausaha Informatika
39 E31180509 freelance kerja lepas Wirausaha Informatika
40 E31180724 lain lain Non-IT
41 E31170050 it staff teknisi Programmer
42 E31180707 freelance kerja lepas Programmer
43 E31181971 digital marketing strategist Non-IT
44 E31192175 tugas barang bukti Non-IT
45 E31190980 staff Programmer
46 E31192116 freelance kerja lepas Wirausaha Informatika
47 E31192029 freelance kerja lepas Programmer
48 E31190341 staff Programmer
49 E31202314 staf laksana Non-IT
50 E31202562 staff Non-IT
51 E31191800 checker angkut barang Non-IT
52 E31172055 maintenance and technical support retail Non-IT
53 E31170468 teknisi support Programmer
54 E31202038 freelance kerja lepas Wirausaha Informatika
55 E31201422 freelance kerja lepas Wirausaha Informatika
56 E31181032 freelance kerja lepas Wirausaha Informatika
57 E31202278 founder Wirausaha Informatika
58 E31171733 staff divisi suhsi Non-IT
59 E31200561 freelance kerja lepas Data Analyst
60 E31171396 programmer developer Programmer
61 E31192024 teknisi laboratorium Programmer
62 E31181929 administrasi bmn Non-IT
63 E31190987 staff administrasi Non-IT
64 E31201009 staff Non-IT
65 E31191792 admin and finance staff Non-IT
66 E31170253 admin Non-IT
67 E31171904 freelance illustrator Wirausaha Informatika
68 E31180575 designer Programmer
69 E31160852 interviewer Non-IT
70 E31191848 fullstack developer Programmer
71 E31200909 founder Wirausaha Informatika
72 E31191660 founder Wirausaha Informatika
73 E31171317 web developer Programmer
74 E31202451 staff Non-IT
75 E31170570 software engineer Programmer
76 E31192159 staff data Non-IT
77 E31171106 phl Non-IT
78 E31202523 freelance kerja lepas Wirausaha Informatika
79 E31191793 founder Wirausaha Informatika
80 E31192043 freelance kerja lepas Wirausaha Informatika
81 E31201706 freelance kerja lepas Wirausaha Informatika
82 E31180303 staff Non-IT
83 E31192416 freelance kerja lepas Wirausaha Informatika
84 E31190286 staff Non-IT
85 E31190800 staff Non-IT
86 E31181976 teller Non-IT
87 E31201954 freelance kerja lepas Wirausaha Informatika
88 E31161636 web developer Programmer
89 E3117228 programmer developer Programmer
90 E31192264 founder Wirausaha Informatika
91 E31201573 staff Non-IT
92 E31160170 pranata teknologi informasi komputer Programmer
93 E31180686 founder Wirausaha Informatika
94 E31201020 staff Non-IT
95 E31190302 teknisi laboratorium Programmer
96 E31201300 freelance kerja lepas Wirausaha Informatika
97 E31192047 founder Wirausaha Informatika
98 E31140976 staff admin Non-IT
99 E31191621 staff Non-IT
100 E31180504 founder Wirausaha Informatika
101 E31180735 programer Programmer
102 E31172173 administrator Non-IT
103 E31180207 founder Wirausaha Informatika
104 E31160707 staf programmer Programmer
105 E31200700 staff Wirausaha Informatika
106 E31191272 staff Wirausaha Informatika
107 E31190068 staff administrasi produksi Non-IT
108 E31191006 freelance kerja lepas Wirausaha Informatika
109 E31180678 asisten tenaga latih Non-IT

109
data/processed/test_set_3.csv
View File

@ -1,109 +0,0 @@
nim;job_text_raw;label
E31200637;freelance kerja lepas;Wirausaha Informatika
E31200880;freelance kerja lepas;Wirausaha Informatika
E31160787;software programmer;Programmer
E31171733;staff divisi suhsi;Non-IT
E31170445;it staff teknisi;Programmer
E31200364;freelance kerja lepas;Data Analyst
E31191677;freelance kerja lepas;Wirausaha Informatika
E31201951;staff;Non-IT
E31200912;staff;Non-IT
E31191460;freelance kerja lepas;Wirausaha Informatika
E31170468;teknisi support;Programmer
E31181709;staff;Programmer
E31201634;staff;Non-IT
E31201095;freelance kerja lepas;Wirausaha Informatika
E31180926;freelance kerja lepas;Wirausaha Informatika
E31192259;owner;Wirausaha Informatika
E31192120;freelance kerja lepas;Wirausaha Informatika
E31191394;founder;Wirausaha Informatika
E31170772;engineer product;Programmer
E31140976;staff admin;Non-IT
E31180222;freelance kerja lepas;Wirausaha Informatika
E31190851;founder;Wirausaha Informatika
E31190430;freelance kerja lepas;Wirausaha Informatika
E31190801;staff;Non-IT
E31200917;freelance kerja lepas;Wirausaha Informatika
E31161636;web developer;Programmer
E31171992;it staff teknisi;Programmer
E31200680;founder;Wirausaha Informatika
E31191309;freelance kerja lepas;Programmer
E31180276;staff;Wirausaha Informatika
E31171904;freelance illustrator;Wirausaha Informatika
E31200909;founder;Wirausaha Informatika
E31181885;staff admin;Non-IT
E31181091;management informatika;Programmer
E31190197;founder;Wirausaha Informatika
E31160901;logistik;Non-IT
E31200549;staff;Non-IT
E31172055;maintenance and technical support retail;Non-IT
E31201372;founder;Wirausaha Informatika
E31191982;staff;Non-IT
E31190993;staff;Non-IT
E31192391;staff;Programmer
E31191195;founder;Wirausaha Informatika
E31181295;staf;Programmer
E31200561;freelance kerja lepas;Data Analyst
E31192079;staff;Programmer
E31180299;it staff teknisi;Programmer
E31192264;founder;Wirausaha Informatika
E31180735;programer;Programmer
E31202451;staff;Non-IT
E31170253;admin;Non-IT
E31190397;freelance kerja lepas;Wirausaha Informatika
E31201177;freelance kerja lepas;Wirausaha Informatika
E31192347;freelance kerja lepas;Wirausaha Informatika
E31181452;staff admin;Non-IT
E31190731;founder;Programmer
E31191821;founder;Wirausaha Informatika
E31160852;interviewer;Non-IT
E31190596;staff;Non-IT
E31180696;founder;Wirausaha Informatika
E31201459;freelance kerja lepas;Wirausaha Informatika
E31181942;freelance kerja lepas;Programmer
E31171849;programmer developer;Programmer
E31171103;it staff teknisi;Programmer
E31190658;staff;Non-IT
E31200910;staff;Non-IT
E31200592;guru;Non-IT
E31200649;human resource;Non-IT
E31170050;it staff teknisi;Programmer
E31180365;founder;Wirausaha Informatika
E31171799;web programmer;Programmer
E31200254;sekretaris kurikulum;Non-IT
E31181597;it staff teknisi;Programmer
E31170570;software engineer;Programmer
E31201690;staff;Non-IT
E31202490;freelance kerja lepas;Wirausaha Informatika
E31171333;it staff teknisi;Programmer
E31190068;staff administrasi produksi;Non-IT
E31180112;it staff teknisi;Programmer
E31181032;freelance kerja lepas;Wirausaha Informatika
E31201228;founder;Wirausaha Informatika
E31180837;freelance kerja lepas;Programmer
E31200061;marketing officer;Non-IT
E31160170;pranata teknologi informasi komputer;Programmer
E31192072;staff;Non-IT
E31171106;phl;Non-IT
E31181567;founder;Wirausaha Informatika
E31171281;direktur utama;Wirausaha Informatika
E31180260;staff;Non-IT
E31180509;freelance kerja lepas;Wirausaha Informatika
E31190270;staff;Non-IT
E31191250;founder;Wirausaha Informatika
E31180464;staff;Wirausaha Informatika
E31191708;desainer grafis;Non-IT
E31190449;staff;Non-IT
E31191272;staff;Wirausaha Informatika
E31190679;staff;Non-IT
E31200001;freelance kerja lepas;Wirausaha Informatika
E31171396;programmer developer;Programmer
E31160707;staf programmer;Programmer
E31180838;wirausaha founder;Wirausaha Informatika
E31180989;information technology services;Programmer
E31181280;tenaga;Programmer
E31172173;administrator;Non-IT
E31202463;staff;Non-IT
E31190672;freelance kerja lepas;Programmer
E31192116;freelance kerja lepas;Wirausaha Informatika
E31192221;freelance kerja lepas;Wirausaha Informatika
1 nim job_text_raw label
2 E31200637 freelance kerja lepas Wirausaha Informatika
3 E31200880 freelance kerja lepas Wirausaha Informatika
4 E31160787 software programmer Programmer
5 E31171733 staff divisi suhsi Non-IT
6 E31170445 it staff teknisi Programmer
7 E31200364 freelance kerja lepas Data Analyst
8 E31191677 freelance kerja lepas Wirausaha Informatika
9 E31201951 staff Non-IT
10 E31200912 staff Non-IT
11 E31191460 freelance kerja lepas Wirausaha Informatika
12 E31170468 teknisi support Programmer
13 E31181709 staff Programmer
14 E31201634 staff Non-IT
15 E31201095 freelance kerja lepas Wirausaha Informatika
16 E31180926 freelance kerja lepas Wirausaha Informatika
17 E31192259 owner Wirausaha Informatika
18 E31192120 freelance kerja lepas Wirausaha Informatika
19 E31191394 founder Wirausaha Informatika
20 E31170772 engineer product Programmer
21 E31140976 staff admin Non-IT
22 E31180222 freelance kerja lepas Wirausaha Informatika
23 E31190851 founder Wirausaha Informatika
24 E31190430 freelance kerja lepas Wirausaha Informatika
25 E31190801 staff Non-IT
26 E31200917 freelance kerja lepas Wirausaha Informatika
27 E31161636 web developer Programmer
28 E31171992 it staff teknisi Programmer
29 E31200680 founder Wirausaha Informatika
30 E31191309 freelance kerja lepas Programmer
31 E31180276 staff Wirausaha Informatika
32 E31171904 freelance illustrator Wirausaha Informatika
33 E31200909 founder Wirausaha Informatika
34 E31181885 staff admin Non-IT
35 E31181091 management informatika Programmer
36 E31190197 founder Wirausaha Informatika
37 E31160901 logistik Non-IT
38 E31200549 staff Non-IT
39 E31172055 maintenance and technical support retail Non-IT
40 E31201372 founder Wirausaha Informatika
41 E31191982 staff Non-IT
42 E31190993 staff Non-IT
43 E31192391 staff Programmer
44 E31191195 founder Wirausaha Informatika
45 E31181295 staf Programmer
46 E31200561 freelance kerja lepas Data Analyst
47 E31192079 staff Programmer
48 E31180299 it staff teknisi Programmer
49 E31192264 founder Wirausaha Informatika
50 E31180735 programer Programmer
51 E31202451 staff Non-IT
52 E31170253 admin Non-IT
53 E31190397 freelance kerja lepas Wirausaha Informatika
54 E31201177 freelance kerja lepas Wirausaha Informatika
55 E31192347 freelance kerja lepas Wirausaha Informatika
56 E31181452 staff admin Non-IT
57 E31190731 founder Programmer
58 E31191821 founder Wirausaha Informatika
59 E31160852 interviewer Non-IT
60 E31190596 staff Non-IT
61 E31180696 founder Wirausaha Informatika
62 E31201459 freelance kerja lepas Wirausaha Informatika
63 E31181942 freelance kerja lepas Programmer
64 E31171849 programmer developer Programmer
65 E31171103 it staff teknisi Programmer
66 E31190658 staff Non-IT
67 E31200910 staff Non-IT
68 E31200592 guru Non-IT
69 E31200649 human resource Non-IT
70 E31170050 it staff teknisi Programmer
71 E31180365 founder Wirausaha Informatika
72 E31171799 web programmer Programmer
73 E31200254 sekretaris kurikulum Non-IT
74 E31181597 it staff teknisi Programmer
75 E31170570 software engineer Programmer
76 E31201690 staff Non-IT
77 E31202490 freelance kerja lepas Wirausaha Informatika
78 E31171333 it staff teknisi Programmer
79 E31190068 staff administrasi produksi Non-IT
80 E31180112 it staff teknisi Programmer
81 E31181032 freelance kerja lepas Wirausaha Informatika
82 E31201228 founder Wirausaha Informatika
83 E31180837 freelance kerja lepas Programmer
84 E31200061 marketing officer Non-IT
85 E31160170 pranata teknologi informasi komputer Programmer
86 E31192072 staff Non-IT
87 E31171106 phl Non-IT
88 E31181567 founder Wirausaha Informatika
89 E31171281 direktur utama Wirausaha Informatika
90 E31180260 staff Non-IT
91 E31180509 freelance kerja lepas Wirausaha Informatika
92 E31190270 staff Non-IT
93 E31191250 founder Wirausaha Informatika
94 E31180464 staff Wirausaha Informatika
95 E31191708 desainer grafis Non-IT
96 E31190449 staff Non-IT
97 E31191272 staff Wirausaha Informatika
98 E31190679 staff Non-IT
99 E31200001 freelance kerja lepas Wirausaha Informatika
100 E31171396 programmer developer Programmer
101 E31160707 staf programmer Programmer
102 E31180838 wirausaha founder Wirausaha Informatika
103 E31180989 information technology services Programmer
104 E31181280 tenaga Programmer
105 E31172173 administrator Non-IT
106 E31202463 staff Non-IT
107 E31190672 freelance kerja lepas Programmer
108 E31192116 freelance kerja lepas Wirausaha Informatika
109 E31192221 freelance kerja lepas Wirausaha Informatika

475
data/processed/training_corpus_3.csv → data/processed/training_corpus.csv
View File

@ -1,250 +1,251 @@
nim;job_text_raw;label nim;job_text_raw;label
E31200864;freelance kerja lepas;Wirausaha Informatika
E31202562;staff;Non-IT
E31202523;freelance kerja lepas;Wirausaha Informatika
E31201845;founder;Wirausaha Informatika
E31201199;founder;Wirausaha Informatika
E31171680;quality control specialist;Non-IT
E31192166;founder;Wirausaha Informatika
E31201422;freelance kerja lepas;Wirausaha Informatika
E31190648;founder;Programmer
E31190939;founder;Wirausaha Informatika
E31201950;marketing officer;Non-IT
E31190800;staff;Non-IT
E31192123;customer engineer;Programmer
E31201237;staff;Non-IT
E31171791;pranata komputer;Non-IT
E31192213;freelance kerja lepas;Programmer
E31200102;freelance kerja lepas;Wirausaha Informatika
E31170389;field coll;Non-IT
E31190785;freelance kerja lepas;Wirausaha Informatika
E31192403;freelance kerja lepas;Wirausaha Informatika
E31201365;founder;Programmer
E31162061;web developer;Programmer
E31192304;founder;Wirausaha Informatika
E31180306;staff;Data Analyst
E31190980;staff;Programmer
E31181553;staff admin;Non-IT
E31181158;freelance kerja lepas;Programmer
E31171868;information technology staff;Non-IT
E31200700;staff;Wirausaha Informatika
E31192175;petugas barang bukti;Non-IT
E31201555;freelance kerja lepas;Wirausaha Informatika
E31200251;ppic;Non-IT
E31171935;machine operator;Non-IT
E31170763;document control specialist;Non-IT
E31160622;programmer;Programmer
E31201573;staff;Non-IT
E31201020;staff;Non-IT
E31192043;freelance kerja lepas;Wirausaha Informatika
E31192202;admin produksi;Non-IT
E31171444;wirausaha founder;Wirausaha Informatika
E31180373;staff;Non-IT
E31171427;pranata komputer;Non-IT
E31182030;freelance kerja lepas;Programmer
E31171243;it staff teknisi;Programmer
E31201687;staff graphic designer;Non-IT
E31190631;staff;Non-IT
E31180332;founder;Wirausaha Informatika
E31202038;freelance kerja lepas;Wirausaha Informatika
E31170217;human resources development officer;Non-IT
E31191547;staf sdm parmas;Non-IT
E31171076;support specialist;Programmer
E31172262;it staff teknisi;Programmer
E31170087;design development;Non-IT
E31191300;staff;Non-IT
E31200238;staff kontrak;Non-IT
E31201300;freelance kerja lepas;Wirausaha Informatika
E31171431;guru tik;Non-IT
E31181991;wirausaha founder;Wirausaha Informatika
E31171777;programmer developer;Programmer
E31201932;freelance kerja lepas;Wirausaha Informatika
E31201954;freelance kerja lepas;Wirausaha Informatika
E31191441;freelance kerja lepas;Wirausaha Informatika
E31192311;founder;Wirausaha Informatika
E31180878;freelance kerja lepas;Programmer
E31200293;staff;Non-IT
E31180456;it staff teknisi;Programmer
E31192292;staff;Non-IT
E31171367;office administrator;Non-IT
E31180748;freelance kerja lepas;Wirausaha Informatika
E31201939;freelance kerja lepas;Programmer
E31171151;sales executive;Non-IT
E31182101;founder;Wirausaha Informatika
E31170734;administrative assistant;Non-IT
E31191621;staff;Non-IT
E31192416;freelance kerja lepas;Wirausaha Informatika
E31192353;founder;Wirausaha Informatika
E31171236;it staff teknisi;Programmer
E31201647;founder;Wirausaha Informatika
E31190527;founder;Wirausaha Informatika E31190527;founder;Wirausaha Informatika
E31190801;staff;Non-IT
E31190672;freelance kerja lepas;Programmer
E31192382;staff;Non-IT E31192382;staff;Non-IT
E31200245;staff;Non-IT E31201634;staff;Non-IT
E31191124;founder;Wirausaha Informatika E31172208;fullstack developer web app;Programmer
E31160690;graphic designer;Non-IT
E31202345;founder;Wirausaha Informatika
E31181929;administrasi bmn;Non-IT
E31190865;founder;Programmer
E31190698;staff;Non-IT
E31181100;freelance kerja lepas;Wirausaha Informatika
E31201431;freelance kerja lepas;Data Analyst
E31171296;cutting machine operator;Non-IT
E31191159;founder;Wirausaha Informatika
E31191793;founder;Wirausaha Informatika
E31200456;data analyst;Data Analyst
E31180678;freelance kerja lepas;Wirausaha Informatika
E31181204;founder;Wirausaha Informatika
E31190408;freelance kerja lepas;Wirausaha Informatika
E31191228;it staff teknisi;Programmer
E31202278;founder;Wirausaha Informatika
E31201773;freelance kerja lepas;Programmer
E31191006;freelance kerja lepas;Wirausaha Informatika
E31192064;freelance kerja lepas;Wirausaha Informatika
E31180207;founder;Wirausaha Informatika
E31180513;it staff teknisi;Programmer
E31202314;staf pelaksana;Non-IT
E31201546;freelance kerja lepas;Wirausaha Informatika
E31200888;freelance kerja lepas;Wirausaha Informatika
E31191849;founder;Programmer
E31200284;staff;Non-IT
E31192029;freelance kerja lepas;Programmer
E31180638;founder;Wirausaha Informatika
E31180539;staff;Non-IT
E31181496;programmer developer;Programmer
E31192293;freelance kerja lepas;Wirausaha Informatika
E31180504;founder;Wirausaha Informatika
E31170392;it staff teknisi;Programmer
E31201537;freelance kerja lepas;Wirausaha Informatika
E31200837;founder;Wirausaha Informatika
E31192060;freelance kerja lepas;Wirausaha Informatika
E31180256;freelance kerja lepas;Wirausaha Informatika
E31171971;it staff teknisi;Programmer
E31181474;founder;Wirausaha Informatika
E31181955;freelance kerja lepas;Programmer
E31191879;founder;Wirausaha Informatika
E31170933;tester;Programmer
E31182109;freelance kerja lepas;Wirausaha Informatika E31182109;freelance kerja lepas;Wirausaha Informatika
E31160524;kepala urusan perencanaan;Non-IT
E31191234;staff;Non-IT
E31201056;freelance kerja lepas;Wirausaha Informatika
E31170906;wirausaha founder;Wirausaha Informatika
E31171618;customer service staff;Non-IT
E31181049;freelance kerja lepas;Programmer
E31160306;admin stnk bpkb;Non-IT E31160306;admin stnk bpkb;Non-IT
E31171470;staf;Non-IT E31160524;kepala urus rencana;Non-IT
E31190445;staff;Non-IT E31191401;staff;Programmer
E31171869;programmer developer;Programmer E31190679;staff;Non-IT
E31200822;staff;Wirausaha Informatika E31181567;founder;Wirausaha Informatika
E31191120;freelance kerja lepas;Programmer E31191677;freelance kerja lepas;Wirausaha Informatika
E31161169;customer service;Non-IT E31161776;web developer;Programmer
E31190279;founder;Programmer E31200254;sekretaris kurikulum;Non-IT
E31191894;fullstack developer;Programmer E31201237;staff;Non-IT
E31180703;it staff teknisi;Programmer E31170087;design development;Non-IT
E31201829;staff;Non-IT E31192064;freelance kerja lepas;Wirausaha Informatika
E31182077;data analyst;Data Analyst E31160622;programmer;Programmer
E31200356;founder;Programmer E31191394;founder;Wirausaha Informatika
E31180644;freelance kerja lepas;Programmer E31191234;staff;Non-IT
E31200973;programmer;Programmer E31201365;founder;Programmer
E31171313;it staff teknisi;Programmer
E31192472;freelance kerja lepas;Wirausaha Informatika
E31202319;freelance kerja lepas;Wirausaha Informatika
E31191800;checker angkutan barang;Non-IT
E31170805;senior account manager;Non-IT
E31192024;teknisi laboratorium;Programmer
E31192285;founder;Programmer
E31191660;founder;Wirausaha Informatika
E31191433;founder;Wirausaha Informatika
E31160880;teller;Non-IT E31160880;teller;Non-IT
E31201706;freelance kerja lepas;Wirausaha Informatika E31202345;founder;Wirausaha Informatika
E31190341;staff;Programmer E31180299;teller;Non-IT
E31202279;freelance kerja lepas;Programmer E31170763;document control specialist;Non-IT
E31190883;staff;Programmer E31180464;staff;Wirausaha Informatika
E31200997;freelance kerja lepas;Wirausaha Informatika E31180112;it staff teknisi;Programmer
E31182133;freelance kerja lepas;Programmer E31200888;freelance kerja lepas;Wirausaha Informatika
E31180707;freelance kerja lepas;Programmer E31180260;staff;Non-IT
E31180295;freelance kerja lepas;Wirausaha Informatika E31190070;staff administrator;Non-IT
E31200117;freelance kerja lepas;Wirausaha Informatika E31191124;founder;Wirausaha Informatika
E31190134;freelance kerja lepas;Wirausaha Informatika E31201690;staff;Non-IT
E31180724;lain lain;Non-IT E31200910;staff;Non-IT
E31190286;staff;Non-IT E31190957;freelance kerja lepas;Wirausaha Informatika
E31180686;founder;Wirausaha Informatika E31170715;administrasi;Non-IT
E31171096;customer service officer;Non-IT E31200284;staff;Non-IT
E31200880;freelance kerja lepas;Wirausaha Informatika
E31181885;staff admin;Non-IT
E31200549;staff;Non-IT
E31180703;it staff teknisi;Programmer
E31171367;office administrator;Non-IT
E31160910;administration assistant;Wirausaha Informatika
E31201459;freelance kerja lepas;Wirausaha Informatika
E31201095;freelance kerja lepas;Wirausaha Informatika
E31160241;web developer;Programmer E31160241;web developer;Programmer
E31191460;freelance kerja lepas;Wirausaha Informatika
E31200245;staff;Non-IT
E31190851;founder;Wirausaha Informatika
E31200238;staff kontrak;Non-IT
E31190055;staff usaha labuh;Wirausaha Informatika
E31200293;staff;Non-IT
E31200356;founder;Programmer
E31191894;fullstack developer;Programmer
E31170392;it staff teknisi;Programmer
E31171791;pranata komputer;Non-IT
E31201177;freelance kerja lepas;Wirausaha Informatika
E31191286;staff;Non-IT
E31171680;quality control specialist;Non-IT
E31171700;administration;Non-IT
E31171618;customer service staff;Non-IT
E31180306;fulfillment admin;Data Analyst
E31171935;machine operator;Non-IT
E31191159;founder;Wirausaha Informatika
E31190939;founder;Wirausaha Informatika
E31190197;founder;Wirausaha Informatika
E31180878;management information system staf;Programmer
E31191888;staf;Non-IT
E31202319;freelance kerja lepas;Wirausaha Informatika
E31181597;cashier;Non-IT
E31181991;digital marketing specialist;Wirausaha Informatika
E31180513;contact center agent;Non-IT
E31171243;it staff teknisi;Programmer
E31192304;founder;Wirausaha Informatika
E31201431;freelance kerja lepas;Data Analyst
E31200001;freelance kerja lepas;Wirausaha Informatika
E31192311;founder;Wirausaha Informatika
E31200456;data analyst;Data Analyst
E31170933;tester;Programmer
E31190883;staff;Programmer
E31201773;freelance kerja lepas;Programmer
E31181100;freelance kerja lepas;Wirausaha Informatika
E31200592;guru;Non-IT
E31200680;founder;Wirausaha Informatika
E31190397;freelance kerja lepas;Wirausaha Informatika
E31192391;staff;Programmer
E31170372;administrative assistant pdpt upt tik;Non-IT
E31180837;designer;Programmer
E31190631;staff;Non-IT
E31190430;freelance kerja lepas;Wirausaha Informatika
E31171296;cutting machine operator;Non-IT
E31192353;founder;Wirausaha Informatika
E31201537;freelance kerja lepas;Wirausaha Informatika
E31191309;freelance kerja lepas;Programmer
E31181436;tenaga ajar informatika;Programmer
E31171061;back end developer;Programmer
E31191228;it staff teknisi;Programmer
E31192463;founder;Wirausaha Informatika
E31180365;founder;Wirausaha Informatika
E31181496;game programmer;Programmer
E31191879;founder;Wirausaha Informatika
E31191434;staff;Programmer
E31171868;information technology staff;Non-IT
E31200649;human resource;Non-IT
E31191551;freelance kerja lepas;Wirausaha Informatika
E31171444;wirausaha founder;Wirausaha Informatika
E31192259;owner;Wirausaha Informatika
E31170906;wirausaha founder;Wirausaha Informatika
E31191250;founder;Wirausaha Informatika
E31191116;freelance kerja lepas;Wirausaha Informatika
E31201246;digital marketing;Non-IT
E31180370;mitra bps;Wirausaha Informatika
E31200102;freelance kerja lepas;Wirausaha Informatika
E31202394;staff;Non-IT
E31190449;staff;Non-IT
E31172013;wirausaha founder;Wirausaha Informatika
E31190658;staff;Non-IT
E31171333;it staff teknisi;Programmer
E31181452;management trainer;Non-IT
E31170389;field coll;Non-IT
E31201939;freelance kerja lepas;Programmer
E31181091;management informatika;Programmer
E31182235;design director;Programmer
E31191433;founder;Wirausaha Informatika
E31180373;back office assistant;Non-IT
E31190445;staff;Non-IT
E31170772;engineer product;Programmer
E31171103;it staff teknisi;Programmer
E31192079;staff;Programmer
E31192281;mitra rider operational;Non-IT
E31182077;system analyst;Data Analyst
E31162061;web developer;Programmer
E31192214;staff;Programmer
E31171777;programmer developer;Programmer
E31180696;founder;Wirausaha Informatika
E31160690;graphic designer;Non-IT
E31171257;admin dan digital marketing;Non-IT
E31201546;freelance kerja lepas;Wirausaha Informatika
E31181373;sosial media and website admin;Non-IT
E31202490;freelance kerja lepas;Wirausaha Informatika
E31191195;founder;Wirausaha Informatika
E31190698;staff;Non-IT
E31200997;freelance kerja lepas;Wirausaha Informatika
E31171236;it staff teknisi;Programmer
E31182299;staff noc;Programmer E31182299;staff noc;Programmer
E31201536;freelance kerja lepas;Programmer E31201536;freelance kerja lepas;Programmer
E31202536;freelance kerja lepas;Data Analyst E31180638;founder;Wirausaha Informatika
E31171061;back end developer;Programmer E31200822;staff;Wirausaha Informatika
E31181131;founder;Data Analyst E31200837;founder;Wirausaha Informatika
E31200844;staff;Programmer E31190134;freelance kerja lepas;Wirausaha Informatika
E31192134;freelance kerja lepas;Wirausaha Informatika E31180539;staff;Non-IT
E31201009;staff;Non-IT E31190893;staff;Non-IT
E31180370;freelance kerja lepas;Wirausaha Informatika E31180748;freelance kerja lepas;Wirausaha Informatika
E31181436;staff;Programmer E31181263;staff;Non-IT
E31180691;founder;Wirausaha Informatika E31171971;it staff teknisi;Programmer
E31170372;administrative assistant pdpt upt tik;Non-IT
E31190070;staff administrator;Non-IT
E31190416;freelance kerja lepas;Wirausaha Informatika
E31191551;freelance kerja lepas;Wirausaha Informatika
E31170715;administrasi;Non-IT
E31190957;freelance kerja lepas;Wirausaha Informatika
E31191919;freelance kerja lepas;Wirausaha Informatika
E31201146;freelance kerja lepas;Wirausaha Informatika E31201146;freelance kerja lepas;Wirausaha Informatika
E31171350;programmer developer;Programmer E31191849;founder;Programmer
E31190302;teknisi laboratorium;Programmer E31192293;freelance kerja lepas;Wirausaha Informatika
E31180369;freelance kerja lepas;Wirausaha Informatika E31181955;support technician;Programmer
E31191888;staf;Non-IT E31181131;system analyst;Data Analyst
E31181373;freelance kerja lepas;Wirausaha Informatika E31171076;support specialist;Programmer
E31191447;freelance kerja lepas;Wirausaha Informatika E31192120;freelance kerja lepas;Wirausaha Informatika
E31171996;beauty advisor you;Non-IT E31171096;customer service officer;Non-IT
E31160910;founder;Wirausaha Informatika E31180276;staff;Wirausaha Informatika
E31191116;freelance kerja lepas;Wirausaha Informatika E31171869;programmer developer;Programmer
E31201794;founder;Wirausaha Informatika E31201199;founder;Wirausaha Informatika
E31191300;staff;Non-IT
E31170805;senior account manager;Non-IT
E31200912;staff;Non-IT
E31192472;freelance kerja lepas;Wirausaha Informatika
E31181297;staff;Wirausaha Informatika
E31191310;founder;Wirausaha Informatika
E31200844;staff;Programmer
E31190731;founder;Programmer
E31190596;staff;Non-IT
E31200864;freelance kerja lepas;Wirausaha Informatika
E31190993;staff;Non-IT
E31180369;admin;Wirausaha Informatika
E31192117;freelance kerja lepas;Wirausaha Informatika
E31180297;adminsitrasi marketing;Non-IT
E31182140;programer;Programmer
E31181553;staff admin;Non-IT
E31180884;staff;Programmer
E31181158;web developer;Programmer
E31170445;it staff teknisi;Programmer
E31172262;it staff teknisi;Programmer
E31181391;staff;Non-IT
E31180256;admin zoom;Non-IT
E31170217;human resources development officer;Non-IT
E31180222;freelance kerja lepas;Wirausaha Informatika
E31170384;customer service;Non-IT
E31202463;staff;Non-IT
E31180989;information technology services;Programmer
E31191708;desainer grafis;Non-IT
E31190270;staff;Non-IT
E31190279;founder;Programmer
E31171313;it staff teknisi;Programmer
E31171151;sales executive;Non-IT
E31181474;founder;Wirausaha Informatika
E31191982;staff;Non-IT
E31200917;freelance kerja lepas;Wirausaha Informatika
E31181097;call center officer;Non-IT
E31191821;founder;Wirausaha Informatika
E31192072;staff;Non-IT
E31202279;freelance kerja lepas;Programmer
E31201845;founder;Wirausaha Informatika
E31201056;freelance kerja lepas;Wirausaha Informatika
E31192202;admin produksi;Non-IT
E31182133;freelance kerja lepas;Programmer
E31180456;it staff teknisi;Programmer
E31191120;freelance kerja lepas;Programmer
E31202339;freelance kerja lepas;Wirausaha Informatika
E31190648;founder;Programmer
E31190933;staff;Non-IT E31190933;staff;Non-IT
E31161776;web developer;Programmer E31192221;freelance kerja lepas;Wirausaha Informatika
E31182235;programmer developer;Programmer E31180825;staff;Non-IT
E31192285;founder;Programmer
E31180691;sales executive;Wirausaha Informatika
E31200637;freelance kerja lepas;Wirausaha Informatika
E31181049;programmer;Programmer
E31201555;freelance kerja lepas;Wirausaha Informatika
E31171996;beauty advisor you;Non-IT
E31161169;customer service;Non-IT
E31200918;freelance kerja lepas;Wirausaha Informatika
E31191709;staf;Non-IT
E31171470;staf;Non-IT
E31192033;staff;Programmer
E31171431;guru tik;Non-IT
E31171350;programmer developer;Programmer
E31201647;founder;Wirausaha Informatika
E31181280;admin;Programmer
E31201794;founder;Wirausaha Informatika
E31192347;freelance kerja lepas;Wirausaha Informatika
E31160548;senior backend engineer;Programmer
E31201228;founder;Wirausaha Informatika
E31201158;freelance kerja lepas;Wirausaha Informatika
E31191919;freelance kerja lepas;Wirausaha Informatika
E31190865;founder;Programmer
E31180332;founder;Wirausaha Informatika
E31171281;direktur utama;Wirausaha Informatika
E31190320;staff;Wirausaha Informatika
E31200251;ppic;Non-IT
E31190237;staff;Non-IT
E31171427;pranata komputer;Non-IT
E31182101;founder;Wirausaha Informatika
E31192254;freelance kerja lepas;Programmer E31192254;freelance kerja lepas;Programmer
E31201887;freelance kerja lepas;Programmer E31201887;freelance kerja lepas;Programmer
E31191310;founder;Wirausaha Informatika E31202536;freelance kerja lepas;Data Analyst
E31180575;staff;Programmer E31171799;web programmer;Programmer
E31160548;senior backend engineer;Programmer E31170734;administrative assistant;Non-IT
E31181097;call center officer;Non-IT
E31180825;staff;Non-IT
E31202394;staff;Non-IT
E31201158;freelance kerja lepas;Wirausaha Informatika
E31201246;digital marketing;Non-IT
E31181391;staff;Non-IT
E31172208;fullstack developer web app;Programmer
E31171700;administration;Non-IT
E31191286;staff;Non-IT
E31180949;programmer;Programmer
E3117228;programmer developer;Programmer
E31190237;staff;Non-IT
E31182140;programer;Programmer
E31170384;customer service;Non-IT
E31192463;founder;Wirausaha Informatika
E31181297;staff;Wirausaha Informatika
E31191792;admin and finance staff;Non-IT
E31172013;wirausaha founder;Wirausaha Informatika
E31191709;staf;Non-IT
E31191434;staff;Programmer
E31200918;freelance kerja lepas;Wirausaha Informatika
E31190055;staff usaha pelabuhan;Wirausaha Informatika
E31190987;staff administrasi;Non-IT
E31192117;freelance kerja lepas;Wirausaha Informatika
E31180884;staff;Programmer
E31192033;staff;Programmer
E31191401;staff;Programmer
E31181971;freelance kerja lepas;Wirausaha Informatika
E31171317;web developer;Programmer
E31192159;staff data;Non-IT
E31180297;founder;Wirausaha Informatika
E31181263;staff;Non-IT
E31192214;staff;Programmer
E31190893;staff;Non-IT
E31192047;founder;Wirausaha Informatika
E31191848;fullstack developer;Programmer
E31192281;mitra rider operational;Non-IT
E31180303;staff;Non-IT
E31171257;admin dan digital marketing;Non-IT
E31190320;staff;Wirausaha Informatika
E31202339;freelance kerja lepas;Wirausaha Informatika
1 nim job_text_raw label
E31200864 freelance kerja lepas Wirausaha Informatika
E31202562 staff Non-IT
E31202523 freelance kerja lepas Wirausaha Informatika
E31201845 founder Wirausaha Informatika
E31201199 founder Wirausaha Informatika
E31171680 quality control specialist Non-IT
E31192166 founder Wirausaha Informatika
E31201422 freelance kerja lepas Wirausaha Informatika
E31190648 founder Programmer
E31190939 founder Wirausaha Informatika
E31201950 marketing officer Non-IT
E31190800 staff Non-IT
E31192123 customer engineer Programmer
E31201237 staff Non-IT
E31171791 pranata komputer Non-IT
E31192213 freelance kerja lepas Programmer
E31200102 freelance kerja lepas Wirausaha Informatika
E31170389 field coll Non-IT
E31190785 freelance kerja lepas Wirausaha Informatika
E31192403 freelance kerja lepas Wirausaha Informatika
E31201365 founder Programmer
E31162061 web developer Programmer
E31192304 founder Wirausaha Informatika
E31180306 staff Data Analyst
E31190980 staff Programmer
E31181553 staff admin Non-IT
E31181158 freelance kerja lepas Programmer
E31171868 information technology staff Non-IT
E31200700 staff Wirausaha Informatika
E31192175 petugas barang bukti Non-IT
E31201555 freelance kerja lepas Wirausaha Informatika
E31200251 ppic Non-IT
E31171935 machine operator Non-IT
E31170763 document control specialist Non-IT
E31160622 programmer Programmer
E31201573 staff Non-IT
E31201020 staff Non-IT
E31192043 freelance kerja lepas Wirausaha Informatika
E31192202 admin produksi Non-IT
E31171444 wirausaha founder Wirausaha Informatika
E31180373 staff Non-IT
E31171427 pranata komputer Non-IT
E31182030 freelance kerja lepas Programmer
E31171243 it staff teknisi Programmer
E31201687 staff graphic designer Non-IT
E31190631 staff Non-IT
E31180332 founder Wirausaha Informatika
E31202038 freelance kerja lepas Wirausaha Informatika
E31170217 human resources development officer Non-IT
E31191547 staf sdm parmas Non-IT
E31171076 support specialist Programmer
E31172262 it staff teknisi Programmer
E31170087 design development Non-IT
E31191300 staff Non-IT
E31200238 staff kontrak Non-IT
E31201300 freelance kerja lepas Wirausaha Informatika
E31171431 guru tik Non-IT
E31181991 wirausaha founder Wirausaha Informatika
E31171777 programmer developer Programmer
E31201932 freelance kerja lepas Wirausaha Informatika
E31201954 freelance kerja lepas Wirausaha Informatika
E31191441 freelance kerja lepas Wirausaha Informatika
E31192311 founder Wirausaha Informatika
E31180878 freelance kerja lepas Programmer
E31200293 staff Non-IT
E31180456 it staff teknisi Programmer
E31192292 staff Non-IT
E31171367 office administrator Non-IT
E31180748 freelance kerja lepas Wirausaha Informatika
E31201939 freelance kerja lepas Programmer
E31171151 sales executive Non-IT
E31182101 founder Wirausaha Informatika
E31170734 administrative assistant Non-IT
E31191621 staff Non-IT
E31192416 freelance kerja lepas Wirausaha Informatika
E31192353 founder Wirausaha Informatika
E31171236 it staff teknisi Programmer
E31201647 founder Wirausaha Informatika
2 E31190527 founder Wirausaha Informatika
3 E31190801 staff Non-IT
4 E31190672 freelance kerja lepas Programmer
5 E31192382 staff Non-IT
6 E31200245 E31201634 staff Non-IT
7 E31191124 E31172208 founder fullstack developer web app Wirausaha Informatika Programmer
E31160690 graphic designer Non-IT
E31202345 founder Wirausaha Informatika
E31181929 administrasi bmn Non-IT
E31190865 founder Programmer
E31190698 staff Non-IT
E31181100 freelance kerja lepas Wirausaha Informatika
E31201431 freelance kerja lepas Data Analyst
E31171296 cutting machine operator Non-IT
E31191159 founder Wirausaha Informatika
E31191793 founder Wirausaha Informatika
E31200456 data analyst Data Analyst
E31180678 freelance kerja lepas Wirausaha Informatika
E31181204 founder Wirausaha Informatika
E31190408 freelance kerja lepas Wirausaha Informatika
E31191228 it staff teknisi Programmer
E31202278 founder Wirausaha Informatika
E31201773 freelance kerja lepas Programmer
E31191006 freelance kerja lepas Wirausaha Informatika
E31192064 freelance kerja lepas Wirausaha Informatika
E31180207 founder Wirausaha Informatika
E31180513 it staff teknisi Programmer
E31202314 staf pelaksana Non-IT
E31201546 freelance kerja lepas Wirausaha Informatika
E31200888 freelance kerja lepas Wirausaha Informatika
E31191849 founder Programmer
E31200284 staff Non-IT
E31192029 freelance kerja lepas Programmer
E31180638 founder Wirausaha Informatika
E31180539 staff Non-IT
E31181496 programmer developer Programmer
E31192293 freelance kerja lepas Wirausaha Informatika
E31180504 founder Wirausaha Informatika
E31170392 it staff teknisi Programmer
E31201537 freelance kerja lepas Wirausaha Informatika
E31200837 founder Wirausaha Informatika
E31192060 freelance kerja lepas Wirausaha Informatika
E31180256 freelance kerja lepas Wirausaha Informatika
E31171971 it staff teknisi Programmer
E31181474 founder Wirausaha Informatika
E31181955 freelance kerja lepas Programmer
E31191879 founder Wirausaha Informatika
E31170933 tester Programmer
8 E31182109 freelance kerja lepas Wirausaha Informatika
E31160524 kepala urusan perencanaan Non-IT
E31191234 staff Non-IT
E31201056 freelance kerja lepas Wirausaha Informatika
E31170906 wirausaha founder Wirausaha Informatika
E31171618 customer service staff Non-IT
E31181049 freelance kerja lepas Programmer
9 E31160306 admin stnk bpkb Non-IT
10 E31171470 E31160524 staf kepala urus rencana Non-IT
11 E31190445 E31191401 staff Non-IT Programmer
12 E31171869 E31190679 programmer developer staff Programmer Non-IT
13 E31200822 E31181567 staff founder Wirausaha Informatika
14 E31191120 E31191677 freelance kerja lepas Programmer Wirausaha Informatika
15 E31161169 E31161776 customer service web developer Non-IT Programmer
16 E31190279 E31200254 founder sekretaris kurikulum Programmer Non-IT
17 E31191894 E31201237 fullstack developer staff Programmer Non-IT
18 E31180703 E31170087 it staff teknisi design development Programmer Non-IT
19 E31201829 E31192064 staff freelance kerja lepas Non-IT Wirausaha Informatika
20 E31182077 E31160622 data analyst programmer Data Analyst Programmer
21 E31200356 E31191394 founder Programmer Wirausaha Informatika
22 E31180644 E31191234 freelance kerja lepas staff Programmer Non-IT
23 E31200973 E31201365 programmer founder Programmer
E31171313 it staff teknisi Programmer
E31192472 freelance kerja lepas Wirausaha Informatika
E31202319 freelance kerja lepas Wirausaha Informatika
E31191800 checker angkutan barang Non-IT
E31170805 senior account manager Non-IT
E31192024 teknisi laboratorium Programmer
E31192285 founder Programmer
E31191660 founder Wirausaha Informatika
E31191433 founder Wirausaha Informatika
24 E31160880 teller Non-IT
25 E31201706 E31202345 freelance kerja lepas founder Wirausaha Informatika
26 E31190341 E31180299 staff teller Programmer Non-IT
27 E31202279 E31170763 freelance kerja lepas document control specialist Programmer Non-IT
28 E31190883 E31180464 staff Programmer Wirausaha Informatika
29 E31200997 E31180112 freelance kerja lepas it staff teknisi Wirausaha Informatika Programmer
30 E31182133 E31200888 freelance kerja lepas Programmer Wirausaha Informatika
31 E31180707 E31180260 freelance kerja lepas staff Programmer Non-IT
32 E31180295 E31190070 freelance kerja lepas staff administrator Wirausaha Informatika Non-IT
33 E31200117 E31191124 freelance kerja lepas founder Wirausaha Informatika
34 E31190134 E31201690 freelance kerja lepas staff Wirausaha Informatika Non-IT
35 E31180724 E31200910 lain lain staff Non-IT
36 E31190286 E31190957 staff freelance kerja lepas Non-IT Wirausaha Informatika
37 E31180686 E31170715 founder administrasi Wirausaha Informatika Non-IT
38 E31171096 E31200284 customer service officer staff Non-IT
39 E31200880 freelance kerja lepas Wirausaha Informatika
40 E31181885 staff admin Non-IT
41 E31200549 staff Non-IT
42 E31180703 it staff teknisi Programmer
43 E31171367 office administrator Non-IT
44 E31160910 administration assistant Wirausaha Informatika
45 E31201459 freelance kerja lepas Wirausaha Informatika
46 E31201095 freelance kerja lepas Wirausaha Informatika
47 E31160241 web developer Programmer
48 E31191460 freelance kerja lepas Wirausaha Informatika
49 E31200245 staff Non-IT
50 E31190851 founder Wirausaha Informatika
51 E31200238 staff kontrak Non-IT
52 E31190055 staff usaha labuh Wirausaha Informatika
53 E31200293 staff Non-IT
54 E31200356 founder Programmer
55 E31191894 fullstack developer Programmer
56 E31170392 it staff teknisi Programmer
57 E31171791 pranata komputer Non-IT
58 E31201177 freelance kerja lepas Wirausaha Informatika
59 E31191286 staff Non-IT
60 E31171680 quality control specialist Non-IT
61 E31171700 administration Non-IT
62 E31171618 customer service staff Non-IT
63 E31180306 fulfillment admin Data Analyst
64 E31171935 machine operator Non-IT
65 E31191159 founder Wirausaha Informatika
66 E31190939 founder Wirausaha Informatika
67 E31190197 founder Wirausaha Informatika
68 E31180878 management information system staf Programmer
69 E31191888 staf Non-IT
70 E31202319 freelance kerja lepas Wirausaha Informatika
71 E31181597 cashier Non-IT
72 E31181991 digital marketing specialist Wirausaha Informatika
73 E31180513 contact center agent Non-IT
74 E31171243 it staff teknisi Programmer
75 E31192304 founder Wirausaha Informatika
76 E31201431 freelance kerja lepas Data Analyst
77 E31200001 freelance kerja lepas Wirausaha Informatika
78 E31192311 founder Wirausaha Informatika
79 E31200456 data analyst Data Analyst
80 E31170933 tester Programmer
81 E31190883 staff Programmer
82 E31201773 freelance kerja lepas Programmer
83 E31181100 freelance kerja lepas Wirausaha Informatika
84 E31200592 guru Non-IT
85 E31200680 founder Wirausaha Informatika
86 E31190397 freelance kerja lepas Wirausaha Informatika
87 E31192391 staff Programmer
88 E31170372 administrative assistant pdpt upt tik Non-IT
89 E31180837 designer Programmer
90 E31190631 staff Non-IT
91 E31190430 freelance kerja lepas Wirausaha Informatika
92 E31171296 cutting machine operator Non-IT
93 E31192353 founder Wirausaha Informatika
94 E31201537 freelance kerja lepas Wirausaha Informatika
95 E31191309 freelance kerja lepas Programmer
96 E31181436 tenaga ajar informatika Programmer
97 E31171061 back end developer Programmer
98 E31191228 it staff teknisi Programmer
99 E31192463 founder Wirausaha Informatika
100 E31180365 founder Wirausaha Informatika
101 E31181496 game programmer Programmer
102 E31191879 founder Wirausaha Informatika
103 E31191434 staff Programmer
104 E31171868 information technology staff Non-IT
105 E31200649 human resource Non-IT
106 E31191551 freelance kerja lepas Wirausaha Informatika
107 E31171444 wirausaha founder Wirausaha Informatika
108 E31192259 owner Wirausaha Informatika
109 E31170906 wirausaha founder Wirausaha Informatika
110 E31191250 founder Wirausaha Informatika
111 E31191116 freelance kerja lepas Wirausaha Informatika
112 E31201246 digital marketing Non-IT
113 E31180370 mitra bps Wirausaha Informatika
114 E31200102 freelance kerja lepas Wirausaha Informatika
115 E31202394 staff Non-IT
116 E31190449 staff Non-IT
117 E31172013 wirausaha founder Wirausaha Informatika
118 E31190658 staff Non-IT
119 E31171333 it staff teknisi Programmer
120 E31181452 management trainer Non-IT
121 E31170389 field coll Non-IT
122 E31201939 freelance kerja lepas Programmer
123 E31181091 management informatika Programmer
124 E31182235 design director Programmer
125 E31191433 founder Wirausaha Informatika
126 E31180373 back office assistant Non-IT
127 E31190445 staff Non-IT
128 E31170772 engineer product Programmer
129 E31171103 it staff teknisi Programmer
130 E31192079 staff Programmer
131 E31192281 mitra rider operational Non-IT
132 E31182077 system analyst Data Analyst
133 E31162061 web developer Programmer
134 E31192214 staff Programmer
135 E31171777 programmer developer Programmer
136 E31180696 founder Wirausaha Informatika
137 E31160690 graphic designer Non-IT
138 E31171257 admin dan digital marketing Non-IT
139 E31201546 freelance kerja lepas Wirausaha Informatika
140 E31181373 sosial media and website admin Non-IT
141 E31202490 freelance kerja lepas Wirausaha Informatika
142 E31191195 founder Wirausaha Informatika
143 E31190698 staff Non-IT
144 E31200997 freelance kerja lepas Wirausaha Informatika
145 E31171236 it staff teknisi Programmer
146 E31182299 staff noc Programmer
147 E31201536 freelance kerja lepas Programmer
148 E31202536 E31180638 freelance kerja lepas founder Data Analyst Wirausaha Informatika
149 E31171061 E31200822 back end developer staff Programmer Wirausaha Informatika
150 E31181131 E31200837 founder Data Analyst Wirausaha Informatika
151 E31200844 E31190134 staff freelance kerja lepas Programmer Wirausaha Informatika
152 E31192134 E31180539 freelance kerja lepas staff Wirausaha Informatika Non-IT
153 E31201009 E31190893 staff Non-IT
154 E31180370 E31180748 freelance kerja lepas Wirausaha Informatika
155 E31181436 E31181263 staff Programmer Non-IT
156 E31180691 E31171971 founder it staff teknisi Wirausaha Informatika Programmer
E31170372 administrative assistant pdpt upt tik Non-IT
E31190070 staff administrator Non-IT
E31190416 freelance kerja lepas Wirausaha Informatika
E31191551 freelance kerja lepas Wirausaha Informatika
E31170715 administrasi Non-IT
E31190957 freelance kerja lepas Wirausaha Informatika
E31191919 freelance kerja lepas Wirausaha Informatika
157 E31201146 freelance kerja lepas Wirausaha Informatika
158 E31171350 E31191849 programmer developer founder Programmer
159 E31190302 E31192293 teknisi laboratorium freelance kerja lepas Programmer Wirausaha Informatika
160 E31180369 E31181955 freelance kerja lepas support technician Wirausaha Informatika Programmer
161 E31191888 E31181131 staf system analyst Non-IT Data Analyst
162 E31181373 E31171076 freelance kerja lepas support specialist Wirausaha Informatika Programmer
163 E31191447 E31192120 freelance kerja lepas Wirausaha Informatika
164 E31171996 E31171096 beauty advisor you customer service officer Non-IT
165 E31160910 E31180276 founder staff Wirausaha Informatika
166 E31191116 E31171869 freelance kerja lepas programmer developer Wirausaha Informatika Programmer
167 E31201794 E31201199 founder Wirausaha Informatika
168 E31191300 staff Non-IT
169 E31170805 senior account manager Non-IT
170 E31200912 staff Non-IT
171 E31192472 freelance kerja lepas Wirausaha Informatika
172 E31181297 staff Wirausaha Informatika
173 E31191310 founder Wirausaha Informatika
174 E31200844 staff Programmer
175 E31190731 founder Programmer
176 E31190596 staff Non-IT
177 E31200864 freelance kerja lepas Wirausaha Informatika
178 E31190993 staff Non-IT
179 E31180369 admin Wirausaha Informatika
180 E31192117 freelance kerja lepas Wirausaha Informatika
181 E31180297 adminsitrasi marketing Non-IT
182 E31182140 programer Programmer
183 E31181553 staff admin Non-IT
184 E31180884 staff Programmer
185 E31181158 web developer Programmer
186 E31170445 it staff teknisi Programmer
187 E31172262 it staff teknisi Programmer
188 E31181391 staff Non-IT
189 E31180256 admin zoom Non-IT
190 E31170217 human resources development officer Non-IT
191 E31180222 freelance kerja lepas Wirausaha Informatika
192 E31170384 customer service Non-IT
193 E31202463 staff Non-IT
194 E31180989 information technology services Programmer
195 E31191708 desainer grafis Non-IT
196 E31190270 staff Non-IT
197 E31190279 founder Programmer
198 E31171313 it staff teknisi Programmer
199 E31171151 sales executive Non-IT
200 E31181474 founder Wirausaha Informatika
201 E31191982 staff Non-IT
202 E31200917 freelance kerja lepas Wirausaha Informatika
203 E31181097 call center officer Non-IT
204 E31191821 founder Wirausaha Informatika
205 E31192072 staff Non-IT
206 E31202279 freelance kerja lepas Programmer
207 E31201845 founder Wirausaha Informatika
208 E31201056 freelance kerja lepas Wirausaha Informatika
209 E31192202 admin produksi Non-IT
210 E31182133 freelance kerja lepas Programmer
211 E31180456 it staff teknisi Programmer
212 E31191120 freelance kerja lepas Programmer
213 E31202339 freelance kerja lepas Wirausaha Informatika
214 E31190648 founder Programmer
215 E31190933 staff Non-IT
216 E31161776 E31192221 web developer freelance kerja lepas Programmer Wirausaha Informatika
217 E31182235 E31180825 programmer developer staff Programmer Non-IT
218 E31192285 founder Programmer
219 E31180691 sales executive Wirausaha Informatika
220 E31200637 freelance kerja lepas Wirausaha Informatika
221 E31181049 programmer Programmer
222 E31201555 freelance kerja lepas Wirausaha Informatika
223 E31171996 beauty advisor you Non-IT
224 E31161169 customer service Non-IT
225 E31200918 freelance kerja lepas Wirausaha Informatika
226 E31191709 staf Non-IT
227 E31171470 staf Non-IT
228 E31192033 staff Programmer
229 E31171431 guru tik Non-IT
230 E31171350 programmer developer Programmer
231 E31201647 founder Wirausaha Informatika
232 E31181280 admin Programmer
233 E31201794 founder Wirausaha Informatika
234 E31192347 freelance kerja lepas Wirausaha Informatika
235 E31160548 senior backend engineer Programmer
236 E31201228 founder Wirausaha Informatika
237 E31201158 freelance kerja lepas Wirausaha Informatika
238 E31191919 freelance kerja lepas Wirausaha Informatika
239 E31190865 founder Programmer
240 E31180332 founder Wirausaha Informatika
241 E31171281 direktur utama Wirausaha Informatika
242 E31190320 staff Wirausaha Informatika
243 E31200251 ppic Non-IT
244 E31190237 staff Non-IT
245 E31171427 pranata komputer Non-IT
246 E31182101 founder Wirausaha Informatika
247 E31192254 freelance kerja lepas Programmer
248 E31201887 freelance kerja lepas Programmer
249 E31191310 E31202536 founder freelance kerja lepas Wirausaha Informatika Data Analyst
250 E31180575 E31171799 staff web programmer Programmer
251 E31160548 E31170734 senior backend engineer administrative assistant Programmer Non-IT
E31181097 call center officer Non-IT
E31180825 staff Non-IT
E31202394 staff Non-IT
E31201158 freelance kerja lepas Wirausaha Informatika
E31201246 digital marketing Non-IT
E31181391 staff Non-IT
E31172208 fullstack developer web app Programmer
E31171700 administration Non-IT
E31191286 staff Non-IT
E31180949 programmer Programmer
E3117228 programmer developer Programmer
E31190237 staff Non-IT
E31182140 programer Programmer
E31170384 customer service Non-IT
E31192463 founder Wirausaha Informatika
E31181297 staff Wirausaha Informatika
E31191792 admin and finance staff Non-IT
E31172013 wirausaha founder Wirausaha Informatika
E31191709 staf Non-IT
E31191434 staff Programmer
E31200918 freelance kerja lepas Wirausaha Informatika
E31190055 staff usaha pelabuhan Wirausaha Informatika
E31190987 staff administrasi Non-IT
E31192117 freelance kerja lepas Wirausaha Informatika
E31180884 staff Programmer
E31192033 staff Programmer
E31191401 staff Programmer
E31181971 freelance kerja lepas Wirausaha Informatika
E31171317 web developer Programmer
E31192159 staff data Non-IT
E31180297 founder Wirausaha Informatika
E31181263 staff Non-IT
E31192214 staff Programmer
E31190893 staff Non-IT
E31192047 founder Wirausaha Informatika
E31191848 fullstack developer Programmer
E31192281 mitra rider operational Non-IT
E31180303 staff Non-IT
E31171257 admin dan digital marketing Non-IT
E31190320 staff Wirausaha Informatika
E31202339 freelance kerja lepas Wirausaha Informatika

177
data/train_model.py Normal file
View File

@ -0,0 +1,177 @@
import pandas as pd
import numpy as np
import re
import joblib
import json
import warnings
from pathlib import Path
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import classification_report, confusion_matrix, ConfusionMatrixDisplay
from sklearn.pipeline import Pipeline
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
warnings.filterwarnings('ignore')
BASE_DIR = Path(__file__).parent
TRAIN_FILE = BASE_DIR / "processed" / "training_corpus.csv"
TEST_FILE = BASE_DIR / "processed" / "test_set.csv"
ML_DIR = BASE_DIR.parent / "fastapi" / "ml_assets"
ML_DIR.mkdir(parents=True, exist_ok=True)
TARGET_CLASSES = ["Programmer", "Data Analyst", "Wirausaha Informatika", "Non-IT"]
CONFIDENCE_THRESHOLD = 0.50
# [v4] Tidak import Sastrawi — corpus sudah di-stem oleh prepare_corpus.py
# Stemming dua kali pada data yang sama dapat mendegradasi teks.
STOPWORDS = {
"yang", "di", "ke", "dari", "dan", "atau", "dengan", "untuk", "pada", "dalam",
"adalah", "ini", "itu", "tidak", "juga", "sudah", "akan", "bisa", "ada", "oleh",
"karena", "secara", "serta", "sebagai", "bagi", "telah", "maka", "namun", "sehingga",
"jika", "agar", "ketika", "saat", "sebelum", "sesudah", "hingga", "sampai", "antara",
"sekitar", "hanya", "saja", "belum", "masih", "lagi", "pun", "justru", "walaupun",
"meskipun", "bahkan", "cukup", "sangat", "paling", "lebih", "kurang", "lain",
"macam", "cara", "hal", "tentang", "mengenai", "terhadap", "kepada", "menuju",
"kecuali", "selain", "tanpa", "demi", "guna", "khususnya", "umumnya", "kebanyakan",
"sebagian", "beberapa", "semua", "setiap", "tiap", "satu", "dua", "tiga", "empat",
"lima", "enam", "tujuh", "delapan", "sembilan", "sepuluh", "ratus", "ribu", "juta"
}
def preprocess_text(text):
"""Pembersihan dasar — tanpa stemming ulang karena korpus sudah di-stem."""
if pd.isna(text) or not isinstance(text, str): return ""
text = text.lower().strip()
if text in {"nan", "none", "null", "-", "0", "", "tidak diisi"}: return ""
text = text.replace('-', ' ').replace('/', ' ').replace('_', ' ').replace(',', ' ')
text = re.sub(r'[^\w\s]', '', text)
text = re.sub(r'\d+', '', text)
words = [w for w in text.split() if w not in STOPWORDS and len(w) > 2]
return " ".join(words)
def main():
if not TRAIN_FILE.exists():
print("File training_corpus.csv belum ada. Jalankan prepare_corpus.py terlebih dahulu."); return
df_train = pd.read_csv(TRAIN_FILE, sep=";", dtype=str).dropna(subset=["job_text_raw", "label"])
df_test = pd.read_csv(TEST_FILE, sep=";", dtype=str).dropna(subset=["job_text_raw", "label"]) if TEST_FILE.exists() else None
print("Sedang memproses teks (cleaning dasar tanpa stemming redundan)...")
X_train = df_train["job_text_raw"].apply(preprocess_text)
y_train = df_train["label"]
mask = X_train.str.len() > 0
X_train, y_train = X_train[mask], y_train[mask]
X_test, y_test = pd.Series(dtype=str), pd.Series(dtype=str)
if df_test is not None:
X_test = df_test["job_text_raw"].apply(preprocess_text)
y_test = df_test["label"]
mask_t = X_test.str.len() > 0
X_test, y_test = X_test[mask_t], y_test[mask_t]
print(f"Jumlah data latih: {len(X_train)} baris | Data uji: {len(X_test)} baris\n")
# [KEEP v3] K-Fold cross validation — estimasi variance performa model
print("Melakukan pengujian K-Fold (5 putaran) pada data latih...")
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
fold_metrics = []
for i, (tr_idx, te_idx) in enumerate(skf.split(X_train, y_train)):
model = Pipeline([
('tfidf', TfidfVectorizer(max_features=3000, ngram_range=(1,2), sublinear_tf=True, min_df=1)),
('clf', LogisticRegression(max_iter=1000, class_weight='balanced', solver='lbfgs'))
])
model.fit(X_train.iloc[tr_idx], y_train.iloc[tr_idx])
y_pred = model.predict(X_train.iloc[te_idx])
rep = classification_report(y_train.iloc[te_idx], y_pred, output_dict=True, zero_division=0)
fold_metrics.append(rep)
print(f" Akurasi putaran ke-{i+1}: {rep['accuracy']:.4f}")
avg_acc = np.mean([f['accuracy'] for f in fold_metrics])
print(f"\n RATA-RATA PENGUJIAN K-FOLD:")
print(f" Akurasi Keseluruhan : {avg_acc:.4f} ± {np.std([f['accuracy'] for f in fold_metrics]):.4f}")
for cls in TARGET_CLASSES:
p = np.mean([f.get(cls, {}).get('precision', 0) for f in fold_metrics])
r = np.mean([f.get(cls, {}).get('recall', 0) for f in fold_metrics])
f1 = np.mean([f.get(cls, {}).get('f1-score', 0) for f in fold_metrics])
print(f" {cls:25} | P: {p:.3f} | R: {r:.3f} | F1: {f1:.3f}")
print("\n Membuat model final dari seluruh data latih yang tersedia...")
final_model = Pipeline([
('tfidf', TfidfVectorizer(max_features=3000, ngram_range=(1,2), sublinear_tf=True, min_df=1)),
('clf', LogisticRegression(max_iter=1000, class_weight='balanced', solver='lbfgs'))
])
final_model.fit(X_train, y_train)
metrics_test = None
if len(X_test) > 0:
y_pred = final_model.predict(X_test)
rep_test = classification_report(y_test, y_pred, output_dict=True, zero_division=0)
metrics_test = rep_test
print("\n HASIL PENGUJIAN PADA DATA TEST:")
print(classification_report(y_test, y_pred, zero_division=0))
for cls in TARGET_CLASSES:
sup = rep_test[cls]['support'] if cls in rep_test else 0
if sup < 5: print(f"Catatan: Kelas '{cls}' cuma punya {sup} data uji — skor F1-nya mungkin kurang akurat.")
# [KEEP v3] Confusion matrix
cm = confusion_matrix(y_test, y_pred, labels=TARGET_CLASSES)
disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=TARGET_CLASSES)
disp.plot(cmap='Blues', values_format='d', xticks_rotation=45, colorbar=False)
plt.title('Confusion Matrix - Hold-Out Test'); plt.tight_layout()
plt.savefig(ML_DIR / 'confusion_matrix_test.png', dpi=300); plt.close()
print("Grafik confusion matrix berhasil disimpan ke confusion_matrix_test.png")
# [BUGFIX] Feature importance per kelas
# clf.coef_ diurutkan mengikuti clf.classes_ (alfabetikal sklearn),
# BUKAN urutan TARGET_CLASSES yang hardcoded. Pakai clf.classes_ sebagai
# iterator agar index i selalu selaras dengan baris koefisien yang benar.
vec, clf = final_model.named_steps['tfidf'], final_model.named_steps['clf']
feats, coeffs = vec.get_feature_names_out(), clf.coef_
print("\n5 KATA PALING BERPENGARUH UNTUK TIAP KELAS:")
for i, cls in enumerate(clf.classes_): # <-- clf.classes_, bukan TARGET_CLASSES
top = [(feats[j], coeffs[i][j]) for j in coeffs[i].argsort()[-5:][::-1] if coeffs[i][j] > 0]
print(f" [{cls}] " + ", ".join([f"{w}({c:.2f})" for w, c in top]))
# [KEEP v3] Confidence distribution
proba = final_model.predict_proba(X_test)
max_p = np.max(proba, axis=1)
plt.hist(max_p, bins=15, edgecolor='black', alpha=0.7, color='teal')
plt.axvline(x=CONFIDENCE_THRESHOLD, color='red', linestyle='--', linewidth=2, label=f'Threshold {CONFIDENCE_THRESHOLD}')
plt.xlabel('Confidence Score'); plt.ylabel('Count'); plt.title('Distribusi Confidence Score')
plt.legend(); plt.grid(axis='y', alpha=0.3); plt.tight_layout()
plt.savefig(ML_DIR / 'confidence_distribution.png', dpi=300); plt.close()
below = np.sum(max_p < CONFIDENCE_THRESHOLD)
print(f"\nPengecekan Keyakinan Model: {below} dari {len(max_p)} prediksi ({below/len(max_p)*100:.1f}%) di bawah threshold {CONFIDENCE_THRESHOLD}.")
# [KEEP v3] Daftar prediksi yang meleset
mis = np.where(y_test.values != y_pred)[0]
if len(mis) > 0:
print("\n DAFTAR PREDIKSI YANG MELESET:")
for idx in mis[:min(6, len(mis))]:
txt = df_test.iloc[idx]['job_text_raw'][:80]
print(f" • Seharusnya: {y_test.iloc[idx]:20} | Ditebak: {y_pred[idx]:20} | Teks: '{txt}...'")
else:
print("\n Semua prediksi pada test set benar.")
model_path = ML_DIR / "ml_pipeline_internal.pkl"
joblib.dump(final_model, model_path)
metrics = {
"methodology": "Internal MIF only",
"k_fold": {
"accuracy_mean": float(avg_acc),
"accuracy_std": float(np.std([f['accuracy'] for f in fold_metrics])),
"folds": fold_metrics
},
"threshold_config": CONFIDENCE_THRESHOLD
}
if metrics_test: metrics["hold_out_test"] = metrics_test
with open(ML_DIR / "metrics_internal_only.json", 'w') as f: json.dump(metrics, f, indent=2)
print(f"\n Model berhasil disimpan di: {model_path}")
print(f"Laporan metrik disimpan di: {ML_DIR / 'metrics_internal_only.json'}")
if __name__ == "__main__":
main()

384
data/unused/main.py Normal file
View File

@ -0,0 +1,384 @@
from fastapi import FastAPI, UploadFile, File, HTTPException, BackgroundTasks, Depends, Security
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
from fastapi.responses import JSONResponse
import pandas as pd
import numpy as np
import joblib
import re
import io
import json
import uuid
import shutil
import subprocess
import sys
import os
import logging
from contextlib import asynccontextmanager
from pathlib import Path
from typing import List, Dict, Any
from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
# KONFIGURASI & KONSTANTA
BASE_DIR = Path(__file__).parent.parent
ML_DIR = BASE_DIR / "ml_assets"
PIPELINE_PATH = ML_DIR / "ml_pipeline_internal.pkl"
STATUS_PATH = ML_DIR / "retrain_status.json"
RETRAIN_WORKER = Path(__file__).parent / "retrain_worker.py"
TEMP_DIR = ML_DIR / "tmp"
TEMP_DIR.mkdir(parents=True, exist_ok=True)
CONFIDENCE_THRESHOLD = 0.50
TARGET_CLASSES = ["Programmer", "Data Analyst", "Wirausaha Informatika", "Non-IT"]
KEYWORD_RULES = {
"Programmer": ["programmer", "developer", "engineer", "fullstack", "backend", "frontend", "mobile", "android", "ios", "software", "web dev", "coding", "it staff", "teknisi", "sistem informasi", "application", "network", "devops", "qa", "tester", "ui", "ux", "swe"],
"Data Analyst": ["data analyst", "analis data", "data science", "business analyst", "research", "statistik", "bi analyst", "reporting", "database", "sql", "etl", "data engineer", "big data", "analyst", "data mining", "machine learning", "data visual", "power bi", "tableau", "looker", "business intelligence", "bi developer", "data warehouse"],
"Wirausaha Informatika": ["founder", "owner", "ceo", "wiraswasta", "startup", "freelance", "freelancer", "wirausaha", "bisnis", "usaha mandiri", "konsultan", "co founder", "entrepreneur", "self employed", "owner toko", "usaha", "dagang online", "tokopedia", "shopee", "dropship", "reseller"]
}
F5C_MAP = {"1": "founder owner wirausaha startup", "2": "co-founder partner wirausaha", "3": "staff karyawan pegawai", "4": "freelance kerja lepas lepasan"}
F1101_MAP = {"1": "instansi pemerintah dinas kementerian", "2": "non-profit lsm yayasan", "3": "perusahaan swasta corporate", "4": "wiraswasta usaha mandiri", "6": "bumn bumd pemerintah", "7": "multilateral internasional"}
logging.basicConfig(level=logging.INFO, format="%(asctime)s | %(levelname)s | %(message)s")
logger = logging.getLogger("tracer_worker")
stemmer = StemmerFactory().create_stemmer()
STOPWORDS = {
"yang", "di", "ke", "dari", "dan", "atau", "dengan", "untuk", "pada", "dalam",
"adalah", "ini", "itu", "tidak", "juga", "sudah", "akan", "bisa", "ada", "oleh",
"karena", "secara", "serta", "sebagai", "bagi", "telah", "maka", "namun", "sehingga",
"jika", "agar", "ketika", "saat", "sebelum", "sesudah", "hingga", "sampai", "antara",
"sekitar", "hanya", "saja", "belum", "masih", "lagi", "pun", "justru", "walaupun",
"meskipun", "bahkan", "cukup", "sangat", "paling", "lebih", "kurang", "lain",
"macam", "cara", "hal", "tentang", "mengenai", "terhadap", "kepada", "menuju",
"kecuali", "selain", "tanpa", "demi", "guna", "khususnya", "umumnya", "kebanyakan",
"sebagian", "beberapa", "semua", "setiap", "tiap", "satu", "dua", "tiga", "empat",
"lima", "enam", "tujuh", "delapan", "sembilan", "sepuluh", "ratus", "ribu", "juta"
}
# MODEL LOADING
@asynccontextmanager
async def lifespan(app: FastAPI):
try:
app.state.pipeline = joblib.load(PIPELINE_PATH)
logger.info(f" ML Pipeline loaded successfully from {PIPELINE_PATH}")
except Exception as e:
logger.error(f" Failed to load ML pipeline: {e}")
app.state.pipeline = None
yield
app = FastAPI(title="Tracer Study Classification Worker", lifespan=lifespan)
# --- SECURITY SETUP ---
security = HTTPBearer()
EXPECTED_TOKEN = os.getenv("FASTAPI_SECRET_KEY", "")
def verify_token(credentials: HTTPAuthorizationCredentials = Security(security)):
if not EXPECTED_TOKEN:
return credentials.credentials
if credentials.credentials != EXPECTED_TOKEN:
raise HTTPException(status_code=401, detail="Unauthorized - Invalid Token")
return credentials.credentials
# ----------------------
def clean_text(text: str) -> str:
if pd.isna(text) or not isinstance(text, str):
return ""
text = text.lower().strip()
if text in {"nan", "none", "null", "-", "0", "", "tidak diisi"}:
return ""
text = text.replace('-', ' ').replace('/', ' ').replace('_', ' ').replace(',', ' ')
text = re.sub(r'[^\w\s]', '', text)
text = re.sub(r'\d+', '', text)
words = [w for w in text.split() if w not in STOPWORDS and len(w) > 2]
return " ".join([stemmer.stem(w) for w in words])
def find_column_by_code(df_columns: list, code: str) -> str | None:
for col in df_columns:
if code.lower() in col.lower():
return col
return None
def safe_get(row: pd.Series, df_columns: list, code: str, default: str = "") -> str:
actual_col = find_column_by_code(df_columns, code)
if actual_col and actual_col in row:
val = row[actual_col]
if pd.isna(val) or val is None:
return default
return str(val).strip()
return default
# CLASSIFICATION LOGIC
def classify_rule(job_text: str) -> dict:
clean = clean_text(job_text)
if not clean or len(clean) < 3:
return {"profile": "Tidak Diketahui", "confidence": 0.65, "method": "rule_based_fallback"}
for profile, keywords in KEYWORD_RULES.items():
if any(kw in clean for kw in keywords):
return {"profile": profile, "confidence": 0.65, "method": "rule_based"}
return None
def classify_ml(job_text: str, pipeline) -> dict:
clean = clean_text(job_text)
if not clean:
return {"profile": "Tidak Diketahui", "confidence": 0.0, "method": "ml_empty_input"}
try:
proba = pipeline.predict_proba([clean])[0]
max_conf = float(np.max(proba))
pred_class = pipeline.classes_[np.argmax(proba)]
method = "ml_fallback" if max_conf >= CONFIDENCE_THRESHOLD else "manual_review"
return {"profile": pred_class, "confidence": round(max_conf, 4), "method": method}
except Exception as e:
logger.error(f"ML_INFER_ERROR | job_text='{job_text[:50]}' | err={e}")
raise HTTPException(status_code=500, detail=f"ML inference failed: {str(e)}")
def extract_kemendik_text(row: pd.Series, df_columns: list) -> str:
f5b_col = find_column_by_code(df_columns, "f5b")
f5c_col = find_column_by_code(df_columns, "f5c")
f1101_col = find_column_by_code(df_columns, "f1101")
f1102_col = find_column_by_code(df_columns, "f1102")
f5b = clean_text(row.get(f5b_col, "") if f5b_col else "")
f1102 = clean_text(row.get(f1102_col, "") if f1102_col else "")
f5c_raw = str(row.get(f5c_col, "")).strip() if f5c_col else ""
f5c_code = re.match(r'^(\d+)', f5c_raw)
f5c_text = F5C_MAP.get(f5c_code.group(1), "") if f5c_code else ""
f1101_raw = str(row.get(f1101_col, "")).strip() if f1101_col else ""
f1101_code = re.match(r'^(\d+)', f1101_raw)
f1101_text = F1101_MAP.get(f1101_code.group(1), "") if f1101_code else ""
return " ".join(p for p in [f5b, f5c_text, f1101_text, f1102] if p).strip()
# SOURCE DETECTION
def detect_source(df: pd.DataFrame) -> str:
cols_lower = [c.lower().strip() for c in df.columns]
kemendik_codes = ["f5b", "f5c", "f8", "f1101", "nimhsmsmh", "nmmhsmsmh"]
for col in cols_lower:
if any(code in col for code in kemendik_codes):
return "kemendik"
if any("jabatan" in c for c in cols_lower):
return "internal_mif"
return "unknown"
# ROUTES
@app.get("/health")
def health_check():
return {"status": "healthy", "pipeline_loaded": app.state.pipeline is not None}
# RE-TRAINING ENDPOINTS
@app.post("/api/v1/retrain", dependencies=[Depends(verify_token)])
async def trigger_retrain(file: UploadFile = File(None)):
if STATUS_PATH.exists():
try:
current = json.loads(STATUS_PATH.read_text())
if current.get("stage") in ["started", "backup", "loading_data",
"preprocessing", "training", "evaluating", "promoting"]:
raise HTTPException(
status_code=409,
detail="Re-training sedang berjalan. Tunggu hingga selesai."
)
except HTTPException:
raise
except Exception:
pass
extra_csv_path = ""
if file and file.filename:
job_id = str(uuid.uuid4())[:8]
extra_csv_path = str(TEMP_DIR / f"extra_{job_id}.csv")
contents = await file.read()
Path(extra_csv_path).write_bytes(contents)
logger.info(f"Extra CSV disimpan: {extra_csv_path}")
STATUS_PATH.write_text(json.dumps({
"stage": "started",
"message": "Memulai proses re-training...",
"timestamp": __import__('datetime').datetime.now().isoformat(),
}, ensure_ascii=False))
cmd = [sys.executable, str(RETRAIN_WORKER)]
if extra_csv_path:
cmd.append(extra_csv_path)
try:
subprocess.Popen(
cmd,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
start_new_session=True
)
logger.info(f"Retrain subprocess spawned: {' '.join(cmd)}")
except Exception as e:
STATUS_PATH.write_text(json.dumps({
"stage": "failed",
"message": f"Gagal spawn subprocess: {str(e)}",
}, ensure_ascii=False))
raise HTTPException(status_code=500, detail=f"Gagal memulai training: {str(e)}")
return JSONResponse(content={"status": "started", "message": "Re-training dimulai di background."})
@app.get("/api/v1/retrain/status", dependencies=[Depends(verify_token)])
def retrain_status():
if not STATUS_PATH.exists():
return JSONResponse(content={"stage": "idle", "message": "Belum ada proses re-training."})
try:
data = json.loads(STATUS_PATH.read_text())
except Exception:
return JSONResponse(content={"stage": "unknown", "message": "Status tidak terbaca."})
terminal_stages = {"promoted", "rolled_back", "failed"}
if data.get("stage") in terminal_stages and not data.get("_reloaded"):
try:
app.state.pipeline = joblib.load(PIPELINE_PATH)
data["_reloaded"] = True
STATUS_PATH.write_text(json.dumps(data, indent=2, ensure_ascii=False))
logger.info(f"Pipeline di-reload setelah retrain (stage={data['stage']})")
except Exception as e:
logger.error(f"Gagal reload pipeline: {e}")
return JSONResponse(content=data)
@app.post("/api/v1/retrain/reload", dependencies=[Depends(verify_token)])
def reload_model():
try:
app.state.pipeline = joblib.load(PIPELINE_PATH)
logger.info("Pipeline di-reload secara manual.")
return JSONResponse(content={"status": "ok", "message": "Model berhasil di-reload."})
except Exception as e:
raise HTTPException(status_code=500, detail=f"Gagal reload model: {str(e)}")
@app.post("/api/v1/classify", dependencies=[Depends(verify_token)])
async def classify_tracer(file: UploadFile = File(...)):
if not file.filename:
raise HTTPException(status_code=400, detail="No file provided")
try:
contents = await file.read()
if file.filename.endswith(".xlsx") or file.filename.endswith(".xls"):
try:
df = pd.read_excel(io.BytesIO(contents), dtype=str, engine="openpyxl")
except ImportError:
logger.error("openpyxl not installed")
raise HTTPException(status_code=500, detail="Server misconfiguration: openpyxl missing")
except Exception as excel_err:
logger.error(f"Excel parsing error: {excel_err}")
try:
df = pd.read_csv(io.BytesIO(contents), sep=";", encoding="utf-8-sig", dtype=str)
except:
df = pd.read_csv(io.BytesIO(contents), sep=";", encoding="latin-1", dtype=str)
else:
try:
df = pd.read_csv(io.BytesIO(contents), sep=";", encoding="utf-8-sig", dtype=str)
except UnicodeDecodeError:
df = pd.read_csv(io.BytesIO(contents), sep=";", encoding="latin-1", dtype=str)
df.columns = df.columns.str.strip()
except Exception as e:
logger.error(f"File parsing error: {type(e).__name__}: {str(e)}")
raise HTTPException(status_code=400, detail=f"Failed to parse file: {type(e).__name__}: {str(e)[:200]}")
source_type = detect_source(df)
if source_type == "unknown":
raise HTTPException(status_code=400, detail="Unrecognized file format.")
if source_type == "internal_mif" and app.state.pipeline is None:
logger.error("ML pipeline not loaded")
raise HTTPException(status_code=503, detail="ML model not available.")
results = []
pipeline = app.state.pipeline
logger.info(f"Processing {len(df)} rows | Source: {source_type}")
find_col_cached = lambda p: next((c for c in df.columns if p.lower() in c.lower()), None)
col_f5a1 = find_col_cached("f5a1")
col_f5c = find_col_cached("f5c")
col_f1101 = find_col_cached("f1101")
col_f5b = find_col_cached("f5b")
col_f1102 = find_col_cached("f1102")
col_nim_kem = find_col_cached("nimhsmsmh")
col_nama_kem = find_col_cached("nmmhsmsmh")
col_tahun_kem = find_col_cached("tahun_lulus")
col_nim_int = find_col_cached("nim")
col_nama_int = find_col_cached("nama_lengkap")
col_tahun_int = find_col_cached("tahun_lulus")
for _, row in df.iterrows():
try:
def safe_str(val):
if pd.isna(val) or val is None: return ""
return str(val).strip()
if source_type == "kemendik":
nim = safe_str(row.get(col_nim_kem))
nama = safe_str(row.get(col_nama_kem))
tahun = safe_str(row.get(col_tahun_kem))
else:
nim = safe_str(row.get(col_nim_int))
nama = safe_str(row.get(col_nama_int))
tahun = safe_str(row.get(col_tahun_int))
if source_type == "kemendik":
job_text = extract_kemendik_text(row, df.columns.tolist())
raw_data_payload = {str(k): safe_str(v) for k, v in row.to_dict().items()}
raw_data_payload.update({
"F5a1": safe_str(row.get(col_f5a1)),
"F5c": safe_str(row.get(col_f5c)),
"F1101": safe_str(row.get(col_f1101)),
"F5b": safe_str(row.get(col_f5b)),
"F1102": safe_str(row.get(col_f1102))
})
results.append({
"nim": nim, "nama": nama, "tahun_lulus": tahun,
"job_text_raw": job_text, "source_type": "kemendik",
"predicted_profile": None, "confidence_score": None,
"classification_method": "dashboard_only", "status": "processed",
"raw_data": raw_data_payload
})
else:
col_jabatan = find_col_cached("jabatan")
col_perusahaan = find_col_cached("perusahaan")
col_deskripsi = find_col_cached("deskripsi")
job_text = clean_text(safe_str(row.get(col_jabatan))) if col_jabatan else ""
if not job_text and col_perusahaan and col_deskripsi:
job_text = clean_text(f"{safe_str(row.get(col_perusahaan))} {safe_str(row.get(col_deskripsi))}".strip())
elif not job_text and col_perusahaan:
job_text = clean_text(safe_str(row.get(col_perusahaan)))
rule_res = classify_rule(job_text)
res = rule_res or (classify_ml(job_text, pipeline) if pipeline else {"profile": "Non-IT", "confidence": 0.0, "method": "ml_unavailable"})
status = "auto_classified" if res["method"] in ["rule_based", "ml_fallback"] or res["profile"] == "Tidak Diketahui" else "needs_review"
raw_data_payload = {str(k): safe_str(v) for k, v in row.to_dict().items()}
results.append({
"nim": nim, "nama": nama, "tahun_lulus": tahun,
"job_text_raw": job_text, "source_type": "internal_mif",
"predicted_profile": res["profile"], "confidence_score": res["confidence"],
"classification_method": res["method"], "status": status,
"raw_data": raw_data_payload
})
except Exception as row_err:
logger.warning(f"Row error: {row_err}")
results.append({"nim": nim if 'nim' in locals() else "", "status": "failed", "error_detail": str(row_err)[:100]})
return JSONResponse(content={"status": "success", "total_rows": len(df), "processed_rows": len(results), "source_type": source_type, "results": results})
if __name__ == "__main__":
import uvicorn
uvicorn.run("app.main:app", host="127.0.0.1", port=8000, reload=True)

310
data/unused/main2.py Normal file
View File

@ -0,0 +1,310 @@
from fastapi import FastAPI, UploadFile, File, HTTPException, BackgroundTasks, Depends, Security
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
from fastapi.responses import JSONResponse
import pandas as pd
import numpy as np
import joblib
import re
import io
import json
import uuid
import shutil
import subprocess
import sys
import os
import logging
from contextlib import asynccontextmanager
from pathlib import Path
from typing import List, Dict, Any
from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
from filelock import FileLock, Timeout # PENAMBAHAN KUNCI MUTEX
# KONFIGURASI & KONSTANTA
BASE_DIR = Path(__file__).parent.parent
ML_DIR = BASE_DIR / "ml_assets"
PIPELINE_PATH = ML_DIR / "ml_pipeline_internal.pkl"
STATUS_PATH = ML_DIR / "retrain_status.json"
LOCK_PATH = ML_DIR / "retrain_status.lock"
RETRAIN_WORKER = Path(__file__).parent / "retrain_worker.py"
TEMP_DIR = ML_DIR / "tmp"
TEMP_DIR.mkdir(parents=True, exist_ok=True)
CONFIDENCE_THRESHOLD = 0.50
TARGET_CLASSES = ["Programmer", "Data Analyst", "Wirausaha Informatika", "Non-IT"]
KEYWORD_RULES = {
"Programmer": ["programmer", "developer", "engineer", "fullstack", "backend", "frontend", "mobile", "android", "ios", "software", "web dev", "coding", "it staff", "teknisi", "sistem informasi", "application", "network", "devops", "qa", "tester", "ui", "ux", "swe"],
"Data Analyst": ["data analyst", "analis data", "data science", "business analyst", "research", "statistik", "bi analyst", "reporting", "database", "sql", "etl", "data engineer", "big data", "analyst", "data mining", "machine learning", "data visual", "power bi", "tableau", "looker", "business intelligence", "bi developer", "data warehouse"],
"Wirausaha Informatika": ["founder", "owner", "ceo", "wiraswasta", "startup", "freelance", "freelancer", "wirausaha", "bisnis", "usaha mandiri", "konsultan", "co founder", "entrepreneur", "self employed", "owner toko", "usaha", "dagang online", "tokopedia", "shopee", "dropship", "reseller"]
}
F5C_MAP = {"1": "founder owner wirausaha startup", "2": "co-founder partner wirausaha", "3": "staff karyawan pegawai", "4": "freelance kerja lepas lepasan"}
F1101_MAP = {"1": "instansi pemerintah dinas kementerian", "2": "non-profit lsm yayasan", "3": "perusahaan swasta corporate", "4": "wiraswasta usaha mandiri", "6": "bumn bumd pemerintah", "7": "multilateral internasional"}
logging.basicConfig(level=logging.INFO, format="%(asctime)s | %(levelname)s | %(message)s")
logger = logging.getLogger("tracer_worker")
stemmer = StemmerFactory().create_stemmer()
STOPWORDS = {
"yang", "di", "ke", "dari", "dan", "atau", "dengan", "untuk", "pada", "dalam",
"adalah", "ini", "itu", "tidak", "juga", "sudah", "akan", "bisa", "ada", "oleh",
"karena", "secara", "serta", "sebagai", "bagi", "telah", "maka", "namun", "sehingga",
"jika", "agar", "ketika", "saat", "sebelum", "sesudah", "hingga", "sampai", "antara",
"sekitar", "hanya", "saja", "belum", "masih", "lagi", "pun", "justru", "walaupun",
"meskipun", "bahkan", "cukup", "sangat", "paling", "lebih", "kurang", "lain",
"macam", "cara", "hal", "tentang", "mengenai", "terhadap", "kepada", "menuju",
"kecuali", "selain", "tanpa", "demi", "guna", "khususnya", "umumnya", "kebanyakan",
"sebagian", "beberapa", "semua", "setiap", "tiap", "satu", "dua", "tiga", "empat",
"lima", "enam", "tujuh", "delapan", "sembilan", "sepuluh", "ratus", "ribu", "juta"
}
@asynccontextmanager
async def lifespan(app: FastAPI):
try:
app.state.pipeline = joblib.load(PIPELINE_PATH)
logger.info(f" ML Pipeline loaded successfully from {PIPELINE_PATH}")
except Exception as e:
logger.error(f" Failed to load ML pipeline: {e}")
app.state.pipeline = None
yield
app = FastAPI(title="Tracer Study Classification Worker", lifespan=lifespan)
security = HTTPBearer()
EXPECTED_TOKEN = os.getenv("FASTAPI_SECRET_KEY", "")
def verify_token(credentials: HTTPAuthorizationCredentials = Security(security)):
if not EXPECTED_TOKEN: return credentials.credentials
if credentials.credentials != EXPECTED_TOKEN:
raise HTTPException(status_code=401, detail="Unauthorized - Invalid Token")
return credentials.credentials
def clean_text(text: str) -> str:
# Stemmer HANYA berjalan pada saat ada permintaan inferensi baru
if pd.isna(text) or not isinstance(text, str): return ""
text = text.lower().strip()
if text in {"nan", "none", "null", "-", "0", "", "tidak diisi"}: return ""
text = text.replace('-', ' ').replace('/', ' ').replace('_', ' ').replace(',', ' ')
text = re.sub(r'[^\w\s]', '', text)
text = re.sub(r'\d+', '', text)
words = [w for w in text.split() if w not in STOPWORDS and len(w) > 2]
return " ".join([stemmer.stem(w) for w in words])
def classify_rule(job_text: str) -> dict:
clean = clean_text(job_text)
if not clean or len(clean) < 3:
return {"profile": "Tidak Diketahui", "confidence": 0.65, "method": "rule_based_fallback"}
for profile, keywords in KEYWORD_RULES.items():
if any(kw in clean for kw in keywords):
return {"profile": profile, "confidence": 0.65, "method": "rule_based"}
return None
def classify_ml(job_text: str, pipeline) -> dict:
clean = clean_text(job_text)
if not clean: return {"profile": "Tidak Diketahui", "confidence": 0.0, "method": "ml_empty_input"}
try:
proba = pipeline.predict_proba([clean])[0]
max_conf = float(np.max(proba))
pred_class = pipeline.classes_[np.argmax(proba)]
method = "ml_fallback" if max_conf >= CONFIDENCE_THRESHOLD else "manual_review"
return {"profile": pred_class, "confidence": round(max_conf, 4), "method": method}
except Exception as e:
logger.error(f"ML_INFER_ERROR | job_text='{job_text[:50]}' | err={e}")
raise HTTPException(status_code=500, detail=f"ML inference failed: {str(e)}")
def detect_source(df: pd.DataFrame) -> str:
cols_lower = [c.lower().strip() for c in df.columns]
kemendik_codes = ["f5b", "f5c", "f8", "f1101", "nimhsmsmh", "nmmhsmsmh"]
for col in cols_lower:
if any(code in col for code in kemendik_codes): return "kemendik"
if any("jabatan" in c for c in cols_lower): return "internal_mif"
return "unknown"
@app.get("/health")
def health_check():
return {"status": "healthy", "pipeline_loaded": app.state.pipeline is not None}
@app.post("/api/v1/retrain", dependencies=[Depends(verify_token)])
async def trigger_retrain(file: UploadFile = File(None)):
lock = FileLock(LOCK_PATH, timeout=2) # Timeout 2 detik jika bentrok
try:
with lock:
if STATUS_PATH.exists():
try:
current = json.loads(STATUS_PATH.read_text())
if current.get("stage") in ["started", "backup", "loading_data", "preprocessing", "training", "evaluating", "promoting"]:
raise HTTPException(status_code=409, detail="Re-training sedang berjalan. Tunggu hingga selesai.")
except json.JSONDecodeError:
pass
extra_csv_path = ""
if file and file.filename:
job_id = str(uuid.uuid4())[:8]
extra_csv_path = str(TEMP_DIR / f"extra_{job_id}.csv")
contents = await file.read()
Path(extra_csv_path).write_bytes(contents)
logger.info(f"Extra CSV disimpan: {extra_csv_path}")
STATUS_PATH.write_text(json.dumps({
"stage": "started",
"message": "Memulai proses re-training...",
"timestamp": __import__('datetime').datetime.now().isoformat(),
}, ensure_ascii=False))
cmd = [sys.executable, str(RETRAIN_WORKER)]
if extra_csv_path: cmd.append(extra_csv_path)
subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True)
logger.info(f"Retrain subprocess spawned: {' '.join(cmd)}")
return JSONResponse(content={"status": "started", "message": "Re-training dimulai di background."})
except Timeout:
raise HTTPException(status_code=429, detail="Sistem sibuk. Permintaan retraining lain sedang diproses.")
except Exception as e:
if not isinstance(e, HTTPException):
raise HTTPException(status_code=500, detail=f"Gagal memulai training: {str(e)}")
raise e
@app.get("/api/v1/retrain/status", dependencies=[Depends(verify_token)])
def retrain_status():
if not STATUS_PATH.exists(): return JSONResponse(content={"stage": "idle", "message": "Belum ada proses re-training."})
try:
lock = FileLock(LOCK_PATH, timeout=2)
with lock:
data = json.loads(STATUS_PATH.read_text())
terminal_stages = {"promoted", "rolled_back", "failed"}
if data.get("stage") in terminal_stages and not data.get("_reloaded"):
try:
app.state.pipeline = joblib.load(PIPELINE_PATH)
data["_reloaded"] = True
STATUS_PATH.write_text(json.dumps(data, indent=2, ensure_ascii=False))
logger.info("Pipeline di-reload setelah retrain selesai.")
except Exception as e:
logger.error(f"Gagal reload pipeline: {e}")
return JSONResponse(content=data)
except Timeout:
return JSONResponse(status_code=429, content={"stage": "locked", "message": "Status sedang diupdate sistem."})
except Exception:
return JSONResponse(content={"stage": "unknown", "message": "Status tidak terbaca."})
@app.post("/api/v1/classify", dependencies=[Depends(verify_token)])
async def classify_tracer(file: UploadFile = File(...)):
if not file.filename:
raise HTTPException(status_code=400, detail="No file provided")
try:
contents = await file.read()
if file.filename.endswith(".xlsx") or file.filename.endswith(".xls"):
try:
df = pd.read_excel(io.BytesIO(contents), dtype=str, engine="openpyxl")
except ImportError:
logger.error("openpyxl not installed")
raise HTTPException(status_code=500, detail="Server misconfiguration: openpyxl missing")
except Exception as excel_err:
logger.error(f"Excel parsing error: {excel_err}")
try:
df = pd.read_csv(io.BytesIO(contents), sep=";", encoding="utf-8-sig", dtype=str)
except:
df = pd.read_csv(io.BytesIO(contents), sep=";", encoding="latin-1", dtype=str)
else:
try:
df = pd.read_csv(io.BytesIO(contents), sep=";", encoding="utf-8-sig", dtype=str)
except UnicodeDecodeError:
df = pd.read_csv(io.BytesIO(contents), sep=";", encoding="latin-1", dtype=str)
df.columns = df.columns.str.strip()
except Exception as e:
logger.error(f"File parsing error: {type(e).__name__}: {str(e)}")
raise HTTPException(status_code=400, detail=f"Failed to parse file: {type(e).__name__}: {str(e)[:200]}")
source_type = detect_source(df)
if source_type == "unknown":
raise HTTPException(status_code=400, detail="Unrecognized file format.")
if source_type == "internal_mif" and app.state.pipeline is None:
logger.error("ML pipeline not loaded")
raise HTTPException(status_code=503, detail="ML model not available.")
results = []
pipeline = app.state.pipeline
logger.info(f"Processing {len(df)} rows | Source: {source_type}")
find_col_cached = lambda p: next((c for c in df.columns if p.lower() in c.lower()), None)
col_f5a1 = find_col_cached("f5a1")
col_f5c = find_col_cached("f5c")
col_f1101 = find_col_cached("f1101")
col_f5b = find_col_cached("f5b")
col_f1102 = find_col_cached("f1102")
col_nim_kem = find_col_cached("nimhsmsmh")
col_nama_kem = find_col_cached("nmmhsmsmh")
col_tahun_kem = find_col_cached("tahun_lulus")
col_nim_int = find_col_cached("nim")
col_nama_int = find_col_cached("nama_lengkap")
col_tahun_int = find_col_cached("tahun_lulus")
for _, row in df.iterrows():
try:
def safe_str(val):
if pd.isna(val) or val is None: return ""
return str(val).strip()
if source_type == "kemendik":
nim = safe_str(row.get(col_nim_kem))
nama = safe_str(row.get(col_nama_kem))
tahun = safe_str(row.get(col_tahun_kem))
else:
nim = safe_str(row.get(col_nim_int))
nama = safe_str(row.get(col_nama_int))
tahun = safe_str(row.get(col_tahun_int))
if source_type == "kemendik":
job_text = extract_kemendik_text(row, df.columns.tolist())
raw_data_payload = {str(k): safe_str(v) for k, v in row.to_dict().items()}
raw_data_payload.update({
"F5a1": safe_str(row.get(col_f5a1)),
"F5c": safe_str(row.get(col_f5c)),
"F1101": safe_str(row.get(col_f1101)),
"F5b": safe_str(row.get(col_f5b)),
"F1102": safe_str(row.get(col_f1102))
})
results.append({
"nim": nim, "nama": nama, "tahun_lulus": tahun,
"job_text_raw": job_text, "source_type": "kemendik",
"predicted_profile": None, "confidence_score": None,
"classification_method": "dashboard_only", "status": "processed",
"raw_data": raw_data_payload
})
else:
col_jabatan = find_col_cached("jabatan")
col_perusahaan = find_col_cached("perusahaan")
col_deskripsi = find_col_cached("deskripsi")
job_text = clean_text(safe_str(row.get(col_jabatan))) if col_jabatan else ""
if not job_text and col_perusahaan and col_deskripsi:
job_text = clean_text(f"{safe_str(row.get(col_perusahaan))} {safe_str(row.get(col_deskripsi))}".strip())
elif not job_text and col_perusahaan:
job_text = clean_text(safe_str(row.get(col_perusahaan)))
rule_res = classify_rule(job_text)
res = rule_res or (classify_ml(job_text, pipeline) if pipeline else {"profile": "Non-IT", "confidence": 0.0, "method": "ml_unavailable"})
status = "auto_classified" if res["method"] in ["rule_based", "ml_fallback"] or res["profile"] == "Tidak Diketahui" else "needs_review"
raw_data_payload = {str(k): safe_str(v) for k, v in row.to_dict().items()}
results.append({
"nim": nim, "nama": nama, "tahun_lulus": tahun,
"job_text_raw": job_text, "source_type": "internal_mif",
"predicted_profile": res["profile"], "confidence_score": res["confidence"],
"classification_method": res["method"], "status": status,
"raw_data": raw_data_payload
})
except Exception as row_err:
logger.warning(f"Row error: {row_err}")
results.append({"nim": nim if 'nim' in locals() else "", "status": "failed", "error_detail": str(row_err)[:100]})
return JSONResponse(content={"status": "success", "total_rows": len(df), "processed_rows": len(results), "source_type": source_type, "results": results})
if __name__ == "__main__":
import uvicorn
uvicorn.run("app.main:app", host="127.0.0.1", port=8000, reload=True)

20
data/prepare_corpus_3.py → data/unused/prepare_corpus_3.py
View File

@ -4,6 +4,10 @@ import re
import sys import sys
from pathlib import Path from pathlib import Path
from sklearn.model_selection import train_test_split from sklearn.model_selection import train_test_split
from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
factory = StemmerFactory()
stemmer = factory.create_stemmer()
BASE_DIR = Path(__file__).parent BASE_DIR = Path(__file__).parent
DATA_DIR = BASE_DIR DATA_DIR = BASE_DIR
@ -57,7 +61,7 @@ def clean_text(text: str) -> str:
text = re.sub(r'[^\w\s]', '', text) text = re.sub(r'[^\w\s]', '', text)
text = re.sub(r'\d+', '', text) text = re.sub(r'\d+', '', text)
tokens = [w for w in text.split() if w not in COMPANY_STOPWORDS and len(w) >= 3] tokens = [w for w in text.split() if w not in COMPANY_STOPWORDS and len(w) >= 3]
return " ".join(tokens) return " ".join([stemmer.stem(w) for w in tokens])
def is_likely_name(text: str, full_name: str) -> bool: def is_likely_name(text: str, full_name: str) -> bool:
if not text or not full_name: return False if not text or not full_name: return False
@ -79,11 +83,21 @@ def main():
df.columns = df.columns.str.strip() df.columns = df.columns.str.strip()
col_nim = find_col(df, ["nim"]) col_nim = find_col(df, ["nim"])
col_nama = find_col(df, ["nama", "lengkap"]) col_nama = find_col(df, ["nama", "lengkap"])
col_jabatan = find_col(df, ["jabatan"]) col_jab_lama = find_col(df, ["jabatan"])
col_jab_baru = find_col(df, ["jabatan_terupdate"])
col_klasifikasi = find_col(df, ["klasifikasi"]) col_klasifikasi = find_col(df, ["klasifikasi"])
col_status = find_col(df, ["status", "kerja"]) col_status = find_col(df, ["status", "kerja"])
if not all([col_nim, col_nama, col_jabatan]):
if not all([col_nim, col_nama, col_jab_lama]):
print(" Kolom esensial tidak ditemukan"); print(df.columns.tolist()[:10]); sys.exit(1) print(" Kolom esensial tidak ditemukan"); print(df.columns.tolist()[:10]); sys.exit(1)
if col_jab_baru:
mask_empty = df[col_jab_baru].isna() | df[col_jab_baru].astype(str).str.strip().str.lower().isin(["", "-", "0", "nan", "none", "null", "tidak diisi", "tidak diketahui"])
df["jabatan_final"] = df[col_jab_baru].where(~mask_empty, df[col_jab_lama])
else:
df["jabatan_final"] = df[col_jab_lama]
col_jabatan = "jabatan_final"
if col_status: if col_status:
blacklist = ["tidah diketahui", "tidak bekerja", "pelajar", "melanjutkan pendidikan", "nan", ""] blacklist = ["tidah diketahui", "tidak bekerja", "pelajar", "melanjutkan pendidikan", "nan", ""]
mask = ~df[col_status].str.lower().str.strip().isin(blacklist) mask = ~df[col_status].str.lower().str.strip().isin(blacklist)

152
data/unused/prepare_corpus_4.py Normal file
View File

@ -0,0 +1,152 @@
import pandas as pd
import numpy as np
import re
import sys
from pathlib import Path
from sklearn.model_selection import train_test_split
from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
factory = StemmerFactory()
stemmer = factory.create_stemmer()
BASE_DIR = Path(__file__).parent
DATA_DIR = BASE_DIR
OUTPUT_DIR = DATA_DIR / "processed"
OUTPUT_DIR.mkdir(exist_ok=True)
FILE_INTERNAL = DATA_DIR / "ts_internal_mif.xlsx"
SEPARATOR = ";"
ENCODING = "utf-8-sig"
TARGET_CLASSES = ["Programmer", "Data Analyst", "Wirausaha Informatika", "Non-IT"]
KLASIFIKASI_MAP = {
"Programmer": "Programmer",
"Data Analyst": "Data Analyst",
"Wirausaha IT": "Wirausaha Informatika",
"Wirausaha": "Wirausaha Informatika",
"Non-IT": "Non-IT",
"Infokom": None, "Pelajar": None, "Tidak Bekerja": None, "TIdak diketahui": None
}
KEYWORD_RULES = {
"Programmer": ["programmer", "developer", "engineer", "fullstack", "backend", "frontend", "mobile", "android", "ios", "software", "web dev", "coding", "it staff", "teknisi", "sistem informasi", "application", "network", "devops", "qa", "tester", "ui", "ux", "swe"],
"Data Analyst": ["data analyst", "analis data", "data science", "business analyst", "research", "statistik", "bi analyst", "reporting", "database", "sql", "etl", "data engineer", "big data", "analyst", "data mining", "machine learning", "data visual", "power bi", "tableau", "looker", "business intelligence", "bi developer", "data warehouse"],
"Wirausaha Informatika": ["founder", "owner", "ceo", "wiraswasta", "startup", "freelance", "freelancer", "wirausaha", "bisnis", "usaha mandiri", "konsultan", "co founder", "entrepreneur", "self employed", "owner toko", "usaha", "dagang online", "tokopedia", "shopee", "dropship", "reseller"]
}
COMPANY_STOPWORDS = {
"pt", "cv", "ud", "tbk", "persero", "corp", "inc", "ltd", "koperasi", "bumn", "bumd",
"dinas", "kantor", "pemkab", "pemprov", "politeknik", "universitas", "sekolah", "sma", "smk", "sd",
"bank", "bpr", "rs", "rumah sakit", "klinik", "apotek", "hotel", "restoran", "cafe", "toko", "konter",
"foundation", "yayasan", "perkumpulan", "organisasi", "agency", "studio", "consulting", "group", "holding"
}
def load_data_file(file_path: str) -> pd.DataFrame:
ext = Path(file_path).suffix.lower()
if ext == '.csv':
try: return pd.read_csv(file_path, sep=SEPARATOR, encoding=ENCODING, dtype=str, on_bad_lines='skip', engine='python')
except UnicodeDecodeError: return pd.read_csv(file_path, sep=SEPARATOR, encoding='latin1', dtype=str, on_bad_lines='skip', engine='python')
elif ext == '.xlsx':
return pd.read_excel(file_path, dtype=str)
else:
raise ValueError(f"Format tidak didukung: {ext}")
def find_col(df: pd.DataFrame, keywords: list) -> str | None:
for col in df.columns:
col_clean = str(col).strip().lower()
if any(k.strip().lower() in col_clean for k in keywords): return col
return None
def clean_text(text: str) -> str:
if pd.isna(text) or str(text).strip().lower() in ["nan", "none", "null", "-", "0", "", "tidak diisi", "tidak diketahui"]: return ""
text = str(text).strip().lower()
text = re.sub(r'^\d+\s*[-:/]\s*', '', text)
text = text.replace('-', ' ').replace('/', ' ').replace('_', ' ').replace(',', ' ')
text = re.sub(r'[^\w\s]', '', text)
text = re.sub(r'\d+', '', text)
tokens = [w for w in text.split() if w not in COMPANY_STOPWORDS and len(w) >= 3]
return " ".join([stemmer.stem(w) for w in tokens])
def is_likely_name(text: str, full_name: str) -> bool:
if not text or not full_name: return False
text_clean = re.sub(r'[^\w\s]', '', text.lower())
name_clean = re.sub(r'[^\w\s]', '', str(full_name).lower())
text_parts = set(text_clean.split())
name_parts = set(name_clean.split())
if len(text_parts) == 0: return False
# Logika yang lebih ketat: Cek apakah input hanya berisi komponen nama (indikasi bocor nama)
if len(text_parts) <= 3 and text_parts.issubset(name_parts):
return True
return False
def classify_rule_based(text: str) -> str:
if not text or len(text) < 3: return "Non-IT"
for profile in ["Programmer", "Data Analyst", "Wirausaha Informatika"]:
if any(kw in text for kw in KEYWORD_RULES[profile]): return profile
return "Non-IT"
def main():
if not FILE_INTERNAL.exists():
print(f"Berkas tidak ditemukan: {FILE_INTERNAL}"); sys.exit(1)
print("[1/4] Memuat & Membersihkan Data (Strict Cleaning)...")
df = load_data_file(str(FILE_INTERNAL))
df.columns = df.columns.str.strip()
col_nim = find_col(df, ["nim"])
col_nama = find_col(df, ["nama", "lengkap"])
col_jab_lama = find_col(df, ["jabatan"])
col_jab_baru = find_col(df, ["jabatan_terupdate"])
col_klasifikasi = find_col(df, ["klasifikasi"])
col_status = find_col(df, ["status", "kerja"])
if not all([col_nim, col_nama, col_jab_lama]):
print(" Kolom esensial tidak ditemukan"); print(df.columns.tolist()[:10]); sys.exit(1)
if col_jab_baru:
mask_empty = df[col_jab_baru].isna() | df[col_jab_baru].astype(str).str.strip().str.lower().isin(["", "-", "0", "nan", "none", "null", "tidak diisi", "tidak diketahui"])
df["jabatan_final"] = df[col_jab_baru].where(~mask_empty, df[col_jab_lama])
else:
df["jabatan_final"] = df[col_jab_lama]
col_jabatan = "jabatan_final"
if col_status:
blacklist = ["tidah diketahui", "tidak bekerja", "pelajar", "melanjutkan pendidikan", "nan", ""]
mask = ~df[col_status].str.lower().str.strip().isin(blacklist)
df = df[mask].copy()
# Stemming berat terjadi HANYA di sini
df["job_text_raw"] = df[col_jabatan].apply(clean_text)
if col_klasifikasi:
fallback_map = {"Programmer": "programmer developer", "Data Analyst": "data analyst", "Wirausaha IT": "wirausaha founder", "Wirausaha": "wirausaha founder", "Non IT": "staff admin", "Infokom": "it staff teknisi", "TIdah diketahui": "", "Pelajar": "", "Tidak Bekerja": ""}
empty_mask = df["job_text_raw"] == ""
if empty_mask.any(): df.loc[empty_mask, "job_text_raw"] = df.loc[empty_mask, col_klasifikasi].map(fallback_map).fillna("")
if col_nama:
name_leak_mask = df.apply(lambda row: is_likely_name(row["job_text_raw"], row[col_nama]), axis=1)
leaked_count = name_leak_mask.sum()
if leaked_count > 0: print(f" Mengabaikan {leaked_count} baris yang terindikasi mengandung nama pribadi.")
df = df[~name_leak_mask].copy()
if col_klasifikasi:
df["label"] = df[col_klasifikasi].str.strip().map(KLASIFIKASI_MAP)
missing = df["label"].isna()
if missing.any(): df.loc[missing, "label"] = df.loc[missing, "job_text_raw"].apply(classify_rule_based)
else: df["label"] = df["job_text_raw"].apply(classify_rule_based)
df = df[df["job_text_raw"].str.len() >= 3].copy()
result = df[[col_nim, col_jabatan, "job_text_raw", "label"]].rename(columns={col_nim: "nim"}).dropna(subset=["nim", "label"]).drop_duplicates(subset=["nim"], keep="first")
print(f"Memuat {len(result)} data valid (berdasarkan teks pekerjaan)\n")
cols_out = ["nim", "job_text_raw", "label"]
# Kita tetap simpan data uji ke CSV (jika butuh untuk sanity check statis), tapi tidak akan digunakan
# oleh retrain_worker secara buta lagi.
min_count = result["label"].value_counts().min()
if min_count < 2: train_df, test_df = train_test_split(result, test_size=0.30, random_state=42)
else: train_df, test_df = train_test_split(result, test_size=0.30, stratify=result["label"], random_state=42)
train_df[cols_out].to_csv(OUTPUT_DIR / "training_corpus_4.csv", index=False, sep=";", encoding=ENCODING)
test_df[cols_out].to_csv(OUTPUT_DIR / "test_set_4.csv", index=False, sep=";", encoding=ENCODING)
print("DATA PELATIHAN (TRAINING SET):")
print(train_df["label"].value_counts().to_string())
print(f"\nData berhasil disimpan ke {OUTPUT_DIR}")
if __name__ == "__main__":
main()

394
data/unused/retrain_worker.py Normal file
View File

@ -0,0 +1,394 @@
#!/usr/bin/env python3
"""
TRACER STUDY - RE-TRAINING WORKER (SUBPROCESS)
Dipanggil oleh FastAPI sebagai background subprocess.
Alur:
1. Backup pkl lama → pkl.bak
2. Merge corpus asli + data manual_override baru
3. Train model baru (candidate)
4. Evaluasi: bandingkan weighted F1-score baru vs lama
5. Promote jika lebih baik; rollback jika tidak
6. Update retrain_status.json di setiap tahap
"""
import sys
import json
import shutil
import logging
import warnings
import re
import os
import pandas as pd
import numpy as np
import joblib
from pathlib import Path
from datetime import datetime
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import StratifiedKFold, train_test_split
from sklearn.metrics import classification_report
from sklearn.pipeline import Pipeline
from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
warnings.filterwarnings("ignore")
# ──────────────────────────────────────────────────────────────
# KONFIGURASI
# ──────────────────────────────────────────────────────────────
BASE_DIR = Path(__file__).parent.parent
ML_DIR = BASE_DIR / "ml_assets"
DATA_DIR = BASE_DIR.parent / "data" / "processed"
PIPELINE_PATH = ML_DIR / "ml_pipeline_internal.pkl"
PIPELINE_BAK_PATH = ML_DIR / "ml_pipeline_internal.pkl.bak"
CANDIDATE_PATH = ML_DIR / "ml_pipeline_candidate.pkl"
METRICS_PATH = ML_DIR / "metrics_internal_only.json"
STATUS_PATH = ML_DIR / "retrain_status.json"
CORPUS_PATH = DATA_DIR / "training_corpus_3.csv"
TEST_PATH = DATA_DIR / "test_set_3.csv"
TARGET_CLASSES = ["Programmer", "Data Analyst", "Wirausaha Informatika", "Non-IT"]
# Threshold: model baru harus lebih baik minimal MIN_IMPROVEMENT dari model lama
MIN_IMPROVEMENT_THRESHOLD = 0.01 # 1% weighted F1
MAX_REGRESSION_ALLOWED = 0.02 # Toleransi: model baru boleh lebih buruk max 2% (di luar ini = rollback keras)
STOPWORDS = {
"yang", "di", "ke", "dari", "dan", "atau", "dengan", "untuk", "pada", "dalam",
"adalah", "ini", "itu", "tidak", "juga", "sudah", "akan", "bisa", "ada", "oleh",
"karena", "secara", "serta", "sebagai", "bagi", "telah", "maka", "namun", "sehingga",
"jika", "agar", "ketika", "saat", "sebelum", "sesudah", "hingga", "sampai", "antara",
"sekitar", "hanya", "saja", "belum", "masih", "lagi", "pun", "justru", "walaupun",
"meskipun", "bahkan", "cukup", "sangat", "paling", "lebih", "kurang", "lain",
"macam", "cara", "hal", "tentang", "mengenai", "terhadap", "kepada", "menuju",
"kecuali", "selain", "tanpa", "demi", "guna", "khususnya", "umumnya", "kebanyakan",
"sebagian", "beberapa", "semua", "setiap", "tiap", "satu", "dua", "tiga", "empat",
"lima", "enam", "tujuh", "delapan", "sembilan", "sepuluh", "ratus", "ribu", "juta"
}
stemmer = StemmerFactory().create_stemmer()
# ──────────────────────────────────────────────────────────────
# LOGGING
# ──────────────────────────────────────────────────────────────
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s | %(levelname)s | %(message)s",
handlers=[logging.StreamHandler(sys.stdout)]
)
logger = logging.getLogger("retrain_worker")
# ──────────────────────────────────────────────────────────────
# STATUS WRITER
# ──────────────────────────────────────────────────────────────
def write_status(stage: str, message: str, extra: dict = None):
payload = {
"stage": stage,
"message": message,
"timestamp": datetime.now().isoformat(),
}
if extra:
payload.update(extra)
STATUS_PATH.write_text(json.dumps(payload, indent=2, ensure_ascii=False))
logger.info(f"[{stage}] {message}")
# ──────────────────────────────────────────────────────────────
# PREPROCESSING — IDENTIK dengan main.py
# ──────────────────────────────────────────────────────────────
def preprocess_text(text: str) -> str:
if pd.isna(text) or not isinstance(text, str):
return ""
text = text.lower().strip()
if text in {"nan", "none", "null", "-", "0", "", "tidak diisi"}:
return ""
text = text.replace('-', ' ').replace('/', ' ').replace('_', ' ').replace(',', ' ')
text = re.sub(r'[^\w\s]', '', text)
text = re.sub(r'\d+', '', text)
words = [w for w in text.split() if w not in STOPWORDS and len(w) > 2]
return " ".join([stemmer.stem(w) for w in words])
# ──────────────────────────────────────────────────────────────
# EVALUASI MODEL — weighted F1 pada hold-out test
# ──────────────────────────────────────────────────────────────
def evaluate_model(model, X_test: pd.Series, y_test: pd.Series) -> dict:
"""
Evaluasi model pada hold-out test set.
Return: dict berisi weighted F1, accuracy, dan per-class metrics.
"""
if len(X_test) == 0:
return {"weighted_f1": 0.0, "accuracy": 0.0, "per_class": {}}
y_pred = model.predict(X_test)
report = classification_report(
y_test, y_pred,
output_dict=True,
zero_division=0,
labels=TARGET_CLASSES
)
return {
"weighted_f1": round(report.get("weighted avg", {}).get("f1-score", 0.0), 4),
"accuracy": round(report.get("accuracy", 0.0), 4),
"per_class": {
cls: {
"precision": round(report.get(cls, {}).get("precision", 0.0), 3),
"recall": round(report.get(cls, {}).get("recall", 0.0), 3),
"f1": round(report.get(cls, {}).get("f1-score", 0.0), 3),
"support": int(report.get(cls, {}).get("support", 0)),
}
for cls in TARGET_CLASSES
}
}
# ──────────────────────────────────────────────────────────────
# AMBIL BASELINE DARI METRICS JSON
# ──────────────────────────────────────────────────────────────
def get_baseline_f1() -> float:
"""
Baca weighted F1 model lama dari metrics_internal_only.json.
Fallback ke 0 jika file tidak ada.
"""
try:
if METRICS_PATH.exists():
m = json.loads(METRICS_PATH.read_text())
return float(m.get("hold_out_test", {}).get("weighted avg", {}).get("f1-score", 0.0))
except Exception as e:
logger.warning(f"Gagal baca baseline metrics: {e}")
return 0.0
# ──────────────────────────────────────────────────────────────
# ROLLBACK
# ──────────────────────────────────────────────────────────────
def do_rollback(reason: str, old_f1: float, new_f1: float):
"""Kembalikan pkl aktif ke backup."""
# Bersihkan candidate jika ada
if CANDIDATE_PATH.exists():
CANDIDATE_PATH.unlink()
# Restore dari backup
if PIPELINE_BAK_PATH.exists():
shutil.copy2(PIPELINE_BAK_PATH, PIPELINE_PATH)
logger.info(f"Rollback berhasil: pkl lama dipulihkan dari .bak")
else:
logger.warning("File .bak tidak ditemukan, pkl aktif dibiarkan.")
write_status(
stage="rolled_back",
message=f"Model lama dipertahankan. {reason}",
extra={
"result": "rolled_back",
"reason": reason,
"old_f1": old_f1,
"new_f1": new_f1,
}
)
# ──────────────────────────────────────────────────────────────
# MAIN
# ──────────────────────────────────────────────────────────────
def main(extra_csv_path: str = None):
write_status("started", "Worker dimulai")
# ── STEP 1: BACKUP ──────────────────────────────────────
write_status("backup", "Membuat backup model lama...")
if PIPELINE_PATH.exists():
shutil.copy2(PIPELINE_PATH, PIPELINE_BAK_PATH)
logger.info(f"Backup tersimpan: {PIPELINE_BAK_PATH}")
else:
write_status("failed", "File pkl aktif tidak ditemukan, tidak bisa backup.")
sys.exit(1)
baseline_f1 = get_baseline_f1()
logger.info(f"Baseline weighted F1 (model lama): {baseline_f1:.4f}")
try:
# ── STEP 2: LOAD & MERGE DATA ────────────────────────
write_status("loading_data", "Memuat dan menggabungkan data training...")
if not CORPUS_PATH.exists():
raise FileNotFoundError(f"Corpus asli tidak ditemukan: {CORPUS_PATH}")
df_corpus = pd.read_csv(CORPUS_PATH, sep=";", dtype=str).dropna(subset=["job_text_raw", "label"])
logger.info(f"Corpus asli: {len(df_corpus)} baris")
df_extra = pd.DataFrame()
if extra_csv_path and Path(extra_csv_path).exists():
df_extra = pd.read_csv(extra_csv_path, sep=";", dtype=str).dropna(subset=["job_text_raw", "label"])
# Validasi label
valid_labels = set(TARGET_CLASSES)
df_extra = df_extra[df_extra["label"].isin(valid_labels)]
logger.info(f"Data manual_override: {len(df_extra)} baris valid")
if len(df_extra) > 0:
df_all = pd.concat([df_corpus, df_extra], ignore_index=True)
# Deduplicate: preferensikan data extra (manual override) jika job_text_raw sama
df_all = df_all.drop_duplicates(subset=["job_text_raw"], keep="last")
else:
df_all = df_corpus.copy()
logger.info("Tidak ada data tambahan — menggunakan corpus asli saja")
logger.info(f"Total setelah merge & deduplicate: {len(df_all)} baris")
if len(df_all) < 20:
raise ValueError(f"Data terlalu sedikit untuk training: {len(df_all)} baris (minimum 20)")
# ── STEP 3: PREPROCESSING ────────────────────────────
write_status("preprocessing", "Preprocessing teks...")
X_all = df_all["job_text_raw"].apply(preprocess_text)
y_all = df_all["label"]
mask = X_all.str.len() > 0
X_all, y_all = X_all[mask], y_all[mask]
logger.info(f"Setelah filter kosong: {len(X_all)} baris")
# ── STEP 4: SPLIT TRAIN / TEST ───────────────────────
# Cek apakah test_set_3.csv ada untuk hold-out
if TEST_PATH.exists():
df_test = pd.read_csv(TEST_PATH, sep=";", dtype=str).dropna(subset=["job_text_raw", "label"])
X_test = df_test["job_text_raw"].apply(preprocess_text)
y_test = df_test["label"]
mask_t = X_test.str.len() > 0
X_test, y_test = X_test[mask_t], y_test[mask_t]
X_train, y_train = X_all, y_all
logger.info(f"Hold-out test dari file: {len(X_test)} baris")
else:
# Fallback: split 70/30 dari data gabungan
X_train, X_test, y_train, y_test = train_test_split(
X_all, y_all, test_size=0.3, random_state=42, stratify=y_all
)
logger.info(f"Hold-out test dari split 30%: {len(X_test)} baris")
# ── STEP 5: TRAINING ─────────────────────────────────
write_status("training", f"Training model baru... ({len(X_train)} sampel training)")
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
fold_metrics = []
for i, (tr_idx, te_idx) in enumerate(skf.split(X_train, y_train)):
m = Pipeline([
('tfidf', TfidfVectorizer(max_features=3000, ngram_range=(1, 2), sublinear_tf=True, min_df=1)),
('clf', LogisticRegression(max_iter=1000, class_weight='balanced', solver='lbfgs'))
])
m.fit(X_train.iloc[tr_idx], y_train.iloc[tr_idx])
y_pred = m.predict(X_train.iloc[te_idx])
rep = classification_report(y_train.iloc[te_idx], y_pred, output_dict=True, zero_division=0)
fold_metrics.append(rep)
write_status("training", f"K-Fold selesai: fold {i+1}/5 | acc={rep['accuracy']:.4f}")
# Train final model on full training set
write_status("training", "Training model final pada seluruh data training...")
candidate_model = Pipeline([
('tfidf', TfidfVectorizer(max_features=3000, ngram_range=(1, 2), sublinear_tf=True, min_df=1)),
('clf', LogisticRegression(max_iter=1000, class_weight='balanced', solver='lbfgs'))
])
candidate_model.fit(X_train, y_train)
# Simpan sebagai candidate (belum overwrite aktif)
joblib.dump(candidate_model, CANDIDATE_PATH)
logger.info(f"Candidate model tersimpan: {CANDIDATE_PATH}")
# ── STEP 6: EVALUASI & KEPUTUSAN ─────────────────────
write_status("evaluating", "Mengevaluasi model baru vs model lama...")
new_metrics = evaluate_model(candidate_model, X_test, y_test)
new_f1 = new_metrics["weighted_f1"]
delta = new_f1 - baseline_f1
logger.info(f"Baseline F1 : {baseline_f1:.4f}")
logger.info(f"Candidate F1: {new_f1:.4f} (delta: {delta:+.4f})")
# ── KEPUTUSAN ────────────────────────────────────────
if baseline_f1 == 0.0:
# Tidak ada baseline (model lama belum pernah dievaluasi) → langsung promote
reason_promote = "Tidak ada baseline metrics → model baru dipromote"
should_promote = True
elif delta >= MIN_IMPROVEMENT_THRESHOLD:
should_promote = True
reason_promote = f"Model baru lebih baik (+{delta*100:.2f}% weighted F1)"
elif delta < -MAX_REGRESSION_ALLOWED:
should_promote = False
reason_rollback = f"Model baru lebih buruk secara signifikan ({delta*100:.2f}% weighted F1)"
else:
should_promote = False
reason_rollback = (
f"Peningkatan tidak signifikan ({delta*100:.2f}% weighted F1, "
f"minimum dibutuhkan: +{MIN_IMPROVEMENT_THRESHOLD*100:.1f}%)"
)
if not should_promote:
do_rollback(reason_rollback, baseline_f1, new_f1)
return
# ── STEP 7: PROMOTE ──────────────────────────────────
write_status("promoting", "Model baru lebih baik — mempromote model baru...")
# Atomic rename: candidate → aktif
shutil.move(str(CANDIDATE_PATH), str(PIPELINE_PATH))
logger.info(f"Model baru dipromote: {PIPELINE_PATH}")
# Update metrics JSON
avg_acc = float(np.mean([f["accuracy"] for f in fold_metrics]))
y_pred_final = candidate_model.predict(X_test)
new_metrics_full = {
"methodology": "Internal MIF + manual_override",
"retrained_at": datetime.now().isoformat(),
"extra_samples_added": len(df_extra),
"total_training_samples": len(X_train),
"k_fold": {
"accuracy_mean": avg_acc,
"accuracy_std": float(np.std([f["accuracy"] for f in fold_metrics])),
"folds": fold_metrics,
},
"threshold_config": 0.50,
"hold_out_test": classification_report(
y_test,
y_pred_final,
output_dict=True,
zero_division=0
),
"comparison": {
"old_weighted_f1": baseline_f1,
"new_weighted_f1": new_f1,
"delta": round(delta, 4),
}
}
METRICS_PATH.write_text(json.dumps(new_metrics_full, indent=2, ensure_ascii=False))
write_status(
stage="promoted",
message=f"Model baru berhasil dipromote. {reason_promote}",
extra={
"result": "promoted",
"old_f1": baseline_f1,
"new_f1": new_f1,
"delta": round(delta, 4),
"new_accuracy": new_metrics["accuracy"],
"per_class": new_metrics["per_class"],
"extra_samples_added": len(df_extra),
"total_training_samples": len(X_train),
}
)
except Exception as e:
logger.error(f"ERROR saat training: {type(e).__name__}: {e}", exc_info=True)
do_rollback(
reason=f"Training gagal karena error: {type(e).__name__}: {str(e)[:200]}",
old_f1=baseline_f1,
new_f1=0.0
)
# Override stage ke 'failed' agar UI tahu ini bukan rollback biasa
status = json.loads(STATUS_PATH.read_text())
status["stage"] = "failed"
STATUS_PATH.write_text(json.dumps(status, indent=2, ensure_ascii=False))
sys.exit(1)
if __name__ == "__main__":
# Argumen: [extra_csv_path]
extra_csv = sys.argv[1] if len(sys.argv) > 1 else None
main(extra_csv)

178
data/unused/retrain_worker2.py Normal file
View File

@ -0,0 +1,178 @@
#!/usr/bin/env python3
import sys
import json
import shutil
import logging
import warnings
import re
import os
import pandas as pd
import numpy as np
import joblib
from pathlib import Path
from datetime import datetime
from filelock import FileLock # PENAMBAHAN KUNCI MUTEX
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import StratifiedKFold, train_test_split
from sklearn.metrics import classification_report
from sklearn.pipeline import Pipeline
from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
warnings.filterwarnings("ignore")
BASE_DIR = Path(__file__).parent.parent
ML_DIR = BASE_DIR / "ml_assets"
DATA_DIR = BASE_DIR.parent / "data" / "processed"
PIPELINE_PATH = ML_DIR / "ml_pipeline_internal.pkl"
PIPELINE_BAK_PATH = ML_DIR / "ml_pipeline_internal.pkl.bak"
CANDIDATE_PATH = ML_DIR / "ml_pipeline_candidate.pkl"
METRICS_PATH = ML_DIR / "metrics_internal_only.json"
STATUS_PATH = ML_DIR / "retrain_status.json"
LOCK_PATH = ML_DIR / "retrain_status.lock"
CORPUS_PATH = DATA_DIR / "training_corpus_3.csv"
TARGET_CLASSES = ["Programmer", "Data Analyst", "Wirausaha Informatika", "Non-IT"]
MIN_IMPROVEMENT_THRESHOLD = 0.01
MAX_REGRESSION_ALLOWED = 0.02
STOPWORDS = {
"yang", "di", "ke", "dari", "dan", "atau", "dengan", "untuk", "pada", "dalam",
"adalah", "ini", "itu", "tidak", "juga", "sudah", "akan", "bisa", "ada", "oleh"
}
stemmer = StemmerFactory().create_stemmer()
logging.basicConfig(level=logging.INFO, format="%(asctime)s | %(levelname)s | %(message)s", handlers=[logging.StreamHandler(sys.stdout)])
logger = logging.getLogger("retrain_worker")
def write_status(stage: str, message: str, extra: dict = None):
# MENULIS STATUS SECARA ATOMIK
lock = FileLock(LOCK_PATH, timeout=10)
with lock:
payload = {"stage": stage, "message": message, "timestamp": datetime.now().isoformat()}
if extra: payload.update(extra)
STATUS_PATH.write_text(json.dumps(payload, indent=2, ensure_ascii=False))
logger.info(f"[{stage}] {message}")
def preprocess_raw_text(text: str) -> str:
# Stemming HANYA untuk data manual override baru
if pd.isna(text) or not isinstance(text, str): return ""
text = text.lower().strip()
text = re.sub(r'[^\w\s]', '', text.replace('-', ' ').replace('/', ' '))
text = re.sub(r'\d+', '', text)
words = [w for w in text.split() if w not in STOPWORDS and len(w) > 2]
return " ".join([stemmer.stem(w) for w in words])
def preprocess_stemmed_text(text: str) -> str:
# Pembersihan dasar tanpa memanggil Sastrawi untuk korpus lama
if pd.isna(text) or not isinstance(text, str): return ""
text = text.lower().strip()
text = re.sub(r'[^\w\s]', '', text.replace('-', ' ').replace('/', ' '))
words = [w for w in text.split() if w not in STOPWORDS and len(w) > 2]
return " ".join(words)
def evaluate_model(model, X_test: pd.Series, y_test: pd.Series) -> dict:
if len(X_test) == 0: return {"weighted_f1": 0.0, "accuracy": 0.0, "per_class": {}}
y_pred = model.predict(X_test)
report = classification_report(y_test, y_pred, output_dict=True, zero_division=0, labels=TARGET_CLASSES)
return {
"weighted_f1": round(report.get("weighted avg", {}).get("f1-score", 0.0), 4),
"accuracy": round(report.get("accuracy", 0.0), 4),
"per_class": {cls: {"f1": round(report.get(cls, {}).get("f1-score", 0.0), 3)} for cls in TARGET_CLASSES}
}
def get_baseline_f1() -> float:
try:
if METRICS_PATH.exists():
return float(json.loads(METRICS_PATH.read_text()).get("hold_out_test", {}).get("weighted avg", {}).get("f1-score", 0.0))
except Exception: pass
return 0.0
def do_rollback(reason: str, old_f1: float, new_f1: float):
if CANDIDATE_PATH.exists(): CANDIDATE_PATH.unlink()
if PIPELINE_BAK_PATH.exists():
shutil.copy2(PIPELINE_BAK_PATH, PIPELINE_PATH)
write_status("rolled_back", f"Model lama dipertahankan. {reason}", {"result": "rolled_back", "old_f1": old_f1, "new_f1": new_f1})
def main(extra_csv_path: str = None):
write_status("started", "Worker dimulai")
write_status("backup", "Membuat backup model lama...")
if PIPELINE_PATH.exists(): shutil.copy2(PIPELINE_PATH, PIPELINE_BAK_PATH)
baseline_f1 = get_baseline_f1()
try:
write_status("loading_data", "Memuat dan menggabungkan data training...")
if not CORPUS_PATH.exists(): raise FileNotFoundError(f"Corpus asli tidak ditemukan: {CORPUS_PATH}")
df_corpus = pd.read_csv(CORPUS_PATH, sep=";", dtype=str).dropna(subset=["job_text_raw", "label"])
df_corpus["features"] = df_corpus["job_text_raw"].apply(preprocess_stemmed_text)
df_extra = pd.DataFrame()
if extra_csv_path and Path(extra_csv_path).exists():
df_extra = pd.read_csv(extra_csv_path, sep=";", dtype=str).dropna(subset=["job_text_raw", "label"])
df_extra = df_extra[df_extra["label"].isin(TARGET_CLASSES)]
df_extra["features"] = df_extra["job_text_raw"].apply(preprocess_raw_text)
# Gabungkan tanpa drop_duplicates kasar. Model butuh tahu frekuensi/probabilitas kelas untuk teks yang bias.
df_all = pd.concat([df_corpus, df_extra], ignore_index=True)
mask = df_all["features"].str.len() > 0
df_all = df_all[mask]
if len(df_all) < 20: raise ValueError("Data terlalu sedikit (minimum 20 baris).")
X_all = df_all["features"]
y_all = df_all["label"]
# SPLIT DINAMIS BARU: Tidak ada lagi static test-set.
X_train, X_test, y_train, y_test = train_test_split(X_all, y_all, test_size=0.3, random_state=42, stratify=y_all)
write_status("training", "Training model final pada data gabungan dinamis...")
candidate_model = Pipeline([
('tfidf', TfidfVectorizer(max_features=3000, ngram_range=(1, 2), sublinear_tf=True, min_df=1)),
('clf', LogisticRegression(max_iter=1000, class_weight='balanced', solver='lbfgs'))
])
candidate_model.fit(X_train, y_train)
joblib.dump(candidate_model, CANDIDATE_PATH)
write_status("evaluating", "Mengevaluasi model baru vs model lama...")
new_metrics = evaluate_model(candidate_model, X_test, y_test)
new_f1 = new_metrics["weighted_f1"]
delta = new_f1 - baseline_f1
if baseline_f1 == 0.0 or delta >= MIN_IMPROVEMENT_THRESHOLD:
should_promote, reason = True, "Model baru dipromote (Peningkatan Signifikan)"
elif delta < -MAX_REGRESSION_ALLOWED:
should_promote, reason = False, "Regresi skor terlalu tinggi. Rollback."
else:
should_promote, reason = False, "Peningkatan tidak memenuhi threshold minimal. Rollback."
if not should_promote:
do_rollback(reason, baseline_f1, new_f1)
return
write_status("promoting", "Mem-promote model baru...")
shutil.move(str(CANDIDATE_PATH), str(PIPELINE_PATH))
new_metrics_full = {
"methodology": "Dynamic Split Retrain",
"comparison": {"old_weighted_f1": baseline_f1, "new_weighted_f1": new_f1, "delta": round(delta, 4)},
"hold_out_test": classification_report(y_test, candidate_model.predict(X_test), output_dict=True, zero_division=0)
}
METRICS_PATH.write_text(json.dumps(new_metrics_full, indent=2, ensure_ascii=False))
write_status("promoted", f"Selesai. {reason}")
except Exception as e:
logger.error(f"ERROR: {e}", exc_info=True)
do_rollback(f"Error sistem: {str(e)[:100]}", baseline_f1, 0.0)
lock = FileLock(LOCK_PATH, timeout=10)
with lock:
status = json.loads(STATUS_PATH.read_text())
status["stage"] = "failed"
STATUS_PATH.write_text(json.dumps(status, indent=2, ensure_ascii=False))
sys.exit(1)
if __name__ == "__main__":
main(sys.argv[1] if len(sys.argv) > 1 else None)

0
data/train_model_3.py → data/unused/train_model_3.py
View File

114
data/unused/train_model_4.py Normal file
View File

@ -0,0 +1,114 @@
import pandas as pd
import numpy as np
import re
import joblib
import json
import warnings
from pathlib import Path
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import classification_report, confusion_matrix, ConfusionMatrixDisplay
from sklearn.pipeline import Pipeline
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
warnings.filterwarnings('ignore')
BASE_DIR = Path(__file__).parent
TRAIN_FILE = BASE_DIR / "processed" / "training_corpus_4.csv"
TEST_FILE = BASE_DIR / "processed" / "test_set_4.csv"
ML_DIR = BASE_DIR.parent / "fastapi" / "ml_assets"
ML_DIR.mkdir(parents=True, exist_ok=True)
TARGET_CLASSES = ["Programmer", "Data Analyst", "Wirausaha Informatika", "Non-IT"]
CONFIDENCE_THRESHOLD = 0.50
STOPWORDS = {
"yang", "di", "ke", "dari", "dan", "atau", "dengan", "untuk", "pada", "dalam",
"adalah", "ini", "itu", "tidak", "juga", "sudah", "akan", "bisa", "ada", "oleh",
"karena", "secara", "serta", "sebagai", "bagi", "telah", "maka", "namun", "sehingga",
"jika", "agar", "ketika", "saat", "sebelum", "sesudah", "hingga", "sampai", "antara",
"sekitar", "hanya", "saja", "belum", "masih", "lagi", "pun", "justru", "walaupun",
"meskipun", "bahkan", "cukup", "sangat", "paling", "lebih", "kurang", "lain",
"macam", "cara", "hal", "tentang", "mengenai", "terhadap", "kepada", "menuju",
"kecuali", "selain", "tanpa", "demi", "guna", "khususnya", "umumnya", "kebanyakan",
"sebagian", "beberapa", "semua", "setiap", "tiap", "satu", "dua", "tiga", "empat",
"lima", "enam", "tujuh", "delapan", "sembilan", "sepuluh", "ratus", "ribu", "juta"
}
def preprocess_text(text):
# Tidak ada Sastrawi! Teks korpus SUDAH di-stem.
if pd.isna(text) or not isinstance(text, str): return ""
text = text.lower().strip()
if text in {"nan", "none", "null", "-", "0", "", "tidak diisi"}: return ""
text = text.replace('-', ' ').replace('/', ' ').replace('_', ' ').replace(',', ' ')
text = re.sub(r'[^\w\s]', '', text)
text = re.sub(r'\d+', '', text)
words = [w for w in text.split() if w not in STOPWORDS and len(w) > 2]
return " ".join(words)
def main():
if not TRAIN_FILE.exists():
print("File training_corpus_4.csv belum ada"); return
df_train = pd.read_csv(TRAIN_FILE, sep=";", dtype=str).dropna(subset=["job_text_raw", "label"])
df_test = pd.read_csv(TEST_FILE, sep=";", dtype=str).dropna(subset=["job_text_raw", "label"]) if TEST_FILE.exists() else None
print("Sedang memproses teks (cleaning dasar tanpa stemming redundan)...")
X_train = df_train["job_text_raw"].apply(preprocess_text)
y_train = df_train["label"]
mask = X_train.str.len() > 0
X_train, y_train = X_train[mask], y_train[mask]
X_test, y_test = pd.Series(dtype=str), pd.Series(dtype=str)
if df_test is not None:
X_test = df_test["job_text_raw"].apply(preprocess_text)
y_test = df_test["label"]
mask_t = X_test.str.len() > 0
X_test, y_test = X_test[mask_t], y_test[mask_t]
print(f"Jumlah data latih: {len(X_train)} baris | Data uji: {len(X_test)} baris\n")
print("Melakukan pengujian K-Fold (5 putaran) pada data latih...")
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
fold_metrics = []
for i, (tr_idx, te_idx) in enumerate(skf.split(X_train, y_train)):
model = Pipeline([
('tfidf', TfidfVectorizer(max_features=3000, ngram_range=(1,2), sublinear_tf=True, min_df=1)),
('clf', LogisticRegression(max_iter=1000, class_weight='balanced', solver='lbfgs'))
])
model.fit(X_train.iloc[tr_idx], y_train.iloc[tr_idx])
y_pred = model.predict(X_train.iloc[te_idx])
rep = classification_report(y_train.iloc[te_idx], y_pred, output_dict=True, zero_division=0)
fold_metrics.append(rep)
print(f" Akurasi putaran ke-{i+1}: {rep['accuracy']:.4f}")
avg_acc = np.mean([f['accuracy'] for f in fold_metrics])
print(f"\n RATA-RATA PENGUJIAN K-FOLD:")
print(f" Akurasi Keseluruhan : {avg_acc:.4f} ± {np.std([f['accuracy'] for f in fold_metrics]):.4f}")
print("\n Membuat model final dari seluruh data latih yang tersedia...")
final_model = Pipeline([
('tfidf', TfidfVectorizer(max_features=3000, ngram_range=(1,2), sublinear_tf=True, min_df=1)),
('clf', LogisticRegression(max_iter=1000, class_weight='balanced', solver='lbfgs'))
])
final_model.fit(X_train, y_train)
metrics_test = None
if len(X_test) > 0:
y_pred = final_model.predict(X_test)
rep_test = classification_report(y_test, y_pred, output_dict=True, zero_division=0)
metrics_test = rep_test
print("\n HASIL PENGUJIAN PADA DATA TEST:")
print(classification_report(y_test, y_pred, zero_division=0))
model_path = ML_DIR / "ml_pipeline_internal.pkl"
joblib.dump(final_model, model_path)
metrics = {"methodology": "Internal MIF only", "k_fold": {"accuracy_mean": float(avg_acc), "accuracy_std": float(np.std([f['accuracy'] for f in fold_metrics])), "folds": fold_metrics}, "threshold_config": CONFIDENCE_THRESHOLD}
if metrics_test: metrics["hold_out_test"] = metrics_test
with open(ML_DIR / "metrics_internal_only.json", 'w') as f: json.dump(metrics, f, indent=2)
print(f"\n Model berhasil disimpan di: {model_path}")
if __name__ == "__main__":
main()

209
fastapi/app/main.py
View File

@ -1,4 +1,4 @@
from fastapi import FastAPI, UploadFile, File, HTTPException, BackgroundTasks, Depends, Security from fastapi import FastAPI, UploadFile, File, HTTPException, Depends, Security, BackgroundTasks
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
from fastapi.responses import JSONResponse from fastapi.responses import JSONResponse
import pandas as pd import pandas as pd
@ -8,7 +8,6 @@ import re
import io import io
import json import json
import uuid import uuid
import shutil
import subprocess import subprocess
import sys import sys
import os import os
@ -17,17 +16,21 @@ from contextlib import asynccontextmanager
from pathlib import Path from pathlib import Path
from typing import List, Dict, Any from typing import List, Dict, Any
from Sastrawi.Stemmer.StemmerFactory import StemmerFactory from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
from filelock import FileLock, Timeout
# ──────────────────────────────────────────────────────────────
# KONFIGURASI & KONSTANTA # KONFIGURASI & KONSTANTA
# ──────────────────────────────────────────────────────────────
BASE_DIR = Path(__file__).parent.parent BASE_DIR = Path(__file__).parent.parent
ML_DIR = BASE_DIR / "ml_assets" ML_DIR = BASE_DIR / "ml_assets"
PIPELINE_PATH = ML_DIR / "ml_pipeline_internal.pkl" PIPELINE_PATH = ML_DIR / "ml_pipeline_internal.pkl"
STATUS_PATH = ML_DIR / "retrain_status.json" STATUS_PATH = ML_DIR / "retrain_status.json"
LOCK_PATH = ML_DIR / "retrain_status.lock"
RETRAIN_WORKER = Path(__file__).parent / "retrain_worker.py" RETRAIN_WORKER = Path(__file__).parent / "retrain_worker.py"
TEMP_DIR = ML_DIR / "tmp" TEMP_DIR = ML_DIR / "tmp"
TEMP_DIR.mkdir(parents=True, exist_ok=True) TEMP_DIR.mkdir(parents=True, exist_ok=True)
JOBS_DIR = ML_DIR / "jobs"
JOBS_DIR.mkdir(parents=True, exist_ok=True)
CONFIDENCE_THRESHOLD = 0.50 CONFIDENCE_THRESHOLD = 0.50
TARGET_CLASSES = ["Programmer", "Data Analyst", "Wirausaha Informatika", "Non-IT"] TARGET_CLASSES = ["Programmer", "Data Analyst", "Wirausaha Informatika", "Non-IT"]
@ -36,7 +39,6 @@ KEYWORD_RULES = {
"Data Analyst": ["data analyst", "analis data", "data science", "business analyst", "research", "statistik", "bi analyst", "reporting", "database", "sql", "etl", "data engineer", "big data", "analyst", "data mining", "machine learning", "data visual", "power bi", "tableau", "looker", "business intelligence", "bi developer", "data warehouse"], "Data Analyst": ["data analyst", "analis data", "data science", "business analyst", "research", "statistik", "bi analyst", "reporting", "database", "sql", "etl", "data engineer", "big data", "analyst", "data mining", "machine learning", "data visual", "power bi", "tableau", "looker", "business intelligence", "bi developer", "data warehouse"],
"Wirausaha Informatika": ["founder", "owner", "ceo", "wiraswasta", "startup", "freelance", "freelancer", "wirausaha", "bisnis", "usaha mandiri", "konsultan", "co founder", "entrepreneur", "self employed", "owner toko", "usaha", "dagang online", "tokopedia", "shopee", "dropship", "reseller"] "Wirausaha Informatika": ["founder", "owner", "ceo", "wiraswasta", "startup", "freelance", "freelancer", "wirausaha", "bisnis", "usaha mandiri", "konsultan", "co founder", "entrepreneur", "self employed", "owner toko", "usaha", "dagang online", "tokopedia", "shopee", "dropship", "reseller"]
} }
F5C_MAP = {"1": "founder owner wirausaha startup", "2": "co-founder partner wirausaha", "3": "staff karyawan pegawai", "4": "freelance kerja lepas lepasan"} F5C_MAP = {"1": "founder owner wirausaha startup", "2": "co-founder partner wirausaha", "3": "staff karyawan pegawai", "4": "freelance kerja lepas lepasan"}
F1101_MAP = {"1": "instansi pemerintah dinas kementerian", "2": "non-profit lsm yayasan", "3": "perusahaan swasta corporate", "4": "wiraswasta usaha mandiri", "6": "bumn bumd pemerintah", "7": "multilateral internasional"} F1101_MAP = {"1": "instansi pemerintah dinas kementerian", "2": "non-profit lsm yayasan", "3": "perusahaan swasta corporate", "4": "wiraswasta usaha mandiri", "6": "bumn bumd pemerintah", "7": "multilateral internasional"}
@ -57,8 +59,9 @@ STOPWORDS = {
"lima", "enam", "tujuh", "delapan", "sembilan", "sepuluh", "ratus", "ribu", "juta" "lima", "enam", "tujuh", "delapan", "sembilan", "sepuluh", "ratus", "ribu", "juta"
} }
# ──────────────────────────────────────────────────────────────
# MODEL LOADING # MODEL LOADING
# ──────────────────────────────────────────────────────────────
@asynccontextmanager @asynccontextmanager
async def lifespan(app: FastAPI): async def lifespan(app: FastAPI):
try: try:
@ -71,24 +74,25 @@ async def lifespan(app: FastAPI):
app = FastAPI(title="Tracer Study Classification Worker", lifespan=lifespan) app = FastAPI(title="Tracer Study Classification Worker", lifespan=lifespan)
# --- SECURITY SETUP --- # ──────────────────────────────────────────────────────────────
# SECURITY
# ──────────────────────────────────────────────────────────────
security = HTTPBearer() security = HTTPBearer()
EXPECTED_TOKEN = os.getenv("FASTAPI_SECRET_KEY", "") EXPECTED_TOKEN = os.getenv("FASTAPI_SECRET_KEY", "")
def verify_token(credentials: HTTPAuthorizationCredentials = Security(security)): def verify_token(credentials: HTTPAuthorizationCredentials = Security(security)):
if not EXPECTED_TOKEN: if not EXPECTED_TOKEN: return credentials.credentials
return credentials.credentials
if credentials.credentials != EXPECTED_TOKEN: if credentials.credentials != EXPECTED_TOKEN:
raise HTTPException(status_code=401, detail="Unauthorized - Invalid Token") raise HTTPException(status_code=401, detail="Unauthorized - Invalid Token")
return credentials.credentials return credentials.credentials
# ----------------------
# ──────────────────────────────────────────────────────────────
# TEXT PROCESSING
# ──────────────────────────────────────────────────────────────
def clean_text(text: str) -> str: def clean_text(text: str) -> str:
if pd.isna(text) or not isinstance(text, str): if pd.isna(text) or not isinstance(text, str): return ""
return ""
text = text.lower().strip() text = text.lower().strip()
if text in {"nan", "none", "null", "-", "0", "", "tidak diisi"}: if text in {"nan", "none", "null", "-", "0", "", "tidak diisi"}: return ""
return ""
text = text.replace('-', ' ').replace('/', ' ').replace('_', ' ').replace(',', ' ') text = text.replace('-', ' ').replace('/', ' ').replace('_', ' ').replace(',', ' ')
text = re.sub(r'[^\w\s]', '', text) text = re.sub(r'[^\w\s]', '', text)
text = re.sub(r'\d+', '', text) text = re.sub(r'\d+', '', text)
@ -97,27 +101,24 @@ def clean_text(text: str) -> str:
def find_column_by_code(df_columns: list, code: str) -> str | None: def find_column_by_code(df_columns: list, code: str) -> str | None:
for col in df_columns: for col in df_columns:
if code.lower() in col.lower(): if code.lower() in col.lower(): return col
return col
return None return None
def safe_get(row: pd.Series, df_columns: list, code: str, default: str = "") -> str: def safe_get(row: pd.Series, df_columns: list, code: str, default: str = "") -> str:
actual_col = find_column_by_code(df_columns, code) actual_col = find_column_by_code(df_columns, code)
if actual_col and actual_col in row: if actual_col and actual_col in row:
val = row[actual_col] val = row[actual_col]
if pd.isna(val) or val is None: if pd.isna(val) or val is None: return default
return default
return str(val).strip() return str(val).strip()
return default return default
# ──────────────────────────────────────────────────────────────
# CLASSIFICATION LOGIC # CLASSIFICATION LOGIC
# ──────────────────────────────────────────────────────────────
def classify_rule(job_text: str) -> dict: def classify_rule(job_text: str) -> dict:
clean = clean_text(job_text) clean = clean_text(job_text)
if not clean or len(clean) < 3: if not clean or len(clean) < 3:
return {"profile": "Tidak Diketahui", "confidence": 0.65, "method": "rule_based_fallback"} return {"profile": "Tidak Diketahui", "confidence": 0.65, "method": "rule_based_fallback"}
for profile, keywords in KEYWORD_RULES.items(): for profile, keywords in KEYWORD_RULES.items():
if any(kw in clean for kw in keywords): if any(kw in clean for kw in keywords):
return {"profile": profile, "confidence": 0.65, "method": "rule_based"} return {"profile": profile, "confidence": 0.65, "method": "rule_based"}
@ -137,6 +138,7 @@ def classify_ml(job_text: str, pipeline) -> dict:
logger.error(f"ML_INFER_ERROR | job_text='{job_text[:50]}' | err={e}") logger.error(f"ML_INFER_ERROR | job_text='{job_text[:50]}' | err={e}")
raise HTTPException(status_code=500, detail=f"ML inference failed: {str(e)}") raise HTTPException(status_code=500, detail=f"ML inference failed: {str(e)}")
# [KEEP v3] Ekstraksi teks dari format Kemendikbud
def extract_kemendik_text(row: pd.Series, df_columns: list) -> str: def extract_kemendik_text(row: pd.Series, df_columns: list) -> str:
f5b_col = find_column_by_code(df_columns, "f5b") f5b_col = find_column_by_code(df_columns, "f5b")
f5c_col = find_column_by_code(df_columns, "f5c") f5c_col = find_column_by_code(df_columns, "f5c")
@ -150,48 +152,41 @@ def extract_kemendik_text(row: pd.Series, df_columns: list) -> str:
f1101_raw = str(row.get(f1101_col, "")).strip() if f1101_col else "" f1101_raw = str(row.get(f1101_col, "")).strip() if f1101_col else ""
f1101_code = re.match(r'^(\d+)', f1101_raw) f1101_code = re.match(r'^(\d+)', f1101_raw)
f1101_text = F1101_MAP.get(f1101_code.group(1), "") if f1101_code else "" f1101_text = F1101_MAP.get(f1101_code.group(1), "") if f1101_code else ""
return " ".join(p for p in [f5b, f5c_text, f1101_text, f1102] if p).strip() return " ".join(p for p in [f5b, f5c_text, f1101_text, f1102] if p).strip()
# ──────────────────────────────────────────────────────────────
# SOURCE DETECTION # SOURCE DETECTION
# ──────────────────────────────────────────────────────────────
def detect_source(df: pd.DataFrame) -> str: def detect_source(df: pd.DataFrame) -> str:
cols_lower = [c.lower().strip() for c in df.columns] cols_lower = [c.lower().strip() for c in df.columns]
kemendik_codes = ["f5b", "f5c", "f8", "f1101", "nimhsmsmh", "nmmhsmsmh"] kemendik_codes = ["f5b", "f5c", "f8", "f1101", "nimhsmsmh", "nmmhsmsmh"]
for col in cols_lower: for col in cols_lower:
if any(code in col for code in kemendik_codes): if any(code in col for code in kemendik_codes): return "kemendik"
return "kemendik" if any("jabatan" in c for c in cols_lower): return "internal_mif"
if any("jabatan" in c for c in cols_lower):
return "internal_mif"
return "unknown" return "unknown"
# ──────────────────────────────────────────────────────────────
# ROUTES # ROUTES
# ──────────────────────────────────────────────────────────────
@app.get("/health") @app.get("/health")
def health_check(): def health_check():
return {"status": "healthy", "pipeline_loaded": app.state.pipeline is not None} return {"status": "healthy", "pipeline_loaded": app.state.pipeline is not None}
# RE-TRAINING ENDPOINTS
@app.post("/api/v1/retrain", dependencies=[Depends(verify_token)]) @app.post("/api/v1/retrain", dependencies=[Depends(verify_token)])
async def trigger_retrain(file: UploadFile = File(None)): async def trigger_retrain(file: UploadFile = File(None)):
# [v4] FileLock mencegah race condition dari concurrent requests
lock = FileLock(LOCK_PATH, timeout=2)
try:
with lock:
if STATUS_PATH.exists(): if STATUS_PATH.exists():
try: try:
current = json.loads(STATUS_PATH.read_text()) current = json.loads(STATUS_PATH.read_text())
if current.get("stage") in ["started", "backup", "loading_data", if current.get("stage") in ["started", "backup", "loading_data", "preprocessing", "training", "evaluating", "promoting"]:
"preprocessing", "training", "evaluating", "promoting"]: raise HTTPException(status_code=409, detail="Re-training sedang berjalan. Tunggu hingga selesai.")
raise HTTPException( except json.JSONDecodeError:
status_code=409,
detail="Re-training sedang berjalan. Tunggu hingga selesai."
)
except HTTPException:
raise
except Exception:
pass pass
extra_csv_path = "" extra_csv_path = ""
if file and file.filename: if file and file.filename:
job_id = str(uuid.uuid4())[:8] job_id = str(uuid.uuid4())[:8]
@ -207,37 +202,29 @@ async def trigger_retrain(file: UploadFile = File(None)):
}, ensure_ascii=False)) }, ensure_ascii=False))
cmd = [sys.executable, str(RETRAIN_WORKER)] cmd = [sys.executable, str(RETRAIN_WORKER)]
if extra_csv_path: if extra_csv_path: cmd.append(extra_csv_path)
cmd.append(extra_csv_path)
try: subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True)
subprocess.Popen(
cmd,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
start_new_session=True
)
logger.info(f"Retrain subprocess spawned: {' '.join(cmd)}") logger.info(f"Retrain subprocess spawned: {' '.join(cmd)}")
except Exception as e:
STATUS_PATH.write_text(json.dumps({
"stage": "failed",
"message": f"Gagal spawn subprocess: {str(e)}",
}, ensure_ascii=False))
raise HTTPException(status_code=500, detail=f"Gagal memulai training: {str(e)}")
return JSONResponse(content={"status": "started", "message": "Re-training dimulai di background."}) return JSONResponse(content={"status": "started", "message": "Re-training dimulai di background."})
except Timeout:
raise HTTPException(status_code=429, detail="Sistem sibuk. Permintaan retraining lain sedang diproses.")
except Exception as e:
if not isinstance(e, HTTPException):
raise HTTPException(status_code=500, detail=f"Gagal memulai training: {str(e)}")
raise e
@app.get("/api/v1/retrain/status", dependencies=[Depends(verify_token)]) @app.get("/api/v1/retrain/status", dependencies=[Depends(verify_token)])
def retrain_status(): def retrain_status():
if not STATUS_PATH.exists(): if not STATUS_PATH.exists():
return JSONResponse(content={"stage": "idle", "message": "Belum ada proses re-training."}) return JSONResponse(content={"stage": "idle", "message": "Belum ada proses re-training."})
try: try:
# [v4] Lock saat baca untuk menghindari partial write dari worker
lock = FileLock(LOCK_PATH, timeout=2)
with lock:
data = json.loads(STATUS_PATH.read_text()) data = json.loads(STATUS_PATH.read_text())
except Exception:
return JSONResponse(content={"stage": "unknown", "message": "Status tidak terbaca."})
terminal_stages = {"promoted", "rolled_back", "failed"} terminal_stages = {"promoted", "rolled_back", "failed"}
if data.get("stage") in terminal_stages and not data.get("_reloaded"): if data.get("stage") in terminal_stages and not data.get("_reloaded"):
try: try:
@ -247,10 +234,14 @@ def retrain_status():
logger.info(f"Pipeline di-reload setelah retrain (stage={data['stage']})") logger.info(f"Pipeline di-reload setelah retrain (stage={data['stage']})")
except Exception as e: except Exception as e:
logger.error(f"Gagal reload pipeline: {e}") logger.error(f"Gagal reload pipeline: {e}")
return JSONResponse(content=data) return JSONResponse(content=data)
except Timeout:
return JSONResponse(status_code=429, content={"stage": "locked", "message": "Status sedang diupdate sistem."})
except Exception:
return JSONResponse(content={"stage": "unknown", "message": "Status tidak terbaca."})
# [KEEP v3] Endpoint reload manual — berguna jika pipeline perlu di-reload tanpa restart server
@app.post("/api/v1/retrain/reload", dependencies=[Depends(verify_token)]) @app.post("/api/v1/retrain/reload", dependencies=[Depends(verify_token)])
def reload_model(): def reload_model():
try: try:
@ -260,25 +251,33 @@ def reload_model():
except Exception as e: except Exception as e:
raise HTTPException(status_code=500, detail=f"Gagal reload model: {str(e)}") raise HTTPException(status_code=500, detail=f"Gagal reload model: {str(e)}")
@app.post("/api/v1/classify", dependencies=[Depends(verify_token)])
async def classify_tracer(file: UploadFile = File(...)): def process_classification_bg(job_id: str, file_path: Path, filename: str, pipeline):
if not file.filename: status_file = JOBS_DIR / f"{job_id}.json"
raise HTTPException(status_code=400, detail="No file provided") def update_status(status, message, results=None, processed_rows=0, total_rows=0, source_type="unknown"):
data = {
"status": status, "message": message,
"processed_rows": processed_rows, "total_rows": total_rows,
"source_type": source_type
}
if results is not None:
data["results"] = results
status_file.write_text(json.dumps(data, ensure_ascii=False))
try: try:
contents = await file.read() update_status("processing", "Parsing file excel/csv...")
contents = file_path.read_bytes()
if file.filename.endswith(".xlsx") or file.filename.endswith(".xls"): if filename.endswith(".xlsx") or filename.endswith(".xls"):
try: try:
df = pd.read_excel(io.BytesIO(contents), dtype=str, engine="openpyxl") df = pd.read_excel(io.BytesIO(contents), dtype=str, engine="openpyxl")
except ImportError: except ImportError:
logger.error("openpyxl not installed") update_status("error", "Server misconfiguration: openpyxl missing")
raise HTTPException(status_code=500, detail="Server misconfiguration: openpyxl missing") return
except Exception as excel_err: except Exception as excel_err:
logger.error(f"Excel parsing error: {excel_err}") logger.error(f"Excel parsing error: {excel_err}")
try: try:
df = pd.read_csv(io.BytesIO(contents), sep=";", encoding="utf-8-sig", dtype=str) df = pd.read_csv(io.BytesIO(contents), sep=";", encoding="utf-8-sig", dtype=str)
except: except Exception:
df = pd.read_csv(io.BytesIO(contents), sep=";", encoding="latin-1", dtype=str) df = pd.read_csv(io.BytesIO(contents), sep=";", encoding="latin-1", dtype=str)
else: else:
try: try:
@ -287,21 +286,17 @@ async def classify_tracer(file: UploadFile = File(...)):
df = pd.read_csv(io.BytesIO(contents), sep=";", encoding="latin-1", dtype=str) df = pd.read_csv(io.BytesIO(contents), sep=";", encoding="latin-1", dtype=str)
df.columns = df.columns.str.strip() df.columns = df.columns.str.strip()
except Exception as e:
logger.error(f"File parsing error: {type(e).__name__}: {str(e)}")
raise HTTPException(status_code=400, detail=f"Failed to parse file: {type(e).__name__}: {str(e)[:200]}")
source_type = detect_source(df) source_type = detect_source(df)
if source_type == "unknown": if source_type == "unknown":
raise HTTPException(status_code=400, detail="Unrecognized file format.") update_status("error", "Unrecognized file format.")
return
if source_type == "internal_mif" and pipeline is None:
update_status("error", "ML model not available.")
return
if source_type == "internal_mif" and app.state.pipeline is None: update_status("processing", f"Processing {len(df)} rows...", total_rows=len(df), source_type=source_type)
logger.error("ML pipeline not loaded")
raise HTTPException(status_code=503, detail="ML model not available.")
results = [] results = []
pipeline = app.state.pipeline
logger.info(f"Processing {len(df)} rows | Source: {source_type}") logger.info(f"Processing {len(df)} rows | Source: {source_type}")
find_col_cached = lambda p: next((c for c in df.columns if p.lower() in c.lower()), None) find_col_cached = lambda p: next((c for c in df.columns if p.lower() in c.lower()), None)
@ -318,6 +313,7 @@ async def classify_tracer(file: UploadFile = File(...)):
col_tahun_int = find_col_cached("tahun_lulus") col_tahun_int = find_col_cached("tahun_lulus")
for _, row in df.iterrows(): for _, row in df.iterrows():
nim = ""
try: try:
def safe_str(val): def safe_str(val):
if pd.isna(val) or val is None: return "" if pd.isna(val) or val is None: return ""
@ -363,7 +359,6 @@ async def classify_tracer(file: UploadFile = File(...)):
rule_res = classify_rule(job_text) rule_res = classify_rule(job_text)
res = rule_res or (classify_ml(job_text, pipeline) if pipeline else {"profile": "Non-IT", "confidence": 0.0, "method": "ml_unavailable"}) res = rule_res or (classify_ml(job_text, pipeline) if pipeline else {"profile": "Non-IT", "confidence": 0.0, "method": "ml_unavailable"})
status = "auto_classified" if res["method"] in ["rule_based", "ml_fallback"] or res["profile"] == "Tidak Diketahui" else "needs_review" status = "auto_classified" if res["method"] in ["rule_based", "ml_fallback"] or res["profile"] == "Tidak Diketahui" else "needs_review"
raw_data_payload = {str(k): safe_str(v) for k, v in row.to_dict().items()} raw_data_payload = {str(k): safe_str(v) for k, v in row.to_dict().items()}
results.append({ results.append({
@ -375,9 +370,51 @@ async def classify_tracer(file: UploadFile = File(...)):
}) })
except Exception as row_err: except Exception as row_err:
logger.warning(f"Row error: {row_err}") logger.warning(f"Row error: {row_err}")
results.append({"nim": nim if 'nim' in locals() else "", "status": "failed", "error_detail": str(row_err)[:100]}) results.append({"nim": nim, "status": "failed", "error_detail": str(row_err)[:100]})
update_status("success", "Selesai", results=results, processed_rows=len(results), total_rows=len(df), source_type=source_type)
except Exception as e:
logger.error(f"Job {job_id} error: {e}")
update_status("error", f"Processing failed: {str(e)}")
finally:
if file_path.exists():
try:
file_path.unlink()
except Exception as unlink_err:
logger.warning(f"Failed to delete temp file {file_path}: {unlink_err}")
@app.post("/api/v1/classify", dependencies=[Depends(verify_token)])
async def classify_tracer(background_tasks: BackgroundTasks, file: UploadFile = File(...)):
if not file.filename:
raise HTTPException(status_code=400, detail="No file provided")
job_id = str(uuid.uuid4())
temp_file = JOBS_DIR / f"raw_{job_id}_{file.filename}"
contents = await file.read()
temp_file.write_bytes(contents)
status_file = JOBS_DIR / f"{job_id}.json"
status_file.write_text(json.dumps({"status": "pending", "message": "Job masuk antrean"}))
background_tasks.add_task(process_classification_bg, job_id, temp_file, file.filename, app.state.pipeline)
return JSONResponse(content={"job_id": job_id, "status": "pending", "message": "Classification started in background."})
@app.get("/api/v1/classify/status/{job_id}", dependencies=[Depends(verify_token)])
def classify_status(job_id: str):
status_file = JOBS_DIR / f"{job_id}.json"
if not status_file.exists():
raise HTTPException(status_code=404, detail="Job not found")
try:
data = json.loads(status_file.read_text())
return JSONResponse(content=data)
except json.JSONDecodeError:
return JSONResponse(content={"status": "processing", "message": "Updating status..."})
return JSONResponse(content={"status": "success", "total_rows": len(df), "processed_rows": len(results), "source_type": source_type, "results": results})
if __name__ == "__main__": if __name__ == "__main__":
import uvicorn import uvicorn

353
fastapi/app/retrain_worker.py
View File

@ -1,14 +1,18 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
""" """
TRACER STUDY - RE-TRAINING WORKER (SUBPROCESS) TRACER STUDY - RE-TRAINING WORKER FINAL (SUBPROCESS)
Dipanggil oleh FastAPI sebagai background subprocess. Dipanggil oleh FastAPI (main3.py) sebagai background subprocess.
Alur: Alur:
1. Backup pkl lama → pkl.bak 1. Backup pkl lama → pkl.bak
2. Merge corpus asli + data manual_override baru 2. Merge corpus + manual_override (TANPA drop_duplicates)
3. Train model baru (candidate) Override diberi sample_weight lebih tinggi, bukan menghapus data historis.
4. Evaluasi: bandingkan weighted F1-score baru vs lama 3. Dynamic train/test split dari data gabungan (selalu fresh)
5. Promote jika lebih baik; rollback jika tidak 4. Evaluasi MODEL LAMA pada test set yang sama (fair comparison)
6. Update retrain_status.json di setiap tahap 5. K-Fold + train model final (candidate) dengan sample_weight
6. Evaluasi candidate pada test set yang sama → delta terhadap old model
7. Promote jika lebih baik; rollback jika tidak
8. Setiap write status dilindungi FileLock (atomic)
""" """
import sys import sys
@ -17,12 +21,13 @@ import shutil
import logging import logging
import warnings import warnings
import re import re
import os import math
import pandas as pd import pandas as pd
import numpy as np import numpy as np
import joblib import joblib
from pathlib import Path from pathlib import Path
from datetime import datetime from datetime import datetime
from filelock import FileLock
from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression from sklearn.linear_model import LogisticRegression
@ -45,15 +50,50 @@ PIPELINE_BAK_PATH = ML_DIR / "ml_pipeline_internal.pkl.bak"
CANDIDATE_PATH = ML_DIR / "ml_pipeline_candidate.pkl" CANDIDATE_PATH = ML_DIR / "ml_pipeline_candidate.pkl"
METRICS_PATH = ML_DIR / "metrics_internal_only.json" METRICS_PATH = ML_DIR / "metrics_internal_only.json"
STATUS_PATH = ML_DIR / "retrain_status.json" STATUS_PATH = ML_DIR / "retrain_status.json"
CORPUS_PATH = DATA_DIR / "training_corpus_3.csv" LOCK_PATH = ML_DIR / "retrain_status.lock"
TEST_PATH = DATA_DIR / "test_set_3.csv" CORPUS_PATH = DATA_DIR / "training_corpus.csv"
TEST_PATH = DATA_DIR / "test_set.csv"
TARGET_CLASSES = ["Programmer", "Data Analyst", "Wirausaha Informatika", "Non-IT"] TARGET_CLASSES = ["Programmer", "Data Analyst", "Wirausaha Informatika", "Non-IT"]
MIN_IMPROVEMENT_THRESHOLD = 0.01 # Model baru harus lebih baik minimal +1% weighted F1
MAX_REGRESSION_ALLOWED = 0.02 # Toleransi degradasi maksimal 2% sebelum rollback keras
# Threshold: model baru harus lebih baik minimal MIN_IMPROVEMENT dari model lama # ── KONFIGURASI BOBOT OVERRIDE ──────────────────────────────────────────────
MIN_IMPROVEMENT_THRESHOLD = 0.01 # 1% weighted F1 #
MAX_REGRESSION_ALLOWED = 0.02 # Toleransi: model baru boleh lebih buruk max 2% (di luar ini = rollback keras) # Masalah yang diselesaikan:
# Pada skala besar (N_corpus >> N_override), bobot statis kehilangan daya
# akibat dilusi. Formula proporsi murni w = t*N / (M*(1-t)) menyelesaikan
# dilusi, tapi menghasilkan w=42.85 pada skenario 10k/100 — yang berisiko
# overfitting ekstrem pada noise override.
#
# Solusi: Logarithmic damping — tanpa tembok statis.
# w_raw = formula proporsi (jaminan 30% jika tidak di-damp)
# w = MIN + ln(1 + max(0, w_raw - MIN))
#
# Fase linear (w_raw rendah): w ≈ w_raw → proporsi terpenuhi
# Fase log (w_raw tinggi): w tumbuh tapi melambat → damp alami
#
# Implikasi jujur:
# Target 30% TIDAK dipertahankan di skala ekstrem. Ini trade-off yang
# disengaja: degradasi gradual lebih aman daripada overfitting ke 10
# baris override berbobot 42x. Tanpa tembok statis, redaman terjadi
# secara natural mengikuti kurva logaritmik, bukan menabrak batas arbitrer.
#
# Perilaku nyata (MIN=2.0, TARGET=0.30):
# w_raw │ w_log │ influence aktual
# ──────────────────────────────────
# 2.00 │ 2.000 │ formula <= MIN, pakai floor
# 4.29 │ 3.178 │ (1k corpus, 100 override) → ~24%
# 8.57 │ 3.999 │ (100 korpus, 5 override) → ~17%
# 42.86 │ 5.723 │ (10k corpus, 100 override)→ ~5.4%
# 428.60 │ 8.063 │ (10k corpus, 10 override) → ~0.8%
#
# Jika angka influence aktual dianggap terlalu kecil → naikkan TARGET.
# Jika model terlalu sensitif ke override → naikkan MIN agar floor lebih tinggi.
OVERRIDE_INFLUENCE_TARGET = 0.30 # Titik acuan proporsi (valid di skala normal)
OVERRIDE_MIN_WEIGHT = 2.0 # Lantai: override selalu minimal 2× korpus
# [FIX v4] STOPWORDS lengkap — versi retrain_worker2 hanya punya 10 kata (bug terpotong)
STOPWORDS = { STOPWORDS = {
"yang", "di", "ke", "dari", "dan", "atau", "dengan", "untuk", "pada", "dalam", "yang", "di", "ke", "dari", "dan", "atau", "dengan", "untuk", "pada", "dalam",
"adalah", "ini", "itu", "tidak", "juga", "sudah", "akan", "bisa", "ada", "oleh", "adalah", "ini", "itu", "tidak", "juga", "sudah", "akan", "bisa", "ada", "oleh",
@ -69,6 +109,39 @@ STOPWORDS = {
stemmer = StemmerFactory().create_stemmer() stemmer = StemmerFactory().create_stemmer()
# ──────────────────────────────────────────────────────────────
# DYNAMIC WEIGHT CALCULATOR
# ──────────────────────────────────────────────────────────────
def compute_override_weight(n_corpus: int, n_override: int) -> float:
"""
Hitung bobot override dengan logarithmic damping.
TIDAK ada OVERRIDE_MAX_WEIGHT — tembok statis menghancurkan jaminan
pengaruh tepat saat skala besar membutuhkannya. Sebagai gantinya,
fungsi ln(1+x) menyediakan redaman alami:
w_raw = (t * n_corpus) / (n_override * (1 - t)) # proporsi murni
w = MIN + ln(1 + max(0, w_raw - MIN)) # log damping
Jaminan matematis:
- w selalu >= OVERRIDE_MIN_WEIGHT (floor tetap ada)
- w tidak pernah meledak ke infinity karena ln tumbuh O(log n)
- Tidak ada tembok statis yang membuat influence kolaps tiba-tiba
Catatan interaksi:
LogisticRegression dipanggil dengan class_weight='balanced' DAN
sample_weight. Keduanya dikalikan oleh sklearn secara internal.
Artinya sampel override dari kelas minoritas mendapat boost ganda.
Pantau per-class F1 di metrics JSON untuk mendeteksi efek ini.
"""
if n_override == 0:
return 1.0
t = OVERRIDE_INFLUENCE_TARGET
w_raw = (t * n_corpus) / (n_override * (1.0 - t))
# ln damping: linear di zona rendah, melambat secara alami di skala besar
w_log = OVERRIDE_MIN_WEIGHT + math.log1p(max(0.0, w_raw - OVERRIDE_MIN_WEIGHT))
return round(w_log, 4)
# ────────────────────────────────────────────────────────────── # ──────────────────────────────────────────────────────────────
# LOGGING # LOGGING
# ────────────────────────────────────────────────────────────── # ──────────────────────────────────────────────────────────────
@ -79,56 +152,50 @@ logging.basicConfig(
) )
logger = logging.getLogger("retrain_worker") logger = logging.getLogger("retrain_worker")
# ────────────────────────────────────────────────────────────── # ──────────────────────────────────────────────────────────────
# STATUS WRITER # STATUS WRITER — atomic dengan FileLock (v4)
# ────────────────────────────────────────────────────────────── # ──────────────────────────────────────────────────────────────
def write_status(stage: str, message: str, extra: dict = None): def write_status(stage: str, message: str, extra: dict = None):
payload = { lock = FileLock(LOCK_PATH, timeout=10)
"stage": stage, with lock:
"message": message, payload = {"stage": stage, "message": message, "timestamp": datetime.now().isoformat()}
"timestamp": datetime.now().isoformat(), if extra: payload.update(extra)
}
if extra:
payload.update(extra)
STATUS_PATH.write_text(json.dumps(payload, indent=2, ensure_ascii=False)) STATUS_PATH.write_text(json.dumps(payload, indent=2, ensure_ascii=False))
logger.info(f"[{stage}] {message}") logger.info(f"[{stage}] {message}")
# ────────────────────────────────────────────────────────────── # ──────────────────────────────────────────────────────────────
# PREPROCESSING — IDENTIK dengan main.py # PREPROCESSING — dua fungsi terpisah (v4)
# ────────────────────────────────────────────────────────────── # ──────────────────────────────────────────────────────────────
def preprocess_text(text: str) -> str: def preprocess_stemmed(text: str) -> str:
if pd.isna(text) or not isinstance(text, str): """Untuk data korpus yang SUDAH di-stem oleh prepare_corpus — tidak re-stem."""
return "" if pd.isna(text) or not isinstance(text, str): return ""
text = text.lower().strip() text = text.lower().strip()
if text in {"nan", "none", "null", "-", "0", "", "tidak diisi"}: if text in {"nan", "none", "null", "-", "0", "", "tidak diisi"}: return ""
return "" text = text.replace('-', ' ').replace('/', ' ').replace('_', ' ').replace(',', ' ')
text = re.sub(r'[^\w\s]', '', text)
text = re.sub(r'\d+', '', text)
words = [w for w in text.split() if w not in STOPWORDS and len(w) > 2]
return " ".join(words)
def preprocess_raw(text: str) -> str:
"""Untuk data manual_override yang BELUM di-stem — jalankan Sastrawi."""
if pd.isna(text) or not isinstance(text, str): return ""
text = text.lower().strip()
if text in {"nan", "none", "null", "-", "0", "", "tidak diisi"}: return ""
text = text.replace('-', ' ').replace('/', ' ').replace('_', ' ').replace(',', ' ') text = text.replace('-', ' ').replace('/', ' ').replace('_', ' ').replace(',', ' ')
text = re.sub(r'[^\w\s]', '', text) text = re.sub(r'[^\w\s]', '', text)
text = re.sub(r'\d+', '', text) text = re.sub(r'\d+', '', text)
words = [w for w in text.split() if w not in STOPWORDS and len(w) > 2] words = [w for w in text.split() if w not in STOPWORDS and len(w) > 2]
return " ".join([stemmer.stem(w) for w in words]) return " ".join([stemmer.stem(w) for w in words])
# ────────────────────────────────────────────────────────────── # ──────────────────────────────────────────────────────────────
# EVALUASI MODEL — weighted F1 pada hold-out test # EVALUASI MODEL
# ────────────────────────────────────────────────────────────── # ──────────────────────────────────────────────────────────────
def evaluate_model(model, X_test: pd.Series, y_test: pd.Series) -> dict: def evaluate_model(model, X_test: pd.Series, y_test: pd.Series) -> dict:
"""
Evaluasi model pada hold-out test set.
Return: dict berisi weighted F1, accuracy, dan per-class metrics.
"""
if len(X_test) == 0: if len(X_test) == 0:
return {"weighted_f1": 0.0, "accuracy": 0.0, "per_class": {}} return {"weighted_f1": 0.0, "accuracy": 0.0, "per_class": {}}
y_pred = model.predict(X_test) y_pred = model.predict(X_test)
report = classification_report( report = classification_report(y_test, y_pred, output_dict=True, zero_division=0, labels=TARGET_CLASSES)
y_test, y_pred,
output_dict=True,
zero_division=0,
labels=TARGET_CLASSES
)
return { return {
"weighted_f1": round(report.get("weighted avg", {}).get("f1-score", 0.0), 4), "weighted_f1": round(report.get("weighted avg", {}).get("f1-score", 0.0), 4),
"accuracy": round(report.get("accuracy", 0.0), 4), "accuracy": round(report.get("accuracy", 0.0), 4),
@ -143,52 +210,46 @@ def evaluate_model(model, X_test: pd.Series, y_test: pd.Series) -> dict:
} }
} }
# ──────────────────────────────────────────────────────────────
# BASELINE F1 — re-evaluasi model lama pada test set yang sama
# ──────────────────────────────────────────────────────────────
def get_baseline_f1_on_testset(X_test: pd.Series, y_test: pd.Series) -> float:
"""
Evaluasi model lama (backup) pada X_test/y_test yang SAMA dengan yang
akan digunakan mengevaluasi candidate. Ini satu-satunya cara yang adil
(apples-to-apples) karena distribusi test set berubah setiap retrain.
# ────────────────────────────────────────────────────────────── Membandingkan F1 baru dengan angka JSON lama (dihitung di distribusi berbeda)
# AMBIL BASELINE DARI METRICS JSON adalah perbandingan apel vs jeruk — tidak valid untuk keputusan promote/rollback.
# ──────────────────────────────────────────────────────────────
def get_baseline_f1() -> float:
""" """
Baca weighted F1 model lama dari metrics_internal_only.json. if not PIPELINE_BAK_PATH.exists():
Fallback ke 0 jika file tidak ada. logger.warning("Backup .bak tidak ditemukan — baseline F1 diasumsikan 0.0")
""" return 0.0
try: try:
if METRICS_PATH.exists(): old_model = joblib.load(PIPELINE_BAK_PATH)
m = json.loads(METRICS_PATH.read_text()) result = evaluate_model(old_model, X_test, y_test)
return float(m.get("hold_out_test", {}).get("weighted avg", {}).get("f1-score", 0.0)) logger.info(f"Baseline F1 (old model, same test set): {result['weighted_f1']:.4f}")
except Exception as e: return result["weighted_f1"]
logger.warning(f"Gagal baca baseline metrics: {e}") except Exception as e:
logger.warning(f"Gagal load/evaluasi model lama untuk baseline: {e}")
return 0.0 return 0.0
# ────────────────────────────────────────────────────────────── # ──────────────────────────────────────────────────────────────
# ROLLBACK # ROLLBACK
# ────────────────────────────────────────────────────────────── # ──────────────────────────────────────────────────────────────
def do_rollback(reason: str, old_f1: float, new_f1: float): def do_rollback(reason: str, old_f1: float, new_f1: float):
"""Kembalikan pkl aktif ke backup.""" if CANDIDATE_PATH.exists(): CANDIDATE_PATH.unlink()
# Bersihkan candidate jika ada
if CANDIDATE_PATH.exists():
CANDIDATE_PATH.unlink()
# Restore dari backup
if PIPELINE_BAK_PATH.exists(): if PIPELINE_BAK_PATH.exists():
shutil.copy2(PIPELINE_BAK_PATH, PIPELINE_PATH) shutil.copy2(PIPELINE_BAK_PATH, PIPELINE_PATH)
logger.info(f"Rollback berhasil: pkl lama dipulihkan dari .bak") logger.info("Rollback berhasil: pkl lama dipulihkan dari .bak")
else: else:
logger.warning("File .bak tidak ditemukan, pkl aktif dibiarkan.") logger.warning("File .bak tidak ditemukan — pkl aktif dibiarkan.")
write_status( write_status(
stage="rolled_back", stage="rolled_back",
message=f"Model lama dipertahankan. {reason}", message=f"Model lama dipertahankan. {reason}",
extra={ extra={"result": "rolled_back", "reason": reason, "old_f1": old_f1, "new_f1": new_f1}
"result": "rolled_back",
"reason": reason,
"old_f1": old_f1,
"new_f1": new_f1,
}
) )
# ────────────────────────────────────────────────────────────── # ──────────────────────────────────────────────────────────────
# MAIN # MAIN
# ────────────────────────────────────────────────────────────── # ──────────────────────────────────────────────────────────────
@ -197,15 +258,15 @@ def main(extra_csv_path: str = None):
# ── STEP 1: BACKUP ────────────────────────────────────── # ── STEP 1: BACKUP ──────────────────────────────────────
write_status("backup", "Membuat backup model lama...") write_status("backup", "Membuat backup model lama...")
if PIPELINE_PATH.exists(): if not PIPELINE_PATH.exists():
shutil.copy2(PIPELINE_PATH, PIPELINE_BAK_PATH)
logger.info(f"Backup tersimpan: {PIPELINE_BAK_PATH}")
else:
write_status("failed", "File pkl aktif tidak ditemukan, tidak bisa backup.") write_status("failed", "File pkl aktif tidak ditemukan, tidak bisa backup.")
sys.exit(1) sys.exit(1)
shutil.copy2(PIPELINE_PATH, PIPELINE_BAK_PATH)
logger.info(f"Backup tersimpan: {PIPELINE_BAK_PATH}")
baseline_f1 = get_baseline_f1() # baseline_f1 akan dihitung setelah split dinamis terbentuk
logger.info(f"Baseline weighted F1 (model lama): {baseline_f1:.4f}") # (evaluasi model lama pada test set yang sama dengan candidate)
baseline_f1 = 0.0
try: try:
# ── STEP 2: LOAD & MERGE DATA ──────────────────────── # ── STEP 2: LOAD & MERGE DATA ────────────────────────
@ -215,57 +276,70 @@ def main(extra_csv_path: str = None):
raise FileNotFoundError(f"Corpus asli tidak ditemukan: {CORPUS_PATH}") raise FileNotFoundError(f"Corpus asli tidak ditemukan: {CORPUS_PATH}")
df_corpus = pd.read_csv(CORPUS_PATH, sep=";", dtype=str).dropna(subset=["job_text_raw", "label"]) df_corpus = pd.read_csv(CORPUS_PATH, sep=";", dtype=str).dropna(subset=["job_text_raw", "label"])
df_corpus["features"] = df_corpus["job_text_raw"].apply(preprocess_stemmed)
df_corpus["_weight"] = 1.0 # Bobot normal untuk data historis
logger.info(f"Corpus asli: {len(df_corpus)} baris") logger.info(f"Corpus asli: {len(df_corpus)} baris")
df_extra = pd.DataFrame() df_extra = pd.DataFrame()
if extra_csv_path and Path(extra_csv_path).exists(): if extra_csv_path and Path(extra_csv_path).exists():
df_extra = pd.read_csv(extra_csv_path, sep=";", dtype=str).dropna(subset=["job_text_raw", "label"]) df_extra = pd.read_csv(extra_csv_path, sep=";", dtype=str).dropna(subset=["job_text_raw", "label"])
# Validasi label df_extra = df_extra[df_extra["label"].isin(set(TARGET_CLASSES))]
valid_labels = set(TARGET_CLASSES) df_extra["features"] = df_extra["job_text_raw"].apply(preprocess_raw)
df_extra = df_extra[df_extra["label"].isin(valid_labels)] override_w = compute_override_weight(len(df_corpus), len(df_extra))
logger.info(f"Data manual_override: {len(df_extra)} baris valid") df_extra["_weight"] = override_w
# Pre-compute total bobot sekali — dipakai di logging & metrics JSON.
# .sum() lebih benar daripada len() * .iloc[0] karena tidak mengasumsikan
# homogenitas bobot di seluruh baris (future-proof jika bobot per-baris ditambahkan).
total_corpus_weight = df_corpus["_weight"].sum()
total_override_weight = df_extra["_weight"].sum() if len(df_extra) > 0 else 0.0
actual_influence_pct = (
total_override_weight / (total_corpus_weight + total_override_weight) * 100
if total_override_weight > 0 else 0.0
)
if len(df_extra) > 0: if len(df_extra) > 0:
df_all = pd.concat([df_corpus, df_extra], ignore_index=True) logger.info(
# Deduplicate: preferensikan data extra (manual override) jika job_text_raw sama f"Data manual_override: {len(df_extra)} baris | "
df_all = df_all.drop_duplicates(subset=["job_text_raw"], keep="last") f"weight={override_w:.4f}x (log-damped) | "
f"influence aktual={actual_influence_pct:.1f}% "
f"(target nominal {OVERRIDE_INFLUENCE_TARGET*100:.0f}%)"
)
else: else:
df_all = df_corpus.copy()
logger.info("Tidak ada data tambahan — menggunakan corpus asli saja") logger.info("Tidak ada data tambahan — menggunakan corpus asli saja")
logger.info(f"Total setelah merge & deduplicate: {len(df_all)} baris") # Concat tanpa deduplication — semua frekuensi historis dipertahankan
df_all = pd.concat([df_corpus, df_extra], ignore_index=True) if len(df_extra) > 0 else df_corpus.copy()
mask = df_all["features"].str.len() > 0
df_all = df_all[mask]
logger.info(f"Total setelah merge (tanpa deduplicate): {len(df_all)} baris")
if len(df_all) < 20: if len(df_all) < 20:
raise ValueError(f"Data terlalu sedikit untuk training: {len(df_all)} baris (minimum 20)") raise ValueError(f"Data terlalu sedikit untuk training: {len(df_all)} baris (minimum 20)")
# ── STEP 3: PREPROCESSING ──────────────────────────── X_all = df_all["features"]
write_status("preprocessing", "Preprocessing teks...")
X_all = df_all["job_text_raw"].apply(preprocess_text)
y_all = df_all["label"] y_all = df_all["label"]
mask = X_all.str.len() > 0 w_all = df_all["_weight"]
X_all, y_all = X_all[mask], y_all[mask]
logger.info(f"Setelah filter kosong: {len(X_all)} baris")
# ── STEP 4: SPLIT TRAIN / TEST ─────────────────────── # ── STEP 3: DYNAMIC SPLIT — selalu dari data gabungan terkini ────
# Cek apakah test_set_3.csv ada untuk hold-out # Test set statis (test_set.csv) tidak digunakan karena:
if TEST_PATH.exists(): # (a) tidak mencerminkan pola baru dari data override
df_test = pd.read_csv(TEST_PATH, sep=";", dtype=str).dropna(subset=["job_text_raw", "label"]) # (b) baseline_f1 dari JSON dihitung pada distribusi berbeda →
X_test = df_test["job_text_raw"].apply(preprocess_text) # perbandingan apel vs jeruk, tidak valid untuk keputusan promote.
y_test = df_test["label"] # Solusi: split dinamis, lalu evaluasi model LAMA pada test set YANG SAMA.
mask_t = X_test.str.len() > 0 X_train, X_test, y_train, y_test, w_train, _ = train_test_split(
X_test, y_test = X_test[mask_t], y_test[mask_t] X_all, y_all, w_all, test_size=0.3, random_state=42, stratify=y_all
X_train, y_train = X_all, y_all
logger.info(f"Hold-out test dari file: {len(X_test)} baris")
else:
# Fallback: split 70/30 dari data gabungan
X_train, X_test, y_train, y_test = train_test_split(
X_all, y_all, test_size=0.3, random_state=42, stratify=y_all
) )
logger.info(f"Hold-out test dari split 30%: {len(X_test)} baris") logger.info(f"Dynamic split: {len(X_train)} train | {len(X_test)} test")
# ── STEP 5: TRAINING ───────────────────────────────── # ── STEP 4: BASELINE — evaluasi model LAMA pada test set yang sama ──
write_status("training", f"Training model baru... ({len(X_train)} sampel training)") # Ini satu-satunya cara perbandingan yang jujur (apples-to-apples).
write_status("evaluating", "Mengevaluasi model lama pada test set baru...")
baseline_f1 = get_baseline_f1_on_testset(X_test, y_test)
# ── STEP 5: K-FOLD VALIDATION ────────────────────────
write_status("training", f"K-Fold validation & training... ({len(X_train)} sampel)")
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42) skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
fold_metrics = [] fold_metrics = []
for i, (tr_idx, te_idx) in enumerate(skf.split(X_train, y_train)): for i, (tr_idx, te_idx) in enumerate(skf.split(X_train, y_train)):
@ -273,70 +347,68 @@ def main(extra_csv_path: str = None):
('tfidf', TfidfVectorizer(max_features=3000, ngram_range=(1, 2), sublinear_tf=True, min_df=1)), ('tfidf', TfidfVectorizer(max_features=3000, ngram_range=(1, 2), sublinear_tf=True, min_df=1)),
('clf', LogisticRegression(max_iter=1000, class_weight='balanced', solver='lbfgs')) ('clf', LogisticRegression(max_iter=1000, class_weight='balanced', solver='lbfgs'))
]) ])
m.fit(X_train.iloc[tr_idx], y_train.iloc[tr_idx]) # sample_weight diteruskan ke step 'clf' via Pipeline naming convention
m.fit(
X_train.iloc[tr_idx], y_train.iloc[tr_idx],
clf__sample_weight=w_train.iloc[tr_idx].values
)
y_pred = m.predict(X_train.iloc[te_idx]) y_pred = m.predict(X_train.iloc[te_idx])
rep = classification_report(y_train.iloc[te_idx], y_pred, output_dict=True, zero_division=0) rep = classification_report(y_train.iloc[te_idx], y_pred, output_dict=True, zero_division=0)
fold_metrics.append(rep) fold_metrics.append(rep)
write_status("training", f"K-Fold selesai: fold {i+1}/5 | acc={rep['accuracy']:.4f}") write_status("training", f"K-Fold selesai: fold {i+1}/5 | acc={rep['accuracy']:.4f}")
# Train final model on full training set # ── STEP 6: TRAINING MODEL FINAL ─────────────────────
write_status("training", "Training model final pada seluruh data training...") write_status("training", "Training model final pada seluruh data training...")
candidate_model = Pipeline([ candidate_model = Pipeline([
('tfidf', TfidfVectorizer(max_features=3000, ngram_range=(1, 2), sublinear_tf=True, min_df=1)), ('tfidf', TfidfVectorizer(max_features=3000, ngram_range=(1, 2), sublinear_tf=True, min_df=1)),
('clf', LogisticRegression(max_iter=1000, class_weight='balanced', solver='lbfgs')) ('clf', LogisticRegression(max_iter=1000, class_weight='balanced', solver='lbfgs'))
]) ])
candidate_model.fit(X_train, y_train) candidate_model.fit(X_train, y_train, clf__sample_weight=w_train.values)
# Simpan sebagai candidate (belum overwrite aktif)
joblib.dump(candidate_model, CANDIDATE_PATH) joblib.dump(candidate_model, CANDIDATE_PATH)
logger.info(f"Candidate model tersimpan: {CANDIDATE_PATH}") logger.info(f"Candidate model tersimpan: {CANDIDATE_PATH}")
# ── STEP 6: EVALUASI & KEPUTUSAN ───────────────────── # ── STEP 7: EVALUASI & KEPUTUSAN ─────────────────────
write_status("evaluating", "Mengevaluasi model baru vs model lama...") write_status("evaluating", "Mengevaluasi model baru vs model lama (test set sama)...")
new_metrics = evaluate_model(candidate_model, X_test, y_test) new_metrics = evaluate_model(candidate_model, X_test, y_test)
new_f1 = new_metrics["weighted_f1"] new_f1 = new_metrics["weighted_f1"]
delta = new_f1 - baseline_f1 delta = new_f1 - baseline_f1
logger.info(f"Baseline F1 : {baseline_f1:.4f}") logger.info(f"Baseline F1 (old model, same test) : {baseline_f1:.4f}")
logger.info(f"Candidate F1: {new_f1:.4f} (delta: {delta:+.4f})") logger.info(f"Candidate F1 (new model, same test): {new_f1:.4f} (delta: {delta:+.4f})")
# ── KEPUTUSAN ────────────────────────────────────────
if baseline_f1 == 0.0: if baseline_f1 == 0.0:
# Tidak ada baseline (model lama belum pernah dievaluasi) → langsung promote
reason_promote = "Tidak ada baseline metrics → model baru dipromote"
should_promote = True should_promote = True
reason = "Model lama tidak bisa dievaluasi (backup tidak ada) → model baru dipromote"
elif delta >= MIN_IMPROVEMENT_THRESHOLD: elif delta >= MIN_IMPROVEMENT_THRESHOLD:
should_promote = True should_promote = True
reason_promote = f"Model baru lebih baik (+{delta*100:.2f}% weighted F1)" reason = f"Model baru lebih baik (+{delta*100:.2f}% weighted F1)"
elif delta < -MAX_REGRESSION_ALLOWED: elif delta < -MAX_REGRESSION_ALLOWED:
should_promote = False should_promote = False
reason_rollback = f"Model baru lebih buruk secara signifikan ({delta*100:.2f}% weighted F1)" reason = f"Model baru lebih buruk secara signifikan ({delta*100:.2f}% weighted F1)"
else: else:
should_promote = False should_promote = False
reason_rollback = ( reason = (
f"Peningkatan tidak signifikan ({delta*100:.2f}% weighted F1, " f"Peningkatan tidak signifikan ({delta*100:.2f}% weighted F1, "
f"minimum dibutuhkan: +{MIN_IMPROVEMENT_THRESHOLD*100:.1f}%)" f"minimum dibutuhkan: +{MIN_IMPROVEMENT_THRESHOLD*100:.1f}%)"
) )
if not should_promote: if not should_promote:
do_rollback(reason_rollback, baseline_f1, new_f1) do_rollback(reason, baseline_f1, new_f1)
return return
# ── STEP 7: PROMOTE ────────────────────────────────── # ── STEP 7: PROMOTE ──────────────────────────────────
write_status("promoting", "Model baru lebih baik — mempromote model baru...") write_status("promoting", "Model baru lebih baik — mempromote model baru...")
# Atomic rename: candidate → aktif
shutil.move(str(CANDIDATE_PATH), str(PIPELINE_PATH)) shutil.move(str(CANDIDATE_PATH), str(PIPELINE_PATH))
logger.info(f"Model baru dipromote: {PIPELINE_PATH}") logger.info(f"Model baru dipromote: {PIPELINE_PATH}")
# Update metrics JSON
avg_acc = float(np.mean([f["accuracy"] for f in fold_metrics])) avg_acc = float(np.mean([f["accuracy"] for f in fold_metrics]))
y_pred_final = candidate_model.predict(X_test)
new_metrics_full = { new_metrics_full = {
"methodology": "Internal MIF + manual_override", "methodology": "Internal MIF + manual_override (sample_weight, dynamic split)",
"retrained_at": datetime.now().isoformat(), "retrained_at": datetime.now().isoformat(),
"extra_samples_added": len(df_extra), "extra_samples_added": len(df_extra),
"override_weight_used": round(df_extra["_weight"].iloc[0], 4) if len(df_extra) > 0 else 1.0,
"override_influence_pct_actual": round(actual_influence_pct, 2),
"override_influence_target_pct": OVERRIDE_INFLUENCE_TARGET * 100,
"total_training_samples": len(X_train), "total_training_samples": len(X_train),
"k_fold": { "k_fold": {
"accuracy_mean": avg_acc, "accuracy_mean": avg_acc,
@ -345,12 +417,12 @@ def main(extra_csv_path: str = None):
}, },
"threshold_config": 0.50, "threshold_config": 0.50,
"hold_out_test": classification_report( "hold_out_test": classification_report(
y_test, y_test, candidate_model.predict(X_test),
y_pred_final, output_dict=True, zero_division=0
output_dict=True,
zero_division=0
), ),
# Perbandingan fair: kedua model dievaluasi pada test set yang SAMA
"comparison": { "comparison": {
"note": "Both models evaluated on identical dynamic test set",
"old_weighted_f1": baseline_f1, "old_weighted_f1": baseline_f1,
"new_weighted_f1": new_f1, "new_weighted_f1": new_f1,
"delta": round(delta, 4), "delta": round(delta, 4),
@ -358,10 +430,9 @@ def main(extra_csv_path: str = None):
} }
METRICS_PATH.write_text(json.dumps(new_metrics_full, indent=2, ensure_ascii=False)) METRICS_PATH.write_text(json.dumps(new_metrics_full, indent=2, ensure_ascii=False))
write_status( write_status(
stage="promoted", stage="promoted",
message=f"Model baru berhasil dipromote. {reason_promote}", message=f"Model baru berhasil dipromote. {reason}",
extra={ extra={
"result": "promoted", "result": "promoted",
"old_f1": baseline_f1, "old_f1": baseline_f1,
@ -382,6 +453,8 @@ def main(extra_csv_path: str = None):
new_f1=0.0 new_f1=0.0
) )
# Override stage ke 'failed' agar UI tahu ini bukan rollback biasa # Override stage ke 'failed' agar UI tahu ini bukan rollback biasa
lock = FileLock(LOCK_PATH, timeout=10)
with lock:
status = json.loads(STATUS_PATH.read_text()) status = json.loads(STATUS_PATH.read_text())
status["stage"] = "failed" status["stage"] = "failed"
STATUS_PATH.write_text(json.dumps(status, indent=2, ensure_ascii=False)) STATUS_PATH.write_text(json.dumps(status, indent=2, ensure_ascii=False))
@ -389,6 +462,4 @@ def main(extra_csv_path: str = None):
if __name__ == "__main__": if __name__ == "__main__":
# Argumen: [extra_csv_path] main(sys.argv[1] if len(sys.argv) > 1 else None)
extra_csv = sys.argv[1] if len(sys.argv) > 1 else None
main(extra_csv)

BIN
fastapi/ml_assets/confidence_distribution.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 60 KiB

After

Width:  |  Height:  |  Size: 62 KiB

BIN
fastapi/ml_assets/confusion_matrix_test.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 113 KiB

After

Width:  |  Height:  |  Size: 114 KiB

230
fastapi/ml_assets/metrics_internal_only.json
View File

@ -1,8 +1,8 @@
{ {
"methodology": "Internal MIF only", "methodology": "Internal MIF only",
"k_fold": { "k_fold": {
"accuracy_mean": 0.7902040816326531, "accuracy_mean": 0.728,
"accuracy_std": 0.046079445366646855, "accuracy_std": 0.08634813257969161,
"folds": [ "folds": [
{ {
"Data Analyst": { "Data Analyst": {
@ -12,34 +12,34 @@
"support": 1.0 "support": 1.0
}, },
"Non-IT": { "Non-IT": {
"precision": 0.7619047619047619, "precision": 0.8,
"recall": 1.0, "recall": 0.8888888888888888,
"f1-score": 0.8648648648648649, "f1-score": 0.8421052631578947,
"support": 16.0 "support": 18.0
}, },
"Programmer": { "Programmer": {
"precision": 1.0, "precision": 0.8,
"recall": 0.5714285714285714, "recall": 0.6153846153846154,
"f1-score": 0.7272727272727273, "f1-score": 0.6956521739130435,
"support": 14.0 "support": 13.0
}, },
"Wirausaha Informatika": { "Wirausaha Informatika": {
"precision": 0.8571428571428571, "precision": 0.8,
"recall": 0.9473684210526315, "recall": 0.8888888888888888,
"f1-score": 0.9, "f1-score": 0.8421052631578947,
"support": 19.0 "support": 18.0
}, },
"accuracy": 0.84, "accuracy": 0.8,
"macro avg": { "macro avg": {
"precision": 0.6547619047619048, "precision": 0.6000000000000001,
"recall": 0.6296992481203008, "recall": 0.5982905982905983,
"f1-score": 0.6230343980343981, "f1-score": 0.5949656750572082,
"support": 50.0 "support": 50.0
}, },
"weighted avg": { "weighted avg": {
"precision": 0.8495238095238095, "precision": 0.784,
"recall": 0.84, "recall": 0.8,
"f1-score": 0.8223931203931204, "f1-score": 0.7871853546910755,
"support": 50.0 "support": 50.0
} }
}, },
@ -51,10 +51,10 @@
"support": 1.0 "support": 1.0
}, },
"Non-IT": { "Non-IT": {
"precision": 0.8, "precision": 0.7727272727272727,
"recall": 1.0, "recall": 1.0,
"f1-score": 0.8888888888888888, "f1-score": 0.8717948717948718,
"support": 16.0 "support": 17.0
}, },
"Programmer": { "Programmer": {
"precision": 1.0, "precision": 1.0,
@ -63,37 +63,37 @@
"support": 14.0 "support": 14.0
}, },
"Wirausaha Informatika": { "Wirausaha Informatika": {
"precision": 0.7083333333333334, "precision": 0.6818181818181818,
"recall": 0.8947368421052632, "recall": 0.8333333333333334,
"f1-score": 0.7906976744186046, "f1-score": 0.75,
"support": 19.0 "support": 18.0
}, },
"accuracy": 0.78, "accuracy": 0.76,
"macro avg": { "macro avg": {
"precision": 0.6270833333333333, "precision": 0.6136363636363636,
"recall": 0.5808270676691729, "recall": 0.5654761904761905,
"f1-score": 0.5698966408268734, "f1-score": 0.555448717948718,
"support": 50.0 "support": 50.0
}, },
"weighted avg": { "weighted avg": {
"precision": 0.8051666666666667, "precision": 0.7881818181818181,
"recall": 0.78, "recall": 0.76,
"f1-score": 0.7529095607235142, "f1-score": 0.7344102564102565,
"support": 50.0 "support": 50.0
} }
}, },
{ {
"Data Analyst": { "Data Analyst": {
"precision": 0.0, "precision": 1.0,
"recall": 0.0, "recall": 1.0,
"f1-score": 0.0, "f1-score": 1.0,
"support": 2.0 "support": 1.0
}, },
"Non-IT": { "Non-IT": {
"precision": 0.7142857142857143, "precision": 0.6296296296296297,
"recall": 1.0, "recall": 1.0,
"f1-score": 0.8333333333333334, "f1-score": 0.7727272727272727,
"support": 15.0 "support": 17.0
}, },
"Programmer": { "Programmer": {
"precision": 1.0, "precision": 1.0,
@ -102,101 +102,101 @@
"support": 14.0 "support": 14.0
}, },
"Wirausaha Informatika": { "Wirausaha Informatika": {
"precision": 0.7916666666666666, "precision": 0.9411764705882353,
"recall": 1.0, "recall": 0.8888888888888888,
"f1-score": 0.8837209302325582, "f1-score": 0.9142857142857143,
"support": 19.0 "support": 18.0
}, },
"accuracy": 0.78, "accuracy": 0.78,
"macro avg": { "macro avg": {
"precision": 0.6264880952380952, "precision": 0.8927015250544663,
"recall": 0.5892857142857143, "recall": 0.8115079365079365,
"f1-score": 0.560842513259894, "f1-score": 0.8033321941216678,
"support": 50.0 "support": 50.0
}, },
"weighted avg": { "weighted avg": {
"precision": 0.7951190476190476, "precision": 0.8528976034858389,
"recall": 0.78, "recall": 0.78,
"f1-score": 0.7331823745410037, "f1-score": 0.7592385509227614,
"support": 50.0 "support": 50.0
} }
}, },
{ {
"Data Analyst": { "Data Analyst": {
"precision": 0.0, "precision": 1.0,
"recall": 0.0, "recall": 1.0,
"f1-score": 0.0, "f1-score": 1.0,
"support": 1.0 "support": 1.0
}, },
"Non-IT": { "Non-IT": {
"precision": 0.7272727272727273, "precision": 0.6538461538461539,
"recall": 1.0, "recall": 1.0,
"f1-score": 0.8421052631578947, "f1-score": 0.7906976744186046,
"support": 16.0 "support": 17.0
}, },
"Programmer": { "Programmer": {
"precision": 1.0, "precision": 1.0,
"recall": 0.5, "recall": 0.42857142857142855,
"f1-score": 0.6666666666666666, "f1-score": 0.6,
"support": 14.0 "support": 14.0
}, },
"Wirausaha Informatika": { "Wirausaha Informatika": {
"precision": 0.9, "precision": 0.7647058823529411,
"recall": 1.0, "recall": 0.7222222222222222,
"f1-score": 0.9473684210526315, "f1-score": 0.7428571428571429,
"support": 18.0 "support": 18.0
}, },
"accuracy": 0.8367346938775511, "accuracy": 0.74,
"macro avg": { "macro avg": {
"precision": 0.6568181818181819, "precision": 0.8546380090497737,
"recall": 0.625, "recall": 0.7876984126984127,
"f1-score": 0.6140350877192982, "f1-score": 0.7833887043189369,
"support": 49.0 "support": 50.0
}, },
"weighted avg": { "weighted avg": {
"precision": 0.8538033395176252, "precision": 0.797601809954751,
"recall": 0.8367346938775511, "recall": 0.74,
"f1-score": 0.8134622269960615, "f1-score": 0.724265780730897,
"support": 49.0 "support": 50.0
} }
}, },
{ {
"Data Analyst": { "Data Analyst": {
"precision": 0.0, "precision": 0.07692307692307693,
"recall": 0.0, "recall": 0.5,
"f1-score": 0.0, "f1-score": 0.13333333333333333,
"support": 1.0 "support": 2.0
}, },
"Non-IT": { "Non-IT": {
"precision": 0.8, "precision": 0.6923076923076923,
"recall": 1.0, "recall": 1.0,
"f1-score": 0.8888888888888888, "f1-score": 0.8181818181818182,
"support": 16.0 "support": 18.0
}, },
"Programmer": { "Programmer": {
"precision": 1.0, "precision": 1.0,
"recall": 0.21428571428571427, "recall": 0.23076923076923078,
"f1-score": 0.35294117647058826, "f1-score": 0.375,
"support": 14.0 "support": 13.0
}, },
"Wirausaha Informatika": { "Wirausaha Informatika": {
"precision": 0.6153846153846154, "precision": 0.75,
"recall": 0.8888888888888888, "recall": 0.35294117647058826,
"f1-score": 0.7272727272727273, "f1-score": 0.48,
"support": 18.0 "support": 17.0
}, },
"accuracy": 0.7142857142857143, "accuracy": 0.56,
"macro avg": { "macro avg": {
"precision": 0.6038461538461539, "precision": 0.6298076923076923,
"recall": 0.5257936507936507, "recall": 0.5209276018099548,
"f1-score": 0.4922756981580511, "f1-score": 0.4516287878787879,
"support": 49.0 "support": 50.0
}, },
"weighted avg": { "weighted avg": {
"precision": 0.7729984301412873, "precision": 0.7673076923076922,
"recall": 0.7142857142857143, "recall": 0.56,
"f1-score": 0.6582511792595825, "f1-score": 0.5605787878787879,
"support": 49.0 "support": 50.0
} }
} }
] ]
@ -210,35 +210,35 @@
"support": 2.0 "support": 2.0
}, },
"Non-IT": { "Non-IT": {
"precision": 0.7083333333333334, "precision": 0.7777777777777778,
"recall": 1.0, "recall": 0.9722222222222222,
"f1-score": 0.8292682926829268, "f1-score": 0.8641975308641975,
"support": 34.0 "support": 36.0
}, },
"Programmer": { "Programmer": {
"precision": 0.9444444444444444, "precision": 0.95,
"recall": 0.5483870967741935, "recall": 0.6333333333333333,
"f1-score": 0.6938775510204082, "f1-score": 0.76,
"support": 31.0 "support": 30.0
}, },
"Wirausaha Informatika": { "Wirausaha Informatika": {
"precision": 0.8333333333333334, "precision": 0.8571428571428571,
"recall": 0.8536585365853658, "recall": 0.9230769230769231,
"f1-score": 0.8433734939759037, "f1-score": 0.8888888888888888,
"support": 41.0 "support": 39.0
}, },
"accuracy": 0.7962962962962963, "accuracy": 0.8411214953271028,
"macro avg": { "macro avg": {
"precision": 0.6215277777777778, "precision": 0.6462301587301588,
"recall": 0.6005114083398898, "recall": 0.6321581196581196,
"f1-score": 0.5916298344198097, "f1-score": 0.6282716049382716,
"support": 108.0 "support": 107.0
}, },
"weighted avg": { "weighted avg": {
"precision": 0.8104423868312758, "precision": 0.8404539385847796,
"recall": 0.7962962962962963, "recall": 0.8411214953271028,
"f1-score": 0.7804040674617057, "f1-score": 0.8278296988577363,
"support": 108.0 "support": 107.0
} }
} }
} }

BIN
fastapi/ml_assets/ml_pipeline_internal.pkl
View File

Binary file not shown.

10
fastapi/ml_assets/retrain_status.json
View File

@ -1,10 +1,10 @@
{ {
"stage": "rolled_back", "stage": "rolled_back",
"message": "Model lama dipertahankan. Model baru lebih buruk secara signifikan (-12.92% weighted F1)", "message": "Model lama dipertahankan. Model baru lebih buruk secara signifikan (-22.52% weighted F1)",
"timestamp": "2026-05-25T06:51:07.653672", "timestamp": "2026-05-28T16:37:10.275000",
"result": "rolled_back", "result": "rolled_back",
"reason": "Model baru lebih buruk secara signifikan (-12.92% weighted F1)", "reason": "Model baru lebih buruk secara signifikan (-22.52% weighted F1)",
"old_f1": 0.7804040674617057, "old_f1": 0.8788,
"new_f1": 0.6512, "new_f1": 0.6536,
"_reloaded": true "_reloaded": true
} }

BIN
fastapi/requirements.txt
View File

Binary file not shown.

80
sista_mif/app/Http/Controllers/ClassificationTaskController.php
View File

@ -18,22 +18,46 @@ public function __construct(protected FastApiWorkerService $fastApi) {}
public function upload(Request $request) { public function upload(Request $request) {
$request->validate(['file' => 'required|file|mimes:csv,xlsx,xls|max:20480']); $request->validate(['file' => 'required|file|mimes:csv,xlsx,xls|max:20480']);
$result = $this->fastApi->classifyFile($request->file('file')->getRealPath(), $request->file('file')->getClientOriginalName()); $file = $request->file('file');
$result = $this->fastApi->classifyFile($file->getRealPath(), $file->getClientOriginalName());
if (!isset($result['results']) && isset($result['error'])) { if (isset($result['error'])) {
if ($request->wantsJson()) return response()->json(['success' => false, 'message' => $result['error']], 400);
return back()->with('error', $result['error']); return back()->with('error', $result['error']);
} }
if (empty($result['results'])) { if ($request->wantsJson()) {
return back()->with('success', "Selesai. 0 data berhasil diproses."); return response()->json([
'success' => true,
'job_id' => $result['job_id'] ?? null,
'message' => $result['message'] ?? 'Proses dimulai...'
]);
}
return back()->with('success', 'File sedang diproses. Silakan refresh beberapa saat lagi.');
} }
DB::transaction(function () use ($result) { public function checkStatus(Request $request, string $jobId)
{
$status = $this->fastApi->getClassifyStatus($jobId);
if (!isset($status['status']) || in_array($status['status'], ['pending', 'processing'])) {
return response()->json($status);
}
if ($status['status'] === 'error') {
return response()->json($status);
}
if ($status['status'] === 'success' && isset($status['results'])) {
try {
DB::transaction(function () use ($status) {
$chunks = array_chunk($status['results'], 500);
foreach ($chunks as $chunk) {
$kemendikUpsertData = []; $kemendikUpsertData = [];
$internalResultsUpsertData = []; $internalResultsUpsertData = [];
$internalRawUpsertData = []; $internalRawUpsertData = [];
foreach ($result['results'] as $row) { foreach ($chunk as $row) {
if ($row['source_type'] === 'kemendik') { if ($row['source_type'] === 'kemendik') {
$raw = $row['raw_data'] ?? []; $raw = $row['raw_data'] ?? [];
$kemendikUpsertData[] = [ $kemendikUpsertData[] = [
@ -52,8 +76,6 @@ public function upload(Request $request) {
$internalResultsUpsertData[] = [ $internalResultsUpsertData[] = [
'nim' => $row['nim'], 'nim' => $row['nim'],
'source_type' => 'internal_mif', 'source_type' => 'internal_mif',
'nama' => $row['nama'],
'tahun_lulus' => $row['tahun_lulus'] ?? null,
'job_text_raw' => $row['job_text_raw'] ?? '', 'job_text_raw' => $row['job_text_raw'] ?? '',
'predicted_profile' => $row['predicted_profile'], 'predicted_profile' => $row['predicted_profile'],
'confidence_score' => $row['confidence_score'] ?? 0, 'confidence_score' => $row['confidence_score'] ?? 0,
@ -85,24 +107,20 @@ public function upload(Request $request) {
} }
if (!empty($kemendikUpsertData)) { if (!empty($kemendikUpsertData)) {
foreach (array_chunk($kemendikUpsertData, 500) as $chunk) {
KemendikRawData::upsert( KemendikRawData::upsert(
$chunk, $kemendikUpsertData,
['nimhsmsmh', 'source_type'], ['nimhsmsmh', 'source_type'],
['nmmhsmsmh', 'tahun_lulus', 'f5c_jabatan_kode', 'f5c_jabatan_text', 'f1101_jenis_instansi_kode', 'f1101_jenis_instansi_text', 'f5a1_provinsi', 'raw_data'] ['nmmhsmsmh', 'tahun_lulus', 'f5c_jabatan_kode', 'f5c_jabatan_text', 'f1101_jenis_instansi_kode', 'f1101_jenis_instansi_text', 'f5a1_provinsi', 'raw_data']
); );
} }
}
if (!empty($internalResultsUpsertData)) { if (!empty($internalResultsUpsertData)) {
foreach (array_chunk($internalResultsUpsertData, 500) as $chunk) {
ClassificationResult::upsert( ClassificationResult::upsert(
$chunk, $internalResultsUpsertData,
['nim', 'source_type'], ['nim', 'source_type'],
['nama', 'tahun_lulus', 'job_text_raw', 'predicted_profile', 'confidence_score', 'classification_method', 'status', 'error_detail'] ['job_text_raw', 'predicted_profile', 'confidence_score', 'classification_method', 'status', 'error_detail']
); );
} }
}
if (!empty($internalRawUpsertData)) { if (!empty($internalRawUpsertData)) {
$cols = [ $cols = [
@ -116,20 +134,29 @@ public function upload(Request $request) {
'jabatan_terupdate', 'tahun_bekerja', 'status_pekerjaan_terupdate', 'jabatan_terupdate', 'tahun_bekerja', 'status_pekerjaan_terupdate',
'linkedin_profile', 'sosmed_ig' 'linkedin_profile', 'sosmed_ig'
]; ];
foreach (array_chunk($internalRawUpsertData, 500) as $chunk) { InternalRawData::upsert($internalRawUpsertData, ['nim'], $cols);
InternalRawData::upsert($chunk, ['nim'], $cols);
} }
} }
}); });
$isKemendik = $result['results'][0]['source_type'] === 'kemendik'; $source_type = $status['source_type'] ?? 'unknown';
if ($isKemendik) { if ($source_type === 'kemendik') {
$count = collect($result['results'])->where('source_type','kemendik')->count(); $count = collect($status['results'])->where('source_type','kemendik')->count();
return back()->with('success', "Selesai. {$result['processed_rows']} baris diproses. {$count} data Kemendik berhasil masuk."); $msg = "Selesai. {$status['processed_rows']} baris diproses. {$count} data Kemendik masuk.";
} else { } else {
$classified = collect($result['results'])->where('source_type','internal_mif')->where('status','auto_classified')->count(); $classified = collect($status['results'])->where('source_type','internal_mif')->where('status','auto_classified')->count();
return back()->with('success', "Selesai. {$result['processed_rows']} data diproses. {$classified} diklasifikasi otomatis (Internal MIF)."); $msg = "Selesai. {$status['processed_rows']} data diproses. {$classified} otomatis (MIF).";
} }
// Hapus data results dari file status agar tidak berat saat diambil UI selanjutnya? Tidak perlu.
return response()->json(['status' => 'completed', 'message' => $msg]);
} catch (\Exception $e) {
Log::error("DB Upsert failed: " . $e->getMessage());
return response()->json(['status' => 'error', 'message' => "Gagal menyimpan database: " . $e->getMessage()]);
}
}
return response()->json(['status' => 'error', 'message' => 'Status tidak dikenali.']);
} }
private function mapF5c(?string $c): ?string { return ['1'=>'Founder','2'=>'Co-Founder','3'=>'Staff','4'=>'Freelance'][$c] ?? null; } private function mapF5c(?string $c): ?string { return ['1'=>'Founder','2'=>'Co-Founder','3'=>'Staff','4'=>'Freelance'][$c] ?? null; }
@ -141,16 +168,17 @@ public function retrain(Request $request)
->where('status', 'manual_override') ->where('status', 'manual_override')
->whereNotNull('job_text_raw') ->whereNotNull('job_text_raw')
->whereNotNull('predicted_profile') ->whereNotNull('predicted_profile')
->get(['job_text_raw', 'predicted_profile']); ->get(['nim', 'job_text_raw', 'predicted_profile']);
$csvPath = null; $csvPath = null;
if ($overrideData->count() > 0) { if ($overrideData->count() > 0) {
$csvLines = ['job_text_raw;label']; $csvLines = ['nim;job_text_raw;label'];
foreach ($overrideData as $row) { foreach ($overrideData as $row) {
$nim = $row->nim;
$text = str_replace([';', "\n", "\r"], [',', ' ', ' '], $row->job_text_raw); $text = str_replace([';', "\n", "\r"], [',', ' ', ' '], $row->job_text_raw);
$label = $row->predicted_profile; $label = $row->predicted_profile;
$csvLines[] = "{$text};{$label}"; $csvLines[] = "{$nim};{$text};{$label}";
} }
$tmpDir = storage_path('app/temp'); $tmpDir = storage_path('app/temp');

38
sista_mif/app/Http/Controllers/InternalDataController.php
View File

@ -21,10 +21,24 @@ public function index(Request $request)
$query->orderByRaw("FIELD(status, 'needs_review', 'manual_override', 'auto_classified', 'failed')"); $query->orderByRaw("FIELD(status, 'needs_review', 'manual_override', 'auto_classified', 'failed')");
if ($request->filled('search')) {
$query->where(function($q) use ($request) {
$q->where('job_text_raw', 'LIKE', '%' . $request->search . '%')
->orWhere('nim', 'LIKE', '%' . $request->search . '%')
->orWhereHas('internalRaw', function($q) use ($request) {
$q->where('nama_lengkap', 'LIKE', '%' . $request->search . '%');
});
});
}
if ($request->has('sort')) { if ($request->has('sort')) {
$allowedSorts = ['nim', 'nama', 'job_text_raw', 'predicted_profile', 'status']; $allowedSorts = ['nim', 'job_text_raw', 'predicted_profile', 'status'];
if (in_array($request->sort, $allowedSorts)) { if ($request->sort === 'nama') {
$query->orderBy($request->sort, $request->direction === 'desc' ? 'desc' : 'asc'); $query->join('internal_raw_data', 'classification_results.nim', '=', 'internal_raw_data.nim')
->orderBy('internal_raw_data.nama_lengkap', $request->direction === 'desc' ? 'desc' : 'asc')
->select('classification_results.*');
} elseif (in_array($request->sort, $allowedSorts)) {
$query->orderBy('classification_results.'.$request->sort, $request->direction === 'desc' ? 'desc' : 'asc');
} }
} else { } else {
$query->latest(); $query->latest();
@ -119,7 +133,7 @@ public function update(Request $request, $id)
'status' => 'manual_override' 'status' => 'manual_override'
]); ]);
return back()->with('success', 'Data berhasil diperbarui!'); return redirect(url()->previous() . '#tabel-data')->with('success', 'Data berhasil diperbarui!');
} }
public function destroy($id) public function destroy($id)
@ -127,7 +141,7 @@ public function destroy($id)
$record = ClassificationResult::findOrFail($id); $record = ClassificationResult::findOrFail($id);
$record->delete(); $record->delete();
return back()->with('success', 'Data berhasil dihapus!'); return redirect(url()->previous() . '#tabel-data')->with('success', 'Data berhasil dihapus!');
} }
public function bulkDestroy(Request $request) public function bulkDestroy(Request $request)
@ -139,7 +153,7 @@ public function bulkDestroy(Request $request)
ClassificationResult::whereIn('id', $request->ids)->delete(); ClassificationResult::whereIn('id', $request->ids)->delete();
return back()->with('success', count($request->ids) . ' data berhasil dihapus!'); return redirect(url()->previous() . '#tabel-data')->with('success', count($request->ids) . ' data berhasil dihapus!');
} }
public function bulkUpdate(Request $request) public function bulkUpdate(Request $request)
@ -155,7 +169,7 @@ public function bulkUpdate(Request $request)
'status' => 'manual_override' 'status' => 'manual_override'
]); ]);
return back()->with('success', count($request->ids) . ' data berhasil diubah profilnya!'); return redirect(url()->previous() . '#tabel-data')->with('success', count($request->ids) . ' data berhasil diubah profilnya!');
} }
public function exportPdf(Request $request) public function exportPdf(Request $request)
@ -164,9 +178,13 @@ public function exportPdf(Request $request)
$query->orderByRaw("FIELD(status, 'needs_review', 'manual_override', 'auto_classified', 'failed')"); $query->orderByRaw("FIELD(status, 'needs_review', 'manual_override', 'auto_classified', 'failed')");
if ($request->has('sort')) { if ($request->has('sort')) {
$allowedSorts = ['nim', 'nama', 'job_text_raw', 'predicted_profile', 'status']; $allowedSorts = ['nim', 'job_text_raw', 'predicted_profile', 'status'];
if (in_array($request->sort, $allowedSorts)) { if ($request->sort === 'nama') {
$query->orderBy($request->sort, $request->direction === 'desc' ? 'desc' : 'asc'); $query->join('internal_raw_data', 'classification_results.nim', '=', 'internal_raw_data.nim')
->orderBy('internal_raw_data.nama_lengkap', $request->direction === 'desc' ? 'desc' : 'asc')
->select('classification_results.*');
} elseif (in_array($request->sort, $allowedSorts)) {
$query->orderBy('classification_results.'.$request->sort, $request->direction === 'desc' ? 'desc' : 'asc');
} }
} else { } else {
$query->latest(); $query->latest();

10
sista_mif/app/Models/ClassificationResult.php
View File

@ -10,5 +10,15 @@ class ClassificationResult extends Model
protected $casts = ['confidence_score' => 'float']; protected $casts = ['confidence_score' => 'float'];
public function scopeInternal($q) { return $q->where('source_type', 'internal_mif'); } public function scopeInternal($q) { return $q->where('source_type', 'internal_mif'); }
public function scopeNeedsReview($q) { return $q->where('status', 'needs_review'); } public function scopeNeedsReview($q) { return $q->where('status', 'needs_review'); }
public function internalRaw()
{
return $this->belongsTo(InternalRawData::class, 'nim', 'nim');
}
public function kemendikRaw()
{
return $this->belongsTo(KemendikRawData::class, 'nim', 'nimhsmsmh');
}
} }

27
sista_mif/app/Services/FastApiWorkerService.php
View File

@ -39,7 +39,7 @@ public function classifyFile(string $filePath, string $filename): array
throw new \Exception("Tidak dapat membaca file: {$filePath}"); throw new \Exception("Tidak dapat membaca file: {$filePath}");
} }
$response = $this->getClient(config('services.fastapi.timeout', 120)) $response = $this->getClient(15)
->attach('file', $fileStream, $filename) ->attach('file', $fileStream, $filename)
->post($this->baseUrl . '/api/v1/classify'); ->post($this->baseUrl . '/api/v1/classify');
@ -61,6 +61,24 @@ public function classifyFile(string $filePath, string $filename): array
} }
} }
public function getClassifyStatus(string $jobId): array
{
try {
$response = $this->getClient(10)->get($this->baseUrl . "/api/v1/classify/status/{$jobId}");
if ($response->failed()) {
if ($response->status() === 404) {
return ['status' => 'error', 'message' => 'Job ID tidak ditemukan di server.'];
}
return ['status' => 'error', 'message' => "Gagal menghubungi server: HTTP {$response->status()}"];
}
return $response->json();
} catch (ConnectionException $e) {
return ['status' => 'error', 'message' => 'Koneksi ke server AI terputus.'];
} catch (\Exception $e) {
return ['status' => 'error', 'message' => "Kesalahan sistem: {$e->getMessage()}"];
}
}
/** /**
* Kirim CSV data manual_override ke FastAPI untuk memulai re-training. * Kirim CSV data manual_override ke FastAPI untuk memulai re-training.
* CSV bersifat opsional — training tetap berjalan dengan corpus asli jika tidak ada. * CSV bersifat opsional — training tetap berjalan dengan corpus asli jika tidak ada.
@ -85,6 +103,10 @@ public function triggerRetrain(?string $csvPath = null): array
return ['error' => 'Re-training sedang berjalan. Tunggu hingga selesai.']; return ['error' => 'Re-training sedang berjalan. Tunggu hingga selesai.'];
} }
if ($response->status() === 429) {
return ['error' => 'Sistem sibuk. Permintaan retraining lain sedang diproses.'];
}
if ($response->failed()) { if ($response->failed()) {
Log::error("FastAPI Retrain Error: {$response->status()} | {$response->body()}"); Log::error("FastAPI Retrain Error: {$response->status()} | {$response->body()}");
return ['error' => "Gagal memulai re-training: HTTP {$response->status()}"]; return ['error' => "Gagal memulai re-training: HTTP {$response->status()}"];
@ -113,6 +135,9 @@ public function getRetrainStatus(): array
$response = $this->getClient(10)->get($this->baseUrl . '/api/v1/retrain/status'); $response = $this->getClient(10)->get($this->baseUrl . '/api/v1/retrain/status');
if ($response->failed()) { if ($response->failed()) {
if ($response->status() === 429) {
return $response->json(); // Return stage locked dari FastAPI
}
return ['stage' => 'unknown', 'message' => 'Tidak dapat membaca status dari FastAPI.']; return ['stage' => 'unknown', 'message' => 'Tidak dapat membaca status dari FastAPI.'];
} }

2
sista_mif/config/services.php
View File

@ -36,7 +36,7 @@
], ],
'fastapi' => [ 'fastapi' => [
'worker_url' => env('FASTAPI_WORKER_URL', 'http://127.0.0.1:8001'), 'worker_url' => env('FASTAPI_WORKER_URL', ''),
'secret_key' => env('FASTAPI_SECRET_KEY'), 'secret_key' => env('FASTAPI_SECRET_KEY'),
'timeout' => env('FASTAPI_TIMEOUT', 120), 'timeout' => env('FASTAPI_TIMEOUT', 120),
], ],

2
sista_mif/database/migrations/2026_05_23_132404_add_role_to_users_table.php
View File

@ -12,7 +12,7 @@
public function up(): void public function up(): void
{ {
Schema::table('users', function (Blueprint $table) { Schema::table('users', function (Blueprint $table) {
$table->enum('role', ['admin', 'cdc'])->default('cdc')->after('email'); $table->string('role')->default('cdc');
$table->boolean('is_active')->default(true)->after('role'); $table->boolean('is_active')->default(true)->after('role');
}); });
} }

27
sista_mif/database/migrations/2026_05_28_105600_drop_redundant_columns_and_add_indexes_to_classification_results.php Normal file
View File

@ -0,0 +1,27 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
public function up(): void
{
Schema::table('classification_results', function (Blueprint $table) {
$table->dropColumn(['nama', 'tahun_lulus']);
$table->index(['source_type', 'status']);
$table->index('predicted_profile');
});
}
public function down(): void
{
Schema::table('classification_results', function (Blueprint $table) {
$table->string('nama')->nullable();
$table->string('tahun_lulus')->nullable();
$table->dropIndex(['source_type', 'status']);
$table->dropIndex(['predicted_profile']);
});
}
};

188
sista_mif/resources/views/dashboard.blade.php
View File

@ -103,20 +103,9 @@
<p class="text-sm text-slate-500 mt-1">Unggah file Excel/CSV untuk memproses klasifikasi otomatis menggunakan Machine Learning.</p> <p class="text-sm text-slate-500 mt-1">Unggah file Excel/CSV untuk memproses klasifikasi otomatis menggunakan Machine Learning.</p>
</div> </div>
@if(session('success'))
<div class="bg-emerald-50 border border-emerald-200 text-emerald-700 px-4 py-3 rounded-xl mb-6 flex items-center space-x-3">
<svg class="w-5 h-5" fill="currentColor" viewBox="0 0 20 20"><path fill-rule="evenodd" d="M10 18a8 8 0 100-16 8 8 0 000 16zm3.707-9.293a1 1 0 00-1.414-1.414L9 10.586 7.707 9.293a1 1 0 00-1.414 1.414l2 2a1 1 0 001.414 0l4-4z" clip-rule="evenodd"></path></svg>
<span>{{ session('success') }}</span>
</div>
@endif
@if(session('error'))
<div class="bg-rose-50 border border-rose-200 text-rose-700 px-4 py-3 rounded-xl mb-6 flex items-center space-x-3">
<svg class="w-5 h-5" fill="currentColor" viewBox="0 0 20 20"><path fill-rule="evenodd" d="M10 18a8 8 0 100-16 8 8 0 000 16zM8.707 7.293a1 1 0 00-1.414 1.414L8.586 10l-1.293 1.293a1 1 0 101.414 1.414L10 11.414l1.293 1.293a1 1 0 001.414-1.414L11.414 10l1.293-1.293a1 1 0 00-1.414-1.414L10 8.586 8.707 7.293z" clip-rule="evenodd"></path></svg>
<span>{{ session('error') }}</span>
</div>
@endif
<form action="{{ route('upload.process') }}" method="POST" enctype="multipart/form-data" class="flex flex-col md:flex-row gap-6 items-end">
<form id="uploadForm" action="{{ route('upload.process') }}" method="POST" enctype="multipart/form-data" class="flex flex-col md:flex-row gap-6 items-end">
@csrf @csrf
<input type="hidden" name="source_type" value="internal_mif"> <input type="hidden" name="source_type" value="internal_mif">
@ -339,13 +328,24 @@ class="inline-flex items-center gap-2 px-6 py-3 rounded-xl text-sm font-semibold
</div> </div>
<!-- Data Table --> <!-- Data Table -->
<div class="bg-white rounded-2xl shadow-xl shadow-slate-200/50 border border-slate-100 overflow-hidden"> <div id="tabel-data" class="bg-white rounded-2xl shadow-xl shadow-slate-200/50 border border-slate-100 overflow-hidden">
<div class="px-6 py-5 border-b border-slate-100 bg-slate-50/50 flex justify-between items-center"> <div class="px-6 py-5 border-b border-slate-100 bg-slate-50/50 flex justify-between items-center">
<h3 class="text-lg font-bold text-slate-800">Tabel Hasil Klasifikasi</h3> <h3 class="text-lg font-bold text-slate-800">Tabel Hasil Klasifikasi</h3>
<div class="flex items-center space-x-3"> <div class="flex items-center space-x-3 w-full md:w-auto mt-4 md:mt-0">
<form action="{{ route('dashboard') }}" method="GET" class="relative flex items-center">
@if(request('sort')) <input type="hidden" name="sort" value="{{ request('sort') }}"> @endif
@if(request('direction')) <input type="hidden" name="direction" value="{{ request('direction') }}"> @endif
<input type="text" name="search" value="{{ request('search') }}" placeholder="Cari job / nim / nama..." class="rounded-l-lg border-slate-300 text-sm focus:ring-blue-500 focus:border-blue-500 pl-4 py-2 w-full md:w-56 lg:w-64">
<button type="submit" class="bg-blue-600 hover:bg-blue-700 text-white px-4 py-2 rounded-r-lg text-sm font-semibold border border-blue-600 transition-colors">Cari</button>
@if(request('search'))
<a href="{{ route('dashboard', request()->except('search', 'page')) }}" class="ml-2 text-rose-500 hover:text-rose-700 text-sm font-medium whitespace-nowrap">Reset</a>
@endif
</form>
<a href="{{ route('internal.download_pdf', request()->query()) }}" class="inline-flex items-center px-4 py-2 bg-emerald-600 text-white text-sm font-semibold rounded-lg hover:bg-emerald-700 transition-colors shadow-sm"> <a href="{{ route('internal.download_pdf', request()->query()) }}" class="inline-flex items-center px-4 py-2 bg-emerald-600 text-white text-sm font-semibold rounded-lg hover:bg-emerald-700 transition-colors shadow-sm">
<svg class="w-4 h-4 mr-2" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M12 10v6m0 0l-3-3m3 3l3-3m2 8H7a2 2 0 01-2-2V5a2 2 0 012-2h5.586a1 1 0 01.707.293l5.414 5.414a1 1 0 01.293.707V19a2 2 0 01-2 2z"></path></svg> <svg class="w-4 h-4 mr-2" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M12 10v6m0 0l-3-3m3 3l3-3m2 8H7a2 2 0 01-2-2V5a2 2 0 012-2h5.586a1 1 0 01.707.293l5.414 5.414a1 1 0 01.293.707V19a2 2 0 01-2 2z"></path></svg>
Download PDF PDF
</a> </a>
<div x-show="selectedIds.length > 0" style="display: none;" class="flex items-center space-x-2"> <div x-show="selectedIds.length > 0" style="display: none;" class="flex items-center space-x-2">
<span class="text-sm text-slate-600 font-medium mr-2"><span x-text="selectedIds.length"></span> dipilih</span> <span class="text-sm text-slate-600 font-medium mr-2"><span x-text="selectedIds.length"></span> dipilih</span>
@ -355,7 +355,7 @@ class="inline-flex items-center gap-2 px-6 py-3 rounded-xl text-sm font-semibold
Edit Edit
</button> </button>
<form action="{{ route('internal.bulk_destroy') }}" method="POST" onsubmit="return confirm('Yakin ingin menghapus data yang dipilih?');" class="inline-block"> <form action="{{ route('internal.bulk_destroy') }}" method="POST" class="inline-block bulk-delete-form">
@csrf @csrf
<template x-for="id in selectedIds" :key="id"> <template x-for="id in selectedIds" :key="id">
<input type="hidden" name="ids[]" :value="id"> <input type="hidden" name="ids[]" :value="id">
@ -410,7 +410,7 @@ class="inline-flex items-center gap-2 px-6 py-3 rounded-xl text-sm font-semibold
</td> </td>
<td class="px-6 py-4 whitespace-nowrap"> <td class="px-6 py-4 whitespace-nowrap">
<div class="text-sm font-semibold text-slate-800">{{ $row->nim }}</div> <div class="text-sm font-semibold text-slate-800">{{ $row->nim }}</div>
<div class="text-sm text-slate-500">{{ $row->nama }}</div> <div class="text-sm text-slate-500">{{ $row->internalRaw->nama_lengkap ?? '-' }}</div>
</td> </td>
<td class="px-6 py-4 text-sm text-slate-600 max-w-xs"> <td class="px-6 py-4 text-sm text-slate-600 max-w-xs">
<div class="line-clamp-2" title="{{ $row->job_text_raw }}"> <div class="line-clamp-2" title="{{ $row->job_text_raw }}">
@ -454,8 +454,8 @@ class="inline-flex items-center gap-2 px-6 py-3 rounded-xl text-sm font-semibold
@endif @endif
</td> </td>
<td class="px-6 py-4 whitespace-nowrap text-right text-sm font-medium space-x-2"> <td class="px-6 py-4 whitespace-nowrap text-right text-sm font-medium space-x-2">
<button @click="openEdit({{ $row->id }}, '{{ $row->predicted_profile }}', '{{ addslashes($row->nama) }}')" class="text-blue-600 hover:text-blue-900 transition-colors">Edit</button> <button @click="openEdit({{ $row->id }}, '{{ $row->predicted_profile }}', '{{ addslashes($row->internalRaw->nama_lengkap ?? '-') }}')" class="text-blue-600 hover:text-blue-900 transition-colors">Edit</button>
<form action="{{ route('internal.destroy', $row->id) }}" method="POST" class="inline-block" onsubmit="return confirm('Yakin ingin menghapus data ini?');"> <form action="{{ route('internal.destroy', $row->id) }}" method="POST" class="inline-block delete-form">
@csrf @csrf
@method('DELETE') @method('DELETE')
<button type="submit" class="text-rose-600 hover:text-rose-900 transition-colors">Hapus</button> <button type="submit" class="text-rose-600 hover:text-rose-900 transition-colors">Hapus</button>
@ -472,7 +472,7 @@ class="inline-flex items-center gap-2 px-6 py-3 rounded-xl text-sm font-semibold
<div class="bg-white px-4 pt-5 pb-4 sm:p-6 sm:pb-4 border-b border-slate-100"> <div class="bg-white px-4 pt-5 pb-4 sm:p-6 sm:pb-4 border-b border-slate-100">
<div class="flex justify-between items-center mb-4"> <div class="flex justify-between items-center mb-4">
<h3 class="text-lg leading-6 font-bold text-slate-800" id="modal-title"> <h3 class="text-lg leading-6 font-bold text-slate-800" id="modal-title">
Detail Data Internal - {{ $row->nama }} Detail Data Internal - {{ $row->internalRaw->nama_lengkap ?? '-' }}
</h3> </h3>
<button @click="openDetail = false" class="text-slate-400 hover:text-slate-600 focus:outline-none"> <button @click="openDetail = false" class="text-slate-400 hover:text-slate-600 focus:outline-none">
<svg class="h-6 w-6" fill="none" viewBox="0 0 24 24" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M6 18L18 6M6 6l12 12" /></svg> <svg class="h-6 w-6" fill="none" viewBox="0 0 24 24" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M6 18L18 6M6 6l12 12" /></svg>
@ -850,4 +850,150 @@ function retrainPanel() {
}; };
} }
</script> </script>
<script>
document.addEventListener('DOMContentLoaded', () => {
const tableContainer = document.getElementById('tabel-data');
if (tableContainer) {
tableContainer.addEventListener('click', e => {
const link = e.target.closest('nav[role="navigation"] a');
if (link && link.href && !link.closest('thead')) {
e.preventDefault();
const url = link.href;
tableContainer.classList.add('opacity-50', 'pointer-events-none', 'transition-opacity');
fetch(url, { headers: { 'X-Requested-With': 'XMLHttpRequest' } })
.then(res => res.text())
.then(html => {
const doc = new DOMParser().parseFromString(html, 'text/html');
const newTable = doc.getElementById('tabel-data');
if (newTable) {
tableContainer.innerHTML = newTable.innerHTML;
window.history.pushState({ path: url }, '', url);
}
})
.catch(err => console.error('Gagal memuat halaman tabel:', err))
.finally(() => tableContainer.classList.remove('opacity-50', 'pointer-events-none'));
}
});
window.addEventListener('popstate', () => window.location.reload());
}
// SweetAlert Confirmations
document.querySelectorAll('.delete-form').forEach(form => {
form.addEventListener('submit', function(e) {
e.preventDefault();
Swal.fire({
title: 'Hapus Data?',
text: "Data yang dihapus tidak dapat dikembalikan!",
icon: 'warning',
showCancelButton: true,
confirmButtonColor: '#ef4444',
cancelButtonColor: '#64748b',
confirmButtonText: 'Ya, Hapus!',
cancelButtonText: 'Batal'
}).then((result) => {
if (result.isConfirmed) form.submit();
});
});
});
document.querySelectorAll('.bulk-delete-form').forEach(form => {
form.addEventListener('submit', function(e) {
e.preventDefault();
Swal.fire({
title: 'Hapus Data Massal?',
text: "Semua data yang dipilih akan dihapus permanen!",
icon: 'warning',
showCancelButton: true,
confirmButtonColor: '#ef4444',
cancelButtonColor: '#64748b',
confirmButtonText: 'Ya, Hapus Semua!',
cancelButtonText: 'Batal'
}).then((result) => {
if (result.isConfirmed) form.submit();
});
});
});
// AJAX Upload & Polling
const uploadForm = document.getElementById('uploadForm');
if (uploadForm) {
uploadForm.addEventListener('submit', async function(e) {
e.preventDefault();
const formData = new FormData(this);
const submitBtn = this.querySelector('button[type="submit"]');
const originalBtnText = submitBtn.innerHTML;
submitBtn.disabled = true;
submitBtn.innerHTML = '<svg class="animate-spin h-5 w-5 mr-3" viewBox="0 0 24 24"><circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4" fill="none"></circle><path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path></svg> Mengirim...';
try {
const response = await fetch(this.action, {
method: 'POST',
body: formData,
headers: { 'Accept': 'application/json' }
});
const result = await response.json();
if (!result.success) {
Swal.fire('Error', result.message, 'error');
submitBtn.disabled = false;
submitBtn.innerHTML = originalBtnText;
return;
}
Swal.fire({
title: 'Memproses Data...',
html: result.message || 'Mohon tunggu, sistem sedang memproses file Anda.',
allowOutsideClick: false,
didOpen: () => {
Swal.showLoading();
}
});
pollStatus(result.job_id, submitBtn, originalBtnText);
} catch (err) {
Swal.fire('Error', 'Gagal mengirim file.', 'error');
submitBtn.disabled = false;
submitBtn.innerHTML = originalBtnText;
}
});
}
function pollStatus(jobId, submitBtn, originalBtnText) {
const interval = setInterval(async () => {
try {
const res = await fetch(`/upload/status/${jobId}`);
const data = await res.json();
if (data.status === 'completed') {
clearInterval(interval);
Swal.fire('Selesai!', data.message, 'success').then(() => {
window.location.reload();
});
} else if (data.status === 'error') {
clearInterval(interval);
Swal.fire('Gagal', data.message, 'error');
if(submitBtn) {
submitBtn.disabled = false;
submitBtn.innerHTML = originalBtnText;
}
} else {
if (data.message) {
Swal.getHtmlContainer().textContent = data.message;
}
}
} catch (err) {
clearInterval(interval);
Swal.fire('Error', 'Koneksi ke server terputus saat mengecek status.', 'error');
if(submitBtn) {
submitBtn.disabled = false;
submitBtn.innerHTML = originalBtnText;
}
}
}, 3000);
}
});
</script>
</x-app-layout> </x-app-layout>

126
sista_mif/resources/views/kemendik.blade.php
View File

@ -29,20 +29,9 @@
<p class="text-sm text-slate-500 mt-1">Unggah file Excel/CSV data Kemendik untuk agregasi dan mapping (tanpa proses NLP klasifikasi).</p> <p class="text-sm text-slate-500 mt-1">Unggah file Excel/CSV data Kemendik untuk agregasi dan mapping (tanpa proses NLP klasifikasi).</p>
</div> </div>
@if(session('success'))
<div class="bg-emerald-50 border border-emerald-200 text-emerald-700 px-4 py-3 rounded-xl mb-6 flex items-center space-x-3">
<svg class="w-5 h-5" fill="currentColor" viewBox="0 0 20 20"><path fill-rule="evenodd" d="M10 18a8 8 0 100-16 8 8 0 000 16zm3.707-9.293a1 1 0 00-1.414-1.414L9 10.586 7.707 9.293a1 1 0 00-1.414 1.414l2 2a1 1 0 001.414 0l4-4z" clip-rule="evenodd"></path></svg>
<span>{{ session('success') }}</span>
</div>
@endif
@if(session('error'))
<div class="bg-rose-50 border border-rose-200 text-rose-700 px-4 py-3 rounded-xl mb-6 flex items-center space-x-3">
<svg class="w-5 h-5" fill="currentColor" viewBox="0 0 20 20"><path fill-rule="evenodd" d="M10 18a8 8 0 100-16 8 8 0 000 16zM8.707 7.293a1 1 0 00-1.414 1.414L8.586 10l-1.293 1.293a1 1 0 101.414 1.414L10 11.414l1.293 1.293a1 1 0 001.414-1.414L11.414 10l1.293-1.293a1 1 0 00-1.414-1.414L10 8.586 8.707 7.293z" clip-rule="evenodd"></path></svg>
<span>{{ session('error') }}</span>
</div>
@endif
<form action="{{ route('upload.process') }}" method="POST" enctype="multipart/form-data" class="flex flex-col md:flex-row gap-6 items-end">
<form id="uploadForm" action="{{ route('upload.process') }}" method="POST" enctype="multipart/form-data" class="flex flex-col md:flex-row gap-6 items-end">
@csrf @csrf
<input type="hidden" name="source_type" value="kemendik"> <input type="hidden" name="source_type" value="kemendik">
@ -121,7 +110,7 @@
@endif @endif
<!-- Data Table --> <!-- Data Table -->
<div class="bg-white rounded-2xl shadow-xl shadow-purple-100/50 border border-purple-50 overflow-hidden"> <div id="tabel-data" class="bg-white rounded-2xl shadow-xl shadow-purple-100/50 border border-purple-50 overflow-hidden">
<div class="px-6 py-5 border-b border-purple-50 bg-purple-50/20 flex justify-between items-center"> <div class="px-6 py-5 border-b border-purple-50 bg-purple-50/20 flex justify-between items-center">
<h3 class="text-lg font-bold text-slate-800">Tabel Data Kemendiktisaintek</h3> <h3 class="text-lg font-bold text-slate-800">Tabel Data Kemendiktisaintek</h3>
</div> </div>
@ -324,4 +313,113 @@
}); });
</script> </script>
@endif @endif
<script>
document.addEventListener('DOMContentLoaded', () => {
const tableContainer = document.getElementById('tabel-data');
if (tableContainer) {
tableContainer.addEventListener('click', e => {
const link = e.target.closest('nav[role="navigation"] a');
if (link && link.href && !link.closest('thead')) {
e.preventDefault();
const url = link.href;
tableContainer.classList.add('opacity-50', 'pointer-events-none', 'transition-opacity');
fetch(url, { headers: { 'X-Requested-With': 'XMLHttpRequest' } })
.then(res => res.text())
.then(html => {
const doc = new DOMParser().parseFromString(html, 'text/html');
const newTable = doc.getElementById('tabel-data');
if (newTable) {
tableContainer.innerHTML = newTable.innerHTML;
window.history.pushState({ path: url }, '', url);
}
})
.catch(err => console.error('Gagal memuat halaman tabel:', err))
.finally(() => tableContainer.classList.remove('opacity-50', 'pointer-events-none'));
}
});
window.addEventListener('popstate', () => window.location.reload());
}
// AJAX Upload & Polling
const uploadForm = document.getElementById('uploadForm');
if (uploadForm) {
uploadForm.addEventListener('submit', async function(e) {
e.preventDefault();
const formData = new FormData(this);
const submitBtn = this.querySelector('button[type="submit"]');
const originalBtnText = submitBtn.innerHTML;
submitBtn.disabled = true;
submitBtn.innerHTML = '<svg class="animate-spin h-5 w-5 mr-3" viewBox="0 0 24 24"><circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4" fill="none"></circle><path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path></svg> Mengirim...';
try {
const response = await fetch(this.action, {
method: 'POST',
body: formData,
headers: { 'Accept': 'application/json' }
});
const result = await response.json();
if (!result.success) {
Swal.fire('Error', result.message, 'error');
submitBtn.disabled = false;
submitBtn.innerHTML = originalBtnText;
return;
}
Swal.fire({
title: 'Memproses Data...',
html: result.message || 'Mohon tunggu, sistem sedang memproses file Anda.',
allowOutsideClick: false,
didOpen: () => {
Swal.showLoading();
}
});
pollStatus(result.job_id, submitBtn, originalBtnText);
} catch (err) {
Swal.fire('Error', 'Gagal mengirim file.', 'error');
submitBtn.disabled = false;
submitBtn.innerHTML = originalBtnText;
}
});
}
function pollStatus(jobId, submitBtn, originalBtnText) {
const interval = setInterval(async () => {
try {
const res = await fetch(`/upload/status/${jobId}`);
const data = await res.json();
if (data.status === 'completed') {
clearInterval(interval);
Swal.fire('Selesai!', data.message, 'success').then(() => {
window.location.reload();
});
} else if (data.status === 'error') {
clearInterval(interval);
Swal.fire('Gagal', data.message, 'error');
if(submitBtn) {
submitBtn.disabled = false;
submitBtn.innerHTML = originalBtnText;
}
} else {
if (data.message) {
Swal.getHtmlContainer().textContent = data.message;
}
}
} catch (err) {
clearInterval(interval);
Swal.fire('Error', 'Koneksi ke server terputus saat mengecek status.', 'error');
if(submitBtn) {
submitBtn.disabled = false;
submitBtn.innerHTML = originalBtnText;
}
}
}, 3000);
}
});
</script>
</x-app-layout> </x-app-layout>

31
sista_mif/resources/views/layouts/app.blade.php
View File

@ -32,5 +32,36 @@
{{ $slot }} {{ $slot }}
</main> </main>
</div> </div>
<!-- SweetAlert2 -->
<script src="https://cdn.jsdelivr.net/npm/sweetalert2@11"></script>
<script>
document.addEventListener("DOMContentLoaded", () => {
const Toast = Swal.mixin({
toast: true,
position: 'top-end',
showConfirmButton: false,
timer: 3000,
timerProgressBar: true,
didOpen: (toast) => {
toast.addEventListener('mouseenter', Swal.stopTimer)
toast.addEventListener('mouseleave', Swal.resumeTimer)
}
});
@if(session('success'))
Toast.fire({
icon: 'success',
title: "{!! session('success') !!}"
});
@endif
@if(session('error'))
Toast.fire({
icon: 'error',
title: "{!! session('error') !!}"
});
@endif
});
</script>
</body> </body>
</html> </html>

2
sista_mif/resources/views/pdf/classification_results.blade.php
View File

@ -48,7 +48,7 @@
<tr> <tr>
<td>{{ $index + 1 }}</td> <td>{{ $index + 1 }}</td>
<td>{{ $row->nim }}</td> <td>{{ $row->nim }}</td>
<td>{{ $row->nama }}</td> <td>{{ $row->internalRaw->nama_lengkap ?? '-' }}</td>
<td>{{ $row->job_text_raw }}</td> <td>{{ $row->job_text_raw }}</td>
<td> <td>
{{ $row->predicted_profile ?? '-' }} {{ $row->predicted_profile ?? '-' }}

9
sista_mif/routes/web.php
View File

@ -11,21 +11,18 @@
Route::middleware(['auth'])->group(function () { Route::middleware(['auth'])->group(function () {
Route::get('/dashboard', [InternalDataController::class, 'index'])->name('dashboard'); Route::get('/dashboard', [InternalDataController::class, 'index'])->name('dashboard');
Route::get('/kemendik', [KemendikDataController::class, 'index'])->name('kemendik'); Route::get('/kemendik', [KemendikDataController::class, 'index'])->name('kemendik');
// Import Data
Route::post('/upload', [ClassificationTaskController::class, 'upload'])->name('upload.process'); Route::post('/upload', [ClassificationTaskController::class, 'upload'])->name('upload.process');
Route::get('/upload/status/{job_id}', [ClassificationTaskController::class, 'checkStatus'])->name('upload.status');
// Internal Data Management Route::middleware('admin')->group(function () {
Route::put('/internal/{id}', [InternalDataController::class, 'update'])->name('internal.update'); Route::put('/internal/{id}', [InternalDataController::class, 'update'])->name('internal.update');
Route::delete('/internal/{id}', [InternalDataController::class, 'destroy'])->name('internal.destroy'); Route::delete('/internal/{id}', [InternalDataController::class, 'destroy'])->name('internal.destroy');
Route::post('/internal/bulk-delete', [InternalDataController::class, 'bulkDestroy'])->name('internal.bulk_destroy'); Route::post('/internal/bulk-delete', [InternalDataController::class, 'bulkDestroy'])->name('internal.bulk_destroy');
Route::post('/internal/bulk-update', [InternalDataController::class, 'bulkUpdate'])->name('internal.bulk_update'); Route::post('/internal/bulk-update', [InternalDataController::class, 'bulkUpdate'])->name('internal.bulk_update');
Route::get('/internal/download-pdf', [InternalDataController::class, 'exportPdf'])->name('internal.download_pdf'); Route::get('/internal/download-pdf', [InternalDataController::class, 'exportPdf'])->name('internal.download_pdf');
// ML Task
Route::post('/retrain', [ClassificationTaskController::class, 'retrain'])->name('retrain.trigger'); Route::post('/retrain', [ClassificationTaskController::class, 'retrain'])->name('retrain.trigger');
Route::get('/retrain/status', [ClassificationTaskController::class, 'retrainStatus'])->name('retrain.status'); Route::get('/retrain/status', [ClassificationTaskController::class, 'retrainStatus'])->name('retrain.status');
});
Route::prefix('profile')->group(function () { Route::prefix('profile')->group(function () {
Route::get('/', [ProfileController::class, 'edit'])->name('profile.edit'); Route::get('/', [ProfileController::class, 'edit'])->name('profile.edit');