09-06-26
This commit is contained in:
parent
b5d5a9b63a
commit
2f52a2e70c
|
|
@ -8,7 +8,7 @@ code_to_name = {}
|
||||||
with open('wilayah_dapodik_final.csv', mode='r', encoding='utf-8') as f:
|
with open('wilayah_dapodik_final.csv', mode='r', encoding='utf-8') as f:
|
||||||
reader = csv.DictReader(f)
|
reader = csv.DictReader(f)
|
||||||
for row in reader:
|
for row in reader:
|
||||||
# Bersihkan nama kabupaten/kota dan provinsi agar pencarian lebih akurat
|
|
||||||
kab = row['Kabupaten_Kota'].replace('Kab. ', '').replace('Kota Adm. ', '').replace('Kota ', '')
|
kab = row['Kabupaten_Kota'].replace('Kab. ', '').replace('Kota Adm. ', '').replace('Kota ', '')
|
||||||
prov = row['Provinsi'].replace('Prov. ', '')
|
prov = row['Provinsi'].replace('Prov. ', '')
|
||||||
|
|
||||||
|
|
@ -20,7 +20,6 @@ with open('wilayah_dapodik_final.csv', mode='r', encoding='utf-8') as f:
|
||||||
if kab == 'Adm. Kep. Seribu':
|
if kab == 'Adm. Kep. Seribu':
|
||||||
kab = 'Kepulauan Seribu'
|
kab = 'Kepulauan Seribu'
|
||||||
|
|
||||||
# Format query pencarian, contoh: "Malang, Jawa Timur, Indonesia"
|
|
||||||
if prov == 'Luar Negeri':
|
if prov == 'Luar Negeri':
|
||||||
query = kab
|
query = kab
|
||||||
else:
|
else:
|
||||||
|
|
@ -35,22 +34,20 @@ coords = {}
|
||||||
|
|
||||||
print(f"Mulai mencari koordinat untuk {len(code_to_name)} wilayah...")
|
print(f"Mulai mencari koordinat untuk {len(code_to_name)} wilayah...")
|
||||||
|
|
||||||
# Looping melalui semua data yang ada di CSV
|
|
||||||
for code, info in code_to_name.items():
|
for code, info in code_to_name.items():
|
||||||
query = info['query']
|
query = info['query']
|
||||||
name = info['name']
|
name = info['name']
|
||||||
max_retries = 3
|
max_retries = 3
|
||||||
for attempt in range(max_retries):
|
for attempt in range(max_retries):
|
||||||
try:
|
try:
|
||||||
time.sleep(1) # Jeda 1 detik agar tidak terkena limit API Nominatim
|
time.sleep(1)
|
||||||
# Menambahkan parameter timeout eksplisit jika diperlukan, default biasanya 1 detik
|
|
||||||
location = geolocator.geocode(query, timeout=5)
|
location = geolocator.geocode(query, timeout=5)
|
||||||
if location:
|
if location:
|
||||||
coords[code] = {"name": name, "coords": [location.latitude, location.longitude]}
|
coords[code] = {"name": name, "coords": [location.latitude, location.longitude]}
|
||||||
print(f"Geocoded: {code} -> {query} -> {coords[code]['coords']}")
|
print(f"Geocoded: {code} -> {query} -> {coords[code]['coords']}")
|
||||||
else:
|
else:
|
||||||
print(f"NOT FOUND: {code} -> {query}")
|
print(f"NOT FOUND: {code} -> {query}")
|
||||||
break # Berhasil diproses (entah ketemu atau tidak ketemu), hentikan loop retry
|
break
|
||||||
except GeocoderTimedOut:
|
except GeocoderTimedOut:
|
||||||
print(f"TIMEOUT (Percobaan {attempt + 1}/{max_retries}): {code} -> {query}")
|
print(f"TIMEOUT (Percobaan {attempt + 1}/{max_retries}): {code} -> {query}")
|
||||||
if attempt == max_retries - 1:
|
if attempt == max_retries - 1:
|
||||||
|
|
|
||||||
File diff suppressed because one or more lines are too long
|
|
@ -63,8 +63,6 @@ def clean_text(text: str) -> str:
|
||||||
tokens = [w for w in text.split() if w not in COMPANY_STOPWORDS and len(w) >= 3]
|
tokens = [w for w in text.split() if w not in COMPANY_STOPWORDS and len(w) >= 3]
|
||||||
return " ".join([stemmer.stem(w) for w in tokens])
|
return " ".join([stemmer.stem(w) for w in tokens])
|
||||||
|
|
||||||
# [FIX v4] Logika lebih ketat: hanya flag jika token teks sepenuhnya subset dari nama
|
|
||||||
# (max 3 token) — bukan sekadar rasio overlap 50% seperti v3 yang rawan false positive.
|
|
||||||
def is_likely_name(text: str, full_name: str) -> bool:
|
def is_likely_name(text: str, full_name: str) -> bool:
|
||||||
if not text or not full_name: return False
|
if not text or not full_name: return False
|
||||||
text_clean = re.sub(r'[^\w\s]', '', text.lower())
|
text_clean = re.sub(r'[^\w\s]', '', text.lower())
|
||||||
|
|
@ -110,7 +108,6 @@ def main():
|
||||||
mask = ~df[col_status].str.lower().str.strip().isin(blacklist)
|
mask = ~df[col_status].str.lower().str.strip().isin(blacklist)
|
||||||
df = df[mask].copy()
|
df = df[mask].copy()
|
||||||
|
|
||||||
# Stemming berat terjadi HANYA di sini — satu kali saat corpus preparation
|
|
||||||
df["job_text_raw"] = df[col_jabatan].apply(clean_text)
|
df["job_text_raw"] = df[col_jabatan].apply(clean_text)
|
||||||
|
|
||||||
if col_klasifikasi:
|
if col_klasifikasi:
|
||||||
|
|
@ -153,7 +150,6 @@ def main():
|
||||||
print("DATA PELATIHAN (TRAINING SET):")
|
print("DATA PELATIHAN (TRAINING SET):")
|
||||||
print(train_df["label"].value_counts().to_string())
|
print(train_df["label"].value_counts().to_string())
|
||||||
|
|
||||||
# [KEEP v3] Distribusi per kelas di test set — berguna untuk deteksi class imbalance
|
|
||||||
print("\nDATA PENGUJIAN (TEST SET):")
|
print("\nDATA PENGUJIAN (TEST SET):")
|
||||||
for cls in TARGET_CLASSES:
|
for cls in TARGET_CLASSES:
|
||||||
c = test_df["label"].value_counts().get(cls, 0)
|
c = test_df["label"].value_counts().get(cls, 0)
|
||||||
|
|
|
||||||
File diff suppressed because one or more lines are too long
|
|
@ -1,181 +1,3 @@
|
||||||
# import pandas as pd
|
|
||||||
# import numpy as np
|
|
||||||
# import re
|
|
||||||
# import joblib
|
|
||||||
# import json
|
|
||||||
# import warnings
|
|
||||||
# from pathlib import Path
|
|
||||||
# from sklearn.feature_extraction.text import TfidfVectorizer
|
|
||||||
# from sklearn.linear_model import LogisticRegression
|
|
||||||
# from sklearn.model_selection import StratifiedKFold
|
|
||||||
# from sklearn.metrics import classification_report, confusion_matrix, ConfusionMatrixDisplay
|
|
||||||
# from sklearn.pipeline import Pipeline
|
|
||||||
# import matplotlib
|
|
||||||
# matplotlib.use('Agg')
|
|
||||||
# import matplotlib.pyplot as plt
|
|
||||||
# warnings.filterwarnings('ignore')
|
|
||||||
|
|
||||||
# BASE_DIR = Path(__file__).parent
|
|
||||||
# TRAIN_FILE = BASE_DIR / "processed" / "training_corpus.csv"
|
|
||||||
# TEST_FILE = BASE_DIR / "processed" / "test_set.csv"
|
|
||||||
# ML_DIR = BASE_DIR.parent / "fastapi" / "ml_assets"
|
|
||||||
# ML_DIR.mkdir(parents=True, exist_ok=True)
|
|
||||||
# TARGET_CLASSES = ["Programmer", "Data Analyst", "Wirausaha Informatika", "Non-IT"]
|
|
||||||
# CONFIDENCE_THRESHOLD = 0.50
|
|
||||||
|
|
||||||
# # [v4] Tidak import Sastrawi — corpus sudah di-stem oleh prepare_corpus.py
|
|
||||||
# # Stemming dua kali pada data yang sama dapat mendegradasi teks.
|
|
||||||
# STOPWORDS = {
|
|
||||||
# "yang", "di", "ke", "dari", "dan", "atau", "dengan", "untuk", "pada", "dalam",
|
|
||||||
# "adalah", "ini", "itu", "tidak", "juga", "sudah", "akan", "bisa", "ada", "oleh",
|
|
||||||
# "karena", "secara", "serta", "sebagai", "bagi", "telah", "maka", "namun", "sehingga",
|
|
||||||
# "jika", "agar", "ketika", "saat", "sebelum", "sesudah", "hingga", "sampai", "antara",
|
|
||||||
# "sekitar", "hanya", "saja", "belum", "masih", "lagi", "pun", "justru", "walaupun",
|
|
||||||
# "meskipun", "bahkan", "cukup", "sangat", "paling", "lebih", "kurang", "lain",
|
|
||||||
# "macam", "cara", "hal", "tentang", "mengenai", "terhadap", "kepada", "menuju",
|
|
||||||
# "kecuali", "selain", "tanpa", "demi", "guna", "khususnya", "umumnya", "kebanyakan",
|
|
||||||
# "sebagian", "beberapa", "semua", "setiap", "tiap", "satu", "dua", "tiga", "empat",
|
|
||||||
# "lima", "enam", "tujuh", "delapan", "sembilan", "sepuluh", "ratus", "ribu", "juta"
|
|
||||||
# }
|
|
||||||
|
|
||||||
# def preprocess_text(text):
|
|
||||||
# """Pembersihan dasar — tanpa stemming ulang karena korpus sudah di-stem."""
|
|
||||||
# if pd.isna(text) or not isinstance(text, str): return ""
|
|
||||||
# text = text.lower().strip()
|
|
||||||
# if text in {"nan", "none", "null", "-", "0", "", "tidak diisi"}: return ""
|
|
||||||
# text = text.replace('-', ' ').replace('/', ' ').replace('_', ' ').replace(',', ' ')
|
|
||||||
# text = re.sub(r'[^\w\s]', '', text)
|
|
||||||
# text = re.sub(r'\d+', '', text)
|
|
||||||
# words = [w for w in text.split() if w not in STOPWORDS and len(w) > 2]
|
|
||||||
# return " ".join(words)
|
|
||||||
|
|
||||||
# def main():
|
|
||||||
# if not TRAIN_FILE.exists():
|
|
||||||
# print("File training_corpus.csv belum ada. Jalankan prepare_corpus.py terlebih dahulu."); return
|
|
||||||
|
|
||||||
# df_train = pd.read_csv(TRAIN_FILE, sep=";", dtype=str).dropna(subset=["job_text_raw", "label"])
|
|
||||||
# df_test = pd.read_csv(TEST_FILE, sep=";", dtype=str).dropna(subset=["job_text_raw", "label"]) if TEST_FILE.exists() else None
|
|
||||||
|
|
||||||
# print("Sedang memproses teks (cleaning dasar tanpa stemming redundan)...")
|
|
||||||
# X_train = df_train["job_text_raw"].apply(preprocess_text)
|
|
||||||
# y_train = df_train["label"]
|
|
||||||
# mask = X_train.str.len() > 0
|
|
||||||
# X_train, y_train = X_train[mask], y_train[mask]
|
|
||||||
|
|
||||||
# X_test, y_test = pd.Series(dtype=str), pd.Series(dtype=str)
|
|
||||||
# if df_test is not None:
|
|
||||||
# X_test = df_test["job_text_raw"].apply(preprocess_text)
|
|
||||||
# y_test = df_test["label"]
|
|
||||||
# mask_t = X_test.str.len() > 0
|
|
||||||
# X_test, y_test = X_test[mask_t], y_test[mask_t]
|
|
||||||
|
|
||||||
# print(f"Jumlah data latih: {len(X_train)} baris | Data uji: {len(X_test)} baris\n")
|
|
||||||
|
|
||||||
# # [KEEP v3] K-Fold cross validation — estimasi variance performa model
|
|
||||||
# print("Melakukan pengujian K-Fold (5 putaran) pada data latih...")
|
|
||||||
# skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
|
|
||||||
# fold_metrics = []
|
|
||||||
# for i, (tr_idx, te_idx) in enumerate(skf.split(X_train, y_train)):
|
|
||||||
# model = Pipeline([
|
|
||||||
# ('tfidf', TfidfVectorizer(max_features=3000, ngram_range=(1,2), sublinear_tf=True, min_df=1)),
|
|
||||||
# ('clf', LogisticRegression(max_iter=1000, class_weight='balanced', solver='lbfgs'))
|
|
||||||
# ])
|
|
||||||
# model.fit(X_train.iloc[tr_idx], y_train.iloc[tr_idx])
|
|
||||||
# y_pred = model.predict(X_train.iloc[te_idx])
|
|
||||||
# rep = classification_report(y_train.iloc[te_idx], y_pred, output_dict=True, zero_division=0)
|
|
||||||
# fold_metrics.append(rep)
|
|
||||||
# print(f" Akurasi putaran ke-{i+1}: {rep['accuracy']:.4f}")
|
|
||||||
|
|
||||||
# avg_acc = np.mean([f['accuracy'] for f in fold_metrics])
|
|
||||||
# print(f"\n RATA-RATA PENGUJIAN K-FOLD:")
|
|
||||||
# print(f" Akurasi Keseluruhan : {avg_acc:.4f} ± {np.std([f['accuracy'] for f in fold_metrics]):.4f}")
|
|
||||||
# for cls in TARGET_CLASSES:
|
|
||||||
# p = np.mean([f.get(cls, {}).get('precision', 0) for f in fold_metrics])
|
|
||||||
# r = np.mean([f.get(cls, {}).get('recall', 0) for f in fold_metrics])
|
|
||||||
# f1 = np.mean([f.get(cls, {}).get('f1-score', 0) for f in fold_metrics])
|
|
||||||
# print(f" {cls:25} | P: {p:.3f} | R: {r:.3f} | F1: {f1:.3f}")
|
|
||||||
|
|
||||||
# print("\n Membuat model final dari seluruh data latih yang tersedia...")
|
|
||||||
# final_model = Pipeline([
|
|
||||||
# ('tfidf', TfidfVectorizer(max_features=3000, ngram_range=(1,2), sublinear_tf=True, min_df=1)),
|
|
||||||
# ('clf', LogisticRegression(max_iter=1000, class_weight='balanced', solver='lbfgs'))
|
|
||||||
# ])
|
|
||||||
# final_model.fit(X_train, y_train)
|
|
||||||
|
|
||||||
# metrics_test = None
|
|
||||||
# if len(X_test) > 0:
|
|
||||||
# y_pred = final_model.predict(X_test)
|
|
||||||
# rep_test = classification_report(y_test, y_pred, output_dict=True, zero_division=0)
|
|
||||||
# metrics_test = rep_test
|
|
||||||
|
|
||||||
# print("\n HASIL PENGUJIAN PADA DATA TEST:")
|
|
||||||
# print(classification_report(y_test, y_pred, zero_division=0))
|
|
||||||
|
|
||||||
# for cls in TARGET_CLASSES:
|
|
||||||
# sup = rep_test[cls]['support'] if cls in rep_test else 0
|
|
||||||
# if sup < 5: print(f"Catatan: Kelas '{cls}' cuma punya {sup} data uji — skor F1-nya mungkin kurang akurat.")
|
|
||||||
|
|
||||||
# # [KEEP v3] Confusion matrix
|
|
||||||
# cm = confusion_matrix(y_test, y_pred, labels=TARGET_CLASSES)
|
|
||||||
# disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=TARGET_CLASSES)
|
|
||||||
# disp.plot(cmap='Blues', values_format='d', xticks_rotation=45, colorbar=False)
|
|
||||||
# plt.title('Confusion Matrix - Hold-Out Test'); plt.tight_layout()
|
|
||||||
# plt.savefig(ML_DIR / 'confusion_matrix_test.png', dpi=300); plt.close()
|
|
||||||
# print("Grafik confusion matrix berhasil disimpan ke confusion_matrix_test.png")
|
|
||||||
|
|
||||||
# # [BUGFIX] Feature importance per kelas
|
|
||||||
# # clf.coef_ diurutkan mengikuti clf.classes_ (alfabetikal sklearn),
|
|
||||||
# # BUKAN urutan TARGET_CLASSES yang hardcoded. Pakai clf.classes_ sebagai
|
|
||||||
# # iterator agar index i selalu selaras dengan baris koefisien yang benar.
|
|
||||||
# vec, clf = final_model.named_steps['tfidf'], final_model.named_steps['clf']
|
|
||||||
# feats, coeffs = vec.get_feature_names_out(), clf.coef_
|
|
||||||
# print("\n5 KATA PALING BERPENGARUH UNTUK TIAP KELAS:")
|
|
||||||
# for i, cls in enumerate(clf.classes_): # <-- clf.classes_, bukan TARGET_CLASSES
|
|
||||||
# top = [(feats[j], coeffs[i][j]) for j in coeffs[i].argsort()[-5:][::-1] if coeffs[i][j] > 0]
|
|
||||||
# print(f" [{cls}] " + ", ".join([f"{w}({c:.2f})" for w, c in top]))
|
|
||||||
|
|
||||||
# # [KEEP v3] Confidence distribution
|
|
||||||
# proba = final_model.predict_proba(X_test)
|
|
||||||
# max_p = np.max(proba, axis=1)
|
|
||||||
# plt.hist(max_p, bins=15, edgecolor='black', alpha=0.7, color='teal')
|
|
||||||
# plt.axvline(x=CONFIDENCE_THRESHOLD, color='red', linestyle='--', linewidth=2, label=f'Threshold {CONFIDENCE_THRESHOLD}')
|
|
||||||
# plt.xlabel('Confidence Score'); plt.ylabel('Count'); plt.title('Distribusi Confidence Score')
|
|
||||||
# plt.legend(); plt.grid(axis='y', alpha=0.3); plt.tight_layout()
|
|
||||||
# plt.savefig(ML_DIR / 'confidence_distribution.png', dpi=300); plt.close()
|
|
||||||
# below = np.sum(max_p < CONFIDENCE_THRESHOLD)
|
|
||||||
# print(f"\nPengecekan Keyakinan Model: {below} dari {len(max_p)} prediksi ({below/len(max_p)*100:.1f}%) di bawah threshold {CONFIDENCE_THRESHOLD}.")
|
|
||||||
|
|
||||||
# # [KEEP v3] Daftar prediksi yang meleset
|
|
||||||
# mis = np.where(y_test.values != y_pred)[0]
|
|
||||||
# if len(mis) > 0:
|
|
||||||
# print("\n DAFTAR PREDIKSI YANG MELESET:")
|
|
||||||
# for idx in mis[:min(6, len(mis))]:
|
|
||||||
# txt = df_test.iloc[idx]['job_text_raw'][:80]
|
|
||||||
# print(f" • Seharusnya: {y_test.iloc[idx]:20} | Ditebak: {y_pred[idx]:20} | Teks: '{txt}...'")
|
|
||||||
# else:
|
|
||||||
# print("\n Semua prediksi pada test set benar.")
|
|
||||||
|
|
||||||
# model_path = ML_DIR / "ml_pipeline_internal.pkl"
|
|
||||||
# joblib.dump(final_model, model_path)
|
|
||||||
|
|
||||||
# metrics = {
|
|
||||||
# "methodology": "Internal MIF only",
|
|
||||||
# "k_fold": {
|
|
||||||
# "accuracy_mean": float(avg_acc),
|
|
||||||
# "accuracy_std": float(np.std([f['accuracy'] for f in fold_metrics])),
|
|
||||||
# "folds": fold_metrics
|
|
||||||
# },
|
|
||||||
# "threshold_config": CONFIDENCE_THRESHOLD
|
|
||||||
# }
|
|
||||||
# if metrics_test: metrics["hold_out_test"] = metrics_test
|
|
||||||
# with open(ML_DIR / "metrics_internal_only.json", 'w') as f: json.dump(metrics, f, indent=2)
|
|
||||||
|
|
||||||
# print(f"\n Model berhasil disimpan di: {model_path}")
|
|
||||||
# print(f"Laporan metrik disimpan di: {ML_DIR / 'metrics_internal_only.json'}")
|
|
||||||
|
|
||||||
# if __name__ == "__main__":
|
|
||||||
# main()
|
|
||||||
|
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import re
|
import re
|
||||||
|
|
@ -192,9 +14,6 @@ import matplotlib
|
||||||
matplotlib.use('Agg')
|
matplotlib.use('Agg')
|
||||||
import matplotlib.pyplot as plt
|
import matplotlib.pyplot as plt
|
||||||
|
|
||||||
# PERBAIKAN 1: Hapus warnings.filterwarnings('ignore') secara global.
|
|
||||||
# Biarkan warning muncul. Jika ada ConvergenceWarning, Anda akan tahu bahwa max_iter perlu dinaikkan lagi.
|
|
||||||
|
|
||||||
BASE_DIR = Path(__file__).parent
|
BASE_DIR = Path(__file__).parent
|
||||||
TRAIN_FILE = BASE_DIR / "processed" / "training_corpus.csv"
|
TRAIN_FILE = BASE_DIR / "processed" / "training_corpus.csv"
|
||||||
TEST_FILE = BASE_DIR / "processed" / "test_set.csv"
|
TEST_FILE = BASE_DIR / "processed" / "test_set.csv"
|
||||||
|
|
@ -223,10 +42,6 @@ def preprocess_text(text):
|
||||||
if text in {"nan", "none", "null", "-", "0", "", "tidak diisi"}: return ""
|
if text in {"nan", "none", "null", "-", "0", "", "tidak diisi"}: return ""
|
||||||
text = text.replace('-', ' ').replace('/', ' ').replace('_', ' ').replace(',', ' ')
|
text = text.replace('-', ' ').replace('/', ' ').replace('_', ' ').replace(',', ' ')
|
||||||
text = re.sub(r'[^\w\s]', '', text)
|
text = re.sub(r'[^\w\s]', '', text)
|
||||||
|
|
||||||
# PERBAIKAN 2: Penambahan komentar eksplisit.
|
|
||||||
# Anda harus sadar bahwa menghapus angka menghilangkan sinyal seperti "Python 3" atau "5 tahun".
|
|
||||||
# Jika ini keputusan sadar, biarkan. Jika tidak, hapus baris di bawah ini.
|
|
||||||
text = re.sub(r'\d+', '', text)
|
text = re.sub(r'\d+', '', text)
|
||||||
|
|
||||||
words = [w for w in text.split() if w not in STOPWORDS and len(w) > 2]
|
words = [w for w in text.split() if w not in STOPWORDS and len(w) > 2]
|
||||||
|
|
@ -239,14 +54,14 @@ def get_pipeline():
|
||||||
max_features=3000,
|
max_features=3000,
|
||||||
ngram_range=(1, 2),
|
ngram_range=(1, 2),
|
||||||
sublinear_tf=True,
|
sublinear_tf=True,
|
||||||
min_df=2, # PERBAIKAN 3: Diubah dari 1 ke 2 untuk mencegah overfitting pada noise/typo.
|
min_df=2,
|
||||||
norm='l2' # PERBAIKAN 4: Eksplisit menyatakan normalisasi L2 (default, tapi ditulis agar transparan).
|
norm='l2'
|
||||||
)),
|
)),
|
||||||
('clf', LogisticRegression(
|
('clf', LogisticRegression(
|
||||||
max_iter=2000, # PERBAIKAN 5: Dinaikkan dari 1000 untuk mencegah ConvergenceWarning pada data sparse.
|
max_iter=2000,
|
||||||
class_weight='balanced',
|
class_weight='balanced',
|
||||||
solver='lbfgs',
|
solver='lbfgs',
|
||||||
random_state=42 # PERBAIKAN 6: Ditambahkan untuk reproducibility (hasil yang konsisten).
|
random_state=42
|
||||||
))
|
))
|
||||||
])
|
])
|
||||||
|
|
||||||
|
|
@ -279,7 +94,6 @@ def main():
|
||||||
fold_metrics = []
|
fold_metrics = []
|
||||||
|
|
||||||
for i, (tr_idx, te_idx) in enumerate(skf.split(X_train, y_train)):
|
for i, (tr_idx, te_idx) in enumerate(skf.split(X_train, y_train)):
|
||||||
# PERBAIKAN 7: Menggunakan fungsi get_pipeline() untuk menghindari duplikasi kode dan inkonsistensi
|
|
||||||
model = get_pipeline()
|
model = get_pipeline()
|
||||||
model.fit(X_train.iloc[tr_idx], y_train.iloc[tr_idx])
|
model.fit(X_train.iloc[tr_idx], y_train.iloc[tr_idx])
|
||||||
y_pred = model.predict(X_train.iloc[te_idx])
|
y_pred = model.predict(X_train.iloc[te_idx])
|
||||||
|
|
@ -297,7 +111,6 @@ def main():
|
||||||
print(f" {cls:25} | P: {p:.3f} | R: {r:.3f} | F1: {f1:.3f}")
|
print(f" {cls:25} | P: {p:.3f} | R: {r:.3f} | F1: {f1:.3f}")
|
||||||
|
|
||||||
print("\n Membuat model final dari seluruh data latih yang tersedia...")
|
print("\n Membuat model final dari seluruh data latih yang tersedia...")
|
||||||
# PERBAIKAN 8: Menggunakan fungsi get_pipeline()
|
|
||||||
final_model = get_pipeline()
|
final_model = get_pipeline()
|
||||||
final_model.fit(X_train, y_train)
|
final_model.fit(X_train, y_train)
|
||||||
|
|
||||||
|
|
@ -325,7 +138,6 @@ def main():
|
||||||
plt.close()
|
plt.close()
|
||||||
print("Grafik confusion matrix berhasil disimpan ke confusion_matrix_test.png")
|
print("Grafik confusion matrix berhasil disimpan ke confusion_matrix_test.png")
|
||||||
|
|
||||||
# PERBAIKAN 9: Feature importance yang menampilkan koefisien POSITIF dan NEGATIF
|
|
||||||
vec = final_model.named_steps['tfidf']
|
vec = final_model.named_steps['tfidf']
|
||||||
clf = final_model.named_steps['clf']
|
clf = final_model.named_steps['clf']
|
||||||
feats = vec.get_feature_names_out()
|
feats = vec.get_feature_names_out()
|
||||||
|
|
|
||||||
|
|
@ -138,7 +138,6 @@ def classify_ml(job_text: str, pipeline) -> dict:
|
||||||
logger.error(f"ML_INFER_ERROR | job_text='{job_text[:50]}' | err={e}")
|
logger.error(f"ML_INFER_ERROR | job_text='{job_text[:50]}' | err={e}")
|
||||||
raise HTTPException(status_code=500, detail=f"ML inference failed: {str(e)}")
|
raise HTTPException(status_code=500, detail=f"ML inference failed: {str(e)}")
|
||||||
|
|
||||||
# [KEEP v3] Ekstraksi teks dari format Kemendikbud
|
|
||||||
def extract_kemendik_text(row: pd.Series, df_columns: list) -> str:
|
def extract_kemendik_text(row: pd.Series, df_columns: list) -> str:
|
||||||
f5b_col = find_column_by_code(df_columns, "f5b")
|
f5b_col = find_column_by_code(df_columns, "f5b")
|
||||||
f5c_col = find_column_by_code(df_columns, "f5c")
|
f5c_col = find_column_by_code(df_columns, "f5c")
|
||||||
|
|
@ -175,7 +174,6 @@ def health_check():
|
||||||
|
|
||||||
@app.post("/api/v1/retrain", dependencies=[Depends(verify_token)])
|
@app.post("/api/v1/retrain", dependencies=[Depends(verify_token)])
|
||||||
async def trigger_retrain(file: UploadFile = File(None)):
|
async def trigger_retrain(file: UploadFile = File(None)):
|
||||||
# [v4] FileLock mencegah race condition dari concurrent requests
|
|
||||||
lock = FileLock(LOCK_PATH, timeout=2)
|
lock = FileLock(LOCK_PATH, timeout=2)
|
||||||
try:
|
try:
|
||||||
with lock:
|
with lock:
|
||||||
|
|
@ -221,7 +219,6 @@ def retrain_status():
|
||||||
if not STATUS_PATH.exists():
|
if not STATUS_PATH.exists():
|
||||||
return JSONResponse(content={"stage": "idle", "message": "Belum ada proses re-training."})
|
return JSONResponse(content={"stage": "idle", "message": "Belum ada proses re-training."})
|
||||||
try:
|
try:
|
||||||
# [v4] Lock saat baca untuk menghindari partial write dari worker
|
|
||||||
lock = FileLock(LOCK_PATH, timeout=2)
|
lock = FileLock(LOCK_PATH, timeout=2)
|
||||||
with lock:
|
with lock:
|
||||||
data = json.loads(STATUS_PATH.read_text(encoding="utf-8"))
|
data = json.loads(STATUS_PATH.read_text(encoding="utf-8"))
|
||||||
|
|
@ -241,7 +238,6 @@ def retrain_status():
|
||||||
return JSONResponse(content={"stage": "unknown", "message": "Status tidak terbaca."})
|
return JSONResponse(content={"stage": "unknown", "message": "Status tidak terbaca."})
|
||||||
|
|
||||||
|
|
||||||
# [KEEP v3] Endpoint reload manual — berguna jika pipeline perlu di-reload tanpa restart server
|
|
||||||
@app.post("/api/v1/retrain/reload", dependencies=[Depends(verify_token)])
|
@app.post("/api/v1/retrain/reload", dependencies=[Depends(verify_token)])
|
||||||
def reload_model():
|
def reload_model():
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -11,7 +11,6 @@ Alur:
|
||||||
6. Evaluasi candidate pada test set yang sama → delta terhadap old model
|
6. Evaluasi candidate pada test set yang sama → delta terhadap old model
|
||||||
7. Promote jika lebih baik; rollback jika tidak
|
7. Promote jika lebih baik; rollback jika tidak
|
||||||
8. Setiap write status dilindungi FileLock (atomic)
|
8. Setiap write status dilindungi FileLock (atomic)
|
||||||
9. [v5] Semua event log disimpan ke file JSONL terpisah untuk audit BAB 4
|
|
||||||
"""
|
"""
|
||||||
import sys
|
import sys
|
||||||
import json
|
import json
|
||||||
|
|
@ -42,7 +41,7 @@ warnings.filterwarnings("ignore")
|
||||||
BASE_DIR = Path(__file__).parent.parent
|
BASE_DIR = Path(__file__).parent.parent
|
||||||
ML_DIR = BASE_DIR / "ml_assets"
|
ML_DIR = BASE_DIR / "ml_assets"
|
||||||
DATA_DIR = BASE_DIR.parent / "data" / "processed"
|
DATA_DIR = BASE_DIR.parent / "data" / "processed"
|
||||||
LOG_DIR = ML_DIR / "logs" # [v5] Direktori log terpisah
|
LOG_DIR = ML_DIR / "logs"
|
||||||
LOG_DIR.mkdir(parents=True, exist_ok=True)
|
LOG_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
PIPELINE_PATH = ML_DIR / "ml_pipeline_internal.pkl"
|
PIPELINE_PATH = ML_DIR / "ml_pipeline_internal.pkl"
|
||||||
|
|
@ -54,7 +53,6 @@ LOCK_PATH = ML_DIR / "retrain_status.lock"
|
||||||
CORPUS_PATH = DATA_DIR / "training_corpus.csv"
|
CORPUS_PATH = DATA_DIR / "training_corpus.csv"
|
||||||
TEST_PATH = DATA_DIR / "test_set.csv"
|
TEST_PATH = DATA_DIR / "test_set.csv"
|
||||||
|
|
||||||
# [v5] Nama file log berbasis timestamp — satu file per sesi retraining
|
|
||||||
SESSION_TS = datetime.now().strftime("%Y%m%d_%H%M%S")
|
SESSION_TS = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||||
LOG_FILE_PATH = LOG_DIR / f"retrain_log_{SESSION_TS}.jsonl"
|
LOG_FILE_PATH = LOG_DIR / f"retrain_log_{SESSION_TS}.jsonl"
|
||||||
|
|
||||||
|
|
@ -81,7 +79,7 @@ STOPWORDS = {
|
||||||
stemmer = StemmerFactory().create_stemmer()
|
stemmer = StemmerFactory().create_stemmer()
|
||||||
|
|
||||||
# ──────────────────────────────────────────────────────────────
|
# ──────────────────────────────────────────────────────────────
|
||||||
# [v5] CUSTOM JSON FORMATTER — setiap log entry jadi JSON valid
|
# CUSTOM JSON FORMATTER
|
||||||
# ──────────────────────────────────────────────────────────────
|
# ──────────────────────────────────────────────────────────────
|
||||||
class JsonFormatter(logging.Formatter):
|
class JsonFormatter(logging.Formatter):
|
||||||
"""Format log record sebagai JSON satu baris (JSONL)."""
|
"""Format log record sebagai JSON satu baris (JSONL)."""
|
||||||
|
|
@ -92,7 +90,6 @@ class JsonFormatter(logging.Formatter):
|
||||||
"logger": record.name,
|
"logger": record.name,
|
||||||
"message": record.getMessage(),
|
"message": record.getMessage(),
|
||||||
}
|
}
|
||||||
# [v5] Jika ada extra data (stage, metrics, dll), gabungkan ke root
|
|
||||||
if hasattr(record, "stage"):
|
if hasattr(record, "stage"):
|
||||||
log_entry["stage"] = record.stage
|
log_entry["stage"] = record.stage
|
||||||
if hasattr(record, "event_type"):
|
if hasattr(record, "event_type"):
|
||||||
|
|
@ -102,7 +99,7 @@ class JsonFormatter(logging.Formatter):
|
||||||
return json.dumps(log_entry, ensure_ascii=False)
|
return json.dumps(log_entry, ensure_ascii=False)
|
||||||
|
|
||||||
# ──────────────────────────────────────────────────────────────
|
# ──────────────────────────────────────────────────────────────
|
||||||
# [v5] DUAL LOGGER SETUP — console + file JSONL
|
# DUAL LOGGER SETUP
|
||||||
# ──────────────────────────────────────────────────────────────
|
# ──────────────────────────────────────────────────────────────
|
||||||
logger = logging.getLogger("retrain_worker")
|
logger = logging.getLogger("retrain_worker")
|
||||||
logger.setLevel(logging.INFO)
|
logger.setLevel(logging.INFO)
|
||||||
|
|
@ -123,7 +120,7 @@ logger.addHandler(file_handler)
|
||||||
logger.info(f"[init] Log file tersimpan di: {LOG_FILE_PATH}")
|
logger.info(f"[init] Log file tersimpan di: {LOG_FILE_PATH}")
|
||||||
|
|
||||||
# ──────────────────────────────────────────────────────────────
|
# ──────────────────────────────────────────────────────────────
|
||||||
# [v5] STRUCTURED LOG HELPER — untuk event penting dengan metadata
|
# STRUCTURED LOG HELPER
|
||||||
# ──────────────────────────────────────────────────────────────
|
# ──────────────────────────────────────────────────────────────
|
||||||
def log_event(event_type: str, message: str, stage: str = "info", metadata: dict = None):
|
def log_event(event_type: str, message: str, stage: str = "info", metadata: dict = None):
|
||||||
"""
|
"""
|
||||||
|
|
@ -148,7 +145,7 @@ def compute_override_weight(n_corpus: int, n_override: int) -> float:
|
||||||
return round(w_log, 4)
|
return round(w_log, 4)
|
||||||
|
|
||||||
# ──────────────────────────────────────────────────────────────
|
# ──────────────────────────────────────────────────────────────
|
||||||
# STATUS WRITER — atomic dengan FileLock (v4)
|
# STATUS WRITER
|
||||||
# ──────────────────────────────────────────────────────────────
|
# ──────────────────────────────────────────────────────────────
|
||||||
def write_status(stage: str, message: str, extra: dict = None):
|
def write_status(stage: str, message: str, extra: dict = None):
|
||||||
lock = FileLock(LOCK_PATH, timeout=10)
|
lock = FileLock(LOCK_PATH, timeout=10)
|
||||||
|
|
@ -156,7 +153,6 @@ def write_status(stage: str, message: str, extra: dict = None):
|
||||||
payload = {"stage": stage, "message": message, "timestamp": datetime.now().isoformat()}
|
payload = {"stage": stage, "message": message, "timestamp": datetime.now().isoformat()}
|
||||||
if extra: payload.update(extra)
|
if extra: payload.update(extra)
|
||||||
STATUS_PATH.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8")
|
STATUS_PATH.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||||
# [v5] Tulis juga ke log file dengan stage tracking
|
|
||||||
logger.info(f"[{stage}] {message}", extra={"stage": stage})
|
logger.info(f"[{stage}] {message}", extra={"stage": stage})
|
||||||
|
|
||||||
# ──────────────────────────────────────────────────────────────
|
# ──────────────────────────────────────────────────────────────
|
||||||
|
|
@ -228,7 +224,6 @@ def do_rollback(reason: str, old_f1: float, new_f1: float):
|
||||||
else:
|
else:
|
||||||
logger.warning("File .bak tidak ditemukan — pkl aktif dibiarkan.")
|
logger.warning("File .bak tidak ditemukan — pkl aktif dibiarkan.")
|
||||||
|
|
||||||
# [v5] Log event rollback dengan metadata lengkap
|
|
||||||
log_event(
|
log_event(
|
||||||
event_type="rollback",
|
event_type="rollback",
|
||||||
message=f"Model lama dipertahankan. {reason}",
|
message=f"Model lama dipertahankan. {reason}",
|
||||||
|
|
@ -246,7 +241,6 @@ def do_rollback(reason: str, old_f1: float, new_f1: float):
|
||||||
# MAIN
|
# MAIN
|
||||||
# ──────────────────────────────────────────────────────────────
|
# ──────────────────────────────────────────────────────────────
|
||||||
def main(extra_csv_path: str = None):
|
def main(extra_csv_path: str = None):
|
||||||
# [v5] Log awal sesi — penting untuk audit trail
|
|
||||||
log_event(
|
log_event(
|
||||||
event_type="session_start",
|
event_type="session_start",
|
||||||
message=f"Retraining session dimulai. Log file: {LOG_FILE_PATH.name}",
|
message=f"Retraining session dimulai. Log file: {LOG_FILE_PATH.name}",
|
||||||
|
|
@ -314,7 +308,6 @@ def main(extra_csv_path: str = None):
|
||||||
y_all = df_all["label"]
|
y_all = df_all["label"]
|
||||||
w_all = df_all["_weight"]
|
w_all = df_all["_weight"]
|
||||||
|
|
||||||
# [v5] Log event data merge dengan metadata
|
|
||||||
log_event(
|
log_event(
|
||||||
event_type="data_merged",
|
event_type="data_merged",
|
||||||
message=f"Data training siap: {len(df_all)} baris",
|
message=f"Data training siap: {len(df_all)} baris",
|
||||||
|
|
@ -342,7 +335,7 @@ def main(extra_csv_path: str = None):
|
||||||
write_status("training", f"K-Fold validation & training... ({len(X_train)} sampel)")
|
write_status("training", f"K-Fold validation & training... ({len(X_train)} sampel)")
|
||||||
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
|
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
|
||||||
fold_metrics = []
|
fold_metrics = []
|
||||||
fold_accuracies = [] # [v5] untuk log ringkas
|
fold_accuracies = []
|
||||||
for i, (tr_idx, te_idx) in enumerate(skf.split(X_train, y_train)):
|
for i, (tr_idx, te_idx) in enumerate(skf.split(X_train, y_train)):
|
||||||
m = Pipeline([
|
m = Pipeline([
|
||||||
('tfidf', TfidfVectorizer(max_features=3000, ngram_range=(1, 2), sublinear_tf=True, min_df=1)),
|
('tfidf', TfidfVectorizer(max_features=3000, ngram_range=(1, 2), sublinear_tf=True, min_df=1)),
|
||||||
|
|
@ -358,7 +351,6 @@ def main(extra_csv_path: str = None):
|
||||||
fold_accuracies.append(round(rep['accuracy'], 4))
|
fold_accuracies.append(round(rep['accuracy'], 4))
|
||||||
write_status("training", f"K-Fold selesai: fold {i+1}/5 | acc={rep['accuracy']:.4f}")
|
write_status("training", f"K-Fold selesai: fold {i+1}/5 | acc={rep['accuracy']:.4f}")
|
||||||
|
|
||||||
# [v5] Log ringkas hasil K-Fold
|
|
||||||
log_event(
|
log_event(
|
||||||
event_type="kfold_completed",
|
event_type="kfold_completed",
|
||||||
message=f"K-Fold 5 lipatan selesai. Akurasi per fold: {fold_accuracies}",
|
message=f"K-Fold 5 lipatan selesai. Akurasi per fold: {fold_accuracies}",
|
||||||
|
|
@ -405,7 +397,6 @@ def main(extra_csv_path: str = None):
|
||||||
f"minimum dibutuhkan: +{MIN_IMPROVEMENT_THRESHOLD*100:.1f}%)"
|
f"minimum dibutuhkan: +{MIN_IMPROVEMENT_THRESHOLD*100:.1f}%)"
|
||||||
)
|
)
|
||||||
|
|
||||||
# [v5] Log event evaluasi dengan metadata lengkap — ini yang akan dikutip di BAB 4
|
|
||||||
log_event(
|
log_event(
|
||||||
event_type="evaluation_completed",
|
event_type="evaluation_completed",
|
||||||
message=f"Evaluasi selesai. Delta F1: {delta:+.4f} ({'promote' if should_promote else 'rollback'})",
|
message=f"Evaluasi selesai. Delta F1: {delta:+.4f} ({'promote' if should_promote else 'rollback'})",
|
||||||
|
|
@ -460,7 +451,6 @@ def main(extra_csv_path: str = None):
|
||||||
}
|
}
|
||||||
METRICS_PATH.write_text(json.dumps(new_metrics_full, indent=2, ensure_ascii=False), encoding="utf-8")
|
METRICS_PATH.write_text(json.dumps(new_metrics_full, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||||
|
|
||||||
# [v5] Log event promote final
|
|
||||||
log_event(
|
log_event(
|
||||||
event_type="model_promoted",
|
event_type="model_promoted",
|
||||||
message=f"Model baru berhasil dipromote. {reason}",
|
message=f"Model baru berhasil dipromote. {reason}",
|
||||||
|
|
@ -492,7 +482,6 @@ def main(extra_csv_path: str = None):
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"ERROR saat training: {type(e).__name__}: {e}", exc_info=True)
|
logger.error(f"ERROR saat training: {type(e).__name__}: {e}", exc_info=True)
|
||||||
|
|
||||||
# [v5] Log event error dengan traceback info
|
|
||||||
log_event(
|
log_event(
|
||||||
event_type="training_error",
|
event_type="training_error",
|
||||||
message=f"Training gagal: {type(e).__name__}: {str(e)[:200]}",
|
message=f"Training gagal: {type(e).__name__}: {str(e)[:200]}",
|
||||||
|
|
|
||||||
Binary file not shown.
|
Before Width: | Height: | Size: 62 KiB After Width: | Height: | Size: 92 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 114 KiB After Width: | Height: | Size: 137 KiB |
Binary file not shown.
Loading…
Reference in New Issue