09-06-26
This commit is contained in:
parent
b5d5a9b63a
commit
2f52a2e70c
|
|
@ -8,7 +8,7 @@ code_to_name = {}
|
|||
with open('wilayah_dapodik_final.csv', mode='r', encoding='utf-8') as f:
|
||||
reader = csv.DictReader(f)
|
||||
for row in reader:
|
||||
# Bersihkan nama kabupaten/kota dan provinsi agar pencarian lebih akurat
|
||||
|
||||
kab = row['Kabupaten_Kota'].replace('Kab. ', '').replace('Kota Adm. ', '').replace('Kota ', '')
|
||||
prov = row['Provinsi'].replace('Prov. ', '')
|
||||
|
||||
|
|
@ -20,7 +20,6 @@ with open('wilayah_dapodik_final.csv', mode='r', encoding='utf-8') as f:
|
|||
if kab == 'Adm. Kep. Seribu':
|
||||
kab = 'Kepulauan Seribu'
|
||||
|
||||
# Format query pencarian, contoh: "Malang, Jawa Timur, Indonesia"
|
||||
if prov == 'Luar Negeri':
|
||||
query = kab
|
||||
else:
|
||||
|
|
@ -35,22 +34,20 @@ coords = {}
|
|||
|
||||
print(f"Mulai mencari koordinat untuk {len(code_to_name)} wilayah...")
|
||||
|
||||
# Looping melalui semua data yang ada di CSV
|
||||
for code, info in code_to_name.items():
|
||||
query = info['query']
|
||||
name = info['name']
|
||||
max_retries = 3
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
time.sleep(1) # Jeda 1 detik agar tidak terkena limit API Nominatim
|
||||
# Menambahkan parameter timeout eksplisit jika diperlukan, default biasanya 1 detik
|
||||
time.sleep(1)
|
||||
location = geolocator.geocode(query, timeout=5)
|
||||
if location:
|
||||
coords[code] = {"name": name, "coords": [location.latitude, location.longitude]}
|
||||
print(f"Geocoded: {code} -> {query} -> {coords[code]['coords']}")
|
||||
else:
|
||||
print(f"NOT FOUND: {code} -> {query}")
|
||||
break # Berhasil diproses (entah ketemu atau tidak ketemu), hentikan loop retry
|
||||
break
|
||||
except GeocoderTimedOut:
|
||||
print(f"TIMEOUT (Percobaan {attempt + 1}/{max_retries}): {code} -> {query}")
|
||||
if attempt == max_retries - 1:
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -63,8 +63,6 @@ def clean_text(text: str) -> str:
|
|||
tokens = [w for w in text.split() if w not in COMPANY_STOPWORDS and len(w) >= 3]
|
||||
return " ".join([stemmer.stem(w) for w in tokens])
|
||||
|
||||
# [FIX v4] Logika lebih ketat: hanya flag jika token teks sepenuhnya subset dari nama
|
||||
# (max 3 token) — bukan sekadar rasio overlap 50% seperti v3 yang rawan false positive.
|
||||
def is_likely_name(text: str, full_name: str) -> bool:
|
||||
if not text or not full_name: return False
|
||||
text_clean = re.sub(r'[^\w\s]', '', text.lower())
|
||||
|
|
@ -110,7 +108,6 @@ def main():
|
|||
mask = ~df[col_status].str.lower().str.strip().isin(blacklist)
|
||||
df = df[mask].copy()
|
||||
|
||||
# Stemming berat terjadi HANYA di sini — satu kali saat corpus preparation
|
||||
df["job_text_raw"] = df[col_jabatan].apply(clean_text)
|
||||
|
||||
if col_klasifikasi:
|
||||
|
|
@ -153,7 +150,6 @@ def main():
|
|||
print("DATA PELATIHAN (TRAINING SET):")
|
||||
print(train_df["label"].value_counts().to_string())
|
||||
|
||||
# [KEEP v3] Distribusi per kelas di test set — berguna untuk deteksi class imbalance
|
||||
print("\nDATA PENGUJIAN (TEST SET):")
|
||||
for cls in TARGET_CLASSES:
|
||||
c = test_df["label"].value_counts().get(cls, 0)
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -1,181 +1,3 @@
|
|||
# import pandas as pd
|
||||
# import numpy as np
|
||||
# import re
|
||||
# import joblib
|
||||
# import json
|
||||
# import warnings
|
||||
# from pathlib import Path
|
||||
# from sklearn.feature_extraction.text import TfidfVectorizer
|
||||
# from sklearn.linear_model import LogisticRegression
|
||||
# from sklearn.model_selection import StratifiedKFold
|
||||
# from sklearn.metrics import classification_report, confusion_matrix, ConfusionMatrixDisplay
|
||||
# from sklearn.pipeline import Pipeline
|
||||
# import matplotlib
|
||||
# matplotlib.use('Agg')
|
||||
# import matplotlib.pyplot as plt
|
||||
# warnings.filterwarnings('ignore')
|
||||
|
||||
# BASE_DIR = Path(__file__).parent
|
||||
# TRAIN_FILE = BASE_DIR / "processed" / "training_corpus.csv"
|
||||
# TEST_FILE = BASE_DIR / "processed" / "test_set.csv"
|
||||
# ML_DIR = BASE_DIR.parent / "fastapi" / "ml_assets"
|
||||
# ML_DIR.mkdir(parents=True, exist_ok=True)
|
||||
# TARGET_CLASSES = ["Programmer", "Data Analyst", "Wirausaha Informatika", "Non-IT"]
|
||||
# CONFIDENCE_THRESHOLD = 0.50
|
||||
|
||||
# # [v4] Tidak import Sastrawi — corpus sudah di-stem oleh prepare_corpus.py
|
||||
# # Stemming dua kali pada data yang sama dapat mendegradasi teks.
|
||||
# STOPWORDS = {
|
||||
# "yang", "di", "ke", "dari", "dan", "atau", "dengan", "untuk", "pada", "dalam",
|
||||
# "adalah", "ini", "itu", "tidak", "juga", "sudah", "akan", "bisa", "ada", "oleh",
|
||||
# "karena", "secara", "serta", "sebagai", "bagi", "telah", "maka", "namun", "sehingga",
|
||||
# "jika", "agar", "ketika", "saat", "sebelum", "sesudah", "hingga", "sampai", "antara",
|
||||
# "sekitar", "hanya", "saja", "belum", "masih", "lagi", "pun", "justru", "walaupun",
|
||||
# "meskipun", "bahkan", "cukup", "sangat", "paling", "lebih", "kurang", "lain",
|
||||
# "macam", "cara", "hal", "tentang", "mengenai", "terhadap", "kepada", "menuju",
|
||||
# "kecuali", "selain", "tanpa", "demi", "guna", "khususnya", "umumnya", "kebanyakan",
|
||||
# "sebagian", "beberapa", "semua", "setiap", "tiap", "satu", "dua", "tiga", "empat",
|
||||
# "lima", "enam", "tujuh", "delapan", "sembilan", "sepuluh", "ratus", "ribu", "juta"
|
||||
# }
|
||||
|
||||
# def preprocess_text(text):
|
||||
# """Pembersihan dasar — tanpa stemming ulang karena korpus sudah di-stem."""
|
||||
# if pd.isna(text) or not isinstance(text, str): return ""
|
||||
# text = text.lower().strip()
|
||||
# if text in {"nan", "none", "null", "-", "0", "", "tidak diisi"}: return ""
|
||||
# text = text.replace('-', ' ').replace('/', ' ').replace('_', ' ').replace(',', ' ')
|
||||
# text = re.sub(r'[^\w\s]', '', text)
|
||||
# text = re.sub(r'\d+', '', text)
|
||||
# words = [w for w in text.split() if w not in STOPWORDS and len(w) > 2]
|
||||
# return " ".join(words)
|
||||
|
||||
# def main():
|
||||
# if not TRAIN_FILE.exists():
|
||||
# print("File training_corpus.csv belum ada. Jalankan prepare_corpus.py terlebih dahulu."); return
|
||||
|
||||
# df_train = pd.read_csv(TRAIN_FILE, sep=";", dtype=str).dropna(subset=["job_text_raw", "label"])
|
||||
# df_test = pd.read_csv(TEST_FILE, sep=";", dtype=str).dropna(subset=["job_text_raw", "label"]) if TEST_FILE.exists() else None
|
||||
|
||||
# print("Sedang memproses teks (cleaning dasar tanpa stemming redundan)...")
|
||||
# X_train = df_train["job_text_raw"].apply(preprocess_text)
|
||||
# y_train = df_train["label"]
|
||||
# mask = X_train.str.len() > 0
|
||||
# X_train, y_train = X_train[mask], y_train[mask]
|
||||
|
||||
# X_test, y_test = pd.Series(dtype=str), pd.Series(dtype=str)
|
||||
# if df_test is not None:
|
||||
# X_test = df_test["job_text_raw"].apply(preprocess_text)
|
||||
# y_test = df_test["label"]
|
||||
# mask_t = X_test.str.len() > 0
|
||||
# X_test, y_test = X_test[mask_t], y_test[mask_t]
|
||||
|
||||
# print(f"Jumlah data latih: {len(X_train)} baris | Data uji: {len(X_test)} baris\n")
|
||||
|
||||
# # [KEEP v3] K-Fold cross validation — estimasi variance performa model
|
||||
# print("Melakukan pengujian K-Fold (5 putaran) pada data latih...")
|
||||
# skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
|
||||
# fold_metrics = []
|
||||
# for i, (tr_idx, te_idx) in enumerate(skf.split(X_train, y_train)):
|
||||
# model = Pipeline([
|
||||
# ('tfidf', TfidfVectorizer(max_features=3000, ngram_range=(1,2), sublinear_tf=True, min_df=1)),
|
||||
# ('clf', LogisticRegression(max_iter=1000, class_weight='balanced', solver='lbfgs'))
|
||||
# ])
|
||||
# model.fit(X_train.iloc[tr_idx], y_train.iloc[tr_idx])
|
||||
# y_pred = model.predict(X_train.iloc[te_idx])
|
||||
# rep = classification_report(y_train.iloc[te_idx], y_pred, output_dict=True, zero_division=0)
|
||||
# fold_metrics.append(rep)
|
||||
# print(f" Akurasi putaran ke-{i+1}: {rep['accuracy']:.4f}")
|
||||
|
||||
# avg_acc = np.mean([f['accuracy'] for f in fold_metrics])
|
||||
# print(f"\n RATA-RATA PENGUJIAN K-FOLD:")
|
||||
# print(f" Akurasi Keseluruhan : {avg_acc:.4f} ± {np.std([f['accuracy'] for f in fold_metrics]):.4f}")
|
||||
# for cls in TARGET_CLASSES:
|
||||
# p = np.mean([f.get(cls, {}).get('precision', 0) for f in fold_metrics])
|
||||
# r = np.mean([f.get(cls, {}).get('recall', 0) for f in fold_metrics])
|
||||
# f1 = np.mean([f.get(cls, {}).get('f1-score', 0) for f in fold_metrics])
|
||||
# print(f" {cls:25} | P: {p:.3f} | R: {r:.3f} | F1: {f1:.3f}")
|
||||
|
||||
# print("\n Membuat model final dari seluruh data latih yang tersedia...")
|
||||
# final_model = Pipeline([
|
||||
# ('tfidf', TfidfVectorizer(max_features=3000, ngram_range=(1,2), sublinear_tf=True, min_df=1)),
|
||||
# ('clf', LogisticRegression(max_iter=1000, class_weight='balanced', solver='lbfgs'))
|
||||
# ])
|
||||
# final_model.fit(X_train, y_train)
|
||||
|
||||
# metrics_test = None
|
||||
# if len(X_test) > 0:
|
||||
# y_pred = final_model.predict(X_test)
|
||||
# rep_test = classification_report(y_test, y_pred, output_dict=True, zero_division=0)
|
||||
# metrics_test = rep_test
|
||||
|
||||
# print("\n HASIL PENGUJIAN PADA DATA TEST:")
|
||||
# print(classification_report(y_test, y_pred, zero_division=0))
|
||||
|
||||
# for cls in TARGET_CLASSES:
|
||||
# sup = rep_test[cls]['support'] if cls in rep_test else 0
|
||||
# if sup < 5: print(f"Catatan: Kelas '{cls}' cuma punya {sup} data uji — skor F1-nya mungkin kurang akurat.")
|
||||
|
||||
# # [KEEP v3] Confusion matrix
|
||||
# cm = confusion_matrix(y_test, y_pred, labels=TARGET_CLASSES)
|
||||
# disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=TARGET_CLASSES)
|
||||
# disp.plot(cmap='Blues', values_format='d', xticks_rotation=45, colorbar=False)
|
||||
# plt.title('Confusion Matrix - Hold-Out Test'); plt.tight_layout()
|
||||
# plt.savefig(ML_DIR / 'confusion_matrix_test.png', dpi=300); plt.close()
|
||||
# print("Grafik confusion matrix berhasil disimpan ke confusion_matrix_test.png")
|
||||
|
||||
# # [BUGFIX] Feature importance per kelas
|
||||
# # clf.coef_ diurutkan mengikuti clf.classes_ (alfabetikal sklearn),
|
||||
# # BUKAN urutan TARGET_CLASSES yang hardcoded. Pakai clf.classes_ sebagai
|
||||
# # iterator agar index i selalu selaras dengan baris koefisien yang benar.
|
||||
# vec, clf = final_model.named_steps['tfidf'], final_model.named_steps['clf']
|
||||
# feats, coeffs = vec.get_feature_names_out(), clf.coef_
|
||||
# print("\n5 KATA PALING BERPENGARUH UNTUK TIAP KELAS:")
|
||||
# for i, cls in enumerate(clf.classes_): # <-- clf.classes_, bukan TARGET_CLASSES
|
||||
# top = [(feats[j], coeffs[i][j]) for j in coeffs[i].argsort()[-5:][::-1] if coeffs[i][j] > 0]
|
||||
# print(f" [{cls}] " + ", ".join([f"{w}({c:.2f})" for w, c in top]))
|
||||
|
||||
# # [KEEP v3] Confidence distribution
|
||||
# proba = final_model.predict_proba(X_test)
|
||||
# max_p = np.max(proba, axis=1)
|
||||
# plt.hist(max_p, bins=15, edgecolor='black', alpha=0.7, color='teal')
|
||||
# plt.axvline(x=CONFIDENCE_THRESHOLD, color='red', linestyle='--', linewidth=2, label=f'Threshold {CONFIDENCE_THRESHOLD}')
|
||||
# plt.xlabel('Confidence Score'); plt.ylabel('Count'); plt.title('Distribusi Confidence Score')
|
||||
# plt.legend(); plt.grid(axis='y', alpha=0.3); plt.tight_layout()
|
||||
# plt.savefig(ML_DIR / 'confidence_distribution.png', dpi=300); plt.close()
|
||||
# below = np.sum(max_p < CONFIDENCE_THRESHOLD)
|
||||
# print(f"\nPengecekan Keyakinan Model: {below} dari {len(max_p)} prediksi ({below/len(max_p)*100:.1f}%) di bawah threshold {CONFIDENCE_THRESHOLD}.")
|
||||
|
||||
# # [KEEP v3] Daftar prediksi yang meleset
|
||||
# mis = np.where(y_test.values != y_pred)[0]
|
||||
# if len(mis) > 0:
|
||||
# print("\n DAFTAR PREDIKSI YANG MELESET:")
|
||||
# for idx in mis[:min(6, len(mis))]:
|
||||
# txt = df_test.iloc[idx]['job_text_raw'][:80]
|
||||
# print(f" • Seharusnya: {y_test.iloc[idx]:20} | Ditebak: {y_pred[idx]:20} | Teks: '{txt}...'")
|
||||
# else:
|
||||
# print("\n Semua prediksi pada test set benar.")
|
||||
|
||||
# model_path = ML_DIR / "ml_pipeline_internal.pkl"
|
||||
# joblib.dump(final_model, model_path)
|
||||
|
||||
# metrics = {
|
||||
# "methodology": "Internal MIF only",
|
||||
# "k_fold": {
|
||||
# "accuracy_mean": float(avg_acc),
|
||||
# "accuracy_std": float(np.std([f['accuracy'] for f in fold_metrics])),
|
||||
# "folds": fold_metrics
|
||||
# },
|
||||
# "threshold_config": CONFIDENCE_THRESHOLD
|
||||
# }
|
||||
# if metrics_test: metrics["hold_out_test"] = metrics_test
|
||||
# with open(ML_DIR / "metrics_internal_only.json", 'w') as f: json.dump(metrics, f, indent=2)
|
||||
|
||||
# print(f"\n Model berhasil disimpan di: {model_path}")
|
||||
# print(f"Laporan metrik disimpan di: {ML_DIR / 'metrics_internal_only.json'}")
|
||||
|
||||
# if __name__ == "__main__":
|
||||
# main()
|
||||
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import re
|
||||
|
|
@ -192,9 +14,6 @@ import matplotlib
|
|||
matplotlib.use('Agg')
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
# PERBAIKAN 1: Hapus warnings.filterwarnings('ignore') secara global.
|
||||
# Biarkan warning muncul. Jika ada ConvergenceWarning, Anda akan tahu bahwa max_iter perlu dinaikkan lagi.
|
||||
|
||||
BASE_DIR = Path(__file__).parent
|
||||
TRAIN_FILE = BASE_DIR / "processed" / "training_corpus.csv"
|
||||
TEST_FILE = BASE_DIR / "processed" / "test_set.csv"
|
||||
|
|
@ -223,10 +42,6 @@ def preprocess_text(text):
|
|||
if text in {"nan", "none", "null", "-", "0", "", "tidak diisi"}: return ""
|
||||
text = text.replace('-', ' ').replace('/', ' ').replace('_', ' ').replace(',', ' ')
|
||||
text = re.sub(r'[^\w\s]', '', text)
|
||||
|
||||
# PERBAIKAN 2: Penambahan komentar eksplisit.
|
||||
# Anda harus sadar bahwa menghapus angka menghilangkan sinyal seperti "Python 3" atau "5 tahun".
|
||||
# Jika ini keputusan sadar, biarkan. Jika tidak, hapus baris di bawah ini.
|
||||
text = re.sub(r'\d+', '', text)
|
||||
|
||||
words = [w for w in text.split() if w not in STOPWORDS and len(w) > 2]
|
||||
|
|
@ -239,14 +54,14 @@ def get_pipeline():
|
|||
max_features=3000,
|
||||
ngram_range=(1, 2),
|
||||
sublinear_tf=True,
|
||||
min_df=2, # PERBAIKAN 3: Diubah dari 1 ke 2 untuk mencegah overfitting pada noise/typo.
|
||||
norm='l2' # PERBAIKAN 4: Eksplisit menyatakan normalisasi L2 (default, tapi ditulis agar transparan).
|
||||
min_df=2,
|
||||
norm='l2'
|
||||
)),
|
||||
('clf', LogisticRegression(
|
||||
max_iter=2000, # PERBAIKAN 5: Dinaikkan dari 1000 untuk mencegah ConvergenceWarning pada data sparse.
|
||||
max_iter=2000,
|
||||
class_weight='balanced',
|
||||
solver='lbfgs',
|
||||
random_state=42 # PERBAIKAN 6: Ditambahkan untuk reproducibility (hasil yang konsisten).
|
||||
random_state=42
|
||||
))
|
||||
])
|
||||
|
||||
|
|
@ -279,7 +94,6 @@ def main():
|
|||
fold_metrics = []
|
||||
|
||||
for i, (tr_idx, te_idx) in enumerate(skf.split(X_train, y_train)):
|
||||
# PERBAIKAN 7: Menggunakan fungsi get_pipeline() untuk menghindari duplikasi kode dan inkonsistensi
|
||||
model = get_pipeline()
|
||||
model.fit(X_train.iloc[tr_idx], y_train.iloc[tr_idx])
|
||||
y_pred = model.predict(X_train.iloc[te_idx])
|
||||
|
|
@ -297,7 +111,6 @@ def main():
|
|||
print(f" {cls:25} | P: {p:.3f} | R: {r:.3f} | F1: {f1:.3f}")
|
||||
|
||||
print("\n Membuat model final dari seluruh data latih yang tersedia...")
|
||||
# PERBAIKAN 8: Menggunakan fungsi get_pipeline()
|
||||
final_model = get_pipeline()
|
||||
final_model.fit(X_train, y_train)
|
||||
|
||||
|
|
@ -325,7 +138,6 @@ def main():
|
|||
plt.close()
|
||||
print("Grafik confusion matrix berhasil disimpan ke confusion_matrix_test.png")
|
||||
|
||||
# PERBAIKAN 9: Feature importance yang menampilkan koefisien POSITIF dan NEGATIF
|
||||
vec = final_model.named_steps['tfidf']
|
||||
clf = final_model.named_steps['clf']
|
||||
feats = vec.get_feature_names_out()
|
||||
|
|
|
|||
|
|
@ -138,7 +138,6 @@ def classify_ml(job_text: str, pipeline) -> dict:
|
|||
logger.error(f"ML_INFER_ERROR | job_text='{job_text[:50]}' | err={e}")
|
||||
raise HTTPException(status_code=500, detail=f"ML inference failed: {str(e)}")
|
||||
|
||||
# [KEEP v3] Ekstraksi teks dari format Kemendikbud
|
||||
def extract_kemendik_text(row: pd.Series, df_columns: list) -> str:
|
||||
f5b_col = find_column_by_code(df_columns, "f5b")
|
||||
f5c_col = find_column_by_code(df_columns, "f5c")
|
||||
|
|
@ -175,7 +174,6 @@ def health_check():
|
|||
|
||||
@app.post("/api/v1/retrain", dependencies=[Depends(verify_token)])
|
||||
async def trigger_retrain(file: UploadFile = File(None)):
|
||||
# [v4] FileLock mencegah race condition dari concurrent requests
|
||||
lock = FileLock(LOCK_PATH, timeout=2)
|
||||
try:
|
||||
with lock:
|
||||
|
|
@ -221,7 +219,6 @@ def retrain_status():
|
|||
if not STATUS_PATH.exists():
|
||||
return JSONResponse(content={"stage": "idle", "message": "Belum ada proses re-training."})
|
||||
try:
|
||||
# [v4] Lock saat baca untuk menghindari partial write dari worker
|
||||
lock = FileLock(LOCK_PATH, timeout=2)
|
||||
with lock:
|
||||
data = json.loads(STATUS_PATH.read_text(encoding="utf-8"))
|
||||
|
|
@ -241,7 +238,6 @@ def retrain_status():
|
|||
return JSONResponse(content={"stage": "unknown", "message": "Status tidak terbaca."})
|
||||
|
||||
|
||||
# [KEEP v3] Endpoint reload manual — berguna jika pipeline perlu di-reload tanpa restart server
|
||||
@app.post("/api/v1/retrain/reload", dependencies=[Depends(verify_token)])
|
||||
def reload_model():
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@ Alur:
|
|||
6. Evaluasi candidate pada test set yang sama → delta terhadap old model
|
||||
7. Promote jika lebih baik; rollback jika tidak
|
||||
8. Setiap write status dilindungi FileLock (atomic)
|
||||
9. [v5] Semua event log disimpan ke file JSONL terpisah untuk audit BAB 4
|
||||
"""
|
||||
import sys
|
||||
import json
|
||||
|
|
@ -42,7 +41,7 @@ warnings.filterwarnings("ignore")
|
|||
BASE_DIR = Path(__file__).parent.parent
|
||||
ML_DIR = BASE_DIR / "ml_assets"
|
||||
DATA_DIR = BASE_DIR.parent / "data" / "processed"
|
||||
LOG_DIR = ML_DIR / "logs" # [v5] Direktori log terpisah
|
||||
LOG_DIR = ML_DIR / "logs"
|
||||
LOG_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
PIPELINE_PATH = ML_DIR / "ml_pipeline_internal.pkl"
|
||||
|
|
@ -54,7 +53,6 @@ LOCK_PATH = ML_DIR / "retrain_status.lock"
|
|||
CORPUS_PATH = DATA_DIR / "training_corpus.csv"
|
||||
TEST_PATH = DATA_DIR / "test_set.csv"
|
||||
|
||||
# [v5] Nama file log berbasis timestamp — satu file per sesi retraining
|
||||
SESSION_TS = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
LOG_FILE_PATH = LOG_DIR / f"retrain_log_{SESSION_TS}.jsonl"
|
||||
|
||||
|
|
@ -81,7 +79,7 @@ STOPWORDS = {
|
|||
stemmer = StemmerFactory().create_stemmer()
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# [v5] CUSTOM JSON FORMATTER — setiap log entry jadi JSON valid
|
||||
# CUSTOM JSON FORMATTER
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
class JsonFormatter(logging.Formatter):
|
||||
"""Format log record sebagai JSON satu baris (JSONL)."""
|
||||
|
|
@ -92,7 +90,6 @@ class JsonFormatter(logging.Formatter):
|
|||
"logger": record.name,
|
||||
"message": record.getMessage(),
|
||||
}
|
||||
# [v5] Jika ada extra data (stage, metrics, dll), gabungkan ke root
|
||||
if hasattr(record, "stage"):
|
||||
log_entry["stage"] = record.stage
|
||||
if hasattr(record, "event_type"):
|
||||
|
|
@ -102,7 +99,7 @@ class JsonFormatter(logging.Formatter):
|
|||
return json.dumps(log_entry, ensure_ascii=False)
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# [v5] DUAL LOGGER SETUP — console + file JSONL
|
||||
# DUAL LOGGER SETUP
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
logger = logging.getLogger("retrain_worker")
|
||||
logger.setLevel(logging.INFO)
|
||||
|
|
@ -123,7 +120,7 @@ logger.addHandler(file_handler)
|
|||
logger.info(f"[init] Log file tersimpan di: {LOG_FILE_PATH}")
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# [v5] STRUCTURED LOG HELPER — untuk event penting dengan metadata
|
||||
# STRUCTURED LOG HELPER
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
def log_event(event_type: str, message: str, stage: str = "info", metadata: dict = None):
|
||||
"""
|
||||
|
|
@ -148,7 +145,7 @@ def compute_override_weight(n_corpus: int, n_override: int) -> float:
|
|||
return round(w_log, 4)
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# STATUS WRITER — atomic dengan FileLock (v4)
|
||||
# STATUS WRITER
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
def write_status(stage: str, message: str, extra: dict = None):
|
||||
lock = FileLock(LOCK_PATH, timeout=10)
|
||||
|
|
@ -156,7 +153,6 @@ def write_status(stage: str, message: str, extra: dict = None):
|
|||
payload = {"stage": stage, "message": message, "timestamp": datetime.now().isoformat()}
|
||||
if extra: payload.update(extra)
|
||||
STATUS_PATH.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
# [v5] Tulis juga ke log file dengan stage tracking
|
||||
logger.info(f"[{stage}] {message}", extra={"stage": stage})
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
|
|
@ -228,7 +224,6 @@ def do_rollback(reason: str, old_f1: float, new_f1: float):
|
|||
else:
|
||||
logger.warning("File .bak tidak ditemukan — pkl aktif dibiarkan.")
|
||||
|
||||
# [v5] Log event rollback dengan metadata lengkap
|
||||
log_event(
|
||||
event_type="rollback",
|
||||
message=f"Model lama dipertahankan. {reason}",
|
||||
|
|
@ -246,7 +241,6 @@ def do_rollback(reason: str, old_f1: float, new_f1: float):
|
|||
# MAIN
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
def main(extra_csv_path: str = None):
|
||||
# [v5] Log awal sesi — penting untuk audit trail
|
||||
log_event(
|
||||
event_type="session_start",
|
||||
message=f"Retraining session dimulai. Log file: {LOG_FILE_PATH.name}",
|
||||
|
|
@ -314,7 +308,6 @@ def main(extra_csv_path: str = None):
|
|||
y_all = df_all["label"]
|
||||
w_all = df_all["_weight"]
|
||||
|
||||
# [v5] Log event data merge dengan metadata
|
||||
log_event(
|
||||
event_type="data_merged",
|
||||
message=f"Data training siap: {len(df_all)} baris",
|
||||
|
|
@ -342,7 +335,7 @@ def main(extra_csv_path: str = None):
|
|||
write_status("training", f"K-Fold validation & training... ({len(X_train)} sampel)")
|
||||
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
|
||||
fold_metrics = []
|
||||
fold_accuracies = [] # [v5] untuk log ringkas
|
||||
fold_accuracies = []
|
||||
for i, (tr_idx, te_idx) in enumerate(skf.split(X_train, y_train)):
|
||||
m = Pipeline([
|
||||
('tfidf', TfidfVectorizer(max_features=3000, ngram_range=(1, 2), sublinear_tf=True, min_df=1)),
|
||||
|
|
@ -358,7 +351,6 @@ def main(extra_csv_path: str = None):
|
|||
fold_accuracies.append(round(rep['accuracy'], 4))
|
||||
write_status("training", f"K-Fold selesai: fold {i+1}/5 | acc={rep['accuracy']:.4f}")
|
||||
|
||||
# [v5] Log ringkas hasil K-Fold
|
||||
log_event(
|
||||
event_type="kfold_completed",
|
||||
message=f"K-Fold 5 lipatan selesai. Akurasi per fold: {fold_accuracies}",
|
||||
|
|
@ -405,7 +397,6 @@ def main(extra_csv_path: str = None):
|
|||
f"minimum dibutuhkan: +{MIN_IMPROVEMENT_THRESHOLD*100:.1f}%)"
|
||||
)
|
||||
|
||||
# [v5] Log event evaluasi dengan metadata lengkap — ini yang akan dikutip di BAB 4
|
||||
log_event(
|
||||
event_type="evaluation_completed",
|
||||
message=f"Evaluasi selesai. Delta F1: {delta:+.4f} ({'promote' if should_promote else 'rollback'})",
|
||||
|
|
@ -460,7 +451,6 @@ def main(extra_csv_path: str = None):
|
|||
}
|
||||
METRICS_PATH.write_text(json.dumps(new_metrics_full, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
|
||||
# [v5] Log event promote final
|
||||
log_event(
|
||||
event_type="model_promoted",
|
||||
message=f"Model baru berhasil dipromote. {reason}",
|
||||
|
|
@ -492,7 +482,6 @@ def main(extra_csv_path: str = None):
|
|||
except Exception as e:
|
||||
logger.error(f"ERROR saat training: {type(e).__name__}: {e}", exc_info=True)
|
||||
|
||||
# [v5] Log event error dengan traceback info
|
||||
log_event(
|
||||
event_type="training_error",
|
||||
message=f"Training gagal: {type(e).__name__}: {str(e)[:200]}",
|
||||
|
|
|
|||
Binary file not shown.
|
Before Width: | Height: | Size: 62 KiB After Width: | Height: | Size: 92 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 114 KiB After Width: | Height: | Size: 137 KiB |
Binary file not shown.
Loading…
Reference in New Issue