Merge branch 'main' of https://github.com/mhilya/sista_mif_ta
This commit is contained in:
commit
b5d5a9b63a
File diff suppressed because one or more lines are too long
|
|
@ -2,19 +2,17 @@
|
|||
"""
|
||||
TRACER STUDY - RE-TRAINING WORKER FINAL (SUBPROCESS)
|
||||
Dipanggil oleh FastAPI (main3.py) sebagai background subprocess.
|
||||
|
||||
Alur:
|
||||
1. Backup pkl lama → pkl.bak
|
||||
2. Merge corpus + manual_override (TANPA drop_duplicates)
|
||||
Override diberi sample_weight lebih tinggi, bukan menghapus data historis.
|
||||
3. Dynamic train/test split dari data gabungan (selalu fresh)
|
||||
4. Evaluasi MODEL LAMA pada test set yang sama (fair comparison)
|
||||
5. K-Fold + train model final (candidate) dengan sample_weight
|
||||
6. Evaluasi candidate pada test set yang sama → delta terhadap old model
|
||||
7. Promote jika lebih baik; rollback jika tidak
|
||||
8. Setiap write status dilindungi FileLock (atomic)
|
||||
9. [v5] Semua event log disimpan ke file JSONL terpisah untuk audit BAB 4
|
||||
"""
|
||||
|
||||
import sys
|
||||
import json
|
||||
import shutil
|
||||
|
|
@ -39,11 +37,13 @@ from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
|
|||
warnings.filterwarnings("ignore")
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# KONFIGURASI
|
||||
# KONFIGURASI PATH
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
BASE_DIR = Path(__file__).parent.parent
|
||||
ML_DIR = BASE_DIR / "ml_assets"
|
||||
DATA_DIR = BASE_DIR.parent / "data" / "processed"
|
||||
LOG_DIR = ML_DIR / "logs" # [v5] Direktori log terpisah
|
||||
LOG_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
PIPELINE_PATH = ML_DIR / "ml_pipeline_internal.pkl"
|
||||
PIPELINE_BAK_PATH = ML_DIR / "ml_pipeline_internal.pkl.bak"
|
||||
|
|
@ -54,46 +54,17 @@ LOCK_PATH = ML_DIR / "retrain_status.lock"
|
|||
CORPUS_PATH = DATA_DIR / "training_corpus.csv"
|
||||
TEST_PATH = DATA_DIR / "test_set.csv"
|
||||
|
||||
# [v5] Nama file log berbasis timestamp — satu file per sesi retraining
|
||||
SESSION_TS = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
LOG_FILE_PATH = LOG_DIR / f"retrain_log_{SESSION_TS}.jsonl"
|
||||
|
||||
TARGET_CLASSES = ["Programmer", "Data Analyst", "Wirausaha Informatika", "Non-IT"]
|
||||
MIN_IMPROVEMENT_THRESHOLD = 0.01 # Model baru harus lebih baik minimal +1% weighted F1
|
||||
MAX_REGRESSION_ALLOWED = 0.02 # Toleransi degradasi maksimal 2% sebelum rollback keras
|
||||
MIN_IMPROVEMENT_THRESHOLD = 0.01
|
||||
MAX_REGRESSION_ALLOWED = 0.02
|
||||
|
||||
# ── KONFIGURASI BOBOT OVERRIDE ──────────────────────────────────────────────
|
||||
#
|
||||
# Masalah yang diselesaikan:
|
||||
# Pada skala besar (N_corpus >> N_override), bobot statis kehilangan daya
|
||||
# akibat dilusi. Formula proporsi murni w = t*N / (M*(1-t)) menyelesaikan
|
||||
# dilusi, tapi menghasilkan w=42.85 pada skenario 10k/100 — yang berisiko
|
||||
# overfitting ekstrem pada noise override.
|
||||
#
|
||||
# Solusi: Logarithmic damping — tanpa tembok statis.
|
||||
# w_raw = formula proporsi (jaminan 30% jika tidak di-damp)
|
||||
# w = MIN + ln(1 + max(0, w_raw - MIN))
|
||||
#
|
||||
# Fase linear (w_raw rendah): w ≈ w_raw → proporsi terpenuhi
|
||||
# Fase log (w_raw tinggi): w tumbuh tapi melambat → damp alami
|
||||
#
|
||||
# Implikasi jujur:
|
||||
# Target 30% TIDAK dipertahankan di skala ekstrem. Ini trade-off yang
|
||||
# disengaja: degradasi gradual lebih aman daripada overfitting ke 10
|
||||
# baris override berbobot 42x. Tanpa tembok statis, redaman terjadi
|
||||
# secara natural mengikuti kurva logaritmik, bukan menabrak batas arbitrer.
|
||||
#
|
||||
# Perilaku nyata (MIN=2.0, TARGET=0.30):
|
||||
# w_raw │ w_log │ influence aktual
|
||||
# ──────────────────────────────────
|
||||
# 2.00 │ 2.000 │ formula <= MIN, pakai floor
|
||||
# 4.29 │ 3.178 │ (1k corpus, 100 override) → ~24%
|
||||
# 8.57 │ 3.999 │ (100 korpus, 5 override) → ~17%
|
||||
# 42.86 │ 5.723 │ (10k corpus, 100 override)→ ~5.4%
|
||||
# 428.60 │ 8.063 │ (10k corpus, 10 override) → ~0.8%
|
||||
#
|
||||
# Jika angka influence aktual dianggap terlalu kecil → naikkan TARGET.
|
||||
# Jika model terlalu sensitif ke override → naikkan MIN agar floor lebih tinggi.
|
||||
OVERRIDE_INFLUENCE_TARGET = 0.30 # Titik acuan proporsi (valid di skala normal)
|
||||
OVERRIDE_MIN_WEIGHT = 2.0 # Lantai: override selalu minimal 2× korpus
|
||||
OVERRIDE_INFLUENCE_TARGET = 0.30
|
||||
OVERRIDE_MIN_WEIGHT = 2.0
|
||||
|
||||
# [FIX v4] STOPWORDS lengkap — versi retrain_worker2 hanya punya 10 kata (bug terpotong)
|
||||
STOPWORDS = {
|
||||
"yang", "di", "ke", "dari", "dan", "atau", "dengan", "untuk", "pada", "dalam",
|
||||
"adalah", "ini", "itu", "tidak", "juga", "sudah", "akan", "bisa", "ada", "oleh",
|
||||
|
|
@ -109,49 +80,73 @@ STOPWORDS = {
|
|||
|
||||
stemmer = StemmerFactory().create_stemmer()
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# [v5] CUSTOM JSON FORMATTER — setiap log entry jadi JSON valid
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
class JsonFormatter(logging.Formatter):
|
||||
"""Format log record sebagai JSON satu baris (JSONL)."""
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
log_entry = {
|
||||
"timestamp": datetime.fromtimestamp(record.created).isoformat(),
|
||||
"level": record.levelname,
|
||||
"logger": record.name,
|
||||
"message": record.getMessage(),
|
||||
}
|
||||
# [v5] Jika ada extra data (stage, metrics, dll), gabungkan ke root
|
||||
if hasattr(record, "stage"):
|
||||
log_entry["stage"] = record.stage
|
||||
if hasattr(record, "event_type"):
|
||||
log_entry["event_type"] = record.event_type
|
||||
if hasattr(record, "metadata") and isinstance(record.metadata, dict):
|
||||
log_entry["metadata"] = record.metadata
|
||||
return json.dumps(log_entry, ensure_ascii=False)
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# [v5] DUAL LOGGER SETUP — console + file JSONL
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
logger = logging.getLogger("retrain_worker")
|
||||
logger.setLevel(logging.INFO)
|
||||
logger.propagate = False # mencegah duplikasi ke root logger
|
||||
|
||||
# Handler 1: Console (untuk dev/debugging)
|
||||
console_handler = logging.StreamHandler(sys.stdout)
|
||||
console_handler.setFormatter(logging.Formatter(
|
||||
"%(asctime)s | %(levelname)s | %(message)s"
|
||||
))
|
||||
logger.addHandler(console_handler)
|
||||
|
||||
# Handler 2: File JSONL (untuk audit BAB 4 & debugging production)
|
||||
file_handler = logging.FileHandler(LOG_FILE_PATH, mode="w", encoding="utf-8")
|
||||
file_handler.setFormatter(JsonFormatter())
|
||||
logger.addHandler(file_handler)
|
||||
|
||||
logger.info(f"[init] Log file tersimpan di: {LOG_FILE_PATH}")
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# [v5] STRUCTURED LOG HELPER — untuk event penting dengan metadata
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
def log_event(event_type: str, message: str, stage: str = "info", metadata: dict = None):
|
||||
"""
|
||||
Tulis event terstruktur ke log file.
|
||||
Berbeda dengan logger.info biasa, fungsi ini menyertakan metadata
|
||||
yang bisa diparse programmatically (misal: old_f1, new_f1, delta).
|
||||
"""
|
||||
extra = {"event_type": event_type, "stage": stage}
|
||||
if metadata:
|
||||
extra["metadata"] = metadata
|
||||
logger.info(message, extra=extra)
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# DYNAMIC WEIGHT CALCULATOR
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
def compute_override_weight(n_corpus: int, n_override: int) -> float:
|
||||
"""
|
||||
Hitung bobot override dengan logarithmic damping.
|
||||
|
||||
TIDAK ada OVERRIDE_MAX_WEIGHT — tembok statis menghancurkan jaminan
|
||||
pengaruh tepat saat skala besar membutuhkannya. Sebagai gantinya,
|
||||
fungsi ln(1+x) menyediakan redaman alami:
|
||||
|
||||
w_raw = (t * n_corpus) / (n_override * (1 - t)) # proporsi murni
|
||||
w = MIN + ln(1 + max(0, w_raw - MIN)) # log damping
|
||||
|
||||
Jaminan matematis:
|
||||
- w selalu >= OVERRIDE_MIN_WEIGHT (floor tetap ada)
|
||||
- w tidak pernah meledak ke infinity karena ln tumbuh O(log n)
|
||||
- Tidak ada tembok statis yang membuat influence kolaps tiba-tiba
|
||||
|
||||
Catatan interaksi:
|
||||
LogisticRegression dipanggil dengan class_weight='balanced' DAN
|
||||
sample_weight. Keduanya dikalikan oleh sklearn secara internal.
|
||||
Artinya sampel override dari kelas minoritas mendapat boost ganda.
|
||||
Pantau per-class F1 di metrics JSON untuk mendeteksi efek ini.
|
||||
"""
|
||||
if n_override == 0:
|
||||
return 1.0
|
||||
t = OVERRIDE_INFLUENCE_TARGET
|
||||
w_raw = (t * n_corpus) / (n_override * (1.0 - t))
|
||||
# ln damping: linear di zona rendah, melambat secara alami di skala besar
|
||||
w_log = OVERRIDE_MIN_WEIGHT + math.log1p(max(0.0, w_raw - OVERRIDE_MIN_WEIGHT))
|
||||
return round(w_log, 4)
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# LOGGING
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s | %(levelname)s | %(message)s",
|
||||
handlers=[logging.StreamHandler(sys.stdout)]
|
||||
)
|
||||
logger = logging.getLogger("retrain_worker")
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# STATUS WRITER — atomic dengan FileLock (v4)
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
|
|
@ -161,13 +156,13 @@ def write_status(stage: str, message: str, extra: dict = None):
|
|||
payload = {"stage": stage, "message": message, "timestamp": datetime.now().isoformat()}
|
||||
if extra: payload.update(extra)
|
||||
STATUS_PATH.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
logger.info(f"[{stage}] {message}")
|
||||
# [v5] Tulis juga ke log file dengan stage tracking
|
||||
logger.info(f"[{stage}] {message}", extra={"stage": stage})
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# PREPROCESSING — dua fungsi terpisah (v4)
|
||||
# PREPROCESSING
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
def preprocess_stemmed(text: str) -> str:
|
||||
"""Untuk data korpus yang SUDAH di-stem oleh prepare_corpus — tidak re-stem."""
|
||||
if pd.isna(text) or not isinstance(text, str): return ""
|
||||
text = text.lower().strip()
|
||||
if text in {"nan", "none", "null", "-", "0", "", "tidak diisi"}: return ""
|
||||
|
|
@ -178,7 +173,6 @@ def preprocess_stemmed(text: str) -> str:
|
|||
return " ".join(words)
|
||||
|
||||
def preprocess_raw(text: str) -> str:
|
||||
"""Untuk data manual_override yang BELUM di-stem — jalankan Sastrawi."""
|
||||
if pd.isna(text) or not isinstance(text, str): return ""
|
||||
text = text.lower().strip()
|
||||
if text in {"nan", "none", "null", "-", "0", "", "tidak diisi"}: return ""
|
||||
|
|
@ -210,18 +204,7 @@ def evaluate_model(model, X_test: pd.Series, y_test: pd.Series) -> dict:
|
|||
}
|
||||
}
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# BASELINE F1 — re-evaluasi model lama pada test set yang sama
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
def get_baseline_f1_on_testset(X_test: pd.Series, y_test: pd.Series) -> float:
|
||||
"""
|
||||
Evaluasi model lama (backup) pada X_test/y_test yang SAMA dengan yang
|
||||
akan digunakan mengevaluasi candidate. Ini satu-satunya cara yang adil
|
||||
(apples-to-apples) karena distribusi test set berubah setiap retrain.
|
||||
|
||||
Membandingkan F1 baru dengan angka JSON lama (dihitung di distribusi berbeda)
|
||||
adalah perbandingan apel vs jeruk — tidak valid untuk keputusan promote/rollback.
|
||||
"""
|
||||
if not PIPELINE_BAK_PATH.exists():
|
||||
logger.warning("Backup .bak tidak ditemukan — baseline F1 diasumsikan 0.0")
|
||||
return 0.0
|
||||
|
|
@ -244,6 +227,15 @@ def do_rollback(reason: str, old_f1: float, new_f1: float):
|
|||
logger.info("Rollback berhasil: pkl lama dipulihkan dari .bak")
|
||||
else:
|
||||
logger.warning("File .bak tidak ditemukan — pkl aktif dibiarkan.")
|
||||
|
||||
# [v5] Log event rollback dengan metadata lengkap
|
||||
log_event(
|
||||
event_type="rollback",
|
||||
message=f"Model lama dipertahankan. {reason}",
|
||||
stage="rolled_back",
|
||||
metadata={"reason": reason, "old_f1": old_f1, "new_f1": new_f1}
|
||||
)
|
||||
|
||||
write_status(
|
||||
stage="rolled_back",
|
||||
message=f"Model lama dipertahankan. {reason}",
|
||||
|
|
@ -254,6 +246,13 @@ def do_rollback(reason: str, old_f1: float, new_f1: float):
|
|||
# MAIN
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
def main(extra_csv_path: str = None):
|
||||
# [v5] Log awal sesi — penting untuk audit trail
|
||||
log_event(
|
||||
event_type="session_start",
|
||||
message=f"Retraining session dimulai. Log file: {LOG_FILE_PATH.name}",
|
||||
stage="started",
|
||||
metadata={"extra_csv_provided": extra_csv_path is not None}
|
||||
)
|
||||
write_status("started", "Worker dimulai")
|
||||
|
||||
# ── STEP 1: BACKUP ──────────────────────────────────────
|
||||
|
|
@ -264,8 +263,6 @@ def main(extra_csv_path: str = None):
|
|||
shutil.copy2(PIPELINE_PATH, PIPELINE_BAK_PATH)
|
||||
logger.info(f"Backup tersimpan: {PIPELINE_BAK_PATH}")
|
||||
|
||||
# baseline_f1 akan dihitung setelah split dinamis terbentuk
|
||||
# (evaluasi model lama pada test set yang sama dengan candidate)
|
||||
baseline_f1 = 0.0
|
||||
|
||||
try:
|
||||
|
|
@ -277,7 +274,7 @@ def main(extra_csv_path: str = None):
|
|||
|
||||
df_corpus = pd.read_csv(CORPUS_PATH, sep=";", dtype=str).dropna(subset=["job_text_raw", "label"])
|
||||
df_corpus["features"] = df_corpus["job_text_raw"].apply(preprocess_stemmed)
|
||||
df_corpus["_weight"] = 1.0 # Bobot normal untuk data historis
|
||||
df_corpus["_weight"] = 1.0
|
||||
logger.info(f"Corpus asli: {len(df_corpus)} baris")
|
||||
|
||||
df_extra = pd.DataFrame()
|
||||
|
|
@ -288,9 +285,6 @@ def main(extra_csv_path: str = None):
|
|||
override_w = compute_override_weight(len(df_corpus), len(df_extra))
|
||||
df_extra["_weight"] = override_w
|
||||
|
||||
# Pre-compute total bobot sekali — dipakai di logging & metrics JSON.
|
||||
# .sum() lebih benar daripada len() * .iloc[0] karena tidak mengasumsikan
|
||||
# homogenitas bobot di seluruh baris (future-proof jika bobot per-baris ditambahkan).
|
||||
total_corpus_weight = df_corpus["_weight"].sum()
|
||||
total_override_weight = df_extra["_weight"].sum() if len(df_extra) > 0 else 0.0
|
||||
actual_influence_pct = (
|
||||
|
|
@ -308,9 +302,7 @@ def main(extra_csv_path: str = None):
|
|||
else:
|
||||
logger.info("Tidak ada data tambahan — menggunakan corpus asli saja")
|
||||
|
||||
# Concat tanpa deduplication — semua frekuensi historis dipertahankan
|
||||
df_all = pd.concat([df_corpus, df_extra], ignore_index=True) if len(df_extra) > 0 else df_corpus.copy()
|
||||
|
||||
mask = df_all["features"].str.len() > 0
|
||||
df_all = df_all[mask]
|
||||
logger.info(f"Total setelah merge (tanpa deduplicate): {len(df_all)} baris")
|
||||
|
|
@ -322,19 +314,27 @@ def main(extra_csv_path: str = None):
|
|||
y_all = df_all["label"]
|
||||
w_all = df_all["_weight"]
|
||||
|
||||
# ── STEP 3: DYNAMIC SPLIT — selalu dari data gabungan terkini ────
|
||||
# Test set statis (test_set.csv) tidak digunakan karena:
|
||||
# (a) tidak mencerminkan pola baru dari data override
|
||||
# (b) baseline_f1 dari JSON dihitung pada distribusi berbeda →
|
||||
# perbandingan apel vs jeruk, tidak valid untuk keputusan promote.
|
||||
# Solusi: split dinamis, lalu evaluasi model LAMA pada test set YANG SAMA.
|
||||
# [v5] Log event data merge dengan metadata
|
||||
log_event(
|
||||
event_type="data_merged",
|
||||
message=f"Data training siap: {len(df_all)} baris",
|
||||
stage="loading_data",
|
||||
metadata={
|
||||
"corpus_rows": len(df_corpus),
|
||||
"override_rows": len(df_extra),
|
||||
"override_weight": override_w if len(df_extra) > 0 else 1.0,
|
||||
"influence_pct": round(actual_influence_pct, 2),
|
||||
"total_merged": len(df_all),
|
||||
}
|
||||
)
|
||||
|
||||
# ── STEP 3: DYNAMIC SPLIT ────────────────────────────
|
||||
X_train, X_test, y_train, y_test, w_train, _ = train_test_split(
|
||||
X_all, y_all, w_all, test_size=0.3, random_state=42, stratify=y_all
|
||||
)
|
||||
logger.info(f"Dynamic split: {len(X_train)} train | {len(X_test)} test")
|
||||
|
||||
# ── STEP 4: BASELINE — evaluasi model LAMA pada test set yang sama ──
|
||||
# Ini satu-satunya cara perbandingan yang jujur (apples-to-apples).
|
||||
# ── STEP 4: BASELINE ─────────────────────────────────
|
||||
write_status("evaluating", "Mengevaluasi model lama pada test set baru...")
|
||||
baseline_f1 = get_baseline_f1_on_testset(X_test, y_test)
|
||||
|
||||
|
|
@ -342,12 +342,12 @@ def main(extra_csv_path: str = None):
|
|||
write_status("training", f"K-Fold validation & training... ({len(X_train)} sampel)")
|
||||
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
|
||||
fold_metrics = []
|
||||
fold_accuracies = [] # [v5] untuk log ringkas
|
||||
for i, (tr_idx, te_idx) in enumerate(skf.split(X_train, y_train)):
|
||||
m = Pipeline([
|
||||
('tfidf', TfidfVectorizer(max_features=3000, ngram_range=(1, 2), sublinear_tf=True, min_df=1)),
|
||||
('clf', LogisticRegression(max_iter=1000, class_weight='balanced', solver='lbfgs'))
|
||||
])
|
||||
# sample_weight diteruskan ke step 'clf' via Pipeline naming convention
|
||||
m.fit(
|
||||
X_train.iloc[tr_idx], y_train.iloc[tr_idx],
|
||||
clf__sample_weight=w_train.iloc[tr_idx].values
|
||||
|
|
@ -355,8 +355,21 @@ def main(extra_csv_path: str = None):
|
|||
y_pred = m.predict(X_train.iloc[te_idx])
|
||||
rep = classification_report(y_train.iloc[te_idx], y_pred, output_dict=True, zero_division=0)
|
||||
fold_metrics.append(rep)
|
||||
fold_accuracies.append(round(rep['accuracy'], 4))
|
||||
write_status("training", f"K-Fold selesai: fold {i+1}/5 | acc={rep['accuracy']:.4f}")
|
||||
|
||||
# [v5] Log ringkas hasil K-Fold
|
||||
log_event(
|
||||
event_type="kfold_completed",
|
||||
message=f"K-Fold 5 lipatan selesai. Akurasi per fold: {fold_accuracies}",
|
||||
stage="training",
|
||||
metadata={
|
||||
"fold_accuracies": fold_accuracies,
|
||||
"mean_accuracy": round(float(np.mean(fold_accuracies)), 4),
|
||||
"std_accuracy": round(float(np.std(fold_accuracies)), 4),
|
||||
}
|
||||
)
|
||||
|
||||
# ── STEP 6: TRAINING MODEL FINAL ─────────────────────
|
||||
write_status("training", "Training model final pada seluruh data training...")
|
||||
candidate_model = Pipeline([
|
||||
|
|
@ -392,11 +405,29 @@ def main(extra_csv_path: str = None):
|
|||
f"minimum dibutuhkan: +{MIN_IMPROVEMENT_THRESHOLD*100:.1f}%)"
|
||||
)
|
||||
|
||||
# [v5] Log event evaluasi dengan metadata lengkap — ini yang akan dikutip di BAB 4
|
||||
log_event(
|
||||
event_type="evaluation_completed",
|
||||
message=f"Evaluasi selesai. Delta F1: {delta:+.4f} ({'promote' if should_promote else 'rollback'})",
|
||||
stage="evaluating",
|
||||
metadata={
|
||||
"baseline_f1": baseline_f1,
|
||||
"candidate_f1": new_f1,
|
||||
"delta": round(delta, 4),
|
||||
"delta_pct": round(delta * 100, 2),
|
||||
"threshold_min": MIN_IMPROVEMENT_THRESHOLD,
|
||||
"decision": "promote" if should_promote else "rollback",
|
||||
"reason": reason,
|
||||
"new_accuracy": new_metrics["accuracy"],
|
||||
"per_class_f1": {cls: v["f1"] for cls, v in new_metrics["per_class"].items()},
|
||||
}
|
||||
)
|
||||
|
||||
if not should_promote:
|
||||
do_rollback(reason, baseline_f1, new_f1)
|
||||
return
|
||||
|
||||
# ── STEP 7: PROMOTE ──────────────────────────────────
|
||||
# ── STEP 8: PROMOTE ──────────────────────────────────
|
||||
write_status("promoting", "Model baru lebih baik — mempromote model baru...")
|
||||
shutil.move(str(CANDIDATE_PATH), str(PIPELINE_PATH))
|
||||
logger.info(f"Model baru dipromote: {PIPELINE_PATH}")
|
||||
|
|
@ -420,7 +451,6 @@ def main(extra_csv_path: str = None):
|
|||
y_test, candidate_model.predict(X_test),
|
||||
output_dict=True, zero_division=0
|
||||
),
|
||||
# Perbandingan fair: kedua model dievaluasi pada test set yang SAMA
|
||||
"comparison": {
|
||||
"note": "Both models evaluated on identical dynamic test set",
|
||||
"old_weighted_f1": baseline_f1,
|
||||
|
|
@ -430,6 +460,20 @@ def main(extra_csv_path: str = None):
|
|||
}
|
||||
METRICS_PATH.write_text(json.dumps(new_metrics_full, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
|
||||
# [v5] Log event promote final
|
||||
log_event(
|
||||
event_type="model_promoted",
|
||||
message=f"Model baru berhasil dipromote. {reason}",
|
||||
stage="promoted",
|
||||
metadata={
|
||||
"old_f1": baseline_f1,
|
||||
"new_f1": new_f1,
|
||||
"delta": round(delta, 4),
|
||||
"log_file": LOG_FILE_PATH.name,
|
||||
"metrics_file": METRICS_PATH.name,
|
||||
}
|
||||
)
|
||||
|
||||
write_status(
|
||||
stage="promoted",
|
||||
message=f"Model baru berhasil dipromote. {reason}",
|
||||
|
|
@ -447,12 +491,20 @@ def main(extra_csv_path: str = None):
|
|||
|
||||
except Exception as e:
|
||||
logger.error(f"ERROR saat training: {type(e).__name__}: {e}", exc_info=True)
|
||||
|
||||
# [v5] Log event error dengan traceback info
|
||||
log_event(
|
||||
event_type="training_error",
|
||||
message=f"Training gagal: {type(e).__name__}: {str(e)[:200]}",
|
||||
stage="failed",
|
||||
metadata={"error_type": type(e).__name__, "error_message": str(e)[:500]}
|
||||
)
|
||||
|
||||
do_rollback(
|
||||
reason=f"Training gagal karena error: {type(e).__name__}: {str(e)[:200]}",
|
||||
old_f1=baseline_f1,
|
||||
new_f1=0.0
|
||||
)
|
||||
# Override stage ke 'failed' agar UI tahu ini bukan rollback biasa
|
||||
lock = FileLock(LOCK_PATH, timeout=10)
|
||||
with lock:
|
||||
status = json.loads(STATUS_PATH.read_text(encoding="utf-8"))
|
||||
|
|
|
|||
|
|
@ -0,0 +1,11 @@
|
|||
import joblib
|
||||
pipeline = joblib.load('ml_assets/ml_pipeline_internal.pkl')
|
||||
vec = pipeline.named_steps['tfidf']
|
||||
clf = pipeline.named_steps['clf']
|
||||
|
||||
# Ambil 5 fitur dengan bobot tertinggi untuk kelas Programmer
|
||||
feats = vec.get_feature_names_out()
|
||||
prog_idx = list(clf.classes_).index('Programmer')
|
||||
top5 = sorted(zip(feats, clf.coef_[prog_idx]), key=lambda x: -x[1])[:10]
|
||||
for w, c in top5:
|
||||
print(f"{w}: {c:.4f}")
|
||||
|
|
@ -0,0 +1 @@
|
|||
{"status": "error", "message": "Processing failed: 'charmap' codec can't encode character '\\U0001f602' in position 885944: character maps to <undefined>", "processed_rows": 0, "total_rows": 0, "source_type": "unknown"}
|
||||
|
|
@ -0,0 +1 @@
|
|||
{"status": "error", "message": "Processing failed: 'charmap' codec can't encode character '\\U0001f602' in position 885944: character maps to <undefined>", "processed_rows": 0, "total_rows": 0, "source_type": "unknown"}
|
||||
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1,29 @@
|
|||
{"timestamp": "2026-06-05T19:08:42.267450", "level": "INFO", "logger": "retrain_worker", "message": "[init] Log file tersimpan di: D:\\msaif\\Project\\Laravel\\sista_mif_ta\\fastapi\\ml_assets\\logs\\retrain_log_20260605_190842.jsonl"}
|
||||
{"timestamp": "2026-06-05T19:08:42.267450", "level": "INFO", "logger": "retrain_worker", "message": "Retraining session dimulai. Log file: retrain_log_20260605_190842.jsonl", "stage": "started", "event_type": "session_start", "metadata": {"extra_csv_provided": true}}
|
||||
{"timestamp": "2026-06-05T19:08:42.268452", "level": "INFO", "logger": "retrain_worker", "message": "[started] Worker dimulai", "stage": "started"}
|
||||
{"timestamp": "2026-06-05T19:08:42.269454", "level": "INFO", "logger": "retrain_worker", "message": "[backup] Membuat backup model lama...", "stage": "backup"}
|
||||
{"timestamp": "2026-06-05T19:08:42.271473", "level": "INFO", "logger": "retrain_worker", "message": "Backup tersimpan: D:\\msaif\\Project\\Laravel\\sista_mif_ta\\fastapi\\ml_assets\\ml_pipeline_internal.pkl.bak"}
|
||||
{"timestamp": "2026-06-05T19:08:42.271473", "level": "INFO", "logger": "retrain_worker", "message": "[loading_data] Memuat dan menggabungkan data training...", "stage": "loading_data"}
|
||||
{"timestamp": "2026-06-05T19:08:42.275472", "level": "INFO", "logger": "retrain_worker", "message": "Corpus asli: 250 baris"}
|
||||
{"timestamp": "2026-06-05T19:08:42.445530", "level": "INFO", "logger": "retrain_worker", "message": "Data manual_override: 1 baris | weight=6.6648x (log-damped) | influence aktual=2.6% (target nominal 30%)"}
|
||||
{"timestamp": "2026-06-05T19:08:42.465047", "level": "INFO", "logger": "retrain_worker", "message": "Total setelah merge (tanpa deduplicate): 251 baris"}
|
||||
{"timestamp": "2026-06-05T19:08:42.465047", "level": "INFO", "logger": "retrain_worker", "message": "Data training siap: 251 baris", "stage": "loading_data", "event_type": "data_merged", "metadata": {"corpus_rows": 250, "override_rows": 1, "override_weight": 6.6648, "influence_pct": 2.6, "total_merged": 251}}
|
||||
{"timestamp": "2026-06-05T19:08:42.479066", "level": "INFO", "logger": "retrain_worker", "message": "Dynamic split: 175 train | 76 test"}
|
||||
{"timestamp": "2026-06-05T19:08:42.480074", "level": "INFO", "logger": "retrain_worker", "message": "[evaluating] Mengevaluasi model lama pada test set baru...", "stage": "evaluating"}
|
||||
{"timestamp": "2026-06-05T19:08:42.492120", "level": "INFO", "logger": "retrain_worker", "message": "Baseline F1 (old model, same test set): 0.9059"}
|
||||
{"timestamp": "2026-06-05T19:08:42.494119", "level": "INFO", "logger": "retrain_worker", "message": "[training] K-Fold validation & training... (175 sampel)", "stage": "training"}
|
||||
{"timestamp": "2026-06-05T19:08:42.534414", "level": "INFO", "logger": "retrain_worker", "message": "[training] K-Fold selesai: fold 1/5 | acc=0.5143", "stage": "training"}
|
||||
{"timestamp": "2026-06-05T19:08:42.542922", "level": "INFO", "logger": "retrain_worker", "message": "[training] K-Fold selesai: fold 2/5 | acc=0.6857", "stage": "training"}
|
||||
{"timestamp": "2026-06-05T19:08:42.552262", "level": "INFO", "logger": "retrain_worker", "message": "[training] K-Fold selesai: fold 3/5 | acc=0.5429", "stage": "training"}
|
||||
{"timestamp": "2026-06-05T19:08:42.561273", "level": "INFO", "logger": "retrain_worker", "message": "[training] K-Fold selesai: fold 4/5 | acc=0.5714", "stage": "training"}
|
||||
{"timestamp": "2026-06-05T19:08:42.569782", "level": "INFO", "logger": "retrain_worker", "message": "[training] K-Fold selesai: fold 5/5 | acc=0.7143", "stage": "training"}
|
||||
{"timestamp": "2026-06-05T19:08:42.569782", "level": "INFO", "logger": "retrain_worker", "message": "K-Fold 5 lipatan selesai. Akurasi per fold: [0.5143, 0.6857, 0.5429, 0.5714, 0.7143]", "stage": "training", "event_type": "kfold_completed", "metadata": {"fold_accuracies": [0.5143, 0.6857, 0.5429, 0.5714, 0.7143], "mean_accuracy": 0.6057, "std_accuracy": 0.0796}}
|
||||
{"timestamp": "2026-06-05T19:08:42.570788", "level": "INFO", "logger": "retrain_worker", "message": "[training] Training model final pada seluruh data training...", "stage": "training"}
|
||||
{"timestamp": "2026-06-05T19:08:42.579235", "level": "INFO", "logger": "retrain_worker", "message": "Candidate model tersimpan: D:\\msaif\\Project\\Laravel\\sista_mif_ta\\fastapi\\ml_assets\\ml_pipeline_candidate.pkl"}
|
||||
{"timestamp": "2026-06-05T19:08:42.589948", "level": "INFO", "logger": "retrain_worker", "message": "[evaluating] Mengevaluasi model baru vs model lama (test set sama)...", "stage": "evaluating"}
|
||||
{"timestamp": "2026-06-05T19:08:42.593983", "level": "INFO", "logger": "retrain_worker", "message": "Baseline F1 (old model, same test) : 0.9059"}
|
||||
{"timestamp": "2026-06-05T19:08:42.593983", "level": "INFO", "logger": "retrain_worker", "message": "Candidate F1 (new model, same test): 0.7011 (delta: -0.2048)"}
|
||||
{"timestamp": "2026-06-05T19:08:42.593983", "level": "INFO", "logger": "retrain_worker", "message": "Evaluasi selesai. Delta F1: -0.2048 (rollback)", "stage": "evaluating", "event_type": "evaluation_completed", "metadata": {"baseline_f1": 0.9059, "candidate_f1": 0.7011, "delta": -0.2048, "delta_pct": -20.48, "threshold_min": 0.01, "decision": "rollback", "reason": "Model baru lebih buruk secara signifikan (-20.48% weighted F1)", "new_accuracy": 0.6711, "per_class_f1": {"Programmer": 0.8, "Data Analyst": 0.19, "Wirausaha Informatika": 0.474, "Non-IT": 0.897}}}
|
||||
{"timestamp": "2026-06-05T19:08:42.593983", "level": "INFO", "logger": "retrain_worker", "message": "Rollback berhasil: pkl lama dipulihkan dari .bak"}
|
||||
{"timestamp": "2026-06-05T19:08:42.593983", "level": "INFO", "logger": "retrain_worker", "message": "Model lama dipertahankan. Model baru lebih buruk secara signifikan (-20.48% weighted F1)", "stage": "rolled_back", "event_type": "rollback", "metadata": {"reason": "Model baru lebih buruk secara signifikan (-20.48% weighted F1)", "old_f1": 0.9059, "new_f1": 0.7011}}
|
||||
{"timestamp": "2026-06-05T19:08:42.594976", "level": "INFO", "logger": "retrain_worker", "message": "[rolled_back] Model lama dipertahankan. Model baru lebih buruk secara signifikan (-20.48% weighted F1)", "stage": "rolled_back"}
|
||||
Loading…
Reference in New Issue