import pandas as pd
import glob
import sys
sys.path.append('Backend')
from app.parsers.column_finder import find_column, get_value
from app.parsers.mappings import PEKERJAAN_MAPPING
from app.services.import_service import _parse_status_pekerjaan

files_2025 = sorted(glob.glob('Data_Tracerstudy/*2025*.csv'))
total_rows = 0
null_count = 0
val_counts = {}

for f in files_2025:
    df = pd.read_csv(f, sep=';', encoding='utf-8-sig').dropna(how='all')
    source_cols = list(df.columns)
    col = find_column(source_cols, PEKERJAAN_MAPPING['status_saat_ini'])
    print(f"\n{f.split('/')[-1].split(chr(92))[-1]}: {len(df)} rows")
    for idx, row in df.iterrows():
        total_rows += 1
        raw_val = get_value(row, source_cols, PEKERJAAN_MAPPING['status_saat_ini'], '')
        parsed = _parse_status_pekerjaan(raw_val)
        val_counts[raw_val] = val_counts.get(raw_val, 0) + 1
        if parsed is None:
            null_count += 1
            # print sample
            if null_count <= 5:
                print(f"  NULL parsed from raw: {repr(raw_val)}")

print("\n" + "="*60)
print(f"Total 2025 rows: {total_rows}")
print(f"Total NULL status: {null_count} ({(null_count/total_rows)*100:.1f}%)")
print("\nBreakdown of raw F8 values in 2025:")
for val, cnt in sorted(val_counts.items(), key=lambda x: x[1], reverse=True):
    parsed = _parse_status_pekerjaan(val)
    print(f"  {cnt:3d}x | raw: {repr(val)[:45]:<45} -> parsed: {repr(parsed)}")