import sys
import pandas as pd
sys.path.append('Backend')

from app.parsers.column_finder import find_column, get_value
from app.parsers.mappings import (
    YEAR_CONFIG, IDENTITAS_MAPPING, PEKERJAAN_MAPPING, STUDI_LANJUT_MAPPING,
    KESESUAIAN_MAPPING, KOMPETENSI_LULUS_MAPPING, KOMPETENSI_SAAT_INI_MAPPING,
    METODE_PEMBELAJARAN_MAPPING, WAKTU_CARI_KERJA_MAPPING, CARA_CARI_KERJA_MAPPING,
    LAMARAN_KEAKTIFAN_MAPPING, KESESUAIAN_PEKERJAAN_MAPPING, get_field_code
)
from app.parsers.converters import konversi_jawaban, parse_2024_value

all_mapping = {}
all_mapping.update(PEKERJAAN_MAPPING)
all_mapping.update(STUDI_LANJUT_MAPPING)
all_mapping.update(KESESUAIAN_MAPPING)
all_mapping.update(KOMPETENSI_LULUS_MAPPING)
all_mapping.update(KOMPETENSI_SAAT_INI_MAPPING)
all_mapping.update(METODE_PEMBELAJARAN_MAPPING)
all_mapping.update(WAKTU_CARI_KERJA_MAPPING)
all_mapping.update(CARA_CARI_KERJA_MAPPING)
all_mapping.update(LAMARAN_KEAKTIFAN_MAPPING)
all_mapping.update(KESESUAIAN_PEKERJAAN_MAPPING)

years = ['2019', '2020', '2021', '2022', '2023']

print("=== CHECKING ALL RAW NON-EMPTY VALUES THAT BECOME NULL ===", flush=True)
for yr in years:
    fpath = f'Data_Tracerstudy/Data_{yr}_Tracer_study.csv'
    df = pd.read_csv(fpath, sep=';', encoding='utf-8-sig').dropna(how='all')
    source_cols = list(df.columns)
    
    issues = {}
    for idx, row in df.iterrows():
        for field, cols in all_mapping.items():
            mapped_col = find_column(source_cols, cols)
            if not mapped_col:
                continue
            raw_val = row.get(mapped_col)
            if pd.isna(raw_val):
                continue
            raw_str = str(raw_val).strip()
            if raw_str in ['', 'nan', 'None']:
                continue
                
            fc = get_field_code(field, all_mapping)
            fc_upper = str(fc).upper() if fc else ''
            
            # Simulate old format processing in import_service
            converted = konversi_jawaban(fc, raw_val)
            
            is_saved = False
            if field in ['bulan_dapat_kerja', 'bulan_dapat_kerja_setelah', 'bulan_cari_sebelum_lulus', 'bulan_cari_setelah_wisuda']:
                try:
                    int(float(raw_str))
                    is_saved = True
                except:
                    pass
            elif converted is not None:
                is_saved = True
            elif raw_str in ['0', '0.0'] and fc_upper in ['F1761', 'F1762', 'F1763', 'F1764', 'F1765', 'F1766', 'F1767', 'F1768', 'F1769', 'F1770', 'F1771', 'F1772', 'F1773', 'F1774', 'F21', 'F22', 'F23', 'F24', 'F25', 'F26', 'F27']:
                # Intentionally null for 0
                is_saved = True
            elif raw_str:
                # In import_service line 442: else tracer_data[field] = str(val).strip()
                # BUT wait! What does line 442 do?
                pass
                
            if converted is None and not is_saved:
                key = (field, fc, mapped_col)
                if key not in issues:
                    issues[key] = set()
                issues[key].add(raw_str)
                
    print(f"\n--- TAHUN {yr} (Unconverted non-empty raw values) ---", flush=True)
    if issues:
        for (field, fc, col), vals in sorted(issues.items()):
            print(f"  Field '{field}' ({fc}) from col '{col}': {list(vals)[:10]}", flush=True)
    else:
        print("  None! All non-empty raw values are converted.", flush=True)