import pandas as pd
import glob
import sys
from scripts.tests.test_f8_all_years import parse_status_pekerjaan_new

sys.path.append('Backend')
from app.parsers.column_finder import find_column, get_value
from app.parsers.mappings import PEKERJAAN_MAPPING

files_2025 = sorted(glob.glob('Data_Tracerstudy/*2025*.csv'))
total_rows = 0
parsed_counts = {}
unparsed_list = []

for f in files_2025:
    df = pd.read_csv(f, sep=';', encoding='utf-8-sig').dropna(how='all')
    source_cols = list(df.columns)
    col = find_column(source_cols, PEKERJAAN_MAPPING['status_saat_ini'])
    for idx, row in df.iterrows():
        total_rows += 1
        raw_val = get_value(row, source_cols, PEKERJAAN_MAPPING['status_saat_ini'], '')
        parsed = parse_status_pekerjaan_new(raw_val)
        parsed_counts[parsed] = parsed_counts.get(parsed, 0) + 1
        if parsed is None:
            unparsed_list.append((f, idx, raw_val))

print(f"Total 2025 rows processed: {total_rows}")
print("\nParsed status distribution:")
for status, cnt in sorted(parsed_counts.items(), key=lambda x: str(x[0])):
    print(f"  {str(status):<20}: {cnt} ({(cnt/total_rows)*100:.1f}%)")

if unparsed_list:
    print(f"\nUnparsed values ({len(unparsed_list)}):")
    for u in unparsed_list:
        print(" ", u)
else:
    print("\nSUCCESS! 0 NULL values across all 2025 datasets!")