38 lines
1.3 KiB
Python
38 lines
1.3 KiB
Python
import pandas as pd
|
|
import glob
|
|
import sys
|
|
from scripts.tests.test_f8_all_years import parse_status_pekerjaan_new
|
|
|
|
sys.path.append('Backend')
|
|
from app.parsers.column_finder import find_column, get_value
|
|
from app.parsers.mappings import PEKERJAAN_MAPPING
|
|
|
|
files_2025 = sorted(glob.glob('Data_Tracerstudy/*2025*.csv'))
|
|
total_rows = 0
|
|
parsed_counts = {}
|
|
unparsed_list = []
|
|
|
|
for f in files_2025:
|
|
df = pd.read_csv(f, sep=';', encoding='utf-8-sig').dropna(how='all')
|
|
source_cols = list(df.columns)
|
|
col = find_column(source_cols, PEKERJAAN_MAPPING['status_saat_ini'])
|
|
for idx, row in df.iterrows():
|
|
total_rows += 1
|
|
raw_val = get_value(row, source_cols, PEKERJAAN_MAPPING['status_saat_ini'], '')
|
|
parsed = parse_status_pekerjaan_new(raw_val)
|
|
parsed_counts[parsed] = parsed_counts.get(parsed, 0) + 1
|
|
if parsed is None:
|
|
unparsed_list.append((f, idx, raw_val))
|
|
|
|
print(f"Total 2025 rows processed: {total_rows}")
|
|
print("\nParsed status distribution:")
|
|
for status, cnt in sorted(parsed_counts.items(), key=lambda x: str(x[0])):
|
|
print(f" {str(status):<20}: {cnt} ({(cnt/total_rows)*100:.1f}%)")
|
|
|
|
if unparsed_list:
|
|
print(f"\nUnparsed values ({len(unparsed_list)}):")
|
|
for u in unparsed_list:
|
|
print(" ", u)
|
|
else:
|
|
print("\nSUCCESS! 0 NULL values across all 2025 datasets!")
|