37 lines
1.4 KiB
Python
37 lines
1.4 KiB
Python
import pandas as pd
|
|
import glob
|
|
import sys
|
|
sys.path.append('Backend')
|
|
from app.parsers.column_finder import find_column, get_value
|
|
from app.parsers.mappings import PEKERJAAN_MAPPING
|
|
from app.services.import_service import _parse_status_pekerjaan
|
|
|
|
files_2025 = sorted(glob.glob('Data_Tracerstudy/*2025*.csv'))
|
|
total_rows = 0
|
|
null_count = 0
|
|
val_counts = {}
|
|
|
|
for f in files_2025:
|
|
df = pd.read_csv(f, sep=';', encoding='utf-8-sig').dropna(how='all')
|
|
source_cols = list(df.columns)
|
|
col = find_column(source_cols, PEKERJAAN_MAPPING['status_saat_ini'])
|
|
print(f"\n{f.split('/')[-1].split(chr(92))[-1]}: {len(df)} rows")
|
|
for idx, row in df.iterrows():
|
|
total_rows += 1
|
|
raw_val = get_value(row, source_cols, PEKERJAAN_MAPPING['status_saat_ini'], '')
|
|
parsed = _parse_status_pekerjaan(raw_val)
|
|
val_counts[raw_val] = val_counts.get(raw_val, 0) + 1
|
|
if parsed is None:
|
|
null_count += 1
|
|
# print sample
|
|
if null_count <= 5:
|
|
print(f" NULL parsed from raw: {repr(raw_val)}")
|
|
|
|
print("\n" + "="*60)
|
|
print(f"Total 2025 rows: {total_rows}")
|
|
print(f"Total NULL status: {null_count} ({(null_count/total_rows)*100:.1f}%)")
|
|
print("\nBreakdown of raw F8 values in 2025:")
|
|
for val, cnt in sorted(val_counts.items(), key=lambda x: x[1], reverse=True):
|
|
parsed = _parse_status_pekerjaan(val)
|
|
print(f" {cnt:3d}x | raw: {repr(val)[:45]:<45} -> parsed: {repr(parsed)}")
|