67 lines
3.0 KiB
Python
67 lines
3.0 KiB
Python
import pandas as pd
|
|
import sys
|
|
sys.path.append('Backend')
|
|
|
|
from app.parsers.column_finder import find_column, get_value
|
|
from app.parsers.mappings import (
|
|
YEAR_CONFIG, IDENTITAS_MAPPING, PEKERJAAN_MAPPING, STUDI_LANJUT_MAPPING,
|
|
KESESUAIAN_MAPPING, KOMPETENSI_LULUS_MAPPING, KOMPETENSI_SAAT_INI_MAPPING,
|
|
METODE_PEMBELAJARAN_MAPPING, WAKTU_CARI_KERJA_MAPPING, CARA_CARI_KERJA_MAPPING,
|
|
LAMARAN_KEAKTIFAN_MAPPING, KESESUAIAN_PEKERJAAN_MAPPING, get_field_code
|
|
)
|
|
from app.parsers.converters import konversi_jawaban
|
|
|
|
all_mapping = {}
|
|
all_mapping.update(IDENTITAS_MAPPING)
|
|
all_mapping.update(PEKERJAAN_MAPPING)
|
|
all_mapping.update(STUDI_LANJUT_MAPPING)
|
|
all_mapping.update(KESESUAIAN_MAPPING)
|
|
all_mapping.update(KOMPETENSI_LULUS_MAPPING)
|
|
all_mapping.update(KOMPETENSI_SAAT_INI_MAPPING)
|
|
all_mapping.update(METODE_PEMBELAJARAN_MAPPING)
|
|
all_mapping.update(WAKTU_CARI_KERJA_MAPPING)
|
|
all_mapping.update(CARA_CARI_KERJA_MAPPING)
|
|
all_mapping.update(LAMARAN_KEAKTIFAN_MAPPING)
|
|
all_mapping.update(KESESUAIAN_PEKERJAAN_MAPPING)
|
|
|
|
for yr in ['2019', '2020', '2021', '2022', '2023']:
|
|
fpath = f'Data_Tracerstudy/Data_{yr}_Tracer_study.csv'
|
|
df = pd.read_csv(fpath, sep=';', encoding='utf-8-sig').dropna(how='all')
|
|
source_cols = list(df.columns)
|
|
|
|
conversion_loss = []
|
|
for field, cols in all_mapping.items():
|
|
if field in ['nim', 'nama', 'tahun_lulus', 'kode_prodi']:
|
|
continue
|
|
mapped_col = find_column(source_cols, cols)
|
|
if not mapped_col:
|
|
continue
|
|
fc = get_field_code(field, all_mapping)
|
|
fc_upper = str(fc).upper() if fc else ''
|
|
|
|
# Check non-null in CSV vs converted
|
|
raw_series = df[mapped_col].dropna()
|
|
# filter out empty strings
|
|
valid_raw = [v for v in raw_series if str(v).strip() not in ['', 'nan', 'None']]
|
|
|
|
lost_values = []
|
|
for val in valid_raw:
|
|
converted = konversi_jawaban(fc, val)
|
|
if converted is None:
|
|
# check if it was intentionally 0 for competency
|
|
if str(val).strip() in ['0', '0.0'] and fc_upper in ['F1761', 'F1762', 'F1763', 'F1764', 'F1765', 'F1766', 'F1767', 'F1768', 'F1769', 'F1770', 'F1771', 'F1772', 'F1773', 'F1774', 'F21', 'F22', 'F23', 'F24', 'F25', 'F26', 'F27']:
|
|
continue
|
|
lost_values.append(val)
|
|
|
|
if lost_values:
|
|
unique_lost = list(set(lost_values))[:5]
|
|
conversion_loss.append((field, fc, mapped_col, len(lost_values), len(valid_raw), unique_lost))
|
|
|
|
print(f"\n==================== TAHUN {yr} CONVERSION LOSS ====================")
|
|
if conversion_loss:
|
|
print(f"Found {len(conversion_loss)} fields with potential conversion loss (data exists in CSV but konversi_jawaban returned None):")
|
|
for cl in conversion_loss:
|
|
print(f" Field '{cl[0]}' (code {cl[1]}, col '{cl[2]}'): {cl[3]}/{cl[4]} lost! Sample lost raw values: {cl[5]}")
|
|
else:
|
|
print("ZERO conversion loss! Every non-empty value in CSV converted properly.")
|