82 lines
3.4 KiB
Python
82 lines
3.4 KiB
Python
import sys
|
|
import pandas as pd
|
|
sys.path.append('Backend')
|
|
|
|
from app.parsers.column_finder import find_column, get_value
|
|
from app.parsers.mappings import (
|
|
YEAR_CONFIG, IDENTITAS_MAPPING, PEKERJAAN_MAPPING, STUDI_LANJUT_MAPPING,
|
|
KESESUAIAN_MAPPING, KOMPETENSI_LULUS_MAPPING, KOMPETENSI_SAAT_INI_MAPPING,
|
|
METODE_PEMBELAJARAN_MAPPING, WAKTU_CARI_KERJA_MAPPING, CARA_CARI_KERJA_MAPPING,
|
|
LAMARAN_KEAKTIFAN_MAPPING, KESESUAIAN_PEKERJAAN_MAPPING, get_field_code
|
|
)
|
|
from app.parsers.converters import konversi_jawaban, parse_2024_value
|
|
|
|
all_mapping = {}
|
|
all_mapping.update(PEKERJAAN_MAPPING)
|
|
all_mapping.update(STUDI_LANJUT_MAPPING)
|
|
all_mapping.update(KESESUAIAN_MAPPING)
|
|
all_mapping.update(KOMPETENSI_LULUS_MAPPING)
|
|
all_mapping.update(KOMPETENSI_SAAT_INI_MAPPING)
|
|
all_mapping.update(METODE_PEMBELAJARAN_MAPPING)
|
|
all_mapping.update(WAKTU_CARI_KERJA_MAPPING)
|
|
all_mapping.update(CARA_CARI_KERJA_MAPPING)
|
|
all_mapping.update(LAMARAN_KEAKTIFAN_MAPPING)
|
|
all_mapping.update(KESESUAIAN_PEKERJAAN_MAPPING)
|
|
|
|
years = ['2019', '2020', '2021', '2022', '2023']
|
|
|
|
print("=== CHECKING ALL RAW NON-EMPTY VALUES THAT BECOME NULL ===", flush=True)
|
|
for yr in years:
|
|
fpath = f'Data_Tracerstudy/Data_{yr}_Tracer_study.csv'
|
|
df = pd.read_csv(fpath, sep=';', encoding='utf-8-sig').dropna(how='all')
|
|
source_cols = list(df.columns)
|
|
|
|
issues = {}
|
|
for idx, row in df.iterrows():
|
|
for field, cols in all_mapping.items():
|
|
mapped_col = find_column(source_cols, cols)
|
|
if not mapped_col:
|
|
continue
|
|
raw_val = row.get(mapped_col)
|
|
if pd.isna(raw_val):
|
|
continue
|
|
raw_str = str(raw_val).strip()
|
|
if raw_str in ['', 'nan', 'None']:
|
|
continue
|
|
|
|
fc = get_field_code(field, all_mapping)
|
|
fc_upper = str(fc).upper() if fc else ''
|
|
|
|
# Simulate old format processing in import_service
|
|
converted = konversi_jawaban(fc, raw_val)
|
|
|
|
is_saved = False
|
|
if field in ['bulan_dapat_kerja', 'bulan_dapat_kerja_setelah', 'bulan_cari_sebelum_lulus', 'bulan_cari_setelah_wisuda']:
|
|
try:
|
|
int(float(raw_str))
|
|
is_saved = True
|
|
except:
|
|
pass
|
|
elif converted is not None:
|
|
is_saved = True
|
|
elif raw_str in ['0', '0.0'] and fc_upper in ['F1761', 'F1762', 'F1763', 'F1764', 'F1765', 'F1766', 'F1767', 'F1768', 'F1769', 'F1770', 'F1771', 'F1772', 'F1773', 'F1774', 'F21', 'F22', 'F23', 'F24', 'F25', 'F26', 'F27']:
|
|
# Intentionally null for 0
|
|
is_saved = True
|
|
elif raw_str:
|
|
# In import_service line 442: else tracer_data[field] = str(val).strip()
|
|
# BUT wait! What does line 442 do?
|
|
pass
|
|
|
|
if converted is None and not is_saved:
|
|
key = (field, fc, mapped_col)
|
|
if key not in issues:
|
|
issues[key] = set()
|
|
issues[key].add(raw_str)
|
|
|
|
print(f"\n--- TAHUN {yr} (Unconverted non-empty raw values) ---", flush=True)
|
|
if issues:
|
|
for (field, fc, col), vals in sorted(issues.items()):
|
|
print(f" Field '{field}' ({fc}) from col '{col}': {list(vals)[:10]}", flush=True)
|
|
else:
|
|
print(" None! All non-empty raw values are converted.", flush=True)
|