47 lines
1.8 KiB
Python
47 lines
1.8 KiB
Python
import sys
|
|
import glob
|
|
sys.path.append('Backend')
|
|
from app.config.database import SessionLocal
|
|
from app.services.import_service import import_data
|
|
from app.parsers.mappings import YEAR_CONFIG, IDENTITAS_MAPPING
|
|
from app.parsers.column_finder import find_column, get_value
|
|
import pandas as pd
|
|
|
|
files_2025 = glob.glob('Data_Tracerstudy/*2025*.csv') + glob.glob('Data_Tracerstudy/*2025*.xlsx')
|
|
print(f"Found {len(files_2025)} files for 2025:")
|
|
for f in files_2025:
|
|
print(" -", f)
|
|
|
|
for fpath in glob.glob('Data_Tracerstudy/*2025*.csv'):
|
|
print("\n" + "="*60)
|
|
print(f"Testing import simulation for: {fpath}")
|
|
print("="*60)
|
|
|
|
# Check pandas read
|
|
try:
|
|
df = pd.read_csv(fpath, sep=';', encoding='utf-8-sig')
|
|
print(f"DF shape: {df.shape}")
|
|
|
|
# Check NIM column finding
|
|
source_cols = list(df.columns)
|
|
nim_col = find_column(source_cols, IDENTITAS_MAPPING['nim'])
|
|
print(f"Found NIM column: {repr(nim_col)}")
|
|
|
|
# Check first 5 NIM values
|
|
if nim_col:
|
|
print("First 5 NIM values:", df[nim_col].dropna().head().tolist())
|
|
print(f"Total rows in df: {len(df)}")
|
|
print(f"Non-null NIMs: {df[nim_col].notna().sum()}")
|
|
empty_nims = df[df[nim_col].isna() | (df[nim_col].astype(str).str.strip() == '')]
|
|
print(f"Empty NIM count: {len(empty_nims)}")
|
|
if len(empty_nims) > 0:
|
|
print("Index of empty NIMs:", empty_nims.index.tolist()[:10])
|
|
print("Sample empty NIM rows preview:")
|
|
print(df.iloc[empty_nims.index[:3]][df.columns[:5]])
|
|
else:
|
|
print("WARNING: NIM column NOT FOUND!")
|
|
print("Columns containing 'nim':", [c for c in source_cols if 'nim' in c.lower()])
|
|
|
|
except Exception as e:
|
|
print(f"Error reading CSV {fpath}: {e}")
|