436 lines
13 KiB
Python
436 lines
13 KiB
Python
"""
|
|
=============================================================
|
|
TEXT PREPROCESSING - TEKS PEKERJAAN
|
|
=============================================================
|
|
|
|
Tujuan:
|
|
- Membentuk teks_pekerjaan dari jenis_perusahaan + nama_perusahaan + jabatan
|
|
- Melakukan text preprocessing: case folding, cleaning, tokenizing, stopword, stemming
|
|
- Menyimpan hasil preprocessing
|
|
|
|
PRD Reference: Section 12, 13
|
|
=============================================================
|
|
"""
|
|
|
|
import pandas as pd
|
|
import re
|
|
import os
|
|
import json
|
|
from datetime import datetime
|
|
|
|
# Sastrawi untuk stemming Bahasa Indonesia
|
|
try:
|
|
from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
|
|
factory = StemmerFactory()
|
|
stemmer = factory.create_stemmer()
|
|
SASTRAWI_AVAILABLE = True
|
|
except ImportError:
|
|
SASTRAWI_AVAILABLE = False
|
|
print("[WARNING] Sastrawi tidak tersedia. Stemming tidak akan dilakukan.")
|
|
|
|
# NLTK untuk stopwords
|
|
try:
|
|
import nltk
|
|
from nltk.corpus import stopwords
|
|
from nltk.tokenize import word_tokenize
|
|
nltk.download('stopwords', quiet=True)
|
|
nltk.download('punkt', quiet=True)
|
|
nltk.download('punkt_tab', quiet=True)
|
|
NLTK_AVAILABLE = True
|
|
except:
|
|
NLTK_AVAILABLE = False
|
|
|
|
# Path configuration
|
|
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
|
|
DATA_INPUT = os.path.join(BASE_DIR, '..', 'data', 'standardized', 'unified_dataset.csv')
|
|
DATA_OUTPUT = os.path.join(BASE_DIR, '..', 'data', 'processed')
|
|
|
|
# Mapping status pekerjaan dari angka ke teks
|
|
STATUS_MAPPING = {
|
|
1: "bekerja",
|
|
2: "belum_kerja",
|
|
3: "wiraswasta",
|
|
4: "melanjutkan_studi",
|
|
5: "mencari_kerja"
|
|
}
|
|
|
|
# Mapping jenis_perusahaan dari angka ke teks
|
|
JENIS_PERUSAHAAN_MAPPING = {
|
|
1: "pemerintah",
|
|
2: "bnpp",
|
|
3: "swasta",
|
|
4: "wiraswasta",
|
|
5: "lainnya",
|
|
6: "bumn",
|
|
7: "nirlaba",
|
|
0: ""
|
|
}
|
|
|
|
# Stopwords Bahasa Indonesia (extended)
|
|
INDONESIAN_STOPWORDS = {
|
|
'yang', 'dan', 'di', 'dari', 'ini', 'itu', 'dengan', 'untuk', 'pada',
|
|
'adalah', 'ke', 'atau', 'akan', 'juga', 'tidak', 'dalam', 'oleh',
|
|
'sebagai', 'lebih', 'sudah', 'ada', 'bisa', 'telah', 'antara', 'maka',
|
|
'serta', 'namun', 'secara', 'seperti', 'jika', 'hanya', 'tetapi', 'saat',
|
|
'sedang', 'sebelumnya', 'tersebut', 'yaitu', 'saya', 'kami', 'kita',
|
|
'anda', 'dia', 'mereka', 'kami', 'kita', 'nya', 'lah', 'kah', 'pun',
|
|
'sih', 'deh', 'dong', 'gitu', 'aja', 'emang', 'nih', 'tuh', 'kok',
|
|
'yg', 'dg', 'dr', 'dlm', 'dgn', 'utk', 'pd', 'sdh', 'blm', 'jg',
|
|
'krn', 'sm', 'tp', 'jd', 'sd', 'bt', 'spy', 'msh', 'dl', 'bs', 'bsa',
|
|
'pt', 'cv', 'tdk', 'bg', 'smua', 'atl', 'krn', 'sprti', 'lngkp', 'sm',
|
|
# Tambahan umum
|
|
'per', 'bagian', 'ibu', 'bapak', 'kakak', 'adik', 'pak', 'bu', 'mas',
|
|
'mbak', 'pak', 'bu', 'pak', 'pak', 'dan', 'atau', 'maupun', 'hal', 'satu',
|
|
'dua', 'tiga', 'empat', 'lima', 'enam', 'tujuh', 'delapan', 'sembilan', 'sepuluh'
|
|
}
|
|
|
|
# Pattern untuk cleaning
|
|
CLEANING_PATTERNS = {
|
|
'email': r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b',
|
|
'url': r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+',
|
|
'phone': r'\b\d{10,15}\b',
|
|
'special_chars': r'[^\w\s]',
|
|
'extra_spaces': r'\s+',
|
|
'numbers': r'\b\d+\b'
|
|
}
|
|
|
|
|
|
def clean_text(text):
|
|
"""
|
|
Basic text cleaning:
|
|
- Hapus email, URL, phone
|
|
- Hapus special characters
|
|
- Hapus standalone numbers (tidak bermakna)
|
|
- Hapus extra spaces
|
|
"""
|
|
if pd.isna(text) or text == '' or text == 'nan':
|
|
return ''
|
|
|
|
text = str(text).lower()
|
|
|
|
# Hapus email
|
|
text = re.sub(CLEANING_PATTERNS['email'], ' ', text)
|
|
# Hapus URL
|
|
text = re.sub(CLEANING_PATTERNS['url'], ' ', text)
|
|
# Hapus phone
|
|
text = re.sub(CLEANING_PATTERNS['phone'], ' ', text)
|
|
# Hapus special chars (simpan spasi dan alfanumerik)
|
|
text = re.sub(r'[^\w\s]', ' ', text)
|
|
# Hapus extra spaces
|
|
text = re.sub(r'\s+', ' ', text)
|
|
# Trim
|
|
text = text.strip()
|
|
|
|
return text
|
|
|
|
|
|
def remove_stopwords(tokens):
|
|
"""
|
|
Hapus stopwords dari list tokens.
|
|
Also removes standalone numbers.
|
|
"""
|
|
result = []
|
|
for token in tokens:
|
|
t = token.lower()
|
|
# Skip if it's a stopword
|
|
if t in INDONESIAN_STOPWORDS:
|
|
continue
|
|
# Skip if it's just a number
|
|
if t.isdigit():
|
|
continue
|
|
# Skip if it's too short (1-2 chars)
|
|
if len(t) <= 2:
|
|
continue
|
|
result.append(token)
|
|
return result
|
|
|
|
|
|
_STEM_CACHE: dict = {}
|
|
|
|
def stemming(tokens):
|
|
"""
|
|
Apply stemming menggunakan Sastrawi dengan memoization cache.
|
|
"""
|
|
if not SASTRAWI_AVAILABLE:
|
|
return tokens
|
|
|
|
stemmed = []
|
|
for token in tokens:
|
|
if token in _STEM_CACHE:
|
|
stemmed.append(_STEM_CACHE[token])
|
|
continue
|
|
try:
|
|
stemmed_token = stemmer.stem(token)
|
|
_STEM_CACHE[token] = stemmed_token
|
|
stemmed.append(stemmed_token)
|
|
except Exception:
|
|
_STEM_CACHE[token] = token
|
|
stemmed.append(token)
|
|
|
|
return stemmed
|
|
|
|
|
|
def tokenize(text):
|
|
"""
|
|
Tokenize text menjadi list of words.
|
|
"""
|
|
if not text or text == '':
|
|
return []
|
|
|
|
# Split berdasarkan whitespace
|
|
tokens = text.split()
|
|
|
|
return tokens
|
|
|
|
|
|
def form_teks_pekerjaan(row):
|
|
"""
|
|
Membentuk teks_pekerjaan dari:
|
|
status_pekerjaan + jenis_perusahaan + nama_perusahaan + jabatan + tingkatan_tempat_kerja
|
|
|
|
PRD Section 11:
|
|
'bekerja perusahaan swasta pt abc teknologi software engineer nasional wiraswasta berbadan hukum'
|
|
"""
|
|
parts = []
|
|
|
|
def clean_val(v):
|
|
if pd.isna(v) or not v:
|
|
return ''
|
|
s = str(v).strip()
|
|
if s.lower() in ['-', 'none', 'null', 'kosong', 'tidak ada', '0']:
|
|
return ''
|
|
return s
|
|
|
|
# 1. status_pekerjaan
|
|
status = clean_val(row.get('status_pekerjaan', row.get('status_kerja', '')))
|
|
if status:
|
|
parts.append(status)
|
|
|
|
# 2. jenis_perusahaan
|
|
jenis = row.get('jenis_perusahaan', '')
|
|
if pd.notna(jenis) and str(jenis).strip() != '':
|
|
try:
|
|
jenis_num = int(float(str(jenis)))
|
|
jenis_text = JENIS_PERUSAHAAN_MAPPING.get(jenis_num, str(jenis))
|
|
except (ValueError, TypeError):
|
|
jenis_text = str(jenis)
|
|
j_clean = clean_val(jenis_text)
|
|
if j_clean:
|
|
parts.append(j_clean)
|
|
|
|
# 3. nama_perusahaan
|
|
nama = clean_val(row.get('nama_perusahaan', ''))
|
|
if nama:
|
|
parts.append(nama)
|
|
|
|
# 4. jabatan
|
|
jabatan = clean_val(row.get('jabatan', ''))
|
|
if jabatan:
|
|
parts.append(jabatan)
|
|
|
|
# Gabungkan 4 kolom dengan spasi
|
|
teks_pekerjaan = ' '.join(parts)
|
|
|
|
return teks_pekerjaan
|
|
|
|
|
|
def preprocess_text(text):
|
|
"""
|
|
Full text preprocessing pipeline:
|
|
1. Case Folding
|
|
2. Cleaning
|
|
3. Tokenizing
|
|
4. Stopword Removal
|
|
5. Stemming
|
|
|
|
PRD Section 13:
|
|
Urutan: Case Folding -> Cleaning -> Tokenizing -> Stopword Removal -> Stemming
|
|
"""
|
|
# 1. Case Folding (sudah di dalam clean_text)
|
|
# 2. Cleaning
|
|
text = clean_text(text)
|
|
|
|
if not text:
|
|
return ''
|
|
|
|
# 3. Tokenizing
|
|
tokens = tokenize(text)
|
|
|
|
# 4. Stopword Removal
|
|
tokens = remove_stopwords(tokens)
|
|
|
|
# 5. Stemming
|
|
if SASTRAWI_AVAILABLE:
|
|
tokens = stemming(tokens)
|
|
|
|
# Join kembali
|
|
processed_text = ' '.join(tokens)
|
|
|
|
return processed_text
|
|
|
|
|
|
def preprocess_dataframe(df, text_column='teks_pekerjaan'):
|
|
"""
|
|
Menjalankan preprocessing pada kolom teks di seluruh DataFrame.
|
|
"""
|
|
df_copy = df.copy()
|
|
if text_column in df_copy.columns:
|
|
df_copy[text_column] = df_copy[text_column].apply(lambda x: preprocess_text(str(x)) if pd.notna(x) else '')
|
|
return df_copy
|
|
|
|
|
|
|
|
def classify_has_pekerjaan(teks_pekerjaan, status):
|
|
"""
|
|
Klasifikasi apakah alumni memiliki informasi pekerjaan yang cukup.
|
|
|
|
PRD Section 10:
|
|
- Memiliki informasi pekerjaan jika: nama perusahaan ADA ATAU jabatan ADA ATAU jenis perusahaan ADA
|
|
"""
|
|
if status in ['belum_kerja', 'belum_bekerja', 'mencari_kerja', 'tidak_kerja']:
|
|
return 'tidak_berpengalaman'
|
|
|
|
if not teks_pekerjaan or teks_pekerjaan == '':
|
|
return 'tidak_berpengalaman'
|
|
|
|
# Cek apakah teks terlalu pendek (hanya 1-2 kata)
|
|
words = teks_pekerjaan.split()
|
|
if len(words) < 2:
|
|
return 'tidak_berpengalaman'
|
|
|
|
return 'berpengalaman'
|
|
|
|
|
|
def main():
|
|
"""
|
|
Main function untuk text preprocessing.
|
|
"""
|
|
print("=" * 60)
|
|
print("TEXT PREPROCESSING - TEKS PEKERJAAN")
|
|
print("=" * 60)
|
|
|
|
# Load unified dataset
|
|
print("\n>> Loading unified dataset...")
|
|
df = pd.read_csv(DATA_INPUT, encoding='utf-8-sig')
|
|
print(f" Total data: {len(df)} alumni")
|
|
|
|
# 1. Form teks_pekerjaan
|
|
print("\n>> Membentuk teks_pekerjaan...")
|
|
df['teks_pekerjaan_raw'] = df.apply(form_teks_pekerjaan, axis=1)
|
|
|
|
# Statistik
|
|
empty_teks = (df['teks_pekerjaan_raw'] == '').sum()
|
|
print(f" Teks pekerjaan kosong: {empty_teks}")
|
|
print(f" Teks pekerjaan terisi: {len(df) - empty_teks}")
|
|
|
|
# 2. Text Preprocessing
|
|
print("\n>> Melakukan text preprocessing...")
|
|
print(f" Sastrawi available: {SASTRAWI_AVAILABLE}")
|
|
print(f" NLTK available: {NLTK_AVAILABLE}")
|
|
|
|
df['teks_pekerjaan'] = df['teks_pekerjaan_raw'].apply(preprocess_text)
|
|
|
|
# 3. Klasifikasi pengalaman kerja
|
|
print("\n>> Mengklasifikasi pengalaman kerja...")
|
|
df['kategori_pengalaman'] = df.apply(
|
|
lambda row: classify_has_pekerjaan(row['teks_pekerjaan'], row['status_pekerjaan']),
|
|
axis=1
|
|
)
|
|
|
|
# Statistik
|
|
pengalaman_counts = df['kategori_pengalaman'].value_counts()
|
|
print("\n Distribusi Kategori Pengalaman:")
|
|
for cat, count in pengalaman_counts.items():
|
|
pct = (count / len(df)) * 100
|
|
print(f" {cat}: {count} ({pct:.1f}%)")
|
|
|
|
# 4. Pisahkan data untuk clustering
|
|
print("\n>> Memisahkan data untuk clustering...")
|
|
|
|
# Data yang akan di-cluster: memiliki teks pekerjaan yang cukup
|
|
df_untuk_clustering = df[
|
|
(df['kategori_pengalaman'] == 'berpengalaman') &
|
|
(df['teks_pekerjaan'].str.len() > 0)
|
|
].copy()
|
|
|
|
# Data C3: belum/tidak bekerja
|
|
df_c3 = df[
|
|
(df['status_pekerjaan'].isin(['belum_kerja', 'mencari_kerja'])) |
|
|
(df['kategori_pengalaman'] == 'tidak_berpengalaman')
|
|
].copy()
|
|
|
|
print(f" Data untuk clustering: {len(df_untuk_clustering)}")
|
|
print(f" Data C3 (tidak untuk clustering): {len(df_c3)}")
|
|
|
|
# 5. Generate Summary
|
|
print("\n" + "=" * 60)
|
|
print("SUMMARY")
|
|
print("=" * 60)
|
|
print(f"Total alumni: {len(df)}")
|
|
print(f"Data untuk clustering: {len(df_untuk_clustering)}")
|
|
print(f"Data C3: {len(df_c3)}")
|
|
|
|
print("\n>> Distribusi Status Pekerjaan:")
|
|
for status, count in df['status_pekerjaan'].value_counts().items():
|
|
pct = (count / len(df)) * 100
|
|
print(f" {status}: {count} ({pct:.1f}%)")
|
|
|
|
print("\n>> Distribusi Prodi:")
|
|
for prodi, count in df['kode_prodi'].value_counts().items():
|
|
pct = (count / len(df)) * 100
|
|
print(f" {prodi}: {count} ({pct:.1f}%)")
|
|
|
|
print("\n>> Sample teks_pekerjaan (raw):")
|
|
for idx, row in df_untuk_clustering.head(5).iterrows():
|
|
print(f" [{row['nama']}] {row['teks_pekerjaan_raw'][:80]}...")
|
|
|
|
print("\n>> Sample teks_pekerjaan (processed):")
|
|
for idx, row in df_untuk_clustering.head(5).iterrows():
|
|
print(f" [{row['nama']}] {row['teks_pekerjaan'][:80]}...")
|
|
|
|
# 6. Save results
|
|
print("\n>> Saving results...")
|
|
|
|
os.makedirs(DATA_OUTPUT, exist_ok=True)
|
|
|
|
# Save full dataset
|
|
full_output = os.path.join(DATA_OUTPUT, 'processed_dataset.csv')
|
|
df.to_csv(full_output, index=False, encoding='utf-8-sig')
|
|
print(f" Full dataset: {full_output}")
|
|
|
|
# Save dataset untuk clustering
|
|
clustering_output = os.path.join(DATA_OUTPUT, 'dataset_untuk_clustering.csv')
|
|
df_untuk_clustering.to_csv(clustering_output, index=False, encoding='utf-8-sig')
|
|
print(f" Dataset clustering: {clustering_output}")
|
|
|
|
# Save C3 data
|
|
c3_output = os.path.join(DATA_OUTPUT, 'dataset_c3.csv')
|
|
df_c3.to_csv(c3_output, index=False, encoding='utf-8-sig')
|
|
print(f" Dataset C3: {c3_output}")
|
|
|
|
# Save summary
|
|
summary = {
|
|
'tanggal': datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
|
|
'total_alumni': len(df),
|
|
'total_untuk_clustering': len(df_untuk_clustering),
|
|
'total_c3': len(df_c3),
|
|
'sastrawi_available': SASTRAWI_AVAILABLE,
|
|
'distribusi_status': df['status_pekerjaan'].value_counts().to_dict(),
|
|
'distribusi_prodi': df['kode_prodi'].value_counts().to_dict()
|
|
}
|
|
|
|
summary_output = os.path.join(DATA_OUTPUT, 'preprocessing_summary.json')
|
|
with open(summary_output, 'w', encoding='utf-8') as f:
|
|
json.dump(summary, f, indent=2, ensure_ascii=False)
|
|
print(f" Summary: {summary_output}")
|
|
|
|
print("\n" + "=" * 60)
|
|
print("[SUCCESS] TEXT PREPROCESSING SELESAI!")
|
|
print("=" * 60)
|
|
|
|
return df, df_untuk_clustering, df_c3
|
|
|
|
|
|
if __name__ == "__main__":
|
|
df, df_clustering, df_c3 = main()
|