372 lines
14 KiB
Plaintext
372 lines
14 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 1,
|
|
"id": "adf2c795",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import pandas as pd\n",
|
|
"import re\n",
|
|
"import string"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "e2dbcc87",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"data = pd.read_csv('dataset\\dataYangDiPakai\\data_sound_horeg_total_mei.csv')\n",
|
|
"data.head(20)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "8e082cd4",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"\n",
|
|
"# 1. Pastikan 'favorite_count' TIDAK ada di daftar kolom yang dibuang\n",
|
|
"kolomDibuang = [\n",
|
|
" 'conversation_id_str', 'id_str', 'image_url', 'in_reply_to_screen_name',\n",
|
|
" 'lang', 'location', 'quote_count', 'reply_count', 'retweet_count',\n",
|
|
" 'tweet_url', 'user_id_str', 'username'\n",
|
|
"]\n",
|
|
"\n",
|
|
"# 2. Hapus kolom sampah\n",
|
|
"dataPembersihan = data.drop(columns=kolomDibuang)\n",
|
|
"\n",
|
|
"# 3. Hapus baris kosong agar tidak error\n",
|
|
"dataPembersihan = dataPembersihan.dropna(subset=['full_text'])\n",
|
|
"\n",
|
|
"# 4. Fungsi Pembersihan (Tetap sama)\n",
|
|
"def cleaningKhusus(text):\n",
|
|
" text = re.sub(r'http\\S+|www\\S+|https\\S+', '', text, flags=re.MULTILINE)\n",
|
|
" text = re.sub(r'\\@\\w+|\\#','', text)\n",
|
|
" text = text.translate(str.maketrans('', '', string.punctuation))\n",
|
|
" text = re.sub(r'\\d+', '', text)\n",
|
|
" text = text.strip()\n",
|
|
" return text\n",
|
|
"\n",
|
|
"def cleaningUmum(text):\n",
|
|
" text = text.lower()\n",
|
|
" text = re.sub(r'\\s+', ' ', text)\n",
|
|
" text = text.encode('ascii', 'ignore').decode('ascii')\n",
|
|
" text = ' '.join([word for word in text.split() if len(word) > 1])\n",
|
|
" return text\n",
|
|
"\n",
|
|
"# 5. BERSIHKAN TEKS dan langsung simpan kembali ke kolom 'full_text'\n",
|
|
"# Ini akan menimpa teks asli dengan teks yang sudah bersih\n",
|
|
"dataPembersihan['full_text'] = dataPembersihan['full_text'].apply(cleaningKhusus).apply(cleaningUmum)\n",
|
|
"\n",
|
|
"# 6. Pilih hanya 3 kolom yang Anda minta\n",
|
|
"df_hasil_akhir = dataPembersihan[['created_at', 'favorite_count', 'full_text']]\n",
|
|
"\n",
|
|
"# 7. Simpan ke CSV\n",
|
|
"df_hasil_akhir.to_csv('dataset\\dataYangDiPakai\\data_sound_horeg_total_mei_dibersihkan.csv', index=False)\n",
|
|
"\n",
|
|
"# Tampilkan hasil\n",
|
|
"print(\"Berhasil! Kolom sekarang hanya: created_at, favorite_count, dan full_text (sudah bersih).\")\n",
|
|
"print(df_hasil_akhir.head(20))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"id": "ee52f2ae",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"pd.set_option('display.max_colwidth', None)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "414378d3",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"df = pd.read_csv('dataset\\dataYangDiPakai\\hasil_preprocessing_intoleransi.csv')\n",
|
|
"df.head(20)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "69e6b2a9",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#membersihkan data\n",
|
|
"import pandas as pd\n",
|
|
"import re\n",
|
|
"from Sastrawi.StopWordRemover.StopWordRemoverFactory import StopWordRemoverFactory, StopWordRemover, ArrayDictionary\n",
|
|
"from Sastrawi.Stemmer.StemmerFactory import StemmerFactory\n",
|
|
"\n",
|
|
"# --- PENGATURAN TAMPILAN ---\n",
|
|
"pd.set_option('display.max_colwidth', None)\n",
|
|
"\n",
|
|
"# 1. LOAD DATA\n",
|
|
"# Path file sesuai dengan yang kamu berikan\n",
|
|
"filename = '/content/drive/MyDrive/proyek skiprsi sound horeg/hasil_data_bersih_lengkap2.csv'\n",
|
|
"\n",
|
|
"try:\n",
|
|
" df = pd.read_csv(filename)\n",
|
|
" print(f\"Data berhasil dimuat: {len(df)} baris\")\n",
|
|
"except FileNotFoundError:\n",
|
|
" print(\"Error: File tidak ditemukan. Pastikan path di Google Drive sudah benar.\")\n",
|
|
" # Dummy data untuk antisipasi error jika dijalankan orang lain\n",
|
|
" df = pd.DataFrame({'full_text': ['sound horeg jancok gak enak', 'asu tenan', 'saya tidak setuju']})\n",
|
|
"\n",
|
|
"# Hapus Duplikat\n",
|
|
"df = df.drop_duplicates(subset=['full_text'])\n",
|
|
"\n",
|
|
"# 2. DEFINISI KAMUS (Kamus Alay + Istilah Intoleransi + Bahasa Jawa)\n",
|
|
"kamus_alay = {\n",
|
|
" # --- A. KATA GANTI & SINGKATAN UMUM ---\n",
|
|
" 'org': 'orang', 'yg': 'yang', 'jg': 'juga', 'ga': 'tidak', 'udh': 'sudah',\n",
|
|
" 'jatim': 'jawa timur', 'bgt': 'banget', 'wong': 'orang', 'tak': 'tidak',\n",
|
|
" 'utk': 'untuk', 'trs': 'terus', 'gak': 'tidak', 'tu': 'itu', 'gimana': 'bagaimana',\n",
|
|
" 'sampe': 'sampai', 'ampe': 'sampai', 'jd': 'jadi', 'gw': 'aku', 'tau': 'tahu',\n",
|
|
" 'gara': 'karena', 'trus': 'terus', 'sm': 'sama', 'pake': 'pakai', 'klo': 'kalau',\n",
|
|
" 'gue': 'aku', 'tp': 'tapi', 'dr': 'dari', 'jgn': 'jangan', 'fasum': 'fasilitas umum',\n",
|
|
" 'gini': 'ini', 'ama': 'sama', 'knp': 'kenapa', 'cm': 'cuma', 'udah': 'sudah',\n",
|
|
" 'gada': 'tidak ada', 'gmn': 'bagaimana', 'emg': 'memang', 'krn': 'karena',\n",
|
|
" 'sdh': 'sudah', 'aja': 'saja', 'dlm': 'dalam', 'blm': 'belum', 'dgn': 'dengan',\n",
|
|
" 'scr': 'secara', 'adlh': 'adalah', 'tdk': 'tidak', 'skrg': 'sekarang',\n",
|
|
" 'bkn': 'bukan', 'sbg': 'sebagai', 'kalo': 'kalau', 'buanter': 'kencang',\n",
|
|
"\n",
|
|
" # --- B. ISTILAH KONFLIK & INTOLERANSI ---\n",
|
|
" 'brisik': 'berisik', 'bising': 'berisik', 'budeg': 'tuli', 'brebeken': 'berisik',\n",
|
|
" 'pekok': 'bodoh', 'goblok': 'bodoh', 'tolol': 'bodoh', 'edan': 'gila',\n",
|
|
" 'gendeng': 'gila', 'stress': 'gila', 'rusuh': 'rusak', 'ancur': 'hancur',\n",
|
|
" 'bakar': 'bakar', 'matek': 'mati', 'modar': 'mati', 'sampah': 'buruk',\n",
|
|
" 'sdm': 'sumber daya manusia', 'rendah': 'buruk', 'norak': 'kampungan',\n",
|
|
" 'ganggu': 'mengganggu', 'keganggu': 'terganggu',\n",
|
|
"\n",
|
|
" # --- C. BAHASA JAWA TIMURAN & KATA KASAR (UPDATED) ---\n",
|
|
" 'nek': 'kalau', 'iso': 'bisa', 'ra': 'tidak', 'ora': 'tidak', 'ae': 'saja',\n",
|
|
" 'wae': 'saja', 'akeh': 'banyak', 'seng': 'yang', 'sing': 'yang', 'wes': 'sudah',\n",
|
|
" 'wis': 'sudah', 'urung': 'belum', 'durung': 'belum', 'lapo': 'kenapa',\n",
|
|
" 'opo': 'apa', 'iki': 'ini', 'kuwi': 'itu', 'kae': 'itu', 'elek': 'jelek',\n",
|
|
" 'apik': 'bagus', 'onok': 'ada', 'karo': 'sama',\n",
|
|
"\n",
|
|
" # PERBAIKAN DI SINI:\n",
|
|
" 'cok': 'jancok', # Singkatan disamakan ke jancok\n",
|
|
" 'dancok': 'jancok', # Varian disamakan ke jancok\n",
|
|
" # 'jancok' KITA BIARKAN (TIDAK ADA DI KAMUS) AGAR TIDAK BERUBAH\n",
|
|
"\n",
|
|
" 'asu': 'anjing', # Hewan tetap diterjemahkan\n",
|
|
"\n",
|
|
" # --- D. PERBAIKAN TYPO ---\n",
|
|
" 'gabisa': 'tidak bisa', 'gaenak': 'tidak enak', 'soundhoreg': 'sound horeg',\n",
|
|
" 'soundsystem': 'sound system', 'gasemua': 'tidak semua',\n",
|
|
"\n",
|
|
" # --- E. PENGHAPUSAN (Kata tanpa makna) ---\n",
|
|
" 'wkwkwkw': '', 'wkwkw': '', 'wkwkwkwk': '', 'wkwk': '', 'sih': '', 'nya': ''\n",
|
|
"}\n",
|
|
"\n",
|
|
"# 3. PERSIAPAN SASTRAWI (MODIFIKASI KHUSUS INTOLERANSI)\n",
|
|
"factory_stop = StopWordRemoverFactory()\n",
|
|
"stopwords_list = factory_stop.get_stop_words()\n",
|
|
"\n",
|
|
"# WHITELIST: Kata yang HARAM dihapus\n",
|
|
"whitelist = [\n",
|
|
" 'tidak', 'enggak', 'bukan', 'jangan', 'tapi',\n",
|
|
" 'masalah', 'kurang', 'belum', 'tak', 'tanpa',\n",
|
|
" 'soal', 'sebab', 'karena', 'akibat', 'padahal'\n",
|
|
"]\n",
|
|
"\n",
|
|
"# Hapus whitelist dari daftar stopword bawaan\n",
|
|
"for word in whitelist:\n",
|
|
" if word in stopwords_list:\n",
|
|
" stopwords_list.remove(word)\n",
|
|
"\n",
|
|
"# Tambahkan stopword sampah\n",
|
|
"stopwords_list.extend(['min', 'kak', 'gan', 'sis', 'guys', 'halo', 'hai'])\n",
|
|
"\n",
|
|
"dictionary = ArrayDictionary(stopwords_list)\n",
|
|
"stopword_remover = StopWordRemover(dictionary)\n",
|
|
"\n",
|
|
"factory_stem = StemmerFactory()\n",
|
|
"stemmer = factory_stem.create_stemmer()\n",
|
|
"\n",
|
|
"# 4. FUNGSI PEMBERSIH UTAMA\n",
|
|
"def clean_text_complete(text):\n",
|
|
" if not isinstance(text, str):\n",
|
|
" return \"\"\n",
|
|
"\n",
|
|
" # A. Case Folding\n",
|
|
" text = text.lower()\n",
|
|
"\n",
|
|
" # B. Ganti simbol dengan spasi\n",
|
|
" text = re.sub(r'[^a-zA-Z0-9]', ' ', text)\n",
|
|
"\n",
|
|
" # C. Normalisasi Kata\n",
|
|
" words = text.split()\n",
|
|
" normalized_words = []\n",
|
|
" for w in words:\n",
|
|
" if w in kamus_alay:\n",
|
|
" # Jika ada di kamus, ganti. Jika replacement '', kata dihapus.\n",
|
|
" if kamus_alay[w] != '':\n",
|
|
" normalized_words.append(kamus_alay[w])\n",
|
|
" else:\n",
|
|
" # Jika tidak ada di kamus (misal: 'jancok'), biarkan apa adanya\n",
|
|
" normalized_words.append(w)\n",
|
|
"\n",
|
|
" text = ' '.join(normalized_words)\n",
|
|
"\n",
|
|
" # D. Stopword Removal\n",
|
|
" text = stopword_remover.remove(text)\n",
|
|
"\n",
|
|
" # E. Stemming\n",
|
|
" text = stemmer.stem(text)\n",
|
|
"\n",
|
|
" # F. Rapikan Spasi\n",
|
|
" text = re.sub(r'\\s+', ' ', text).strip()\n",
|
|
"\n",
|
|
" return text\n",
|
|
"\n",
|
|
"# 5. EKSEKUSI\n",
|
|
"print(\"Sedang memproses teks... (Mohon tunggu)\")\n",
|
|
"df['full_text_clean'] = df['full_text'].apply(clean_text_complete)\n",
|
|
"\n",
|
|
"# 6. SIMPAN HASIL\n",
|
|
"columns_to_save = ['created_at', 'favorite_count', 'full_text_clean']\n",
|
|
"valid_columns = [col for col in columns_to_save if col in df.columns]\n",
|
|
"df_final = df[valid_columns]\n",
|
|
"\n",
|
|
"# Rename kolom hasil bersih menjadi 'full_text' agar siap dipakai\n",
|
|
"df_final = df_final.rename(columns={'full_text_clean': 'full_text'})\n",
|
|
"\n",
|
|
"# Hapus baris kosong\n",
|
|
"df_final = df_final[df_final['full_text'].str.strip() != '']\n",
|
|
"\n",
|
|
"# Simpan\n",
|
|
"output_file = 'hasil_preprocessing_intoleransi_final.csv'\n",
|
|
"df_final.to_csv(output_file, index=False)\n",
|
|
"\n",
|
|
"print(\"\\n--- SELESAI! ---\")\n",
|
|
"print(f\"File siap disimpan sebagai: {output_file}\")\n",
|
|
"print(\"\\nContoh Hasil (5 baris pertama):\")\n",
|
|
"print(df_final.head())"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "3d1636b6",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#labeling data ke 3 kategori\n",
|
|
"\n",
|
|
"import pandas as pd\n",
|
|
"# 1. MOUNT DRIVE\n",
|
|
"# 2. LOAD DATA BERSIH\n",
|
|
"# Pastikan path file benar\n",
|
|
"input_filename = '/content/drive/MyDrive/proyek skiprsi sound horeg/hasil_preprocessing_intoleransi_final.csv'\n",
|
|
"df = pd.read_csv(input_filename)\n",
|
|
"\n",
|
|
"# 3. DEFINISI KATA KUNCI (3 KATEGORI)\n",
|
|
"\n",
|
|
"# KATA KUNCI RASIONAL (Label 1)\n",
|
|
"# Fokus: Dampak fisik, gangguan situasi, penolakan logis\n",
|
|
"keywords_rasional = [\n",
|
|
" 'ganggu', 'bising', 'berisik', 'brisik', 'polusi', 'tuli', 'budeg',\n",
|
|
" 'pecah', 'getar', 'runtuh', 'rusak', 'macet', 'blokir', 'tutup jalan',\n",
|
|
" 'sakit', 'pusing', 'jantung', 'bayi', 'orang tua', 'anak', 'nangis',\n",
|
|
" 'tidak setuju', 'tidak suka', 'tolak', 'keberatan', 'komplain',\n",
|
|
" 'aturan', 'izin', 'waktu', 'jam', 'solusi', 'saran', 'uang', 'biaya',\n",
|
|
" 'sebab', 'karena', 'gara', 'akibat', 'dampak', 'bikin', 'buat',\n",
|
|
" 'tidur', 'istirahat', 'belajar', 'ibadah', 'sholat', 'ngaji'\n",
|
|
"]\n",
|
|
"\n",
|
|
"# KATA KUNCI CACI MAKI (Label 2)\n",
|
|
"# Fokus: Hinaan personal, hewan, kotoran, ancaman kosong\n",
|
|
"keywords_cacian = [\n",
|
|
" 'jancok', 'cok', 'dancok', 'asu', 'anjing', 'bangsat', 'bajingan',\n",
|
|
" 'goblok', 'tolol', 'pekok', 'bodoh', 'bego', 'idiot', 'setan', 'iblis',\n",
|
|
" 'sakit jiwa', 'gila', 'edan', 'gendeng', 'sdm rendah', 'kampungan',\n",
|
|
" 'bakar', 'musnah', 'usir', 'mati', 'modar', 'sampah', 'norak',\n",
|
|
" 'jelek', 'buruk'\n",
|
|
"]\n",
|
|
"\n",
|
|
"# 4. FUNGSI LABELING 3 KATEGORI\n",
|
|
"def auto_label_3_class(text):\n",
|
|
" if not isinstance(text, str):\n",
|
|
" return 0 # Default Netral\n",
|
|
"\n",
|
|
" text_check = f\" {text} \"\n",
|
|
"\n",
|
|
" # PRIORITAS 1: Apakah ini Kritik Rasional? (Label 1)\n",
|
|
" # Aturan: Walaupun kasar, kalau ada poin rasional, masuk sini.\n",
|
|
" for word in keywords_rasional:\n",
|
|
" if f\" {word} \" in text_check:\n",
|
|
" return 1\n",
|
|
"\n",
|
|
" # PRIORITAS 2: Apakah ini Murni Caci Maki? (Label 2)\n",
|
|
" # Aturan: Kasar tapi tidak ada alasan jelas.\n",
|
|
" for word in keywords_cacian:\n",
|
|
" if f\" {word} \" in text_check:\n",
|
|
" return 2\n",
|
|
"\n",
|
|
" # PRIORITAS 3: Sisanya adalah Netral/Info (Label 0)\n",
|
|
" return 0\n",
|
|
"\n",
|
|
"# 5. EKSEKUSI\n",
|
|
"print(\"Sedang melabeli data menjadi 3 Kategori...\")\n",
|
|
"print(\"0: Netral | 1: Kritik Rasional | 2: Caci Maki\")\n",
|
|
"df['label'] = df['full_text'].apply(auto_label_3_class)\n",
|
|
"\n",
|
|
"# 6. CEK HASIL SEBARAN\n",
|
|
"counts = df['label'].value_counts().sort_index()\n",
|
|
"print(\"\\n--- Statistik Label Sementara ---\")\n",
|
|
"print(f\"Label 0 (Netral/Info) : {counts.get(0, 0)} data\")\n",
|
|
"print(f\"Label 1 (Kritik Rasional) : {counts.get(1, 0)} data\")\n",
|
|
"print(f\"Label 2 (Caci Maki Murni) : {counts.get(2, 0)} data\")\n",
|
|
"\n",
|
|
"# 7. SIMPAN KE FILE BARU\n",
|
|
"output_file = '/content/drive/MyDrive/proyek skiprsi sound horeg/data_label_3_kategori.csv'\n",
|
|
"df.to_csv(output_file, index=False)\n",
|
|
"print(f\"\\n[SUKSES] File siap diverifikasi manual: {output_file}\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "dd1ab0e0",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "virtualEnvironment",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.11.9"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|