{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "adf2c795",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import re\n",
    "import string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e2dbcc87",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('dataset\\dataYangDiPakai\\data_sound_horeg_total_mei.csv')\n",
    "data.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8e082cd4",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# 1. Pastikan 'favorite_count' TIDAK ada di daftar kolom yang dibuang\n",
    "kolomDibuang = [\n",
    "    'conversation_id_str', 'id_str', 'image_url', 'in_reply_to_screen_name',\n",
    "    'lang', 'location', 'quote_count', 'reply_count', 'retweet_count',\n",
    "    'tweet_url', 'user_id_str', 'username'\n",
    "]\n",
    "\n",
    "# 2. Hapus kolom sampah\n",
    "dataPembersihan = data.drop(columns=kolomDibuang)\n",
    "\n",
    "# 3. Hapus baris kosong agar tidak error\n",
    "dataPembersihan = dataPembersihan.dropna(subset=['full_text'])\n",
    "\n",
    "# 4. Fungsi Pembersihan (Tetap sama)\n",
    "def cleaningKhusus(text):\n",
    "    text = re.sub(r'http\\S+|www\\S+|https\\S+', '', text, flags=re.MULTILINE)\n",
    "    text = re.sub(r'\\@\\w+|\\#','', text)\n",
    "    text = text.translate(str.maketrans('', '', string.punctuation))\n",
    "    text = re.sub(r'\\d+', '', text)\n",
    "    text = text.strip()\n",
    "    return text\n",
    "\n",
    "def cleaningUmum(text):\n",
    "    text = text.lower()\n",
    "    text = re.sub(r'\\s+', ' ', text)\n",
    "    text = text.encode('ascii', 'ignore').decode('ascii')\n",
    "    text = ' '.join([word for word in text.split() if len(word) > 1])\n",
    "    return text\n",
    "\n",
    "# 5. BERSIHKAN TEKS dan langsung simpan kembali ke kolom 'full_text'\n",
    "# Ini akan menimpa teks asli dengan teks yang sudah bersih\n",
    "dataPembersihan['full_text'] = dataPembersihan['full_text'].apply(cleaningKhusus).apply(cleaningUmum)\n",
    "\n",
    "# 6. Pilih hanya 3 kolom yang Anda minta\n",
    "df_hasil_akhir = dataPembersihan[['created_at', 'favorite_count', 'full_text']]\n",
    "\n",
    "# 7. Simpan ke CSV\n",
    "df_hasil_akhir.to_csv('dataset\\dataYangDiPakai\\data_sound_horeg_total_mei_dibersihkan.csv', index=False)\n",
    "\n",
    "# Tampilkan hasil\n",
    "print(\"Berhasil! Kolom sekarang hanya: created_at, favorite_count, dan full_text (sudah bersih).\")\n",
    "print(df_hasil_akhir.head(20))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ee52f2ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_colwidth', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "414378d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('dataset\\dataYangDiPakai\\hasil_preprocessing_intoleransi.csv')\n",
    "df.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "69e6b2a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "#membersihkan data\n",
    "import pandas as pd\n",
    "import re\n",
    "from Sastrawi.StopWordRemover.StopWordRemoverFactory import StopWordRemoverFactory, StopWordRemover, ArrayDictionary\n",
    "from Sastrawi.Stemmer.StemmerFactory import StemmerFactory\n",
    "\n",
    "# --- PENGATURAN TAMPILAN ---\n",
    "pd.set_option('display.max_colwidth', None)\n",
    "\n",
    "# 1. LOAD DATA\n",
    "# Path file sesuai dengan yang kamu berikan\n",
    "filename = '/content/drive/MyDrive/proyek skiprsi sound horeg/hasil_data_bersih_lengkap2.csv'\n",
    "\n",
    "try:\n",
    "    df = pd.read_csv(filename)\n",
    "    print(f\"Data berhasil dimuat: {len(df)} baris\")\n",
    "except FileNotFoundError:\n",
    "    print(\"Error: File tidak ditemukan. Pastikan path di Google Drive sudah benar.\")\n",
    "    # Dummy data untuk antisipasi error jika dijalankan orang lain\n",
    "    df = pd.DataFrame({'full_text': ['sound horeg jancok gak enak', 'asu tenan', 'saya tidak setuju']})\n",
    "\n",
    "# Hapus Duplikat\n",
    "df = df.drop_duplicates(subset=['full_text'])\n",
    "\n",
    "# 2. DEFINISI KAMUS (Kamus Alay + Istilah Intoleransi + Bahasa Jawa)\n",
    "kamus_alay = {\n",
    "    # --- A. KATA GANTI & SINGKATAN UMUM ---\n",
    "    'org': 'orang', 'yg': 'yang', 'jg': 'juga', 'ga': 'tidak', 'udh': 'sudah',\n",
    "    'jatim': 'jawa timur', 'bgt': 'banget', 'wong': 'orang', 'tak': 'tidak',\n",
    "    'utk': 'untuk', 'trs': 'terus', 'gak': 'tidak', 'tu': 'itu', 'gimana': 'bagaimana',\n",
    "    'sampe': 'sampai', 'ampe': 'sampai', 'jd': 'jadi', 'gw': 'aku', 'tau': 'tahu',\n",
    "    'gara': 'karena', 'trus': 'terus', 'sm': 'sama', 'pake': 'pakai', 'klo': 'kalau',\n",
    "    'gue': 'aku', 'tp': 'tapi', 'dr': 'dari', 'jgn': 'jangan', 'fasum': 'fasilitas umum',\n",
    "    'gini': 'ini', 'ama': 'sama', 'knp': 'kenapa', 'cm': 'cuma', 'udah': 'sudah',\n",
    "    'gada': 'tidak ada', 'gmn': 'bagaimana', 'emg': 'memang', 'krn': 'karena',\n",
    "    'sdh': 'sudah', 'aja': 'saja', 'dlm': 'dalam', 'blm': 'belum', 'dgn': 'dengan',\n",
    "    'scr': 'secara', 'adlh': 'adalah', 'tdk': 'tidak', 'skrg': 'sekarang',\n",
    "    'bkn': 'bukan', 'sbg': 'sebagai', 'kalo': 'kalau', 'buanter': 'kencang',\n",
    "\n",
    "    # --- B. ISTILAH KONFLIK & INTOLERANSI ---\n",
    "    'brisik': 'berisik', 'bising': 'berisik', 'budeg': 'tuli', 'brebeken': 'berisik',\n",
    "    'pekok': 'bodoh', 'goblok': 'bodoh', 'tolol': 'bodoh', 'edan': 'gila',\n",
    "    'gendeng': 'gila', 'stress': 'gila', 'rusuh': 'rusak', 'ancur': 'hancur',\n",
    "    'bakar': 'bakar', 'matek': 'mati', 'modar': 'mati', 'sampah': 'buruk',\n",
    "    'sdm': 'sumber daya manusia', 'rendah': 'buruk', 'norak': 'kampungan',\n",
    "    'ganggu': 'mengganggu', 'keganggu': 'terganggu',\n",
    "\n",
    "    # --- C. BAHASA JAWA TIMURAN & KATA KASAR (UPDATED) ---\n",
    "    'nek': 'kalau', 'iso': 'bisa', 'ra': 'tidak', 'ora': 'tidak', 'ae': 'saja',\n",
    "    'wae': 'saja', 'akeh': 'banyak', 'seng': 'yang', 'sing': 'yang', 'wes': 'sudah',\n",
    "    'wis': 'sudah', 'urung': 'belum', 'durung': 'belum', 'lapo': 'kenapa',\n",
    "    'opo': 'apa', 'iki': 'ini', 'kuwi': 'itu', 'kae': 'itu', 'elek': 'jelek',\n",
    "    'apik': 'bagus', 'onok': 'ada', 'karo': 'sama',\n",
    "\n",
    "    # PERBAIKAN DI SINI:\n",
    "    'cok': 'jancok',      # Singkatan disamakan ke jancok\n",
    "    'dancok': 'jancok',   # Varian disamakan ke jancok\n",
    "    # 'jancok' KITA BIARKAN (TIDAK ADA DI KAMUS) AGAR TIDAK BERUBAH\n",
    "\n",
    "    'asu': 'anjing',      # Hewan tetap diterjemahkan\n",
    "\n",
    "    # --- D. PERBAIKAN TYPO ---\n",
    "    'gabisa': 'tidak bisa', 'gaenak': 'tidak enak', 'soundhoreg': 'sound horeg',\n",
    "    'soundsystem': 'sound system', 'gasemua': 'tidak semua',\n",
    "\n",
    "    # --- E. PENGHAPUSAN (Kata tanpa makna) ---\n",
    "    'wkwkwkw': '', 'wkwkw': '', 'wkwkwkwk': '', 'wkwk': '', 'sih': '', 'nya': ''\n",
    "}\n",
    "\n",
    "# 3. PERSIAPAN SASTRAWI (MODIFIKASI KHUSUS INTOLERANSI)\n",
    "factory_stop = StopWordRemoverFactory()\n",
    "stopwords_list = factory_stop.get_stop_words()\n",
    "\n",
    "# WHITELIST: Kata yang HARAM dihapus\n",
    "whitelist = [\n",
    "    'tidak', 'enggak', 'bukan', 'jangan', 'tapi',\n",
    "    'masalah', 'kurang', 'belum', 'tak', 'tanpa',\n",
    "    'soal', 'sebab', 'karena', 'akibat', 'padahal'\n",
    "]\n",
    "\n",
    "# Hapus whitelist dari daftar stopword bawaan\n",
    "for word in whitelist:\n",
    "    if word in stopwords_list:\n",
    "        stopwords_list.remove(word)\n",
    "\n",
    "# Tambahkan stopword sampah\n",
    "stopwords_list.extend(['min', 'kak', 'gan', 'sis', 'guys', 'halo', 'hai'])\n",
    "\n",
    "dictionary = ArrayDictionary(stopwords_list)\n",
    "stopword_remover = StopWordRemover(dictionary)\n",
    "\n",
    "factory_stem = StemmerFactory()\n",
    "stemmer = factory_stem.create_stemmer()\n",
    "\n",
    "# 4. FUNGSI PEMBERSIH UTAMA\n",
    "def clean_text_complete(text):\n",
    "    if not isinstance(text, str):\n",
    "        return \"\"\n",
    "\n",
    "    # A. Case Folding\n",
    "    text = text.lower()\n",
    "\n",
    "    # B. Ganti simbol dengan spasi\n",
    "    text = re.sub(r'[^a-zA-Z0-9]', ' ', text)\n",
    "\n",
    "    # C. Normalisasi Kata\n",
    "    words = text.split()\n",
    "    normalized_words = []\n",
    "    for w in words:\n",
    "        if w in kamus_alay:\n",
    "            # Jika ada di kamus, ganti. Jika replacement '', kata dihapus.\n",
    "            if kamus_alay[w] != '':\n",
    "                normalized_words.append(kamus_alay[w])\n",
    "        else:\n",
    "            # Jika tidak ada di kamus (misal: 'jancok'), biarkan apa adanya\n",
    "            normalized_words.append(w)\n",
    "\n",
    "    text = ' '.join(normalized_words)\n",
    "\n",
    "    # D. Stopword Removal\n",
    "    text = stopword_remover.remove(text)\n",
    "\n",
    "    # E. Stemming\n",
    "    text = stemmer.stem(text)\n",
    "\n",
    "    # F. Rapikan Spasi\n",
    "    text = re.sub(r'\\s+', ' ', text).strip()\n",
    "\n",
    "    return text\n",
    "\n",
    "# 5. EKSEKUSI\n",
    "print(\"Sedang memproses teks... (Mohon tunggu)\")\n",
    "df['full_text_clean'] = df['full_text'].apply(clean_text_complete)\n",
    "\n",
    "# 6. SIMPAN HASIL\n",
    "columns_to_save = ['created_at', 'favorite_count', 'full_text_clean']\n",
    "valid_columns = [col for col in columns_to_save if col in df.columns]\n",
    "df_final = df[valid_columns]\n",
    "\n",
    "# Rename kolom hasil bersih menjadi 'full_text' agar siap dipakai\n",
    "df_final = df_final.rename(columns={'full_text_clean': 'full_text'})\n",
    "\n",
    "# Hapus baris kosong\n",
    "df_final = df_final[df_final['full_text'].str.strip() != '']\n",
    "\n",
    "# Simpan\n",
    "output_file = 'hasil_preprocessing_intoleransi_final.csv'\n",
    "df_final.to_csv(output_file, index=False)\n",
    "\n",
    "print(\"\\n--- SELESAI! ---\")\n",
    "print(f\"File siap disimpan sebagai: {output_file}\")\n",
    "print(\"\\nContoh Hasil (5 baris pertama):\")\n",
    "print(df_final.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d1636b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "#labeling data ke 3 kategori\n",
    "\n",
    "import pandas as pd\n",
    "# 1. MOUNT DRIVE\n",
    "# 2. LOAD DATA BERSIH\n",
    "# Pastikan path file benar\n",
    "input_filename = '/content/drive/MyDrive/proyek skiprsi sound horeg/hasil_preprocessing_intoleransi_final.csv'\n",
    "df = pd.read_csv(input_filename)\n",
    "\n",
    "# 3. DEFINISI KATA KUNCI (3 KATEGORI)\n",
    "\n",
    "# KATA KUNCI RASIONAL (Label 1)\n",
    "# Fokus: Dampak fisik, gangguan situasi, penolakan logis\n",
    "keywords_rasional = [\n",
    "    'ganggu', 'bising', 'berisik', 'brisik', 'polusi', 'tuli', 'budeg',\n",
    "    'pecah', 'getar', 'runtuh', 'rusak', 'macet', 'blokir', 'tutup jalan',\n",
    "    'sakit', 'pusing', 'jantung', 'bayi', 'orang tua', 'anak', 'nangis',\n",
    "    'tidak setuju', 'tidak suka', 'tolak', 'keberatan', 'komplain',\n",
    "    'aturan', 'izin', 'waktu', 'jam', 'solusi', 'saran', 'uang', 'biaya',\n",
    "    'sebab', 'karena', 'gara', 'akibat', 'dampak', 'bikin', 'buat',\n",
    "    'tidur', 'istirahat', 'belajar', 'ibadah', 'sholat', 'ngaji'\n",
    "]\n",
    "\n",
    "# KATA KUNCI CACI MAKI (Label 2)\n",
    "# Fokus: Hinaan personal, hewan, kotoran, ancaman kosong\n",
    "keywords_cacian = [\n",
    "    'jancok', 'cok', 'dancok', 'asu', 'anjing', 'bangsat', 'bajingan',\n",
    "    'goblok', 'tolol', 'pekok', 'bodoh', 'bego', 'idiot', 'setan', 'iblis',\n",
    "    'sakit jiwa', 'gila', 'edan', 'gendeng', 'sdm rendah', 'kampungan',\n",
    "    'bakar', 'musnah', 'usir', 'mati', 'modar', 'sampah', 'norak',\n",
    "    'jelek', 'buruk'\n",
    "]\n",
    "\n",
    "# 4. FUNGSI LABELING 3 KATEGORI\n",
    "def auto_label_3_class(text):\n",
    "    if not isinstance(text, str):\n",
    "        return 0 # Default Netral\n",
    "\n",
    "    text_check = f\" {text} \"\n",
    "\n",
    "    # PRIORITAS 1: Apakah ini Kritik Rasional? (Label 1)\n",
    "    # Aturan: Walaupun kasar, kalau ada poin rasional, masuk sini.\n",
    "    for word in keywords_rasional:\n",
    "        if f\" {word} \" in text_check:\n",
    "            return 1\n",
    "\n",
    "    # PRIORITAS 2: Apakah ini Murni Caci Maki? (Label 2)\n",
    "    # Aturan: Kasar tapi tidak ada alasan jelas.\n",
    "    for word in keywords_cacian:\n",
    "        if f\" {word} \" in text_check:\n",
    "            return 2\n",
    "\n",
    "    # PRIORITAS 3: Sisanya adalah Netral/Info (Label 0)\n",
    "    return 0\n",
    "\n",
    "# 5. EKSEKUSI\n",
    "print(\"Sedang melabeli data menjadi 3 Kategori...\")\n",
    "print(\"0: Netral | 1: Kritik Rasional | 2: Caci Maki\")\n",
    "df['label'] = df['full_text'].apply(auto_label_3_class)\n",
    "\n",
    "# 6. CEK HASIL SEBARAN\n",
    "counts = df['label'].value_counts().sort_index()\n",
    "print(\"\\n--- Statistik Label Sementara ---\")\n",
    "print(f\"Label 0 (Netral/Info)      : {counts.get(0, 0)} data\")\n",
    "print(f\"Label 1 (Kritik Rasional)  : {counts.get(1, 0)} data\")\n",
    "print(f\"Label 2 (Caci Maki Murni)  : {counts.get(2, 0)} data\")\n",
    "\n",
    "# 7. SIMPAN KE FILE BARU\n",
    "output_file = '/content/drive/MyDrive/proyek skiprsi sound horeg/data_label_3_kategori.csv'\n",
    "df.to_csv(output_file, index=False)\n",
    "print(f\"\\n[SUKSES] File siap diverifikasi manual: {output_file}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd1ab0e0",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "virtualEnvironment",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}