import re import nltk from nltk.tokenize import word_tokenize from Sastrawi.Stemmer.StemmerFactory import ( StemmerFactory ) from Sastrawi.StopWordRemover.StopWordRemoverFactory import ( StopWordRemoverFactory ) # ========================================================= # NLTK # ========================================================= nltk.download("punkt", quiet=True) nltk.download("punkt_tab", quiet=True) # ========================================================= # STEMMER # ========================================================= factory_stemmer = StemmerFactory() stemmer = ( factory_stemmer .create_stemmer() ) # ========================================================= # STOPWORD # ========================================================= factory_stopword = StopWordRemoverFactory() stopwords = set( factory_stopword.get_stop_words() ) # ========================================================= # KATA PENTING # ========================================================= # Kata-kata ini dikeluarkan dari daftar stopword # agar tetap dipertahankan karena memiliki konteks # terhadap sentimen. kata_penting = { # ------------------------- # Negasi # ------------------------- "tidak", "bukan", "belum", "jangan", "kurang", # ------------------------- # Konteks waktu dan kondisi # ------------------------- "masih", "sudah", "sering", "jarang", "kadang", "lagi", "sebelum", "setelah", "kembali", # ------------------------- # Konjungsi yang membawa konteks # ------------------------- "namun", "tetapi", "meskipun", "walaupun", # ------------------------- # Intensitas # ------------------------- "sangat", "cukup", "terlalu", # ------------------------- # Kondisi dan kemampuan # ------------------------- "hanya", "tanpa", "harus", "bisa", "dapat", "perlu", "seharusnya", # ------------------------- # Keinginan # ------------------------- "ingin" } # Hapus kata penting dari stopword stopwords = stopwords - kata_penting # Kata tambahan yang tidak membawa informasi sentimen stopwords.update({ "nya" }) # ========================================================= # KAMUS NORMALISASI # ========================================================= kamus_normalisasi = { # ------------------------- # Singkatan # ------------------------- "yg": "yang", "yng": "yang", "dgn": "dengan", "utk": "untuk", # ------------------------- # Negasi # ------------------------- "tdk": "tidak", "gak": "tidak", "ga": "tidak", "nggak": "tidak", "ngga": "tidak", "enggak": "tidak", # ------------------------- # Kata umum # ------------------------- "krn": "karena", "karna": "karena", "dr": "dari", "jd": "jadi", "jg": "juga", "tp": "tetapi", "sm": "sama", "sdh": "sudah", "udh": "sudah", "udah": "sudah", "blm": "belum", "belom": "belum", "bgt": "banget", "aja": "saja", "sbg": "sebagai", "dpt": "dapat", "bbrp": "beberapa", # ------------------------- # Kata tanya # ------------------------- "gmn": "bagaimana", "gimana": "bagaimana", # ------------------------- # Kata kondisi # ------------------------- "klo": "kalau", "kalo": "kalau", "kalok": "kalau", # ------------------------- # Kata informal # ------------------------- "emg": "memang", "emang": "memang", "kayak": "seperti", "kaya": "seperti", # ------------------------- # Kata khusus penelitian # ------------------------- "terkadang": "kadang" } # ========================================================= # CASE FOLDING # ========================================================= def case_folding(text): return str(text).lower() # ========================================================= # NORMALISASI # ========================================================= def normalisasi(text): text = str(text).lower() # ===================================================== # S-LEARN # ===================================================== # S-Learn # S Learn # Slearn # s-learn text = re.sub( r"\bs[\s-]*learn\b", "slearn", text ) # ===================================================== # LOGIN # ===================================================== # log in # log-in # login text = re.sub( r"\blog[\s-]*in\b", "login", text ) # ===================================================== # LOGOUT # ===================================================== # log out # log-out # logout text = re.sub( r"\blog[\s-]*out\b", "logout", text ) # ===================================================== # CHECK IN # ===================================================== # check in # check-in # checkin text = re.sub( r"\bcheck[\s-]+in\b", "checkin", text ) # ===================================================== # HP-NYA # ===================================================== text = re.sub( r"\bhpnya\b", "hp nya", text ) text = re.sub( r"\bhp\s+nya\b", "hp nya", text ) # ===================================================== # PENGULANGAN HURUF # ===================================================== # # bagusss -> bagus # lamaaa -> lama # lambattt -> lambat # # Hanya huruf yang berulang 3 kali atau lebih # yang dikurangi. text = re.sub( r"([a-z])\1{2,}", r"\1", text ) # ===================================================== # KAMUS NORMALISASI # ===================================================== words = text.split() words = [ kamus_normalisasi.get( word, word ) for word in words ] return " ".join(words) # ========================================================= # CLEANING # ========================================================= def cleaning(text): text = str(text) # ===================================================== # URL # ===================================================== text = re.sub( r"http\S+|www\S+", " ", text ) # ===================================================== # MENTION # ===================================================== text = re.sub( r"@\w+", " ", text ) # ===================================================== # HASHTAG # ===================================================== text = re.sub( r"#\w+", " ", text ) # ===================================================== # ANGKA # ===================================================== text = re.sub( r"\d+", " ", text ) # ===================================================== # SIMBOL DAN TANDA BACA # ===================================================== text = re.sub( r"[^a-zA-Z\s]", " ", text ) # ===================================================== # SPASI BERLEBIH # ===================================================== text = re.sub( r"\s+", " ", text ).strip() return text # ========================================================= # TOKENIZATION # ========================================================= def tokenisasi(text): if not str(text).strip(): return [] return word_tokenize( str(text) ) # ========================================================= # STOPWORD REMOVAL # ========================================================= def stopword_removal(tokens): return [ word for word in tokens if word not in stopwords ] # ========================================================= # ISTILAH TEKNIS # ========================================================= # Istilah berikut tidak dilakukan stemming karena # merupakan istilah teknis yang digunakan pada # konteks aplikasi S-Learn. istilah_teknis = { "slearn", "mobile", "error", "lag", "bug", "online", "offline", "deadline", "server", "loading", "website", "login", "logout", "ios", "android", "update", "ui", "ux", "user", "interface", "upload", "download", "menu", "button", "file", "link", "feedback", "friendly", "dashboard", "password", "username", "checkin", "refresh", "performance", "responsive", "submit", "session", "support" } # ========================================================= # KAMUS PENGECUALIAN STEMMING # ========================================================= kamus_pengecualian_stemming = { # ===================================================== # KATA YANG JANGAN DIUBAH # KARENA STEMMING DAPAT MERUSAK MAKNA # ===================================================== "terasa": "terasa", "terdapat": "terdapat", "pengalaman": "pengalaman", "mengenai": "mengenai", "pengumuman": "pengumuman", "bersamaan": "bersamaan", "berbagai": "berbagai", "melalui": "melalui", "sebelumnya": "sebelumnya", "pembaruan": "pembaruan", "kegiatan": "kegiatan", "tersedia": "tersedia", "tampilan": "tampilan", "pengguna": "pengguna", "ketergantungan": "ketergantungan", "dibandingkan": "dibandingkan", "menarik": "menarik", "melakukan": "melakukan", "setalah": "setelah", "terjadi": "terjadi", "digunakan": "digunakan", "tampilannya": "tampilan", "dikembangkan": "dikembangkan", "melakukan": "melakukan", "berikutnya": "berikutnya", "diperbaiki": "diperbaiki", "diperbarui": "diperbarui", "dilakukan": "dilakukan", "pengembangan": "pengembangan", "sebagian": "sebagian", "menggunakannya": "menggunakan", "menggunakan": "menggunakan", "semoga": "semoga", "memadai": "memadai", "lemot": "lemot", # ===================================================== # KATA YANG MEMANG DIINGINKAN # MENJADI BENTUK TERTENTU # ===================================================== "berurutan": "urut", "leluasa": "luas" } # ========================================================= # STEMMING # ========================================================= def stemming(tokens): hasil = [] for word in tokens: # ------------------------------------------------- # 1. Istilah teknis Inggris # ------------------------------------------------- if word in istilah_teknis: hasil.append( word ) # ------------------------------------------------- # 2. Kata yang harus dipertahankan # ------------------------------------------------- elif word in kamus_pengecualian_stemming: hasil.append( kamus_pengecualian_stemming[word] ) # ------------------------------------------------- # 3. Stemming normal Sastrawi # ------------------------------------------------- else: hasil.append( stemmer.stem(word) ) return hasil # ========================================================= # FULL PREPROCESSING # ========================================================= def preprocess_text(text): # ===================================================== # 1. CASE FOLDING # ===================================================== hasil_case_folding = case_folding( text ) # ===================================================== # 2. NORMALISASI # ===================================================== hasil_normalisasi = normalisasi( hasil_case_folding ) # ===================================================== # 3. CLEANING # ===================================================== hasil_cleaning = cleaning( hasil_normalisasi ) # ===================================================== # 4. TOKENIZATION # ===================================================== hasil_tokenization = tokenisasi( hasil_cleaning ) # ===================================================== # 5. STOPWORD REMOVAL # ===================================================== hasil_stopword = stopword_removal( hasil_tokenization ) # ===================================================== # 6. STEMMING # ===================================================== hasil_stemming = stemming( hasil_stopword ) # ===================================================== # 7. TEXT PREPROCESSED # ===================================================== hasil_preprocessed = " ".join( hasil_stemming ) # ===================================================== # RETURN HASIL SETIAP TAHAP # ===================================================== return { "case_folding": hasil_case_folding, "normalisasi": hasil_normalisasi, "cleaning": hasil_cleaning, "tokenization": " ".join( hasil_tokenization ), "stopword_removal": " ".join( hasil_stopword ), "stemming": " ".join( hasil_stemming ), "text_preprocessed": hasil_preprocessed }