from flask import Flask, render_template, url_for, request, flash import tweepy import re, string, csv, pickle, os from os.path import join, dirname, realpath import pandas as pd import numpy as np from Sastrawi.Stemmer.StemmerFactory import StemmerFactory from Sastrawi.StopWordRemover.StopWordRemoverFactory import StopWordRemoverFactory, StopWordRemover, ArrayDictionary import nltk from nltk.corpus import stopwords from nltk.tokenize import word_tokenize from googletrans import Translator from textblob import TextBlob from sklearn.metrics import accuracy_score, precision_score, recall_score from sklearn.metrics import classification_report from sklearn.model_selection import train_test_split from sklearn.metrics import confusion_matrix from sklearn.feature_extraction.text import TfidfVectorizer from PIL import Image import urllib.request import matplotlib.pyplot as plt from wordcloud import WordCloud from sklearn.naive_bayes import MultinomialNB nltk.download('punkt') nltk.download('stopwords') #Preprocessing Twitter hasil_preprocessing = [] def preprocessing_twitter(): translator = Translator() hasil_preprocessing.clear() # Siapkan file CSV untuk menampung hasil with open('static/files/Data Preprocessing.csv', 'w', newline='', encoding='utf-8') as file: writer = csv.writer(file) # Tulis header CSV writer.writerow([ 'Tanggal', 'Username', 'Tweet', 'Cleansing', 'Case Folding', 'Normalisasi', 'Tokenizing', 'Stopword', 'Stemming', 'Translate' ]) # Baca data input dari file scraping with open("static/files/Data Scraping.csv", "r", encoding='utf-8') as csvfile: readCSV = csv.reader(csvfile, delimiter=',') for row in readCSV: # Ambil data asli tanggal = row[0] username = row[1] tweet = row[2] # --- CLEANSING (versi disamakan dengan Google Colab) --- clean = tweet clean = re.sub(r'@[A-Za-z0-9_]+', '', clean) # hapus mention clean = re.sub(r'#\w+', '', clean) # hapus hashtag clean = re.sub(r'RT[\s]+', '', clean) # hapus RT clean = re.sub(r'https?://\S+', '', clean) # hapus link clean = re.sub(r'[^A-Za-z0-9 ]', '', clean) # hapus karakter selain alfanumerik dan spasi clean = re.sub(r'\s+', ' ', clean).strip() # hilangkan spasi berlebih # --- CASE FOLDING --- casefold = clean.casefold() # --- NORMALISASI --- normalized_text = normalize_text(casefold) # --- TOKENIZING --- tokenizing = nltk.tokenize.word_tokenize(normalized_text) # --- STOPWORD REMOVAL --- stop_factory = StopWordRemoverFactory().get_stop_words() more_stop_word = ['tidak'] all_stopwords = stop_factory + more_stop_word dictionary = ArrayDictionary(all_stopwords) stopword_remover = StopWordRemover(dictionary) stopword_removed = nltk.tokenize.word_tokenize(stopword_remover.remove(normalized_text)) # --- STEMMING --- kalimat = ' '.join(stopword_removed) factory = StemmerFactory() stemmer = factory.create_stemmer() stemming = stemmer.stem(kalimat) # --- TRANSLATE --- try: translation = translator.translate(stemming, dest='en') translated_text = translation.text except: translated_text = "Terjemahan gagal" # --- SIMPAN SEMUA HASIL --- tweets = [ tanggal, username, tweet, clean, casefold, normalized_text, tokenizing, stopword_removed, stemming, translated_text ] hasil_preprocessing.append(tweets) writer.writerow(tweets) flash('Preprocessing Berhasil', 'preprocessing_data') def normalize_text(text): # Normalisasi manual kata tidak baku ke bentuk baku (berdasarkan kamus dari Google Colab) text = re.sub(r'sdh', 'sudah', text) text = re.sub(r' yg ', ' yang ', text) text = re.sub(r' nggak ', ' tidak ', text) text = re.sub(r' gak ', ' tidak ', text) text = re.sub(r' bangetdari ', ' banget dari ', text) text = re.sub(r'vibes ', 'suasana ', text) text = re.sub(r'mantab ', 'mantap ', text) text = re.sub(r' benarsetuju ', ' benar setuju ', text) text = re.sub(r' ganjarmahfud ', ' ganjar mahfud ', text) text = re.sub(r' stylish ', ' bergaya ', text) text = re.sub(r' ngapusi ', ' bohong ', text) text = re.sub(r' gede ', ' besar ', text) text = re.sub(r' all in ', ' yakin ', text) text = re.sub(r' blokkkkk ', ' goblok ', text) text = re.sub(r' blokkkk ', ' goblok ', text) text = re.sub(r' blokkk ', ' goblok ', text) text = re.sub(r' blokk ', ' goblok ', text) text = re.sub(r' blok ', ' goblok ', text) text = re.sub(r' ri ', ' republik indonesia ', text) text = re.sub(r' kem3nangan ', ' kemenangan ', text) text = re.sub(r' sat set ', ' cepat ', text) text = re.sub(r' ala ', ' dari ', text) text = re.sub(r' best ', ' terbaik ', text) text = re.sub(r' bgttt ', ' banget ', text) text = re.sub(r' gue ', ' saya ', text) text = re.sub(r' hrs ', ' harus ', text) text = re.sub(r' fixed ', ' tetap ', text) text = re.sub(r' blom ', ' belum ', text) text = re.sub(r' aing ', ' aku ', text) text = re.sub(r' tehnologi ', ' teknologi ', text) text = re.sub(r' jd ', ' jadi ', text) text = re.sub(r' dg ', ' dengan ', text) text = re.sub(r' kudu ', ' harus ', text) text = re.sub(r' jk ', ' jika ', text) text = re.sub(r' problem ', ' masalah ', text) text = re.sub(r' iru ', ' itu ', text) text = re.sub(r' duit ', ' uang ', text) text = re.sub(r' duid ', ' uang ', text) text = re.sub(r' bgsd ', ' bangsat ', text) text = re.sub(r' jt ', ' juta ', text) text = re.sub(r' stop ', ' berhenti ', text) text = re.sub(r' ngeri ', ' seram ', text) text = re.sub(r' turu ', ' tidur ', text) text = re.sub(r' early ', ' awal ', text) text = re.sub(r' pertamna ', ' pertamina ', text) text = re.sub(r' yg ', ' yang ', text) text = re.sub(r' mnurut ', ' menurut ', text) text = re.sub(r' trus ', ' terus ', text) text = re.sub(r' msh ', ' masih ', text) text = re.sub(r' simple ', ' mudah ', text) text = re.sub(r' worth ', ' layak ', text) text = re.sub(r'problem ', 'masalah ', text) text = re.sub(r' hny ', ' hanya ', text) text = re.sub(r' dn ', ' dan ', text) text = re.sub(r' jln ', ' jalan ', text) text = re.sub(r' bgt ', ' banget ', text) text = re.sub(r' yg ', ' yang ', text) text = re.sub(r' ga ', ' tidak ', text) text = re.sub(r' text ', ' teks ', text) text = re.sub(r' end ', ' selesai ', text) text = re.sub(r' kelen ', ' kalian ', text) text = re.sub(r' jd ', ' jadi ', text) text = re.sub(r' tuk ', ' untuk ', text) text = re.sub(r' kk ', ' kakak ', text) return text # Labeling 5 Kelas hasil_labeling = [] def labeling_twitter(): hasil_labeling.clear() with open("static/files/Data Preprocessing.csv", "r", encoding='utf-8') as csvfile: readCSV = csv.reader(csvfile, delimiter=',') next(readCSV) # Lewati header CSV with open('static/files/Data Labeling.csv', 'w', newline='', encoding='utf-8') as file: writer = csv.writer(file) # Header file hasil labeling writer.writerow(['Tanggal', 'Username', 'Tweet', 'Stemming', 'Translate', 'Label']) for row in readCSV: tanggal = row[0] username = row[1] tweet_asli = row[2] stemming = row[8] translated = row[9] # hasil translate try: analysis = TextBlob(translated) score = analysis.sentiment.polarity except Exception as e: score = 0.0 # Jika gagal, asumsikan netral # Penentuan label berdasarkan polaritas if score >= 0.6: label = 'Sangat Mendukung' elif 0.2 <= score < 0.6: label = 'Mendukung' elif -0.2 < score < 0.2: label = 'Netral' elif -0.6 < score <= -0.2: label = 'Tidak Mendukung' else: label = 'Sangat Tidak Mendukung' hasil = [tanggal, username, tweet_asli, stemming, translated, label] hasil_labeling.append(hasil) writer.writerow(hasil) flash('Labeling 5 Kelas Berhasil', 'labeling_data') #Klasifikasi # Membuat variabel df df = None df2 = None # menentukan akurasi 0 akurasi = 0 def proses_klasifikasi(): global df global df2 global akurasi tweet = [] y = [] # Baca data labeling with open("static/files/Data Labeling.csv", encoding='utf-8') as csvfile: readCSV = csv.reader(csvfile, delimiter=',') next(readCSV) # Lewati header for row in readCSV: tweet_text = row[4] # kolom Translate label = row[5] # kolom Label if tweet_text.lower() != "terjemahan gagal": # filter data gagal translate tweet.append(tweet_text) y.append(label) # TF-IDF vectorizer = TfidfVectorizer() x = vectorizer.fit_transform(tweet) # Split data training dan testing x_train, x_test, y_train, y_test = train_test_split( x, y, test_size=0.2, random_state=42 ) # Naive Bayes clf = MultinomialNB() clf.fit(x_train, y_train) predict = clf.predict(x_test) # Simpan classification report ke CSV report = classification_report(y_test, predict, output_dict=True) clsf_report = pd.DataFrame(report).transpose() clsf_report.to_csv('static/files/Data Klasifikasi.csv', index=True) # Simpan model dan vectorizer pickle.dump(vectorizer, open('static/files/vec.pkl', 'wb')) pickle.dump(x, open('static/files/tfidf.pkl', 'wb')) pickle.dump(clf, open('static/files/model.pkl', 'wb')) # Confusion Matrix unique_label = np.unique([y_test, predict]) cmtx = pd.DataFrame( confusion_matrix(y_test, predict, labels=unique_label), index=['pred:{:}'.format(x) for x in unique_label], columns=['true:{:}'.format(x) for x in unique_label] ) cmtx.to_csv('static/files/Data Confusion Matrix.csv', index=True) # Baca ulang hasil evaluasi df = pd.read_csv('static/files/Data Confusion Matrix.csv', sep=",") df.rename(columns={'Unnamed: 0': ''}, inplace=True) df2 = pd.read_csv('static/files/Data Klasifikasi.csv', sep=",") df2.rename(columns={'Unnamed: 0': ''}, inplace=True) # Hitung akurasi akurasi = round(accuracy_score(y_test, predict) * 100, 2) # Wordcloud kalimat = "".join(tweet) urllib.request.urlretrieve( "https://firebasestorage.googleapis.com/v0/b/sentimen-97d49.appspot.com/o/Circle-icon.png?alt=media&token=b9647ca7-dfdb-46cd-80a9-cfcaa45a1ee4", 'circle.png') mask = np.array(Image.open("circle.png")) wordcloud = WordCloud(width=1600, height=800, max_font_size=200, background_color='white', mask=mask) wordcloud.generate(kalimat) plt.figure(figsize=(12, 10)) plt.imshow(wordcloud, interpolation='bilinear') plt.axis("off") plt.savefig('static/files/wordcloud.png') flash('Klasifikasi Berhasil', 'klasifikasi_data') app = Flask(__name__) app.config['SECRET_KEY'] = 'farez' # Upload folder UPLOAD_FOLDER = 'static/files' ALLOWED_EXTENSION = set(['csv']) app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER def allowed_file(filename): return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSION @app.route('/') def index(): return render_template('index.html') @app.route('/preprocessing', methods=['GET', 'POST']) def preprocessing(): if request.method == 'POST': if request.form.get('upload') == 'Upload Data': hasil_preprocessing.clear() file = request.files['file'] if not allowed_file(file.filename): flash('Format file tidak diperbolehkan', 'upload_gagal') return render_template('preprocessing.html', value=hasil_preprocessing) if 'file' not in request.files: flash('File tidak boleh kosong', 'upload_gagal') return render_template('preprocessing.html', value=hasil_preprocessing) if file.filename == '': flash('File tidak boleh kosong', 'upload_gagal') return render_template('preprocessing.html', value=hasil_preprocessing) if file and allowed_file(file.filename): file.filename = "Data Scraping.csv" file.save(os.path.join(app.config['UPLOAD_FOLDER'], file.filename)) hasil_preprocessing.clear() flash('File Berhasil di upload', 'upload_berhasil') return render_template('preprocessing.html') if request.form.get('preprocess') == 'Preprocessing Data': preprocessing_twitter() return render_template('preprocessing.html', value=hasil_preprocessing) return render_template('preprocessing.html', value=hasil_preprocessing) @app.route('/labeling', methods=['GET', 'POST']) def labeling(): if request.method == 'POST': if request.form.get('upload') == 'Upload Data': hasil_labeling.clear() file = request.files['file'] if not allowed_file(file.filename): flash('Format file tidak diperbolehkan', 'upload_gagal') return render_template('labeling.html', value=hasil_labeling) if 'file' not in request.files: flash('File tidak boleh kosong', 'upload_gagal') return render_template('labeling.html', value=hasil_labeling) if file.filename == '': flash('File tidak boleh kosong', 'upload_gagal') return render_template('labeling.html', value=hasil_labeling) if file and allowed_file(file.filename): file.filename = "Data Preprocessing.csv" file.save(os.path.join(app.config['UPLOAD_FOLDER'], file.filename)) hasil_labeling.clear() flash('File Berhasil di upload', 'upload_berhasil') return render_template('labeling.html') if request.form.get('labeling') == 'Labeling Data': labeling_twitter() return render_template('labeling.html', value=hasil_labeling) return render_template('labeling.html', value=hasil_labeling) @app.route('/klasifikasi', methods=['GET', 'POST']) def klasifikasi(): if request.method == 'POST': if request.form.get('upload') == 'Upload Data': file = request.files['file'] if not allowed_file(file.filename): flash('Format file tidak diperbolehkan', 'upload_gagal') return render_template('klasifikasi.html') if 'file' not in request.files: flash('File tidak boleh kosong', 'upload_gagal') return render_template('klasifikasi.html',) if file.filename == '': flash('File tidak boleh kosong', 'upload_gagal') return render_template('klasifikasi.html') if file and allowed_file(file.filename): file.filename = "Data Labeling.csv" file.save(os.path.join(app.config['UPLOAD_FOLDER'], file.filename)) flash('File Berhasil di upload', 'upload_berhasil') return render_template('klasifikasi.html') if request.form.get('klasifikasi') == 'Klasifikasi Data': proses_klasifikasi() return render_template('klasifikasi.html', accuracy=akurasi, tables=[df.to_html(classes='table table-bordered', index=False, justify='left')], titles=df.columns.values, tables2=[df2.to_html(classes='table table-bordered', index=False, justify='left')], titles2=df2.columns.values) if akurasi == 0: return render_template('klasifikasi.html') else: return render_template('klasifikasi.html', accuracy=akurasi, tables=[df.to_html(classes='table table-bordered', index=False, justify='left')], titles=df.columns.values, tables2=[df2.to_html(classes='table table-bordered', index=False, justify='left')], titles2=df2.columns.values) @app.route('/visualisasi') def visualisasi(): return render_template('visualisasi.html') if __name__ == "__main__": app.run(debug=True)