Mufrida_Farah
/
MIF_E31231226
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update fitur

This commit is contained in:
MufridaFaraDiani27 2026-04-21 18:33:47 +07:00
parent bcf6326dee
commit d5f3f33a83
9 changed files with 456 additions and 315 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/Http/Controllers/AnalisisController.php
View File

@ -30,4 +30,4 @@ public function index(Request $request)
'wisataList' 'wisataList'
)); ));
} }
} }

99
app/Http/Controllers/DashboardController.php
View File

@ -12,89 +12,101 @@ public function index(Request $request)
$wisata = $request->wisata; $wisata = $request->wisata;
// ============================= // =============================
// TOTAL DATA // TOTAL ULASAN (DARI TABEL ULASAN)
// ============================= // =============================
$total = DB::table('hasil_analisis') $totalUlasan = DB::table('ulasan')
->when($wisata, function ($q) use ($wisata) { ->when($wisata && $wisata != 'Semua Destinasi', function ($q) use ($wisata) {
$q->where('wisata', $wisata); $q->where('wisata', $wisata);
}) })
->count(); ->count();
// ============================= // =============================
// HITUNG SENTIMEN // CEK ADA ANALISIS ATAU BELUM
// =============================
$hasAnalisis = DB::table('hasil_analisis')->count() > 0;
// =============================
// HITUNG SENTIMEN (DARI HASIL ANALISIS)
// ============================= // =============================
$positif = DB::table('hasil_analisis') $positif = DB::table('hasil_analisis')
->where('sentimen', 'Positif') ->where('sentimen', 'Positif')
->when($wisata, function ($q) use ($wisata) { ->when($wisata && $wisata != 'Semua Destinasi', function ($q) use ($wisata) {
$q->where('wisata', $wisata); $q->where('wisata', $wisata);
}) })
->count(); ->count();
$negatif = DB::table('hasil_analisis') $negatif = DB::table('hasil_analisis')
->where('sentimen', 'Negatif') ->where('sentimen', 'Negatif')
->when($wisata, function ($q) use ($wisata) { ->when($wisata && $wisata != 'Semua Destinasi', function ($q) use ($wisata) {
$q->where('wisata', $wisata); $q->where('wisata', $wisata);
}) })
->count(); ->count();
$netral = DB::table('hasil_analisis') $netral = DB::table('hasil_analisis')
->where('sentimen', 'Netral') ->where('sentimen', 'Netral')
->when($wisata, function ($q) use ($wisata) { ->when($wisata && $wisata != 'Semua Destinasi', function ($q) use ($wisata) {
$q->where('wisata', $wisata); $q->where('wisata', $wisata);
}) })
->count(); ->count();
// ============================= // =============================
// PERSENTASE // PERSENTASE SENTIMEN
// ============================= // =============================
$totalSentimen = $positif + $negatif + $netral;
$stats = [ $stats = [
'total' => $total, 'total' => $totalUlasan,
'positif_persen' => $total ? round(($positif / $total) * 100) : 0, 'positif_persen' => $totalSentimen > 0 ? round(($positif / $totalSentimen) * 100) : 0,
'negatif_persen' => $total ? round(($negatif / $total) * 100) : 0, 'negatif_persen' => $totalSentimen > 0 ? round(($negatif / $totalSentimen) * 100) : 0,
'netral_persen' => $total ? round(($netral / $total) * 100) : 0, 'netral_persen' => $totalSentimen > 0 ? round(($netral / $totalSentimen) * 100) : 0,
]; ];
// ============================= // =============================
// GRAFIK PIE (Distribusi Sentimen) // GRAFIK PIE (Distribusi Sentimen)
// ============================= // =============================
$chartSentimen = DB::table('hasil_analisis') $chartSentimen = $hasAnalisis
->select('sentimen', DB::raw('count(*) as total')) ? DB::table('hasil_analisis')
->when($wisata, function ($q) use ($wisata) { ->select('sentimen', DB::raw('count(*) as total'))
$q->where('wisata', $wisata); ->when($wisata && $wisata != 'Semua Destinasi', function ($q) use ($wisata) {
}) $q->where('wisata', $wisata);
->groupBy('sentimen') })
->get(); ->groupBy('sentimen')
->orderByRaw("FIELD(sentimen, 'Positif', 'Negatif', 'Netral')") // 🔥 INI KUNCINYA
->get()
: collect();
// ============================= // =============================
// GRAFIK BAR (per destinasi) // GRAFIK BAR (Sentimen per Destinasi)
// ============================= // =============================
$chartDestinasi = DB::table('hasil_analisis') $chartDestinasi = $hasAnalisis
->select( ? DB::table('hasil_analisis')
'wisata', ->select(
DB::raw("SUM(CASE WHEN sentimen = 'Positif' THEN 1 ELSE 0 END) as positif"), 'wisata',
DB::raw("SUM(CASE WHEN sentimen = 'Negatif' THEN 1 ELSE 0 END) as negatif"), DB::raw("SUM(CASE WHEN sentimen = 'Positif' THEN 1 ELSE 0 END) as positif"),
DB::raw("SUM(CASE WHEN sentimen = 'Netral' THEN 1 ELSE 0 END) as netral") DB::raw("SUM(CASE WHEN sentimen = 'Negatif' THEN 1 ELSE 0 END) as negatif"),
) DB::raw("SUM(CASE WHEN sentimen = 'Netral' THEN 1 ELSE 0 END) as netral")
)
->when($wisata && $wisata != 'Semua Destinasi', function ($q) use ($wisata) {
$q->where('wisata', $wisata);
})
->groupBy('wisata')
->get()
: collect();
// =============================
// TOTAL DATA PER DESTINASI (DARI ULASAN)
// =============================
$totalPerWisata = DB::table('ulasan')
->select('wisata', DB::raw('count(*) as total'))
->groupBy('wisata') ->groupBy('wisata')
->get(); ->get();
// ============================= // =============================
// TOTAL DATA PER DESTINASI // WORD CLOUD (DARI PREPROCESSING)
// ============================= // =============================
$totalPerWisata = DB::table('hasil_analisis') $texts = DB::table('preprocessing_data')->pluck('stemming')->toArray();
->select('wisata', DB::raw('count(*) as total')) $allText = count($texts) > 0 ? implode(' ', $texts) : '';
->groupBy('wisata')
->get()
->toArray();
// =============================
// WORD CLOUD
// =============================
$texts = DB::table('preprocessing_data')
->pluck('stemming')
->toArray(); // 🔥 WAJIB
$allText = implode(' ', $texts);
// ============================= // =============================
// LAST UPDATE // LAST UPDATE
@ -109,7 +121,8 @@ public function index(Request $request)
'chartDestinasi', 'chartDestinasi',
'totalPerWisata', 'totalPerWisata',
'allText', 'allText',
'lastUpdate' 'lastUpdate',
'hasAnalisis'
)); ));
} }
} }

2
app/Http/Controllers/RekomendasiController.php
View File

@ -12,7 +12,7 @@ public function index(Request $request)
// dropdown destinasi unik // dropdown destinasi unik
$destinasiList = HasilAnalisis::select('wisata') $destinasiList = HasilAnalisis::select('wisata')
->distinct() ->distinct()
->pluck('scraping_pipeline.py'); ->pluck('wisata');
// query utama // query utama
$query = HasilAnalisis::query(); $query = HasilAnalisis::query();

18
app/Http/Controllers/UlasanController.php
View File

@ -25,7 +25,7 @@ public function index(Request $request)
$mentah = $query->latest()->paginate(50)->withQueryString(); $mentah = $query->latest()->paginate(50)->withQueryString();
// preprocessing // preprocessing
$preprocessing = PreprocessingData::latest()->paginate(50); $preprocessing = PreprocessingData::latest()->paginate(30, ['*'], 'pre_page');
return view('ulasan.index', compact('mentah', 'preprocessing')); return view('ulasan.index', compact('mentah', 'preprocessing'));
} }
@ -41,14 +41,14 @@ public function ambilData()
->with('success','Data berhasil diambil'); ->with('success','Data berhasil diambil');
} }
public function analisisData() public function analisisData()
{ {
$pythonPath = '"C:\\Users\\Mufrida Farah\\AppData\\Local\\Programs\\Python\\Python312\\python.exe"'; $pythonPath = '"C:\\Users\\Mufrida Farah\\AppData\\Local\\Programs\\Python\\Python312\\python.exe"';
$scriptPath = '"C:\\laragon\\www\\Sentara\\scraper\\preprocessing.py"'; $scriptPath = '"C:\\laragon\\www\\Sentara\\scraper\\preprocessing.py"';
shell_exec($pythonPath . " " . $scriptPath . " 2>&1"); $output = shell_exec($pythonPath . " " . $scriptPath . " 2>&1");
return redirect()->route('ulasan.index') return redirect()->route('ulasan.index')
->with('success','Analisis berhasil'); ->with('success','Analisis berhasil');
} }
} }

43
resources/views/analisis/index.blade.php
View File

@ -98,9 +98,46 @@ class="bg-white border border-gray-300 text-sm rounded-lg px-4 py-2 shadow-sm
</div> </div>
<!-- PAGINATION --> <!-- PAGINATION -->
<div class="mt-6"> <div class="text-sm text-gray-500 mt-4 text-center">
{{ $hasil->links() }} Showing {{ $hasil->firstItem() }} to {{ $hasil->lastItem() }}
</div> of {{ $hasil->total() }} results
</div>
<div class="flex justify-center mt-3 items-center gap-2 text-sm">
{{-- PREV --}}
@if ($hasil->onFirstPage())
<span class="text-gray-400">‹</span>
@else
<a href="{{ $hasil->previousPageUrl() }}">‹</a>
@endif
@php
$start = max($hasil->currentPage() - 2, 1);
$end = min($hasil->currentPage() + 2, $hasil->lastPage());
@endphp
{{-- NUMBER --}}
@for ($i = $start; $i <= $end; $i++)
@if ($i == $hasil->currentPage())
<span class="px-3 py-1 bg-blue-600 text-white rounded-full">
{{ $i }}
</span>
@else
<a href="{{ $hasil->url($i) }}" class="px-2 hover:underline">
{{ $i }}
</a>
@endif
@endfor
{{-- NEXT --}}
@if ($hasil->hasMorePages())
<a href="{{ $hasil->nextPageUrl() }}">›</a>
@else
<span class="text-gray-400">›</span>
@endif
</div>
</div> </div>

202
resources/views/dashboard.blade.php
View File

@ -4,37 +4,12 @@
<div class="max-w-7xl mx-auto"> <div class="max-w-7xl mx-auto">
{{-- Header --}} {{-- ALERT --}}
<div class="flex justify-between items-start mb-8"> @if(!$hasAnalisis)
<div> <div class="bg-yellow-100 border-l-4 border-yellow-500 text-yellow-700 p-4 mb-6 rounded">
<h2 class="text-2xl font-semibold text-gray-800"> Data ulasan sudah tersedia, tetapi belum dilakukan analisis sentimen.
Dashboard </div>
</h2> @endif
<!-- <p class="text-sm text-gray-500 mt-1">
Terakhir diperbarui: 27 Februari 2026, 22:45
</p>
<p class="text-xs text-gray-400">
Periode Data: Februari 2026
</p> -->
</div>
<!-- Filter Destinasi -->
<form method="GET" action="" class="ml-4">
<select name="destinasi"
class="bg-white border border-gray-300 text-sm rounded-lg px-4 py-2 shadow-sm
focus:ring-2 focus:ring-blue-400 focus:outline-none">
<option value="">Semua Destinasi</option>
<option value="papuma">Pantai Papuma</option>
<option value="watuulo">Pantai Watu Ulo</option>
<option value="teluklove">Teluk Love (Payangan)</option>
<option value="gununggambir">Kebun Teh Gunung Gambir </option>
</select>
</form>
</div>
{{-- Statistik Cards --}} {{-- Statistik Cards --}}
<div class="grid grid-cols-1 md:grid-cols-4 gap-6 mb-8"> <div class="grid grid-cols-1 md:grid-cols-4 gap-6 mb-8">
@ -69,78 +44,133 @@ class="bg-white border border-gray-300 text-sm rounded-lg px-4 py-2 shadow-sm
</div> </div>
{{-- Chart Section --}} {{-- Chart Section --}}
<div class="grid grid-cols-1 lg:grid-cols-2 gap-6 mb-8"> <div class="grid grid-cols-1 lg:grid-cols-2 gap-6 mb-8">
<!-- Pie Chart --> <!-- Pie Chart -->
<div class="bg-white rounded-xl shadow p-6"> <div class="bg-white rounded-xl shadow p-6">
<h3 class="font-semibold mb-4 text-gray-700"> <h3 class="font-semibold mb-4 text-gray-700">Distribusi Sentimen</h3>
Distribusi Sentimen <div style="height:300px;">
</h3> <canvas id="pieChart"></canvas>
<div class="h-[280px] flex justify-center items-center"> </div>
<canvas id="pieChart"></canvas>
</div> </div>
<!-- Bar Chart -->
<div class="bg-white rounded-xl shadow p-6">
<h3 class="font-semibold mb-4 text-gray-700">Sentimen per Destinasi</h3>
<div style="height:300px;">
<canvas id="barChart"></canvas>
</div>
</div>
</div> </div>
<!-- Bar Chart --> {{-- Section Bawah --}}
<div class="bg-white rounded-xl shadow p-6"> <div class="grid grid-cols-1 lg:grid-cols-2 gap-6">
<h3 class="font-semibold mb-4 text-gray-700">
Sentimen per Destinasi
</h3>
<div class="h-[280px]">
<canvas id="barChart"></canvas>
</div>
</div>
</div> {{-- TUTUP GRID CHART --}} <!-- Total Data per Destinasi -->
<div class="bg-white rounded-xl shadow p-6 h-[320px]">
<h3 class="font-semibold mb-6 text-gray-700">
Total Data per Destinasi
</h3>
<div class="space-y-4 text-sm text-gray-600">
{{-- Section Bawah --}} @foreach($totalPerWisata as $item)
<div class="grid grid-cols-1 lg:grid-cols-2 gap-6"> <div class="flex justify-between border-b pb-3">
<span>{{ $item->wisata }}</span>
<!-- Total Data per Destinasi --> <span class="font-semibold text-gray-800">
<div class="bg-white rounded-xl shadow p-6 h-[320px]"> {{ $item->total }} ulasan
<h3 class="font-semibold mb-6 text-gray-700"> </span>
Total Data per Destinasi </div>
</h3> @endforeach
<div class="space-y-4 text-sm text-gray-600">
<div class="flex justify-between border-b pb-3">
<span>Pantai Papuma</span>
<span class="font-semibold text-gray-800">120 ulasan</span>
</div>
<div class="flex justify-between border-b pb-3">
<span>Pantai Watu Ulo</span>
<span class="font-semibold text-gray-800">85 ulasan</span>
</div>
<div class="flex justify-between border-b pb-3">
<span>Teluk Love (Payangan)</span>
<span class="font-semibold text-gray-800">95 ulasan</span>
</div>
<div class="flex justify-between">
<span>Kebun Teh Gunung Gambir</span>
<span class="font-semibold text-gray-800">60 ulasan</span>
</div> </div>
</div> </div>
</div>
<!-- WORD CLOUD --> <!-- WORD CLOUD -->
<div class="bg-white rounded-xl shadow p-6 h-[320px]"> <div class="bg-white rounded-xl shadow p-6 h-[320px]">
<h3 class="font-semibold mb-6 text-gray-700"> <h3 class="font-semibold mb-6 text-gray-700">
Word Cloud Word Cloud
</h3> </h3>
<div id="wordCloud" class="w-full h-[240px] overflow-hidden"></div>
</div>
<div id="wordCloud" class="w-full h-[240px]"></div>
</div> </div>
</div> </div>
</div> {{-- ========================= --}}
{{-- SCRIPT SECTION --}}
{{-- ========================= --}}
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/wordcloud2.js/1.1.2/wordcloud2.min.js"></script>
</div> <script>
</div> document.addEventListener("DOMContentLoaded", function () {
// =============================
// PIE CHART
// =============================
const pieData = JSON.parse('{!! json_encode($chartSentimen) !!}');
const pieLabels = pieData.map(item => item.sentimen);
const pieValues = pieData.map(item => item.total);
new Chart(document.getElementById('pieChart'), {
type: 'pie',
data: {
labels: pieLabels,
datasets: [{
data: pieValues
}]
}
});
// =============================
// BAR CHART
// =============================
const barData = JSON.parse('{!! json_encode($chartDestinasi) !!}');
const labelsBar = barData.map(item => item.wisata);
const positif = barData.map(item => item.positif);
const negatif = barData.map(item => item.negatif);
const netral = barData.map(item => item.netral);
new Chart(document.getElementById('barChart'), {
type: 'bar',
data: {
labels: labelsBar,
datasets: [
{ label: 'Positif', data: positif },
{ label: 'Negatif', data: negatif },
{ label: 'Netral', data: netral }
]
}
});
// =============================
// WORD CLOUD
// =============================
const text = `{!! $allText !!}`;
if(text.length > 0){
const words = text.split(" ");
const wordCount = {};
words.forEach(word => {
if(word.length > 3){
wordCount[word] = (wordCount[word] || 0) + 1;
}
});
const list = Object.entries(wordCount);
WordCloud(document.getElementById('wordCloud'), {
list: list
});
}
});
</script>
@endsection @endsection

155
resources/views/ulasan/index.blade.php
View File

@ -12,7 +12,7 @@
<option value="">Semua Destinasi</option> <option value="">Semua Destinasi</option>
<option value="Pantai Papuma" {{ request('wisata')=='Pantai Papuma'?'selected':'' }}>Pantai Papuma</option> <option value="Pantai Papuma" {{ request('wisata')=='Pantai Papuma'?'selected':'' }}>Pantai Papuma</option>
<option value="Pantai Watu Ulo" {{ request('wisata')=='Pantai Watu Ulo'?'selected':'' }}>Pantai Watu Ulo</option> <option value="Pantai Watu Ulo" {{ request('wisata')=='Pantai Watu Ulo'?'selected':'' }}>Pantai Watu Ulo</option>
<option value="Teluk Love (Payangan)" {{ request('wisata')=='Teluk Love (Payangan)'?'selected':'' }}>Teluk Love</option> <option value="Teluk Love" {{ request('wisata')=='Teluk Love'?'selected':'' }}>Teluk Love</option>
<option value="Kebun Teh Gunung Gambir" {{ request('wisata')=='Kebun Teh Gunung Gambir'?'selected':'' }}>Kebun Teh Gunung Gambir</option> <option value="Kebun Teh Gunung Gambir" {{ request('wisata')=='Kebun Teh Gunung Gambir'?'selected':'' }}>Kebun Teh Gunung Gambir</option>
</select> </select>
</div> </div>
@ -155,82 +155,80 @@
<div class="bg-white rounded-xl shadow p-6"> <div class="bg-white rounded-xl shadow p-6">
<h3 class="font-semibold mb-4">Hasil Preprocessing</h3> <h3 class="font-semibold mb-4">Hasil Preprocessing</h3>
<div class="max-h-[300px] overflow-y-auto"> <div id="preprocessingTabel">
<table class="w-full text-sm">
<thead class="bg-gray-100 sticky top-0">
<tr>
<th class="px-4 py-2">Wisata</th>
<th class="px-4 py-2">Ulasan Mentah</th>
<th class="px-4 py-2">Ulasan Bersih</th>
</tr>
</thead>
<tbody>
@forelse($preprocessing as $row)
<tr class="border-b">
<td class="px-4 py-2">{{ $row->wisata }}</td>
<td class="px-4 py-2">{{ $row->ulasan_asli }}</td>
<td class="px-4 py-2">{{ $row->stemming }}</td>
</tr>
@empty
<tr>
<td colspan="3" class="text-center py-4 text-gray-400">
Belum ada data
</td>
</tr>
<div class="flex justify-center mt-4 text-sm">
<!-- INFO -->
<div class="text-sm text-gray-500 mt-4 text-center">
Showing {{ $preprocessing->firstItem() }} to {{ $preprocessing->lastItem() }}
of {{ $preprocessing->total() }} results
</div>
<!-- PAGINATION --> <!-- TABLE -->
<div class="flex justify-center mt-6 items-center gap-2 text-sm"> <div class="max-h-[300px] overflow-y-auto">
<table class="w-full text-sm">
<thead class="bg-gray-100 sticky top-0">
<tr>
<th class="px-4 py-2">Wisata</th>
<th class="px-4 py-2">Ulasan Mentah</th>
<th class="px-4 py-2">Ulasan Bersih</th>
</tr>
</thead>
<tbody>
@forelse($preprocessing as $row)
<tr class="border-b">
<td class="px-4 py-2">{{ $row->wisata }}</td>
<td class="px-4 py-2">{{ $row->ulasan_asli }}</td>
<td class="px-4 py-2">{{ $row->stemming }}</td>
</tr>
@empty
<tr>
<td colspan="3" class="text-center py-4 text-gray-400">
Belum ada data
</td>
</tr>
@endforelse
</tbody>
</table>
</div>
{{-- FIRST --}} <!-- INFO -->
@if ($preprocessing->currentPage() > 1) <div class="text-sm text-gray-500 mt-4 text-center">
<a href="{{ $preprocessing->url(1) }}">«</a> Showing {{ $preprocessing->firstItem() }} to {{ $preprocessing->lastItem() }}
@endif of {{ $preprocessing->total() }} results
</div>
{{-- PREV --}} <!-- PAGINATION -->
@if ($preprocessing->onFirstPage()) <div class="flex justify-center mt-4 items-center gap-2 text-sm">
<span class="text-gray-400">‹</span>
@else
<a href="{{ $preprocessing->previousPageUrl() }}">‹</a>
@endif
@php {{-- PREV --}}
$start = max($preprocessing->currentPage() - 2, 1); @if ($preprocessing->onFirstPage())
$end = min($preprocessing->currentPage() + 2, $preprocessing->lastPage()); <span class="text-gray-400">‹</span>
@endphp @else
<a href="{{ $preprocessing->previousPageUrl() }}">‹</a>
@endif
{{-- NUMBER DINAMIS --}} @php
@for ($i = $start; $i <= $end; $i++) $start = max($preprocessing->currentPage() - 2, 1);
@if ($i == $preprocessing->currentPage()) $end = min($preprocessing->currentPage() + 2, $preprocessing->lastPage());
<span class="px-3 py-1 bg-blue-600 text-white rounded-full">{{ $i }}</span> @endphp
@else
<a href="{{ $preprocessing->url($i) }}" class="px-3 py-1 hover:underline">
{{ $i }}
</a>
@endif
@endfor
{{-- NEXT --}} {{-- NUMBER --}}
@if ($preprocessing->hasMorePages()) @for ($i = $start; $i <= $end; $i++)
<a href="{{ $preprocessing->nextPageUrl() }}">›</a> @if ($i == $preprocessing->currentPage())
@else <span class="px-3 py-1 bg-blue-600 text-white rounded-full">
<span class="text-gray-400">›</span> {{ $i }}
@endif </span>
@else
<a href="{{ $preprocessing->url($i) }}" class="px-3 py-1 hover:underline">
{{ $i }}
</a>
@endif
@endfor
{{-- LAST --}} {{-- NEXT --}}
@if ($preprocessing->currentPage() < $preprocessing->lastPage()) @if ($preprocessing->hasMorePages())
<a href="{{ $preprocessing->url($preprocessing->lastPage()) }}">»</a> <a href="{{ $preprocessing->nextPageUrl() }}">›</a>
@endif @else
<span class="text-gray-400">›</span>
@endif
</div> </div>
</div> </div>
<script> <script>
document.getElementById("filterWisata").addEventListener("change", function () { document.getElementById("filterWisata").addEventListener("change", function () {
let wisata = this.value; let wisata = this.value;
@ -244,12 +242,31 @@
}); });
}); });
</script> </script>
@endforelse
</tbody> </tbody>
</table> </table>
</div> </div>
</div> </div>
</div>
</div> </div>
<script>
document.addEventListener("click", function(e) {
if(e.target.closest("#preprocessingTable a")) {
e.preventDefault();
let url = e.target.closest("a").getAttribute("href");
fetch(url)
.then(res => res.text())
.then(html => {
let doc = new DOMParser().parseFromString(html, 'text/html');
let newContent = doc.querySelector("#preprocessingTable").innerHTML;
document.querySelector("#preprocessingTable").innerHTML = newContent;
});
}
});
</script>
@endsection @endsection

247
scraper/preprocessing.py
View File

@ -1,8 +1,20 @@
import pandas as pd
import mysql.connector import mysql.connector
import re import re
import math import string
from collections import Counter import matplotlib.pyplot as plt
from sklearn.utils import resample
from Sastrawi.Stemmer.StemmerFactory import StemmerFactory from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
from Sastrawi.StopWordRemover.StopWordRemoverFactory import StopWordRemoverFactory
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import accuracy_score, classification_report
from wordcloud import WordCloud
# ============================= # =============================
# DATABASE # DATABASE
@ -14,130 +26,161 @@ conn = mysql.connector.connect(
password="" password=""
) )
cursor = conn.cursor() query = "SELECT wisata, ulasan, rating FROM ulasan"
df = pd.read_sql(query, conn)
print("Total data:", len(df))
# ============================= # =============================
# STEMMER # STEMMER & STOPWORD
# ============================= # =============================
factory = StemmerFactory() factory = StemmerFactory()
stemmer = factory.create_stemmer() stemmer = factory.create_stemmer()
# ============================= factory_stop = StopWordRemoverFactory()
# DATA TRAINING stopword = factory_stop.create_stop_word_remover()
# =============================
data_train = [
("bagus bersih indah nyaman sejuk", "positif"),
("indah nyaman segar bagus", "positif"),
("kotor jelek rusak mahal sempit", "negatif"),
("jalan rusak sempit mahal", "negatif")
]
# ============================= # =============================
# TRAIN NAIVE BAYES # PREPROCESSING
# ============================= # =============================
def train_nb(data): # kamus slang
word_counts = {"positif": Counter(), "negatif": Counter()} slang_dict = {
class_counts = {"positif": 0, "negatif": 0} "gk": "tidak",
total_words = {"positif": 0, "negatif": 0} "ga": "tidak",
"bgt": "banget",
"tp": "tapi",
"dr": "dari",
"yg": "yang"
}
for text, label in data: def normalize_slang(text):
words = text.split()
class_counts[label] += 1
word_counts[label].update(words)
total_words[label] += len(words)
return word_counts, class_counts, total_words
word_counts, class_counts, total_words = train_nb(data_train)
# =============================
# PREDICT
# =============================
def predict_nb(text):
words = text.split() words = text.split()
return ' '.join([slang_dict.get(w, w) for w in words])
vocab = set(word_counts["positif"]).union(set(word_counts["negatif"]))
log_pos = math.log(class_counts["positif"] / sum(class_counts.values())) def preprocess(text):
log_neg = math.log(class_counts["negatif"] / sum(class_counts.values())) text = str(text)
text = text.lower()
for word in words: # cleaning
prob_pos = (word_counts["positif"][word] + 1) / (total_words["positif"] + len(vocab)) text = re.sub(r'[^a-zA-Z\s]', '', text)
prob_neg = (word_counts["negatif"][word] + 1) / (total_words["negatif"] + len(vocab))
log_pos += math.log(prob_pos) # 🔥 TAMBAHKAN DI SINI
log_neg += math.log(prob_neg) text = normalize_slang(text)
if log_pos > log_neg: # stopword
return "Positif", log_pos text = stopword.remove(text)
# stemming
text = stemmer.stem(text)
return text
def preprocess(text):
text = str(text)
text = text.lower()
text = re.sub(r'[^a-zA-Z\s]', '', text)
text = stopword.remove(text)
text = stemmer.stem(text)
return text
df['clean_text'] = df['ulasan'].apply(preprocess)
# =============================
# LABELING
# =============================
def label_sentimen(rating):
if rating >= 4:
return "positif"
elif rating == 3:
return "negatif" # 🔥 ubah ini
else: else:
return "Negatif", log_neg return "negatif"
df['label'] = df['rating'].apply(label_sentimen)
print("\nDistribusi Label:")
print(df['label'].value_counts())
# ============================= # =============================
# HAPUS DATA LAMA (PENTING!) # SPLIT DATA
# ============================= # =============================
X_train, X_test, y_train, y_test = train_test_split(
df['clean_text'],
df['label'],
test_size=0.2,
random_state=42
)
# =============================
# TF-IDF
# =============================
vectorizer = TfidfVectorizer()
X_train_tfidf = vectorizer.fit_transform(X_train)
X_test_tfidf = vectorizer.transform(X_test)
# =============================
# MODEL NAIVE BAYES
# =============================
model = MultinomialNB()
model.fit(X_train_tfidf, y_train)
# =============================
# EVALUASI
# =============================
y_pred = model.predict(X_test_tfidf)
print("\nAccuracy:", accuracy_score(y_test, y_pred))
print("\nClassification Report:\n", classification_report(y_test, y_pred))
# =============================
# PREDIKSI SEMUA DATA
# =============================
print("\nMulai prediksi semua data...")
X_all_tfidf = vectorizer.transform(df['clean_text'])
df['prediksi'] = model.predict(X_all_tfidf)
# =============================
# SIMPAN KE DATABASE
# =============================
cursor = conn.cursor()
cursor.execute("DELETE FROM preprocessing_data") cursor.execute("DELETE FROM preprocessing_data")
cursor.execute("DELETE FROM hasil_analisis") cursor.execute("DELETE FROM hasil_analisis")
# ============================= for i, row in df.iterrows():
# AMBIL DATA try:
# ============================= # preprocessing
cursor.execute("SELECT wisata, ulasan FROM ulasan") cursor.execute("""
data = cursor.fetchall() INSERT INTO preprocessing_data
(wisata, ulasan_asli, cleaning, tokenizing, stemming, final_text)
print("Total ulasan:", len(data)) VALUES (%s, %s, %s, %s, %s, %s)
""", (
# ============================= row['wisata'],
# PROSES row['ulasan'],
# ============================= row['clean_text'],
for row in data: row['clean_text'],
row['clean_text'],
wisata = row[0] row['clean_text']
text = row[1]
if not text:
continue
# CLEANING
text_clean = re.sub(r'[^a-zA-Z\s]', '', text)
text_clean = text_clean.lower()
# TOKENIZING
tokens = text_clean.split()
text_token = ' '.join(tokens)
# STEMMING
text_stem = stemmer.stem(text_clean)
# PREDIKSI
sentimen, probabilitas = predict_nb(text_stem)
# INSERT PREPROCESSING
cursor.execute("""
INSERT INTO preprocessing_data
(wisata, ulasan_asli, cleaning, tokenizing, stemming)
VALUES (%s,%s,%s,%s,%s)
""",
(
wisata,
text,
text_clean,
text_token,
text_stem
)) ))
# INSERT HASIL ANALISIS # hasil analisis
cursor.execute(""" cursor.execute("""
INSERT INTO hasil_analisis INSERT INTO hasil_analisis
(wisata, ulasan_terolah, hasil_preprocessing, sentimen, probabilitas) (wisata, ulasan_terolah, hasil_preprocessing, sentimen, probabilitas)
VALUES (%s,%s,%s,%s,%s) VALUES (%s, %s, %s, %s, %s)
""", (wisata, text, text_stem, sentimen, probabilitas)) """, (
row['wisata'],
row['clean_text'],
row['clean_text'],
row['prediksi'],
0.0
))
except Exception as e:
print("Error insert:", e)
# =============================
# COMMIT SEKALI SAJA
# =============================
conn.commit() conn.commit()
print(" Data berhasil disimpan")
print(" Preprocessing & Analisis Sentimen selesai")
conn.close()

3
scraper/scraping_pipeline.py
View File

@ -17,7 +17,8 @@ cursor = conn.cursor()
# CHROME # CHROME
options = Options() options = Options()
options.add_argument("--start-maximized") options.add_argument("--start-maximized")
options.add_argument('--lang=id')
Options.add_argument("accept-language-id-ID,id")
driver = webdriver.Chrome(options=options) driver = webdriver.Chrome(options=options)
destinations = { destinations = {