From dc393edbbaafc53d59696687cd5fd7cc0e984ede Mon Sep 17 00:00:00 2001 From: MufridaFaraDiani27 Date: Sat, 4 Apr 2026 12:44:55 +0700 Subject: [PATCH] update fitur --- app/Http/Controllers/AnalisisController.php | 10 +- app/Http/Controllers/DashboardController.php | 114 +++++- .../Controllers/RekomendasiController.php | 6 +- app/Http/Controllers/UlasanController.php | 73 ++-- app/Models/HasilAnalisis.php | 2 +- app/Models/HasilSentimen.php | 2 +- app/Models/Ulasan.php | 2 +- ...2_11_071732_create_ulasan_mentah_table.php | 2 +- ..._16_172121_create_hasil_analisis_table.php | 2 +- resources/css/app.css | 7 + resources/views/analisis/index.blade.php | 10 +- resources/views/dashboard.blade.php | 254 +++++++------- resources/views/ulasan/index.blade.php | 328 ++++++++++++------ scraper/prepocessing.py | 70 ---- scraper/preprocessing.py | 143 ++++++++ scraper/scraping_pipeline.py | 237 +++++-------- 16 files changed, 734 insertions(+), 528 deletions(-) delete mode 100644 scraper/prepocessing.py create mode 100644 scraper/preprocessing.py diff --git a/app/Http/Controllers/AnalisisController.php b/app/Http/Controllers/AnalisisController.php index d2ffcfd..04bcf50 100644 --- a/app/Http/Controllers/AnalisisController.php +++ b/app/Http/Controllers/AnalisisController.php @@ -10,16 +10,16 @@ class AnalisisController extends Controller public function index(Request $request) { // Dropdown list destinasi unik - $destinasiList = HasilAnalisis::select('nama_wisata') + $wisataList = HasilAnalisis::select('wisata') ->distinct() - ->pluck('nama_wisata'); + ->pluck('wisata'); // Query utama tabel $query = HasilAnalisis::query(); // Filter kalau user pilih destinasi tertentu - if ($request->filled('destinasi')) { - $query->where('nama_wisata', $request->destinasi); + if ($request->filled('wisata')) { + $query->where('wisata', $request->wisata); } // Data hasil analisis (pakai paginate biar rapi) @@ -27,7 +27,7 @@ public function index(Request $request) return view('analisis.index', compact( 'hasil', - 'destinasiList' + 'wisataList' )); } } diff --git a/app/Http/Controllers/DashboardController.php b/app/Http/Controllers/DashboardController.php index 54a71d4..1f90b08 100644 --- a/app/Http/Controllers/DashboardController.php +++ b/app/Http/Controllers/DashboardController.php @@ -3,19 +3,113 @@ namespace App\Http\Controllers; use Illuminate\Http\Request; +use Illuminate\Support\Facades\DB; class DashboardController extends Controller { - public function index() + public function index(Request $request) { - $stats = [ - 'total' => 520, - 'positif_persen' => 68, - 'negatif_persen' => 20, - 'netral_persen' => 12, - ]; - $lastUpdate = '27 Februari 2026, 22:45'; + $wisata = $request->wisata; - return view('dashboard', compact('stats', 'lastUpdate')); + // ============================= + // TOTAL DATA + // ============================= + $total = DB::table('hasil_analisis') + ->when($wisata, function ($q) use ($wisata) { + $q->where('wisata', $wisata); + }) + ->count(); + + // ============================= + // HITUNG SENTIMEN + // ============================= + $positif = DB::table('hasil_analisis') + ->where('sentimen', 'Positif') + ->when($wisata, function ($q) use ($wisata) { + $q->where('wisata', $wisata); + }) + ->count(); + + $negatif = DB::table('hasil_analisis') + ->where('sentimen', 'Negatif') + ->when($wisata, function ($q) use ($wisata) { + $q->where('wisata', $wisata); + }) + ->count(); + + $netral = DB::table('hasil_analisis') + ->where('sentimen', 'Netral') + ->when($wisata, function ($q) use ($wisata) { + $q->where('wisata', $wisata); + }) + ->count(); + + // ============================= + // PERSENTASE + // ============================= + $stats = [ + 'total' => $total, + 'positif_persen' => $total ? round(($positif / $total) * 100) : 0, + 'negatif_persen' => $total ? round(($negatif / $total) * 100) : 0, + 'netral_persen' => $total ? round(($netral / $total) * 100) : 0, + ]; + + // ============================= + // GRAFIK PIE (Distribusi Sentimen) + // ============================= + $chartSentimen = DB::table('hasil_analisis') + ->select('sentimen', DB::raw('count(*) as total')) + ->when($wisata, function ($q) use ($wisata) { + $q->where('wisata', $wisata); + }) + ->groupBy('sentimen') + ->get(); + + // ============================= + // GRAFIK BAR (per destinasi) + // ============================= + $chartDestinasi = DB::table('hasil_analisis') + ->select( + 'wisata', + DB::raw("SUM(CASE WHEN sentimen = 'Positif' THEN 1 ELSE 0 END) as positif"), + DB::raw("SUM(CASE WHEN sentimen = 'Negatif' THEN 1 ELSE 0 END) as negatif"), + DB::raw("SUM(CASE WHEN sentimen = 'Netral' THEN 1 ELSE 0 END) as netral") + ) + ->groupBy('wisata') + ->get(); + + // ============================= + // TOTAL DATA PER DESTINASI + // ============================= + $totalPerWisata = DB::table('hasil_analisis') + ->select('wisata', DB::raw('count(*) as total')) + ->groupBy('wisata') + ->get() + ->toArray(); + + // ============================= + // WORD CLOUD + // ============================= + $texts = DB::table('preprocessing_data') + ->pluck('stemming') + ->toArray(); // 🔥 WAJIB + + $allText = implode(' ', $texts); + + // ============================= + // LAST UPDATE + // ============================= + $lastUpdate = DB::table('hasil_analisis') + ->latest('created_at') + ->value('created_at'); + + return view('dashboard', compact( + 'stats', + 'chartSentimen', + 'chartDestinasi', + 'totalPerWisata', + 'allText', + 'lastUpdate' + )); } -} +} \ No newline at end of file diff --git a/app/Http/Controllers/RekomendasiController.php b/app/Http/Controllers/RekomendasiController.php index e683945..69e9c95 100644 --- a/app/Http/Controllers/RekomendasiController.php +++ b/app/Http/Controllers/RekomendasiController.php @@ -10,16 +10,16 @@ class RekomendasiController extends Controller public function index(Request $request) { // dropdown destinasi unik - $destinasiList = HasilAnalisis::select('nama_wisata') + $destinasiList = HasilAnalisis::select('wisata') ->distinct() - ->pluck('nama_wisata'); + ->pluck('scraping_pipeline.py'); // query utama $query = HasilAnalisis::query(); // filter destinasi jika dipilih if ($request->filled('destinasi')) { - $query->where('nama_wisata', $request->destinasi); + $query->where('wisata', $request->destinasi); } // ambil ulasan negatif diff --git a/app/Http/Controllers/UlasanController.php b/app/Http/Controllers/UlasanController.php index d79f7ee..35eeb8a 100644 --- a/app/Http/Controllers/UlasanController.php +++ b/app/Http/Controllers/UlasanController.php @@ -4,58 +4,51 @@ use Illuminate\Http\Request; use App\Models\Ulasan; -// use App\Models\HasilSentimen; use App\Models\PreprocessingData; - class UlasanController extends Controller { - // public function index() - // { - // $mentah = Ulasan::where('is_processed', 0)->get(); - // // $hasil = HasilSentimen::latest()->get(); + public function index(Request $request) + { + $wisata = $request->wisata; - // // return view('ulasan.index', compact('mentah', 'hasil')); - // } - -public function index() -{ - // Data ulasan mentah (belum diproses) - $mentah = Ulasan::all(); - - // Data hasil preprocessing - $preprocessing = PreprocessingData::latest()->get(); - - return view('ulasan.index', compact('mentah', 'preprocessing')); + $query = Ulasan::query(); + if ($request->filled('search')) { + $query->where('ulasan', 'LIKE', '%' . $request->search . '%'); } + if ($wisata) { + $query->where('wisata', 'LIKE', '%' . $wisata . '%'); + } - // Tombol ambil data terbaru - public function ambilData() -{ - $pythonPath = '"C:\\Users\\Mufrida Farah\\AppData\\Local\\Programs\\Python\\Python312\\python.exe"'; - $scriptPath = base_path('scraper/scraping_pipeline.py'); + // ✅ PAGINATION 50 DATA + $mentah = $query->latest()->paginate(50)->withQueryString(); - $command = $pythonPath . " " . $scriptPath . " 2>&1"; + // preprocessing + $preprocessing = PreprocessingData::latest()->paginate(50); - shell_exec($command); + return view('ulasan.index', compact('mentah', 'preprocessing')); + } - return redirect()->route('ulasan.index') - ->with('success', 'Data berhasil diambil dari Google Maps'); -} + public function ambilData() + { + $pythonPath = "C:\\Users\\Mufrida Farah\\AppData\\Local\\Programs\\Python\\Python312\\python.exe"; + $scriptPath = base_path('scraper/scraping_pipeline.py'); + + shell_exec($pythonPath . " " . $scriptPath); + + return redirect()->route('ulasan.index') + ->with('success','Data berhasil diambil'); + } - // Tombol analisis data public function analisisData() -{ - $pythonPath = '"C:\\Users\\Mufrida Farah\\AppData\\Local\\Programs\\Python\\Python312\\python.exe"'; + { + $pythonPath = '"C:\\Users\\Mufrida Farah\\AppData\\Local\\Programs\\Python\\Python312\\python.exe"'; + $scriptPath = '"C:\\laragon\\www\\Sentara\\scraper\\preprocessing.py"'; - $scriptPath = base_path('scraper/preprocessing.py'); + shell_exec($pythonPath . " " . $scriptPath . " 2>&1"); - $command = $pythonPath . " " . $scriptPath . " 2>&1"; - - shell_exec($command); - - return redirect()->route('ulasan.index') - ->with('success','Preprocessing berhasil dilakukan'); -} -} + return redirect()->route('ulasan.index') + ->with('success','Analisis berhasil'); + } +} \ No newline at end of file diff --git a/app/Models/HasilAnalisis.php b/app/Models/HasilAnalisis.php index 45cb373..eb60391 100644 --- a/app/Models/HasilAnalisis.php +++ b/app/Models/HasilAnalisis.php @@ -12,7 +12,7 @@ class HasilAnalisis extends Model protected $table = 'hasil_analisis'; protected $fillable = [ - 'nama_wisata', + 'wisata', 'ulasan_terolah', 'sentimen', 'probabilitas' diff --git a/app/Models/HasilSentimen.php b/app/Models/HasilSentimen.php index 9eb07eb..6f8ecb6 100644 --- a/app/Models/HasilSentimen.php +++ b/app/Models/HasilSentimen.php @@ -12,7 +12,7 @@ class HasilSentimen extends Model protected $table = 'hasil_sentimen'; protected $fillable = [ - 'nama_wisata', + 'wisata', 'ulasan', 'sentimen' ]; diff --git a/app/Models/Ulasan.php b/app/Models/Ulasan.php index da7e75f..5cbc998 100644 --- a/app/Models/Ulasan.php +++ b/app/Models/Ulasan.php @@ -12,7 +12,7 @@ class Ulasan extends Model protected $table = 'ulasan'; protected $fillable = [ - 'nama_wisata', + 'wisata', 'reviewer', 'rating', 'ulasan', diff --git a/database/migrations/2026_02_11_071732_create_ulasan_mentah_table.php b/database/migrations/2026_02_11_071732_create_ulasan_mentah_table.php index 8a0f1f5..3805b76 100644 --- a/database/migrations/2026_02_11_071732_create_ulasan_mentah_table.php +++ b/database/migrations/2026_02_11_071732_create_ulasan_mentah_table.php @@ -13,7 +13,7 @@ public function up(): void { Schema::create('ulasan_mentah', function (Blueprint $table) { $table->id(); - $table->string('nama_wisata'); + $table->string('wisata'); $table->string('reviewer_name')->nullable(); $table->integer('rating')->nullable(); $table->text('ulasan'); diff --git a/database/migrations/2026_02_16_172121_create_hasil_analisis_table.php b/database/migrations/2026_02_16_172121_create_hasil_analisis_table.php index 24cc56f..a018453 100644 --- a/database/migrations/2026_02_16_172121_create_hasil_analisis_table.php +++ b/database/migrations/2026_02_16_172121_create_hasil_analisis_table.php @@ -13,7 +13,7 @@ public function up(): void { Schema::create('hasil_analisis', function (Blueprint $table) { $table->id(); - $table->string('nama_wisata'); + $table->string('wisata'); $table->text('ulasan_terolah'); $table->string('sentimen'); $table->float('probabilitas'); diff --git a/resources/css/app.css b/resources/css/app.css index d1dd59e..fa8b31d 100644 --- a/resources/css/app.css +++ b/resources/css/app.css @@ -26,3 +26,10 @@ .sidebar-menu .sidebar-link { .sidebar-menu .sidebar-link:hover { background: rgba(255, 255, 255, 0.15); } + +.line-clamp-3 { + display: -webkit-box; + -webkit-line-clamp: 3; + -webkit-box-orient: vertical; + overflow: hidden; +} \ No newline at end of file diff --git a/resources/views/analisis/index.blade.php b/resources/views/analisis/index.blade.php index f73b4c8..3cb34d4 100644 --- a/resources/views/analisis/index.blade.php +++ b/resources/views/analisis/index.blade.php @@ -12,16 +12,16 @@
- + +
+

Data Ulasan

+ +
+ + + + + + @csrf +
- - {{-- Statistik Cards --}} -
- -
-

Total Ulasan

-

- {{ $stats['total'] }} -

-
- -
-

Sentimen Positif

-

- {{ $stats['positif_persen'] }}% -

-
- -
-

Sentimen Negatif

-

- {{ $stats['negatif_persen'] }}% -

-
- -
-

Sentimen Netral

-

- {{ $stats['netral_persen'] }}% -

-
- -
- - {{-- Chart Section --}} -
- - +
-

- Distribusi Sentimen -

-
- + +

Data Ulasan Mentah

+ +
+ + + + + + + + + + + +
WisataRatingUlasanTanggal
+ + +
+ + + @forelse($mentah as $item) + + + + + + + @empty + + + + @endforelse + +
{{ $item->wisata }}{{ $item->rating }} + {{ $item->ulasan }} + {{ $item->tanggal }}
+ Tidak ada data +
+
+
+ + +
+ + @if($mentah->onFirstPage()) + Prev + @else + Prev + @endif + + @for ($i = 1; $i <= $mentah->lastPage(); $i++) + @if ($i == $mentah->currentPage()) + {{ $i }} + @elseif ($i <= 5 || $i == $mentah->lastPage()) + {{ $i }} + @elseif ($i == 6) + ... + @endif + @endfor + + @if($mentah->hasMorePages()) + Next + @else + Next + @endif + +
+
- + +
+ @csrf + +
+ +
-

- Sentimen per Destinasi -

-
- +

Hasil Preprocessing

+ +
+ + + + + + + + + + @forelse($preprocessing as $row) + + + + + + @empty + + + + @endforelse + +
WisataUlasan MentahUlasan Bersih
{{ $row->wisata }}{{ $row->ulasan_asli }}{{ $row->stemming }}
+ Belum ada data +
-
{{-- TUTUP GRID CHART --}} - - -{{-- Section Bawah --}} -
- - -
-

- Total Data per Destinasi -

- -
-
- Pantai Papuma - 120 ulasan -
-
- Pantai Watu Ulo - 85 ulasan -
-
- Teluk Love (Payangan) - 95 ulasan -
-
- Kebun Teh Gunung Gambir - 60 ulasan -
-
-
- - - -
-
- - -
- - - - -@endsection +@endsection \ No newline at end of file diff --git a/resources/views/ulasan/index.blade.php b/resources/views/ulasan/index.blade.php index 6739824..27ffc16 100644 --- a/resources/views/ulasan/index.blade.php +++ b/resources/views/ulasan/index.blade.php @@ -6,136 +6,250 @@
-

- Data Ulasan -

+

Data Ulasan

- + + + + +
- -
-
- @csrf - -
+ + + +
+ @csrf + +
+ + +
+ +

Data Ulasan Mentah

+ +
+ + + + + + + + + + + +
WisataRatingUlasanTanggal
+ + +
+ + + + + @foreach($mentah as $item) + + + + + + + + + + + + + + + + @endforeach + + +
+ {{ $item->wisata }} + + {{ $item->rating }} + +
+ {{ $item->ulasan }} +
+ 		+ {{ $item->tanggal }} +
- -
-

- Data Ulasan Mentah -

+
-
- - - - - - - - - + +
+ Showing {{ $mentah->firstItem() }} to {{ $mentah->lastItem() }} + of {{ $mentah->total() }} results +
+ + +
+ + {{-- FIRST --}} + @if ($mentah->currentPage() > 1) + « + @endif + + {{-- PREV --}} + @if ($mentah->onFirstPage()) + ‹ + @else + ‹ + @endif + + @php + $start = max($mentah->currentPage() - 2, 1); + $end = min($mentah->currentPage() + 2, $mentah->lastPage()); + @endphp + + {{-- NUMBER DINAMIS --}} + @for ($i = $start; $i <= $end; $i++) + @if ($i == $mentah->currentPage()) + {{ $i }} + @else + + {{ $i }} + + @endif + @endfor + + {{-- NEXT --}} + @if ($mentah->hasMorePages()) + › + @else + › + @endif + + {{-- LAST --}} + @if ($mentah->currentPage() < $mentah->lastPage()) + » + @endif + +
+ + + + + + @csrf + + + + +
+

Hasil Preprocessing

+ +
+
DestinasiReviewerRatingUlasanTanggal
+ + + + + - -@forelse($mentah as $data) - - - - - - - - -@empty - - - -@endforelse - + @forelse($preprocessing as $row) + + + + + + @empty + + + +
+ + +
+ Showing {{ $preprocessing->firstItem() }} to {{ $preprocessing->lastItem() }} + of {{ $preprocessing->total() }} results +
-
WisataUlasan MentahUlasan Bersih
{{ $data->destinasi }}{{ $data->reviewer }}{{ $data->rating }}{{ $data->ulasan }}{{ $data->tanggal }}
- Belum ada data mentah. -
{{ $row->wisata }}{{ $row->ulasan_asli }}{{ $row->stemming }}
+ Belum ada data +
-
-
+ +
- -
-
- @csrf - -
-
+ {{-- FIRST --}} + @if ($preprocessing->currentPage() > 1) + « + @endif - -
-

- Hasil Preprocessing Data -

+ {{-- PREV --}} + @if ($preprocessing->onFirstPage()) + ‹ + @else + ‹ + @endif -
- - - - - - - - + @php + $start = max($preprocessing->currentPage() - 2, 1); + $end = min($preprocessing->currentPage() + 2, $preprocessing->lastPage()); + @endphp - - @forelse($preprocessing as $row) - + {{-- NUMBER DINAMIS --}} + @for ($i = $start; $i <= $end; $i++) + @if ($i == $preprocessing->currentPage()) + {{ $i }} + @else + + {{ $i }} + + @endif + @endfor - + {{-- NEXT --}} + @if ($preprocessing->hasMorePages()) + › + @else + › + @endif - + {{-- LAST --}} + @if ($preprocessing->currentPage() < $preprocessing->lastPage()) + » + @endif - + - - - @empty - - - - - - @endforelse - + + + @endforelse +
DestinasiUlasan MentahUlasan Bersih
- {{ $row->destinasi }} - - {{ $row->ulasan_asli }} - - {{ $row->stemming }} -
- Belum ada hasil preprocessing. -
-@endsection +@endsection \ No newline at end of file diff --git a/scraper/prepocessing.py b/scraper/prepocessing.py deleted file mode 100644 index 40c5876..0000000 --- a/scraper/prepocessing.py +++ /dev/null @@ -1,70 +0,0 @@ -import mysql.connector -import re -from Sastrawi.Stemmer.StemmerFactory import StemmerFactory - -# ============================= -# DATABASE -# ============================= -conn = mysql.connector.connect( - host="localhost", - database="sentara", - user="root", - password="" -) - -cursor = conn.cursor() - -# ============================= -# STEMMER -# ============================= -factory = StemmerFactory() -stemmer = factory.create_stemmer() - -# ============================= -# AMBIL DATA ULASAN -# ============================= -cursor.execute("SELECT id, ulasan FROM ulasan") - -data = cursor.fetchall() - -print("Total ulasan:", len(data)) - -for row in data: - - id_ulasan = row[0] - text = row[1] - - # ============================= - # CLEANING - # ============================= - text_clean = re.sub(r'[^a-zA-Z\s]', '', text) - - # ============================= - # CASE FOLDING - # ============================= - text_clean = text_clean.lower() - - # ============================= - # STEMMING - # ============================= - text_stem = stemmer.stem(text_clean) - - # ============================= - # INSERT KE DATABASE - # ============================= - cursor.execute(""" - INSERT INTO preprocessing_data - (ulasan_asli, cleaning, stemming) - VALUES (%s,%s,%s) - """, - ( - text, - text_clean, - text_stem - )) - - conn.commit() - -print("Preprocessing selesai") - -conn.close() \ No newline at end of file diff --git a/scraper/preprocessing.py b/scraper/preprocessing.py new file mode 100644 index 0000000..0e9c20e --- /dev/null +++ b/scraper/preprocessing.py @@ -0,0 +1,143 @@ +import mysql.connector +import re +import math +from collections import Counter +from Sastrawi.Stemmer.StemmerFactory import StemmerFactory + +# ============================= +# DATABASE +# ============================= +conn = mysql.connector.connect( + host="localhost", + database="sentara", + user="root", + password="" +) + +cursor = conn.cursor() + +# ============================= +# STEMMER +# ============================= +factory = StemmerFactory() +stemmer = factory.create_stemmer() + +# ============================= +# DATA TRAINING +# ============================= +data_train = [ + ("bagus bersih indah nyaman sejuk", "positif"), + ("indah nyaman segar bagus", "positif"), + ("kotor jelek rusak mahal sempit", "negatif"), + ("jalan rusak sempit mahal", "negatif") +] + +# ============================= +# TRAIN NAIVE BAYES +# ============================= +def train_nb(data): + word_counts = {"positif": Counter(), "negatif": Counter()} + class_counts = {"positif": 0, "negatif": 0} + total_words = {"positif": 0, "negatif": 0} + + for text, label in data: + words = text.split() + class_counts[label] += 1 + word_counts[label].update(words) + total_words[label] += len(words) + + return word_counts, class_counts, total_words + +word_counts, class_counts, total_words = train_nb(data_train) + +# ============================= +# PREDICT +# ============================= +def predict_nb(text): + words = text.split() + + vocab = set(word_counts["positif"]).union(set(word_counts["negatif"])) + + log_pos = math.log(class_counts["positif"] / sum(class_counts.values())) + log_neg = math.log(class_counts["negatif"] / sum(class_counts.values())) + + for word in words: + prob_pos = (word_counts["positif"][word] + 1) / (total_words["positif"] + len(vocab)) + prob_neg = (word_counts["negatif"][word] + 1) / (total_words["negatif"] + len(vocab)) + + log_pos += math.log(prob_pos) + log_neg += math.log(prob_neg) + + if log_pos > log_neg: + return "Positif", log_pos + else: + return "Negatif", log_neg + +# ============================= +# HAPUS DATA LAMA (PENTING!) +# ============================= +cursor.execute("DELETE FROM preprocessing_data") +cursor.execute("DELETE FROM hasil_analisis") + +# ============================= +# AMBIL DATA +# ============================= +cursor.execute("SELECT wisata, ulasan FROM ulasan") +data = cursor.fetchall() + +print("Total ulasan:", len(data)) + +# ============================= +# PROSES +# ============================= +for row in data: + + wisata = row[0] + text = row[1] + + if not text: + continue + + # CLEANING + text_clean = re.sub(r'[^a-zA-Z\s]', '', text) + text_clean = text_clean.lower() + + # TOKENIZING + tokens = text_clean.split() + text_token = ' '.join(tokens) + + # STEMMING + text_stem = stemmer.stem(text_clean) + + # PREDIKSI + sentimen, probabilitas = predict_nb(text_stem) + + # INSERT PREPROCESSING + cursor.execute(""" + INSERT INTO preprocessing_data + (wisata, ulasan_asli, cleaning, tokenizing, stemming) + VALUES (%s,%s,%s,%s,%s) + """, + ( + wisata, + text, + text_clean, + text_token, + text_stem + )) + + # INSERT HASIL ANALISIS + cursor.execute(""" + INSERT INTO hasil_analisis + (wisata, ulasan_terolah, hasil_preprocessing, sentimen, probabilitas) + VALUES (%s,%s,%s,%s,%s) + """, (wisata, text, text_stem, sentimen, probabilitas)) + +# ============================= +# COMMIT SEKALI SAJA +# ============================= +conn.commit() + +print(" Preprocessing & Analisis Sentimen selesai") + +conn.close() \ No newline at end of file diff --git a/scraper/scraping_pipeline.py b/scraper/scraping_pipeline.py index 220b2b7..fac71f1 100644 --- a/scraper/scraping_pipeline.py +++ b/scraper/scraping_pipeline.py @@ -1,214 +1,143 @@ from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.chrome.options import Options -from selenium.webdriver.support.ui import WebDriverWait -from selenium.webdriver.support import expected_conditions as EC import mysql.connector import time from datetime import datetime -# ============================= # DATABASE -# ============================= conn = mysql.connector.connect( host="localhost", database="sentara", user="root", password="" ) - cursor = conn.cursor() -# ============================= # CHROME -# ============================= options = Options() options.add_argument("--start-maximized") driver = webdriver.Chrome(options=options) -wait = WebDriverWait(driver, 15) -# ============================= -# DESTINASI -# ============================= destinations = { - "Pantai Papuma": "Pantai Papuma Jember", - "Pantai Watu Ulo": "Pantai Watu Ulo Jember", - "Teluk Love": "Teluk Love Jember", - "Kebun Teh Gunung Gambir": "Kebun Teh Gunung Gambir Jember" + "Pantai Papuma":"https://www.google.com/maps/place/Pantai+Papuma+Jember", + "Pantai Watu Ulo":"https://www.google.com/maps/place/Pantai+Watu+Ulo+Jember", + "Teluk Love":"https://www.google.com/maps/place/Teluk+Love+Jember", + "Kebun Teh Gunung Gambir":"https://www.google.com/maps/place/Kebun+Teh+Gunung+Gambir" } -# ============================= -# LOOP DESTINASI -# ============================= -for nama_wisata, keyword in destinations.items(): +for wisata, url in destinations.items(): - saved = 0 - print("\n====================") - print("Scraping:", nama_wisata) - print("====================") + print("\nScraping:", wisata) - driver.get("https://www.google.com/maps") - time.sleep(5) + driver.get(url) + time.sleep(10) - # ============================= - # CARI TEMPAT - # ============================= + # ========================= + # BUKA ULASAN (FIX) + # ========================= try: - search = driver.find_element(By.ID, "searchboxinput") - except: - search = driver.find_element(By.XPATH, "//input[@aria-label='Search Google Maps']") + buttons = driver.find_elements(By.XPATH, "//button") - search.clear() - search.send_keys(keyword) + ketemu = False - driver.find_element(By.ID, "searchbox-searchbutton").click() + for btn in buttons: + try: + label = btn.get_attribute("aria-label") - time.sleep(6) + if label and ("ulasan" in label.lower() or "review" in label.lower()): + driver.execute_script("arguments[0].click();", btn) + time.sleep(5) + print("Berhasil buka ulasan") + ketemu = True + break + except: + pass - # ============================= - # KLIK HASIL PERTAMA - # ============================= - try: - first_result = wait.until( - EC.element_to_be_clickable((By.XPATH, "(//a[contains(@href,'/maps/place')])[1]")) - ) - first_result.click() - time.sleep(5) + if not ketemu: + print("Tombol ulasan tidak ditemukan") + continue - except: - print("Tempat tidak ditemukan") + except Exception as e: + print("Error buka ulasan:", e) continue - # ============================= - # BUKA TAB REVIEW - # ============================= - try: - review_button = wait.until( - EC.element_to_be_clickable((By.XPATH, "//button[contains(@aria-label,'review')]")) - ) - - review_button.click() - time.sleep(5) - - except: - print("Review button tidak ditemukan") - continue - - # ============================= - # SCROLL REVIEW - # ============================= + # ========================= + # SCROLL + # ========================= try: scrollable_div = driver.find_element(By.XPATH, "//div[@role='region']") except: - print("Container review tidak ditemukan") + print("Container tidak ditemukan") continue - last_count = 0 - same_count = 0 - reviews = [] - - for i in range(200): + last_height = 0 + for i in range(50): driver.execute_script( "arguments[0].scrollTop = arguments[0].scrollHeight", scrollable_div ) - time.sleep(2) - reviews = driver.find_elements(By.XPATH, "//div[@data-review-id]") + new_height = driver.execute_script( + "return arguments[0].scrollHeight", + scrollable_div + ) - current_count = len(reviews) + print("Scroll ke-", i) - print("Scroll", i, "Review:", current_count) - - if current_count == last_count: - same_count += 1 - else: - same_count = 0 - - if same_count >= 6: - print("✔ Semua review sudah termuat") + if new_height == last_height: + print("Sudah mentok") break - last_count = current_count - - # ============================= - # AMBIL DATA REVIEW - # ============================= - for review in reviews: + last_height = new_height + # ========================= + # EXPAND + # ========================= + buttons = driver.find_elements(By.XPATH, "//button[contains(text(),'Selengkapnya')]") + for b in buttons: try: + driver.execute_script("arguments[0].click();", b) + except: + pass - review_id = review.get_attribute("data-review-id") + time.sleep(2) - if not review_id: + # ========================= + # AMBIL DATA + # ========================= + reviews = driver.find_elements(By.XPATH, "//div[@data-review-id]") + + print("Total review ditemukan:", len(reviews)) + + saved = 0 + + for review in reviews: + try: + reviewer = review.find_element(By.XPATH, ".//div[contains(@class,'d4r55')]").text + + rating = review.find_element(By.XPATH, ".//span[@role='img']").get_attribute("aria-label") + + ulasan = review.find_element(By.XPATH, ".//span[contains(@class,'wiI7pd')]").text + + tanggal = review.find_element(By.XPATH, ".//span[contains(@class,'rsqaWe')]").text + + if ulasan.strip() == "": continue - # cek duplikasi - cursor.execute( - "SELECT COUNT(*) FROM ulasan WHERE review_id=%s", - (review_id,) - ) - - exists = cursor.fetchone()[0] - - if exists > 0: - continue - - # reviewer - try: - reviewer = review.find_element( - By.XPATH, ".//div[contains(@class,'d4r55')]" - ).text - except: - reviewer = "Unknown" - - # rating - try: - rating_text = review.find_element( - By.XPATH, ".//span[@role='img']" - ).get_attribute("aria-label") - - rating = float(rating_text.split()[0]) - except: - rating = 0 - - # ulasan - try: - review_text = review.find_element( - By.XPATH, ".//span[contains(@class,'wiI7pd')]" - ).text - except: - review_text = "" - - # tanggal - try: - review_date = review.find_element( - By.XPATH, ".//span[contains(@class,'rsqaWe')]" - ).text - except: - review_date = "" - - if review_text == "": - continue - - # ============================= - # INSERT DATABASE - # ============================= cursor.execute(""" INSERT INTO ulasan - (review_id,destinasi,reviewer,ulasan,rating,tanggal,scraping_date) - VALUES(%s,%s,%s,%s,%s,%s,%s) - """, - ( - review_id, - nama_wisata, + (destinasi, reviewer, ulasan, rating, tanggal, scraping_date) + VALUES (%s,%s,%s,%s,%s,%s) + """, ( + wisata, reviewer, - review_text, + ulasan, rating, - review_date, + tanggal, datetime.now() )) @@ -218,11 +147,9 @@ for nama_wisata, keyword in destinations.items(): except Exception as e: print("skip:", e) - print("Total tersimpan:", saved) - -print("\n====================") -print("SCRAPING SELESAI") -print("====================") + print("Berhasil simpan:", saved) driver.quit() -conn.close() \ No newline at end of file +conn.close() + +print("\nSCRAPING SELESAI") \ No newline at end of file