periode
This commit is contained in:
parent
387df6bdef
commit
338d089fca
|
|
@ -66,6 +66,13 @@ public function index(Request $request)
|
||||||
$wisata = $tab === 'analisis' ? $request->wisata : null;
|
$wisata = $tab === 'analisis' ? $request->wisata : null;
|
||||||
$destinasi = $tab === 'rekomendasi' ? $request->destinasi : null;
|
$destinasi = $tab === 'rekomendasi' ? $request->destinasi : null;
|
||||||
|
|
||||||
|
// ================================================================
|
||||||
|
// PERIODE BULAN YANG DIPILIH
|
||||||
|
// ================================================================
|
||||||
|
$periodeBulan = $request->input('periode_bulan', now()->format('Y-m'));
|
||||||
|
|
||||||
|
[$tahun, $bulan] = explode('-', $periodeBulan);
|
||||||
|
|
||||||
// ================================================================
|
// ================================================================
|
||||||
// AMBIL SEMUA PERIODE & PERIODE AKTIF
|
// AMBIL SEMUA PERIODE & PERIODE AKTIF
|
||||||
// ================================================================
|
// ================================================================
|
||||||
|
|
@ -83,11 +90,21 @@ public function index(Request $request)
|
||||||
->orderBy('p.id', 'desc')
|
->orderBy('p.id', 'desc')
|
||||||
->select('p.*')
|
->select('p.*')
|
||||||
->get();
|
->get();
|
||||||
$availablePeriodeIds = $periodeList->pluck('id')->map(fn($id) => (string) $id);
|
|
||||||
$periodeAktif = $request->periode_id && $availablePeriodeIds->contains((string) $request->periode_id)
|
// Cocokkan pakai kolom bulan & tahun
|
||||||
? $periodeList->firstWhere('id', (int) $request->periode_id)
|
$periodeAktif = $periodeList->first(function ($p) use ($tahun, $bulan) {
|
||||||
: $periodeList->first();
|
return $p->tahun == (int)$tahun && $p->bulan == (int)$bulan;
|
||||||
$periodeId = $periodeAktif->id ?? null;
|
});
|
||||||
|
|
||||||
|
// Jika tidak ada data di bulan tsb
|
||||||
|
$noDataPesan = null;
|
||||||
|
if (!$periodeAktif) {
|
||||||
|
$namaBulanDipilih = \Carbon\Carbon::createFromDate($tahun, $bulan, 1)
|
||||||
|
->locale('id')->isoFormat('MMMM YYYY');
|
||||||
|
$noDataPesan = "Tidak ada data untuk bulan {$namaBulanDipilih}.";
|
||||||
|
}
|
||||||
|
|
||||||
|
$periodeId = $periodeAktif->id ?? null;
|
||||||
|
|
||||||
$evaluasi = DB::table('evaluasi_model')
|
$evaluasi = DB::table('evaluasi_model')
|
||||||
->when($periodeId, fn($q) => $q->where('periode_id', $periodeId))
|
->when($periodeId, fn($q) => $q->where('periode_id', $periodeId))
|
||||||
|
|
@ -345,8 +362,9 @@ public function index(Request $request)
|
||||||
'kataDominan',
|
'kataDominan',
|
||||||
'saranPerbaikan',
|
'saranPerbaikan',
|
||||||
'prioritas',
|
'prioritas',
|
||||||
'periodeList', // ← tambahan
|
'periodeList',
|
||||||
'periodeAktif', // ← tambahan
|
'periodeAktif',
|
||||||
|
'noDataPesan', // ← tambahan baru
|
||||||
))->with([
|
))->with([
|
||||||
'scraperDestinations' => $this->scraperDestinations(),
|
'scraperDestinations' => $this->scraperDestinations(),
|
||||||
]);
|
]);
|
||||||
|
|
@ -361,4 +379,4 @@ private function scraperDestinations(): array
|
||||||
'Kebun Teh Gunung Gambir',
|
'Kebun Teh Gunung Gambir',
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -237,6 +237,43 @@
|
||||||
width:150px;
|
width:150px;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@media(max-width:900px){
|
||||||
|
.container{
|
||||||
|
flex-direction:column;
|
||||||
|
height:auto;
|
||||||
|
}
|
||||||
|
|
||||||
|
.left{
|
||||||
|
width:100%;
|
||||||
|
padding:40px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.right{
|
||||||
|
width:100%;
|
||||||
|
height:260px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.overlay p{
|
||||||
|
font-size:20px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.logo img{
|
||||||
|
width:150px;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* HILANGKAN ICON PASSWORD BAWAAN BROWSER */
|
||||||
|
input[type="password"]::-ms-reveal,
|
||||||
|
input[type="password"]::-ms-clear {
|
||||||
|
display: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
input::-ms-reveal,
|
||||||
|
input::-ms-clear {
|
||||||
|
display: none;
|
||||||
|
}
|
||||||
|
|
||||||
</style>
|
</style>
|
||||||
</head>
|
</head>
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -15,6 +15,14 @@
|
||||||
{{ session('error') }}
|
{{ session('error') }}
|
||||||
</div>
|
</div>
|
||||||
@endif
|
@endif
|
||||||
|
|
||||||
|
{{-- PESAN TIDAK ADA DATA --}}
|
||||||
|
@if($noDataPesan ?? null)
|
||||||
|
<div class="bg-blue-50 border-l-4 border-blue-400 text-blue-700 p-4 mb-6 rounded flex items-center gap-2">
|
||||||
|
<span>📅</span> <span>{{ $noDataPesan }}</span>
|
||||||
|
</div>
|
||||||
|
@endif
|
||||||
|
|
||||||
@if($periodeAktif && !$hasAnalisis)
|
@if($periodeAktif && !$hasAnalisis)
|
||||||
<div class="bg-yellow-100 border-l-4 border-yellow-500 text-yellow-700 p-4 mb-6 rounded">
|
<div class="bg-yellow-100 border-l-4 border-yellow-500 text-yellow-700 p-4 mb-6 rounded">
|
||||||
Data ulasan sudah tersedia, tetapi belum dilakukan analisis sentimen.
|
Data ulasan sudah tersedia, tetapi belum dilakukan analisis sentimen.
|
||||||
|
|
@ -36,20 +44,21 @@
|
||||||
Terakhir update: {{ $lastUpdate ? \Carbon\Carbon::parse($lastUpdate)->format('d M Y H:i') : '-' }}
|
Terakhir update: {{ $lastUpdate ? \Carbon\Carbon::parse($lastUpdate)->format('d M Y H:i') : '-' }}
|
||||||
</span>
|
</span>
|
||||||
@endif
|
@endif
|
||||||
<form action="{{ route('ulasan.ambil') }}" method="POST" class="flex flex-col md:flex-row gap-3 md:items-center">
|
<form method="GET"
|
||||||
@csrf
|
action="{{ route('dashboard') }}"
|
||||||
<input type="hidden" name="redirect_to" value="dashboard">
|
class="flex flex-col md:flex-row gap-3 md:items-center">
|
||||||
<input type="month" name="periode_bulan" value="{{ now()->format('Y-m') }}"
|
|
||||||
|
{{-- HAPUS hidden periode_id, pakai periode_bulan saja --}}
|
||||||
|
|
||||||
|
<input type="month"
|
||||||
|
name="periode_bulan"
|
||||||
|
value="{{ request('periode_bulan', now()->format('Y-m')) }}"
|
||||||
|
max="{{ now()->format('Y-m') }}"
|
||||||
|
onchange="this.form.submit()"
|
||||||
class="border rounded-lg px-3 py-2 text-sm bg-white shadow-sm min-w-[150px]">
|
class="border rounded-lg px-3 py-2 text-sm bg-white shadow-sm min-w-[150px]">
|
||||||
<select name="wisata" class="border rounded-lg pl-3 pr-9 py-2 text-sm bg-white shadow-sm min-w-[220px]">
|
|
||||||
<option value="">Semua Lokasi</option>
|
|
||||||
@foreach(($scraperDestinations ?? []) as $scraperDestination)
|
|
||||||
<option value="{{ $scraperDestination }}">{{ $scraperDestination }}</option>
|
|
||||||
@endforeach
|
|
||||||
</select>
|
|
||||||
<button class="bg-blue-600 hover:bg-blue-700 text-white px-4 py-2 rounded-lg text-sm shadow">
|
|
||||||
Ambil Data
|
|
||||||
</button>
|
|
||||||
</form>
|
</form>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -335,4 +344,4 @@ class="flex-1 py-3 rounded-lg text-sm font-medium transition">
|
||||||
});
|
});
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
@endsection
|
@endsection
|
||||||
|
|
@ -17,7 +17,9 @@
|
||||||
<form action="{{ route('ulasan.ambil') }}" method="POST" class="flex flex-col md:flex-row gap-3 md:items-center">
|
<form action="{{ route('ulasan.ambil') }}" method="POST" class="flex flex-col md:flex-row gap-3 md:items-center">
|
||||||
@csrf
|
@csrf
|
||||||
<input type="hidden" name="redirect_to" value="ulasan">
|
<input type="hidden" name="redirect_to" value="ulasan">
|
||||||
<input type="month" name="periode_bulan" value="{{ now()->format('Y-m') }}"
|
<input type="month" name="periode_bulan" value="{{ $periodeAktif
|
||||||
|
? $periodeAktif->tahun . '-' . str_pad($periodeAktif->bulan, 2, '0', STR_PAD_LEFT)
|
||||||
|
: now()->format('Y-m') }}"
|
||||||
class="border rounded-lg px-3 py-2 text-sm bg-white shadow-sm min-w-[150px]">
|
class="border rounded-lg px-3 py-2 text-sm bg-white shadow-sm min-w-[150px]">
|
||||||
<select name="wisata" class="border rounded-lg pl-3 pr-9 py-2 text-sm bg-white shadow-sm min-w-[220px]">
|
<select name="wisata" class="border rounded-lg pl-3 pr-9 py-2 text-sm bg-white shadow-sm min-w-[220px]">
|
||||||
<option value="">Semua Lokasi</option>
|
<option value="">Semua Lokasi</option>
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,14 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
import argparse
|
import argparse
|
||||||
import os
|
import os
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
from networkx import config
|
||||||
|
import undetected_chromedriver as uc
|
||||||
|
|
||||||
|
from jmespath import Options
|
||||||
|
from pandas import options
|
||||||
|
from selenium import webdriver
|
||||||
|
|
||||||
if 'PATH' not in os.environ:
|
if 'PATH' not in os.environ:
|
||||||
os.environ['PATH'] = r"C:\Windows\System32;C:\Windows"
|
os.environ['PATH'] = r"C:\Windows\System32;C:\Windows"
|
||||||
|
|
@ -297,6 +305,9 @@ def build_chrome_options(config):
|
||||||
from selenium.webdriver.chrome.options import Options
|
from selenium.webdriver.chrome.options import Options
|
||||||
|
|
||||||
options = Options()
|
options = Options()
|
||||||
|
# options = uc.ChromeOptions()
|
||||||
|
options.binary_location = r"C:\Program Files\Google\Chrome\Application\chrome.exe"
|
||||||
|
|
||||||
options.add_argument("--lang=id")
|
options.add_argument("--lang=id")
|
||||||
options.add_argument("--accept-language=id-ID,id")
|
options.add_argument("--accept-language=id-ID,id")
|
||||||
options.add_argument("--window-size=1366,900")
|
options.add_argument("--window-size=1366,900")
|
||||||
|
|
@ -306,19 +317,28 @@ def build_chrome_options(config):
|
||||||
options.add_argument("--no-first-run")
|
options.add_argument("--no-first-run")
|
||||||
options.add_argument("--no-default-browser-check")
|
options.add_argument("--no-default-browser-check")
|
||||||
|
|
||||||
|
options.add_argument("--disable-gpu")
|
||||||
|
options.add_argument("--disable-software-rasterizer")
|
||||||
|
options.add_argument("--remote-debugging-port=9222")
|
||||||
|
options.add_argument("--disable-extensions")
|
||||||
|
options.add_argument("--disable-dev-shm-usage")
|
||||||
|
options.add_argument("--disable-gpu")
|
||||||
|
|
||||||
|
options.add_argument(f"--user-data-dir={tempfile.mkdtemp()}")
|
||||||
|
|
||||||
if config["headless"]:
|
if config["headless"]:
|
||||||
options.add_argument("--headless=new")
|
options.add_argument("--headless=new")
|
||||||
else:
|
else:
|
||||||
options.add_argument("--start-maximized")
|
options.add_argument("--start-maximized")
|
||||||
|
|
||||||
if config["user_data_dir"]:
|
# if config["user_data_dir"]:
|
||||||
options.add_argument(f"--user-data-dir={config['user_data_dir']}")
|
# options.add_argument(f"--user-data-dir={config['user_data_dir']}")
|
||||||
|
|
||||||
if config["profile"]:
|
# if config["profile"]:
|
||||||
options.add_argument(f"--profile-directory={config['profile']}")
|
# options.add_argument(f"--profile-directory={config['profile']}")
|
||||||
|
|
||||||
return options
|
return options
|
||||||
|
driver = uc.Chrome(options=options)
|
||||||
|
|
||||||
def is_login_or_consent_page(driver):
|
def is_login_or_consent_page(driver):
|
||||||
current_url = driver.current_url.lower()
|
current_url = driver.current_url.lower()
|
||||||
|
|
@ -729,7 +749,7 @@ def find_reviews_scroll_container(driver, wait):
|
||||||
|
|
||||||
|
|
||||||
def scrape_with_selenium(cursor, conn, periode_id, start_date, end_date, config, destinations):
|
def scrape_with_selenium(cursor, conn, periode_id, start_date, end_date, config, destinations):
|
||||||
from selenium import webdriver
|
|
||||||
from selenium.webdriver.common.by import By
|
from selenium.webdriver.common.by import By
|
||||||
from selenium.webdriver.support import expected_conditions as EC
|
from selenium.webdriver.support import expected_conditions as EC
|
||||||
from selenium.webdriver.support.ui import WebDriverWait
|
from selenium.webdriver.support.ui import WebDriverWait
|
||||||
|
|
@ -742,7 +762,18 @@ def scrape_with_selenium(cursor, conn, periode_id, start_date, end_date, config,
|
||||||
+ ("headless" if config["headless"] else "visible")
|
+ ("headless" if config["headless"] else "visible")
|
||||||
+ (f", profile={config['profile']}" if config["profile"] else ""),
|
+ (f", profile={config['profile']}" if config["profile"] else ""),
|
||||||
)
|
)
|
||||||
driver = webdriver.Chrome(options=build_chrome_options(config))
|
|
||||||
|
from selenium.webdriver.chrome.service import Service
|
||||||
|
from webdriver_manager.chrome import ChromeDriverManager
|
||||||
|
|
||||||
|
# driver = webdriver.Chrome(
|
||||||
|
# service = Service(r"C:\Users\Mufrida Farah\Downloads\chromedriver-win64\chromedriver-win64\chromedriver.exe")
|
||||||
|
# driver = webdriver.Chrome(service=service, options=build_chrome_options(config))
|
||||||
|
# options=build_chrome_options(config)
|
||||||
|
# )
|
||||||
|
|
||||||
|
service = Service(r"C:\Users\Mufrida Farah\Downloads\chromedriver-win64\chromedriver-win64\chromedriver.exe")
|
||||||
|
driver = webdriver.Chrome(service=service, options=build_chrome_options(config))
|
||||||
|
|
||||||
total_saved = 0
|
total_saved = 0
|
||||||
total_skipped_duplicate = 0
|
total_skipped_duplicate = 0
|
||||||
|
|
@ -755,6 +786,13 @@ def scrape_with_selenium(cursor, conn, periode_id, start_date, end_date, config,
|
||||||
try:
|
try:
|
||||||
for wisata, url in destinations.items():
|
for wisata, url in destinations.items():
|
||||||
log("INFO", f"Scraping: {wisata}")
|
log("INFO", f"Scraping: {wisata}")
|
||||||
|
|
||||||
|
main_window = driver.window_handles[0]
|
||||||
|
for handle in driver.window_handles[1:]:
|
||||||
|
driver.switch_to.window(handle)
|
||||||
|
driver.close()
|
||||||
|
driver.switch_to.window(main_window)
|
||||||
|
|
||||||
driver.get(url)
|
driver.get(url)
|
||||||
wait = WebDriverWait(driver, 25)
|
wait = WebDriverWait(driver, 25)
|
||||||
time.sleep(5)
|
time.sleep(5)
|
||||||
|
|
@ -771,6 +809,14 @@ def scrape_with_selenium(cursor, conn, periode_id, start_date, end_date, config,
|
||||||
log("WARNING", f"Tombol/tab ulasan tidak ditemukan untuk {wisata}")
|
log("WARNING", f"Tombol/tab ulasan tidak ditemukan untuk {wisata}")
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
time.sleep(2)
|
||||||
|
all_windows = driver.window_handles
|
||||||
|
if len(all_windows) > 1:
|
||||||
|
driver.switch_to.window(all_windows[-1])
|
||||||
|
log("INFO", f"{wisata}: pindah ke tab baru")
|
||||||
|
|
||||||
|
click_sort_newest(driver)
|
||||||
|
|
||||||
click_sort_newest(driver)
|
click_sort_newest(driver)
|
||||||
|
|
||||||
scrollable_div = find_reviews_scroll_container(driver, wait)
|
scrollable_div = find_reviews_scroll_container(driver, wait)
|
||||||
|
|
@ -779,14 +825,29 @@ def scrape_with_selenium(cursor, conn, periode_id, start_date, end_date, config,
|
||||||
continue
|
continue
|
||||||
|
|
||||||
last_height = 0
|
last_height = 0
|
||||||
|
|
||||||
for i in range(config["scroll_limit"]):
|
for i in range(config["scroll_limit"]):
|
||||||
driver.execute_script("arguments[0].scrollTop = arguments[0].scrollHeight", scrollable_div)
|
try:
|
||||||
time.sleep(2)
|
driver.execute_script("arguments[0].scrollTop = arguments[0].scrollHeight", scrollable_div)
|
||||||
new_height = driver.execute_script("return arguments[0].scrollHeight", scrollable_div)
|
|
||||||
log("INFO", f"{wisata}: scroll ke-{i + 1}")
|
time.sleep(2)
|
||||||
if new_height == last_height:
|
|
||||||
|
new_height = driver.execute_script("return arguments[0].scrollHeight", scrollable_div)
|
||||||
|
log("INFO", f"{wisata}: scroll ke-{i + 1}")
|
||||||
|
|
||||||
|
if new_height == last_height:
|
||||||
|
break
|
||||||
|
last_height = new_height
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
log("ERROR", f"{wisata}: Error occurred while scrolling: {e}")
|
||||||
|
|
||||||
|
scrollable_div = find_reviews_scroll_container(driver, wait)
|
||||||
|
|
||||||
|
if scrollable_div is None:
|
||||||
break
|
break
|
||||||
last_height = new_height
|
|
||||||
|
continue
|
||||||
|
|
||||||
for button in driver.find_elements(By.XPATH, "//button[contains(text(),'Selengkapnya')]"):
|
for button in driver.find_elements(By.XPATH, "//button[contains(text(),'Selengkapnya')]"):
|
||||||
try:
|
try:
|
||||||
|
|
@ -893,7 +954,7 @@ def main():
|
||||||
log("INFO", "Lokasi scraping: " + ", ".join(destinations.keys()))
|
log("INFO", "Lokasi scraping: " + ", ".join(destinations.keys()))
|
||||||
if scraper["filter_date_range"]:
|
if scraper["filter_date_range"]:
|
||||||
log("INFO", f"Filter aktif: ulasan {start_date.strftime('%Y-%m-%d')} sampai {end_date.strftime('%Y-%m-%d')}")
|
log("INFO", f"Filter aktif: ulasan {start_date.strftime('%Y-%m-%d')} sampai {end_date.strftime('%Y-%m-%d')}")
|
||||||
purge_out_of_range_reviews(cursor, conn, periode_id, start_date, end_date)
|
# purge_out_of_range_reviews(cursor, conn, periode_id, start_date, end_date)
|
||||||
|
|
||||||
if scraper["provider"] == "apify":
|
if scraper["provider"] == "apify":
|
||||||
scrape_with_apify(cursor, conn, periode_id, start_date, end_date, scraper, destinations)
|
scrape_with_apify(cursor, conn, periode_id, start_date, end_date, scraper, destinations)
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue