194 lines
8.0 KiB
Python
194 lines
8.0 KiB
Python
from datetime import date, timedelta
|
|
from io import StringIO
|
|
from typing import Dict, List
|
|
|
|
import pandas as pd
|
|
import requests
|
|
|
|
|
|
class ScrapingError(Exception):
|
|
pass
|
|
|
|
|
|
class SiskaperbapoScraper:
|
|
MARKET_ALIASES = {
|
|
"pasar_tanjung": "Pasar Tanjung",
|
|
"pasar_kreongan": "Pasar Kreongan",
|
|
"pasar_tegal_besar": "Pasar Tegal Besar",
|
|
"pasar_mangli": "Pasar Mangli",
|
|
"pasar_wirolegi": "Pasar Wirolegi",
|
|
}
|
|
MARKET_IDS = {
|
|
"pasar_tanjung": "11",
|
|
"pasar_kreongan": "12",
|
|
"pasar_tegal_besar": "13",
|
|
"pasar_mangli": "148",
|
|
"pasar_wirolegi": "149",
|
|
}
|
|
|
|
def __init__(self, url: str, commodity_param: str, timeout: int = 30) -> None:
|
|
self.url = url
|
|
self.commodity_param = commodity_param
|
|
self.timeout = timeout
|
|
|
|
@staticmethod
|
|
def _parse_price(value) -> float:
|
|
if isinstance(value, (int, float)) and not pd.isna(value):
|
|
return float(value)
|
|
text = str(value).strip().replace("Rp", "").replace(" ", "")
|
|
if not text or text.lower() in {"nan", "none", "-", "0"}:
|
|
raise ValueError("harga kosong")
|
|
text = text.replace(".", "").replace(",", ".")
|
|
return float(text)
|
|
|
|
@staticmethod
|
|
def _price_in_rupiah(value) -> float:
|
|
price = SiskaperbapoScraper._parse_price(value)
|
|
return price * 1000 if 0 < price < 1000 else price
|
|
|
|
def _read_tables(self, commodity: str) -> List[pd.DataFrame]:
|
|
try:
|
|
response = requests.get(
|
|
self.url,
|
|
params={self.commodity_param: commodity},
|
|
timeout=self.timeout,
|
|
headers={"User-Agent": "PanenHarga/1.0"},
|
|
)
|
|
response.raise_for_status()
|
|
return pd.read_html(StringIO(response.text))
|
|
except (requests.RequestException, ValueError) as error:
|
|
raise ScrapingError(f"Gagal membaca halaman SISKAPERBAPO: {error}") from error
|
|
|
|
def _read_historical_table(self, price_date: date, market_id: str) -> List[pd.DataFrame]:
|
|
try:
|
|
response = requests.post(
|
|
self.url.replace("/harga-komoditas", "/harga/tabel.nodesign"),
|
|
data={"tanggal": price_date.isoformat(), "kabkota": "jemberkab", "pasar": market_id},
|
|
timeout=self.timeout,
|
|
headers={"User-Agent": "PanenHarga/1.0"},
|
|
)
|
|
response.raise_for_status()
|
|
return pd.read_html(StringIO(response.text))
|
|
except (requests.RequestException, ValueError) as error:
|
|
raise ScrapingError(f"Gagal membaca histori SISKAPERBAPO: {error}") from error
|
|
|
|
def scrape_range(self, commodities: Dict[str, str], start_date: date, end_date: date, on_progress=None) -> List[dict]:
|
|
observations = []
|
|
dates = [
|
|
start_date + timedelta(days=offset)
|
|
for offset in range((end_date - start_date).days + 1)
|
|
]
|
|
total = len(dates) * len(self.MARKET_IDS)
|
|
completed = 0
|
|
row_patterns = {
|
|
"rawit": "cabe rawit merah",
|
|
"merah": "cabe merah besar",
|
|
"keriting": "cabe merah keriting",
|
|
}
|
|
for price_date in dates:
|
|
for market_slug, market_id in self.MARKET_IDS.items():
|
|
tables = self._read_historical_table(price_date, market_id)
|
|
for table in tables:
|
|
if table.empty:
|
|
continue
|
|
for _, row in table.iterrows():
|
|
text = " ".join(str(value).strip() for value in row.tolist())
|
|
normalized = text.casefold().replace("-", " ")
|
|
commodity_slug = next(
|
|
(
|
|
slug for slug in commodities
|
|
if row_patterns.get(slug, "") in normalized
|
|
),
|
|
None,
|
|
)
|
|
if not commodity_slug:
|
|
continue
|
|
values = [
|
|
value for value in row.tolist()
|
|
if str(value).strip().replace(".", "").replace(",", "").isdigit()
|
|
]
|
|
if len(values) < 2:
|
|
continue
|
|
try:
|
|
price = self._price_in_rupiah(values[1])
|
|
except (TypeError, ValueError):
|
|
continue
|
|
observations.append(
|
|
{
|
|
"commodity": commodity_slug,
|
|
"market": market_slug,
|
|
"date": price_date,
|
|
"price": price,
|
|
"source": "siskaperbapo",
|
|
}
|
|
)
|
|
break
|
|
completed += 1
|
|
if on_progress:
|
|
on_progress(
|
|
10 + int(completed / max(total, 1) * 55),
|
|
f"Membaca {price_date.isoformat()} · {self.MARKET_ALIASES[market_slug]}",
|
|
)
|
|
if not observations:
|
|
raise ScrapingError(
|
|
f"Tidak ada data SISKAPERBAPO untuk periode {start_date.isoformat()} sampai {end_date.isoformat()}."
|
|
)
|
|
return observations
|
|
|
|
def scrape(self, commodities: Dict[str, str], on_progress=None) -> List[dict]:
|
|
observations = []
|
|
total_commodities = len(commodities)
|
|
for commodity_index, commodity_slug in enumerate(commodities, start=1):
|
|
if on_progress:
|
|
on_progress(
|
|
10 + int((commodity_index - 1) / max(total_commodities, 1) * 55),
|
|
f"Membaca data {commodity_slug}",
|
|
)
|
|
tables = self._read_tables(commodity_slug)
|
|
for table in tables:
|
|
if table.empty:
|
|
continue
|
|
location_column = next(
|
|
(column for column in table.columns if "lokasi" in str(column).lower() or "pasar" in str(column).lower()),
|
|
None,
|
|
)
|
|
if location_column is None:
|
|
continue
|
|
date_columns = []
|
|
for column in table.columns:
|
|
try:
|
|
parsed = pd.to_datetime(str(column), errors="raise")
|
|
except (TypeError, ValueError):
|
|
continue
|
|
if not pd.isna(parsed):
|
|
date_columns.append((column, parsed.date()))
|
|
if not date_columns:
|
|
continue
|
|
for market_slug, market_name in self.MARKET_ALIASES.items():
|
|
matches = table[table[location_column].astype(str).str.strip().str.casefold() == market_name.casefold()]
|
|
if matches.empty:
|
|
continue
|
|
row = matches.iloc[0]
|
|
for column, price_date in date_columns:
|
|
try:
|
|
price = self._price_in_rupiah(row[column])
|
|
except (TypeError, ValueError):
|
|
continue
|
|
observations.append(
|
|
{
|
|
"commodity": commodity_slug,
|
|
"market": market_slug,
|
|
"date": price_date,
|
|
"price": price,
|
|
"source": "siskaperbapo",
|
|
}
|
|
)
|
|
if on_progress:
|
|
on_progress(
|
|
10 + int(commodity_index / max(total_commodities, 1) * 55),
|
|
f"Data {commodity_slug} selesai dibaca",
|
|
)
|
|
if not observations:
|
|
raise ScrapingError("Tidak ada data pasar Jember yang cocok dari SISKAPERBAPO.")
|
|
return observations
|