from datetime import date, timedelta from io import StringIO from typing import Dict, List import pandas as pd import requests class ScrapingError(Exception): pass class SiskaperbapoScraper: MARKET_ALIASES = { "pasar_tanjung": "Pasar Tanjung", "pasar_kreongan": "Pasar Kreongan", "pasar_tegal_besar": "Pasar Tegal Besar", "pasar_mangli": "Pasar Mangli", "pasar_wirolegi": "Pasar Wirolegi", } MARKET_IDS = { "pasar_tanjung": "11", "pasar_kreongan": "12", "pasar_tegal_besar": "13", "pasar_mangli": "148", "pasar_wirolegi": "149", } def __init__(self, url: str, commodity_param: str, timeout: int = 30) -> None: self.url = url self.commodity_param = commodity_param self.timeout = timeout @staticmethod def _parse_price(value) -> float: if isinstance(value, (int, float)) and not pd.isna(value): return float(value) text = str(value).strip().replace("Rp", "").replace(" ", "") if not text or text.lower() in {"nan", "none", "-", "0"}: raise ValueError("harga kosong") text = text.replace(".", "").replace(",", ".") return float(text) @staticmethod def _price_in_rupiah(value) -> float: price = SiskaperbapoScraper._parse_price(value) return price * 1000 if 0 < price < 1000 else price def _read_tables(self, commodity: str) -> List[pd.DataFrame]: try: response = requests.get( self.url, params={self.commodity_param: commodity}, timeout=self.timeout, headers={"User-Agent": "PanenHarga/1.0"}, ) response.raise_for_status() return pd.read_html(StringIO(response.text)) except (requests.RequestException, ValueError) as error: raise ScrapingError(f"Gagal membaca halaman SISKAPERBAPO: {error}") from error def _read_historical_table(self, price_date: date, market_id: str) -> List[pd.DataFrame]: try: response = requests.post( self.url.replace("/harga-komoditas", "/harga/tabel.nodesign"), data={"tanggal": price_date.isoformat(), "kabkota": "jemberkab", "pasar": market_id}, timeout=self.timeout, headers={"User-Agent": "PanenHarga/1.0"}, ) response.raise_for_status() return pd.read_html(StringIO(response.text)) except (requests.RequestException, ValueError) as error: raise ScrapingError(f"Gagal membaca histori SISKAPERBAPO: {error}") from error def scrape_range(self, commodities: Dict[str, str], start_date: date, end_date: date, on_progress=None) -> List[dict]: observations = [] dates = [ start_date + timedelta(days=offset) for offset in range((end_date - start_date).days + 1) ] total = len(dates) * len(self.MARKET_IDS) completed = 0 row_patterns = { "rawit": "cabe rawit merah", "merah": "cabe merah besar", "keriting": "cabe merah keriting", } for price_date in dates: for market_slug, market_id in self.MARKET_IDS.items(): tables = self._read_historical_table(price_date, market_id) for table in tables: if table.empty: continue for _, row in table.iterrows(): text = " ".join(str(value).strip() for value in row.tolist()) normalized = text.casefold().replace("-", " ") commodity_slug = next( ( slug for slug in commodities if row_patterns.get(slug, "") in normalized ), None, ) if not commodity_slug: continue values = [ value for value in row.tolist() if str(value).strip().replace(".", "").replace(",", "").isdigit() ] if len(values) < 2: continue try: price = self._price_in_rupiah(values[1]) except (TypeError, ValueError): continue observations.append( { "commodity": commodity_slug, "market": market_slug, "date": price_date, "price": price, "source": "siskaperbapo", } ) break completed += 1 if on_progress: on_progress( 10 + int(completed / max(total, 1) * 55), f"Membaca {price_date.isoformat()} ยท {self.MARKET_ALIASES[market_slug]}", ) if not observations: raise ScrapingError( f"Tidak ada data SISKAPERBAPO untuk periode {start_date.isoformat()} sampai {end_date.isoformat()}." ) return observations def scrape(self, commodities: Dict[str, str], on_progress=None) -> List[dict]: observations = [] total_commodities = len(commodities) for commodity_index, commodity_slug in enumerate(commodities, start=1): if on_progress: on_progress( 10 + int((commodity_index - 1) / max(total_commodities, 1) * 55), f"Membaca data {commodity_slug}", ) tables = self._read_tables(commodity_slug) for table in tables: if table.empty: continue location_column = next( (column for column in table.columns if "lokasi" in str(column).lower() or "pasar" in str(column).lower()), None, ) if location_column is None: continue date_columns = [] for column in table.columns: try: parsed = pd.to_datetime(str(column), errors="raise") except (TypeError, ValueError): continue if not pd.isna(parsed): date_columns.append((column, parsed.date())) if not date_columns: continue for market_slug, market_name in self.MARKET_ALIASES.items(): matches = table[table[location_column].astype(str).str.strip().str.casefold() == market_name.casefold()] if matches.empty: continue row = matches.iloc[0] for column, price_date in date_columns: try: price = self._price_in_rupiah(row[column]) except (TypeError, ValueError): continue observations.append( { "commodity": commodity_slug, "market": market_slug, "date": price_date, "price": price, "source": "siskaperbapo", } ) if on_progress: on_progress( 10 + int(commodity_index / max(total_commodities, 1) * 55), f"Data {commodity_slug} selesai dibaca", ) if not observations: raise ScrapingError("Tidak ada data pasar Jember yang cocok dari SISKAPERBAPO.") return observations