first commit
This commit is contained in:
commit
de876df2a3
|
|
@ -0,0 +1,3 @@
|
|||
{
|
||||
"python-envs.defaultEnvManager": "ms-python.python:system"
|
||||
}
|
||||
Binary file not shown.
|
|
@ -0,0 +1,712 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import threading
|
||||
import time
|
||||
from datetime import date, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from pydantic import BaseModel, Field
|
||||
from sklearn.preprocessing import MinMaxScaler
|
||||
from torch.utils.data import DataLoader, TensorDataset
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.chrome.options import Options
|
||||
from selenium.webdriver.common.by import By
|
||||
from selenium.webdriver.support.ui import WebDriverWait
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DB_PATH = Path(os.getenv("NBEATS_DB_PATH", ROOT / "data deploy hasil.sqlite"))
|
||||
MODEL_PATH = Path(os.getenv("NBEATS_MODEL_PATH", ROOT / "best_nbeats_model.pth"))
|
||||
HYPERPARAMETERS_PATH = Path(
|
||||
os.getenv("NBEATS_HYPERPARAMETERS_PATH", ROOT / "best_nbeats_hyperparameters.json")
|
||||
)
|
||||
TABLE_NAME = "data_deploy_hasil"
|
||||
VALUE_COLUMNS = ("Harga Medium 1", "Harga Medium 2", "rata2")
|
||||
DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
MODEL_LOCK = threading.RLock()
|
||||
UPDATE_LOCK = threading.Lock()
|
||||
SCRAPING_URL = "https://www.bi.go.id/hargapangan/TabelHarga/PasarTradisionalDaerah"
|
||||
SCRAPING_COMMODITIES = ("Beras Kualitas Medium I", "Beras Kualitas Medium II")
|
||||
SCRAPING_PROVINCE = "Jawa Timur"
|
||||
SCRAPING_REGENCY = "Kab. Banyuwangi"
|
||||
FORECAST_HISTORY_DAYS = 90
|
||||
|
||||
|
||||
def trend_basis(theta: torch.Tensor, size: int) -> torch.Tensor:
|
||||
time = torch.linspace(0, 1, steps=size, device=theta.device)
|
||||
basis = torch.stack([time**i for i in range(theta.shape[1])], dim=0)
|
||||
return theta @ basis
|
||||
|
||||
|
||||
def seasonality_basis(theta: torch.Tensor, size: int) -> torch.Tensor:
|
||||
harmonics = theta.shape[1] // 2
|
||||
time = torch.linspace(0, 2 * np.pi, steps=size, device=theta.device)
|
||||
sine = torch.stack([torch.sin((i + 1) * time) for i in range(harmonics)], dim=0)
|
||||
cosine = torch.stack([torch.cos((i + 1) * time) for i in range(harmonics)], dim=0)
|
||||
return theta @ torch.cat([sine, cosine], dim=0)
|
||||
|
||||
|
||||
class NBeatsBlock(nn.Module):
|
||||
def __init__(self, input_size: int, output_size: int, hidden_size: int,
|
||||
theta_size: int, basis_type: str) -> None:
|
||||
super().__init__()
|
||||
self.input_size = input_size
|
||||
self.output_size = output_size
|
||||
self.basis_type = basis_type
|
||||
self.fc = nn.Sequential(
|
||||
nn.Linear(input_size, hidden_size), nn.GELU(),
|
||||
nn.Linear(hidden_size, hidden_size), nn.GELU(),
|
||||
nn.Linear(hidden_size, hidden_size), nn.GELU(),
|
||||
nn.Linear(hidden_size, hidden_size), nn.GELU(),
|
||||
)
|
||||
self.theta_back = nn.Linear(hidden_size, theta_size)
|
||||
self.theta_fore = nn.Linear(hidden_size, theta_size)
|
||||
|
||||
def forward(self, values: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
hidden = self.fc(values)
|
||||
theta_back = self.theta_back(hidden)
|
||||
theta_fore = self.theta_fore(hidden)
|
||||
if self.basis_type == "trend":
|
||||
return trend_basis(theta_back, self.input_size), trend_basis(theta_fore, self.output_size)
|
||||
return seasonality_basis(theta_back, self.input_size), seasonality_basis(theta_fore, self.output_size)
|
||||
|
||||
|
||||
class NBeatsStack(nn.Module):
|
||||
def __init__(self, input_size: int, output_size: int, hidden_size: int,
|
||||
theta_size: int, blocks: int, basis_type: str) -> None:
|
||||
super().__init__()
|
||||
self.blocks = nn.ModuleList(
|
||||
NBeatsBlock(input_size, output_size, hidden_size, theta_size, basis_type)
|
||||
for _ in range(blocks)
|
||||
)
|
||||
|
||||
def forward(self, values: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor, list[torch.Tensor], list[torch.Tensor]]:
|
||||
residual = values
|
||||
forecast = torch.zeros((values.size(0), self.blocks[0].output_size), device=values.device)
|
||||
backcasts, inputs = [], []
|
||||
for block in self.blocks:
|
||||
backcast, block_forecast = block(residual)
|
||||
inputs.append(residual.detach())
|
||||
backcasts.append(backcast)
|
||||
residual = residual - backcast
|
||||
forecast = forecast + block_forecast
|
||||
return residual, forecast, backcasts, inputs
|
||||
|
||||
|
||||
class NBeats(nn.Module):
|
||||
def __init__(self, input_size: int, output_size: int, hidden_size: int,
|
||||
n_stacks_trend: int, n_stacks_seasonal: int,
|
||||
blocks_per_stack: int, theta_size_trend: int,
|
||||
theta_size_season: int) -> None:
|
||||
super().__init__()
|
||||
self.output_size = output_size
|
||||
self.trend_stacks = nn.ModuleList(
|
||||
NBeatsStack(input_size, output_size, hidden_size, theta_size_trend,
|
||||
blocks_per_stack, "trend") for _ in range(n_stacks_trend)
|
||||
)
|
||||
self.season_stacks = nn.ModuleList(
|
||||
NBeatsStack(input_size, output_size, hidden_size, theta_size_season * 2,
|
||||
blocks_per_stack, "seasonality") for _ in range(n_stacks_seasonal)
|
||||
)
|
||||
|
||||
def forward(self, values: torch.Tensor) -> tuple[torch.Tensor, list[torch.Tensor], list[torch.Tensor]]:
|
||||
residual = values
|
||||
forecast = torch.zeros((values.size(0), self.output_size), device=values.device)
|
||||
backcasts, inputs = [], []
|
||||
for stack in [*self.trend_stacks, *self.season_stacks]:
|
||||
residual, stack_forecast, stack_backcasts, stack_inputs = stack(residual)
|
||||
forecast = forecast + stack_forecast
|
||||
backcasts.extend(stack_backcasts)
|
||||
inputs.extend(stack_inputs)
|
||||
return forecast, backcasts, inputs
|
||||
|
||||
|
||||
def load_hyperparameters() -> dict[str, Any]:
|
||||
# Load tuned parameters when available; otherwise use the built-in defaults.
|
||||
if HYPERPARAMETERS_PATH.exists():
|
||||
return json.loads(HYPERPARAMETERS_PATH.read_text(encoding="utf-8"))
|
||||
return {
|
||||
"input_size": 7, "output_size": 1, "hidden_size": 512,
|
||||
"n_stacks_trend": 3, "n_stacks_seasonal": 4,
|
||||
"blocks_per_stack": 3, "theta_size_trend": 2, "theta_size_season": 4,
|
||||
}
|
||||
|
||||
|
||||
def build_model(hyperparameters: dict[str, Any]) -> NBeats:
|
||||
# Construct the model architecture from the selected hyperparameters.
|
||||
model = NBeats(
|
||||
input_size=int(hyperparameters["input_size"]),
|
||||
output_size=int(hyperparameters["output_size"]),
|
||||
hidden_size=int(hyperparameters["hidden_size"]),
|
||||
n_stacks_trend=int(hyperparameters["n_stacks_trend"]),
|
||||
n_stacks_seasonal=int(hyperparameters["n_stacks_seasonal"]),
|
||||
blocks_per_stack=int(hyperparameters["blocks_per_stack"]),
|
||||
theta_size_trend=int(hyperparameters["theta_size_trend"]),
|
||||
theta_size_season=int(hyperparameters["theta_size_season"]),
|
||||
).to(DEVICE)
|
||||
return model
|
||||
|
||||
|
||||
def clean_data(data: pd.DataFrame, value_column: str) -> pd.DataFrame:
|
||||
# Normalize dates and values before sorting and removing duplicate dates.
|
||||
data["Tanggal"] = pd.to_datetime(data["Tanggal"], errors="coerce")
|
||||
data[value_column] = pd.to_numeric(data[value_column], errors="coerce")
|
||||
data = (
|
||||
data
|
||||
.dropna(subset=["Tanggal"])
|
||||
.sort_values("Tanggal")
|
||||
.drop_duplicates(subset=["Tanggal"], keep="last")
|
||||
.reset_index(drop=True)
|
||||
)
|
||||
|
||||
if data.empty:
|
||||
return data
|
||||
|
||||
window = 3
|
||||
previous_mean = (
|
||||
data[value_column]
|
||||
.shift(1)
|
||||
.rolling(window=window, min_periods=1)
|
||||
.mean()
|
||||
)
|
||||
next_mean = (
|
||||
data[value_column]
|
||||
.shift(-1)
|
||||
.rolling(window=window, min_periods=1)
|
||||
.mean()
|
||||
)
|
||||
data[value_column] = (
|
||||
# Fill missing values using neighboring trends, then interpolation as fallback.
|
||||
data[value_column]
|
||||
.fillna((previous_mean + next_mean) / 2)
|
||||
.fillna(previous_mean)
|
||||
.fillna(next_mean)
|
||||
.interpolate(method="linear")
|
||||
.bfill()
|
||||
.ffill()
|
||||
)
|
||||
|
||||
q1 = data[value_column].quantile(0.25)
|
||||
q3 = data[value_column].quantile(0.75)
|
||||
iqr = q3 - q1
|
||||
lower_bound = q1 - 1.5 * iqr
|
||||
upper_bound = q3 + 1.5 * iqr
|
||||
outlier_mask = (
|
||||
(data[value_column] < lower_bound)
|
||||
| (data[value_column] > upper_bound)
|
||||
)
|
||||
# Replace IQR outliers and interpolate them using surrounding observations.
|
||||
data.loc[outlier_mask, value_column] = np.nan
|
||||
data[value_column] = (
|
||||
data[value_column]
|
||||
.interpolate(method="linear")
|
||||
.bfill()
|
||||
.ffill()
|
||||
)
|
||||
return data
|
||||
|
||||
|
||||
def read_data(value_column: str) -> pd.DataFrame:
|
||||
# Read one selected series for forecasting or retraining.
|
||||
if not DB_PATH.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Database tidak ditemukan: {DB_PATH}")
|
||||
try:
|
||||
with sqlite3.connect(DB_PATH) as connection:
|
||||
data = pd.read_sql_query(
|
||||
f'SELECT "Tanggal", "{value_column}" FROM "{TABLE_NAME}" ORDER BY "Tanggal"',
|
||||
connection,
|
||||
)
|
||||
except (sqlite3.Error, pd.errors.DatabaseError) as error:
|
||||
raise HTTPException(status_code=400, detail=f"Gagal membaca database: {error}") from error
|
||||
data = clean_data(data, value_column)
|
||||
if data.empty:
|
||||
raise HTTPException(status_code=422, detail="Tidak ada data numerik yang dapat digunakan.")
|
||||
if data[value_column].isna().any():
|
||||
raise HTTPException(status_code=422, detail="Data masih memiliki nilai kosong setelah preprocessing.")
|
||||
return data
|
||||
|
||||
|
||||
def read_all_data() -> pd.DataFrame:
|
||||
"""Read every column and row from SQLite without preprocessing."""
|
||||
# Keep the raw database response unchanged for GET /data.
|
||||
if not DB_PATH.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Database tidak ditemukan: {DB_PATH}")
|
||||
try:
|
||||
with sqlite3.connect(DB_PATH) as connection:
|
||||
return pd.read_sql_query(
|
||||
f'SELECT * FROM "{TABLE_NAME}"',
|
||||
connection,
|
||||
)
|
||||
except (sqlite3.Error, pd.errors.DatabaseError) as error:
|
||||
raise HTTPException(status_code=400, detail=f"Gagal membaca database: {error}") from error
|
||||
|
||||
|
||||
def preprocess_all_data(data: pd.DataFrame) -> pd.DataFrame:
|
||||
"""Clean dates and numeric columns, then add a sequential index."""
|
||||
# Validate the complete schema before applying transformations.
|
||||
required_columns = {"Tanggal", *VALUE_COLUMNS}
|
||||
missing_columns = required_columns.difference(data.columns)
|
||||
if missing_columns:
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail=f"Kolom wajib tidak ditemukan: {', '.join(sorted(missing_columns))}",
|
||||
)
|
||||
|
||||
data = data.copy()
|
||||
data["Tanggal"] = pd.to_datetime(data["Tanggal"], errors="coerce")
|
||||
for column in VALUE_COLUMNS:
|
||||
data[column] = pd.to_numeric(data[column], errors="coerce")
|
||||
data = (
|
||||
# Keep the latest record for duplicate dates after chronological sorting.
|
||||
data
|
||||
.dropna(subset=["Tanggal"])
|
||||
.sort_values("Tanggal")
|
||||
.drop_duplicates(subset=["Tanggal"], keep="last")
|
||||
.reset_index(drop=True)
|
||||
)
|
||||
if data.empty:
|
||||
raise HTTPException(status_code=422, detail="Tidak ada data dengan tanggal valid.")
|
||||
|
||||
for column in VALUE_COLUMNS:
|
||||
# Fill missing values first so IQR bounds are calculated from usable data.
|
||||
data[column] = (
|
||||
data[column]
|
||||
.interpolate(method="linear", limit_direction="both")
|
||||
.bfill()
|
||||
.ffill()
|
||||
)
|
||||
q1 = data[column].quantile(0.25)
|
||||
q3 = data[column].quantile(0.75)
|
||||
iqr = q3 - q1
|
||||
lower_bound = q1 - 1.5 * iqr
|
||||
upper_bound = q3 + 1.5 * iqr
|
||||
data.loc[
|
||||
(data[column] < lower_bound) | (data[column] > upper_bound),
|
||||
column,
|
||||
] = np.nan
|
||||
# Fill values identified as outliers and verify that no missing values remain.
|
||||
data[column] = data[column].interpolate(
|
||||
method="linear", limit_direction="both"
|
||||
).bfill().ffill()
|
||||
if data[column].isna().any():
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail=f"Kolom {column} tidak memiliki nilai valid untuk mengisi missing value.",
|
||||
)
|
||||
|
||||
data["Tanggal"] = data["Tanggal"].dt.date.astype(str)
|
||||
data.insert(0, "index", range(len(data)))
|
||||
return data
|
||||
|
||||
|
||||
def records_from_dataframe(data: pd.DataFrame) -> list[dict[str, Any]]:
|
||||
"""Convert a dataframe to JSON-safe records, including null values."""
|
||||
records = data.astype(object).where(pd.notna(data), None).to_dict(orient="records")
|
||||
for record in records:
|
||||
for key, value in record.items():
|
||||
if isinstance(value, (np.integer, np.floating)):
|
||||
record[key] = value.item()
|
||||
return records
|
||||
|
||||
|
||||
def _click_scraping_tree_item(driver: webdriver.Chrome, wait: WebDriverWait,
|
||||
tree_id: str, item_name: str) -> None:
|
||||
def find_checkbox(browser: webdriver.Chrome):
|
||||
rows = browser.find_elements(By.CSS_SELECTOR, f"#{tree_id} tr.dx-data-row")
|
||||
for row in rows:
|
||||
if row.text.replace("\u2003", " ").strip() == item_name:
|
||||
return row.find_element(By.CSS_SELECTOR, ".dx-select-checkbox")
|
||||
return False
|
||||
|
||||
checkbox = wait.until(find_checkbox)
|
||||
if checkbox.get_attribute("aria-checked") != "true":
|
||||
driver.execute_script(
|
||||
"arguments[0].scrollIntoView({block: 'center'});", checkbox
|
||||
)
|
||||
checkbox.click()
|
||||
wait.until(lambda browser: checkbox.get_attribute("aria-checked") == "true")
|
||||
|
||||
|
||||
def _set_scraping_date(driver: webdriver.Chrome, date_box_id: str, value: date) -> None:
|
||||
driver.execute_script(
|
||||
"window.jQuery(arguments[0]).dxDateBox('instance').option('value', arguments[1]);",
|
||||
f"#{date_box_id}",
|
||||
value.isoformat(),
|
||||
)
|
||||
|
||||
|
||||
def _read_scraped_prices(driver: webdriver.Chrome) -> pd.DataFrame:
|
||||
header_row = next(
|
||||
row for row in driver.find_elements(
|
||||
By.CSS_SELECTOR, ".dx-datagrid-headers tr.dx-header-row"
|
||||
)
|
||||
if any(
|
||||
"/" in (cell.get_attribute("aria-label") or "")
|
||||
for cell in row.find_elements(By.CSS_SELECTOR, 'td[role="columnheader"]')
|
||||
)
|
||||
)
|
||||
headers = [
|
||||
(cell.get_attribute("aria-label") or "").replace("Column ", "").strip()
|
||||
for cell in header_row.find_elements(By.CSS_SELECTOR, 'td[role="columnheader"]')
|
||||
if (cell.get_attribute("aria-label") or "").startswith("Column ")
|
||||
]
|
||||
label_rows: list[list[str]] = []
|
||||
value_rows: list[list[str]] = []
|
||||
for table in driver.find_elements(By.CSS_SELECTOR, ".dx-datagrid-rowsview table"):
|
||||
rows = table.find_elements(By.CSS_SELECTOR, "tbody tr.dx-data-row")
|
||||
parsed_rows = [
|
||||
[cell.text.replace("\u2003", " ").strip() for cell in row.find_elements(By.CSS_SELECTOR, "td")]
|
||||
for row in rows
|
||||
]
|
||||
if any(
|
||||
any(commodity in " ".join(row) for commodity in SCRAPING_COMMODITIES)
|
||||
for row in parsed_rows
|
||||
):
|
||||
label_rows = parsed_rows
|
||||
elif parsed_rows:
|
||||
value_rows = parsed_rows
|
||||
|
||||
if not headers or not label_rows or not value_rows:
|
||||
raise RuntimeError("Tabel harga belum menampilkan data.")
|
||||
|
||||
records: list[dict[str, str]] = []
|
||||
for label_row, value_row in zip(label_rows, value_rows):
|
||||
if len(label_row) < 2:
|
||||
continue
|
||||
commodity = label_row[1]
|
||||
commodity = commodity.split(" ", 1)[1] if commodity[:1].isdigit() else commodity
|
||||
if commodity not in SCRAPING_COMMODITIES:
|
||||
continue
|
||||
for header, value in zip(headers[2:], value_row[2:]):
|
||||
if value and value != "-":
|
||||
records.append({"komoditas": commodity, "tanggal": header, "harga": value})
|
||||
return pd.DataFrame(records)
|
||||
|
||||
|
||||
def scrape_missing_prices(start_date: date, end_date: date) -> pd.DataFrame:
|
||||
options = Options()
|
||||
options.add_argument("--headless=new")
|
||||
options.add_argument("--window-size=1920,1080")
|
||||
driver = webdriver.Chrome(options=options)
|
||||
wait = WebDriverWait(driver, 30)
|
||||
try:
|
||||
driver.get(SCRAPING_URL)
|
||||
wait.until(lambda browser: browser.find_element(By.TAG_NAME, "body"))
|
||||
for commodity in SCRAPING_COMMODITIES:
|
||||
_click_scraping_tree_item(driver, wait, "CommodityTree", commodity)
|
||||
_click_scraping_tree_item(driver, wait, "cboProvince", SCRAPING_PROVINCE)
|
||||
wait.until(
|
||||
lambda browser: browser.find_elements(
|
||||
By.CSS_SELECTOR, "#cboRegency tr.dx-data-row"
|
||||
)
|
||||
)
|
||||
_click_scraping_tree_item(driver, wait, "cboRegency", SCRAPING_REGENCY)
|
||||
_set_scraping_date(driver, "dboDateMulai", start_date)
|
||||
_set_scraping_date(driver, "dboDateSelesai", end_date)
|
||||
driver.find_element(By.ID, "btnReport").click()
|
||||
wait.until(
|
||||
lambda browser: browser.find_elements(
|
||||
By.CSS_SELECTOR,
|
||||
".dx-datagrid-rowsview table tbody tr.dx-data-row",
|
||||
)
|
||||
)
|
||||
time.sleep(1)
|
||||
return _read_scraped_prices(driver)
|
||||
finally:
|
||||
driver.quit()
|
||||
|
||||
|
||||
def merge_scraped_prices(existing: pd.DataFrame, scraped: pd.DataFrame) -> pd.DataFrame:
|
||||
expected_columns = ["Tanggal", "Harga Medium 1", "Harga Medium 2", "rata2"]
|
||||
current = existing[expected_columns].copy()
|
||||
current["Tanggal"] = pd.to_datetime(current["Tanggal"], errors="coerce")
|
||||
current = current.dropna(subset=["Tanggal"]).set_index("Tanggal")
|
||||
if not scraped.empty:
|
||||
incoming = scraped.copy()
|
||||
incoming["Tanggal"] = pd.to_datetime(
|
||||
incoming["tanggal"].astype(str).str.replace(" ", "", regex=False),
|
||||
format="%d/%m/%Y",
|
||||
errors="coerce",
|
||||
)
|
||||
incoming["harga"] = pd.to_numeric(
|
||||
incoming["harga"].astype(str)
|
||||
.str.replace(",", "", regex=False)
|
||||
.str.replace("Rp", "", regex=False)
|
||||
.str.strip(),
|
||||
errors="coerce",
|
||||
)
|
||||
incoming = incoming.dropna(subset=["Tanggal", "harga"])
|
||||
new_values = incoming.pivot_table(
|
||||
index="Tanggal",
|
||||
columns="komoditas",
|
||||
values="harga",
|
||||
aggfunc="last",
|
||||
).rename(columns={
|
||||
"Beras Kualitas Medium I": "Harga Medium 1",
|
||||
"Beras Kualitas Medium II": "Harga Medium 2",
|
||||
})
|
||||
new_values = new_values.reindex(columns=["Harga Medium 1", "Harga Medium 2"])
|
||||
current = pd.concat([current.drop(index=new_values.index, errors="ignore"), new_values])
|
||||
|
||||
if current.empty:
|
||||
raise HTTPException(status_code=422, detail="Tidak ada data tanggal yang valid.")
|
||||
current = current.sort_index()
|
||||
full_dates = pd.date_range(current.index.min(), current.index.max(), freq="D")
|
||||
current = current.reindex(full_dates)
|
||||
current.index.name = "Tanggal"
|
||||
price_columns = ["Harga Medium 1", "Harga Medium 2"]
|
||||
current[price_columns] = current[price_columns].interpolate(
|
||||
method="linear", limit_direction="both"
|
||||
)
|
||||
current["rata2"] = current[price_columns].mean(axis=1)
|
||||
return current.reset_index()[expected_columns]
|
||||
|
||||
|
||||
def update_database_from_scraping() -> dict[str, Any]:
|
||||
existing = read_all_data()
|
||||
if existing.empty:
|
||||
raise HTTPException(status_code=422, detail="Database tidak memiliki data awal.")
|
||||
existing["Tanggal"] = pd.to_datetime(existing["Tanggal"], errors="coerce")
|
||||
existing = existing.dropna(subset=["Tanggal"])
|
||||
last_date = existing["Tanggal"].max().date()
|
||||
today = date.today()
|
||||
allowed_date = today if datetime.now().hour >= 13 else today - timedelta(days=1)
|
||||
if last_date >= allowed_date:
|
||||
return {
|
||||
"status": "up_to_date",
|
||||
"message": "Data sudah diperbarui sampai batas waktu yang diizinkan.",
|
||||
"last_date": last_date.isoformat(),
|
||||
"allowed_date": allowed_date.isoformat(),
|
||||
"updated_rows": 0,
|
||||
}
|
||||
|
||||
start_date = last_date + timedelta(days=1)
|
||||
scraped = scrape_missing_prices(start_date, allowed_date)
|
||||
if scraped.empty:
|
||||
raise HTTPException(status_code=502, detail="Situs sumber tidak mengembalikan data.")
|
||||
merged = merge_scraped_prices(existing, scraped)
|
||||
with sqlite3.connect(DB_PATH) as connection:
|
||||
merged.to_sql(TABLE_NAME, connection, if_exists="replace", index=False)
|
||||
return {
|
||||
"status": "updated",
|
||||
"message": "Data berhasil diperbarui dari situs sumber.",
|
||||
"start_date": start_date.isoformat(),
|
||||
"last_date": last_date.isoformat(),
|
||||
"allowed_date": allowed_date.isoformat(),
|
||||
"updated_rows": len(scraped),
|
||||
"total_rows": len(merged),
|
||||
}
|
||||
|
||||
|
||||
def dataset(values: np.ndarray, input_size: int) -> tuple[np.ndarray, np.ndarray]:
|
||||
# Convert the time series into sliding windows and next-step targets.
|
||||
if len(values) <= input_size:
|
||||
raise ValueError(f"Minimal {input_size + 1} data diperlukan.")
|
||||
inputs = np.array([values[i:i + input_size] for i in range(len(values) - input_size)])
|
||||
targets = np.array([values[i + input_size] for i in range(len(values) - input_size)])[:, None]
|
||||
return inputs, targets
|
||||
|
||||
|
||||
class ForecastRequest(BaseModel):
|
||||
horizon: int = Field(default=7, ge=1, le=90)
|
||||
value_column: Literal["rata2", "Harga Medium 1", "Harga Medium 2"] = "rata2"
|
||||
|
||||
|
||||
class RetrainRequest(BaseModel):
|
||||
train_ratio: float = Field(default=0.8, gt=0.5, lt=1.0)
|
||||
epochs: int = Field(default=20, ge=1, le=1000)
|
||||
batch_size: int = Field(default=32, ge=1, le=1024)
|
||||
learning_rate: float = Field(default=0.0001, gt=0, le=1)
|
||||
value_column: Literal["rata2", "Harga Medium 1", "Harga Medium 2"] = "rata2"
|
||||
|
||||
|
||||
app = FastAPI(title="N-BEATS Forecast API", version="1.0.0")
|
||||
HYPERPARAMETERS = load_hyperparameters()
|
||||
MODEL = build_model(HYPERPARAMETERS)
|
||||
CHECKPOINT: dict[str, Any] | None = None
|
||||
|
||||
|
||||
def load_saved_model() -> None:
|
||||
global CHECKPOINT
|
||||
if not MODEL_PATH.exists():
|
||||
return
|
||||
# Restore model weights and preprocessing scaler from the saved checkpoint.
|
||||
checkpoint = torch.load(MODEL_PATH, map_location=DEVICE, weights_only=False)
|
||||
MODEL.load_state_dict(checkpoint["model_state_dict"])
|
||||
MODEL.eval()
|
||||
CHECKPOINT = checkpoint
|
||||
|
||||
|
||||
try:
|
||||
load_saved_model()
|
||||
except Exception as error:
|
||||
raise RuntimeError(f"Checkpoint tidak kompatibel dengan arsitektur model: {error}") from error
|
||||
|
||||
|
||||
def scaler_from_checkpoint(data: pd.DataFrame, value_column: str) -> MinMaxScaler:
|
||||
# Reuse the training scaler when it matches the requested value column.
|
||||
scaler = MinMaxScaler()
|
||||
saved = CHECKPOINT.get("scaler") if CHECKPOINT else None
|
||||
if saved and saved.get("value_column") == value_column:
|
||||
scaler.min_ = np.asarray(saved["min_"], dtype=float)
|
||||
scaler.scale_ = np.asarray(saved["scale_"], dtype=float)
|
||||
scaler.data_min_ = np.asarray(saved["data_min_"], dtype=float)
|
||||
scaler.data_max_ = np.asarray(saved["data_max_"], dtype=float)
|
||||
scaler.data_range_ = np.asarray(saved["data_range_"], dtype=float)
|
||||
scaler.n_features_in_ = 1
|
||||
scaler.n_samples_seen_ = int(saved.get("n_samples_seen_", len(data)))
|
||||
return scaler
|
||||
scaler.fit(data[[value_column]].to_numpy())
|
||||
return scaler
|
||||
|
||||
|
||||
# Endpoint GET /data: mengambil seluruh data mentah dari SQLite.
|
||||
@app.get("/data")
|
||||
def all_data() -> dict[str, Any]:
|
||||
"""Return all rows and columns exactly as stored in SQLite."""
|
||||
# Serialize the raw dataframe into JSON-safe records for the website.
|
||||
data = read_all_data()
|
||||
return {"total_rows": len(data), "columns": list(data.columns), "rows": records_from_dataframe(data)}
|
||||
|
||||
|
||||
# Endpoint GET /data/preprocessed: mengambil seluruh data setelah preprocessing.
|
||||
@app.get("/data/preprocessed")
|
||||
def preprocess_data() -> dict[str, Any]:
|
||||
"""Return the complete dataset after date, missing-value, and outlier handling."""
|
||||
# Apply the complete cleaning pipeline before returning website data.
|
||||
data = preprocess_all_data(read_all_data())
|
||||
return {
|
||||
"total_rows": len(data),
|
||||
"columns": list(data.columns),
|
||||
"rows": records_from_dataframe(data),
|
||||
}
|
||||
|
||||
|
||||
# Endpoint POST /data/update: mengambil data terbaru dan memperbarui SQLite.
|
||||
@app.post("/data/update")
|
||||
def update_data() -> dict[str, Any]:
|
||||
# Serialize updates so concurrent requests cannot overwrite each other.
|
||||
if not DB_PATH.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Database tidak ditemukan: {DB_PATH}")
|
||||
if not UPDATE_LOCK.acquire(blocking=False):
|
||||
raise HTTPException(status_code=409, detail="Proses update data sedang berjalan.")
|
||||
try:
|
||||
try:
|
||||
return update_database_from_scraping()
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as error:
|
||||
raise HTTPException(
|
||||
status_code=502,
|
||||
detail=f"Gagal mengambil atau menyimpan data scraping: {error}",
|
||||
) from error
|
||||
finally:
|
||||
UPDATE_LOCK.release()
|
||||
|
||||
|
||||
# Endpoint POST /forecast: menghasilkan prediksi harga menggunakan model N-BEATS.
|
||||
@app.post("/forecast")
|
||||
def forecast(request: ForecastRequest) -> dict[str, Any]:
|
||||
# Normalize the latest 90 calendar days, predict one step repeatedly, and inverse-transform results.
|
||||
data = read_data(request.value_column)
|
||||
last_available_date = data["Tanggal"].iloc[-1]
|
||||
first_forecast_date = last_available_date - timedelta(days=FORECAST_HISTORY_DAYS - 1)
|
||||
data = data[data["Tanggal"] >= first_forecast_date].reset_index(drop=True)
|
||||
input_size = int(HYPERPARAMETERS["input_size"])
|
||||
if len(data) < input_size:
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail=(
|
||||
f"Minimal {input_size} data valid dari {FORECAST_HISTORY_DAYS} hari terakhir "
|
||||
"diperlukan."
|
||||
),
|
||||
)
|
||||
scaler = scaler_from_checkpoint(data, request.value_column)
|
||||
history = scaler.transform(data[[request.value_column]].to_numpy()).ravel().tolist()
|
||||
predictions: list[float] = []
|
||||
with MODEL_LOCK, torch.no_grad():
|
||||
MODEL.eval()
|
||||
for _ in range(request.horizon):
|
||||
# Use the latest input window so each prediction feeds the next one.
|
||||
values = torch.tensor([history[-input_size:]], dtype=torch.float32, device=DEVICE)
|
||||
predicted = float(MODEL(values)[0][0, 0].cpu())
|
||||
history.append(predicted)
|
||||
predictions.append(float(scaler.inverse_transform([[predicted]])[0, 0]))
|
||||
last_date = data["Tanggal"].iloc[-1].date()
|
||||
return {"value_column": request.value_column, "horizon": request.horizon,
|
||||
"forecast": [{"date": (last_date + timedelta(days=i)).isoformat(), "value": value}
|
||||
for i, value in enumerate(predictions, start=1)]}
|
||||
|
||||
|
||||
# Endpoint POST /retrain: melatih ulang model dan menyimpan checkpoint terbaru.
|
||||
@app.post("/retrain")
|
||||
def retrain(request: RetrainRequest) -> dict[str, Any]:
|
||||
# Prepare sliding-window training and validation data from the selected series.
|
||||
data = read_data(request.value_column)
|
||||
input_size = int(HYPERPARAMETERS["input_size"])
|
||||
scaler = MinMaxScaler().fit(data[[request.value_column]].iloc[: int(len(data) * request.train_ratio)])
|
||||
scaled = scaler.transform(data[[request.value_column]]).ravel()
|
||||
inputs, targets = dataset(scaled, input_size)
|
||||
split_index = max(1, int(len(inputs) * request.train_ratio))
|
||||
if split_index >= len(inputs):
|
||||
raise HTTPException(status_code=422, detail="Split menghasilkan data validasi kosong.")
|
||||
train_inputs, val_inputs = inputs[:split_index], inputs[split_index:]
|
||||
train_targets, val_targets = targets[:split_index], targets[split_index:]
|
||||
train_loader = DataLoader(TensorDataset(torch.tensor(train_inputs, dtype=torch.float32),
|
||||
torch.tensor(train_targets, dtype=torch.float32)),
|
||||
batch_size=request.batch_size, shuffle=False)
|
||||
model = build_model(HYPERPARAMETERS)
|
||||
optimizer = torch.optim.AdamW(model.parameters(), lr=request.learning_rate, weight_decay=1e-4)
|
||||
loss_fn = nn.MSELoss()
|
||||
history: list[dict[str, float]] = []
|
||||
for epoch in range(request.epochs):
|
||||
# Optimize model weights on training batches, then measure validation loss.
|
||||
model.train()
|
||||
train_loss = 0.0
|
||||
for batch_inputs, batch_targets in train_loader:
|
||||
batch_inputs, batch_targets = batch_inputs.to(DEVICE), batch_targets.to(DEVICE)
|
||||
prediction, backcasts, block_inputs = model(batch_inputs)
|
||||
loss = loss_fn(prediction, batch_targets)
|
||||
loss = loss + float(HYPERPARAMETERS.get("alpha", 0.1)) * sum(
|
||||
loss_fn(backcast, block_input) for backcast, block_input in zip(backcasts, block_inputs)
|
||||
) / len(backcasts)
|
||||
optimizer.zero_grad()
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
train_loss += loss.item() * len(batch_inputs)
|
||||
model.eval()
|
||||
with torch.no_grad():
|
||||
validation_prediction = model(torch.tensor(val_inputs, dtype=torch.float32, device=DEVICE))[0]
|
||||
validation_loss = loss_fn(validation_prediction, torch.tensor(val_targets, dtype=torch.float32, device=DEVICE)).item()
|
||||
history.append({"epoch": epoch + 1, "train_loss": train_loss / len(train_inputs), "validation_loss": validation_loss})
|
||||
checkpoint = {"model_state_dict": model.state_dict(), "optimizer_state_dict": optimizer.state_dict(),
|
||||
"epoch": request.epochs, "best_val_loss": history[-1]["validation_loss"],
|
||||
"hyperparameters": HYPERPARAMETERS, "scaler": {"value_column": request.value_column,
|
||||
**{key: getattr(scaler, key).tolist() for key in ("min_", "scale_", "data_min_", "data_max_", "data_range_")},
|
||||
"n_samples_seen_": int(scaler.n_samples_seen_)}}
|
||||
temporary_path = MODEL_PATH.with_suffix(".tmp")
|
||||
with MODEL_LOCK:
|
||||
# Atomically replace the checkpoint and activate the newly trained model.
|
||||
torch.save(checkpoint, temporary_path)
|
||||
temporary_path.replace(MODEL_PATH)
|
||||
MODEL.load_state_dict(model.state_dict())
|
||||
MODEL.eval()
|
||||
globals()["CHECKPOINT"] = checkpoint
|
||||
return {"message": "Model berhasil diretrain.", "train_rows": len(train_inputs),
|
||||
"validation_rows": len(val_inputs), "train_ratio": request.train_ratio,
|
||||
"epochs": request.epochs, "final_validation_loss": history[-1]["validation_loss"]}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
|
||||
uvicorn.run(app, host="0.0.0.0", port=8000, reload=False)
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
# Logs
|
||||
logs
|
||||
*.log
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
pnpm-debug.log*
|
||||
lerna-debug.log*
|
||||
|
||||
node_modules
|
||||
dist
|
||||
dist-ssr
|
||||
*.local
|
||||
|
||||
# Editor directories and files
|
||||
.vscode/*
|
||||
!.vscode/extensions.json
|
||||
.idea
|
||||
.DS_Store
|
||||
*.suo
|
||||
*.ntvs*
|
||||
*.njsproj
|
||||
*.sln
|
||||
*.sw?
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
{
|
||||
"$schema": "./node_modules/oxlint/configuration_schema.json",
|
||||
"plugins": ["react", "oxc"],
|
||||
"rules": {
|
||||
"react/rules-of-hooks": "error",
|
||||
"react/only-export-components": ["warn", { "allowConstantExport": true }]
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
# React + Vite
|
||||
|
||||
This template provides a minimal setup to get React working in Vite with HMR and some Oxlint rules.
|
||||
|
||||
Currently, two official plugins are available:
|
||||
|
||||
- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react) uses [Oxc](https://oxc.rs)
|
||||
- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react-swc) uses [SWC](https://swc.rs/)
|
||||
|
||||
## React Compiler
|
||||
|
||||
The React Compiler is not enabled on this template because of its impact on dev & build performances. To add it, see [this documentation](https://react.dev/learn/react-compiler/installation).
|
||||
|
||||
## Expanding the Oxlint configuration
|
||||
|
||||
If you are developing a production application, we recommend using TypeScript with type-aware lint rules enabled. Check out the [TS template](https://github.com/vitejs/vite/tree/main/packages/create-vite/template-react-ts) for information on how to integrate TypeScript and Oxlint's TypeScript related rules in your project.
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>web</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
<script type="module" src="/src/main.jsx"></script>
|
||||
</body>
|
||||
</html>
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,25 @@
|
|||
{
|
||||
"name": "web",
|
||||
"private": true,
|
||||
"version": "0.0.0",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"build": "vite build",
|
||||
"lint": "oxlint",
|
||||
"preview": "vite preview"
|
||||
},
|
||||
"dependencies": {
|
||||
"lucide-react": "^1.50.0",
|
||||
"react": "^19.2.8",
|
||||
"react-dom": "^19.2.8",
|
||||
"recharts": "^3.10.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/react": "^19.2.18",
|
||||
"@types/react-dom": "^19.2.7",
|
||||
"@vitejs/plugin-react": "^6.1.1",
|
||||
"oxlint": "^1.81.0",
|
||||
"vite": "^8.3.0"
|
||||
}
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 9.3 KiB |
|
|
@ -0,0 +1,24 @@
|
|||
<svg xmlns="http://www.w3.org/2000/svg">
|
||||
<symbol id="bluesky-icon" viewBox="0 0 16 17">
|
||||
<g clip-path="url(#bluesky-clip)"><path fill="#08060d" d="M7.75 7.735c-.693-1.348-2.58-3.86-4.334-5.097-1.68-1.187-2.32-.981-2.74-.79C.188 2.065.1 2.812.1 3.251s.241 3.602.398 4.13c.52 1.744 2.367 2.333 4.07 2.145-2.495.37-4.71 1.278-1.805 4.512 3.196 3.309 4.38-.71 4.987-2.746.608 2.036 1.307 5.91 4.93 2.746 2.72-2.746.747-4.143-1.747-4.512 1.702.189 3.55-.4 4.07-2.145.156-.528.397-3.691.397-4.13s-.088-1.186-.575-1.406c-.42-.19-1.06-.395-2.741.79-1.755 1.24-3.64 3.752-4.334 5.099"/></g>
|
||||
<defs><clipPath id="bluesky-clip"><path fill="#fff" d="M.1.85h15.3v15.3H.1z"/></clipPath></defs>
|
||||
</symbol>
|
||||
<symbol id="discord-icon" viewBox="0 0 20 19">
|
||||
<path fill="#08060d" d="M16.224 3.768a14.5 14.5 0 0 0-3.67-1.153c-.158.286-.343.67-.47.976a13.5 13.5 0 0 0-4.067 0c-.128-.306-.317-.69-.476-.976A14.4 14.4 0 0 0 3.868 3.77C1.546 7.28.916 10.703 1.231 14.077a14.7 14.7 0 0 0 4.5 2.306q.545-.748.965-1.587a9.5 9.5 0 0 1-1.518-.74q.191-.14.372-.293c2.927 1.369 6.107 1.369 8.999 0q.183.152.372.294-.723.437-1.52.74.418.838.963 1.588a14.6 14.6 0 0 0 4.504-2.308c.37-3.911-.63-7.302-2.644-10.309m-9.13 8.234c-.878 0-1.599-.82-1.599-1.82 0-.998.705-1.82 1.6-1.82.894 0 1.614.82 1.599 1.82.001 1-.705 1.82-1.6 1.82m5.91 0c-.878 0-1.599-.82-1.599-1.82 0-.998.705-1.82 1.6-1.82.893 0 1.614.82 1.599 1.82 0 1-.706 1.82-1.6 1.82"/>
|
||||
</symbol>
|
||||
<symbol id="documentation-icon" viewBox="0 0 21 20">
|
||||
<path fill="none" stroke="#aa3bff" stroke-linecap="round" stroke-linejoin="round" stroke-width="1.35" d="m15.5 13.333 1.533 1.322c.645.555.967.833.967 1.178s-.322.623-.967 1.179L15.5 18.333m-3.333-5-1.534 1.322c-.644.555-.966.833-.966 1.178s.322.623.966 1.179l1.534 1.321"/>
|
||||
<path fill="none" stroke="#aa3bff" stroke-linecap="round" stroke-linejoin="round" stroke-width="1.35" d="M17.167 10.836v-4.32c0-1.41 0-2.117-.224-2.68-.359-.906-1.118-1.621-2.08-1.96-.599-.21-1.349-.21-2.848-.21-2.623 0-3.935 0-4.983.369-1.684.591-3.013 1.842-3.641 3.428C3 6.449 3 7.684 3 10.154v2.122c0 2.558 0 3.838.706 4.726q.306.383.713.671c.76.536 1.79.64 3.581.66"/>
|
||||
<path fill="none" stroke="#aa3bff" stroke-linecap="round" stroke-linejoin="round" stroke-width="1.35" d="M3 10a2.78 2.78 0 0 1 2.778-2.778c.555 0 1.209.097 1.748-.047.48-.129.854-.503.982-.982.145-.54.048-1.194.048-1.749a2.78 2.78 0 0 1 2.777-2.777"/>
|
||||
</symbol>
|
||||
<symbol id="github-icon" viewBox="0 0 19 19">
|
||||
<path fill="#08060d" fill-rule="evenodd" d="M9.356 1.85C5.05 1.85 1.57 5.356 1.57 9.694a7.84 7.84 0 0 0 5.324 7.44c.387.079.528-.168.528-.376 0-.182-.013-.805-.013-1.454-2.165.467-2.616-.935-2.616-.935-.349-.91-.864-1.143-.864-1.143-.71-.48.051-.48.051-.48.787.051 1.2.805 1.2.805.695 1.194 1.817.857 2.268.649.064-.507.27-.857.49-1.052-1.728-.182-3.545-.857-3.545-3.87 0-.857.31-1.558.8-2.104-.078-.195-.349-1 .077-2.078 0 0 .657-.208 2.14.805a7.5 7.5 0 0 1 1.946-.26c.657 0 1.328.092 1.946.26 1.483-1.013 2.14-.805 2.14-.805.426 1.078.155 1.883.078 2.078.502.546.799 1.247.799 2.104 0 3.013-1.818 3.675-3.558 3.87.284.247.528.714.528 1.454 0 1.052-.012 1.896-.012 2.156 0 .208.142.455.528.377a7.84 7.84 0 0 0 5.324-7.441c.013-4.338-3.48-7.844-7.773-7.844" clip-rule="evenodd"/>
|
||||
</symbol>
|
||||
<symbol id="social-icon" viewBox="0 0 20 20">
|
||||
<path fill="none" stroke="#aa3bff" stroke-linecap="round" stroke-linejoin="round" stroke-width="1.35" d="M12.5 6.667a4.167 4.167 0 1 0-8.334 0 4.167 4.167 0 0 0 8.334 0"/>
|
||||
<path fill="none" stroke="#aa3bff" stroke-linecap="round" stroke-linejoin="round" stroke-width="1.35" d="M2.5 16.667a5.833 5.833 0 0 1 8.75-5.053m3.837.474.513 1.035c.07.144.257.282.414.309l.93.155c.596.1.736.536.307.965l-.723.73a.64.64 0 0 0-.152.531l.207.903c.164.715-.213.991-.84.618l-.872-.52a.63.63 0 0 0-.577 0l-.872.52c-.624.373-1.003.094-.84-.618l.207-.903a.64.64 0 0 0-.152-.532l-.723-.729c-.426-.43-.289-.864.306-.964l.93-.156a.64.64 0 0 0 .412-.31l.513-1.034c.28-.562.735-.562 1.012 0"/>
|
||||
</symbol>
|
||||
<symbol id="x-icon" viewBox="0 0 19 19">
|
||||
<path fill="#08060d" fill-rule="evenodd" d="M1.893 1.98c.052.072 1.245 1.769 2.653 3.77l2.892 4.114c.183.261.333.48.333.486s-.068.089-.152.183l-.522.593-.765.867-3.597 4.087c-.375.426-.734.834-.798.905a1 1 0 0 0-.118.148c0 .01.236.017.664.017h.663l.729-.83c.4-.457.796-.906.879-.999a692 692 0 0 0 1.794-2.038c.034-.037.301-.34.594-.675l.551-.624.345-.392a7 7 0 0 1 .34-.374c.006 0 .93 1.306 2.052 2.903l2.084 2.965.045.063h2.275c1.87 0 2.273-.003 2.266-.021-.008-.02-1.098-1.572-3.894-5.547-2.013-2.862-2.28-3.246-2.273-3.266.008-.019.282-.332 2.085-2.38l2-2.274 1.567-1.782c.022-.028-.016-.03-.65-.03h-.674l-.3.342a871 871 0 0 1-1.782 2.025c-.067.075-.405.458-.75.852a100 100 0 0 1-.803.91c-.148.172-.299.344-.99 1.127-.304.343-.32.358-.345.327-.015-.019-.904-1.282-1.976-2.808L6.365 1.85H1.8zm1.782.91 8.078 11.294c.772 1.08 1.413 1.973 1.425 1.984.016.017.241.02 1.05.017l1.03-.004-2.694-3.766L7.796 5.75 5.722 2.852l-1.039-.004-1.039-.004z" clip-rule="evenodd"/>
|
||||
</symbol>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 4.9 KiB |
|
|
@ -0,0 +1,184 @@
|
|||
.counter {
|
||||
font-size: 16px;
|
||||
padding: 5px 10px;
|
||||
border-radius: 5px;
|
||||
color: var(--accent);
|
||||
background: var(--accent-bg);
|
||||
border: 2px solid transparent;
|
||||
transition: border-color 0.3s;
|
||||
margin-bottom: 24px;
|
||||
|
||||
&:hover {
|
||||
border-color: var(--accent-border);
|
||||
}
|
||||
&:focus-visible {
|
||||
outline: 2px solid var(--accent);
|
||||
outline-offset: 2px;
|
||||
}
|
||||
}
|
||||
|
||||
.hero {
|
||||
position: relative;
|
||||
|
||||
.base,
|
||||
.framework,
|
||||
.vite {
|
||||
inset-inline: 0;
|
||||
margin: 0 auto;
|
||||
}
|
||||
|
||||
.base {
|
||||
width: 170px;
|
||||
position: relative;
|
||||
z-index: 0;
|
||||
}
|
||||
|
||||
.framework,
|
||||
.vite {
|
||||
position: absolute;
|
||||
}
|
||||
|
||||
.framework {
|
||||
z-index: 1;
|
||||
top: 34px;
|
||||
height: 28px;
|
||||
transform: perspective(2000px) rotateZ(300deg) rotateX(44deg) rotateY(39deg)
|
||||
scale(1.4);
|
||||
}
|
||||
|
||||
.vite {
|
||||
z-index: 0;
|
||||
top: 107px;
|
||||
height: 26px;
|
||||
width: auto;
|
||||
transform: perspective(2000px) rotateZ(300deg) rotateX(40deg) rotateY(39deg)
|
||||
scale(0.8);
|
||||
}
|
||||
}
|
||||
|
||||
#center {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 25px;
|
||||
place-content: center;
|
||||
place-items: center;
|
||||
flex-grow: 1;
|
||||
|
||||
@media (max-width: 1024px) {
|
||||
padding: 32px 20px 24px;
|
||||
gap: 18px;
|
||||
}
|
||||
}
|
||||
|
||||
#next-steps {
|
||||
display: flex;
|
||||
border-top: 1px solid var(--border);
|
||||
text-align: left;
|
||||
|
||||
& > div {
|
||||
flex: 1 1 0;
|
||||
padding: 32px;
|
||||
@media (max-width: 1024px) {
|
||||
padding: 24px 20px;
|
||||
}
|
||||
}
|
||||
|
||||
.icon {
|
||||
margin-bottom: 16px;
|
||||
width: 22px;
|
||||
height: 22px;
|
||||
}
|
||||
|
||||
@media (max-width: 1024px) {
|
||||
flex-direction: column;
|
||||
text-align: center;
|
||||
}
|
||||
}
|
||||
|
||||
#docs {
|
||||
border-right: 1px solid var(--border);
|
||||
|
||||
@media (max-width: 1024px) {
|
||||
border-right: none;
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
}
|
||||
|
||||
#next-steps ul {
|
||||
list-style: none;
|
||||
padding: 0;
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
margin: 32px 0 0;
|
||||
|
||||
.logo {
|
||||
height: 18px;
|
||||
}
|
||||
|
||||
a {
|
||||
color: var(--text-h);
|
||||
font-size: 16px;
|
||||
border-radius: 6px;
|
||||
background: var(--social-bg);
|
||||
display: flex;
|
||||
padding: 6px 12px;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
text-decoration: none;
|
||||
transition: box-shadow 0.3s;
|
||||
|
||||
&:hover {
|
||||
box-shadow: var(--shadow);
|
||||
}
|
||||
.button-icon {
|
||||
height: 18px;
|
||||
width: 18px;
|
||||
}
|
||||
}
|
||||
|
||||
@media (max-width: 1024px) {
|
||||
margin-top: 20px;
|
||||
flex-wrap: wrap;
|
||||
justify-content: center;
|
||||
|
||||
li {
|
||||
flex: 1 1 calc(50% - 8px);
|
||||
}
|
||||
|
||||
a {
|
||||
width: 100%;
|
||||
justify-content: center;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#spacer {
|
||||
height: 88px;
|
||||
border-top: 1px solid var(--border);
|
||||
@media (max-width: 1024px) {
|
||||
height: 48px;
|
||||
}
|
||||
}
|
||||
|
||||
.ticks {
|
||||
position: relative;
|
||||
width: 100%;
|
||||
|
||||
&::before,
|
||||
&::after {
|
||||
content: '';
|
||||
position: absolute;
|
||||
top: -4.5px;
|
||||
border: 5px solid transparent;
|
||||
}
|
||||
|
||||
&::before {
|
||||
left: 0;
|
||||
border-left-color: var(--border);
|
||||
}
|
||||
&::after {
|
||||
right: 0;
|
||||
border-right-color: var(--border);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,695 @@
|
|||
import { useState, useEffect, useMemo, useCallback, useRef } from 'react';
|
||||
import {
|
||||
LineChart, Line, XAxis, YAxis, CartesianGrid, Tooltip,
|
||||
Legend, ResponsiveContainer, ReferenceLine
|
||||
} from 'recharts';
|
||||
import {
|
||||
AlertCircle, TrendingUp, RefreshCw, CheckCircle2, XCircle, WifiOff,
|
||||
LineChart as LineChartIcon, Table2
|
||||
} from 'lucide-react';
|
||||
|
||||
// ─── Toast Notification Component ─────────────────────────────────────────
|
||||
function Toast({ toasts, onDismiss }) {
|
||||
if (!toasts.length) return null;
|
||||
return (
|
||||
<div className="toast-container">
|
||||
{toasts.map(t => (
|
||||
<div key={t.id} className={`toast toast--${t.type}`}>
|
||||
<span className="toast-icon">
|
||||
{t.type === 'success' && <CheckCircle2 size={18} />}
|
||||
{t.type === 'error' && <XCircle size={18} />}
|
||||
{t.type === 'warning' && <WifiOff size={18} />}
|
||||
</span>
|
||||
<div className="toast-body">
|
||||
<strong>{t.title}</strong>
|
||||
{t.message && <span>{t.message}</span>}
|
||||
</div>
|
||||
<button className="toast-close" onClick={() => onDismiss(t.id)} aria-label="Tutup">×</button>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const API_BASE_URL = '/api';
|
||||
|
||||
// ─── Helpers ───────────────────────────────────────────────────────────────
|
||||
const formatRp = (v) =>
|
||||
v != null ? `Rp ${Number(v).toLocaleString('id-ID', { minimumFractionDigits: 0, maximumFractionDigits: 0 })}` : '-';
|
||||
|
||||
const normalizeForecastValue = (value) => {
|
||||
const numericValue = Number(value);
|
||||
if (!Number.isFinite(numericValue) || numericValue < 1) return null;
|
||||
return Math.trunc(numericValue);
|
||||
};
|
||||
|
||||
const subDays = (dateStr, n) => {
|
||||
const d = new Date(dateStr);
|
||||
d.setDate(d.getDate() - n);
|
||||
return d.toISOString().split('T')[0];
|
||||
};
|
||||
|
||||
// ─── Custom Tooltip ────────────────────────────────────────────────────────
|
||||
const CustomTooltip = ({ active, payload, label }) => {
|
||||
if (!active || !payload?.length) return null;
|
||||
return (
|
||||
<div className="custom-tooltip">
|
||||
<p className="tooltip-date">{label}</p>
|
||||
{payload.map((entry, i) => (
|
||||
<p key={i} className="tooltip-value" style={{ color: entry.color }}>
|
||||
<span>{entry.name}:</span>
|
||||
<strong>{formatRp(entry.value)}</strong>
|
||||
</p>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
// ─── Quick-filter pill buttons ─────────────────────────────────────────────
|
||||
const QUICK_FILTERS = [
|
||||
{ label: '1 Minggu', days: 7 },
|
||||
{ label: '1 Bulan', days: 30 },
|
||||
{ label: '6 Bulan', days: 180 },
|
||||
{ label: '1 Tahun', days: 365 },
|
||||
{ label: 'Semua', days: null },
|
||||
];
|
||||
|
||||
// ─── Raw Data Chart Section ────────────────────────────────────────────────
|
||||
function RawDataChart({ allRawData, onUpdateData, isUpdating, updateResult, onDismissUpdate }) {
|
||||
const [activeFilter, setActiveFilter] = useState('1 Tahun');
|
||||
const [startDate, setStartDate] = useState('');
|
||||
const [endDate, setEndDate] = useState('');
|
||||
const [useCustomRange, setUseCustomRange] = useState(false);
|
||||
const [viewMode, setViewMode] = useState('chart'); // 'chart' | 'table'
|
||||
const [trendFilter, setTrendFilter] = useState('all'); // 'all' | 'up' | 'down'
|
||||
const [tablePage, setTablePage] = useState(1);
|
||||
const TABLE_PAGE_SIZE = 20;
|
||||
|
||||
const latestDate = allRawData.length > 0 ? allRawData[allRawData.length - 1].date : null;
|
||||
|
||||
const applyQuickFilter = (filter) => {
|
||||
setActiveFilter(filter.label);
|
||||
setUseCustomRange(false);
|
||||
setStartDate('');
|
||||
setEndDate('');
|
||||
setTablePage(1);
|
||||
};
|
||||
|
||||
const applyCustomRange = () => {
|
||||
if (startDate && endDate) { setUseCustomRange(true); setTablePage(1); }
|
||||
};
|
||||
|
||||
const clearCustomRange = () => {
|
||||
setUseCustomRange(false);
|
||||
setStartDate('');
|
||||
setEndDate('');
|
||||
setActiveFilter('1 Tahun');
|
||||
setTablePage(1);
|
||||
};
|
||||
|
||||
const applyTrendFilter = (filter) => {
|
||||
setTrendFilter(filter);
|
||||
setTablePage(1);
|
||||
};
|
||||
|
||||
const filteredData = useMemo(() => {
|
||||
if (allRawData.length === 0) return [];
|
||||
if (useCustomRange && startDate && endDate) {
|
||||
return allRawData.filter(d => d.date >= startDate && d.date <= endDate);
|
||||
}
|
||||
const filter = QUICK_FILTERS.find(f => f.label === activeFilter);
|
||||
if (!filter || filter.days === null) return allRawData;
|
||||
const cutoff = subDays(latestDate, filter.days);
|
||||
return allRawData.filter(d => d.date >= cutoff);
|
||||
}, [allRawData, activeFilter, useCustomRange, startDate, endDate, latestDate]);
|
||||
|
||||
// Table pagination – show newest first
|
||||
const sortedDesc = useMemo(() => [...filteredData].reverse(), [filteredData]);
|
||||
const tableRows = useMemo(() => sortedDesc.map((row, index) => {
|
||||
const curr = row['Rata-rata'];
|
||||
const prev = sortedDesc[index + 1]?.['Rata-rata'];
|
||||
return {
|
||||
...row,
|
||||
delta: curr == null || prev == null ? null : curr - prev,
|
||||
previousAverage: prev,
|
||||
};
|
||||
}), [sortedDesc]);
|
||||
const visibleRows = useMemo(() => {
|
||||
if (trendFilter === 'up') return tableRows.filter(row => row.delta > 0);
|
||||
if (trendFilter === 'down') return tableRows.filter(row => row.delta < 0);
|
||||
return tableRows;
|
||||
}, [tableRows, trendFilter]);
|
||||
const totalPages = Math.max(1, Math.ceil(visibleRows.length / TABLE_PAGE_SIZE));
|
||||
const pagedRows = visibleRows.slice((tablePage - 1) * TABLE_PAGE_SIZE, tablePage * TABLE_PAGE_SIZE);
|
||||
const globalStart = visibleRows.length - (tablePage - 1) * TABLE_PAGE_SIZE; // row number (desc)
|
||||
|
||||
return (
|
||||
<div className="glass-panel raw-chart-panel" style={{ marginTop: '2rem' }}>
|
||||
<div className="raw-chart-header">
|
||||
<div style={{ display: 'flex', alignItems: 'center', gap: '0.75rem' }}>
|
||||
<div>
|
||||
<h2 className="section-title">Data Historis Harga</h2>
|
||||
<p className="subtitle">Harga Medium 1 · Harga Medium 2 · Rata-rata</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div style={{ display: 'flex', gap: '0.6rem', alignItems: 'center', flexWrap: 'wrap' }}>
|
||||
{/* ── View Toggle ── */}
|
||||
<div className="view-toggle">
|
||||
<button
|
||||
className={`view-toggle-btn${viewMode === 'chart' ? ' active' : ''}`}
|
||||
onClick={() => setViewMode('chart')}
|
||||
title="Tampilkan Grafik"
|
||||
>
|
||||
<LineChartIcon size={15} /> Grafik
|
||||
</button>
|
||||
<button
|
||||
className={`view-toggle-btn${viewMode === 'table' ? ' active' : ''}`}
|
||||
onClick={() => setViewMode('table')}
|
||||
title="Tampilkan Tabel"
|
||||
>
|
||||
<Table2 size={15} /> Tabel
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* ── Update Data Button ── */}
|
||||
<button
|
||||
id="btn-update-data"
|
||||
className={`update-btn${isUpdating ? ' updating' : ''}`}
|
||||
onClick={onUpdateData}
|
||||
disabled={isUpdating}
|
||||
title="Ambil data terbaru dari situs sumber"
|
||||
>
|
||||
<RefreshCw size={15} className={isUpdating ? 'spin-icon' : ''} />
|
||||
{isUpdating ? 'Memperbarui...' : 'Perbarui Data'}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* ── Progress Bar ── */}
|
||||
{isUpdating && (
|
||||
<div className="update-progress-wrap">
|
||||
<div className="update-progress-bar" />
|
||||
<span className="update-progress-label">
|
||||
<RefreshCw size={12} className="spin-icon" />
|
||||
Mengambil data dari situs sumber, harap tunggu...
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* ── Update Result Banner ── */}
|
||||
{updateResult && (
|
||||
<div className={`update-banner update-banner--${updateResult.status}`}>
|
||||
<span className="update-banner-emoji">
|
||||
{updateResult.status === 'updated' ? '✅' : 'ℹ️'}
|
||||
</span>
|
||||
<div className="update-banner-body">
|
||||
<strong>{updateResult.message}</strong>
|
||||
<span className="update-banner-meta">
|
||||
{updateResult.status === 'updated'
|
||||
? `+${updateResult.updated_rows} baris baru · Total: ${updateResult.total_rows} baris · s/d ${updateResult.last_date}`
|
||||
: `Data sudah terkini s/d ${updateResult.last_date}`
|
||||
}
|
||||
</span>
|
||||
</div>
|
||||
<button className="update-banner-close" onClick={onDismissUpdate} aria-label="Tutup">×</button>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* ── Filter Bar (only relevant for chart view) ── */}
|
||||
<div className="filter-bar">
|
||||
<div className="filter-pills">
|
||||
{QUICK_FILTERS.map(f => (
|
||||
<button
|
||||
key={f.label}
|
||||
className={`pill-btn ${!useCustomRange && activeFilter === f.label ? 'active' : ''}`}
|
||||
onClick={() => applyQuickFilter(f)}
|
||||
>
|
||||
{f.label}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
|
||||
<div className="date-range-group">
|
||||
<div className="control-group">
|
||||
<label>Dari</label>
|
||||
<input type="date" value={startDate} max={endDate || latestDate || ''}
|
||||
onChange={e => setStartDate(e.target.value)} />
|
||||
</div>
|
||||
<div className="control-group">
|
||||
<label>Sampai</label>
|
||||
<input type="date" value={endDate} min={startDate || ''} max={latestDate || ''}
|
||||
onChange={e => setEndDate(e.target.value)} />
|
||||
</div>
|
||||
<button onClick={applyCustomRange} disabled={!startDate || !endDate} style={{ marginTop: '1.4rem' }}>
|
||||
Terapkan
|
||||
</button>
|
||||
{useCustomRange && (
|
||||
<button onClick={clearCustomRange} style={{ marginTop: '1.4rem', background: 'rgba(100,116,139,0.4)' }}>
|
||||
Reset
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* ══ CHART VIEW ══ */}
|
||||
{viewMode === 'chart' && (
|
||||
<>
|
||||
<div className="chart-container" style={{ height: '380px', marginTop: '1.5rem' }}>
|
||||
{filteredData.length > 0 ? (
|
||||
<ResponsiveContainer width="100%" height="100%">
|
||||
<LineChart data={filteredData} margin={{ top: 10, right: 30, left: 20, bottom: 30 }}>
|
||||
<CartesianGrid strokeDasharray="3 3" stroke="var(--border)" vertical={false} />
|
||||
<XAxis dataKey="date" stroke="var(--text-muted)"
|
||||
tick={{ fill: 'var(--text-muted)', fontSize: 12 }} tickMargin={12} minTickGap={40} />
|
||||
<YAxis stroke="var(--text-muted)"
|
||||
tick={{ fill: 'var(--text-muted)', fontSize: 12 }}
|
||||
tickFormatter={v => `Rp ${(v / 1000).toFixed(0)}k`} domain={['auto', 'auto']} />
|
||||
<Tooltip content={<CustomTooltip />} />
|
||||
<Legend verticalAlign="top" height={36} wrapperStyle={{ paddingBottom: '20px' }} />
|
||||
<Line type="monotone" dataKey="Harga Medium 1" stroke="#60a5fa" strokeWidth={2} dot={false} activeDot={{ r: 5 }} />
|
||||
<Line type="monotone" dataKey="Harga Medium 2" stroke="#a78bfa" strokeWidth={2} dot={false} activeDot={{ r: 5 }} />
|
||||
<Line type="monotone" dataKey="Rata-rata" stroke="#34d399" strokeWidth={2} dot={false} activeDot={{ r: 5 }} />
|
||||
</LineChart>
|
||||
</ResponsiveContainer>
|
||||
) : (
|
||||
<div style={{ height: '100%', display: 'flex', alignItems: 'center', justifyContent: 'center', color: 'var(--text-muted)' }}>
|
||||
Tidak ada data pada rentang yang dipilih.
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
{filteredData.length > 0 && (
|
||||
<p className="data-range-info">
|
||||
Menampilkan <strong>{filteredData.length}</strong> hari data
|
||||
{filteredData[0]?.date && ` · ${filteredData[0].date}`}
|
||||
{filteredData[filteredData.length - 1]?.date && ` — ${filteredData[filteredData.length - 1].date}`}
|
||||
</p>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
|
||||
{/* ══ TABLE VIEW ══ */}
|
||||
{viewMode === 'table' && (
|
||||
<div className="hist-table-wrap" style={{ marginTop: '1.5rem' }}>
|
||||
<div className="filter-bar" style={{ marginBottom: '1rem' }}>
|
||||
<div>
|
||||
<span className="filter-label">Perubahan harian</span>
|
||||
<div className="filter-pills">
|
||||
{[
|
||||
{ value: 'all', label: 'Semua' },
|
||||
{ value: 'up', label: '▲ Naik' },
|
||||
{ value: 'down', label: '▼ Turun' },
|
||||
].map(filter => (
|
||||
<button
|
||||
key={filter.value}
|
||||
className={`pill-btn${trendFilter === filter.value ? ' active' : ''}`}
|
||||
onClick={() => applyTrendFilter(filter.value)}
|
||||
>
|
||||
{filter.label}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{visibleRows.length > 0 ? (
|
||||
<>
|
||||
<div className="hist-table-scroll">
|
||||
<table className="hist-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>#</th>
|
||||
<th>Tanggal</th>
|
||||
<th>Harga Medium 1</th>
|
||||
<th>Harga Medium 2</th>
|
||||
<th>Rata-rata</th>
|
||||
<th>Δ Harian</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{pagedRows.map((row, i) => {
|
||||
const delta = row.delta;
|
||||
const isUp = delta > 0;
|
||||
const isDown = delta < 0;
|
||||
return (
|
||||
<tr key={row.date}>
|
||||
<td className="row-num">{globalStart - i}</td>
|
||||
<td className="date-cell">{row.date}</td>
|
||||
<td>{formatRp(row['Harga Medium 1'])}</td>
|
||||
<td>{formatRp(row['Harga Medium 2'])}</td>
|
||||
<td className="avg-cell">{formatRp(row['Rata-rata'])}</td>
|
||||
<td className={`delta-cell${isUp ? ' delta-up' : isDown ? ' delta-down' : ' delta-flat'}`}>
|
||||
{delta == null ? (
|
||||
<span className="delta-null">—</span>
|
||||
) : !isUp && !isDown ? (
|
||||
<span className="delta-badge">Tetap</span>
|
||||
) : (
|
||||
<span className="delta-badge">
|
||||
{isUp ? '▲' : '▼'}
|
||||
{' '}{formatRp(Math.abs(delta))}
|
||||
{' '}({delta >= 0 ? '+' : ''}{((delta / (row.previousAverage || 1)) * 100).toFixed(2)}%)
|
||||
</span>
|
||||
)}
|
||||
</td>
|
||||
</tr>
|
||||
);
|
||||
})}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
{/* Pagination */}
|
||||
<div className="table-pagination">
|
||||
<span className="table-info">
|
||||
{sortedDesc.length} baris · halaman {tablePage} dari {totalPages}
|
||||
</span>
|
||||
<div className="table-pager">
|
||||
<button className="pager-btn" onClick={() => setTablePage(1)} disabled={tablePage === 1}>«</button>
|
||||
<button className="pager-btn" onClick={() => setTablePage(p => Math.max(1, p - 1))} disabled={tablePage === 1}>‹</button>
|
||||
{Array.from({ length: Math.min(5, totalPages) }, (_, i) => {
|
||||
const offset = Math.min(Math.max(tablePage - 3, 1), Math.max(totalPages - 4, 1));
|
||||
const pg = offset + i;
|
||||
return pg <= totalPages ? (
|
||||
<button key={pg} className={`pager-btn${pg === tablePage ? ' active' : ''}`}
|
||||
onClick={() => setTablePage(pg)}>{pg}</button>
|
||||
) : null;
|
||||
})}
|
||||
<button className="pager-btn" onClick={() => setTablePage(p => Math.min(totalPages, p + 1))} disabled={tablePage === totalPages}>›</button>
|
||||
<button className="pager-btn" onClick={() => setTablePage(totalPages)} disabled={tablePage === totalPages}>»</button>
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
) : (
|
||||
<div style={{ textAlign: 'center', padding: '3rem 0', color: 'var(--text-muted)' }}>
|
||||
Tidak ada data pada rentang yang dipilih.
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// ─── Main App ──────────────────────────────────────────────────────────────
|
||||
function App() {
|
||||
const [rawData, setRawData] = useState([]); // all rows for raw chart
|
||||
const [data, setData] = useState([]); // mapped { date, actual }
|
||||
const [forecast, setForecast] = useState([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [isForecasting, setIsForecasting] = useState(false);
|
||||
const [error, setError] = useState(null);
|
||||
const [horizon, setHorizon] = useState(7);
|
||||
const [targetColumn, setTargetColumn] = useState('rata2');
|
||||
const [isUpdating, setIsUpdating] = useState(false);
|
||||
const [updateResult, setUpdateResult] = useState(null);
|
||||
const [toasts, setToasts] = useState([]);
|
||||
const toastTimers = useRef({});
|
||||
|
||||
const showToast = useCallback((type, title, message, duration = 6000) => {
|
||||
const id = Date.now();
|
||||
setToasts(prev => [...prev, { id, type, title, message }]);
|
||||
if (duration > 0) {
|
||||
toastTimers.current[id] = setTimeout(() => {
|
||||
setToasts(prev => prev.filter(t => t.id !== id));
|
||||
delete toastTimers.current[id];
|
||||
}, duration);
|
||||
}
|
||||
return id;
|
||||
}, []);
|
||||
|
||||
const dismissToast = useCallback((id) => {
|
||||
clearTimeout(toastTimers.current[id]);
|
||||
delete toastTimers.current[id];
|
||||
setToasts(prev => prev.filter(t => t.id !== id));
|
||||
}, []);
|
||||
|
||||
|
||||
const fetchAllData = useCallback(async () => {
|
||||
try {
|
||||
setLoading(true);
|
||||
setError(null);
|
||||
|
||||
const res = await fetch(`${API_BASE_URL}/data/preprocessed`);
|
||||
if (!res.ok) throw new Error(`HTTP error! status: ${res.status}`);
|
||||
const result = await res.json();
|
||||
|
||||
// raw chart rows – keep all three price columns
|
||||
const raw = result.rows.map(row => ({
|
||||
date: row.Tanggal.split('T')[0],
|
||||
'Harga Medium 1': row['Harga Medium 1'],
|
||||
'Harga Medium 2': row['Harga Medium 2'],
|
||||
'Rata-rata': row['rata2'],
|
||||
}));
|
||||
setRawData(raw);
|
||||
|
||||
// forecast chart rows
|
||||
const mapped = result.rows.map(row => ({
|
||||
date: row.Tanggal.split('T')[0],
|
||||
actual: row[targetColumn],
|
||||
}));
|
||||
setData(mapped);
|
||||
} catch (err) {
|
||||
console.error('Failed to fetch data:', err);
|
||||
setError('Gagal memuat data dari server API. Pastikan server API aktif di http://127.0.0.1:8000.');
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}, [targetColumn]);
|
||||
|
||||
useEffect(() => {
|
||||
fetchAllData();
|
||||
}, [fetchAllData]);
|
||||
|
||||
const handleUpdateData = async () => {
|
||||
try {
|
||||
setIsUpdating(true);
|
||||
setUpdateResult(null);
|
||||
|
||||
const res = await fetch(`${API_BASE_URL}/data/update`, { method: 'POST' });
|
||||
const body = await res.json().catch(() => ({}));
|
||||
|
||||
if (!res.ok) {
|
||||
// 502 = situs sumber tidak mengembalikan data
|
||||
if (res.status === 502) {
|
||||
showToast(
|
||||
'warning',
|
||||
'Data Belum Tersedia',
|
||||
'Situs sumber (BI Harga Pangan) belum menyediakan data untuk tanggal yang diminta. Coba lagi nanti.',
|
||||
0 // tidak auto-dismiss
|
||||
);
|
||||
} else {
|
||||
showToast(
|
||||
'error',
|
||||
'Gagal Memperbarui Data',
|
||||
body.detail || `HTTP ${res.status} – terjadi kesalahan pada server.`,
|
||||
0
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
setUpdateResult(body);
|
||||
|
||||
if (body.status === 'updated' && body.updated_rows > 0) {
|
||||
// Refresh grafik lalu tampilkan toast sukses
|
||||
await fetchAllData();
|
||||
showToast(
|
||||
'success',
|
||||
'Data Berhasil Diperbarui',
|
||||
`+${body.updated_rows} baris baru ditambahkan · Total: ${body.total_rows} baris · s/d ${body.last_date}`,
|
||||
7000
|
||||
);
|
||||
} else {
|
||||
// up_to_date – tidak perlu refresh
|
||||
showToast(
|
||||
'success',
|
||||
'Data Sudah Terkini',
|
||||
`Data sudah tersedia s/d ${body.last_date}. Tidak ada baris baru.`,
|
||||
5000
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Update error:', err);
|
||||
showToast(
|
||||
'error',
|
||||
'Koneksi Gagal',
|
||||
`Tidak dapat menghubungi server: ${err.message}`,
|
||||
0
|
||||
);
|
||||
} finally {
|
||||
setIsUpdating(false);
|
||||
}
|
||||
};
|
||||
|
||||
const handleDismissUpdate = () => setUpdateResult(null);
|
||||
|
||||
const handleForecast = async () => {
|
||||
try {
|
||||
setIsForecasting(true);
|
||||
setError(null);
|
||||
|
||||
const res = await fetch(`${API_BASE_URL}/forecast`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ horizon: parseInt(horizon), value_column: targetColumn }),
|
||||
});
|
||||
if (!res.ok) throw new Error(`HTTP error! status: ${res.status}`);
|
||||
const result = await res.json();
|
||||
setForecast(result.forecast);
|
||||
} catch (err) {
|
||||
console.error('Forecast error:', err);
|
||||
setError('Gagal memuat prediksi. Pastikan server API berjalan dan dapat diakses.');
|
||||
} finally {
|
||||
setIsForecasting(false);
|
||||
}
|
||||
};
|
||||
|
||||
// Combine last 90 days + forecast for the top chart
|
||||
const chartData = useMemo(() => {
|
||||
if (data.length === 0) return [];
|
||||
const recent = data.slice(-90).map(d => ({ date: d.date, Actual: d.actual, Forecast: null }));
|
||||
const validForecast = forecast
|
||||
.map(f => ({ ...f, value: normalizeForecastValue(f.value) }))
|
||||
.filter(f => f.value !== null);
|
||||
|
||||
if (validForecast.length > 0) {
|
||||
const last = recent[recent.length - 1];
|
||||
if (last) last.Forecast = last.Actual;
|
||||
const fData = validForecast.map(f => ({
|
||||
date: f.date.split('T')[0], Actual: null, Forecast: f.value,
|
||||
}));
|
||||
return [...recent, ...fData];
|
||||
}
|
||||
return recent;
|
||||
}, [data, forecast]);
|
||||
|
||||
const hasValidForecast = forecast.some(f => normalizeForecastValue(f.value) !== null);
|
||||
const lastPrice = data.length > 0 ? data[data.length - 1].actual : 0;
|
||||
const lastDate = data.length > 0 ? data[data.length - 1].date : '-';
|
||||
|
||||
return (
|
||||
<div className="dashboard-container">
|
||||
{/* ── Toast Notifications ── */}
|
||||
<Toast toasts={toasts} onDismiss={dismissToast} />
|
||||
{loading && (
|
||||
<div className="loading-overlay">
|
||||
<div className="spinner"></div>
|
||||
<h2>Memuat Data...</h2>
|
||||
<p style={{ color: 'var(--text-muted)', marginTop: '0.5rem' }}>Mengambil data historis dari API</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<header>
|
||||
<div>
|
||||
<h1>Prediksi Harga Beras Medium Di Banyuwangi</h1>
|
||||
<p className="subtitle">Real-time time series prediction monitoring</p>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
{error && (
|
||||
<div className="error-message">
|
||||
<AlertCircle size={20} />
|
||||
<span>{error}</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* ── Stat Cards ── */}
|
||||
<div className="stats-grid" style={{ marginBottom: '2rem' }}>
|
||||
<div className="glass-panel stat-card">
|
||||
<div className="stat-info">
|
||||
<h3>Total Data Historis</h3>
|
||||
<p>{data.length} <span style={{ fontSize: '0.8rem', color: 'var(--text-muted)', fontWeight: 400 }}>hari</span></p>
|
||||
</div>
|
||||
</div>
|
||||
<div className="glass-panel stat-card">
|
||||
<div className="stat-info">
|
||||
<h3>Harga Terakhir ({targetColumn})</h3>
|
||||
<p>{formatRp(lastPrice)}</p>
|
||||
</div>
|
||||
</div>
|
||||
<div className="glass-panel stat-card">
|
||||
<div className="stat-info">
|
||||
<h3>Update Terakhir</h3>
|
||||
<p style={{ fontSize: '1.2rem' }}>{lastDate}</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* ── Forecast Chart ── */}
|
||||
<div className="glass-panel" style={{ marginTop: '2rem' }}>
|
||||
<div className="raw-chart-header">
|
||||
<div>
|
||||
<h2 className="section-title">Prediksi Harga</h2>
|
||||
<p className="subtitle">90 hari terakhir + forecast ke depan</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="controls" style={{ marginTop: '1.25rem' }}>
|
||||
<div className="control-group">
|
||||
<label>Variabel Target</label>
|
||||
<select
|
||||
value={targetColumn}
|
||||
onChange={e => { setTargetColumn(e.target.value); setForecast([]); }}
|
||||
disabled={isForecasting}
|
||||
>
|
||||
<option value="rata2">Rata-rata (rata2)</option>
|
||||
<option value="Harga Medium 1">Harga Medium 1</option>
|
||||
<option value="Harga Medium 2">Harga Medium 2</option>
|
||||
</select>
|
||||
</div>
|
||||
<div className="control-group">
|
||||
<label>Horizon (Hari)</label>
|
||||
<input
|
||||
type="number" min="1" max="90"
|
||||
value={horizon}
|
||||
onChange={e => setHorizon(e.target.value)}
|
||||
disabled={isForecasting}
|
||||
style={{ width: '120px' }}
|
||||
/>
|
||||
</div>
|
||||
<button onClick={handleForecast} disabled={isForecasting || loading}>
|
||||
{isForecasting ? (
|
||||
<><div className="spinner" style={{ width: '16px', height: '16px', marginBottom: 0, borderWidth: '2px' }}></div>Memproses...</>
|
||||
) : (
|
||||
<><TrendingUp size={18} />Generate Forecast</>
|
||||
)}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div className="chart-container">
|
||||
{chartData.length > 0 ? (
|
||||
<ResponsiveContainer width="100%" height="100%">
|
||||
<LineChart data={chartData} margin={{ top: 10, right: 30, left: 20, bottom: 30 }}>
|
||||
<CartesianGrid strokeDasharray="3 3" stroke="var(--border)" vertical={false} />
|
||||
<XAxis dataKey="date" stroke="var(--text-muted)" tick={{ fill: 'var(--text-muted)' }} tickMargin={15} minTickGap={30} />
|
||||
<YAxis stroke="var(--text-muted)" tick={{ fill: 'var(--text-muted)' }} tickFormatter={v => `Rp ${(v/1000).toFixed(0)}k`} domain={['auto','auto']} />
|
||||
<Tooltip content={<CustomTooltip />} />
|
||||
<Legend verticalAlign="top" height={36} wrapperStyle={{ paddingBottom: '20px' }} />
|
||||
{hasValidForecast && (
|
||||
<ReferenceLine x={lastDate} stroke="rgba(255,255,255,0.2)" strokeDasharray="3 3"
|
||||
label={{ position: 'top', value: 'Hari Ini', fill: 'var(--text-muted)', fontSize: 12 }} />
|
||||
)}
|
||||
<Line type="monotone" dataKey="Actual" stroke="#3b82f6" strokeWidth={3} dot={false} activeDot={{ r: 6, fill: '#3b82f6', stroke: '#0f172a', strokeWidth: 2 }} connectNulls />
|
||||
<Line type="monotone" dataKey="Forecast" stroke="#f43f5e" strokeWidth={3} strokeDasharray="5 5" dot={{ r: 4, fill: '#f43f5e', stroke: 'none' }} activeDot={{ r: 6, fill: '#f43f5e', stroke: '#0f172a', strokeWidth: 2 }} connectNulls />
|
||||
</LineChart>
|
||||
</ResponsiveContainer>
|
||||
) : (
|
||||
<div style={{ height: '100%', display: 'flex', alignItems: 'center', justifyContent: 'center', color: 'var(--text-muted)' }}>
|
||||
Klik <strong style={{ margin: '0 4px' }}>Generate Forecast</strong> untuk memulai prediksi.
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* ── Raw Data Historical Chart ── */}
|
||||
<RawDataChart
|
||||
allRawData={rawData}
|
||||
onUpdateData={handleUpdateData}
|
||||
isUpdating={isUpdating}
|
||||
updateResult={updateResult}
|
||||
onDismissUpdate={handleDismissUpdate}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export default App;
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 13 KiB |
|
|
@ -0,0 +1 @@
|
|||
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="35.93" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 228"><path fill="#00D8FF" d="M210.483 73.824a171.49 171.49 0 0 0-8.24-2.597c.465-1.9.893-3.777 1.273-5.621c6.238-30.281 2.16-54.676-11.769-62.708c-13.355-7.7-35.196.329-57.254 19.526a171.23 171.23 0 0 0-6.375 5.848a155.866 155.866 0 0 0-4.241-3.917C100.759 3.829 77.587-4.822 63.673 3.233C50.33 10.957 46.379 33.89 51.995 62.588a170.974 170.974 0 0 0 1.892 8.48c-3.28.932-6.445 1.924-9.474 2.98C17.309 83.498 0 98.307 0 113.668c0 15.865 18.582 31.778 46.812 41.427a145.52 145.52 0 0 0 6.921 2.165a167.467 167.467 0 0 0-2.01 9.138c-5.354 28.2-1.173 50.591 12.134 58.266c13.744 7.926 36.812-.22 59.273-19.855a145.567 145.567 0 0 0 5.342-4.923a168.064 168.064 0 0 0 6.92 6.314c21.758 18.722 43.246 26.282 56.54 18.586c13.731-7.949 18.194-32.003 12.4-61.268a145.016 145.016 0 0 0-1.535-6.842c1.62-.48 3.21-.974 4.76-1.488c29.348-9.723 48.443-25.443 48.443-41.52c0-15.417-17.868-30.326-45.517-39.844Zm-6.365 70.984c-1.4.463-2.836.91-4.3 1.345c-3.24-10.257-7.612-21.163-12.963-32.432c5.106-11 9.31-21.767 12.459-31.957c2.619.758 5.16 1.557 7.61 2.4c23.69 8.156 38.14 20.213 38.14 29.504c0 9.896-15.606 22.743-40.946 31.14Zm-10.514 20.834c2.562 12.94 2.927 24.64 1.23 33.787c-1.524 8.219-4.59 13.698-8.382 15.893c-8.067 4.67-25.32-1.4-43.927-17.412a156.726 156.726 0 0 1-6.437-5.87c7.214-7.889 14.423-17.06 21.459-27.246c12.376-1.098 24.068-2.894 34.671-5.345a134.17 134.17 0 0 1 1.386 6.193ZM87.276 214.515c-7.882 2.783-14.16 2.863-17.955.675c-8.075-4.657-11.432-22.636-6.853-46.752a156.923 156.923 0 0 1 1.869-8.499c10.486 2.32 22.093 3.988 34.498 4.994c7.084 9.967 14.501 19.128 21.976 27.15a134.668 134.668 0 0 1-4.877 4.492c-9.933 8.682-19.886 14.842-28.658 17.94ZM50.35 144.747c-12.483-4.267-22.792-9.812-29.858-15.863c-6.35-5.437-9.555-10.836-9.555-15.216c0-9.322 13.897-21.212 37.076-29.293c2.813-.98 5.757-1.905 8.812-2.773c3.204 10.42 7.406 21.315 12.477 32.332c-5.137 11.18-9.399 22.249-12.634 32.792a134.718 134.718 0 0 1-6.318-1.979Zm12.378-84.26c-4.811-24.587-1.616-43.134 6.425-47.789c8.564-4.958 27.502 2.111 47.463 19.835a144.318 144.318 0 0 1 3.841 3.545c-7.438 7.987-14.787 17.08-21.808 26.988c-12.04 1.116-23.565 2.908-34.161 5.309a160.342 160.342 0 0 1-1.76-7.887Zm110.427 27.268a347.8 347.8 0 0 0-7.785-12.803c8.168 1.033 15.994 2.404 23.343 4.08c-2.206 7.072-4.956 14.465-8.193 22.045a381.151 381.151 0 0 0-7.365-13.322Zm-45.032-43.861c5.044 5.465 10.096 11.566 15.065 18.186a322.04 322.04 0 0 0-30.257-.006c4.974-6.559 10.069-12.652 15.192-18.18ZM82.802 87.83a323.167 323.167 0 0 0-7.227 13.238c-3.184-7.553-5.909-14.98-8.134-22.152c7.304-1.634 15.093-2.97 23.209-3.984a321.524 321.524 0 0 0-7.848 12.897Zm8.081 65.352c-8.385-.936-16.291-2.203-23.593-3.793c2.26-7.3 5.045-14.885 8.298-22.6a321.187 321.187 0 0 0 7.257 13.246c2.594 4.48 5.28 8.868 8.038 13.147Zm37.542 31.03c-5.184-5.592-10.354-11.779-15.403-18.433c4.902.192 9.899.29 14.978.29c5.218 0 10.376-.117 15.453-.343c-4.985 6.774-10.018 12.97-15.028 18.486Zm52.198-57.817c3.422 7.8 6.306 15.345 8.596 22.52c-7.422 1.694-15.436 3.058-23.88 4.071a382.417 382.417 0 0 0 7.859-13.026a347.403 347.403 0 0 0 7.425-13.565Zm-16.898 8.101a358.557 358.557 0 0 1-12.281 19.815a329.4 329.4 0 0 1-23.444.823c-7.967 0-15.716-.248-23.178-.732a310.202 310.202 0 0 1-12.513-19.846h.001a307.41 307.41 0 0 1-10.923-20.627a310.278 310.278 0 0 1 10.89-20.637l-.001.001a307.318 307.318 0 0 1 12.413-19.761c7.613-.576 15.42-.876 23.31-.876H128c7.926 0 15.743.303 23.354.883a329.357 329.357 0 0 1 12.335 19.695a358.489 358.489 0 0 1 11.036 20.54a329.472 329.472 0 0 1-11 20.722Zm22.56-122.124c8.572 4.944 11.906 24.881 6.52 51.026c-.344 1.668-.73 3.367-1.15 5.09c-10.622-2.452-22.155-4.275-34.23-5.408c-7.034-10.017-14.323-19.124-21.64-27.008a160.789 160.789 0 0 1 5.888-5.4c18.9-16.447 36.564-22.941 44.612-18.3ZM128 90.808c12.625 0 22.86 10.235 22.86 22.86s-10.235 22.86-22.86 22.86s-22.86-10.235-22.86-22.86s10.235-22.86 22.86-22.86Z"></path></svg>
|
||||
|
After Width: | Height: | Size: 4.0 KiB |
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 8.5 KiB |
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,10 @@
|
|||
import { StrictMode } from 'react'
|
||||
import { createRoot } from 'react-dom/client'
|
||||
import './index.css'
|
||||
import App from './App.jsx'
|
||||
|
||||
createRoot(document.getElementById('root')).render(
|
||||
<StrictMode>
|
||||
<App />
|
||||
</StrictMode>,
|
||||
)
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
import react from '@vitejs/plugin-react'
|
||||
import { defineConfig } from 'vite'
|
||||
|
||||
// https://vite.dev/config/
|
||||
export default defineConfig({
|
||||
plugins: [react()],
|
||||
server: {
|
||||
proxy: {
|
||||
'/api': {
|
||||
target: 'http://127.0.0.1:8000',
|
||||
changeOrigin: true,
|
||||
rewrite: (path) => path.replace(/^\/api/, '')
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -0,0 +1,37 @@
|
|||
{
|
||||
"input_size": 7,
|
||||
"output_size": 1,
|
||||
"hidden_size": 512,
|
||||
"n_stacks_trend": 3,
|
||||
"n_stacks_seasonal": 4,
|
||||
"blocks_per_stack": 3,
|
||||
"theta_size_trend": 2,
|
||||
"theta_size_season": 4,
|
||||
"epochs": 300,
|
||||
"batch_size": 32,
|
||||
"learning_rate": 0.0001,
|
||||
"loss_type": "mse",
|
||||
"alpha": 0.1,
|
||||
"neuron_function": "GELU(approximate='none')",
|
||||
"device": "cuda",
|
||||
"train_samples": 1971,
|
||||
"validation_samples": 494,
|
||||
"train_input_shape": [
|
||||
1971,
|
||||
7
|
||||
],
|
||||
"train_target_shape": [
|
||||
1971,
|
||||
1
|
||||
],
|
||||
"validation_input_shape": [
|
||||
494,
|
||||
7
|
||||
],
|
||||
"validation_target_shape": [
|
||||
494,
|
||||
1
|
||||
],
|
||||
"best_epoch": 300,
|
||||
"best_validation_loss": 4.962215825798921e-05
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -0,0 +1,500 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "5d345c6c",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Manipulasi `deploy_data.xlsx`\n",
|
||||
"\n",
|
||||
"Notebook ini mengubah data deploy berbentuk wide menjadi format long yang sama dengan `data testing.xlsx`: `Tanggal`, `Harga Medium 1`, `Harga Medium 2`, dan `rata2`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "ffcf2b96",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"WindowsPath('c:/Users/LENOVO/Documents/TA/Nbeats/deploy_data.xlsx')"
|
||||
]
|
||||
},
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from pathlib import Path\n",
|
||||
"import re\n",
|
||||
"import numpy as np\n",
|
||||
"import pandas as pd\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"\n",
|
||||
"ROOT = Path.cwd()\n",
|
||||
"if ROOT.name == 'Streamlit':\n",
|
||||
" ROOT = ROOT.parent\n",
|
||||
"INPUT_FILE = ROOT / 'deploy_data.xlsx'\n",
|
||||
"REFERENCE_FILE = ROOT / 'data testing.xlsx'\n",
|
||||
"OUTPUT_XLSX = ROOT / 'data deploy hasil.xlsx'\n",
|
||||
"OUTPUT_CSV = ROOT / 'data deploy hasil.csv'\n",
|
||||
"MEDIUM_1_LABEL = 'Beras Kualitas Medium I'\n",
|
||||
"MEDIUM_2_LABEL = 'Beras Kualitas Medium II'\n",
|
||||
"USE_DAILY_CALENDAR = True\n",
|
||||
"INPUT_FILE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "15b6be12",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Ukuran deploy: (3, 194)\n",
|
||||
"Kolom deploy: ['No', 'Komoditas (Rp)', '01/ 01/ 2026', '02/ 01/ 2026', '05/ 01/ 2026'] ... ['23/ 09/ 2026', '24/ 09/ 2026', '25/ 09/ 2026']\n",
|
||||
"Baris komoditas:\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div>\n",
|
||||
"<style scoped>\n",
|
||||
" .dataframe tbody tr th:only-of-type {\n",
|
||||
" vertical-align: middle;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe tbody tr th {\n",
|
||||
" vertical-align: top;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe thead th {\n",
|
||||
" text-align: right;\n",
|
||||
" }\n",
|
||||
"</style>\n",
|
||||
"<table border=\"1\" class=\"dataframe\">\n",
|
||||
" <thead>\n",
|
||||
" <tr style=\"text-align: right;\">\n",
|
||||
" <th></th>\n",
|
||||
" <th>No</th>\n",
|
||||
" <th>Komoditas (Rp)</th>\n",
|
||||
" </tr>\n",
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>0</th>\n",
|
||||
" <td>I</td>\n",
|
||||
" <td>Beras</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1</th>\n",
|
||||
" <td>1</td>\n",
|
||||
" <td>Beras Kualitas Medium I</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2</th>\n",
|
||||
" <td>2</td>\n",
|
||||
" <td>Beras Kualitas Medium II</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
" No Komoditas (Rp)\n",
|
||||
"0 I Beras\n",
|
||||
"1 1 Beras Kualitas Medium I\n",
|
||||
"2 2 Beras Kualitas Medium II"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Schema referensi: ['Tanggal', 'Harga Medium 1', 'Harga Medium 2', 'rata2']\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"deploy_raw = pd.read_excel(INPUT_FILE)\n",
|
||||
"reference = pd.read_excel(REFERENCE_FILE)\n",
|
||||
"print('Ukuran deploy:', deploy_raw.shape)\n",
|
||||
"print('Kolom deploy:', list(deploy_raw.columns[:5]), '...', list(deploy_raw.columns[-3:]))\n",
|
||||
"print('Baris komoditas:')\n",
|
||||
"display(deploy_raw.iloc[:, :2])\n",
|
||||
"print('Schema referensi:', reference.columns.tolist())"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "09ac66e7",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def parse_number(value):\n",
|
||||
" if pd.isna(value):\n",
|
||||
" return np.nan\n",
|
||||
" text = str(value).strip().replace('Rp', '').replace(' ', '')\n",
|
||||
" if not text:\n",
|
||||
" return np.nan\n",
|
||||
" # Sumber memakai koma sebagai pemisah ribuan, misalnya 13,850.\n",
|
||||
" text = re.sub(r'[^0-9,.-]', '', text)\n",
|
||||
" if text.count(',') > 0 and '.' not in text:\n",
|
||||
" text = text.replace(',', '')\n",
|
||||
" else:\n",
|
||||
" text = text.replace(',', '')\n",
|
||||
" return pd.to_numeric(text, errors='coerce')\n",
|
||||
"\n",
|
||||
"def parse_deploy_date(value):\n",
|
||||
" text = str(value).strip().replace(' ', '')\n",
|
||||
" return pd.to_datetime(text, format='%d/%m/%Y', errors='coerce')\n",
|
||||
"\n",
|
||||
"def find_row(label):\n",
|
||||
" labels = deploy_raw['Komoditas (Rp)'].astype(str).str.strip()\n",
|
||||
" matches = deploy_raw.loc[labels.eq(label)]\n",
|
||||
" if matches.empty:\n",
|
||||
" raise ValueError(f'Baris komoditas tidak ditemukan: {label}')\n",
|
||||
" return matches.iloc[0]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "0c6bdf3e",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div>\n",
|
||||
"<style scoped>\n",
|
||||
" .dataframe tbody tr th:only-of-type {\n",
|
||||
" vertical-align: middle;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe tbody tr th {\n",
|
||||
" vertical-align: top;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe thead th {\n",
|
||||
" text-align: right;\n",
|
||||
" }\n",
|
||||
"</style>\n",
|
||||
"<table border=\"1\" class=\"dataframe\">\n",
|
||||
" <thead>\n",
|
||||
" <tr style=\"text-align: right;\">\n",
|
||||
" <th></th>\n",
|
||||
" <th>Tanggal</th>\n",
|
||||
" <th>Harga Medium 1</th>\n",
|
||||
" <th>Harga Medium 2</th>\n",
|
||||
" <th>rata2</th>\n",
|
||||
" </tr>\n",
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>0</th>\n",
|
||||
" <td>2026-01-01</td>\n",
|
||||
" <td>13500.0</td>\n",
|
||||
" <td>12250.0</td>\n",
|
||||
" <td>12875.0</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1</th>\n",
|
||||
" <td>2026-01-02</td>\n",
|
||||
" <td>13500.0</td>\n",
|
||||
" <td>12250.0</td>\n",
|
||||
" <td>12875.0</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2</th>\n",
|
||||
" <td>2026-01-03</td>\n",
|
||||
" <td>13500.0</td>\n",
|
||||
" <td>12250.0</td>\n",
|
||||
" <td>12875.0</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>3</th>\n",
|
||||
" <td>2026-01-04</td>\n",
|
||||
" <td>13500.0</td>\n",
|
||||
" <td>12250.0</td>\n",
|
||||
" <td>12875.0</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>4</th>\n",
|
||||
" <td>2026-01-05</td>\n",
|
||||
" <td>13500.0</td>\n",
|
||||
" <td>12250.0</td>\n",
|
||||
" <td>12875.0</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
" Tanggal Harga Medium 1 Harga Medium 2 rata2\n",
|
||||
"0 2026-01-01 13500.0 12250.0 12875.0\n",
|
||||
"1 2026-01-02 13500.0 12250.0 12875.0\n",
|
||||
"2 2026-01-03 13500.0 12250.0 12875.0\n",
|
||||
"3 2026-01-04 13500.0 12250.0 12875.0\n",
|
||||
"4 2026-01-05 13500.0 12250.0 12875.0"
|
||||
]
|
||||
},
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"medium_1 = find_row(MEDIUM_1_LABEL)\n",
|
||||
"medium_2 = find_row(MEDIUM_2_LABEL)\n",
|
||||
"date_columns = [column for column in deploy_raw.columns if parse_deploy_date(column) is not pd.NaT]\n",
|
||||
"dates = pd.Series([parse_deploy_date(column) for column in date_columns], name='Tanggal')\n",
|
||||
"result = pd.DataFrame({\n",
|
||||
" 'Tanggal': dates,\n",
|
||||
" 'Harga Medium 1': [parse_number(medium_1[column]) for column in date_columns],\n",
|
||||
" 'Harga Medium 2': [parse_number(medium_2[column]) for column in date_columns],\n",
|
||||
"})\n",
|
||||
"result = result.sort_values('Tanggal').drop_duplicates('Tanggal').set_index('Tanggal')\n",
|
||||
"\n",
|
||||
"if USE_DAILY_CALENDAR:\n",
|
||||
" full_dates = pd.date_range(result.index.min(), result.index.max(), freq='D')\n",
|
||||
" result = result.reindex(full_dates)\n",
|
||||
" result.index.name = 'Tanggal'\n",
|
||||
" result[['Harga Medium 1', 'Harga Medium 2']] = result[['Harga Medium 1', 'Harga Medium 2']].interpolate(\n",
|
||||
" method='linear', limit_direction='both'\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"result['rata2'] = result[['Harga Medium 1', 'Harga Medium 2']].mean(axis=1)\n",
|
||||
"result = result.reset_index()\n",
|
||||
"result = result[['Tanggal', 'Harga Medium 1', 'Harga Medium 2', 'rata2']]\n",
|
||||
"result.head()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "4a0c5328",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Valid: 268 baris\n",
|
||||
"Rentang: 2026-01-01 sampai 2026-09-25\n",
|
||||
"Kolom: ['Tanggal', 'Harga Medium 1', 'Harga Medium 2', 'rata2']\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div>\n",
|
||||
"<style scoped>\n",
|
||||
" .dataframe tbody tr th:only-of-type {\n",
|
||||
" vertical-align: middle;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe tbody tr th {\n",
|
||||
" vertical-align: top;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe thead th {\n",
|
||||
" text-align: right;\n",
|
||||
" }\n",
|
||||
"</style>\n",
|
||||
"<table border=\"1\" class=\"dataframe\">\n",
|
||||
" <thead>\n",
|
||||
" <tr style=\"text-align: right;\">\n",
|
||||
" <th></th>\n",
|
||||
" <th>Tanggal</th>\n",
|
||||
" <th>Harga Medium 1</th>\n",
|
||||
" <th>Harga Medium 2</th>\n",
|
||||
" <th>rata2</th>\n",
|
||||
" </tr>\n",
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>0</th>\n",
|
||||
" <td>2026-01-01</td>\n",
|
||||
" <td>13500.0</td>\n",
|
||||
" <td>12250.0</td>\n",
|
||||
" <td>12875.0</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1</th>\n",
|
||||
" <td>2026-01-02</td>\n",
|
||||
" <td>13500.0</td>\n",
|
||||
" <td>12250.0</td>\n",
|
||||
" <td>12875.0</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2</th>\n",
|
||||
" <td>2026-01-03</td>\n",
|
||||
" <td>13500.0</td>\n",
|
||||
" <td>12250.0</td>\n",
|
||||
" <td>12875.0</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
" Tanggal Harga Medium 1 Harga Medium 2 rata2\n",
|
||||
"0 2026-01-01 13500.0 12250.0 12875.0\n",
|
||||
"1 2026-01-02 13500.0 12250.0 12875.0\n",
|
||||
"2 2026-01-03 13500.0 12250.0 12875.0"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div>\n",
|
||||
"<style scoped>\n",
|
||||
" .dataframe tbody tr th:only-of-type {\n",
|
||||
" vertical-align: middle;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe tbody tr th {\n",
|
||||
" vertical-align: top;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe thead th {\n",
|
||||
" text-align: right;\n",
|
||||
" }\n",
|
||||
"</style>\n",
|
||||
"<table border=\"1\" class=\"dataframe\">\n",
|
||||
" <thead>\n",
|
||||
" <tr style=\"text-align: right;\">\n",
|
||||
" <th></th>\n",
|
||||
" <th>Tanggal</th>\n",
|
||||
" <th>Harga Medium 1</th>\n",
|
||||
" <th>Harga Medium 2</th>\n",
|
||||
" <th>rata2</th>\n",
|
||||
" </tr>\n",
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>265</th>\n",
|
||||
" <td>2026-09-23</td>\n",
|
||||
" <td>14500.0</td>\n",
|
||||
" <td>13150.0</td>\n",
|
||||
" <td>13825.0</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>266</th>\n",
|
||||
" <td>2026-09-24</td>\n",
|
||||
" <td>14500.0</td>\n",
|
||||
" <td>13150.0</td>\n",
|
||||
" <td>13825.0</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>267</th>\n",
|
||||
" <td>2026-09-25</td>\n",
|
||||
" <td>14500.0</td>\n",
|
||||
" <td>13150.0</td>\n",
|
||||
" <td>13825.0</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
" Tanggal Harga Medium 1 Harga Medium 2 rata2\n",
|
||||
"265 2026-09-23 14500.0 13150.0 13825.0\n",
|
||||
"266 2026-09-24 14500.0 13150.0 13825.0\n",
|
||||
"267 2026-09-25 14500.0 13150.0 13825.0"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"expected_columns = ['Tanggal', 'Harga Medium 1', 'Harga Medium 2', 'rata2']\n",
|
||||
"assert result.columns.tolist() == expected_columns\n",
|
||||
"assert pd.api.types.is_datetime64_any_dtype(result['Tanggal'])\n",
|
||||
"assert result['Tanggal'].is_monotonic_increasing\n",
|
||||
"assert result[expected_columns[1:]].notna().all().all()\n",
|
||||
"assert result['Tanggal'].duplicated().sum() == 0\n",
|
||||
"print(f'Valid: {len(result):,} baris')\n",
|
||||
"print('Rentang:', result['Tanggal'].min().date(), 'sampai', result['Tanggal'].max().date())\n",
|
||||
"print('Kolom:', result.columns.tolist())\n",
|
||||
"display(result.head(3))\n",
|
||||
"display(result.tail(3))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "7d9e6fef",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"ax = result.set_index('Tanggal')[['Harga Medium 1', 'Harga Medium 2', 'rata2']].plot(figsize=(14, 5))\n",
|
||||
"ax.set_title('Data deploy setelah manipulasi')\n",
|
||||
"ax.set_ylabel('Harga')\n",
|
||||
"ax.grid(alpha=0.25)\n",
|
||||
"plt.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "ba97555c",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Tersimpan:\n",
|
||||
"c:\\Users\\LENOVO\\Documents\\TA\\Nbeats\\data deploy hasil.xlsx\n",
|
||||
"c:\\Users\\LENOVO\\Documents\\TA\\Nbeats\\data deploy hasil.csv\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"result.to_excel(OUTPUT_XLSX, index=False)\n",
|
||||
"# result.to_csv(OUTPUT_CSV, index=False, date_format='%Y-%m-%d')\n",
|
||||
"print('Tersimpan:')\n",
|
||||
"print(OUTPUT_XLSX)\n",
|
||||
"print(OUTPUT_CSV)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.6"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
|
|
@ -0,0 +1,181 @@
|
|||
{
|
||||
"input_size": 7,
|
||||
"output_size": 1,
|
||||
"hidden_size": 512,
|
||||
"n_stacks_trend": 3,
|
||||
"n_stacks_seasonal": 4,
|
||||
"blocks_per_stack": 3,
|
||||
"theta_size_trend": 2,
|
||||
"theta_size_season": 4,
|
||||
"epochs": 300,
|
||||
"batch_size": 32,
|
||||
"learning_rate": 0.0001,
|
||||
"loss_type": "mse",
|
||||
"alpha": 0.1,
|
||||
"neuron_function": "GELU(approximate='none')",
|
||||
"device": "cuda",
|
||||
"train_samples": 1971,
|
||||
"validation_samples": 494,
|
||||
"train_input_shape": [
|
||||
1971,
|
||||
7
|
||||
],
|
||||
"train_target_shape": [
|
||||
1971,
|
||||
1
|
||||
],
|
||||
"validation_input_shape": [
|
||||
494,
|
||||
7
|
||||
],
|
||||
"validation_target_shape": [
|
||||
494,
|
||||
1
|
||||
],
|
||||
"best_epoch": 300,
|
||||
"best_validation_loss": 4.962215825798921e-05,
|
||||
"time_complexity": {
|
||||
"forward": "O(N * B * H^2)",
|
||||
"training_step_approx": "O(N * B * H^2)",
|
||||
"gradient_norm_calculation": "O(P)",
|
||||
"N_batch_size": 32,
|
||||
"B_total_blocks": 21,
|
||||
"H_hidden_size": 512,
|
||||
"I_input_size": 7,
|
||||
"O_output_size": 1,
|
||||
"total_parameters": 16750308
|
||||
},
|
||||
"big_o_empirical_test": {
|
||||
"benchmark_batch_size": 8,
|
||||
"device": "cuda",
|
||||
"hidden_size_scaling": {
|
||||
"expected_complexity": "O(H^2)",
|
||||
"results": [
|
||||
{
|
||||
"hidden_size": 128,
|
||||
"total_blocks": 21,
|
||||
"batch_size": 8,
|
||||
"time_seconds_median": 0.01987130008637905,
|
||||
"time_milliseconds_median": 19.87130008637905,
|
||||
"theoretical_relative_cost_vs_first": 1.0
|
||||
},
|
||||
{
|
||||
"hidden_size": 256,
|
||||
"total_blocks": 21,
|
||||
"batch_size": 8,
|
||||
"time_seconds_median": 0.01879219990223646,
|
||||
"time_milliseconds_median": 18.79219990223646,
|
||||
"theoretical_relative_cost_vs_first": 4.0
|
||||
},
|
||||
{
|
||||
"hidden_size": 512,
|
||||
"total_blocks": 21,
|
||||
"batch_size": 8,
|
||||
"time_seconds_median": 0.019390400033444166,
|
||||
"time_milliseconds_median": 19.390400033444166,
|
||||
"theoretical_relative_cost_vs_first": 16.0
|
||||
}
|
||||
],
|
||||
"log_log_fit": {
|
||||
"exponent": -0.017671848609463185,
|
||||
"intercept": -3.847262919698698,
|
||||
"r2": 0.1915588923022309
|
||||
}
|
||||
},
|
||||
"block_scaling": {
|
||||
"expected_complexity": "O(B)",
|
||||
"results": [
|
||||
{
|
||||
"blocks_per_stack": 10,
|
||||
"total_blocks": 70,
|
||||
"hidden_size": 512,
|
||||
"batch_size": 8,
|
||||
"time_seconds_median": 0.06300389999523759,
|
||||
"time_milliseconds_median": 63.00389999523759,
|
||||
"theoretical_relative_cost_vs_first": 1.0
|
||||
},
|
||||
{
|
||||
"blocks_per_stack": 20,
|
||||
"total_blocks": 140,
|
||||
"hidden_size": 512,
|
||||
"batch_size": 8,
|
||||
"time_seconds_median": 0.12464780011214316,
|
||||
"time_milliseconds_median": 124.64780011214316,
|
||||
"theoretical_relative_cost_vs_first": 2.0
|
||||
},
|
||||
{
|
||||
"blocks_per_stack": 30,
|
||||
"total_blocks": 210,
|
||||
"hidden_size": 512,
|
||||
"batch_size": 8,
|
||||
"time_seconds_median": 0.18604329996742308,
|
||||
"time_milliseconds_median": 186.04329996742308,
|
||||
"theoretical_relative_cost_vs_first": 3.0
|
||||
}
|
||||
],
|
||||
"log_log_fit": {
|
||||
"exponent": 0.9854571097543345,
|
||||
"intercept": -6.951476412790202,
|
||||
"r2": 0.9999991876654754
|
||||
}
|
||||
},
|
||||
"input_batch_scaling": {
|
||||
"expected_complexity": "O(N)",
|
||||
"results": [
|
||||
{
|
||||
"batch_size": 1,
|
||||
"hidden_size": 512,
|
||||
"total_blocks": 21,
|
||||
"time_seconds_median": 0.01942260004580021,
|
||||
"time_milliseconds_median": 19.42260004580021,
|
||||
"theoretical_relative_cost_vs_first": 1.0
|
||||
},
|
||||
{
|
||||
"batch_size": 2,
|
||||
"hidden_size": 512,
|
||||
"total_blocks": 21,
|
||||
"time_seconds_median": 0.019125699996948242,
|
||||
"time_milliseconds_median": 19.125699996948242,
|
||||
"theoretical_relative_cost_vs_first": 2.0
|
||||
},
|
||||
{
|
||||
"batch_size": 4,
|
||||
"hidden_size": 512,
|
||||
"total_blocks": 21,
|
||||
"time_seconds_median": 0.018762200139462948,
|
||||
"time_milliseconds_median": 18.762200139462948,
|
||||
"theoretical_relative_cost_vs_first": 4.0
|
||||
},
|
||||
{
|
||||
"batch_size": 8,
|
||||
"hidden_size": 512,
|
||||
"total_blocks": 21,
|
||||
"time_seconds_median": 0.01835409994237125,
|
||||
"time_milliseconds_median": 18.35409994237125,
|
||||
"theoretical_relative_cost_vs_first": 8.0
|
||||
},
|
||||
{
|
||||
"batch_size": 16,
|
||||
"hidden_size": 512,
|
||||
"total_blocks": 21,
|
||||
"time_seconds_median": 0.019410799955949187,
|
||||
"time_milliseconds_median": 19.410799955949187,
|
||||
"theoretical_relative_cost_vs_first": 16.0
|
||||
},
|
||||
{
|
||||
"batch_size": 32,
|
||||
"hidden_size": 512,
|
||||
"total_blocks": 21,
|
||||
"time_seconds_median": 0.01993660000152886,
|
||||
"time_milliseconds_median": 19.93660000152886,
|
||||
"theoretical_relative_cost_vs_first": 32.0
|
||||
}
|
||||
],
|
||||
"log_log_fit": {
|
||||
"exponent": 0.006306562161759777,
|
||||
"intercept": -3.965757991421921,
|
||||
"r2": 0.07918679416134755
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
{
|
||||
"source": "data deploy hasil.sqlite",
|
||||
"table": "data_deploy_hasil",
|
||||
"model_path": "best_nbeats_model_retrained_db.pth",
|
||||
"input_size": 7,
|
||||
"output_size": 1,
|
||||
"hidden_size": 512,
|
||||
"stacks": 7,
|
||||
"blocks_per_stack": 3,
|
||||
"learning_rate": 0.0001,
|
||||
"epochs": 700,
|
||||
"best_val_loss": 0.0005434106569737196
|
||||
}
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
streamlit>=1.35
|
||||
pandas>=2.0
|
||||
numpy>=1.24
|
||||
scikit-learn>=1.3
|
||||
openpyxl>=3.1
|
||||
selenium>=4.20
|
||||
torch>=2.1
|
||||
fastapi>=0.115
|
||||
uvicorn[standard]>=0.30
|
||||
requests>=2.31
|
||||
|
|
@ -0,0 +1,350 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "a8479860",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Scraping Harga Beras Medium BI\n",
|
||||
"\n",
|
||||
"Mengambil harga terbaru untuk **Beras Kualitas Medium I dan II** pada **Jawa Timur - Kab. Banyuwangi** dari situs PIHPS Bank Indonesia. ID komoditas dan wilayah diambil dari endpoint referensi agar tetap mengikuti perubahan data di situs."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "c3db673e",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"<selenium.webdriver.remote.webelement.WebElement (session=\"8e127cd331be0c4b70b419cca364b1b3\", element=\"f.5733A7B02DBE6EF2537F2F51FB65F501.d.61B78F598BD215CED9E06D85340C50EB.e.55\")>"
|
||||
]
|
||||
},
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from datetime import date, datetime, timedelta\n",
|
||||
"from pathlib import Path\n",
|
||||
"import sqlite3\n",
|
||||
"import time\n",
|
||||
"\n",
|
||||
"import pandas as pd\n",
|
||||
"from IPython.display import display\n",
|
||||
"from selenium import webdriver\n",
|
||||
"from selenium.common.exceptions import TimeoutException\n",
|
||||
"from selenium.webdriver.common.by import By\n",
|
||||
"from selenium.webdriver.chrome.options import Options\n",
|
||||
"from selenium.webdriver.support.ui import WebDriverWait\n",
|
||||
"\n",
|
||||
"BASE_URL = 'https://www.bi.go.id/hargapangan'\n",
|
||||
"HALAMAN_URL = f'{BASE_URL}/TabelHarga/PasarTradisionalDaerah'\n",
|
||||
"OUTPUT_FILE = Path('deploy_data.xlsx')\n",
|
||||
"RESULT_DB_FILE = Path('data deploy hasil.sqlite')\n",
|
||||
"RESULT_TABLE = 'data_deploy_hasil'\n",
|
||||
"LEGACY_RESULT_FILE = Path('data deploy hasil.xlsx')\n",
|
||||
"COMMODITIES = ('Beras Kualitas Medium I', 'Beras Kualitas Medium II')\n",
|
||||
"PROVINCE = 'Jawa Timur'\n",
|
||||
"REGENCY = 'Kab. Banyuwangi'\n",
|
||||
"\n",
|
||||
"options = Options()\n",
|
||||
"options.add_argument('--headless=new')\n",
|
||||
"options.add_argument('--window-size=1920,1080')\n",
|
||||
"driver = webdriver.Chrome(options=options)\n",
|
||||
"driver.get(HALAMAN_URL)\n",
|
||||
"wait = WebDriverWait(driver, 30)\n",
|
||||
"wait.until(lambda browser: browser.find_element(By.TAG_NAME, 'body'))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "adabf437",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def click_tree_item(tree_id, item_name):\n",
|
||||
" def find_checkbox(browser):\n",
|
||||
" rows = browser.find_elements(By.CSS_SELECTOR, f'#{tree_id} tr.dx-data-row')\n",
|
||||
" for row in rows:\n",
|
||||
" if row.text.replace('\\u2003', ' ').strip() == item_name:\n",
|
||||
" return row.find_element(By.CSS_SELECTOR, '.dx-select-checkbox')\n",
|
||||
" return False\n",
|
||||
"\n",
|
||||
" checkbox = wait.until(find_checkbox)\n",
|
||||
" if checkbox.get_attribute('aria-checked') != 'true':\n",
|
||||
" driver.execute_script(\"arguments[0].scrollIntoView({block: 'center'});\", checkbox)\n",
|
||||
" checkbox.click()\n",
|
||||
" wait.until(lambda browser: checkbox.get_attribute('aria-checked') == 'true')\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def set_date(date_box_id, value):\n",
|
||||
" driver.execute_script(\n",
|
||||
" \"window.jQuery(arguments[0]).dxDateBox('instance').option('value', arguments[1]);\",\n",
|
||||
" f'#{date_box_id}',\n",
|
||||
" value.isoformat(),\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def configure_filters():\n",
|
||||
" for commodity in COMMODITIES:\n",
|
||||
" click_tree_item('CommodityTree', commodity)\n",
|
||||
" click_tree_item('cboProvince', PROVINCE)\n",
|
||||
" wait.until(lambda browser: browser.find_elements(By.CSS_SELECTOR, '#cboRegency tr.dx-data-row'))\n",
|
||||
" click_tree_item('cboRegency', REGENCY)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def read_visible_prices():\n",
|
||||
" header_row = next(\n",
|
||||
" row for row in driver.find_elements(By.CSS_SELECTOR, '.dx-datagrid-headers tr.dx-header-row')\n",
|
||||
" if any('/' in (cell.get_attribute('aria-label') or '') for cell in row.find_elements(By.CSS_SELECTOR, 'td[role=\"columnheader\"]'))\n",
|
||||
" )\n",
|
||||
" headers = [\n",
|
||||
" (cell.get_attribute('aria-label') or '').replace('Column ', '').strip()\n",
|
||||
" for cell in header_row.find_elements(By.CSS_SELECTOR, 'td[role=\"columnheader\"]')\n",
|
||||
" if (cell.get_attribute('aria-label') or '').startswith('Column ')\n",
|
||||
" ]\n",
|
||||
"\n",
|
||||
" tables = driver.find_elements(By.CSS_SELECTOR, '.dx-datagrid-rowsview table')\n",
|
||||
" label_rows = []\n",
|
||||
" value_rows = []\n",
|
||||
" for table in tables:\n",
|
||||
" rows = table.find_elements(By.CSS_SELECTOR, 'tbody tr.dx-data-row')\n",
|
||||
" parsed_rows = [\n",
|
||||
" [cell.text.replace('\\u2003', ' ').strip() for cell in row.find_elements(By.CSS_SELECTOR, 'td')]\n",
|
||||
" for row in rows\n",
|
||||
" ]\n",
|
||||
" if any(any(commodity in ' '.join(row) for commodity in COMMODITIES) for row in parsed_rows):\n",
|
||||
" label_rows = parsed_rows\n",
|
||||
" elif parsed_rows:\n",
|
||||
" value_rows = parsed_rows\n",
|
||||
"\n",
|
||||
" if not headers or not label_rows or not value_rows:\n",
|
||||
" raise RuntimeError('Tabel harga belum menampilkan data.')\n",
|
||||
"\n",
|
||||
" records = []\n",
|
||||
" for label_row, value_row in zip(label_rows, value_rows):\n",
|
||||
" if len(label_row) < 2:\n",
|
||||
" continue\n",
|
||||
" commodity = label_row[1]\n",
|
||||
" commodity = commodity.split(' ', 1)[1] if commodity[:1].isdigit() else commodity\n",
|
||||
" if commodity not in COMMODITIES:\n",
|
||||
" continue\n",
|
||||
" for header, value in zip(headers[2:], value_row[2:]):\n",
|
||||
" if value and value != '-':\n",
|
||||
" records.append({'komoditas': commodity, 'tanggal': header, 'harga': value})\n",
|
||||
" return pd.DataFrame(records)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def scrape_harga(tanggal_mulai, tanggal_selesai):\n",
|
||||
" set_date('dboDateMulai', tanggal_mulai)\n",
|
||||
" set_date('dboDateSelesai', tanggal_selesai)\n",
|
||||
" driver.find_element(By.ID, 'btnReport').click()\n",
|
||||
" wait.until(\n",
|
||||
" lambda browser: browser.find_elements(\n",
|
||||
" By.CSS_SELECTOR,\n",
|
||||
" '.dx-datagrid-rowsview table tbody tr.dx-data-row',\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
" time.sleep(1)\n",
|
||||
" return read_visible_prices()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "651fb356",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Filter Selenium siap: Beras Kualitas Medium I, Beras Kualitas Medium II | Jawa Timur | Kab. Banyuwangi\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"configure_filters()\n",
|
||||
"print('Filter Selenium siap:', ', '.join(COMMODITIES), '|', PROVINCE, '|', REGENCY)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "f31d1817",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"ename": "RuntimeError",
|
||||
"evalue": "Tabel harga belum menampilkan data.",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[1;31mRuntimeError\u001b[0m Traceback (most recent call last)",
|
||||
"Cell \u001b[1;32mIn[4], line 112\u001b[0m\n\u001b[0;32m 110\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m last_date \u001b[38;5;241m<\u001b[39m tanggal_terakhir_diizinkan:\n\u001b[0;32m 111\u001b[0m tanggal_mulai \u001b[38;5;241m=\u001b[39m last_date \u001b[38;5;241m+\u001b[39m timedelta(days\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m--> 112\u001b[0m hasil \u001b[38;5;241m=\u001b[39m \u001b[43mscrape_harga\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtanggal_mulai\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtanggal_terakhir_diizinkan\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 113\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m hasil\u001b[38;5;241m.\u001b[39mempty:\n\u001b[0;32m 114\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mTidak ada harga pada rentang tanggal tersebut.\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
|
||||
"Cell \u001b[1;32mIn[2], line 85\u001b[0m, in \u001b[0;36mscrape_harga\u001b[1;34m(tanggal_mulai, tanggal_selesai)\u001b[0m\n\u001b[0;32m 78\u001b[0m wait\u001b[38;5;241m.\u001b[39muntil(\n\u001b[0;32m 79\u001b[0m \u001b[38;5;28;01mlambda\u001b[39;00m browser: browser\u001b[38;5;241m.\u001b[39mfind_elements(\n\u001b[0;32m 80\u001b[0m By\u001b[38;5;241m.\u001b[39mCSS_SELECTOR,\n\u001b[0;32m 81\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m.dx-datagrid-rowsview table tbody tr.dx-data-row\u001b[39m\u001b[38;5;124m'\u001b[39m,\n\u001b[0;32m 82\u001b[0m )\n\u001b[0;32m 83\u001b[0m )\n\u001b[0;32m 84\u001b[0m time\u001b[38;5;241m.\u001b[39msleep(\u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m---> 85\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mread_visible_prices\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
|
||||
"Cell \u001b[1;32mIn[2], line 58\u001b[0m, in \u001b[0;36mread_visible_prices\u001b[1;34m()\u001b[0m\n\u001b[0;32m 55\u001b[0m value_rows \u001b[38;5;241m=\u001b[39m parsed_rows\n\u001b[0;32m 57\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m headers \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m label_rows \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m value_rows:\n\u001b[1;32m---> 58\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mTabel harga belum menampilkan data.\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m 60\u001b[0m records \u001b[38;5;241m=\u001b[39m []\n\u001b[0;32m 61\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m label_row, value_row \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(label_rows, value_rows):\n",
|
||||
"\u001b[1;31mRuntimeError\u001b[0m: Tabel harga belum menampilkan data."
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"def parse_deploy_date(value):\n",
|
||||
" return pd.to_datetime(str(value).replace(' ', ''), format='%d/%m/%Y', errors='coerce')\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def format_price(value):\n",
|
||||
" number = pd.to_numeric(str(value).replace(',', '').replace('Rp', '').strip(), errors='coerce')\n",
|
||||
" if pd.isna(number):\n",
|
||||
" return value\n",
|
||||
" return f'{number:,.0f}'\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def update_deploy_workbook(deploy, data):\n",
|
||||
" deploy_labels = deploy['Komoditas (Rp)'].astype(str).str.strip()\n",
|
||||
" updated_columns = []\n",
|
||||
" for record in data.to_dict('records'):\n",
|
||||
" commodity = record['komoditas'].strip()\n",
|
||||
" date_column = record['tanggal'].strip()\n",
|
||||
" row_index = deploy.index[deploy_labels.eq(commodity)]\n",
|
||||
" if row_index.empty:\n",
|
||||
" raise ValueError(f'Baris {commodity!r} tidak ditemukan di {OUTPUT_FILE}.')\n",
|
||||
" deploy.loc[row_index[0], date_column] = format_price(record['harga'])\n",
|
||||
" if date_column not in updated_columns:\n",
|
||||
" updated_columns.append(date_column)\n",
|
||||
" return updated_columns\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def load_result_store():\n",
|
||||
" if RESULT_DB_FILE.exists():\n",
|
||||
" with sqlite3.connect(RESULT_DB_FILE) as connection:\n",
|
||||
" result = pd.read_sql_query(\n",
|
||||
" f'SELECT \"Tanggal\", \"Harga Medium 1\", \"Harga Medium 2\", \"rata2\" FROM {RESULT_TABLE}',\n",
|
||||
" connection,\n",
|
||||
" )\n",
|
||||
" else:\n",
|
||||
" if not LEGACY_RESULT_FILE.exists():\n",
|
||||
" raise FileNotFoundError(f'Sumber migrasi tidak ditemukan: {LEGACY_RESULT_FILE}')\n",
|
||||
" result = pd.read_excel(LEGACY_RESULT_FILE)\n",
|
||||
" result['Tanggal'] = pd.to_datetime(result['Tanggal'], errors='coerce')\n",
|
||||
" return result.dropna(subset=['Tanggal'])\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def save_result_store(result):\n",
|
||||
" stored = result.copy()\n",
|
||||
" stored['Tanggal'] = pd.to_datetime(stored['Tanggal']).dt.strftime('%Y-%m-%d')\n",
|
||||
" with sqlite3.connect(RESULT_DB_FILE) as connection:\n",
|
||||
" stored.to_sql(RESULT_TABLE, connection, if_exists='replace', index=False)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def reconstruct_deploy_result(scraped_data):\n",
|
||||
" expected_columns = ['Tanggal', 'Harga Medium 1', 'Harga Medium 2', 'rata2']\n",
|
||||
" existing = load_result_store()[expected_columns].set_index('Tanggal')\n",
|
||||
"\n",
|
||||
" if scraped_data.empty:\n",
|
||||
" combined = existing\n",
|
||||
" else:\n",
|
||||
" scraped = scraped_data.copy()\n",
|
||||
" scraped['Tanggal'] = pd.to_datetime(\n",
|
||||
" scraped['tanggal'].astype(str).str.replace(' ', '', regex=False),\n",
|
||||
" format='%d/%m/%Y',\n",
|
||||
" errors='coerce',\n",
|
||||
" )\n",
|
||||
" scraped['harga'] = pd.to_numeric(\n",
|
||||
" scraped['harga'].astype(str).str.replace(',', '', regex=False).str.replace('Rp', '', regex=False),\n",
|
||||
" errors='coerce',\n",
|
||||
" )\n",
|
||||
" scraped = scraped.dropna(subset=['Tanggal', 'harga'])\n",
|
||||
" new_values = scraped.pivot_table(\n",
|
||||
" index='Tanggal',\n",
|
||||
" columns='komoditas',\n",
|
||||
" values='harga',\n",
|
||||
" aggfunc='last',\n",
|
||||
" ).rename(columns={\n",
|
||||
" 'Beras Kualitas Medium I': 'Harga Medium 1',\n",
|
||||
" 'Beras Kualitas Medium II': 'Harga Medium 2',\n",
|
||||
" })\n",
|
||||
" new_values = new_values.reindex(columns=['Harga Medium 1', 'Harga Medium 2'])\n",
|
||||
" combined = pd.concat([\n",
|
||||
" existing.drop(index=new_values.index, errors='ignore'),\n",
|
||||
" new_values,\n",
|
||||
" ], axis=0)\n",
|
||||
"\n",
|
||||
" combined = combined.sort_index()\n",
|
||||
" full_dates = pd.date_range(combined.index.min(), combined.index.max(), freq='D')\n",
|
||||
" combined = combined.reindex(full_dates)\n",
|
||||
" combined.index.name = 'Tanggal'\n",
|
||||
" price_columns = ['Harga Medium 1', 'Harga Medium 2']\n",
|
||||
" combined[price_columns] = combined[price_columns].interpolate(method='linear', limit_direction='both')\n",
|
||||
" combined['rata2'] = combined[price_columns].mean(axis=1)\n",
|
||||
" result = combined.reset_index()[expected_columns]\n",
|
||||
" save_result_store(result)\n",
|
||||
" return result\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"deploy = pd.read_excel(OUTPUT_FILE)\n",
|
||||
"date_columns = [column for column in deploy.columns if pd.notna(parse_deploy_date(column))]\n",
|
||||
"if not date_columns:\n",
|
||||
" raise ValueError(f'Tidak ada kolom tanggal yang ditemukan di {OUTPUT_FILE}.')\n",
|
||||
"\n",
|
||||
"last_date = max(parse_deploy_date(column).date() for column in date_columns)\n",
|
||||
"tanggal_sekarang = date.today()\n",
|
||||
"tanggal_terakhir_diizinkan = (\n",
|
||||
" tanggal_sekarang\n",
|
||||
" if datetime.now().hour >= 13\n",
|
||||
" else tanggal_sekarang - timedelta(days=1)\n",
|
||||
")\n",
|
||||
"hasil = pd.DataFrame(columns=['komoditas', 'tanggal', 'harga'])\n",
|
||||
"tanggal_diisi = []\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" if last_date < tanggal_terakhir_diizinkan:\n",
|
||||
" tanggal_mulai = last_date + timedelta(days=1)\n",
|
||||
" hasil = scrape_harga(tanggal_mulai, tanggal_terakhir_diizinkan)\n",
|
||||
" if hasil.empty:\n",
|
||||
" raise RuntimeError('Tidak ada harga pada rentang tanggal tersebut.')\n",
|
||||
" tanggal_diisi = update_deploy_workbook(deploy, hasil)\n",
|
||||
" print(f'{tanggal_mulai} sampai {tanggal_terakhir_diizinkan}: {len(hasil)} harga diterima.')\n",
|
||||
" else:\n",
|
||||
" print(f'Excel sudah terisi sampai batas data: {tanggal_terakhir_diizinkan}.')\n",
|
||||
"finally:\n",
|
||||
" driver.quit()\n",
|
||||
"\n",
|
||||
"deploy.to_excel(OUTPUT_FILE, index=False)\n",
|
||||
"result = reconstruct_deploy_result(hasil)\n",
|
||||
"print(f'Workbook wide diperbarui: {OUTPUT_FILE.resolve()}')\n",
|
||||
"print(f'Data hasil scraping disimpan ke SQLite: {RESULT_DB_FILE.resolve()}')\n",
|
||||
"print(f'Tabel SQLite: {RESULT_TABLE}')\n",
|
||||
"print(f'Tanggal baru/diperbarui: {len(tanggal_diisi)}')\n",
|
||||
"print(f'Ukuran data deploy hasil: {result.shape[0]} baris')\n",
|
||||
"display(result.tail())"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.6"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
Loading…
Reference in New Issue