diff --git a/sigap-website/app/_utils/common.ts b/sigap-website/app/_utils/common.ts index 0cfaab2..bca4d16 100644 --- a/sigap-website/app/_utils/common.ts +++ b/sigap-website/app/_utils/common.ts @@ -374,6 +374,24 @@ function formatDateV2(date: Date, formatStr: string): string { .replace('ss', pad(date.getSeconds())); } +/** + * Universal Custom ID Generator + * Creates structured, readable IDs for any system or entity + * + * @param {Object} options - Configuration options + * @param {string} options.prefix - Primary identifier prefix (e.g., "CRIME", "USER", "INVOICE") + * @param {Object} options.segments - Collection of ID segments to include + * @param {string[]} options.segments.codes - Array of short codes (e.g., region codes, department codes) + * @param {number} options.segments.year - Year to include in the ID + * @param {number} options.segments.sequentialDigits - Number of digits for sequential number + * @param {boolean} options.segments.includeDate - Whether to include current date + * @param {string} options.segments.dateFormat - Format for date (e.g., "yyyy-MM-dd", "dd/MM/yyyy") + * @param {boolean} options.segments.includeTime - Whether to include timestamp + * @param {string} options.format - Custom format string for ID structure + * @param {string} options.separator - Character to separate ID components + * @param {boolean} options.upperCase - Convert result to uppercase + * @returns {string} - Generated custom ID + */ /** * Universal Custom ID Generator * Creates structured, readable IDs for any system or entity @@ -527,7 +545,7 @@ export function generateId( config.segments.codes.length > 0 ? config.segments.codes.join(config.separator) : '', - // year: yearValue, + year: yearValue, // Added the year value to components sequence: sequentialNum, date: dateString, time: timeString, @@ -568,7 +586,7 @@ export function generateId( const parts = []; if (components.prefix) parts.push(components.prefix); if (components.codes) parts.push(components.codes); - // if (components.year) parts.push(components.year); + if (components.year) parts.push(components.year); if (components.date) parts.push(components.date); if (components.time) parts.push(components.time); if (components.sequence) parts.push(components.sequence); diff --git a/sigap-website/prisma/data/excels/crimes/district_summary_2020_2024.csv b/sigap-website/prisma/data/excels/crimes/district_summary_2020_2024.csv new file mode 100644 index 0000000..1182f76 --- /dev/null +++ b/sigap-website/prisma/data/excels/crimes/district_summary_2020_2024.csv @@ -0,0 +1,32 @@ +district_id,district_name,crime_total,crime_cleared,avg_crime,avg_score,level +350901,Jombang,118,110,23.6,79,low +350902,Kencong,91,74,18.2,84,low +350903,Sumberbaru,157,130,31.4,72,high +350904,Gumukmas,91,78,18.2,84,low +350905,Umbulsari,115,88,23.0,79,low +350906,Tanggul,266,213,53.2,52,high +350907,Semboro,94,89,18.8,83,low +350908,Puger,180,160,36.0,67,high +350909,Bangsalsari,154,132,30.8,72,high +350910,Balung,278,223,55.6,49,high +350911,Wuluhan,216,176,43.2,61,high +350912,Ambulu,157,124,31.4,72,high +350913,Rambipuji,278,170,55.6,49,low +350914,Panti,139,109,27.8,75,low +350915,Sukorambi,77,55,15.4,86,low +350916,Jenggawah,235,224,47.0,57,high +350917,Ajung,88,77,17.6,84,low +350918,Tempurejo,74,48,14.8,87,low +350919,Kaliwates,194,139,38.8,65,medium +350920,Patrang,202,145,40.4,63,medium +350921,Sumbersari,217,138,43.4,61,medium +350922,Arjasa,116,81,23.2,79,low +350923,Mumbulsari,99,81,19.8,82,low +350924,Pakusari,152,129,30.4,73,low +350925,Jelbuk,132,90,26.4,76,low +350926,Mayang,89,60,17.8,84,low +350927,Kalisat,270,163,54.0,51,high +350928,Ledokombo,103,76,20.6,82,low +350929,Sukowono,171,125,34.2,69,low +350930,Silo,143,85,28.6,74,high +350931,Sumberjambe,109,94,21.8,80,low diff --git a/sigap-website/prisma/migrations/20250423141344_init_migration/migration.sql b/sigap-website/prisma/migrations/20250423141344_init_migration/migration.sql index 8063902..ddb9056 100644 --- a/sigap-website/prisma/migrations/20250423141344_init_migration/migration.sql +++ b/sigap-website/prisma/migrations/20250423141344_init_migration/migration.sql @@ -1,10 +1,29 @@ -- CreateExtension -CREATE EXTENSION IF NOT EXISTS "pgcrypto"; - CREATE SCHEMA IF NOT EXISTS "extensions"; CREATE SCHEMA IF NOT EXISTS "gis"; +CREATE SCHEMA IF NOT EXISTS "pgsodium"; + +CREATE SCHEMA IF NOT EXISTS "vault"; + +CREATE SCHEMA IF NOT EXISTS graphql; + +CREATE EXTENSION IF NOT EXISTS pg_graphql WITH SCHEMA graphql; + +CREATE EXTENSION IF NOT EXISTS pg_net WITH SCHEMA "extensions"; + +CREATE EXTENSION IF NOT EXISTS pgsodium WITH SCHEMA "pgsodium"; +CREATE EXTENSION IF NOT EXISTS supabase_vault WITH SCHEMA "vault"; +CREATE EXTENSION IF NOT EXISTS pg_stat_statements WITH SCHEMA extensions; + +CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA extensions; + +CREATE EXTENSION IF NOT EXISTS pgjwt WITH SCHEMA extensions; + +CREATE EXTENSION IF NOT EXISTS "pgcrypto"; + + CREATE EXTENSION IF NOT EXISTS "postgis" WITH SCHEMA "gis"; -- CreateExtension @@ -175,7 +194,7 @@ CREATE TABLE "crimes" ( "number_of_crime" INTEGER NOT NULL DEFAULT 0, "score" DOUBLE PRECISION NOT NULL DEFAULT 0, "updated_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP, - "year" INTEGER NOT NULL, + "year" INTEGER, CONSTRAINT "crimes_pkey" PRIMARY KEY ("id") ); diff --git a/sigap-website/prisma/migrations/20250423155055_add_phone_and_create_polres_jemebr/migration.sql b/sigap-website/prisma/migrations/20250423155055_add_phone_and_create_polres_jemebr/migration.sql index 59237c5..f7d061c 100644 --- a/sigap-website/prisma/migrations/20250423155055_add_phone_and_create_polres_jemebr/migration.sql +++ b/sigap-website/prisma/migrations/20250423155055_add_phone_and_create_polres_jemebr/migration.sql @@ -1,2 +1,14 @@ -- AlterTable ALTER TABLE "units" ADD COLUMN "phone" TEXT; + +grant all privileges on all tables in schema public to postgres, anon, authenticated, service_role, prisma; +grant all privileges on all functions in schema public to postgres, anon, authenticated, service_role, prisma; +grant all privileges on all sequences in schema public to postgres, anon, authenticated, service_role, prisma; + +alter default privileges in schema public grant all on tables to postgres, anon, authenticated, service_role, prisma; +alter default privileges in schema public grant all on functions to postgres, anon, authenticated, service_role, prisma; +alter default privileges in schema public grant all on sequences to postgres, anon, authenticated, service_role, prisma; + +grant usage on schema "public" to anon; +grant usage on schema "public" to authenticated; +grant usage on schema "public" to prisma; \ No newline at end of file diff --git a/sigap-website/prisma/schema.prisma b/sigap-website/prisma/schema.prisma index 240b12b..a446e83 100644 --- a/sigap-website/prisma/schema.prisma +++ b/sigap-website/prisma/schema.prisma @@ -173,7 +173,7 @@ model crimes { number_of_crime Int @default(0) score Float @default(0) updated_at DateTime? @default(now()) @db.Timestamptz(6) - year Int + year Int? crime_incidents crime_incidents[] districts districts @relation(fields: [district_id], references: [id], onDelete: Cascade, onUpdate: NoAction) diff --git a/sigap-website/prisma/seed.ts b/sigap-website/prisma/seed.ts index b945bed..ce48853 100644 --- a/sigap-website/prisma/seed.ts +++ b/sigap-website/prisma/seed.ts @@ -7,7 +7,7 @@ import { GeoJSONSeeder } from './seeds/geographic'; import { execSync } from 'child_process'; import { DemographicsSeeder } from './seeds/demographic'; import { CrimeCategoriesSeeder } from './seeds/crime-category'; -import { CrimeIncidentsSeeder } from './seeds/crime-incident'; + import { UnitSeeder } from './seeds/units'; import { CrimesSeeder } from './seeds/crimes'; import { CrimeIncidentsSeeder as DetailedCrimeIncidentsSeeder } from './seeds/crime-incidents'; @@ -29,13 +29,13 @@ class DatabaseSeeder { // Daftar semua seeders di sini this.seeders = [ - // new RoleSeeder(prisma), - // new ResourceSeeder(prisma), - // new PermissionSeeder(prisma), - // new CrimeCategoriesSeeder(prisma), - // new GeoJSONSeeder(prisma), - // new UnitSeeder(prisma), - // new DemographicsSeeder(prisma), + new RoleSeeder(prisma), + new ResourceSeeder(prisma), + new PermissionSeeder(prisma), + new CrimeCategoriesSeeder(prisma), + new GeoJSONSeeder(prisma), + new UnitSeeder(prisma), + new DemographicsSeeder(prisma), new CrimesSeeder(prisma), new DetailedCrimeIncidentsSeeder(prisma), // Add the new crime incidents seeder // new CrimeIncidentsSeederV2(prisma), diff --git a/sigap-website/prisma/seeds/crime-incident.ts b/sigap-website/prisma/seeds/crime-incident.ts deleted file mode 100644 index b18ff3c..0000000 --- a/sigap-website/prisma/seeds/crime-incident.ts +++ /dev/null @@ -1,1594 +0,0 @@ -// Modified CrimeIncidentsSeeder.ts to use real JSON data with specific district mapping -import { createClient } from '../../app/_utils/supabase/client'; -import { generateCode, generateId } from '../../app/_utils/common'; -import { PrismaClient, crime_status } from '@prisma/client'; -import axios from 'axios'; -import { kmeans } from 'ml-kmeans'; -import path from 'path'; -import fs from 'fs'; - -// Define proper interfaces to replace "any" types -interface District { - id: string; - name: string; - city_id: string; -} - -interface City { - id: string; - name: string; -} - -interface CrimeCategory { - id: string; - name: string; -} - -interface Crime { - id: string; - district_id: string; - year: number; - month: number | null; - number_of_crime: number; - level: 'low' | 'medium' | 'high'; - score: number; -} - -interface Location { - id: string; - district_id: string; - name?: string; - description?: string; - type?: string; - year?: number; - latitude: number; - longitude: number; - address?: string; - location?: string; -} - -interface Demographics { - district_id: string; - year: number; - population_density: number; - number_of_unemployed: number; -} - -interface CategoryDistribution { - category: string; - id?: string; - count: number; - total: number; - probability: number; -} - -interface NormalizationParams { - year: number; - crimes: { min: number; max: number; range: number }; - density: { min: number; max: number; range: number }; - unemployment: { min: number; max: number; range: number }; -} - -interface KMeansModel { - centroids: number[][]; - clusters: Record; - normalization?: NormalizationParams; -} - -interface DistrictData { - numberOfCrimes: number; - populationDensity: number; - unemploymentRate: number; -} - -interface CreateLocationDto { - district_id: string; - event_id: string; - name?: string; - description?: string; - address?: string; - type?: string; - latitude: number; - longitude: number; - land_area?: number; - polygon?: Record; - geometry?: Record; - location: any; - year: number; // Added year field -} - -export class CrimeIncidentsSeeder { - private mapboxToken: string; - private totalIncidentsCreated: number = 0; - - // Add tracking properties for statistics - private districtStats: Record< - string, - { - crimeRecords: number; - crimeIncidents: number; - missingRecords: string[]; - } - > = {}; - - // Store district demographic data to avoid repeated queries - private districtDemographicCache: Record< - string, - Record< - number, - { - populationDensity: number; - unemployment: number; - } - > - > = {}; - - // Store the k-means model for each year - private kmeansModels: Record = {}; - - private normalizationParams: NormalizationParams | null = null; - - // Map police unit (SEK X) to district names - private unitToDistrictMap: Record = {}; - - // Store events and sessions by year to reuse them - private yearEvents: Record = - {}; - - // Add progress tracking counters - private progressCounters = { - totalCrimes: 0, - totalIncidents: 0, - currentDistrict: '', - currentYear: 0, - currentMonth: '', - categoriesLoaded: 0, - }; - - // Add a log throttling mechanism to avoid excessive logging - private logThrottleCounter = 0; - private readonly LOG_THROTTLE_RATE = 100; // Log every 100 operations - - // Add a tracking property for summary data - private crimeSummaryData: Record< - number, - { - crime_total: number; - crime_cleared: number; - clearance_rate: number; - } - > = {}; - - constructor( - private prisma: PrismaClient, - private supabase = createClient() - ) { - this.mapboxToken = process.env.NEXT_PUBLIC_MAPBOX_ACCESS_TOKEN || ''; - } - - async run(): Promise { - console.log(`🔵 Starting crime incidents seeding for 2020-2024...`); - - // Load the summary data first - await this.loadCrimeSummaryData(); - - // Get all districts, cities and categories - const districts = await this.prisma.districts.findMany(); - const cities = await this.prisma.cities.findMany(); - const crimeCategories = await this.prisma.crime_categories.findMany(); - - // Initialize statistics tracking for each district - districts.forEach((district) => { - this.districtStats[district.id] = { - crimeRecords: 0, - crimeIncidents: 0, - missingRecords: [], - }; - }); - - // Create a map of category names to category objects - const categoryMap: Record = {}; - crimeCategories.forEach((category) => { - categoryMap[category.name.toLowerCase()] = category; - }); - - // Initialize police unit to district mapping - this.initializeUnitToDistrictMap(districts); - - // Pre-load all demographics data for faster access - await this.preloadDemographicData(districts); - - // Clear existing data - await this.prisma.locations.deleteMany({}); - await this.prisma.crime_incidents.deleteMany({}); - await this.prisma.crimes.deleteMany({}); - await this.prisma.sessions.deleteMany({}); - await this.prisma.events.deleteMany({}); - - // Define year range (2020-2024) - const years = [2020]; - - // Create one event and session for each year - await this.createYearlyEventsAndSessions(years); - - // Define months mapping for JSON data - const monthsMap: Record = this.initializeMonthsMap(); - - // Initialize counters - const districtCrimeCount: Record< - string, - Record> - > = {}; - const cityCrimeCount: Record> = {}; - - // Initialize counters for all districts and cities - districts.forEach((district) => { - districtCrimeCount[district.id] = {}; - years.forEach((year) => { - districtCrimeCount[district.id][year] = {}; - // Initialize for each month (1-12) and for yearly total ("annual") - for (let month = 1; month <= 12; month++) { - districtCrimeCount[district.id][year][month] = 0; - } - - // Use a string like "annual" instead of null - districtCrimeCount[district.id][year][''] = 0; - }); - }); - - cities.forEach((city) => { - cityCrimeCount[city.id] = {}; - years.forEach((year) => { - cityCrimeCount[city.id][year] = 0; - }); - }); - - // First create all crime records for districts and years - both yearly and monthly entries - for (const district of districts) { - const city = await this.prisma.cities.findFirst({ - where: { id: district.city_id }, - }); - - if (!city) { - throw new Error(`City not found for district ID: ${district.name}`); - } - - this.progressCounters.currentDistrict = district.name; - - for (const year of years) { - this.progressCounters.currentYear = year; - - // Create the yearly entry (month = null) - await this.createCrimeRecord(district, city, year, null); - - // Create monthly entries (month = 1-12) - for (let month = 1; month <= 12; month++) { - await this.createCrimeRecord(district, city, year, month); - } - - this.progressCounters.totalCrimes += 13; // 1 annual + 12 monthly records - - if (this.progressCounters.totalCrimes % 50 === 0) { - console.log( - `✓ CR: Created ${this.progressCounters.totalCrimes} crime records (latest: ${district.name}, ${year})` - ); - } - } - } - - // Add this: Load crime category distribution from JSON files - const crimeCategoryDistribution: Record = - {}; - - // Track all unique categories across all years - const allUniqueCategories: Record< - string, - { totalCount: number; years: Set } - > = {}; - - for (const year of years) { - try { - // Load crime category distribution from JSON instead of CSV - const categoryDistPath = path.resolve( - `prisma/data/jsons/crimes/crime ${year}.json` - ); - - if (fs.existsSync(categoryDistPath)) { - const categoryDistData = fs.readFileSync(categoryDistPath, 'utf8'); - const categoryRecords = JSON.parse(categoryDistData); - - // Process the data to create weighted distribution - const categories: { category: string; count: number }[] = []; - let totalCrimes = 0; - - categoryRecords.forEach((record: any) => { - // Skip the "Jumlah" (total) category - if (record.category === 'Jumlah') return; - - const category = record.category; - const totalCount = record.sum_ct || 0; - - if (category && totalCount > 0) { - categories.push({ category, count: totalCount }); - totalCrimes += totalCount; - - // Track in our master category list - if (!allUniqueCategories[category]) { - allUniqueCategories[category] = { - totalCount: 0, - years: new Set(), - }; - } - allUniqueCategories[category].totalCount += totalCount; - allUniqueCategories[category].years.add(year); - } - }); - - // Calculate probabilities for weighted selection - crimeCategoryDistribution[year] = categories.map((cat) => ({ - category: cat.category, - count: cat.count, - total: totalCrimes, - probability: cat.count / totalCrimes, - })); - - this.progressCounters.categoriesLoaded += categories.length; - console.log( - `📊 Year ${year}: Loaded ${categories.length} crime categories (total: ${this.progressCounters.categoriesLoaded} raw entries)` - ); - } else { - console.log(`❌ No crime category JSON file found for ${year}`); - } - } catch (error) { - console.error( - `❌ Error processing crime categories for year ${year}:`, - error - ); - } - } - - // Normalize and deduplicate categories - const normalizedCategories = - this.normalizeCrimeCategories(allUniqueCategories); - console.log( - `🔍 Found ${Object.keys(normalizedCategories).length} unique crime categories after normalization` - ); - - // Replace the yearly distributions with normalized versions - for (const year of years) { - if (crimeCategoryDistribution[year]) { - crimeCategoryDistribution[year] = this.createNormalizedDistribution( - crimeCategoryDistribution[year], - normalizedCategories - ); - } - } - - // Process actual crime data per police unit (district) - for (const year of years) { - try { - // Read JSON file for this year (e.g., units 2020.json) - const jsonFilePath = path.resolve( - `prisma/data/jsons/units/units ${year}.json` - ); - - // Check if file exists - if (!fs.existsSync(jsonFilePath)) { - console.log(`JSON file for year ${year} not found. Skipping.`); - continue; - } - - const jsonData = fs.readFileSync(jsonFilePath, 'utf8'); - const unitRecords = JSON.parse(jsonData); - - console.log( - `🔍 Year ${year}: Processing ${unitRecords.length} police unit records...` - ); - - // Process each police unit in the JSON - for (const record of unitRecords) { - const unitName = `SEK ${record.unit}`; - - // Skip if unit name is empty or "Ajung" (which has all zeros) - if (!record.unit || record.unit === 'Ajung') { - continue; - } - - // Find the matching district - const districtName = this.getDistrictNameFromUnit(unitName); - - const district = districts.find( - (d) => d.name.toLowerCase() === districtName.toLowerCase() - ); - - if (!district) { - continue; - } - - // Get the crime record for this district and year - const crime = (await this.prisma.crimes.findFirst({ - where: { - district_id: district.id, - year, - }, - })) as Crime | null; - - if (!crime) { - console.error( - `Crime record not found for district ID: ${district.id}, year: ${year}` - ); - continue; - } - - // Increment crime records counter for this district - this.districtStats[district.id].crimeRecords++; - - // Get geographic data for the district - const geoData = (await this.prisma.geographics.findFirst({ - where: { district_id: district.id }, - })) as Location | null; - - if (!geoData) { - console.error( - `Geographic data not found for district ID: ${district.id}` - ); - continue; - } - - // Process each month's data for this police unit - for (const [monthKey, monthIndex] of Object.entries(monthsMap)) { - // Use the JSON format keys: jan_ct, jan_cc, etc. - const ctKey = `${monthKey}_ct`; - const ccKey = `${monthKey}_cc`; - - // Check if the keys exist in the record - if (!(ctKey in record) || !(ccKey in record)) { - continue; - } - - // Parse crime counts from the JSON data - const crimeTotal = parseInt(record[ctKey], 10) || 0; - const crimeCleared = parseInt(record[ccKey], 10) || 0; - - if (crimeTotal <= 0) { - continue; - } - - // Update monthly and yearly counters - const monthNumber = monthIndex + 1; // Convert zero-based index to 1-12 month format - districtCrimeCount[district.id][year][monthNumber] += crimeTotal; - districtCrimeCount[district.id][year]['annual'] += crimeTotal; // Update yearly total - cityCrimeCount[district.city_id][year] += crimeTotal; - - this.progressCounters.currentDistrict = district.name; - this.progressCounters.currentYear = year; - this.progressCounters.currentMonth = monthKey; - - // Only log periodically to reduce console output - if (this.logThrottleCounter % this.LOG_THROTTLE_RATE === 0) { - console.log( - `🔹 ${district.name}, ${year}-${monthKey}: CT=${crimeTotal}, CC=${crimeCleared}` - ); - } - this.logThrottleCounter++; - - // Create resolved incidents (CC) using the year's event and session - for (let i = 0; i < crimeCleared; i++) { - await this.createCrimeIncident( - district, - crime, - crimeCategoryDistribution[year] || [], - geoData, - year, - monthIndex, - 'resolved' - ); - - this.totalIncidentsCreated++; - this.progressCounters.totalIncidents++; - this.districtStats[district.id].crimeIncidents++; - - // Log progress periodically - if (this.progressCounters.totalIncidents % 500 === 0) { - console.log( - `✅ CI: Created ${this.progressCounters.totalIncidents} incidents (${this.progressCounters.currentDistrict}, ${this.progressCounters.currentYear}-${this.progressCounters.currentMonth})` - ); - } - } - - // Create closed incidents (CT - CC) using the year's event and session - for (let i = 0; i < crimeTotal - crimeCleared; i++) { - await this.createCrimeIncident( - district, - crime, - crimeCategoryDistribution[year] || [], - geoData, - year, - monthIndex, - 'closed' - ); - - this.totalIncidentsCreated++; - this.progressCounters.totalIncidents++; - this.districtStats[district.id].crimeIncidents++; - - // Log progress periodically - if (this.progressCounters.totalIncidents % 500 === 0) { - console.log( - `✅ CI: Created ${this.progressCounters.totalIncidents} incidents (${this.progressCounters.currentDistrict}, ${this.progressCounters.currentYear}-${this.progressCounters.currentMonth})` - ); - } - } - } - } - } catch (error) { - console.error(`❌ Error processing JSON for year ${year}:`, error); - } - } - - // Process each year to run K-means clustering and update crime records - for (const year of years) { - // Collect district data for K-means clustering - const districtData: Record = {}; - - for (const district of districts) { - const crimeCount = districtCrimeCount[district.id][year]['annual'] || 0; - const demographics = this.districtDemographicCache[district.id][year]; - const populationDensity = demographics.populationDensity; - const unemploymentRate = demographics.unemployment; - - districtData[district.id] = { - numberOfCrimes: crimeCount, - populationDensity, - unemploymentRate, - }; - } - - // Run K-means clustering for this year - const clustersCreated = await this.runKMeansClustering( - districtData, - year - ); - - if (!clustersCreated) { - console.error( - `❌ Failed to create K-means clusters for year ${year}. Skipping updates.` - ); - continue; - } - - console.log(`🧮 K-means clustering completed for year ${year}`); - - // Verify all districts have clusters assigned before proceeding - const missingClusters = []; - for (const district of districts) { - if ( - !this.kmeansModels[year] || - !this.kmeansModels[year].clusters[district.id] - ) { - missingClusters.push(district.name); - } - } - - if (missingClusters.length > 0) { - console.error( - `❌ Missing clusters for ${missingClusters.length} districts in year ${year}: ${missingClusters.join(', ')}` - ); - console.error( - 'Cannot proceed without complete K-means clustering. Skipping year.' - ); - continue; - } - - // Update all district crime records with counts and rates - both monthly and yearly - let updatedRecords = 0; - - for (const district of districts) { - // First verify that we have all required data for this district and year - if (!districtCrimeCount[district.id]) { - console.error( - `❌ No crime count data found for district ${district.name}, id: ${district.id}` - ); - continue; - } - - if (!districtCrimeCount[district.id][year]) { - console.error( - `❌ No crime count data found for year ${year} in district ${district.name}` - ); - continue; - } - - // Get the yearly total (annual) - if (!('annual' in districtCrimeCount[district.id][year])) { - // If 'annual' key doesn't exist, check for empty string key (we used that earlier) - if (!('' in districtCrimeCount[district.id][year])) { - console.error( - `❌ No annual crime count found for district ${district.name}, year ${year}` - ); - continue; - } - // Use the empty string key value for annual total - const yearlyTotal = parseInt( - String(districtCrimeCount[district.id][year]['']), - 10 - ); - // Add it also as 'annual' for consistency - districtCrimeCount[district.id][year]['annual'] = yearlyTotal; - } - - // No fallbacks - get actual data or report error - const yearlyTotal = parseInt( - String(districtCrimeCount[district.id][year]['annual']), - 10 - ); - - // Check for demographics data - if ( - !this.districtDemographicCache[district.id] || - !this.districtDemographicCache[district.id][year] - ) { - console.error( - `❌ No demographic data found for district ${district.name}, year ${year}` - ); - continue; - } - - const demographics = this.districtDemographicCache[district.id][year]; - - // Ensure we have a valid cluster assignment from K-means - const yearlyRate = this.kmeansModels[year].clusters[district.id]; - - if (!yearlyRate) { - console.error( - `❌ No cluster assigned for district ${district.name} in year ${year}` - ); - continue; - } - - const yearlyScore = this.calculateCrimeScore( - yearlyTotal, - demographics.populationDensity, - demographics.unemployment, - year - ); - - // Only log a small sample of updates to avoid flooding the console - if (updatedRecords % 100 === 0) { - console.log( - `📝 CR Update: ${district.name}, ${year} - Count=${yearlyTotal}, Level=${yearlyRate}, Score=${yearlyScore}` - ); - } - updatedRecords++; - - try { - await this.prisma.crimes.updateMany({ - where: { - district_id: district.id, - year, - month: null, - }, - data: { - number_of_crime: yearlyTotal, - level: yearlyRate, - score: yearlyScore, - }, - }); - } catch (error) { - console.error( - `❌ Error updating crime record for district ${district.name}, year ${year}:`, - error - ); - continue; - } - - // Update each monthly record - for (let month = 1; month <= 12; month++) { - // Verify monthly data exists - if (!(month in districtCrimeCount[district.id][year])) { - console.error( - `❌ No crime count data found for month ${month} in district ${district.name}, year ${year}` - ); - continue; - } - - // No fallbacks - get actual data - const monthlyTotal = parseInt( - String(districtCrimeCount[district.id][year][month]), - 10 - ); - - // Use the same rate as yearly for consistency - const monthlyRate = yearlyRate; - - try { - const monthlyScore = this.calculateCrimeScore( - monthlyTotal, - demographics.populationDensity, - demographics.unemployment, - year - ); - - await this.prisma.crimes.updateMany({ - where: { - district_id: district.id, - year, - month, - }, - data: { - number_of_crime: monthlyTotal, - level: monthlyRate, - score: monthlyScore, - }, - }); - } catch (error) { - console.error( - `❌ Error updating monthly crime record for district ${district.name}, year ${year}, month ${month}:`, - error - ); - continue; - } - } - } - - console.log( - `📊 Updated ${updatedRecords} crime records for year ${year}` - ); - } - - console.log( - `🎉 Seeding completed! Created ${this.totalIncidentsCreated} crime incidents` - ); - - // Display summary statistics at the end - this.displaySummary(); - } - - /** - * Normalize crime categories to handle similar entries - */ - private normalizeCrimeCategories( - categories: Record }> - ): Record { - const normalized: Record = - {}; - - // Create mapping for similar categories - const categoryMapping: Record = { - pencurian: 'pencurian', - 'pencurian dengan kekerasan': 'pencurian dengan kekerasan', - 'pencurian dengan pemberatan': 'pencurian dengan pemberatan', - penipuan: 'penipuan', - penganiayaan: 'penganiayaan', - narkotika: 'narkotika', - pembunuhan: 'pembunuhan', - kdrt: 'kekerasan dalam rumah tangga', - 'kekerasan dalam rumah tangga': 'kekerasan dalam rumah tangga', - perjudian: 'perjudian', - korupsi: 'korupsi', - pemerasan: 'pemerasan', - pemerkosaan: 'pemerkosaan', - penggelapan: 'penggelapan', - penadahan: 'penadahan', - perzinahan: 'perzinahan', - pelecehan: 'pelecehan', - penculikan: 'penculikan', - pemalsuan: 'pemalsuan', - 'kekerasan terhadap anak': 'kekerasan terhadap anak', - }; - - // Process each category - for (const [category, data] of Object.entries(categories)) { - // Get normalized name or use original - let normalizedName = category.toLowerCase(); - - // Find best match in mapping - for (const [pattern, mappedName] of Object.entries(categoryMapping)) { - if (normalizedName.includes(pattern)) { - normalizedName = mappedName; - break; - } - } - - // Add to normalized categories or increment count - if (!normalized[normalizedName]) { - normalized[normalizedName] = { - normalized: normalizedName, - count: data.totalCount, - }; - } else { - normalized[normalizedName].count += data.totalCount; - } - } - - console.log( - `Normalized ${Object.keys(categories).length} categories into ${Object.keys(normalized).length} unique categories` - ); - return normalized; - } - - /** - * Create a normalized distribution from raw category data - */ - private createNormalizedDistribution( - rawDistribution: CategoryDistribution[], - normalizedCategories: Record - ): CategoryDistribution[] { - // Group raw distributions by normalized category - const groupedData: Record = {}; - let totalCrimes = 0; - - // Sum up counts for each normalized category - for (const catData of rawDistribution) { - const originalName = catData.category.toLowerCase(); - let normalizedName = originalName; - - // Find matching normalized name - for (const [normName, data] of Object.entries(normalizedCategories)) { - if ( - originalName.includes(normName) || - normName.includes(originalName) - ) { - normalizedName = data.normalized; - break; - } - } - - // Add to grouped data - if (!groupedData[normalizedName]) { - groupedData[normalizedName] = 0; - } - groupedData[normalizedName] += catData.count; - totalCrimes += catData.count; - } - - // Create new distribution - const normalizedDist: CategoryDistribution[] = []; - - for (const [category, count] of Object.entries(groupedData)) { - normalizedDist.push({ - category, - count, - total: totalCrimes, - probability: count / totalCrimes, - }); - } - - return normalizedDist; - } - - /** - * Load crime summary data from JSON file to ensure we match the correct totals - */ - private async loadCrimeSummaryData(): Promise { - try { - const summaryFilePath = path.resolve( - `prisma/data/jsons/units/units summary.json` - ); - - if (fs.existsSync(summaryFilePath)) { - const summaryData = fs.readFileSync(summaryFilePath, 'utf8'); - const summaryRecords = JSON.parse(summaryData); - - // Convert array to record with year as key - summaryRecords.forEach((record: any) => { - this.crimeSummaryData[record.year] = { - crime_total: record.crime_total, - crime_cleared: record.crime_cleared, - clearance_rate: record.clearance_rate, - }; - }); - - console.log( - `📊 Loaded crime summary data for ${Object.keys(this.crimeSummaryData).length} years` - ); - } else { - console.error('❌ Crime summary data file not found!'); - } - } catch (error) { - console.error('❌ Error loading crime summary data:', error); - } - } - - /** - * Calculate safety score based on crime count, population density, and unemployment - * Score ranges from 100-0, where higher score means SAFER area (reversed from original) - * 100 = safest, 0 = least safe - */ - private calculateCrimeScore( - crimeCount: number, - populationDensity: number, - unemploymentRate: number, - year: number - ): number { - // Ensure inputs are valid numbers with no fallbacks - if ( - isNaN(crimeCount) || - isNaN(populationDensity) || - isNaN(unemploymentRate) - ) { - console.error( - `❌ Invalid inputs for crime score calculation: crimeCount=${crimeCount}, populationDensity=${populationDensity}, unemploymentRate=${unemploymentRate}` - ); - throw new Error('Invalid inputs for crime score calculation'); - } - - // Get the normalization params for the year - const normParams = this.kmeansModels[year].normalization; - if (!normParams) { - console.error(`❌ No normalization parameters found for year ${year}`); - throw new Error(`No normalization parameters found for year ${year}`); - } - - // Ensure the normalization parameters have valid ranges - if ( - !normParams.crimes.range || - !normParams.density.range || - !normParams.unemployment.range - ) { - console.error(`❌ Invalid normalization ranges for year ${year}`); - throw new Error(`Invalid normalization ranges for year ${year}`); - } - - // Normalize the features using min-max scaling (ensuring we don't divide by zero) - const normalizedCrimes = - (crimeCount - normParams.crimes.min) / normParams.crimes.range; - const normalizedDensity = - (populationDensity - normParams.density.min) / normParams.density.range; - const normalizedUnemployment = - (unemploymentRate - normParams.unemployment.min) / - normParams.unemployment.range; - - // Custom weighting for safety score (100-0): - // - Crime count has the highest impact (60%) - // - Population density has moderate impact (25%) - higher density can lead to more crime opportunities - // - Unemployment has some impact (15%) - higher unemployment can correlate with crime rates - const crimeWeight = 0.6; - const densityWeight = 0.25; - const unemploymentWeight = 0.15; - - // Non-linear transformation to better represent safety - // Using exponential function to emphasize areas with high crime rates - const crimeFactor = Math.pow(normalizedCrimes, 1.2); // Slightly exponential - - // Calculate weighted score of unsafety factors - const unsafetyScore = - crimeFactor * crimeWeight + - normalizedDensity * densityWeight + - normalizedUnemployment * unemploymentWeight; - - // Invert the score to get safety score (100 = safest, 0 = least safe) - // Scale to 0-100 range and ensure the result is always a valid integer - const safetyScore = Math.min( - Math.max(Math.round(100 - unsafetyScore * 100), 0), - 100 - ); - - return safetyScore; - } - - /** - * Create one event and session for each year - */ - private async createYearlyEventsAndSessions(years: number[]): Promise { - console.log('🗓️ Creating one event and session for each year...'); - - // Get the admin user - const admin = await this.prisma.users.findUnique({ - where: { email: 'polresjember@gmail.com' }, - }); - - if (!admin) { - console.error(`Admin user not found. Cannot create events and sessions.`); - return; - } - - // Create one event and session for each year - for (const year of years) { - // Create an event for this year - const event = await this.prisma.events.create({ - data: { - user_id: admin.id, - name: `Crime incidents for year ${year}`, - description: `Collection of all crime incidents that occurred in ${year}`, - }, - }); - - // Create a session for this year - const session = await this.prisma.sessions.create({ - data: { - user_id: admin.id, - event_id: event.id, - status: 'completed', - }, - }); - - // Store references for later use - this.yearEvents[year] = { - eventId: event.id, - sessionId: session.id, - }; - - console.log(`✓ Created event and session for year ${year}`); - } - } - - /** - * Display summary statistics after seeding is complete - */ - private displaySummary(): void { - console.log('\n=== 📊 CRIME DATA SEEDING SUMMARY 📊 ==='); - console.log( - `🔹 Total Crime Incidents (CI) created: ${this.totalIncidentsCreated}` - ); - console.log( - `🔹 Total Events created: ${Object.keys(this.yearEvents).length}` - ); - console.log( - `🔹 Total Sessions created: ${Object.keys(this.yearEvents).length}` - ); - console.log( - `🔹 Total Crime Records (CR) created: ${this.progressCounters.totalCrimes}` - ); - - // Get district names for better output - this.prisma.districts.findMany().then((districts) => { - const districtNames: Record = {}; - districts.forEach((d) => (districtNames[d.id] = d.name)); - - console.log('\n=== STATS PER DISTRICT ==='); - for (const [districtId, stats] of Object.entries(this.districtStats)) { - const districtName = districtNames[districtId] || districtId; - console.log(`\n${districtName}:`); - console.log(` - Crime Records (CR): ${stats.crimeRecords}`); - console.log(` - Crime Incidents (CI): ${stats.crimeIncidents}`); - - if (stats.missingRecords.length > 0) { - console.log(` - Missing records: ${stats.missingRecords.length}`); - // Display up to 3 missing records as examples - const samplesToShow = Math.min(3, stats.missingRecords.length); - for (let i = 0; i < samplesToShow; i++) { - console.log(` * ${stats.missingRecords[i]}`); - } - if (stats.missingRecords.length > samplesToShow) { - console.log( - ` * ... and ${stats.missingRecords.length - samplesToShow} more` - ); - } - } - } - - // Display districts with no crime incidents - const noIncidentsDistricts = Object.entries(this.districtStats) - .filter(([_, stats]) => stats.crimeIncidents === 0) - .map(([id, _]) => districtNames[id] || id); - - if (noIncidentsDistricts.length > 0) { - console.log('\n=== DISTRICTS WITH NO CRIME INCIDENTS ==='); - noIncidentsDistricts.forEach((name) => console.log(`- ${name}`)); - } - - console.log('\n=== END OF SUMMARY ==='); - }); - } - - /** - * Create a crime record for a district with optional month - */ - private async createCrimeRecord( - district: District, - city: City, - year: number, - month: number | null - ): Promise { - // Generate a unique ID for the crime record - const newCrimeId = generateId({ - prefix: 'CR', - segments: { - codes: [city.id], - sequentialDigits: 4, - year, - }, - format: '{prefix}-{codes}-{sequence}-{year}', - separator: '-', - randomSequence: false, - uniquenessStrategy: 'counter', - }); - - // Description should indicate if it's a monthly or yearly record - const description = month - ? `Crime data for ${district.name} in ${year}-${month}` - : `Annual crime data for ${district.name} in ${year}`; - - await this.prisma.crimes.create({ - data: { - id: newCrimeId, - district_id: district.id, - year, - month, - method: 'KMeans', - number_of_crime: 0, // Initialize with zero - level: 'low', - score: 0, // Initialize with zero - }, - }); - } - - /** - * Initialize police unit to district name mapping - */ - private initializeUnitToDistrictMap(districts: District[]): void { - console.log('Initializing unit to district mapping...'); - - // Extract district names from police unit names (remove "SEK " prefix) - districts.forEach((district) => { - let districtName = district.name.toLowerCase(); - - this.unitToDistrictMap[`SEK ${districtName}`] = districtName; - }); - } - - /** - * Extract district name from police unit name - */ - private getDistrictNameFromUnit(unitName: string): string { - if (unitName === 'SEK SEMPOLAN') unitName = 'SEK SILO'; - - const knownDistrict = this.unitToDistrictMap[unitName]; - if (knownDistrict) { - return knownDistrict; - } - - // If not found, try extracting from SEK X format by removing "SEK " prefix - if (unitName.startsWith('SEK ')) { - return unitName.substring(4).toLowerCase(); - } - - // Return original as fallback - return unitName; - } - - /** - * Update the monthsMap to match the JSON data's month naming - */ - private initializeMonthsMap(): Record { - return { - jan: 0, - feb: 1, - mar: 2, - apr: 3, - mei: 4, - jun: 5, - jul: 6, - aug: 7, - sep: 8, - okt: 9, - nov: 10, - des: 11, - }; - } - - /** - * Create a crime incident with randomly assigned category based on weighted distribution - */ - private async createCrimeIncident( - district: District, - crime: Crime, - crimeCategoryDistribution: CategoryDistribution[], - geoData: Location, - year: number, - monthIndex: number, - status: crime_status - ): Promise { - if (!geoData) { - console.error( - `Geographic data not found for district ID: ${district.id}` - ); - return; - } - - let crimeCategory: CrimeCategory | null = null; - let location = district.name; - - if (!crimeCategoryDistribution || crimeCategoryDistribution.length === 0) { - console.error( - `No crime category distribution available for year ${year}` - ); - return; - } - - // Use weighted random selection based on actual crime statistics - const rand = Math.random(); - let cumulativeProbability = 0; - - for (const catData of crimeCategoryDistribution) { - cumulativeProbability += catData.probability; - - if (rand <= cumulativeProbability) { - // Find the corresponding category in the database - crimeCategory = (await this.prisma.crime_categories.findFirst({ - where: { - name: { - contains: catData.category, - mode: 'insensitive', - }, - }, - })) as CrimeCategory | null; - - if (!crimeCategory) { - console.error( - `No crime category found for distribution data: ${catData.category}` - ); - return; - } - - // Generate random coordinates within district boundary (with small variation) - const radius = 0.02; // Approximately 2km - const angle = Math.random() * Math.PI * 2; // Random angle in radians - const distance = Math.sqrt(Math.random()) * radius; // Random distance within circle - - // Calculate offset using simple approximation (not exact but good enough for this purpose) - const latOffset = distance * Math.cos(angle); - const lngOffset = distance * Math.sin(angle); - - // Apply offset to base coordinates - const latitude = geoData.latitude + latOffset; - const longitude = geoData.longitude + lngOffset; - - // Generate random date and time within the specified month of the year - const startOfMonth = new Date(year, monthIndex, 1); - const endOfMonth = new Date(year, monthIndex + 1, 0); // Last day of month - const randomDate = new Date( - this.getRandomNumber(startOfMonth.getTime(), endOfMonth.getTime()) - ); - - // Generate random time with more realistic distribution - // Most crimes happen in the evening or at night - let hours; - const timeDistribution = Math.random(); - if (timeDistribution < 0.1) { - // 10% in early morning (00:00-06:00) - hours = Math.floor(this.getRandomNumber(0, 6)); - } else if (timeDistribution < 0.3) { - // 20% in daytime (06:00-18:00) - hours = Math.floor(this.getRandomNumber(6, 18)); - } else { - // 70% in evening/night (18:00-24:00) - hours = Math.floor(this.getRandomNumber(18, 24)); - } - - const minutes = Math.floor(this.getRandomNumber(0, 59)); - const randomTime = new Date( - randomDate.getFullYear(), - randomDate.getMonth(), - randomDate.getDate(), - hours, - minutes, - 0 - ); - - // Randomly determine victim count based on crime category - let victimCount = 0; - const categoryName = crimeCategory.name.toLowerCase(); - if (categoryName.includes('pembunuhan')) { - victimCount = Math.floor(this.getRandomNumber(1, 3)); // 1-2 victims for murder - } else if ( - categoryName.includes('penganiayaan') || - categoryName.includes('kekerasan') || - categoryName.includes('penculikan') - ) { - victimCount = Math.floor(this.getRandomNumber(1, 4)); // 1-3 victims for assault - } else if ( - categoryName.includes('pencurian') || - categoryName.includes('penipuan') || - categoryName.includes('pemerasan') - ) { - victimCount = Math.floor(this.getRandomNumber(0, 3)); // 0-2 victims for theft/fraud - } - - // Generate more realistic description based on crime category - const locations = [ - `di daerah ${district.name}`, - `di sekitar ${district.name}`, - `di wilayah ${district.name}`, - `di jalan utama ${district.name}`, - `di perumahan ${district.name}`, - `di pasar ${district.name}`, - `di perbatasan ${district.name}`, - ]; - - const randomLocation = - locations[Math.floor(Math.random() * locations.length)]; - - const timeOfDay = - hours < 6 - ? 'dini hari' - : hours < 12 - ? 'pagi' - : hours < 18 - ? 'sore' - : 'malam'; - - const descriptions = [ - `Kasus ${crimeCategory.name.toLowerCase()} ${randomLocation}`, - `Laporan ${crimeCategory.name.toLowerCase()} terjadi pada ${timeOfDay} ${randomLocation}`, - `${crimeCategory.name} dilaporkan ${randomLocation}`, - `Insiden ${crimeCategory.name.toLowerCase()} terjadi ${randomLocation}`, - `Kejadian ${crimeCategory.name.toLowerCase()} ${randomLocation}`, - ]; - - const randomDescription = - descriptions[Math.floor(Math.random() * descriptions.length)]; - - // Use the existing event and session for this year instead of creating new ones - const yearEventData = this.yearEvents[year]; - if (!yearEventData) { - console.error(`No event found for year ${year}`); - return; - } - - // Generate a location ID for this incident - const locationId = generateId({ - prefix: 'LOC', - segments: { - codes: [district.id], - sequentialDigits: 4, - }, - format: '{prefix}-{codes}-{sequence}', - separator: '-', - randomSequence: false, - uniquenessStrategy: 'counter', - }); - - // Create the location record - let { data: locationData, error: locationError } = await this.supabase - .from('locations') - .insert({ - id: locationId, - district_id: district.id, - event_id: yearEventData.eventId, - type: 'crime incident', - latitude, - longitude, - address: location, - location: `POINT(${longitude} ${latitude})`, - year: year, - }) - .select(); - - if (locationError) { - console.error( - `Error inserting location for district ${district.name} (${year}):`, - locationError - ); - return; - } - - // Generate ID for the crime incident - const newCrimeIncidentId = generateId({ - prefix: 'CI', - segments: { - codes: [district.id], - sequentialDigits: 4, - year: year, - }, - format: '{prefix}-{codes}-{sequence}-{year}', - separator: '-', - randomSequence: false, - uniquenessStrategy: 'counter', - }); - - // Create incident with the new location - await this.prisma.crime_incidents.create({ - data: { - id: newCrimeIncidentId, - crime_id: crime.id, - crime_category_id: crimeCategory.id, - location_id: locationId, - timestamp: randomTime, - description: randomDescription, - victim_count: victimCount, - status, - }, - }); - - break; // Exit the loop once we found our category - } - } - } - - private async runKMeansClustering( - districtData: Record, - year: number - ): Promise { - // Convert to array format needed by kmeans library - const data: number[][] = []; - const districtIds: string[] = []; - - // Extract all values for each feature to calculate statistics - const allCrimes: number[] = []; - const allDensities: number[] = []; - const allUnemployment: number[] = []; - - // First pass: collect all values - for (const [districtId, values] of Object.entries(districtData)) { - allCrimes.push(values.numberOfCrimes); - allDensities.push(values.populationDensity); - allUnemployment.push(values.unemploymentRate); - districtIds.push(districtId); - } - - // Calculate statistics for normalization - // Find min and max for each feature - const crimeStats = { - min: Math.min(...allCrimes), - max: Math.max(...allCrimes), - range: 0, - }; - crimeStats.range = crimeStats.max - crimeStats.min || 1; // Avoid division by zero - - const densityStats = { - min: Math.min(...allDensities), - max: Math.max(...allDensities), - range: 0, - }; - densityStats.range = densityStats.max - densityStats.min || 1; - - const unemploymentStats = { - min: Math.min(...allUnemployment), - max: Math.max(...allUnemployment), - range: 0, - }; - unemploymentStats.range = - unemploymentStats.max - unemploymentStats.min || 1; - - // Store normalization params for later prediction - this.normalizationParams = { - year, - crimes: crimeStats, - density: densityStats, - unemployment: unemploymentStats, - }; - - // Second pass: normalize using min-max scaling - for (const [districtId, values] of Object.entries(districtData)) { - // Min-max scaling: (value - min) / range -> scales to [0,1] - const normalizedCrimes = - (values.numberOfCrimes - crimeStats.min) / crimeStats.range; - const normalizedDensity = - (values.populationDensity - densityStats.min) / densityStats.range; - const normalizedUnemployment = - (values.unemploymentRate - unemploymentStats.min) / - unemploymentStats.range; - - data.push([normalizedCrimes, normalizedDensity, normalizedUnemployment]); - } - - if (data.length === 0) { - console.error(`❌ No data for K-means clustering for year ${year}`); - return false; - } - - try { - // Run K-means with 3 clusters (low, medium, high) - const result = kmeans(data, 3, { - initialization: 'kmeans++', - maxIterations: 100, - }); - - // Determine which cluster corresponds to which label (low, medium, high) - const clusterCentroids = result.centroids; - - // Sort clusters by the sum of their centroids (higher sum = higher crime rate) - const clusterSums = clusterCentroids.map((centroid) => - centroid.reduce((sum, val) => sum + val, 0) - ); - - const sortedIndices = clusterSums - .map((sum, index) => ({ sum, index })) - .sort((a, b) => a.sum - b.sum) - .map((item) => item.index); - - // Map sorted indices to labels - const labelMap: Record = { - [sortedIndices[0]]: 'low', - [sortedIndices[1]]: 'medium', - [sortedIndices[2]]: 'high', - }; - - // Create mapping from district ID to cluster label - const clusters: Record = {}; - for (let i = 0; i < districtIds.length; i++) { - const clusterId = result.clusters[i]; - clusters[districtIds[i]] = labelMap[clusterId]; - } - - // Verify that all districts have a cluster assigned - const clusterCount = Object.keys(clusters).length; - const districtCount = Object.keys(districtData).length; - - if (clusterCount !== districtCount) { - console.error( - `❌ K-means clustering failed to assign clusters to all districts. Expected ${districtCount}, got ${clusterCount}` - ); - return false; - } - - // Store the K-means model and normalization params for this year - this.kmeansModels[year] = { - centroids: clusterCentroids, - clusters: clusters, - normalization: this.normalizationParams, - }; - - return true; - } catch (error) { - console.error( - `❌ Error running K-means clustering for year ${year}:`, - error - ); - return false; - } - } - - /** - * Preload demographic data for all districts and years - */ - private async preloadDemographicData(districts: District[]): Promise { - console.log('Preloading demographic data...'); - - for (const district of districts) { - this.districtDemographicCache[district.id] = {}; - - const demographics = (await this.prisma.demographics.findMany({ - where: { district_id: district.id }, - })) as Demographics[]; - - for (const demo of demographics) { - // Ensure populationDensity is properly retrieved - const populationDensity = demo.population_density || 0; - - this.districtDemographicCache[district.id][demo.year] = { - populationDensity: populationDensity, - unemployment: demo.number_of_unemployed || 5, - }; - } - } - - console.log('Demographic data preloaded'); - } - - /** - * Get street name from Mapbox API based on coordinates - */ - private async getStreetFromMapbox(lng: number, lat: number): Promise { - try { - const response = await axios.get( - `https://api.mapbox.com/search/geocode/v6/reverse?longitude=${lng}&latitude=${lat}&access_token=${this.mapboxToken}` - ); - - if ( - response.data && - response.data.features && - response.data.features.length > 0 - ) { - // Extract full_address from the first feature - const fullAddress = response.data.features[0].properties.full_address; - return ( - fullAddress || - `Jalan Tidak Diketahui No. ${Math.floor(this.getRandomNumber(1, 100))}` - ); - } - - // Fallback if no address found - return `Jalan Tidak Diketahui No. ${Math.floor(this.getRandomNumber(1, 100))}`; - } catch (error) { - console.error('Error fetching street from Mapbox:', error); - throw error; - } - } - - private getRandomNumber(min: number, max: number): number { - return Math.random() * (max - min) + min; - } -} diff --git a/sigap-website/prisma/seeds/crime-incidents.ts b/sigap-website/prisma/seeds/crime-incidents.ts index 864a98a..3199bd5 100644 --- a/sigap-website/prisma/seeds/crime-incidents.ts +++ b/sigap-website/prisma/seeds/crime-incidents.ts @@ -285,6 +285,14 @@ export class CrimeIncidentsSeeder { // Calculate a date within the crime's month const year = crime.year; + + if (!year) { + console.error( + `Year is not defined for crime record ${crime.id}, skipping.` + ); + return []; + } + const month = (crime.month as number) - 1; // JavaScript months are 0-indexed const maxDay = new Date(year, month + 1, 0).getDate(); // Get last day of month const day = Math.floor(Math.random() * maxDay) + 1; @@ -299,7 +307,7 @@ export class CrimeIncidentsSeeder { segments: { codes: [district.cities.id], sequentialDigits: 4, - year: crime.year, + year, }, format: '{prefix}-{codes}-{sequence}-{year}', separator: '-', diff --git a/sigap-website/prisma/seeds/crimes.ts b/sigap-website/prisma/seeds/crimes.ts index aac4eb1..c1317c6 100644 --- a/sigap-website/prisma/seeds/crimes.ts +++ b/sigap-website/prisma/seeds/crimes.ts @@ -16,6 +16,8 @@ export class CrimesSeeder { async run(): Promise { console.log('🌱 Seeding crimes data...'); + // Clear existing data + try { // Create test user const user = await this.createUsers(); @@ -32,6 +34,9 @@ export class CrimesSeeder { // Import yearly crime data from CSV file await this.importYearlyCrimeData(); + // Import all-year crime summaries (2020-2024) + await this.importAllYearSummaries(); + console.log('✅ Crime seeding completed successfully.'); } catch (error) { console.error('❌ Error seeding crimes:', error); @@ -276,137 +281,80 @@ export class CrimesSeeder { console.log(`Imported ${records.length} yearly crime records.`); } - // private async generateYearlyCrimeSummaries() { - // console.log('Generating yearly crime summaries...'); + private async importAllYearSummaries() { + console.log('Importing all-year (2020-2024) crime summaries...'); - // // Check if yearly summaries already exist (records with null month) - // const existingYearlySummary = await this.prisma.crimes.findFirst({ - // where: { month: null }, - // }); + // Check if all-year summaries already exist (records with null month and null year) + const existingAllYearSummaries = await this.prisma.crimes.findFirst({ + where: { month: null, year: null }, + }); - // if (existingYearlySummary) { - // console.log('Yearly summaries already exist, skipping generation.'); - // return; - // } + if (existingAllYearSummaries) { + console.log('All-year crime summaries already exist, skipping import.'); + return; + } - // // Get all districts and years combinations - // const districtsYears = await this.prisma.crimes.findMany({ - // select: { - // district_id: true, - // year: true, - // }, - // distinct: ['district_id', 'year'], - // }); + // Read CSV file + const csvFilePath = path.resolve( + __dirname, + '../data/excels/crimes/district_summary_2020_2024.csv' + ); + const fileContent = fs.readFileSync(csvFilePath, { encoding: 'utf-8' }); - // // For each district and year, calculate yearly summary - // for (const { district_id, year } of districtsYears) { - // // Calculate sum of crimes for the district and year - // const result = await this.prisma.crimes.aggregate({ - // _sum: { - // number_of_crime: true, - // }, - // where: { - // district_id: district_id, - // year: year, - // month: { - // not: null, - // }, - // }, - // }); + // Parse CSV + const records = parse(fileContent, { + columns: true, + skip_empty_lines: true, + }); - // if (!result || !result._sum.number_of_crime) { - // console.log( - // `No monthly data found for district ${district_id} in year ${year}. Skipping...` - // ); - // continue; - // } + for (const record of records) { + const crimeRate = record.level.toLowerCase() as crime_rates; + const districtId = record.district_id; - // const totalCrimes = result._sum.number_of_crime; + const city = await this.prisma.cities.findFirst({ + where: { + districts: { + some: { + id: districtId, + }, + }, + }, + }); - // // Get average level based on monthly data (use the most common level) - // const levelResult = await this.prisma.crimes - // .groupBy({ - // by: ['level'], - // where: { - // district_id: district_id, - // year: year, - // month: { - // not: null, - // }, - // }, - // _count: { - // level: true, - // }, - // orderBy: { - // _count: { - // level: 'desc', - // }, - // }, - // take: 1, - // }) - // .then((results) => - // results.map((result) => ({ - // level: result.level, - // count: result._count.level, - // })) - // ); + if (!city) { + console.error(`City not found for district ${districtId}`); + continue; + } - // if (levelResult.length === 0) { - // console.log( - // `No level data found for district ${district_id} in year ${year}. Skipping...` - // ); - // continue; - // } + // Create a unique ID for all-year summary data + const crimeId = generateId({ + prefix: 'CR', + segments: { + codes: [city.id], + sequentialDigits: 4, + }, + format: '{prefix}-{codes}-{sequence}', + separator: '-', + randomSequence: false, + uniquenessStrategy: 'counter', + }); - // const level = levelResult[0].level || 'low'; + await this.prisma.crimes.create({ + data: { + id: crimeId, + district_id: districtId, + level: crimeRate, + method: 'kmeans', + month: null, + year: null, + number_of_crime: parseInt(record.crime_total), + score: parseFloat(record.avg_score), + }, + }); + } - // const city = await this.prisma.cities.findFirst({ - // where: { - // districts: { - // some: { - // id: district_id, - // }, - // }, - // }, - // }); - - // if (!city) { - // console.error(`City not found for district ${district_id}`); - // continue; - // } - - // // Create yearly summary record - // const newCrimeId = generateId({ - // prefix: 'CR', - // segments: { - // codes: [city.id], - // sequentialDigits: 4, - // year, - // }, - // format: '{prefix}-{codes}-{sequence}-{year}', - // separator: '-', - // randomSequence: false, - // uniquenessStrategy: 'counter', - // }); - - // await this.prisma.crimes.create({ - // data: { - // id: newCrimeId, - // district_id: district_id as string, - // level: level as crime_rates, - // method: 'kmeans', - // month: null, - // year: year as number, - // number_of_crime: totalCrimes, - // score: 100 - Math.min(totalCrimes, 100), // Simple score calculation - // }, - // }); - // } - - // console.log( - // `Generated yearly summaries for ${districtsYears.length} district-year combinations.` - // ); - // } + console.log(`Imported ${records.length} all-year crime summaries.`); + } } // This allows the file to be executed standalone for testing diff --git a/sigap-website/prisma/seeds/geographic.ts b/sigap-website/prisma/seeds/geographic.ts index a0e8baf..b996293 100644 --- a/sigap-website/prisma/seeds/geographic.ts +++ b/sigap-website/prisma/seeds/geographic.ts @@ -105,9 +105,10 @@ export class GeoJSONSeeder { async run(): Promise { console.log('Seeding GeoJSON data...'); - await this.prisma.geographics.deleteMany({}); + await this.prisma.units.deleteMany({}); await this.prisma.districts.deleteMany({}); await this.prisma.cities.deleteMany({}); + await this.prisma.geographics.deleteMany({}); try { // Load GeoJSON file diff --git a/sigap-website/prisma/seeds/role.ts b/sigap-website/prisma/seeds/role.ts index 162ed4b..ff7d39c 100644 --- a/sigap-website/prisma/seeds/role.ts +++ b/sigap-website/prisma/seeds/role.ts @@ -8,6 +8,7 @@ export class RoleSeeder { console.log('Seeding roles...'); await this.prisma.sessions.deleteMany({}); + await this.prisma.locations.deleteMany({}); await this.prisma.events.deleteMany({}); await this.prisma.permissions.deleteMany({}); await this.prisma.users.deleteMany({});