feat: enhance crime incidents seeder with year validation and import all-year summaries

- Added year validation in CrimeIncidentsSeeder to skip records with undefined year. - Implemented importAllYearSummaries method in CrimesSeeder to import crime summaries from CSV for 2020-2024. - Cleared existing data for units before seeding geographic data. - Cleared locations data in role seeder before seeding roles. - Added district summary CSV file for crime data from 2020 to 2024.
2025-04-25 16:58:32 +07:00 · 2025-04-25 16:58:32 +07:00 · bcd71c6cad
parent 63b0721859
commit bcd71c6cad
11 changed files with 176 additions and 1731 deletions
--- a/sigap-website/app/_utils/common.ts
+++ b/sigap-website/app/_utils/common.ts
@ -374,6 +374,24 @@ function formatDateV2(date: Date, formatStr: string): string {
    .replace('ss', pad(date.getSeconds()));
 }

+/**
+ * Universal Custom ID Generator
+ * Creates structured, readable IDs for any system or entity
+ *
+ * @param {Object} options - Configuration options
+ * @param {string} options.prefix - Primary identifier prefix (e.g., "CRIME", "USER", "INVOICE")
+ * @param {Object} options.segments - Collection of ID segments to include
+ * @param {string[]} options.segments.codes - Array of short codes (e.g., region codes, department codes)
+ * @param {number} options.segments.year - Year to include in the ID
+ * @param {number} options.segments.sequentialDigits - Number of digits for sequential number
+ * @param {boolean} options.segments.includeDate - Whether to include current date
+ * @param {string} options.segments.dateFormat - Format for date (e.g., "yyyy-MM-dd", "dd/MM/yyyy")
+ * @param {boolean} options.segments.includeTime - Whether to include timestamp
+ * @param {string} options.format - Custom format string for ID structure
+ * @param {string} options.separator - Character to separate ID components
+ * @param {boolean} options.upperCase - Convert result to uppercase
+ * @returns {string} - Generated custom ID
+ */
 /**
 * Universal Custom ID Generator
 * Creates structured, readable IDs for any system or entity
@ -527,7 +545,7 @@ export function generateId(
      config.segments.codes.length > 0
        ? config.segments.codes.join(config.separator)
        : '',
-    // year: yearValue,
+    year: yearValue, // Added the year value to components
    sequence: sequentialNum,
    date: dateString,
    time: timeString,
@ -568,7 +586,7 @@ export function generateId(
    const parts = [];
    if (components.prefix) parts.push(components.prefix);
    if (components.codes) parts.push(components.codes);
-    // if (components.year) parts.push(components.year);
+    if (components.year) parts.push(components.year);
    if (components.date) parts.push(components.date);
    if (components.time) parts.push(components.time);
    if (components.sequence) parts.push(components.sequence);
--- a/sigap-website/prisma/data/excels/crimes/district_summary_2020_2024.csv
+++ b/sigap-website/prisma/data/excels/crimes/district_summary_2020_2024.csv
@ -0,0 +1,32 @@
+district_id,district_name,crime_total,crime_cleared,avg_crime,avg_score,level
+350901,Jombang,118,110,23.6,79,low
+350902,Kencong,91,74,18.2,84,low
+350903,Sumberbaru,157,130,31.4,72,high
+350904,Gumukmas,91,78,18.2,84,low
+350905,Umbulsari,115,88,23.0,79,low
+350906,Tanggul,266,213,53.2,52,high
+350907,Semboro,94,89,18.8,83,low
+350908,Puger,180,160,36.0,67,high
+350909,Bangsalsari,154,132,30.8,72,high
+350910,Balung,278,223,55.6,49,high
+350911,Wuluhan,216,176,43.2,61,high
+350912,Ambulu,157,124,31.4,72,high
+350913,Rambipuji,278,170,55.6,49,low
+350914,Panti,139,109,27.8,75,low
+350915,Sukorambi,77,55,15.4,86,low
+350916,Jenggawah,235,224,47.0,57,high
+350917,Ajung,88,77,17.6,84,low
+350918,Tempurejo,74,48,14.8,87,low
+350919,Kaliwates,194,139,38.8,65,medium
+350920,Patrang,202,145,40.4,63,medium
+350921,Sumbersari,217,138,43.4,61,medium
+350922,Arjasa,116,81,23.2,79,low
+350923,Mumbulsari,99,81,19.8,82,low
+350924,Pakusari,152,129,30.4,73,low
+350925,Jelbuk,132,90,26.4,76,low
+350926,Mayang,89,60,17.8,84,low
+350927,Kalisat,270,163,54.0,51,high
+350928,Ledokombo,103,76,20.6,82,low
+350929,Sukowono,171,125,34.2,69,low
+350930,Silo,143,85,28.6,74,high
+350931,Sumberjambe,109,94,21.8,80,low
--- a/sigap-website/prisma/migrations/20250423141344_init_migration/migration.sql
+++ b/sigap-website/prisma/migrations/20250423141344_init_migration/migration.sql
@ -1,10 +1,29 @@
 -- CreateExtension
-CREATE EXTENSION IF NOT EXISTS "pgcrypto";
-
 CREATE SCHEMA IF NOT EXISTS "extensions";

 CREATE SCHEMA IF NOT EXISTS "gis";

+CREATE SCHEMA IF NOT EXISTS "pgsodium";
+
+CREATE SCHEMA IF NOT EXISTS "vault";
+
+CREATE SCHEMA IF NOT EXISTS graphql;
+
+CREATE EXTENSION IF NOT EXISTS pg_graphql WITH SCHEMA graphql;
+
+CREATE EXTENSION IF NOT EXISTS pg_net WITH SCHEMA "extensions";
+
+CREATE EXTENSION IF NOT EXISTS pgsodium WITH SCHEMA "pgsodium";
+CREATE EXTENSION IF NOT EXISTS supabase_vault WITH SCHEMA "vault";
+CREATE EXTENSION IF NOT EXISTS pg_stat_statements WITH SCHEMA extensions;
+
+CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA extensions;
+
+CREATE EXTENSION IF NOT EXISTS pgjwt WITH SCHEMA extensions;
+
+CREATE EXTENSION IF NOT EXISTS "pgcrypto";
+
+
 CREATE EXTENSION IF NOT EXISTS "postgis" WITH SCHEMA "gis";

 -- CreateExtension
@ -175,7 +194,7 @@ CREATE TABLE "crimes" (
    "number_of_crime" INTEGER NOT NULL DEFAULT 0,
    "score" DOUBLE PRECISION NOT NULL DEFAULT 0,
    "updated_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP,
-    "year" INTEGER NOT NULL,
+    "year" INTEGER,

    CONSTRAINT "crimes_pkey" PRIMARY KEY ("id")
 );
--- a/sigap-website/prisma/migrations/20250423155055_add_phone_and_create_polres_jemebr/migration.sql
+++ b/sigap-website/prisma/migrations/20250423155055_add_phone_and_create_polres_jemebr/migration.sql
@ -1,2 +1,14 @@
 -- AlterTable
 ALTER TABLE "units" ADD COLUMN     "phone" TEXT;
+
+grant all privileges on all tables in schema public to postgres, anon, authenticated, service_role, prisma;
+grant all privileges on all functions in schema public to postgres, anon, authenticated, service_role, prisma;
+grant all privileges on all sequences in schema public to postgres, anon, authenticated, service_role, prisma;
+
+alter default privileges in schema public grant all on tables to postgres, anon, authenticated, service_role, prisma;
+alter default privileges in schema public grant all on functions to postgres, anon, authenticated, service_role, prisma;
+alter default privileges in schema public grant all on sequences to postgres, anon, authenticated, service_role, prisma;
+
+grant usage on schema "public" to anon;
+grant usage on schema "public" to authenticated;
+grant usage on schema "public" to prisma;
--- a/sigap-website/prisma/schema.prisma
+++ b/sigap-website/prisma/schema.prisma
@ -173,7 +173,7 @@ model crimes {
  number_of_crime Int               @default(0)
  score           Float             @default(0)
  updated_at      DateTime?         @default(now()) @db.Timestamptz(6)
-  year            Int
+  year            Int?
  crime_incidents crime_incidents[]
  districts       districts         @relation(fields: [district_id], references: [id], onDelete: Cascade, onUpdate: NoAction)

--- a/sigap-website/prisma/seed.ts
+++ b/sigap-website/prisma/seed.ts
@ -7,7 +7,7 @@ import { GeoJSONSeeder } from './seeds/geographic';
 import { execSync } from 'child_process';
 import { DemographicsSeeder } from './seeds/demographic';
 import { CrimeCategoriesSeeder } from './seeds/crime-category';
-import { CrimeIncidentsSeeder } from './seeds/crime-incident';
+ 
 import { UnitSeeder } from './seeds/units';
 import { CrimesSeeder } from './seeds/crimes';
 import { CrimeIncidentsSeeder as DetailedCrimeIncidentsSeeder } from './seeds/crime-incidents';
@ -29,13 +29,13 @@ class DatabaseSeeder {

    // Daftar semua seeders di sini
    this.seeders = [
-      // new RoleSeeder(prisma),
-      // new ResourceSeeder(prisma),
-      // new PermissionSeeder(prisma),
-      // new CrimeCategoriesSeeder(prisma),
-      // new GeoJSONSeeder(prisma),
-      // new UnitSeeder(prisma),
-      // new DemographicsSeeder(prisma),
+      new RoleSeeder(prisma),
+      new ResourceSeeder(prisma),
+      new PermissionSeeder(prisma),
+      new CrimeCategoriesSeeder(prisma),
+      new GeoJSONSeeder(prisma),
+      new UnitSeeder(prisma),
+      new DemographicsSeeder(prisma),
      new CrimesSeeder(prisma),
      new DetailedCrimeIncidentsSeeder(prisma), // Add the new crime incidents seeder
      // new CrimeIncidentsSeederV2(prisma),
--- a/sigap-website/prisma/seeds/crime-incident.ts
+++ b/sigap-website/prisma/seeds/crime-incident.ts
--- a/sigap-website/prisma/seeds/crime-incidents.ts
+++ b/sigap-website/prisma/seeds/crime-incidents.ts
@ -285,6 +285,14 @@ export class CrimeIncidentsSeeder {

      // Calculate a date within the crime's month
      const year = crime.year;
+
+      if (!year) {
+        console.error(
+          `Year is not defined for crime record ${crime.id}, skipping.`
+        );
+        return [];
+      }
+
      const month = (crime.month as number) - 1; // JavaScript months are 0-indexed
      const maxDay = new Date(year, month + 1, 0).getDate(); // Get last day of month
      const day = Math.floor(Math.random() * maxDay) + 1;
@ -299,7 +307,7 @@ export class CrimeIncidentsSeeder {
        segments: {
          codes: [district.cities.id],
          sequentialDigits: 4,
-          year: crime.year,
+          year,
        },
        format: '{prefix}-{codes}-{sequence}-{year}',
        separator: '-',
--- a/sigap-website/prisma/seeds/crimes.ts
+++ b/sigap-website/prisma/seeds/crimes.ts
@ -16,6 +16,8 @@ export class CrimesSeeder {
  async run(): Promise<void> {
    console.log('🌱 Seeding crimes data...');

+    // Clear existing data
+
    try {
      // Create test user
      const user = await this.createUsers();
@ -32,6 +34,9 @@ export class CrimesSeeder {
      // Import yearly crime data from CSV file
      await this.importYearlyCrimeData();

+      // Import all-year crime summaries (2020-2024)
+      await this.importAllYearSummaries();
+
      console.log('✅ Crime seeding completed successfully.');
    } catch (error) {
      console.error('❌ Error seeding crimes:', error);
@ -276,137 +281,80 @@ export class CrimesSeeder {
    console.log(`Imported ${records.length} yearly crime records.`);
  }

-  //   private async generateYearlyCrimeSummaries() {
-  //     console.log('Generating yearly crime summaries...');
+  private async importAllYearSummaries() {
+    console.log('Importing all-year (2020-2024) crime summaries...');

-  //     // Check if yearly summaries already exist (records with null month)
-  //     const existingYearlySummary = await this.prisma.crimes.findFirst({
-  //       where: { month: null },
-  //     });
+    // Check if all-year summaries already exist (records with null month and null year)
+    const existingAllYearSummaries = await this.prisma.crimes.findFirst({
+      where: { month: null, year: null },
+    });

-  //     if (existingYearlySummary) {
-  //       console.log('Yearly summaries already exist, skipping generation.');
-  //       return;
-  //     }
+    if (existingAllYearSummaries) {
+      console.log('All-year crime summaries already exist, skipping import.');
+      return;
+    }

-  //     // Get all districts and years combinations
-  //     const districtsYears = await this.prisma.crimes.findMany({
-  //       select: {
-  //         district_id: true,
-  //         year: true,
-  //       },
-  //       distinct: ['district_id', 'year'],
-  //     });
+    // Read CSV file
+    const csvFilePath = path.resolve(
+      __dirname,
+      '../data/excels/crimes/district_summary_2020_2024.csv'
+    );
+    const fileContent = fs.readFileSync(csvFilePath, { encoding: 'utf-8' });

-  //     // For each district and year, calculate yearly summary
-  //     for (const { district_id, year } of districtsYears) {
-  //       // Calculate sum of crimes for the district and year
-  //       const result = await this.prisma.crimes.aggregate({
-  //         _sum: {
-  //           number_of_crime: true,
-  //         },
-  //         where: {
-  //           district_id: district_id,
-  //           year: year,
-  //           month: {
-  //             not: null,
-  //           },
-  //         },
-  //       });
+    // Parse CSV
+    const records = parse(fileContent, {
+      columns: true,
+      skip_empty_lines: true,
+    });

-  //       if (!result || !result._sum.number_of_crime) {
-  //         console.log(
-  //           `No monthly data found for district ${district_id} in year ${year}. Skipping...`
-  //         );
-  //         continue;
-  //       }
+    for (const record of records) {
+      const crimeRate = record.level.toLowerCase() as crime_rates;
+      const districtId = record.district_id;

-  //       const totalCrimes = result._sum.number_of_crime;
+      const city = await this.prisma.cities.findFirst({
+        where: {
+          districts: {
+            some: {
+              id: districtId,
+            },
+          },
+        },
+      });

-  //       // Get average level based on monthly data (use the most common level)
-  //       const levelResult = await this.prisma.crimes
-  //         .groupBy({
-  //           by: ['level'],
-  //           where: {
-  //             district_id: district_id,
-  //             year: year,
-  //             month: {
-  //               not: null,
-  //             },
-  //           },
-  //           _count: {
-  //             level: true,
-  //           },
-  //           orderBy: {
-  //             _count: {
-  //               level: 'desc',
-  //             },
-  //           },
-  //           take: 1,
-  //         })
-  //         .then((results) =>
-  //           results.map((result) => ({
-  //             level: result.level,
-  //             count: result._count.level,
-  //           }))
-  //         );
+      if (!city) {
+        console.error(`City not found for district ${districtId}`);
+        continue;
+      }

-  //       if (levelResult.length === 0) {
-  //         console.log(
-  //           `No level data found for district ${district_id} in year ${year}. Skipping...`
-  //         );
-  //         continue;
-  //       }
+      // Create a unique ID for all-year summary data
+      const crimeId = generateId({
+        prefix: 'CR',
+        segments: {
+          codes: [city.id],
+          sequentialDigits: 4,
+        },
+        format: '{prefix}-{codes}-{sequence}',
+        separator: '-',
+        randomSequence: false,
+        uniquenessStrategy: 'counter',
+      });

-  //       const level = levelResult[0].level || 'low';
+      await this.prisma.crimes.create({
+        data: {
+          id: crimeId,
+          district_id: districtId,
+          level: crimeRate,
+          method: 'kmeans',
+          month: null,
+          year: null,
+          number_of_crime: parseInt(record.crime_total),
+          score: parseFloat(record.avg_score),
+        },
+      });
+    }

-  //       const city = await this.prisma.cities.findFirst({
-  //         where: {
-  //           districts: {
-  //             some: {
-  //               id: district_id,
-  //             },
-  //           },
-  //         },
-  //       });
-
-  //       if (!city) {
-  //         console.error(`City not found for district ${district_id}`);
-  //         continue;
-  //       }
-
-  //       // Create yearly summary record
-  //       const newCrimeId = generateId({
-  //         prefix: 'CR',
-  //         segments: {
-  //           codes: [city.id],
-  //           sequentialDigits: 4,
-  //           year,
-  //         },
-  //         format: '{prefix}-{codes}-{sequence}-{year}',
-  //         separator: '-',
-  //         randomSequence: false,
-  //         uniquenessStrategy: 'counter',
-  //       });
-
-  //       await this.prisma.crimes.create({
-  //         data: {
-  //           id: newCrimeId,
-  //           district_id: district_id as string,
-  //           level: level as crime_rates,
-  //           method: 'kmeans',
-  //           month: null,
-  //           year: year as number,
-  //           number_of_crime: totalCrimes,
-  //           score: 100 - Math.min(totalCrimes, 100), // Simple score calculation
-  //         },
-  //       });
-  //     }
-
-  //     console.log(
-  //       `Generated yearly summaries for ${districtsYears.length} district-year combinations.`
-  //     );
-  //   }
+    console.log(`Imported ${records.length} all-year crime summaries.`);
+  }
 }

 // This allows the file to be executed standalone for testing
--- a/sigap-website/prisma/seeds/geographic.ts
+++ b/sigap-website/prisma/seeds/geographic.ts
@ -105,9 +105,10 @@ export class GeoJSONSeeder {
  async run(): Promise<void> {
    console.log('Seeding GeoJSON data...');

-    await this.prisma.geographics.deleteMany({});
+    await this.prisma.units.deleteMany({});
    await this.prisma.districts.deleteMany({});
    await this.prisma.cities.deleteMany({});
+    await this.prisma.geographics.deleteMany({});

    try {
      // Load GeoJSON file
--- a/sigap-website/prisma/seeds/role.ts
+++ b/sigap-website/prisma/seeds/role.ts
@ -8,6 +8,7 @@ export class RoleSeeder {
    console.log('Seeding roles...');

    await this.prisma.sessions.deleteMany({});
+    await this.prisma.locations.deleteMany({});
    await this.prisma.events.deleteMany({});
    await this.prisma.permissions.deleteMany({});
    await this.prisma.users.deleteMany({});