From 70c52bc0a985865b1a1f39d5c3894c750157b75a Mon Sep 17 00:00:00 2001 From: zeeshanalico Date: Sat, 15 Mar 2025 13:22:29 +0500 Subject: [PATCH 1/3] feat: location module, their relevant api's and the schema changed --- Backend/package-lock.json | 16 ++- Backend/package.json | 2 +- .../migration.sql | 52 ++++++++ Backend/prisma/schema.prisma | 32 ++++- Backend/src/app.module.ts | 3 + Backend/src/modules/auth/auth.service.ts | 27 ++-- Backend/src/modules/auth/auth.type.ts | 4 +- Backend/src/modules/location/dto/city.dto.ts | 16 +++ Backend/src/modules/location/dto/state.dto.ts | 17 +++ .../modules/location/location.controller.ts | 87 +++++++++++++ .../src/modules/location/location.module.ts | 12 ++ .../src/modules/location/location.service.ts | 121 ++++++++++++++++++ Backend/src/modules/user/user.service.ts | 17 +-- 13 files changed, 365 insertions(+), 41 deletions(-) create mode 100644 Backend/prisma/migrations/20250315075044_state_and_location_tables_added/migration.sql create mode 100644 Backend/src/modules/location/dto/city.dto.ts create mode 100644 Backend/src/modules/location/dto/state.dto.ts create mode 100644 Backend/src/modules/location/location.controller.ts create mode 100644 Backend/src/modules/location/location.module.ts create mode 100644 Backend/src/modules/location/location.service.ts diff --git a/Backend/package-lock.json b/Backend/package-lock.json index 2a17eb8..5f0da20 100644 --- a/Backend/package-lock.json +++ b/Backend/package-lock.json @@ -19,7 +19,7 @@ "@nestjs/jwt": "^10.2.0", "@nestjs/platform-express": "^10.4.4", "@nestjs/swagger": "^8.1.0", - "@prisma/client": "^6.0.1", + "@prisma/client": "^6.5.0", "bcrypt": "^5.1.1", "class-transformer": "^0.5.1", "class-validator": "^0.14.1", @@ -3045,20 +3045,24 @@ } }, "node_modules/@prisma/client": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/@prisma/client/-/client-6.0.1.tgz", - "integrity": "sha512-60w7kL6bUxz7M6Gs/V+OWMhwy94FshpngVmOY05TmGD0Lhk+Ac0ZgtjlL6Wll9TD4G03t4Sq1wZekNVy+Xdlbg==", + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/@prisma/client/-/client-6.5.0.tgz", + "integrity": "sha512-M6w1Ql/BeiGoZmhMdAZUXHu5sz5HubyVcKukbLs3l0ELcQb8hTUJxtGEChhv4SVJ0QJlwtLnwOLgIRQhpsm9dw==", "hasInstallScript": true, "license": "Apache-2.0", "engines": { "node": ">=18.18" }, "peerDependencies": { - "prisma": "*" + "prisma": "*", + "typescript": ">=5.1.0" }, "peerDependenciesMeta": { "prisma": { "optional": true + }, + "typescript": { + "optional": true } } }, @@ -13578,7 +13582,7 @@ "version": "5.3.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz", "integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==", - "dev": true, + "devOptional": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" diff --git a/Backend/package.json b/Backend/package.json index 660368e..03858fd 100644 --- a/Backend/package.json +++ b/Backend/package.json @@ -40,7 +40,7 @@ "@nestjs/jwt": "^10.2.0", "@nestjs/platform-express": "^10.4.4", "@nestjs/swagger": "^8.1.0", - "@prisma/client": "^6.0.1", + "@prisma/client": "^6.5.0", "bcrypt": "^5.1.1", "class-transformer": "^0.5.1", "class-validator": "^0.14.1", diff --git a/Backend/prisma/migrations/20250315075044_state_and_location_tables_added/migration.sql b/Backend/prisma/migrations/20250315075044_state_and_location_tables_added/migration.sql new file mode 100644 index 0000000..89fe49c --- /dev/null +++ b/Backend/prisma/migrations/20250315075044_state_and_location_tables_added/migration.sql @@ -0,0 +1,52 @@ +/* + Warnings: + + - You are about to drop the column `city` on the `Business` table. All the data in the column will be lost. + - You are about to drop the column `state` on the `Business` table. All the data in the column will be lost. + +*/ +-- AlterTable +ALTER TABLE "Business" DROP COLUMN "city", +DROP COLUMN "state", +ADD COLUMN "city_id" INTEGER, +ADD COLUMN "state_id" INTEGER; + +-- CreateTable +CREATE TABLE "State" ( + "id" SERIAL NOT NULL, + "name" VARCHAR(255) NOT NULL, + "code" VARCHAR(10) NOT NULL, + "created_at" TIMESTAMP(6) DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(6) DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "State_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "City" ( + "id" SERIAL NOT NULL, + "name" VARCHAR(255) NOT NULL, + "state_id" INTEGER NOT NULL, + "created_at" TIMESTAMP(6) DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(6) DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "City_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "State_name_key" ON "State"("name"); + +-- CreateIndex +CREATE UNIQUE INDEX "State_code_key" ON "State"("code"); + +-- CreateIndex +CREATE UNIQUE INDEX "City_name_state_id_key" ON "City"("name", "state_id"); + +-- AddForeignKey +ALTER TABLE "City" ADD CONSTRAINT "City_state_id_fkey" FOREIGN KEY ("state_id") REFERENCES "State"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "Business" ADD CONSTRAINT "Business_state_id_fkey" FOREIGN KEY ("state_id") REFERENCES "State"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "Business" ADD CONSTRAINT "Business_city_id_fkey" FOREIGN KEY ("city_id") REFERENCES "City"("id") ON DELETE SET NULL ON UPDATE CASCADE; diff --git a/Backend/prisma/schema.prisma b/Backend/prisma/schema.prisma index dbdc826..f2a674f 100644 --- a/Backend/prisma/schema.prisma +++ b/Backend/prisma/schema.prisma @@ -7,6 +7,28 @@ datasource db { url = env("DATABASE_URL") } +model State { + id Int @id @default(autoincrement()) + name String @unique @db.VarChar(255) + code String @unique @db.VarChar(10) + cities City[] + businesses Business[] + created_at DateTime? @default(now()) @db.Timestamp(6) + updated_at DateTime? @default(now()) @db.Timestamp(6) +} + +model City { + id Int @id @default(autoincrement()) + name String @db.VarChar(255) + state_id Int + state State @relation(fields: [state_id], references: [id], onDelete: Cascade) + businesses Business[] + created_at DateTime? @default(now()) @db.Timestamp(6) + updated_at DateTime? @default(now()) @db.Timestamp(6) + + @@unique([name, state_id]) +} + model Business { id Int @id @default(autoincrement()) name String @db.VarChar(255) @@ -21,12 +43,14 @@ model Business { location String? @db.VarChar(255) business_type BUSINESS_TYPE? website String? @db.VarChar(255) - city String? @db.VarChar(255) - state String? @db.VarChar(255) + state_id Int? + city_id Int? + state State? @relation(fields: [state_id], references: [id]) + city City? @relation(fields: [city_id], references: [id]) zip_code String? @db.VarChar(255) description String? - acc_status ACC_STATUS? @default(inactive) - Ingested_data Ingested_data[] + acc_status ACC_STATUS? @default(inactive) + Ingested_data Ingested_data[] otp_validation Otp_validation[] } diff --git a/Backend/src/app.module.ts b/Backend/src/app.module.ts index cee91c6..791e89d 100644 --- a/Backend/src/app.module.ts +++ b/Backend/src/app.module.ts @@ -13,12 +13,15 @@ import { APP_INTERCEPTOR } from '@nestjs/core'; import { ResponseInterceptor } from './common/interceptors/response.interceptor'; import { ConfigModule, ConfigService } from '@nestjs/config'; import { DataIngestorModule } from './modules/data-ingestor/data-ingestor.module'; +import { LocationModule } from './modules/location/location.module'; + const imports = [ AuthModule, UserModule, PrismaModule, LoggerModule, DataIngestorModule, + LocationModule, JwtModule.register({ global: true, secret: new ConfigService().get('SECRET_KEY'), diff --git a/Backend/src/modules/auth/auth.service.ts b/Backend/src/modules/auth/auth.service.ts index e781eec..eb6e750 100644 --- a/Backend/src/modules/auth/auth.service.ts +++ b/Backend/src/modules/auth/auth.service.ts @@ -7,7 +7,7 @@ import { JwtService } from '@nestjs/jwt'; import { ConfigService } from '@nestjs/config'; import { OtpService } from './otp.service'; import { IRegisterBusiness } from './auth.type'; -import { Business, ACC_STATUS } from '@prisma/client'; +import { Business, ACC_STATUS, Prisma } from '@prisma/client'; @Injectable() @@ -46,9 +46,9 @@ export class AuthService { password, phone_number, location, - city, + city_id, description, - state, + state_id, zip_code, website, }: IRegisterBusiness = businessData; @@ -61,31 +61,34 @@ export class AuthService { const password_hash = await bcrypt.hash(password, 10); - const business = { + const business: Prisma.BusinessCreateInput = { name, email, phone_number, password_hash, - city, description, - state, zip_code, website, location, + ...(state_id && { + state: { + connect: { id: state_id } + } + }), + ...(city_id && { + city: { + connect: { id: city_id } + } + }) }; // Save Business and OTP in a transaction await this.prisma.$transaction(async (trx) => { const createdBusiness = await this.userService.createBusiness({ trx, - business, + business, }); - // await this.otpService.generateAndSendOtpToMessage({ - // business_id: createdBusiness.id, - // phoneNumber: phone_number, - // trx, - // }); await this.otpService.generateAndSendOtpToMail({ business_id: createdBusiness.id, to: email, diff --git a/Backend/src/modules/auth/auth.type.ts b/Backend/src/modules/auth/auth.type.ts index 7ad0f8e..9c48916 100644 --- a/Backend/src/modules/auth/auth.type.ts +++ b/Backend/src/modules/auth/auth.type.ts @@ -5,8 +5,8 @@ export interface IRegisterBusiness { phone_number: string; location?: string; website?: string; - city?: string; - state?: string; + state_id?: number; + city_id?: number; zip_code?: string; description?: string; } diff --git a/Backend/src/modules/location/dto/city.dto.ts b/Backend/src/modules/location/dto/city.dto.ts new file mode 100644 index 0000000..68fbddc --- /dev/null +++ b/Backend/src/modules/location/dto/city.dto.ts @@ -0,0 +1,16 @@ +import { IsString, IsNotEmpty, IsNumber } from 'class-validator'; +import { ApiProperty } from '@nestjs/swagger'; + +export class CreateCityDto { + @ApiProperty({ example: 'San Francisco', description: 'Name of the city' }) + @IsString() + @IsNotEmpty() + name: string; + + @ApiProperty({ example: 1, description: 'ID of the state this city belongs to' }) + @IsNumber() + @IsNotEmpty() + state_id: number; +} + +export class UpdateCityDto extends CreateCityDto {} \ No newline at end of file diff --git a/Backend/src/modules/location/dto/state.dto.ts b/Backend/src/modules/location/dto/state.dto.ts new file mode 100644 index 0000000..bd8c0e0 --- /dev/null +++ b/Backend/src/modules/location/dto/state.dto.ts @@ -0,0 +1,17 @@ +import { IsString, IsNotEmpty, Length } from 'class-validator'; +import { ApiProperty } from '@nestjs/swagger'; + +export class CreateStateDto { + @ApiProperty({ example: 'California', description: 'Name of the state' }) + @IsString() + @IsNotEmpty() + name: string; + + @ApiProperty({ example: 'CA', description: 'State code' }) + @IsString() + @IsNotEmpty() + @Length(2, 10) + code: string; +} + +export class UpdateStateDto extends CreateStateDto {} \ No newline at end of file diff --git a/Backend/src/modules/location/location.controller.ts b/Backend/src/modules/location/location.controller.ts new file mode 100644 index 0000000..4935c28 --- /dev/null +++ b/Backend/src/modules/location/location.controller.ts @@ -0,0 +1,87 @@ +import { Controller, Get, Post, Put, Delete, Body, Param, ParseIntPipe } from '@nestjs/common'; +import { ApiTags, ApiOperation, ApiResponse } from '@nestjs/swagger'; +import { LocationService } from './location.service'; +import { CreateStateDto, UpdateStateDto } from './dto/state.dto'; +import { CreateCityDto, UpdateCityDto } from './dto/city.dto'; + +@ApiTags('Location') +@Controller('location') +export class LocationController { + constructor(private readonly locationService: LocationService) {} + + // State endpoints + @Post('states') + @ApiOperation({ summary: 'Create a new state' }) + @ApiResponse({ status: 201, description: 'State created successfully' }) + createState(@Body() createStateDto: CreateStateDto) { + return this.locationService.createState(createStateDto); + } + + @Get('states') + @ApiOperation({ summary: 'Get all states' }) + getAllStates() { + return this.locationService.getAllStates(); + } + + @Get('states/:id') + @ApiOperation({ summary: 'Get a state by ID' }) + getStateById(@Param('id', ParseIntPipe) id: number) { + return this.locationService.getStateById(id); + } + + @Put('states/:id') + @ApiOperation({ summary: 'Update a state' }) + updateState( + @Param('id', ParseIntPipe) id: number, + @Body() updateStateDto: UpdateStateDto, + ) { + return this.locationService.updateState(id, updateStateDto); + } + + @Delete('states/:id') + @ApiOperation({ summary: 'Delete a state' }) + deleteState(@Param('id', ParseIntPipe) id: number) { + return this.locationService.deleteState(id); + } + + // City endpoints + @Post('cities') + @ApiOperation({ summary: 'Create a new city' }) + @ApiResponse({ status: 201, description: 'City created successfully' }) + createCity(@Body() createCityDto: CreateCityDto) { + return this.locationService.createCity(createCityDto); + } + + @Get('cities') + @ApiOperation({ summary: 'Get all cities' }) + getAllCities() { + return this.locationService.getAllCities(); + } + + @Get('states/:stateId/cities') + @ApiOperation({ summary: 'Get all cities in a state' }) + getCitiesByState(@Param('stateId', ParseIntPipe) stateId: number) { + return this.locationService.getCitiesByState(stateId); + } + + @Get('cities/:id') + @ApiOperation({ summary: 'Get a city by ID' }) + getCityById(@Param('id', ParseIntPipe) id: number) { + return this.locationService.getCityById(id); + } + + @Put('cities/:id') + @ApiOperation({ summary: 'Update a city' }) + updateCity( + @Param('id', ParseIntPipe) id: number, + @Body() updateCityDto: UpdateCityDto, + ) { + return this.locationService.updateCity(id, updateCityDto); + } + + @Delete('cities/:id') + @ApiOperation({ summary: 'Delete a city' }) + deleteCity(@Param('id', ParseIntPipe) id: number) { + return this.locationService.deleteCity(id); + } +} \ No newline at end of file diff --git a/Backend/src/modules/location/location.module.ts b/Backend/src/modules/location/location.module.ts new file mode 100644 index 0000000..469819a --- /dev/null +++ b/Backend/src/modules/location/location.module.ts @@ -0,0 +1,12 @@ +import { Module } from '@nestjs/common'; +import { LocationService } from './location.service'; +import { LocationController } from './location.controller'; +import { PrismaModule } from '../prisma/prisma.module'; + +@Module({ + imports: [PrismaModule], + controllers: [LocationController], + providers: [LocationService], + exports: [LocationService], +}) +export class LocationModule {} \ No newline at end of file diff --git a/Backend/src/modules/location/location.service.ts b/Backend/src/modules/location/location.service.ts new file mode 100644 index 0000000..f730652 --- /dev/null +++ b/Backend/src/modules/location/location.service.ts @@ -0,0 +1,121 @@ +import { Injectable, NotFoundException } from '@nestjs/common'; +import { PrismaService } from '../prisma/prisma.service'; +import { CreateStateDto, UpdateStateDto } from './dto/state.dto'; +import { CreateCityDto, UpdateCityDto } from './dto/city.dto'; + +@Injectable() +export class LocationService { + constructor(private prisma: PrismaService) {} + + // State operations + async createState(createStateDto: CreateStateDto) { + return this.prisma.state.create({ + data: createStateDto, + }); + } + + async getAllStates() { + return this.prisma.state.findMany({ + include: { + cities: true, + }, + }); + } + + async getStateById(id: number) { + const state = await this.prisma.state.findUnique({ + where: { id }, + include: { + cities: true, + }, + }); + + if (!state) { + throw new NotFoundException(`State with ID ${id} not found`); + } + + return state; + } + + async updateState(id: number, updateStateDto: UpdateStateDto) { + return this.prisma.state.update({ + where: { id }, + data: updateStateDto, + }); + } + + async deleteState(id: number) { + return this.prisma.state.delete({ + where: { id }, + }); + } + + // City operations + async createCity(createCityDto: CreateCityDto) { + // Check if state exists + const state = await this.prisma.state.findUnique({ + where: { id: createCityDto.state_id }, + }); + + if (!state) { + throw new NotFoundException(`State with ID ${createCityDto.state_id} not found`); + } + + return this.prisma.city.create({ + data: createCityDto, + include: { + state: true, + }, + }); + } + + async getAllCities() { + return this.prisma.city.findMany({ + include: { + state: true, + }, + }); + } + + async getCitiesByState(stateId: number) { + return this.prisma.city.findMany({ + where: { + state_id: stateId, + }, + include: { + state: true, + }, + }); + } + + async getCityById(id: number) { + const city = await this.prisma.city.findUnique({ + where: { id }, + include: { + state: true, + }, + }); + + if (!city) { + throw new NotFoundException(`City with ID ${id} not found`); + } + + return city; + } + + async updateCity(id: number, updateCityDto: UpdateCityDto) { + return this.prisma.city.update({ + where: { id }, + data: updateCityDto, + include: { + state: true, + }, + }); + } + + async deleteCity(id: number) { + return this.prisma.city.delete({ + where: { id }, + }); + } +} \ No newline at end of file diff --git a/Backend/src/modules/user/user.service.ts b/Backend/src/modules/user/user.service.ts index 3eed437..7a2c7ab 100644 --- a/Backend/src/modules/user/user.service.ts +++ b/Backend/src/modules/user/user.service.ts @@ -1,18 +1,8 @@ import { Injectable } from '@nestjs/common'; import { PrismaService } from '../prisma/prisma.service'; -import { Business, ACC_STATUS } from '@prisma/client'; -import { Prisma } from '@prisma/client'; +import { Business, ACC_STATUS, Prisma } from '@prisma/client'; import { NotFoundException, HttpException, HttpStatus } from '@nestjs/common'; -export type Business_metadataCreateInput = { - location?: string | null - website?: string | null - city?: string | null - state?: string | null - zip_code?: string | null - description?: string | null -} - @Injectable() export class UserService { constructor(private readonly prisma: PrismaService) { } @@ -23,16 +13,11 @@ export class UserService { trx?: Prisma.TransactionClient; business: Prisma.BusinessCreateInput; }): Promise { - const transaction = trx || this.prisma; - const createdBusiness = await transaction.business.create({ data: { ...business }, }); - - - return transaction.business.findUnique({ where: { id: createdBusiness.id }, }); From 7392858903d488b3cbb0aa071a1bc8850db02028 Mon Sep 17 00:00:00 2001 From: zeeshanalico Date: Sun, 16 Mar 2025 15:53:25 +0500 Subject: [PATCH 2/3] feat: data ingestion grouping, ingesting, sheets, refine and transform etc --- Backend/package-lock.json | 92 ++-- Backend/package.json | 8 +- .../migration.sql | 60 +++ Backend/prisma/schema.prisma | 51 ++ .../common/filters/catchEverything.filter.ts | 6 +- Backend/src/main.ts | 1 + .../data-ingestor/data-ingestor.controller.ts | 235 +++++++-- .../data-ingestor/data-ingestor.module.ts | 34 +- .../data-ingestor.service.spec.ts | 288 +++++++++++ .../data-ingestor/data-ingestor.service.ts | 464 +++++++++++++++++- .../data-ingestor/dto/file-group.dto.ts | 62 +++ .../data-ingestor/dto/ingest-file.dto.ts | 27 + .../interfaces/responses.interface.ts | 57 +++ .../interfaces/sheet-data.interface.ts | 16 + .../data-ingestor/parser/csv.parser.ts | 41 ++ .../data-ingestor/parser/parser.factory.ts | 26 +- .../data-ingestor/parser/parser.interface.ts | 17 + .../data-ingestor/parser/xlsx.parser.ts | 42 ++ .../services/sheet-cleaner.service.ts | 131 +++++ Backend/src/modules/location/dto/city.dto.ts | 2 +- 20 files changed, 1529 insertions(+), 131 deletions(-) create mode 100644 Backend/prisma/migrations/20250315095903_ingestion_tables_created/migration.sql create mode 100644 Backend/src/modules/data-ingestor/data-ingestor.service.spec.ts create mode 100644 Backend/src/modules/data-ingestor/dto/file-group.dto.ts create mode 100644 Backend/src/modules/data-ingestor/dto/ingest-file.dto.ts create mode 100644 Backend/src/modules/data-ingestor/interfaces/responses.interface.ts create mode 100644 Backend/src/modules/data-ingestor/interfaces/sheet-data.interface.ts create mode 100644 Backend/src/modules/data-ingestor/parser/csv.parser.ts create mode 100644 Backend/src/modules/data-ingestor/parser/parser.interface.ts create mode 100644 Backend/src/modules/data-ingestor/parser/xlsx.parser.ts create mode 100644 Backend/src/modules/data-ingestor/services/sheet-cleaner.service.ts diff --git a/Backend/package-lock.json b/Backend/package-lock.json index 5f0da20..c92b8c3 100644 --- a/Backend/package-lock.json +++ b/Backend/package-lock.json @@ -36,12 +36,12 @@ "devDependencies": { "@nestjs/cli": "^10.4.5", "@nestjs/schematics": "^10.1.0", - "@nestjs/testing": "^10.3.2", + "@nestjs/testing": "^10.4.15", "@swc/cli": "^0.3.9", "@swc/core": "^1.4.0", "@types/bcrypt": "^5.0.2", "@types/express": "^4.17.21", - "@types/jest": "^29.5.12", + "@types/jest": "^29.5.14", "@types/multer": "^1.4.12", "@types/node": "^20.11.16", "@types/supertest": "^6.0.2", @@ -56,7 +56,7 @@ "prisma": "^6.1.0", "source-map-support": "^0.5.21", "supertest": "^6.3.4", - "ts-jest": "^29.1.2", + "ts-jest": "^29.2.6", "ts-loader": "^9.5.1", "ts-node": "^10.9.2", "tsconfig-paths": "^4.2.0", @@ -2941,12 +2941,13 @@ } }, "node_modules/@nestjs/testing": { - "version": "10.3.2", - "resolved": "https://registry.npmjs.org/@nestjs/testing/-/testing-10.3.2.tgz", - "integrity": "sha512-jetqEPqOPuxmhBinkizmJQg4UZ2IRFrUoMrBDSgg0ogQClokKjnLgkoC5de+Jfm2kub/VpqorHB0me8cCr5jEQ==", + "version": "10.4.15", + "resolved": "https://registry.npmjs.org/@nestjs/testing/-/testing-10.4.15.tgz", + "integrity": "sha512-eGlWESkACMKti+iZk1hs6FUY/UqObmMaa8HAN9JLnaYkoLf1Jeh+EuHlGnfqo/Rq77oznNLIyaA3PFjrFDlNUg==", "dev": true, + "license": "MIT", "dependencies": { - "tslib": "2.6.2" + "tslib": "2.8.1" }, "funding": { "type": "opencollective", @@ -2967,6 +2968,13 @@ } } }, + "node_modules/@nestjs/testing/node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true, + "license": "0BSD" + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -3734,10 +3742,11 @@ } }, "node_modules/@types/jest": { - "version": "29.5.12", - "resolved": "https://registry.npmjs.org/@types/jest/-/jest-29.5.12.tgz", - "integrity": "sha512-eDC8bTvT/QhYdxJAulQikueigY5AsdBRH2yDKW3yveW7svY3+DzN84/2NUgkw10RTiJbWqZrTtoGVdYlvFJdLw==", + "version": "29.5.14", + "resolved": "https://registry.npmjs.org/@types/jest/-/jest-29.5.14.tgz", + "integrity": "sha512-ZN+4sdnLUbo8EVvVc2ao0GFW6oVrQRPn4K2lglySj7APvSrgzxHiNNK99us4WDMi57xxA2yggblIAMNhXOotLQ==", "dev": true, + "license": "MIT", "dependencies": { "expect": "^29.0.0", "pretty-format": "^29.0.0" @@ -6461,8 +6470,8 @@ "version": "3.1.10", "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.10.tgz", "integrity": "sha512-UeJmFfOrAQS8OJWPZ4qtgHyWExa088/MtK5UEyoJGFH67cDEXkZSviOiKRCZ4Xij0zxI3JECgYs3oKx+AizQBA==", + "devOptional": true, "license": "Apache-2.0", - "optional": true, "dependencies": { "jake": "^10.8.5" }, @@ -7274,8 +7283,8 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz", "integrity": "sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q==", + "devOptional": true, "license": "Apache-2.0", - "optional": true, "dependencies": { "minimatch": "^5.0.1" } @@ -7284,8 +7293,8 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "devOptional": true, "license": "MIT", - "optional": true, "dependencies": { "balanced-match": "^1.0.0" } @@ -7294,8 +7303,8 @@ "version": "5.1.6", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "devOptional": true, "license": "ISC", - "optional": true, "dependencies": { "brace-expansion": "^2.0.1" }, @@ -8655,8 +8664,8 @@ "version": "10.9.2", "resolved": "https://registry.npmjs.org/jake/-/jake-10.9.2.tgz", "integrity": "sha512-2P4SQ0HrLQ+fw6llpLnOaGAvN2Zu6778SJMrCUwns4fOoG9ayrTiZk3VV8sCPkVZF8ab0zksVpS8FDY5pRCNBA==", + "devOptional": true, "license": "Apache-2.0", - "optional": true, "dependencies": { "async": "^3.2.3", "chalk": "^4.0.2", @@ -8675,6 +8684,7 @@ "resolved": "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz", "integrity": "sha512-NIy3oAFp9shda19hy4HK0HRTWKtPJmGdnvywu01nOqNC2vZg+Z+fvJDxpMQA88eb2I9EcafcdjYgsDthnYTvGw==", "dev": true, + "license": "MIT", "dependencies": { "@jest/core": "^29.7.0", "@jest/types": "^29.6.3", @@ -12392,12 +12402,10 @@ } }, "node_modules/semver": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", - "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", - "dependencies": { - "lru-cache": "^6.0.0" - }, + "version": "7.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.1.tgz", + "integrity": "sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA==", + "license": "ISC", "bin": { "semver": "bin/semver.js" }, @@ -12432,22 +12440,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/semver/node_modules/lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/semver/node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" - }, "node_modules/send": { "version": "0.19.0", "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz", @@ -13363,28 +13355,31 @@ } }, "node_modules/ts-jest": { - "version": "29.1.2", - "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-29.1.2.tgz", - "integrity": "sha512-br6GJoH/WUX4pu7FbZXuWGKGNDuU7b8Uj77g/Sp7puZV6EXzuByl6JrECvm0MzVzSTkSHWTihsXt+5XYER5b+g==", + "version": "29.2.6", + "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-29.2.6.tgz", + "integrity": "sha512-yTNZVZqc8lSixm+QGVFcPe6+yj7+TWZwIesuOWvfcn4B9bz5x4NDzVCQQjOs7Hfouu36aEqfEbo9Qpo+gq8dDg==", "dev": true, + "license": "MIT", "dependencies": { - "bs-logger": "0.x", - "fast-json-stable-stringify": "2.x", + "bs-logger": "^0.2.6", + "ejs": "^3.1.10", + "fast-json-stable-stringify": "^2.1.0", "jest-util": "^29.0.0", "json5": "^2.2.3", - "lodash.memoize": "4.x", - "make-error": "1.x", - "semver": "^7.5.3", - "yargs-parser": "^21.0.1" + "lodash.memoize": "^4.1.2", + "make-error": "^1.3.6", + "semver": "^7.7.1", + "yargs-parser": "^21.1.1" }, "bin": { "ts-jest": "cli.js" }, "engines": { - "node": "^16.10.0 || ^18.0.0 || >=20.0.0" + "node": "^14.15.0 || ^16.10.0 || ^18.0.0 || >=20.0.0" }, "peerDependencies": { "@babel/core": ">=7.0.0-beta.0 <8", + "@jest/transform": "^29.0.0", "@jest/types": "^29.0.0", "babel-jest": "^29.0.0", "jest": "^29.0.0", @@ -13394,6 +13389,9 @@ "@babel/core": { "optional": true }, + "@jest/transform": { + "optional": true + }, "@jest/types": { "optional": true }, diff --git a/Backend/package.json b/Backend/package.json index 03858fd..594e531 100644 --- a/Backend/package.json +++ b/Backend/package.json @@ -15,7 +15,7 @@ "test": "jest", "test:watch": "jest --watch", "test:cov": "jest --coverage", - "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/jest/bin/jest --runInBand", + "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand", "test:e2e": "jest --config ./test/jest-e2e.json", "prisma:generate": "dotenv -e .env.development -- npx prisma generate", "prisma:create": "dotenv -e .env.development -- npx prisma migrate dev --create-only", @@ -57,12 +57,12 @@ "devDependencies": { "@nestjs/cli": "^10.4.5", "@nestjs/schematics": "^10.1.0", - "@nestjs/testing": "^10.3.2", + "@nestjs/testing": "^10.4.15", "@swc/cli": "^0.3.9", "@swc/core": "^1.4.0", "@types/bcrypt": "^5.0.2", "@types/express": "^4.17.21", - "@types/jest": "^29.5.12", + "@types/jest": "^29.5.14", "@types/multer": "^1.4.12", "@types/node": "^20.11.16", "@types/supertest": "^6.0.2", @@ -77,7 +77,7 @@ "prisma": "^6.1.0", "source-map-support": "^0.5.21", "supertest": "^6.3.4", - "ts-jest": "^29.1.2", + "ts-jest": "^29.2.6", "ts-loader": "^9.5.1", "ts-node": "^10.9.2", "tsconfig-paths": "^4.2.0", diff --git a/Backend/prisma/migrations/20250315095903_ingestion_tables_created/migration.sql b/Backend/prisma/migrations/20250315095903_ingestion_tables_created/migration.sql new file mode 100644 index 0000000..7395173 --- /dev/null +++ b/Backend/prisma/migrations/20250315095903_ingestion_tables_created/migration.sql @@ -0,0 +1,60 @@ +-- CreateEnum +CREATE TYPE "FILE_TYPE" AS ENUM ('XLSX', 'CSV', 'JSON', 'XML'); + +-- CreateTable +CREATE TABLE "File_group" ( + "id" SERIAL NOT NULL, + "business_id" INTEGER NOT NULL, + "name" VARCHAR(255) NOT NULL, + "description" TEXT, + "created_at" TIMESTAMP(6) DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(6) DEFAULT CURRENT_TIMESTAMP, + "deleted_at" TIMESTAMP(6), + + CONSTRAINT "File_group_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "File_upload" ( + "id" SERIAL NOT NULL, + "group_id" INTEGER NOT NULL, + "business_id" INTEGER NOT NULL, + "filename" VARCHAR(255) NOT NULL, + "file_type" "FILE_TYPE" NOT NULL, + "created_at" TIMESTAMP(6) DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(6) DEFAULT CURRENT_TIMESTAMP, + "deleted_at" TIMESTAMP(6), + + CONSTRAINT "File_upload_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "Sheet" ( + "id" SERIAL NOT NULL, + "file_upload_id" INTEGER NOT NULL, + "sheet_name" VARCHAR(255) NOT NULL, + "row_count" INTEGER NOT NULL, + "column_count" INTEGER NOT NULL, + "headers" JSONB NOT NULL, + "data" JSONB NOT NULL, + "created_at" TIMESTAMP(6) DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(6) DEFAULT CURRENT_TIMESTAMP, + "deleted_at" TIMESTAMP(6), + + CONSTRAINT "Sheet_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "Sheet_file_upload_id_sheet_name_key" ON "Sheet"("file_upload_id", "sheet_name"); + +-- AddForeignKey +ALTER TABLE "File_group" ADD CONSTRAINT "File_group_business_id_fkey" FOREIGN KEY ("business_id") REFERENCES "Business"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "File_upload" ADD CONSTRAINT "File_upload_group_id_fkey" FOREIGN KEY ("group_id") REFERENCES "File_group"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "File_upload" ADD CONSTRAINT "File_upload_business_id_fkey" FOREIGN KEY ("business_id") REFERENCES "Business"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "Sheet" ADD CONSTRAINT "Sheet_file_upload_id_fkey" FOREIGN KEY ("file_upload_id") REFERENCES "File_upload"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/Backend/prisma/schema.prisma b/Backend/prisma/schema.prisma index f2a674f..73c5f4a 100644 --- a/Backend/prisma/schema.prisma +++ b/Backend/prisma/schema.prisma @@ -52,6 +52,8 @@ model Business { acc_status ACC_STATUS? @default(inactive) Ingested_data Ingested_data[] otp_validation Otp_validation[] + file_groups File_group[] + file_uploads File_upload[] } model Otp_validation { @@ -78,6 +80,48 @@ model Ingested_data { Business Business @relation(fields: [business_id], references: [id], onDelete: Cascade, onUpdate: NoAction) } +model File_group { + id Int @id @default(autoincrement()) + business_id Int + name String @db.VarChar(255) // Group name (e.g., "Sales Reports 2024") + description String? @db.Text + created_at DateTime? @default(now()) @db.Timestamp(6) + updated_at DateTime? @default(now()) @db.Timestamp(6) + deleted_at DateTime? @db.Timestamp(6) + files File_upload[] + Business Business @relation(fields: [business_id], references: [id], onDelete: Cascade) +} + +model File_upload { + id Int @id @default(autoincrement()) + group_id Int + business_id Int + filename String @db.VarChar(255) + file_type FILE_TYPE + created_at DateTime? @default(now()) @db.Timestamp(6) + updated_at DateTime? @default(now()) @db.Timestamp(6) + deleted_at DateTime? @db.Timestamp(6) + sheets Sheet[] + file_group File_group @relation(fields: [group_id], references: [id], onDelete: Cascade) + Business Business @relation(fields: [business_id], references: [id], onDelete: Cascade) +} + +model Sheet { + id Int @id @default(autoincrement()) + file_upload_id Int + sheet_name String @db.VarChar(255) + row_count Int + column_count Int + headers Json // Store column headers + data Json // Store actual data + created_at DateTime? @default(now()) @db.Timestamp(6) + updated_at DateTime? @default(now()) @db.Timestamp(6) + deleted_at DateTime? @db.Timestamp(6) + file_upload File_upload @relation(fields: [file_upload_id], references: [id], onDelete: Cascade) + + @@unique([file_upload_id, sheet_name]) +} + enum ACC_STATUS { active inactive @@ -100,3 +144,10 @@ enum SOURCE_TYPE { database none } + +enum FILE_TYPE { + XLSX + CSV + JSON + XML +} diff --git a/Backend/src/common/filters/catchEverything.filter.ts b/Backend/src/common/filters/catchEverything.filter.ts index 64b0fb0..f6a12ea 100644 --- a/Backend/src/common/filters/catchEverything.filter.ts +++ b/Backend/src/common/filters/catchEverything.filter.ts @@ -41,9 +41,7 @@ export class CatchEverythingFilter implements ExceptionFilter { }; if (exception instanceof HttpException) { const status = exception.getStatus(); - const response = exception.getResponse(); - responseBody.m1 = - typeof response === 'string' ? response : (response as any).message || 'An error occurred.'; + responseBody.m1 =exception.message || 'An error occurred.'; responseBody.m2 = exception.stack || null; responseBody.status = status; } else if (exception instanceof PrismaClientValidationError) { @@ -59,7 +57,7 @@ export class CatchEverythingFilter implements ExceptionFilter { responseBody.testing = 'This attribute is added only for testing purpose: PrismaClientKnownRequestError'; } else if (exception instanceof Error) { - responseBody.m1 = exception.message || 'An error occurred.'; + responseBody.m1 = 'An error occurred on the server.'; responseBody.m2 = exception.stack || null; responseBody.status = HttpStatus.INTERNAL_SERVER_ERROR; responseBody.testing = 'This attribute is added only for testing purpose: Error'; diff --git a/Backend/src/main.ts b/Backend/src/main.ts index b922a0e..ec35f26 100644 --- a/Backend/src/main.ts +++ b/Backend/src/main.ts @@ -10,6 +10,7 @@ async function bootstrap() { .setTitle('Backend') .setDescription('The backend of Data visualization and analysis application') .setVersion('1.0') + .addBearerAuth() // .addTag('back') .build(); const documentFactory = () => SwaggerModule.createDocument(app, config); diff --git a/Backend/src/modules/data-ingestor/data-ingestor.controller.ts b/Backend/src/modules/data-ingestor/data-ingestor.controller.ts index 8d38ad8..f4b194b 100644 --- a/Backend/src/modules/data-ingestor/data-ingestor.controller.ts +++ b/Backend/src/modules/data-ingestor/data-ingestor.controller.ts @@ -1,75 +1,212 @@ import { Controller, Post, + Get, + Delete, UseInterceptors, UploadedFile, - BadRequestException, + Param, + ParseIntPipe, + Body, UseGuards, - Inject, - Req + Request, + UploadedFiles, + BadRequestException, + UnsupportedMediaTypeException } from '@nestjs/common'; -import { FileInterceptor } from '@nestjs/platform-express'; -import { ResponseI } from 'src/common/interceptors/response.interceptor'; -import { Parser } from './parser/data-parser'; -import { ApiTags, ApiConsumes, ApiBody, ApiHeader } from '@nestjs/swagger'; -import { ApiDescription, ApiResponseDocs } from 'src/utils/api-docs.decorators'; -import { AuthGuard } from '../auth/auth.guard'; +import { FileInterceptor, FilesInterceptor } from '@nestjs/platform-express'; +import { ApiTags, ApiOperation, ApiConsumes, ApiBody, ApiBearerAuth, ApiResponse } from '@nestjs/swagger'; import { DataIngestorService } from './data-ingestor.service'; -@ApiTags("Data Ingestor API's") +import { FileMetadataDto, IngestFileResponseDto } from './dto/ingest-file.dto'; +// import { JwtAuthGuard } from '../auth/guards/jwt-auth.guard'; +import { CreateFileGroupDto, UploadFilesToGroupDto } from './dto/file-group.dto'; +import { AuthGuard } from '../auth/auth.guard'; +import { ResponseI } from 'src/common/interceptors/response.interceptor'; + + +@ApiTags('Data Ingestion') +@Controller('data-ingestor') @UseGuards(AuthGuard) -@Controller('upload-data') +@ApiBearerAuth() export class DataIngestorController { - constructor(@Inject('PARSER_FACTORY') private readonly parserFactory: (fileType: string) => Parser, private readonly dataIngestorService: DataIngestorService) { } + constructor(private readonly dataIngestorService: DataIngestorService) { } - @ApiDescription('Upload Data', 'Uploads a file and processes it based on the file type, supports XLSX, CSV, SQL; add in body like {file: file}') - @ApiResponseDocs({ - success: 'File processed successfully.', - badRequest: 'No file uploaded or unable to determine file type.', + @Post('groups') + @ApiOperation({ + summary: 'Create a new file group', + description: 'Create a group to organize related files', }) - @ApiHeader({ - name: 'Authorization', - description: 'Bearer token for authentication', - required: true, - schema: { - type: 'string', - example: 'Bearer ', - }, + async createGroup( + @Body() createGroupDto: CreateFileGroupDto, + @Request() req, + ): Promise { + const res = await this.dataIngestorService.createFileGroup(req.payload.id, createGroupDto); + return { + m1: 'File group created successfully', + data: res, + }; + } + + @Post('groups/:groupId/files') + @ApiOperation({ + summary: 'Upload multiple files to a group', + description: 'Upload multiple XLSX files to an existing group', }) - @ApiBody({ - schema: { - type: 'object', - properties: { - file: { - type: 'string', - format: 'binary', - }, - }, + @ApiConsumes('multipart/form-data') + @UseInterceptors(FilesInterceptor('files', 10, { + fileFilter: (req, file, callback) => { + const allowedMimeTypes = [ + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', // xlsx + 'application/vnd.ms-excel', // xls + 'text/csv', // csv + ]; + if (!allowedMimeTypes.includes(file.mimetype)) { + callback(new UnsupportedMediaTypeException( + `File type ${file.mimetype} is not supported. Supported types: XLSX, XLS, CSV` + ), false); + } + callback(null, true); }, - }) - @ApiConsumes('multipart/form-data') // Specify the media type - - @Post() - @UseInterceptors(FileInterceptor('file')) // Handle single file upload - async uploadData(@UploadedFile() file: Express.Multer.File, @Req() req: Request): Promise { + })) + async uploadFiles( + @UploadedFiles() files: Express.Multer.File[], + @Body() metadata: UploadFilesToGroupDto, + @Param('groupId', ParseIntPipe) groupId: number, + @Request() req, + ): Promise { + if (!files || files.length === 0) { + throw new BadRequestException('No files were uploaded'); + } - if (!file) throw new BadRequestException('No file uploaded'); + const res = await this.dataIngestorService.ingestFiles( + files, + metadata.fileType, + req.payload.id, + groupId, + ); - const fileType = file.originalname.split('.').pop()?.toLowerCase(); - if (!fileType) throw new BadRequestException('Unable to determine file type'); + return { + m1: 'Files uploaded successfully', + data: res, + }; + } - const parser = this.parserFactory(fileType); - const parsedData = await parser.parseToJSONAsync(file.buffer); // Assuming the file is in memory buffer - const business_id: number = req['payload']?.business_id - if (!business_id) throw new BadRequestException('Business ID not found in request') - this.dataIngestorService.ingestData({ data: parsedData, business_id }); + @Get('groups') + @ApiOperation({ + summary: 'Get all file groups', + description: 'Retrieve all file groups with their files and sheets', + }) + async getGroups(@Request() req): Promise { + const res = await this.dataIngestorService.getFileGroups(req.payload.id); return { - m1: 'File processed successfully', - data: parsedData, + m1: 'File groups retrieved successfully', + data: res, + }; + } + + @Get('groups/:groupId') + @ApiOperation({ + summary: 'Get a specific file group', + description: 'Retrieve details of a specific file group with all its files and sheets', + }) + async getGroup( + @Param('groupId', ParseIntPipe) groupId: number, + @Request() req, + ): Promise { + const res = await this.dataIngestorService.getFileGroup(groupId, req.payload.id); + return { + m1: 'File group retrieved successfully', + data: res, }; } + @Post('upload') + @ApiOperation({ + summary: 'Upload and process a file', + description: 'Upload an XLSX/CSV file for processing. Each sheet will be processed separately.', + }) + @ApiConsumes('multipart/form-data') + @ApiBody({ + type: FileMetadataDto, + }) + @UseInterceptors(FileInterceptor('file', { + fileFilter: (req, file, callback) => { + const allowedMimeTypes = [ + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', // xlsx + 'application/vnd.ms-excel', // xls + 'text/csv', // csv + ]; + if (!allowedMimeTypes.includes(file.mimetype)) { + callback(new UnsupportedMediaTypeException( + `File type ${file.mimetype} is not supported. Supported types: XLSX, XLS, CSV` + ), false); + } + callback(null, true); + }, + })) + async uploadFile( + @UploadedFile() file: Express.Multer.File, + @Body() metadata: FileMetadataDto, + @Request() req, + ): Promise { + if (!file) { + throw new BadRequestException('No file was uploaded'); + } + const res = await this.dataIngestorService.ingestFile( + file.buffer, + metadata.fileType, + req.payload.id, + file.originalname, + ); + return { + m1: 'File processed successfully', + data: res, + }; + } + @Get('files') + @ApiOperation({ + summary: 'Get all files', + description: 'Retrieve all files uploaded by the business', + }) + async getFiles(@Request() req): Promise { + const res = await this.dataIngestorService.getBusinessFiles(req.payload.id); + return { + m1: 'Files retrieved successfully', + data: res, + }; + } + @Get('sheets/:sheetId') + @ApiOperation({ + summary: 'Get sheet data', + description: 'Retrieve data from a specific sheet', + }) + async getSheetData( + @Param('sheetId', ParseIntPipe) sheetId: number, + @Request() req, + ): Promise { + const res = await this.dataIngestorService.getSheetData(sheetId, req.payload.id); + return { + m1: 'Sheet data retrieved successfully', + data: res, + }; + } + + @Delete('files/:fileId') + @ApiOperation({ + summary: 'Delete file', + description: 'Delete a file and all its associated sheets', + }) + async deleteFile( + @Param('fileId', ParseIntPipe) fileId: number, + @Request() req, + ): Promise { + const res = await this.dataIngestorService.deleteFile(fileId, req.payload.id); + return { + m1: res.message, + }; + } } diff --git a/Backend/src/modules/data-ingestor/data-ingestor.module.ts b/Backend/src/modules/data-ingestor/data-ingestor.module.ts index 1bf2308..2984661 100644 --- a/Backend/src/modules/data-ingestor/data-ingestor.module.ts +++ b/Backend/src/modules/data-ingestor/data-ingestor.module.ts @@ -1,12 +1,36 @@ -import {Module} from '@nestjs/common'; -import {DataIngestorController} from './data-ingestor.controller'; -import {DataIngestorService} from './data-ingestor.service'; +import { Module, BadRequestException } from '@nestjs/common'; +import { DataIngestorController } from './data-ingestor.controller'; +import { DataIngestorService } from './data-ingestor.service'; +import { SheetCleanerService } from './services/sheet-cleaner.service'; import { parserFactory } from './parser/parser.factory'; +import { PrismaModule } from '../prisma/prisma.module'; +import { MulterModule } from '@nestjs/platform-express'; + @Module({ + imports: [ + PrismaModule, + MulterModule.register({ + limits: { + fileSize: 10 * 1024 * 1024, // 10MB limit per file + files: 10, // Maximum number of files + fieldSize: 20 * 1024 * 1024, // 20MB total upload size + }, + fileFilter: (req, file, callback) => { + const minSize = 10; // Minimum 10 bytes + if (parseInt(req.headers['content-length']) < minSize) { + callback(new BadRequestException('File is too small'), false); + return; + } + callback(null, true); + }, + }), + ], controllers: [DataIngestorController], providers: [ DataIngestorService, - parserFactory, + SheetCleanerService, + parserFactory, ], + exports: [DataIngestorService], }) -export class DataIngestorModule {} +export class DataIngestorModule { } diff --git a/Backend/src/modules/data-ingestor/data-ingestor.service.spec.ts b/Backend/src/modules/data-ingestor/data-ingestor.service.spec.ts new file mode 100644 index 0000000..054703c --- /dev/null +++ b/Backend/src/modules/data-ingestor/data-ingestor.service.spec.ts @@ -0,0 +1,288 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { DataIngestorService } from './data-ingestor.service'; +import { PrismaService } from '../prisma/prisma.service'; +import { SheetCleanerService } from './services/sheet-cleaner.service'; +import { BadRequestException, ForbiddenException, NotFoundException } from '@nestjs/common'; +import { FILE_TYPE } from '@prisma/client'; +import { ParserFactory } from './parser/parser.factory'; + +// Mock PrismaService +const mockPrismaService = { + $transaction: jest.fn((callback) => callback(mockPrismaService)), + file_group: { + findFirst: jest.fn(), + create: jest.fn(), + findMany: jest.fn(), + findUnique: jest.fn(), + }, + file_upload: { + create: jest.fn(), + findMany: jest.fn(), + findFirst: jest.fn(), + update: jest.fn(), + }, + sheet: { + create: jest.fn(), + findUnique: jest.fn(), + findFirst: jest.fn(), + updateMany: jest.fn(), + }, +}; + +// Mock SheetCleanerService +const mockSheetCleanerService = { + cleanSheetData: jest.fn(), + validateData: jest.fn(), +}; + +// Mock ParserFactory +jest.mock('./parser/parser.factory', () => ({ + ParserFactory: { + getParser: jest.fn(), + }, +})); + +describe('DataIngestorService', () => { + let service: DataIngestorService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ + DataIngestorService, + { + provide: PrismaService, + useValue: mockPrismaService, + }, + { + provide: SheetCleanerService, + useValue: mockSheetCleanerService, + }, + ], + }).compile(); + + service = module.get(DataIngestorService); + jest.clearAllMocks(); + }); + + describe('ingestFile', () => { + const mockFile = Buffer.from('test'); + const mockBusinessId = 1; + const mockFilename = 'test.xlsx'; + + it('should successfully ingest a file', async () => { + const mockParser = { + parseFile: jest.fn().mockResolvedValue([{ + sheetName: 'Sheet1', + headers: ['header1'], + data: [['data1']], + }]), + }; + (ParserFactory.getParser as jest.Mock).mockReturnValue(mockParser); + + mockPrismaService.file_group.findFirst.mockResolvedValue({ id: 1 }); + mockPrismaService.file_upload.create.mockResolvedValue({ id: 1 }); + mockPrismaService.sheet.create.mockResolvedValue({ + id: 1, + sheet_name: 'Sheet1', + row_count: 1, + column_count: 1, + }); + + const result = await service.ingestFile(mockFile, FILE_TYPE.XLSX, mockBusinessId, mockFilename); + + expect(result).toEqual({ + fileId: 1, + sheets: expect.any(Array), + message: expect.any(String), + }); + }); + + it('should throw BadRequestException when file processing fails', async () => { + const mockError = new Error('Processing failed'); + mockPrismaService.file_group.findFirst.mockRejectedValue(mockError); + + await expect( + service.ingestFile(mockFile, FILE_TYPE.XLSX, mockBusinessId, mockFilename) + ).rejects.toThrow(BadRequestException); + }); + }); + + describe('getBusinessFiles', () => { + const mockBusinessId = 1; + + it('should return business files', async () => { + const mockFiles = [{ + id: 1, + filename: 'test.xlsx', + file_type: FILE_TYPE.XLSX, + created_at: new Date(), + sheets: [{ + sheet_name: 'Sheet1', + row_count: 1, + column_count: 1, + headers: ['header1'], + }], + }]; + + mockPrismaService.file_upload.findMany.mockResolvedValue(mockFiles); + + const result = await service.getBusinessFiles(mockBusinessId); + + expect(result).toEqual(expect.arrayContaining([ + expect.objectContaining({ + file_id: expect.any(Number), + filename: expect.any(String), + }), + ])); + }); + }); + + describe('getSheetData', () => { + const mockSheetId = 1; + const mockBusinessId = 1; + + it('should return sheet data when access is allowed', async () => { + const mockSheet = { + id: 1, + data: [['data']], + headers: ['header'], + sheet_name: 'Sheet1', + file_upload: { filename: 'test.xlsx' }, + }; + + mockPrismaService.sheet.findUnique.mockResolvedValue(mockSheet); + mockPrismaService.sheet.findFirst.mockResolvedValue(mockSheet); + mockSheetCleanerService.cleanSheetData.mockReturnValue({}); + mockSheetCleanerService.validateData.mockReturnValue(true); + + const result = await service.getSheetData(mockSheetId, mockBusinessId); + + expect(result).toHaveProperty('filename'); + }); + + it('should throw NotFoundException when sheet does not exist', async () => { + mockPrismaService.sheet.findUnique.mockResolvedValue(null); + + await expect( + service.getSheetData(mockSheetId, mockBusinessId) + ).rejects.toThrow(NotFoundException); + }); + + it('should throw ForbiddenException when access is denied', async () => { + mockPrismaService.sheet.findUnique.mockResolvedValue({ id: 1 }); + mockPrismaService.sheet.findFirst.mockResolvedValue(null); + + await expect( + service.getSheetData(mockSheetId, mockBusinessId) + ).rejects.toThrow(ForbiddenException); + }); + }); + + describe('deleteFile', () => { + const mockFileId = 1; + const mockBusinessId = 1; + + it('should successfully delete a file', async () => { + mockPrismaService.file_upload.findFirst.mockResolvedValue({ + id: 1, + sheets: [{ id: 1 }, { id: 2 }], + }); + + const result = await service.deleteFile(mockFileId, mockBusinessId); + + expect(result).toEqual({ + message: expect.stringContaining('deleted successfully'), + }); + expect(mockPrismaService.sheet.updateMany).toHaveBeenCalled(); + expect(mockPrismaService.file_upload.update).toHaveBeenCalled(); + }); + + it('should throw NotFoundException when file does not exist', async () => { + mockPrismaService.file_upload.findFirst.mockResolvedValue(null); + + await expect( + service.deleteFile(mockFileId, mockBusinessId) + ).rejects.toThrow(NotFoundException); + }); + }); + + describe('createFileGroup', () => { + it('should create a file group', async () => { + const mockData = { name: 'Test Group' }; + const mockBusinessId = 1; + const mockGroup = { id: 1, ...mockData }; + + mockPrismaService.file_group.create.mockResolvedValue(mockGroup); + + const result = await service.createFileGroup(mockBusinessId, mockData); + + expect(result).toEqual(mockGroup); + expect(mockPrismaService.file_group.create).toHaveBeenCalledWith({ + data: expect.objectContaining({ + business_id: mockBusinessId, + }), + }); + }); + }); + + describe('ingestFiles', () => { + const mockFiles = [ + { buffer: Buffer.from('test1'), originalname: 'test1.xlsx' }, + { buffer: Buffer.from('test2'), originalname: 'test2.xlsx' }, + ] as Express.Multer.File[]; + const mockBusinessId = 1; + const mockGroupId = 1; + + it('should successfully ingest multiple files', async () => { + const mockParser = { + parseFile: jest.fn().mockResolvedValue([{ + sheetName: 'Sheet1', + headers: ['header1'], + data: [['data1']], + rowCount: 1, + columnCount: 1, + }]), + }; + (ParserFactory.getParser as jest.Mock).mockReturnValue(mockParser); + + mockPrismaService.file_group.findFirst.mockResolvedValue({ id: 1 }); + mockPrismaService.file_upload.create.mockResolvedValue({ id: 1, filename: 'test.xlsx' }); + mockPrismaService.sheet.create.mockResolvedValue({ + id: 1, + sheet_name: 'Sheet1', + row_count: 1, + column_count: 1, + }); + mockPrismaService.file_group.findUnique.mockResolvedValue({ + id: 1, + name: 'Test Group', + description: 'Test Description', + files: [], + }); + + const result = await service.ingestFiles(mockFiles, FILE_TYPE.XLSX, mockBusinessId, mockGroupId); + + expect(result).toHaveProperty('group_id'); + expect(result).toHaveProperty('files'); + expect(result).toHaveProperty('processingResults'); + }); + + it('should throw NotFoundException when group does not exist', async () => { + mockPrismaService.file_group.findFirst.mockResolvedValue(null); + + await expect( + service.ingestFiles(mockFiles, FILE_TYPE.XLSX, mockBusinessId, mockGroupId) + ).rejects.toThrow(NotFoundException); + }); + + it('should throw BadRequestException when no files are processed successfully', async () => { + mockPrismaService.file_group.findFirst.mockResolvedValue({ id: 1 }); + const mockError = new Error('Processing failed'); + mockPrismaService.file_upload.create.mockRejectedValue(mockError); + + await expect( + service.ingestFiles(mockFiles, FILE_TYPE.XLSX, mockBusinessId, mockGroupId) + ).rejects.toThrow(BadRequestException); + }); + }); +}); \ No newline at end of file diff --git a/Backend/src/modules/data-ingestor/data-ingestor.service.ts b/Backend/src/modules/data-ingestor/data-ingestor.service.ts index c53c9dc..5a3188a 100644 --- a/Backend/src/modules/data-ingestor/data-ingestor.service.ts +++ b/Backend/src/modules/data-ingestor/data-ingestor.service.ts @@ -1,17 +1,465 @@ -import { Injectable } from '@nestjs/common'; +import { Injectable, BadRequestException, NotFoundException, ForbiddenException } from '@nestjs/common'; import { PrismaService } from '../prisma/prisma.service'; +import { FILE_TYPE, File_group, File_upload, Prisma, Sheet } from '@prisma/client'; +import * as XLSX from 'xlsx'; +import { IngestFileResponseDto } from './dto/ingest-file.dto'; +import { CreateFileGroupDto, FileGroupResponseDto } from './dto/file-group.dto'; +import { SheetSummaryResponse } from './interfaces/responses.interface'; +import { ParserFactory } from './parser/parser.factory'; +import { SheetCleanerService } from './services/sheet-cleaner.service'; +import { BusinessFileResponse, FileGroupResponse, SheetDataResponse, ProcessedFile } from './interfaces/responses.interface'; + @Injectable() export class DataIngestorService { - constructor(private readonly prisma: PrismaService) { } - async ingestData({ business_id, data }: { business_id: number, data: any }): Promise { - const insertedData = await this.prisma.ingested_data.create({ + constructor( + private readonly prisma: PrismaService, + private readonly sheetCleaner: SheetCleanerService, + ) { } + + /** + * Process and store an uploaded file + * @param file The uploaded file buffer + * @param fileType Type of the file (XLSX, CSV, etc.) + * @param businessId ID of the business uploading the file + * @param filename Original filename + */ + async ingestFile( + file: Buffer, + fileType: FILE_TYPE, + businessId: number, + filename: string, + ): Promise { + try { + const parser = ParserFactory.getParser(fileType); + let fileUpload: File_upload; + let sheetSummaries = []; + + // Use transaction to ensure data consistency + await this.prisma.$transaction(async (prisma) => { + // Find or create a default group for this business + const defaultGroup = await prisma.file_group.findFirst({ + where: { + business_id: businessId, + name: 'Default Group', + deleted_at: null, + }, + }); + + let groupId: number; + if (!defaultGroup) { + const newGroup = await prisma.file_group.create({ + data: { + business_id: businessId, + name: 'Default Group', + description: 'Default group for single file uploads' + }, + }); + groupId = newGroup.id; + } else { + groupId = defaultGroup.id; + } + + // Read the workbook + const workbook = XLSX.read(file, { type: 'buffer' }); + + // Create file upload record + fileUpload = await prisma.file_upload.create({ + data: { + business_id: businessId, + filename, + file_type: fileType, + group_id: groupId, + }, + }); + + // Process each sheet + for (const sheetName of workbook.SheetNames) { + const worksheet = workbook.Sheets[sheetName]; + + // Convert sheet to JSON + const jsonData = XLSX.utils.sheet_to_json(worksheet, { header: 1 }) as any[][]; + + if (jsonData.length === 0) { + continue; // Skip empty sheets + } + + const headers = jsonData[0] as string[]; + const data = jsonData.slice(1) as any[][]; + + // Create sheet record + const sheet = await prisma.sheet.create({ + data: { + file_upload_id: fileUpload.id, + sheet_name: sheetName, + row_count: data.length, + column_count: headers.length, + headers: headers as any, + data: data as any, + }, + }); + + sheetSummaries.push({ + sheetName: sheet.sheet_name, + rowCount: sheet.row_count, + columnCount: sheet.column_count, + }); + } + }); + + return { + fileId: fileUpload.id, + sheets: sheetSummaries, + message: `File processed successfully with ${sheetSummaries.length} sheets`, + }; + } catch (error) { + throw new BadRequestException(`Failed to process file: ${error.message}`); + } + } + + /** + * Retrieve all files uploaded by a business + * @param businessId ID of the business + */ + async getBusinessFiles(businessId: number): Promise { + const files = await this.prisma.file_upload.findMany({ + where: { + business_id: businessId, + deleted_at: null, + }, + select: { + id: true, + filename: true, + file_type: true, + created_at: true, + sheets: { + select: { + sheet_name: true, + row_count: true, + column_count: true, + headers: true, + }, + }, + }, + }); + + return files.map(({ id: file_id, filename, file_type, created_at, sheets }) => ({ + file_id, + file_name: filename, + file_type, + created_at, + sheets: sheets.map(({ + sheet_name, + row_count, + column_count, + headers + }) => ({ + sheet_name, + row_count, + column_count, + headers: headers as string[] + })) + })); + } + + /** + * Retrieve data from a specific sheet + * @param sheetId ID of the sheet + * @param businessId ID of the business (for authorization) + */ + async getSheetData(sheetId: number, businessId: number): Promise { + // First check if the sheet exists + const sheetExists = await this.prisma.sheet.findUnique({ + where: { id: sheetId }, + }); + + if (!sheetExists) { + throw new NotFoundException(`Sheet with ID ${sheetId} not found`); + } + + // Then check if the user has access to it + const sheet = await this.prisma.sheet.findFirst({ + where: { + id: sheetId, + file_upload: { + business_id: businessId, + }, + }, + include: { + file_upload: true, + }, + }); + + if (!sheet) { + throw new ForbiddenException(`Access denied to sheet with ID ${sheetId}`); + } + + // Clean and transform the sheet data + const cleanedData = this.sheetCleaner.cleanSheetData( + sheet.data as any[], + sheet.headers as string[], + sheet.sheet_name + ); + + // Validate the cleaned data + if (!this.sheetCleaner.validateData(cleanedData)) { + throw new BadRequestException('Sheet data quality does not meet minimum requirements'); + } + + return { + filename: sheet.file_upload.filename, + ...cleanedData + }; + } + + /** + * Delete a file and all its sheets + * @param fileId ID of the file to delete + * @param businessId ID of the business (for authorization) + */ + async deleteFile(fileId: number, businessId: number): Promise<{ message: string }> { + // First check if file exists and user has access + const file = await this.prisma.file_upload.findFirst({ + where: { + id: fileId, + business_id: businessId, + deleted_at: null, + }, + include: { + sheets: true + } + }); + + if (!file) { + throw new NotFoundException('File not found or already deleted'); + } + + // Use transaction to ensure atomic deletion + await this.prisma.$transaction(async (prisma) => { + // Mark all associated sheets as deleted + await prisma.sheet.updateMany({ + where: { + file_upload_id: fileId + }, + data: { + deleted_at: new Date() + } + }); + + // Mark the file as deleted + await prisma.file_upload.update({ + where: { id: fileId }, + data: { deleted_at: new Date() } + }); + }); + + return { + message: `File and ${file.sheets.length} associated sheets deleted successfully` + }; + } + + async createFileGroup(businessId: number, data: CreateFileGroupDto): Promise { + return this.prisma.file_group.create({ data: { - data: data, - business_id + ...data, + business_id: businessId, + }, + }); + } + + async ingestFiles( + files: Express.Multer.File[], + fileType: FILE_TYPE, + businessId: number, + groupId: number, + ): Promise { + const existingGroup = await this.getFileGroup(groupId, businessId); + if (!existingGroup) { + throw new NotFoundException(`File group with ID ${groupId} not found or you don't have access to it`); + } + + const parser = ParserFactory.getParser(fileType); + const results: ProcessedFile[] = []; + const successfulFiles = []; + + // Use transaction to ensure data consistency + await this.prisma.$transaction(async (prisma) => { + // Process each file + for (const file of files) { + try { + // Create file upload record + const fileUpload = await prisma.file_upload.create({ + data: { + business_id: businessId, + group_id: groupId, + filename: file.originalname, + file_type: fileType, + }, + }); + + const parsedSheets = await parser.parseFile(file.buffer); + const sheetSummaries = []; + + // Store each sheet + for (const parsedSheet of parsedSheets) { + const sheet = await prisma.sheet.create({ + data: { + file_upload_id: fileUpload.id, + sheet_name: parsedSheet.sheetName, + row_count: parsedSheet.rowCount, + column_count: parsedSheet.columnCount, + headers: parsedSheet.headers, + data: parsedSheet.data, + }, + }); + + sheetSummaries.push({ + sheetName: sheet.sheet_name, + rowCount: sheet.row_count, + columnCount: sheet.column_count, + }); + } + + successfulFiles.push({ + file_id: fileUpload.id, + file_name: fileUpload.filename, + sheets: sheetSummaries, + }); + + results.push({ + success: true, + file_id: fileUpload.id, + file_name: file.originalname, + sheets: sheetSummaries + }); + } catch (error) { + results.push({ + success: false, + file_id: null, + file_name: file.originalname, + error: error.message + }); + } } - }) - return insertedData; + }); + + // If no files were processed successfully, throw an error + if (successfulFiles.length === 0) { + throw new BadRequestException('No files were processed successfully'); + } + + const group = await this.prisma.file_group.findUnique({ + where: { id: groupId }, + include: { + files: { + where: { + deleted_at: null, + }, + include: { + sheets: { + select: { + sheet_name: true, + row_count: true, + column_count: true, + }, + }, + }, + }, + }, + }); + + return { + group_id: group.id, + group_name: group.name, + description: group.description, + files: successfulFiles, + processingResults: results + }; + } + + async getFileGroups(businessId: number): Promise { + const groups = await this.prisma.file_group.findMany({ + where: { + business_id: businessId, + deleted_at: null, + }, + include: { + files: { + where: { + deleted_at: null, + }, + include: { + sheets: { + select: { + sheet_name: true, + row_count: true, + column_count: true, + }, + }, + }, + }, + }, + }); + + // Properly map the data to match FileGroupResponse interface + return groups.map(group => ({ + group_id: group.id, + group_name: group.name, + description: group.description, + files: group.files.map(file => ({ + file_id: file.id, + file_name: file.filename, + sheets: file.sheets.map(sheet => ({ + sheetName: sheet.sheet_name, + rowCount: sheet.row_count, + columnCount: sheet.column_count, + })), + })), + })); } + async getFileGroup(groupId: number, businessId: number): Promise { + const group = await this.prisma.file_group.findFirst({ + where: { + id: groupId, + business_id: businessId, + deleted_at: null, + }, + include: { + files: { + where: { + deleted_at: null, + }, + include: { + sheets: { + select: { + sheet_name: true, + row_count: true, + column_count: true, + }, + }, + }, + }, + }, + }); + + if (!group) { + return null; + } + // Properly map the data to match FileGroupResponse interface + return { + group_id: group.id, + group_name: group.name, + description: group.description, + files: group.files.map(file => ({ + file_id: file.id, + file_name: file.filename, + sheets: file.sheets.map(sheet => ({ + sheetName: sheet.sheet_name, + rowCount: sheet.row_count, + columnCount: sheet.column_count, + })), + })), + }; + } } diff --git a/Backend/src/modules/data-ingestor/dto/file-group.dto.ts b/Backend/src/modules/data-ingestor/dto/file-group.dto.ts new file mode 100644 index 0000000..9701e53 --- /dev/null +++ b/Backend/src/modules/data-ingestor/dto/file-group.dto.ts @@ -0,0 +1,62 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { IsString, IsOptional, IsArray, IsEnum } from 'class-validator'; +import { FILE_TYPE } from '@prisma/client'; + +export class CreateFileGroupDto { + @ApiProperty({ example: 'Sales Reports 2024', description: 'Name of the file group' }) + @IsString() + name: string; + + @ApiProperty({ example: 'Monthly sales reports for 2024', description: 'Description of the file group' }) + @IsOptional() + @IsString() + description?: string; +} + +export class UploadFilesToGroupDto { + @ApiProperty({ enum: FILE_TYPE, description: 'Type of the files being uploaded' }) + @IsEnum(FILE_TYPE) + fileType: FILE_TYPE; + + @ApiProperty({ + type: 'array', + items: { + type: 'string', + format: 'binary' + }, + description: 'Multiple files to be uploaded' + }) + files: any[]; +} + +export class FileGroupResponseDto { + @ApiProperty({ example: 1, description: 'ID of the file group' }) + id: number; + + @ApiProperty({ example: 'Sales Reports 2024' }) + name: string; + + @ApiProperty({ example: 'Monthly sales reports for 2024' }) + description?: string; + + @ApiProperty({ + example: [ + { + id: 1, + filename: 'january.xlsx', + sheets: [ + { sheetName: 'Sheet1', rowCount: 100 } + ] + } + ] + }) + files: { + id: number; + filename: string; + sheets: { + sheetName: string; + rowCount: number; + columnCount: number; + }[]; + }[]; +} \ No newline at end of file diff --git a/Backend/src/modules/data-ingestor/dto/ingest-file.dto.ts b/Backend/src/modules/data-ingestor/dto/ingest-file.dto.ts new file mode 100644 index 0000000..8095501 --- /dev/null +++ b/Backend/src/modules/data-ingestor/dto/ingest-file.dto.ts @@ -0,0 +1,27 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { IsEnum } from 'class-validator'; +import { FILE_TYPE } from '@prisma/client'; + +export class FileMetadataDto { + @ApiProperty({ enum: FILE_TYPE, description: 'Type of the file being uploaded' }) + @IsEnum(FILE_TYPE) + fileType: FILE_TYPE; + + @ApiProperty({ type: 'string', format: 'binary' }) + file: any; +} + +export class IngestFileResponseDto { + @ApiProperty({ example: 1 }) + fileId: number; + + @ApiProperty() + sheets: { + sheetName: string; + rowCount: number; + columnCount: number; + }[]; + + @ApiProperty() + message: string; +} \ No newline at end of file diff --git a/Backend/src/modules/data-ingestor/interfaces/responses.interface.ts b/Backend/src/modules/data-ingestor/interfaces/responses.interface.ts new file mode 100644 index 0000000..3ea240f --- /dev/null +++ b/Backend/src/modules/data-ingestor/interfaces/responses.interface.ts @@ -0,0 +1,57 @@ +import { FILE_TYPE } from '@prisma/client'; + +export interface SheetSummaryResponse { + sheetName: string; + rowCount: number; + columnCount: number; +} + +export interface FileUploadResponse { + file_id: number; + file_name: string; + sheets: SheetSummaryResponse[]; +} + +export interface ProcessedFile { + success: boolean; + file_id: number | null; + file_name: string | null; + sheets?: SheetSummaryResponse[]; + error?: string; +} +export interface FileGroupResponse { + group_id: number; + group_name: string; + description: string | null; + files: FileUploadResponse[]; + processingResults?: ProcessedFile[]; +} + +export interface BusinessFileResponse { + file_id: number; + file_name: string; + file_type: FILE_TYPE; + created_at: Date; + sheets: { + sheet_name: string; + row_count: number; + column_count: number; + headers: string[]; + }[]; +} + +export interface SheetDataResponse { + filename: string; + sheetName: string; + rowCount: number; + columnCount: number; + headers: string[]; + data: Record[]; + summary: { + totalRows: number; + totalColumns: number; + emptyRows: number; + emptyCells: number; + dataTypes: Record; + }; +} \ No newline at end of file diff --git a/Backend/src/modules/data-ingestor/interfaces/sheet-data.interface.ts b/Backend/src/modules/data-ingestor/interfaces/sheet-data.interface.ts new file mode 100644 index 0000000..08f9f74 --- /dev/null +++ b/Backend/src/modules/data-ingestor/interfaces/sheet-data.interface.ts @@ -0,0 +1,16 @@ +export interface CleanedSheetData { + sheetName: string; + rowCount: number; + columnCount: number; + headers: string[]; + data: Record[]; + summary: SheetSummary; +} + +export interface SheetSummary { + totalRows: number; + totalColumns: number; + emptyRows: number; + emptyCells: number; + dataTypes: Record; +} \ No newline at end of file diff --git a/Backend/src/modules/data-ingestor/parser/csv.parser.ts b/Backend/src/modules/data-ingestor/parser/csv.parser.ts new file mode 100644 index 0000000..93fa2e4 --- /dev/null +++ b/Backend/src/modules/data-ingestor/parser/csv.parser.ts @@ -0,0 +1,41 @@ +import { Parser, SheetData } from './parser.interface'; +import { parse } from 'csv-parse'; + +export class CSVParser implements Parser { + async parseFile(buffer: Buffer): Promise { + return new Promise((resolve, reject) => { + parse(buffer, { + columns: false, + skip_empty_lines: true, + trim: true, + }, (err, records: any[][]) => { + if (err) reject(err); + + if (records.length === 0) { + resolve([]); + return; + } + + const headers = records[0].map(String); + const data = records.slice(1); + + const filteredData = data.filter(row => + row.some(cell => cell !== null && cell !== undefined && cell.toString().trim() !== '') + ); + + if (filteredData.length === 0) { + resolve([]); + return; + } + + resolve([{ + sheetName: 'Sheet1', + rowCount: filteredData.length, + columnCount: headers.length, + headers, + data: filteredData + }]); + }); + }); + } +} \ No newline at end of file diff --git a/Backend/src/modules/data-ingestor/parser/parser.factory.ts b/Backend/src/modules/data-ingestor/parser/parser.factory.ts index 244e158..7d09cb8 100644 --- a/Backend/src/modules/data-ingestor/parser/parser.factory.ts +++ b/Backend/src/modules/data-ingestor/parser/parser.factory.ts @@ -1,27 +1,27 @@ import { Injectable, HttpException, HttpStatus } from '@nestjs/common'; -import { Parser, CSVParser, XLSXParser, } from './data-parser'; +import { FILE_TYPE } from '@prisma/client'; +import { Parser } from './parser.interface'; +import { XLSXParser } from './xlsx.parser'; +import { CSVParser } from './csv.parser'; + @Injectable() -class ParserFactory { - static getParser(fileType: string): Parser { - switch (fileType.toLowerCase()) { - case 'xlsx': +export class ParserFactory { + static getParser(fileType: FILE_TYPE): Parser { + switch (fileType) { + case FILE_TYPE.XLSX: return new XLSXParser(); - case 'csv': + case FILE_TYPE.CSV: return new CSVParser(); - // case 'sql': - // return new SQLParser(); default: - throw new HttpException( - `No parser available for file type: ${fileType}`, - HttpStatus.BAD_REQUEST, - ); + throw new Error(`Unsupported file type: ${fileType}`); } } } + //if you have multiple methods in ParserFactory, you can use useClass instead of useFactory in the provider export const parserFactory = { provide: 'PARSER_FACTORY', useFactory: () => { - return (fileType: string) => ParserFactory.getParser(fileType); + return (fileType: FILE_TYPE) => ParserFactory.getParser(fileType); }, }; diff --git a/Backend/src/modules/data-ingestor/parser/parser.interface.ts b/Backend/src/modules/data-ingestor/parser/parser.interface.ts new file mode 100644 index 0000000..4580e16 --- /dev/null +++ b/Backend/src/modules/data-ingestor/parser/parser.interface.ts @@ -0,0 +1,17 @@ +export interface Parser { + parseFile(buffer: Buffer): Promise<{ + sheetName: string; + rowCount: number; + columnCount: number; + headers: string[]; + data: any[]; + }[]>; +} + +export type SheetData = { + sheetName: string; + rowCount: number; + columnCount: number; + headers: string[]; + data: any[]; +}; \ No newline at end of file diff --git a/Backend/src/modules/data-ingestor/parser/xlsx.parser.ts b/Backend/src/modules/data-ingestor/parser/xlsx.parser.ts new file mode 100644 index 0000000..31359a5 --- /dev/null +++ b/Backend/src/modules/data-ingestor/parser/xlsx.parser.ts @@ -0,0 +1,42 @@ +import * as XLSX from 'xlsx'; +import { SheetData, Parser } from './parser.interface'; + +export class XLSXParser implements Parser { + async parseFile(buffer: Buffer): Promise { + const workbook = XLSX.read(buffer, { type: 'buffer' }); + const sheets = []; + + for (const sheetName of workbook.SheetNames) { + const worksheet = workbook.Sheets[sheetName]; + const jsonData = XLSX.utils.sheet_to_json(worksheet, { + header: 1, + blankrows: false, + defval: null + }) as any[][]; + + if (jsonData.length === 0) continue; + + const headers = jsonData[0].map(header => + header === null || header === undefined ? '' : String(header).trim() + ); + const data = jsonData.slice(1); + + // Filter out completely empty rows + const filteredData = data.filter(row => + row.some(cell => cell !== null && cell !== undefined && cell.toString().trim() !== '') + ); + + if (filteredData.length === 0) continue; + + sheets.push({ + sheetName, + rowCount: filteredData.length, + columnCount: headers.length, + headers, + data: filteredData, + }); + } + + return sheets; + } +} \ No newline at end of file diff --git a/Backend/src/modules/data-ingestor/services/sheet-cleaner.service.ts b/Backend/src/modules/data-ingestor/services/sheet-cleaner.service.ts new file mode 100644 index 0000000..da3919d --- /dev/null +++ b/Backend/src/modules/data-ingestor/services/sheet-cleaner.service.ts @@ -0,0 +1,131 @@ +import { Injectable } from '@nestjs/common'; +import { CleanedSheetData, SheetSummary } from '../interfaces/sheet-data.interface'; + +@Injectable() +export class SheetCleanerService { + cleanSheetData(rawData: any[], headers: string[], sheetName: string): CleanedSheetData { + // Clean headers + const cleanedHeaders = this.cleanHeaders(headers); + + // Clean and transform data + const cleanedData = this.transformData(rawData, cleanedHeaders); + + // Generate summary + const summary = this.generateSummary(cleanedData, cleanedHeaders); + + return { + sheetName, + rowCount: cleanedData.length, + columnCount: cleanedHeaders.length, + headers: cleanedHeaders, + data: cleanedData, + summary + }; + } + + private cleanHeaders(headers: string[]): string[] { + return headers.map(header => { + // Remove special characters and normalize spaces + let cleaned = header + .trim() + .toLowerCase() + .replace(/[^\w\s]/g, '') + .replace(/\s+/g, '_'); + + // Ensure header is not empty + if (!cleaned) { + cleaned = 'column_' + (headers.indexOf(header) + 1); + } + + return cleaned; + }); + } + + private transformData(rawData: any[], headers: string[]): Record[] { + return rawData.map(row => { + const transformedRow: Record = {}; + + headers.forEach((header, index) => { + let value = row[index]; + + // Clean and type convert values + if (typeof value === 'string') { + value = value.trim(); + + // Convert to number if possible + if (!isNaN(value as any) && value !== '') { + value = Number(value); + } + + // Convert to boolean if applicable + if (value.toLowerCase() === 'true') value = true; + if (value.toLowerCase() === 'false') value = false; + + // Convert to null if empty + if (value === '') value = null; + } + + transformedRow[header] = value; + }); + + return transformedRow; + }).filter(row => Object.values(row).some(value => value !== null)); // Remove completely empty rows + } + + private generateSummary(data: Record[], headers: string[]): SheetSummary { + let emptyCells = 0; + let emptyRows = 0; + const dataTypes: Record = {}; + + // Initialize dataTypes + headers.forEach(header => { + dataTypes[header] = 'mixed'; + }); + + // Analyze data + data.forEach(row => { + let rowEmpty = true; + + headers.forEach(header => { + const value = row[header]; + + if (value === null || value === undefined) { + emptyCells++; + } else { + rowEmpty = false; + + // Determine data type + const type = typeof value; + if (dataTypes[header] === 'mixed') { + dataTypes[header] = type; + } else if (dataTypes[header] !== type) { + dataTypes[header] = 'mixed'; + } + } + }); + + if (rowEmpty) emptyRows++; + }); + + return { + totalRows: data.length, + totalColumns: headers.length, + emptyRows, + emptyCells, + dataTypes + }; + } + + /** + * Validate if the data meets quality standards + */ + validateData(cleanedData: CleanedSheetData): boolean { + const { summary } = cleanedData; + + // Example validation rules + const emptyRowPercentage = (summary.emptyRows / summary.totalRows) * 100; + const emptyCellPercentage = (summary.emptyCells / (summary.totalRows * summary.totalColumns)) * 100; + + return emptyRowPercentage < 50 && emptyCellPercentage < 30; + } +} \ No newline at end of file diff --git a/Backend/src/modules/location/dto/city.dto.ts b/Backend/src/modules/location/dto/city.dto.ts index 68fbddc..dcb5be5 100644 --- a/Backend/src/modules/location/dto/city.dto.ts +++ b/Backend/src/modules/location/dto/city.dto.ts @@ -13,4 +13,4 @@ export class CreateCityDto { state_id: number; } -export class UpdateCityDto extends CreateCityDto {} \ No newline at end of file +export class UpdateCityDto extends CreateCityDto {} \ No newline at end of file From 22921750c57888cd79567f341598543e93c16e49 Mon Sep 17 00:00:00 2001 From: zeeshanalico Date: Fri, 28 Mar 2025 02:19:35 +0500 Subject: [PATCH 3/3] config: vercel.json and db configuration with neon --- Backend/package-lock.json | 42 ++++++++++++++++++++++++++++----------- Backend/package.json | 9 ++++++--- Backend/vercel.json | 5 +++++ 3 files changed, 41 insertions(+), 15 deletions(-) create mode 100644 Backend/vercel.json diff --git a/Backend/package-lock.json b/Backend/package-lock.json index c92b8c3..b663964 100644 --- a/Backend/package-lock.json +++ b/Backend/package-lock.json @@ -43,7 +43,7 @@ "@types/express": "^4.17.21", "@types/jest": "^29.5.14", "@types/multer": "^1.4.12", - "@types/node": "^20.11.16", + "@types/node": "^20.17.28", "@types/supertest": "^6.0.2", "@typescript-eslint/eslint-plugin": "^6.21.0", "@typescript-eslint/parser": "^6.21.0", @@ -60,7 +60,7 @@ "ts-loader": "^9.5.1", "ts-node": "^10.9.2", "tsconfig-paths": "^4.2.0", - "typescript": "^5.3.3" + "typescript": "^5.8.2" }, "engines": { "node": ">=16.0.0", @@ -2512,6 +2512,20 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/@nestjs/cli/node_modules/typescript": { + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz", + "integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, "node_modules/@nestjs/common": { "version": "10.3.2", "resolved": "https://registry.npmjs.org/@nestjs/common/-/common-10.3.2.tgz", @@ -3821,11 +3835,12 @@ } }, "node_modules/@types/node": { - "version": "20.11.16", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.16.tgz", - "integrity": "sha512-gKb0enTmRCzXSSUJDq6/sPcqrfCv2mkkG6Jt/clpn5eiCbKTY+SgZUxo+p8ZKMof5dCp9vHQUAB7wOUTod22wQ==", + "version": "20.17.28", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.17.28.tgz", + "integrity": "sha512-DHlH/fNL6Mho38jTy7/JT7sn2wnXI+wULR6PV4gy4VHLVvnrV/d3pHAMQHhc4gjdLmK2ZiPoMxzp6B3yRajLSQ==", + "license": "MIT", "dependencies": { - "undici-types": "~5.26.4" + "undici-types": "~6.19.2" } }, "node_modules/@types/node-fetch": { @@ -13428,6 +13443,7 @@ "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", "dev": true, + "license": "MIT", "dependencies": { "@cspotcode/source-map-support": "^0.8.0", "@tsconfig/node10": "^1.0.7", @@ -13577,10 +13593,11 @@ "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==" }, "node_modules/typescript": { - "version": "5.3.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz", - "integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==", + "version": "5.8.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.2.tgz", + "integrity": "sha512-aJn6wq13/afZp/jT9QZmwEjDqqvSGp1VT5GVg+f/t6/oVyrgXM6BY1h9BRh/O5p3PlUPAe+WuiEZOmb/49RqoQ==", "devOptional": true, + "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -13621,9 +13638,10 @@ } }, "node_modules/undici-types": { - "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" + "version": "6.19.8", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", + "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==", + "license": "MIT" }, "node_modules/universalify": { "version": "2.0.1", diff --git a/Backend/package.json b/Backend/package.json index 594e531..515e2b6 100644 --- a/Backend/package.json +++ b/Backend/package.json @@ -23,7 +23,10 @@ "prisma:pull": "dotenv -e .env.development -- npx prisma db pull", "prisma:studio": "dotenv -e .env.development -- npx prisma studio", "prisma:migrate:reset": "dotenv -e .env.development -- npx prisma migrate reset", - "prisma:seed": "dotenv -e .env.development -- ts-node prisma/seed.ts" + "prisma:seed": "dotenv -e .env.development -- ts-node prisma/seed.ts", + "prisma:prod:generate": "dotenv -e .env.production -- npx prisma generate", + "prisma:prod:migrate": "dotenv -e .env.production -- npx prisma migrate deploy", + "prisma:prod:seed": "dotenv -e .env.production -- ts-node prisma/seed.ts" }, "engines": { "npm": ">=8.0.0", @@ -64,7 +67,7 @@ "@types/express": "^4.17.21", "@types/jest": "^29.5.14", "@types/multer": "^1.4.12", - "@types/node": "^20.11.16", + "@types/node": "^20.17.28", "@types/supertest": "^6.0.2", "@typescript-eslint/eslint-plugin": "^6.21.0", "@typescript-eslint/parser": "^6.21.0", @@ -81,7 +84,7 @@ "ts-loader": "^9.5.1", "ts-node": "^10.9.2", "tsconfig-paths": "^4.2.0", - "typescript": "^5.3.3" + "typescript": "^5.8.2" }, "jest": { "moduleFileExtensions": [ diff --git a/Backend/vercel.json b/Backend/vercel.json new file mode 100644 index 0000000..67d564b --- /dev/null +++ b/Backend/vercel.json @@ -0,0 +1,5 @@ +{ + "rewrites": [ + {"source": "/(.*)", "destination": "/"} + ] + } \ No newline at end of file