diff --git a/api/src/config/index.ts b/api/src/config/index.ts index 68898396..5a76f4ce 100644 --- a/api/src/config/index.ts +++ b/api/src/config/index.ts @@ -110,7 +110,7 @@ export const config: Config = { storageMode: 'disk', maxUploadSize: process.env.UPLOAD_MAX_SIZE_IN_BYTES ? Number(process.env.UPLOAD_MAX_SIZE_IN_BYTES) - : 2000000, + : 50 * 1024 * 1024, // 50 MB in bytes appName: 'Hexabot.ai', }, pagination: { diff --git a/api/src/nlp/controllers/nlp-sample.controller.spec.ts b/api/src/nlp/controllers/nlp-sample.controller.spec.ts index 6b0fae31..b8d51f9f 100644 --- a/api/src/nlp/controllers/nlp-sample.controller.spec.ts +++ b/api/src/nlp/controllers/nlp-sample.controller.spec.ts @@ -6,17 +6,12 @@ * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file). */ -import fs from 'fs'; - import { CACHE_MANAGER } from '@nestjs/cache-manager'; import { BadRequestException, NotFoundException } from '@nestjs/common'; import { EventEmitter2 } from '@nestjs/event-emitter'; import { MongooseModule } from '@nestjs/mongoose'; import { Test, TestingModule } from '@nestjs/testing'; -import { AttachmentRepository } from '@/attachment/repositories/attachment.repository'; -import { AttachmentModel } from '@/attachment/schemas/attachment.schema'; -import { AttachmentService } from '@/attachment/services/attachment.service'; import { HelperService } from '@/helper/helper.service'; import { LanguageRepository } from '@/i18n/repositories/language.repository'; import { Language, LanguageModel } from '@/i18n/schemas/language.schema'; @@ -50,7 +45,6 @@ import { NlpEntityService } from '../services/nlp-entity.service'; import { NlpSampleEntityService } from '../services/nlp-sample-entity.service'; import { NlpSampleService } from '../services/nlp-sample.service'; import { NlpValueService } from '../services/nlp-value.service'; -import { NlpService } from '../services/nlp.service'; import { NlpSampleController } from './nlp-sample.controller'; @@ -60,7 +54,6 @@ describe('NlpSampleController', () => { let nlpSampleService: NlpSampleService; let nlpEntityService: NlpEntityService; let nlpValueService: NlpValueService; - let attachmentService: AttachmentService; let languageService: LanguageService; let byeJhonSampleId: string; let languages: Language[]; @@ -76,7 +69,6 @@ describe('NlpSampleController', () => { MongooseModule.forFeature([ NlpSampleModel, NlpSampleEntityModel, - AttachmentModel, NlpEntityModel, NlpValueModel, SettingModel, @@ -87,9 +79,7 @@ describe('NlpSampleController', () => { LoggerService, NlpSampleRepository, NlpSampleEntityRepository, - AttachmentService, NlpEntityService, - AttachmentRepository, NlpEntityRepository, NlpValueService, NlpValueRepository, @@ -98,7 +88,6 @@ describe('NlpSampleController', () => { LanguageRepository, LanguageService, EventEmitter2, - NlpService, HelperService, SettingRepository, SettingService, @@ -131,7 +120,6 @@ describe('NlpSampleController', () => { text: 'Bye Jhon', }) ).id; - attachmentService = module.get(AttachmentService); languageService = module.get(LanguageService); languages = await languageService.findAll(); }); @@ -315,83 +303,44 @@ describe('NlpSampleController', () => { }); }); - describe('import', () => { - it('should throw exception when attachment is not found', async () => { - const invalidattachmentId = ( - await attachmentService.findOne({ - name: 'store2.jpg', - }) - ).id; - await attachmentService.deleteOne({ name: 'store2.jpg' }); - await expect( - nlpSampleController.import(invalidattachmentId), - ).rejects.toThrow(NotFoundException); - }); - - it('should throw exception when file location is not present', async () => { - const attachmentId = ( - await attachmentService.findOne({ - name: 'store1.jpg', - }) - ).id; - jest.spyOn(fs, 'existsSync').mockReturnValueOnce(false); - await expect(nlpSampleController.import(attachmentId)).rejects.toThrow( - NotFoundException, + describe('importFile', () => { + it('should throw exception when something is wrong with the upload', async () => { + const file = { + buffer: Buffer.from('', 'utf-8'), + size: 0, + mimetype: 'text/csv', + } as Express.Multer.File; + await expect(nlpSampleController.importFile(file)).rejects.toThrow( + 'Bad Request Exception', ); }); it('should return a failure if an error occurs when parsing csv file ', async () => { const mockCsvDataWithErrors: string = `intent,entities,lang,question greeting,person,en`; - jest.spyOn(fs, 'existsSync').mockReturnValueOnce(true); - jest.spyOn(fs, 'readFileSync').mockReturnValueOnce(mockCsvDataWithErrors); - const attachmentId = ( - await attachmentService.findOne({ - name: 'store1.jpg', - }) - ).id; - const mockParsedCsvDataWithErrors = { - data: [{ intent: 'greeting', entities: 'person', lang: 'en' }], - errors: [ - { - type: 'FieldMismatch', - code: 'TooFewFields', - message: 'Too few fields: expected 4 fields but parsed 3', - row: 0, - }, - ], - meta: { - delimiter: ',', - linebreak: '\n', - aborted: false, - truncated: false, - cursor: 49, - fields: ['intent', 'entities', 'lang', 'question'], - }, - }; - await expect(nlpSampleController.import(attachmentId)).rejects.toThrow( - new BadRequestException({ - cause: mockParsedCsvDataWithErrors.errors, - description: 'Error while parsing CSV', - }), - ); + const buffer = Buffer.from(mockCsvDataWithErrors, 'utf-8'); + const file = { + buffer, + size: buffer.length, + mimetype: 'text/csv', + } as Express.Multer.File; + await expect(nlpSampleController.importFile(file)).rejects.toThrow(); }); it('should import data from a CSV file', async () => { - const attachmentId = ( - await attachmentService.findOne({ - name: 'store1.jpg', - }) - ).id; const mockCsvData: string = [ `text,intent,language`, `How much does a BMW cost?,price,en`, ].join('\n'); - jest.spyOn(fs, 'existsSync').mockReturnValueOnce(true); - jest.spyOn(fs, 'readFileSync').mockReturnValueOnce(mockCsvData); - const result = await nlpSampleController.import(attachmentId); + const buffer = Buffer.from(mockCsvData, 'utf-8'); + const file = { + buffer, + size: buffer.length, + mimetype: 'text/csv', + } as Express.Multer.File; + const result = await nlpSampleController.importFile(file); const intentEntityResult = await nlpEntityService.findOne({ name: 'intent', }); @@ -429,9 +378,10 @@ describe('NlpSampleController', () => { expect(intentEntityResult).toEqualPayload(intentEntity); expect(priceValueResult).toEqualPayload(priceValue); expect(textSampleResult).toEqualPayload(textSample); - expect(result).toEqual({ success: true }); + expect(result).toEqualPayload([textSample]); }); }); + describe('deleteMany', () => { it('should delete multiple nlp samples', async () => { const samplesToDelete = [ diff --git a/api/src/nlp/controllers/nlp-sample.controller.ts b/api/src/nlp/controllers/nlp-sample.controller.ts index 6fa35bc3..fbeed58b 100644 --- a/api/src/nlp/controllers/nlp-sample.controller.ts +++ b/api/src/nlp/controllers/nlp-sample.controller.ts @@ -6,8 +6,6 @@ * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file). */ -import fs from 'fs'; -import { join } from 'path'; import { Readable } from 'stream'; import { @@ -25,14 +23,13 @@ import { Query, Res, StreamableFile, + UploadedFile, UseInterceptors, } from '@nestjs/common'; +import { FileInterceptor } from '@nestjs/platform-express'; import { CsrfCheck } from '@tekuconcept/nestjs-csrf'; import { Response } from 'express'; -import Papa from 'papaparse'; -import { AttachmentService } from '@/attachment/services/attachment.service'; -import { config } from '@/config'; import { HelperService } from '@/helper/helper.service'; import { LanguageService } from '@/i18n/services/language.service'; import { CsrfInterceptor } from '@/interceptors/csrf.interceptor'; @@ -45,18 +42,17 @@ import { PopulatePipe } from '@/utils/pipes/populate.pipe'; import { SearchFilterPipe } from '@/utils/pipes/search-filter.pipe'; import { TFilterQuery } from '@/utils/types/filter.types'; -import { NlpSampleCreateDto, NlpSampleDto } from '../dto/nlp-sample.dto'; +import { NlpSampleDto } from '../dto/nlp-sample.dto'; import { NlpSample, NlpSampleFull, NlpSamplePopulate, NlpSampleStub, } from '../schemas/nlp-sample.schema'; -import { NlpSampleEntityValue, NlpSampleState } from '../schemas/types'; +import { NlpSampleState } from '../schemas/types'; import { NlpEntityService } from '../services/nlp-entity.service'; import { NlpSampleEntityService } from '../services/nlp-sample-entity.service'; import { NlpSampleService } from '../services/nlp-sample.service'; -import { NlpService } from '../services/nlp.service'; @UseInterceptors(CsrfInterceptor) @Controller('nlpsample') @@ -68,11 +64,9 @@ export class NlpSampleController extends BaseController< > { constructor( private readonly nlpSampleService: NlpSampleService, - private readonly attachmentService: AttachmentService, private readonly nlpSampleEntityService: NlpSampleEntityService, private readonly nlpEntityService: NlpEntityService, private readonly logger: LoggerService, - private readonly nlpService: NlpService, private readonly languageService: LanguageService, private readonly helperService: HelperService, ) { @@ -369,129 +363,11 @@ export class NlpSampleController extends BaseController< return deleteResult; } - /** - * Imports NLP samples from a CSV file. - * - * @param file - The file path or ID of the CSV file to import. - * - * @returns A success message after the import process is completed. - */ @CsrfCheck(true) - @Post('import/:file') - async import( - @Param('file') - file: string, - ) { - // Check if file is present - const importedFile = await this.attachmentService.findOne(file); - if (!importedFile) { - throw new NotFoundException('Missing file!'); - } - const filePath = importedFile - ? join(config.parameters.uploadDir, importedFile.location) - : undefined; - - // Check if file location is present - if (!fs.existsSync(filePath)) { - throw new NotFoundException('File does not exist'); - } - - const allEntities = await this.nlpEntityService.findAll(); - - // Check if file location is present - if (allEntities.length === 0) { - throw new NotFoundException( - 'No entities found, please create them first.', - ); - } - - // Read file content - const data = fs.readFileSync(filePath, 'utf8'); - - // Parse local CSV file - const result: { - errors: any[]; - data: Array>; - } = Papa.parse(data, { - header: true, - skipEmptyLines: true, - }); - - if (result.errors && result.errors.length > 0) { - this.logger.warn( - `Errors parsing the file: ${JSON.stringify(result.errors)}`, - ); - throw new BadRequestException(result.errors, { - cause: result.errors, - description: 'Error while parsing CSV', - }); - } - // Remove data with no intent - const filteredData = result.data.filter((d) => d.intent !== 'none'); - const languages = await this.languageService.getLanguages(); - const defaultLanguage = await this.languageService.getDefaultLanguage(); - // Reduce function to ensure executing promises one by one - for (const d of filteredData) { - try { - // Check if a sample with the same text already exists - const existingSamples = await this.nlpSampleService.find({ - text: d.text, - }); - - // Skip if sample already exists - if (Array.isArray(existingSamples) && existingSamples.length > 0) { - continue; - } - - // Fallback to default language if 'language' is missing or invalid - if (!d.language || !(d.language in languages)) { - if (d.language) { - this.logger.warn( - `Language "${d.language}" does not exist, falling back to default.`, - ); - } - d.language = defaultLanguage.code; - } - - // Create a new sample dto - const sample: NlpSampleCreateDto = { - text: d.text, - trained: false, - language: languages[d.language].id, - }; - - // Create a new sample entity dto - const entities: NlpSampleEntityValue[] = allEntities - .filter(({ name }) => name in d) - .map(({ name }) => { - return { - entity: name, - value: d[name], - }; - }); - - // Store any new entity/value - const storedEntities = await this.nlpEntityService.storeNewEntities( - sample.text, - entities, - ['trait'], - ); - // Store sample - const createdSample = await this.nlpSampleService.create(sample); - // Map and assign the sample ID to each stored entity - const sampleEntities = storedEntities.map((se) => ({ - ...se, - sample: createdSample?.id, - })); - - // Store sample entities - await this.nlpSampleEntityService.createMany(sampleEntities); - } catch (err) { - this.logger.error('Error occurred when extracting data. ', err); - } - } - - this.logger.log('Import process completed successfully.'); - return { success: true }; + @Post('import') + @UseInterceptors(FileInterceptor('file')) + async importFile(@UploadedFile() file: Express.Multer.File) { + const datasetContent = file.buffer.toString('utf-8'); + return await this.nlpSampleService.parseAndSaveDataset(datasetContent); } } diff --git a/api/src/nlp/services/nlp-sample.service.spec.ts b/api/src/nlp/services/nlp-sample.service.spec.ts index 0a95e92b..3b62973b 100644 --- a/api/src/nlp/services/nlp-sample.service.spec.ts +++ b/api/src/nlp/services/nlp-sample.service.spec.ts @@ -7,6 +7,7 @@ */ import { CACHE_MANAGER } from '@nestjs/cache-manager'; +import { BadRequestException, NotFoundException } from '@nestjs/common'; import { EventEmitter2 } from '@nestjs/event-emitter'; import { MongooseModule } from '@nestjs/mongoose'; import { Test, TestingModule } from '@nestjs/testing'; @@ -27,7 +28,7 @@ import { NlpEntityRepository } from '../repositories/nlp-entity.repository'; import { NlpSampleEntityRepository } from '../repositories/nlp-sample-entity.repository'; import { NlpSampleRepository } from '../repositories/nlp-sample.repository'; import { NlpValueRepository } from '../repositories/nlp-value.repository'; -import { NlpEntityModel } from '../schemas/nlp-entity.schema'; +import { NlpEntity, NlpEntityModel } from '../schemas/nlp-entity.schema'; import { NlpSampleEntity, NlpSampleEntityModel, @@ -41,7 +42,10 @@ import { NlpSampleService } from './nlp-sample.service'; import { NlpValueService } from './nlp-value.service'; describe('NlpSampleService', () => { + let nlpEntityService: NlpEntityService; let nlpSampleService: NlpSampleService; + let nlpSampleEntityService: NlpSampleEntityService; + let languageService: LanguageService; let nlpSampleEntityRepository: NlpSampleEntityRepository; let nlpSampleRepository: NlpSampleRepository; let languageRepository: LanguageRepository; @@ -84,7 +88,11 @@ describe('NlpSampleService', () => { }, ], }).compile(); + nlpEntityService = module.get(NlpEntityService); nlpSampleService = module.get(NlpSampleService); + nlpSampleEntityService = module.get( + NlpSampleEntityService, + ); nlpSampleRepository = module.get(NlpSampleRepository); nlpSampleEntityRepository = module.get( NlpSampleEntityRepository, @@ -92,6 +100,7 @@ describe('NlpSampleService', () => { nlpSampleEntityRepository = module.get( NlpSampleEntityRepository, ); + languageService = module.get(LanguageService); languageRepository = module.get(LanguageRepository); noNlpSample = await nlpSampleService.findOne({ text: 'No' }); nlpSampleEntity = await nlpSampleEntityRepository.findOne({ @@ -162,4 +171,104 @@ describe('NlpSampleService', () => { expect(result.deletedCount).toEqual(1); }); }); + + describe('parseAndSaveDataset', () => { + it('should throw NotFoundException if no entities are found', async () => { + jest.spyOn(nlpEntityService, 'findAll').mockResolvedValue([]); + + await expect( + nlpSampleService.parseAndSaveDataset( + 'text,intent,language\nHello,none,en', + ), + ).rejects.toThrow(NotFoundException); + + expect(nlpEntityService.findAll).toHaveBeenCalled(); + }); + + it('should throw BadRequestException if CSV parsing fails', async () => { + const invalidCSV = 'text,intent,language\n"Hello,none'; // Malformed CSV + jest + .spyOn(nlpEntityService, 'findAll') + .mockResolvedValue([{ name: 'intent' } as NlpEntity]); + jest.spyOn(languageService, 'getLanguages').mockResolvedValue({}); + jest + .spyOn(languageService, 'getDefaultLanguage') + .mockResolvedValue({ code: 'en' } as Language); + + await expect( + nlpSampleService.parseAndSaveDataset(invalidCSV), + ).rejects.toThrow(BadRequestException); + }); + + it('should filter out rows with "none" as intent', async () => { + const mockData = 'text,intent,language\nHello,none,en\nHi,greet,en'; + jest + .spyOn(nlpEntityService, 'findAll') + .mockResolvedValue([{ name: 'intent' } as NlpEntity]); + jest + .spyOn(languageService, 'getLanguages') + .mockResolvedValue({ en: { id: '1' } }); + jest + .spyOn(languageService, 'getDefaultLanguage') + .mockResolvedValue({ code: 'en' } as Language); + jest.spyOn(nlpSampleService, 'find').mockResolvedValue([]); + jest + .spyOn(nlpSampleService, 'create') + .mockResolvedValue({ id: '1', text: 'Hi' } as NlpSample); + jest.spyOn(nlpSampleEntityService, 'createMany').mockResolvedValue([]); + + const result = await nlpSampleService.parseAndSaveDataset(mockData); + + expect(result).toHaveLength(1); + expect(result[0].text).toEqual('Hi'); + }); + + it('should fallback to the default language if the language is invalid', async () => { + const mockData = 'text,intent,language\nHi,greet,invalidLang'; + jest + .spyOn(nlpEntityService, 'findAll') + .mockResolvedValue([{ name: 'intent' } as NlpEntity]); + jest + .spyOn(languageService, 'getLanguages') + .mockResolvedValue({ en: { id: '1' } }); + jest + .spyOn(languageService, 'getDefaultLanguage') + .mockResolvedValue({ code: 'en' } as Language); + jest.spyOn(nlpSampleService, 'find').mockResolvedValue([]); + jest + .spyOn(nlpSampleService, 'create') + .mockResolvedValue({ id: '1', text: 'Hi' } as NlpSample); + jest.spyOn(nlpSampleEntityService, 'createMany').mockResolvedValue([]); + + const result = await nlpSampleService.parseAndSaveDataset(mockData); + + expect(result).toHaveLength(1); + expect(result[0].text).toEqual('Hi'); + }); + + it('should successfully process and save valid dataset rows', async () => { + const mockData = 'text,intent,language\nHi,greet,en\nBye,bye,en'; + const mockLanguages = { en: { id: '1' } }; + + jest + .spyOn(languageService, 'getLanguages') + .mockResolvedValue(mockLanguages); + jest + .spyOn(languageService, 'getDefaultLanguage') + .mockResolvedValue({ code: 'en' } as Language); + jest.spyOn(nlpSampleService, 'find').mockResolvedValue([]); + let id = 0; + jest.spyOn(nlpSampleService, 'create').mockImplementation((s) => { + return Promise.resolve({ id: (++id).toString(), ...s } as NlpSample); + }); + jest.spyOn(nlpSampleEntityService, 'createMany').mockResolvedValue([]); + + const result = await nlpSampleService.parseAndSaveDataset(mockData); + + expect(nlpSampleEntityService.createMany).toHaveBeenCalledTimes(2); + expect(result).toHaveLength(2); + expect(result[0].text).toEqual('Hi'); + expect(result[1].text).toEqual('Bye'); + }); + }); }); diff --git a/api/src/nlp/services/nlp-sample.service.ts b/api/src/nlp/services/nlp-sample.service.ts index 17e49034..ca7384b1 100644 --- a/api/src/nlp/services/nlp-sample.service.ts +++ b/api/src/nlp/services/nlp-sample.service.ts @@ -6,8 +6,13 @@ * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file). */ -import { Injectable } from '@nestjs/common'; +import { + BadRequestException, + Injectable, + NotFoundException, +} from '@nestjs/common'; import { OnEvent } from '@nestjs/event-emitter'; +import Papa from 'papaparse'; import { Message } from '@/chat/schemas/message.schema'; import { Language } from '@/i18n/schemas/language.schema'; @@ -23,7 +28,10 @@ import { NlpSampleFull, NlpSamplePopulate, } from '../schemas/nlp-sample.schema'; -import { NlpSampleState } from '../schemas/types'; +import { NlpSampleEntityValue, NlpSampleState } from '../schemas/types'; + +import { NlpEntityService } from './nlp-entity.service'; +import { NlpSampleEntityService } from './nlp-sample-entity.service'; @Injectable() export class NlpSampleService extends BaseService< @@ -33,6 +41,8 @@ export class NlpSampleService extends BaseService< > { constructor( readonly repository: NlpSampleRepository, + private readonly nlpSampleEntityService: NlpSampleEntityService, + private readonly nlpEntityService: NlpEntityService, private readonly languageService: LanguageService, private readonly logger: LoggerService, ) { @@ -50,6 +60,110 @@ export class NlpSampleService extends BaseService< return await this.repository.deleteOne(id); } + /** + * This function is responsible for parsing a CSV dataset string and saving the parsed data into the database. + * It ensures that all necessary entities and languages exist, validates the dataset, and processes it row by row + * to create NLP samples and associated entities in the system. + * + * @param data - The raw CSV dataset as a string. + * @returns A promise that resolves to an array of created NLP samples. + */ + async parseAndSaveDataset(data: string) { + const allEntities = await this.nlpEntityService.findAll(); + // Check if file location is present + if (allEntities.length === 0) { + throw new NotFoundException( + 'No entities found, please create them first.', + ); + } + + // Parse local CSV file + const result: { + errors: any[]; + data: Array>; + } = Papa.parse(data, { + header: true, + skipEmptyLines: true, + }); + + if (result.errors && result.errors.length > 0) { + this.logger.warn( + `Errors parsing the file: ${JSON.stringify(result.errors)}`, + ); + throw new BadRequestException(result.errors, { + cause: result.errors, + description: 'Error while parsing CSV', + }); + } + // Remove data with no intent + const filteredData = result.data.filter((d) => d.intent !== 'none'); + const languages = await this.languageService.getLanguages(); + const defaultLanguage = await this.languageService.getDefaultLanguage(); + const nlpSamples: NlpSample[] = []; + // Reduce function to ensure executing promises one by one + for (const d of filteredData) { + try { + // Check if a sample with the same text already exists + const existingSamples = await this.find({ + text: d.text, + }); + + // Skip if sample already exists + if (Array.isArray(existingSamples) && existingSamples.length > 0) { + continue; + } + + // Fallback to default language if 'language' is missing or invalid + if (!d.language || !(d.language in languages)) { + if (d.language) { + this.logger.warn( + `Language "${d.language}" does not exist, falling back to default.`, + ); + } + d.language = defaultLanguage.code; + } + + // Create a new sample dto + const sample: NlpSampleCreateDto = { + text: d.text, + trained: false, + language: languages[d.language].id, + }; + + // Create a new sample entity dto + const entities: NlpSampleEntityValue[] = allEntities + .filter(({ name }) => name in d) + .map(({ name }) => ({ + entity: name, + value: d[name], + })); + + // Store any new entity/value + const storedEntities = await this.nlpEntityService.storeNewEntities( + sample.text, + entities, + ['trait'], + ); + + // Store sample + const createdSample = await this.create(sample); + nlpSamples.push(createdSample); + // Map and assign the sample ID to each stored entity + const sampleEntities = storedEntities.map((storedEntity) => ({ + ...storedEntity, + sample: createdSample?.id, + })); + + // Store sample entities + await this.nlpSampleEntityService.createMany(sampleEntities); + } catch (err) { + this.logger.error('Error occurred when extracting data. ', err); + } + } + + return nlpSamples; + } + /** * When a language gets deleted, we need to set related samples to null * diff --git a/frontend/public/locales/en/translation.json b/frontend/public/locales/en/translation.json index 780bb87c..2ee95216 100644 --- a/frontend/public/locales/en/translation.json +++ b/frontend/public/locales/en/translation.json @@ -58,8 +58,8 @@ "custom_code_is_invalid": "Custom code seems to contain some errors.", "attachment_failure_format": "Attachment has invalid format", "drop_file_here": "Drop file here or click to upload", - "file_max_size": "File must have a size less than 25MB", - "attachment_failure_size": "Invalid size! File must have a size less than 25MB", + "file_max_size": "The file exceeds the maximum allowed size. Please ensure your file is within the size limit and try again.", + "attachment_failure_size": "The file exceeds the maximum allowed size. Please ensure your file is within the size limit and try again.", "upload_failed": "Unable to upload the file!", "value_is_required": "NLU Value is required", "nlp_entity_name_is_invalid": "NLU Entity name format is invalid! Only `A-z`, `0-9` and `_` are allowed.", @@ -81,7 +81,9 @@ "subtitle_is_required": "Subtitle is required", "category_is_required": "Flow is required", "attachment_is_required": "Attachment is required", - "success_import": "Content has been successfuly imported!", + "success_import": "Content has been successfully imported!", + "import_failed": "Import failed", + "import_duplicated_data": "Data already exists", "attachment_not_synced": "- Pending Sync. -", "success_translation_refresh": "Translations has been successfully refreshed!", "message_tag_is_required": "You need to specify a message tag.", @@ -106,7 +108,7 @@ "no_label_found": "No label found", "code_is_required": "Language code is required", "text_is_required": "Text is required", - "invalid_file_type": "Invalid file type", + "invalid_file_type": "Invalid file type. Please select a file in the supported format.", "select_category": "Select a flow" }, "menu": { @@ -341,6 +343,7 @@ "precision": "Precision", "recall": "Recall", "f1score": "F1 Score", + "all": "All", "train": "Train", "test": "Test", "inbox": "Inbox", diff --git a/frontend/public/locales/fr/translation.json b/frontend/public/locales/fr/translation.json index efc23e5e..51ee928a 100644 --- a/frontend/public/locales/fr/translation.json +++ b/frontend/public/locales/fr/translation.json @@ -58,8 +58,8 @@ "custom_code_is_invalid": "Le code personnalisé semble contenir quelques erreurs.", "attachment_failure_format": "La pièce jointe a un format invalide", "drop_file_here": "Déposez le fichier ici ou cliquez pour télécharger", - "file_max_size": "Le fichier doit avoir une taille inférieure à 25 Mo", - "attachment_failure_size": "Taille invalide! Le fichier doit avoir une taille inférieure à 25 Mo", + "file_max_size": "Le fichier dépasse la taille maximale autorisée. Veuillez vérifier que votre fichier respecte la limite de taille et réessayez.", + "attachment_failure_size": "Le fichier dépasse la taille maximale autorisée. Veuillez vérifier que votre fichier respecte la limite de taille et réessayez.", "upload_failed": "Impossible d'envoyer le fichier au serveur!", "value_is_required": "La valeur NLU est requise", "nlp_entity_name_is_invalid": "Le nom d'entité NLU n'est pas valide! Seuls `A-z`,` 0-9` et `_` sont autorisés.", @@ -83,6 +83,8 @@ "category_is_required": "La catégorie est requise", "attachment_is_required": "L'attachement est obligatoire", "success_import": "Le contenu a été importé avec succès!", + "import_failed": "Échec de l'importation", + "import_duplicated_data": "Les données existent déjà", "attachment_not_synced": "- En attente de Sync. -", "success_translation_refresh": "Les traductions ont été actualisées avec succès!", "message_tag_is_required": "Vous devez spécifier le tag de message.", @@ -106,7 +108,7 @@ "no_label_found": "Aucune étiquette trouvée", "code_is_required": "Le code est requis", "text_is_required": "Texte requis", - "invalid_file_type": "Type de fichier invalide", + "invalid_file_type": "Type de fichier invalide. Veuillez choisir un fichier dans un format pris en charge.", "select_category": "Sélectionner une catégorie" }, "menu": { @@ -341,6 +343,7 @@ "precision": "Précision", "recall": "Rappel", "f1score": "F1-Score", + "all": "Tout", "train": "Apprentissage", "test": "Evaluation", "inbox": "Boîte de réception", diff --git a/frontend/src/app-components/inputs/FileInput.tsx b/frontend/src/app-components/inputs/FileInput.tsx new file mode 100644 index 00000000..f6d52d58 --- /dev/null +++ b/frontend/src/app-components/inputs/FileInput.tsx @@ -0,0 +1,82 @@ +/* + * Copyright © 2024 Hexastack. All rights reserved. + * + * Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms: + * 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission. + * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file). + */ + +import UploadIcon from "@mui/icons-material/Upload"; +import { Button, CircularProgress } from "@mui/material"; +import { ChangeEvent, forwardRef } from "react"; + +import { useConfig } from "@/hooks/useConfig"; +import { useToast } from "@/hooks/useToast"; +import { useTranslate } from "@/hooks/useTranslate"; + +import { Input } from "./Input"; + +export type FileUploadButtonProps = { + label: string; + accept?: string; + onChange: (file: File) => void; + isLoading?: boolean; + error?: boolean; + helperText?: string; +}; + +const FileUploadButton = forwardRef( + ({ label, accept, isLoading = true, onChange }, ref) => { + const config = useConfig(); + const { toast } = useToast(); + const { t } = useTranslate(); + const handleImportChange = async (event: ChangeEvent) => { + if (event.target.files?.length) { + const file = event.target.files.item(0); + + if (!file) return false; + + if (accept && !accept.split(",").includes(file.type)) { + toast.error(t("message.invalid_file_type")); + + return false; + } + + if (config.maxUploadSize && file.size > config.maxUploadSize) { + toast.error(t("message.file_max_size")); + + return false; + } + + onChange(file); + } + }; + + return ( + <> + + + + ); + }, +); + +FileUploadButton.displayName = "FileUploadButton"; + +export default FileUploadButton; diff --git a/frontend/src/components/nlp/components/NlpSample.tsx b/frontend/src/components/nlp/components/NlpSample.tsx index 564c6c28..201ef022 100644 --- a/frontend/src/components/nlp/components/NlpSample.tsx +++ b/frontend/src/components/nlp/components/NlpSample.tsx @@ -7,9 +7,9 @@ */ import CircleIcon from "@mui/icons-material/Circle"; +import ClearIcon from "@mui/icons-material/Clear"; import DeleteIcon from "@mui/icons-material/Delete"; import DownloadIcon from "@mui/icons-material/Download"; -import UploadIcon from "@mui/icons-material/Upload"; import { Box, Button, @@ -17,15 +17,19 @@ import { Chip, Grid, IconButton, + InputAdornment, MenuItem, Stack, + Typography, } from "@mui/material"; import { GridColDef, GridRowSelectionModel } from "@mui/x-data-grid"; import { useState } from "react"; +import { useQueryClient } from "react-query"; import { DeleteDialog } from "@/app-components/dialogs"; import { ChipEntity } from "@/app-components/displays/ChipEntity"; import AutoCompleteEntitySelect from "@/app-components/inputs/AutoCompleteEntitySelect"; +import FileUploadButton from "@/app-components/inputs/FileInput"; import { FilterTextfield } from "@/app-components/inputs/FilterTextfield"; import { Input } from "@/app-components/inputs/Input"; import { @@ -34,10 +38,12 @@ import { } from "@/app-components/tables/columns/getColumns"; import { renderHeader } from "@/app-components/tables/columns/renderHeader"; import { DataGrid } from "@/app-components/tables/DataGrid"; +import { isSameEntity } from "@/hooks/crud/helpers"; import { useDelete } from "@/hooks/crud/useDelete"; import { useDeleteMany } from "@/hooks/crud/useDeleteMany"; import { useFind } from "@/hooks/crud/useFind"; import { useGetFromCache } from "@/hooks/crud/useGet"; +import { useImport } from "@/hooks/crud/useImport"; import { useConfig } from "@/hooks/useConfig"; import { getDisplayDialogs, useDialog } from "@/hooks/useDialog"; import { useHasPermission } from "@/hooks/useHasPermission"; @@ -60,6 +66,7 @@ import { NlpImportDialog } from "../NlpImportDialog"; import { NlpSampleDialog } from "../NlpSampleDialog"; const NLP_SAMPLE_TYPE_COLORS = { + all: "#fff", test: "#e6a23c", train: "#67c23a", inbox: "#909399", @@ -69,7 +76,8 @@ export default function NlpSample() { const { apiUrl } = useConfig(); const { toast } = useToast(); const { t } = useTranslate(); - const [type, setType] = useState(undefined); + const queryClient = useQueryClient(); + const [type, setType] = useState("all"); const [language, setLanguage] = useState(undefined); const hasPermission = useHasPermission(); const getNlpEntityFromCache = useGetFromCache(EntityType.NLP_ENTITY); @@ -79,7 +87,10 @@ export default function NlpSample() { ); const getLanguageFromCache = useGetFromCache(EntityType.LANGUAGE); const { onSearch, searchPayload } = useSearch({ - $eq: [...(type ? [{ type }] : []), ...(language ? [{ language }] : [])], + $eq: [ + ...(type !== "all" ? [{ type }] : []), + ...(language ? [{ language }] : []), + ], $iLike: ["text"], }); const { mutateAsync: deleteNlpSample } = useDelete(EntityType.NLP_SAMPLE, { @@ -104,6 +115,32 @@ export default function NlpSample() { }, }, ); + const { mutateAsync: importDataset, isLoading } = useImport( + EntityType.NLP_SAMPLE, + { + onError: () => { + toast.error(t("message.import_failed")); + }, + onSuccess: (data) => { + queryClient.removeQueries({ + predicate: ({ queryKey }) => { + const [_qType, qEntity] = queryKey; + + return ( + isSameEntity(qEntity, EntityType.NLP_SAMPLE_ENTITY) || + isSameEntity(qEntity, EntityType.NLP_ENTITY) || + isSameEntity(qEntity, EntityType.NLP_VALUE) + ); + }, + }); + if (data.length) { + toast.success(t("message.success_import")); + } else { + toast.error(t("message.import_duplicated_data")); + } + }, + }, + ); const [selectedNlpSamples, setSelectedNlpSamples] = useState([]); const { dataGridProps } = useFind( { entity: EntityType.NLP_SAMPLE, format: Format.FULL }, @@ -259,6 +296,9 @@ export default function NlpSample() { const handleSelectionChange = (selection: GridRowSelectionModel) => { setSelectedNlpSamples(selection as string[]); }; + const handleImportChange = async (file: File) => { + await importDataset(file); + }; return ( @@ -292,7 +332,7 @@ export default function NlpSample() { fullWidth={false} sx={{ - minWidth: "150px", + minWidth: "256px", }} autoFocus searchFields={["title", "code"]} @@ -307,35 +347,38 @@ export default function NlpSample() { select fullWidth={false} sx={{ - minWidth: "150px", + minWidth: "256px", }} label={t("label.dataset")} value={type} onChange={(e) => setType(e.target.value as NlpSampleType)} SelectProps={{ ...(type && { - IconComponent: () => ( - setType(undefined)}> - - + endAdornment: ( + + setType("all")}> + + + ), }), renderValue: (value) => {t(`label.${value}`)}, }} > - {Object.values(NlpSampleType).map((nlpSampleType, index) => ( - - - + {["all", ...Object.values(NlpSampleType)].map( + (nlpSampleType, index) => ( + + - - {nlpSampleType} - - - ))} + {t(`label.${nlpSampleType}`)} + + + ), + )} {hasPermission(EntityType.NLP_SAMPLE, PermissionAction.CREATE) && @@ -343,13 +386,12 @@ export default function NlpSample() { EntityType.NLP_SAMPLE_ENTITY, PermissionAction.CREATE, ) ? ( - + ) : null} {hasPermission(EntityType.NLP_SAMPLE, PermissionAction.READ) && hasPermission( diff --git a/frontend/src/contexts/config.context.tsx b/frontend/src/contexts/config.context.tsx index c18f0198..55172f92 100644 --- a/frontend/src/contexts/config.context.tsx +++ b/frontend/src/contexts/config.context.tsx @@ -13,6 +13,7 @@ export const ConfigContext = createContext(null); export interface IConfig { apiUrl: string; ssoEnabled: boolean; + maxUploadSize: number; } export const ConfigProvider = ({ children }) => { diff --git a/frontend/src/hooks/crud/useImport.tsx b/frontend/src/hooks/crud/useImport.tsx new file mode 100644 index 00000000..4444362a --- /dev/null +++ b/frontend/src/hooks/crud/useImport.tsx @@ -0,0 +1,59 @@ +/* + * Copyright © 2024 Hexastack. All rights reserved. + * + * Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms: + * 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission. + * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file). + */ + +import { useMutation, useQueryClient } from "react-query"; + +import { QueryType, TMutationOptions } from "@/services/types"; +import { IBaseSchema, IDynamicProps, TType } from "@/types/base.types"; + +import { useEntityApiClient } from "../useApiClient"; + +import { isSameEntity, useNormalizeAndCache } from "./helpers"; + +export const useImport = < + TEntity extends IDynamicProps["entity"], + TAttr extends File = File, + TBasic extends IBaseSchema = TType["basic"], +>( + entity: TEntity, + options: Omit< + TMutationOptions, + "mutationFn" | "mutationKey" + > = {}, +) => { + const api = useEntityApiClient(entity); + const queryClient = useQueryClient(); + const normalizeAndCache = useNormalizeAndCache( + entity, + ); + const { invalidate = true, ...rest } = options; + + return useMutation({ + mutationFn: async (variables) => { + const data = await api.import(variables); + const { result, entities } = normalizeAndCache(data); + + // Invalidate current entity count and collection + if (invalidate) { + queryClient.invalidateQueries({ + predicate: ({ queryKey }) => { + const [qType, qEntity] = queryKey; + + return ( + (qType === QueryType.count || qType === QueryType.collection) && + isSameEntity(qEntity, entity) + ); + }, + }); + } + + return result.map((id) => entities[entity][id]); + }, + ...rest, + }); +}; diff --git a/frontend/src/pages/api/config.ts b/frontend/src/pages/api/config.ts index 3f872b46..ce57a910 100644 --- a/frontend/src/pages/api/config.ts +++ b/frontend/src/pages/api/config.ts @@ -11,6 +11,7 @@ import type { NextApiRequest, NextApiResponse } from "next"; type ResponseData = { apiUrl: string; ssoEnabled: boolean; + maxUploadSize: number; }; export default function handler( @@ -20,5 +21,8 @@ export default function handler( res.status(200).json({ apiUrl: process.env.NEXT_PUBLIC_API_ORIGIN || "http://localhost:4000", ssoEnabled: process.env.NEXT_PUBLIC_SSO_ENABLED === "true" || false, + maxUploadSize: process.env.UPLOAD_MAX_SIZE_IN_BYTES + ? Number(process.env.UPLOAD_MAX_SIZE_IN_BYTES) + : 50 * 1024 * 1024, // 50 MB in bytes }); } diff --git a/frontend/src/services/api.class.ts b/frontend/src/services/api.class.ts index a29e824e..b6370c4b 100644 --- a/frontend/src/services/api.class.ts +++ b/frontend/src/services/api.class.ts @@ -269,6 +269,20 @@ export class EntityApiClient extends ApiClient { return data; } + async import(file: File) { + const { _csrf } = await this.getCsrf(); + const formData = new FormData(); + + formData.append("file", file); + + const { data } = await this.request.post, FormData>( + `${ROUTES[this.type]}/import?_csrf=${_csrf}`, + formData, + ); + + return data; + } + async upload(file: File) { const { _csrf } = await this.getCsrf(); const formData = new FormData();