feat: implement QuestionBank CRUD with pagination and template query
- Add pagination support to findAll (page, limit query params) - Add findByTemplateId method to service - Add GET /by-template/:templateId endpoint to controller - Service already includes CRUD for QuestionBank and QuestionBankItem
This commit is contained in:
Developer
@@ -0,0 +1,57 @@
|
||||
import {
|
||||
Controller,
|
||||
Post,
|
||||
Get,
|
||||
Delete,
|
||||
Param,
|
||||
Body,
|
||||
Request,
|
||||
UseGuards,
|
||||
Query,
|
||||
} from '@nestjs/common';
|
||||
import { ImportTaskService } from './import-task.service';
|
||||
import { CombinedAuthGuard } from '../auth/combined-auth.guard';
|
||||
import { RolesGuard } from '../auth/roles.guard';
|
||||
import { Roles } from '../auth/roles.decorator';
|
||||
import { UserRole } from '../user/user-role.enum';
|
||||
|
||||
@Controller('import-tasks')
|
||||
@UseGuards(CombinedAuthGuard, RolesGuard)
|
||||
export class ImportTaskController {
|
||||
constructor(private readonly taskService: ImportTaskService) {}
|
||||
|
||||
@Post()
|
||||
@Roles(UserRole.TENANT_ADMIN, UserRole.SUPER_ADMIN)
|
||||
async create(@Request() req, @Body() body: any) {
|
||||
return this.taskService.create({
|
||||
sourcePath: body.sourcePath,
|
||||
targetGroupId: body.targetGroupId,
|
||||
targetGroupName: body.targetGroupName,
|
||||
embeddingModelId: body.embeddingModelId,
|
||||
scheduledAt: body.scheduledAt ? new Date(body.scheduledAt) : undefined,
|
||||
chunkSize: body.chunkSize,
|
||||
chunkOverlap: body.chunkOverlap,
|
||||
mode: body.mode,
|
||||
useHierarchy: body.useHierarchy ?? false,
|
||||
userId: req.user.id,
|
||||
tenantId: req.user.tenantId,
|
||||
});
|
||||
}
|
||||
|
||||
@Get()
|
||||
async findAll(
|
||||
@Request() req,
|
||||
@Query('page') page?: number,
|
||||
@Query('limit') limit?: number,
|
||||
) {
|
||||
return this.taskService.findAll(req.user.id, {
|
||||
page: page ? Number(page) : undefined,
|
||||
limit: limit ? Number(limit) : undefined,
|
||||
});
|
||||
}
|
||||
|
||||
@Delete(':id')
|
||||
async delete(@Param('id') id: string, @Request() req) {
|
||||
return this.taskService.delete(id, req.user.id);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
import {
|
||||
Entity,
|
||||
Column,
|
||||
PrimaryGeneratedColumn,
|
||||
CreateDateColumn,
|
||||
UpdateDateColumn,
|
||||
} from 'typeorm';
|
||||
|
||||
@Entity()
|
||||
export class ImportTask {
|
||||
@PrimaryGeneratedColumn('uuid')
|
||||
id: string;
|
||||
|
||||
@Column()
|
||||
sourcePath: string;
|
||||
|
||||
@Column({ nullable: true })
|
||||
targetGroupId: string; // If null, creates new group
|
||||
|
||||
@Column({ nullable: true })
|
||||
targetGroupName: string; // Used if creating new group
|
||||
|
||||
@Column()
|
||||
userId: string;
|
||||
|
||||
@Column({ nullable: true })
|
||||
tenantId: string;
|
||||
|
||||
@Column({ nullable: true })
|
||||
scheduledAt: Date;
|
||||
|
||||
@Column({ default: 'PENDING' })
|
||||
status: 'PENDING' | 'PROCESSING' | 'COMPLETED' | 'FAILED';
|
||||
|
||||
@Column({ type: 'text', nullable: true })
|
||||
logs: string;
|
||||
|
||||
@Column({ nullable: true })
|
||||
embeddingModelId: string;
|
||||
|
||||
@Column({ nullable: true, default: 500 })
|
||||
chunkSize: number;
|
||||
|
||||
@Column({ nullable: true, default: 50 })
|
||||
chunkOverlap: number;
|
||||
|
||||
@Column({ nullable: true, default: 'fast' })
|
||||
mode: string;
|
||||
|
||||
/** When true, sub-directories become sub-categories mirroring the folder hierarchy */
|
||||
@Column({ default: false })
|
||||
useHierarchy: boolean;
|
||||
|
||||
@CreateDateColumn()
|
||||
createdAt: Date;
|
||||
|
||||
@UpdateDateColumn()
|
||||
updatedAt: Date;
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
import { ImportTask } from './import-task.entity';
|
||||
import { ImportTaskService } from './import-task.service';
|
||||
import { ImportTaskController } from './import-task.controller';
|
||||
import { KnowledgeBaseModule } from '../knowledge-base/knowledge-base.module';
|
||||
import { KnowledgeGroupModule } from '../knowledge-group/knowledge-group.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
TypeOrmModule.forFeature([ImportTask]),
|
||||
KnowledgeBaseModule,
|
||||
KnowledgeGroupModule,
|
||||
],
|
||||
controllers: [ImportTaskController],
|
||||
providers: [ImportTaskService],
|
||||
})
|
||||
export class ImportTaskModule {}
|
||||
@@ -0,0 +1,422 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Repository, LessThanOrEqual } from 'typeorm';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { ImportTask } from './import-task.entity';
|
||||
import { Cron, CronExpression } from '@nestjs/schedule';
|
||||
import { KnowledgeBaseService } from '../knowledge-base/knowledge-base.service';
|
||||
import { KnowledgeGroupService } from '../knowledge-group/knowledge-group.service';
|
||||
import { I18nService } from '../i18n/i18n.service';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
export interface PaginatedImportTasks {
|
||||
items: ImportTask[];
|
||||
total: number;
|
||||
page: number;
|
||||
limit: number;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class ImportTaskService {
|
||||
private readonly logger = new Logger(ImportTaskService.name);
|
||||
|
||||
constructor(
|
||||
@InjectRepository(ImportTask)
|
||||
private taskRepository: Repository<ImportTask>,
|
||||
private kbService: KnowledgeBaseService,
|
||||
private groupService: KnowledgeGroupService,
|
||||
private configService: ConfigService,
|
||||
private i18nService: I18nService,
|
||||
) {}
|
||||
|
||||
async create(taskData: Partial<ImportTask>): Promise<ImportTask> {
|
||||
const task = this.taskRepository.create(taskData);
|
||||
const savedTask = await this.taskRepository.save(task);
|
||||
|
||||
// If no scheduled time or scheduled time is in the past, execute immediately (async)
|
||||
if (!task.scheduledAt || task.scheduledAt <= new Date()) {
|
||||
this.executeTask(savedTask.id).catch((err) =>
|
||||
this.logger.error(
|
||||
`Immediate execution failed to start for task ${savedTask.id}`,
|
||||
err,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
return savedTask;
|
||||
}
|
||||
|
||||
async findAll(
|
||||
userId: string,
|
||||
options: { page?: number; limit?: number } = {},
|
||||
): Promise<PaginatedImportTasks> {
|
||||
const { page = 1, limit = 12 } = options;
|
||||
const skip = (page - 1) * limit;
|
||||
|
||||
const [items, total] = await this.taskRepository.findAndCount({
|
||||
where: { userId },
|
||||
order: { createdAt: 'DESC' },
|
||||
skip,
|
||||
take: limit,
|
||||
});
|
||||
|
||||
return {
|
||||
items,
|
||||
total,
|
||||
page,
|
||||
limit,
|
||||
};
|
||||
}
|
||||
|
||||
async delete(taskId: string, userId: string): Promise<void> {
|
||||
const task = await this.taskRepository.findOne({
|
||||
where: { id: taskId, userId },
|
||||
});
|
||||
if (!task) {
|
||||
throw new Error(this.i18nService.getMessage('importTaskNotFound'));
|
||||
}
|
||||
await this.taskRepository.remove(task);
|
||||
}
|
||||
|
||||
@Cron(CronExpression.EVERY_MINUTE)
|
||||
async handleScheduledTasks() {
|
||||
this.logger.debug('Checking for scheduled import tasks...');
|
||||
const now = new Date();
|
||||
|
||||
const tasks = await this.taskRepository.find({
|
||||
where: {
|
||||
status: 'PENDING',
|
||||
scheduledAt: LessThanOrEqual(now),
|
||||
},
|
||||
});
|
||||
|
||||
for (const task of tasks) {
|
||||
this.logger.log(`Starting scheduled task ${task.id}`);
|
||||
this.executeTask(task.id).catch((err) =>
|
||||
this.logger.error(
|
||||
`Scheduled execution failed to start for task ${task.id}`,
|
||||
err,
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async executeTask(taskId: string) {
|
||||
this.logger.debug(`Executing task ${taskId}`);
|
||||
const task = await this.taskRepository.findOne({ where: { id: taskId } });
|
||||
if (!task) {
|
||||
this.logger.warn(`Task ${taskId} not found.`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (task.status === 'PROCESSING' || task.status === 'COMPLETED') {
|
||||
this.logger.debug(
|
||||
`Task ${taskId} is already ${task.status}, skipping execution.`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
await this.updateStatus(taskId, 'PROCESSING', 'Starting import...');
|
||||
|
||||
try {
|
||||
if (!fs.existsSync(task.sourcePath)) {
|
||||
throw new Error(
|
||||
this.i18nService.formatMessage('sourcePathNotFound', {
|
||||
path: task.sourcePath,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
const uploadPath = this.configService.get<string>(
|
||||
'UPLOAD_FILE_PATH',
|
||||
'./uploads',
|
||||
);
|
||||
const importTargetDir = path.join(uploadPath, 'imported', taskId);
|
||||
|
||||
if (!fs.existsSync(importTargetDir)) {
|
||||
fs.mkdirSync(importTargetDir, { recursive: true });
|
||||
}
|
||||
|
||||
let successCount = 0;
|
||||
let failCount = 0;
|
||||
|
||||
if (task.useHierarchy) {
|
||||
// ---- Hierarchy mode: create sub-groups matching folder structure ----
|
||||
await this.appendLog(
|
||||
taskId,
|
||||
`Scanning directory with hierarchy: ${task.sourcePath}`,
|
||||
);
|
||||
|
||||
// Determine root group
|
||||
let rootGroupId = task.targetGroupId;
|
||||
if (!rootGroupId) {
|
||||
const rootName =
|
||||
task.targetGroupName || path.basename(task.sourcePath);
|
||||
const rootGroup = await this.groupService.create(
|
||||
task.userId,
|
||||
task.tenantId || 'default',
|
||||
{
|
||||
name: rootName,
|
||||
description: `Imported from ${task.sourcePath}`,
|
||||
color: '#0078D4',
|
||||
},
|
||||
);
|
||||
rootGroupId = rootGroup.id;
|
||||
await this.appendLog(taskId, `Created root group: ${rootName}`);
|
||||
}
|
||||
|
||||
// Map from relative dir path -> groupId
|
||||
const dirToGroupId = new Map<string, string>();
|
||||
dirToGroupId.set('.', rootGroupId);
|
||||
|
||||
// Collect all files first
|
||||
const allFiles = this.scanDir(task.sourcePath);
|
||||
await this.appendLog(taskId, `Found ${allFiles.length} files.`);
|
||||
|
||||
for (let i = 0; i < allFiles.length; i++) {
|
||||
const filePath = allFiles[i];
|
||||
const relativeDir = path.relative(
|
||||
task.sourcePath,
|
||||
path.dirname(filePath),
|
||||
);
|
||||
const normalizedDir = relativeDir || '.';
|
||||
|
||||
// Ensure group exists for this directory
|
||||
const groupId = await this.ensureHierarchyGroup(
|
||||
task.userId,
|
||||
task.tenantId || 'default',
|
||||
normalizedDir,
|
||||
dirToGroupId,
|
||||
task.sourcePath,
|
||||
taskId,
|
||||
);
|
||||
|
||||
try {
|
||||
const kb = await this.importSingleFile(
|
||||
filePath,
|
||||
task,
|
||||
importTargetDir,
|
||||
i,
|
||||
allFiles.length,
|
||||
);
|
||||
await this.groupService.addFilesToGroup(
|
||||
kb.id,
|
||||
[groupId],
|
||||
task.userId,
|
||||
task.tenantId || 'default',
|
||||
);
|
||||
successCount++;
|
||||
if (successCount % 10 === 0) {
|
||||
await this.appendLog(taskId, `Imported ${successCount} files...`);
|
||||
}
|
||||
} catch (e) {
|
||||
failCount++;
|
||||
await this.appendLog(
|
||||
taskId,
|
||||
`Failed to import ${path.basename(filePath)}: ${e.message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// ---- Single-group mode (original behavior) ----
|
||||
let groupId = task.targetGroupId;
|
||||
if (!groupId && task.targetGroupName) {
|
||||
const group = await this.groupService.create(
|
||||
task.userId,
|
||||
task.tenantId || 'default',
|
||||
{
|
||||
name: task.targetGroupName,
|
||||
description: `Imported from ${task.sourcePath}`,
|
||||
color: '#0078D4',
|
||||
},
|
||||
);
|
||||
groupId = group.id;
|
||||
await this.appendLog(
|
||||
taskId,
|
||||
`Created new group: ${task.targetGroupName}`,
|
||||
);
|
||||
} else if (!groupId) {
|
||||
throw new Error(this.i18nService.getMessage('targetGroupRequired'));
|
||||
}
|
||||
|
||||
await this.appendLog(taskId, `Scanning directory: ${task.sourcePath}`);
|
||||
const filesToImport = this.scanDir(task.sourcePath);
|
||||
await this.appendLog(taskId, `Found ${filesToImport.length} files.`);
|
||||
|
||||
for (let i = 0; i < filesToImport.length; i++) {
|
||||
const filePath = filesToImport[i];
|
||||
try {
|
||||
const kb = await this.importSingleFile(
|
||||
filePath,
|
||||
task,
|
||||
importTargetDir,
|
||||
i,
|
||||
filesToImport.length,
|
||||
);
|
||||
await this.groupService.addFilesToGroup(
|
||||
kb.id,
|
||||
[groupId],
|
||||
task.userId,
|
||||
task.tenantId || 'default',
|
||||
);
|
||||
successCount++;
|
||||
if (successCount % 10 === 0) {
|
||||
await this.appendLog(taskId, `Imported ${successCount} files...`);
|
||||
}
|
||||
} catch (e) {
|
||||
failCount++;
|
||||
await this.appendLog(
|
||||
taskId,
|
||||
`Failed to import ${path.basename(filePath)}: ${e.message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await this.updateStatus(
|
||||
taskId,
|
||||
'COMPLETED',
|
||||
`Import finished. Success: ${successCount}, Failed: ${failCount}`,
|
||||
);
|
||||
} catch (error) {
|
||||
await this.updateStatus(
|
||||
taskId,
|
||||
'FAILED',
|
||||
`Fatal error: ${error.message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure a KnowledgeGroup exists for each segment of the relative directory path.
|
||||
* Returns the groupId for the leaf directory.
|
||||
*/
|
||||
private async ensureHierarchyGroup(
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
relativeDir: string,
|
||||
dirToGroupId: Map<string, string>,
|
||||
_sourcePath: string,
|
||||
taskId: string,
|
||||
): Promise<string> {
|
||||
if (dirToGroupId.has(relativeDir)) {
|
||||
return dirToGroupId.get(relativeDir)!;
|
||||
}
|
||||
|
||||
const segments = relativeDir.split(path.sep);
|
||||
let currentPath = '';
|
||||
let parentGroupId =
|
||||
dirToGroupId.get('.') ?? dirToGroupId.values().next().value;
|
||||
|
||||
for (const segment of segments) {
|
||||
currentPath = currentPath ? path.join(currentPath, segment) : segment;
|
||||
if (dirToGroupId.has(currentPath)) {
|
||||
parentGroupId = dirToGroupId.get(currentPath)!;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Create a group for this directory segment
|
||||
const group = await this.groupService.findOrCreate(
|
||||
userId,
|
||||
tenantId,
|
||||
segment,
|
||||
parentGroupId,
|
||||
`Sub-folder: ${currentPath}`,
|
||||
);
|
||||
dirToGroupId.set(currentPath, group.id);
|
||||
await this.appendLog(taskId, `Created sub-group: ${segment}`);
|
||||
parentGroupId = group.id;
|
||||
}
|
||||
|
||||
return parentGroupId;
|
||||
}
|
||||
|
||||
/** Copy file to safe location and index it */
|
||||
private async importSingleFile(
|
||||
filePath: string,
|
||||
task: ImportTask,
|
||||
importTargetDir: string,
|
||||
index: number,
|
||||
total: number,
|
||||
) {
|
||||
const filename = path.basename(filePath);
|
||||
const storedFilename = `imported-${Date.now()}-${Math.random().toString(36).substr(2, 5)}-${filename}`;
|
||||
const targetPath = path.join(importTargetDir, storedFilename);
|
||||
|
||||
fs.copyFileSync(filePath, targetPath);
|
||||
const stats = fs.statSync(targetPath);
|
||||
|
||||
const fileInfo = {
|
||||
filename: storedFilename,
|
||||
originalname: filename,
|
||||
path: targetPath,
|
||||
mimetype: 'text/markdown',
|
||||
size: stats.size,
|
||||
};
|
||||
|
||||
const indexingConfig = {
|
||||
chunkSize: task.chunkSize || 500,
|
||||
chunkOverlap: task.chunkOverlap || 50,
|
||||
embeddingModelId: task.embeddingModelId,
|
||||
mode: (task.mode || 'fast') as 'fast' | 'precise',
|
||||
};
|
||||
|
||||
this.logger.log(`Processing file ${index + 1}/${total}: ${filename}`);
|
||||
const kb = await this.kbService.createAndIndex(
|
||||
fileInfo,
|
||||
task.userId,
|
||||
task.tenantId || 'default',
|
||||
{
|
||||
...indexingConfig,
|
||||
waitForCompletion: true,
|
||||
} as any,
|
||||
);
|
||||
this.logger.log(
|
||||
`File ${index + 1}/${total} processing completed: ${filename}`,
|
||||
);
|
||||
return kb;
|
||||
}
|
||||
|
||||
private scanDir(directory: string): string[] {
|
||||
let results: string[] = [];
|
||||
const items = fs.readdirSync(directory);
|
||||
for (const item of items) {
|
||||
const fullPath = path.join(directory, item);
|
||||
const stat = fs.statSync(fullPath);
|
||||
if (stat.isDirectory()) {
|
||||
results = results.concat(this.scanDir(fullPath));
|
||||
} else {
|
||||
if (item.match(/\.(md|txt|html|json|pdf|docx|xlsx|pptx|csv)$/i)) {
|
||||
results.push(fullPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
private async updateStatus(
|
||||
id: string,
|
||||
status: ImportTask['status'],
|
||||
logMessage?: string,
|
||||
) {
|
||||
const task = await this.taskRepository.findOne({ where: { id } });
|
||||
if (task) {
|
||||
task.status = status;
|
||||
if (logMessage) {
|
||||
task.logs =
|
||||
(task.logs || '') + `[${new Date().toISOString()}] ${logMessage}\n`;
|
||||
}
|
||||
await this.taskRepository.save(task);
|
||||
}
|
||||
}
|
||||
|
||||
private async appendLog(id: string, message: string) {
|
||||
const task = await this.taskRepository.findOne({ where: { id } });
|
||||
if (task) {
|
||||
task.logs =
|
||||
(task.logs || '') + `[${new Date().toISOString()}] ${message}\n`;
|
||||
await this.taskRepository.save(task);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user