feat: implement QuestionBank CRUD with pagination and template query

- Add pagination support to findAll (page, limit query params)
- Add findByTemplateId method to service
- Add GET /by-template/:templateId endpoint to controller
- Service already includes CRUD for QuestionBank and QuestionBankItem
This commit is contained in:
Developer
2026-04-23 17:19:11 +08:00
commit 0a9588abb7
492 changed files with 112453 additions and 0 deletions
+320
View File
@@ -0,0 +1,320 @@
import {
BadRequestException,
Body,
Controller,
Post,
Request,
UploadedFile,
UseGuards,
UseInterceptors,
Logger,
} from '@nestjs/common';
import { FileInterceptor } from '@nestjs/platform-express';
import { UploadService } from './upload.service';
import { KnowledgeBaseService } from '../knowledge-base/knowledge-base.service';
import { CombinedAuthGuard } from '../auth/combined-auth.guard';
import { RolesGuard } from '../auth/roles.guard';
import { Roles } from '../auth/roles.decorator';
import { UserRole } from '../user/user-role.enum';
import { errorMessages } from '../i18n/messages';
import {
DEFAULT_CHUNK_SIZE,
MIN_CHUNK_SIZE,
MAX_CHUNK_SIZE,
DEFAULT_CHUNK_OVERLAP,
DEFAULT_MAX_OVERLAP_RATIO,
MAX_FILE_SIZE,
DEFAULT_LANGUAGE,
} from '../common/constants';
import { I18nService } from '../i18n/i18n.service';
import {
isAllowedByExtension,
IMAGE_MIME_TYPES,
} from '../common/file-support.constants';
export interface UploadConfigDto {
chunkSize?: string;
chunkOverlap?: string;
embeddingModelId?: string;
mode?: 'fast' | 'precise'; // Processing mode
groupIds?: string; // JSON string of group IDs
}
@Controller('upload')
@UseGuards(CombinedAuthGuard, RolesGuard)
export class UploadController {
private readonly logger = new Logger(UploadController.name);
constructor(
private readonly uploadService: UploadService,
private readonly knowledgeBaseService: KnowledgeBaseService,
private readonly i18nService: I18nService,
) {}
@Post('local-folder')
@Roles(UserRole.TENANT_ADMIN, UserRole.SUPER_ADMIN)
async importLocalFolder(
@Request() req,
@Body()
body: {
sourcePath: string;
embeddingModelId: string;
chunkSize?: string;
chunkOverlap?: string;
mode?: 'fast' | 'precise';
useHierarchy?: boolean;
groupIds?: string[];
},
) {
if (!body.sourcePath) {
throw new BadRequestException(
this.i18nService.getMessage('sourcePathRequired' as any) ||
'Source path is required',
);
}
if (!body.embeddingModelId) {
throw new BadRequestException(
this.i18nService.getMessage('uploadModelRequired'),
);
}
const indexingConfig = {
chunkSize: body.chunkSize ? parseInt(body.chunkSize) : DEFAULT_CHUNK_SIZE,
chunkOverlap: body.chunkOverlap
? parseInt(body.chunkOverlap)
: DEFAULT_CHUNK_OVERLAP,
embeddingModelId: body.embeddingModelId,
mode: body.mode || 'fast',
useHierarchy: body.useHierarchy ?? false,
groupIds: body.groupIds || [],
};
const result = await this.uploadService.importLocalFolder(
body.sourcePath,
req.user.id,
req.user.tenantId,
indexingConfig,
);
return {
message:
this.i18nService.getMessage('importStarted' as any) || 'Import started',
...result,
};
}
@Post()
@Roles(UserRole.TENANT_ADMIN, UserRole.SUPER_ADMIN)
@UseInterceptors(
FileInterceptor('file', {
fileFilter: (req, file, cb) => {
// Check by image MIME type or extension
const isAllowed =
IMAGE_MIME_TYPES.includes(file.mimetype) ||
isAllowedByExtension(file.originalname);
if (isAllowed) {
cb(null, true);
} else {
cb(
new BadRequestException(
(
errorMessages[DEFAULT_LANGUAGE]?.uploadTypeUnsupported ||
errorMessages['ja'].uploadTypeUnsupported
).replace('{type}', file?.mimetype ?? 'unknown'),
),
false,
);
}
},
}),
)
async uploadFile(
@UploadedFile() file: Express.Multer.File,
@Request() req,
@Body() config: UploadConfigDto,
) {
if (!file) {
throw new BadRequestException(
this.i18nService.getMessage('uploadNoFile'),
);
}
// Validate file sizefrontend limit + backend validation
const maxFileSize = parseInt(
process.env.MAX_FILE_SIZE || String(MAX_FILE_SIZE),
); // 100MB
if (file.size > maxFileSize) {
throw new BadRequestException(
this.i18nService.formatMessage('uploadSizeExceeded', {
size: this.formatBytes(file.size),
max: this.formatBytes(maxFileSize),
}),
);
}
// Validate embedding model config
if (!config.embeddingModelId) {
throw new BadRequestException(
this.i18nService.getMessage('uploadModelRequired'),
);
}
this.logger.log(
`User ${req.user.id} uploaded file: ${file.originalname} (${this.formatBytes(file.size)})`,
);
const fileInfo = await this.uploadService.processUploadedFile(file);
// Parse config parameters and set safe default values
const indexingConfig = {
chunkSize: config.chunkSize
? parseInt(config.chunkSize)
: DEFAULT_CHUNK_SIZE,
chunkOverlap: config.chunkOverlap
? parseInt(config.chunkOverlap)
: DEFAULT_CHUNK_OVERLAP,
embeddingModelId: config.embeddingModelId || null,
groupIds: config.groupIds ? JSON.parse(config.groupIds) : [],
};
// Ensure overlap <= 50% of chunk size
if (
indexingConfig.chunkOverlap >
indexingConfig.chunkSize * DEFAULT_MAX_OVERLAP_RATIO
) {
indexingConfig.chunkOverlap = Math.floor(
indexingConfig.chunkSize * DEFAULT_MAX_OVERLAP_RATIO,
);
this.logger.warn(
this.i18nService.formatMessage('overlapAdjusted', {
newSize: indexingConfig.chunkOverlap,
}),
);
}
// Save to database and trigger indexingasync
const kb = await this.knowledgeBaseService.createAndIndex(
fileInfo,
req.user.id,
req.user.tenantId, // Ensure tenantId is passed down
{
...indexingConfig,
mode: config.mode || 'fast',
} as any, // Bypass strict type check for now or cast to correct type
);
return {
message: this.i18nService.getMessage('uploadSuccess'),
id: kb.id,
filename: fileInfo.filename,
originalname: fileInfo.originalname,
status: kb.status,
mode: config.mode || 'fast',
config: indexingConfig,
estimatedChunks: Math.ceil(file.size / (indexingConfig.chunkSize * 4)), // Estimated chunk count
};
}
@Post('text')
@Roles(UserRole.TENANT_ADMIN, UserRole.SUPER_ADMIN)
async uploadText(
@Request() req,
@Body()
body: {
content: string;
title: string;
chunkSize?: string;
chunkOverlap?: string;
embeddingModelId?: string;
mode?: 'fast' | 'precise';
},
) {
if (!body.content || !body.title) {
throw new BadRequestException(
this.i18nService.getMessage('contentAndTitleRequired'),
);
}
const fs = await import('fs');
const path = await import('path');
const uploadPath = process.env.UPLOAD_FILE_PATH || './uploads';
if (!fs.existsSync(uploadPath)) {
fs.mkdirSync(uploadPath, { recursive: true });
}
const uniqueSuffix = Date.now() + '-' + Math.round(Math.random() * 1e9);
const filename = `note-${uniqueSuffix}.md`;
const filePath = path.join(uploadPath, filename);
// Save content to file (mocking a file upload)
fs.writeFileSync(filePath, body.content, 'utf8');
const fileSize = Buffer.byteLength(body.content, 'utf8');
// Mimic Multer file info
const fileInfo = {
filename: filename,
originalname: body.title.endsWith('.md')
? body.title
: `${body.title}.md`,
size: fileSize,
mimetype: 'text/markdown',
path: filePath,
};
// Validating Config
if (!body.embeddingModelId) {
throw new BadRequestException(
this.i18nService.getMessage('uploadModelRequired'),
);
}
const indexingConfig = {
chunkSize: body.chunkSize ? parseInt(body.chunkSize) : DEFAULT_CHUNK_SIZE,
chunkOverlap: body.chunkOverlap
? parseInt(body.chunkOverlap)
: DEFAULT_CHUNK_OVERLAP,
embeddingModelId: body.embeddingModelId || null,
};
if (
indexingConfig.chunkOverlap >
indexingConfig.chunkSize * DEFAULT_MAX_OVERLAP_RATIO
) {
indexingConfig.chunkOverlap = Math.floor(
indexingConfig.chunkSize * DEFAULT_MAX_OVERLAP_RATIO,
);
}
const kb = await this.knowledgeBaseService.createAndIndex(
fileInfo,
req.user.id,
req.user.tenantId,
{
...indexingConfig,
mode: body.mode || 'fast',
} as any,
);
return {
message: this.i18nService.getMessage('uploadTextSuccess'),
id: kb.id,
filename: fileInfo.filename,
originalname: fileInfo.originalname,
status: kb.status,
config: indexingConfig,
};
}
private formatBytes(bytes: number): string {
if (bytes === 0) return '0 B';
const k = 1024;
const sizes = ['B', 'KB', 'MB', 'GB'];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
}
}
+72
View File
@@ -0,0 +1,72 @@
import { Module } from '@nestjs/common';
import { UploadService } from './upload.service';
import { UploadController } from './upload.controller';
import { MulterModule } from '@nestjs/platform-express';
import { ConfigModule, ConfigService } from '@nestjs/config';
import { KnowledgeBaseModule } from '../knowledge-base/knowledge-base.module'; // Import KnowledgeBaseModule
import { KnowledgeGroupModule } from '../knowledge-group/knowledge-group.module';
import * as multer from 'multer';
import * as fs from 'fs';
import * as path from 'path';
import { UserModule } from '../user/user.module';
@Module({
imports: [
KnowledgeBaseModule,
KnowledgeGroupModule,
UserModule,
MulterModule.registerAsync({
imports: [ConfigModule],
useFactory: (configService: ConfigService) => {
const uploadPath = configService.get<string>(
'UPLOAD_FILE_PATH',
'./uploads',
); // Get upload path from env varor use default './uploads' use
// Ensure upload directory exists
if (!fs.existsSync(uploadPath)) {
fs.mkdirSync(uploadPath, { recursive: true });
}
// Get max file size from env var, default 100MB
const maxFileSize = parseInt(
configService.get<string>('MAX_FILE_SIZE', '104857600'), // 100MB in bytes
);
return {
storage: multer.diskStorage({
destination: (req: any, file, cb) => {
const tenantId = req.user?.tenantId || 'default';
const fullPath = path.join(uploadPath, tenantId);
if (!fs.existsSync(fullPath)) {
fs.mkdirSync(fullPath, { recursive: true });
}
cb(null, fullPath);
},
filename: (req, file, cb) => {
// Fix Chinese filename garbling
file.originalname = Buffer.from(
file.originalname,
'latin1',
).toString('utf8');
const uniqueSuffix =
Date.now() + '-' + Math.round(Math.random() * 1e9);
cb(
null,
`${file.fieldname}-${uniqueSuffix}${path.extname(file.originalname)}`,
);
},
}),
limits: {
fileSize: maxFileSize, // File size limit
},
};
},
inject: [ConfigService],
}),
],
controllers: [UploadController],
providers: [UploadService],
})
export class UploadModule {}
+246
View File
@@ -0,0 +1,246 @@
import { Injectable, Logger, BadRequestException } from '@nestjs/common';
import { KnowledgeBaseService } from '../knowledge-base/knowledge-base.service';
import { KnowledgeGroupService } from '../knowledge-group/knowledge-group.service';
import * as fs from 'fs';
import * as path from 'path';
@Injectable()
export class UploadService {
private readonly logger = new Logger(UploadService.name);
constructor(
private kbService: KnowledgeBaseService,
private groupService: KnowledgeGroupService,
) {}
async processUploadedFile(file: Express.Multer.File) {
// Add more business logic here. Example:
// - Save file info to database
// - Call other services to process file (Tika text extraction, ES indexing etc.)
// - Validate file format or analyze content
// Currently only return basic file info
return {
filename: file.filename,
originalname: file.originalname,
size: file.size,
mimetype: file.mimetype,
path: file.path, // After Multer saves file, full path is in file.path
};
}
async importLocalFolder(
sourcePath: string,
userId: string,
tenantId: string,
config: any,
) {
if (!fs.existsSync(sourcePath)) {
throw new BadRequestException(`Directory not found: ${sourcePath}`);
}
const stat = fs.statSync(sourcePath);
if (!stat.isDirectory()) {
throw new BadRequestException(`Path is not a directory: ${sourcePath}`);
}
// Determine root group for hierarchy or single group
let rootGroupId: string | null = null;
if (config.groupIds && config.groupIds.length > 0) {
rootGroupId = config.groupIds[0];
}
this.logger.log(
`Starting local folder import: ${sourcePath} for user ${userId}, tenant ${tenantId}`,
);
// Trigger scanning and processing asynchronously to not block the request
this.executeLocalImport(
sourcePath,
userId,
tenantId,
config,
rootGroupId,
).catch((err) => {
this.logger.error(`Local folder import failed for ${sourcePath}`, err);
});
return {
sourcePath,
status: 'PROCESSING',
};
}
private async executeLocalImport(
sourcePath: string,
userId: string,
tenantId: string,
config: any,
rootGroupId: string | null,
) {
const files = this.scanDir(sourcePath);
this.logger.log(`Found ${files.length} files in ${sourcePath}`);
const dirToGroupId = new Map<string, string>();
if (rootGroupId) {
dirToGroupId.set('.', rootGroupId);
} else {
// Create a root group based on folder name if none provided
const rootName = path.basename(sourcePath);
const rootGroup = await this.groupService.create(userId, tenantId, {
name: rootName,
description: `Imported from local path: ${sourcePath}`,
});
rootGroupId = rootGroup.id;
dirToGroupId.set('.', rootGroupId);
}
const uploadBaseDir = process.env.UPLOAD_FILE_PATH || './uploads';
for (const filePath of files) {
try {
const relativeDir = path.relative(sourcePath, path.dirname(filePath));
const normalizedDir = relativeDir || '.';
let targetGroupId = rootGroupId;
if (config.useHierarchy) {
targetGroupId = await this.ensureHierarchy(
userId,
tenantId,
normalizedDir,
dirToGroupId,
rootGroupId,
);
}
const filename = path.basename(filePath);
const uniqueSuffix = Date.now() + '-' + Math.round(Math.random() * 1e9);
const storedFilename = `local-${uniqueSuffix}-${filename}`;
// Ensure tenant directory exists
const tenantDir = path.join(uploadBaseDir, tenantId);
if (!fs.existsSync(tenantDir)) {
fs.mkdirSync(tenantDir, { recursive: true });
}
const targetPath = path.join(tenantDir, storedFilename);
fs.copyFileSync(filePath, targetPath);
const stats = fs.statSync(targetPath);
const fileInfo = {
filename: storedFilename,
originalname: filename,
path: targetPath,
size: stats.size,
mimetype: this.getMimeType(filename),
};
await this.kbService.createAndIndex(fileInfo, userId, tenantId, {
...config,
groupIds: [targetGroupId],
});
} catch (err) {
this.logger.error(`Failed to process local file: ${filePath}`, err);
}
}
this.logger.log(`Local folder import completed: ${sourcePath}`);
}
private async ensureHierarchy(
userId: string,
tenantId: string,
relativeDir: string,
dirToGroupId: Map<string, string>,
rootGroupId: string,
): Promise<string> {
if (dirToGroupId.has(relativeDir)) {
return dirToGroupId.get(relativeDir)!;
}
const segments = relativeDir.split(path.sep);
let currentPath = '';
let parentId = rootGroupId;
for (const segment of segments) {
if (!segment || segment === '.') continue;
currentPath = currentPath ? path.join(currentPath, segment) : segment;
if (dirToGroupId.has(currentPath)) {
parentId = dirToGroupId.get(currentPath)!;
continue;
}
const group = await this.groupService.findOrCreate(
userId,
tenantId,
segment,
parentId,
`Sub-folder from local import: ${currentPath}`,
);
dirToGroupId.set(currentPath, group.id);
parentId = group.id;
}
return parentId;
}
private scanDir(directory: string): string[] {
let results: string[] = [];
if (!fs.existsSync(directory)) return results;
const items = fs.readdirSync(directory);
for (const item of items) {
const fullPath = path.join(directory, item);
const stat = fs.statSync(fullPath);
if (stat.isDirectory()) {
results = results.concat(this.scanDir(fullPath));
} else {
// Only include supported document and code extensions
const ext = path.extname(item).toLowerCase().slice(1);
if (
[
'pdf',
'doc',
'docx',
'xls',
'xlsx',
'ppt',
'pptx',
'rtf',
'csv',
'txt',
'md',
'html',
'json',
'xml',
'js',
'ts',
'py',
'java',
'sql',
].includes(ext)
) {
results.push(fullPath);
}
}
}
return results;
}
private getMimeType(filename: string): string {
const ext = path.extname(filename).toLowerCase();
const mimeMap: Record<string, string> = {
'.pdf': 'application/pdf',
'.doc': 'application/msword',
'.docx':
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
'.md': 'text/markdown',
'.txt': 'text/plain',
'.json': 'application/json',
'.html': 'text/html',
'.csv': 'text/csv',
};
return mimeMap[ext] || 'application/octet-stream';
}
}