feat: implement QuestionBank CRUD with pagination and template query
- Add pagination support to findAll (page, limit query params) - Add findByTemplateId method to service - Add GET /by-template/:templateId endpoint to controller - Service already includes CRUD for QuestionBank and QuestionBankItem
This commit is contained in:
Developer
@@ -0,0 +1,85 @@
|
||||
import {
|
||||
Controller,
|
||||
Get,
|
||||
Post,
|
||||
Put,
|
||||
Body,
|
||||
UseGuards,
|
||||
Request,
|
||||
Query,
|
||||
UseInterceptors,
|
||||
UploadedFile,
|
||||
Res,
|
||||
} from '@nestjs/common';
|
||||
import { FileInterceptor } from '@nestjs/platform-express';
|
||||
import { Response } from 'express';
|
||||
import { AdminService } from './admin.service';
|
||||
import { CombinedAuthGuard } from '../auth/combined-auth.guard';
|
||||
import { RolesGuard } from '../auth/roles.guard';
|
||||
import { Roles } from '../auth/roles.decorator';
|
||||
import { UserRole } from '../user/user-role.enum';
|
||||
|
||||
@Controller('v1/admin')
|
||||
@UseGuards(CombinedAuthGuard, RolesGuard)
|
||||
export class AdminController {
|
||||
constructor(private readonly adminService: AdminService) {}
|
||||
|
||||
@Get('users')
|
||||
@Roles(UserRole.TENANT_ADMIN, UserRole.SUPER_ADMIN)
|
||||
async getUsers(
|
||||
@Request() req: any,
|
||||
@Query('page') page?: string,
|
||||
@Query('limit') limit?: string,
|
||||
) {
|
||||
const isSuperAdmin = req.user.role === UserRole.SUPER_ADMIN;
|
||||
return this.adminService.getTenantUsers(
|
||||
isSuperAdmin ? undefined : req.user.tenantId,
|
||||
page ? parseInt(page) : undefined,
|
||||
limit ? parseInt(limit) : undefined,
|
||||
);
|
||||
}
|
||||
|
||||
@Get('users/export')
|
||||
@Roles(UserRole.TENANT_ADMIN, UserRole.SUPER_ADMIN)
|
||||
async getUsersExport(@Request() req: any, @Res() res: Response) {
|
||||
const isSuperAdmin = req.user.role === UserRole.SUPER_ADMIN;
|
||||
const buffer = await this.adminService.exportUsers(
|
||||
isSuperAdmin ? undefined : req.user.tenantId,
|
||||
);
|
||||
res.set({
|
||||
'Content-Type':
|
||||
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
||||
'Content-Disposition': 'attachment; filename="users_export.xlsx"',
|
||||
'Content-Length': buffer.length,
|
||||
});
|
||||
res.end(buffer);
|
||||
}
|
||||
|
||||
@Post('users/import')
|
||||
@Roles(UserRole.TENANT_ADMIN, UserRole.SUPER_ADMIN)
|
||||
@UseInterceptors(FileInterceptor('file'))
|
||||
async importUsers(@Request() req: any, @UploadedFile() file: any) {
|
||||
const isSuperAdmin = req.user.role === UserRole.SUPER_ADMIN;
|
||||
return this.adminService.importUsers(
|
||||
isSuperAdmin ? undefined : req.user.tenantId,
|
||||
file,
|
||||
);
|
||||
}
|
||||
|
||||
@Get('settings')
|
||||
async getSettings(@Request() req: any) {
|
||||
return this.adminService.getTenantSettings(req.user.tenantId);
|
||||
}
|
||||
|
||||
@Put('settings')
|
||||
@Roles(UserRole.TENANT_ADMIN, UserRole.SUPER_ADMIN)
|
||||
async updateSettings(@Request() req: any, @Body() body: any) {
|
||||
return this.adminService.updateTenantSettings(req.user.tenantId, body);
|
||||
}
|
||||
|
||||
@Get('pending-shares')
|
||||
@Roles(UserRole.TENANT_ADMIN, UserRole.SUPER_ADMIN)
|
||||
async getPendingShares(@Request() req: any) {
|
||||
return this.adminService.getPendingShares(req.user.tenantId);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { AdminController } from './admin.controller';
|
||||
import { AdminService } from './admin.service';
|
||||
import { UserModule } from '../user/user.module';
|
||||
import { TenantModule } from '../tenant/tenant.module';
|
||||
|
||||
@Module({
|
||||
imports: [UserModule, TenantModule],
|
||||
controllers: [AdminController],
|
||||
providers: [AdminService],
|
||||
})
|
||||
export class AdminModule {}
|
||||
@@ -0,0 +1,146 @@
|
||||
import {
|
||||
Injectable,
|
||||
NotFoundException,
|
||||
ForbiddenException,
|
||||
BadRequestException,
|
||||
} from '@nestjs/common';
|
||||
import * as XLSX from 'xlsx';
|
||||
import { UserService } from '../user/user.service';
|
||||
import { TenantService } from '../tenant/tenant.service';
|
||||
import { I18nService } from '../i18n/i18n.service';
|
||||
|
||||
interface UserImportRow {
|
||||
Username?: string | number;
|
||||
username?: string | number;
|
||||
DisplayName?: string | number;
|
||||
displayName?: string | number;
|
||||
Name?: string | number;
|
||||
name?: string | number;
|
||||
Password?: string | number;
|
||||
password?: string | number;
|
||||
IsAdmin?: string | number | boolean;
|
||||
isAdmin?: string | number | boolean;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class AdminService {
|
||||
constructor(
|
||||
private readonly userService: UserService,
|
||||
private readonly tenantService: TenantService,
|
||||
private readonly i18nService: I18nService,
|
||||
) {}
|
||||
|
||||
async getTenantUsers(tenantId?: string, page?: number, limit?: number) {
|
||||
if (!tenantId) {
|
||||
return this.userService.findAll(page, limit);
|
||||
}
|
||||
return this.userService.findByTenantId(tenantId, page, limit);
|
||||
}
|
||||
|
||||
async exportUsers(tenantId?: string): Promise<Buffer> {
|
||||
const { data: users } = tenantId
|
||||
? await this.userService.findByTenantId(tenantId)
|
||||
: await this.userService.findAll();
|
||||
|
||||
const worksheet = XLSX.utils.json_to_sheet(
|
||||
users.map((u) => ({
|
||||
Username: u.username,
|
||||
DisplayName: u.displayName || '',
|
||||
IsAdmin: u.isAdmin ? 'Yes' : 'No',
|
||||
CreatedAt: u.createdAt,
|
||||
Password: '', // Placeholder for new users
|
||||
})),
|
||||
);
|
||||
|
||||
const workbook = XLSX.utils.book_new();
|
||||
XLSX.utils.book_append_sheet(workbook, worksheet, 'Users');
|
||||
|
||||
return XLSX.write(workbook, { type: 'buffer', bookType: 'xlsx' });
|
||||
}
|
||||
|
||||
async importUsers(tenantId?: string, file?: any) {
|
||||
if (!file)
|
||||
throw new BadRequestException(
|
||||
this.i18nService.getMessage('uploadNoFile'),
|
||||
);
|
||||
|
||||
const workbook = XLSX.read(file.buffer, { type: 'buffer' });
|
||||
const sheetName = workbook.SheetNames[0];
|
||||
const worksheet = workbook.Sheets[sheetName];
|
||||
const data = XLSX.utils.sheet_to_json<UserImportRow>(worksheet);
|
||||
|
||||
const results = {
|
||||
success: 0,
|
||||
failed: 0,
|
||||
errors: [] as string[],
|
||||
};
|
||||
|
||||
for (const row of data) {
|
||||
try {
|
||||
const username = (row.Username || row.username)?.toString();
|
||||
const displayName = (
|
||||
row.DisplayName ||
|
||||
row.displayName ||
|
||||
row.Name ||
|
||||
row.name
|
||||
)?.toString();
|
||||
const password = (row.Password || row.password)?.toString();
|
||||
const isAdminStr = (row.IsAdmin || row.isAdmin || 'No').toString();
|
||||
const isAdmin =
|
||||
isAdminStr.toLowerCase() === 'yes' ||
|
||||
isAdminStr === 'true' ||
|
||||
isAdminStr === '1';
|
||||
|
||||
if (!username) {
|
||||
throw new Error(this.i18nService.getMessage('usernameRequired'));
|
||||
}
|
||||
|
||||
const existingUser = await this.userService.findOneByUsername(username);
|
||||
|
||||
if (existingUser) {
|
||||
await this.userService.updateUser(existingUser.id, {
|
||||
displayName: displayName || existingUser.displayName,
|
||||
password: password || undefined,
|
||||
// We avoid changing isAdmin status via import for security unless explicitly required
|
||||
});
|
||||
} else {
|
||||
if (!password) {
|
||||
throw new Error(
|
||||
this.i18nService.formatMessage('passwordRequiredForNewUser', {
|
||||
username,
|
||||
}),
|
||||
);
|
||||
}
|
||||
await this.userService.createUser(
|
||||
username,
|
||||
password,
|
||||
isAdmin,
|
||||
tenantId,
|
||||
displayName,
|
||||
);
|
||||
}
|
||||
results.success++;
|
||||
} catch (e: any) {
|
||||
results.failed++;
|
||||
results.errors.push(`${row.Username || 'Unknown'}: ${e.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
async getTenantSettings(tenantId: string) {
|
||||
return this.tenantService.getSettings(tenantId);
|
||||
}
|
||||
|
||||
async updateTenantSettings(tenantId: string, data: any) {
|
||||
return this.tenantService.updateSettings(tenantId, data);
|
||||
}
|
||||
|
||||
// Notebook sharing approval and model assignments would go here
|
||||
async getPendingShares(tenantId: string) {
|
||||
// Mock implementation for pending shares to satisfy UI.
|
||||
// Needs proper schema/entity support in the future.
|
||||
return [];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { EmbeddingService } from './embedding.service';
|
||||
|
||||
@Module({
|
||||
providers: [EmbeddingService],
|
||||
exports: [EmbeddingService],
|
||||
})
|
||||
export class AiModule {}
|
||||
@@ -0,0 +1,29 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { OpenAIEmbeddings } from '@langchain/openai';
|
||||
import { ModelConfig } from '../model-config/model-config.entity';
|
||||
|
||||
@Injectable()
|
||||
export class EmbeddingService {
|
||||
private readonly logger = new Logger(EmbeddingService.name);
|
||||
|
||||
async getEmbeddings(text: string, config: ModelConfig): Promise<number[]> {
|
||||
try {
|
||||
const embeddings = new OpenAIEmbeddings({
|
||||
openAIApiKey: config.apiKey || 'ollama',
|
||||
configuration: {
|
||||
baseURL: config.baseUrl,
|
||||
},
|
||||
modelName: config.modelId,
|
||||
});
|
||||
|
||||
const vector = await embeddings.embedQuery(text);
|
||||
return vector;
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Failed to generate embeddings using model ${config.modelId}`,
|
||||
error,
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,297 @@
|
||||
import {
|
||||
Body,
|
||||
Controller,
|
||||
Delete,
|
||||
Get,
|
||||
Param,
|
||||
Post,
|
||||
Request,
|
||||
Res,
|
||||
UploadedFile,
|
||||
UseGuards,
|
||||
UseInterceptors,
|
||||
} from '@nestjs/common';
|
||||
import { FileInterceptor } from '@nestjs/platform-express';
|
||||
import { Response } from 'express';
|
||||
import { ApiKeyGuard } from '../auth/api-key.guard';
|
||||
import { RagService } from '../rag/rag.service';
|
||||
import { ChatService } from '../chat/chat.service';
|
||||
import { KnowledgeBaseService } from '../knowledge-base/knowledge-base.service';
|
||||
import { ModelConfigService } from '../model-config/model-config.service';
|
||||
import { TenantService } from '../tenant/tenant.service';
|
||||
import { UserSettingService } from '../user/user-setting.service';
|
||||
import { I18nService } from '../i18n/i18n.service';
|
||||
|
||||
@Controller('v1')
|
||||
@UseGuards(ApiKeyGuard)
|
||||
export class ApiV1Controller {
|
||||
constructor(
|
||||
private readonly ragService: RagService,
|
||||
private readonly chatService: ChatService,
|
||||
private readonly knowledgeBaseService: KnowledgeBaseService,
|
||||
private readonly modelConfigService: ModelConfigService,
|
||||
private readonly tenantService: TenantService,
|
||||
private readonly userSettingService: UserSettingService,
|
||||
private readonly i18nService: I18nService,
|
||||
) {}
|
||||
|
||||
// ========== Chat / RAG ==========
|
||||
/**
|
||||
* POST /api/v1/chat
|
||||
* Tenant-scoped RAG chat. Supports both streaming (SSE) and standard JSON responses.
|
||||
* Body: { message, stream?, selectedGroups?, selectedFiles? }
|
||||
*/
|
||||
@Post('chat')
|
||||
async chat(
|
||||
@Request() req,
|
||||
@Body()
|
||||
body: {
|
||||
message: string;
|
||||
stream?: boolean;
|
||||
selectedGroups?: string[];
|
||||
selectedFiles?: string[];
|
||||
},
|
||||
@Res() res: Response,
|
||||
) {
|
||||
const { message, stream = false, selectedGroups, selectedFiles } = body;
|
||||
const user = req.user;
|
||||
|
||||
if (!message) {
|
||||
return res.status(400).json({ error: 'message is required' });
|
||||
}
|
||||
|
||||
// Get organization settings and model configuration
|
||||
const tenantSettings = await this.tenantService.getSettings(user.tenantId);
|
||||
const userSetting = await this.userSettingService.getByUser(user.id);
|
||||
const models = await this.modelConfigService.findAll();
|
||||
const llmModel =
|
||||
models.find((m) => m.id === tenantSettings?.selectedLLMId) ??
|
||||
models.find((m) => m.type === 'llm' && m.isDefault);
|
||||
|
||||
if (!llmModel) {
|
||||
return res
|
||||
.status(400)
|
||||
.json({ error: 'No LLM model configured for this user' });
|
||||
}
|
||||
|
||||
const modelConfig = llmModel as any;
|
||||
|
||||
if (stream) {
|
||||
res.setHeader('Content-Type', 'text/event-stream');
|
||||
res.setHeader('Cache-Control', 'no-cache');
|
||||
res.setHeader('Connection', 'keep-alive');
|
||||
|
||||
try {
|
||||
const stream = this.chatService.streamChat(
|
||||
message,
|
||||
[], // history
|
||||
user.id,
|
||||
modelConfig,
|
||||
userSetting?.language ?? 'zh', // userLanguage
|
||||
tenantSettings?.selectedEmbeddingId, // selectedEmbeddingId
|
||||
selectedGroups, // selectedGroups
|
||||
selectedFiles, // selectedFiles
|
||||
undefined, // historyId
|
||||
tenantSettings?.enableRerank ?? false, // enableRerank
|
||||
tenantSettings?.selectedRerankId, // selectedRerankId
|
||||
tenantSettings?.temperature, // temperature
|
||||
tenantSettings?.maxTokens, // maxTokens
|
||||
tenantSettings?.topK ?? 5, // topK
|
||||
tenantSettings?.similarityThreshold ?? 0.3, // similarityThreshold
|
||||
tenantSettings?.rerankSimilarityThreshold ?? 0.5, // rerankSimilarityThreshold
|
||||
tenantSettings?.enableQueryExpansion ?? false, // enableQueryExpansion
|
||||
tenantSettings?.enableHyDE ?? false, // enableHyDE
|
||||
user.tenantId, // Passing tenantId correctly
|
||||
);
|
||||
|
||||
for await (const chunk of stream) {
|
||||
res.write(`data: ${JSON.stringify(chunk)}\n\n`);
|
||||
}
|
||||
res.write('data: [DONE]\n\n');
|
||||
res.end();
|
||||
} catch (error) {
|
||||
res.write(
|
||||
`data: ${JSON.stringify({ type: 'error', data: error.message })}\n\n`,
|
||||
);
|
||||
res.end();
|
||||
}
|
||||
} else {
|
||||
// Non-streaming: collect all chunks and return as JSON
|
||||
try {
|
||||
let fullContent = '';
|
||||
const sources: any[] = [];
|
||||
let historyId: string | undefined;
|
||||
|
||||
const chatStream = this.chatService.streamChat(
|
||||
message,
|
||||
[],
|
||||
user.id,
|
||||
modelConfig,
|
||||
userSetting?.language ?? 'zh',
|
||||
tenantSettings?.selectedEmbeddingId,
|
||||
selectedGroups,
|
||||
selectedFiles,
|
||||
undefined, // historyId
|
||||
tenantSettings?.enableRerank ?? false,
|
||||
tenantSettings?.selectedRerankId,
|
||||
tenantSettings?.temperature,
|
||||
tenantSettings?.maxTokens,
|
||||
tenantSettings?.topK ?? 5,
|
||||
tenantSettings?.similarityThreshold ?? 0.3,
|
||||
tenantSettings?.rerankSimilarityThreshold ?? 0.5,
|
||||
tenantSettings?.enableQueryExpansion ?? false,
|
||||
tenantSettings?.enableHyDE ?? false,
|
||||
user.tenantId, // Passing tenantId correctly
|
||||
);
|
||||
|
||||
for await (const chunk of chatStream) {
|
||||
if (chunk.type === 'content') fullContent += chunk.data;
|
||||
else if (chunk.type === 'sources') sources.push(...chunk.data);
|
||||
else if (chunk.type === 'historyId') historyId = chunk.data;
|
||||
}
|
||||
|
||||
return res.json({ content: fullContent, sources, historyId });
|
||||
} catch (error) {
|
||||
return res.status(500).json({ error: error.message });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ========== Search ==========
|
||||
/**
|
||||
* POST /api/v1/search
|
||||
* Tenant-scoped hybrid search across knowledge base.
|
||||
* Body: { query, topK?, threshold?, selectedGroups?, selectedFiles? }
|
||||
*/
|
||||
@Post('search')
|
||||
async search(
|
||||
@Request() req,
|
||||
@Body()
|
||||
body: {
|
||||
query: string;
|
||||
topK?: number;
|
||||
threshold?: number;
|
||||
selectedGroups?: string[];
|
||||
selectedFiles?: string[];
|
||||
},
|
||||
) {
|
||||
const {
|
||||
query,
|
||||
topK = 5,
|
||||
threshold = 0.3,
|
||||
selectedGroups,
|
||||
selectedFiles,
|
||||
} = body;
|
||||
const user = req.user;
|
||||
|
||||
if (!query) return { error: 'query is required' };
|
||||
|
||||
const userSetting = await this.tenantService.getSettings(user.tenantId);
|
||||
|
||||
const results = await this.ragService.searchKnowledge(
|
||||
query,
|
||||
user.id,
|
||||
topK,
|
||||
threshold,
|
||||
userSetting?.selectedEmbeddingId,
|
||||
userSetting?.enableFullTextSearch ?? false,
|
||||
userSetting?.enableRerank ?? false,
|
||||
userSetting?.selectedRerankId,
|
||||
selectedGroups,
|
||||
selectedFiles,
|
||||
userSetting?.rerankSimilarityThreshold ?? 0.5,
|
||||
user.tenantId,
|
||||
userSetting?.enableQueryExpansion ?? false,
|
||||
userSetting?.enableHyDE ?? false,
|
||||
);
|
||||
|
||||
return { results, total: results.length };
|
||||
}
|
||||
|
||||
// ========== Knowledge Base ==========
|
||||
/**
|
||||
* GET /api/v1/knowledge-bases
|
||||
* List all files belonging to the caller's tenant.
|
||||
*/
|
||||
@Get('knowledge-bases')
|
||||
async listFiles(@Request() req) {
|
||||
const user = req.user;
|
||||
const files = await this.knowledgeBaseService.findAll(
|
||||
user.id,
|
||||
user.tenantId,
|
||||
);
|
||||
return {
|
||||
files: files.map((f) => ({
|
||||
id: f.id,
|
||||
name: f.originalName,
|
||||
title: f.title,
|
||||
status: f.status,
|
||||
size: f.size,
|
||||
mimetype: f.mimetype,
|
||||
createdAt: f.createdAt,
|
||||
})),
|
||||
total: files.length,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* POST /api/v1/knowledge-bases/upload
|
||||
* Upload and index a file into the caller's tenant knowledge base.
|
||||
*/
|
||||
@Post('knowledge-bases/upload')
|
||||
@UseInterceptors(FileInterceptor('file'))
|
||||
async uploadFile(
|
||||
@Request() req,
|
||||
@UploadedFile() file: Express.Multer.File,
|
||||
@Body()
|
||||
body: {
|
||||
mode?: 'fast' | 'precise';
|
||||
chunkSize?: number;
|
||||
chunkOverlap?: number;
|
||||
},
|
||||
) {
|
||||
if (!file) return { error: 'file is required' };
|
||||
const user = req.user;
|
||||
|
||||
const kb = await this.knowledgeBaseService.createAndIndex(
|
||||
file,
|
||||
user.id,
|
||||
user.tenantId,
|
||||
{
|
||||
mode: body.mode ?? 'fast',
|
||||
chunkSize: body.chunkSize ? Number(body.chunkSize) : 1000,
|
||||
chunkOverlap: body.chunkOverlap ? Number(body.chunkOverlap) : 200,
|
||||
},
|
||||
);
|
||||
|
||||
return {
|
||||
id: kb.id,
|
||||
name: kb.originalName,
|
||||
status: kb.status,
|
||||
message: 'File uploaded and indexing started',
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* DELETE /api/v1/knowledge-bases/:id
|
||||
* Delete a specific file from the knowledge base.
|
||||
*/
|
||||
@Delete('knowledge-bases/:id')
|
||||
async deleteFile(@Request() req, @Param('id') id: string) {
|
||||
const user = req.user;
|
||||
await this.knowledgeBaseService.deleteFile(id, user.id, user.tenantId);
|
||||
return { message: this.i18nService.getMessage('fileDeleted') };
|
||||
}
|
||||
|
||||
@Get('knowledge-bases/:id')
|
||||
async getFile(@Request() req, @Param('id') id: string) {
|
||||
const user = req.user;
|
||||
const files = await this.knowledgeBaseService.findAll(
|
||||
user.id,
|
||||
user.tenantId,
|
||||
);
|
||||
const file = files.find((f) => f.id === id);
|
||||
if (!file) return { error: 'File not found' };
|
||||
return file;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
import {
|
||||
Body,
|
||||
Controller,
|
||||
Get,
|
||||
HttpCode,
|
||||
HttpStatus,
|
||||
Post,
|
||||
Request,
|
||||
UseGuards,
|
||||
} from '@nestjs/common';
|
||||
import { ApiService } from './api.service';
|
||||
import { CombinedAuthGuard } from '../auth/combined-auth.guard';
|
||||
import { ModelConfigService } from '../model-config/model-config.service';
|
||||
import { I18nService } from '../i18n/i18n.service';
|
||||
|
||||
class ChatDto {
|
||||
prompt: string;
|
||||
}
|
||||
|
||||
@Controller()
|
||||
export class ApiController {
|
||||
constructor(
|
||||
private readonly apiService: ApiService,
|
||||
private readonly modelConfigService: ModelConfigService,
|
||||
private readonly i18nService: I18nService,
|
||||
) {}
|
||||
|
||||
@Get('health')
|
||||
healthCheck() {
|
||||
return this.apiService.healthCheck();
|
||||
}
|
||||
|
||||
@Post('chat')
|
||||
@UseGuards(CombinedAuthGuard)
|
||||
@HttpCode(HttpStatus.OK)
|
||||
async chat(@Request() req, @Body() chatDto: ChatDto) {
|
||||
const { prompt } = chatDto;
|
||||
if (!prompt) {
|
||||
throw new Error(this.i18nService.getMessage('promptRequired'));
|
||||
}
|
||||
|
||||
try {
|
||||
// ユーザーの LLM モデル設定を取得
|
||||
const models = await this.modelConfigService.findAll();
|
||||
const llmModel = models.find((m) => m.type === 'llm');
|
||||
if (!llmModel) {
|
||||
throw new Error(this.i18nService.getMessage('addLLMConfig'));
|
||||
}
|
||||
|
||||
// API key is optional - allows local models
|
||||
|
||||
const modelConfigForService = {
|
||||
id: llmModel.id,
|
||||
name: llmModel.name,
|
||||
modelId: llmModel.modelId,
|
||||
baseUrl: llmModel.baseUrl,
|
||||
apiKey: llmModel.apiKey,
|
||||
type: llmModel.type as any,
|
||||
};
|
||||
|
||||
const response = await this.apiService.getChatCompletion(
|
||||
prompt,
|
||||
modelConfigForService,
|
||||
);
|
||||
return { response };
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
error.message || this.i18nService.getMessage('internalServerError'),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { ApiController } from './api.controller';
|
||||
import { ApiService } from './api.service';
|
||||
import { ApiV1Controller } from './api-v1.controller';
|
||||
import { KnowledgeBaseModule } from '../knowledge-base/knowledge-base.module';
|
||||
import { AuthModule } from '../auth/auth.module';
|
||||
import { TenantModule } from '../tenant/tenant.module';
|
||||
import { ModelConfigModule } from '../model-config/model-config.module';
|
||||
import { RagModule } from '../rag/rag.module';
|
||||
import { ChatModule } from '../chat/chat.module';
|
||||
import { UserModule } from '../user/user.module';
|
||||
import { MulterModule } from '@nestjs/platform-express';
|
||||
import { memoryStorage } from 'multer';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
KnowledgeBaseModule,
|
||||
AuthModule,
|
||||
ModelConfigModule,
|
||||
RagModule,
|
||||
ChatModule,
|
||||
UserModule,
|
||||
TenantModule,
|
||||
MulterModule.register({ storage: memoryStorage() }),
|
||||
],
|
||||
controllers: [ApiController, ApiV1Controller],
|
||||
providers: [ApiService],
|
||||
})
|
||||
export class ApiModule {}
|
||||
@@ -0,0 +1,48 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { ChatOpenAI } from '@langchain/openai';
|
||||
import { ModelConfig } from '../types';
|
||||
import { I18nService } from '../i18n/i18n.service';
|
||||
|
||||
@Injectable()
|
||||
export class ApiService {
|
||||
constructor(private i18nService: I18nService) {}
|
||||
|
||||
// Simple health check method
|
||||
healthCheck() {
|
||||
return { status: 'ok', message: 'API is healthy' };
|
||||
}
|
||||
|
||||
async getChatCompletion(
|
||||
prompt: string,
|
||||
modelConfig: ModelConfig,
|
||||
): Promise<string> {
|
||||
// API key is optional - allows local models
|
||||
|
||||
try {
|
||||
const llm = this.createLLM(modelConfig);
|
||||
const response = await llm.invoke(prompt);
|
||||
return response.content.toString();
|
||||
} catch (error) {
|
||||
console.error('LangChain call failed:', error);
|
||||
if (error.message?.includes('401')) {
|
||||
throw new Error(this.i18nService.getMessage('invalidApiKey'));
|
||||
}
|
||||
throw new Error(
|
||||
this.i18nService.formatMessage('apiCallFailed', {
|
||||
message: error.message,
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private createLLM(modelConfig: ModelConfig): ChatOpenAI {
|
||||
return new ChatOpenAI({
|
||||
temperature: 0.7,
|
||||
apiKey: modelConfig.apiKey,
|
||||
modelName: modelConfig.modelId,
|
||||
configuration: {
|
||||
baseURL: modelConfig.baseUrl || 'http://localhost:11434/v1',
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import { Test, TestingModule } from '@nestjs/testing';
|
||||
import { AppController } from './app.controller';
|
||||
import { AppService } from './app.service';
|
||||
|
||||
describe('AppController', () => {
|
||||
let appController: AppController;
|
||||
|
||||
beforeEach(async () => {
|
||||
const app: TestingModule = await Test.createTestingModule({
|
||||
controllers: [AppController],
|
||||
providers: [AppService],
|
||||
}).compile();
|
||||
|
||||
appController = app.get<AppController>(AppController);
|
||||
});
|
||||
|
||||
describe('root', () => {
|
||||
it('should return "Hello World!"', () => {
|
||||
expect(appController.getHello()).toBe('Hello World!');
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,12 @@
|
||||
import { Controller, Get } from '@nestjs/common';
|
||||
import { AppService } from './app.service';
|
||||
|
||||
@Controller()
|
||||
export class AppController {
|
||||
constructor(private readonly appService: AppService) {}
|
||||
|
||||
@Get()
|
||||
getHello(): string {
|
||||
return this.appService.getHello();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
import { MiddlewareConsumer, Module, NestModule } from '@nestjs/common';
|
||||
import { ConfigModule, ConfigService } from '@nestjs/config';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
import { ScheduleModule } from '@nestjs/schedule';
|
||||
import { ServeStaticModule } from '@nestjs/serve-static';
|
||||
import { join } from 'path';
|
||||
import { APP_GUARD, APP_INTERCEPTOR } from '@nestjs/core';
|
||||
import { AppController } from './app.controller';
|
||||
import { AppService } from './app.service';
|
||||
import { ApiModule } from './api/api.module';
|
||||
import { ElasticsearchModule } from './elasticsearch/elasticsearch.module';
|
||||
import { UploadModule } from './upload/upload.module';
|
||||
import { ChatModule } from './chat/chat.module';
|
||||
import { AuthModule } from './auth/auth.module';
|
||||
import { I18nModule } from './i18n/i18n.module';
|
||||
import { JwtAuthGuard } from './auth/jwt-auth.guard';
|
||||
import { CombinedAuthGuard } from './auth/combined-auth.guard';
|
||||
import { KnowledgeBaseModule } from './knowledge-base/knowledge-base.module';
|
||||
import { ModelConfigModule } from './model-config/model-config.module';
|
||||
import { UserModule } from './user/user.module';
|
||||
import { TikaModule } from './tika/tika.module';
|
||||
import { VisionModule } from './vision/vision.module';
|
||||
import { LibreOfficeModule } from './libreoffice/libreoffice.module';
|
||||
import { Pdf2ImageModule } from './pdf2image/pdf2image.module';
|
||||
import { VisionPipelineModule } from './vision-pipeline/vision-pipeline.module';
|
||||
import { KnowledgeGroupModule } from './knowledge-group/knowledge-group.module';
|
||||
import { SearchHistoryModule } from './search-history/search-history.module';
|
||||
import { NoteModule } from './note/note.module';
|
||||
import { PodcastModule } from './podcasts/podcast.module';
|
||||
import { ImportTaskModule } from './import-task/import-task.module';
|
||||
import { AssessmentModule } from './assessment/assessment.module';
|
||||
import { I18nMiddleware } from './i18n/i18n.middleware';
|
||||
import { TenantMiddleware } from './tenant/tenant.middleware';
|
||||
import { User } from './user/user.entity';
|
||||
import { UserSetting } from './user/user-setting.entity';
|
||||
import { ModelConfig } from './model-config/model-config.entity';
|
||||
import { KnowledgeBase } from './knowledge-base/knowledge-base.entity';
|
||||
import { KnowledgeGroup } from './knowledge-group/knowledge-group.entity';
|
||||
import { SearchHistory } from './search-history/search-history.entity';
|
||||
import { ChatMessage } from './search-history/chat-message.entity';
|
||||
import { Note } from './note/note.entity';
|
||||
import { NoteCategory } from './note/note-category.entity';
|
||||
import { PodcastEpisode } from './podcasts/entities/podcast-episode.entity';
|
||||
import { ImportTask } from './import-task/import-task.entity';
|
||||
import { AssessmentSession } from './assessment/entities/assessment-session.entity';
|
||||
import { AssessmentQuestion } from './assessment/entities/assessment-question.entity';
|
||||
import { AssessmentAnswer } from './assessment/entities/assessment-answer.entity';
|
||||
import { AssessmentTemplate } from './assessment/entities/assessment-template.entity';
|
||||
import { Tenant } from './tenant/tenant.entity';
|
||||
import { TenantSetting } from './tenant/tenant-setting.entity';
|
||||
import { ApiKey } from './auth/entities/api-key.entity';
|
||||
import { TenantMember } from './tenant/tenant-member.entity';
|
||||
import { TenantModule } from './tenant/tenant.module';
|
||||
import { SuperAdminModule } from './super-admin/super-admin.module';
|
||||
import { AdminModule } from './admin/admin.module';
|
||||
import { FeishuModule } from './feishu/feishu.module';
|
||||
import { FeishuBot } from './feishu/entities/feishu-bot.entity';
|
||||
import { FeishuAssessmentSession } from './feishu/entities/feishu-assessment-session.entity';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
ConfigModule.forRoot({
|
||||
isGlobal: true,
|
||||
ignoreEnvFile: false,
|
||||
}),
|
||||
ServeStaticModule.forRoot({
|
||||
rootPath: join(process.cwd(), 'uploads'),
|
||||
serveRoot: '/uploads',
|
||||
}),
|
||||
ScheduleModule.forRoot(),
|
||||
TypeOrmModule.forRootAsync({
|
||||
imports: [ConfigModule],
|
||||
inject: [ConfigService],
|
||||
useFactory: (configService: ConfigService) => ({
|
||||
type: 'better-sqlite3',
|
||||
database: configService.get<string>('DATABASE_PATH'),
|
||||
entities: [
|
||||
User,
|
||||
UserSetting,
|
||||
ModelConfig,
|
||||
KnowledgeBase,
|
||||
KnowledgeGroup,
|
||||
SearchHistory,
|
||||
ChatMessage,
|
||||
Note,
|
||||
NoteCategory,
|
||||
PodcastEpisode,
|
||||
ImportTask,
|
||||
AssessmentSession,
|
||||
AssessmentQuestion,
|
||||
AssessmentAnswer,
|
||||
AssessmentTemplate,
|
||||
Tenant,
|
||||
TenantSetting,
|
||||
TenantMember,
|
||||
ApiKey,
|
||||
FeishuBot,
|
||||
FeishuAssessmentSession,
|
||||
],
|
||||
synchronize: true, // Auto-create database schema. Disable in production.
|
||||
}),
|
||||
}),
|
||||
AuthModule,
|
||||
I18nModule,
|
||||
UserModule,
|
||||
TenantModule,
|
||||
ModelConfigModule,
|
||||
KnowledgeBaseModule,
|
||||
KnowledgeGroupModule,
|
||||
SearchHistoryModule,
|
||||
NoteModule,
|
||||
PodcastModule,
|
||||
TikaModule,
|
||||
VisionModule,
|
||||
LibreOfficeModule,
|
||||
Pdf2ImageModule,
|
||||
VisionPipelineModule,
|
||||
ApiModule,
|
||||
ElasticsearchModule,
|
||||
UploadModule,
|
||||
ChatModule,
|
||||
ImportTaskModule,
|
||||
AssessmentModule,
|
||||
SuperAdminModule,
|
||||
AdminModule,
|
||||
FeishuModule,
|
||||
],
|
||||
controllers: [AppController],
|
||||
providers: [
|
||||
AppService,
|
||||
{
|
||||
provide: APP_GUARD,
|
||||
useClass: CombinedAuthGuard,
|
||||
},
|
||||
],
|
||||
})
|
||||
export class AppModule implements NestModule {
|
||||
configure(consumer: MiddlewareConsumer) {
|
||||
consumer.apply(I18nMiddleware, TenantMiddleware).forRoutes('*');
|
||||
}
|
||||
}
|
||||
// Trigger restart correct
|
||||
@@ -0,0 +1,8 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
@Injectable()
|
||||
export class AppService {
|
||||
getHello(): string {
|
||||
return 'Hello World!';
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
import { Test, TestingModule } from '@nestjs/testing';
|
||||
import { Reflector } from '@nestjs/core';
|
||||
import { AssessmentController } from './assessment.controller';
|
||||
import { AssessmentService } from './assessment.service';
|
||||
import { TenantService } from '../tenant/tenant.service';
|
||||
import { UserService } from '../user/user.service';
|
||||
import { CombinedAuthGuard } from '../auth/combined-auth.guard';
|
||||
|
||||
describe('AssessmentController', () => {
|
||||
let controller: AssessmentController;
|
||||
|
||||
const mockService = () => ({});
|
||||
const mockReflector = () => ({
|
||||
get: jest.fn(),
|
||||
getAllAndOverride: jest.fn(),
|
||||
});
|
||||
const mockGuard = () => ({
|
||||
canActivate: jest.fn().mockReturnValue(true),
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
controllers: [AssessmentController],
|
||||
providers: [
|
||||
{ provide: AssessmentService, useFactory: mockService },
|
||||
{ provide: 'UserService', useFactory: mockService },
|
||||
{ provide: TenantService, useFactory: mockService },
|
||||
{ provide: Reflector, useFactory: mockReflector },
|
||||
{ provide: CombinedAuthGuard, useFactory: mockGuard },
|
||||
],
|
||||
}).compile();
|
||||
|
||||
controller = module.get<AssessmentController>(AssessmentController);
|
||||
});
|
||||
|
||||
it('should be defined', () => {
|
||||
expect(controller).toBeDefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,161 @@
|
||||
import {
|
||||
Controller,
|
||||
Post,
|
||||
Body,
|
||||
Get,
|
||||
Param,
|
||||
UseGuards,
|
||||
Request,
|
||||
Sse,
|
||||
MessageEvent,
|
||||
Query,
|
||||
Delete,
|
||||
} from '@nestjs/common';
|
||||
import { map } from 'rxjs/operators';
|
||||
import { AssessmentService } from './assessment.service';
|
||||
import { CombinedAuthGuard } from '../auth/combined-auth.guard';
|
||||
import { ApiTags, ApiOperation, ApiResponse } from '@nestjs/swagger';
|
||||
|
||||
@ApiTags('Assessment')
|
||||
@Controller('assessment')
|
||||
@UseGuards(CombinedAuthGuard)
|
||||
export class AssessmentController {
|
||||
constructor(private readonly assessmentService: AssessmentService) {}
|
||||
|
||||
@Post('start')
|
||||
@ApiOperation({ summary: 'Start a new assessment session' })
|
||||
async startSession(
|
||||
@Request() req: any,
|
||||
@Body()
|
||||
body: { knowledgeBaseId?: string; language?: string; templateId?: string },
|
||||
) {
|
||||
const { id: userId, tenantId } = req.user;
|
||||
console.log(
|
||||
`[AssessmentController] startSession: user=${userId}, tenant=${tenantId}, templateId=${body.templateId}, kbId=${body.knowledgeBaseId}`,
|
||||
);
|
||||
return this.assessmentService.startSession(
|
||||
userId,
|
||||
body.knowledgeBaseId,
|
||||
tenantId,
|
||||
body.language,
|
||||
body.templateId,
|
||||
);
|
||||
}
|
||||
|
||||
@Post(':id/answer')
|
||||
@ApiOperation({ summary: 'Submit an answer to the current question' })
|
||||
async submitAnswer(
|
||||
@Request() req: any,
|
||||
@Param('id') sessionId: string,
|
||||
@Body() body: { answer: string; language?: string },
|
||||
) {
|
||||
const { id: userId } = req.user;
|
||||
console.log(
|
||||
`[AssessmentController] >>> submitAnswer CALLED: user=${userId}, session=${sessionId}, answerLen=${body.answer?.length}`,
|
||||
);
|
||||
return this.assessmentService.submitAnswer(
|
||||
sessionId,
|
||||
userId,
|
||||
body.answer,
|
||||
body.language,
|
||||
);
|
||||
}
|
||||
|
||||
@Sse(':id/start-stream')
|
||||
@ApiOperation({ summary: 'Stream initial session generation' })
|
||||
startSessionStream(@Request() req: any, @Param('id') sessionId: string) {
|
||||
const { id: userId } = req.user;
|
||||
console.log(
|
||||
`[AssessmentController] startSessionStream: user=${userId}, session=${sessionId}`,
|
||||
);
|
||||
return this.assessmentService
|
||||
.startSessionStream(sessionId, userId)
|
||||
.pipe(map((data) => ({ data }) as MessageEvent));
|
||||
}
|
||||
|
||||
@Sse(':id/answer-stream')
|
||||
@ApiOperation({
|
||||
summary: 'Stream answer evaluation and next question generation',
|
||||
})
|
||||
submitAnswerStream(
|
||||
@Request() req: any,
|
||||
@Param('id') sessionId: string,
|
||||
@Query('answer') answer: string,
|
||||
@Query('language') language?: string,
|
||||
) {
|
||||
const { id: userId } = req.user;
|
||||
console.log(
|
||||
`[AssessmentController] >>> submitAnswerStream CALLED: user=${userId}, session=${sessionId}, answerLen=${answer?.length}, lang=${language}`,
|
||||
);
|
||||
return this.assessmentService
|
||||
.submitAnswerStream(sessionId, userId, answer, language)
|
||||
.pipe(map((data) => ({ data }) as MessageEvent));
|
||||
}
|
||||
|
||||
@Get(':id/state')
|
||||
@ApiOperation({ summary: 'Get the current state of an assessment session' })
|
||||
async getSessionState(@Request() req: any, @Param('id') sessionId: string) {
|
||||
const { id: userId } = req.user;
|
||||
console.log(
|
||||
`[AssessmentController] getSessionState: user=${userId}, session=${sessionId}`,
|
||||
);
|
||||
return this.assessmentService.getSessionState(sessionId, userId);
|
||||
}
|
||||
|
||||
@Get()
|
||||
@ApiOperation({ summary: 'Get assessment session history' })
|
||||
async getHistory(@Request() req: any) {
|
||||
const { id: userId, tenantId } = req.user;
|
||||
console.log(
|
||||
`[AssessmentController] getHistory: user=${userId}, tenant=${tenantId}`,
|
||||
);
|
||||
return this.assessmentService.getHistory(userId, tenantId);
|
||||
}
|
||||
|
||||
@Delete(':id')
|
||||
@ApiOperation({ summary: 'Delete an assessment session' })
|
||||
async deleteSession(@Request() req: any, @Param('id') sessionId: string) {
|
||||
const user = req.user;
|
||||
console.log(
|
||||
`[AssessmentController] deleteSession: user=${user.id}, role=${user.role}, session=${sessionId}`,
|
||||
);
|
||||
return this.assessmentService.deleteSession(sessionId, user);
|
||||
}
|
||||
|
||||
@Get(':id/certificate')
|
||||
@ApiOperation({ summary: 'Get certificate for completed assessment' })
|
||||
async getCertificate(
|
||||
@Request() req: any,
|
||||
@Param('id') sessionId: string,
|
||||
) {
|
||||
const { id: userId, tenantId } = req.user;
|
||||
console.log(
|
||||
`[AssessmentController] getCertificate: user=${userId}, session=${sessionId}`,
|
||||
);
|
||||
return this.assessmentService.generateCertificate(sessionId, userId, tenantId);
|
||||
}
|
||||
|
||||
@Get('stats')
|
||||
@ApiOperation({ summary: 'Get assessment statistics for admin' })
|
||||
async getStats(
|
||||
@Request() req: any,
|
||||
@Query('startDate') startDate?: string,
|
||||
@Query('endDate') endDate?: string,
|
||||
@Query('templateId') templateId?: string,
|
||||
@Query('knowledgeGroupId') knowledgeGroupId?: string,
|
||||
) {
|
||||
const { id: userId, tenantId, role } = req.user;
|
||||
console.log(
|
||||
`[AssessmentController] getStats: user=${userId}, role=${role}, tenant=${tenantId}`,
|
||||
);
|
||||
return this.assessmentService.getStats(
|
||||
userId,
|
||||
tenantId,
|
||||
role,
|
||||
startDate,
|
||||
endDate,
|
||||
templateId,
|
||||
knowledgeGroupId,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
import { Module, forwardRef } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
import { AssessmentService } from './assessment.service';
|
||||
import { AssessmentController } from './assessment.controller';
|
||||
import { AssessmentSession } from './entities/assessment-session.entity';
|
||||
import { AssessmentQuestion } from './entities/assessment-question.entity';
|
||||
import { AssessmentAnswer } from './entities/assessment-answer.entity';
|
||||
import { AssessmentTemplate } from './entities/assessment-template.entity';
|
||||
import { AssessmentCertificate } from './entities/assessment-certificate.entity';
|
||||
import { QuestionBank } from './entities/question-bank.entity';
|
||||
import { QuestionBankItem } from './entities/question-bank-item.entity';
|
||||
import { KnowledgeBaseModule } from '../knowledge-base/knowledge-base.module';
|
||||
import { KnowledgeGroupModule } from '../knowledge-group/knowledge-group.module';
|
||||
import { ModelConfigModule } from '../model-config/model-config.module';
|
||||
import { ChatModule } from '../chat/chat.module';
|
||||
import { ElasticsearchModule } from '../elasticsearch/elasticsearch.module';
|
||||
import { RagModule } from '../rag/rag.module';
|
||||
import { TenantModule } from '../tenant/tenant.module';
|
||||
import { TemplateService } from './services/template.service';
|
||||
import { TemplateController } from './controllers/template.controller';
|
||||
import { QuestionBankController } from './controllers/question-bank.controller';
|
||||
import { ContentFilterService } from './services/content-filter.service';
|
||||
import { QuestionOutlineService } from './services/question-outline.service';
|
||||
import { QuestionBankService } from './services/question-bank.service';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
TypeOrmModule.forFeature([
|
||||
AssessmentSession,
|
||||
AssessmentQuestion,
|
||||
AssessmentAnswer,
|
||||
AssessmentTemplate,
|
||||
AssessmentCertificate,
|
||||
QuestionBank,
|
||||
QuestionBankItem,
|
||||
]),
|
||||
forwardRef(() => KnowledgeBaseModule),
|
||||
forwardRef(() => KnowledgeGroupModule),
|
||||
forwardRef(() => ModelConfigModule),
|
||||
forwardRef(() => ChatModule),
|
||||
ElasticsearchModule,
|
||||
RagModule,
|
||||
TenantModule,
|
||||
],
|
||||
controllers: [AssessmentController, TemplateController, QuestionBankController],
|
||||
providers: [
|
||||
AssessmentService,
|
||||
TemplateService,
|
||||
ContentFilterService,
|
||||
QuestionOutlineService,
|
||||
QuestionBankService,
|
||||
],
|
||||
exports: [AssessmentService, TemplateService, QuestionOutlineService, QuestionBankService],
|
||||
})
|
||||
export class AssessmentModule {}
|
||||
@@ -0,0 +1,78 @@
|
||||
import { Test, TestingModule } from '@nestjs/testing';
|
||||
import { getRepositoryToken } from '@nestjs/typeorm';
|
||||
import { AssessmentService } from './assessment.service';
|
||||
import { AssessmentSession } from './entities/assessment-session.entity';
|
||||
import { AssessmentQuestion } from './entities/assessment-question.entity';
|
||||
import { AssessmentAnswer } from './entities/assessment-answer.entity';
|
||||
import { AssessmentCertificate } from './entities/assessment-certificate.entity';
|
||||
import { KnowledgeBaseService } from '../knowledge-base/knowledge-base.service';
|
||||
import { KnowledgeGroupService } from '../knowledge-group/knowledge-group.service';
|
||||
import { ModelConfigService } from '../model-config/model-config.service';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { TemplateService } from './services/template.service';
|
||||
import { ContentFilterService } from './services/content-filter.service';
|
||||
import { QuestionOutlineService } from './services/question-outline.service';
|
||||
import { QuestionBankService } from './services/question-bank.service';
|
||||
import { RagService } from '../rag/rag.service';
|
||||
import { ChatService } from '../chat/chat.service';
|
||||
import { I18nService } from '../i18n/i18n.service';
|
||||
import { TenantService } from '../tenant/tenant.service';
|
||||
import { NotFoundException } from '@nestjs/common';
|
||||
|
||||
describe('AssessmentService', () => {
|
||||
let service: AssessmentService;
|
||||
let sessionRepository: any;
|
||||
|
||||
const mockRepository = () => ({
|
||||
delete: jest.fn(),
|
||||
find: jest.fn(),
|
||||
findOne: jest.fn(),
|
||||
save: jest.fn(),
|
||||
});
|
||||
|
||||
const mockService = () => ({});
|
||||
|
||||
beforeEach(async () => {
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
providers: [
|
||||
AssessmentService,
|
||||
{ provide: getRepositoryToken(AssessmentSession), useFactory: mockRepository },
|
||||
{ provide: getRepositoryToken(AssessmentQuestion), useFactory: mockRepository },
|
||||
{ provide: getRepositoryToken(AssessmentAnswer), useFactory: mockRepository },
|
||||
{ provide: getRepositoryToken(AssessmentCertificate), useFactory: mockRepository },
|
||||
{ provide: KnowledgeBaseService, useFactory: mockService },
|
||||
{ provide: KnowledgeGroupService, useFactory: mockService },
|
||||
{ provide: ModelConfigService, useFactory: mockService },
|
||||
{ provide: ConfigService, useFactory: mockService },
|
||||
{ provide: TemplateService, useFactory: mockService },
|
||||
{ provide: ContentFilterService, useFactory: mockService },
|
||||
{ provide: QuestionOutlineService, useFactory: mockService },
|
||||
{ provide: QuestionBankService, useFactory: mockService },
|
||||
{ provide: RagService, useFactory: mockService },
|
||||
{ provide: ChatService, useFactory: mockService },
|
||||
{ provide: I18nService, useFactory: mockService },
|
||||
{ provide: TenantService, useFactory: mockService },
|
||||
],
|
||||
}).compile();
|
||||
|
||||
service = module.get<AssessmentService>(AssessmentService);
|
||||
sessionRepository = module.get(getRepositoryToken(AssessmentSession));
|
||||
});
|
||||
|
||||
it('should be defined', () => {
|
||||
expect(service).toBeDefined();
|
||||
});
|
||||
|
||||
describe('deleteSession', () => {
|
||||
it('should delete a session if it exists and belongs to the user', async () => {
|
||||
sessionRepository.delete.mockResolvedValue({ affected: 1 });
|
||||
await expect(service.deleteSession('session-id', 'user-id')).resolves.not.toThrow();
|
||||
expect(sessionRepository.delete).toHaveBeenCalledWith({ id: 'session-id', userId: 'user-id' });
|
||||
});
|
||||
|
||||
it('should throw NotFoundException if no session was affected', async () => {
|
||||
sessionRepository.delete.mockResolvedValue({ affected: 0 });
|
||||
await expect(service.deleteSession('non-existent', 'user-id')).rejects.toThrow(NotFoundException);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,1441 @@
|
||||
import {
|
||||
Injectable,
|
||||
Logger,
|
||||
NotFoundException,
|
||||
Inject,
|
||||
forwardRef,
|
||||
} from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Repository, DeepPartial, In } from 'typeorm';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { ChatOpenAI } from '@langchain/openai';
|
||||
import {
|
||||
HumanMessage,
|
||||
BaseMessage,
|
||||
AIMessage,
|
||||
SystemMessage,
|
||||
} from '@langchain/core/messages';
|
||||
import { Observable, from, map, mergeMap, concatMap } from 'rxjs';
|
||||
import {
|
||||
AssessmentSession,
|
||||
AssessmentStatus,
|
||||
} from './entities/assessment-session.entity';
|
||||
import { AssessmentQuestion } from './entities/assessment-question.entity';
|
||||
import { AssessmentAnswer } from './entities/assessment-answer.entity';
|
||||
import { AssessmentTemplate } from './entities/assessment-template.entity';
|
||||
import { AssessmentCertificate } from './entities/assessment-certificate.entity';
|
||||
import { QuestionBank, QuestionBankStatus } from './entities/question-bank.entity';
|
||||
import { QuestionBankItem } from './entities/question-bank-item.entity';
|
||||
import { KnowledgeBaseService } from '../knowledge-base/knowledge-base.service';
|
||||
import { KnowledgeGroupService } from '../knowledge-group/knowledge-group.service';
|
||||
import { ModelConfigService } from '../model-config/model-config.service';
|
||||
import { ModelType } from '../types';
|
||||
import { ElasticsearchService } from '../elasticsearch/elasticsearch.service';
|
||||
import { RagService } from '../rag/rag.service';
|
||||
import { ChatService } from '../chat/chat.service';
|
||||
import { createEvaluationGraph } from './graph/builder';
|
||||
import { EvaluationState } from './graph/state';
|
||||
import { TemplateService } from './services/template.service';
|
||||
import { ContentFilterService } from './services/content-filter.service';
|
||||
import { QuestionOutlineService } from './services/question-outline.service';
|
||||
import { QuestionBankService } from './services/question-bank.service';
|
||||
import { I18nService } from '../i18n/i18n.service';
|
||||
import { TenantService } from '../tenant/tenant.service';
|
||||
|
||||
@Injectable()
|
||||
export class AssessmentService {
|
||||
private readonly logger = new Logger(AssessmentService.name);
|
||||
private readonly graph = createEvaluationGraph();
|
||||
|
||||
constructor(
|
||||
@InjectRepository(AssessmentSession)
|
||||
private sessionRepository: Repository<AssessmentSession>,
|
||||
@InjectRepository(AssessmentQuestion)
|
||||
private questionRepository: Repository<AssessmentQuestion>,
|
||||
@InjectRepository(AssessmentAnswer)
|
||||
private answerRepository: Repository<AssessmentAnswer>,
|
||||
@InjectRepository(AssessmentCertificate)
|
||||
private certificateRepository: Repository<AssessmentCertificate>,
|
||||
@InjectRepository(QuestionBank)
|
||||
private questionBankRepository: Repository<QuestionBank>,
|
||||
@InjectRepository(QuestionBankItem)
|
||||
private questionBankItemRepository: Repository<QuestionBankItem>,
|
||||
@Inject(forwardRef(() => KnowledgeBaseService))
|
||||
private kbService: KnowledgeBaseService,
|
||||
@Inject(forwardRef(() => KnowledgeGroupService))
|
||||
private groupService: KnowledgeGroupService,
|
||||
@Inject(forwardRef(() => ModelConfigService))
|
||||
private modelConfigService: ModelConfigService,
|
||||
private configService: ConfigService,
|
||||
private templateService: TemplateService,
|
||||
private contentFilterService: ContentFilterService,
|
||||
private questionOutlineService: QuestionOutlineService,
|
||||
private questionBankService: QuestionBankService,
|
||||
private ragService: RagService,
|
||||
@Inject(forwardRef(() => ChatService))
|
||||
private chatService: ChatService,
|
||||
private i18nService: I18nService,
|
||||
private tenantService: TenantService,
|
||||
) {}
|
||||
|
||||
private async getModel(tenantId: string): Promise<ChatOpenAI> {
|
||||
const config = await this.modelConfigService.findDefaultByType(
|
||||
tenantId,
|
||||
ModelType.LLM,
|
||||
);
|
||||
this.logger.debug(`[getModel] config: modelId=${config.modelId}, baseUrl=${config.baseUrl}, hasApiKey=${!!config.apiKey}`);
|
||||
return new ChatOpenAI({
|
||||
apiKey: config.apiKey || 'ollama',
|
||||
modelName: config.modelId,
|
||||
temperature: 0.7,
|
||||
configuration: {
|
||||
baseURL: config.baseUrl || 'https://api.deepseek.com/v1',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
private async getMultiGroupContent(
|
||||
groupIds: string[],
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
templateJson?: any,
|
||||
): Promise<string> {
|
||||
this.logger.log(`[getMultiGroupContent] Starting for ${groupIds.length} groups`);
|
||||
const contents: string[] = [];
|
||||
const dimensionMap: Record<string, string> = {
|
||||
prompt: '技术能力-提示词',
|
||||
llm: '技术能力-LLM',
|
||||
ide: 'IDE协作能力',
|
||||
devPattern: 'AI开发范式',
|
||||
workCapability: '工作能力-安全',
|
||||
};
|
||||
|
||||
for (let i = 0; i < groupIds.length; i++) {
|
||||
const groupId = groupIds[i];
|
||||
try {
|
||||
const files = await this.groupService.getGroupFiles(groupId, userId, tenantId);
|
||||
const groupContent = files
|
||||
.filter((f: any) => f.content)
|
||||
.map((f: any) => {
|
||||
const dimension = Object.keys(dimensionMap)[i] || '工作能力-安全';
|
||||
return `=== [${dimension}] ===\n${f.content}`;
|
||||
})
|
||||
.join('\n\n');
|
||||
if (groupContent) {
|
||||
contents.push(groupContent);
|
||||
}
|
||||
} catch (err) {
|
||||
this.logger.warn(`[getMultiGroupContent] Failed to get files for group ${groupId}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
const result = contents.join('\n\n');
|
||||
this.logger.log(`[getMultiGroupContent] Total content length: ${result.length}`);
|
||||
return result;
|
||||
}
|
||||
|
||||
private calculateScores(
|
||||
questions: any[],
|
||||
scores: Record<string, number>,
|
||||
weightConfig: { prompt: number; other: number },
|
||||
): { finalScore: number; dimensionScores: Record<string, number>; radarData: Record<string, number> } {
|
||||
console.log('[calculateScores] Input:', {
|
||||
questionsCount: questions.length,
|
||||
scores,
|
||||
weightConfig,
|
||||
});
|
||||
|
||||
const dimensionScoresMap: Record<string, number[]> = {
|
||||
prompt: [],
|
||||
llm: [],
|
||||
ide: [],
|
||||
devPattern: [],
|
||||
workCapability: [],
|
||||
};
|
||||
|
||||
questions.forEach((q: any, idx: number) => {
|
||||
const dimension = q.dimension || 'workCapability';
|
||||
const score = scores[q.id || idx.toString()] || 0;
|
||||
if (dimensionScoresMap[dimension]) {
|
||||
dimensionScoresMap[dimension].push(score);
|
||||
} else {
|
||||
dimensionScoresMap.workCapability.push(score);
|
||||
}
|
||||
});
|
||||
|
||||
const dimensionAverages: Record<string, number> = {};
|
||||
Object.keys(dimensionScoresMap).forEach(dim => {
|
||||
const arr = dimensionScoresMap[dim];
|
||||
dimensionAverages[dim] = arr.length > 0 ? arr.reduce((a, b) => a + b, 0) / arr.length : 0;
|
||||
});
|
||||
|
||||
const promptAvg = dimensionAverages.prompt || 0;
|
||||
// 只计算有题目的维度，不要把0分算进去
|
||||
const otherDims = ['llm', 'ide', 'devPattern', 'workCapability'];
|
||||
const otherDimsWithScores = otherDims.filter(dim => dimensionScoresMap[dim]?.length > 0);
|
||||
const otherAvg = otherDimsWithScores.length > 0
|
||||
? otherDimsWithScores.reduce((sum, dim) => sum + (dimensionAverages[dim] || 0), 0) / otherDimsWithScores.length
|
||||
: 0;
|
||||
|
||||
console.log('[calculateScores] Scoring debug:', { promptAvg, otherDimsWithScores, otherAvg, workCapability: dimensionAverages.workCapability });
|
||||
|
||||
const finalScore = promptAvg * (weightConfig.prompt / 100) + otherAvg * (weightConfig.other / 100);
|
||||
|
||||
const radarData: Record<string, number> = {};
|
||||
Object.keys(dimensionAverages).forEach(dim => {
|
||||
radarData[dim] = Math.round(dimensionAverages[dim] * 10) / 10;
|
||||
});
|
||||
|
||||
console.log('[calculateScores] Result:', {
|
||||
finalScore: Math.round(finalScore * 10) / 10,
|
||||
dimensionScores: dimensionAverages,
|
||||
promptAvg,
|
||||
otherAvg,
|
||||
});
|
||||
|
||||
return {
|
||||
finalScore: Math.round(finalScore * 10) / 10,
|
||||
dimensionScores: dimensionAverages,
|
||||
radarData,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts a new assessment session.
|
||||
*/
|
||||
|
||||
private async getSessionContent(session: {
|
||||
knowledgeBaseId?: string | null;
|
||||
knowledgeGroupId?: string | null;
|
||||
userId: string;
|
||||
tenantId: string;
|
||||
templateJson?: any;
|
||||
}): Promise<string> {
|
||||
const linkedGroupIds = session.templateJson?.linkedGroupIds;
|
||||
if (linkedGroupIds && linkedGroupIds.length > 0) {
|
||||
return this.getMultiGroupContent(linkedGroupIds, session.userId, session.tenantId, session.templateJson);
|
||||
}
|
||||
|
||||
const kbId = session.knowledgeBaseId || session.knowledgeGroupId;
|
||||
this.logger.log(`[getSessionContent] Starting for KB/Group ID: ${kbId}`);
|
||||
if (!kbId) {
|
||||
this.logger.warn(`[getSessionContent] No KB/Group ID provided`);
|
||||
return '';
|
||||
}
|
||||
|
||||
const keywords = session.templateJson?.keywords || [];
|
||||
|
||||
// If keywords are provided, use RagService (Hybrid Search) to find relevant content
|
||||
if (keywords.length > 0) {
|
||||
this.logger.log(
|
||||
`[getSessionContent] Keywords detected, performing hybrid search via RagService: ${keywords.join(', ')}`,
|
||||
);
|
||||
|
||||
try {
|
||||
// 1. Determine file IDs to include in search
|
||||
let fileIds: string[] = [];
|
||||
if (session.knowledgeBaseId) {
|
||||
fileIds = [session.knowledgeBaseId];
|
||||
} else if (session.knowledgeGroupId) {
|
||||
fileIds = await this.groupService.getFileIdsByGroups(
|
||||
[session.knowledgeGroupId],
|
||||
session.userId,
|
||||
session.tenantId,
|
||||
);
|
||||
}
|
||||
|
||||
if (fileIds.length > 0) {
|
||||
const query = keywords.join(' ');
|
||||
this.logger.log(
|
||||
`[getSessionContent] Performing high-fidelity grounded search (streamChat-style). Keywords: "${query}"`,
|
||||
);
|
||||
|
||||
// 1. Get default embedding model (strict logic from streamChat)
|
||||
const embeddingModel =
|
||||
await this.modelConfigService.findDefaultByType(
|
||||
session.tenantId || 'default',
|
||||
ModelType.EMBEDDING,
|
||||
);
|
||||
|
||||
// 2. Perform advanced RAG search
|
||||
const ragResults = await this.ragService.searchKnowledge(
|
||||
query,
|
||||
session.userId,
|
||||
20, // Increased topK to 20 for broader question coverage
|
||||
0.1, // Lenient similarityThreshold (Chat/Rag defaults are 0.3)
|
||||
embeddingModel?.id,
|
||||
true, // enableFullTextSearch
|
||||
true, // enableRerank
|
||||
undefined, // selectedRerankId
|
||||
undefined, // selectedGroups
|
||||
fileIds,
|
||||
0.3, // Lenient rerankSimilarityThreshold (Chat/Rag defaults are 0.5)
|
||||
session.tenantId,
|
||||
);
|
||||
|
||||
// 3. Format context using localized labels (equivalent to buildContext)
|
||||
const language = session.templateJson?.language || 'zh';
|
||||
const searchContent = ragResults
|
||||
.map((result, index) => {
|
||||
// this.logger.debug(`[getSessionContent] Found chunk [${index + 1}]: score=${result.score.toFixed(4)}, file=${result.fileName}, contentPreview=${result.content}...`);
|
||||
return `[${index + 1}] ${this.i18nService.getMessage('file', language)}：${result.fileName}\n${this.i18nService.getMessage('content', language)}：${result.content}\n`;
|
||||
})
|
||||
.join('\n');
|
||||
|
||||
if (searchContent && searchContent.trim().length > 0) {
|
||||
this.logger.log(
|
||||
`[getSessionContent] SUCCESS: Found ${ragResults.length} relevant chunks. Total length: ${searchContent.length}`,
|
||||
);
|
||||
// this.logger.log(`[getSessionContent] --- AI Context Start ---\n${searchContent}\n[getSessionContent] --- AI Context End ---`);
|
||||
return searchContent;
|
||||
} else {
|
||||
this.logger.warn(
|
||||
`[getSessionContent] High-fidelity search returned no results for query: "${query}".`,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
this.logger.warn(
|
||||
`[getSessionContent] No files found for search scope (KB: ${session.knowledgeBaseId}, Group: ${session.knowledgeGroupId})`,
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
this.logger.error(
|
||||
`[getSessionContent] Grounded search failed unexpectedly: ${err.message}`,
|
||||
err.stack,
|
||||
);
|
||||
}
|
||||
|
||||
this.logger.warn(
|
||||
`[getSessionContent] Grounded search failed or returned nothing. One common reason is that the keywords are not present in the indexed documents.`,
|
||||
);
|
||||
}
|
||||
|
||||
// Fallback or No Keywords: Original behavior (full content retrieval)
|
||||
let content = '';
|
||||
|
||||
if (session.knowledgeBaseId) {
|
||||
this.logger.debug(
|
||||
`[getSessionContent] Fetching content for KnowledgeBase: ${kbId}`,
|
||||
);
|
||||
const kb = await (this.kbService as any).kbRepository.findOne({
|
||||
where: { id: kbId, tenantId: session.tenantId },
|
||||
});
|
||||
if (kb) {
|
||||
content = kb.content || '';
|
||||
this.logger.debug(
|
||||
`[getSessionContent] Found KB content, length: ${content.length}`,
|
||||
);
|
||||
} else {
|
||||
this.logger.warn(
|
||||
`[getSessionContent] KnowledgeBase not found: ${kbId}`,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
this.logger.debug(
|
||||
`[getSessionContent] Fetching content for KnowledgeGroup: ${kbId}`,
|
||||
);
|
||||
const groupFiles = await this.groupService.getGroupFiles(
|
||||
kbId,
|
||||
session.userId,
|
||||
session.tenantId,
|
||||
);
|
||||
this.logger.debug(
|
||||
`[getSessionContent] Found ${groupFiles.length} files in group`,
|
||||
);
|
||||
content = groupFiles
|
||||
.filter((f) => f.content)
|
||||
.map((f) => {
|
||||
this.logger.debug(
|
||||
`[getSessionContent] Including file: ${f.title || f.originalName}, content length: ${f.content?.length || 0}`,
|
||||
);
|
||||
return `--- Document: ${f.title || f.originalName} ---\n${f.content}`;
|
||||
})
|
||||
.join('\n\n');
|
||||
this.logger.debug(
|
||||
`[getSessionContent] Total group content length: ${content.length}`,
|
||||
);
|
||||
} catch (err) {
|
||||
this.logger.error(
|
||||
`[getSessionContent] Failed to get group files: ${err.message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Apply keyword filter (regex based) as an extra layer if still using full content
|
||||
if (content && keywords.length > 0) {
|
||||
this.logger.debug(
|
||||
`[getSessionContent] Applying fallback keyword filters: ${keywords.join(', ')}`,
|
||||
);
|
||||
const prevLen = content.length;
|
||||
content = this.contentFilterService.filterContent(content, keywords);
|
||||
this.logger.debug(
|
||||
`[getSessionContent] After filtering, content length: ${content.length} (was ${prevLen})`,
|
||||
);
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`[getSessionContent] Final content for AI generation (Length: ${content.length})`,
|
||||
);
|
||||
this.logger.debug(
|
||||
`[getSessionContent] Content Preview: ${content.substring(0, 500)}...`,
|
||||
);
|
||||
|
||||
return content;
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts a new assessment session.
|
||||
* kbId can be a KnowledgeBase ID or a KnowledgeGroup ID.
|
||||
*/
|
||||
async startSession(
|
||||
userId: string,
|
||||
kbId: string | undefined,
|
||||
tenantId: string,
|
||||
language: string = 'en',
|
||||
templateId?: string,
|
||||
): Promise<AssessmentSession> {
|
||||
this.logger.log(
|
||||
`[startSession] Starting session for user ${userId}, templateId: ${templateId}, kbId: ${kbId}`,
|
||||
);
|
||||
let template: AssessmentTemplate | null = null;
|
||||
if (templateId) {
|
||||
template = await this.templateService.findOne(
|
||||
templateId,
|
||||
userId,
|
||||
tenantId,
|
||||
);
|
||||
this.logger.debug(
|
||||
`[startSession] Found template: ${template?.name}, linked group: ${template?.knowledgeGroupId}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Use kbId if provided, otherwise fall back to template's group ID
|
||||
const activeKbId = kbId || template?.knowledgeGroupId;
|
||||
this.logger.log(`[startSession] activeKbId resolved to: ${activeKbId}`);
|
||||
if (!activeKbId) {
|
||||
this.logger.error(`[startSession] No knowledge source resolved`);
|
||||
throw new Error('Knowledge source (ID or Template) must be provided.');
|
||||
}
|
||||
|
||||
// Try to determine if it's a KB or Group and check permissions
|
||||
let isKb = false;
|
||||
try {
|
||||
await this.kbService.findOne(activeKbId, userId, tenantId);
|
||||
isKb = true;
|
||||
} catch (kbError) {
|
||||
if (kbError instanceof NotFoundException) {
|
||||
// Try finding it as a Group
|
||||
try {
|
||||
await this.groupService.findOne(activeKbId, userId, tenantId);
|
||||
} catch (groupError) {
|
||||
this.logger.error(
|
||||
`[startSession] Knowledge source ${activeKbId} not found as KB or Group`,
|
||||
);
|
||||
throw new NotFoundException(
|
||||
this.i18nService.getMessage('knowledgeSourceNotFound') ||
|
||||
'Knowledge source not found',
|
||||
);
|
||||
}
|
||||
} else {
|
||||
throw kbError; // e.g. ForbiddenException
|
||||
}
|
||||
}
|
||||
this.logger.debug(`[startSession] isKb: ${isKb}`);
|
||||
|
||||
const templateData = template
|
||||
? {
|
||||
name: template.name,
|
||||
keywords: template.keywords,
|
||||
questionCount: template.questionCount,
|
||||
questionCountMin: template.questionCountMin,
|
||||
questionCountMax: template.questionCountMax,
|
||||
difficultyDistribution: template.difficultyDistribution,
|
||||
difficultyConfig: template.difficultyConfig,
|
||||
weightConfig: template.weightConfig,
|
||||
passingScore: template.passingScore,
|
||||
style: template.style,
|
||||
linkedGroupIds: template.linkedGroupIds,
|
||||
}
|
||||
: undefined;
|
||||
|
||||
let questionsFromBank: any[] = [];
|
||||
let questionSource: 'bank' | 'generator' = 'generator';
|
||||
|
||||
if (templateId) {
|
||||
try {
|
||||
const targetCount = template?.questionCount || 5;
|
||||
const publishedBanks = await this.questionBankRepository.find({
|
||||
where: { templateId, status: QuestionBankStatus.PUBLISHED },
|
||||
});
|
||||
|
||||
if (publishedBanks.length > 0) {
|
||||
const bankIds = publishedBanks.map(b => b.id);
|
||||
const questionCount = await this.questionBankItemRepository.count({
|
||||
where: { bankId: In(bankIds) },
|
||||
});
|
||||
|
||||
this.logger.log(
|
||||
`[startSession] Found ${publishedBanks.length} published banks with ${questionCount} questions, target: ${targetCount}`,
|
||||
);
|
||||
|
||||
if (questionCount >= targetCount) {
|
||||
const bankId = publishedBanks[0].id;
|
||||
const selectedItems = await this.questionBankService.selectQuestions(
|
||||
bankId,
|
||||
targetCount,
|
||||
);
|
||||
|
||||
questionsFromBank = selectedItems.map(item => ({
|
||||
id: item.id,
|
||||
questionText: item.questionText,
|
||||
questionType: item.questionType,
|
||||
keyPoints: item.keyPoints,
|
||||
difficulty: item.difficulty,
|
||||
dimension: item.dimension,
|
||||
basis: item.basis,
|
||||
}));
|
||||
|
||||
questionSource = 'bank';
|
||||
this.logger.log(
|
||||
`[startSession] Selected ${questionsFromBank.length} questions from question bank`,
|
||||
);
|
||||
} else {
|
||||
this.logger.log(
|
||||
`[startSession] Question bank has insufficient questions (${questionCount} < ${targetCount}), will use LLM generation`,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
this.logger.log(
|
||||
`[startSession] No published question banks found for template ${templateId}, will use LLM generation`,
|
||||
);
|
||||
}
|
||||
} catch (err: any) {
|
||||
this.logger.warn(`Bank query failed: ${err.message}, falling back to LLM generation`);
|
||||
}
|
||||
}
|
||||
|
||||
const sessionData: any = {
|
||||
userId,
|
||||
tenantId,
|
||||
knowledgeBaseId: isKb ? activeKbId : undefined,
|
||||
knowledgeGroupId: isKb ? undefined : activeKbId,
|
||||
templateId,
|
||||
templateJson: templateData,
|
||||
status: AssessmentStatus.IN_PROGRESS,
|
||||
language,
|
||||
questions_json: questionsFromBank.length > 0 ? questionsFromBank : [],
|
||||
questionSource,
|
||||
};
|
||||
|
||||
const content = await this.getSessionContent(sessionData);
|
||||
|
||||
if (!content || content.trim().length < 10) {
|
||||
this.logger.error(
|
||||
`[startSession] Insufficient content length: ${content?.length || 0}`,
|
||||
);
|
||||
throw new Error(
|
||||
'Selected knowledge source has no sufficient content for evaluation.',
|
||||
);
|
||||
}
|
||||
|
||||
const session = this.sessionRepository.create(
|
||||
sessionData as DeepPartial<AssessmentSession>,
|
||||
);
|
||||
const savedSession = (await this.sessionRepository.save(
|
||||
session as any,
|
||||
)) as AssessmentSession;
|
||||
|
||||
// Thread ID for LangGraph is the session ID
|
||||
savedSession.threadId = savedSession.id;
|
||||
await this.sessionRepository.save(savedSession);
|
||||
|
||||
this.logger.log(
|
||||
`[startSession] Session ${savedSession.id} created and saved`,
|
||||
);
|
||||
return savedSession;
|
||||
}
|
||||
|
||||
/**
|
||||
* Specialized streaming start for initial generation.
|
||||
*/
|
||||
startSessionStream(sessionId: string, userId: string): Observable<any> {
|
||||
return new Observable((observer) => {
|
||||
(async () => {
|
||||
try {
|
||||
const session = await this.sessionRepository.findOne({
|
||||
where: { id: sessionId, userId },
|
||||
});
|
||||
if (!session) {
|
||||
observer.error(new NotFoundException('Session not found'));
|
||||
return;
|
||||
}
|
||||
|
||||
const model = await this.getModel(session.tenantId);
|
||||
const content = await this.getSessionContent(session);
|
||||
|
||||
// Check if questions already exist in session (from question bank)
|
||||
const existingQuestions = session.questions_json || [];
|
||||
const hasExistingQuestions = existingQuestions.length > 0;
|
||||
|
||||
// Check if we already have state
|
||||
const existingState = await this.graph.getState({
|
||||
configurable: { thread_id: sessionId },
|
||||
});
|
||||
if (
|
||||
existingState &&
|
||||
existingState.values &&
|
||||
existingState.values.questions?.length > 0
|
||||
) {
|
||||
this.logger.log(
|
||||
`Session ${sessionId} already has state, skipping generation.`,
|
||||
);
|
||||
const mappedData = { ...existingState.values };
|
||||
mappedData.messages = this.mapMessages(mappedData.messages || []);
|
||||
mappedData.feedbackHistory = this.mapMessages(
|
||||
mappedData.feedbackHistory || [],
|
||||
);
|
||||
observer.next({ type: 'final', data: mappedData });
|
||||
observer.complete();
|
||||
return;
|
||||
}
|
||||
|
||||
const initialState: Partial<EvaluationState> = {
|
||||
assessmentSessionId: sessionId,
|
||||
knowledgeBaseId:
|
||||
session.knowledgeBaseId || session.knowledgeGroupId || '',
|
||||
messages: [],
|
||||
questionCount: session.templateJson?.questionCount,
|
||||
|
||||
difficultyDistribution:
|
||||
session.templateJson?.difficultyDistribution,
|
||||
|
||||
style: session.templateJson?.style,
|
||||
keywords: session.templateJson?.keywords,
|
||||
currentQuestionIndex: 0,
|
||||
};
|
||||
|
||||
const isZh = (session.language || 'en') === 'zh';
|
||||
const isJa = session.language === 'ja';
|
||||
|
||||
const hasQuestionsFromBank = hasExistingQuestions;
|
||||
|
||||
if (hasQuestionsFromBank) {
|
||||
this.logger.log(
|
||||
`[startSessionStream] Using ${existingQuestions.length} questions from question bank`,
|
||||
);
|
||||
initialState.questions = existingQuestions;
|
||||
initialState.messages = [
|
||||
new HumanMessage(
|
||||
isZh ? '我已准备好回答问题。' : isJa ? '質問への回答準備ができています。' : 'I am ready to answer the questions.',
|
||||
),
|
||||
];
|
||||
}
|
||||
|
||||
const initialMsg = isZh
|
||||
? '现在生成评估问题。请务必使用中文。'
|
||||
: isJa
|
||||
? '今すぐアセスメント問題を生成してください。必ず日本語で回答してください。'
|
||||
: 'Generate the assessment questions now. Please strictly respond in English.';
|
||||
|
||||
this.logger.log(
|
||||
`[startSessionStream] Starting stream for session ${sessionId}`,
|
||||
);
|
||||
const stream = await this.graph.stream(
|
||||
{
|
||||
...initialState,
|
||||
language: session.language || 'en',
|
||||
messages: hasQuestionsFromBank
|
||||
? initialState.messages
|
||||
: [new HumanMessage(initialMsg)],
|
||||
},
|
||||
{
|
||||
configurable: {
|
||||
thread_id: sessionId,
|
||||
model,
|
||||
knowledgeBaseContent: content,
|
||||
language: session.language || 'en',
|
||||
targetCount: session.templateJson?.questionCount || 5,
|
||||
questionCount: session.templateJson?.questionCount,
|
||||
|
||||
difficultyDistribution:
|
||||
session.templateJson?.difficultyDistribution,
|
||||
|
||||
style: session.templateJson?.style,
|
||||
keywords: session.templateJson?.keywords,
|
||||
},
|
||||
streamMode: ['values', 'updates'],
|
||||
},
|
||||
);
|
||||
|
||||
this.logger.debug(`[startSessionStream] Graph stream started`);
|
||||
|
||||
let hasEmittedQuestion = false;
|
||||
|
||||
for await (const [mode, data] of stream) {
|
||||
if (mode === 'updates') {
|
||||
const node = Object.keys(data)[0];
|
||||
const updateData = { ...data[node] };
|
||||
if (updateData.messages) {
|
||||
updateData.messages = this.mapMessages(updateData.messages);
|
||||
}
|
||||
if (updateData.feedbackHistory) {
|
||||
updateData.feedbackHistory = this.mapMessages(
|
||||
updateData.feedbackHistory,
|
||||
);
|
||||
}
|
||||
if (node === 'interviewer' && !hasEmittedQuestion && hasQuestionsFromBank) {
|
||||
updateData.questions = existingQuestions;
|
||||
hasEmittedQuestion = true;
|
||||
}
|
||||
observer.next({ type: 'node', node, data: updateData });
|
||||
}
|
||||
}
|
||||
|
||||
// After stream, get the latest authoritative state from checkpointer
|
||||
const fullState = await this.graph.getState({
|
||||
configurable: { thread_id: sessionId },
|
||||
});
|
||||
const finalData = fullState.values as EvaluationState;
|
||||
|
||||
if (finalData && finalData.messages) {
|
||||
console.log(
|
||||
`[AssessmentService] startSessionStream Final Authoritative State messages:`,
|
||||
finalData.messages.length,
|
||||
);
|
||||
session.messages = finalData.messages;
|
||||
session.feedbackHistory = finalData.feedbackHistory || [];
|
||||
session.questions_json = hasQuestionsFromBank && existingQuestions.length > 0
|
||||
? existingQuestions
|
||||
: finalData.questions;
|
||||
session.currentQuestionIndex = finalData.currentQuestionIndex;
|
||||
session.followUpCount = finalData.followUpCount || 0;
|
||||
|
||||
if (finalData.report) {
|
||||
session.status = AssessmentStatus.COMPLETED;
|
||||
session.finalReport = finalData.report;
|
||||
const scores = finalData.scores;
|
||||
const questions = finalData.questions || [];
|
||||
const weightConfig = session.templateJson?.weightConfig || { prompt: 50, other: 50 };
|
||||
const passingScore = session.templateJson?.passingScore || 90;
|
||||
|
||||
if (questions.length > 0 && Object.keys(scores).length > 0) {
|
||||
const { finalScore, dimensionScores, radarData } = this.calculateScores(
|
||||
questions,
|
||||
scores,
|
||||
weightConfig,
|
||||
);
|
||||
session.finalScore = finalScore;
|
||||
(session as any).dimensionScores = dimensionScores;
|
||||
(session as any).radarData = radarData;
|
||||
(session as any).passed = finalScore >= passingScore;
|
||||
}
|
||||
}
|
||||
await this.sessionRepository.save(session);
|
||||
|
||||
const mappedData: any = { ...finalData };
|
||||
mappedData.messages = this.mapMessages(finalData.messages);
|
||||
mappedData.feedbackHistory = this.mapMessages(
|
||||
finalData.feedbackHistory || [],
|
||||
);
|
||||
mappedData.status = session.status;
|
||||
mappedData.report = session.finalReport;
|
||||
mappedData.finalScore = session.finalScore;
|
||||
observer.next({ type: 'final', data: mappedData });
|
||||
}
|
||||
|
||||
observer.complete();
|
||||
} catch (err) {
|
||||
observer.error(err);
|
||||
}
|
||||
})();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Submits a user's answer and continues the assessment.
|
||||
*/
|
||||
async submitAnswer(
|
||||
sessionId: string,
|
||||
userId: string,
|
||||
answer: string,
|
||||
language: string = 'en',
|
||||
): Promise<any> {
|
||||
const session = await this.sessionRepository.findOne({
|
||||
where: { id: sessionId, userId },
|
||||
relations: ['template'],
|
||||
});
|
||||
if (!session) throw new NotFoundException('Session not found');
|
||||
|
||||
const model = await this.getModel(session.tenantId);
|
||||
await this.ensureGraphState(sessionId, session);
|
||||
const content = await this.getSessionContent(session);
|
||||
|
||||
// Update state with human message first to ensure it's in history before resumption
|
||||
await this.graph.updateState(
|
||||
{ configurable: { thread_id: sessionId } },
|
||||
{ messages: [new HumanMessage(answer)] },
|
||||
);
|
||||
|
||||
this.logger.debug(`[submitAnswer] Resuming graph for session ${sessionId}`);
|
||||
|
||||
let finalResult: any = null;
|
||||
const weightConfig = session.templateJson?.weightConfig || { prompt: 50, other: 50 };
|
||||
const passingScore = session.templateJson?.passingScore || 90;
|
||||
|
||||
// Resume from the last interrupt (typically after interviewer)
|
||||
const stream = await this.graph.stream(null, {
|
||||
configurable: {
|
||||
thread_id: sessionId,
|
||||
model,
|
||||
knowledgeBaseContent: content,
|
||||
language: session.language || language,
|
||||
targetCount: session.templateJson?.questionCount || 5,
|
||||
questionCount: session.templateJson?.questionCount,
|
||||
|
||||
difficultyDistribution: session.templateJson?.difficultyDistribution,
|
||||
weightConfig: weightConfig,
|
||||
passingScore: passingScore,
|
||||
style: session.templateJson?.style,
|
||||
keywords: session.templateJson?.keywords,
|
||||
},
|
||||
streamMode: ['values', 'updates'],
|
||||
});
|
||||
|
||||
for await (const [mode, data] of stream) {
|
||||
if (mode === 'values') {
|
||||
// This might be the interrupt info if interrupted
|
||||
finalResult = data;
|
||||
} else if (mode === 'updates') {
|
||||
const nodeName = Object.keys(data)[0];
|
||||
this.logger.debug(`[submitAnswer] Node completed: ${nodeName}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Always get the latest authoritative state from checkpointer after the stream
|
||||
const fullState = await this.graph.getState({
|
||||
configurable: { thread_id: sessionId },
|
||||
});
|
||||
finalResult = fullState.values as EvaluationState;
|
||||
|
||||
this.logger.log(
|
||||
`[submitAnswer] Stream finished. State Index: ${finalResult.currentQuestionIndex}, Questions: ${finalResult.questions?.length}, HasReport: ${!!finalResult.report}`,
|
||||
);
|
||||
|
||||
if (finalResult && (finalResult.messages || finalResult.questions)) {
|
||||
session.messages = finalResult.messages;
|
||||
session.questions_json = finalResult.questions;
|
||||
session.currentQuestionIndex = finalResult.currentQuestionIndex;
|
||||
session.followUpCount = finalResult.followUpCount || 0;
|
||||
|
||||
if (finalResult.report) {
|
||||
session.status = AssessmentStatus.COMPLETED;
|
||||
session.finalReport = finalResult.report;
|
||||
const scores = finalResult.scores as Record<string, number>;
|
||||
const questions = finalResult.questions || [];
|
||||
const weightConfig = session.templateJson?.weightConfig || { prompt: 50, other: 50 };
|
||||
const passingScore = session.templateJson?.passingScore || 90;
|
||||
|
||||
if (questions.length > 0 && Object.keys(scores).length > 0) {
|
||||
const { finalScore, dimensionScores, radarData } = this.calculateScores(
|
||||
questions,
|
||||
scores,
|
||||
weightConfig,
|
||||
);
|
||||
session.finalScore = finalScore;
|
||||
(session as any).dimensionScores = dimensionScores;
|
||||
(session as any).radarData = radarData;
|
||||
(session as any).passed = finalScore >= passingScore;
|
||||
}
|
||||
}
|
||||
|
||||
session.feedbackHistory = finalResult.feedbackHistory || [];
|
||||
await this.sessionRepository.save(session);
|
||||
|
||||
// Map result for return
|
||||
finalResult.messages = this.mapMessages(finalResult.messages);
|
||||
finalResult.feedbackHistory = this.mapMessages(
|
||||
finalResult.feedbackHistory || [],
|
||||
);
|
||||
finalResult.report = session.finalReport;
|
||||
finalResult.finalScore = session.finalScore;
|
||||
finalResult.dimensionScores = (session as any).dimensionScores;
|
||||
finalResult.radarData = (session as any).radarData;
|
||||
finalResult.passed = (session as any).passed;
|
||||
|
||||
this.logger.log(
|
||||
`[submitAnswer] session saved. DB Status: ${session.status}, Index: ${session.currentQuestionIndex}`,
|
||||
);
|
||||
this.logger.log(
|
||||
`[submitAnswer] finalResult check: hasQuestions=${!!finalResult.questions}, questionsLen=${finalResult.questions?.length}, hasReport=${!!finalResult.report}`,
|
||||
);
|
||||
this.logger.debug(
|
||||
`[submitAnswer] finalResult keys: ${Object.keys(finalResult).join(', ')}`,
|
||||
);
|
||||
this.logger.log(
|
||||
`[submitAnswer] session updated: status=${session.status}, index=${session.currentQuestionIndex}`,
|
||||
);
|
||||
} else {
|
||||
this.logger.warn(
|
||||
`[submitAnswer] finalResult has no usable data! Keys: ${Object.keys(finalResult || {}).join(', ')}`,
|
||||
);
|
||||
}
|
||||
|
||||
return finalResult;
|
||||
}
|
||||
|
||||
/**
|
||||
* Streaming version of submitAnswer.
|
||||
*/
|
||||
submitAnswerStream(
|
||||
sessionId: string,
|
||||
userId: string,
|
||||
answer: string,
|
||||
language: string = 'en',
|
||||
): Observable<any> {
|
||||
console.log('[submitAnswerStream] START - sessionId:', sessionId, 'answer length:', answer?.length);
|
||||
let emittedNextQuestion = false;
|
||||
let hasEmittedNodes = false;
|
||||
return new Observable((observer) => {
|
||||
(async () => {
|
||||
try {
|
||||
console.log('[submitAnswerStream] After Observable - sessionId:', sessionId);
|
||||
const session = await this.sessionRepository.findOne({
|
||||
where: { id: sessionId, userId },
|
||||
});
|
||||
if (!session) {
|
||||
observer.error(new NotFoundException('Session not found'));
|
||||
return;
|
||||
}
|
||||
|
||||
const model = await this.getModel(session.tenantId);
|
||||
const content = await this.getSessionContent(session);
|
||||
await this.ensureGraphState(sessionId, session);
|
||||
const graphState = await this.graph.getState({
|
||||
configurable: { thread_id: sessionId },
|
||||
});
|
||||
const hasState =
|
||||
graphState &&
|
||||
graphState.values &&
|
||||
Object.keys(graphState.values).length > 0;
|
||||
console.log(
|
||||
`[AssessmentService] submitAnswerStream: sessionId=${sessionId}, hasState=${hasState}, nextNodes=[${graphState.next || ''}]`,
|
||||
);
|
||||
|
||||
// Update state with human message first to ensure it's in history
|
||||
await this.graph.updateState(
|
||||
{ configurable: { thread_id: sessionId } },
|
||||
{ messages: [new HumanMessage(answer)] },
|
||||
);
|
||||
|
||||
// Resume from the last interrupt
|
||||
const stream = await this.graph.stream(null, {
|
||||
configurable: {
|
||||
thread_id: sessionId,
|
||||
model,
|
||||
knowledgeBaseContent: content,
|
||||
language: session.language || language,
|
||||
targetCount: session.templateJson?.questionCount || 5,
|
||||
},
|
||||
streamMode: ['values', 'updates'],
|
||||
});
|
||||
|
||||
let streamCount = 0;
|
||||
let hasEmittedNodes = false;
|
||||
for await (const [mode, data] of stream) {
|
||||
streamCount++;
|
||||
console.log('[submitAnswerStream] Stream event:', streamCount, mode, Object.keys(data || {}));
|
||||
console.log('[submitAnswerStream] Data detail:', JSON.stringify(data).substring(0, 500));
|
||||
if (mode === 'updates') {
|
||||
hasEmittedNodes = true;
|
||||
const node = Object.keys(data)[0];
|
||||
const updateData = { ...data[node] };
|
||||
|
||||
// Skip interrupt nodes - they have no useful data
|
||||
if (node === '__interrupt__' || !updateData || Object.keys(updateData).length === 0) {
|
||||
console.log('[submitAnswerStream] Skipping empty interrupt node');
|
||||
continue;
|
||||
}
|
||||
|
||||
console.log('[submitAnswerStream] Node update:', node, {
|
||||
hasMessages: !!updateData.messages,
|
||||
messageCount: updateData.messages?.length,
|
||||
currentIndex: updateData.currentQuestionIndex,
|
||||
dataKeys: Object.keys(updateData).join(',')
|
||||
});
|
||||
console.log('[submitAnswerStream] Sending to frontend:', JSON.stringify(updateData).substring(0, 500));
|
||||
if (updateData.messages) {
|
||||
updateData.messages = this.mapMessages(updateData.messages);
|
||||
}
|
||||
if (updateData.feedbackHistory) {
|
||||
updateData.feedbackHistory = this.mapMessages(
|
||||
updateData.feedbackHistory,
|
||||
);
|
||||
}
|
||||
observer.next({ type: 'node', node, data: updateData });
|
||||
} else if (mode === 'values') {
|
||||
console.log('[submitAnswerStream] Values update - keys:', Object.keys(data || {}));
|
||||
}
|
||||
}
|
||||
|
||||
// After stream, get authoritative state
|
||||
const fullState = await this.graph.getState({
|
||||
configurable: { thread_id: sessionId },
|
||||
});
|
||||
const finalData = fullState.values as EvaluationState;
|
||||
|
||||
// Force emit the next question if stream didn't emit updates (hasEmittedNodes is false)
|
||||
console.log('[submitAnswerStream] Force check:', { hasEmittedNodes, hasFinalData: !!finalData, hasQuestions: !!finalData?.questions, qLen: finalData?.questions?.length, emittedNextQuestion });
|
||||
if (!hasEmittedNodes && finalData && finalData.questions && finalData.questions.length > 0 && !emittedNextQuestion) {
|
||||
const currentIndex = finalData.currentQuestionIndex || 0;
|
||||
const nextQuestion = finalData.questions[currentIndex];
|
||||
if (nextQuestion) {
|
||||
const questionText = nextQuestion.questionText || '';
|
||||
console.log('[submitAnswerStream] Forcing emit next question:', {
|
||||
currentIndex,
|
||||
questionPreview: questionText.substring(0, 50)
|
||||
});
|
||||
const { HumanMessage, AIMessage } = await import('@langchain/core/messages');
|
||||
observer.next({
|
||||
type: 'node',
|
||||
node: 'interviewer',
|
||||
data: {
|
||||
messages: [new AIMessage(`问题 ${currentIndex + 1}: ${questionText}\n\n请提供您的回答。`)],
|
||||
currentQuestionIndex: currentIndex,
|
||||
questions: finalData.questions,
|
||||
shouldFollowUp: false,
|
||||
}
|
||||
});
|
||||
emittedNextQuestion = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (finalData && finalData.messages) {
|
||||
console.log(
|
||||
`[AssessmentService] submitAnswerStream Final Authoritative State messages:`,
|
||||
finalData.messages.length,
|
||||
);
|
||||
session.messages = finalData.messages;
|
||||
session.feedbackHistory = finalData.feedbackHistory || [];
|
||||
session.questions_json = finalData.questions;
|
||||
session.currentQuestionIndex = finalData.currentQuestionIndex;
|
||||
session.followUpCount = finalData.followUpCount || 0;
|
||||
|
||||
if (finalData.report) {
|
||||
session.status = AssessmentStatus.COMPLETED;
|
||||
session.finalReport = finalData.report;
|
||||
const scores = finalData.scores;
|
||||
const questions = finalData.questions || [];
|
||||
const weightConfig = session.templateJson?.weightConfig || { prompt: 50, other: 50 };
|
||||
const passingScore = session.templateJson?.passingScore || 90;
|
||||
|
||||
if (questions.length > 0 && Object.keys(scores).length > 0) {
|
||||
const { finalScore, dimensionScores, radarData } = this.calculateScores(
|
||||
questions,
|
||||
scores,
|
||||
weightConfig,
|
||||
);
|
||||
session.finalScore = finalScore;
|
||||
(session as any).dimensionScores = dimensionScores;
|
||||
(session as any).radarData = radarData;
|
||||
(session as any).passed = finalScore >= passingScore;
|
||||
this.logger.log(
|
||||
`[DimensionScoring] Session ${sessionId} Final Score: ${finalScore}, Passed: ${finalScore >= passingScore}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
await this.sessionRepository.save(session);
|
||||
|
||||
const mappedData: any = { ...finalData };
|
||||
mappedData.messages = this.mapMessages(finalData.messages);
|
||||
mappedData.feedbackHistory = this.mapMessages(
|
||||
finalData.feedbackHistory || [],
|
||||
);
|
||||
mappedData.status = session.status;
|
||||
mappedData.report = session.finalReport;
|
||||
observer.next({ type: 'final', data: mappedData });
|
||||
}
|
||||
|
||||
observer.complete();
|
||||
} catch (err) {
|
||||
observer.error(err);
|
||||
}
|
||||
})();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves the current state of a session.
|
||||
*/
|
||||
async getSessionState(sessionId: string, userId: string): Promise<any> {
|
||||
this.logger.log(
|
||||
`Retrieving state for session ${sessionId} for user ${userId}`,
|
||||
);
|
||||
const session = await this.sessionRepository.findOne({
|
||||
where: { id: sessionId, userId },
|
||||
relations: ['template'],
|
||||
});
|
||||
if (!session) throw new NotFoundException('Session not found');
|
||||
|
||||
// Ensure graph has state (lazy init or recovery)
|
||||
await this.ensureGraphState(sessionId, session);
|
||||
|
||||
const state = await this.graph.getState({
|
||||
configurable: { thread_id: sessionId },
|
||||
});
|
||||
const values = { ...state.values };
|
||||
|
||||
if (values.messages) {
|
||||
values.messages = this.mapMessages(values.messages);
|
||||
}
|
||||
if (values.feedbackHistory) {
|
||||
values.feedbackHistory = this.mapMessages(values.feedbackHistory);
|
||||
}
|
||||
|
||||
return values;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves assessment session history for a user.
|
||||
*/
|
||||
async getHistory(
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
): Promise<AssessmentSession[]> {
|
||||
const history = await this.sessionRepository.find({
|
||||
where: { userId, tenantId },
|
||||
order: { createdAt: 'DESC' },
|
||||
relations: ['knowledgeBase', 'knowledgeGroup'],
|
||||
});
|
||||
|
||||
// Map questions_json to questions for frontend compatibility
|
||||
const mappedHistory = history.map((session) => ({
|
||||
...session,
|
||||
questions: session.questions_json || [],
|
||||
})) as any;
|
||||
|
||||
this.logger.log(`Found ${history.length} historical sessions`);
|
||||
return mappedHistory;
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes an assessment session.
|
||||
*/
|
||||
async deleteSession(sessionId: string, user: any): Promise<void> {
|
||||
this.logger.log(
|
||||
`Deleting session ${sessionId} for user ${user.id} (role: ${user.role})`,
|
||||
);
|
||||
|
||||
const userId = user.id;
|
||||
const isAdmin = user.role === 'super_admin' || user.role === 'admin';
|
||||
|
||||
const deleteCondition: any = { id: sessionId };
|
||||
if (!isAdmin) {
|
||||
deleteCondition.userId = userId;
|
||||
}
|
||||
|
||||
const result = await this.sessionRepository.delete(deleteCondition);
|
||||
if (result.affected === 0) {
|
||||
throw new NotFoundException(
|
||||
'Session not found or you do not have permission to delete it',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensures the graph checkpointer has the state for the given session.
|
||||
* Useful for lazy initialization and recovery after server restarts.
|
||||
*/
|
||||
private async ensureGraphState(
|
||||
sessionId: string,
|
||||
session: AssessmentSession,
|
||||
): Promise<void> {
|
||||
const state = await this.graph.getState({
|
||||
configurable: { thread_id: sessionId },
|
||||
});
|
||||
|
||||
if (
|
||||
!state.values ||
|
||||
Object.keys(state.values).length === 0 ||
|
||||
!state.values.messages ||
|
||||
state.values.messages.length === 0
|
||||
) {
|
||||
const hasHistory = session.messages && session.messages.length > 0;
|
||||
|
||||
if (hasHistory) {
|
||||
this.logger.log(
|
||||
`[ensureGraphState] Recovering state from DB for session ${sessionId}`,
|
||||
);
|
||||
const historicalMessages = this.hydrateMessages(session.messages);
|
||||
const existingQuestions = session.questions_json || [];
|
||||
const hasQuestionsFromBank = existingQuestions.length > 0;
|
||||
|
||||
if (hasQuestionsFromBank) {
|
||||
this.logger.log(
|
||||
`[ensureGraphState] Using ${existingQuestions.length} questions from question bank`,
|
||||
);
|
||||
await this.graph.updateState(
|
||||
{ configurable: { thread_id: sessionId } },
|
||||
{
|
||||
assessmentSessionId: sessionId,
|
||||
knowledgeBaseId:
|
||||
session.knowledgeBaseId || session.knowledgeGroupId || '',
|
||||
messages: historicalMessages,
|
||||
feedbackHistory: this.hydrateMessages(
|
||||
session.feedbackHistory || [],
|
||||
),
|
||||
questions: existingQuestions,
|
||||
currentQuestionIndex: session.currentQuestionIndex || 0,
|
||||
followUpCount: session.followUpCount || 0,
|
||||
questionCount: session.templateJson?.questionCount || 5,
|
||||
difficultyDistribution:
|
||||
session.templateJson?.difficultyDistribution,
|
||||
style: session.templateJson?.style,
|
||||
keywords: session.templateJson?.keywords,
|
||||
},
|
||||
'grader',
|
||||
);
|
||||
} else {
|
||||
await this.graph.updateState(
|
||||
{ configurable: { thread_id: sessionId } },
|
||||
{
|
||||
assessmentSessionId: sessionId,
|
||||
knowledgeBaseId:
|
||||
session.knowledgeBaseId || session.knowledgeGroupId || '',
|
||||
messages: historicalMessages,
|
||||
feedbackHistory: this.hydrateMessages(
|
||||
session.feedbackHistory || [],
|
||||
),
|
||||
questions: session.questions_json || [],
|
||||
currentQuestionIndex: session.currentQuestionIndex || 0,
|
||||
followUpCount: session.followUpCount || 0,
|
||||
questionCount: session.templateJson?.questionCount || 5,
|
||||
difficultyDistribution:
|
||||
session.templateJson?.difficultyDistribution,
|
||||
style: session.templateJson?.style,
|
||||
keywords: session.templateJson?.keywords,
|
||||
},
|
||||
'grader',
|
||||
);
|
||||
}
|
||||
} else {
|
||||
this.logger.log(`Initializing new state for session ${sessionId}`);
|
||||
const content = await this.getSessionContent(session);
|
||||
const model = await this.getModel(session.tenantId);
|
||||
|
||||
const initialState: Partial<EvaluationState> = {
|
||||
assessmentSessionId: sessionId,
|
||||
knowledgeBaseId:
|
||||
session.knowledgeBaseId || session.knowledgeGroupId || '',
|
||||
messages: [],
|
||||
questionCount: session.templateJson?.questionCount,
|
||||
difficultyDistribution: session.templateJson?.difficultyDistribution,
|
||||
style: session.templateJson?.style,
|
||||
keywords: session.templateJson?.keywords,
|
||||
language: session.language || 'en',
|
||||
};
|
||||
|
||||
this.logger.log(
|
||||
`[ensureGraphState] Initializing with questionCount=${initialState.questionCount}, keywords=${initialState.keywords?.join(',')}, style=${initialState.style}`,
|
||||
);
|
||||
|
||||
const resultStream = await this.graph.stream(initialState, {
|
||||
configurable: {
|
||||
thread_id: sessionId,
|
||||
model,
|
||||
knowledgeBaseContent: content,
|
||||
language: session.language || 'en',
|
||||
targetCount: session.templateJson?.questionCount || 5,
|
||||
keywords: session.templateJson?.keywords,
|
||||
questionCount: session.templateJson?.questionCount,
|
||||
difficultyDistribution:
|
||||
session.templateJson?.difficultyDistribution,
|
||||
style: session.templateJson?.style,
|
||||
},
|
||||
streamMode: ['values', 'updates'],
|
||||
});
|
||||
|
||||
let finalInvokeResult: any = null;
|
||||
const nodes: string[] = [];
|
||||
for await (const [mode, data] of resultStream) {
|
||||
if (mode === 'values') finalInvokeResult = data;
|
||||
else if (mode === 'updates') nodes.push(...Object.keys(data));
|
||||
}
|
||||
|
||||
if (finalInvokeResult.messages) {
|
||||
session.messages = finalInvokeResult.messages;
|
||||
session.feedbackHistory = finalInvokeResult.feedbackHistory || [];
|
||||
session.questions_json = finalInvokeResult.questions;
|
||||
session.currentQuestionIndex = finalInvokeResult.currentQuestionIndex;
|
||||
session.followUpCount = finalInvokeResult.followUpCount || 0;
|
||||
await this.sessionRepository.save(session);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-hydrates plain objects from DB into LangChain message instances.
|
||||
*/
|
||||
private hydrateMessages(messages: any[]): BaseMessage[] {
|
||||
if (!messages) return [];
|
||||
return messages.map((m) => {
|
||||
if (m instanceof BaseMessage) return m;
|
||||
|
||||
const content = m.content || m.text || (typeof m === 'string' ? m : '');
|
||||
const type = m.role || m.type || m._getType?.() || 'ai';
|
||||
|
||||
if (type === 'human' || type === 'user') {
|
||||
return new HumanMessage(content);
|
||||
} else if (type === 'ai' || type === 'assistant') {
|
||||
return new AIMessage(content);
|
||||
} else if (type === 'system') {
|
||||
return new SystemMessage(content);
|
||||
}
|
||||
return new AIMessage(content);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps LangChain messages to a simple format for the frontend and storage.
|
||||
*/
|
||||
private mapMessages(messages: BaseMessage[]): any[] {
|
||||
if (!messages) return [];
|
||||
return messages.map((msg) => {
|
||||
const type = msg._getType();
|
||||
let role: 'user' | 'assistant' | 'system' = 'system';
|
||||
|
||||
if (type === 'human') role = 'user';
|
||||
else if (type === 'ai') role = 'assistant';
|
||||
else if (type === 'system') role = 'system';
|
||||
|
||||
return {
|
||||
role,
|
||||
content: msg.content,
|
||||
type, // Also store the LangChain type for easier hydration
|
||||
timestamp: (msg as any).timestamp || Date.now(),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async generateCertificate(
|
||||
sessionId: string,
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
): Promise<AssessmentCertificate> {
|
||||
const session = await this.sessionRepository.findOne({
|
||||
where: { id: sessionId, userId },
|
||||
});
|
||||
if (!session) {
|
||||
throw new NotFoundException('Session not found');
|
||||
}
|
||||
|
||||
if (session.status !== AssessmentStatus.COMPLETED) {
|
||||
throw new Error('Session not completed');
|
||||
}
|
||||
|
||||
const existing = await this.certificateRepository.findOne({
|
||||
where: { sessionId },
|
||||
});
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
|
||||
const level = this.determineLevel(session.finalScore || 0);
|
||||
const qrCode = `cert://${sessionId}-${Date.now()}`;
|
||||
|
||||
const certificate = this.certificateRepository.create({
|
||||
userId,
|
||||
sessionId,
|
||||
templateId: session.templateId || '',
|
||||
level,
|
||||
totalScore: session.finalScore || 0,
|
||||
qrCode,
|
||||
dimensionScores: (session as any).dimensionScores,
|
||||
radarData: (session as any).radarData,
|
||||
passed: (session as any).passed || false,
|
||||
});
|
||||
|
||||
return this.certificateRepository.save(certificate);
|
||||
}
|
||||
|
||||
private determineLevel(score: number): string {
|
||||
if (score >= 90) return 'Expert';
|
||||
if (score >= 75) return 'Advanced';
|
||||
if (score >= 60) return 'Proficient';
|
||||
return 'Novice';
|
||||
}
|
||||
|
||||
async getStats(
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
role: string,
|
||||
startDate?: string,
|
||||
endDate?: string,
|
||||
templateId?: string,
|
||||
knowledgeGroupId?: string,
|
||||
): Promise<any> {
|
||||
const isAdmin = role === 'super_admin' || role === 'admin';
|
||||
|
||||
const qb = this.sessionRepository.createQueryBuilder('session');
|
||||
qb.where('session.tenantId = :tenantId', { tenantId });
|
||||
|
||||
if (!isAdmin) {
|
||||
qb.andWhere('session.userId = :userId', { userId });
|
||||
}
|
||||
|
||||
if (startDate) {
|
||||
qb.andWhere('session.createdAt >= :startDate', { startDate: new Date(startDate) });
|
||||
}
|
||||
if (endDate) {
|
||||
qb.andWhere('session.createdAt <= :endDate', { endDate: new Date(endDate) });
|
||||
}
|
||||
if (templateId) {
|
||||
qb.andWhere('session.templateId = :templateId', { templateId });
|
||||
}
|
||||
if (knowledgeGroupId) {
|
||||
qb.andWhere('session.knowledgeGroupId = :knowledgeGroupId', { knowledgeGroupId });
|
||||
}
|
||||
|
||||
const sessions = await qb
|
||||
.leftJoinAndSelect('session.template', 'template')
|
||||
.leftJoinAndSelect('session.knowledgeGroup', 'knowledgeGroup')
|
||||
.orderBy('session.createdAt', 'DESC')
|
||||
.take(100)
|
||||
.getMany();
|
||||
|
||||
const totalAttempts = sessions.length;
|
||||
const completedSessions = sessions.filter(s => s.status === AssessmentStatus.COMPLETED);
|
||||
const completedCount = completedSessions.length;
|
||||
const scores = completedSessions
|
||||
.map(s => s.finalScore)
|
||||
.filter((score): score is number => score !== null && score !== undefined);
|
||||
|
||||
const highestScore = scores.length > 0 ? Math.max(...scores) : 0;
|
||||
const averageScore = scores.length > 0
|
||||
? Math.round((scores.reduce((a, b) => a + b, 0) / scores.length) * 10) / 10
|
||||
: 0;
|
||||
const completionRate = totalAttempts > 0
|
||||
? Math.round((completedCount / totalAttempts) * 1000) / 10
|
||||
: 0;
|
||||
|
||||
const recentRecords = sessions.slice(0, 20).map(session => ({
|
||||
id: session.id,
|
||||
userId: session.userId,
|
||||
knowledgeBase: session.knowledgeBase?.name || session.knowledgeGroup?.name || '-',
|
||||
template: session.template?.name || '-',
|
||||
score: session.finalScore || null,
|
||||
status: session.status,
|
||||
createdAt: session.createdAt,
|
||||
user: isAdmin ? { id: session.userId } : undefined,
|
||||
}));
|
||||
|
||||
return {
|
||||
totalAttempts,
|
||||
highestScore,
|
||||
averageScore,
|
||||
completionRate,
|
||||
recentRecords,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
import {
|
||||
Controller,
|
||||
Get,
|
||||
Post,
|
||||
Put,
|
||||
Delete,
|
||||
Body,
|
||||
Param,
|
||||
Query,
|
||||
UseGuards,
|
||||
UsePipes,
|
||||
ValidationPipe,
|
||||
Req,
|
||||
Logger,
|
||||
} from '@nestjs/common';
|
||||
import { QuestionBankService } from '../services/question-bank.service';
|
||||
import {
|
||||
CreateQuestionBankDto,
|
||||
UpdateQuestionBankDto,
|
||||
CreateQuestionBankItemDto,
|
||||
UpdateQuestionBankItemDto,
|
||||
ReviewDto,
|
||||
} from '../services/question-bank.service';
|
||||
import { CombinedAuthGuard } from '../../auth/combined-auth.guard';
|
||||
|
||||
@Controller('question-banks')
|
||||
@UseGuards(CombinedAuthGuard)
|
||||
@UsePipes(ValidationPipe)
|
||||
export class QuestionBankController {
|
||||
private readonly logger = new Logger(QuestionBankController.name);
|
||||
|
||||
constructor(private readonly questionBankService: QuestionBankService) {}
|
||||
|
||||
@Post()
|
||||
create(@Body() createDto: CreateQuestionBankDto, @Req() req: any) {
|
||||
this.logger.log(`Creating question bank: ${createDto.name}`);
|
||||
return this.questionBankService.create(createDto, req.user.id, req.user.tenantId);
|
||||
}
|
||||
|
||||
@Get()
|
||||
findAll(
|
||||
@Req() req: any,
|
||||
@Query('page') page?: string,
|
||||
@Query('limit') limit?: string,
|
||||
) {
|
||||
const pageNum = page ? parseInt(page, 10) : undefined;
|
||||
const limitNum = limit ? parseInt(limit, 10) : undefined;
|
||||
return this.questionBankService.findAll(
|
||||
req.user.id,
|
||||
req.user.tenantId,
|
||||
pageNum,
|
||||
limitNum,
|
||||
);
|
||||
}
|
||||
|
||||
@Get('by-template/:templateId')
|
||||
async findByTemplateId(@Param('templateId') templateId: string) {
|
||||
return this.questionBankService.findByTemplateId(templateId);
|
||||
}
|
||||
|
||||
@Get(':id')
|
||||
async findOne(@Param('id') id: string) {
|
||||
return this.questionBankService.findOne(id);
|
||||
}
|
||||
|
||||
@Put(':id')
|
||||
async update(@Param('id') id: string, @Body() updateDto: UpdateQuestionBankDto) {
|
||||
return this.questionBankService.update(id, updateDto);
|
||||
}
|
||||
|
||||
@Delete(':id')
|
||||
async remove(@Param('id') id: string) {
|
||||
return this.questionBankService.remove(id);
|
||||
}
|
||||
|
||||
@Put(':id/submit')
|
||||
async submitForReview(@Param('id') id: string, @Req() req: any) {
|
||||
return this.questionBankService.submitForReview(id, req.user.id);
|
||||
}
|
||||
|
||||
@Put(':id/review')
|
||||
async review(@Param('id') id: string, @Body() reviewDto: ReviewDto, @Req() req: any) {
|
||||
return this.questionBankService.review(id, reviewDto, req.user.id);
|
||||
}
|
||||
|
||||
@Put(':id/publish')
|
||||
async publish(@Param('id') id: string) {
|
||||
return this.questionBankService.publish(id);
|
||||
}
|
||||
|
||||
@Post(':bankId/items')
|
||||
async addItem(
|
||||
@Param('bankId') bankId: string,
|
||||
@Body() createDto: CreateQuestionBankItemDto,
|
||||
) {
|
||||
return this.questionBankService.addItem(bankId, createDto);
|
||||
}
|
||||
|
||||
@Put(':bankId/items/:id')
|
||||
async updateItem(
|
||||
@Param('bankId') bankId: string,
|
||||
@Param('id') id: string,
|
||||
@Body() updateDto: UpdateQuestionBankItemDto,
|
||||
) {
|
||||
return this.questionBankService.updateItem(bankId, id, updateDto);
|
||||
}
|
||||
|
||||
@Delete(':bankId/items/:id')
|
||||
async removeItem(
|
||||
@Param('bankId') bankId: string,
|
||||
@Param('id') id: string,
|
||||
) {
|
||||
return this.questionBankService.removeItem(bankId, id);
|
||||
}
|
||||
|
||||
@Post(':bankId/generate')
|
||||
async generate(
|
||||
@Param('bankId') bankId: string,
|
||||
@Body() body: { count: number; knowledgeBaseContent?: string },
|
||||
@Req() req: any,
|
||||
) {
|
||||
this.logger.log(`[generate] Generating ${body.count} questions for bank ${bankId}`);
|
||||
return this.questionBankService.generateQuestions(
|
||||
bankId,
|
||||
body.count,
|
||||
body.knowledgeBaseContent || '',
|
||||
req.user.tenantId,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
import {
|
||||
Controller,
|
||||
Get,
|
||||
Post,
|
||||
Body,
|
||||
Put,
|
||||
Param,
|
||||
Delete,
|
||||
UseGuards,
|
||||
Req,
|
||||
} from '@nestjs/common';
|
||||
import { TemplateService } from '../services/template.service';
|
||||
import { CreateTemplateDto } from '../dto/create-template.dto';
|
||||
import { UpdateTemplateDto } from '../dto/update-template.dto';
|
||||
import { CombinedAuthGuard } from '../../auth/combined-auth.guard';
|
||||
|
||||
@Controller('assessment/templates')
|
||||
@UseGuards(CombinedAuthGuard)
|
||||
export class TemplateController {
|
||||
constructor(private readonly templateService: TemplateService) {}
|
||||
|
||||
@Post()
|
||||
create(@Body() createDto: CreateTemplateDto, @Req() req: any) {
|
||||
return this.templateService.create(
|
||||
createDto,
|
||||
req.user.id,
|
||||
req.user.tenantId,
|
||||
);
|
||||
}
|
||||
|
||||
@Get()
|
||||
findAll(@Req() req: any) {
|
||||
return this.templateService.findAll(req.user.tenantId);
|
||||
}
|
||||
|
||||
@Get(':id')
|
||||
async findOne(@Param('id') id: string, @Req() req: any) {
|
||||
return this.templateService.findOne(id, req.user.id, req.user.tenantId);
|
||||
}
|
||||
|
||||
@Put(':id')
|
||||
async update(
|
||||
@Param('id') id: string,
|
||||
@Body() updateDto: UpdateTemplateDto,
|
||||
@Req() req: any,
|
||||
) {
|
||||
return this.templateService.update(
|
||||
id,
|
||||
updateDto,
|
||||
req.user.id,
|
||||
req.user.tenantId,
|
||||
);
|
||||
}
|
||||
|
||||
@Delete(':id')
|
||||
async remove(@Param('id') id: string, @Req() req: any) {
|
||||
return this.templateService.remove(id, req.user.id, req.user.tenantId);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
import {
|
||||
IsString,
|
||||
IsNotEmpty,
|
||||
IsOptional,
|
||||
IsArray,
|
||||
IsInt,
|
||||
Min,
|
||||
Max,
|
||||
IsObject,
|
||||
IsBoolean,
|
||||
} from 'class-validator';
|
||||
|
||||
export class CreateTemplateDto {
|
||||
@IsString()
|
||||
@IsNotEmpty()
|
||||
name: string;
|
||||
|
||||
@IsString()
|
||||
@IsOptional()
|
||||
description?: string;
|
||||
|
||||
@IsArray()
|
||||
@IsString({ each: true })
|
||||
@IsOptional()
|
||||
keywords?: string[];
|
||||
|
||||
@IsInt()
|
||||
@Min(1)
|
||||
@Max(20)
|
||||
@IsOptional()
|
||||
questionCount?: number = 5;
|
||||
|
||||
@IsObject()
|
||||
@IsOptional()
|
||||
difficultyDistribution?: {
|
||||
standard: number;
|
||||
advanced: number;
|
||||
specialist: number;
|
||||
};
|
||||
|
||||
@IsString()
|
||||
@IsOptional()
|
||||
style?: string = 'technical';
|
||||
|
||||
@IsString()
|
||||
@IsOptional()
|
||||
knowledgeBaseId?: string;
|
||||
|
||||
@IsString()
|
||||
@IsOptional()
|
||||
knowledgeGroupId?: string;
|
||||
|
||||
@IsBoolean()
|
||||
@IsOptional()
|
||||
isActive?: boolean = true;
|
||||
|
||||
@IsArray()
|
||||
@IsString({ each: true })
|
||||
@IsOptional()
|
||||
linkedGroupIds?: string[];
|
||||
|
||||
@IsObject()
|
||||
@IsOptional()
|
||||
weightConfig?: {
|
||||
prompt: number;
|
||||
other: number;
|
||||
};
|
||||
|
||||
@IsObject()
|
||||
@IsOptional()
|
||||
difficultyConfig?: {
|
||||
standard: number;
|
||||
advanced: number;
|
||||
specialist: number;
|
||||
};
|
||||
|
||||
@IsInt()
|
||||
@Min(1)
|
||||
@Max(20)
|
||||
@IsOptional()
|
||||
questionCountMin?: number;
|
||||
|
||||
@IsInt()
|
||||
@Min(1)
|
||||
@Max(20)
|
||||
@IsOptional()
|
||||
questionCountMax?: number;
|
||||
|
||||
@IsInt()
|
||||
@Min(0)
|
||||
@Max(100)
|
||||
@IsOptional()
|
||||
passingScore?: number;
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
import { PartialType } from '@nestjs/mapped-types';
|
||||
import { CreateTemplateDto } from './create-template.dto';
|
||||
|
||||
export class UpdateTemplateDto extends PartialType(CreateTemplateDto) {}
|
||||
@@ -0,0 +1,45 @@
|
||||
import {
|
||||
Entity,
|
||||
PrimaryGeneratedColumn,
|
||||
Column,
|
||||
CreateDateColumn,
|
||||
UpdateDateColumn,
|
||||
ManyToOne,
|
||||
JoinColumn,
|
||||
} from 'typeorm';
|
||||
import type { AssessmentQuestion } from './assessment-question.entity';
|
||||
|
||||
@Entity('assessment_answers')
|
||||
export class AssessmentAnswer {
|
||||
@PrimaryGeneratedColumn('uuid')
|
||||
id: string;
|
||||
|
||||
@Column({ name: 'question_id' })
|
||||
questionId: string;
|
||||
|
||||
@ManyToOne(
|
||||
'AssessmentQuestion',
|
||||
(question: AssessmentQuestion) => question.answers,
|
||||
{ onDelete: 'CASCADE' },
|
||||
)
|
||||
@JoinColumn({ name: 'question_id' })
|
||||
question: AssessmentQuestion;
|
||||
|
||||
@Column({ type: 'text', name: 'user_answer' })
|
||||
userAnswer: string;
|
||||
|
||||
@Column({ type: 'float', nullable: true })
|
||||
score: number;
|
||||
|
||||
@Column({ type: 'text', nullable: true })
|
||||
feedback: string;
|
||||
|
||||
@Column({ type: 'boolean', name: 'is_follow_up', default: false })
|
||||
isFollowUp: boolean;
|
||||
|
||||
@CreateDateColumn({ name: 'created_at' })
|
||||
createdAt: Date;
|
||||
|
||||
@UpdateDateColumn({ name: 'updated_at' })
|
||||
updatedAt: Date;
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
import {
|
||||
Entity,
|
||||
PrimaryGeneratedColumn,
|
||||
Column,
|
||||
CreateDateColumn,
|
||||
ManyToOne,
|
||||
JoinColumn,
|
||||
} from 'typeorm';
|
||||
import { User } from '../../user/user.entity';
|
||||
|
||||
@Entity('assessment_certificates')
|
||||
export class AssessmentCertificate {
|
||||
@PrimaryGeneratedColumn('uuid')
|
||||
id: string;
|
||||
|
||||
@Column({ name: 'user_id' })
|
||||
userId: string;
|
||||
|
||||
@ManyToOne(() => User)
|
||||
@JoinColumn({ name: 'user_id' })
|
||||
user: User;
|
||||
|
||||
@Column({ name: 'session_id' })
|
||||
sessionId: string;
|
||||
|
||||
@Column({ name: 'template_id' })
|
||||
templateId: string;
|
||||
|
||||
@Column()
|
||||
level: string;
|
||||
|
||||
@Column({ type: 'float', name: 'total_score' })
|
||||
totalScore: number;
|
||||
|
||||
@Column({ name: 'qr_code', nullable: true })
|
||||
qrCode: string;
|
||||
|
||||
@Column({ name: 'dimension_scores', type: 'simple-json', nullable: true })
|
||||
dimensionScores: Record<string, number>;
|
||||
|
||||
@Column({ name: 'radar_data', type: 'simple-json', nullable: true })
|
||||
radarData: Record<string, number>;
|
||||
|
||||
@Column({ name: 'passed', type: 'boolean', default: false })
|
||||
passed: boolean;
|
||||
|
||||
@CreateDateColumn({ name: 'issued_at' })
|
||||
issuedAt: Date;
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import {
|
||||
Entity,
|
||||
PrimaryGeneratedColumn,
|
||||
Column,
|
||||
CreateDateColumn,
|
||||
UpdateDateColumn,
|
||||
ManyToOne,
|
||||
JoinColumn,
|
||||
OneToMany,
|
||||
} from 'typeorm';
|
||||
import type { AssessmentSession } from './assessment-session.entity';
|
||||
import type { AssessmentAnswer } from './assessment-answer.entity';
|
||||
|
||||
@Entity('assessment_questions')
|
||||
export class AssessmentQuestion {
|
||||
@PrimaryGeneratedColumn('uuid')
|
||||
id: string;
|
||||
|
||||
@Column({ name: 'session_id' })
|
||||
sessionId: string;
|
||||
|
||||
@ManyToOne(
|
||||
'AssessmentSession',
|
||||
(session: AssessmentSession) => session.questions,
|
||||
{ onDelete: 'CASCADE' },
|
||||
)
|
||||
@JoinColumn({ name: 'session_id' })
|
||||
session: AssessmentSession;
|
||||
|
||||
@Column({ type: 'text', name: 'question_text' })
|
||||
questionText: string;
|
||||
|
||||
@Column({ type: 'simple-json', name: 'key_points', nullable: true })
|
||||
keyPoints: string[];
|
||||
|
||||
@Column({ type: 'varchar', nullable: true })
|
||||
difficulty: string;
|
||||
|
||||
@OneToMany('AssessmentAnswer', (answer: AssessmentAnswer) => answer.question)
|
||||
answers: AssessmentAnswer[];
|
||||
|
||||
@CreateDateColumn({ name: 'created_at' })
|
||||
createdAt: Date;
|
||||
|
||||
@UpdateDateColumn({ name: 'updated_at' })
|
||||
updatedAt: Date;
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
import {
|
||||
Entity,
|
||||
PrimaryGeneratedColumn,
|
||||
Column,
|
||||
CreateDateColumn,
|
||||
UpdateDateColumn,
|
||||
ManyToOne,
|
||||
JoinColumn,
|
||||
OneToMany,
|
||||
} from 'typeorm';
|
||||
import { User } from '../../user/user.entity';
|
||||
import { KnowledgeBase } from '../../knowledge-base/knowledge-base.entity';
|
||||
import { KnowledgeGroup } from '../../knowledge-group/knowledge-group.entity';
|
||||
import type { AssessmentQuestion } from './assessment-question.entity';
|
||||
import { AssessmentTemplate } from './assessment-template.entity';
|
||||
|
||||
export enum AssessmentStatus {
|
||||
IN_PROGRESS = 'IN_PROGRESS',
|
||||
COMPLETED = 'COMPLETED',
|
||||
}
|
||||
|
||||
@Entity('assessment_sessions')
|
||||
export class AssessmentSession {
|
||||
@PrimaryGeneratedColumn('uuid')
|
||||
id: string;
|
||||
|
||||
@Column({ name: 'user_id' })
|
||||
userId: string;
|
||||
|
||||
@ManyToOne(() => User)
|
||||
@JoinColumn({ name: 'user_id' })
|
||||
user: User;
|
||||
|
||||
@Column({ name: 'tenant_id', nullable: true })
|
||||
tenantId: string;
|
||||
|
||||
@Column({ name: 'knowledge_base_id', nullable: true })
|
||||
knowledgeBaseId: string | null;
|
||||
|
||||
@ManyToOne(() => KnowledgeBase, { nullable: true })
|
||||
@JoinColumn({ name: 'knowledge_base_id' })
|
||||
knowledgeBase: KnowledgeBase;
|
||||
|
||||
@Column({ name: 'knowledge_group_id', nullable: true })
|
||||
knowledgeGroupId: string | null;
|
||||
|
||||
@ManyToOne(() => KnowledgeGroup, { nullable: true })
|
||||
@JoinColumn({ name: 'knowledge_group_id' })
|
||||
knowledgeGroup: KnowledgeGroup;
|
||||
|
||||
@Column({ name: 'thread_id', nullable: true })
|
||||
threadId: string;
|
||||
|
||||
@Column({
|
||||
type: 'varchar',
|
||||
enum: AssessmentStatus,
|
||||
default: AssessmentStatus.IN_PROGRESS,
|
||||
})
|
||||
status: AssessmentStatus;
|
||||
|
||||
@Column({ type: 'float', name: 'final_score', nullable: true })
|
||||
finalScore: number;
|
||||
|
||||
@Column({ type: 'text', name: 'final_report', nullable: true })
|
||||
finalReport: string;
|
||||
|
||||
@Column({ type: 'simple-json', nullable: true })
|
||||
messages: any[];
|
||||
|
||||
@Column({ type: 'simple-json', name: 'feedback_history', nullable: true })
|
||||
feedbackHistory: any[];
|
||||
|
||||
@Column({ type: 'int', name: 'current_question_index', default: 0 })
|
||||
currentQuestionIndex: number;
|
||||
|
||||
@Column({ type: 'int', name: 'follow_up_count', default: 0 })
|
||||
followUpCount: number;
|
||||
|
||||
@Column({ type: 'simple-json', nullable: true })
|
||||
questions_json: any[];
|
||||
|
||||
@Column({ type: 'varchar', length: 10, default: 'zh' })
|
||||
language: string;
|
||||
|
||||
@Column({ name: 'template_id', nullable: true })
|
||||
templateId: string;
|
||||
|
||||
@ManyToOne(() => AssessmentTemplate, { nullable: true })
|
||||
@JoinColumn({ name: 'template_id' })
|
||||
template: AssessmentTemplate;
|
||||
|
||||
@Column({ type: 'simple-json', name: 'template_json', nullable: true })
|
||||
templateJson: any;
|
||||
|
||||
@OneToMany(
|
||||
'AssessmentQuestion',
|
||||
(question: AssessmentQuestion) => question.session,
|
||||
)
|
||||
questions: AssessmentQuestion[];
|
||||
|
||||
@CreateDateColumn({ name: 'created_at' })
|
||||
createdAt: Date;
|
||||
|
||||
@UpdateDateColumn({ name: 'updated_at' })
|
||||
updatedAt: Date;
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
import {
|
||||
Entity,
|
||||
PrimaryGeneratedColumn,
|
||||
Column,
|
||||
CreateDateColumn,
|
||||
UpdateDateColumn,
|
||||
ManyToOne,
|
||||
JoinColumn,
|
||||
} from 'typeorm';
|
||||
import { Tenant } from '../../tenant/tenant.entity';
|
||||
import { KnowledgeBase } from '../../knowledge-base/knowledge-base.entity';
|
||||
import { KnowledgeGroup } from '../../knowledge-group/knowledge-group.entity';
|
||||
|
||||
@Entity('assessment_templates')
|
||||
export class AssessmentTemplate {
|
||||
@PrimaryGeneratedColumn('uuid')
|
||||
id: string;
|
||||
|
||||
@Column({ name: 'tenant_id', nullable: true })
|
||||
tenantId: string;
|
||||
|
||||
@ManyToOne(() => Tenant, { nullable: true, onDelete: 'CASCADE' })
|
||||
@JoinColumn({ name: 'tenant_id' })
|
||||
tenant: Tenant;
|
||||
|
||||
@Column()
|
||||
name: string;
|
||||
|
||||
@Column({ type: 'text', nullable: true })
|
||||
description: string;
|
||||
|
||||
@Column({ type: 'simple-json', nullable: true })
|
||||
keywords: string[];
|
||||
|
||||
@Column({ type: 'int', name: 'question_count', default: 5 })
|
||||
questionCount: number;
|
||||
|
||||
@Column({
|
||||
type: 'simple-json',
|
||||
name: 'difficulty_distribution',
|
||||
nullable: true,
|
||||
})
|
||||
difficultyDistribution: {
|
||||
standard: number;
|
||||
advanced: number;
|
||||
specialist: number;
|
||||
};
|
||||
|
||||
@Column({ type: 'varchar', default: 'technical' })
|
||||
style: string;
|
||||
|
||||
@Column({ name: 'knowledge_base_id', nullable: true })
|
||||
knowledgeBaseId: string | null;
|
||||
|
||||
@ManyToOne(() => KnowledgeBase, { nullable: true })
|
||||
@JoinColumn({ name: 'knowledge_base_id' })
|
||||
knowledgeBase: KnowledgeBase;
|
||||
|
||||
@Column({ name: 'knowledge_group_id', nullable: true })
|
||||
knowledgeGroupId: string | null;
|
||||
|
||||
@ManyToOne(() => KnowledgeGroup, { nullable: true })
|
||||
@JoinColumn({ name: 'knowledge_group_id' })
|
||||
knowledgeGroup: KnowledgeGroup;
|
||||
|
||||
@Column({ type: 'boolean', name: 'is_active', default: true })
|
||||
isActive: boolean;
|
||||
|
||||
@Column({ type: 'int', default: 1 })
|
||||
version: number;
|
||||
|
||||
@Column({ name: 'created_by', nullable: true })
|
||||
createdBy: string;
|
||||
|
||||
@Column({ type: 'simple-json', name: 'linked_group_ids', nullable: true })
|
||||
linkedGroupIds: string[];
|
||||
|
||||
@Column({ type: 'simple-json', name: 'weight_config', nullable: true })
|
||||
weightConfig: {
|
||||
prompt: number;
|
||||
other: number;
|
||||
};
|
||||
|
||||
@Column({ type: 'simple-json', name: 'difficulty_config', nullable: true })
|
||||
difficultyConfig: {
|
||||
standard: number;
|
||||
advanced: number;
|
||||
specialist: number;
|
||||
};
|
||||
|
||||
@Column({ type: 'int', name: 'question_count_min', default: 8 })
|
||||
questionCountMin: number;
|
||||
|
||||
@Column({ type: 'int', name: 'question_count_max', default: 10 })
|
||||
questionCountMax: number;
|
||||
|
||||
@Column({ type: 'int', name: 'passing_score', default: 90 })
|
||||
passingScore: number;
|
||||
|
||||
@CreateDateColumn({ name: 'created_at' })
|
||||
createdAt: Date;
|
||||
|
||||
@UpdateDateColumn({ name: 'updated_at' })
|
||||
updatedAt: Date;
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
import {
|
||||
Entity,
|
||||
PrimaryGeneratedColumn,
|
||||
Column,
|
||||
CreateDateColumn,
|
||||
UpdateDateColumn,
|
||||
ManyToOne,
|
||||
JoinColumn,
|
||||
} from 'typeorm';
|
||||
import { QuestionBank } from './question-bank.entity';
|
||||
|
||||
export enum QuestionBankItemStatus {
|
||||
PENDING_REVIEW = 'PENDING_REVIEW',
|
||||
PUBLISHED = 'PUBLISHED',
|
||||
}
|
||||
|
||||
export enum QuestionType {
|
||||
SHORT_ANSWER = 'SHORT_ANSWER',
|
||||
MULTIPLE_CHOICE = 'MULTIPLE_CHOICE',
|
||||
TRUE_FALSE = 'TRUE_FALSE',
|
||||
}
|
||||
|
||||
export enum QuestionDifficulty {
|
||||
STANDARD = 'STANDARD',
|
||||
ADVANCED = 'ADVANCED',
|
||||
SPECIALIST = 'SPECIALIST',
|
||||
}
|
||||
|
||||
export enum QuestionDimension {
|
||||
PROMPT = 'PROMPT',
|
||||
LLM = 'LLM',
|
||||
IDE = 'IDE',
|
||||
DEV_PATTERN = 'DEV_PATTERN',
|
||||
WORK_CAPABILITY = 'WORK_CAPABILITY',
|
||||
}
|
||||
|
||||
@Entity('question_bank_items')
|
||||
export class QuestionBankItem {
|
||||
@PrimaryGeneratedColumn('uuid')
|
||||
id: string;
|
||||
|
||||
@Column({ name: 'bank_id' })
|
||||
bankId: string;
|
||||
|
||||
@ManyToOne(
|
||||
() => QuestionBank,
|
||||
(bank: QuestionBank) => bank.items,
|
||||
{ onDelete: 'CASCADE' },
|
||||
)
|
||||
@JoinColumn({ name: 'bank_id' })
|
||||
bank: QuestionBank;
|
||||
|
||||
@Column({ type: 'text', name: 'question_text' })
|
||||
questionText: string;
|
||||
|
||||
@Column({
|
||||
type: 'enum',
|
||||
enum: QuestionType,
|
||||
default: QuestionType.SHORT_ANSWER,
|
||||
})
|
||||
questionType: QuestionType;
|
||||
|
||||
@Column({ type: 'simple-json', nullable: true })
|
||||
options: string[] | null;
|
||||
|
||||
@Column({ type: 'text', nullable: true })
|
||||
correctAnswer: string | null;
|
||||
|
||||
@Column({ type: 'simple-json', name: 'key_points' })
|
||||
keyPoints: string[];
|
||||
|
||||
@Column({
|
||||
type: 'enum',
|
||||
enum: QuestionDifficulty,
|
||||
default: QuestionDifficulty.STANDARD,
|
||||
})
|
||||
difficulty: QuestionDifficulty;
|
||||
|
||||
@Column({
|
||||
type: 'enum',
|
||||
enum: QuestionDimension,
|
||||
default: QuestionDimension.PROMPT,
|
||||
})
|
||||
dimension: QuestionDimension;
|
||||
|
||||
@Column({ type: 'text', nullable: true })
|
||||
basis: string | null;
|
||||
|
||||
@Column({ name: 'created_by', nullable: true })
|
||||
createdBy: string | null;
|
||||
|
||||
@Column({
|
||||
type: 'enum',
|
||||
enum: QuestionBankItemStatus,
|
||||
default: QuestionBankItemStatus.PENDING_REVIEW,
|
||||
})
|
||||
status: QuestionBankItemStatus;
|
||||
|
||||
@CreateDateColumn({ name: 'created_at' })
|
||||
createdAt: Date;
|
||||
|
||||
@UpdateDateColumn({ name: 'updated_at' })
|
||||
updatedAt: Date;
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
import {
|
||||
Entity,
|
||||
PrimaryGeneratedColumn,
|
||||
Column,
|
||||
CreateDateColumn,
|
||||
UpdateDateColumn,
|
||||
ManyToOne,
|
||||
OneToOne,
|
||||
JoinColumn,
|
||||
OneToMany,
|
||||
Unique,
|
||||
} from 'typeorm';
|
||||
import { Tenant } from '../../tenant/tenant.entity';
|
||||
import { AssessmentTemplate } from './assessment-template.entity';
|
||||
import type { QuestionBankItem } from './question-bank-item.entity';
|
||||
|
||||
export enum QuestionBankStatus {
|
||||
DRAFT = 'DRAFT',
|
||||
PENDING_REVIEW = 'PENDING_REVIEW',
|
||||
PUBLISHED = 'PUBLISHED',
|
||||
REJECTED = 'REJECTED',
|
||||
}
|
||||
|
||||
@Entity('question_banks')
|
||||
@Unique(['templateId'])
|
||||
export class QuestionBank {
|
||||
@PrimaryGeneratedColumn('uuid')
|
||||
id: string;
|
||||
|
||||
@Column({ name: 'tenant_id', nullable: true })
|
||||
tenantId: string | null;
|
||||
|
||||
@ManyToOne(() => Tenant, { nullable: true, onDelete: 'CASCADE' })
|
||||
@JoinColumn({ name: 'tenant_id' })
|
||||
tenant: Tenant;
|
||||
|
||||
@Column({ name: 'template_id', nullable: true })
|
||||
templateId: string | null;
|
||||
|
||||
@OneToOne(() => AssessmentTemplate, { nullable: true })
|
||||
@JoinColumn({ name: 'template_id' })
|
||||
template: AssessmentTemplate;
|
||||
|
||||
@Column()
|
||||
name: string;
|
||||
|
||||
@Column({ type: 'text', nullable: true })
|
||||
description: string | null;
|
||||
|
||||
@Column({
|
||||
type: 'enum',
|
||||
enum: QuestionBankStatus,
|
||||
default: QuestionBankStatus.DRAFT,
|
||||
})
|
||||
status: QuestionBankStatus;
|
||||
|
||||
@Column({ name: 'created_by', nullable: true })
|
||||
createdBy: string | null;
|
||||
|
||||
@Column({ name: 'reviewed_by', nullable: true })
|
||||
reviewedBy: string | null;
|
||||
|
||||
@Column({ name: 'reviewed_at', nullable: true })
|
||||
reviewedAt: Date | null;
|
||||
|
||||
@Column({ name: 'review_comment', nullable: true })
|
||||
reviewComment: string | null;
|
||||
|
||||
@OneToMany(
|
||||
'QuestionBankItem',
|
||||
(item: QuestionBankItem) => item.bank,
|
||||
)
|
||||
items: QuestionBankItem[];
|
||||
|
||||
@CreateDateColumn({ name: 'created_at' })
|
||||
createdAt: Date;
|
||||
|
||||
@UpdateDateColumn({ name: 'updated_at' })
|
||||
updatedAt: Date;
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
import { StateGraph, MemorySaver } from '@langchain/langgraph';
|
||||
import { EvaluationAnnotation } from './state';
|
||||
import { questionGeneratorNode } from './nodes/generator.node';
|
||||
import { interviewerNode } from './nodes/interviewer.node';
|
||||
import { graderNode } from './nodes/grader.node';
|
||||
import { reportAnalyzerNode } from './nodes/analyzer.node';
|
||||
|
||||
/**
|
||||
* Conditional routing logic for the Grader node.
|
||||
*/
|
||||
const routeAfterGrading = (state: typeof EvaluationAnnotation.State) => {
|
||||
const targetCount = state.questionCount || 5;
|
||||
const questionsLen = state.questions?.length || 0;
|
||||
|
||||
console.log('[Router] Evaluation Result:', {
|
||||
currentIndex: state.currentQuestionIndex,
|
||||
shouldFollowUp: state.shouldFollowUp,
|
||||
numQuestions: questionsLen,
|
||||
targetCount,
|
||||
});
|
||||
|
||||
if (state.shouldFollowUp) {
|
||||
console.log('[Router] Routing to follow-up interviewer');
|
||||
return 'interviewer';
|
||||
}
|
||||
|
||||
if (state.currentQuestionIndex < targetCount) {
|
||||
// If the next question isn't generated yet, go back to generator
|
||||
if (state.currentQuestionIndex >= questionsLen) {
|
||||
console.log('[Router] Index >= Questions, routing to generator');
|
||||
return 'generator';
|
||||
}
|
||||
// If it is generated, go to interviewer
|
||||
console.log('[Router] Index < Questions, routing to interviewer');
|
||||
return 'interviewer';
|
||||
}
|
||||
|
||||
console.log('[Router] Assessment complete, routing to analyzer');
|
||||
return 'analyzer';
|
||||
};
|
||||
|
||||
/**
|
||||
* Builds and compiles the Evaluation Graph.
|
||||
*/
|
||||
export const createEvaluationGraph = () => {
|
||||
const workflow = new StateGraph(EvaluationAnnotation)
|
||||
.addNode('generator', questionGeneratorNode)
|
||||
.addNode('interviewer', interviewerNode)
|
||||
.addNode('grader', graderNode)
|
||||
.addNode('analyzer', reportAnalyzerNode)
|
||||
|
||||
// Flow definition
|
||||
.addEdge('__start__', 'generator')
|
||||
.addEdge('generator', 'interviewer')
|
||||
|
||||
// After interviewer, the graph will naturally pause for user input
|
||||
// if we use it in a thread-safe way with interrupts or simple invocation.
|
||||
.addEdge('interviewer', 'grader')
|
||||
|
||||
// After grading, decide where to go
|
||||
.addConditionalEdges('grader', routeAfterGrading, {
|
||||
interviewer: 'interviewer',
|
||||
generator: 'generator',
|
||||
analyzer: 'analyzer',
|
||||
})
|
||||
|
||||
.addEdge('analyzer', '__end__');
|
||||
|
||||
// Using MemorySaver for thread-based persistence
|
||||
const checkpointer = new MemorySaver();
|
||||
|
||||
return workflow.compile({
|
||||
checkpointer,
|
||||
// We want the graph to stop after the interviewer presents the question
|
||||
interruptAfter: ['interviewer'],
|
||||
});
|
||||
};
|
||||
@@ -0,0 +1,162 @@
|
||||
import { ChatOpenAI } from '@langchain/openai';
|
||||
import { SystemMessage, HumanMessage } from '@langchain/core/messages';
|
||||
import { RunnableConfig } from '@langchain/core/runnables';
|
||||
import { EvaluationState } from '../state';
|
||||
|
||||
/**
|
||||
* Node responsible for generating the final mastery report at the end of the session.
|
||||
*/
|
||||
export const reportAnalyzerNode = async (
|
||||
state: EvaluationState,
|
||||
config?: RunnableConfig,
|
||||
): Promise<Partial<EvaluationState>> => {
|
||||
const { model } = (config?.configurable as any) || {};
|
||||
const { scores, messages } = state;
|
||||
const questionList = state.questions || [];
|
||||
|
||||
console.log('[AnalyzerNode] Entering node...', {
|
||||
numScores: Object.keys(scores || {}).length,
|
||||
numMessages: messages?.length,
|
||||
scores,
|
||||
});
|
||||
|
||||
if (!model) {
|
||||
throw new Error('Missing model in node configuration');
|
||||
}
|
||||
|
||||
const scoreSummary = Object.entries(scores)
|
||||
.map(([qId, score]) => {
|
||||
const displayId = isNaN(parseInt(qId))
|
||||
? qId
|
||||
: (parseInt(qId) + 1).toString();
|
||||
return `Question ${displayId}: Score ${score}/10`;
|
||||
})
|
||||
.join('\n');
|
||||
|
||||
const dimensionSummary = questionList.reduce((acc: Record<string, number[]>, q: any) => {
|
||||
const dim = q.dimension || 'workCapability';
|
||||
const score = scores[q.id] || 0;
|
||||
if (!acc[dim]) acc[dim] = [];
|
||||
acc[dim].push(score);
|
||||
return acc;
|
||||
}, {});
|
||||
|
||||
const dimensionAvg = Object.entries(dimensionSummary).map(([dim, arr]: [string, any]) => {
|
||||
const avg = arr.reduce((a: number, b: number) => a + b, 0) / arr.length;
|
||||
return `${dim}: ${avg.toFixed(1)}/10`;
|
||||
}).join('\n');
|
||||
|
||||
const isZh = state.language === 'zh';
|
||||
const isJa = state.language === 'ja';
|
||||
|
||||
const systemPromptZh = `你是一位客观且严谨的高级教育顾问。
|
||||
请审查以下评估结果，并为员工提供一份严谨的掌握程度报告。
|
||||
|
||||
重要提示：
|
||||
1. **你必须使用以下语言生成报告：中文 (Simplified Chinese)**。
|
||||
2. **严禁夹杂日文**。即使对话记录中包含日文，报告内容也必须全中文。
|
||||
3. 报告的第一行必须严格遵守此格式："LEVEL: [Novice/Proficient/Advanced/Expert]"。
|
||||
4. 必须保持客观。如果用户没有提供有效的回答或得分为 0，你必须将其识别为 'Novice'，并明确指出他们尚未证明其掌握程度。
|
||||
5. 不要虚构或幻想优点（如"潜力"或"好奇心"），如果用户明确表示"不知道"或未提供实质内容。
|
||||
6. 专注于对话记录中已证明的事实。
|
||||
|
||||
各维度得分：
|
||||
${dimensionAvg}
|
||||
|
||||
问题与得分：
|
||||
${scoreSummary}
|
||||
|
||||
对话记录：
|
||||
${messages
|
||||
.filter((m: any) => m._getType() !== 'system')
|
||||
.map((m: any) => `${m.role || m._getType()}: ${m.content}`)
|
||||
.join('\n')}
|
||||
|
||||
报告结构：
|
||||
1. 总体级别（已在顶部指定）
|
||||
2. 各维度得分分析（提示词、LLM、IDE、开发范式、工作能力）
|
||||
3. 薄弱环节识别
|
||||
4. 针对性改进建议
|
||||
5. 推荐的学习路径。`;
|
||||
|
||||
const systemPromptJa = `あなたは客観的で厳格なシニア教育コンサルタントです。
|
||||
以下の評価結果をレビューし、従業員に対して厳格な習熟度レポートを提供してください。
|
||||
|
||||
重要事項：
|
||||
1. **レポートは必ず次の言語で生成してください：日本語**。
|
||||
2. **中国語を混ぜないでください**。会話ログに中国語が含まれていても、レポートの内容はすべて日本語で記述してください。
|
||||
3. レポートの最初の行は, 必ず次の形式に従ってください："LEVEL: [Novice/Proficient/Advanced/Expert]"。
|
||||
4. 客観的であること。ユーザーが有効な回答を提供しなかった場合、またはスコアが 0 の場合、'Novice' と判定し、習熟度が証明されていないことを明示してください。
|
||||
5. ユーザーが「わからない」と言ったり、内容を提供しなかった場合に、長所（「ポテンシャル」や「好奇心」など）を捏造しないでください。
|
||||
6. 会話ログで証明された事実に集中してください。
|
||||
|
||||
各ディメンションスコア：
|
||||
${dimensionAvg}
|
||||
|
||||
質問とスコア：
|
||||
${scoreSummary}
|
||||
|
||||
会話ログ：
|
||||
${messages
|
||||
.filter((m: any) => m._getType() !== 'system')
|
||||
.map((m: any) => `${m.role || m._getType()}: ${m.content}`)
|
||||
.join('\n')}
|
||||
|
||||
レポート構成：
|
||||
1. 総合レベル（一番上に指定済み）
|
||||
2. 各ディメンション分析（提示詞、LLM、IDE、開発範式、工作能力）
|
||||
3. 薄弱环节识别
|
||||
4. 推奨される学習パス。`;
|
||||
|
||||
const systemPromptEn = `You are an objective and critical seniority education consultant.
|
||||
Review the following assessment results and provide a rigorous mastery report for the employee.
|
||||
|
||||
IMPORTANT:
|
||||
1. **You MUST generate the report strictly in English.**
|
||||
2. START the report with exactly this format: "LEVEL: [Novice/Proficient/Advanced/Expert]" on the first line.
|
||||
3. Be OBJECTIVE. If the user provided no valid answers or scores are 0, you MUST identify them as 'Novice' and explicitly state they have NOT demonstrated mastery.
|
||||
4. DO NOT invent or hallucinate strengths (like 'potential' or 'curiosity') if the user explicitly said "I don't know" or provided no content.
|
||||
5. Focus on what was PROVEN in the conversation logs.
|
||||
|
||||
DIMENSION SCORES:
|
||||
${dimensionAvg}
|
||||
|
||||
QUESTIONS AND SCORES:
|
||||
${scoreSummary}
|
||||
|
||||
CONVERSATION LOGS:
|
||||
${messages
|
||||
.filter((m: any) => m._getType() !== 'system')
|
||||
.map((m: any) => `${m.role || m._getType()}: ${m.content}`)
|
||||
.join('\n')}
|
||||
|
||||
REPORT STRUCTURE:
|
||||
1. Overall Level (Already specified at top)
|
||||
2. Dimension Analysis (Prompt, LLM, IDE, DevPattern, WorkCapability)
|
||||
3. Weak Areas Identification
|
||||
4. Targeted Learning Recommendations.`;
|
||||
|
||||
const systemPrompt = isZh
|
||||
? systemPromptZh
|
||||
: isJa
|
||||
? systemPromptJa
|
||||
: systemPromptEn;
|
||||
const humanMsg = isZh
|
||||
? '生成最终掌握程度报告。'
|
||||
: isJa
|
||||
? '最終的な習熟度レポートを生成してください。'
|
||||
: 'Generate the final mastery report.';
|
||||
|
||||
const response = await model.invoke([
|
||||
new SystemMessage(systemPrompt),
|
||||
new HumanMessage(humanMsg),
|
||||
]);
|
||||
|
||||
console.log(
|
||||
'[AnalyzerNode] Report generated successfully. Length:',
|
||||
response.content?.toString().length,
|
||||
);
|
||||
return {
|
||||
report: response.content as string,
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,246 @@
|
||||
import { ChatOpenAI } from '@langchain/openai';
|
||||
import { SystemMessage, HumanMessage } from '@langchain/core/messages';
|
||||
import { RunnableConfig } from '@langchain/core/runnables';
|
||||
import { EvaluationState } from '../state';
|
||||
import { safeParseJson } from '../../../common/json-utils';
|
||||
|
||||
/**
|
||||
* Node responsible for generating assessment questions based on the knowledge base content.
|
||||
*/
|
||||
export const questionGeneratorNode = async (
|
||||
state: EvaluationState,
|
||||
config?: RunnableConfig,
|
||||
): Promise<Partial<EvaluationState>> => {
|
||||
const { model, knowledgeBaseContent, targetCount } = (config?.configurable as any) || {};
|
||||
const limitCount = targetCount || 5;
|
||||
|
||||
console.log('[GeneratorNode] Starting generation...', {
|
||||
language: state.language,
|
||||
hasModel: !!model,
|
||||
contentLength: knowledgeBaseContent?.length,
|
||||
keywords: state.keywords || [],
|
||||
targetCount: limitCount,
|
||||
});
|
||||
|
||||
if (!model || !knowledgeBaseContent) {
|
||||
console.error('[GeneratorNode] Missing model or knowledgeBaseContent');
|
||||
throw new Error(
|
||||
'Missing model or knowledgeBaseContent in node configuration',
|
||||
);
|
||||
}
|
||||
|
||||
const isZh = state.language === 'zh';
|
||||
const isJa = state.language === 'ja';
|
||||
|
||||
const style = state.style || 'technical';
|
||||
const difficultyText = state.difficultyDistribution
|
||||
? JSON.stringify(state.difficultyDistribution)
|
||||
: isZh
|
||||
? '随机分布'
|
||||
: isJa
|
||||
? 'ランダム分布'
|
||||
: 'Random distribution';
|
||||
const keywords = state.keywords || [];
|
||||
const hasKeywords = keywords.length > 0;
|
||||
const keywordText = hasKeywords ? keywords.join(', ') : '';
|
||||
|
||||
const rulesZh = [
|
||||
`**禁止重复**：绝对禁止生成与下方“禁止重复列表”中相似的题目。`,
|
||||
`**深度挖掘**：如果之前的题目考查了核心定义，新题目必须考查具体的应用案例、对比分析或隐藏的细节。`,
|
||||
hasKeywords
|
||||
? `**关键词权重**：必须围绕关键词 (${keywordText}) 展开，但要从关键词的不同侧面（如流程、限制、优缺点、具体参数等）进行挖掘。`
|
||||
: null,
|
||||
`**随机扰动**：即使对于相同的主题或关键词，也要尝试从不同的逻辑链条（如“因为...所以...” vs “如果没有...会怎样”）出发。`,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.map((r, i) => `${i + 1}. ${r}`)
|
||||
.join('\n');
|
||||
|
||||
const rulesJa = [
|
||||
`**重複禁止**：下記の「作成済み問題リスト」と類似した内容は絶対に避けてください。`,
|
||||
`**多角的アプローチ**：前回が定義だった場合は、今回は応用方法、制限事項、具体的な数値などに焦点を当ててください。`,
|
||||
hasKeywords
|
||||
? `**キーワードの深掘り**：キーワード (${keywordText}) の異なる側面から出題してください。`
|
||||
: null,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.map((r, i) => `${i + 1}. ${r}`)
|
||||
.join('\n');
|
||||
|
||||
const rulesEn = [
|
||||
`**NO REPETITION**: Strictly avoid any conceptual overlap with the "Previous Questions" list below.`,
|
||||
`**New Facets**: If previous questions were about definitions, focus on applications, edge cases, or specific details.`,
|
||||
hasKeywords
|
||||
? `**Keyword Variety**: Center on (${keywordText}), but explore different aspects (process, pros/cons, requirements).`
|
||||
: null,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.map((r, i) => `${i + 1}. ${r}`)
|
||||
.join('\n');
|
||||
|
||||
const existingQuestions = state.questions || [];
|
||||
const existingQuestionsText = existingQuestions
|
||||
.map((q, i) => `Q${i + 1}: ${q.questionText}`)
|
||||
.join('\n');
|
||||
|
||||
const systemPromptZh = `你是一位专业的知识评估专家。请根据提供的知识库片段生成 1 个唯一的测试题目。
|
||||
|
||||
### 强制性语言规则：
|
||||
**必须使用中文 (Simplified Chinese) 进行回复**。即使知识库内容是英文或其他语言，问题(question_text)和关键点(key_points)也必须使用中文。
|
||||
|
||||
### 强制性多样性规则：
|
||||
${rulesZh}
|
||||
|
||||
### 禁止重复列表（已出过）：
|
||||
${existingQuestionsText || '无'}
|
||||
|
||||
### 任务：
|
||||
${hasKeywords ? `目标关键词：${keywordText}\n` : ''}出题风格：${style}
|
||||
难度：${difficultyText}
|
||||
|
||||
请以 JSON 数组格式返回 1 个问题：
|
||||
[
|
||||
{
|
||||
"question_text": "...",
|
||||
"key_points": ["点1", "点2"],
|
||||
"difficulty": "...",
|
||||
"dimension": "prompt/llm/ide/devPattern/workCapability",
|
||||
"basis": "[n] 引用原文..."
|
||||
}
|
||||
]`;
|
||||
// dimension取值：prompt=提示词, llm=LLM原理, ide=IDE协作, devPattern=开发范式, workCapability=工作能力
|
||||
|
||||
const systemPromptJa = `あなたは専門的なアセスメントエキスパートです。提供されたナレッジベースに基づいて、ユニークな問題を 1 つ作成してください。
|
||||
|
||||
### 言語ルール（最重要）：
|
||||
**必ず日本語で作成してください**。提供されたナレッジベースが英語や中国語、その他の言語であっても、質問文(question_text)およびキーポイント(key_points)は必ず日本語で回答してください。中国語が混ざらないように厳格に注意してください。
|
||||
|
||||
### 多様性ルール：
|
||||
${rulesJa}
|
||||
|
||||
### 作成済み問題リスト：
|
||||
${existingQuestionsText || 'なし'}
|
||||
|
||||
### 任務：
|
||||
${hasKeywords ? `目標キーワード：${keywordText}\n` : ''}出題スタイル：${style}
|
||||
難易度：${difficultyText}
|
||||
|
||||
以下のJSON配列形式で問題を1つ返してください：
|
||||
[
|
||||
{
|
||||
"question_text": "...",
|
||||
"key_points": ["ポイント1", "ポイント2"],
|
||||
"difficulty": "...",
|
||||
"dimension": "prompt/llm/ide/devPattern/workCapability",
|
||||
"basis": "[n] 引用箇所..."
|
||||
}
|
||||
]`;
|
||||
|
||||
const systemPromptEn = `You are an expert examiner. Generate 1 UNIQUE question based on the provided context.
|
||||
|
||||
### Language Rule:
|
||||
**You MUST generate the question and key points in English.**
|
||||
|
||||
### Diversity Rules:
|
||||
${rulesEn}
|
||||
|
||||
### Previous Questions (DO NOT REPEAT):
|
||||
${existingQuestionsText || 'None'}
|
||||
|
||||
Return 1 question as a JSON array with format:
|
||||
[
|
||||
{
|
||||
"question_text": "...",
|
||||
"key_points": ["point1", "point2"],
|
||||
"difficulty": "...",
|
||||
"dimension": "prompt/llm/ide/devPattern/workCapability",
|
||||
"basis": "[n] citation..."
|
||||
}
|
||||
]`;
|
||||
|
||||
// dimension values: prompt=prompt engineering, llm=LLM principles, ide=IDE collaboration, devPattern=development paradigm, workCapability=work capability
|
||||
|
||||
const systemPrompt = isZh
|
||||
? systemPromptZh
|
||||
: isJa
|
||||
? systemPromptJa
|
||||
: systemPromptEn;
|
||||
const humanMsg = isZh
|
||||
? `请使用中文基于以下内容生成题目：\n\n${knowledgeBaseContent}`
|
||||
: isJa
|
||||
? `以下の内容に基づいて、必ず日本語でアセスメント問題を作成してください：\n\n${knowledgeBaseContent}`
|
||||
: `Generate evaluation question in English based on:\n\n${knowledgeBaseContent}`;
|
||||
|
||||
try {
|
||||
const response = await model.invoke([
|
||||
new SystemMessage(systemPrompt),
|
||||
new HumanMessage(humanMsg),
|
||||
]);
|
||||
|
||||
try {
|
||||
let newQuestions = safeParseJson<any>(response.content as string);
|
||||
|
||||
if (!newQuestions) {
|
||||
console.error('[GeneratorNode] Failed to parse JSON. Raw content:', response.content);
|
||||
throw new Error('Invalid JSON format from AI');
|
||||
}
|
||||
|
||||
// Handle both array and single object
|
||||
if (!Array.isArray(newQuestions)) {
|
||||
newQuestions = [newQuestions];
|
||||
}
|
||||
|
||||
const dimensionMap: Record<string, string> = {
|
||||
// 中文
|
||||
'技术能力-提示词': 'prompt',
|
||||
'提示词': 'prompt',
|
||||
'技术能力-LLM': 'llm',
|
||||
'LLM': 'llm',
|
||||
'IDE协作能力': 'ide',
|
||||
'IDE': 'ide',
|
||||
'AI开发范式': 'devPattern',
|
||||
'开发范式': 'devPattern',
|
||||
'工作能力-安全': 'workCapability',
|
||||
'工作能力': 'workCapability',
|
||||
// 英文直接映射
|
||||
'prompt': 'prompt',
|
||||
'llm': 'llm',
|
||||
'ide': 'ide',
|
||||
'devPattern': 'devPattern',
|
||||
'workCapability': 'workCapability',
|
||||
};
|
||||
|
||||
const mappedNewQuestions = newQuestions.map((q: any) => {
|
||||
let inferredDimension = 'workCapability';
|
||||
const dimValue = q.dimension?.toString().toLowerCase().trim();
|
||||
if (dimValue) {
|
||||
inferredDimension = dimensionMap[dimValue] || 'workCapability';
|
||||
console.log('[GeneratorNode] Dimension mapping:', { original: q.dimension, mapped: inferredDimension });
|
||||
}
|
||||
return {
|
||||
id: (existingQuestions.length + 1).toString(),
|
||||
questionText: q.question_text,
|
||||
keyPoints: q.key_points,
|
||||
difficulty: q.difficulty,
|
||||
basis: q.basis,
|
||||
dimension: inferredDimension,
|
||||
};
|
||||
});
|
||||
|
||||
const questionsToGenerate = Math.max(1, limitCount - existingQuestions.length);
|
||||
const limitedNewQuestions = mappedNewQuestions.slice(0, questionsToGenerate);
|
||||
|
||||
console.log('[GeneratorNode] Generated questions:', mappedNewQuestions.length, 'Limit:', questionsToGenerate);
|
||||
|
||||
return {
|
||||
questions: [...existingQuestions, ...limitedNewQuestions],
|
||||
};
|
||||
} catch (error) {
|
||||
console.error('[GeneratorNode] Parse error:', error);
|
||||
return { questions: existingQuestions };
|
||||
}
|
||||
} catch (invokeError) {
|
||||
console.error('[GeneratorNode] Invoke error:', invokeError);
|
||||
throw invokeError;
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,252 @@
|
||||
import { ChatOpenAI } from '@langchain/openai';
|
||||
import {
|
||||
SystemMessage,
|
||||
HumanMessage,
|
||||
AIMessage,
|
||||
} from '@langchain/core/messages';
|
||||
import { RunnableConfig } from '@langchain/core/runnables';
|
||||
import { EvaluationState } from '../state';
|
||||
import { safeParseJson } from '../../../common/json-utils';
|
||||
|
||||
/**
|
||||
* Node responsible for grading the user's answer and deciding if a follow-up is needed.
|
||||
*/
|
||||
export const graderNode = async (
|
||||
state: EvaluationState,
|
||||
config?: RunnableConfig,
|
||||
): Promise<Partial<EvaluationState>> => {
|
||||
const { model } = (config?.configurable as any) || {};
|
||||
const { questions, currentQuestionIndex, messages } = state;
|
||||
const currentFollowUpCount = state.followUpCount || 0;
|
||||
|
||||
console.log('[GraderNode] Entering node...', {
|
||||
currentIndex: currentQuestionIndex,
|
||||
numMessages: messages?.length,
|
||||
questionCount: state.questionCount,
|
||||
hasQuestions: !!questions?.length,
|
||||
});
|
||||
|
||||
if (!model) {
|
||||
throw new Error('Missing model in node configuration');
|
||||
}
|
||||
|
||||
const lastUserMessage = messages[messages.length - 1];
|
||||
|
||||
console.log('[GraderNode] Incoming Messages Count:', messages.length);
|
||||
if (lastUserMessage) {
|
||||
console.log(
|
||||
'[GraderNode] Last Message Type:',
|
||||
lastUserMessage.constructor.name,
|
||||
);
|
||||
// Safely extract content for logging
|
||||
const logContent =
|
||||
typeof lastUserMessage.content === 'string'
|
||||
? lastUserMessage.content
|
||||
: JSON.stringify(lastUserMessage.content);
|
||||
console.log(
|
||||
'[GraderNode] Last Message Content:',
|
||||
logContent.substring(0, 50),
|
||||
);
|
||||
}
|
||||
|
||||
if (!(lastUserMessage instanceof HumanMessage)) {
|
||||
console.log(
|
||||
'[GraderNode] Last message is not HumanMessage, skipping grading.',
|
||||
);
|
||||
return {};
|
||||
}
|
||||
|
||||
const isZh = state.language === 'zh';
|
||||
const isJa = state.language === 'ja';
|
||||
|
||||
const currentQuestion = questions[currentQuestionIndex];
|
||||
if (!currentQuestion) {
|
||||
console.error(
|
||||
`[GraderNode] Question at index ${currentQuestionIndex} not found!`,
|
||||
);
|
||||
return { currentQuestionIndex: currentQuestionIndex + 1 };
|
||||
}
|
||||
|
||||
const systemPromptZh = `你是一位专业的考官。
|
||||
请根据以下问题和关键点对用户的回答进行评分。
|
||||
|
||||
重要提示：
|
||||
1. **你必须使用以下语言提供反馈：中文 (Simplified Chinese)**。
|
||||
2. 即使用户的回答或知识库内容涉及其他语言，请确保你的反馈和解释依然严格使用中文。不要夹杂日文。
|
||||
|
||||
问题：${currentQuestion.questionText}
|
||||
预期的关键点：${currentQuestion.keyPoints.join(', ')}
|
||||
|
||||
评估标准：
|
||||
1. 准确性：他们是否正确覆盖了关键点？
|
||||
2. 完整性：他们是否遗漏了任何重要内容？
|
||||
3. 深度：解释是否充分？
|
||||
|
||||
请提供：
|
||||
1. 0 到 10 的评分。
|
||||
2. 建设性的反馈。
|
||||
3. 如果回答不完整或不清晰，需要进一步解释，请将 'should_follow_up' 标志设为 true。
|
||||
|
||||
请以 JSON 格式返回响应：
|
||||
{
|
||||
"score": 8,
|
||||
"feedback": "...",
|
||||
"should_follow_up": false
|
||||
}`;
|
||||
|
||||
const systemPromptJa = `あなたは専門的な試験官です。
|
||||
以下の質問とキーポイントに基づいて、ユーザーの回答を採点してください。
|
||||
|
||||
重要事項：
|
||||
1. **フィードバックは必ず次の言語で提供してください：日本語**。
|
||||
2. ユーザーの回答やナレッジベースの内容に他の言語（中国語や英語など）が含まれている場合でも、フィードバックと説明は必ず日本語のみで行ってください。中国語が混ざらないよう厳格に注意してください。
|
||||
|
||||
質問：${currentQuestion.questionText}
|
||||
期待されるキーポイント：${currentQuestion.keyPoints.join(', ')}
|
||||
|
||||
評価基準：
|
||||
1. 正確性：キーポイントを正確に網羅していますか？
|
||||
2. 網羅性：重要な内容が欠落していませんか？
|
||||
3. 深さ：説明は十分ですか？
|
||||
|
||||
以下を提供してください：
|
||||
1. 0 から 10 までのスコア。
|
||||
2. 建設的なフィードバック。
|
||||
3. 回答が不完全または不明確で、さらなる説明が必要な場合は、'should_follow_up' フラグを true に設定してください。
|
||||
|
||||
JSON 形式で回答してください：
|
||||
{
|
||||
"score": 8,
|
||||
"feedback": "...",
|
||||
"should_follow_up": false
|
||||
}`;
|
||||
|
||||
const systemPromptEn = `You are an expert examiner.
|
||||
Grade the user's answer based on the following question and key points.
|
||||
|
||||
IMPORTANT:
|
||||
1. **You MUST provide the feedback in English.**
|
||||
2. If the user's answer or knowledge base content references other languages, ensure your feedback and explanation remain strictly in English.
|
||||
|
||||
QUESTION: ${currentQuestion.questionText}
|
||||
EXPECTED KEY POINTS: ${currentQuestion.keyPoints.join(', ')}
|
||||
|
||||
Evaluate:
|
||||
1. Accuracy: Did they cover the key points correctly?
|
||||
2. Completeness: Did they miss anything important?
|
||||
3. Depth: Is the explanation sufficient?
|
||||
|
||||
Provide:
|
||||
1. A score from 0 to 10.
|
||||
2. Constructive feedback.
|
||||
3. A boolean flag 'should_follow_up' if the answer is incomplete or unclear and needs further clarification.
|
||||
|
||||
Format your response as JSON:
|
||||
{
|
||||
"score": 8,
|
||||
"feedback": "...",
|
||||
"should_follow_up": false
|
||||
}`;
|
||||
|
||||
const systemPrompt = isZh
|
||||
? systemPromptZh
|
||||
: isJa
|
||||
? systemPromptJa
|
||||
: systemPromptEn;
|
||||
|
||||
const userContentText =
|
||||
typeof lastUserMessage.content === 'string'
|
||||
? lastUserMessage.content
|
||||
: JSON.stringify(lastUserMessage.content);
|
||||
|
||||
console.log('[GraderNode] === START GRADING ===');
|
||||
console.log('[GraderNode] User answer length:', userContentText.length);
|
||||
console.log('[GraderNode] Question:', currentQuestion?.questionText?.substring(0, 100));
|
||||
console.log('[GraderNode] Target dimension:', currentQuestion?.dimension);
|
||||
|
||||
const response = await model.invoke([
|
||||
new SystemMessage(systemPrompt),
|
||||
new HumanMessage(userContentText),
|
||||
]);
|
||||
|
||||
console.log('[GraderNode] LLM invoke completed');
|
||||
try {
|
||||
const rawContent = response.content as string;
|
||||
console.log('[GraderNode] Raw AI response length:', rawContent.length);
|
||||
console.log('[GraderNode] Raw AI response:', rawContent.substring(0, 800));
|
||||
|
||||
const result = safeParseJson<any>(rawContent);
|
||||
if (!result) {
|
||||
console.error('[GraderNode] Failed to parse JSON. Raw content:', rawContent);
|
||||
throw new Error('Invalid JSON format from AI');
|
||||
}
|
||||
console.log('[GraderNode] === GRADING RESULT ===');
|
||||
console.log('[GraderNode] Parsed result:', JSON.stringify(result, null, 2));
|
||||
console.log('[GraderNode] Score value:', result.score);
|
||||
console.log('[GraderNode] Feedback value:', result.feedback?.substring(0, 200));
|
||||
|
||||
const scoreLabel = isZh ? '得分' : isJa ? 'スコア' : 'Score';
|
||||
const feedbackLabel = isZh ? '反馈' : isJa ? 'フィードバック' : 'Feedback';
|
||||
|
||||
const feedbackMessage = new AIMessage(
|
||||
`${scoreLabel}: ${result.score}/10\n\n${feedbackLabel}: ${result.feedback}`,
|
||||
);
|
||||
|
||||
const newScores = {
|
||||
...state.scores,
|
||||
[currentQuestion.id || currentQuestionIndex.toString()]: result.score,
|
||||
};
|
||||
|
||||
let shouldFollowUp = result.should_follow_up === true;
|
||||
|
||||
// Breakout logic:
|
||||
// 1. Max 1 follow-up per question
|
||||
// 2. If score is decent (>= 8), don't follow up
|
||||
// 3. If answer is short "don't know", don't follow up
|
||||
const normalizedContent = userContentText.trim().toLowerCase();
|
||||
const saysIDontKnow =
|
||||
normalizedContent.length < 10 &&
|
||||
(normalizedContent.includes('不知道') ||
|
||||
normalizedContent.includes('不会') ||
|
||||
normalizedContent.includes("don't know") ||
|
||||
normalizedContent.includes('no idea') ||
|
||||
normalizedContent.includes('不知') ||
|
||||
normalizedContent.includes('わかりません') ||
|
||||
normalizedContent.includes('わからん') ||
|
||||
normalizedContent.includes('知らない') ||
|
||||
normalizedContent.includes('不明') ||
|
||||
normalizedContent.includes('わからない'));
|
||||
|
||||
if (currentFollowUpCount >= 2 || result.score >= 8 || saysIDontKnow) {
|
||||
shouldFollowUp = false;
|
||||
}
|
||||
|
||||
console.log('[GraderNode] Final State decision:', {
|
||||
shouldFollowUp,
|
||||
nextIndex: shouldFollowUp
|
||||
? currentQuestionIndex
|
||||
: currentQuestionIndex + 1,
|
||||
score: result.score,
|
||||
saysIDontKnow,
|
||||
});
|
||||
|
||||
return {
|
||||
feedbackHistory: [feedbackMessage],
|
||||
scores: newScores,
|
||||
shouldFollowUp: shouldFollowUp,
|
||||
followUpCount: shouldFollowUp ? currentFollowUpCount + 1 : 0,
|
||||
currentQuestionIndex: shouldFollowUp
|
||||
? currentQuestionIndex
|
||||
: currentQuestionIndex + 1,
|
||||
} as any;
|
||||
} catch (error) {
|
||||
console.error('Failed to parse grade from AI response:', error);
|
||||
return {
|
||||
feedbackHistory: [
|
||||
new AIMessage("I had some trouble grading that, but let's move on."),
|
||||
],
|
||||
currentQuestionIndex: currentQuestionIndex + 1,
|
||||
shouldFollowUp: false,
|
||||
} as any;
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,99 @@
|
||||
import { AIMessage } from '@langchain/core/messages';
|
||||
import { RunnableConfig } from '@langchain/core/runnables';
|
||||
import { EvaluationState } from '../state';
|
||||
|
||||
/**
|
||||
* Node responsible for presenting the current question or follow-up to the user.
|
||||
*/
|
||||
export const interviewerNode = async (
|
||||
state: EvaluationState,
|
||||
config?: RunnableConfig,
|
||||
): Promise<Partial<EvaluationState>> => {
|
||||
const { questions, currentQuestionIndex, shouldFollowUp, messages } = state;
|
||||
|
||||
console.log('[InterviewerNode] Entering node...', {
|
||||
numQuestions: questions?.length,
|
||||
currentIndex: currentQuestionIndex,
|
||||
shouldFollowUp,
|
||||
numMessages: messages?.length,
|
||||
});
|
||||
|
||||
if (!questions || questions.length === 0) {
|
||||
const isZh = state.language === 'zh';
|
||||
const isJa = state.language === 'ja';
|
||||
const msg = isZh
|
||||
? '很抱歉，我无法为此会话生成任何问题。'
|
||||
: isJa
|
||||
? '申し訳ありませんが、このセッションの問題を生成できませんでした。'
|
||||
: "I'm sorry, I couldn't generate any questions for this session.";
|
||||
return {
|
||||
messages: [new AIMessage(msg)],
|
||||
};
|
||||
}
|
||||
|
||||
const currentQuestion = questions[currentQuestionIndex];
|
||||
|
||||
// If it's a follow-up, we add a prefix to the label later.
|
||||
// If we've run out of questions and no follow-up requested, we shouldn't be here, but let's be safe.
|
||||
if (currentQuestionIndex >= questions.length) {
|
||||
return { shouldFollowUp: false };
|
||||
}
|
||||
|
||||
const isZh = state.language === 'zh';
|
||||
const isJa = state.language === 'ja';
|
||||
|
||||
let prompt = '';
|
||||
|
||||
if (
|
||||
shouldFollowUp &&
|
||||
state.feedbackHistory &&
|
||||
state.feedbackHistory.length > 0
|
||||
) {
|
||||
// Construct a follow-up prompt based on last feedback
|
||||
const lastFeedbackMsg =
|
||||
state.feedbackHistory[state.feedbackHistory.length - 1];
|
||||
const feedbackText = lastFeedbackMsg.content.toString();
|
||||
|
||||
// Extract the "Feedback: ..." part if possible, otherwise use whole text
|
||||
const feedbackMatch = feedbackText.match(
|
||||
/(?:Feedback|反馈|フィードバック): ([\s\S]*)/i,
|
||||
);
|
||||
const specificFeedback = feedbackMatch
|
||||
? feedbackMatch[1].trim()
|
||||
: feedbackText;
|
||||
|
||||
const followUpLabel = isZh
|
||||
? '补充追问'
|
||||
: isJa
|
||||
? '追加の質問'
|
||||
: 'Follow-up Clarification';
|
||||
const followUpInstruction = isZh
|
||||
? '根据以上反馈，请补充更具体的信息：'
|
||||
: isJa
|
||||
? '上記のフィードバックに基づき、より具体的な情報を追加してください：'
|
||||
: 'Based on the feedback above, please provide more specific details:';
|
||||
|
||||
prompt = `${followUpLabel}\n\n${specificFeedback}\n\n${followUpInstruction}`;
|
||||
} else {
|
||||
// Standard question presentation
|
||||
const label = isZh
|
||||
? `问题 ${currentQuestionIndex + 1}`
|
||||
: isJa
|
||||
? `質問 ${currentQuestionIndex + 1}`
|
||||
: `Question ${currentQuestionIndex + 1}`;
|
||||
|
||||
const instruction = isZh
|
||||
? '请提供您的回答。'
|
||||
: isJa
|
||||
? '回答を入力してください。'
|
||||
: 'Please provide your answer.';
|
||||
|
||||
prompt = `${label}: ${currentQuestion.questionText}\n\n${instruction}`;
|
||||
}
|
||||
|
||||
console.log('[InterviewerNode] Returning question:', { currentQuestionIndex, questionText: currentQuestion?.questionText });
|
||||
return {
|
||||
messages: [new AIMessage(prompt)],
|
||||
shouldFollowUp: false,
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,124 @@
|
||||
import { Annotation, MessagesAnnotation } from '@langchain/langgraph';
|
||||
import { BaseMessage } from '@langchain/core/messages';
|
||||
|
||||
/**
|
||||
* State representing the evaluation session using LangGraph Annotation.
|
||||
*/
|
||||
export const EvaluationAnnotation = Annotation.Root({
|
||||
/**
|
||||
* The message history of the conversation.
|
||||
* Inherits from MessagesAnnotation to handle message merging.
|
||||
*/
|
||||
...MessagesAnnotation.spec,
|
||||
|
||||
/**
|
||||
* Historical evaluation feedback from the grader.
|
||||
* Separated from main messages to keep conversation context clean.
|
||||
*/
|
||||
feedbackHistory: Annotation<BaseMessage[]>({
|
||||
reducer: (prev, next) => [...(prev || []), ...(next || [])],
|
||||
default: () => [],
|
||||
}),
|
||||
|
||||
/**
|
||||
* The database ID of the current assessment session.
|
||||
*/
|
||||
assessmentSessionId: Annotation<string>(),
|
||||
|
||||
/**
|
||||
* The knowledge base ID used as ground truth for this evaluation.
|
||||
*/
|
||||
knowledgeBaseId: Annotation<string>(),
|
||||
|
||||
/**
|
||||
* List of questions generated for this session.
|
||||
*/
|
||||
questions: Annotation<any[]>({
|
||||
reducer: (prev, next) => next ?? prev,
|
||||
default: () => [],
|
||||
}),
|
||||
|
||||
/**
|
||||
* Index of the current question being discussed.
|
||||
*/
|
||||
currentQuestionIndex: Annotation<number>({
|
||||
reducer: (prev, next) => next ?? prev,
|
||||
default: () => 0,
|
||||
}),
|
||||
|
||||
/**
|
||||
* Flag indicating if the Grader believes a follow-up question is needed for clarity.
|
||||
*/
|
||||
shouldFollowUp: Annotation<boolean>({
|
||||
reducer: (prev, next) => next ?? prev,
|
||||
default: () => false,
|
||||
}),
|
||||
|
||||
/**
|
||||
* Map of scores for each question.
|
||||
*/
|
||||
scores: Annotation<Record<string, number>>({
|
||||
reducer: (prev, next) => ({ ...prev, ...next }),
|
||||
default: () => ({}),
|
||||
}),
|
||||
|
||||
/**
|
||||
* Final report generated by the ReportAnalyzer.
|
||||
*/
|
||||
report: Annotation<string | undefined>({
|
||||
reducer: (prev, next) => next ?? prev,
|
||||
}),
|
||||
|
||||
/**
|
||||
* Context chunks retrieved from the knowledge base for grounding.
|
||||
*/
|
||||
context: Annotation<string[] | undefined>({
|
||||
reducer: (prev, next) => next ?? prev,
|
||||
}),
|
||||
|
||||
/**
|
||||
* Preferred language for the assessment (zh, en, ja).
|
||||
*/
|
||||
language: Annotation<string>({
|
||||
reducer: (prev, next) => next ?? prev,
|
||||
default: () => 'zh',
|
||||
}),
|
||||
|
||||
/**
|
||||
* Number of times we have followed up on the current question.
|
||||
*/
|
||||
followUpCount: Annotation<number>({
|
||||
reducer: (prev, next) => next ?? prev,
|
||||
default: () => 0,
|
||||
}),
|
||||
|
||||
/**
|
||||
* Number of questions to generate.
|
||||
*/
|
||||
questionCount: Annotation<number | undefined>({
|
||||
reducer: (prev, next) => next ?? prev,
|
||||
}),
|
||||
|
||||
/**
|
||||
* Desired difficulty distribution.
|
||||
*/
|
||||
difficultyDistribution: Annotation<any | undefined>({
|
||||
reducer: (prev, next) => next ?? prev,
|
||||
}),
|
||||
|
||||
/**
|
||||
* Desired question style.
|
||||
*/
|
||||
style: Annotation<string | undefined>({
|
||||
reducer: (prev, next) => next ?? prev,
|
||||
}),
|
||||
|
||||
/**
|
||||
* Target keywords for question generation.
|
||||
*/
|
||||
keywords: Annotation<string[] | undefined>({
|
||||
reducer: (prev, next) => next ?? prev,
|
||||
}),
|
||||
});
|
||||
|
||||
export type EvaluationState = typeof EvaluationAnnotation.State;
|
||||
@@ -0,0 +1,45 @@
|
||||
-- Migration: Create Question Banks Tables
|
||||
-- Run this SQL to create the question_banks and question_bank_items tables
|
||||
|
||||
-- Create QuestionBanks table
|
||||
CREATE TABLE IF NOT EXISTS question_banks (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
tenant_id VARCHAR(255),
|
||||
template_id UUID,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
description TEXT,
|
||||
status VARCHAR(50) DEFAULT 'DRAFT',
|
||||
created_by VARCHAR(255),
|
||||
reviewed_by VARCHAR(255),
|
||||
reviewed_at TIMESTAMP,
|
||||
review_comment TEXT,
|
||||
created_at TIMESTAMP DEFAULT NOW(),
|
||||
updated_at TIMESTAMP DEFAULT NOW(),
|
||||
CONSTRAINT fk_template FOREIGN KEY (template_id) REFERENCES assessment_templates(id) ON DELETE SET NULL
|
||||
);
|
||||
|
||||
-- Create QuestionBankItems table
|
||||
CREATE TABLE IF NOT EXISTS question_bank_items (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
bank_id UUID NOT NULL,
|
||||
question_text TEXT NOT NULL,
|
||||
question_type VARCHAR(50) DEFAULT 'SHORT_ANSWER',
|
||||
options JSONB,
|
||||
correct_answer TEXT,
|
||||
key_points JSONB NOT NULL,
|
||||
difficulty VARCHAR(50) DEFAULT 'STANDARD',
|
||||
dimension VARCHAR(50) DEFAULT 'PROMPT',
|
||||
basis TEXT,
|
||||
created_by VARCHAR(255),
|
||||
created_at TIMESTAMP DEFAULT NOW(),
|
||||
updated_at TIMESTAMP DEFAULT NOW(),
|
||||
CONSTRAINT fk_bank FOREIGN KEY (bank_id) REFERENCES question_banks(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
-- Create indexes for performance
|
||||
CREATE INDEX IF NOT EXISTS idx_question_banks_tenant_id ON question_banks(tenant_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_question_banks_status ON question_banks(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_question_banks_created_by ON question_banks(created_by);
|
||||
CREATE INDEX IF NOT EXISTS idx_question_bank_items_bank_id ON question_bank_items(bank_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_question_bank_items_difficulty ON question_bank_items(difficulty);
|
||||
CREATE INDEX IF NOT EXISTS idx_question_bank_items_dimension ON question_bank_items(dimension);
|
||||
@@ -0,0 +1,60 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
@Injectable()
|
||||
export class ContentFilterService {
|
||||
private readonly logger = new Logger(ContentFilterService.name);
|
||||
|
||||
/**
|
||||
* Filters knowledge base content based on keywords.
|
||||
* In a real implementation, this might use semantic search or simple keyword filtering.
|
||||
* For now, we'll implement a simple relevance-based filtering.
|
||||
*/
|
||||
filterContent(content: string, keywords: string[]): string {
|
||||
if (!keywords || keywords.length === 0) {
|
||||
return content;
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`Filtering content with ${keywords.length} keywords: ${keywords.join(', ')}`,
|
||||
);
|
||||
|
||||
// Split content into paragraphs or sections
|
||||
const sections = content.split(/\n\n+/);
|
||||
|
||||
// Score each section based on keyword matches (case-insensitive)
|
||||
const scoredSections = sections.map((section) => {
|
||||
let score = 0;
|
||||
const lowerSection = section.toLowerCase();
|
||||
|
||||
keywords.forEach((keyword) => {
|
||||
const lowerKeyword = keyword.toLowerCase();
|
||||
const matches = lowerSection.split(lowerKeyword).length - 1;
|
||||
score += matches;
|
||||
});
|
||||
|
||||
return { section, score };
|
||||
});
|
||||
|
||||
// Sort sections by score and take the most relevant ones
|
||||
// If content is huge, we might want to limit the total length
|
||||
const relevantSections = scoredSections
|
||||
.filter((s) => s.score > 0)
|
||||
.sort((a, b) => b.score - a.score)
|
||||
.map((s) => s.section);
|
||||
|
||||
// If no sections matched, return a sample or the original content
|
||||
if (relevantSections.length === 0) {
|
||||
this.logger.warn(
|
||||
'No sections matched keywords, returning first 5000 characters',
|
||||
);
|
||||
return content.substring(0, 5000);
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`Found ${relevantSections.length} relevant sections out of ${sections.length}`,
|
||||
);
|
||||
|
||||
// Return combined relevant sections (up to a reasonable limit)
|
||||
return relevantSections.join('\n\n').substring(0, 50000);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,393 @@
|
||||
import {
|
||||
Injectable,
|
||||
NotFoundException,
|
||||
ForbiddenException,
|
||||
Logger,
|
||||
} from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Repository } from 'typeorm';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { ChatOpenAI } from '@langchain/openai';
|
||||
import { SystemMessage, HumanMessage } from '@langchain/core/messages';
|
||||
import { QuestionBank, QuestionBankStatus } from '../entities/question-bank.entity';
|
||||
import {
|
||||
QuestionBankItem,
|
||||
QuestionType,
|
||||
QuestionDifficulty,
|
||||
QuestionDimension,
|
||||
} from '../entities/question-bank-item.entity';
|
||||
import { ModelConfigService } from '../model-config/model-config.service';
|
||||
import { ModelType } from '../types';
|
||||
import { safeParseJson } from '../../common/json-utils';
|
||||
|
||||
export interface CreateQuestionBankDto {
|
||||
templateId?: string;
|
||||
name: string;
|
||||
description?: string;
|
||||
}
|
||||
|
||||
export interface UpdateQuestionBankDto {
|
||||
name?: string;
|
||||
description?: string;
|
||||
status?: QuestionBankStatus;
|
||||
}
|
||||
|
||||
export interface CreateQuestionBankItemDto {
|
||||
questionText: string;
|
||||
questionType?: QuestionType;
|
||||
options?: string[];
|
||||
correctAnswer?: string;
|
||||
keyPoints: string[];
|
||||
difficulty?: QuestionDifficulty;
|
||||
dimension?: QuestionDimension;
|
||||
basis?: string;
|
||||
}
|
||||
|
||||
export interface UpdateQuestionBankItemDto {
|
||||
questionText?: string;
|
||||
questionType?: QuestionType;
|
||||
options?: string[];
|
||||
correctAnswer?: string;
|
||||
keyPoints?: string[];
|
||||
difficulty?: QuestionDifficulty;
|
||||
dimension?: QuestionDimension;
|
||||
basis?: string;
|
||||
}
|
||||
|
||||
export interface ReviewDto {
|
||||
approved: boolean;
|
||||
comment?: string;
|
||||
}
|
||||
|
||||
const DIMENSIONS = [
|
||||
QuestionDimension.PROMPT,
|
||||
QuestionDimension.LLM,
|
||||
QuestionDimension.IDE,
|
||||
QuestionDimension.DEV_PATTERN,
|
||||
QuestionDimension.WORK_CAPABILITY,
|
||||
];
|
||||
|
||||
@Injectable()
|
||||
export class QuestionBankService {
|
||||
private readonly logger = new Logger(QuestionBankService.name);
|
||||
|
||||
constructor(
|
||||
@InjectRepository(QuestionBank)
|
||||
private readonly bankRepository: Repository<QuestionBank>,
|
||||
@InjectRepository(QuestionBankItem)
|
||||
private readonly itemRepository: Repository<QuestionBankItem>,
|
||||
private readonly modelConfigService: ModelConfigService,
|
||||
private readonly configService: ConfigService,
|
||||
) {}
|
||||
|
||||
async create(
|
||||
createDto: CreateQuestionBankDto,
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
): Promise<QuestionBank> {
|
||||
const bank = this.bankRepository.create({
|
||||
...createDto,
|
||||
createdBy: userId,
|
||||
tenantId,
|
||||
status: QuestionBankStatus.DRAFT,
|
||||
});
|
||||
return this.bankRepository.save(bank);
|
||||
}
|
||||
|
||||
async findAll(
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
page?: number,
|
||||
limit?: number,
|
||||
): Promise<{ data: QuestionBank[]; total: number } | QuestionBank[]> {
|
||||
const queryBuilder = this.bankRepository
|
||||
.createQueryBuilder('bank')
|
||||
.leftJoinAndSelect('bank.template', 'template')
|
||||
.where('bank.tenantId = :tenantId', { tenantId })
|
||||
.orderBy('bank.createdAt', 'DESC');
|
||||
|
||||
if (page !== undefined && limit !== undefined) {
|
||||
const [data, total] = await queryBuilder
|
||||
.skip((page - 1) * limit)
|
||||
.take(limit)
|
||||
.getManyAndCount();
|
||||
return { data, total };
|
||||
}
|
||||
|
||||
return queryBuilder.getMany();
|
||||
}
|
||||
|
||||
async findByTemplateId(templateId: string): Promise<QuestionBank | null> {
|
||||
return this.bankRepository.findOne({
|
||||
where: { templateId },
|
||||
relations: ['template', 'items'],
|
||||
});
|
||||
}
|
||||
|
||||
async findOne(id: string): Promise<QuestionBank> {
|
||||
const bank = await this.bankRepository.findOne({
|
||||
where: { id },
|
||||
relations: ['template', 'items'],
|
||||
});
|
||||
if (!bank) {
|
||||
throw new NotFoundException(`QuestionBank with ID "${id}" not found`);
|
||||
}
|
||||
return bank;
|
||||
}
|
||||
|
||||
async update(
|
||||
id: string,
|
||||
updateDto: UpdateQuestionBankDto,
|
||||
): Promise<QuestionBank> {
|
||||
const bank = await this.findOne(id);
|
||||
Object.assign(bank, updateDto);
|
||||
return this.bankRepository.save(bank);
|
||||
}
|
||||
|
||||
async remove(id: string): Promise<void> {
|
||||
const bank = await this.findOne(id);
|
||||
await this.bankRepository.remove(bank);
|
||||
}
|
||||
|
||||
async submitForReview(id: string, userId: string): Promise<QuestionBank> {
|
||||
const bank = await this.findOne(id);
|
||||
if (bank.status !== QuestionBankStatus.DRAFT) {
|
||||
throw new ForbiddenException(
|
||||
'Only DRAFT status can be submitted for review',
|
||||
);
|
||||
}
|
||||
bank.status = QuestionBankStatus.PENDING_REVIEW;
|
||||
return this.bankRepository.save(bank);
|
||||
}
|
||||
|
||||
async review(
|
||||
id: string,
|
||||
reviewDto: ReviewDto,
|
||||
reviewerId: string,
|
||||
): Promise<QuestionBank> {
|
||||
const bank = await this.findOne(id);
|
||||
if (bank.status !== QuestionBankStatus.PENDING_REVIEW) {
|
||||
throw new ForbiddenException(
|
||||
'Only PENDING_REVIEW status can be reviewed',
|
||||
);
|
||||
}
|
||||
bank.reviewedBy = reviewerId;
|
||||
bank.reviewedAt = new Date();
|
||||
bank.reviewComment = reviewDto.comment || null;
|
||||
bank.status = reviewDto.approved
|
||||
? QuestionBankStatus.PUBLISHED
|
||||
: QuestionBankStatus.REJECTED;
|
||||
return this.bankRepository.save(bank);
|
||||
}
|
||||
|
||||
async publish(id: string): Promise<QuestionBank> {
|
||||
const bank = await this.findOne(id);
|
||||
if (bank.status === QuestionBankStatus.PUBLISHED) {
|
||||
return bank;
|
||||
}
|
||||
bank.status = QuestionBankStatus.PUBLISHED;
|
||||
this.logger.log(`QuestionBank ${id} published`);
|
||||
return this.bankRepository.save(bank);
|
||||
}
|
||||
|
||||
async addItem(
|
||||
bankId: string,
|
||||
createDto: CreateQuestionBankItemDto,
|
||||
): Promise<QuestionBankItem> {
|
||||
await this.findOne(bankId);
|
||||
const item = this.itemRepository.create({
|
||||
...createDto,
|
||||
bankId,
|
||||
questionType: createDto.questionType || QuestionType.SHORT_ANSWER,
|
||||
difficulty: createDto.difficulty || QuestionDifficulty.STANDARD,
|
||||
dimension: createDto.dimension || QuestionDimension.PROMPT,
|
||||
});
|
||||
return this.itemRepository.save(item);
|
||||
}
|
||||
|
||||
async updateItem(
|
||||
bankId: string,
|
||||
itemId: string,
|
||||
updateDto: UpdateQuestionBankItemDto,
|
||||
): Promise<QuestionBankItem> {
|
||||
await this.findOne(bankId);
|
||||
const item = await this.itemRepository.findOne({
|
||||
where: { id: itemId, bankId },
|
||||
});
|
||||
if (!item) {
|
||||
throw new NotFoundException(`QuestionBankItem with ID "${itemId}" not found`);
|
||||
}
|
||||
Object.assign(item, updateDto);
|
||||
return this.itemRepository.save(item);
|
||||
}
|
||||
|
||||
async removeItem(bankId: string, itemId: string): Promise<void> {
|
||||
await this.findOne(bankId);
|
||||
const item = await this.itemRepository.findOne({
|
||||
where: { id: itemId, bankId },
|
||||
});
|
||||
if (!item) {
|
||||
throw new NotFoundException(`QuestionBankItem with ID "${itemId}" not found`);
|
||||
}
|
||||
await this.itemRepository.remove(item);
|
||||
}
|
||||
|
||||
async generateQuestions(
|
||||
bankId: string,
|
||||
count: number,
|
||||
knowledgeBaseContent: string,
|
||||
tenantId: string,
|
||||
): Promise<QuestionBankItem[]> {
|
||||
const bank = await this.findOne(bankId);
|
||||
this.logger.log(`[generateQuestions] Starting AI generation for bank ${bankId}, count: ${count}`);
|
||||
|
||||
const modelConfig = await this.modelConfigService.findDefaultByType(
|
||||
tenantId,
|
||||
ModelType.LLM,
|
||||
);
|
||||
const model = new ChatOpenAI({
|
||||
apiKey: modelConfig.apiKey || 'ollama',
|
||||
modelName: modelConfig.modelId,
|
||||
temperature: 0.7,
|
||||
configuration: {
|
||||
baseURL: modelConfig.baseUrl || 'https://api.deepseek.com/v1',
|
||||
},
|
||||
});
|
||||
|
||||
const systemPrompt = `你是一位专业的知识评估专家。请根据提供的知识库片段生成 ${count} 个唯一的测试题目。
|
||||
|
||||
### 强制性语言规则：
|
||||
**必须使用中文 (Simplified Chinese) 进行回复**。即使知识库内容是英文或其他语言，问题(question_text)和关键点(key_points)也必须使用中文。
|
||||
|
||||
### 多样性规则：
|
||||
1. 禁止重复：绝对禁止生成相似的题目
|
||||
2. 深度挖掘：从不同的角度出题，如流程、限制、优缺点、具体参数等
|
||||
3. 随机扰动：从不同的逻辑链条出发
|
||||
|
||||
### 任务：
|
||||
请以 JSON 数组格式返回 ${count} 个问题：
|
||||
[
|
||||
{
|
||||
"question_text": "问题内容",
|
||||
"key_points": ["要点1", "要点2"],
|
||||
"difficulty": "STANDARD|ADVANCED|SPECIALIST",
|
||||
"dimension": "prompt|llm|ide|devPattern|workCapability",
|
||||
"basis": "[n] 引用原文..."
|
||||
}
|
||||
]`;
|
||||
|
||||
const humanMsg = `请使用中文基于以下内容生成题目：\n\n${knowledgeBaseContent}`;
|
||||
|
||||
try {
|
||||
const response = await model.invoke([
|
||||
new SystemMessage(systemPrompt),
|
||||
new HumanMessage(humanMsg),
|
||||
]);
|
||||
|
||||
let parsedQuestions = safeParseJson<any>(response.content as string);
|
||||
if (!parsedQuestions) {
|
||||
this.logger.error('[generateQuestions] Failed to parse JSON from AI response');
|
||||
throw new Error('Invalid JSON format from AI');
|
||||
}
|
||||
|
||||
if (!Array.isArray(parsedQuestions)) {
|
||||
parsedQuestions = [parsedQuestions];
|
||||
}
|
||||
|
||||
const dimensionMap: Record<string, string> = {
|
||||
'prompt': 'PROMPT',
|
||||
'llm': 'LLM',
|
||||
'ide': 'IDE',
|
||||
'devPattern': 'DEV_PATTERN',
|
||||
'workCapability': 'WORK_CAPABILITY',
|
||||
};
|
||||
|
||||
const difficultyMap: Record<string, string> = {
|
||||
'STANDARD': 'STANDARD',
|
||||
'ADVANCED': 'ADVANCED',
|
||||
'SPECIALIST': 'SPECIALIST',
|
||||
};
|
||||
|
||||
const items: QuestionBankItem[] = [];
|
||||
for (const q of parsedQuestions) {
|
||||
const dimension = dimensionMap[q.dimension?.toLowerCase()] || 'WORK_CAPABILITY';
|
||||
const difficulty = difficultyMap[q.difficulty?.toUpperCase()] || 'STANDARD';
|
||||
|
||||
const item = this.itemRepository.create({
|
||||
bankId,
|
||||
questionText: q.question_text,
|
||||
questionType: QuestionType.SHORT_ANSWER,
|
||||
keyPoints: q.key_points || [],
|
||||
difficulty: difficulty as QuestionDifficulty,
|
||||
dimension: dimension as QuestionDimension,
|
||||
basis: q.basis,
|
||||
});
|
||||
items.push(await this.itemRepository.save(item));
|
||||
}
|
||||
|
||||
this.logger.log(`[generateQuestions] Generated ${items.length} questions for bank ${bankId}`);
|
||||
return items;
|
||||
} catch (error) {
|
||||
this.logger.error('[generateQuestions] Error generating questions:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async selectQuestions(
|
||||
bankId: string,
|
||||
count: number,
|
||||
): Promise<QuestionBankItem[]> {
|
||||
const bank = await this.findOne(bankId);
|
||||
if (bank.status !== QuestionBankStatus.PUBLISHED) {
|
||||
throw new ForbiddenException(
|
||||
'Only PUBLISHED banks can be used for selection',
|
||||
);
|
||||
}
|
||||
|
||||
const allItems = await this.itemRepository.find({
|
||||
where: { bankId },
|
||||
});
|
||||
|
||||
if (allItems.length === 0) {
|
||||
this.logger.warn(`[selectQuestions] No items found for bank ${bankId}`);
|
||||
return [];
|
||||
}
|
||||
|
||||
const usedIds = new Set<string>();
|
||||
const selected: QuestionBankItem[] = [];
|
||||
|
||||
let dimIdx = 0;
|
||||
while (selected.length < count && usedIds.size < allItems.length) {
|
||||
const dim = DIMENSIONS[dimIdx % DIMENSIONS.length];
|
||||
dimIdx++;
|
||||
|
||||
if (selected.length >= count) break;
|
||||
|
||||
const available = allItems.filter(
|
||||
(i) => i.dimension === dim && !usedIds.has(i.id),
|
||||
);
|
||||
|
||||
if (available.length > 0) {
|
||||
const idx = Math.floor(Math.random() * available.length);
|
||||
const item = available[idx];
|
||||
selected.push(item);
|
||||
usedIds.add(item.id);
|
||||
}
|
||||
}
|
||||
|
||||
if (selected.length < count) {
|
||||
const remaining = allItems.filter((i) => !usedIds.has(i.id));
|
||||
const shuffled = remaining.sort(() => Math.random() - 0.5);
|
||||
for (const item of shuffled) {
|
||||
if (selected.length >= count) break;
|
||||
selected.push(item);
|
||||
usedIds.add(item.id);
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`[selectQuestions] Selected ${selected.length} questions from bank ${bankId}`,
|
||||
);
|
||||
return selected;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
@Injectable()
|
||||
export class QuestionOutlineService {}
|
||||
@@ -0,0 +1,89 @@
|
||||
import {
|
||||
Injectable,
|
||||
NotFoundException,
|
||||
ForbiddenException,
|
||||
} from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Repository } from 'typeorm';
|
||||
import { AssessmentTemplate } from '../entities/assessment-template.entity';
|
||||
import { CreateTemplateDto } from '../dto/create-template.dto';
|
||||
import { UpdateTemplateDto } from '../dto/update-template.dto';
|
||||
import { TenantService } from '../../tenant/tenant.service';
|
||||
|
||||
@Injectable()
|
||||
export class TemplateService {
|
||||
constructor(
|
||||
@InjectRepository(AssessmentTemplate)
|
||||
private readonly templateRepository: Repository<AssessmentTemplate>,
|
||||
private readonly tenantService: TenantService,
|
||||
) {}
|
||||
|
||||
async create(
|
||||
createDto: CreateTemplateDto,
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
): Promise<AssessmentTemplate> {
|
||||
const { ...data } = createDto;
|
||||
const template = this.templateRepository.create({
|
||||
...data,
|
||||
createdBy: userId,
|
||||
tenantId,
|
||||
});
|
||||
return this.templateRepository.save(template);
|
||||
}
|
||||
|
||||
async findAll(tenantId: string): Promise<AssessmentTemplate[]> {
|
||||
return this.templateRepository.find({
|
||||
where: { tenantId, isActive: true },
|
||||
relations: ['knowledgeGroup'],
|
||||
order: { createdAt: 'DESC' },
|
||||
});
|
||||
}
|
||||
|
||||
async findOne(
|
||||
id: string,
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
): Promise<AssessmentTemplate> {
|
||||
const template = await this.templateRepository.findOne({
|
||||
where: { id },
|
||||
relations: ['knowledgeGroup'],
|
||||
});
|
||||
|
||||
if (!template) {
|
||||
throw new NotFoundException(`Template with ID "${id}" not found`);
|
||||
}
|
||||
|
||||
// Check permission using TenantService
|
||||
const hasAccess = await this.tenantService.canAccessTenant(
|
||||
userId,
|
||||
template.tenantId,
|
||||
tenantId,
|
||||
);
|
||||
if (!hasAccess) {
|
||||
throw new ForbiddenException(
|
||||
`You do not have permission to access this template`,
|
||||
);
|
||||
}
|
||||
|
||||
return template;
|
||||
}
|
||||
|
||||
async update(
|
||||
id: string,
|
||||
updateDto: UpdateTemplateDto,
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
): Promise<AssessmentTemplate> {
|
||||
const template = await this.findOne(id, userId, tenantId);
|
||||
Object.assign(template, updateDto);
|
||||
return this.templateRepository.save(template);
|
||||
}
|
||||
|
||||
async remove(id: string, userId: string, tenantId: string): Promise<void> {
|
||||
const template = await this.findOne(id, userId, tenantId);
|
||||
// Soft delete by setting isActive to false
|
||||
template.isActive = false;
|
||||
await this.templateRepository.save(template);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import { Injectable, CanActivate, ExecutionContext } from '@nestjs/common';
|
||||
import { UserRole } from '../user/user-role.enum';
|
||||
|
||||
@Injectable()
|
||||
export class AdminGuard implements CanActivate {
|
||||
canActivate(context: ExecutionContext): boolean {
|
||||
const request = context.switchToHttp().getRequest();
|
||||
const user = request.user;
|
||||
|
||||
// Check if user exists and has admin privileges (Super Admin or Tenant Admin)
|
||||
return !!(
|
||||
user &&
|
||||
(user.role === UserRole.SUPER_ADMIN ||
|
||||
user.role === UserRole.TENANT_ADMIN ||
|
||||
user.isAdmin === true)
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
import {
|
||||
Injectable,
|
||||
CanActivate,
|
||||
ExecutionContext,
|
||||
UnauthorizedException,
|
||||
} from '@nestjs/common';
|
||||
import { Reflector } from '@nestjs/core';
|
||||
import { UserService } from '../user/user.service';
|
||||
import { Request } from 'express';
|
||||
import { IS_PUBLIC_KEY } from './public.decorator';
|
||||
|
||||
@Injectable()
|
||||
export class ApiKeyGuard implements CanActivate {
|
||||
constructor(
|
||||
private reflector: Reflector,
|
||||
private userService: UserService,
|
||||
) {}
|
||||
|
||||
async canActivate(context: ExecutionContext): Promise<boolean> {
|
||||
const isPublic = this.reflector.getAllAndOverride<boolean>(IS_PUBLIC_KEY, [
|
||||
context.getHandler(),
|
||||
context.getClass(),
|
||||
]);
|
||||
if (isPublic) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const request = context
|
||||
.switchToHttp()
|
||||
.getRequest<Request & { user?: any; tenantId?: string }>();
|
||||
const apiKey = this.extractApiKeyFromHeader(request);
|
||||
|
||||
if (apiKey) {
|
||||
const user = await this.userService.findByApiKey(apiKey);
|
||||
if (user) {
|
||||
request.user = user;
|
||||
request.tenantId = user.tenantId;
|
||||
return true;
|
||||
}
|
||||
throw new UnauthorizedException('Invalid API key');
|
||||
}
|
||||
|
||||
throw new UnauthorizedException('Missing API key');
|
||||
}
|
||||
|
||||
private extractApiKeyFromHeader(request: Request): string | undefined {
|
||||
const authHeader = request.headers.authorization;
|
||||
if (authHeader && authHeader.startsWith('Bearer kb_')) {
|
||||
return authHeader.substring(7, authHeader.length);
|
||||
}
|
||||
const headerKey = request.headers['x-api-key'] as string;
|
||||
if (headerKey) return headerKey;
|
||||
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import { Controller, Get, Post, Request, UseGuards } from '@nestjs/common';
|
||||
import { AuthService } from './auth.service';
|
||||
import { LocalAuthGuard } from './local-auth.guard';
|
||||
import { CombinedAuthGuard } from './combined-auth.guard';
|
||||
import { Public } from './public.decorator';
|
||||
|
||||
@Controller('auth')
|
||||
export class AuthController {
|
||||
constructor(private authService: AuthService) {}
|
||||
|
||||
@Public()
|
||||
@UseGuards(LocalAuthGuard)
|
||||
@Post('login')
|
||||
async login(@Request() req) {
|
||||
return this.authService.login(req.user);
|
||||
}
|
||||
|
||||
@UseGuards(CombinedAuthGuard)
|
||||
@Get('profile')
|
||||
getProfile(@Request() req) {
|
||||
return req.user;
|
||||
}
|
||||
|
||||
@UseGuards(CombinedAuthGuard)
|
||||
@Get('api-key')
|
||||
async getApiKey(@Request() req) {
|
||||
const apiKey = await this.authService.getOrCreateApiKey(req.user.id);
|
||||
return { apiKey };
|
||||
}
|
||||
|
||||
@UseGuards(CombinedAuthGuard)
|
||||
@Post('api-key/regenerate')
|
||||
async regenerateApiKey(@Request() req) {
|
||||
const apiKey = await this.authService.regenerateApiKey(req.user.id);
|
||||
return { apiKey };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { AuthService } from './auth.service';
|
||||
import { AuthController } from './auth.controller';
|
||||
import { UserModule } from '../user/user.module';
|
||||
import { PassportModule } from '@nestjs/passport';
|
||||
import { JwtModule } from '@nestjs/jwt';
|
||||
import { ConfigModule, ConfigService } from '@nestjs/config';
|
||||
import { LocalStrategy } from './local.strategy';
|
||||
import { JwtStrategy } from './jwt.strategy';
|
||||
import { TenantModule } from '../tenant/tenant.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
UserModule,
|
||||
TenantModule,
|
||||
PassportModule,
|
||||
JwtModule.registerAsync({
|
||||
imports: [ConfigModule],
|
||||
inject: [ConfigService],
|
||||
useFactory: (configService: ConfigService) => ({
|
||||
secret: configService.get<string>('JWT_SECRET'),
|
||||
signOptions: { expiresIn: '1d' }, // Token expires in 1 day
|
||||
}),
|
||||
}),
|
||||
],
|
||||
providers: [AuthService, LocalStrategy, JwtStrategy],
|
||||
controllers: [AuthController],
|
||||
})
|
||||
export class AuthModule {}
|
||||
@@ -0,0 +1,48 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { UserService } from '../user/user.service';
|
||||
import { JwtService } from '@nestjs/jwt';
|
||||
import { User } from '../user/user.entity';
|
||||
import { SafeUser } from '../user/dto/user-safe.dto';
|
||||
|
||||
@Injectable()
|
||||
export class AuthService {
|
||||
constructor(
|
||||
private userService: UserService,
|
||||
private jwtService: JwtService,
|
||||
) {}
|
||||
|
||||
async validateUser(username: string, pass: string): Promise<User | null> {
|
||||
const user = await this.userService.findOneByUsername(username);
|
||||
if (user && (await user.validatePassword(pass))) {
|
||||
return user;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async login(user: SafeUser) {
|
||||
const payload = {
|
||||
username: user.username,
|
||||
sub: user.id,
|
||||
role: user.role,
|
||||
tenantId: user.tenantId,
|
||||
};
|
||||
return {
|
||||
access_token: this.jwtService.sign(payload),
|
||||
user: {
|
||||
id: user.id,
|
||||
username: user.username,
|
||||
role: user.role,
|
||||
tenantId: user.tenantId,
|
||||
displayName: user.displayName,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async getOrCreateApiKey(userId: string) {
|
||||
return this.userService.getOrCreateApiKey(userId);
|
||||
}
|
||||
|
||||
async regenerateApiKey(userId: string) {
|
||||
return this.userService.regenerateApiKey(userId);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
import {
|
||||
Injectable,
|
||||
CanActivate,
|
||||
ExecutionContext,
|
||||
UnauthorizedException,
|
||||
} from '@nestjs/common';
|
||||
import { Reflector } from '@nestjs/core';
|
||||
import { AuthGuard } from '@nestjs/passport';
|
||||
import { UserService } from '../user/user.service';
|
||||
import { Request } from 'express';
|
||||
import { lastValueFrom, Observable } from 'rxjs';
|
||||
import { IS_PUBLIC_KEY } from './public.decorator';
|
||||
import { tenantStore } from '../tenant/tenant.store';
|
||||
import { UserRole } from '../user/user-role.enum';
|
||||
import { TenantService } from '../tenant/tenant.service';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
/**
|
||||
* A combined authentication guard that accepts either:
|
||||
* 1. An API key via the `x-api-key` header (or `Authorization: Bearer kb_...`)
|
||||
* 2. A standard JWT Bearer token
|
||||
*
|
||||
* This replaces JwtAuthGuard on routes that should support both auth methods.
|
||||
*/
|
||||
@Injectable()
|
||||
export class CombinedAuthGuard implements CanActivate {
|
||||
// We extend AuthGuard('jwt') functionality by composition
|
||||
private jwtGuard: ReturnType<typeof AuthGuard>;
|
||||
|
||||
constructor(
|
||||
private reflector: Reflector,
|
||||
private userService: UserService,
|
||||
private tenantService: TenantService,
|
||||
) {
|
||||
// Create a JWT guard instance
|
||||
const JwtGuardClass = AuthGuard('jwt');
|
||||
this.jwtGuard = new JwtGuardClass() as any;
|
||||
}
|
||||
|
||||
async canActivate(context: ExecutionContext): Promise<boolean> {
|
||||
// Allow @Public() decorated routes
|
||||
const isPublic = this.reflector.getAllAndOverride<boolean>(IS_PUBLIC_KEY, [
|
||||
context.getHandler(),
|
||||
context.getClass(),
|
||||
]);
|
||||
|
||||
const request = context
|
||||
.switchToHttp()
|
||||
.getRequest<Request & { user?: any; tenantId?: string }>();
|
||||
const logMsg = `\n[${new Date().toISOString()}] AuthGuard: ${request.method} ${request.url} (isPublic: ${isPublic})\n`;
|
||||
fs.appendFileSync('auth_debug.log', logMsg);
|
||||
|
||||
if (isPublic) {
|
||||
return true;
|
||||
}
|
||||
|
||||
console.log(
|
||||
`[CombinedAuthGuard] Checking auth for route: ${request.method} ${request.url}`,
|
||||
);
|
||||
|
||||
// --- Try API Key first ---
|
||||
const apiKey = this.extractApiKey(request);
|
||||
if (apiKey) {
|
||||
const user = await this.userService.findByApiKey(apiKey);
|
||||
if (user) {
|
||||
// If x-tenant-id is provided, verify membership
|
||||
const requestedTenantId = request.headers['x-tenant-id'] as string;
|
||||
let activeTenantId = user.tenantId;
|
||||
|
||||
if (requestedTenantId) {
|
||||
const memberships = await this.userService.getUserTenants(user.id);
|
||||
const hasAccess = memberships.some(
|
||||
(m) => m.tenantId === requestedTenantId,
|
||||
);
|
||||
|
||||
if (hasAccess || user.isAdmin) {
|
||||
activeTenantId = requestedTenantId;
|
||||
} else {
|
||||
throw new UnauthorizedException(
|
||||
'User does not belong to the requested tenant',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const role = await this.tenantService.getUserRole(
|
||||
user.id,
|
||||
activeTenantId,
|
||||
);
|
||||
|
||||
request.user = {
|
||||
id: user.id,
|
||||
username: user.username,
|
||||
role,
|
||||
tenantId: activeTenantId,
|
||||
};
|
||||
request.tenantId = activeTenantId;
|
||||
|
||||
// Update tenant context store
|
||||
const store = tenantStore.getStore();
|
||||
if (store) {
|
||||
store.tenantId = activeTenantId;
|
||||
store.userId = user.id;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
throw new UnauthorizedException('Invalid API key');
|
||||
}
|
||||
|
||||
// --- Fall back to JWT ---
|
||||
try {
|
||||
const result = await (this.jwtGuard as any).canActivate(context);
|
||||
let hasJwtSession = false;
|
||||
|
||||
if (result instanceof Observable) {
|
||||
hasJwtSession = await lastValueFrom(result);
|
||||
} else {
|
||||
hasJwtSession = result;
|
||||
}
|
||||
|
||||
if (hasJwtSession) {
|
||||
const user = request.user;
|
||||
if (!user) return false;
|
||||
|
||||
const requestedTenantId = request.headers['x-tenant-id'] as string;
|
||||
|
||||
if (requestedTenantId && user.tenantId !== requestedTenantId) {
|
||||
const memberships = await this.userService.getUserTenants(user.id);
|
||||
const hasAccess = memberships.some(
|
||||
(m) => m.tenantId === requestedTenantId,
|
||||
);
|
||||
|
||||
if (hasAccess || user.isAdmin) {
|
||||
user.tenantId = requestedTenantId;
|
||||
} else {
|
||||
throw new UnauthorizedException(
|
||||
'User does not belong to the requested tenant',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch the role for the active tenant
|
||||
const role = await this.tenantService.getUserRole(
|
||||
user.id,
|
||||
user.tenantId,
|
||||
);
|
||||
user.role = role;
|
||||
|
||||
request.tenantId = user.tenantId;
|
||||
|
||||
// Update tenant context store
|
||||
const store = tenantStore.getStore();
|
||||
if (store) {
|
||||
store.tenantId = user.tenantId;
|
||||
store.userId = user.id;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
} catch (e) {
|
||||
console.error(`[CombinedAuthGuard] JWT Auth Error:`, e);
|
||||
throw e instanceof UnauthorizedException
|
||||
? e
|
||||
: new UnauthorizedException('Authentication required');
|
||||
}
|
||||
}
|
||||
|
||||
private extractApiKey(request: Request): string | undefined {
|
||||
// Allow `Authorization: Bearer kb_...` form
|
||||
const authHeader = request.headers.authorization;
|
||||
if (authHeader?.startsWith('Bearer kb_')) {
|
||||
return authHeader.substring(7);
|
||||
}
|
||||
// Or a plain `x-api-key` header
|
||||
const headerKey = request.headers['x-api-key'] as string;
|
||||
if (headerKey) return headerKey;
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
import {
|
||||
Column,
|
||||
CreateDateColumn,
|
||||
Entity,
|
||||
JoinColumn,
|
||||
ManyToOne,
|
||||
PrimaryGeneratedColumn,
|
||||
} from 'typeorm';
|
||||
import { User } from '../../user/user.entity';
|
||||
|
||||
@Entity('api_keys')
|
||||
export class ApiKey {
|
||||
@PrimaryGeneratedColumn('uuid')
|
||||
id: string;
|
||||
|
||||
@Column({ name: 'user_id', type: 'uuid' })
|
||||
userId: string;
|
||||
|
||||
@ManyToOne(() => User, (user) => user.apiKeys, { onDelete: 'CASCADE' })
|
||||
@JoinColumn({ name: 'user_id' })
|
||||
user: User;
|
||||
|
||||
@Column({ type: 'text', unique: true })
|
||||
key: string;
|
||||
|
||||
@CreateDateColumn({ name: 'created_at' })
|
||||
createdAt: Date;
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
import { Injectable, CanActivate, ExecutionContext } from '@nestjs/common';
|
||||
import { Reflector } from '@nestjs/core';
|
||||
import { AuthGuard } from '@nestjs/passport';
|
||||
import { lastValueFrom, Observable } from 'rxjs';
|
||||
import { IS_PUBLIC_KEY } from './public.decorator';
|
||||
import { tenantStore } from '../tenant/tenant.store';
|
||||
|
||||
@Injectable()
|
||||
export class JwtAuthGuard extends AuthGuard('jwt') implements CanActivate {
|
||||
constructor(private reflector: Reflector) {
|
||||
super();
|
||||
}
|
||||
|
||||
async canActivate(context: ExecutionContext): Promise<boolean> {
|
||||
const isPublic = this.reflector.getAllAndOverride<boolean>(IS_PUBLIC_KEY, [
|
||||
context.getHandler(),
|
||||
context.getClass(),
|
||||
]);
|
||||
if (isPublic) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const result = await super.canActivate(context);
|
||||
let canActivate = false;
|
||||
|
||||
if (result instanceof Observable) {
|
||||
canActivate = await lastValueFrom(result);
|
||||
} else {
|
||||
canActivate = result;
|
||||
}
|
||||
|
||||
if (canActivate) {
|
||||
const request = context.switchToHttp().getRequest();
|
||||
const user = request.user;
|
||||
if (user) {
|
||||
const store = tenantStore.getStore();
|
||||
if (store) {
|
||||
store.tenantId = user.tenantId;
|
||||
store.userId = user.id;
|
||||
}
|
||||
}
|
||||
}
|
||||
return canActivate;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { PassportStrategy } from '@nestjs/passport';
|
||||
import { ExtractJwt, Strategy } from 'passport-jwt';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { UserService } from '../user/user.service';
|
||||
import { SafeUser } from '../user/dto/user-safe.dto';
|
||||
import { UserRole } from '../user/user-role.enum';
|
||||
import { TenantService } from '../tenant/tenant.service';
|
||||
|
||||
@Injectable()
|
||||
export class JwtStrategy extends PassportStrategy(Strategy) {
|
||||
constructor(
|
||||
private configService: ConfigService,
|
||||
private userService: UserService,
|
||||
private tenantService: TenantService,
|
||||
) {
|
||||
super({
|
||||
jwtFromRequest: ExtractJwt.fromAuthHeaderAsBearerToken(),
|
||||
ignoreExpiration: false,
|
||||
secretOrKey: configService.get<string>('JWT_SECRET')!,
|
||||
});
|
||||
}
|
||||
|
||||
// Passport first verifies the JWT's signature and expiration, then calls this method.
|
||||
async validate(payload: {
|
||||
sub: string;
|
||||
username: string;
|
||||
role?: string;
|
||||
tenantId?: string;
|
||||
}): Promise<SafeUser | null> {
|
||||
// 1. ALWAYS lookup by ID (sub) for identity stability
|
||||
const user = await this.userService.findOneById(payload.sub);
|
||||
|
||||
if (user) {
|
||||
const { password, ...result } = user;
|
||||
|
||||
// In a multi-tenant setup, the tenantId in the payload is the "default" or "last active" one.
|
||||
// But it can be overridden by the x-tenant-id header in the guard.
|
||||
// Map the backend isAdmin flag to the global UserRole
|
||||
const activeTenantId = payload.tenantId || result.tenantId;
|
||||
|
||||
// Fetch the actual role for this tenant from the database
|
||||
const role = await this.tenantService.getUserRole(
|
||||
result.id,
|
||||
activeTenantId,
|
||||
);
|
||||
|
||||
return {
|
||||
...result,
|
||||
role,
|
||||
tenantId: activeTenantId,
|
||||
} as SafeUser;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { AuthGuard } from '@nestjs/passport';
|
||||
|
||||
@Injectable()
|
||||
export class LocalAuthGuard extends AuthGuard('local') {}
|
||||
@@ -0,0 +1,37 @@
|
||||
import { Strategy } from 'passport-local';
|
||||
import { PassportStrategy } from '@nestjs/passport';
|
||||
import { Injectable, UnauthorizedException } from '@nestjs/common';
|
||||
import { AuthService } from './auth.service';
|
||||
import { SafeUser } from '../user/dto/user-safe.dto'; // Import SafeUser
|
||||
import { I18nService } from '../i18n/i18n.service';
|
||||
import { UserRole } from '../user/user-role.enum';
|
||||
import { TenantService } from '../tenant/tenant.service';
|
||||
|
||||
@Injectable()
|
||||
export class LocalStrategy extends PassportStrategy(Strategy) {
|
||||
constructor(
|
||||
private authService: AuthService,
|
||||
private i18nService: I18nService,
|
||||
private tenantService: TenantService,
|
||||
) {
|
||||
super({ usernameField: 'username' });
|
||||
}
|
||||
|
||||
async validate(username: string, password: string): Promise<SafeUser> {
|
||||
const user = await this.authService.validateUser(username, password);
|
||||
if (!user) {
|
||||
throw new UnauthorizedException(
|
||||
this.i18nService.getMessage('incorrectCredentials'),
|
||||
);
|
||||
}
|
||||
const { password: userPassword, ...result } = user;
|
||||
|
||||
// Fetch the actual role for the user's primary tenant
|
||||
const role = await this.tenantService.getUserRole(user.id, user.tenantId);
|
||||
|
||||
return {
|
||||
...result,
|
||||
role,
|
||||
} as SafeUser;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
import { SetMetadata } from '@nestjs/common';
|
||||
|
||||
export const IS_PUBLIC_KEY = 'isPublic';
|
||||
export const Public = () => SetMetadata(IS_PUBLIC_KEY, true);
|
||||
@@ -0,0 +1,5 @@
|
||||
import { SetMetadata } from '@nestjs/common';
|
||||
import { UserRole } from '../user/user-role.enum';
|
||||
|
||||
export const ROLES_KEY = 'roles';
|
||||
export const Roles = (...roles: UserRole[]) => SetMetadata(ROLES_KEY, roles);
|
||||
@@ -0,0 +1,28 @@
|
||||
import { Injectable, CanActivate, ExecutionContext } from '@nestjs/common';
|
||||
import { Reflector } from '@nestjs/core';
|
||||
import { ROLES_KEY } from './roles.decorator';
|
||||
import { UserRole } from '../user/user-role.enum';
|
||||
|
||||
@Injectable()
|
||||
export class RolesGuard implements CanActivate {
|
||||
constructor(private reflector: Reflector) {}
|
||||
|
||||
canActivate(context: ExecutionContext): boolean {
|
||||
const requiredRoles = this.reflector.getAllAndOverride<UserRole[]>(
|
||||
ROLES_KEY,
|
||||
[context.getHandler(), context.getClass()],
|
||||
);
|
||||
|
||||
if (!requiredRoles) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const { user } = context.switchToHttp().getRequest();
|
||||
// User might not be injected yet if auth guard fails, but auth guard runs first usually.
|
||||
if (!user) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return requiredRoles.includes(user.role);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
import { Injectable, CanActivate, ExecutionContext } from '@nestjs/common';
|
||||
import { UserRole } from '../user/user-role.enum';
|
||||
|
||||
@Injectable()
|
||||
export class SuperAdminGuard implements CanActivate {
|
||||
canActivate(context: ExecutionContext): boolean {
|
||||
const request = context.switchToHttp().getRequest();
|
||||
const user = request.user;
|
||||
return (
|
||||
user && (user.role === UserRole.SUPER_ADMIN || user.isAdmin === true)
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import { Injectable, CanActivate, ExecutionContext } from '@nestjs/common';
|
||||
import { UserRole } from '../user/user-role.enum';
|
||||
|
||||
@Injectable()
|
||||
export class TenantAdminGuard implements CanActivate {
|
||||
canActivate(context: ExecutionContext): boolean {
|
||||
const request = context.switchToHttp().getRequest();
|
||||
const user = request.user;
|
||||
return (
|
||||
user &&
|
||||
(user.role === UserRole.SUPER_ADMIN ||
|
||||
user.role === UserRole.TENANT_ADMIN ||
|
||||
user.isAdmin === true)
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
import {
|
||||
Body,
|
||||
Controller,
|
||||
Post,
|
||||
Request,
|
||||
Res,
|
||||
UseGuards,
|
||||
} from '@nestjs/common';
|
||||
import { Response } from 'express';
|
||||
import { ChatMessage, ChatService } from './chat.service';
|
||||
import { CombinedAuthGuard } from '../auth/combined-auth.guard';
|
||||
import { ModelConfigService } from '../model-config/model-config.service';
|
||||
import { TenantService } from '../tenant/tenant.service';
|
||||
import { ModelType } from '../types';
|
||||
|
||||
class StreamChatDto {
|
||||
message: string;
|
||||
history: ChatMessage[];
|
||||
userLanguage?: string;
|
||||
selectedEmbeddingId?: string;
|
||||
selectedLLMId?: string;
|
||||
selectedGroups?: string[];
|
||||
selectedFiles?: string[];
|
||||
historyId?: string;
|
||||
enableRerank?: boolean;
|
||||
selectedRerankId?: string;
|
||||
temperature?: number;
|
||||
maxTokens?: number;
|
||||
topK?: number;
|
||||
similarityThreshold?: number;
|
||||
rerankSimilarityThreshold?: number;
|
||||
enableQueryExpansion?: boolean;
|
||||
enableHyDE?: boolean;
|
||||
}
|
||||
|
||||
@Controller('chat')
|
||||
@UseGuards(CombinedAuthGuard)
|
||||
export class ChatController {
|
||||
constructor(
|
||||
private chatService: ChatService,
|
||||
private modelConfigService: ModelConfigService,
|
||||
private tenantService: TenantService,
|
||||
) {}
|
||||
|
||||
@Post('stream')
|
||||
async streamChat(
|
||||
@Request() req,
|
||||
@Body() body: StreamChatDto,
|
||||
@Res() res: Response,
|
||||
) {
|
||||
try {
|
||||
console.log('Full Request Body:', JSON.stringify(body, null, 2));
|
||||
const {
|
||||
message,
|
||||
history = [],
|
||||
userLanguage = 'zh',
|
||||
selectedEmbeddingId,
|
||||
selectedLLMId,
|
||||
selectedGroups,
|
||||
selectedFiles,
|
||||
historyId,
|
||||
enableRerank,
|
||||
selectedRerankId,
|
||||
temperature,
|
||||
maxTokens,
|
||||
topK,
|
||||
similarityThreshold,
|
||||
rerankSimilarityThreshold,
|
||||
enableQueryExpansion,
|
||||
enableHyDE,
|
||||
} = body;
|
||||
const userId = req.user.id;
|
||||
|
||||
console.log('=== Chat Debug Info ===');
|
||||
console.log('User ID:', userId);
|
||||
console.log('Message:', message);
|
||||
console.log('User Language:', userLanguage);
|
||||
console.log('Selected Embedding ID:', selectedEmbeddingId);
|
||||
console.log('Selected LLM ID:', selectedLLMId);
|
||||
console.log('Selected Groups:', selectedGroups);
|
||||
console.log('Selected Files:', selectedFiles);
|
||||
console.log('History ID:', historyId);
|
||||
console.log('Temperature:', temperature);
|
||||
console.log('Max Tokens:', maxTokens);
|
||||
console.log('Top K:', topK);
|
||||
console.log('Similarity Threshold:', similarityThreshold);
|
||||
console.log('Rerank Similarity Threshold:', rerankSimilarityThreshold);
|
||||
console.log('Query Expansion:', enableQueryExpansion);
|
||||
console.log('HyDE:', enableHyDE);
|
||||
|
||||
const role = req.user.role;
|
||||
const tenantId = req.user.tenantId;
|
||||
|
||||
// 获取用户的LLM模型配置
|
||||
let models = await this.modelConfigService.findAll();
|
||||
|
||||
if (role !== 'SUPER_ADMIN') {
|
||||
const tenantSettings = await this.tenantService.getSettings(tenantId);
|
||||
const enabledIds = tenantSettings?.enabledModelIds || [];
|
||||
// Only allow models that are enabled by the tenant admin
|
||||
models = models.filter((m) => enabledIds.includes(m.id));
|
||||
}
|
||||
|
||||
let llmModel;
|
||||
if (selectedLLMId) {
|
||||
// Find specifically selected model
|
||||
llmModel = await this.modelConfigService.findOne(selectedLLMId);
|
||||
console.log('使用选中的LLM模型:', llmModel.name);
|
||||
} else {
|
||||
// Use organization's default LLM from Index Chat Config (strict)
|
||||
llmModel = await this.modelConfigService.findDefaultByType(
|
||||
tenantId,
|
||||
ModelType.LLM,
|
||||
);
|
||||
console.log(
|
||||
'最终使用的LLM模型 (默认):',
|
||||
llmModel ? llmModel.name : '无',
|
||||
);
|
||||
}
|
||||
|
||||
// 设置 SSE 响应头
|
||||
res.setHeader('Content-Type', 'text/event-stream');
|
||||
res.setHeader('Cache-Control', 'no-cache');
|
||||
res.setHeader('Connection', 'keep-alive');
|
||||
res.setHeader('Access-Control-Allow-Origin', '*');
|
||||
|
||||
if (!llmModel) {
|
||||
res.write(
|
||||
`data: ${JSON.stringify({ type: 'error', data: 'Please add LLM model and configure API key in model management' })}\n\n`,
|
||||
);
|
||||
res.write('data: [DONE]\n\n');
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
|
||||
const stream = this.chatService.streamChat(
|
||||
message,
|
||||
history,
|
||||
userId,
|
||||
llmModel,
|
||||
userLanguage,
|
||||
selectedEmbeddingId,
|
||||
selectedGroups,
|
||||
selectedFiles,
|
||||
historyId,
|
||||
enableRerank,
|
||||
selectedRerankId,
|
||||
temperature, // 传递 temperature 参数
|
||||
maxTokens, // 传递 maxTokens 参数
|
||||
topK, // 传递 topK 参数
|
||||
similarityThreshold, // 传递 similarityThreshold 参数
|
||||
rerankSimilarityThreshold, // 传递 rerankSimilarityThreshold 参数
|
||||
enableQueryExpansion, // 传递 enableQueryExpansion
|
||||
enableHyDE, // 传递 enableHyDE
|
||||
req.user.tenantId, // Pass tenant ID
|
||||
);
|
||||
|
||||
for await (const chunk of stream) {
|
||||
res.write(`data: ${JSON.stringify(chunk)}\n\n`);
|
||||
}
|
||||
|
||||
res.write('data: [DONE]\n\n');
|
||||
res.end();
|
||||
} catch (error) {
|
||||
console.error('Stream chat error:', error);
|
||||
try {
|
||||
res.write(
|
||||
`data: ${JSON.stringify({ type: 'error', data: error.message || 'Server Error' })}\n\n`,
|
||||
);
|
||||
res.write('data: [DONE]\n\n');
|
||||
res.end();
|
||||
} catch (writeError) {
|
||||
console.error('Failed to write error response:', writeError);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Post('assist')
|
||||
async streamAssist(
|
||||
@Request() req,
|
||||
@Body() body: { instruction: string; context: string },
|
||||
@Res() res: Response,
|
||||
) {
|
||||
try {
|
||||
const { instruction, context } = body;
|
||||
const userId = req.user.id; // Corrected to use req.user.id
|
||||
const tenantId = req.user.tenantId;
|
||||
const role = req.user.role;
|
||||
|
||||
// Use organization's default LLM from Index Chat Config (strict)
|
||||
const llmModel = await this.modelConfigService.findDefaultByType(
|
||||
tenantId,
|
||||
ModelType.LLM,
|
||||
);
|
||||
|
||||
res.setHeader('Content-Type', 'text/event-stream');
|
||||
res.setHeader('Cache-Control', 'no-cache');
|
||||
res.setHeader('Connection', 'keep-alive');
|
||||
res.setHeader('Access-Control-Allow-Origin', '*');
|
||||
|
||||
if (!llmModel) {
|
||||
res.write(
|
||||
`data: ${JSON.stringify({ type: 'error', data: 'LLM model configuration not found' })}\n\n`,
|
||||
);
|
||||
res.write('data: [DONE]\n\n');
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
|
||||
const stream = this.chatService.streamAssist(
|
||||
instruction,
|
||||
context,
|
||||
llmModel as any,
|
||||
);
|
||||
|
||||
for await (const chunk of stream) {
|
||||
res.write(`data: ${JSON.stringify(chunk)}\n\n`);
|
||||
}
|
||||
|
||||
res.write('data: [DONE]\n\n');
|
||||
res.end();
|
||||
} catch (error) {
|
||||
console.error('Stream assist error:', error);
|
||||
res.write(
|
||||
`data: ${JSON.stringify({ type: 'error', data: error.message || 'Server Error' })}\n\n`,
|
||||
);
|
||||
res.write('data: [DONE]\n\n');
|
||||
res.end();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
import { Module, forwardRef } from '@nestjs/common';
|
||||
import { ChatController } from './chat.controller';
|
||||
import { ChatService } from './chat.service';
|
||||
import { ElasticsearchModule } from '../elasticsearch/elasticsearch.module';
|
||||
import { KnowledgeBaseModule } from '../knowledge-base/knowledge-base.module';
|
||||
import { ModelConfigModule } from '../model-config/model-config.module';
|
||||
import { KnowledgeGroupModule } from '../knowledge-group/knowledge-group.module';
|
||||
import { SearchHistoryModule } from '../search-history/search-history.module';
|
||||
import { RagModule } from '../rag/rag.module';
|
||||
import { TenantModule } from '../tenant/tenant.module';
|
||||
import { UserModule } from '../user/user.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
forwardRef(() => ElasticsearchModule),
|
||||
forwardRef(() => KnowledgeBaseModule),
|
||||
ModelConfigModule,
|
||||
forwardRef(() => KnowledgeGroupModule),
|
||||
SearchHistoryModule,
|
||||
RagModule,
|
||||
TenantModule,
|
||||
UserModule,
|
||||
],
|
||||
controllers: [ChatController],
|
||||
providers: [ChatService],
|
||||
exports: [ChatService],
|
||||
})
|
||||
export class ChatModule {}
|
||||
@@ -0,0 +1,714 @@
|
||||
import { Injectable, Logger, Inject, forwardRef } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { ChatOpenAI } from '@langchain/openai';
|
||||
import { PromptTemplate } from '@langchain/core/prompts';
|
||||
import { ElasticsearchService } from '../elasticsearch/elasticsearch.service';
|
||||
import { EmbeddingService } from '../knowledge-base/embedding.service';
|
||||
import { ModelConfigService } from '../model-config/model-config.service';
|
||||
import { KnowledgeGroupService } from '../knowledge-group/knowledge-group.service';
|
||||
import { SearchHistoryService } from '../search-history/search-history.service';
|
||||
import { ModelConfig, ModelType } from '../types';
|
||||
import { RagService } from '../rag/rag.service';
|
||||
|
||||
import {
|
||||
DEFAULT_VECTOR_DIMENSIONS,
|
||||
DEFAULT_LANGUAGE,
|
||||
} from '../common/constants';
|
||||
import { I18nService } from '../i18n/i18n.service';
|
||||
import { TenantService } from '../tenant/tenant.service';
|
||||
import { UserSettingService } from '../user/user-setting.service';
|
||||
|
||||
export interface ChatMessage {
|
||||
role: 'user' | 'assistant';
|
||||
content: string;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class ChatService {
|
||||
private readonly logger = new Logger(ChatService.name);
|
||||
private readonly defaultDimensions: number;
|
||||
|
||||
constructor(
|
||||
@Inject(forwardRef(() => ElasticsearchService))
|
||||
private elasticsearchService: ElasticsearchService,
|
||||
private embeddingService: EmbeddingService,
|
||||
private modelConfigService: ModelConfigService,
|
||||
@Inject(forwardRef(() => KnowledgeGroupService))
|
||||
private knowledgeGroupService: KnowledgeGroupService,
|
||||
private searchHistoryService: SearchHistoryService,
|
||||
private configService: ConfigService,
|
||||
private ragService: RagService,
|
||||
private i18nService: I18nService,
|
||||
private tenantService: TenantService,
|
||||
private userSettingService: UserSettingService,
|
||||
) {
|
||||
this.defaultDimensions = parseInt(
|
||||
this.configService.get<string>(
|
||||
'DEFAULT_VECTOR_DIMENSIONS',
|
||||
String(DEFAULT_VECTOR_DIMENSIONS),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
async *streamChat(
|
||||
message: string,
|
||||
history: ChatMessage[],
|
||||
userId: string,
|
||||
modelConfig: ModelConfig,
|
||||
userLanguage: string = DEFAULT_LANGUAGE,
|
||||
selectedEmbeddingId?: string,
|
||||
selectedGroups?: string[], // New: Selected groups
|
||||
selectedFiles?: string[], // New: Selected files
|
||||
historyId?: string, // New: Chat history ID
|
||||
enableRerank: boolean = false,
|
||||
selectedRerankId?: string,
|
||||
temperature?: number, // New: temperature parameter
|
||||
maxTokens?: number, // New: maxTokens parameter
|
||||
topK?: number, // New: topK parameter
|
||||
similarityThreshold?: number, // New: similarityThreshold parameter
|
||||
rerankSimilarityThreshold?: number, // New: rerankSimilarityThreshold parameter
|
||||
enableQueryExpansion?: boolean, // New
|
||||
enableHyDE?: boolean, // New
|
||||
tenantId?: string, // New: tenant isolation
|
||||
): AsyncGenerator<{ type: 'content' | 'sources' | 'historyId'; data: any }> {
|
||||
console.log('=== ChatService.streamChat ===');
|
||||
console.log('User ID:', userId);
|
||||
console.log('User language:', userLanguage);
|
||||
console.log('Selected embedding model ID:', selectedEmbeddingId);
|
||||
console.log('Selected groups:', selectedGroups);
|
||||
console.log('Selected files:', selectedFiles);
|
||||
console.log('History ID:', historyId);
|
||||
console.log('Temperature:', temperature);
|
||||
console.log('Max Tokens:', maxTokens);
|
||||
console.log('Top K:', topK);
|
||||
console.log('Similarity threshold:', similarityThreshold);
|
||||
console.log('Rerank threshold:', rerankSimilarityThreshold);
|
||||
console.log('Query expansion:', enableQueryExpansion);
|
||||
console.log('HyDE:', enableHyDE);
|
||||
console.log('Model configuration:', {
|
||||
name: modelConfig.name,
|
||||
modelId: modelConfig.modelId,
|
||||
baseUrl: modelConfig.baseUrl,
|
||||
});
|
||||
console.log(
|
||||
'API Key prefix:',
|
||||
modelConfig.apiKey?.substring(0, 10) + '...',
|
||||
);
|
||||
console.log('API Key length:', modelConfig.apiKey?.length);
|
||||
|
||||
// Get current language setting (keeping LANGUAGE_CONFIG for backward compatibility, now uses i18n service)
|
||||
// Use actual language based on user settings
|
||||
const effectiveUserLanguage = userLanguage || DEFAULT_LANGUAGE;
|
||||
|
||||
let currentHistoryId = historyId;
|
||||
let fullResponse = '';
|
||||
|
||||
try {
|
||||
// Create new chat history if no historyId
|
||||
if (!currentHistoryId) {
|
||||
const searchHistory = await this.searchHistoryService.create(
|
||||
userId,
|
||||
tenantId || 'default', // New
|
||||
message,
|
||||
selectedGroups,
|
||||
);
|
||||
currentHistoryId = searchHistory.id;
|
||||
console.log(
|
||||
this.i18nService.getMessage(
|
||||
'creatingHistory',
|
||||
effectiveUserLanguage,
|
||||
) + currentHistoryId,
|
||||
);
|
||||
yield { type: 'historyId', data: currentHistoryId };
|
||||
}
|
||||
|
||||
// Save user message
|
||||
await this.searchHistoryService.addMessage(
|
||||
currentHistoryId,
|
||||
'user',
|
||||
message,
|
||||
);
|
||||
// 1. Get user's embedding model settings
|
||||
let embeddingModel: any;
|
||||
|
||||
if (selectedEmbeddingId) {
|
||||
// Find specifically selected model
|
||||
embeddingModel =
|
||||
await this.modelConfigService.findOne(selectedEmbeddingId);
|
||||
} else {
|
||||
// Use organization's default from Index Chat Config (strict)
|
||||
embeddingModel = await this.modelConfigService.findDefaultByType(
|
||||
tenantId || 'default',
|
||||
ModelType.EMBEDDING,
|
||||
);
|
||||
}
|
||||
|
||||
console.log(
|
||||
this.i18nService.getMessage(
|
||||
'usingEmbeddingModel',
|
||||
effectiveUserLanguage,
|
||||
) +
|
||||
embeddingModel.name +
|
||||
' ' +
|
||||
embeddingModel.modelId +
|
||||
' ID:' +
|
||||
embeddingModel.id,
|
||||
);
|
||||
|
||||
// 2. Search using user's query directly
|
||||
console.log(
|
||||
this.i18nService.getMessage('startingSearch', effectiveUserLanguage),
|
||||
);
|
||||
yield {
|
||||
type: 'content',
|
||||
data:
|
||||
this.i18nService.getMessage('searching', effectiveUserLanguage) +
|
||||
'\n',
|
||||
};
|
||||
|
||||
let searchResults: any[] = [];
|
||||
let context = '';
|
||||
|
||||
try {
|
||||
// 3. If knowledge groups are selected, get file IDs from those groups first
|
||||
let effectiveFileIds = selectedFiles; // Prioritize explicitly specified files
|
||||
if (!effectiveFileIds && selectedGroups && selectedGroups.length > 0) {
|
||||
// Get file IDs from knowledge groups
|
||||
effectiveFileIds =
|
||||
await this.knowledgeGroupService.getFileIdsByGroups(
|
||||
selectedGroups,
|
||||
userId,
|
||||
tenantId as string,
|
||||
);
|
||||
}
|
||||
|
||||
// 3. Use RagService for search (supports hybrid search + Rerank)
|
||||
const ragResults = await this.ragService.searchKnowledge(
|
||||
message,
|
||||
userId,
|
||||
topK,
|
||||
similarityThreshold,
|
||||
embeddingModel.id,
|
||||
true, // enableFullTextSearch (Chat defaults to hybrid)
|
||||
enableRerank,
|
||||
selectedRerankId,
|
||||
undefined, // selectedGroups
|
||||
effectiveFileIds,
|
||||
rerankSimilarityThreshold,
|
||||
tenantId,
|
||||
enableQueryExpansion,
|
||||
enableHyDE,
|
||||
);
|
||||
|
||||
// Convert RagSearchResult to format needed by ChatService (any[])
|
||||
// HybridSearch returns ES hit structure, but RagSearchResult is normalized
|
||||
// BuildContext expects {fileName, content}. RagSearchResult has these
|
||||
searchResults = ragResults;
|
||||
console.log(
|
||||
this.i18nService.getMessage(
|
||||
'searchResultsCount',
|
||||
effectiveUserLanguage,
|
||||
) + searchResults.length,
|
||||
);
|
||||
|
||||
// 4. Build context
|
||||
context = this.buildContext(searchResults, effectiveUserLanguage);
|
||||
|
||||
if (searchResults.length === 0) {
|
||||
if (selectedGroups && selectedGroups.length > 0) {
|
||||
// User selected knowledge groups but no matches found
|
||||
const noMatchMsg = this.i18nService.getMessage(
|
||||
'noMatchInKnowledgeGroup',
|
||||
effectiveUserLanguage,
|
||||
);
|
||||
yield { type: 'content', data: `⚠️ ${noMatchMsg}\n\n` };
|
||||
} else {
|
||||
yield {
|
||||
type: 'content',
|
||||
data:
|
||||
this.i18nService.getMessage(
|
||||
'noResults',
|
||||
effectiveUserLanguage,
|
||||
) + '\n\n',
|
||||
};
|
||||
}
|
||||
yield {
|
||||
type: 'content',
|
||||
data: `[Debug] ${this.i18nService.getMessage('searchScope', effectiveUserLanguage)}: ${selectedFiles ? selectedFiles.length + ' ' + this.i18nService.getMessage('files', effectiveUserLanguage) : selectedGroups ? selectedGroups.length + ' ' + this.i18nService.getMessage('notebooks', effectiveUserLanguage) : this.i18nService.getMessage('all', effectiveUserLanguage)}\n`,
|
||||
};
|
||||
yield {
|
||||
type: 'content',
|
||||
data: `[Debug] ${this.i18nService.getMessage('searchResults', effectiveUserLanguage)}: 0 ${this.i18nService.getMessage('items', effectiveUserLanguage)}\n`,
|
||||
};
|
||||
} else {
|
||||
yield {
|
||||
type: 'content',
|
||||
data: `${searchResults.length} ${this.i18nService.getMessage('relevantInfoFound', effectiveUserLanguage)}。${this.i18nService.getMessage('generatingResponse', effectiveUserLanguage)}...\n\n`,
|
||||
};
|
||||
// Debug info
|
||||
const scores = searchResults
|
||||
.map((r) => {
|
||||
if (
|
||||
r.originalScore !== undefined &&
|
||||
r.originalScore !== r.score
|
||||
) {
|
||||
return `${r.originalScore.toFixed(2)} → ${r.score.toFixed(2)}`;
|
||||
}
|
||||
return r.score.toFixed(2);
|
||||
})
|
||||
.join(', ');
|
||||
const files = [...new Set(searchResults.map((r) => r.fileName))].join(
|
||||
', ',
|
||||
);
|
||||
yield {
|
||||
type: 'content',
|
||||
data: `> [Debug] ${this.i18nService.getMessage('searchHits', effectiveUserLanguage)}: ${searchResults.length} ${this.i18nService.getMessage('items', effectiveUserLanguage)}\n`,
|
||||
};
|
||||
yield {
|
||||
type: 'content',
|
||||
data: `> [Debug] ${this.i18nService.getMessage('relevance', effectiveUserLanguage)}: ${scores}\n`,
|
||||
};
|
||||
yield {
|
||||
type: 'content',
|
||||
data: `> [Debug] ${this.i18nService.getMessage('sourceFiles', effectiveUserLanguage)}: ${files}\n\n---\n\n`,
|
||||
};
|
||||
}
|
||||
} catch (searchError) {
|
||||
console.error(
|
||||
this.i18nService.getMessage(
|
||||
'searchFailedLog',
|
||||
effectiveUserLanguage,
|
||||
) + ':',
|
||||
searchError,
|
||||
);
|
||||
yield {
|
||||
type: 'content',
|
||||
data:
|
||||
this.i18nService.getMessage('searchFailed', effectiveUserLanguage) +
|
||||
'\n\n',
|
||||
};
|
||||
}
|
||||
|
||||
// 5. Stream response generation
|
||||
this.logger.log(
|
||||
this.i18nService.formatMessage(
|
||||
'modelCall',
|
||||
{
|
||||
type: 'LLM',
|
||||
model: `${modelConfig.name} (${modelConfig.modelId})`,
|
||||
user: userId,
|
||||
},
|
||||
effectiveUserLanguage,
|
||||
),
|
||||
);
|
||||
const llm = new ChatOpenAI({
|
||||
apiKey: modelConfig.apiKey || 'ollama',
|
||||
streaming: true,
|
||||
temperature: temperature !== undefined ? temperature : 0.3,
|
||||
maxTokens: maxTokens !== undefined ? maxTokens : undefined,
|
||||
modelName: modelConfig.modelId,
|
||||
configuration: {
|
||||
baseURL: modelConfig.baseUrl || 'http://localhost:11434/v1',
|
||||
},
|
||||
});
|
||||
|
||||
const promptTemplate =
|
||||
context.length > 0
|
||||
? this.i18nService.getPrompt(
|
||||
effectiveUserLanguage,
|
||||
'withContext',
|
||||
selectedGroups && selectedGroups.length > 0,
|
||||
)
|
||||
: this.i18nService.getPrompt(effectiveUserLanguage, 'withoutContext');
|
||||
|
||||
const prompt = PromptTemplate.fromTemplate(promptTemplate);
|
||||
|
||||
const chain = prompt.pipe(llm);
|
||||
|
||||
const stream = await chain.stream({
|
||||
context,
|
||||
history: this.formatHistory(history, userLanguage),
|
||||
question: message,
|
||||
});
|
||||
|
||||
for await (const chunk of stream) {
|
||||
if (chunk.content) {
|
||||
fullResponse += chunk.content;
|
||||
yield { type: 'content', data: chunk.content };
|
||||
}
|
||||
}
|
||||
|
||||
// Save AI response
|
||||
await this.searchHistoryService.addMessage(
|
||||
currentHistoryId,
|
||||
'assistant',
|
||||
fullResponse,
|
||||
searchResults.map((result) => ({
|
||||
fileName: result.fileName,
|
||||
title: result.metadata?.title || result.metadata?.originalName, // ES metadata contains these
|
||||
content: String(result.content).substring(0, 200) + '...',
|
||||
score: result.score,
|
||||
chunkIndex: result.chunkIndex,
|
||||
fileId: result.fileId,
|
||||
})),
|
||||
);
|
||||
|
||||
// 7. Auto-generate chat title (executed after first exchange)
|
||||
const messagesInHistory = await this.searchHistoryService.findOne(
|
||||
currentHistoryId,
|
||||
userId,
|
||||
tenantId,
|
||||
);
|
||||
if (messagesInHistory.messages.length === 2) {
|
||||
this.generateChatTitle(currentHistoryId, userId, tenantId).catch(
|
||||
(err) => {
|
||||
this.logger.error(
|
||||
`Failed to generate chat title for ${currentHistoryId}`,
|
||||
err,
|
||||
);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
// 6. Return sources
|
||||
yield {
|
||||
type: 'sources',
|
||||
data: searchResults.map((result) => ({
|
||||
fileName: result.fileName,
|
||||
content: String(result.content).substring(0, 200) + '...',
|
||||
score: result.score,
|
||||
chunkIndex: result.chunkIndex,
|
||||
fileId: result.fileId,
|
||||
})),
|
||||
};
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
this.i18nService.getMessage('chatStreamError', effectiveUserLanguage),
|
||||
error,
|
||||
);
|
||||
yield {
|
||||
type: 'content',
|
||||
data: `${this.i18nService.getMessage('error', effectiveUserLanguage)}: ${error.message}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async *streamAssist(
|
||||
instruction: string,
|
||||
context: string,
|
||||
modelConfig: ModelConfig,
|
||||
userLanguage: string = DEFAULT_LANGUAGE,
|
||||
): AsyncGenerator<{ type: 'content'; data: any }> {
|
||||
try {
|
||||
this.logger.log(
|
||||
this.i18nService.formatMessage(
|
||||
'modelCall',
|
||||
{
|
||||
type: 'LLM (Assist)',
|
||||
model: `${modelConfig.name} (${modelConfig.modelId})`,
|
||||
user: 'N/A',
|
||||
},
|
||||
userLanguage,
|
||||
),
|
||||
);
|
||||
const llm = new ChatOpenAI({
|
||||
apiKey: modelConfig.apiKey || 'ollama',
|
||||
streaming: true,
|
||||
temperature: 0.7,
|
||||
modelName: modelConfig.modelId,
|
||||
configuration: {
|
||||
baseURL: modelConfig.baseUrl || 'http://localhost:11434/v1',
|
||||
},
|
||||
});
|
||||
|
||||
const systemPrompt = `${this.i18nService.getMessage('intelligentAssistant', userLanguage)}
|
||||
${this.i18nService.getMessage('assistSystemPrompt', userLanguage)}
|
||||
|
||||
${this.i18nService.getMessage('contextLabel', userLanguage)}:
|
||||
${context}
|
||||
|
||||
${this.i18nService.getMessage('userInstructionLabel', userLanguage)}:
|
||||
${instruction}`;
|
||||
|
||||
const stream = await llm.stream(systemPrompt);
|
||||
|
||||
for await (const chunk of stream) {
|
||||
if (chunk.content) {
|
||||
yield { type: 'content', data: chunk.content };
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
this.i18nService.getMessage('assistStreamError', userLanguage),
|
||||
error,
|
||||
);
|
||||
yield {
|
||||
type: 'content',
|
||||
data: `${this.i18nService.getMessage('error', userLanguage)}: ${error.message}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
private async hybridSearch(
|
||||
keywords: string[],
|
||||
userId: string,
|
||||
embeddingModelId?: string,
|
||||
selectedGroups?: string[], // New parameter
|
||||
explicitFileIds?: string[], // New parameter
|
||||
tenantId?: string, // Added
|
||||
userLanguage: string = DEFAULT_LANGUAGE,
|
||||
): Promise<any[]> {
|
||||
try {
|
||||
// Join keywords into search string
|
||||
const combinedQuery = keywords.join(' ');
|
||||
console.log(
|
||||
this.i18nService.getMessage('searchString', userLanguage) +
|
||||
combinedQuery,
|
||||
);
|
||||
|
||||
// Check if embedding model ID is provided
|
||||
if (!embeddingModelId) {
|
||||
console.log(
|
||||
this.i18nService.getMessage(
|
||||
'embeddingModelIdNotProvided',
|
||||
userLanguage,
|
||||
),
|
||||
);
|
||||
return [];
|
||||
}
|
||||
|
||||
// Use actual embedding vector
|
||||
console.log(
|
||||
this.i18nService.getMessage('generatingEmbeddings', userLanguage),
|
||||
);
|
||||
const queryEmbedding = await this.embeddingService.getEmbeddings(
|
||||
[combinedQuery],
|
||||
embeddingModelId,
|
||||
);
|
||||
const queryVector = queryEmbedding[0];
|
||||
console.log(
|
||||
this.i18nService.getMessage('embeddingsGenerated', userLanguage) +
|
||||
this.i18nService.getMessage('dimensions', userLanguage) +
|
||||
':',
|
||||
queryVector.length,
|
||||
);
|
||||
|
||||
// Hybrid search
|
||||
console.log(
|
||||
this.i18nService.getMessage('performingHybridSearch', userLanguage),
|
||||
);
|
||||
const results = await this.elasticsearchService.hybridSearch(
|
||||
queryVector,
|
||||
combinedQuery,
|
||||
userId,
|
||||
10,
|
||||
0.6,
|
||||
selectedGroups, // Pass selected groups
|
||||
explicitFileIds, // Pass explicit file IDs
|
||||
tenantId, // Pass tenant ID
|
||||
);
|
||||
console.log(
|
||||
this.i18nService.getMessage('esSearchCompleted', userLanguage) +
|
||||
this.i18nService.getMessage('resultsCount', userLanguage) +
|
||||
':',
|
||||
results.length,
|
||||
);
|
||||
|
||||
return results.slice(0, 10);
|
||||
} catch (error) {
|
||||
console.error(
|
||||
this.i18nService.getMessage('hybridSearchFailed', userLanguage) + ':',
|
||||
error,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
private buildContext(
|
||||
results: any[],
|
||||
language: string = DEFAULT_LANGUAGE,
|
||||
): string {
|
||||
return results
|
||||
.map(
|
||||
(result, index) =>
|
||||
`[${index + 1}] ${this.i18nService.getMessage('file', language)}：${result.fileName}\n${this.i18nService.getMessage('content', language)}：${result.content}\n`,
|
||||
)
|
||||
.join('\n');
|
||||
}
|
||||
|
||||
private formatHistory(
|
||||
history: ChatMessage[],
|
||||
userLanguage: string = DEFAULT_LANGUAGE,
|
||||
): string {
|
||||
const userLabel = this.i18nService.getMessage('userLabel', userLanguage);
|
||||
const assistantLabel = this.i18nService.getMessage(
|
||||
'assistantLabel',
|
||||
userLanguage,
|
||||
);
|
||||
|
||||
return history
|
||||
.slice(-6)
|
||||
.map(
|
||||
(msg) =>
|
||||
`${msg.role === 'user' ? userLabel : assistantLabel}：${msg.content}`,
|
||||
)
|
||||
.join('\n');
|
||||
}
|
||||
async getContextForTopic(
|
||||
topic: string,
|
||||
userId: string,
|
||||
tenantId?: string,
|
||||
groupId?: string,
|
||||
fileIds?: string[],
|
||||
userLanguage: string = DEFAULT_LANGUAGE,
|
||||
): Promise<string> {
|
||||
try {
|
||||
// Use organization's default embedding from Index Chat Config (strict)
|
||||
const embeddingModel = await this.modelConfigService.findDefaultByType(
|
||||
tenantId || 'default',
|
||||
ModelType.EMBEDDING,
|
||||
);
|
||||
|
||||
const results = await this.hybridSearch(
|
||||
[topic],
|
||||
userId,
|
||||
embeddingModel.id,
|
||||
groupId ? [groupId] : undefined,
|
||||
fileIds,
|
||||
tenantId,
|
||||
userLanguage,
|
||||
);
|
||||
|
||||
return this.buildContext(results);
|
||||
} catch (err) {
|
||||
this.logger.error(
|
||||
`${this.i18nService.getMessage('getContextForTopicFailed', userLanguage)}: ${err.message}`,
|
||||
);
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
async generateSimpleChat(
|
||||
messages: ChatMessage[],
|
||||
userId: string,
|
||||
tenantId?: string,
|
||||
modelConfig?: ModelConfig, // Optional, looks up if not provided
|
||||
userLanguage: string = DEFAULT_LANGUAGE,
|
||||
): Promise<string> {
|
||||
try {
|
||||
let config = modelConfig;
|
||||
if (!config) {
|
||||
// Use organization's default LLM from Index Chat Config (strict)
|
||||
const found = await this.modelConfigService.findDefaultByType(
|
||||
tenantId || 'default',
|
||||
ModelType.LLM,
|
||||
);
|
||||
config = found as unknown as ModelConfig;
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
this.i18nService.formatMessage(
|
||||
'modelCall',
|
||||
{
|
||||
type: 'LLM (Simple)',
|
||||
model: `${config.name} (${config.modelId})`,
|
||||
user: userId,
|
||||
},
|
||||
DEFAULT_LANGUAGE,
|
||||
),
|
||||
);
|
||||
const settings = await this.tenantService.getSettings(
|
||||
tenantId || 'default',
|
||||
);
|
||||
const llm = new ChatOpenAI({
|
||||
apiKey: config.apiKey || 'ollama',
|
||||
temperature: settings?.temperature ?? 0.7,
|
||||
modelName: config.modelId,
|
||||
configuration: {
|
||||
baseURL: config.baseUrl || 'http://localhost:11434/v1',
|
||||
},
|
||||
});
|
||||
|
||||
const response = await llm.invoke(
|
||||
messages.map((m) => [m.role, m.content]),
|
||||
);
|
||||
|
||||
return String(response.content);
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
this.i18nService.getMessage('simpleChatGenerationError', userLanguage),
|
||||
error,
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Automatically generate chat title based on conversation content
|
||||
*/
|
||||
async generateChatTitle(
|
||||
historyId: string,
|
||||
userId: string,
|
||||
tenantId?: string,
|
||||
): Promise<string | null> {
|
||||
this.logger.log(`Generating automatic title for chat session ${historyId}`);
|
||||
|
||||
try {
|
||||
const history = await this.searchHistoryService.findOne(
|
||||
historyId,
|
||||
userId,
|
||||
tenantId || 'default',
|
||||
);
|
||||
if (!history || history.messages.length < 2) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const userMessage =
|
||||
history.messages.find((m) => m.role === 'user')?.content || '';
|
||||
const aiResponse =
|
||||
history.messages.find((m) => m.role === 'assistant')?.content || '';
|
||||
|
||||
if (!userMessage || !aiResponse) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Get language from user settings
|
||||
const userSettings = await this.userSettingService.getByUser(userId);
|
||||
const language = userSettings?.language || DEFAULT_LANGUAGE;
|
||||
|
||||
// Build prompt
|
||||
const prompt = this.i18nService.getChatTitlePrompt(
|
||||
language,
|
||||
userMessage,
|
||||
aiResponse,
|
||||
);
|
||||
|
||||
// Call LLM to generate title
|
||||
const generatedTitle = await this.generateSimpleChat(
|
||||
[{ role: 'user', content: prompt }],
|
||||
userId,
|
||||
tenantId || 'default',
|
||||
);
|
||||
|
||||
if (generatedTitle && generatedTitle.trim().length > 0) {
|
||||
// Remove extra quotes
|
||||
const cleanedTitle = generatedTitle
|
||||
.trim()
|
||||
.replace(/^["']|["']$/g, '')
|
||||
.substring(0, 50);
|
||||
await this.searchHistoryService.updateTitle(historyId, cleanedTitle);
|
||||
this.logger.log(
|
||||
`Successfully generated title for chat ${historyId}: ${cleanedTitle}`,
|
||||
);
|
||||
return cleanedTitle;
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Failed to generate chat title for ${historyId}`,
|
||||
error,
|
||||
);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
export const DEFAULT_CHUNK_SIZE = 200;
|
||||
export const MIN_CHUNK_SIZE = 50;
|
||||
export const MAX_CHUNK_SIZE = 8191;
|
||||
export const DEFAULT_CHUNK_OVERLAP = 40;
|
||||
export const MIN_CHUNK_OVERLAP = 25;
|
||||
export const DEFAULT_MAX_OVERLAP_RATIO = 0.5;
|
||||
|
||||
export const DEFAULT_VECTOR_DIMENSIONS = 1536;
|
||||
|
||||
// File size limit (バイト)
|
||||
export const MAX_FILE_SIZE = 100 * 1024 * 1024; // 100MB
|
||||
|
||||
export const DEFAULT_MAX_BATCH_SIZE = 2048;
|
||||
|
||||
// Supported languages
|
||||
const SUPPORTED_LANGUAGES = ['zh', 'en', 'ja'] as const;
|
||||
|
||||
/**
|
||||
* Get the default language from the environment variable.
|
||||
* Fallback to 'en' (English) if not set or invalid.
|
||||
*/
|
||||
function getDefaultLanguage(): (typeof SUPPORTED_LANGUAGES)[number] {
|
||||
const envValue = process.env.DEFAULT_LANGUAGE?.toLowerCase();
|
||||
|
||||
// Validate: must be one of supported languages
|
||||
if (
|
||||
envValue &&
|
||||
SUPPORTED_LANGUAGES.includes(
|
||||
envValue as (typeof SUPPORTED_LANGUAGES)[number],
|
||||
)
|
||||
) {
|
||||
return envValue as (typeof SUPPORTED_LANGUAGES)[number];
|
||||
}
|
||||
|
||||
// Fallback to English if not set or invalid
|
||||
return 'en';
|
||||
}
|
||||
|
||||
// Default language - read from env, fallback to English
|
||||
export const DEFAULT_LANGUAGE = getDefaultLanguage();
|
||||
export const DEFAULT_LANGUAGE_FALLBACK = 'en';
|
||||
|
||||
// システム全体の共通テナントID（シードデータetc.で使用）
|
||||
export const GLOBAL_TENANT_ID = '00000000-0000-0000-0000-000000000000';
|
||||
@@ -0,0 +1,64 @@
|
||||
export const DOC_EXTENSIONS = [
|
||||
'pdf',
|
||||
'doc',
|
||||
'docx',
|
||||
'xls',
|
||||
'xlsx',
|
||||
'ppt',
|
||||
'pptx',
|
||||
'rtf',
|
||||
'csv',
|
||||
'txt',
|
||||
'md',
|
||||
'html',
|
||||
'json',
|
||||
'xml',
|
||||
'odt',
|
||||
'ods',
|
||||
'odp',
|
||||
];
|
||||
export const CODE_EXTENSIONS = [
|
||||
'js',
|
||||
'jsx',
|
||||
'ts',
|
||||
'tsx',
|
||||
'css',
|
||||
'py',
|
||||
'java',
|
||||
'sql',
|
||||
'cpp',
|
||||
'h',
|
||||
'go',
|
||||
'rs',
|
||||
'php',
|
||||
'rb',
|
||||
];
|
||||
export const IMAGE_EXTENSIONS = [
|
||||
'jpg',
|
||||
'jpeg',
|
||||
'png',
|
||||
'gif',
|
||||
'bmp',
|
||||
'webp',
|
||||
'tiff',
|
||||
];
|
||||
export const IMAGE_MIME_TYPES = [
|
||||
'image/jpeg',
|
||||
'image/png',
|
||||
'image/gif',
|
||||
'image/bmp',
|
||||
'image/webp',
|
||||
'image/tiff',
|
||||
];
|
||||
|
||||
export const ALL_ALLOWED_EXTENSIONS = [
|
||||
...DOC_EXTENSIONS,
|
||||
...CODE_EXTENSIONS,
|
||||
...IMAGE_EXTENSIONS,
|
||||
];
|
||||
|
||||
export const isAllowedByExtension = (filename: string): boolean => {
|
||||
const ext = filename.toLowerCase().split('.').pop();
|
||||
if (!ext) return false;
|
||||
return ALL_ALLOWED_EXTENSIONS.includes(ext);
|
||||
};
|
||||
@@ -0,0 +1,48 @@
|
||||
/**
|
||||
* Safely parses JSON from a string, handling markdown code blocks and leading/trailing text.
|
||||
*/
|
||||
export function safeParseJson<T = any>(text: string): T | null {
|
||||
if (!text) return null;
|
||||
|
||||
let jsonStr = text.trim();
|
||||
|
||||
// 1. Try to extract JSON from markdown code blocks if they exist
|
||||
// Matches ```json ... ``` or just ``` ... ```
|
||||
const codeBlockRegex = /```(?:json)?\s*([\s\S]*?)\s*```/i;
|
||||
const match = jsonStr.match(codeBlockRegex);
|
||||
|
||||
if (match && match[1]) {
|
||||
jsonStr = match[1].trim();
|
||||
} else {
|
||||
// 2. If no markdown block, try to find the start and end of JSON characters
|
||||
// This handles cases where the AI adds introductory or concluding text outside the block
|
||||
const firstOpenBrace = jsonStr.indexOf('{');
|
||||
const firstOpenBracket = jsonStr.indexOf('[');
|
||||
|
||||
let startIndex = -1;
|
||||
if (firstOpenBrace !== -1 && (firstOpenBracket === -1 || firstOpenBrace < firstOpenBracket)) {
|
||||
startIndex = firstOpenBrace;
|
||||
} else if (firstOpenBracket !== -1) {
|
||||
startIndex = firstOpenBracket;
|
||||
}
|
||||
|
||||
if (startIndex !== -1) {
|
||||
const lastCloseBrace = jsonStr.lastIndexOf('}');
|
||||
const lastCloseBracket = jsonStr.lastIndexOf(']');
|
||||
const endIndex = Math.max(lastCloseBrace, lastCloseBracket);
|
||||
|
||||
if (endIndex !== -1 && endIndex > startIndex) {
|
||||
jsonStr = jsonStr.substring(startIndex, endIndex + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
return JSON.parse(jsonStr) as T;
|
||||
} catch (error) {
|
||||
console.error('[safeParseJson] Failed to parse JSON:', error);
|
||||
console.error('[safeParseJson] Original text:', text);
|
||||
console.error('[safeParseJson] Extracted string:', jsonStr);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import { DataSource } from 'typeorm';
|
||||
import { User } from './user/user.entity';
|
||||
// import { UserSetting } from './user-setting/user-setting.entity';
|
||||
import { ModelConfig } from './model-config/model-config.entity';
|
||||
import { KnowledgeBase } from './knowledge-base/knowledge-base.entity';
|
||||
import { KnowledgeGroup } from './knowledge-group/knowledge-group.entity';
|
||||
import { SearchHistory } from './search-history/search-history.entity';
|
||||
import { ChatMessage } from './search-history/chat-message.entity';
|
||||
import { Note } from './note/note.entity';
|
||||
import { PodcastEpisode } from './podcasts/entities/podcast-episode.entity';
|
||||
import { ImportTask } from './import-task/import-task.entity';
|
||||
import { Tenant } from './tenant/tenant.entity';
|
||||
import { TenantSetting } from './tenant/tenant-setting.entity';
|
||||
|
||||
export const AppDataSource = new DataSource({
|
||||
type: 'better-sqlite3',
|
||||
database: './data/knowledge-base.db',
|
||||
synchronize: false,
|
||||
logging: true,
|
||||
entities: [
|
||||
User,
|
||||
// UserSetting,
|
||||
ModelConfig,
|
||||
KnowledgeBase,
|
||||
KnowledgeGroup,
|
||||
SearchHistory,
|
||||
ChatMessage,
|
||||
Note,
|
||||
PodcastEpisode,
|
||||
ImportTask,
|
||||
Tenant,
|
||||
TenantSetting,
|
||||
],
|
||||
migrations: ['src/migrations/**/*.ts'],
|
||||
});
|
||||
@@ -0,0 +1,28 @@
|
||||
// server/src/defaults.ts
|
||||
import { AppSettings, ModelConfig, ModelType } from './types'; // Import from local types
|
||||
|
||||
export const DEFAULT_MODELS: ModelConfig[] = [];
|
||||
|
||||
export const DEFAULT_SETTINGS: AppSettings = {
|
||||
selectedLLMId: '',
|
||||
selectedEmbeddingId: '',
|
||||
selectedRerankId: '',
|
||||
|
||||
temperature: 0.3,
|
||||
maxTokens: 8192,
|
||||
|
||||
enableRerank: false,
|
||||
topK: 4,
|
||||
similarityThreshold: 0.3,
|
||||
rerankSimilarityThreshold: 0.5,
|
||||
enableFullTextSearch: false,
|
||||
hybridVectorWeight: 0.7,
|
||||
|
||||
enableQueryExpansion: false,
|
||||
enableHyDE: false,
|
||||
|
||||
chunkSize: 1000,
|
||||
chunkOverlap: 100,
|
||||
|
||||
language: 'zh',
|
||||
};
|
||||
@@ -0,0 +1,10 @@
|
||||
import { Module, forwardRef } from '@nestjs/common';
|
||||
import { ElasticsearchService } from './elasticsearch.service';
|
||||
import { KnowledgeGroupModule } from '../knowledge-group/knowledge-group.module';
|
||||
|
||||
@Module({
|
||||
imports: [forwardRef(() => KnowledgeGroupModule)],
|
||||
providers: [ElasticsearchService],
|
||||
exports: [ElasticsearchService],
|
||||
})
|
||||
export class ElasticsearchModule {}
|
||||
@@ -0,0 +1,653 @@
|
||||
import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { Client } from '@elastic/elasticsearch';
|
||||
import { I18nService } from '../i18n/i18n.service';
|
||||
|
||||
@Injectable()
|
||||
export class ElasticsearchService implements OnModuleInit {
|
||||
public readonly client: Client;
|
||||
private readonly logger = new Logger(ElasticsearchService.name);
|
||||
private readonly indexName: string;
|
||||
|
||||
constructor(
|
||||
private configService: ConfigService,
|
||||
private i18nService: I18nService,
|
||||
) {
|
||||
const node = this.configService.get<string>('ELASTICSEARCH_HOST'); // Changed from NODE to HOST
|
||||
this.indexName = this.configService.get<string>(
|
||||
'ELASTICSEARCH_INDEX',
|
||||
'knowledge_base',
|
||||
);
|
||||
|
||||
if (!node) {
|
||||
throw new Error(this.i18nService.getMessage('elasticsearchHostRequired'));
|
||||
}
|
||||
|
||||
this.client = new Client({
|
||||
node,
|
||||
});
|
||||
}
|
||||
|
||||
async onModuleInit() {
|
||||
try {
|
||||
const health = await this.client.cluster.health();
|
||||
this.logger.log(`Elasticsearch cluster health is: ${health.status}`);
|
||||
// Index is created dynamically on first use based on the model
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to connect to Elasticsearch', error);
|
||||
}
|
||||
}
|
||||
|
||||
async createIndexIfNotExists(vectorDimensions: number) {
|
||||
const indexExists = await this.client.indices.exists({
|
||||
index: this.indexName,
|
||||
});
|
||||
|
||||
if (!indexExists) {
|
||||
this.logger.log(
|
||||
`Creating index ${this.indexName}. Vector dimensions: ${vectorDimensions}`,
|
||||
);
|
||||
await this.createIndex(vectorDimensions);
|
||||
} else {
|
||||
// Check existing index vector dimensions
|
||||
const mapping = await this.client.indices.getMapping({
|
||||
index: this.indexName,
|
||||
});
|
||||
|
||||
const vectorMapping = mapping[this.indexName]?.mappings?.properties
|
||||
?.vector as any;
|
||||
const existingDims = vectorMapping?.dims;
|
||||
|
||||
if (existingDims && existingDims !== vectorDimensions) {
|
||||
this.logger.warn(
|
||||
`Vector dimensions ${existingDims} of index ${this.indexName} do not match the current model dimensions ${vectorDimensions}.`,
|
||||
);
|
||||
this.logger.warn(
|
||||
`Reason: The embedding model might have been changed to one with different dimensions. The system will automatically recreate the index.`,
|
||||
);
|
||||
|
||||
// Delete existing index and recreate
|
||||
await this.client.indices.delete({ index: this.indexName });
|
||||
this.logger.log(`Successfully deleted old index: ${this.indexName}`);
|
||||
|
||||
await this.createIndex(vectorDimensions);
|
||||
this.logger.log(
|
||||
`Recreated index: ${this.indexName} (Dimensions: ${vectorDimensions})`,
|
||||
);
|
||||
} else {
|
||||
this.logger.log(
|
||||
`Index ${this.indexName} already exists. Vector dimensions: ${existingDims || 'Unknown'}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async indexDocument(
|
||||
documentId: string,
|
||||
content: string,
|
||||
vector: number[],
|
||||
metadata: any,
|
||||
) {
|
||||
this.logger.log(
|
||||
`Indexing document ${documentId}: content=${content.length} chars, vector=${vector?.length} dims`,
|
||||
);
|
||||
|
||||
if (!vector || vector.length === 0) {
|
||||
this.logger.error(`Invalid vector for document ${documentId}`);
|
||||
throw new Error(this.i18nService.getMessage('vectorRequired'));
|
||||
}
|
||||
|
||||
const document = {
|
||||
content,
|
||||
vector,
|
||||
fileId: metadata.fileId,
|
||||
fileName: metadata.originalName,
|
||||
title: metadata.title || metadata.originalName,
|
||||
fileMimeType: metadata.mimetype,
|
||||
chunkIndex: metadata.chunkIndex,
|
||||
startPosition: metadata.startPosition,
|
||||
endPosition: metadata.endPosition,
|
||||
userId: metadata.userId,
|
||||
tenantId: metadata.tenantId,
|
||||
createdAt: new Date(),
|
||||
};
|
||||
|
||||
const result = await this.client.index({
|
||||
index: this.indexName,
|
||||
id: documentId,
|
||||
document,
|
||||
});
|
||||
|
||||
this.logger.log(
|
||||
`Indexed document ${documentId} with ${vector.length}D vector`,
|
||||
);
|
||||
return result;
|
||||
}
|
||||
|
||||
async deleteByFileId(fileId: string, userId: string, tenantId?: string) {
|
||||
const filter: any[] = [{ term: { fileId } }];
|
||||
if (tenantId) {
|
||||
filter.push({ term: { tenantId } });
|
||||
} else {
|
||||
filter.push({ term: { userId } });
|
||||
}
|
||||
|
||||
await this.client.deleteByQuery({
|
||||
index: this.indexName,
|
||||
query: {
|
||||
bool: { filter },
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async updateTitleByFileId(fileId: string, title: string, tenantId?: string) {
|
||||
const filter: any[] = [{ term: { fileId } }];
|
||||
if (tenantId) {
|
||||
filter.push({ term: { tenantId } });
|
||||
}
|
||||
|
||||
await this.client.updateByQuery({
|
||||
index: this.indexName,
|
||||
query: {
|
||||
bool: { filter },
|
||||
},
|
||||
script: {
|
||||
source: 'ctx._source.title = params.title',
|
||||
params: { title },
|
||||
},
|
||||
refresh: true, // Reflect in search immediately
|
||||
});
|
||||
}
|
||||
|
||||
async deleteByUserId(userId: string) {
|
||||
// Note: This method should likely only be used by admin functionality
|
||||
// since it deletes all data for a user
|
||||
await this.client.deleteByQuery({
|
||||
index: this.indexName,
|
||||
query: {
|
||||
term: { userId },
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async searchSimilar(
|
||||
queryVector: number[],
|
||||
userId: string,
|
||||
topK: number = 5,
|
||||
tenantId?: string,
|
||||
) {
|
||||
try {
|
||||
this.logger.log(
|
||||
`Vector search: userId=${userId}, vectorDim=${queryVector?.length}, topK=${topK}`,
|
||||
);
|
||||
|
||||
if (!queryVector || queryVector.length === 0) {
|
||||
this.logger.warn('Empty query vector provided');
|
||||
return [];
|
||||
}
|
||||
|
||||
const filterClauses: any[] = [];
|
||||
if (tenantId) {
|
||||
filterClauses.push({ term: { tenantId } });
|
||||
} else {
|
||||
filterClauses.push({ term: { userId } });
|
||||
}
|
||||
|
||||
const response = await this.client.search({
|
||||
index: this.indexName,
|
||||
knn: {
|
||||
field: 'vector',
|
||||
query_vector: queryVector,
|
||||
k: topK,
|
||||
num_candidates: topK * 2,
|
||||
filter: { bool: { must: filterClauses } },
|
||||
},
|
||||
size: topK,
|
||||
_source: {
|
||||
excludes: ['vector'],
|
||||
},
|
||||
});
|
||||
|
||||
const results = response.hits.hits.map((hit: any) => ({
|
||||
id: hit._id,
|
||||
score: this.normalizeScore(hit._score), // Normalize score
|
||||
content: hit._source?.content,
|
||||
fileId: hit._source?.fileId,
|
||||
fileName: hit._source?.fileName,
|
||||
title: hit._source?.title,
|
||||
chunkIndex: hit._source?.chunkIndex,
|
||||
startPosition: hit._source?.startPosition,
|
||||
endPosition: hit._source?.endPosition,
|
||||
}));
|
||||
|
||||
this.logger.log(
|
||||
`Vector search completed: found ${results.length} results`,
|
||||
);
|
||||
return results;
|
||||
} catch (error) {
|
||||
this.logger.error('Vector search failed:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async searchFullText(
|
||||
query: string,
|
||||
userId: string,
|
||||
topK: number = 5,
|
||||
tenantId?: string,
|
||||
) {
|
||||
try {
|
||||
this.logger.log(
|
||||
`Full-text search: userId=${userId}, query="${query}", topK=${topK}`,
|
||||
);
|
||||
|
||||
if (!query || query.trim().length === 0) {
|
||||
this.logger.warn('Empty query provided for full-text search');
|
||||
return [];
|
||||
}
|
||||
|
||||
const filterClauses: any[] = [];
|
||||
if (tenantId) {
|
||||
filterClauses.push({ term: { tenantId } });
|
||||
} else {
|
||||
filterClauses.push({ term: { userId } });
|
||||
}
|
||||
|
||||
const response = await this.client.search({
|
||||
index: this.indexName,
|
||||
query: {
|
||||
bool: {
|
||||
must: {
|
||||
match: {
|
||||
content: {
|
||||
query: query,
|
||||
fuzziness: 'AUTO',
|
||||
},
|
||||
},
|
||||
},
|
||||
filter: filterClauses,
|
||||
},
|
||||
},
|
||||
size: topK,
|
||||
_source: {
|
||||
excludes: ['vector'],
|
||||
},
|
||||
});
|
||||
|
||||
const results = response.hits.hits.map((hit: any) => ({
|
||||
id: hit._id,
|
||||
score: this.normalizeScore(hit._score), // Normalize score
|
||||
content: hit._source?.content,
|
||||
fileId: hit._source?.fileId,
|
||||
fileName: hit._source?.fileName,
|
||||
title: hit._source?.title,
|
||||
chunkIndex: hit._source?.chunkIndex,
|
||||
startPosition: hit._source?.startPosition,
|
||||
endPosition: hit._source?.endPosition,
|
||||
}));
|
||||
|
||||
this.logger.log(
|
||||
`Full-text search completed: found ${results.length} results`,
|
||||
);
|
||||
return results;
|
||||
} catch (error) {
|
||||
this.logger.error('Full-text search failed:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async hybridSearch(
|
||||
queryVector: number[],
|
||||
query: string,
|
||||
userId: string,
|
||||
topK: number = 5,
|
||||
vectorWeight: number = 0.7,
|
||||
selectedGroups?: string[], // Keep for backward compatibility（未使用）
|
||||
explicitFileIds?: string[], // Explicitly specified file ID list
|
||||
tenantId?: string,
|
||||
) {
|
||||
// selectedGroups is deprecated。呼び出し側で fileIds に変換して explicitFileIds を使用please
|
||||
const fileIds = explicitFileIds;
|
||||
|
||||
if (fileIds && fileIds.length === 0) {
|
||||
this.logger.log('No search target files (count=0), skipping search');
|
||||
return [];
|
||||
}
|
||||
|
||||
if (fileIds) {
|
||||
this.logger.log(`Final search target scope: ${fileIds.length} files`);
|
||||
}
|
||||
|
||||
const [vectorResults, textResults] = await Promise.all([
|
||||
this.searchSimilarWithFileFilter(
|
||||
queryVector,
|
||||
userId,
|
||||
topK,
|
||||
fileIds,
|
||||
tenantId,
|
||||
),
|
||||
this.searchFullTextWithFileFilter(query, userId, topK, fileIds, tenantId),
|
||||
]);
|
||||
|
||||
// Merge results and remove duplicates
|
||||
const combinedResults = new Map();
|
||||
|
||||
// Add vector search results
|
||||
vectorResults.forEach((result) => {
|
||||
combinedResults.set(result.id, {
|
||||
...result,
|
||||
vectorScore: result.score,
|
||||
textScore: 0,
|
||||
combinedScore: result.score * vectorWeight,
|
||||
});
|
||||
});
|
||||
|
||||
// Add full-text search results
|
||||
textResults.forEach((result) => {
|
||||
if (combinedResults.has(result.id)) {
|
||||
const existing = combinedResults.get(result.id);
|
||||
existing.textScore = result.score;
|
||||
existing.combinedScore =
|
||||
existing.vectorScore * vectorWeight +
|
||||
result.score * (1 - vectorWeight);
|
||||
} else {
|
||||
combinedResults.set(result.id, {
|
||||
...result,
|
||||
vectorScore: 0,
|
||||
textScore: result.score,
|
||||
combinedScore: result.score * (1 - vectorWeight),
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// 正規化forにすべての組み合わせスコアを取得
|
||||
const allScores = Array.from(combinedResults.values()).map(
|
||||
(r) => r.combinedScore,
|
||||
);
|
||||
const maxScore = Math.max(...allScores, 1);
|
||||
const minScore = Math.min(...allScores);
|
||||
|
||||
return Array.from(combinedResults.values())
|
||||
.sort((a, b) => b.combinedScore - a.combinedScore)
|
||||
.slice(0, topK)
|
||||
.map((result) => {
|
||||
let finalScore = result.combinedScore;
|
||||
|
||||
finalScore = Math.max(0, Math.min(1.0, finalScore));
|
||||
|
||||
return {
|
||||
id: result.id,
|
||||
score: finalScore,
|
||||
content: result.content,
|
||||
fileId: result.fileId,
|
||||
fileName: result.fileName,
|
||||
title: result.title,
|
||||
chunkIndex: result.chunkIndex,
|
||||
startPosition: result.startPosition,
|
||||
endPosition: result.endPosition,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
private async createIndex(vectorDimensions: number) {
|
||||
const mappings: any = {
|
||||
properties: {
|
||||
content: {
|
||||
type: 'text',
|
||||
analyzer: 'standard',
|
||||
},
|
||||
|
||||
vector: {
|
||||
type: 'dense_vector',
|
||||
dims: vectorDimensions,
|
||||
index: true,
|
||||
similarity: 'cosine',
|
||||
},
|
||||
|
||||
fileId: { type: 'keyword' },
|
||||
fileName: { type: 'keyword' },
|
||||
title: { type: 'text' },
|
||||
fileMimeType: { type: 'keyword' },
|
||||
|
||||
chunkIndex: { type: 'integer' },
|
||||
startPosition: { type: 'integer' },
|
||||
endPosition: { type: 'integer' },
|
||||
|
||||
userId: { type: 'keyword' },
|
||||
|
||||
// テナント情報（マルチテナント分離用）
|
||||
tenantId: { type: 'keyword' },
|
||||
|
||||
// タイムスタンプ
|
||||
createdAt: { type: 'date' },
|
||||
},
|
||||
};
|
||||
|
||||
await this.client.indices.create({
|
||||
index: this.indexName,
|
||||
mappings,
|
||||
});
|
||||
|
||||
this.logger.log(
|
||||
`Successfully created index ${this.indexName}. Vector dimensions: ${vectorDimensions}`,
|
||||
);
|
||||
}
|
||||
|
||||
private normalizeScore(rawScore: number): number {
|
||||
if (!rawScore || rawScore <= 0) return 0;
|
||||
|
||||
return Math.min(1.0, rawScore);
|
||||
}
|
||||
|
||||
private async searchSimilarWithFileFilter(
|
||||
queryVector: number[],
|
||||
userId: string,
|
||||
topK: number = 5,
|
||||
fileIds?: string[],
|
||||
tenantId?: string,
|
||||
) {
|
||||
try {
|
||||
this.logger.log(
|
||||
`Vector search with filter: userId=${userId}, tenantId=${tenantId}, vectorDim=${queryVector?.length}, topK=${topK}, fileIds=${fileIds?.length || 'all'}`,
|
||||
);
|
||||
|
||||
if (!queryVector || queryVector.length === 0) {
|
||||
this.logger.warn('Empty query vector provided');
|
||||
return [];
|
||||
}
|
||||
|
||||
if (fileIds && fileIds.length === 0) {
|
||||
this.logger.log(
|
||||
'Filter resulted in 0 files, returning empty results for vector search',
|
||||
);
|
||||
return [];
|
||||
}
|
||||
|
||||
const filterClauses: any[] = [];
|
||||
if (fileIds && fileIds.length > 0) {
|
||||
filterClauses.push({ terms: { fileId: fileIds } });
|
||||
}
|
||||
// Tenant isolation: when tenantId is provided, enforce it
|
||||
if (tenantId) {
|
||||
filterClauses.push({ term: { tenantId } });
|
||||
} else {
|
||||
// Legacy: fall back to userId-based filter
|
||||
filterClauses.push({ term: { userId } });
|
||||
}
|
||||
|
||||
const filter =
|
||||
filterClauses.length > 0
|
||||
? { bool: { must: filterClauses } }
|
||||
: undefined;
|
||||
|
||||
const queryBody: any = {
|
||||
index: this.indexName,
|
||||
knn: {
|
||||
field: 'vector',
|
||||
query_vector: queryVector,
|
||||
k: topK,
|
||||
num_candidates: topK * 2,
|
||||
},
|
||||
size: topK,
|
||||
_source: {
|
||||
excludes: ['vector'],
|
||||
},
|
||||
};
|
||||
|
||||
if (filter && Object.keys(filter).length > 0) {
|
||||
queryBody.knn.filter = filter;
|
||||
}
|
||||
|
||||
const response = await this.client.search(queryBody);
|
||||
|
||||
const results = response.hits.hits.map((hit: any) => ({
|
||||
id: hit._id,
|
||||
score: this.normalizeScore(hit._score),
|
||||
content: hit._source?.content,
|
||||
fileId: hit._source?.fileId,
|
||||
fileName: hit._source?.fileName,
|
||||
title: hit._source?.title,
|
||||
chunkIndex: hit._source?.chunkIndex,
|
||||
startPosition: hit._source?.startPosition,
|
||||
endPosition: hit._source?.endPosition,
|
||||
}));
|
||||
|
||||
this.logger.log(
|
||||
`Vector search completed: found ${results.length} results`,
|
||||
);
|
||||
return results;
|
||||
} catch (error) {
|
||||
this.logger.error('Vector search failed:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Performs full-text search with file filtering.
|
||||
*/
|
||||
public async searchFullTextWithFileFilter(
|
||||
query: string,
|
||||
userId: string,
|
||||
topK: number = 5,
|
||||
fileIds?: string[],
|
||||
tenantId?: string,
|
||||
) {
|
||||
try {
|
||||
this.logger.log(
|
||||
`Full-text search with filter: userId=${userId}, query="${query}", topK=${topK}, fileIds=${fileIds?.length || 'all'}`,
|
||||
);
|
||||
|
||||
if (!query || query.trim().length === 0) {
|
||||
this.logger.warn('Empty query provided for full-text search');
|
||||
return [];
|
||||
}
|
||||
|
||||
if (fileIds && fileIds.length === 0) {
|
||||
this.logger.log(
|
||||
'Filter resulted in 0 files, returning empty results for full-text search',
|
||||
);
|
||||
return [];
|
||||
}
|
||||
|
||||
const mustClause: any[] = [
|
||||
{
|
||||
match: {
|
||||
content: {
|
||||
query: query,
|
||||
fuzziness: 'AUTO',
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const filter: any[] = [];
|
||||
if (fileIds && fileIds.length > 0) {
|
||||
filter.push({ terms: { fileId: fileIds } });
|
||||
}
|
||||
if (tenantId) {
|
||||
filter.push({ term: { tenantId } });
|
||||
} else {
|
||||
filter.push({ term: { userId } });
|
||||
}
|
||||
|
||||
const queryBody: any = {
|
||||
index: this.indexName,
|
||||
query: {
|
||||
bool: {
|
||||
must: mustClause,
|
||||
filter: filter,
|
||||
},
|
||||
},
|
||||
size: topK,
|
||||
_source: {
|
||||
excludes: ['vector'],
|
||||
},
|
||||
};
|
||||
|
||||
const response = await this.client.search(queryBody);
|
||||
|
||||
const results = response.hits.hits.map((hit: any) => ({
|
||||
id: hit._id,
|
||||
score: this.normalizeScore(hit._score),
|
||||
content: hit._source?.content,
|
||||
fileId: hit._source?.fileId,
|
||||
fileName: hit._source?.fileName,
|
||||
title: hit._source?.title,
|
||||
chunkIndex: hit._source?.chunkIndex,
|
||||
startPosition: hit._source?.startPosition,
|
||||
endPosition: hit._source?.endPosition,
|
||||
}));
|
||||
|
||||
this.logger.log(
|
||||
`Full-text search completed: found ${results.length} results`,
|
||||
);
|
||||
return results;
|
||||
} catch (error) {
|
||||
this.logger.error('Full-text search failed:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 指定されたファイルのすべてのチャンクを取得
|
||||
*/
|
||||
async getFileChunks(fileId: string, userId: string, tenantId?: string) {
|
||||
try {
|
||||
this.logger.log(`Getting chunks for file ${fileId}`);
|
||||
|
||||
const filter: any[] = [{ term: { fileId } }];
|
||||
if (tenantId) {
|
||||
filter.push({ term: { tenantId } });
|
||||
} else {
|
||||
filter.push({ term: { userId } });
|
||||
}
|
||||
|
||||
const response = await this.client.search({
|
||||
index: this.indexName,
|
||||
query: {
|
||||
bool: { filter },
|
||||
},
|
||||
sort: [{ chunkIndex: 'asc' }],
|
||||
size: 10000,
|
||||
_source: {
|
||||
excludes: ['vector'],
|
||||
},
|
||||
});
|
||||
|
||||
const chunks = response.hits.hits.map((hit: any) => ({
|
||||
id: hit._id,
|
||||
chunkIndex: hit._source.chunkIndex,
|
||||
content: hit._source.content,
|
||||
startPosition: hit._source.startPosition,
|
||||
endPosition: hit._source.endPosition,
|
||||
fileName: hit._source.fileName,
|
||||
}));
|
||||
|
||||
this.logger.log(`Found ${chunks.length} chunks for file ${fileId}`);
|
||||
return chunks;
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to get chunks for file ${fileId}`, error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
/**
|
||||
* Assessment Command DTO
|
||||
* 定义飞书机器人测评命令的类型和接口
|
||||
*/
|
||||
|
||||
export enum AssessmentCommandType {
|
||||
START = 'start',
|
||||
ANSWER = 'answer',
|
||||
STATUS = 'status',
|
||||
RESULT = 'result',
|
||||
HELP = 'help',
|
||||
CANCEL = 'cancel',
|
||||
}
|
||||
|
||||
export interface AssessmentCommand {
|
||||
type: AssessmentCommandType;
|
||||
parameters: string[];
|
||||
rawMessage: string;
|
||||
timestamp: Date;
|
||||
}
|
||||
|
||||
export class AssessmentCommandDto {
|
||||
type: AssessmentCommandType;
|
||||
parameters: string[];
|
||||
rawMessage: string;
|
||||
|
||||
constructor(
|
||||
type: AssessmentCommandType,
|
||||
parameters: string[],
|
||||
rawMessage: string,
|
||||
) {
|
||||
this.type = type;
|
||||
this.parameters = parameters;
|
||||
this.rawMessage = rawMessage;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
import { IsString, IsNotEmpty, IsUUID } from 'class-validator';
|
||||
|
||||
export class BindFeishuBotDto {
|
||||
@IsUUID()
|
||||
@IsNotEmpty()
|
||||
botId: string;
|
||||
|
||||
@IsString()
|
||||
@IsNotEmpty()
|
||||
verificationCode?: string; // Optional: used to validate the binding relationship
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import { IsString, IsNotEmpty, IsOptional, IsBoolean } from 'class-validator';
|
||||
|
||||
export class CreateFeishuBotDto {
|
||||
@IsString()
|
||||
@IsNotEmpty()
|
||||
appId: string;
|
||||
|
||||
@IsString()
|
||||
@IsNotEmpty()
|
||||
appSecret: string;
|
||||
|
||||
@IsString()
|
||||
@IsOptional()
|
||||
verificationToken?: string;
|
||||
|
||||
@IsString()
|
||||
@IsOptional()
|
||||
encryptKey?: string;
|
||||
|
||||
@IsString()
|
||||
@IsOptional()
|
||||
botName?: string;
|
||||
|
||||
@IsBoolean()
|
||||
@IsOptional()
|
||||
enabled?: boolean;
|
||||
|
||||
@IsString()
|
||||
@IsOptional()
|
||||
knowledgeBaseId?: string;
|
||||
|
||||
@IsString()
|
||||
@IsOptional()
|
||||
knowledgeGroupId?: string;
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import { IsString, IsOptional } from 'class-validator';
|
||||
|
||||
export class CreateSignatureDto {
|
||||
@IsString()
|
||||
@IsOptional()
|
||||
timestamp?: string;
|
||||
|
||||
@IsString()
|
||||
@IsOptional()
|
||||
nonce?: string;
|
||||
}
|
||||
|
||||
export class VerifyWebhookDto {
|
||||
@IsString()
|
||||
token: string;
|
||||
|
||||
@IsString()
|
||||
@IsOptional()
|
||||
challenge?: string;
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
export enum ConnectionState {
|
||||
DISCONNECTED = 'disconnected',
|
||||
CONNECTING = 'connecting',
|
||||
CONNECTED = 'connected',
|
||||
ERROR = 'error',
|
||||
}
|
||||
|
||||
export interface ConnectionStatus {
|
||||
botId: string;
|
||||
state: ConnectionState;
|
||||
connectedAt?: Date;
|
||||
lastHeartbeat?: Date;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export class WsStatusResponseDto {
|
||||
botId: string;
|
||||
state: ConnectionState;
|
||||
connectedAt?: string;
|
||||
lastHeartbeat?: string;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export class WsConnectResponseDto {
|
||||
success: boolean;
|
||||
botId: string;
|
||||
status: ConnectionState | string;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export class WsDisconnectResponseDto {
|
||||
success: boolean;
|
||||
botId: string;
|
||||
status: ConnectionState | string;
|
||||
}
|
||||
|
||||
export class AllWsStatusResponseDto {
|
||||
connections: WsStatusResponseDto[];
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
import {
|
||||
Entity,
|
||||
PrimaryGeneratedColumn,
|
||||
Column,
|
||||
CreateDateColumn,
|
||||
UpdateDateColumn,
|
||||
ManyToOne,
|
||||
JoinColumn,
|
||||
} from 'typeorm';
|
||||
import { FeishuBot } from './feishu-bot.entity';
|
||||
|
||||
@Entity('feishu_assessment_sessions')
|
||||
export class FeishuAssessmentSession {
|
||||
@PrimaryGeneratedColumn('uuid')
|
||||
id: string;
|
||||
|
||||
@Column({ name: 'bot_id' })
|
||||
botId: string;
|
||||
|
||||
@Column({ name: 'open_id' })
|
||||
openId: string;
|
||||
|
||||
@Column({ name: 'assessment_session_id' })
|
||||
assessmentSessionId: string;
|
||||
|
||||
@Column({
|
||||
type: 'varchar',
|
||||
enum: ['active', 'completed', 'cancelled'],
|
||||
default: 'active',
|
||||
})
|
||||
status: 'active' | 'completed' | 'cancelled';
|
||||
|
||||
@Column({ name: 'current_question_index', default: 0 })
|
||||
currentQuestionIndex: number;
|
||||
|
||||
@CreateDateColumn({ name: 'created_at' })
|
||||
createdAt: Date;
|
||||
|
||||
@UpdateDateColumn({ name: 'updated_at' })
|
||||
updatedAt: Date;
|
||||
|
||||
// 关联关系
|
||||
@ManyToOne(() => FeishuBot, { onDelete: 'CASCADE' })
|
||||
@JoinColumn({ name: 'bot_id' })
|
||||
bot: FeishuBot;
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
import {
|
||||
Entity,
|
||||
PrimaryGeneratedColumn,
|
||||
Column,
|
||||
CreateDateColumn,
|
||||
UpdateDateColumn,
|
||||
ManyToOne,
|
||||
JoinColumn,
|
||||
} from 'typeorm';
|
||||
import { User } from '../../user/user.entity';
|
||||
|
||||
@Entity('feishu_bots')
|
||||
export class FeishuBot {
|
||||
@PrimaryGeneratedColumn('uuid')
|
||||
id: string;
|
||||
|
||||
@Column({ name: 'user_id' })
|
||||
userId: string;
|
||||
|
||||
@Column({ name: 'tenant_id', nullable: true })
|
||||
tenantId: string;
|
||||
|
||||
@ManyToOne(() => User, { onDelete: 'CASCADE' })
|
||||
@JoinColumn({ name: 'user_id' })
|
||||
user: User;
|
||||
|
||||
@Column({ name: 'app_id', length: 64 })
|
||||
appId: string;
|
||||
|
||||
@Column({ name: 'app_secret', length: 256 })
|
||||
appSecret: string;
|
||||
|
||||
@Column({ name: 'tenant_access_token', nullable: true, type: 'text' })
|
||||
tenantAccessToken: string;
|
||||
|
||||
@Column({ name: 'token_expires_at', nullable: true, type: 'datetime' })
|
||||
tokenExpiresAt: Date;
|
||||
|
||||
@Column({ name: 'verification_token', nullable: true, length: 128 })
|
||||
verificationToken: string;
|
||||
|
||||
@Column({ name: 'encrypt_key', nullable: true, length: 256 })
|
||||
encryptKey: string;
|
||||
|
||||
@Column({ name: 'bot_name', nullable: true, length: 128 })
|
||||
botName: string;
|
||||
|
||||
@Column({ default: true })
|
||||
enabled: boolean;
|
||||
|
||||
@Column({ name: 'is_default', default: false })
|
||||
isDefault: boolean;
|
||||
|
||||
@Column({ name: 'webhook_url', nullable: true, type: 'text' })
|
||||
webhookUrl: string;
|
||||
|
||||
@Column({ name: 'use_web_socket', default: false })
|
||||
useWebSocket: boolean;
|
||||
|
||||
@Column({ name: 'ws_connection_state', nullable: true, length: 32 })
|
||||
wsConnectionState: string;
|
||||
|
||||
@Column({ name: 'knowledge_base_id', nullable: true, length: 36 })
|
||||
knowledgeBaseId: string;
|
||||
|
||||
@Column({ name: 'knowledge_group_id', nullable: true, length: 36 })
|
||||
knowledgeGroupId: string;
|
||||
|
||||
@CreateDateColumn({ name: 'created_at' })
|
||||
createdAt: Date;
|
||||
|
||||
@UpdateDateColumn({ name: 'updated_at' })
|
||||
updatedAt: Date;
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import {
|
||||
WSClient,
|
||||
EventDispatcher,
|
||||
LoggerLevel,
|
||||
} from '@larksuiteoapi/node-sdk';
|
||||
import { FeishuBot } from './entities/feishu-bot.entity';
|
||||
import { ConnectionState, ConnectionStatus } from './dto/ws-status.dto';
|
||||
|
||||
interface BotConnection {
|
||||
client: WSClient;
|
||||
status: ConnectionStatus;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class FeishuWsManager {
|
||||
private readonly logger = new Logger(FeishuWsManager.name);
|
||||
private connections: Map<string, BotConnection> = new Map();
|
||||
private reconnectAttempts: Map<string, number> = new Map();
|
||||
private readonly MAX_RECONNECT_ATTEMPTS = 5;
|
||||
private readonly RECONNECT_DELAYS = [1000, 2000, 4000, 8000, 16000];
|
||||
|
||||
// Injected after construction to avoid circular dep
|
||||
private _feishuService: any;
|
||||
|
||||
setFeishuService(service: any): void {
|
||||
this._feishuService = service;
|
||||
}
|
||||
|
||||
/**
|
||||
* Start WebSocket connection for a bot
|
||||
*/
|
||||
async connect(bot: FeishuBot): Promise<void> {
|
||||
const botId = bot.id;
|
||||
|
||||
// Check if already connected or connecting
|
||||
const existing = this.connections.get(botId);
|
||||
if (
|
||||
existing &&
|
||||
(existing.status.state === ConnectionState.CONNECTED ||
|
||||
existing.status.state === ConnectionState.CONNECTING)
|
||||
) {
|
||||
this.logger.warn(`Bot ${botId} is already connecting or connected`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Mark as connecting immediately to prevent race conditions
|
||||
this.connections.set(botId, {
|
||||
client: null as any,
|
||||
status: { botId, state: ConnectionState.CONNECTING },
|
||||
});
|
||||
|
||||
try {
|
||||
const wsClient = new WSClient({
|
||||
appId: bot.appId,
|
||||
appSecret: bot.appSecret,
|
||||
loggerLevel: LoggerLevel.info,
|
||||
logger: {
|
||||
debug: (msg: string) => this.logger.debug(msg),
|
||||
info: (msg: string) => this.logger.log(msg),
|
||||
warn: (msg: string) => this.logger.warn(msg),
|
||||
error: (msg: string) => this.logger.error(msg),
|
||||
trace: (msg: string) => this.logger.verbose(msg),
|
||||
},
|
||||
});
|
||||
|
||||
// Register the main message event handler
|
||||
wsClient.start({
|
||||
eventDispatcher: new EventDispatcher({}).register({
|
||||
'im.message.receive_v1': async (data: any) => {
|
||||
await this._handleMessage(bot, data);
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
const conn = this.connections.get(botId);
|
||||
if (conn) {
|
||||
conn.client = wsClient;
|
||||
conn.status = {
|
||||
botId,
|
||||
state: ConnectionState.CONNECTED,
|
||||
connectedAt: new Date(),
|
||||
};
|
||||
}
|
||||
|
||||
this.reconnectAttempts.set(botId, 0);
|
||||
this.logger.log(`WebSocket connected for bot ${botId}`);
|
||||
} catch (error: any) {
|
||||
this.logger.error(`Failed to connect WebSocket for bot ${botId}`, error);
|
||||
this._setStatus(botId, {
|
||||
botId,
|
||||
state: ConnectionState.ERROR,
|
||||
error: error?.message || 'Connection failed',
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Disconnect WebSocket for a bot
|
||||
*/
|
||||
async disconnect(botId: string): Promise<void> {
|
||||
const connection = this.connections.get(botId);
|
||||
if (!connection) {
|
||||
this.logger.warn(`No connection found for bot ${botId}`);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// Lark.WSClient does not expose a public stop() in all versions;
|
||||
// we simply remove the reference and let GC clean up.
|
||||
this.connections.delete(botId);
|
||||
this.reconnectAttempts.delete(botId);
|
||||
this.logger.log(`WebSocket disconnected for bot ${botId}`);
|
||||
} catch (error) {
|
||||
this.logger.error(`Error disconnecting bot ${botId}`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get connection status for a specific bot
|
||||
*/
|
||||
getStatus(botId: string): ConnectionStatus | null {
|
||||
return this.connections.get(botId)?.status ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all active connection statuses
|
||||
*/
|
||||
getAllStatuses(): ConnectionStatus[] {
|
||||
return Array.from(this.connections.values()).map((c) => c.status);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if the bot has an active connection
|
||||
*/
|
||||
isConnected(botId: string): boolean {
|
||||
return (
|
||||
this.connections.get(botId)?.status.state === ConnectionState.CONNECTED
|
||||
);
|
||||
}
|
||||
|
||||
// ─── Private Helpers ──────────────────────────────────────────────────────
|
||||
|
||||
private _setStatus(botId: string, status: ConnectionStatus): void {
|
||||
const connection = this.connections.get(botId);
|
||||
if (connection) {
|
||||
connection.status = status;
|
||||
}
|
||||
// If connection not stored yet (during initial connect), we just skip —
|
||||
// the status will be set when the connection map entry is created.
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle an incoming im.message.receive_v1 event from Feishu WebSocket.
|
||||
*/
|
||||
private async _handleMessage(bot: FeishuBot, data: any): Promise<void> {
|
||||
this.logger.log(`Received WS message for bot ${bot.id}`);
|
||||
|
||||
try {
|
||||
const event = data?.event ?? data;
|
||||
const message = event?.message;
|
||||
|
||||
if (!message) {
|
||||
this.logger.warn('No message field in WS event');
|
||||
return;
|
||||
}
|
||||
|
||||
const messageId: string = message.message_id;
|
||||
const openId: string | undefined = event?.sender?.sender_id?.open_id;
|
||||
|
||||
if (!openId) {
|
||||
this.logger.warn('No sender open_id in WS event');
|
||||
return;
|
||||
}
|
||||
|
||||
let userText = '';
|
||||
try {
|
||||
const content = JSON.parse(message.content || '{}');
|
||||
userText = content.text || '';
|
||||
} catch {
|
||||
this.logger.warn('Failed to parse WS message content');
|
||||
return;
|
||||
}
|
||||
|
||||
if (!userText.trim()) return;
|
||||
|
||||
if (this._feishuService) {
|
||||
await this._feishuService.handleIncomingMessage(
|
||||
bot,
|
||||
openId,
|
||||
messageId,
|
||||
userText,
|
||||
);
|
||||
} else {
|
||||
this.logger.error('FeishuService not injected into FeishuWsManager');
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error('Error in _handleMessage', error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Schedule an auto-reconnect with exponential backoff.
|
||||
*/
|
||||
async attemptReconnect(bot: FeishuBot): Promise<void> {
|
||||
const botId = bot.id;
|
||||
const attempts = this.reconnectAttempts.get(botId) ?? 0;
|
||||
|
||||
if (attempts >= this.MAX_RECONNECT_ATTEMPTS) {
|
||||
this.logger.error(`Max reconnect attempts reached for bot ${botId}`);
|
||||
this._setStatus(botId, {
|
||||
botId,
|
||||
state: ConnectionState.ERROR,
|
||||
error: 'Max reconnect attempts reached',
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const delay =
|
||||
this.RECONNECT_DELAYS[attempts] ??
|
||||
this.RECONNECT_DELAYS[this.RECONNECT_DELAYS.length - 1];
|
||||
this.logger.log(
|
||||
`Reconnecting bot ${botId} in ${delay}ms (attempt ${attempts + 1})`,
|
||||
);
|
||||
this.reconnectAttempts.set(botId, attempts + 1);
|
||||
|
||||
setTimeout(async () => {
|
||||
try {
|
||||
await this.connect(bot);
|
||||
} catch (error) {
|
||||
this.logger.error(`Reconnect failed for bot ${botId}`, error);
|
||||
}
|
||||
}, delay);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,291 @@
|
||||
import {
|
||||
Controller,
|
||||
Post,
|
||||
Get,
|
||||
Delete,
|
||||
Body,
|
||||
Param,
|
||||
Headers,
|
||||
UseGuards,
|
||||
Request,
|
||||
Logger,
|
||||
Patch,
|
||||
Res,
|
||||
} from '@nestjs/common';
|
||||
import { Response } from 'express';
|
||||
import { FeishuService } from './feishu.service';
|
||||
import { FeishuAssessmentService } from './services/feishu-assessment.service';
|
||||
import { CreateFeishuBotDto } from './dto/create-bot.dto';
|
||||
import { CombinedAuthGuard } from '../auth/combined-auth.guard';
|
||||
import { Public } from '../auth/public.decorator';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
@Controller('feishu')
|
||||
export class FeishuController {
|
||||
private readonly logger = new Logger(FeishuController.name);
|
||||
|
||||
constructor(
|
||||
private readonly feishuService: FeishuService,
|
||||
private readonly feishuAssessmentService: FeishuAssessmentService,
|
||||
) {}
|
||||
|
||||
// ─── Bot Management Endpoints (JWT-protected) ─────────────────────────────
|
||||
|
||||
/** GET /feishu/bots - List user's bots, masking sensitive fields */
|
||||
@Get('bots')
|
||||
@UseGuards(CombinedAuthGuard)
|
||||
async listBots(@Request() req) {
|
||||
const bots = await this.feishuService.getUserBots(
|
||||
req.user.id,
|
||||
req.user.tenantId,
|
||||
);
|
||||
return bots.map((bot) => ({
|
||||
id: bot.id,
|
||||
appId: bot.appId,
|
||||
botName: bot.botName,
|
||||
enabled: bot.enabled,
|
||||
isDefault: bot.isDefault,
|
||||
webhookUrl: `/api/feishu/webhook/${bot.appId}`,
|
||||
createdAt: bot.createdAt,
|
||||
}));
|
||||
}
|
||||
|
||||
/** POST /feishu/bots - Create or update a bot */
|
||||
@Post('bots')
|
||||
@UseGuards(CombinedAuthGuard)
|
||||
async createBot(@Request() req, @Body() dto: CreateFeishuBotDto) {
|
||||
const bot = await this.feishuService.createBot(
|
||||
req.user.id,
|
||||
req.user.tenantId,
|
||||
dto,
|
||||
);
|
||||
return {
|
||||
id: bot.id,
|
||||
appId: bot.appId,
|
||||
botName: bot.botName,
|
||||
enabled: bot.enabled,
|
||||
webhookUrl: `/api/feishu/webhook/${bot.appId}`,
|
||||
};
|
||||
}
|
||||
|
||||
/** PATCH /feishu/bots/:id/toggle - Enable or disable a bot */
|
||||
@Patch('bots/:id/toggle')
|
||||
@UseGuards(CombinedAuthGuard)
|
||||
async toggleBot(
|
||||
@Request() req,
|
||||
@Param('id') botId: string,
|
||||
@Body() body: { enabled: boolean },
|
||||
) {
|
||||
const bot = await this.feishuService.setBotEnabled(botId, body.enabled);
|
||||
return { id: bot.id, enabled: bot.enabled };
|
||||
}
|
||||
|
||||
/** DELETE /feishu/bots/:id - Delete a bot */
|
||||
@Delete('bots/:id')
|
||||
@UseGuards(CombinedAuthGuard)
|
||||
async deleteBot(@Request() req, @Param('id') botId: string) {
|
||||
await this.feishuService.deleteBot(req.user.id, botId);
|
||||
return { success: true };
|
||||
}
|
||||
|
||||
// ─── WebSocket Management Endpoints ────────────────────────────────────
|
||||
|
||||
/** POST /feishu/bots/:id/ws/connect - Start WebSocket connection */
|
||||
@Post('bots/:id/ws/connect')
|
||||
@UseGuards(CombinedAuthGuard)
|
||||
async connectWs(@Request() req, @Param('id') botId: string) {
|
||||
const bot = await this.feishuService.getBotById(botId);
|
||||
if (!bot || bot.userId !== req.user.id) {
|
||||
return { success: false, error: 'Bot not found' };
|
||||
}
|
||||
try {
|
||||
await this.feishuService.startWsConnection(botId);
|
||||
return { success: true, botId, status: 'connecting' };
|
||||
} catch (error: any) {
|
||||
return {
|
||||
success: false,
|
||||
botId,
|
||||
error: error?.message || 'Failed to connect',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/** POST /feishu/bots/:id/ws/disconnect - Stop WebSocket connection */
|
||||
@Post('bots/:id/ws/disconnect')
|
||||
@UseGuards(CombinedAuthGuard)
|
||||
async disconnectWs(@Request() req, @Param('id') botId: string) {
|
||||
const bot = await this.feishuService.getBotById(botId);
|
||||
if (!bot || bot.userId !== req.user.id) {
|
||||
return { success: false, error: 'Bot not found' };
|
||||
}
|
||||
try {
|
||||
await this.feishuService.stopWsConnection(botId);
|
||||
return { success: true, botId, status: 'disconnected' };
|
||||
} catch (error: any) {
|
||||
return {
|
||||
success: false,
|
||||
botId,
|
||||
error: error?.message || 'Failed to disconnect',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/** GET /feishu/bots/:id/ws/status - Get connection status */
|
||||
@Get('bots/:id/ws/status')
|
||||
@UseGuards(CombinedAuthGuard)
|
||||
async getWsStatus(@Request() req, @Param('id') botId: string) {
|
||||
const bot = await this.feishuService.getBotById(botId);
|
||||
if (!bot || bot.userId !== req.user.id) {
|
||||
return { success: false, error: 'Bot not found' };
|
||||
}
|
||||
const status = await this.feishuService.getWsStatus(botId);
|
||||
if (!status) {
|
||||
return { botId, state: 'disconnected' };
|
||||
}
|
||||
return {
|
||||
botId: status.botId,
|
||||
state: status.state,
|
||||
connectedAt: status.connectedAt?.toISOString(),
|
||||
lastHeartbeat: status.lastHeartbeat?.toISOString(),
|
||||
error: status.error,
|
||||
};
|
||||
}
|
||||
|
||||
/** GET /feishu/ws/status - Get all active WS connection statuses */
|
||||
@Get('ws/status')
|
||||
@UseGuards(CombinedAuthGuard)
|
||||
async getAllWsStatus() {
|
||||
const statuses = await this.feishuService.getAllWsStatuses();
|
||||
return {
|
||||
connections: statuses.map((s) => ({
|
||||
botId: s.botId,
|
||||
state: s.state,
|
||||
connectedAt: s.connectedAt?.toISOString(),
|
||||
lastHeartbeat: s.lastHeartbeat?.toISOString(),
|
||||
error: s.error,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Feishu Webhook Endpoint (Public) ────────────────────────────────────
|
||||
@Get('webhook/:appId')
|
||||
@Post('webhook/:appId')
|
||||
@Public()
|
||||
async handleWebhook(
|
||||
@Param('appId') appId: string,
|
||||
@Body() body: any,
|
||||
@Headers() headers: any,
|
||||
@Request() req: any,
|
||||
@Res() res: Response,
|
||||
) {
|
||||
const logEntry = `\n[${new Date().toISOString()}] ${req.method} /api/feishu/webhook/${appId}\nHeaders: ${JSON.stringify(headers)}\nBody: ${JSON.stringify(body)}\n`;
|
||||
fs.appendFileSync('feishu_webhook.log', logEntry);
|
||||
|
||||
this.logger.log(
|
||||
`Incoming Feishu webhook [${req.method}] for appId: ${appId}`,
|
||||
);
|
||||
|
||||
// GET request for simple connection test
|
||||
if (req.method === 'GET') {
|
||||
return res.status(200).json({
|
||||
status: 'ok',
|
||||
message: 'AuraK Feishu Webhook is active.',
|
||||
appId,
|
||||
timestamp: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
// Step 1: URL verification handshake
|
||||
const challenge = body?.challenge || body?.event?.challenge;
|
||||
if (body?.type === 'url_verification' || challenge) {
|
||||
this.logger.log(
|
||||
`URL verification active for appId: ${appId}, challenge: ${challenge}`,
|
||||
);
|
||||
return res.status(200).json({ challenge });
|
||||
}
|
||||
|
||||
// Step 2: Return 200 immediately for all other events
|
||||
res.status(200).json({ success: true });
|
||||
|
||||
// Step 3: Process the event asynchronously
|
||||
if (body?.type === 'event_callback' || body?.header?.event_type) {
|
||||
setImmediate(() =>
|
||||
this._processEvent(appId, body).catch((e) =>
|
||||
this.logger.error('Failed to process Feishu event async', e),
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Private Event Processor ─────────────────────────────────────────────
|
||||
|
||||
private async _processEvent(appId: string, body: any): Promise<void> {
|
||||
const { type, event, header } = body;
|
||||
|
||||
if (type !== 'event_callback') return;
|
||||
|
||||
const eventType = header?.event_type || body.event_type;
|
||||
const eventId = header?.event_id;
|
||||
this.logger.log(
|
||||
`Processing Feishu event [${eventId}]: ${eventType} for appId: ${appId}`,
|
||||
);
|
||||
|
||||
const bot = await this.feishuService.getBotByAppId(appId);
|
||||
if (!bot || !bot.enabled) {
|
||||
this.logger.warn(`Bot not found or disabled for appId: ${appId}`);
|
||||
return;
|
||||
}
|
||||
|
||||
switch (eventType) {
|
||||
case 'im.message.receive_v1':
|
||||
case 'im.message.p2p_msg_received':
|
||||
case 'im.message.group_at_msg_received':
|
||||
await this._handleMessage(bot, event);
|
||||
break;
|
||||
default:
|
||||
this.logger.log(`Unhandled event type: ${eventType}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse incoming IM message and route to chatService via FeishuService.
|
||||
* Implements Chunk 5 integration.
|
||||
*/
|
||||
private async _handleMessage(bot: any, event: any): Promise<void> {
|
||||
const message = event?.message;
|
||||
if (!message) return;
|
||||
|
||||
const messageId = message.message_id;
|
||||
const openId = event?.sender?.sender_id?.open_id;
|
||||
|
||||
if (!openId) {
|
||||
this.logger.warn('No sender open_id found in Feishu event');
|
||||
return;
|
||||
}
|
||||
|
||||
// Parse text content
|
||||
let userText = '';
|
||||
try {
|
||||
const content = JSON.parse(message.content || '{}');
|
||||
userText = content.text || '';
|
||||
} catch {
|
||||
this.logger.warn('Failed to parse Feishu message content');
|
||||
return;
|
||||
}
|
||||
|
||||
if (!userText.trim()) return;
|
||||
|
||||
try {
|
||||
// Centralized routing via FeishuService
|
||||
await this.feishuService.handleIncomingMessage(
|
||||
bot,
|
||||
openId,
|
||||
messageId,
|
||||
userText,
|
||||
);
|
||||
} catch (error) {
|
||||
this.logger.error('Message handling failed', error);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import { Module, forwardRef } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
import { FeishuController } from './feishu.controller';
|
||||
import { FeishuService } from './feishu.service';
|
||||
import { FeishuBot } from './entities/feishu-bot.entity';
|
||||
import { FeishuAssessmentSession } from './entities/feishu-assessment-session.entity';
|
||||
import { FeishuWsManager } from './feishu-ws.manager';
|
||||
import { FeishuAssessmentService } from './services/feishu-assessment.service';
|
||||
import { AssessmentCommandParser } from './services/assessment-command.parser';
|
||||
import { ChatModule } from '../chat/chat.module';
|
||||
import { UserModule } from '../user/user.module';
|
||||
import { ModelConfigModule } from '../model-config/model-config.module';
|
||||
import { AssessmentModule } from '../assessment/assessment.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
TypeOrmModule.forFeature([FeishuBot, FeishuAssessmentSession]),
|
||||
forwardRef(() => ChatModule),
|
||||
forwardRef(() => UserModule),
|
||||
forwardRef(() => ModelConfigModule),
|
||||
forwardRef(() => AssessmentModule),
|
||||
],
|
||||
controllers: [FeishuController],
|
||||
providers: [
|
||||
FeishuService,
|
||||
FeishuWsManager,
|
||||
FeishuAssessmentService,
|
||||
AssessmentCommandParser,
|
||||
],
|
||||
exports: [FeishuService, FeishuAssessmentService, TypeOrmModule],
|
||||
})
|
||||
export class FeishuModule {}
|
||||
@@ -0,0 +1,583 @@
|
||||
import {
|
||||
Injectable,
|
||||
Logger,
|
||||
forwardRef,
|
||||
Inject,
|
||||
OnModuleInit,
|
||||
} from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Repository } from 'typeorm';
|
||||
import axios from 'axios';
|
||||
import { FeishuBot } from './entities/feishu-bot.entity';
|
||||
import { CreateFeishuBotDto } from './dto/create-bot.dto';
|
||||
import { ChatService } from '../chat/chat.service';
|
||||
import { ModelConfigService } from '../model-config/model-config.service';
|
||||
import { UserService } from '../user/user.service';
|
||||
import { ModelType } from '../types';
|
||||
import { FeishuWsManager } from './feishu-ws.manager';
|
||||
import { ConnectionStatus } from './dto/ws-status.dto';
|
||||
import { FeishuAssessmentService } from './services/feishu-assessment.service';
|
||||
import { tenantStore } from '../tenant/tenant.store';
|
||||
import { i18nStore } from '../i18n/i18n.store';
|
||||
|
||||
@Injectable()
|
||||
export class FeishuService implements OnModuleInit {
|
||||
private readonly logger = new Logger(FeishuService.name);
|
||||
private readonly feishuApiBase = 'https://open.feishu.cn/open-apis';
|
||||
|
||||
constructor(
|
||||
@InjectRepository(FeishuBot)
|
||||
private botRepository: Repository<FeishuBot>,
|
||||
@Inject(forwardRef(() => ChatService))
|
||||
private chatService: ChatService,
|
||||
@Inject(forwardRef(() => ModelConfigService))
|
||||
private modelConfigService: ModelConfigService,
|
||||
@Inject(forwardRef(() => UserService))
|
||||
private userService: UserService,
|
||||
private wsManager: FeishuWsManager,
|
||||
@Inject(forwardRef(() => FeishuAssessmentService))
|
||||
private feishuAssessmentService: FeishuAssessmentService,
|
||||
) {}
|
||||
|
||||
onModuleInit(): void {
|
||||
// Break circular dep: inject self into manager after module is ready
|
||||
this.wsManager.setFeishuService(this);
|
||||
}
|
||||
|
||||
async createBot(
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
dto: CreateFeishuBotDto,
|
||||
): Promise<FeishuBot> {
|
||||
const existing = await this.botRepository.findOne({
|
||||
where: { userId, appId: dto.appId, tenantId },
|
||||
});
|
||||
|
||||
if (existing) {
|
||||
Object.assign(existing, dto);
|
||||
return this.botRepository.save(existing);
|
||||
}
|
||||
|
||||
const bot = this.botRepository.create({ userId, tenantId, ...dto });
|
||||
return this.botRepository.save(bot);
|
||||
}
|
||||
|
||||
async getUserBots(userId: string, tenantId: string): Promise<FeishuBot[]> {
|
||||
return this.botRepository.find({
|
||||
where: { userId, tenantId },
|
||||
relations: ['user'],
|
||||
});
|
||||
}
|
||||
|
||||
async getBotById(botId: string): Promise<FeishuBot | null> {
|
||||
return this.botRepository.findOne({
|
||||
where: { id: botId },
|
||||
relations: ['user'],
|
||||
});
|
||||
}
|
||||
|
||||
async getBotByAppId(appId: string): Promise<FeishuBot | null> {
|
||||
return this.botRepository.findOne({
|
||||
where: { appId },
|
||||
relations: ['user'],
|
||||
});
|
||||
}
|
||||
|
||||
async setBotEnabled(botId: string, enabled: boolean): Promise<FeishuBot> {
|
||||
const bot = await this.botRepository.findOne({ where: { id: botId } });
|
||||
if (!bot) throw new Error('Bot not found');
|
||||
bot.enabled = enabled;
|
||||
return this.botRepository.save(bot);
|
||||
}
|
||||
|
||||
async deleteBot(userId: string, botId: string): Promise<void> {
|
||||
await this.botRepository.delete({ id: botId, userId });
|
||||
}
|
||||
|
||||
// ─── Feishu API Calls ─────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Get or refresh tenant_access_token, cached per bot in DB
|
||||
*/
|
||||
async getValidToken(bot: FeishuBot): Promise<string> {
|
||||
if (
|
||||
bot.tokenExpiresAt &&
|
||||
bot.tenantAccessToken &&
|
||||
new Date(bot.tokenExpiresAt) > new Date(Date.now() + 30 * 60 * 1000)
|
||||
) {
|
||||
return bot.tenantAccessToken;
|
||||
}
|
||||
|
||||
this.logger.log(`Refreshing access token for bot: ${bot.appId}`);
|
||||
const { data } = await axios.post<{
|
||||
code: number;
|
||||
msg: string;
|
||||
tenant_access_token: string;
|
||||
expire: number;
|
||||
}>(`${this.feishuApiBase}/auth/v3/tenant_access_token/internal`, {
|
||||
app_id: bot.appId,
|
||||
app_secret: bot.appSecret,
|
||||
});
|
||||
|
||||
if (data.code !== 0) {
|
||||
throw new Error(`Failed to get Feishu token: ${data.msg}`);
|
||||
}
|
||||
|
||||
bot.tenantAccessToken = data.tenant_access_token;
|
||||
bot.tokenExpiresAt = new Date(Date.now() + data.expire * 1000);
|
||||
await this.botRepository.save(bot);
|
||||
|
||||
return data.tenant_access_token;
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a card message to a Feishu user
|
||||
*/
|
||||
async sendCardMessage(
|
||||
bot: FeishuBot,
|
||||
receiveIdType: 'open_id' | 'user_id' | 'chat_id',
|
||||
receiveId: string,
|
||||
card: any,
|
||||
): Promise<string> {
|
||||
const token = await this.getValidToken(bot);
|
||||
|
||||
const { data } = await axios.post<{
|
||||
code: number;
|
||||
msg: string;
|
||||
data: { message_id: string };
|
||||
}>(
|
||||
`${this.feishuApiBase}/im/v1/messages?receive_id_type=${receiveIdType}`,
|
||||
{
|
||||
receive_id: receiveId,
|
||||
msg_type: 'interactive',
|
||||
content: JSON.stringify(card),
|
||||
},
|
||||
{ headers: { Authorization: `Bearer ${token}` } },
|
||||
);
|
||||
|
||||
if (data.code !== 0) {
|
||||
throw new Error(`Failed to send Feishu card: ${data.msg}`);
|
||||
}
|
||||
|
||||
return data.data.message_id;
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a simple text message to a Feishu user
|
||||
*/
|
||||
async sendTextMessage(
|
||||
bot: FeishuBot,
|
||||
receiveIdType: 'open_id' | 'user_id' | 'chat_id',
|
||||
receiveId: string,
|
||||
text: string,
|
||||
): Promise<string> {
|
||||
const token = await this.getValidToken(bot);
|
||||
|
||||
const { data } = await axios.post<{
|
||||
code: number;
|
||||
msg: string;
|
||||
data: { message_id: string };
|
||||
}>(
|
||||
`${this.feishuApiBase}/im/v1/messages?receive_id_type=${receiveIdType}`,
|
||||
{
|
||||
receive_id: receiveId,
|
||||
msg_type: 'text',
|
||||
content: JSON.stringify({ text }),
|
||||
},
|
||||
{ headers: { Authorization: `Bearer ${token}` } },
|
||||
);
|
||||
|
||||
if (data.code !== 0) {
|
||||
throw new Error(`Failed to send Feishu message: ${data.msg}`);
|
||||
}
|
||||
|
||||
return data.data.message_id;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update an already-sent message (supports interactive cards)
|
||||
*/
|
||||
async updateMessage(
|
||||
bot: FeishuBot,
|
||||
messageId: string,
|
||||
content: any,
|
||||
msgType: 'text' | 'interactive' = 'interactive',
|
||||
): Promise<void> {
|
||||
const token = await this.getValidToken(bot);
|
||||
|
||||
const { data } = await axios.patch<{ code: number; msg: string }>(
|
||||
`${this.feishuApiBase}/im/v1/messages/${messageId}`,
|
||||
{ msg_type: msgType, content: JSON.stringify(content) },
|
||||
{ headers: { Authorization: `Bearer ${token}` } },
|
||||
);
|
||||
|
||||
if (data.code !== 0) {
|
||||
this.logger.warn(`Failed to update Feishu message: ${data.msg}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a professional Feishu card
|
||||
*/
|
||||
private buildFeishuCard(
|
||||
content: string,
|
||||
title = 'AuraK AI 助手',
|
||||
isFinal = false,
|
||||
) {
|
||||
return {
|
||||
config: {
|
||||
wide_screen_mode: true,
|
||||
},
|
||||
header: {
|
||||
template: isFinal ? 'blue' : 'orange',
|
||||
title: {
|
||||
content: title + (isFinal ? '' : ' (正在生成...)'),
|
||||
tag: 'plain_text',
|
||||
},
|
||||
},
|
||||
elements: [
|
||||
{
|
||||
tag: 'div',
|
||||
text: {
|
||||
content: content || '...',
|
||||
tag: 'lark_md',
|
||||
},
|
||||
},
|
||||
{
|
||||
tag: 'hr',
|
||||
},
|
||||
{
|
||||
tag: 'note',
|
||||
elements: [
|
||||
{
|
||||
content: `由 AuraK 知识库驱动 · ${new Date().toLocaleTimeString()}`,
|
||||
tag: 'plain_text',
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Chunk 5: ChatService RAG Integration ─────────────────────────────────────
|
||||
|
||||
private processedMessages = new Map<
|
||||
string,
|
||||
{ time: number; responseId?: string }
|
||||
>();
|
||||
|
||||
/**
|
||||
* Check if message is an assessment command
|
||||
*/
|
||||
isAssessmentCommand(message: string): boolean {
|
||||
const trimmed = message.trim().toLowerCase();
|
||||
const commandPrefixes = ['/assessment', '/测评', '/eval', '/测评评估'];
|
||||
return commandPrefixes.some((prefix) =>
|
||||
trimmed.startsWith(prefix.toLowerCase()),
|
||||
);
|
||||
}
|
||||
|
||||
async handleIncomingMessage(
|
||||
bot: FeishuBot,
|
||||
openId: string,
|
||||
messageId: string,
|
||||
userText: string,
|
||||
): Promise<void> {
|
||||
// Strip Feishu AT tags and trim
|
||||
userText = userText.replace(/<at [^>]*><\/at>/g, '').trim();
|
||||
if (!userText) return;
|
||||
|
||||
// 1. Deduplication: check if we are already processing this message
|
||||
const now = Date.now();
|
||||
const existing = this.processedMessages.get(messageId);
|
||||
if (existing && now - existing.time < 1000 * 60 * 10) {
|
||||
this.logger.warn(`Ignoring duplicate Feishu message: ${messageId}`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Mark as being processed
|
||||
this.processedMessages.set(messageId, { time: now });
|
||||
|
||||
// Cleanup old cache (simple)
|
||||
if (this.processedMessages.size > 1000) {
|
||||
for (const [key, val] of this.processedMessages) {
|
||||
if (now - val.time > 1000 * 60 * 30) this.processedMessages.delete(key);
|
||||
}
|
||||
}
|
||||
|
||||
// Get bot owner context
|
||||
const userId = bot.userId;
|
||||
const tenantId = bot.tenantId || 'default';
|
||||
|
||||
// We still fetch the user for language preference, but tenant/user IDs are from bot
|
||||
const user = await this.userService.findOneById(userId);
|
||||
const language = user?.userSetting?.language || 'zh';
|
||||
|
||||
try {
|
||||
// Establish context for all downstream services and TypeORM subscribers
|
||||
await tenantStore.run({ tenantId, userId }, async () => {
|
||||
await i18nStore.run({ language }, async () => {
|
||||
// Check if message is an assessment command
|
||||
if (this.isAssessmentCommand(userText)) {
|
||||
this.logger.log(
|
||||
`Routing assessment command [${messageId}] for bot ${bot.appId}`,
|
||||
);
|
||||
// Delegate to assessment service (will run in the same context)
|
||||
await this.feishuAssessmentService.handleCommand(
|
||||
bot,
|
||||
openId,
|
||||
userText,
|
||||
);
|
||||
} else {
|
||||
// Delegate to standard RAG pipeline (will run in the same context)
|
||||
await this.processChatMessage(
|
||||
bot,
|
||||
openId,
|
||||
messageId,
|
||||
userText,
|
||||
true,
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Message routing failed [${messageId}]: ${error.message}`,
|
||||
error,
|
||||
);
|
||||
try {
|
||||
await this.sendTextMessage(
|
||||
bot,
|
||||
'open_id',
|
||||
openId,
|
||||
'抱歉，处理您的消息时遇到了错误，请稍后重试。',
|
||||
);
|
||||
} catch (sendError) {
|
||||
this.logger.error('Failed to send error message to Feishu', sendError);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a user message via the AuraK RAG pipeline and send the result back.
|
||||
* This is the core of the Feishu integration.
|
||||
*/
|
||||
async processChatMessage(
|
||||
bot: FeishuBot,
|
||||
openId: string,
|
||||
messageId: string,
|
||||
userMessage: string,
|
||||
alreadyDeduplicated = false,
|
||||
): Promise<void> {
|
||||
this.logger.log(
|
||||
`Processing Feishu message [${messageId}] for bot ${bot.appId}`,
|
||||
);
|
||||
const now = Date.now();
|
||||
|
||||
if (!alreadyDeduplicated) {
|
||||
// 1. Deduplication: check if we are already processing this message
|
||||
const now = Date.now();
|
||||
const existing = this.processedMessages.get(messageId);
|
||||
if (existing && now - existing.time < 1000 * 60 * 10) {
|
||||
this.logger.warn(`Ignoring duplicate Feishu message: ${messageId}`);
|
||||
return;
|
||||
}
|
||||
// Mark as being processed
|
||||
this.processedMessages.set(messageId, { time: now });
|
||||
}
|
||||
|
||||
// Cleanup old cache (simple)
|
||||
if (this.processedMessages.size > 1000) {
|
||||
for (const [key, val] of this.processedMessages) {
|
||||
if (now - val.time > 1000 * 60 * 30) this.processedMessages.delete(key);
|
||||
}
|
||||
}
|
||||
|
||||
// Get bot owner context
|
||||
const userId = bot.userId;
|
||||
const tenantId = bot.tenantId || 'default';
|
||||
|
||||
// Fetch user for language preference
|
||||
const user = await this.userService.findOneById(userId);
|
||||
const language = user?.userSetting?.language || 'zh';
|
||||
|
||||
// Get the user's default LLM model
|
||||
const llmModel = await this.modelConfigService.findDefaultByType(
|
||||
tenantId,
|
||||
ModelType.LLM,
|
||||
);
|
||||
|
||||
if (!llmModel) {
|
||||
await this.sendTextMessage(
|
||||
bot,
|
||||
'open_id',
|
||||
openId,
|
||||
'❌ 请先在 AuraK 中配置 LLM 模型才能使用机器人。',
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Send initial "thinking" card
|
||||
const cardTitle = 'AuraK 知识检索';
|
||||
const initialCard = this.buildFeishuCard(
|
||||
'⏳ 正在检索知识库，请稍候...',
|
||||
cardTitle,
|
||||
false,
|
||||
);
|
||||
const msgId = await this.sendCardMessage(
|
||||
bot,
|
||||
'open_id',
|
||||
openId,
|
||||
initialCard,
|
||||
);
|
||||
|
||||
// Save the response message ID for potential future deduplication debugging
|
||||
this.processedMessages.set(messageId, { time: now, responseId: msgId });
|
||||
|
||||
// Run the RAG pipeline in the background so we don't block the Feishu event handler
|
||||
// This prevents Feishu from retrying the event if it takes > 3s.
|
||||
|
||||
// Handle knowledge source selection
|
||||
let selectedFiles: string[] | undefined = undefined;
|
||||
let selectedGroups: string[] | undefined = undefined;
|
||||
|
||||
if (bot.knowledgeBaseId) {
|
||||
selectedFiles = [bot.knowledgeBaseId];
|
||||
} else if (bot.knowledgeGroupId) {
|
||||
selectedGroups = [bot.knowledgeGroupId];
|
||||
}
|
||||
|
||||
this._runRagBackground(
|
||||
bot,
|
||||
msgId,
|
||||
userMessage,
|
||||
userId,
|
||||
llmModel,
|
||||
language,
|
||||
tenantId,
|
||||
cardTitle,
|
||||
selectedFiles,
|
||||
selectedGroups,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal background task for RAG processing
|
||||
*/
|
||||
private async _runRagBackground(
|
||||
bot: FeishuBot,
|
||||
msgId: string,
|
||||
userMessage: string,
|
||||
userId: string,
|
||||
llmModel: any,
|
||||
language: string,
|
||||
tenantId: string,
|
||||
cardTitle: string,
|
||||
selectedFiles?: string[],
|
||||
selectedGroups?: string[],
|
||||
) {
|
||||
let fullResponse = '';
|
||||
let lastUpdateTime = Date.now();
|
||||
const UPDATE_INTERVAL = 1500;
|
||||
|
||||
try {
|
||||
// Stream from ChatService RAG pipeline
|
||||
const stream = this.chatService.streamChat(
|
||||
userMessage,
|
||||
[],
|
||||
userId,
|
||||
llmModel,
|
||||
language,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
false,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
10,
|
||||
0.7,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
tenantId,
|
||||
);
|
||||
|
||||
for await (const chunk of stream) {
|
||||
if (chunk.type === 'content') {
|
||||
fullResponse += chunk.data;
|
||||
|
||||
const now = Date.now();
|
||||
if (
|
||||
now - lastUpdateTime > UPDATE_INTERVAL &&
|
||||
fullResponse.length > 50
|
||||
) {
|
||||
const loadingCard = this.buildFeishuCard(
|
||||
fullResponse,
|
||||
cardTitle,
|
||||
false,
|
||||
);
|
||||
await this.updateMessage(bot, msgId, loadingCard);
|
||||
lastUpdateTime = now;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
this.logger.error('RAG stream error for Feishu message', err);
|
||||
fullResponse = `抱歉，处理您的问题时遇到了错误：${err?.message || '未知错误'}。`;
|
||||
}
|
||||
|
||||
const MAX_LENGTH = 4500;
|
||||
const finalContent =
|
||||
fullResponse.length > MAX_LENGTH
|
||||
? fullResponse.substring(0, MAX_LENGTH) + '\n\n...(内容过长，已截断)'
|
||||
: fullResponse || '抱歉，未能生成有效回复，请稍后再试。';
|
||||
|
||||
const finalCard = this.buildFeishuCard(finalContent, cardTitle, true);
|
||||
await this.updateMessage(bot, msgId, finalCard);
|
||||
}
|
||||
|
||||
// ─── WebSocket Connection Management ─────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Start WebSocket connection for a bot
|
||||
*/
|
||||
async startWsConnection(botId: string): Promise<void> {
|
||||
const bot = await this.getBotById(botId);
|
||||
if (!bot) throw new Error('Bot not found');
|
||||
if (!bot.enabled) throw new Error('Bot is disabled');
|
||||
|
||||
bot.useWebSocket = true;
|
||||
await this.botRepository.save(bot);
|
||||
|
||||
await this.wsManager.connect(bot);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop WebSocket connection for a bot
|
||||
*/
|
||||
async stopWsConnection(botId: string): Promise<void> {
|
||||
const bot = await this.getBotById(botId);
|
||||
if (!bot) throw new Error('Bot not found');
|
||||
|
||||
bot.useWebSocket = false;
|
||||
await this.botRepository.save(bot);
|
||||
|
||||
await this.wsManager.disconnect(botId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get WebSocket connection status for a specific bot
|
||||
*/
|
||||
async getWsStatus(botId: string): Promise<ConnectionStatus | null> {
|
||||
return this.wsManager.getStatus(botId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all WebSocket connection statuses
|
||||
*/
|
||||
async getAllWsStatuses(): Promise<ConnectionStatus[]> {
|
||||
return this.wsManager.getAllStatuses();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
import { AssessmentCommandParser } from './assessment-command.parser';
|
||||
import { AssessmentCommandType } from '../dto/assessment-command.dto';
|
||||
|
||||
describe('AssessmentCommandParser', () => {
|
||||
let parser: AssessmentCommandParser;
|
||||
|
||||
beforeEach(() => {
|
||||
parser = new AssessmentCommandParser();
|
||||
});
|
||||
|
||||
describe('parse', () => {
|
||||
it('should parse start command without parameters', () => {
|
||||
const result = parser.parse('/assessment start');
|
||||
expect(result).toMatchObject({
|
||||
type: AssessmentCommandType.START,
|
||||
parameters: [],
|
||||
});
|
||||
});
|
||||
|
||||
it('should parse start command with parameters', () => {
|
||||
const result = parser.parse('/assessment start kb_123');
|
||||
expect(result).toMatchObject({
|
||||
type: AssessmentCommandType.START,
|
||||
parameters: ['kb_123'],
|
||||
});
|
||||
});
|
||||
|
||||
it('should parse answer command', () => {
|
||||
const result = parser.parse('/assessment answer my answer');
|
||||
expect(result).toMatchObject({
|
||||
type: AssessmentCommandType.ANSWER,
|
||||
parameters: ['my', 'answer'],
|
||||
});
|
||||
});
|
||||
|
||||
it('should parse Chinese commands', () => {
|
||||
const result = parser.parse('/测评 开始');
|
||||
expect(result).toMatchObject({
|
||||
type: AssessmentCommandType.START,
|
||||
parameters: [],
|
||||
});
|
||||
});
|
||||
|
||||
it('should return null for non-assessment commands', () => {
|
||||
const result = parser.parse('hello world');
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('isAssessmentCommand', () => {
|
||||
it('should return true for valid commands', () => {
|
||||
expect(parser.isAssessmentCommand('/assessment status')).toBe(true);
|
||||
expect(parser.isAssessmentCommand('/测评 状态')).toBe(true);
|
||||
});
|
||||
|
||||
it('should return false for invalid commands', () => {
|
||||
expect(parser.isAssessmentCommand('status')).toBe(false);
|
||||
expect(parser.isAssessmentCommand('/status')).toBe(false);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,135 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import {
|
||||
AssessmentCommand,
|
||||
AssessmentCommandType,
|
||||
AssessmentCommandDto,
|
||||
} from '../dto/assessment-command.dto';
|
||||
|
||||
@Injectable()
|
||||
export class AssessmentCommandParser {
|
||||
private readonly logger = new Logger(AssessmentCommandParser.name);
|
||||
|
||||
// 支持的命令前缀
|
||||
private readonly commandPrefixes = [
|
||||
'/assessment',
|
||||
'/测评',
|
||||
'/eval',
|
||||
'/测评评估',
|
||||
];
|
||||
|
||||
// 命令映射
|
||||
private readonly commandMap: Record<string, AssessmentCommandType> = {
|
||||
start: AssessmentCommandType.START,
|
||||
开始: AssessmentCommandType.START,
|
||||
answer: AssessmentCommandType.ANSWER,
|
||||
回答: AssessmentCommandType.ANSWER,
|
||||
status: AssessmentCommandType.STATUS,
|
||||
状态: AssessmentCommandType.STATUS,
|
||||
result: AssessmentCommandType.RESULT,
|
||||
结果: AssessmentCommandType.RESULT,
|
||||
help: AssessmentCommandType.HELP,
|
||||
帮助: AssessmentCommandType.HELP,
|
||||
cancel: AssessmentCommandType.CANCEL,
|
||||
取消: AssessmentCommandType.CANCEL,
|
||||
};
|
||||
|
||||
/**
|
||||
* 解析消息是否为测评命令
|
||||
*/
|
||||
parse(message: string): AssessmentCommand | null {
|
||||
const trimmed = message.trim();
|
||||
|
||||
// 检查是否是测评命令
|
||||
const isCommand = this.commandPrefixes.some((prefix) =>
|
||||
trimmed.toLowerCase().startsWith(prefix.toLowerCase()),
|
||||
);
|
||||
|
||||
if (!isCommand) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
// 解析命令
|
||||
const parts = trimmed.split(/\s+/);
|
||||
const commandTypeStr = parts[1]?.toLowerCase();
|
||||
|
||||
// 查找命令类型
|
||||
const commandType = this.commandMap[commandTypeStr];
|
||||
|
||||
if (!commandType) {
|
||||
// 未知命令，返回帮助
|
||||
return {
|
||||
type: AssessmentCommandType.HELP,
|
||||
parameters: [],
|
||||
rawMessage: message,
|
||||
timestamp: new Date(),
|
||||
};
|
||||
}
|
||||
|
||||
// 获取参数（跳过命令前缀和命令类型）
|
||||
const parameters = parts.slice(2);
|
||||
|
||||
return {
|
||||
type: commandType,
|
||||
parameters,
|
||||
rawMessage: message,
|
||||
timestamp: new Date(),
|
||||
};
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Failed to parse assessment command: ${error.message}`,
|
||||
error,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查消息是否为测评命令
|
||||
*/
|
||||
isAssessmentCommand(message: string): boolean {
|
||||
const trimmed = message.trim().toLowerCase();
|
||||
return this.commandPrefixes.some((prefix) =>
|
||||
trimmed.startsWith(prefix.toLowerCase()),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取命令帮助文本
|
||||
*/
|
||||
getHelpText(language: string = 'zh'): string {
|
||||
if (language === 'zh') {
|
||||
return `
|
||||
**人才测评机器人帮助**
|
||||
|
||||
命令格式：
|
||||
- \`/assessment start [templateId]\` - 开始测评
|
||||
- \`/assessment answer [answer]\` - 提交答案
|
||||
- \`/assessment status\` - 查看测评状态
|
||||
- \`/assessment result\` - 获取测评结果
|
||||
- \`/assessment help\` - 显示帮助
|
||||
- \`/assessment cancel\` - 取消测评
|
||||
|
||||
说明：
|
||||
- 如果未指定知识库/模板，将使用机器人配置的默认知识库
|
||||
- 也可直接回复答案，无需命令前缀
|
||||
`.trim();
|
||||
} else {
|
||||
return `
|
||||
**Assessment Bot Help**
|
||||
|
||||
Commands:
|
||||
- \`/assessment start [templateId]\` - Start assessment
|
||||
- \`/assessment answer [answer]\` - Submit answer
|
||||
- \`/assessment status\` - Check assessment status
|
||||
- \`/assessment result\` - Get assessment results
|
||||
- \`/assessment help\` - Show help
|
||||
- \`/assessment cancel\` - Cancel assessment
|
||||
|
||||
Note:
|
||||
- If no knowledge base/template is specified, the bot's default knowledge base will be used
|
||||
- You can also reply directly with your answer without command prefix
|
||||
`.trim();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,619 @@
|
||||
import { Injectable, Logger, Inject, forwardRef } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Repository } from 'typeorm';
|
||||
import { FeishuBot } from '../entities/feishu-bot.entity';
|
||||
import { FeishuAssessmentSession } from '../entities/feishu-assessment-session.entity';
|
||||
import { FeishuService } from '../feishu.service';
|
||||
import { AssessmentService } from '../../assessment/assessment.service';
|
||||
import { AssessmentCommandParser } from './assessment-command.parser';
|
||||
import { AssessmentCommandType } from '../dto/assessment-command.dto';
|
||||
import { i18nStore } from '../../i18n/i18n.store';
|
||||
import { DEFAULT_LANGUAGE } from '../../common/constants';
|
||||
|
||||
@Injectable()
|
||||
export class FeishuAssessmentService {
|
||||
private readonly logger = new Logger(FeishuAssessmentService.name);
|
||||
|
||||
constructor(
|
||||
@InjectRepository(FeishuAssessmentSession)
|
||||
private sessionRepository: Repository<FeishuAssessmentSession>,
|
||||
@Inject(forwardRef(() => AssessmentService))
|
||||
private assessmentService: AssessmentService,
|
||||
@Inject(forwardRef(() => FeishuService))
|
||||
private feishuService: FeishuService,
|
||||
private commandParser: AssessmentCommandParser,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* 处理测评命令
|
||||
*/
|
||||
async handleCommand(
|
||||
bot: FeishuBot,
|
||||
openId: string,
|
||||
message: string,
|
||||
): Promise<void> {
|
||||
// Ensure bot user relation is loaded (might be missing if fetched from an old cache or WS connection)
|
||||
if (!bot.user) {
|
||||
this.logger.log(`Reloading bot ${bot.id} to fetch user relation`);
|
||||
const loadedBot = await this.feishuService.getBotById(bot.id);
|
||||
if (loadedBot) {
|
||||
bot = loadedBot;
|
||||
}
|
||||
}
|
||||
|
||||
const command = this.commandParser.parse(message);
|
||||
|
||||
if (!command) {
|
||||
// 不是测评命令，使用默认聊天处理
|
||||
await this.feishuService.processChatMessage(bot, openId, '', message);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
switch (command.type) {
|
||||
case AssessmentCommandType.START:
|
||||
await this.startAssessment(bot, openId, command.parameters);
|
||||
break;
|
||||
case AssessmentCommandType.ANSWER:
|
||||
await this.submitAnswer(bot, openId, command.parameters.join(' '));
|
||||
break;
|
||||
case AssessmentCommandType.STATUS:
|
||||
await this.getStatus(bot, openId);
|
||||
break;
|
||||
case AssessmentCommandType.RESULT:
|
||||
await this.getResult(bot, openId);
|
||||
break;
|
||||
case AssessmentCommandType.HELP:
|
||||
await this.sendHelp(bot, openId);
|
||||
break;
|
||||
case AssessmentCommandType.CANCEL:
|
||||
await this.cancelAssessment(bot, openId);
|
||||
break;
|
||||
default:
|
||||
await this.sendHelp(bot, openId);
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Failed to handle assessment command: ${error.message}`,
|
||||
error,
|
||||
);
|
||||
await this.feishuService.sendTextMessage(
|
||||
bot,
|
||||
'open_id',
|
||||
openId,
|
||||
`处理测评命令时出错: ${error.message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 开始测评
|
||||
*/
|
||||
async startAssessment(
|
||||
bot: FeishuBot,
|
||||
openId: string,
|
||||
parameters: string[],
|
||||
): Promise<void> {
|
||||
// 检查是否已有进行中的测评
|
||||
const existingSession = await this.getActiveSession(bot.id, openId);
|
||||
if (existingSession) {
|
||||
await this.feishuService.sendTextMessage(
|
||||
bot,
|
||||
'open_id',
|
||||
openId,
|
||||
'您已有进行中的测评会话，请先完成当前测评或发送 /assessment cancel 取消。',
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// 解析参数
|
||||
const [kbIdOrTemplateId] = parameters;
|
||||
let knowledgeBaseId: string | undefined;
|
||||
let templateId: string | undefined;
|
||||
|
||||
// 统一作为模板ID处理，因为模板包含了更完整的测评配置
|
||||
// 且知识库ID也多为UUID，按长度判断不准确
|
||||
if (kbIdOrTemplateId) {
|
||||
templateId = kbIdOrTemplateId;
|
||||
}
|
||||
|
||||
// 使用机器人配置的知识库或知识组（如果未指定）
|
||||
if (!knowledgeBaseId && !templateId) {
|
||||
if (bot.knowledgeBaseId) {
|
||||
knowledgeBaseId = bot.knowledgeBaseId;
|
||||
} else if (bot.knowledgeGroupId) {
|
||||
knowledgeBaseId = bot.knowledgeGroupId;
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`Starting assessment: bot=${bot.id}, openId=${openId}, kb=${knowledgeBaseId}, template=${templateId}`,
|
||||
);
|
||||
|
||||
// 发送"正在创建"消息
|
||||
await this.feishuService.sendTextMessage(
|
||||
bot,
|
||||
'open_id',
|
||||
openId,
|
||||
'⏳ 正在创建测评会话，请稍候...',
|
||||
);
|
||||
|
||||
try {
|
||||
const language = i18nStore.getStore()?.language || 'zh';
|
||||
const session = await this.assessmentService.startSession(
|
||||
bot.userId,
|
||||
knowledgeBaseId,
|
||||
bot.tenantId || 'default',
|
||||
language,
|
||||
templateId,
|
||||
);
|
||||
|
||||
// 触发问题生成（startSession 仅创建会话，getSessionState 会触发 agent 生成问题）
|
||||
this.logger.log(
|
||||
`Triggering question generation for session ${session.id}`,
|
||||
);
|
||||
const state = await this.assessmentService.getSessionState(
|
||||
session.id,
|
||||
bot.userId,
|
||||
);
|
||||
const questions = state.questions || [];
|
||||
|
||||
// 存储飞书会话关联
|
||||
const feishuSession = this.sessionRepository.create({
|
||||
botId: bot.id,
|
||||
openId,
|
||||
assessmentSessionId: session.id,
|
||||
status: 'active',
|
||||
currentQuestionIndex: 0,
|
||||
});
|
||||
await this.sessionRepository.save(feishuSession);
|
||||
|
||||
// 发送第一个问题
|
||||
if (questions && questions.length > 0) {
|
||||
const firstQuestion = questions[0];
|
||||
const totalQuestions = state.questionCount || questions.length;
|
||||
const card = this.buildQuestionCard(
|
||||
firstQuestion,
|
||||
session.id,
|
||||
1,
|
||||
totalQuestions,
|
||||
);
|
||||
await this.feishuService.sendCardMessage(bot, 'open_id', openId, card);
|
||||
} else {
|
||||
await this.feishuService.sendTextMessage(
|
||||
bot,
|
||||
'open_id',
|
||||
openId,
|
||||
'测评会话已创建，但未能生成问题。',
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to start assessment: ${error.message}`, error);
|
||||
await this.feishuService.sendTextMessage(
|
||||
bot,
|
||||
'open_id',
|
||||
openId,
|
||||
`创建测评会话失败: ${error.message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 提交答案
|
||||
*/
|
||||
async submitAnswer(
|
||||
bot: FeishuBot,
|
||||
openId: string,
|
||||
answer: string,
|
||||
): Promise<void> {
|
||||
const session = await this.getActiveSession(bot.id, openId);
|
||||
|
||||
if (!session) {
|
||||
await this.feishuService.sendTextMessage(
|
||||
bot,
|
||||
'open_id',
|
||||
openId,
|
||||
'没有进行中的测评会话。请发送 /assessment start 开始测评。',
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!answer || answer.trim() === '') {
|
||||
await this.feishuService.sendTextMessage(
|
||||
bot,
|
||||
'open_id',
|
||||
openId,
|
||||
'请提供答案。',
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`Submitting answer for session ${session.assessmentSessionId}`,
|
||||
);
|
||||
|
||||
// 发送"正在评估"消息
|
||||
await this.feishuService.sendTextMessage(
|
||||
bot,
|
||||
'open_id',
|
||||
openId,
|
||||
'⏳ 正在评估答案...',
|
||||
);
|
||||
|
||||
try {
|
||||
const language = i18nStore.getStore()?.language || 'zh';
|
||||
const result = await this.assessmentService.submitAnswer(
|
||||
session.assessmentSessionId,
|
||||
bot.userId,
|
||||
answer,
|
||||
language,
|
||||
);
|
||||
|
||||
this.logger.log(
|
||||
`Assessment result for session ${session.assessmentSessionId}: score=${result.finalScore}, hasReport=${!!result.report}, questionsLen=${result.questions?.length}, questionsJsonLen=${result.questions_json?.length}`,
|
||||
);
|
||||
this.logger.debug(`Result keys: ${Object.keys(result).join(', ')}`);
|
||||
if (result.report) {
|
||||
this.logger.debug(
|
||||
`Result report snippet: ${result.report.substring(0, 100)}...`,
|
||||
);
|
||||
}
|
||||
|
||||
// 更新会话状态
|
||||
session.currentQuestionIndex = result.currentQuestionIndex || 0;
|
||||
|
||||
// 检查是否完成
|
||||
if (result.report) {
|
||||
session.status = 'completed';
|
||||
await this.sessionRepository.save(session);
|
||||
|
||||
// 发送测评结果
|
||||
await this.sendAssessmentResult(bot, openId, result);
|
||||
} else if (result.questions && result.questions.length > 0) {
|
||||
// 更新并保存会话
|
||||
await this.sessionRepository.save(session);
|
||||
|
||||
// 发送下一个问题
|
||||
const currentQuestionIndex = result.currentQuestionIndex || 0;
|
||||
const nextQuestion = result.questions[currentQuestionIndex];
|
||||
const totalQuestions = result.questionCount || result.questions.length;
|
||||
|
||||
const card = this.buildQuestionCard(
|
||||
nextQuestion,
|
||||
session.assessmentSessionId,
|
||||
currentQuestionIndex + 1,
|
||||
totalQuestions,
|
||||
);
|
||||
await this.feishuService.sendCardMessage(bot, 'open_id', openId, card);
|
||||
} else if (result.questions_json && result.questions_json.length > 0) {
|
||||
// 有些版本返回 questions_json
|
||||
await this.sessionRepository.save(session);
|
||||
const currentQuestionIndex = result.currentQuestionIndex || 0;
|
||||
const nextQuestion = result.questions_json[currentQuestionIndex];
|
||||
const totalQuestions =
|
||||
result.questionCount || result.questions_json.length;
|
||||
|
||||
const card = this.buildQuestionCard(
|
||||
nextQuestion,
|
||||
session.assessmentSessionId,
|
||||
currentQuestionIndex + 1,
|
||||
totalQuestions,
|
||||
);
|
||||
await this.feishuService.sendCardMessage(bot, 'open_id', openId, card);
|
||||
} else {
|
||||
// 没有更多问题，完成测评
|
||||
session.status = 'completed';
|
||||
await this.sessionRepository.save(session);
|
||||
await this.sendAssessmentResult(bot, openId, result);
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to submit answer: ${error.message}`, error);
|
||||
await this.feishuService.sendTextMessage(
|
||||
bot,
|
||||
'open_id',
|
||||
openId,
|
||||
`提交答案失败: ${error.message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取测评状态
|
||||
*/
|
||||
async getStatus(bot: FeishuBot, openId: string): Promise<void> {
|
||||
const session = await this.getActiveSession(bot.id, openId);
|
||||
|
||||
if (!session) {
|
||||
await this.feishuService.sendTextMessage(
|
||||
bot,
|
||||
'open_id',
|
||||
openId,
|
||||
'没有进行中的测评会话。',
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const assessmentState = await this.assessmentService.getSessionState(
|
||||
session.assessmentSessionId,
|
||||
bot.userId,
|
||||
);
|
||||
|
||||
const currentQuestionIndex = assessmentState.currentQuestionIndex || 0;
|
||||
const totalQuestions = assessmentState.questions?.length || 0;
|
||||
|
||||
const message =
|
||||
`测评状态:\n` +
|
||||
`- 进度: ${currentQuestionIndex + 1}/${totalQuestions}\n` +
|
||||
`- 状态: ${session.status === 'active' ? '进行中' : '已完成'}\n` +
|
||||
`- 开始时间: ${session.createdAt.toLocaleString('zh-CN')}`;
|
||||
|
||||
await this.feishuService.sendTextMessage(bot, 'open_id', openId, message);
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to get status: ${error.message}`, error);
|
||||
await this.feishuService.sendTextMessage(
|
||||
bot,
|
||||
'open_id',
|
||||
openId,
|
||||
`获取状态失败: ${error.message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取测评结果
|
||||
*/
|
||||
async getResult(bot: FeishuBot, openId: string): Promise<void> {
|
||||
const session = await this.getActiveSession(bot.id, openId);
|
||||
|
||||
if (!session) {
|
||||
await this.feishuService.sendTextMessage(
|
||||
bot,
|
||||
'open_id',
|
||||
openId,
|
||||
'没有进行中的测评会话。',
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (session.status !== 'completed') {
|
||||
await this.feishuService.sendTextMessage(
|
||||
bot,
|
||||
'open_id',
|
||||
openId,
|
||||
'测评尚未完成，请先完成所有问题。',
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const assessmentState = await this.assessmentService.getSessionState(
|
||||
session.assessmentSessionId,
|
||||
bot.userId,
|
||||
);
|
||||
|
||||
await this.sendAssessmentResult(bot, openId, assessmentState);
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to get result: ${error.message}`, error);
|
||||
await this.feishuService.sendTextMessage(
|
||||
bot,
|
||||
'open_id',
|
||||
openId,
|
||||
`获取结果失败: ${error.message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 取消测评
|
||||
*/
|
||||
async cancelAssessment(bot: FeishuBot, openId: string): Promise<void> {
|
||||
const session = await this.getActiveSession(bot.id, openId);
|
||||
|
||||
if (!session) {
|
||||
await this.feishuService.sendTextMessage(
|
||||
bot,
|
||||
'open_id',
|
||||
openId,
|
||||
'没有进行中的测评会话。',
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
session.status = 'cancelled';
|
||||
await this.sessionRepository.save(session);
|
||||
|
||||
await this.feishuService.sendTextMessage(
|
||||
bot,
|
||||
'open_id',
|
||||
openId,
|
||||
'测评会话已取消。发送 /assessment start 开始新的测评。',
|
||||
);
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to cancel assessment: ${error.message}`, error);
|
||||
await this.feishuService.sendTextMessage(
|
||||
bot,
|
||||
'open_id',
|
||||
openId,
|
||||
`取消测评失败: ${error.message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 发送帮助信息
|
||||
*/
|
||||
async sendHelp(bot: FeishuBot, openId: string): Promise<void> {
|
||||
const language = i18nStore.getStore()?.language || 'zh';
|
||||
const helpText = this.commandParser.getHelpText(language);
|
||||
await this.feishuService.sendTextMessage(bot, 'open_id', openId, helpText);
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取活跃会话
|
||||
*/
|
||||
private async getActiveSession(
|
||||
botId: string,
|
||||
openId: string,
|
||||
): Promise<FeishuAssessmentSession | null> {
|
||||
return this.sessionRepository.findOne({
|
||||
where: {
|
||||
botId,
|
||||
openId,
|
||||
status: 'active',
|
||||
},
|
||||
order: { createdAt: 'DESC' },
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建问题卡片
|
||||
*/
|
||||
private buildQuestionCard(
|
||||
question: any,
|
||||
sessionId: string,
|
||||
currentIndex: number,
|
||||
totalQuestions: number,
|
||||
): any {
|
||||
const difficultyColors: Record<string, string> = {
|
||||
简单: 'green',
|
||||
普通: 'blue',
|
||||
困难: 'orange',
|
||||
专家: 'red',
|
||||
Easy: 'green',
|
||||
Medium: 'blue',
|
||||
Hard: 'orange',
|
||||
Advanced: 'orange',
|
||||
Expert: 'red',
|
||||
Specialist: 'red',
|
||||
};
|
||||
|
||||
const difficulty = question.difficulty || '普通';
|
||||
const headerColor = difficultyColors[difficulty] || 'blue';
|
||||
|
||||
return {
|
||||
config: { wide_screen_mode: true },
|
||||
header: {
|
||||
template: headerColor,
|
||||
title: {
|
||||
content: `人才测评 (${currentIndex}/${totalQuestions})`,
|
||||
tag: 'plain_text',
|
||||
},
|
||||
},
|
||||
elements: [
|
||||
{
|
||||
tag: 'div',
|
||||
text: {
|
||||
content: `**问题 ${currentIndex}:** ${question.questionText || question.text || question.content || '无问题内容'}`,
|
||||
tag: 'lark_md',
|
||||
},
|
||||
},
|
||||
...(question.options
|
||||
? [
|
||||
{
|
||||
tag: 'div',
|
||||
text: {
|
||||
content: `选项:\n${question.options
|
||||
.map(
|
||||
(opt: string, i: number) =>
|
||||
`${String.fromCharCode(65 + i)}. ${opt}`,
|
||||
)
|
||||
.join('\n')}`,
|
||||
tag: 'lark_md',
|
||||
},
|
||||
},
|
||||
]
|
||||
: []),
|
||||
{
|
||||
tag: 'div',
|
||||
text: {
|
||||
content: `难度: ${difficulty} | 分值: ${question.score || 1}`,
|
||||
tag: 'lark_md',
|
||||
},
|
||||
},
|
||||
{
|
||||
tag: 'hr',
|
||||
},
|
||||
{
|
||||
tag: 'note',
|
||||
elements: [
|
||||
{
|
||||
content: `直接回复答案或使用 /assessment answer [你的答案]`,
|
||||
tag: 'plain_text',
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 发送测评结果
|
||||
*/
|
||||
private async sendAssessmentResult(
|
||||
bot: FeishuBot,
|
||||
openId: string,
|
||||
result: any,
|
||||
): Promise<void> {
|
||||
const report = result.report || result.finalReport;
|
||||
const score = result.finalScore || result.score;
|
||||
|
||||
const resultCard = {
|
||||
config: { wide_screen_mode: true },
|
||||
header: {
|
||||
template: 'green',
|
||||
title: {
|
||||
content: '测评完成',
|
||||
tag: 'plain_text',
|
||||
},
|
||||
},
|
||||
elements: [
|
||||
{
|
||||
tag: 'div',
|
||||
text: {
|
||||
content: `**测评结果**`,
|
||||
tag: 'lark_md',
|
||||
},
|
||||
},
|
||||
...(score !== undefined
|
||||
? [
|
||||
{
|
||||
tag: 'div',
|
||||
text: {
|
||||
content: `**总分**: ${score.toFixed(1)}`,
|
||||
tag: 'lark_md',
|
||||
},
|
||||
},
|
||||
]
|
||||
: []),
|
||||
{
|
||||
tag: 'div',
|
||||
text: {
|
||||
content: `**报告**:\n${report && report.trim().length > 0 ? report : '未生成详细报告。'}`,
|
||||
tag: 'lark_md',
|
||||
},
|
||||
},
|
||||
{
|
||||
tag: 'hr',
|
||||
},
|
||||
{
|
||||
tag: 'note',
|
||||
elements: [
|
||||
{
|
||||
content: `发送 /assessment start 开始新的测评`,
|
||||
tag: 'plain_text',
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
await this.feishuService.sendCardMessage(
|
||||
bot,
|
||||
'open_id',
|
||||
openId,
|
||||
resultCard,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
import {
|
||||
Injectable,
|
||||
NestInterceptor,
|
||||
ExecutionContext,
|
||||
CallHandler,
|
||||
} from '@nestjs/common';
|
||||
import { Observable } from 'rxjs';
|
||||
import { i18nStore } from './i18n.store';
|
||||
import { DEFAULT_LANGUAGE } from '../common/constants';
|
||||
|
||||
@Injectable()
|
||||
export class I18nInterceptor implements NestInterceptor {
|
||||
intercept(context: ExecutionContext, next: CallHandler): Observable<any> {
|
||||
const request = context.switchToHttp().getRequest();
|
||||
const language = request.headers['x-user-language'] || DEFAULT_LANGUAGE;
|
||||
|
||||
return new Observable((observer) => {
|
||||
i18nStore.run({ language: String(language) }, () => {
|
||||
next.handle().subscribe({
|
||||
next: (value) => observer.next(value),
|
||||
error: (err) => observer.error(err),
|
||||
complete: () => observer.complete(),
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
import { Injectable, NestMiddleware } from '@nestjs/common';
|
||||
import { Request, Response, NextFunction } from 'express';
|
||||
import { i18nStore } from './i18n.store';
|
||||
import { DEFAULT_LANGUAGE } from '../common/constants';
|
||||
|
||||
@Injectable()
|
||||
export class I18nMiddleware implements NestMiddleware {
|
||||
use(req: Request, res: Response, next: NextFunction) {
|
||||
const language = req.headers['x-user-language'] || DEFAULT_LANGUAGE;
|
||||
i18nStore.run({ language: String(language) }, () => {
|
||||
next();
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
import { Global, Module } from '@nestjs/common';
|
||||
import { I18nService } from './i18n.service';
|
||||
|
||||
@Global()
|
||||
@Module({
|
||||
providers: [I18nService],
|
||||
exports: [I18nService],
|
||||
})
|
||||
export class I18nModule {}
|
||||
@@ -0,0 +1,347 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { errorMessages, logMessages, statusMessages } from './messages';
|
||||
import { i18nStore } from './i18n.store';
|
||||
import { DEFAULT_LANGUAGE } from '../common/constants'; // 使用常量定义的默认语言
|
||||
|
||||
@Injectable()
|
||||
export class I18nService {
|
||||
private readonly defaultLanguage = DEFAULT_LANGUAGE; // 使用常量定义的默认语言
|
||||
|
||||
public normalizeLanguage(lang?: string): string {
|
||||
let language = lang;
|
||||
if (!language) {
|
||||
const store = i18nStore.getStore();
|
||||
language = store?.language;
|
||||
}
|
||||
|
||||
if (!language) return this.defaultLanguage;
|
||||
|
||||
// Normalize language codes (e.g., zh-CN -> zh, en-US -> en)
|
||||
const normalized = language.split('-')[0].toLowerCase();
|
||||
return normalized;
|
||||
}
|
||||
|
||||
getErrorMessage(key: string, language?: string): string {
|
||||
const lang = this.normalizeLanguage(language);
|
||||
return (
|
||||
errorMessages[lang]?.[key] ||
|
||||
errorMessages[this.defaultLanguage][key] ||
|
||||
key
|
||||
);
|
||||
}
|
||||
|
||||
getLogMessage(key: string, language?: string): string {
|
||||
const lang = this.normalizeLanguage(language);
|
||||
return (
|
||||
logMessages[lang]?.[key] || logMessages[this.defaultLanguage][key] || key
|
||||
);
|
||||
}
|
||||
|
||||
getStatusMessage(key: string, language?: string): string {
|
||||
const lang = this.normalizeLanguage(language);
|
||||
return (
|
||||
statusMessages[lang]?.[key] ||
|
||||
statusMessages[this.defaultLanguage][key] ||
|
||||
key
|
||||
);
|
||||
}
|
||||
|
||||
// 汎用メッセージ取得メソッド、順次検索
|
||||
getMessage(key: string, language?: string): string {
|
||||
const lang = this.normalizeLanguage(language);
|
||||
// ステータスメッセージ、エラーメッセージ、ログメッセージの順に検索
|
||||
return (
|
||||
statusMessages[lang]?.[key] ||
|
||||
statusMessages[this.defaultLanguage][key] ||
|
||||
errorMessages[lang]?.[key] ||
|
||||
errorMessages[this.defaultLanguage][key] ||
|
||||
logMessages[lang]?.[key] ||
|
||||
logMessages[this.defaultLanguage][key] ||
|
||||
key
|
||||
);
|
||||
}
|
||||
|
||||
// メッセージの取得とフォーマット
|
||||
formatMessage(
|
||||
key: string,
|
||||
args: Record<string, any>,
|
||||
language?: string,
|
||||
): string {
|
||||
let message = this.getMessage(key, language);
|
||||
for (const [argKey, argValue] of Object.entries(args)) {
|
||||
message = message.replace(
|
||||
new RegExp(`\\{${argKey}\\}`, 'g'),
|
||||
String(argValue),
|
||||
);
|
||||
}
|
||||
return message;
|
||||
}
|
||||
|
||||
// サポートされている言語リストを取得
|
||||
getSupportedLanguages(): string[] {
|
||||
return Object.keys(errorMessages);
|
||||
}
|
||||
|
||||
// 言語がサポートされているか確認
|
||||
isLanguageSupported(language: string): boolean {
|
||||
return this.getSupportedLanguages().includes(language);
|
||||
}
|
||||
|
||||
// システムプロンプトを取得
|
||||
getPrompt(
|
||||
lang: string = this.defaultLanguage,
|
||||
type: 'withContext' | 'withoutContext' = 'withContext',
|
||||
hasKnowledgeGroup: boolean = false,
|
||||
): string {
|
||||
const language = this.normalizeLanguage(lang);
|
||||
const noMatchMsg =
|
||||
statusMessages[language]?.noMatchInKnowledgeGroup ||
|
||||
statusMessages[this.defaultLanguage].noMatchInKnowledgeGroup;
|
||||
|
||||
if (language === 'zh') {
|
||||
return type === 'withContext'
|
||||
? `
|
||||
基于以下知识库内容回答用户问题。
|
||||
${
|
||||
hasKnowledgeGroup
|
||||
? `
|
||||
**重要提示**: 用户已选择特定知识组，请严格基于以下知识库内容回答。如果知识库中没有相关信息，请明确告知用户："${noMatchMsg}"，然后再提供答案。
|
||||
`
|
||||
: ''
|
||||
}
|
||||
知识库内容：
|
||||
{context}
|
||||
|
||||
历史对话：
|
||||
{history}
|
||||
|
||||
用户问题：{question}
|
||||
|
||||
请用Chinese回答，并严格遵循以下 Markdown 格式要求：
|
||||
|
||||
1. **段落与结构**：
|
||||
- 使用清晰的段落分隔，每个要点之间空一行
|
||||
- 使用标题（## 或 ###）组织长回答
|
||||
|
||||
2. **文本格式**：
|
||||
- 使用 **粗体** 强调重要概念和关键词
|
||||
- 使用列表（- 或 1.）组织多个要点
|
||||
- 使用 \`代码\` 标记技术术语、命令、文件名
|
||||
|
||||
3. **代码展示**：
|
||||
- 使用代码块展示代码，并指定语言：
|
||||
\`\`\`python
|
||||
def example():
|
||||
return "示例"
|
||||
\`\`\`
|
||||
- 支持语言：python, javascript, typescript, java, bash, sql 等
|
||||
|
||||
4. **图表与可视化**：
|
||||
- 使用 Mermaid 语法绘制流程图、序列图等：
|
||||
\`\`\`mermaid
|
||||
graph LR
|
||||
A[开始] --> B[处理]
|
||||
B --> C[结束]
|
||||
\`\`\`
|
||||
- 适用场景：流程、架构、状态机、时序图
|
||||
|
||||
5. **其他要求**：
|
||||
- 回答精炼准确
|
||||
- 多步骤操作使用有序列表
|
||||
- 对比类信息建议用表格展示（如果适用）
|
||||
`
|
||||
: `
|
||||
作为智能助手，请回答用户的问题。
|
||||
|
||||
历史对话：
|
||||
{history}
|
||||
|
||||
用户问题：{question}
|
||||
|
||||
请用Chinese回答。
|
||||
`;
|
||||
} else if (language === 'ja') {
|
||||
return type === 'withContext'
|
||||
? `
|
||||
以下のナレッジベースの内容に基づいて、ユーザーの質問に答えてください。
|
||||
${
|
||||
hasKnowledgeGroup
|
||||
? `
|
||||
**重要**: ユーザーが特定のナレッジグループを選択しました。以下のナレッジベースの内容に厳密に基づいて回答してください。関連情報がナレッジベースに見つからない場合は、回答を提供する前に、ユーザーに明示的に「${noMatchMsg}」と伝えてください。
|
||||
`
|
||||
: ''
|
||||
}
|
||||
ナレッジベースの内容：
|
||||
{context}
|
||||
|
||||
会話履歴：
|
||||
{history}
|
||||
|
||||
ユーザーの質問：{question}
|
||||
|
||||
日本語で回答し、以下のMarkdown形式のガイドラインに厳密に従ってください。
|
||||
|
||||
1. **段落と構造**:
|
||||
- 明確な段落区切りを使用し、要点の間に空行を入れます
|
||||
- 見出し（## または ###）を使用して長い回答を整理します
|
||||
|
||||
2. **テキスト形式**:
|
||||
- **太字**を使用して重要な概念やキーワードを強調します
|
||||
- リスト（- または 1.）を使用して複数のポイントを整理します
|
||||
- \`コード\`を使用して技術用語、コマンド、ファイル名をマークします
|
||||
|
||||
3. **コード表示**:
|
||||
- 言語指定のあるコードブロックを使用します：
|
||||
\`\`\`python
|
||||
def example():
|
||||
return "示例"
|
||||
\`\`\`
|
||||
- サポートされている言語：python, javascript, typescript, java, bash, sqlなど
|
||||
|
||||
4. **図とチャート**:
|
||||
- フローチャート、シーケンス図などにMermaid構文を使用します：
|
||||
\`\`\`mermaid
|
||||
graph LR
|
||||
A[開始] --> B[処理]
|
||||
B --> C[終了]
|
||||
\`\`\`
|
||||
- 使用例：プロセスフロー、アーキテクチャ図、状態遷移図、シーケンス図
|
||||
|
||||
5. **その他の要件**:
|
||||
- 回答は簡潔かつ明確にします
|
||||
- マルチステップ プロセスには番号付きリストを使用します
|
||||
- 比較情報には表を使用します（該当する場合）
|
||||
`
|
||||
: `
|
||||
インテリジェントなアシスタントとして、ユーザーの質問に答えてください。
|
||||
|
||||
会話履歴：
|
||||
{history}
|
||||
|
||||
ユーザーの質問：{question}
|
||||
|
||||
日本語で回答してください。
|
||||
`;
|
||||
} else {
|
||||
// Fallback to English for any other language
|
||||
return type === 'withContext'
|
||||
? `
|
||||
Answer the user's question based on the following knowledge base content.
|
||||
${
|
||||
hasKnowledgeGroup
|
||||
? `
|
||||
**IMPORTANT**: The user has selected a specific knowledge group. Please answer strictly based on the knowledge base content below. If the relevant information is not found in the knowledge base, explicitly tell the user: "${noMatchMsg}", before providing an answer.
|
||||
`
|
||||
: ''
|
||||
}
|
||||
Knowledge Base CONTENT:
|
||||
{context}
|
||||
|
||||
Conversation history:
|
||||
{history}
|
||||
|
||||
User question: {question}
|
||||
|
||||
Please answer in English and strictly follow these Markdown formatting guidelines:
|
||||
|
||||
1. **Paragraphs & Structure**:
|
||||
- Use clear paragraph breaks with blank lines between key points
|
||||
- Use headings (## or ###) to organize longer answers
|
||||
|
||||
2. **Text Formatting**:
|
||||
- Use **bold** to emphasize important concepts and keywords
|
||||
- Use lists (- or 1.) to organize multiple points
|
||||
- Use \`code\` to mark technical terms, commands, file names
|
||||
|
||||
3. **Code Display**:
|
||||
- Use code blocks with language specification:
|
||||
\`\`\`python
|
||||
def example():
|
||||
return "example"
|
||||
\`\`\`
|
||||
- Supported languages: python, javascript, typescript, java, bash, sql, etc.
|
||||
|
||||
4. **Diagrams & Charts**:
|
||||
- Use Mermaid syntax for flowcharts, sequence diagrams, etc.:
|
||||
\`\`\`mermaid
|
||||
graph LR
|
||||
A[Start] --> B[Process]
|
||||
B --> C[End]
|
||||
\`\`\`
|
||||
- Use cases: process flows, architecture diagrams, state diagrams, sequence diagrams
|
||||
|
||||
5. **Other Requirements**:
|
||||
- Keep answers concise and clear
|
||||
- Use numbered lists for multi-step processes
|
||||
- Use tables for comparison information (if applicable)
|
||||
`
|
||||
: `
|
||||
As an intelligent assistant, please answer the user's question.
|
||||
|
||||
Conversation history:
|
||||
{history}
|
||||
|
||||
User question: {question}
|
||||
|
||||
Please answer in English.
|
||||
`;
|
||||
}
|
||||
}
|
||||
|
||||
// タイトル生成用のプロンプトを取得
|
||||
getDocumentTitlePrompt(
|
||||
lang: string = this.defaultLanguage,
|
||||
contentSample: string,
|
||||
): string {
|
||||
const language = this.normalizeLanguage(lang);
|
||||
if (language === 'zh') {
|
||||
return `你是一个文档分析师。请阅读以下文本（文档开头部分），并生成一个简炼、专业的标题（不超过50个字符）。
|
||||
只返回标题文本。不要包含任何解释性文字或前导词（如“标题是：”）。
|
||||
语言：Chinese
|
||||
文本内容：
|
||||
${contentSample}`;
|
||||
} else if (language === 'ja') {
|
||||
return `あなたは文書分析の専門家です。以下のテキスト（文書の冒頭部分）を読み、簡潔で専門的なタイトル（50文字以内）を生成してください。
|
||||
タイトルのみを返してください。前置きや説明は不要です。
|
||||
言語：Japanese
|
||||
テキスト：
|
||||
${contentSample}`;
|
||||
} else {
|
||||
return `You are a document analyzer. Read the following text (start of a document) and generate a concise, professional title (max 50 chars).
|
||||
Return ONLY the title text. No preamble like "The title is...".
|
||||
Language: English
|
||||
Text:
|
||||
${contentSample}`;
|
||||
}
|
||||
}
|
||||
|
||||
getChatTitlePrompt(
|
||||
lang: string = this.defaultLanguage,
|
||||
userMessage: string,
|
||||
aiResponse: string,
|
||||
): string {
|
||||
const language = this.normalizeLanguage(lang);
|
||||
if (language === 'zh') {
|
||||
return `根据以下对话片段，生成一个简短、描述性的标题（不超过50个字符），总结讨论的主题。
|
||||
只返回标题文本。不要包含任何前导词。
|
||||
语言：Chinese
|
||||
片段：
|
||||
用户: ${userMessage}
|
||||
助手: ${aiResponse}`;
|
||||
} else if (language === 'ja') {
|
||||
return `以下の会話のスニペットに基づいて、話題を要約する短く説明的なタイトル（50文字以内）を生成してください。
|
||||
タイトルのみを返してください。前置きは不要です。
|
||||
言語：Japanese
|
||||
スニペット：
|
||||
ユーザー: ${userMessage}
|
||||
アシスタント: ${aiResponse}`;
|
||||
} else {
|
||||
return `Based on the following conversation snippet, generate a short, descriptive title (max 50 chars) that summarizes the topic.
|
||||
Return ONLY the title. No preamble.
|
||||
Language: English
|
||||
Snippet:
|
||||
User: ${userMessage}
|
||||
Assistant: ${aiResponse}`;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
import { AsyncLocalStorage } from 'async_hooks';
|
||||
|
||||
export interface I18nContext {
|
||||
language: string;
|
||||
}
|
||||
|
||||
export const i18nStore = new AsyncLocalStorage<I18nContext>();
|
||||
@@ -0,0 +1,727 @@
|
||||
export const errorMessages = {
|
||||
zh: {
|
||||
noEmbeddingModel: '请先在系统设置中配置嵌入模型',
|
||||
searchFailed: '搜索知识库失败，将基于一般知识回答...',
|
||||
invalidApiKey: 'API密钥无效',
|
||||
fileNotFound: '未找到文件',
|
||||
insufficientQuota: '配额不足',
|
||||
modelNotConfigured: '未配置模型',
|
||||
visionModelNotConfigured: '未配置视觉模型',
|
||||
embeddingDimensionMismatch: '嵌入维度不匹配',
|
||||
uploadNoFile: '未上传文件',
|
||||
uploadSizeExceeded: '文件大小超过限制: {size}, 最大允许: {max}',
|
||||
uploadModelRequired: '必须选择嵌入模型',
|
||||
uploadTypeUnsupported: '不支持的文件格式: {type}',
|
||||
chunkOverflow: '切片大小 {size} 超过上限 {max} ({reason})。已自动调整',
|
||||
chunkUnderflow: '切片大小 {size} 小于最小值 {min}。已自动调整',
|
||||
overlapOverflow: '重叠大小 {size} 超过上限 {max}。已自动调整',
|
||||
overlapUnderflow: '重叠大小 {size} 小于最小值 {min}。已自动调整',
|
||||
overlapRatioExceeded:
|
||||
'重叠大小 {size} 超过切片大小的50% ({max})。已自动调整',
|
||||
batchOverflowWarning:
|
||||
'建议切片大小不超过 {safeSize} 以避免批量处理溢出 (当前: {size}, 模型限制的 {percent}%)',
|
||||
estimatedChunkCountExcessive: '预计切片数量过多 ({count})，处理可能较慢',
|
||||
contentAndTitleRequired: '内容和标题为必填项',
|
||||
embeddingModelNotFound: '找不到嵌入模型 {id} 或类型不是 embedding',
|
||||
ocrFailed: '提取文本失败: {message}',
|
||||
noImageUploaded: '未上传图片',
|
||||
adminOnlyViewList: '只有管理员可以查看用户列表',
|
||||
passwordsRequired: '当前密码和新密码不能为空',
|
||||
newPasswordMinLength: '新密码长度不能少于6位',
|
||||
adminOnlyCreateUser: '只有管理员可以创建用户',
|
||||
usernamePasswordRequired: '用户名和密码不能为空',
|
||||
passwordMinLength: '密码长度不能少于6位',
|
||||
adminOnlyUpdateUser: '只有管理员可以更新用户信息',
|
||||
userNotFound: '用户不存在',
|
||||
cannotModifyBuiltinAdmin: '无法修改内置管理员账户',
|
||||
adminOnlyDeleteUser: '只有管理员可以删除用户',
|
||||
cannotDeleteSelf: '不能删除自己的账户',
|
||||
cannotDeleteBuiltinAdmin: '无法删除内置管理员账户',
|
||||
invalidMemberRole: '无效的角色。只允许 USER 和 TENANT_ADMIN',
|
||||
incorrectCredentials: '用户名或密码不正确',
|
||||
incorrectCurrentPassword: '当前密码错误',
|
||||
usernameExists: '用户名已存在',
|
||||
noteNotFound: '找不到笔记: {id}',
|
||||
knowledgeGroupNotFound: '找不到知识组: {id}',
|
||||
accessDeniedNoToken: '访问被拒绝：缺少令牌',
|
||||
invalidToken: '无效的令牌',
|
||||
pdfFileNotFound: '找不到 PDF 文件',
|
||||
pdfFileEmpty: 'PDF 文件为空，转换可能失败',
|
||||
pdfConversionFailed: 'PDF 文件不存在或转换失败',
|
||||
pdfConversionFailedDetail: 'PDF 转换失败（文件 ID: {id}），请稍后重试',
|
||||
pdfPreviewNotSupported: '该文件格式不支持预览',
|
||||
pdfServiceUnavailable: 'PDF 服务不可用: {message}',
|
||||
pageImageNotFound: '找不到页面图像',
|
||||
pdfPageImageFailed: '无法获取 PDF 页面图像',
|
||||
someGroupsNotFound: '部分组不存在',
|
||||
promptRequired: '提示词是必填项',
|
||||
addLLMConfig: '请在系统设置中添加 LLM 模型',
|
||||
visionAnalysisFailed: '视觉分析失败: {message}',
|
||||
visionSystemPrompt:
|
||||
'您是专业的文档分析助手。请分析此文档图像，并按以下要求以 JSON 格式返回：\n\n1. 提取所有可读文本（按阅读顺序，保持段落和格式）\n2. 识别图像/图表/表格（描述内容、含义和作用）\n3. 分析页面布局（仅文本/文本和图像混合/表格/图表等）\n4. 评估分析质量 (0-1)\n\n响应格式：\n{\n "text": "完整的文本内容",\n "images": [\n {"type": "图表类型", "description": "详细描述", "position": 1}\n ],\n "layout": "布局说明",\n "confidence": 0.95\n}',
|
||||
retryMechanismError: '重试机制异常',
|
||||
imageLoadError: '无法读取图像: {message}',
|
||||
groupNotFound: '分组不存在',
|
||||
fileDeleted: '文件删除成功',
|
||||
fileDeletedFromGroup: '文件从分组中移除成功',
|
||||
kbCleared: '知识库清除成功',
|
||||
groupSyncSuccess: '分组同步成功',
|
||||
groupDeleted: '分组删除成功',
|
||||
searchHistoryDeleted: '搜索历史删除成功',
|
||||
jwtSecretRequired: 'JWT_SECRET 环境变量未设置',
|
||||
tenantNotFound: '租户不存在',
|
||||
usernameRequired: '用户名是必填项',
|
||||
passwordRequiredForNewUser: '新用户 {username} 需要密码',
|
||||
importTaskNotFound: '导入任务不存在',
|
||||
sourcePathNotFound: '源路径不存在: {path}',
|
||||
targetGroupRequired: '未指定目标分组',
|
||||
modelConfigNotFound: '找不到模型配置: {id}',
|
||||
cannotUpdateOtherTenantModel: '无法更新其他租户的模型',
|
||||
cannotDeleteOtherTenantModel: '无法删除其他租户的模型',
|
||||
elasticsearchHostRequired: 'ELASTICSEARCH_HOST 环境变量未设置',
|
||||
},
|
||||
ja: {
|
||||
noEmbeddingModel: '先にシステム設定で埋め込みモデルを設定してください',
|
||||
searchFailed:
|
||||
'ナレッジベース検索に失敗しました。一般的な知識に基づいて回答します...',
|
||||
invalidApiKey: 'APIキーが無効です',
|
||||
fileNotFound: 'ファイルが見つかりません',
|
||||
insufficientQuota: '利用枠が不足しています',
|
||||
modelNotConfigured: 'モデルが設定されていません',
|
||||
visionModelNotConfigured: 'ビジョンモデルが設定されていません',
|
||||
embeddingDimensionMismatch: '埋め込み次元数が一致しません',
|
||||
uploadNoFile: 'ファイルがアップロードされていません',
|
||||
uploadSizeExceeded: 'ファイルサイズが制限: {size}, 最大許容: {max}',
|
||||
uploadModelRequired: '埋め込みモデルを選択する必要があります',
|
||||
uploadTypeUnsupported: 'サポートされていないファイル形式です: {type}',
|
||||
chunkOverflow:
|
||||
'チャンクサイズ {size} が上限 {max} ({reason}) を超えています。自動調整されました',
|
||||
chunkUnderflow:
|
||||
'チャンクサイズ {size} が最小値 {min} 未満.自動調整されました',
|
||||
overlapOverflow:
|
||||
'重なりサイズ {size} が上限 {max} を超えています。自動調整されました',
|
||||
overlapUnderflow:
|
||||
'重なりサイズ {size} が最小値 {min} 未満.自動調整されました',
|
||||
overlapRatioExceeded:
|
||||
'重なりサイズ {size} がチャンクサイズの50% ({max}) を超えています。自動調整されました',
|
||||
batchOverflowWarning:
|
||||
'バッチ処理のオーバーフローを避けるため、チャンクサイズを {safeSize} 以下にすることをお勧めします (現在: {size}, モデル制限の {percent}%)',
|
||||
estimatedChunkCountExcessive:
|
||||
'推定チャンク数が多すぎます ({count})。処理に時間がかかる可能性があります',
|
||||
contentAndTitleRequired: '内容とタイトルは必須です',
|
||||
embeddingModelNotFound:
|
||||
'埋め込みモデル {id} が見つかりません、またはタイプが embedding ではありません',
|
||||
ocrFailed: 'テキストの抽出に失敗しました: {message}',
|
||||
noImageUploaded: '画像がアップロードされていません',
|
||||
adminOnlyViewList: '管理者のみがユーザーリストを表示できます',
|
||||
passwordsRequired: '現在のパスワードと新しいパスワードは必須です',
|
||||
newPasswordMinLength:
|
||||
'新しいパスワードは少なくとも6文字以上である必要があります',
|
||||
adminOnlyCreateUser: '管理者のみがユーザーを作成できます',
|
||||
usernamePasswordRequired: 'ユーザー名とパスワードは必須です',
|
||||
passwordMinLength: 'パスワードは少なくとも6文字以上である必要があります',
|
||||
adminOnlyUpdateUser: '管理者のみがユーザー情報を更新できます',
|
||||
userNotFound: 'ユーザーが見つかりません',
|
||||
cannotModifyBuiltinAdmin: 'ビルトイン管理者アカウントを変更できません',
|
||||
adminOnlyDeleteUser: '管理者のみがユーザーを削除できます',
|
||||
cannotDeleteSelf: '自分自身のアカウントを削除できません',
|
||||
cannotDeleteBuiltinAdmin: 'ビルトイン管理者アカウントを削除できません',
|
||||
invalidMemberRole:
|
||||
'無効な役割です。USER と TENANT_ADMIN のみ許可されています',
|
||||
incorrectCredentials: 'ユーザー名またはパスワードが間違っています',
|
||||
incorrectCurrentPassword: '現在のパスワードが間違っています',
|
||||
usernameExists: 'ユーザー名が既に存在します',
|
||||
noteNotFound: 'ノートが見つかりません: {id}',
|
||||
knowledgeGroupNotFound: 'ナレッジグループが見つかりません: {id}',
|
||||
accessDeniedNoToken: 'アクセス不許可：トークンがありません',
|
||||
invalidToken: '無効なトークンです',
|
||||
pdfFileNotFound: 'PDF ファイルが見つかりません',
|
||||
pdfFileEmpty: 'PDF ファイルが空.変換に失敗した可能性があります',
|
||||
pdfConversionFailed: 'PDF ファイルが存在しないか、変換に失敗しました',
|
||||
pdfConversionFailedDetail:
|
||||
'PDF 変換に失敗しました（ファイル ID: {id}）。後でもう一度お試しください',
|
||||
pdfPreviewNotSupported:
|
||||
'このファイル形式はプレビューをサポートしていません',
|
||||
pdfServiceUnavailable: 'PDF サービスを利用できません: {message}',
|
||||
pageImageNotFound: 'ページ画像が見つかりません',
|
||||
pdfPageImageFailed: 'PDF ページの画像を取得できませんでした',
|
||||
someGroupsNotFound: '一部のグループが存在しません',
|
||||
promptRequired: 'プロンプトは必須です',
|
||||
addLLMConfig: 'システム設定で LLM モデルを追加してください',
|
||||
visionAnalysisFailed: 'ビジョン分析に失敗しました: {message}',
|
||||
visionSystemPrompt:
|
||||
'あなたはプロフェッショナルなドキュメント分析アシスタントです。このドキュメント画像を分析し、以下の要求に従ってJSON形式で返してください：\n\n1. すべての可読テキストを抽出（読み順で、段落と書式を維持）\n2. 画像/グラフ/テーブルを識別（内容、意味、役割を説明）\n3. ページレイアウトを分析（テキストのみ/テキストと画像混在/テーブル/グラフなど）\n4. 分析品質を評価 (0-1)\n\nレスポンス形式：\n{\n "text": "完全なテキスト内容",\n "images": [\n {"type": "グラフタイプ", "description": "詳細説明", "position": 1}\n ],\n "layout": "レイアウト説明",\n "confidence": 0.95\n}',
|
||||
retryMechanismError: '再試行メカニズムの異常',
|
||||
imageLoadError: '画像を読み込めません: {message}',
|
||||
groupNotFound: 'グループが存在しません',
|
||||
fileDeleted: 'ファイルが削除されました',
|
||||
fileDeletedFromGroup: 'ファイルがグループから削除されました',
|
||||
kbCleared: 'ナレッジベースがクリアされました',
|
||||
groupSyncSuccess: 'グループ同期が完了しました',
|
||||
groupDeleted: 'グループが削除されました',
|
||||
searchHistoryDeleted: '検索履歴が削除されました',
|
||||
jwtSecretRequired: 'JWT_SECRET 環境変数が設定されていません',
|
||||
tenantNotFound: 'テナントが見つかりません',
|
||||
usernameRequired: 'ユーザー名は必須です',
|
||||
passwordRequiredForNewUser:
|
||||
'新しいユーザー {username} のパスワードが必要です',
|
||||
importTaskNotFound: 'インポートタスクが見つかりません',
|
||||
sourcePathNotFound: 'ソースパスが見つかりません: {path}',
|
||||
targetGroupRequired: 'ターゲットグループが指定されていません',
|
||||
modelConfigNotFound: 'モデル設定が見つかりません: {id}',
|
||||
cannotUpdateOtherTenantModel: '他のテナントのモデルは更新できません',
|
||||
cannotDeleteOtherTenantModel: '他のテナントのモデルは削除できません',
|
||||
elasticsearchHostRequired:
|
||||
'ELASTICSEARCH_HOST 環境変数が設定されていません',
|
||||
},
|
||||
en: {
|
||||
noEmbeddingModel:
|
||||
'Please configure embedding model in system settings first',
|
||||
searchFailed:
|
||||
'Knowledge base search failed, will answer based on general knowledge...',
|
||||
invalidApiKey: 'Invalid API key',
|
||||
fileNotFound: 'File not found',
|
||||
insufficientQuota: 'Insufficient quota',
|
||||
modelNotConfigured: 'Model not configured',
|
||||
visionModelNotConfigured: 'Vision model not configured',
|
||||
embeddingDimensionMismatch: 'Embedding dimensions mismatch',
|
||||
uploadNoFile: 'No file uploaded',
|
||||
uploadSizeExceeded: 'File size exceeds limit: {size}, Max allowed: {max}',
|
||||
uploadModelRequired: 'Embedding model must be selected',
|
||||
uploadTypeUnsupported: 'Unsupported file type: {type}',
|
||||
chunkOverflow:
|
||||
'Chunk size {size} exceeds limit {max} ({reason}). Auto-adjusted',
|
||||
chunkUnderflow: 'Chunk size {size} is below minimum {min}. Auto-adjusted',
|
||||
overlapOverflow: 'Overlap size {size} exceeds limit {max}. Auto-adjusted',
|
||||
overlapUnderflow:
|
||||
'Overlap size {size} is below minimum {min}. Auto-adjusted',
|
||||
overlapRatioExceeded:
|
||||
'Overlap size {size} exceeds 50% of chunk size ({max}). Auto-adjusted',
|
||||
batchOverflowWarning:
|
||||
'Recommended chunk size below {safeSize} to avoid batch overflow (Current: {size}, {percent}% of model limit)',
|
||||
estimatedChunkCountExcessive:
|
||||
'Estimated chunk count is too high ({count}). Processing may be slow',
|
||||
contentAndTitleRequired: 'Content and Title are required',
|
||||
embeddingModelNotFound:
|
||||
'Embedding model {id} not found or type is not embedding',
|
||||
ocrFailed: 'Failed to extract text: {message}',
|
||||
noImageUploaded: 'No image uploaded',
|
||||
adminOnlyViewList: 'Only admins can view the user list',
|
||||
passwordsRequired: 'Current and new passwords are required',
|
||||
newPasswordMinLength: 'New password must be at least 6 characters',
|
||||
adminOnlyCreateUser: 'Only admins can create users',
|
||||
usernamePasswordRequired: 'Username and password are required',
|
||||
usernameRequired: 'Username is required',
|
||||
passwordMinLength: 'Password must be at least 6 characters',
|
||||
adminOnlyUpdateUser: 'Only admins can update user info',
|
||||
userNotFound: 'User not found',
|
||||
cannotModifyBuiltinAdmin: 'Cannot modify built-in admin account',
|
||||
adminOnlyDeleteUser: 'Only admins can delete users',
|
||||
cannotDeleteSelf: 'Cannot delete your own account',
|
||||
cannotDeleteBuiltinAdmin: 'Cannot delete built-in admin account',
|
||||
invalidMemberRole: 'Invalid role. Only USER and TENANT_ADMIN are allowed',
|
||||
incorrectCredentials: 'Incorrect username or password',
|
||||
incorrectCurrentPassword: 'Incorrect current password',
|
||||
usernameExists: 'Username already exists',
|
||||
noteNotFound: 'Note with ID {id} not found',
|
||||
knowledgeGroupNotFound: 'Knowledge group with ID {id} not found',
|
||||
accessDeniedNoToken: 'Access Denied: Missing token',
|
||||
invalidToken: 'Invalid token',
|
||||
pdfFileNotFound: 'PDF file not found',
|
||||
pdfFileEmpty: 'PDF file is empty. Conversion may have failed',
|
||||
pdfConversionFailed: 'PDF file does not exist or conversion failed',
|
||||
pdfConversionFailedDetail:
|
||||
'PDF conversion failed for file ID: {id}. Please try again later.',
|
||||
pdfPreviewNotSupported: 'Preview is not supported for this file format',
|
||||
pdfServiceUnavailable: 'PDF service unavailable: {message}',
|
||||
pageImageNotFound: 'Page image not found',
|
||||
pdfPageImageFailed: 'Could not retrieve PDF page image',
|
||||
someGroupsNotFound: 'Some groups not found',
|
||||
promptRequired: 'Prompt is required',
|
||||
addLLMConfig: 'Please add LLM model in system settings',
|
||||
visionAnalysisFailed: 'Vision analysis failed: {message}',
|
||||
visionSystemPrompt:
|
||||
'You are a professional document analysis assistant. Please analyze this document image and return in JSON format according to the following requirements:\n\n1. Extract all readable text (in reading order, preserving paragraphs and formatting)\n2. Identify images/charts/tables (describe content, meaning, and purpose)\n3. Analyze page layout (text only/mixed text and images/tables/charts, etc.)\n4. Evaluate analysis quality (0-1)\n\nResponse format:\n{\n "text": "complete text content",\n "images": [\n {"type": "chart type", "description": "detailed description", "position": 1}\n ],\n "layout": "layout description",\n "confidence": 0.95\n}',
|
||||
retryMechanismError: 'Retry mechanism error',
|
||||
imageLoadError: 'Cannot load image: {message}',
|
||||
groupNotFound: 'Group not found',
|
||||
fileDeleted: 'File deleted successfully',
|
||||
fileDeletedFromGroup: 'File removed from group successfully',
|
||||
kbCleared: 'Knowledge base cleared successfully',
|
||||
groupSyncSuccess: 'Group sync completed successfully',
|
||||
groupDeleted: 'Group deleted successfully',
|
||||
searchHistoryDeleted: 'Search history deleted successfully',
|
||||
jwtSecretRequired:
|
||||
'JWT_SECRET environment variable is required but not set',
|
||||
tenantNotFound: 'Tenant not found',
|
||||
importTaskNotFound: 'Import task not found',
|
||||
sourcePathNotFound: 'Source path not found: {path}',
|
||||
targetGroupRequired: 'Target group not specified',
|
||||
modelConfigNotFound: 'Model config not found: {id}',
|
||||
cannotUpdateOtherTenantModel: 'Cannot update models from another tenant',
|
||||
cannotDeleteOtherTenantModel: 'Cannot delete models from another tenant',
|
||||
elasticsearchHostRequired:
|
||||
'ELASTICSEARCH_HOST environment variable is not set',
|
||||
libreofficeUrlRequired:
|
||||
'LIBREOFFICE_URL environment variable is required but not set',
|
||||
pdfToImageConversionFailed:
|
||||
'PDF to image conversion failed. No images were generated.',
|
||||
pdfPageCountError: 'Could not get PDF page count',
|
||||
parentCategoryNotFound: 'Parent category not found',
|
||||
categoryNotFound: 'Category not found',
|
||||
maxCategoryDepthExceeded: 'Maximum category depth (3 levels) exceeded',
|
||||
userIdRequired: 'User ID is required',
|
||||
podcastNotFound: 'Podcast not found: {id}',
|
||||
scriptGenerationFailed: 'Script generation failed to produce valid JSON',
|
||||
vectorRequired: 'Vector is required for indexing',
|
||||
apiCallFailed: 'API call failed: {message}',
|
||||
tikaHostRequired: 'TIKA_HOST environment variable is required but not set',
|
||||
},
|
||||
};
|
||||
|
||||
export const logMessages = {
|
||||
zh: {
|
||||
processingFile: '处理文件: {name} ({size})',
|
||||
indexingComplete: '索引完成: {id}',
|
||||
vectorizingFile: '向量化文件: ',
|
||||
searchQuery: '搜索查询: ',
|
||||
modelCall: '[模型调用] 类型: {type}, 模型: {model}, 用户: {user}',
|
||||
memoryStatus: '内存状态: ',
|
||||
uploadSuccess: '文件上传成功。正在后台索引',
|
||||
overlapAdjusted: '重叠大小超过切片大小的50%。已自动调整为 {newSize}',
|
||||
environmentLimit: '环境变量限制',
|
||||
modelLimit: '模型限制',
|
||||
configLoaded: '数据库模型配置加载: {name} ({id})',
|
||||
batchSizeAdjusted: '批量大小从 {old} 调整为 {new} (模型限制: {limit})',
|
||||
dimensionMismatch: '模型 {id} 维度不匹配: 预期 {expected}, 实际 {actual}',
|
||||
searchMetadataFailed: '为用户 {userId} 搜索知识库失败',
|
||||
extractedTextTooLarge: '提取的文本内容过大: {size}MB',
|
||||
preciseModeUnsupported: '格式 {ext} 不支持精密模式，回退到快速模式',
|
||||
visionModelNotConfiguredFallback: '未配置视觉模型，回退到快速模式',
|
||||
visionModelInvalidFallback: '视觉模型配置无效，回退到快速模式',
|
||||
visionPipelineFailed: '视觉流水线失败，回退到快速模式',
|
||||
preciseModeComplete: '精密模式提取完成: {pages}页, 费用: ${cost}',
|
||||
skippingEmptyVectorPage: '跳过第 {page} 页（空向量）',
|
||||
pdfPageImageError: '获取 PDF 页面图像失败: {message}',
|
||||
internalServerError: '服务器内部错误',
|
||||
},
|
||||
ja: {
|
||||
processingFile: 'ファイル処理中: {name} ({size})',
|
||||
indexingComplete: 'インデックス完了: {id}',
|
||||
vectorizingFile: 'ファイルベクトル化中: ',
|
||||
searchQuery: '検索クエリ: ',
|
||||
modelCall:
|
||||
'[モデル呼び出し] タイプ: {type}, モデル: {model}, ユーザー: {user}',
|
||||
memoryStatus: 'メモリ状態: ',
|
||||
uploadSuccess:
|
||||
'ファイルが正常にアップロードされました。バックグラウンドでインデックス処理を実行中です',
|
||||
overlapAdjusted:
|
||||
'オーバーラップサイズがチャンクサイズの50%を超えています。自動的に {newSize} に調整されました',
|
||||
environmentLimit: '環境変数の制限',
|
||||
modelLimit: 'モデルの制限',
|
||||
configLoaded: 'データベースからモデル設定を読み込みました: {name} ({id})',
|
||||
batchSizeAdjusted:
|
||||
'バッチサイズを {old} から {new} に調整しました (モデル制限: {limit})',
|
||||
dimensionMismatch:
|
||||
'モデル {id} の次元が一致しません: 期待値 {expected}, 実際 {actual}',
|
||||
searchMetadataFailed:
|
||||
'ユーザー {userId} のナレッジベース検索に失敗しました',
|
||||
extractedTextTooLarge: '抽出されたテキストが大きいです: {size}MB',
|
||||
preciseModeUnsupported:
|
||||
'ファイル形式 {ext} は精密モードをサポートしていません。高速モードにフォールバックします',
|
||||
visionModelNotConfiguredFallback:
|
||||
'ビジョンモデルが設定されていません。高速モードにフォールバックします',
|
||||
visionModelInvalidFallback:
|
||||
'ビジョンモデルの設定が無効です。高速モードにフォールバックします',
|
||||
visionPipelineFailed:
|
||||
'ビジョンパイプラインが失敗しました。高速モードにフォールバックします',
|
||||
preciseModeComplete:
|
||||
'精密モード内容抽出完了: {pages}ページ, コスト: ${cost}',
|
||||
skippingEmptyVectorPage: '第 {page} ページの空ベクトルをスキップします',
|
||||
pdfPageImageError: 'PDF ページの画像取得に失敗しました: {message}',
|
||||
internalServerError: 'サーバー内部エラー',
|
||||
},
|
||||
en: {
|
||||
processingFile: 'Processing file: {name} ({size})',
|
||||
indexingComplete: 'Indexing complete: {id}',
|
||||
vectorizingFile: 'Vectorizing file: ',
|
||||
searchQuery: 'Search query: ',
|
||||
modelCall: '[Model call] Type: {type}, Model: {model}, User: {user}',
|
||||
memoryStatus: 'Memory status: ',
|
||||
uploadSuccess: 'File uploaded successfully. Indexing in background',
|
||||
overlapAdjusted:
|
||||
'Overlap size exceeds 50% of chunk size. Auto-adjusted to {newSize}',
|
||||
environmentLimit: 'Environment variable limit',
|
||||
modelLimit: 'Model limit',
|
||||
configLoaded: 'Model config loaded from DB: {name} ({id})',
|
||||
batchSizeAdjusted:
|
||||
'Batch size adjusted from {old} to {new} (Model limit: {limit})',
|
||||
dimensionMismatch:
|
||||
'Model {id} dimension mismatch: Expected {expected}, Actual {actual}',
|
||||
searchMetadataFailed: 'Failed to search knowledge base for user {userId}',
|
||||
extractedTextTooLarge: 'Extracted text is too large: {size}MB',
|
||||
preciseModeUnsupported:
|
||||
'Format {ext} not supported for precise mode. Falling back to fast mode',
|
||||
visionModelNotConfiguredFallback:
|
||||
'Vision model not configured. Falling back to fast mode',
|
||||
visionModelInvalidFallback:
|
||||
'Vision model config invalid. Falling back to fast mode',
|
||||
visionPipelineFailed: 'Vision pipeline failed. Falling back to fast mode',
|
||||
preciseModeComplete:
|
||||
'Precise mode extraction complete: {pages} pages, cost: ${cost}',
|
||||
skippingEmptyVectorPage: 'Skipping page {page} due to empty vector',
|
||||
pdfPageImageError: 'Failed to retrieve PDF page image: {message}',
|
||||
internalServerError: 'Internal server error',
|
||||
},
|
||||
};
|
||||
|
||||
export const statusMessages = {
|
||||
zh: {
|
||||
searching: '正在搜索知识库...',
|
||||
noResults: '未找到相关知识，将基于一般知识回答...',
|
||||
searchFailed: '知识库搜索失败，将基于一般知识回答...',
|
||||
generatingResponse: '正在生成回答',
|
||||
files: '个文件',
|
||||
notebooks: '个笔记本',
|
||||
all: '全部',
|
||||
items: '个',
|
||||
searchResults: '搜索结果',
|
||||
relevantInfoFound: '条相关信息找到',
|
||||
searchHits: '搜索命中',
|
||||
relevance: '相关度',
|
||||
sourceFiles: '源文件',
|
||||
searchScope: '搜索范围',
|
||||
error: '错误',
|
||||
creatingHistory: '创建新对话历史: ',
|
||||
searchingModelById: '根据ID搜索模型: ',
|
||||
searchModelFallback: '未找到指定的嵌入模型。使用第一个可用模型。',
|
||||
noEmbeddingModelFound: '找不到嵌入模型设置',
|
||||
usingEmbeddingModel: '使用的嵌入模型: ',
|
||||
startingSearch: '开始搜索知识库...',
|
||||
searchResultsCount: '搜索结果数: ',
|
||||
searchFailedLog: '搜索失败',
|
||||
modelCall: '[模型调用]',
|
||||
chatStreamError: '聊天流错误',
|
||||
assistStreamError: '辅助流错误',
|
||||
file: '文件',
|
||||
content: '内容',
|
||||
userLabel: '用户',
|
||||
assistantLabel: '助手',
|
||||
intelligentAssistant: '您是智能写作助手。',
|
||||
assistSystemPrompt:
|
||||
'请根据用户的指示修正或改进提供的文本内容。不要包含问候语或结束语（如"明白了，这是..."等），直接输出修正后的内容。',
|
||||
contextLabel: '上下文（当前内容）',
|
||||
userInstructionLabel: '用户指示',
|
||||
searchString: '搜索字符串: ',
|
||||
embeddingModelIdNotProvided: '未提供嵌入模型ID',
|
||||
generatingEmbeddings: '生成嵌入向量...',
|
||||
embeddingsGenerated: '嵌入向量生成完成',
|
||||
dimensions: '维度',
|
||||
performingHybridSearch: '执行混合搜索...',
|
||||
esSearchCompleted: 'ES搜索完成',
|
||||
resultsCount: '结果数',
|
||||
hybridSearchFailed: '混合搜索失败',
|
||||
getContextForTopicFailed: '获取主题上下文失败',
|
||||
noLLMConfigured: '用户未配置LLM模型',
|
||||
simpleChatGenerationError: '简单聊天生成错误',
|
||||
noMatchInKnowledgeGroup:
|
||||
'所选知识组中未找到相关内容,以下是基于模型的一般性回答:',
|
||||
uploadTextSuccess: '笔记内容已接收。正在后台索引',
|
||||
passwordChanged: '密码已成功修改',
|
||||
userCreated: '用户已成功创建',
|
||||
userInfoUpdated: '用户信息已更新',
|
||||
userDeleted: '用户已删除',
|
||||
pdfNoteTitle: 'PDF 笔记 - {date}',
|
||||
noTextExtracted: '未提取到文本',
|
||||
kbCleared: '知识库已清空',
|
||||
fileDeleted: '文件已删除',
|
||||
pageImageNotFoundDetail: '无法获取 PDF 第 {page} 页’的图像',
|
||||
groupSyncSuccess: '文件分组已更新',
|
||||
fileDeletedFromGroup: '文件已从分组中删除',
|
||||
chunkConfigCorrection: '切片配置已修正: {warnings}',
|
||||
noChunksGenerated: '文件 {id} 未生成任何切片',
|
||||
chunkCountAnomaly:
|
||||
'实际切片数 {actual} 大幅超过预计值 {estimated}，可能存在异常',
|
||||
batchSizeExceeded:
|
||||
'批次 {index} 的大小 {actual} 超过推荐值 {limit}，将拆分处理',
|
||||
skippingEmptyVectorChunk: '跳过文本块 {index} (空向量)',
|
||||
contextLengthErrorFallback:
|
||||
'批次处理发生上下文长度错误，降级到逐条处理模式',
|
||||
chunkLimitExceededForceBatch:
|
||||
'切片数 {actual} 超过模型批次限制 {limit}，强制进行批次处理',
|
||||
noteContentRequired: '笔记内容是必填项',
|
||||
imageAnalysisStarted: '正在使用模型 {id} 分析图像...',
|
||||
batchAnalysisStarted: '正在分析 {count} 张图像...',
|
||||
pageAnalysisFailed: '第 {page} 页分析失败',
|
||||
visionSystemPrompt:
|
||||
'您是专业的文档分析助手。请分析此文档图像，并按以下要求以 JSON 格式返回：\n\n1. 提取所有可读文本（按阅读顺序，保持段落和格式）\n2. 识别图像/图表/表格（描述内容、含义和作用）\n3. 分析页面布局（仅文本/文本和图像混合/表格/图表等）\n4. 评估分析质量 (0-1)\n\n响应格式：\n{\n "text": "完整的文本内容",\n "images": [\n {"type": "图表类型", "description": "详细描述", "position": 1}\n ],\n "layout": "布局说明",\n "confidence": 0.95\n}',
|
||||
visionModelCall: '[模型调用] 类型: Vision, 模型: {model}, 页面: {page}',
|
||||
visionAnalysisSuccess:
|
||||
'✅ 视觉分析完成: {path}{page}, 文本长度: {textLen}, 图像数: {imgCount}, 布局: {layout}, 置信度: {confidence}%',
|
||||
conversationHistoryNotFound: '对话历史不存在',
|
||||
batchContextLengthErrorFallback:
|
||||
'小文件批次处理发生上下文长度错误，降级到逐条处理模式',
|
||||
chunkProcessingFailed: '处理文本块 {index} 失败，已跳过: {message}',
|
||||
singleTextProcessingComplete: '逐条文本处理完成: {count} 个切片',
|
||||
fileVectorizationComplete:
|
||||
'文件 {id} 向量化完成。共处理 {count} 个文本块。最终内存: {memory}MB',
|
||||
fileVectorizationFailed: '文件 {id} 向量化失败',
|
||||
batchProcessingStarted: '开始批次处理: {count} 个项目',
|
||||
batchProcessingProgress: '正在处理批次 {index}/{total}: {count} 个项目',
|
||||
batchProcessingComplete: '批次处理完成: {count} 个项目，耗时 {duration}s',
|
||||
onlyFailedFilesRetryable: '仅允许重试失败的文件 (当前状态: {status})',
|
||||
emptyFileRetryFailed: '文件内容为空，无法重试。请重新上传文件。',
|
||||
ragSystemPrompt:
|
||||
'您是专业的知识库助手。请根据以下提供的文档内容回答用户的问题。',
|
||||
ragRules:
|
||||
'## 规则：\n1. 仅根据提供的文档内容进行回答，请勿编造信息。\n2. 如果文档中没有相关信息，请告知用户。\n3. 请在回答中注明信息来源。格式：[文件名.扩展子]\n4. 如果多个文档中的信息存在矛盾，请进行综合分析或解释不同的观点。\n5. 请使用{lang}进行回答。',
|
||||
ragDocumentContent: '## 文档内容：',
|
||||
ragUserQuestion: '## 用户问题：',
|
||||
ragAnswer: '## 回答：',
|
||||
ragSource: '### 来源：{fileName}',
|
||||
ragSegment: '片段 {index} (相似度: {score}):',
|
||||
ragNoDocumentFound: '未找到相关文档。',
|
||||
queryExpansionPrompt:
|
||||
'您是一个搜索助手。请为以下用户查询生成3个不同的演变版本，以帮助在向量搜索中获得更好的结果。每个版本应包含不同的关键词或表达方式，但保持原始意思。直接输出3行查询，不要有数字或编号：\n\n查询：{query}',
|
||||
hydePrompt:
|
||||
'请为以下用户问题写一段简短、事实性的假设回答（约100字）。不要包含任何引导性文字（如“基于我的分析...”），直接输出答案内容。\n\n问题：{query}',
|
||||
},
|
||||
ja: {
|
||||
searching: 'ナレッジベースを検索中...',
|
||||
noResults:
|
||||
'関連する知識が見つかりませんでした。一般的な知識に基づいて回答します...',
|
||||
searchFailed:
|
||||
'ナレッジベース検索に失敗しました。一般的な知識に基づいて回答します...',
|
||||
generatingResponse: '回答を生成中',
|
||||
files: '個のファイル',
|
||||
notebooks: '個のノートブック',
|
||||
all: 'すべて',
|
||||
items: '件',
|
||||
searchResults: '検索結果',
|
||||
relevantInfoFound: '件の関連情報が見つかりました',
|
||||
searchHits: '検索ヒット',
|
||||
relevance: '関連度',
|
||||
sourceFiles: '元ファイル',
|
||||
searchScope: '検索範囲',
|
||||
error: 'エラー',
|
||||
creatingHistory: '新規対話履歴を作成: ',
|
||||
searchingModelById: 'selectedEmbeddingId に基づいてモデルを検索: ',
|
||||
searchModelFallback:
|
||||
'指定された埋め込みモデルが見つかりません。最初に使用可能なモデルを使用します。',
|
||||
noEmbeddingModelFound: '埋め込みモデルの設定が見つかりません',
|
||||
usingEmbeddingModel: '使用する埋め込みモデル: ',
|
||||
startingSearch: 'ナレッジベースの検索を開始...',
|
||||
searchResultsCount: '検索結果数: ',
|
||||
searchFailedLog: '検索失敗',
|
||||
chatStreamError: 'チャットストリームエラー',
|
||||
assistStreamError: 'アシストストリームエラー',
|
||||
file: 'ファイル',
|
||||
content: '内容',
|
||||
userLabel: 'ユーザー',
|
||||
assistantLabel: 'アシスタント',
|
||||
intelligentAssistant: 'あなたはインテリジェントな執筆アシスタントです。',
|
||||
assistSystemPrompt:
|
||||
'提供されたテキスト内容を、ユーザーの指示に基づいて修正または改善してください。挨拶や結びの言葉（「わかりました、こちらが...」など）は含めず、修正後の内容のみを直接出力してください。',
|
||||
contextLabel: 'コンテキスト（現在の内容）',
|
||||
userInstructionLabel: 'ユーザーの指示',
|
||||
searchString: '検索文字列: ',
|
||||
embeddingModelIdNotProvided: '埋め込みモデルIDが提供されていません',
|
||||
generatingEmbeddings: '埋め込みベクトルを生成中...',
|
||||
embeddingsGenerated: '埋め込みベクトルの生成が完了しました',
|
||||
dimensions: '次元数',
|
||||
performingHybridSearch: 'ES 混合検索を実行中...',
|
||||
esSearchCompleted: 'ES 検索が完了しました',
|
||||
resultsCount: '結果数',
|
||||
hybridSearchFailed: '混合検索に失敗しました',
|
||||
getContextForTopicFailed: 'トピックのコンテキスト取得に失敗しました',
|
||||
noLLMConfigured: 'ユーザーにLLMモデルが設定されていません',
|
||||
simpleChatGenerationError: '簡易チャット生成エラー',
|
||||
noMatchInKnowledgeGroup:
|
||||
'選択された知識グループに関連する内容が見つかりませんでした。以下はモデルに基づく一般的な回答です:',
|
||||
uploadTextSuccess:
|
||||
'ノート内容を受け取りました。バックグラウンドでインデックス処理を実行中です',
|
||||
passwordChanged: 'パスワードが正常に変更されました',
|
||||
userCreated: 'ユーザーが正常に作成されました',
|
||||
userInfoUpdated: 'ユーザー情報が更新されました',
|
||||
userDeleted: 'ユーザーが削除されました',
|
||||
pdfNoteTitle: 'PDF ノート - {date}',
|
||||
noTextExtracted: 'テキストが抽出されませんでした',
|
||||
kbCleared: 'ナレッジベースが空になりました',
|
||||
fileDeleted: 'ファイルが削除されました',
|
||||
pageImageNotFoundDetail: 'PDF の第 {page} ページの画像を取得できません',
|
||||
groupSyncSuccess: 'ファイルグループが更新されました',
|
||||
fileDeletedFromGroup: 'ファイルがグループから削除されました',
|
||||
chunkConfigCorrection: 'チャンク設定の修正: {warnings}',
|
||||
noChunksGenerated:
|
||||
'ファイル {id} からテキストチャンクが生成されませんでした',
|
||||
chunkCountAnomaly:
|
||||
'実際のチャンク数 {actual} が推定値 {estimated} を大幅に超えています。異常がある可能性があります',
|
||||
batchSizeExceeded:
|
||||
'バッチ {index} のサイズ {actual} が推奨値 {limit} を超えています。分割して処理します',
|
||||
skippingEmptyVectorChunk:
|
||||
'空ベクトルのテキストブロック {index} をスキップします',
|
||||
contextLengthErrorFallback:
|
||||
'バッチ処理でコンテキスト長エラーが発生しました。単一テキスト処理モードにダウングレードします',
|
||||
chunkLimitExceededForceBatch:
|
||||
'チャンク数 {actual} がモデルのバッチ制限 {limit} を超えています。強制的にバッチ処理を行います',
|
||||
noteContentRequired: 'ノート内容は必須です',
|
||||
imageAnalysisStarted: 'モデル {id} で画像を分析中...',
|
||||
batchAnalysisStarted: '{count} 枚の画像を分析中...',
|
||||
pageAnalysisFailed: '第 {page} ページの分析に失敗しました',
|
||||
visionSystemPrompt:
|
||||
'あなたは専門的なドキュメント分析アシスタントです。このドキュメント画像を分析し、以下の要求に従って JSON 形式で返してください：\n\n1. すべての読み取り可能なテキストを抽出（読み取り順序に従い、段落と形式を保持）\n2. 画像/グラフ/表の識別（内容、意味、役割を記述）\n3. ページレイアウトの分析（テキストのみ/テキストと画像の混合/表/グラフなど）\n4. 分析品質の評価（0-1）\n\nレスポンス形式：\n{\n "text": "完全なテキスト内容",\n "images": [\n {"type": "グラフの種類", "description": "詳細な記述", "position": 1}\n ],\n "layout": "レイアウトの説明",\n "confidence": 0.95\n}',
|
||||
visionModelCall:
|
||||
'[モデル呼び出し] タイプ: Vision, モデル: {model}, ページ: {page}',
|
||||
visionAnalysisSuccess:
|
||||
'✅ Vision 分析完了: {path}{page}, テキスト長: {textLen}文字, 画像数: {imgCount}, レイアウト: {layout}, 信頼度: {confidence}%',
|
||||
conversationHistoryNotFound: '会話履歴が存在しません',
|
||||
batchContextLengthErrorFallback:
|
||||
'小ファイルバッチ処理でコンテキスト長エラーが発生しました。単一テキスト処理モードにダウングレードします',
|
||||
chunkProcessingFailed:
|
||||
'テキストブロック {index} の処理に失敗しました。スキップします: {message}',
|
||||
singleTextProcessingComplete: '単一テキスト処理完了: {count} チャンク',
|
||||
fileVectorizationComplete:
|
||||
'ファイル {id} ベクトル化完了。{count} 個のテキストブロックを処理しました。最終メモリ: {memory}MB',
|
||||
fileVectorizationFailed: 'ファイル {id} ベクトル化失敗',
|
||||
batchProcessingStarted: 'バッチ処理を開始します: {count} アイテム',
|
||||
batchProcessingProgress:
|
||||
'バッチ {index}/{total} を処理中: {count} 個のアイテム',
|
||||
batchProcessingComplete:
|
||||
'バッチ処理完了: {count} アイテム, 所要時間 {duration}s',
|
||||
onlyFailedFilesRetryable:
|
||||
'失敗したファイルのみ再試行可能です (現在のステータス: {status})',
|
||||
emptyFileRetryFailed:
|
||||
'ファイル内容が空です。再試行できません。ファイルを再アップロードしてください。',
|
||||
ragSystemPrompt:
|
||||
'あなたは専門的なナレッジベースアシスタントです。以下の提供されたドキュメントの内容に基づいて、ユーザーの質問に答えてください。',
|
||||
ragRules:
|
||||
'## ルール：\n1. 提供されたドキュメントの内容のみに基づいて回答し、情報を捏造しないでください。\n2. ドキュメントに関連情報がない場合は、その旨をユーザーに伝えてください。\n3. 回答には情報源を明記してください。形式：[ファイル名.拡張子]\n4. 複数のドキュメントで情報が矛盾している場合は、総合的に分析するか、異なる視点を説明してください。\n5. {lang}で回答してください。',
|
||||
ragDocumentContent: '## ドキュメント内容：',
|
||||
ragUserQuestion: '## ユーザーの質問：',
|
||||
ragAnswer: '## 回答：',
|
||||
ragSource: '### ソース：{fileName}',
|
||||
ragSegment: 'セグメント {index} (類似度: {score}):',
|
||||
ragNoDocumentFound: '関連するドキュメントが見つかりませんでした。',
|
||||
queryExpansionPrompt:
|
||||
'あなたは検索アシスタントです。以下のユーザーのクエリに対して、ベクトル検索でより良い結果を得るために、3つの異なるバリエーションを生成してください。各バリエーションは異なるキーワードや表現を使用しつつ、元の意味を維持する必要があります。数字やプレフィックスなしで、3行のクエリを直接出力してください：\n\nクエリ：{query}',
|
||||
hydePrompt:
|
||||
'以下のユーザーの質問に対して、簡潔で事実に基づいた仮説的な回答（約200文字）を書いてください。「私の分析によると...」などの導入文は含めず、回答内容のみを直接出力してください。\n\n質問：{query}',
|
||||
},
|
||||
en: {
|
||||
searching: 'Searching knowledge base...',
|
||||
noResults:
|
||||
'No relevant knowledge found, will answer based on general knowledge...',
|
||||
searchFailed:
|
||||
'Knowledge base search failed, will answer based on general knowledge...',
|
||||
generatingResponse: 'Generating response',
|
||||
files: ' files',
|
||||
notebooks: ' notebooks',
|
||||
all: 'all',
|
||||
items: '',
|
||||
searchResults: 'Search results',
|
||||
relevantInfoFound: ' relevant info found',
|
||||
searchHits: 'Search hits',
|
||||
relevance: 'Relevance',
|
||||
sourceFiles: 'Source files',
|
||||
searchScope: 'Search scope',
|
||||
error: 'Error',
|
||||
creatingHistory: 'Creating new chat history: ',
|
||||
searchingModelById: 'Searching model by ID: ',
|
||||
searchModelFallback:
|
||||
'Specified embedding model not found. Using first available model.',
|
||||
noEmbeddingModelFound: 'No embedding model settings found',
|
||||
usingEmbeddingModel: 'Using embedding model: ',
|
||||
startingSearch: 'Starting knowledge base search...',
|
||||
searchResultsCount: 'Search results count: ',
|
||||
searchFailedLog: 'Search failed',
|
||||
chatStreamError: 'Chat stream error',
|
||||
assistStreamError: 'Assist stream error',
|
||||
file: 'File',
|
||||
content: 'Content',
|
||||
userLabel: 'User',
|
||||
assistantLabel: 'Assistant',
|
||||
intelligentAssistant: 'You are an intelligent writing assistant.',
|
||||
assistSystemPrompt:
|
||||
'Please revise or improve the provided text content based on the user\'s instructions. Do not include greetings or closing phrases (such as "Understood, here is..." etc.), output only the revised content directly.',
|
||||
contextLabel: 'Context (current content)',
|
||||
userInstructionLabel: 'User instructions',
|
||||
searchString: 'Search string: ',
|
||||
embeddingModelIdNotProvided: 'Embedding model ID not provided',
|
||||
generatingEmbeddings: 'Generating embeddings...',
|
||||
embeddingsGenerated: 'Embeddings generated successfully',
|
||||
dimensions: 'dimensions',
|
||||
performingHybridSearch: 'Performing hybrid search...',
|
||||
esSearchCompleted: 'ES search completed',
|
||||
resultsCount: 'Results count',
|
||||
hybridSearchFailed: 'Hybrid search failed',
|
||||
getContextForTopicFailed: 'getContextForTopic failed',
|
||||
noLLMConfigured: 'No LLM model configured for user',
|
||||
simpleChatGenerationError: 'Simple chat generation error',
|
||||
noMatchInKnowledgeGroup:
|
||||
'No relevant content found in the selected knowledge group. The following is a general answer based on the model:',
|
||||
uploadTextSuccess: 'Note content received. Indexing in background',
|
||||
passwordChanged: 'Password changed successfully',
|
||||
userCreated: 'User created successfully',
|
||||
userInfoUpdated: 'User information updated',
|
||||
userDeleted: 'User deleted',
|
||||
pdfNoteTitle: 'PDF Note - {date}',
|
||||
noTextExtracted: 'No text extracted',
|
||||
kbCleared: 'Knowledge base cleared',
|
||||
fileDeleted: 'File deleted',
|
||||
pageImageNotFoundDetail: 'Could not retrieve image for PDF page {page}',
|
||||
groupSyncSuccess: 'File groups updated',
|
||||
fileDeletedFromGroup: 'File removed from group',
|
||||
chunkConfigCorrection: 'Chunk config corrected: {warnings}',
|
||||
noChunksGenerated: 'No chunks generated for file {id}',
|
||||
chunkCountAnomaly:
|
||||
'Actual chunk count {actual} significantly exceeds estimate {estimated}. Possible anomaly.',
|
||||
batchSizeExceeded:
|
||||
'Batch {index} size {actual} exceeds recommended limit {limit}. Splitting for processing.',
|
||||
skippingEmptyVectorChunk: 'Skipping text block {index} due to empty vector',
|
||||
contextLengthErrorFallback:
|
||||
'Context length error occurred during batch processing. Downgrading to single processing mode.',
|
||||
chunkLimitExceededForceBatch:
|
||||
'Chunk count {actual} exceeds model batch limit {limit}. Forcing batch processing.',
|
||||
noteContentRequired: 'Note content is required',
|
||||
imageAnalysisStarted: 'Analyzing image with model {id}...',
|
||||
batchAnalysisStarted: 'Batch analyzing {count} images...',
|
||||
pageAnalysisFailed: 'Failed to analyze page {page}',
|
||||
visionSystemPrompt:
|
||||
'You are a professional document analysis assistant. Analyze this document image and return in JSON format according to these requirements:\n\n1. Extract all readable text (follow reading order, maintain paragraphs and formatting)\n2. Identify images/graphs/tables (describe content, meaning, and role)\n3. Analyze page layout (text only/mixed/table/graph, etc.)\n4. Evaluate analysis quality (0-1)\n\nResponse format:\n{\n "text": "full text content",\n "images": [\n {"type": "graph type", "description": "detailed description", "position": 1}\n ],\n "layout": "layout description",\n "confidence": 0.95\n}',
|
||||
visionModelCall: '[Model Call] Type: Vision, Model: {model}, Page: {page}',
|
||||
visionAnalysisSuccess:
|
||||
'✅ Vision analysis complete: {path}{page}, Text length: {textLen}, Images: {imgCount}, Layout: {layout}, Confidence: {confidence}%',
|
||||
conversationHistoryNotFound: 'Conversation history not found',
|
||||
batchContextLengthErrorFallback:
|
||||
'Context length error occurred during small file batch processing. Downgrading to single processing mode.',
|
||||
chunkProcessingFailed:
|
||||
'Failed to process text block {index}. Skipping: {message}',
|
||||
singleTextProcessingComplete:
|
||||
'Single text processing complete: {count} chunks',
|
||||
fileVectorizationComplete:
|
||||
'File {id} vectorization complete. Processed {count} text blocks. Final memory: {memory}MB',
|
||||
fileVectorizationFailed: 'File {id} vectorization failed',
|
||||
batchProcessingStarted: 'Batch processing started: {count} items',
|
||||
batchProcessingProgress: 'Processing batch {index}/{total}: {count} items',
|
||||
batchProcessingComplete:
|
||||
'Batch processing complete: {count} items in {duration}s',
|
||||
onlyFailedFilesRetryable:
|
||||
'Only failed files can be retried (current status: {status})',
|
||||
emptyFileRetryFailed:
|
||||
'File content is empty. Cannot retry. Please re-upload the file.',
|
||||
ragSystemPrompt:
|
||||
"You are a professional knowledge base assistant. Please answer the user's question based on the provided document content below.",
|
||||
ragRules:
|
||||
'## Rules:\n1. Answer based only on the provided document content; do not fabricate information.\n2. If there is no relevant information in the documents, please inform the user.\n3. Clearly state the sources in your answer. Format: [filename.ext]\n4. If information in different documents is contradictory, analyze it comprehensively or explain the different perspectives.\n5. Please answer in {lang}.',
|
||||
ragDocumentContent: '## Document Content:',
|
||||
ragUserQuestion: '## User Question:',
|
||||
ragAnswer: '## Answer:',
|
||||
ragSource: '### Source: {fileName}',
|
||||
ragSegment: 'Segment {index} (Similarity: {score}):',
|
||||
ragNoDocumentFound: 'No relevant documents found.',
|
||||
queryExpansionPrompt:
|
||||
'You are a search assistant. Please generate 3 different variations of the following user query to help get better results in vector search. Each variation should use different keywords or phrasing while maintaining the original meaning. Output the 3 queries directly as 3 lines, without numbers or prefixes:\n\nQuery: {query}',
|
||||
hydePrompt:
|
||||
'Please write a brief, factual hypothetical answer (about 100 words) to the following user question. Do not include any introductory text (like "Based on my analysis..."), just output the answer content directly.\n\nQuestion: {query}',
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,57 @@
|
||||
import {
|
||||
Controller,
|
||||
Post,
|
||||
Get,
|
||||
Delete,
|
||||
Param,
|
||||
Body,
|
||||
Request,
|
||||
UseGuards,
|
||||
Query,
|
||||
} from '@nestjs/common';
|
||||
import { ImportTaskService } from './import-task.service';
|
||||
import { CombinedAuthGuard } from '../auth/combined-auth.guard';
|
||||
import { RolesGuard } from '../auth/roles.guard';
|
||||
import { Roles } from '../auth/roles.decorator';
|
||||
import { UserRole } from '../user/user-role.enum';
|
||||
|
||||
@Controller('import-tasks')
|
||||
@UseGuards(CombinedAuthGuard, RolesGuard)
|
||||
export class ImportTaskController {
|
||||
constructor(private readonly taskService: ImportTaskService) {}
|
||||
|
||||
@Post()
|
||||
@Roles(UserRole.TENANT_ADMIN, UserRole.SUPER_ADMIN)
|
||||
async create(@Request() req, @Body() body: any) {
|
||||
return this.taskService.create({
|
||||
sourcePath: body.sourcePath,
|
||||
targetGroupId: body.targetGroupId,
|
||||
targetGroupName: body.targetGroupName,
|
||||
embeddingModelId: body.embeddingModelId,
|
||||
scheduledAt: body.scheduledAt ? new Date(body.scheduledAt) : undefined,
|
||||
chunkSize: body.chunkSize,
|
||||
chunkOverlap: body.chunkOverlap,
|
||||
mode: body.mode,
|
||||
useHierarchy: body.useHierarchy ?? false,
|
||||
userId: req.user.id,
|
||||
tenantId: req.user.tenantId,
|
||||
});
|
||||
}
|
||||
|
||||
@Get()
|
||||
async findAll(
|
||||
@Request() req,
|
||||
@Query('page') page?: number,
|
||||
@Query('limit') limit?: number,
|
||||
) {
|
||||
return this.taskService.findAll(req.user.id, {
|
||||
page: page ? Number(page) : undefined,
|
||||
limit: limit ? Number(limit) : undefined,
|
||||
});
|
||||
}
|
||||
|
||||
@Delete(':id')
|
||||
async delete(@Param('id') id: string, @Request() req) {
|
||||
return this.taskService.delete(id, req.user.id);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
import {
|
||||
Entity,
|
||||
Column,
|
||||
PrimaryGeneratedColumn,
|
||||
CreateDateColumn,
|
||||
UpdateDateColumn,
|
||||
} from 'typeorm';
|
||||
|
||||
@Entity()
|
||||
export class ImportTask {
|
||||
@PrimaryGeneratedColumn('uuid')
|
||||
id: string;
|
||||
|
||||
@Column()
|
||||
sourcePath: string;
|
||||
|
||||
@Column({ nullable: true })
|
||||
targetGroupId: string; // If null, creates new group
|
||||
|
||||
@Column({ nullable: true })
|
||||
targetGroupName: string; // Used if creating new group
|
||||
|
||||
@Column()
|
||||
userId: string;
|
||||
|
||||
@Column({ nullable: true })
|
||||
tenantId: string;
|
||||
|
||||
@Column({ nullable: true })
|
||||
scheduledAt: Date;
|
||||
|
||||
@Column({ default: 'PENDING' })
|
||||
status: 'PENDING' | 'PROCESSING' | 'COMPLETED' | 'FAILED';
|
||||
|
||||
@Column({ type: 'text', nullable: true })
|
||||
logs: string;
|
||||
|
||||
@Column({ nullable: true })
|
||||
embeddingModelId: string;
|
||||
|
||||
@Column({ nullable: true, default: 500 })
|
||||
chunkSize: number;
|
||||
|
||||
@Column({ nullable: true, default: 50 })
|
||||
chunkOverlap: number;
|
||||
|
||||
@Column({ nullable: true, default: 'fast' })
|
||||
mode: string;
|
||||
|
||||
/** When true, sub-directories become sub-categories mirroring the folder hierarchy */
|
||||
@Column({ default: false })
|
||||
useHierarchy: boolean;
|
||||
|
||||
@CreateDateColumn()
|
||||
createdAt: Date;
|
||||
|
||||
@UpdateDateColumn()
|
||||
updatedAt: Date;
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
import { ImportTask } from './import-task.entity';
|
||||
import { ImportTaskService } from './import-task.service';
|
||||
import { ImportTaskController } from './import-task.controller';
|
||||
import { KnowledgeBaseModule } from '../knowledge-base/knowledge-base.module';
|
||||
import { KnowledgeGroupModule } from '../knowledge-group/knowledge-group.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
TypeOrmModule.forFeature([ImportTask]),
|
||||
KnowledgeBaseModule,
|
||||
KnowledgeGroupModule,
|
||||
],
|
||||
controllers: [ImportTaskController],
|
||||
providers: [ImportTaskService],
|
||||
})
|
||||
export class ImportTaskModule {}
|
||||
@@ -0,0 +1,422 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Repository, LessThanOrEqual } from 'typeorm';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { ImportTask } from './import-task.entity';
|
||||
import { Cron, CronExpression } from '@nestjs/schedule';
|
||||
import { KnowledgeBaseService } from '../knowledge-base/knowledge-base.service';
|
||||
import { KnowledgeGroupService } from '../knowledge-group/knowledge-group.service';
|
||||
import { I18nService } from '../i18n/i18n.service';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
export interface PaginatedImportTasks {
|
||||
items: ImportTask[];
|
||||
total: number;
|
||||
page: number;
|
||||
limit: number;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class ImportTaskService {
|
||||
private readonly logger = new Logger(ImportTaskService.name);
|
||||
|
||||
constructor(
|
||||
@InjectRepository(ImportTask)
|
||||
private taskRepository: Repository<ImportTask>,
|
||||
private kbService: KnowledgeBaseService,
|
||||
private groupService: KnowledgeGroupService,
|
||||
private configService: ConfigService,
|
||||
private i18nService: I18nService,
|
||||
) {}
|
||||
|
||||
async create(taskData: Partial<ImportTask>): Promise<ImportTask> {
|
||||
const task = this.taskRepository.create(taskData);
|
||||
const savedTask = await this.taskRepository.save(task);
|
||||
|
||||
// If no scheduled time or scheduled time is in the past, execute immediately (async)
|
||||
if (!task.scheduledAt || task.scheduledAt <= new Date()) {
|
||||
this.executeTask(savedTask.id).catch((err) =>
|
||||
this.logger.error(
|
||||
`Immediate execution failed to start for task ${savedTask.id}`,
|
||||
err,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
return savedTask;
|
||||
}
|
||||
|
||||
async findAll(
|
||||
userId: string,
|
||||
options: { page?: number; limit?: number } = {},
|
||||
): Promise<PaginatedImportTasks> {
|
||||
const { page = 1, limit = 12 } = options;
|
||||
const skip = (page - 1) * limit;
|
||||
|
||||
const [items, total] = await this.taskRepository.findAndCount({
|
||||
where: { userId },
|
||||
order: { createdAt: 'DESC' },
|
||||
skip,
|
||||
take: limit,
|
||||
});
|
||||
|
||||
return {
|
||||
items,
|
||||
total,
|
||||
page,
|
||||
limit,
|
||||
};
|
||||
}
|
||||
|
||||
async delete(taskId: string, userId: string): Promise<void> {
|
||||
const task = await this.taskRepository.findOne({
|
||||
where: { id: taskId, userId },
|
||||
});
|
||||
if (!task) {
|
||||
throw new Error(this.i18nService.getMessage('importTaskNotFound'));
|
||||
}
|
||||
await this.taskRepository.remove(task);
|
||||
}
|
||||
|
||||
@Cron(CronExpression.EVERY_MINUTE)
|
||||
async handleScheduledTasks() {
|
||||
this.logger.debug('Checking for scheduled import tasks...');
|
||||
const now = new Date();
|
||||
|
||||
const tasks = await this.taskRepository.find({
|
||||
where: {
|
||||
status: 'PENDING',
|
||||
scheduledAt: LessThanOrEqual(now),
|
||||
},
|
||||
});
|
||||
|
||||
for (const task of tasks) {
|
||||
this.logger.log(`Starting scheduled task ${task.id}`);
|
||||
this.executeTask(task.id).catch((err) =>
|
||||
this.logger.error(
|
||||
`Scheduled execution failed to start for task ${task.id}`,
|
||||
err,
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async executeTask(taskId: string) {
|
||||
this.logger.debug(`Executing task ${taskId}`);
|
||||
const task = await this.taskRepository.findOne({ where: { id: taskId } });
|
||||
if (!task) {
|
||||
this.logger.warn(`Task ${taskId} not found.`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (task.status === 'PROCESSING' || task.status === 'COMPLETED') {
|
||||
this.logger.debug(
|
||||
`Task ${taskId} is already ${task.status}, skipping execution.`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
await this.updateStatus(taskId, 'PROCESSING', 'Starting import...');
|
||||
|
||||
try {
|
||||
if (!fs.existsSync(task.sourcePath)) {
|
||||
throw new Error(
|
||||
this.i18nService.formatMessage('sourcePathNotFound', {
|
||||
path: task.sourcePath,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
const uploadPath = this.configService.get<string>(
|
||||
'UPLOAD_FILE_PATH',
|
||||
'./uploads',
|
||||
);
|
||||
const importTargetDir = path.join(uploadPath, 'imported', taskId);
|
||||
|
||||
if (!fs.existsSync(importTargetDir)) {
|
||||
fs.mkdirSync(importTargetDir, { recursive: true });
|
||||
}
|
||||
|
||||
let successCount = 0;
|
||||
let failCount = 0;
|
||||
|
||||
if (task.useHierarchy) {
|
||||
// ---- Hierarchy mode: create sub-groups matching folder structure ----
|
||||
await this.appendLog(
|
||||
taskId,
|
||||
`Scanning directory with hierarchy: ${task.sourcePath}`,
|
||||
);
|
||||
|
||||
// Determine root group
|
||||
let rootGroupId = task.targetGroupId;
|
||||
if (!rootGroupId) {
|
||||
const rootName =
|
||||
task.targetGroupName || path.basename(task.sourcePath);
|
||||
const rootGroup = await this.groupService.create(
|
||||
task.userId,
|
||||
task.tenantId || 'default',
|
||||
{
|
||||
name: rootName,
|
||||
description: `Imported from ${task.sourcePath}`,
|
||||
color: '#0078D4',
|
||||
},
|
||||
);
|
||||
rootGroupId = rootGroup.id;
|
||||
await this.appendLog(taskId, `Created root group: ${rootName}`);
|
||||
}
|
||||
|
||||
// Map from relative dir path -> groupId
|
||||
const dirToGroupId = new Map<string, string>();
|
||||
dirToGroupId.set('.', rootGroupId);
|
||||
|
||||
// Collect all files first
|
||||
const allFiles = this.scanDir(task.sourcePath);
|
||||
await this.appendLog(taskId, `Found ${allFiles.length} files.`);
|
||||
|
||||
for (let i = 0; i < allFiles.length; i++) {
|
||||
const filePath = allFiles[i];
|
||||
const relativeDir = path.relative(
|
||||
task.sourcePath,
|
||||
path.dirname(filePath),
|
||||
);
|
||||
const normalizedDir = relativeDir || '.';
|
||||
|
||||
// Ensure group exists for this directory
|
||||
const groupId = await this.ensureHierarchyGroup(
|
||||
task.userId,
|
||||
task.tenantId || 'default',
|
||||
normalizedDir,
|
||||
dirToGroupId,
|
||||
task.sourcePath,
|
||||
taskId,
|
||||
);
|
||||
|
||||
try {
|
||||
const kb = await this.importSingleFile(
|
||||
filePath,
|
||||
task,
|
||||
importTargetDir,
|
||||
i,
|
||||
allFiles.length,
|
||||
);
|
||||
await this.groupService.addFilesToGroup(
|
||||
kb.id,
|
||||
[groupId],
|
||||
task.userId,
|
||||
task.tenantId || 'default',
|
||||
);
|
||||
successCount++;
|
||||
if (successCount % 10 === 0) {
|
||||
await this.appendLog(taskId, `Imported ${successCount} files...`);
|
||||
}
|
||||
} catch (e) {
|
||||
failCount++;
|
||||
await this.appendLog(
|
||||
taskId,
|
||||
`Failed to import ${path.basename(filePath)}: ${e.message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// ---- Single-group mode (original behavior) ----
|
||||
let groupId = task.targetGroupId;
|
||||
if (!groupId && task.targetGroupName) {
|
||||
const group = await this.groupService.create(
|
||||
task.userId,
|
||||
task.tenantId || 'default',
|
||||
{
|
||||
name: task.targetGroupName,
|
||||
description: `Imported from ${task.sourcePath}`,
|
||||
color: '#0078D4',
|
||||
},
|
||||
);
|
||||
groupId = group.id;
|
||||
await this.appendLog(
|
||||
taskId,
|
||||
`Created new group: ${task.targetGroupName}`,
|
||||
);
|
||||
} else if (!groupId) {
|
||||
throw new Error(this.i18nService.getMessage('targetGroupRequired'));
|
||||
}
|
||||
|
||||
await this.appendLog(taskId, `Scanning directory: ${task.sourcePath}`);
|
||||
const filesToImport = this.scanDir(task.sourcePath);
|
||||
await this.appendLog(taskId, `Found ${filesToImport.length} files.`);
|
||||
|
||||
for (let i = 0; i < filesToImport.length; i++) {
|
||||
const filePath = filesToImport[i];
|
||||
try {
|
||||
const kb = await this.importSingleFile(
|
||||
filePath,
|
||||
task,
|
||||
importTargetDir,
|
||||
i,
|
||||
filesToImport.length,
|
||||
);
|
||||
await this.groupService.addFilesToGroup(
|
||||
kb.id,
|
||||
[groupId],
|
||||
task.userId,
|
||||
task.tenantId || 'default',
|
||||
);
|
||||
successCount++;
|
||||
if (successCount % 10 === 0) {
|
||||
await this.appendLog(taskId, `Imported ${successCount} files...`);
|
||||
}
|
||||
} catch (e) {
|
||||
failCount++;
|
||||
await this.appendLog(
|
||||
taskId,
|
||||
`Failed to import ${path.basename(filePath)}: ${e.message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await this.updateStatus(
|
||||
taskId,
|
||||
'COMPLETED',
|
||||
`Import finished. Success: ${successCount}, Failed: ${failCount}`,
|
||||
);
|
||||
} catch (error) {
|
||||
await this.updateStatus(
|
||||
taskId,
|
||||
'FAILED',
|
||||
`Fatal error: ${error.message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure a KnowledgeGroup exists for each segment of the relative directory path.
|
||||
* Returns the groupId for the leaf directory.
|
||||
*/
|
||||
private async ensureHierarchyGroup(
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
relativeDir: string,
|
||||
dirToGroupId: Map<string, string>,
|
||||
_sourcePath: string,
|
||||
taskId: string,
|
||||
): Promise<string> {
|
||||
if (dirToGroupId.has(relativeDir)) {
|
||||
return dirToGroupId.get(relativeDir)!;
|
||||
}
|
||||
|
||||
const segments = relativeDir.split(path.sep);
|
||||
let currentPath = '';
|
||||
let parentGroupId =
|
||||
dirToGroupId.get('.') ?? dirToGroupId.values().next().value;
|
||||
|
||||
for (const segment of segments) {
|
||||
currentPath = currentPath ? path.join(currentPath, segment) : segment;
|
||||
if (dirToGroupId.has(currentPath)) {
|
||||
parentGroupId = dirToGroupId.get(currentPath)!;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Create a group for this directory segment
|
||||
const group = await this.groupService.findOrCreate(
|
||||
userId,
|
||||
tenantId,
|
||||
segment,
|
||||
parentGroupId,
|
||||
`Sub-folder: ${currentPath}`,
|
||||
);
|
||||
dirToGroupId.set(currentPath, group.id);
|
||||
await this.appendLog(taskId, `Created sub-group: ${segment}`);
|
||||
parentGroupId = group.id;
|
||||
}
|
||||
|
||||
return parentGroupId;
|
||||
}
|
||||
|
||||
/** Copy file to safe location and index it */
|
||||
private async importSingleFile(
|
||||
filePath: string,
|
||||
task: ImportTask,
|
||||
importTargetDir: string,
|
||||
index: number,
|
||||
total: number,
|
||||
) {
|
||||
const filename = path.basename(filePath);
|
||||
const storedFilename = `imported-${Date.now()}-${Math.random().toString(36).substr(2, 5)}-${filename}`;
|
||||
const targetPath = path.join(importTargetDir, storedFilename);
|
||||
|
||||
fs.copyFileSync(filePath, targetPath);
|
||||
const stats = fs.statSync(targetPath);
|
||||
|
||||
const fileInfo = {
|
||||
filename: storedFilename,
|
||||
originalname: filename,
|
||||
path: targetPath,
|
||||
mimetype: 'text/markdown',
|
||||
size: stats.size,
|
||||
};
|
||||
|
||||
const indexingConfig = {
|
||||
chunkSize: task.chunkSize || 500,
|
||||
chunkOverlap: task.chunkOverlap || 50,
|
||||
embeddingModelId: task.embeddingModelId,
|
||||
mode: (task.mode || 'fast') as 'fast' | 'precise',
|
||||
};
|
||||
|
||||
this.logger.log(`Processing file ${index + 1}/${total}: ${filename}`);
|
||||
const kb = await this.kbService.createAndIndex(
|
||||
fileInfo,
|
||||
task.userId,
|
||||
task.tenantId || 'default',
|
||||
{
|
||||
...indexingConfig,
|
||||
waitForCompletion: true,
|
||||
} as any,
|
||||
);
|
||||
this.logger.log(
|
||||
`File ${index + 1}/${total} processing completed: ${filename}`,
|
||||
);
|
||||
return kb;
|
||||
}
|
||||
|
||||
private scanDir(directory: string): string[] {
|
||||
let results: string[] = [];
|
||||
const items = fs.readdirSync(directory);
|
||||
for (const item of items) {
|
||||
const fullPath = path.join(directory, item);
|
||||
const stat = fs.statSync(fullPath);
|
||||
if (stat.isDirectory()) {
|
||||
results = results.concat(this.scanDir(fullPath));
|
||||
} else {
|
||||
if (item.match(/\.(md|txt|html|json|pdf|docx|xlsx|pptx|csv)$/i)) {
|
||||
results.push(fullPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
private async updateStatus(
|
||||
id: string,
|
||||
status: ImportTask['status'],
|
||||
logMessage?: string,
|
||||
) {
|
||||
const task = await this.taskRepository.findOne({ where: { id } });
|
||||
if (task) {
|
||||
task.status = status;
|
||||
if (logMessage) {
|
||||
task.logs =
|
||||
(task.logs || '') + `[${new Date().toISOString()}] ${logMessage}\n`;
|
||||
}
|
||||
await this.taskRepository.save(task);
|
||||
}
|
||||
}
|
||||
|
||||
private async appendLog(id: string, message: string) {
|
||||
const task = await this.taskRepository.findOne({ where: { id } });
|
||||
if (task) {
|
||||
task.logs =
|
||||
(task.logs || '') + `[${new Date().toISOString()}] ${message}\n`;
|
||||
await this.taskRepository.save(task);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,393 @@
|
||||
import { Injectable, Logger, BadRequestException } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { ModelConfigService } from '../model-config/model-config.service';
|
||||
import { TenantService } from '../tenant/tenant.service';
|
||||
// import { UserSettingService } from '../user-setting/user-setting.service';
|
||||
|
||||
/**
|
||||
* Chunk config service
|
||||
* Responsible for validating and managing chunk parameters to ensure they conform to model limits and environment variable settings
|
||||
*
|
||||
* Priority of limits:
|
||||
* 1. Environment variables (MAX_CHUNK_SIZE, MAX_OVERLAP_SIZE)
|
||||
* 2. Model settings in database (maxInputTokens, maxBatchSize)
|
||||
* 3. Default values
|
||||
*/
|
||||
import {
|
||||
DEFAULT_CHUNK_SIZE,
|
||||
MIN_CHUNK_SIZE,
|
||||
DEFAULT_CHUNK_OVERLAP,
|
||||
MIN_CHUNK_OVERLAP,
|
||||
DEFAULT_MAX_OVERLAP_RATIO,
|
||||
DEFAULT_MAX_BATCH_SIZE,
|
||||
DEFAULT_VECTOR_DIMENSIONS,
|
||||
} from '../common/constants';
|
||||
import { I18nService } from '../i18n/i18n.service';
|
||||
|
||||
@Injectable()
|
||||
export class ChunkConfigService {
|
||||
private readonly logger = new Logger(ChunkConfigService.name);
|
||||
|
||||
// Default settings
|
||||
private readonly DEFAULTS = {
|
||||
chunkSize: DEFAULT_CHUNK_SIZE,
|
||||
chunkOverlap: DEFAULT_CHUNK_OVERLAP,
|
||||
minChunkSize: MIN_CHUNK_SIZE,
|
||||
minChunkOverlap: MIN_CHUNK_OVERLAP,
|
||||
maxOverlapRatio: DEFAULT_MAX_OVERLAP_RATIO, // Overlap up to 50% of chunk size
|
||||
maxBatchSize: DEFAULT_MAX_BATCH_SIZE, // Default batch limit
|
||||
expectedDimensions: DEFAULT_VECTOR_DIMENSIONS, // Default vector dimensions
|
||||
};
|
||||
|
||||
// Upper limits set by environment variables (used first)
|
||||
private readonly envMaxChunkSize: number;
|
||||
private readonly envMaxOverlapSize: number;
|
||||
|
||||
constructor(
|
||||
private configService: ConfigService,
|
||||
private modelConfigService: ModelConfigService,
|
||||
private i18nService: I18nService,
|
||||
private tenantService: TenantService,
|
||||
) {
|
||||
// Load global limit settings from environment variables
|
||||
this.envMaxChunkSize = parseInt(
|
||||
this.configService.get<string>('MAX_CHUNK_SIZE', '8191'),
|
||||
);
|
||||
this.envMaxOverlapSize = parseInt(
|
||||
this.configService.get<string>('MAX_OVERLAP_SIZE', '2000'),
|
||||
);
|
||||
|
||||
this.logger.log(
|
||||
`Environment variable limits: MAX_CHUNK_SIZE=${this.envMaxChunkSize}, MAX_OVERLAP_SIZE=${this.envMaxOverlapSize}`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get model limit settings (read from database)
|
||||
*/
|
||||
async getModelLimits(modelId: string): Promise<{
|
||||
maxInputTokens: number;
|
||||
maxBatchSize: number;
|
||||
expectedDimensions: number;
|
||||
providerName: string;
|
||||
isVectorModel: boolean;
|
||||
}> {
|
||||
const modelConfig = await this.modelConfigService.findOne(modelId);
|
||||
|
||||
if (!modelConfig || modelConfig.type !== 'embedding') {
|
||||
throw new BadRequestException(
|
||||
this.i18nService.formatMessage('embeddingModelNotFound', {
|
||||
id: modelId,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
// Get limits from database fields and fill with defaults
|
||||
const maxInputTokens = modelConfig.maxInputTokens || this.envMaxChunkSize;
|
||||
const maxBatchSize = modelConfig.maxBatchSize || this.DEFAULTS.maxBatchSize;
|
||||
const expectedDimensions =
|
||||
modelConfig.dimensions ||
|
||||
parseInt(
|
||||
this.configService.get(
|
||||
'DEFAULT_VECTOR_DIMENSIONS',
|
||||
String(this.DEFAULTS.expectedDimensions),
|
||||
),
|
||||
);
|
||||
const providerName = modelConfig.providerName || 'unknown';
|
||||
const isVectorModel = modelConfig.isVectorModel || false;
|
||||
|
||||
this.logger.log(
|
||||
this.i18nService.formatMessage('configLoaded', {
|
||||
name: modelConfig.name,
|
||||
id: modelConfig.modelId,
|
||||
}) +
|
||||
'\n' +
|
||||
` - Provider: ${providerName}\n` +
|
||||
` - Token limit: ${maxInputTokens}\n` +
|
||||
` - Batch limit: ${maxBatchSize}\n` +
|
||||
` - Vector dimensions: ${expectedDimensions}\n` +
|
||||
` - Is vector model: ${isVectorModel}`,
|
||||
);
|
||||
|
||||
return {
|
||||
maxInputTokens,
|
||||
maxBatchSize,
|
||||
expectedDimensions,
|
||||
providerName,
|
||||
isVectorModel,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate and fix chunk config
|
||||
* Priority: Environment variable limits > Model limits > User settings
|
||||
*/
|
||||
async validateChunkConfig(
|
||||
chunkSize: number,
|
||||
chunkOverlap: number,
|
||||
modelId: string,
|
||||
): Promise<{
|
||||
chunkSize: number;
|
||||
chunkOverlap: number;
|
||||
warnings: string[];
|
||||
effectiveMaxChunkSize: number;
|
||||
effectiveMaxOverlapSize: number;
|
||||
}> {
|
||||
const warnings: string[] = [];
|
||||
const limits = await this.getModelLimits(modelId);
|
||||
|
||||
// 1. Calculate final limits (choose smaller of env var and model limit)
|
||||
const effectiveMaxChunkSize = Math.min(
|
||||
this.envMaxChunkSize,
|
||||
limits.maxInputTokens,
|
||||
);
|
||||
|
||||
const effectiveMaxOverlapSize = Math.min(
|
||||
this.envMaxOverlapSize,
|
||||
Math.floor(effectiveMaxChunkSize * this.DEFAULTS.maxOverlapRatio),
|
||||
);
|
||||
|
||||
// 2. Validate chunk size upper limit
|
||||
if (chunkSize > effectiveMaxChunkSize) {
|
||||
const reason =
|
||||
this.envMaxChunkSize < limits.maxInputTokens
|
||||
? `${this.i18nService.getMessage('environmentLimit')} ${this.envMaxChunkSize}`
|
||||
: `${this.i18nService.getMessage('modelLimit')} ${limits.maxInputTokens}`;
|
||||
|
||||
warnings.push(
|
||||
this.i18nService.formatMessage('chunkOverflow', {
|
||||
size: chunkSize,
|
||||
max: effectiveMaxChunkSize,
|
||||
reason,
|
||||
}),
|
||||
);
|
||||
chunkSize = effectiveMaxChunkSize;
|
||||
}
|
||||
|
||||
// 3. Validate chunk size lower limit
|
||||
if (chunkSize < this.DEFAULTS.minChunkSize) {
|
||||
warnings.push(
|
||||
this.i18nService.formatMessage('chunkUnderflow', {
|
||||
size: chunkSize,
|
||||
min: this.DEFAULTS.minChunkSize,
|
||||
}),
|
||||
);
|
||||
chunkSize = this.DEFAULTS.minChunkSize;
|
||||
}
|
||||
|
||||
// 4. Validate overlap size upper limit (env var first)
|
||||
if (chunkOverlap > effectiveMaxOverlapSize) {
|
||||
warnings.push(
|
||||
this.i18nService.formatMessage('overlapOverflow', {
|
||||
size: chunkOverlap,
|
||||
max: effectiveMaxOverlapSize,
|
||||
}),
|
||||
);
|
||||
chunkOverlap = effectiveMaxOverlapSize;
|
||||
}
|
||||
|
||||
// 5. Validate overlap doesn't exceed 50% of chunk size
|
||||
const maxOverlapByRatio = Math.floor(
|
||||
chunkSize * this.DEFAULTS.maxOverlapRatio,
|
||||
);
|
||||
if (chunkOverlap > maxOverlapByRatio) {
|
||||
warnings.push(
|
||||
this.i18nService.formatMessage('overlapRatioExceeded', {
|
||||
size: chunkOverlap,
|
||||
max: maxOverlapByRatio,
|
||||
}),
|
||||
);
|
||||
chunkOverlap = maxOverlapByRatio;
|
||||
}
|
||||
|
||||
if (chunkOverlap < this.DEFAULTS.minChunkOverlap) {
|
||||
warnings.push(
|
||||
this.i18nService.formatMessage('overlapUnderflow', {
|
||||
size: chunkOverlap,
|
||||
min: this.DEFAULTS.minChunkOverlap,
|
||||
}),
|
||||
);
|
||||
chunkOverlap = this.DEFAULTS.minChunkOverlap;
|
||||
}
|
||||
|
||||
// 6. Add safety check for batch processing
|
||||
// During batch processing, ensure total length of multiple texts doesn't exceed model limits
|
||||
const safetyMargin = 0.8; // 80% safety margin to leave space for batch processing
|
||||
const safeChunkSize = Math.floor(effectiveMaxChunkSize * safetyMargin);
|
||||
|
||||
if (chunkSize > safeChunkSize) {
|
||||
warnings.push(
|
||||
this.i18nService.formatMessage('batchOverflowWarning', {
|
||||
safeSize: safeChunkSize,
|
||||
size: chunkSize,
|
||||
percent: Math.round(safetyMargin * 100),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
// 7. Check if estimated chunk count is reasonable
|
||||
const estimatedChunkCount = this.estimateChunkCount(
|
||||
1000000, // Assume 1MB text
|
||||
chunkSize,
|
||||
);
|
||||
|
||||
if (estimatedChunkCount > 50000) {
|
||||
warnings.push(
|
||||
this.i18nService.formatMessage('estimatedChunkCountExcessive', {
|
||||
count: estimatedChunkCount,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
chunkSize,
|
||||
chunkOverlap,
|
||||
warnings,
|
||||
effectiveMaxChunkSize,
|
||||
effectiveMaxOverlapSize,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get recommended batch size
|
||||
*/
|
||||
async getRecommendedBatchSize(
|
||||
modelId: string,
|
||||
currentBatchSize: number = 100,
|
||||
): Promise<number> {
|
||||
const limits = await this.getModelLimits(modelId);
|
||||
|
||||
// Choose smaller of configured value and model limit
|
||||
const recommended = Math.min(
|
||||
currentBatchSize,
|
||||
limits.maxBatchSize,
|
||||
200, // Safety upper limit
|
||||
);
|
||||
|
||||
if (recommended < currentBatchSize) {
|
||||
this.logger.warn(
|
||||
this.i18nService.formatMessage('batchSizeAdjusted', {
|
||||
old: currentBatchSize,
|
||||
new: recommended,
|
||||
limit: limits.maxBatchSize,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
return Math.max(10, recommended); // Minimum 10
|
||||
}
|
||||
|
||||
/**
|
||||
* Estimate chunk count
|
||||
*/
|
||||
estimateChunkCount(textLength: number, chunkSize: number): number {
|
||||
const chunkSizeInChars = chunkSize * 4; // 1 token ≈ 4 chars
|
||||
return Math.ceil(textLength / chunkSizeInChars);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate vector dimensions
|
||||
*/
|
||||
async validateDimensions(
|
||||
modelId: string,
|
||||
actualDimensions: number,
|
||||
): Promise<boolean> {
|
||||
const limits = await this.getModelLimits(modelId);
|
||||
|
||||
if (actualDimensions !== limits.expectedDimensions) {
|
||||
this.logger.warn(
|
||||
this.i18nService.formatMessage('dimensionMismatch', {
|
||||
id: modelId,
|
||||
expected: limits.expectedDimensions,
|
||||
actual: actualDimensions,
|
||||
}),
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get config summary (for logging)
|
||||
*/
|
||||
async getConfigSummary(
|
||||
chunkSize: number,
|
||||
chunkOverlap: number,
|
||||
modelId: string,
|
||||
): Promise<string> {
|
||||
const limits = await this.getModelLimits(modelId);
|
||||
|
||||
return [
|
||||
`Model: ${modelId}`,
|
||||
`Chunk size: ${chunkSize} tokens (limit: ${limits.maxInputTokens})`,
|
||||
`Overlap size: ${chunkOverlap} tokens`,
|
||||
`Batch size: ${limits.maxBatchSize}`,
|
||||
`Vector dimensions: ${limits.expectedDimensions}`,
|
||||
].join(', ');
|
||||
}
|
||||
|
||||
/**
|
||||
* Get config limits for frontend
|
||||
* Used for frontend slider max value settings
|
||||
*/
|
||||
async getFrontendLimits(
|
||||
modelId: string,
|
||||
userId: string,
|
||||
tenantId?: string,
|
||||
): Promise<{
|
||||
maxChunkSize: number;
|
||||
maxOverlapSize: number;
|
||||
minOverlapSize: number;
|
||||
defaultChunkSize: number;
|
||||
defaultOverlapSize: number;
|
||||
modelInfo: {
|
||||
name: string;
|
||||
maxInputTokens: number;
|
||||
maxBatchSize: number;
|
||||
expectedDimensions: number;
|
||||
};
|
||||
}> {
|
||||
const limits = await this.getModelLimits(modelId);
|
||||
|
||||
// Calculate final limits (choose smaller of env var and model limit)
|
||||
const maxChunkSize = Math.min(this.envMaxChunkSize, limits.maxInputTokens);
|
||||
const maxOverlapSize = Math.min(
|
||||
this.envMaxOverlapSize,
|
||||
Math.floor(maxChunkSize * this.DEFAULTS.maxOverlapRatio),
|
||||
);
|
||||
|
||||
// Get model config name
|
||||
const modelConfig = await this.modelConfigService.findOne(modelId);
|
||||
const modelName = modelConfig?.name || 'Unknown';
|
||||
|
||||
// Get defaults from tenant or user settings
|
||||
let defaultChunkSize = this.DEFAULTS.chunkSize;
|
||||
let defaultOverlapSize = this.DEFAULTS.chunkOverlap;
|
||||
|
||||
if (tenantId) {
|
||||
const tenantSettings = await this.tenantService.getSettings(tenantId);
|
||||
if (tenantSettings?.chunkSize)
|
||||
defaultChunkSize = tenantSettings.chunkSize;
|
||||
if (tenantSettings?.chunkOverlap)
|
||||
defaultOverlapSize = tenantSettings.chunkOverlap;
|
||||
}
|
||||
|
||||
return {
|
||||
maxChunkSize,
|
||||
maxOverlapSize,
|
||||
minOverlapSize: this.DEFAULTS.minChunkOverlap,
|
||||
defaultChunkSize: Math.min(defaultChunkSize, maxChunkSize),
|
||||
defaultOverlapSize: Math.max(
|
||||
this.DEFAULTS.minChunkOverlap,
|
||||
Math.min(defaultOverlapSize, maxOverlapSize),
|
||||
),
|
||||
modelInfo: {
|
||||
name: modelName,
|
||||
maxInputTokens: limits.maxInputTokens,
|
||||
maxBatchSize: limits.maxBatchSize,
|
||||
expectedDimensions: limits.expectedDimensions,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
import { IsNotEmpty, IsOptional, IsString } from 'class-validator';
|
||||
|
||||
export class CreateKnowledgeBaseDto {
|
||||
@IsString()
|
||||
@IsNotEmpty()
|
||||
name: string;
|
||||
|
||||
@IsString()
|
||||
@IsOptional()
|
||||
description?: string;
|
||||
}
|
||||
@@ -0,0 +1,286 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { ModelConfigService } from '../model-config/model-config.service';
|
||||
import { I18nService } from '../i18n/i18n.service';
|
||||
|
||||
export interface EmbeddingResponse {
|
||||
data: Array<{
|
||||
embedding: number[];
|
||||
index: number;
|
||||
}>;
|
||||
model: string;
|
||||
usage: {
|
||||
prompt_tokens: number;
|
||||
total_tokens: number;
|
||||
};
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class EmbeddingService {
|
||||
private readonly logger = new Logger(EmbeddingService.name);
|
||||
private readonly defaultDimensions: number;
|
||||
|
||||
constructor(
|
||||
private modelConfigService: ModelConfigService,
|
||||
private configService: ConfigService,
|
||||
private i18nService: I18nService,
|
||||
) {
|
||||
this.defaultDimensions = parseInt(
|
||||
this.configService.get<string>('DEFAULT_VECTOR_DIMENSIONS', '2560'),
|
||||
);
|
||||
this.logger.log(
|
||||
`Default vector dimensions set to ${this.defaultDimensions}`,
|
||||
);
|
||||
}
|
||||
|
||||
async getEmbeddings(
|
||||
texts: string[],
|
||||
embeddingModelConfigId: string,
|
||||
): Promise<number[][]> {
|
||||
this.logger.log(`Generating embeddings for ${texts.length} texts`);
|
||||
|
||||
const modelConfig = await this.modelConfigService.findOne(
|
||||
embeddingModelConfigId,
|
||||
);
|
||||
if (!modelConfig || modelConfig.type !== 'embedding') {
|
||||
throw new Error(
|
||||
this.i18nService.formatMessage('embeddingModelNotFound', {
|
||||
id: embeddingModelConfigId,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
if (modelConfig.isEnabled === false) {
|
||||
throw new Error(
|
||||
`Model ${modelConfig.name} is disabled and cannot generate embeddings`,
|
||||
);
|
||||
}
|
||||
|
||||
if (!modelConfig.baseUrl) {
|
||||
throw new Error(
|
||||
`Model ${modelConfig.name} does not have baseUrl configured`,
|
||||
);
|
||||
}
|
||||
|
||||
// Determine max batch size based on model name
|
||||
const maxBatchSize = this.getMaxBatchSizeForModel(
|
||||
modelConfig.modelId,
|
||||
modelConfig.maxBatchSize,
|
||||
);
|
||||
|
||||
// Split processing if batch size exceeds limit
|
||||
if (texts.length > maxBatchSize) {
|
||||
this.logger.log(
|
||||
`Splitting ${texts.length} texts into batches (model batch limit: ${maxBatchSize})`,
|
||||
);
|
||||
|
||||
const allEmbeddings: number[][] = [];
|
||||
|
||||
for (let i = 0; i < texts.length; i += maxBatchSize) {
|
||||
const batch = texts.slice(i, i + maxBatchSize);
|
||||
const batchEmbeddings = await this.getEmbeddingsForBatch(
|
||||
batch,
|
||||
modelConfig,
|
||||
maxBatchSize,
|
||||
);
|
||||
|
||||
allEmbeddings.push(...batchEmbeddings);
|
||||
|
||||
// Wait briefly to avoid API rate limiting
|
||||
if (i + maxBatchSize < texts.length) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 100)); // Wait 100ms
|
||||
}
|
||||
}
|
||||
|
||||
return allEmbeddings;
|
||||
} else {
|
||||
// Normal processing (within batch size)
|
||||
return await this.getEmbeddingsForBatch(
|
||||
texts,
|
||||
modelConfig,
|
||||
maxBatchSize,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine max batch size based on model ID
|
||||
*/
|
||||
private getMaxBatchSizeForModel(
|
||||
modelId: string,
|
||||
configuredMaxBatchSize?: number,
|
||||
): number {
|
||||
// Model-specific batch size limits
|
||||
if (
|
||||
modelId.includes('text-embedding-004') ||
|
||||
modelId.includes('text-embedding-v4') ||
|
||||
modelId.includes('text-embedding-ada-002')
|
||||
) {
|
||||
return Math.min(10, configuredMaxBatchSize || 100); // Google limit: 10
|
||||
} else if (
|
||||
modelId.includes('text-embedding-3') ||
|
||||
modelId.includes('text-embedding-003')
|
||||
) {
|
||||
return Math.min(2048, configuredMaxBatchSize || 2048); // OpenAI v3 limit: 2048
|
||||
} else {
|
||||
// Default: smaller of configured max or 100
|
||||
return Math.min(configuredMaxBatchSize || 100, 100);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process single batch embedding
|
||||
*/
|
||||
private async getEmbeddingsForBatch(
|
||||
texts: string[],
|
||||
modelConfig: any,
|
||||
maxBatchSize: number,
|
||||
): Promise<number[][]> {
|
||||
const apiUrl = modelConfig.baseUrl.endsWith('/embeddings')
|
||||
? modelConfig.baseUrl
|
||||
: `${modelConfig.baseUrl}/embeddings`;
|
||||
|
||||
let lastError;
|
||||
const MAX_RETRIES = 3;
|
||||
|
||||
for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
|
||||
try {
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => {
|
||||
controller.abort();
|
||||
this.logger.error(`Embedding API timeout after 60s: ${apiUrl}`);
|
||||
}, 60000); // 60s timeout
|
||||
|
||||
this.logger.log(
|
||||
`[Model call] Type: Embedding, Model: ${modelConfig.name} (${modelConfig.modelId}), Text count: ${texts.length}`,
|
||||
);
|
||||
this.logger.log(
|
||||
`Calling embedding API (attempt ${attempt}/${MAX_RETRIES}): ${apiUrl}`,
|
||||
);
|
||||
|
||||
let response;
|
||||
try {
|
||||
response = await fetch(apiUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${modelConfig.apiKey}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
encoding_format: 'float',
|
||||
input: texts,
|
||||
model: modelConfig.modelId,
|
||||
}),
|
||||
signal: controller.signal,
|
||||
});
|
||||
} finally {
|
||||
clearTimeout(timeoutId);
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
|
||||
// Detect batch size limit error
|
||||
if (
|
||||
errorText.includes('batch size is invalid') ||
|
||||
errorText.includes('batch_size') ||
|
||||
errorText.includes('invalid') ||
|
||||
errorText.includes('larger than')
|
||||
) {
|
||||
this.logger.warn(
|
||||
`Batch size limit error detected. Splitting batch in half and retrying: ${maxBatchSize} -> ${Math.floor(maxBatchSize / 2)}`,
|
||||
);
|
||||
|
||||
// Split batch into smaller units and retry
|
||||
if (texts.length > 1) {
|
||||
const midPoint = Math.floor(texts.length / 2);
|
||||
const firstHalf = texts.slice(0, midPoint);
|
||||
const secondHalf = texts.slice(midPoint);
|
||||
|
||||
const firstResult = await this.getEmbeddingsForBatch(
|
||||
firstHalf,
|
||||
modelConfig,
|
||||
Math.floor(maxBatchSize / 2),
|
||||
);
|
||||
const secondResult = await this.getEmbeddingsForBatch(
|
||||
secondHalf,
|
||||
modelConfig,
|
||||
Math.floor(maxBatchSize / 2),
|
||||
);
|
||||
|
||||
return [...firstResult, ...secondResult];
|
||||
}
|
||||
}
|
||||
|
||||
// Detect context length excess error
|
||||
if (
|
||||
errorText.includes('context length') ||
|
||||
errorText.includes('exceeds')
|
||||
) {
|
||||
const avgLength =
|
||||
texts.reduce((s, t) => s + t.length, 0) / texts.length;
|
||||
const totalLength = texts.reduce((s, t) => s + t.length, 0);
|
||||
this.logger.error(
|
||||
`Text length exceeds limit: ${texts.length} texts, ` +
|
||||
`total ${totalLength} characters, average ${Math.round(avgLength)} characters, ` +
|
||||
`model limit: ${modelConfig.maxInputTokens || 8192} tokens`,
|
||||
);
|
||||
throw new Error(
|
||||
`Text length exceeds model limit. ` +
|
||||
`Current: ${texts.length} texts with total ${totalLength} characters, ` +
|
||||
`model limit: ${modelConfig.maxInputTokens || 8192} tokens. ` +
|
||||
`Advice: Reduce chunk size or batch size`,
|
||||
);
|
||||
}
|
||||
|
||||
// Retry on 429 (Too Many Requests) or 5xx (Server Error)
|
||||
if (response.status === 429 || response.status >= 500) {
|
||||
this.logger.warn(
|
||||
`Temporary error from embedding API (${response.status}): ${errorText}`,
|
||||
);
|
||||
throw new Error(`API Error ${response.status}: ${errorText}`);
|
||||
}
|
||||
|
||||
this.logger.error(`Embedding API error details: ${errorText}`);
|
||||
this.logger.error(
|
||||
`Request parameters: model=${modelConfig.modelId}, inputLength=${texts[0]?.length}`,
|
||||
);
|
||||
throw new Error(
|
||||
`Embedding API call failed: ${response.statusText} - ${errorText}`,
|
||||
);
|
||||
}
|
||||
|
||||
const data: EmbeddingResponse = await response.json();
|
||||
const embeddings = data.data.map((item) => item.embedding);
|
||||
|
||||
// Get dimensions from actual response
|
||||
const actualDimensions =
|
||||
embeddings[0]?.length || this.defaultDimensions;
|
||||
this.logger.log(
|
||||
`Got ${embeddings.length} embedding vectors from ${modelConfig.name}. Dimensions: ${actualDimensions}`,
|
||||
);
|
||||
|
||||
return embeddings;
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
|
||||
// If not the last attempt and error appears temporary (or for robustness on all), retry after waiting
|
||||
if (attempt < MAX_RETRIES) {
|
||||
const delay = Math.pow(2, attempt - 1) * 1000; // 1s, 2s, 4s
|
||||
this.logger.warn(
|
||||
`Embedding request failed. Retrying after ${delay}ms: ${error.message}`,
|
||||
);
|
||||
await new Promise((resolve) => setTimeout(resolve, delay));
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError;
|
||||
}
|
||||
|
||||
private getEstimatedDimensions(modelId: string): number {
|
||||
// Use default dimensions from environment variable
|
||||
return this.defaultDimensions;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,362 @@
|
||||
import {
|
||||
Body,
|
||||
Controller,
|
||||
Delete,
|
||||
Get,
|
||||
Param,
|
||||
Post,
|
||||
Query,
|
||||
Request,
|
||||
UseGuards,
|
||||
Res,
|
||||
NotFoundException,
|
||||
InternalServerErrorException,
|
||||
} from '@nestjs/common';
|
||||
import { Response } from 'express';
|
||||
import * as path from 'path';
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { KnowledgeBaseService } from './knowledge-base.service';
|
||||
import { CombinedAuthGuard } from '../auth/combined-auth.guard';
|
||||
import { RolesGuard } from '../auth/roles.guard';
|
||||
import { Roles } from '../auth/roles.decorator';
|
||||
import { UserRole } from '../user/user-role.enum';
|
||||
import { Public } from '../auth/public.decorator';
|
||||
import { KnowledgeBase } from './knowledge-base.entity';
|
||||
import { ChunkConfigService } from './chunk-config.service';
|
||||
import { KnowledgeGroupService } from '../knowledge-group/knowledge-group.service';
|
||||
import { I18nService } from '../i18n/i18n.service';
|
||||
|
||||
@Controller('knowledge-bases')
|
||||
@UseGuards(CombinedAuthGuard, RolesGuard)
|
||||
export class KnowledgeBaseController {
|
||||
private readonly logger = new Logger(KnowledgeBaseController.name);
|
||||
|
||||
constructor(
|
||||
private readonly knowledgeBaseService: KnowledgeBaseService,
|
||||
private readonly chunkConfigService: ChunkConfigService,
|
||||
private readonly knowledgeGroupService: KnowledgeGroupService,
|
||||
private readonly i18nService: I18nService,
|
||||
) {}
|
||||
|
||||
@Get()
|
||||
@UseGuards(CombinedAuthGuard)
|
||||
async findAll(@Request() req): Promise<KnowledgeBase[]> {
|
||||
return this.knowledgeBaseService.findAll(req.user.id, req.user.tenantId);
|
||||
}
|
||||
|
||||
@Get('stats')
|
||||
@UseGuards(CombinedAuthGuard)
|
||||
async getStats(
|
||||
@Request() req,
|
||||
): Promise<{ total: number; uncategorized: number }> {
|
||||
return this.knowledgeBaseService.getStats(req.user.id, req.user.tenantId);
|
||||
}
|
||||
|
||||
@Delete('clear')
|
||||
@Roles(UserRole.TENANT_ADMIN, UserRole.SUPER_ADMIN)
|
||||
async clearAll(@Request() req): Promise<{ message: string }> {
|
||||
await this.knowledgeBaseService.clearAll(req.user.id, req.user.tenantId);
|
||||
return { message: this.i18nService.getMessage('kbCleared') };
|
||||
}
|
||||
|
||||
@Post('search')
|
||||
async search(@Request() req, @Body() body: { query: string; topK?: number }) {
|
||||
return this.knowledgeBaseService.searchKnowledge(
|
||||
req.user.id,
|
||||
req.user.tenantId, // New
|
||||
body.query,
|
||||
body.topK || 5,
|
||||
);
|
||||
}
|
||||
|
||||
@Post('rag-search')
|
||||
async ragSearch(
|
||||
@Request() req,
|
||||
@Body() body: { query: string; settings: any },
|
||||
) {
|
||||
return this.knowledgeBaseService.ragSearch(
|
||||
req.user.id,
|
||||
req.user.tenantId, // New
|
||||
body.query,
|
||||
body.settings,
|
||||
);
|
||||
}
|
||||
|
||||
@Delete(':id')
|
||||
@Roles(UserRole.TENANT_ADMIN, UserRole.SUPER_ADMIN)
|
||||
async deleteFile(
|
||||
@Request() req,
|
||||
@Param('id') fileId: string,
|
||||
): Promise<{ message: string }> {
|
||||
await this.knowledgeBaseService.deleteFile(
|
||||
fileId,
|
||||
req.user.id,
|
||||
req.user.tenantId,
|
||||
);
|
||||
return { message: this.i18nService.getMessage('fileDeleted') };
|
||||
}
|
||||
|
||||
@Post(':id/retry')
|
||||
@Roles(UserRole.TENANT_ADMIN, UserRole.SUPER_ADMIN)
|
||||
async retryFile(
|
||||
@Request() req,
|
||||
@Param('id') fileId: string,
|
||||
): Promise<KnowledgeBase> {
|
||||
return this.knowledgeBaseService.retryFailedFile(
|
||||
fileId,
|
||||
req.user.id,
|
||||
req.user.tenantId,
|
||||
);
|
||||
}
|
||||
|
||||
@Get(':id/chunks')
|
||||
async getFileChunks(@Request() req, @Param('id') fileId: string) {
|
||||
return this.knowledgeBaseService.getFileChunks(
|
||||
fileId,
|
||||
req.user.id,
|
||||
req.user.tenantId,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get chunk config limits (for frontend slider settings)
|
||||
* Query parameter: embeddingModelId - embedding model ID
|
||||
*/
|
||||
@Get('chunk-config/limits')
|
||||
async getChunkConfigLimits(
|
||||
@Request() req,
|
||||
@Query('embeddingModelId') embeddingModelId: string,
|
||||
) {
|
||||
if (!embeddingModelId) {
|
||||
return {
|
||||
maxChunkSize: parseInt(process.env.MAX_CHUNK_SIZE || '8191'),
|
||||
maxOverlapSize: parseInt(process.env.MAX_OVERLAP_SIZE || '2000'),
|
||||
minOverlapSize: 25,
|
||||
defaultChunkSize: 200,
|
||||
defaultOverlapSize: 40,
|
||||
modelInfo: {
|
||||
name: this.i18nService.getMessage('modelNotConfigured'),
|
||||
maxInputTokens: parseInt(process.env.MAX_CHUNK_SIZE || '8191'),
|
||||
maxBatchSize: 2048,
|
||||
expectedDimensions: parseInt(
|
||||
process.env.DEFAULT_VECTOR_DIMENSIONS || '2560',
|
||||
),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
return await this.chunkConfigService.getFrontendLimits(
|
||||
embeddingModelId,
|
||||
req.user.id,
|
||||
req.user.tenantId,
|
||||
);
|
||||
}
|
||||
|
||||
// File group management - requires admin permission
|
||||
@Post(':id/groups')
|
||||
@Roles(UserRole.TENANT_ADMIN, UserRole.SUPER_ADMIN)
|
||||
async addFileToGroups(
|
||||
@Param('id') fileId: string,
|
||||
@Body() body: { groupIds: string[] },
|
||||
@Request() req,
|
||||
) {
|
||||
await this.knowledgeGroupService.addFilesToGroup(
|
||||
fileId,
|
||||
body.groupIds,
|
||||
req.user.id,
|
||||
req.user.tenantId,
|
||||
);
|
||||
return { message: this.i18nService.getMessage('groupSyncSuccess') };
|
||||
}
|
||||
|
||||
@Delete(':id/groups/:groupId')
|
||||
@Roles(UserRole.TENANT_ADMIN, UserRole.SUPER_ADMIN)
|
||||
async removeFileFromGroup(
|
||||
@Param('id') fileId: string,
|
||||
@Param('groupId') groupId: string,
|
||||
@Request() req,
|
||||
) {
|
||||
await this.knowledgeGroupService.removeFileFromGroup(
|
||||
fileId,
|
||||
groupId,
|
||||
req.user.id,
|
||||
req.user.tenantId,
|
||||
);
|
||||
return { message: this.i18nService.getMessage('fileDeletedFromGroup') };
|
||||
}
|
||||
|
||||
// PDF preview - public access
|
||||
@Public()
|
||||
@Get(':id/pdf')
|
||||
async getPDFPreview(
|
||||
@Param('id') fileId: string,
|
||||
@Query('token') token: string,
|
||||
@Res() res: Response,
|
||||
) {
|
||||
try {
|
||||
if (!token) {
|
||||
throw new NotFoundException(
|
||||
this.i18nService.getMessage('accessDeniedNoToken'),
|
||||
);
|
||||
}
|
||||
|
||||
const jwt = await import('jsonwebtoken');
|
||||
const secret = process.env.JWT_SECRET;
|
||||
if (!secret) {
|
||||
throw new InternalServerErrorException(
|
||||
this.i18nService.getMessage('jwtSecretRequired'),
|
||||
);
|
||||
}
|
||||
|
||||
let decoded;
|
||||
try {
|
||||
decoded = jwt.verify(token, secret) as any;
|
||||
} catch {
|
||||
throw new NotFoundException(
|
||||
this.i18nService.getMessage('invalidToken'),
|
||||
);
|
||||
}
|
||||
|
||||
if (decoded.type !== 'pdf-access' || decoded.fileId !== fileId) {
|
||||
throw new NotFoundException(
|
||||
this.i18nService.getMessage('invalidToken'),
|
||||
);
|
||||
}
|
||||
|
||||
const pdfPath = await this.knowledgeBaseService.ensurePDFExists(
|
||||
fileId,
|
||||
decoded.userId,
|
||||
decoded.tenantId, // New
|
||||
);
|
||||
|
||||
const fs = await import('fs');
|
||||
const path = await import('path');
|
||||
|
||||
if (!fs.existsSync(pdfPath)) {
|
||||
throw new NotFoundException(
|
||||
this.i18nService.getMessage('pdfFileNotFound'),
|
||||
);
|
||||
}
|
||||
|
||||
const stat = fs.statSync(pdfPath);
|
||||
const fileName = path.basename(pdfPath);
|
||||
|
||||
if (stat.size === 0) {
|
||||
this.logger.warn(`PDF file is empty: ${pdfPath}`);
|
||||
try {
|
||||
fs.unlinkSync(pdfPath); // Delete empty file
|
||||
} catch (e) {}
|
||||
throw new NotFoundException(
|
||||
this.i18nService.getMessage('pdfFileEmpty'),
|
||||
);
|
||||
}
|
||||
|
||||
res.setHeader('Content-Type', 'application/pdf');
|
||||
res.setHeader('Content-Length', stat.size);
|
||||
|
||||
const stream = fs.createReadStream(pdfPath);
|
||||
stream.pipe(res);
|
||||
} catch (error) {
|
||||
if (error instanceof NotFoundException) {
|
||||
throw error;
|
||||
}
|
||||
this.logger.error(`PDF preview error: ${error.message}`);
|
||||
throw new NotFoundException(
|
||||
this.i18nService.getMessage('pdfConversionFailed'),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Get PDF preview URL
|
||||
@Get(':id/pdf-url')
|
||||
async getPDFUrl(
|
||||
@Param('id') fileId: string,
|
||||
@Query('force') force: string,
|
||||
@Request() req,
|
||||
) {
|
||||
try {
|
||||
// Trigger PDF conversion
|
||||
await this.knowledgeBaseService.ensurePDFExists(
|
||||
fileId,
|
||||
req.user.id,
|
||||
req.user.tenantId,
|
||||
force === 'true',
|
||||
);
|
||||
|
||||
// Generate temporary access token
|
||||
const jwt = await import('jsonwebtoken');
|
||||
|
||||
const secret = process.env.JWT_SECRET;
|
||||
if (!secret) {
|
||||
throw new InternalServerErrorException(
|
||||
this.i18nService.getMessage('jwtSecretRequired'),
|
||||
);
|
||||
}
|
||||
|
||||
const token = jwt.sign(
|
||||
{
|
||||
fileId,
|
||||
userId: req.user.id,
|
||||
tenantId: req.user.tenantId,
|
||||
type: 'pdf-access',
|
||||
},
|
||||
secret,
|
||||
{ expiresIn: '1h' },
|
||||
);
|
||||
|
||||
return {
|
||||
url: `/api/knowledge-bases/${fileId}/pdf?token=${token}`,
|
||||
};
|
||||
} catch (error) {
|
||||
if (error.message.includes('LibreOffice')) {
|
||||
throw new InternalServerErrorException(
|
||||
this.i18nService.formatMessage('pdfServiceUnavailable', {
|
||||
message: error.message,
|
||||
}),
|
||||
);
|
||||
}
|
||||
throw new InternalServerErrorException(error.message);
|
||||
}
|
||||
}
|
||||
|
||||
@Get(':id/pdf-status')
|
||||
async getPDFStatus(@Param('id') fileId: string, @Request() req) {
|
||||
return await this.knowledgeBaseService.getPDFStatus(
|
||||
fileId,
|
||||
req.user.id,
|
||||
req.user.tenantId,
|
||||
);
|
||||
}
|
||||
|
||||
// Get specific page of PDF as image
|
||||
@Get(':id/page/:index')
|
||||
async getPageImage(
|
||||
@Param('id') fileId: string,
|
||||
@Param('index') index: number,
|
||||
@Request() req,
|
||||
@Res() res: Response,
|
||||
) {
|
||||
try {
|
||||
const imagePath = await this.knowledgeBaseService.getPageAsImage(
|
||||
fileId,
|
||||
Number(index),
|
||||
req.user.id,
|
||||
req.user.tenantId,
|
||||
);
|
||||
|
||||
const fs = await import('fs');
|
||||
if (!fs.existsSync(imagePath)) {
|
||||
throw new NotFoundException(
|
||||
this.i18nService.getMessage('pageImageNotFound'),
|
||||
);
|
||||
}
|
||||
|
||||
res.sendFile(path.resolve(imagePath));
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to get PDF page image: ${error.message}`);
|
||||
throw new NotFoundException(
|
||||
this.i18nService.getMessage('pdfPageImageFailed'),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
import {
|
||||
Column,
|
||||
CreateDateColumn,
|
||||
Entity,
|
||||
PrimaryGeneratedColumn,
|
||||
UpdateDateColumn,
|
||||
ManyToMany,
|
||||
ManyToOne,
|
||||
JoinColumn,
|
||||
} from 'typeorm';
|
||||
import { KnowledgeGroup } from '../knowledge-group/knowledge-group.entity';
|
||||
import { Tenant } from '../tenant/tenant.entity';
|
||||
|
||||
export enum FileStatus {
|
||||
PENDING = 'pending',
|
||||
INDEXING = 'indexing',
|
||||
EXTRACTED = 'extracted', // Text extraction completed and saved to database
|
||||
VECTORIZED = 'vectorized', // Vectorization completed and indexed to ES
|
||||
FAILED = 'failed',
|
||||
}
|
||||
|
||||
export enum ProcessingMode {
|
||||
FAST = 'fast', // Fast mode - use Tika
|
||||
PRECISE = 'precise', // Precise mode - use Vision Pipeline
|
||||
}
|
||||
|
||||
@Entity('knowledge_bases')
|
||||
export class KnowledgeBase {
|
||||
@PrimaryGeneratedColumn('uuid')
|
||||
id: string;
|
||||
|
||||
@Column({ name: 'original_name' })
|
||||
originalName: string;
|
||||
|
||||
@Column({ nullable: true })
|
||||
title: string;
|
||||
|
||||
@Column({ name: 'storage_path' })
|
||||
storagePath: string;
|
||||
|
||||
@Column({ type: 'integer', default: 0 })
|
||||
size: number;
|
||||
|
||||
@Column({ length: 100, nullable: true })
|
||||
mimetype: string;
|
||||
|
||||
@Column({
|
||||
type: 'simple-enum',
|
||||
enum: FileStatus,
|
||||
default: FileStatus.PENDING,
|
||||
})
|
||||
status: FileStatus;
|
||||
|
||||
@Column({ name: 'user_id', nullable: true }) // Temporarily allowed empty (for debugging), should be required in future
|
||||
userId: string;
|
||||
|
||||
@Column({ name: 'tenant_id', nullable: true, type: 'text' })
|
||||
tenantId: string;
|
||||
|
||||
@ManyToOne(() => Tenant, { nullable: true, onDelete: 'CASCADE' })
|
||||
@JoinColumn({ name: 'tenant_id' })
|
||||
tenant: Tenant;
|
||||
|
||||
@Column({ type: 'text', nullable: true })
|
||||
content: string; // Stores text content extracted by Tika
|
||||
|
||||
// Index setting parameters
|
||||
@Column({ name: 'chunk_size', type: 'integer', default: 1000 })
|
||||
chunkSize: number;
|
||||
|
||||
@Column({ name: 'chunk_overlap', type: 'integer', default: 200 })
|
||||
chunkOverlap: number;
|
||||
|
||||
@Column({ name: 'embedding_model_id', nullable: true })
|
||||
embeddingModelId: string;
|
||||
|
||||
@Column({
|
||||
type: 'simple-enum',
|
||||
enum: ProcessingMode,
|
||||
default: ProcessingMode.FAST,
|
||||
name: 'processing_mode',
|
||||
})
|
||||
processingMode: ProcessingMode;
|
||||
|
||||
@Column({ type: 'json', nullable: true })
|
||||
metadata: any; // Stores additional metadata (image descriptions, confidence, etc.)
|
||||
|
||||
@Column({ name: 'pdf_path', nullable: true })
|
||||
pdfPath: string; // PDF file path (for preview)
|
||||
|
||||
@ManyToMany(() => KnowledgeGroup, (group) => group.knowledgeBases)
|
||||
groups: KnowledgeGroup[];
|
||||
|
||||
@CreateDateColumn({ name: 'created_at' })
|
||||
createdAt: Date;
|
||||
|
||||
@UpdateDateColumn({ name: 'updated_at' })
|
||||
updatedAt: Date;
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
import { Module, forwardRef } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
import { KnowledgeBase } from './knowledge-base.entity';
|
||||
import { KnowledgeGroup } from '../knowledge-group/knowledge-group.entity';
|
||||
import { KnowledgeBaseService } from './knowledge-base.service';
|
||||
import { KnowledgeBaseController } from './knowledge-base.controller';
|
||||
import { ElasticsearchModule } from '../elasticsearch/elasticsearch.module';
|
||||
import { TikaModule } from '../tika/tika.module';
|
||||
import { ModelConfigModule } from '../model-config/model-config.module';
|
||||
import { EmbeddingService } from './embedding.service';
|
||||
import { TextChunkerService } from './text-chunker.service';
|
||||
import { RagModule } from '../rag/rag.module';
|
||||
import { VisionModule } from '../vision/vision.module';
|
||||
import { MemoryMonitorService } from './memory-monitor.service';
|
||||
import { ChunkConfigService } from './chunk-config.service';
|
||||
import { LibreOfficeModule } from '../libreoffice/libreoffice.module';
|
||||
import { Pdf2ImageModule } from '../pdf2image/pdf2image.module';
|
||||
import { VisionPipelineModule } from '../vision-pipeline/vision-pipeline.module';
|
||||
import { KnowledgeGroupModule } from '../knowledge-group/knowledge-group.module';
|
||||
import { ChatModule } from '../chat/chat.module';
|
||||
import { UserModule } from '../user/user.module';
|
||||
import { TenantModule } from '../tenant/tenant.module';
|
||||
import { CombinedAuthGuard } from '../auth/combined-auth.guard';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
TypeOrmModule.forFeature([KnowledgeBase, KnowledgeGroup]),
|
||||
forwardRef(() => ElasticsearchModule),
|
||||
TikaModule,
|
||||
ModelConfigModule,
|
||||
forwardRef(() => RagModule),
|
||||
VisionModule,
|
||||
LibreOfficeModule,
|
||||
Pdf2ImageModule,
|
||||
VisionPipelineModule,
|
||||
forwardRef(() => KnowledgeGroupModule),
|
||||
forwardRef(() => ChatModule),
|
||||
UserModule,
|
||||
TenantModule,
|
||||
],
|
||||
controllers: [KnowledgeBaseController],
|
||||
providers: [
|
||||
KnowledgeBaseService,
|
||||
EmbeddingService,
|
||||
TextChunkerService,
|
||||
MemoryMonitorService,
|
||||
ChunkConfigService,
|
||||
CombinedAuthGuard,
|
||||
],
|
||||
exports: [KnowledgeBaseService, EmbeddingService],
|
||||
})
|
||||
export class KnowledgeBaseModule {}
|
||||
@@ -0,0 +1,1826 @@
|
||||
import {
|
||||
Injectable,
|
||||
Logger,
|
||||
NotFoundException,
|
||||
ForbiddenException,
|
||||
Inject,
|
||||
forwardRef,
|
||||
} from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { DEFAULT_LANGUAGE } from '../common/constants';
|
||||
import { I18nService } from '../i18n/i18n.service';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Repository, In } from 'typeorm';
|
||||
import {
|
||||
FileStatus,
|
||||
KnowledgeBase,
|
||||
ProcessingMode,
|
||||
} from './knowledge-base.entity';
|
||||
import { KnowledgeGroup } from '../knowledge-group/knowledge-group.entity';
|
||||
import { ElasticsearchService } from '../elasticsearch/elasticsearch.service';
|
||||
import { TikaService } from '../tika/tika.service';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { EmbeddingService } from './embedding.service';
|
||||
import { TextChunkerService } from './text-chunker.service';
|
||||
import { ModelConfigService } from '../model-config/model-config.service';
|
||||
import { RagService } from '../rag/rag.service';
|
||||
import { VisionService } from '../vision/vision.service';
|
||||
import { TenantService } from '../tenant/tenant.service';
|
||||
import { MemoryMonitorService } from './memory-monitor.service';
|
||||
import { ChunkConfigService } from './chunk-config.service';
|
||||
import { VisionPipelineService } from '../vision-pipeline/vision-pipeline.service';
|
||||
import { LibreOfficeService } from '../libreoffice/libreoffice.service';
|
||||
import { Pdf2ImageService } from '../pdf2image/pdf2image.service';
|
||||
import {
|
||||
DOC_EXTENSIONS,
|
||||
IMAGE_EXTENSIONS,
|
||||
} from '../common/file-support.constants';
|
||||
import { ChatService } from '../chat/chat.service';
|
||||
import { UserSettingService } from '../user/user-setting.service';
|
||||
|
||||
export interface PaginatedKnowledgeBase {
|
||||
items: KnowledgeBase[];
|
||||
total: number;
|
||||
page: number;
|
||||
limit: number;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class KnowledgeBaseService {
|
||||
private readonly logger = new Logger(KnowledgeBaseService.name);
|
||||
|
||||
constructor(
|
||||
@InjectRepository(KnowledgeBase)
|
||||
private kbRepository: Repository<KnowledgeBase>,
|
||||
@InjectRepository(KnowledgeGroup)
|
||||
private groupRepository: Repository<KnowledgeGroup>,
|
||||
@Inject(forwardRef(() => ElasticsearchService))
|
||||
private elasticsearchService: ElasticsearchService,
|
||||
private tikaService: TikaService,
|
||||
private embeddingService: EmbeddingService,
|
||||
private textChunkerService: TextChunkerService,
|
||||
private modelConfigService: ModelConfigService,
|
||||
@Inject(forwardRef(() => RagService))
|
||||
private ragService: RagService,
|
||||
private visionService: VisionService,
|
||||
private tenantService: TenantService,
|
||||
private memoryMonitor: MemoryMonitorService,
|
||||
private chunkConfigService: ChunkConfigService,
|
||||
private visionPipelineService: VisionPipelineService,
|
||||
private libreOfficeService: LibreOfficeService,
|
||||
private pdf2ImageService: Pdf2ImageService,
|
||||
private configService: ConfigService,
|
||||
private i18nService: I18nService,
|
||||
@Inject(forwardRef(() => ChatService))
|
||||
private chatService: ChatService,
|
||||
private userSettingService: UserSettingService,
|
||||
) {}
|
||||
|
||||
async createAndIndex(
|
||||
fileInfo: any,
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
config?: any,
|
||||
): Promise<KnowledgeBase> {
|
||||
const mode = config?.mode || 'fast';
|
||||
const processingMode =
|
||||
mode === 'precise' ? ProcessingMode.PRECISE : ProcessingMode.FAST;
|
||||
|
||||
const kb = this.kbRepository.create({
|
||||
originalName: fileInfo.originalname,
|
||||
storagePath: fileInfo.path,
|
||||
size: fileInfo.size,
|
||||
mimetype: fileInfo.mimetype,
|
||||
status: FileStatus.PENDING,
|
||||
userId: userId,
|
||||
tenantId: tenantId,
|
||||
chunkSize: config?.chunkSize || 200,
|
||||
chunkOverlap: config?.chunkOverlap || 40,
|
||||
embeddingModelId: config?.embeddingModelId || null,
|
||||
processingMode: processingMode,
|
||||
});
|
||||
|
||||
// Associate groups
|
||||
if (config?.groupIds && config.groupIds.length > 0) {
|
||||
const groups = await this.groupRepository.find({
|
||||
where: { id: In(config.groupIds), tenantId: tenantId },
|
||||
});
|
||||
kb.groups = groups;
|
||||
}
|
||||
|
||||
const savedKb = await this.kbRepository.save(kb);
|
||||
|
||||
this.logger.log(
|
||||
`Created KB record: ${savedKb.id}, mode: ${mode}, file: ${fileInfo.originalname}`,
|
||||
);
|
||||
|
||||
// ---------------------------------------------------------
|
||||
// Move the file to the final partitioned directory
|
||||
// source: uploads/{tenantId}/{filename} (or wherever it was)
|
||||
// target: uploads/{tenantId}/{savedKb.id}/{filename}
|
||||
// ---------------------------------------------------------
|
||||
const fs = await import('fs');
|
||||
const path = await import('path');
|
||||
const uploadPath = process.env.UPLOAD_FILE_PATH || './uploads';
|
||||
const targetDir = path.join(uploadPath, tenantId || 'default', savedKb.id);
|
||||
const targetPath = path.join(targetDir, fileInfo.filename);
|
||||
|
||||
try {
|
||||
if (!fs.existsSync(targetDir)) {
|
||||
fs.mkdirSync(targetDir, { recursive: true });
|
||||
}
|
||||
if (fs.existsSync(fileInfo.path)) {
|
||||
fs.renameSync(fileInfo.path, targetPath);
|
||||
// Update the DB record with the new path
|
||||
savedKb.storagePath = targetPath;
|
||||
await this.kbRepository.save(savedKb);
|
||||
this.logger.log(`Moved file to partitioned storage: ${targetPath}`);
|
||||
}
|
||||
} catch (fsError) {
|
||||
this.logger.error(
|
||||
`Failed to move file ${savedKb.id} to partitioned storage`,
|
||||
fsError,
|
||||
);
|
||||
// We will let it continue, but the file might be stuck in the temp/root folder
|
||||
}
|
||||
|
||||
// If queue processing is requested, await completion
|
||||
if (config?.waitForCompletion) {
|
||||
await this.processFile(savedKb.id, userId, tenantId, config);
|
||||
} else {
|
||||
// Otherwise trigger asynchronously (default)
|
||||
this.processFile(savedKb.id, userId, tenantId, config).catch((err) => {
|
||||
this.logger.error(`Error processing file ${savedKb.id}`, err);
|
||||
});
|
||||
}
|
||||
|
||||
return savedKb;
|
||||
}
|
||||
|
||||
async findAll(userId: string, tenantId?: string): Promise<KnowledgeBase[]> {
|
||||
const where: any = {};
|
||||
if (tenantId) {
|
||||
where.tenantId = tenantId;
|
||||
} else {
|
||||
where.userId = userId;
|
||||
}
|
||||
return this.kbRepository.find({
|
||||
where,
|
||||
relations: ['groups'], // Load group relations
|
||||
order: { createdAt: 'DESC' },
|
||||
});
|
||||
}
|
||||
|
||||
async findOne(
|
||||
id: string,
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
): Promise<KnowledgeBase> {
|
||||
const kb = await this.kbRepository.findOne({
|
||||
where: { id },
|
||||
relations: ['groups'],
|
||||
});
|
||||
|
||||
if (!kb) {
|
||||
throw new NotFoundException(
|
||||
this.i18nService.getMessage('knowledgeBaseNotFound'),
|
||||
);
|
||||
}
|
||||
|
||||
// Check permission using TenantService
|
||||
const hasAccess = await this.tenantService.canAccessTenant(
|
||||
userId,
|
||||
kb.tenantId,
|
||||
tenantId,
|
||||
);
|
||||
if (!hasAccess) {
|
||||
throw new ForbiddenException(
|
||||
`You do not have permission to access this knowledge base`,
|
||||
);
|
||||
}
|
||||
|
||||
return kb;
|
||||
}
|
||||
|
||||
async getStats(
|
||||
userId: string,
|
||||
tenantId?: string,
|
||||
): Promise<{ total: number; uncategorized: number }> {
|
||||
const where: any = {};
|
||||
if (tenantId) {
|
||||
where.tenantId = tenantId;
|
||||
} else {
|
||||
where.userId = userId;
|
||||
}
|
||||
|
||||
// Get total count
|
||||
const total = await this.kbRepository.count({ where });
|
||||
|
||||
// Get uncategorized count (files with no groups)
|
||||
// We need to use query builder to check for empty groups relation
|
||||
const uncategorizedQuery = this.kbRepository
|
||||
.createQueryBuilder('kb')
|
||||
.leftJoin('kb.groups', 'groups');
|
||||
|
||||
// Apply where conditions
|
||||
if (tenantId) {
|
||||
uncategorizedQuery.where('kb.tenantId = :tenantId', { tenantId });
|
||||
} else {
|
||||
uncategorizedQuery.where('kb.userId = :userId', { userId });
|
||||
}
|
||||
|
||||
// Count files where groups array is empty
|
||||
const uncategorizedCount = await uncategorizedQuery
|
||||
.andWhere('groups.id IS NULL')
|
||||
.getCount();
|
||||
|
||||
return {
|
||||
total,
|
||||
uncategorized: uncategorizedCount,
|
||||
};
|
||||
}
|
||||
|
||||
async searchKnowledge(
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
query: string,
|
||||
topK: number = 5,
|
||||
) {
|
||||
try {
|
||||
// Generate simulation vector using default dimensions from environment variable
|
||||
const defaultDimensions = parseInt(
|
||||
process.env.DEFAULT_VECTOR_DIMENSIONS || '2560',
|
||||
);
|
||||
const mockEmbedding = Array.from(
|
||||
{ length: defaultDimensions },
|
||||
() => Math.random() - 0.5,
|
||||
);
|
||||
const queryVector = mockEmbedding;
|
||||
|
||||
// 2. Search in Elasticsearch
|
||||
const searchResults = await this.elasticsearchService.searchSimilar(
|
||||
queryVector,
|
||||
userId,
|
||||
topK,
|
||||
tenantId, // Ensure shared visibility within tenant
|
||||
);
|
||||
|
||||
// 3. Get file information from database
|
||||
const fileIds = [...new Set(searchResults.map((r) => r.fileId))];
|
||||
const files = await this.kbRepository.findByIds(fileIds);
|
||||
const fileMap = new Map(files.map((f) => [f.id, f]));
|
||||
|
||||
// 4. Combine results with file info
|
||||
const results = searchResults.map((result) => {
|
||||
const file = fileMap.get(result.fileId);
|
||||
return {
|
||||
...result,
|
||||
file: file
|
||||
? {
|
||||
id: file.id,
|
||||
name: file.originalName,
|
||||
mimetype: file.mimetype,
|
||||
size: file.size,
|
||||
createdAt: file.createdAt,
|
||||
}
|
||||
: null,
|
||||
};
|
||||
});
|
||||
|
||||
return {
|
||||
query,
|
||||
results,
|
||||
total: results.length,
|
||||
};
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Metadata search failed for tenant ${tenantId}:`,
|
||||
error.stack || error.message,
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async ragSearch(
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
query: string,
|
||||
settings: any,
|
||||
) {
|
||||
this.logger.log(
|
||||
`RAG search request: userId=${userId}, query="${query}", settings=${JSON.stringify(settings)}`,
|
||||
);
|
||||
|
||||
try {
|
||||
const ragResults = await this.ragService.searchKnowledge(
|
||||
query,
|
||||
userId,
|
||||
settings.topK,
|
||||
settings.similarityThreshold,
|
||||
settings.selectedEmbeddingId,
|
||||
settings.enableFullTextSearch,
|
||||
settings.enableRerank,
|
||||
settings.selectedRerankId,
|
||||
undefined,
|
||||
undefined,
|
||||
settings.rerankSimilarityThreshold,
|
||||
tenantId, // Ensure shared visibility within tenant for RAG
|
||||
);
|
||||
|
||||
const sources = this.ragService.extractSources(ragResults);
|
||||
const ragPrompt = this.ragService.buildRagPrompt(
|
||||
query,
|
||||
ragResults,
|
||||
settings.language || DEFAULT_LANGUAGE,
|
||||
);
|
||||
|
||||
const result = {
|
||||
searchResults: ragResults,
|
||||
sources,
|
||||
ragPrompt,
|
||||
hasRelevantContent: ragResults.length > 0,
|
||||
};
|
||||
|
||||
this.logger.log(
|
||||
`RAG search completed: found ${ragResults.length} results`,
|
||||
);
|
||||
return result;
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`RAG search failed for user ${userId}:`,
|
||||
error.stack || error.message,
|
||||
);
|
||||
// Return empty result instead of throwing error to keep system running
|
||||
return {
|
||||
searchResults: [],
|
||||
sources: [],
|
||||
ragPrompt: query, // Use original query
|
||||
hasRelevantContent: false,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async deleteFile(
|
||||
fileId: string,
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
): Promise<void> {
|
||||
this.logger.log(`Deleting file ${fileId} for user ${userId}`);
|
||||
|
||||
try {
|
||||
// 1. Get file info
|
||||
const file = await this.kbRepository.findOne({
|
||||
where: { id: fileId, tenantId }, // Filter by tenantId
|
||||
});
|
||||
if (!file) {
|
||||
throw new NotFoundException(
|
||||
this.i18nService.getMessage('fileNotFound'),
|
||||
);
|
||||
}
|
||||
|
||||
// 2. Delete file from filesystem
|
||||
const fs = await import('fs');
|
||||
try {
|
||||
if (fs.existsSync(file.storagePath)) {
|
||||
fs.unlinkSync(file.storagePath);
|
||||
this.logger.log(`Deleted file: ${file.storagePath}`);
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.warn(`Failed to delete file ${file.storagePath}:`, error);
|
||||
}
|
||||
|
||||
// 3. Delete from Elasticsearch
|
||||
try {
|
||||
await this.elasticsearchService.deleteByFileId(
|
||||
fileId,
|
||||
userId,
|
||||
tenantId,
|
||||
);
|
||||
this.logger.log(`Deleted ES documents for file ${fileId}`);
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
`Failed to delete ES documents for file ${fileId}:`,
|
||||
error,
|
||||
);
|
||||
}
|
||||
|
||||
// 4. Remove from all groups (cleanup M2M relations)
|
||||
const fileWithGroups = await this.kbRepository.findOne({
|
||||
where: { id: fileId, tenantId },
|
||||
relations: ['groups'],
|
||||
});
|
||||
|
||||
if (
|
||||
fileWithGroups &&
|
||||
fileWithGroups.groups &&
|
||||
fileWithGroups.groups.length > 0
|
||||
) {
|
||||
// Clear groups to remove entries from join table
|
||||
fileWithGroups.groups = [];
|
||||
await this.kbRepository.save(fileWithGroups);
|
||||
this.logger.log(`Cleared group associations for file ${fileId}`);
|
||||
}
|
||||
|
||||
// 5. Delete from SQLite
|
||||
await this.kbRepository.delete({ id: fileId });
|
||||
this.logger.log(`Deleted database record for file ${fileId}`);
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to delete file ${fileId}`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async clearAll(userId: string, tenantId: string): Promise<void> {
|
||||
this.logger.log(
|
||||
`Clearing all knowledge base data for user ${userId} in tenant ${tenantId}`,
|
||||
);
|
||||
|
||||
try {
|
||||
// Get all files for the specific tenant and delete them one by one
|
||||
const files = await this.kbRepository.find({ where: { tenantId } });
|
||||
|
||||
for (const file of files) {
|
||||
await this.deleteFile(file.id, userId, tenantId);
|
||||
}
|
||||
|
||||
this.logger.log(`Cleared all knowledge base data for user ${userId}`);
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Failed to clear knowledge base for user ${userId}`,
|
||||
error,
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async processFile(
|
||||
kbId: string,
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
config?: any,
|
||||
) {
|
||||
this.logger.log(
|
||||
`Starting processing for file ${kbId}, mode: ${config?.mode || 'fast'}`,
|
||||
);
|
||||
await this.updateStatus(kbId, FileStatus.INDEXING);
|
||||
|
||||
try {
|
||||
const kb = await this.kbRepository.findOne({ where: { id: kbId } });
|
||||
if (!kb) {
|
||||
this.logger.error(`KB not found: ${kbId}`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Memory monitor - pre-processing check
|
||||
const memBefore = this.memoryMonitor.getMemoryUsage();
|
||||
this.logger.log(
|
||||
`Memory state - before processing: ${memBefore.heapUsed}/${memBefore.heapTotal}MB`,
|
||||
);
|
||||
|
||||
// Select processing flow based on mode
|
||||
const mode = config?.mode || 'fast';
|
||||
|
||||
if (mode === 'precise') {
|
||||
// Precise mode - use Vision Pipeline
|
||||
await this.processPreciseMode(kb, userId, tenantId, config);
|
||||
} else {
|
||||
// Fast mode - use Tika
|
||||
await this.processFastMode(kb, userId, tenantId, config);
|
||||
}
|
||||
|
||||
this.logger.log(`File ${kbId} processed successfully in ${mode} mode.`);
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to process file ${kbId}`, error);
|
||||
await this.updateStatus(kbId, FileStatus.FAILED);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fast mode processing (existing flow)
|
||||
*/
|
||||
private async processFastMode(
|
||||
kb: KnowledgeBase,
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
config?: any,
|
||||
) {
|
||||
// 1. Extract text using Tika
|
||||
let text = await this.tikaService.extractText(kb.storagePath);
|
||||
|
||||
// Use vision model for image files
|
||||
if (this.visionService.isImageFile(kb.mimetype)) {
|
||||
const settings = await this.tenantService.getSettings(
|
||||
tenantId || 'default',
|
||||
);
|
||||
const visionModelId = settings?.selectedVisionId;
|
||||
if (visionModelId) {
|
||||
const visionModel =
|
||||
await this.modelConfigService.findOne(visionModelId);
|
||||
if (
|
||||
visionModel &&
|
||||
visionModel.type === 'vision' &&
|
||||
visionModel.isEnabled !== false
|
||||
) {
|
||||
text = await this.visionService.extractImageContent(kb.storagePath, {
|
||||
baseUrl: visionModel.baseUrl || '',
|
||||
apiKey: visionModel.apiKey || '',
|
||||
modelId: visionModel.modelId,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!text || text.trim().length === 0) {
|
||||
this.logger.warn(this.i18nService.getMessage('noTextExtracted'));
|
||||
}
|
||||
|
||||
// Check text size
|
||||
const textSizeMB = Math.round(text.length / 1024 / 1024);
|
||||
if (textSizeMB > 50) {
|
||||
this.logger.warn(
|
||||
this.i18nService.formatMessage('extractedTextTooLarge', {
|
||||
size: textSizeMB,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
// Save text to database
|
||||
await this.kbRepository.update(kb.id, { content: text });
|
||||
await this.updateStatus(kb.id, FileStatus.EXTRACTED);
|
||||
|
||||
// Async vectorization
|
||||
await this.vectorizeToElasticsearch(
|
||||
kb.id,
|
||||
userId,
|
||||
tenantId,
|
||||
text,
|
||||
config,
|
||||
).catch((err) => {
|
||||
this.logger.error(`Error vectorizing file ${kb.id}`, err);
|
||||
});
|
||||
|
||||
// Auto-generate title (async execution)
|
||||
this.generateTitle(kb.id).catch((err) => {
|
||||
this.logger.error(`Error generating title for file ${kb.id}`, err);
|
||||
});
|
||||
|
||||
// Trigger PDF conversion asynchronously (for document files)
|
||||
this.ensurePDFExists(kb.id, userId, tenantId).catch((err) => {
|
||||
this.logger.warn(
|
||||
this.i18nService.formatMessage('pdfConversionFailedDetail', {
|
||||
id: kb.id,
|
||||
}),
|
||||
err,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Precise mode processing (new flow)
|
||||
*/
|
||||
private async processPreciseMode(
|
||||
kb: KnowledgeBase,
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
config?: any,
|
||||
) {
|
||||
// Check if precise mode is supported
|
||||
const preciseFormats = ['.pdf', '.doc', '.docx', '.ppt', '.pptx'];
|
||||
const ext = kb.originalName
|
||||
.toLowerCase()
|
||||
.substring(kb.originalName.lastIndexOf('.'));
|
||||
|
||||
if (!preciseFormats.includes(ext)) {
|
||||
this.logger.warn(
|
||||
this.i18nService.formatMessage('preciseModeUnsupported', { ext }),
|
||||
);
|
||||
return this.processFastMode(kb, userId, tenantId, config);
|
||||
}
|
||||
|
||||
// Check if Vision model is configured
|
||||
const settings = await this.tenantService.getSettings(
|
||||
tenantId || 'default',
|
||||
);
|
||||
const visionModelId = settings?.selectedVisionId;
|
||||
if (!visionModelId) {
|
||||
this.logger.warn(
|
||||
this.i18nService.getMessage('visionModelNotConfiguredFallback'),
|
||||
);
|
||||
return this.processFastMode(kb, userId, tenantId, config);
|
||||
}
|
||||
|
||||
const visionModel = await this.modelConfigService.findOne(visionModelId);
|
||||
if (
|
||||
!visionModel ||
|
||||
visionModel.type !== 'vision' ||
|
||||
visionModel.isEnabled === false
|
||||
) {
|
||||
this.logger.warn(
|
||||
this.i18nService.getMessage('visionModelInvalidFallback'),
|
||||
);
|
||||
return this.processFastMode(kb, userId, tenantId, config);
|
||||
}
|
||||
|
||||
// Call Vision Pipeline
|
||||
try {
|
||||
const result = await this.visionPipelineService.processPreciseMode(
|
||||
kb.storagePath,
|
||||
{
|
||||
userId,
|
||||
tenantId, // New
|
||||
modelId: visionModelId,
|
||||
fileId: kb.id,
|
||||
fileName: kb.originalName,
|
||||
skipQualityCheck: false,
|
||||
},
|
||||
);
|
||||
|
||||
if (!result.success) {
|
||||
this.logger.error(`Vision pipeline failed, falling back to fast mode`);
|
||||
this.logger.warn(this.i18nService.getMessage('visionPipelineFailed'));
|
||||
return this.processFastMode(kb, userId, tenantId, config);
|
||||
}
|
||||
|
||||
// Save text content to database
|
||||
const combinedText = result.results.map((r) => r.text).join('\n\n');
|
||||
const metadata = {
|
||||
processedPages: result.processedPages,
|
||||
failedPages: result.failedPages,
|
||||
cost: result.cost,
|
||||
duration: result.duration,
|
||||
results: result.results.map((r) => ({
|
||||
pageIndex: r.pageIndex,
|
||||
confidence: r.confidence,
|
||||
layout: r.layout,
|
||||
imageCount: r.images.length,
|
||||
})),
|
||||
};
|
||||
await this.kbRepository.update(kb.id, {
|
||||
content: combinedText,
|
||||
metadata: metadata as any,
|
||||
});
|
||||
|
||||
await this.updateStatus(kb.id, FileStatus.EXTRACTED);
|
||||
this.logger.log(
|
||||
this.i18nService.formatMessage('preciseModeComplete', {
|
||||
pages: result.processedPages,
|
||||
cost: result.cost.toFixed(2),
|
||||
}),
|
||||
);
|
||||
|
||||
// Async vectorization and Elasticsearch indexing
|
||||
// Create each page as separate document with metadata
|
||||
this.indexPreciseResults(
|
||||
kb,
|
||||
userId,
|
||||
tenantId,
|
||||
kb.embeddingModelId,
|
||||
result.results,
|
||||
).catch((err) => {
|
||||
this.logger.error(`Error indexing precise results for ${kb.id}`, err);
|
||||
});
|
||||
|
||||
// Trigger PDF conversion asynchronously
|
||||
this.ensurePDFExists(kb.id, userId, tenantId).catch((err) => {
|
||||
this.logger.warn(`Initial PDF conversion failed for ${kb.id}`, err);
|
||||
});
|
||||
|
||||
// Auto-generate title (async execution)
|
||||
this.generateTitle(kb.id).catch((err) => {
|
||||
this.logger.error(`Error generating title for file ${kb.id}`, err);
|
||||
});
|
||||
} catch (error) {
|
||||
this.logger.error(`Vision pipeline error: ${error.message}`, error.stack);
|
||||
this.logger.error(`Falling back to fast mode for file ${kb.id}`);
|
||||
return this.processFastMode(kb, userId, tenantId, config);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Index precise mode results
|
||||
*/
|
||||
private async indexPreciseResults(
|
||||
kb: KnowledgeBase,
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
embeddingModelId: string,
|
||||
results: any[],
|
||||
): Promise<void> {
|
||||
this.logger.log(`Indexing ${results.length} precise results for ${kb.id}`);
|
||||
|
||||
// Check index existence - get actual model dimensions
|
||||
const actualDimensions = await this.getActualModelDimensions(
|
||||
embeddingModelId,
|
||||
);
|
||||
await this.elasticsearchService.createIndexIfNotExists(actualDimensions);
|
||||
|
||||
// Batch vectorization and indexing
|
||||
const batchSize = parseInt(process.env.CHUNK_BATCH_SIZE || '50');
|
||||
|
||||
for (let i = 0; i < results.length; i += batchSize) {
|
||||
const batch = results.slice(i, i + batchSize);
|
||||
const texts = batch.map((r) => r.text);
|
||||
|
||||
try {
|
||||
// Generate vectors
|
||||
const embeddings = await this.embeddingService.getEmbeddings(
|
||||
texts,
|
||||
embeddingModelId,
|
||||
);
|
||||
|
||||
// Index each result
|
||||
for (let j = 0; j < batch.length; j++) {
|
||||
const result = batch[j];
|
||||
const embedding = embeddings[j];
|
||||
|
||||
if (!embedding || embedding.length === 0) {
|
||||
this.logger.warn(
|
||||
this.i18nService.formatMessage('skippingEmptyVectorPage', {
|
||||
page: result.pageIndex,
|
||||
}),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
await this.elasticsearchService.indexDocument(
|
||||
`${kb.id}_page_${result.pageIndex}`,
|
||||
result.text,
|
||||
embedding,
|
||||
{
|
||||
fileId: kb.id,
|
||||
originalName: kb.originalName,
|
||||
mimetype: kb.mimetype,
|
||||
userId: userId,
|
||||
tenantId: tenantId, // New
|
||||
pageNumber: result.pageIndex,
|
||||
images: result.images,
|
||||
layout: result.layout,
|
||||
confidence: result.confidence,
|
||||
source: 'precise',
|
||||
mode: 'vision',
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`Batch ${Math.floor(i / batchSize) + 1} completed: ${batch.length} pages`,
|
||||
);
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Batch ${Math.floor(i / batchSize) + 1} processing failed`,
|
||||
error,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
await this.updateStatus(kb.id, FileStatus.VECTORIZED);
|
||||
this.logger.log(`Precise mode indexing completed: ${results.length} pages`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get specific page of PDF as image
|
||||
*/
|
||||
async getPageAsImage(
|
||||
fileId: string,
|
||||
pageIndex: number,
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
): Promise<string> {
|
||||
const pdfPath = await this.ensurePDFExists(fileId, userId, tenantId);
|
||||
|
||||
// Convert specific pages
|
||||
const result = await this.pdf2ImageService.convertToImages(pdfPath, {
|
||||
density: 150,
|
||||
quality: 75,
|
||||
format: 'jpeg',
|
||||
});
|
||||
|
||||
// Find images for corresponding page numbers
|
||||
const pageImage = result.images.find(
|
||||
(img) => img.pageIndex === pageIndex + 1,
|
||||
);
|
||||
if (!pageImage) {
|
||||
throw new NotFoundException(
|
||||
this.i18nService.formatMessage('pageImageNotFoundDetail', {
|
||||
page: pageIndex + 1,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
return pageImage.path;
|
||||
}
|
||||
|
||||
private async vectorizeToElasticsearch(
|
||||
kbId: string,
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
text: string,
|
||||
config?: any,
|
||||
) {
|
||||
try {
|
||||
const kb = await this.kbRepository.findOne({
|
||||
where: { id: kbId, tenantId },
|
||||
});
|
||||
if (!kb) return;
|
||||
|
||||
// Memory monitor - pre-vectorization check
|
||||
const memBeforeChunk = this.memoryMonitor.getMemoryUsage();
|
||||
this.logger.log(
|
||||
`Pre-vectorization memory: ${memBeforeChunk.heapUsed}/${memBeforeChunk.heapTotal}MB`,
|
||||
);
|
||||
|
||||
this.logger.debug(`File ${kbId}: Validating chunk config...`);
|
||||
// 1. Validate and fix chunk config (based on model limits and env vars)
|
||||
const validatedConfig = await this.chunkConfigService.validateChunkConfig(
|
||||
kb.chunkSize,
|
||||
kb.chunkOverlap,
|
||||
kb.embeddingModelId,
|
||||
);
|
||||
this.logger.debug(`File ${kbId}: Chunk config validated.`);
|
||||
|
||||
// If config modified, log warning and update database
|
||||
if (validatedConfig.warnings.length > 0) {
|
||||
this.logger.warn(
|
||||
this.i18nService.formatMessage('chunkConfigCorrection', {
|
||||
warnings: validatedConfig.warnings.join(', '),
|
||||
}),
|
||||
);
|
||||
|
||||
// Update config in database
|
||||
if (
|
||||
validatedConfig.chunkSize !== kb.chunkSize ||
|
||||
validatedConfig.chunkOverlap !== kb.chunkOverlap
|
||||
) {
|
||||
await this.kbRepository.update(kbId, {
|
||||
chunkSize: validatedConfig.chunkSize,
|
||||
chunkOverlap: validatedConfig.chunkOverlap,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Display config summary (including actual limits applied)
|
||||
this.logger.debug(`File ${kbId}: Getting config summary...`);
|
||||
const configSummary = await this.chunkConfigService.getConfigSummary(
|
||||
validatedConfig.chunkSize,
|
||||
validatedConfig.chunkOverlap,
|
||||
kb.embeddingModelId,
|
||||
);
|
||||
this.logger.log(`Chunk config: ${configSummary}`);
|
||||
this.logger.log(
|
||||
`Config limits: chunk=${validatedConfig.effectiveMaxChunkSize}, overlap=${validatedConfig.effectiveMaxOverlapSize}`,
|
||||
);
|
||||
|
||||
// 2. Split text using validated config
|
||||
const chunks = this.textChunkerService.chunkText(
|
||||
text,
|
||||
validatedConfig.chunkSize,
|
||||
validatedConfig.chunkOverlap,
|
||||
);
|
||||
this.logger.log(`File ${kbId} split into ${chunks.length} text blocks`);
|
||||
|
||||
if (chunks.length === 0) {
|
||||
this.logger.warn(
|
||||
this.i18nService.formatMessage('noChunksGenerated', { id: kbId }),
|
||||
);
|
||||
await this.updateStatus(kbId, FileStatus.VECTORIZED);
|
||||
return;
|
||||
}
|
||||
|
||||
// 3. Validate chunk count is reasonable
|
||||
const estimatedChunkCount = this.chunkConfigService.estimateChunkCount(
|
||||
text.length,
|
||||
validatedConfig.chunkSize,
|
||||
);
|
||||
if (chunks.length > estimatedChunkCount * 1.2) {
|
||||
this.logger.warn(
|
||||
this.i18nService.formatMessage('chunkCountAnomaly', {
|
||||
actual: chunks.length,
|
||||
estimated: estimatedChunkCount,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
// 4. Get recommended batch size (based on model limits)
|
||||
const recommendedBatchSize =
|
||||
await this.chunkConfigService.getRecommendedBatchSize(
|
||||
kb.embeddingModelId,
|
||||
parseInt(process.env.CHUNK_BATCH_SIZE || '100'),
|
||||
);
|
||||
|
||||
// 5. Estimate memory usage
|
||||
const avgChunkSize =
|
||||
chunks.reduce((sum, c) => sum + c.content.length, 0) / chunks.length;
|
||||
const estimatedMemory = this.memoryMonitor.estimateMemoryUsage(
|
||||
chunks.length,
|
||||
avgChunkSize,
|
||||
parseInt(process.env.DEFAULT_VECTOR_DIMENSIONS || '2560'),
|
||||
);
|
||||
this.logger.log(
|
||||
`Estimated memory usage: ${estimatedMemory}MB (batch size: ${recommendedBatchSize})`,
|
||||
);
|
||||
|
||||
// 6. Get actual model dimensions and check index exists
|
||||
const actualDimensions = await this.getActualModelDimensions(
|
||||
kb.embeddingModelId,
|
||||
);
|
||||
await this.elasticsearchService.createIndexIfNotExists(actualDimensions);
|
||||
|
||||
// 7. Batch vectorization and indexing
|
||||
const useBatching = this.memoryMonitor.shouldUseBatching(
|
||||
chunks.length,
|
||||
avgChunkSize,
|
||||
actualDimensions,
|
||||
);
|
||||
|
||||
if (useBatching) {
|
||||
try {
|
||||
await this.processInBatches(
|
||||
chunks,
|
||||
async (batch, batchIndex) => {
|
||||
// Verify batch size not exceeding model limit
|
||||
if (batch.length > recommendedBatchSize) {
|
||||
this.logger.warn(
|
||||
this.i18nService.formatMessage('batchSizeExceeded', {
|
||||
index: batchIndex,
|
||||
actual: batch.length,
|
||||
limit: recommendedBatchSize,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
const chunkTexts = batch.map((chunk) => chunk.content);
|
||||
const embeddings = await this.embeddingService.getEmbeddings(
|
||||
chunkTexts,
|
||||
kb.embeddingModelId,
|
||||
);
|
||||
|
||||
// Validate dimension consistency
|
||||
if (
|
||||
embeddings.length > 0 &&
|
||||
embeddings[0].length !== actualDimensions
|
||||
) {
|
||||
this.logger.warn(
|
||||
`Vector dimension mismatch: expected ${actualDimensions}, got ${embeddings[0].length}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Index this batch data immediately
|
||||
for (let i = 0; i < batch.length; i++) {
|
||||
const chunk = batch[i];
|
||||
const embedding = embeddings[i];
|
||||
|
||||
if (!embedding || embedding.length === 0) {
|
||||
this.logger.warn(
|
||||
this.i18nService.formatMessage('skippingEmptyVectorChunk', {
|
||||
index: chunk.index,
|
||||
}),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
await this.elasticsearchService.indexDocument(
|
||||
`${kb.id}_chunk_${chunk.index}`,
|
||||
chunk.content,
|
||||
embedding,
|
||||
{
|
||||
fileId: kb.id,
|
||||
originalName: kb.originalName,
|
||||
mimetype: kb.mimetype,
|
||||
userId: userId,
|
||||
chunkIndex: chunk.index,
|
||||
startPosition: chunk.startPosition,
|
||||
tenantId, // Passing tenantId to ES
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`Batch ${batchIndex} completed: ${batch.length} chunks`,
|
||||
);
|
||||
},
|
||||
{
|
||||
batchSize: recommendedBatchSize,
|
||||
onBatchComplete: (batchIndex, totalBatches) => {
|
||||
const mem = this.memoryMonitor.getMemoryUsage();
|
||||
this.logger.log(
|
||||
`Batch ${batchIndex}/${totalBatches} completed, memory: ${mem.heapUsed}MB`,
|
||||
);
|
||||
},
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
// Detect context length error (supports Japanese/Chinese/English)
|
||||
if (
|
||||
error.message &&
|
||||
(error.message.includes('context length') ||
|
||||
error.message.includes('context length exceeded'))
|
||||
) {
|
||||
this.logger.warn(
|
||||
this.i18nService.getMessage('contextLengthErrorFallback'),
|
||||
);
|
||||
|
||||
// Downgrade to single text processing
|
||||
for (let i = 0; i < chunks.length; i++) {
|
||||
const chunk = chunks[i];
|
||||
|
||||
try {
|
||||
const embeddings = await this.embeddingService.getEmbeddings(
|
||||
[chunk.content], // Single text
|
||||
kb.embeddingModelId,
|
||||
);
|
||||
|
||||
if (!embeddings[0] || embeddings[0].length === 0) {
|
||||
this.logger.warn(
|
||||
this.i18nService.formatMessage('skippingEmptyVectorChunk', {
|
||||
index: chunk.index,
|
||||
}),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
await this.elasticsearchService.indexDocument(
|
||||
`${kb.id}_chunk_${chunk.index}`,
|
||||
chunk.content,
|
||||
embeddings[0],
|
||||
{
|
||||
fileId: kb.id,
|
||||
originalName: kb.originalName,
|
||||
mimetype: kb.mimetype,
|
||||
userId: userId,
|
||||
chunkIndex: chunk.index,
|
||||
startPosition: chunk.startPosition,
|
||||
endPosition: chunk.endPosition,
|
||||
tenantId,
|
||||
},
|
||||
);
|
||||
|
||||
if ((i + 1) % 10 === 0) {
|
||||
this.logger.log(
|
||||
`Single processing progress: ${i + 1}/${chunks.length}`,
|
||||
);
|
||||
}
|
||||
} catch (chunkError) {
|
||||
this.logger.error(
|
||||
`Failed to process text block ${chunk.index}. Skipping: ${chunkError.message}`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`Single text processing completed: ${chunks.length} chunks`,
|
||||
);
|
||||
} else {
|
||||
// Throw other errors directly
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Small files, batch processing (but need to check batch limits)
|
||||
const chunkTexts = chunks.map((chunk) => chunk.content);
|
||||
|
||||
// Force batch processing if chunk count exceeds model batch limit
|
||||
if (chunks.length > recommendedBatchSize) {
|
||||
this.logger.warn(
|
||||
this.i18nService.formatMessage('chunkLimitExceededForceBatch', {
|
||||
actual: chunks.length,
|
||||
limit: recommendedBatchSize,
|
||||
}),
|
||||
);
|
||||
try {
|
||||
await this.processInBatches(chunks, async (batch, batchIndex) => {
|
||||
const batchTexts = batch.map((c) => c.content);
|
||||
const embeddings = await this.embeddingService.getEmbeddings(
|
||||
batchTexts,
|
||||
kb.embeddingModelId,
|
||||
);
|
||||
|
||||
for (let i = 0; i < batch.length; i++) {
|
||||
const chunk = batch[i];
|
||||
const embedding = embeddings[i];
|
||||
|
||||
if (!embedding || embedding.length === 0) {
|
||||
this.logger.warn(
|
||||
`Skipping empty vector text block ${chunk.index}`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
await this.elasticsearchService.indexDocument(
|
||||
`${kb.id}_chunk_${chunk.index}`,
|
||||
chunk.content,
|
||||
embedding,
|
||||
{
|
||||
fileId: kb.id,
|
||||
originalName: kb.originalName,
|
||||
mimetype: kb.mimetype,
|
||||
userId: userId,
|
||||
chunkIndex: chunk.index,
|
||||
startPosition: chunk.startPosition,
|
||||
endPosition: chunk.endPosition,
|
||||
tenantId, // Passing tenantId to ES metadata
|
||||
},
|
||||
);
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
// Detect context length error (supports Japanese/Chinese/English)
|
||||
if (
|
||||
error.message &&
|
||||
(error.message.includes('context length') ||
|
||||
error.message.includes('context length exceeded'))
|
||||
) {
|
||||
this.logger.warn(
|
||||
this.i18nService.getMessage('batchContextLengthErrorFallback'),
|
||||
);
|
||||
|
||||
// Downgrade to single text processing
|
||||
for (let i = 0; i < chunks.length; i++) {
|
||||
const chunk = chunks[i];
|
||||
|
||||
try {
|
||||
const embeddings = await this.embeddingService.getEmbeddings(
|
||||
[chunk.content], // Single text
|
||||
kb.embeddingModelId,
|
||||
);
|
||||
|
||||
if (!embeddings[0] || embeddings[0].length === 0) {
|
||||
this.logger.warn(
|
||||
this.i18nService.formatMessage(
|
||||
'skippingEmptyVectorChunk',
|
||||
{ index: chunk.index },
|
||||
),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
await this.elasticsearchService.indexDocument(
|
||||
`${kb.id}_chunk_${chunk.index}`,
|
||||
chunk.content,
|
||||
embeddings[0],
|
||||
{
|
||||
fileId: kb.id,
|
||||
originalName: kb.originalName,
|
||||
mimetype: kb.mimetype,
|
||||
userId: userId,
|
||||
tenantId, // Added tenantId
|
||||
chunkIndex: chunk.index,
|
||||
startPosition: chunk.startPosition,
|
||||
endPosition: chunk.endPosition,
|
||||
},
|
||||
);
|
||||
|
||||
if ((i + 1) % 10 === 0) {
|
||||
this.logger.log(
|
||||
`Single processing progress: ${i + 1}/${chunks.length}`,
|
||||
);
|
||||
}
|
||||
} catch (chunkError) {
|
||||
this.logger.error(
|
||||
this.i18nService.formatMessage('chunkProcessingFailed', {
|
||||
index: chunk.index,
|
||||
message: chunkError.message,
|
||||
}),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
this.i18nService.formatMessage('singleTextProcessingComplete', {
|
||||
count: chunks.length,
|
||||
}),
|
||||
);
|
||||
} else {
|
||||
// Throw other errors directly
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Process if file is small enough
|
||||
try {
|
||||
const embeddings = await this.embeddingService.getEmbeddings(
|
||||
chunkTexts,
|
||||
kb.embeddingModelId,
|
||||
);
|
||||
|
||||
for (let i = 0; i < chunks.length; i++) {
|
||||
const chunk = chunks[i];
|
||||
const embedding = embeddings[i];
|
||||
|
||||
if (!embedding || embedding.length === 0) {
|
||||
this.logger.warn(
|
||||
this.i18nService.formatMessage('skippingEmptyVectorChunk', {
|
||||
index: chunk.index,
|
||||
}),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
await this.elasticsearchService.indexDocument(
|
||||
`${kb.id}_chunk_${chunk.index}`,
|
||||
chunk.content,
|
||||
embedding,
|
||||
{
|
||||
fileId: kb.id,
|
||||
originalName: kb.originalName,
|
||||
mimetype: kb.mimetype,
|
||||
userId: userId,
|
||||
tenantId, // Added tenantId
|
||||
chunkIndex: chunk.index,
|
||||
startPosition: chunk.startPosition,
|
||||
endPosition: chunk.endPosition,
|
||||
},
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
// Detect context length error (supports Japanese/Chinese/English)
|
||||
if (
|
||||
error.message &&
|
||||
(error.message.includes('context length') ||
|
||||
error.message.includes('context length exceeded'))
|
||||
) {
|
||||
this.logger.warn(
|
||||
this.i18nService.getMessage('batchContextLengthErrorFallback'),
|
||||
);
|
||||
|
||||
// Downgrade to single text processing
|
||||
for (let i = 0; i < chunks.length; i++) {
|
||||
const chunk = chunks[i];
|
||||
|
||||
try {
|
||||
const embeddings = await this.embeddingService.getEmbeddings(
|
||||
[chunk.content], // Single text
|
||||
kb.embeddingModelId,
|
||||
);
|
||||
|
||||
if (!embeddings[0] || embeddings[0].length === 0) {
|
||||
this.logger.warn(
|
||||
`Skipping empty vector text block ${chunk.index}`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
await this.elasticsearchService.indexDocument(
|
||||
`${kb.id}_chunk_${chunk.index}`,
|
||||
chunk.content,
|
||||
embeddings[0],
|
||||
{
|
||||
fileId: kb.id,
|
||||
originalName: kb.originalName,
|
||||
mimetype: kb.mimetype,
|
||||
userId: userId,
|
||||
tenantId, // Added tenantId
|
||||
chunkIndex: chunk.index,
|
||||
startPosition: chunk.startPosition,
|
||||
endPosition: chunk.endPosition,
|
||||
},
|
||||
);
|
||||
|
||||
if ((i + 1) % 10 === 0) {
|
||||
this.logger.log(
|
||||
`Single processing progress: ${i + 1}/${chunks.length}`,
|
||||
);
|
||||
}
|
||||
} catch (chunkError) {
|
||||
this.logger.error(
|
||||
`Failed to process text block ${chunk.index}. Skipping: ${chunkError.message}`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
this.i18nService.formatMessage('singleTextProcessingComplete', {
|
||||
count: chunks.length,
|
||||
}),
|
||||
);
|
||||
} else {
|
||||
// Throw other errors directly
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await this.updateStatus(kbId, FileStatus.VECTORIZED);
|
||||
const memAfter = this.memoryMonitor.getMemoryUsage();
|
||||
this.logger.log(
|
||||
this.i18nService.formatMessage('fileVectorizationComplete', {
|
||||
id: kbId,
|
||||
count: chunks.length,
|
||||
memory: memAfter.heapUsed,
|
||||
}),
|
||||
);
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
this.i18nService.formatMessage('fileVectorizationFailed', { id: kbId }),
|
||||
error,
|
||||
);
|
||||
|
||||
// Save error info to metadata
|
||||
try {
|
||||
const kb = await this.kbRepository.findOne({ where: { id: kbId } });
|
||||
if (kb) {
|
||||
const metadata = kb.metadata || {};
|
||||
metadata.lastError = error.message;
|
||||
metadata.failedAt = new Date().toISOString();
|
||||
await this.kbRepository.update(kbId, { metadata });
|
||||
}
|
||||
} catch (e) {
|
||||
this.logger.warn(
|
||||
`Failed to update metadata for failed file ${kbId}`,
|
||||
e,
|
||||
);
|
||||
}
|
||||
|
||||
await this.updateStatus(kbId, FileStatus.FAILED);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch processing with memory control
|
||||
*/
|
||||
private async processInBatches<T>(
|
||||
items: T[],
|
||||
processor: (batch: T[], batchIndex: number) => Promise<void>,
|
||||
options?: {
|
||||
batchSize?: number;
|
||||
onBatchComplete?: (batchIndex: number, totalBatches: number) => void;
|
||||
},
|
||||
): Promise<void> {
|
||||
const totalItems = items.length;
|
||||
if (totalItems === 0) return;
|
||||
|
||||
const startTime = Date.now();
|
||||
this.logger.log(
|
||||
this.i18nService.formatMessage('batchProcessingStarted', {
|
||||
count: totalItems,
|
||||
}),
|
||||
);
|
||||
|
||||
// Use provided batch size or fallback to env/default
|
||||
const initialBatchSize =
|
||||
options?.batchSize || parseInt(process.env.CHUNK_BATCH_SIZE || '100');
|
||||
const totalBatches = Math.ceil(totalItems / initialBatchSize);
|
||||
|
||||
for (let i = 0; i < totalItems; ) {
|
||||
// Check memory and wait
|
||||
await this.memoryMonitor.waitForMemoryAvailable();
|
||||
|
||||
// Dynamically adjust batch size (start from initialBatchSize, memory monitor can reduce if needed)
|
||||
// Note: memoryMonitor.getDynamicBatchSize may return larger values based on memory situation,
|
||||
// but we must respect model limits (initialBatchSize)
|
||||
const currentMem = this.memoryMonitor.getMemoryUsage().heapUsed;
|
||||
const dynamicBatchSize =
|
||||
this.memoryMonitor.getDynamicBatchSize(currentMem);
|
||||
|
||||
// Ensure we don't exceed the model's limit (initialBatchSize) even if memory allows more
|
||||
const batchSize = Math.min(dynamicBatchSize, initialBatchSize);
|
||||
|
||||
// Get current batch
|
||||
const batch = items.slice(i, i + batchSize);
|
||||
const batchIndex = Math.floor(i / batchSize) + 1;
|
||||
|
||||
this.logger.log(
|
||||
this.i18nService.formatMessage('batchProcessingProgress', {
|
||||
index: batchIndex,
|
||||
total: totalBatches,
|
||||
count: batch.length,
|
||||
}),
|
||||
);
|
||||
|
||||
// Process batch
|
||||
await processor(batch, batchIndex);
|
||||
|
||||
// Callback notification
|
||||
if (options?.onBatchComplete) {
|
||||
options.onBatchComplete(batchIndex, totalBatches);
|
||||
}
|
||||
|
||||
// Force GC (if memory is near threshold)
|
||||
if (currentMem > 800) {
|
||||
this.memoryMonitor.forceGC();
|
||||
}
|
||||
|
||||
// Clear references to help GC
|
||||
batch.length = 0;
|
||||
|
||||
i += batchSize;
|
||||
}
|
||||
|
||||
const duration = ((Date.now() - startTime) / 1000).toFixed(2);
|
||||
this.logger.log(
|
||||
this.i18nService.formatMessage('batchProcessingComplete', {
|
||||
count: totalItems,
|
||||
duration,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Retry vectorization for failed files
|
||||
*/
|
||||
async retryFailedFile(
|
||||
fileId: string,
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
): Promise<KnowledgeBase> {
|
||||
this.logger.log(
|
||||
`Retrying failed file ${fileId} for user ${userId} in tenant ${tenantId}`,
|
||||
);
|
||||
|
||||
// 1. Get file with tenant restriction
|
||||
const kb = await this.kbRepository.findOne({
|
||||
where: { id: fileId, tenantId },
|
||||
});
|
||||
|
||||
if (!kb) {
|
||||
throw new NotFoundException(this.i18nService.getMessage('fileNotFound'));
|
||||
}
|
||||
|
||||
if (kb.status !== FileStatus.FAILED) {
|
||||
throw new Error(
|
||||
this.i18nService.formatMessage('onlyFailedFilesRetryable', {
|
||||
status: kb.status,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
if (!kb.content || kb.content.trim().length === 0) {
|
||||
throw new Error(this.i18nService.getMessage('emptyFileRetryFailed'));
|
||||
}
|
||||
|
||||
// 2. Reset status to INDEXING
|
||||
await this.updateStatus(fileId, FileStatus.INDEXING);
|
||||
|
||||
// 3. Trigger vectorization asynchronously (reuse existing logic)
|
||||
this.vectorizeToElasticsearch(fileId, userId, tenantId, kb.content, {
|
||||
chunkSize: kb.chunkSize,
|
||||
chunkOverlap: kb.chunkOverlap,
|
||||
embeddingModelId: kb.embeddingModelId,
|
||||
}).catch((err) => {
|
||||
this.logger.error(`Retry vectorization failed for file ${fileId}`, err);
|
||||
});
|
||||
|
||||
// 4. Return updated file status
|
||||
const updatedKb = await this.kbRepository.findOne({
|
||||
where: { id: fileId, tenantId },
|
||||
});
|
||||
if (!updatedKb) {
|
||||
throw new NotFoundException(this.i18nService.getMessage('fileNotFound'));
|
||||
}
|
||||
return updatedKb;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all chunk information for a file
|
||||
*/
|
||||
async getFileChunks(fileId: string, userId: string, tenantId: string) {
|
||||
this.logger.log(
|
||||
`Getting chunks for file ${fileId}, user ${userId}, tenant ${tenantId}`,
|
||||
);
|
||||
|
||||
// 1. Get file with tenant check
|
||||
const kb = await this.kbRepository.findOne({
|
||||
where: { id: fileId, tenantId },
|
||||
});
|
||||
|
||||
if (!kb) {
|
||||
throw new NotFoundException(this.i18nService.getMessage('fileNotFound'));
|
||||
}
|
||||
|
||||
// 2. Get all chunks from Elasticsearch
|
||||
const chunks = await this.elasticsearchService.getFileChunks(
|
||||
fileId,
|
||||
userId,
|
||||
tenantId,
|
||||
);
|
||||
|
||||
// 3. Return chunk info
|
||||
return {
|
||||
fileId: kb.id,
|
||||
fileName: kb.originalName,
|
||||
totalChunks: chunks.length,
|
||||
chunkSize: kb.chunkSize,
|
||||
chunkOverlap: kb.chunkOverlap,
|
||||
chunks: chunks.map((chunk) => ({
|
||||
index: chunk.chunkIndex,
|
||||
content: chunk.content,
|
||||
contentLength: chunk.content.length,
|
||||
startPosition: chunk.startPosition,
|
||||
endPosition: chunk.endPosition,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
private async updateStatus(id: string, status: FileStatus) {
|
||||
await this.kbRepository.update(id, { status });
|
||||
}
|
||||
|
||||
// PDF preview related methods
|
||||
async ensurePDFExists(
|
||||
fileId: string,
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
force: boolean = false,
|
||||
): Promise<string> {
|
||||
const kb = await this.kbRepository.findOne({
|
||||
where: { id: fileId, tenantId },
|
||||
});
|
||||
|
||||
if (!kb) {
|
||||
throw new NotFoundException(this.i18nService.getMessage('fileNotFound'));
|
||||
}
|
||||
|
||||
// If original file is PDF, return the original file path directly
|
||||
if (kb.mimetype === 'application/pdf') {
|
||||
return kb.storagePath;
|
||||
}
|
||||
|
||||
// Check if preview conversion is supported (only documents or images allowed)
|
||||
const ext = kb.originalName.toLowerCase().split('.').pop() || '';
|
||||
const isConvertible = [...DOC_EXTENSIONS, ...IMAGE_EXTENSIONS].includes(
|
||||
ext,
|
||||
);
|
||||
|
||||
if (!isConvertible) {
|
||||
this.logger.log(
|
||||
`Skipping PDF conversion for unsupported format: .${ext} (${kb.originalName})`,
|
||||
);
|
||||
throw new Error(this.i18nService.getMessage('pdfPreviewNotSupported'));
|
||||
}
|
||||
|
||||
// Generate PDF field path
|
||||
const path = await import('path');
|
||||
const fs = await import('fs');
|
||||
const uploadDir = path.dirname(kb.storagePath);
|
||||
const baseName = path.basename(
|
||||
kb.storagePath,
|
||||
path.extname(kb.storagePath),
|
||||
);
|
||||
const pdfPath = path.join(uploadDir, `${baseName}.pdf`);
|
||||
|
||||
// Delete if forced regeneration specified and file exists
|
||||
if (force && fs.existsSync(pdfPath)) {
|
||||
try {
|
||||
fs.unlinkSync(pdfPath);
|
||||
this.logger.log(`Forced regeneration: Deleted existing PDF ${pdfPath}`);
|
||||
} catch (e) {
|
||||
this.logger.warn(
|
||||
`Failed to delete existing PDF for regeneration: ${e.message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Check if already converted and regeneration not needed
|
||||
if (fs.existsSync(pdfPath) && !force) {
|
||||
if (!kb.pdfPath) {
|
||||
await this.kbRepository.update(kb.id, { pdfPath: pdfPath });
|
||||
}
|
||||
return pdfPath;
|
||||
}
|
||||
|
||||
// Need to convert to PDF
|
||||
try {
|
||||
this.logger.log(
|
||||
`Starting PDF conversion for ${kb.originalName} at ${kb.storagePath}`,
|
||||
);
|
||||
|
||||
// Convert file
|
||||
await this.libreOfficeService.convertToPDF(kb.storagePath);
|
||||
|
||||
// Check conversion result
|
||||
if (!fs.existsSync(pdfPath)) {
|
||||
throw new Error(
|
||||
`PDF conversion completed but file not found at ${pdfPath}`,
|
||||
);
|
||||
}
|
||||
|
||||
const stats = fs.statSync(pdfPath);
|
||||
if (stats.size === 0) {
|
||||
fs.unlinkSync(pdfPath);
|
||||
throw new Error(`PDF conversion failed: output file is empty`);
|
||||
}
|
||||
|
||||
await this.kbRepository.update(kb.id, { pdfPath: pdfPath });
|
||||
|
||||
this.logger.log(`PDF conversion successful: ${pdfPath}`);
|
||||
return pdfPath;
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`PDF conversion failed for ${fileId}: ${error.message}`,
|
||||
error.stack,
|
||||
);
|
||||
throw new Error(
|
||||
this.i18nService.formatMessage('pdfConversionFailedDetail', {
|
||||
id: fileId,
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async getPDFStatus(fileId: string, userId: string, tenantId: string) {
|
||||
const kb = await this.kbRepository.findOne({
|
||||
where: { id: fileId, tenantId },
|
||||
});
|
||||
|
||||
if (!kb) {
|
||||
throw new NotFoundException(this.i18nService.getMessage('fileNotFound'));
|
||||
}
|
||||
|
||||
// If original file is PDF
|
||||
if (kb.mimetype === 'application/pdf') {
|
||||
const token = this.generateTempToken(fileId, userId, tenantId);
|
||||
return {
|
||||
status: 'ready',
|
||||
url: `/api/knowledge-bases/${fileId}/pdf?token=${token}`,
|
||||
};
|
||||
}
|
||||
|
||||
// Generate PDF file path
|
||||
const path = await import('path');
|
||||
const fs = await import('fs');
|
||||
const uploadDir = path.dirname(kb.storagePath);
|
||||
const baseName = path.basename(
|
||||
kb.storagePath,
|
||||
path.extname(kb.storagePath),
|
||||
);
|
||||
const pdfPath = path.join(uploadDir, `${baseName}.pdf`);
|
||||
|
||||
// Check if converted
|
||||
if (fs.existsSync(pdfPath)) {
|
||||
if (!kb.pdfPath) {
|
||||
kb.pdfPath = pdfPath;
|
||||
await this.kbRepository.save(kb);
|
||||
}
|
||||
const token = this.generateTempToken(fileId, userId, tenantId);
|
||||
return {
|
||||
status: 'ready',
|
||||
url: `/api/knowledge-bases/${fileId}/pdf?token=${token}`,
|
||||
};
|
||||
}
|
||||
|
||||
// Conversion needed
|
||||
return {
|
||||
status: 'pending',
|
||||
};
|
||||
}
|
||||
|
||||
private generateTempToken(
|
||||
fileId: string,
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
): string {
|
||||
const jwt = require('jsonwebtoken');
|
||||
|
||||
const secret = process.env.JWT_SECRET;
|
||||
if (!secret) {
|
||||
throw new Error(this.i18nService.getMessage('jwtSecretRequired'));
|
||||
}
|
||||
|
||||
return jwt.sign({ fileId, userId, tenantId, type: 'pdf-access' }, secret, {
|
||||
expiresIn: '1h',
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get actual model dimensions (with cache check and probe logic)
|
||||
*/
|
||||
private async getActualModelDimensions(
|
||||
embeddingModelId: string,
|
||||
): Promise<number> {
|
||||
const defaultDimensions = parseInt(
|
||||
process.env.DEFAULT_VECTOR_DIMENSIONS || '2560',
|
||||
);
|
||||
|
||||
try {
|
||||
// 1. Prioritize getting from model config
|
||||
const modelConfig =
|
||||
await this.modelConfigService.findOne(embeddingModelId);
|
||||
|
||||
if (modelConfig && modelConfig.dimensions) {
|
||||
this.logger.log(
|
||||
`Got dimensions from ${modelConfig.name} config: ${modelConfig.dimensions}`,
|
||||
);
|
||||
return modelConfig.dimensions;
|
||||
}
|
||||
|
||||
// 2. Otherwise probe for dimensions
|
||||
this.logger.log(`Probing model dimensions: ${embeddingModelId}`);
|
||||
const probeEmbeddings = await this.embeddingService.getEmbeddings(
|
||||
['probe'],
|
||||
embeddingModelId,
|
||||
);
|
||||
|
||||
if (probeEmbeddings.length > 0) {
|
||||
const actualDimensions = probeEmbeddings[0].length;
|
||||
this.logger.log(
|
||||
`Detected actual model dimensions: ${actualDimensions}`,
|
||||
);
|
||||
|
||||
// Update model config for next use
|
||||
if (modelConfig) {
|
||||
try {
|
||||
await this.modelConfigService.update(modelConfig.id, {
|
||||
dimensions: actualDimensions,
|
||||
});
|
||||
this.logger.log(
|
||||
`Updated model ${modelConfig.name} dimension config to ${actualDimensions}`,
|
||||
);
|
||||
} catch (updateErr) {
|
||||
this.logger.warn(
|
||||
`Failed to update model dimension config: ${updateErr.message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return actualDimensions;
|
||||
}
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
`Failed to get dimensions. Using default: ${defaultDimensions}`,
|
||||
err.message,
|
||||
);
|
||||
}
|
||||
|
||||
return defaultDimensions;
|
||||
}
|
||||
|
||||
/**
|
||||
* Auto-generate document title using AI
|
||||
*/
|
||||
async generateTitle(kbId: string): Promise<string | null> {
|
||||
this.logger.log(`Generating automatic title for file ${kbId}`);
|
||||
|
||||
try {
|
||||
const kb = await this.kbRepository.findOne({ where: { id: kbId } });
|
||||
if (!kb || !kb.content || kb.content.trim().length === 0) {
|
||||
return null;
|
||||
}
|
||||
const tenantId = kb.tenantId;
|
||||
|
||||
// Skip if title already exists
|
||||
if (kb.title) {
|
||||
return kb.title;
|
||||
}
|
||||
|
||||
// Get content sample (max 2500 characters)
|
||||
const contentSample = kb.content.substring(0, 2500);
|
||||
|
||||
// Get language from org settings, or use default
|
||||
const userSettings = await this.userSettingService.getByUser(kb.userId);
|
||||
const language = userSettings.language || 'zh';
|
||||
|
||||
// Build prompt
|
||||
const prompt = this.i18nService.getDocumentTitlePrompt(
|
||||
language,
|
||||
contentSample,
|
||||
);
|
||||
|
||||
// Call LLM to generate title
|
||||
let generatedTitle: string | undefined;
|
||||
try {
|
||||
generatedTitle = await this.chatService.generateSimpleChat(
|
||||
[{ role: 'user', content: prompt }],
|
||||
kb.userId,
|
||||
kb.tenantId,
|
||||
);
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
`Failed to generate title for document ${kbId} due to LLM configuration issue: ${err.message}`,
|
||||
);
|
||||
return null; // Skip title generation if LLM is not configured for this tenant
|
||||
}
|
||||
|
||||
if (generatedTitle && generatedTitle.trim().length > 0) {
|
||||
// Remove extra quotes and newlines
|
||||
const cleanedTitle = generatedTitle
|
||||
.trim()
|
||||
.replace(/^["']|["']$/g, '')
|
||||
.substring(0, 100);
|
||||
await this.kbRepository.update(kbId, { title: cleanedTitle });
|
||||
|
||||
// Also update ES chunks
|
||||
await this.elasticsearchService
|
||||
.updateTitleByFileId(kbId, cleanedTitle, tenantId)
|
||||
.catch((err) => {
|
||||
this.logger.error(
|
||||
`Failed to update title in Elasticsearch for ${kbId}`,
|
||||
err,
|
||||
);
|
||||
});
|
||||
|
||||
this.logger.log(
|
||||
`Successfully generated title for ${kbId}: ${cleanedTitle}`,
|
||||
);
|
||||
return cleanedTitle;
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to generate title for ${kbId}`, error);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,255 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
export interface MemoryStats {
|
||||
heapUsed: number; // Used heap memory (MB)
|
||||
heapTotal: number; // Total heap memory (MB)
|
||||
external: number; // External memory (MB)
|
||||
rss: number; // RSS (Resident Set Size) (MB)
|
||||
timestamp: Date;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class MemoryMonitorService {
|
||||
private readonly logger = new Logger(MemoryMonitorService.name);
|
||||
private readonly MAX_MEMORY_MB: number;
|
||||
private readonly BATCH_SIZE: number;
|
||||
private readonly GC_THRESHOLD_MB: number;
|
||||
|
||||
constructor() {
|
||||
// Load config from env vars. Default values for memory optimization
|
||||
this.MAX_MEMORY_MB = parseInt(process.env.MAX_MEMORY_USAGE_MB || '1024'); // 1GB limit
|
||||
this.BATCH_SIZE = parseInt(process.env.CHUNK_BATCH_SIZE || '100'); // 100 chunks per batch
|
||||
this.GC_THRESHOLD_MB = parseInt(process.env.GC_THRESHOLD_MB || '800'); // Trigger GC at 800MB
|
||||
|
||||
this.logger.log(
|
||||
`Memory monitor initialized: limit=${this.MAX_MEMORY_MB}MB, batchSize=${this.BATCH_SIZE}, GCThreshold=${this.GC_THRESHOLD_MB}MB`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current memory usage
|
||||
*/
|
||||
getMemoryUsage(): MemoryStats {
|
||||
const usage = process.memoryUsage();
|
||||
return {
|
||||
heapUsed: Math.round(usage.heapUsed / 1024 / 1024),
|
||||
heapTotal: Math.round(usage.heapTotal / 1024 / 1024),
|
||||
external: Math.round((usage.external || 0) / 1024 / 1024),
|
||||
rss: Math.round(usage.rss / 1024 / 1024),
|
||||
timestamp: new Date(),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if memory is approaching limit
|
||||
*/
|
||||
isMemoryHigh(): boolean {
|
||||
const usage = this.getMemoryUsage();
|
||||
return usage.heapUsed > this.MAX_MEMORY_MB * 0.85; // 85% threshold
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for memory to become available (with timeout)
|
||||
*/
|
||||
async waitForMemoryAvailable(timeoutMs: number = 30000): Promise<void> {
|
||||
const startTime = Date.now();
|
||||
|
||||
while (this.isMemoryHigh()) {
|
||||
if (Date.now() - startTime > timeoutMs) {
|
||||
throw new Error(
|
||||
`Memory wait timeout: current ${this.getMemoryUsage().heapUsed}MB > ${this.MAX_MEMORY_MB * 0.85}MB`,
|
||||
);
|
||||
}
|
||||
|
||||
this.logger.warn(
|
||||
`Memory usage too high. Waiting for release... ${this.getMemoryUsage().heapUsed}/${this.MAX_MEMORY_MB}MB`,
|
||||
);
|
||||
|
||||
// Force garbage collection (if available)
|
||||
if (global.gc) {
|
||||
this.logger.log('Running forced garbage collection...');
|
||||
global.gc();
|
||||
}
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Force garbage collection (if available)
|
||||
*/
|
||||
forceGC(): void {
|
||||
if (global.gc) {
|
||||
const before = this.getMemoryUsage();
|
||||
global.gc();
|
||||
const after = this.getMemoryUsage();
|
||||
this.logger.log(
|
||||
`GC completed: ${before.heapUsed}MB → ${after.heapUsed}MB (${before.heapUsed - after.heapUsed}MB freed)`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Dynamically adjust batch size
|
||||
*/
|
||||
getDynamicBatchSize(currentMemoryMB: number): number {
|
||||
const baseBatchSize = this.BATCH_SIZE;
|
||||
|
||||
if (currentMemoryMB > this.GC_THRESHOLD_MB) {
|
||||
// Memory pressure, reduce batch size
|
||||
const reduced = Math.max(10, Math.floor(baseBatchSize * 0.5));
|
||||
this.logger.warn(
|
||||
`Memory pressure (${currentMemoryMB}MB), adjusting batch size: ${baseBatchSize} → ${reduced}`,
|
||||
);
|
||||
return reduced;
|
||||
} else if (currentMemoryMB < this.MAX_MEMORY_MB * 0.4) {
|
||||
// Enough memory, increase batch size
|
||||
const increased = Math.min(200, Math.floor(baseBatchSize * 1.2));
|
||||
if (increased > baseBatchSize) {
|
||||
this.logger.log(
|
||||
`Memory available (${currentMemoryMB}MB), adjusting batch size: ${baseBatchSize} → ${increased}`,
|
||||
);
|
||||
}
|
||||
return increased;
|
||||
}
|
||||
|
||||
return baseBatchSize;
|
||||
}
|
||||
|
||||
/**
|
||||
* Process large data: auto-batching and memory control
|
||||
*/
|
||||
async processInBatches<T, R>(
|
||||
items: T[],
|
||||
processor: (batch: T[], batchIndex: number) => Promise<R[]>,
|
||||
options?: {
|
||||
onBatchComplete?: (
|
||||
batchIndex: number,
|
||||
totalBatches: number,
|
||||
results: R[],
|
||||
) => Promise<void> | void;
|
||||
maxConcurrency?: number;
|
||||
},
|
||||
): Promise<R[]> {
|
||||
const totalItems = items.length;
|
||||
if (totalItems === 0) return [];
|
||||
|
||||
const startTime = Date.now();
|
||||
this.logger.log(`Starting batch processing: ${totalItems} items`);
|
||||
|
||||
const allResults: R[] = [];
|
||||
let processedCount = 0;
|
||||
|
||||
for (let i = 0; i < totalItems; ) {
|
||||
// Check memory state and wait
|
||||
await this.waitForMemoryAvailable();
|
||||
|
||||
// Dynamically adjust batch size
|
||||
const currentMem = this.getMemoryUsage().heapUsed;
|
||||
const batchSize = this.getDynamicBatchSize(currentMem);
|
||||
|
||||
// Get current batch
|
||||
const batch = items.slice(i, i + batchSize);
|
||||
const batchIndex = Math.floor(i / batchSize) + 1;
|
||||
const totalBatches = Math.ceil(totalItems / batchSize);
|
||||
|
||||
this.logger.log(
|
||||
`Processing batch ${batchIndex}/${totalBatches}: ${batch.length} items (cumulative ${processedCount}/${totalItems})`,
|
||||
);
|
||||
|
||||
// Process batch
|
||||
const batchResults = await processor(batch, batchIndex);
|
||||
allResults.push(...batchResults);
|
||||
processedCount += batch.length;
|
||||
|
||||
// Callback notification
|
||||
if (options?.onBatchComplete) {
|
||||
await options.onBatchComplete(batchIndex, totalBatches, batchResults);
|
||||
}
|
||||
|
||||
// Force GC if memory near threshold
|
||||
if (currentMem > this.GC_THRESHOLD_MB) {
|
||||
this.forceGC();
|
||||
}
|
||||
|
||||
// Clear references to help GC
|
||||
batch.length = 0;
|
||||
|
||||
i += batchSize;
|
||||
}
|
||||
|
||||
const duration = ((Date.now() - startTime) / 1000).toFixed(2);
|
||||
const finalMem = this.getMemoryUsage();
|
||||
this.logger.log(
|
||||
`Batch processing completed: ${totalItems} items, duration ${duration}s, final memory ${finalMem.heapUsed}MB`,
|
||||
);
|
||||
|
||||
return allResults;
|
||||
}
|
||||
|
||||
/**
|
||||
* Estimate memory required for processing
|
||||
*/
|
||||
estimateMemoryUsage(
|
||||
itemCount: number,
|
||||
itemSizeBytes: number,
|
||||
vectorDim: number,
|
||||
): number {
|
||||
// Text content memory
|
||||
const textMemory = itemCount * itemSizeBytes;
|
||||
|
||||
// Vector memory (dimension * 4 bytes per vector)
|
||||
const vectorMemory = itemCount * vectorDim * 4;
|
||||
|
||||
// Object overhead (~100 bytes per object)
|
||||
const overhead = itemCount * 100;
|
||||
|
||||
const totalMB = Math.round(
|
||||
(textMemory + vectorMemory + overhead) / 1024 / 1024,
|
||||
);
|
||||
|
||||
return totalMB;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if batching should be used
|
||||
*/
|
||||
shouldUseBatching(
|
||||
itemCount: number,
|
||||
itemSizeBytes: number,
|
||||
vectorDim: number,
|
||||
): boolean {
|
||||
const estimatedMB = this.estimateMemoryUsage(
|
||||
itemCount,
|
||||
itemSizeBytes,
|
||||
vectorDim,
|
||||
);
|
||||
const threshold = this.MAX_MEMORY_MB * 0.7; // 70% threshold
|
||||
|
||||
if (estimatedMB > threshold) {
|
||||
this.logger.warn(
|
||||
`Estimated memory ${estimatedMB}MB exceeds threshold ${threshold}MB, using batch processing`,
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get recommended batch size
|
||||
*/
|
||||
getRecommendedBatchSize(itemSizeBytes: number, vectorDim: number): number {
|
||||
// Goal: max 200MB memory per batch
|
||||
const targetMemoryMB = 200;
|
||||
const targetMemoryBytes = targetMemoryMB * 1024 * 1024;
|
||||
|
||||
// Memory per item = text + vector + overhead
|
||||
const singleItemMemory = itemSizeBytes + vectorDim * 4 + 100;
|
||||
|
||||
const batchSize = Math.floor(targetMemoryBytes / singleItemMemory);
|
||||
|
||||
// Limit between 10-200
|
||||
return Math.max(10, Math.min(200, batchSize));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
export interface TextChunk {
|
||||
content: string;
|
||||
index: number;
|
||||
startPosition: number;
|
||||
endPosition: number;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class TextChunkerService {
|
||||
chunkText(
|
||||
text: string,
|
||||
chunkSize: number = 1000,
|
||||
overlap: number = 200,
|
||||
): TextChunk[] {
|
||||
if (!text || text.trim().length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const cleanText = text.trim();
|
||||
const chunkSizeInChars = chunkSize * 4; // 1 token ≈ 4 chars
|
||||
const overlapInChars = overlap * 4;
|
||||
|
||||
// If text length <= chunk size, return entire text as one chunk
|
||||
if (cleanText.length <= chunkSizeInChars) {
|
||||
return [
|
||||
{
|
||||
content: cleanText,
|
||||
index: 0,
|
||||
startPosition: 0,
|
||||
endPosition: cleanText.length,
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
const chunks: TextChunk[] = [];
|
||||
let start = 0;
|
||||
let index = 0;
|
||||
|
||||
while (start < cleanText.length) {
|
||||
let end = Math.min(start + chunkSizeInChars, cleanText.length);
|
||||
|
||||
// Split by sentence boundaries
|
||||
if (end < cleanText.length) {
|
||||
const sentenceEnd = this.findSentenceEnd(
|
||||
cleanText,
|
||||
end,
|
||||
start + chunkSizeInChars * 0.8,
|
||||
);
|
||||
if (sentenceEnd > start) {
|
||||
end = sentenceEnd;
|
||||
}
|
||||
}
|
||||
|
||||
const content = cleanText.slice(start, end).trim();
|
||||
if (content.length > 0) {
|
||||
chunks.push({
|
||||
content,
|
||||
index,
|
||||
startPosition: start,
|
||||
endPosition: end,
|
||||
});
|
||||
index++;
|
||||
}
|
||||
|
||||
// Fix infinite loop: if we reached the end, stop here.
|
||||
if (end >= cleanText.length) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Calculate start position of next chunk
|
||||
const newStart = end - overlapInChars;
|
||||
// Protect against infinite loop if overlap is too large or chunk too small
|
||||
if (newStart <= start) {
|
||||
start = end; // Force advance if overlap would cause stagnation
|
||||
} else {
|
||||
start = newStart;
|
||||
}
|
||||
}
|
||||
|
||||
return chunks;
|
||||
}
|
||||
|
||||
private findSentenceEnd(
|
||||
text: string,
|
||||
preferredEnd: number,
|
||||
minEnd: number,
|
||||
): number {
|
||||
const sentenceEnders = ['.', '!', '?', '。', '！', '？'];
|
||||
|
||||
for (let i = preferredEnd; i >= minEnd; i--) {
|
||||
if (sentenceEnders.includes(text[i])) {
|
||||
return i + 1;
|
||||
}
|
||||
}
|
||||
|
||||
for (let i = preferredEnd; i >= minEnd; i--) {
|
||||
if (text[i] === ' ' || text[i] === '\n') {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
|
||||
return preferredEnd;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
import {
|
||||
Controller,
|
||||
Get,
|
||||
Post,
|
||||
Put,
|
||||
Delete,
|
||||
Body,
|
||||
Param,
|
||||
UseGuards,
|
||||
Request,
|
||||
Query,
|
||||
} from '@nestjs/common';
|
||||
import { CombinedAuthGuard } from '../auth/combined-auth.guard';
|
||||
import { RolesGuard } from '../auth/roles.guard';
|
||||
import { Roles } from '../auth/roles.decorator';
|
||||
import { UserRole } from '../user/user-role.enum';
|
||||
import {
|
||||
KnowledgeGroupService,
|
||||
CreateGroupDto,
|
||||
UpdateGroupDto,
|
||||
} from './knowledge-group.service';
|
||||
import { I18nService } from '../i18n/i18n.service';
|
||||
|
||||
@Controller('knowledge-groups')
|
||||
@UseGuards(CombinedAuthGuard, RolesGuard)
|
||||
export class KnowledgeGroupController {
|
||||
constructor(
|
||||
private readonly groupService: KnowledgeGroupService,
|
||||
private readonly i18nService: I18nService,
|
||||
) {}
|
||||
|
||||
@Get()
|
||||
async findAll(@Request() req) {
|
||||
// All users can see all groups for their tenant (returns tree structure)
|
||||
return await this.groupService.findAll(req.user.id, req.user.tenantId);
|
||||
}
|
||||
|
||||
@Get(':id')
|
||||
async findOne(@Param('id') id: string, @Request() req) {
|
||||
return await this.groupService.findOne(id, req.user.id, req.user.tenantId);
|
||||
}
|
||||
|
||||
@Post()
|
||||
@Roles(UserRole.TENANT_ADMIN, UserRole.SUPER_ADMIN)
|
||||
async create(@Body() createGroupDto: CreateGroupDto, @Request() req) {
|
||||
return await this.groupService.create(
|
||||
req.user.id,
|
||||
req.user.tenantId,
|
||||
createGroupDto,
|
||||
);
|
||||
}
|
||||
|
||||
@Put(':id')
|
||||
@Roles(UserRole.TENANT_ADMIN, UserRole.SUPER_ADMIN)
|
||||
async update(
|
||||
@Param('id') id: string,
|
||||
@Body() updateGroupDto: UpdateGroupDto,
|
||||
@Request() req,
|
||||
) {
|
||||
return await this.groupService.update(
|
||||
id,
|
||||
req.user.id,
|
||||
req.user.tenantId,
|
||||
updateGroupDto,
|
||||
);
|
||||
}
|
||||
|
||||
@Delete(':id')
|
||||
@Roles(UserRole.TENANT_ADMIN, UserRole.SUPER_ADMIN)
|
||||
async remove(@Param('id') id: string, @Request() req) {
|
||||
await this.groupService.remove(id, req.user.id, req.user.tenantId);
|
||||
return { message: this.i18nService.getMessage('groupDeleted') };
|
||||
}
|
||||
|
||||
@Get(':id/files')
|
||||
async getGroupFiles(@Param('id') id: string, @Request() req) {
|
||||
const files = await this.groupService.getGroupFiles(
|
||||
id,
|
||||
req.user.id,
|
||||
req.user.tenantId,
|
||||
);
|
||||
return { files };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
import {
|
||||
Entity,
|
||||
PrimaryGeneratedColumn,
|
||||
Column,
|
||||
CreateDateColumn,
|
||||
UpdateDateColumn,
|
||||
ManyToMany,
|
||||
JoinTable,
|
||||
ManyToOne,
|
||||
OneToMany,
|
||||
JoinColumn,
|
||||
} from 'typeorm';
|
||||
import { KnowledgeBase } from '../knowledge-base/knowledge-base.entity';
|
||||
import { Tenant } from '../tenant/tenant.entity';
|
||||
|
||||
@Entity('knowledge_groups')
|
||||
export class KnowledgeGroup {
|
||||
@PrimaryGeneratedColumn('uuid')
|
||||
id: string;
|
||||
|
||||
@Column()
|
||||
name: string;
|
||||
|
||||
@Column({ nullable: true })
|
||||
description: string;
|
||||
|
||||
@Column({ default: '#3B82F6' })
|
||||
color: string;
|
||||
|
||||
// Removed userId field to make groups globally accessible
|
||||
// Tenant scoped: groups are shared within a tenant but isolated across tenants
|
||||
@Column({ name: 'tenant_id', nullable: true, type: 'text' })
|
||||
tenantId: string;
|
||||
|
||||
@ManyToOne(() => Tenant, { nullable: true, onDelete: 'CASCADE' })
|
||||
@JoinColumn({ name: 'tenant_id' })
|
||||
tenant: Tenant;
|
||||
|
||||
// Hierarchical parent-child relationship
|
||||
@Column({ name: 'parent_id', nullable: true, type: 'text' })
|
||||
parentId: string | null;
|
||||
|
||||
@ManyToOne(() => KnowledgeGroup, (group) => group.children, {
|
||||
nullable: true,
|
||||
onDelete: 'SET NULL',
|
||||
})
|
||||
@JoinColumn({ name: 'parent_id' })
|
||||
parent: KnowledgeGroup;
|
||||
|
||||
@OneToMany(() => KnowledgeGroup, (group) => group.parent)
|
||||
children: KnowledgeGroup[];
|
||||
|
||||
@CreateDateColumn({ name: 'created_at' })
|
||||
createdAt: Date;
|
||||
|
||||
@UpdateDateColumn({ name: 'updated_at' })
|
||||
updatedAt: Date;
|
||||
|
||||
@ManyToMany(() => KnowledgeBase, (kb) => kb.groups)
|
||||
@JoinTable({
|
||||
name: 'knowledge_base_groups',
|
||||
joinColumn: { name: 'group_id', referencedColumnName: 'id' },
|
||||
inverseJoinColumn: {
|
||||
name: 'knowledge_base_id',
|
||||
referencedColumnName: 'id',
|
||||
},
|
||||
})
|
||||
knowledgeBases: KnowledgeBase[];
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
import { Module, forwardRef } from '@nestjs/common';
|
||||
import { KnowledgeBaseModule } from '../knowledge-base/knowledge-base.module';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
import { KnowledgeGroup } from './knowledge-group.entity';
|
||||
import { KnowledgeBase } from '../knowledge-base/knowledge-base.entity';
|
||||
import { KnowledgeGroupService } from './knowledge-group.service';
|
||||
import { KnowledgeGroupController } from './knowledge-group.controller';
|
||||
import { ElasticsearchModule } from '../elasticsearch/elasticsearch.module';
|
||||
import { I18nModule } from '../i18n/i18n.module';
|
||||
import { UserModule } from '../user/user.module';
|
||||
import { TenantModule } from '../tenant/tenant.module';
|
||||
import { CombinedAuthGuard } from '../auth/combined-auth.guard';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
TypeOrmModule.forFeature([KnowledgeGroup, KnowledgeBase]),
|
||||
forwardRef(() => ElasticsearchModule),
|
||||
forwardRef(() => KnowledgeBaseModule),
|
||||
I18nModule,
|
||||
UserModule,
|
||||
TenantModule,
|
||||
],
|
||||
controllers: [KnowledgeGroupController],
|
||||
providers: [KnowledgeGroupService, CombinedAuthGuard],
|
||||
exports: [KnowledgeGroupService],
|
||||
})
|
||||
export class KnowledgeGroupModule {}
|
||||
@@ -0,0 +1,401 @@
|
||||
import {
|
||||
Injectable,
|
||||
NotFoundException,
|
||||
ForbiddenException,
|
||||
Inject,
|
||||
forwardRef,
|
||||
} from '@nestjs/common';
|
||||
import { I18nService } from '../i18n/i18n.service';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Repository, In } from 'typeorm';
|
||||
import { KnowledgeGroup } from './knowledge-group.entity';
|
||||
import { KnowledgeBase } from '../knowledge-base/knowledge-base.entity';
|
||||
import { KnowledgeBaseService } from '../knowledge-base/knowledge-base.service';
|
||||
import { TenantService } from '../tenant/tenant.service';
|
||||
|
||||
export interface CreateGroupDto {
|
||||
name: string;
|
||||
description?: string;
|
||||
color?: string;
|
||||
parentId?: string | null;
|
||||
}
|
||||
|
||||
export interface UpdateGroupDto {
|
||||
name?: string;
|
||||
description?: string;
|
||||
color?: string;
|
||||
parentId?: string | null;
|
||||
}
|
||||
|
||||
export interface GroupWithFileCount {
|
||||
id: string;
|
||||
name: string;
|
||||
description?: string;
|
||||
color: string;
|
||||
fileCount: number;
|
||||
parentId?: string | null;
|
||||
children?: GroupWithFileCount[];
|
||||
createdAt: Date;
|
||||
}
|
||||
|
||||
export interface PaginatedGroups {
|
||||
items: GroupWithFileCount[];
|
||||
total: number;
|
||||
page: number;
|
||||
limit: number;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class KnowledgeGroupService {
|
||||
constructor(
|
||||
@InjectRepository(KnowledgeGroup)
|
||||
private groupRepository: Repository<KnowledgeGroup>,
|
||||
@InjectRepository(KnowledgeBase)
|
||||
private knowledgeBaseRepository: Repository<KnowledgeBase>,
|
||||
@Inject(forwardRef(() => KnowledgeBaseService))
|
||||
private knowledgeBaseService: KnowledgeBaseService,
|
||||
private tenantService: TenantService,
|
||||
private i18nService: I18nService,
|
||||
) {}
|
||||
|
||||
async findAll(
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
): Promise<GroupWithFileCount[]> {
|
||||
// Return all groups for the tenant with file counts
|
||||
const groups = await this.groupRepository
|
||||
.createQueryBuilder('group')
|
||||
.leftJoin('group.knowledgeBases', 'kb')
|
||||
.where('group.tenantId = :tenantId', { tenantId })
|
||||
.addSelect('COUNT(kb.id)', 'fileCount')
|
||||
.groupBy('group.id')
|
||||
.orderBy('group.createdAt', 'ASC')
|
||||
.getRawAndEntities();
|
||||
|
||||
const flatList: GroupWithFileCount[] = groups.entities.map(
|
||||
(group, index) => ({
|
||||
id: group.id,
|
||||
name: group.name,
|
||||
description: group.description,
|
||||
color: group.color,
|
||||
parentId: group.parentId ?? null,
|
||||
fileCount: parseInt(groups.raw[index].fileCount) || 0,
|
||||
createdAt: group.createdAt,
|
||||
children: [],
|
||||
}),
|
||||
);
|
||||
|
||||
// Build tree structure
|
||||
return this.buildTree(flatList);
|
||||
}
|
||||
|
||||
/** Build a nested tree from a flat list */
|
||||
private buildTree(items: GroupWithFileCount[]): GroupWithFileCount[] {
|
||||
const map = new Map<string, GroupWithFileCount>();
|
||||
items.forEach((item) => map.set(item.id, { ...item, children: [] }));
|
||||
|
||||
const roots: GroupWithFileCount[] = [];
|
||||
map.forEach((item) => {
|
||||
if (item.parentId && map.has(item.parentId)) {
|
||||
map.get(item.parentId)!.children!.push(item);
|
||||
} else {
|
||||
roots.push(item);
|
||||
}
|
||||
});
|
||||
return roots;
|
||||
}
|
||||
|
||||
async findOne(
|
||||
id: string,
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
): Promise<KnowledgeGroup> {
|
||||
const group = await this.groupRepository.findOne({
|
||||
where: { id },
|
||||
relations: ['knowledgeBases'],
|
||||
});
|
||||
|
||||
if (!group) {
|
||||
throw new NotFoundException(this.i18nService.getMessage('groupNotFound'));
|
||||
}
|
||||
|
||||
// Check permission using TenantService
|
||||
const hasAccess = await this.tenantService.canAccessTenant(
|
||||
userId,
|
||||
group.tenantId,
|
||||
tenantId,
|
||||
);
|
||||
if (!hasAccess) {
|
||||
throw new ForbiddenException(
|
||||
`You do not have permission to access this knowledge group`,
|
||||
);
|
||||
}
|
||||
|
||||
return group;
|
||||
}
|
||||
|
||||
async create(
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
createGroupDto: CreateGroupDto,
|
||||
): Promise<KnowledgeGroup> {
|
||||
const group = this.groupRepository.create({
|
||||
...createGroupDto,
|
||||
parentId: createGroupDto.parentId ?? null,
|
||||
tenantId,
|
||||
});
|
||||
|
||||
return await this.groupRepository.save(group);
|
||||
}
|
||||
|
||||
async update(
|
||||
id: string,
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
updateGroupDto: UpdateGroupDto,
|
||||
): Promise<KnowledgeGroup> {
|
||||
// Update group within the tenant
|
||||
const group = await this.groupRepository.findOne({
|
||||
where: { id, tenantId },
|
||||
});
|
||||
|
||||
if (!group) {
|
||||
throw new NotFoundException(this.i18nService.getMessage('groupNotFound'));
|
||||
}
|
||||
|
||||
Object.assign(group, updateGroupDto);
|
||||
return await this.groupRepository.save(group);
|
||||
}
|
||||
|
||||
async remove(id: string, userId: string, tenantId: string): Promise<void> {
|
||||
const group = await this.groupRepository.findOne({
|
||||
where: { id, tenantId },
|
||||
});
|
||||
|
||||
if (!group) {
|
||||
throw new NotFoundException(this.i18nService.getMessage('groupNotFound'));
|
||||
}
|
||||
|
||||
// Recursively delete this group and all its descendants
|
||||
await this.removeGroupRecursive(id, userId, tenantId);
|
||||
}
|
||||
|
||||
/** Recursively delete a group, all its children, and all associated files */
|
||||
private async removeGroupRecursive(
|
||||
id: string,
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
): Promise<void> {
|
||||
// 1. Find all direct children of this group
|
||||
const children = await this.groupRepository.find({
|
||||
where: { parentId: id, tenantId },
|
||||
});
|
||||
|
||||
// 2. Recurse into each child first (depth-first)
|
||||
for (const child of children) {
|
||||
await this.removeGroupRecursive(child.id, userId, tenantId);
|
||||
}
|
||||
|
||||
// 3. Delete all files belonging to this group
|
||||
const files = await this.knowledgeBaseRepository
|
||||
.createQueryBuilder('kb')
|
||||
.innerJoin('kb.groups', 'group')
|
||||
.where('group.id = :groupId', { groupId: id })
|
||||
.select('kb.id')
|
||||
.getMany();
|
||||
|
||||
for (const file of files) {
|
||||
try {
|
||||
const fullFile = await this.knowledgeBaseRepository.findOne({
|
||||
where: { id: file.id },
|
||||
select: ['id', 'userId', 'tenantId'],
|
||||
});
|
||||
if (fullFile) {
|
||||
await this.knowledgeBaseService.deleteFile(
|
||||
fullFile.id,
|
||||
fullFile.userId,
|
||||
fullFile.tenantId,
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`Failed to delete file ${file.id} when deleting group ${id}`,
|
||||
error,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Delete the group itself
|
||||
const group = await this.groupRepository.findOne({ where: { id } });
|
||||
if (group) {
|
||||
await this.groupRepository.remove(group);
|
||||
}
|
||||
}
|
||||
|
||||
async getGroupFiles(
|
||||
groupId: string,
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
): Promise<KnowledgeBase[]> {
|
||||
const group = await this.groupRepository.findOne({
|
||||
where: { id: groupId },
|
||||
relations: ['knowledgeBases'],
|
||||
});
|
||||
|
||||
if (!group) {
|
||||
throw new NotFoundException(this.i18nService.getMessage('groupNotFound'));
|
||||
}
|
||||
|
||||
// Check permission using TenantService
|
||||
const hasAccess = await this.tenantService.canAccessTenant(
|
||||
userId,
|
||||
group.tenantId,
|
||||
tenantId,
|
||||
);
|
||||
if (!hasAccess) {
|
||||
throw new ForbiddenException(
|
||||
`You do not have permission to access this knowledge group`,
|
||||
);
|
||||
}
|
||||
|
||||
return group.knowledgeBases;
|
||||
}
|
||||
|
||||
async addFilesToGroup(
|
||||
fileId: string,
|
||||
groupIds: string[],
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
): Promise<void> {
|
||||
const file = await this.knowledgeBaseRepository.findOne({
|
||||
where: { id: fileId },
|
||||
relations: ['groups'],
|
||||
});
|
||||
|
||||
if (!file) {
|
||||
throw new NotFoundException(this.i18nService.getMessage('fileNotFound'));
|
||||
}
|
||||
|
||||
// Check permission for file
|
||||
const fileAccess = await this.tenantService.canAccessTenant(
|
||||
userId,
|
||||
file.tenantId,
|
||||
tenantId,
|
||||
);
|
||||
if (!fileAccess) {
|
||||
throw new ForbiddenException(
|
||||
`You do not have permission to modify this knowledge base`,
|
||||
);
|
||||
}
|
||||
|
||||
// Load all groups by ID without user restriction
|
||||
const groups = await this.groupRepository.findBy({ id: In(groupIds) });
|
||||
// Verify each group access
|
||||
for (const g of groups) {
|
||||
const gAccess = await this.tenantService.canAccessTenant(
|
||||
userId,
|
||||
g.tenantId,
|
||||
tenantId,
|
||||
);
|
||||
if (!gAccess) {
|
||||
throw new ForbiddenException(
|
||||
`You do not have permission to access group ${g.name}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (groupIds.length > 0 && groups.length !== groupIds.length) {
|
||||
throw new NotFoundException(
|
||||
this.i18nService.getMessage('someGroupsNotFound'),
|
||||
);
|
||||
}
|
||||
|
||||
file.groups = groups;
|
||||
await this.knowledgeBaseRepository.save(file);
|
||||
}
|
||||
|
||||
async removeFileFromGroup(
|
||||
fileId: string,
|
||||
groupId: string,
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
): Promise<void> {
|
||||
const file = await this.knowledgeBaseRepository.findOne({
|
||||
where: { id: fileId },
|
||||
relations: ['groups'],
|
||||
});
|
||||
|
||||
if (!file) {
|
||||
throw new NotFoundException(this.i18nService.getMessage('fileNotFound'));
|
||||
}
|
||||
|
||||
// Check permission for file
|
||||
const fileAccess = await this.tenantService.canAccessTenant(
|
||||
userId,
|
||||
file.tenantId,
|
||||
tenantId,
|
||||
);
|
||||
if (!fileAccess) {
|
||||
throw new ForbiddenException(
|
||||
`You do not have permission to modify this knowledge base`,
|
||||
);
|
||||
}
|
||||
|
||||
file.groups = file.groups.filter((group) => group.id !== groupId);
|
||||
await this.knowledgeBaseRepository.save(file);
|
||||
}
|
||||
|
||||
async getFileIdsByGroups(
|
||||
groupIds: string[],
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
): Promise<string[]> {
|
||||
if (!groupIds || groupIds.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
// Security check: Verify user has access to these groups
|
||||
const groups = await this.groupRepository.findBy({ id: In(groupIds) });
|
||||
for (const g of groups) {
|
||||
const hasAccess = await this.tenantService.canAccessTenant(
|
||||
userId,
|
||||
g.tenantId,
|
||||
tenantId,
|
||||
);
|
||||
if (!hasAccess) {
|
||||
throw new ForbiddenException(
|
||||
`You do not have permission to access group ${g.name}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const result = await this.knowledgeBaseRepository
|
||||
.createQueryBuilder('kb')
|
||||
.innerJoin('kb.groups', 'group')
|
||||
.where('group.id IN (:...groupIds)', { groupIds })
|
||||
// No extra tenantId check here because we verified the groups above
|
||||
.select('DISTINCT kb.id', 'id')
|
||||
.getRawMany();
|
||||
|
||||
return result.map((row) => row.id);
|
||||
}
|
||||
|
||||
/**
|
||||
* Find or create a group by name and parentId within a tenant.
|
||||
* Used by import tasks to build folder hierarchy.
|
||||
*/
|
||||
async findOrCreate(
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
name: string,
|
||||
parentId: string | null,
|
||||
description?: string,
|
||||
): Promise<KnowledgeGroup> {
|
||||
const existing = await this.groupRepository.findOne({
|
||||
where: { name, tenantId, parentId: parentId ?? undefined },
|
||||
});
|
||||
if (existing) return existing;
|
||||
|
||||
return this.create(userId, tenantId, { name, description, parentId });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
/**
|
||||
* LibreOffice Service Interface Definition
|
||||
*/
|
||||
|
||||
export interface LibreOfficeConvertResponse {
|
||||
pdf_path?: string;
|
||||
pdf_data?: string; // base64 encoded PDF data
|
||||
converted: boolean;
|
||||
original: string;
|
||||
file_size: number;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface LibreOfficeHealthResponse {
|
||||
status: string;
|
||||
service: string;
|
||||
version: string;
|
||||
uptime: number;
|
||||
}
|
||||
|
||||
export interface LibreOfficeVersionResponse {
|
||||
service: string;
|
||||
version: string;
|
||||
framework: string;
|
||||
libreoffice: string;
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { LibreOfficeService } from './libreoffice.service';
|
||||
|
||||
@Module({
|
||||
providers: [LibreOfficeService],
|
||||
exports: [LibreOfficeService],
|
||||
})
|
||||
export class LibreOfficeModule {}
|
||||
@@ -0,0 +1,228 @@
|
||||
import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import * as fs from 'fs/promises';
|
||||
import * as path from 'path';
|
||||
import axios from 'axios';
|
||||
import FormData from 'form-data';
|
||||
import {
|
||||
LibreOfficeConvertResponse,
|
||||
LibreOfficeHealthResponse,
|
||||
} from './libreoffice.interface';
|
||||
import { I18nService } from '../i18n/i18n.service';
|
||||
|
||||
@Injectable()
|
||||
export class LibreOfficeService implements OnModuleInit {
|
||||
private readonly logger = new Logger(LibreOfficeService.name);
|
||||
private baseUrl: string;
|
||||
|
||||
constructor(
|
||||
private configService: ConfigService,
|
||||
private i18nService: I18nService,
|
||||
) {}
|
||||
|
||||
onModuleInit() {
|
||||
const libreofficeUrl = this.configService.get<string>('LIBREOFFICE_URL');
|
||||
if (!libreofficeUrl) {
|
||||
throw new Error(this.i18nService.getMessage('libreofficeUrlRequired'));
|
||||
}
|
||||
this.baseUrl = libreofficeUrl;
|
||||
this.logger.log(
|
||||
`LibreOffice service initialized with base URL: ${this.baseUrl}`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check LibreOffice service health status
|
||||
*/
|
||||
async healthCheck(): Promise<boolean> {
|
||||
try {
|
||||
const response = await axios.get<LibreOfficeHealthResponse>(
|
||||
`${this.baseUrl}/health`,
|
||||
{ timeout: 5000 },
|
||||
);
|
||||
return response.data.status === 'healthy';
|
||||
} catch (error) {
|
||||
this.logger.error('LibreOffice health check failed', error.message);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert document to PDF
|
||||
* @param filePath Path of file to convert
|
||||
* @returns PDF file path
|
||||
*/
|
||||
async convertToPDF(filePath: string): Promise<string> {
|
||||
const fileName = path.basename(filePath);
|
||||
const ext = path.extname(fileName).toLowerCase();
|
||||
|
||||
// Return original path directly if PDF
|
||||
if (ext === '.pdf') {
|
||||
this.logger.log(`File is already PDF: ${filePath}`);
|
||||
return filePath;
|
||||
}
|
||||
|
||||
// Check if file exists
|
||||
try {
|
||||
await fs.access(filePath);
|
||||
} catch {
|
||||
throw new Error(`File does not exist: ${filePath}`);
|
||||
}
|
||||
|
||||
// Generate output PDF path
|
||||
const dir = path.dirname(filePath);
|
||||
const baseName = path.basename(filePath, ext);
|
||||
const targetPdfPath = path.join(dir, `${baseName}.pdf`);
|
||||
|
||||
// Return directly if PDF already exists
|
||||
try {
|
||||
await fs.access(targetPdfPath);
|
||||
this.logger.log(`PDF already exists: ${targetPdfPath}`);
|
||||
return targetPdfPath;
|
||||
} catch {
|
||||
// Need to convert as PDF does not exist
|
||||
}
|
||||
|
||||
// Load file
|
||||
const fileBuffer = await fs.readFile(filePath);
|
||||
|
||||
// Build FormData
|
||||
const formData = new FormData();
|
||||
formData.append('file', fileBuffer, fileName);
|
||||
|
||||
this.logger.log(`Converting ${fileName} to PDF...`);
|
||||
|
||||
// Conversion retry count
|
||||
const maxRetries = 3;
|
||||
let lastError: any;
|
||||
|
||||
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
||||
try {
|
||||
// Call LibreOffice service
|
||||
const response = await axios.post(`${this.baseUrl}/convert`, formData, {
|
||||
headers: formData.getHeaders(),
|
||||
timeout: 300000, // 5 minute timeout
|
||||
responseType: 'stream', // Receive file stream
|
||||
maxRedirects: 5, // Max redirects
|
||||
});
|
||||
|
||||
// Write stream to output file
|
||||
const writer = (await import('fs')).createWriteStream(targetPdfPath);
|
||||
response.data.pipe(writer);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
writer.on('finish', () => {
|
||||
this.logger.log(
|
||||
`Conversion completed: ${fileName} -> ${targetPdfPath}`,
|
||||
);
|
||||
resolve(targetPdfPath);
|
||||
});
|
||||
writer.on('error', (err) => {
|
||||
this.logger.error(`Error writing PDF file: ${err.message}`);
|
||||
reject(err);
|
||||
});
|
||||
});
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Attempt ${attempt} failed for ${fileName}: ${error.message}`,
|
||||
);
|
||||
lastError = error;
|
||||
|
||||
// Wait and retry on socket hang up or connection error
|
||||
if (
|
||||
error.code === 'ECONNRESET' ||
|
||||
error.code === 'ECONNREFUSED' ||
|
||||
error.code === 'ETIMEDOUT' ||
|
||||
error.message.includes('socket hang up')
|
||||
) {
|
||||
if (attempt < maxRetries) {
|
||||
const delay = 2000 * attempt; // Increasing delay
|
||||
this.logger.log(`Waiting ${delay}ms before retry...`);
|
||||
await new Promise((resolve) => setTimeout(resolve, delay));
|
||||
}
|
||||
} else {
|
||||
// Do not retry other errors
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Detailed error handling if all retries fail
|
||||
if (lastError.response) {
|
||||
try {
|
||||
const stream = lastError.response.data;
|
||||
const chunks: any[] = [];
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(chunk);
|
||||
}
|
||||
const errorText = Buffer.concat(chunks).toString('utf8');
|
||||
this.logger.error(`LibreOffice service error details: ${errorText}`);
|
||||
|
||||
let detail = errorText;
|
||||
try {
|
||||
const errorJson = JSON.parse(errorText);
|
||||
detail = errorJson.detail || errorText;
|
||||
} catch {
|
||||
// ignore JSON parse error
|
||||
}
|
||||
|
||||
if (lastError.response.status === 504) {
|
||||
throw new Error('Conversion timed out. The file may be too large.');
|
||||
}
|
||||
throw new Error(`Conversion failed: ${detail}`);
|
||||
} catch (streamError) {
|
||||
this.logger.error('Error reading error stream:', streamError);
|
||||
throw new Error(`Conversion failed: ${lastError.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.error(
|
||||
`Conversion failed for ${fileName} after ${maxRetries} attempts:`,
|
||||
lastError.message,
|
||||
);
|
||||
if (lastError.code === 'ECONNREFUSED') {
|
||||
throw new Error(
|
||||
'LibreOffice service is not running. Please check the service status.',
|
||||
);
|
||||
}
|
||||
if (
|
||||
lastError.code === 'ECONNRESET' ||
|
||||
lastError.message.includes('socket hang up')
|
||||
) {
|
||||
throw new Error(
|
||||
'Connection to LibreOffice service was reset. The service may be unstable.',
|
||||
);
|
||||
}
|
||||
throw new Error(`Conversion failed: ${lastError.message}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch convert files
|
||||
*/
|
||||
async batchConvert(filePaths: string[]): Promise<string[]> {
|
||||
const results: string[] = [];
|
||||
for (const filePath of filePaths) {
|
||||
try {
|
||||
const pdfPath = await this.convertToPDF(filePath);
|
||||
results.push(pdfPath);
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to convert ${filePath}: ${error.message}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get service version information
|
||||
*/
|
||||
async getVersion(): Promise<any> {
|
||||
try {
|
||||
const response = await axios.get(`${this.baseUrl}/version`);
|
||||
return response.data;
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to get version', error.message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import { NestFactory } from '@nestjs/core';
|
||||
import { AppModule } from './app.module';
|
||||
import { ValidationPipe } from '@nestjs/common';
|
||||
import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
|
||||
|
||||
async function bootstrap() {
|
||||
const app = await NestFactory.create(AppModule);
|
||||
app.useGlobalPipes(new ValidationPipe());
|
||||
app.enableCors({
|
||||
origin: true, // Allow all origins
|
||||
methods: 'GET,HEAD,PUT,PATCH,POST,DELETE,OPTIONS',
|
||||
credentials: true,
|
||||
});
|
||||
app.setGlobalPrefix('api'); // Set a global API prefix
|
||||
|
||||
// Swagger / OpenAPI documentation
|
||||
const config = new DocumentBuilder()
|
||||
.setTitle('AuraK API')
|
||||
.setDescription(
|
||||
'External API for accessing AuraK functionalities via API Key',
|
||||
)
|
||||
.setVersion('1.0')
|
||||
.addApiKey({ type: 'apiKey', name: 'x-api-key', in: 'header' }, 'x-api-key')
|
||||
.build();
|
||||
const document = SwaggerModule.createDocument(app, config);
|
||||
SwaggerModule.setup('api/docs', app, document);
|
||||
|
||||
await app.listen(process.env.PORT ?? 3001);
|
||||
|
||||
// Ensure "Default" tenant exists
|
||||
const { TenantService } = await import('./tenant/tenant.service');
|
||||
const tenantService = app.get(TenantService);
|
||||
await tenantService.ensureDefaultTenant();
|
||||
}
|
||||
bootstrap();
|
||||
@@ -0,0 +1,91 @@
|
||||
import { MigrationInterface, QueryRunner } from 'typeorm';
|
||||
|
||||
export class AddKnowledgeBaseEnhancements1737800000000 implements MigrationInterface {
|
||||
name = 'AddKnowledgeBaseEnhancements1737800000000';
|
||||
|
||||
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||
// Create knowledge base group table
|
||||
await queryRunner.query(`
|
||||
CREATE TABLE "knowledge_groups" (
|
||||
"id" varchar PRIMARY KEY NOT NULL,
|
||||
"name" varchar NOT NULL,
|
||||
"description" varchar,
|
||||
"color" varchar NOT NULL DEFAULT '#3B82F6',
|
||||
"user_id" varchar NOT NULL,
|
||||
"created_at" datetime NOT NULL DEFAULT (datetime('now')),
|
||||
"updated_at" datetime NOT NULL DEFAULT (datetime('now'))
|
||||
)
|
||||
`);
|
||||
|
||||
// Create document group related tables
|
||||
await queryRunner.query(`
|
||||
CREATE TABLE "knowledge_base_groups" (
|
||||
"knowledge_base_id" varchar NOT NULL,
|
||||
"group_id" varchar NOT NULL,
|
||||
"created_at" datetime NOT NULL DEFAULT (datetime('now')),
|
||||
PRIMARY KEY ("knowledge_base_id", "group_id"),
|
||||
FOREIGN KEY ("knowledge_base_id") REFERENCES "knowledge_base" ("id") ON DELETE CASCADE,
|
||||
FOREIGN KEY ("group_id") REFERENCES "knowledge_groups" ("id") ON DELETE CASCADE
|
||||
)
|
||||
`);
|
||||
|
||||
// Create search history table
|
||||
await queryRunner.query(`
|
||||
CREATE TABLE "search_history" (
|
||||
"id" varchar PRIMARY KEY NOT NULL,
|
||||
"user_id" varchar NOT NULL,
|
||||
"title" varchar NOT NULL,
|
||||
"selected_groups" text,
|
||||
"created_at" datetime NOT NULL DEFAULT (datetime('now')),
|
||||
"updated_at" datetime NOT NULL DEFAULT (datetime('now'))
|
||||
)
|
||||
`);
|
||||
|
||||
// Create conversation message table
|
||||
await queryRunner.query(`
|
||||
CREATE TABLE "chat_messages" (
|
||||
"id" varchar PRIMARY KEY NOT NULL,
|
||||
"search_history_id" varchar NOT NULL,
|
||||
"role" varchar NOT NULL CHECK ("role" IN ('user', 'assistant')),
|
||||
"content" text NOT NULL,
|
||||
"sources" text,
|
||||
"created_at" datetime NOT NULL DEFAULT (datetime('now')),
|
||||
FOREIGN KEY ("search_history_id") REFERENCES "search_history" ("id") ON DELETE CASCADE
|
||||
)
|
||||
`);
|
||||
|
||||
// Add pdf_path field to knowledge_base table
|
||||
await queryRunner.query(`
|
||||
ALTER TABLE "knowledge_base" ADD COLUMN "pdf_path" varchar
|
||||
`);
|
||||
|
||||
// Create index
|
||||
await queryRunner.query(
|
||||
`CREATE INDEX "IDX_knowledge_groups_user_id" ON "knowledge_groups" ("user_id")`,
|
||||
);
|
||||
await queryRunner.query(
|
||||
`CREATE INDEX "IDX_search_history_user_id" ON "search_history" ("user_id")`,
|
||||
);
|
||||
await queryRunner.query(
|
||||
`CREATE INDEX "IDX_chat_messages_search_history_id" ON "chat_messages" ("search_history_id")`,
|
||||
);
|
||||
}
|
||||
|
||||
public async down(queryRunner: QueryRunner): Promise<void> {
|
||||
// Delete index
|
||||
await queryRunner.query(`DROP INDEX "IDX_chat_messages_search_history_id"`);
|
||||
await queryRunner.query(`DROP INDEX "IDX_search_history_user_id"`);
|
||||
await queryRunner.query(`DROP INDEX "IDX_knowledge_groups_user_id"`);
|
||||
|
||||
// Delete pdf_path field
|
||||
await queryRunner.query(
|
||||
`ALTER TABLE "knowledge_base" DROP COLUMN "pdf_path"`,
|
||||
);
|
||||
|
||||
// Delete table
|
||||
await queryRunner.query(`DROP TABLE "chat_messages"`);
|
||||
await queryRunner.query(`DROP TABLE "search_history"`);
|
||||
await queryRunner.query(`DROP TABLE "knowledge_base_groups"`);
|
||||
await queryRunner.query(`DROP TABLE "knowledge_groups"`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import { MigrationInterface, QueryRunner } from 'typeorm';
|
||||
|
||||
export class RemoveSupportsVisionColumn1739260000000 implements MigrationInterface {
|
||||
name = 'RemoveSupportsVisionColumn1739260000000';
|
||||
|
||||
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||
// Remove supportsVision column from model_configs table
|
||||
// This column is no longer needed as we now use ModelType.VISION instead
|
||||
await queryRunner.query(`
|
||||
ALTER TABLE "model_configs" DROP COLUMN "supportsVision"
|
||||
`);
|
||||
}
|
||||
|
||||
public async down(queryRunner: QueryRunner): Promise<void> {
|
||||
// Restore supportsVision column in case of rollback
|
||||
await queryRunner.query(`
|
||||
ALTER TABLE "model_configs" ADD COLUMN "supportsVision" boolean NOT NULL DEFAULT 0
|
||||
`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
import { MigrationInterface, QueryRunner } from 'typeorm';
|
||||
|
||||
export class AddDefaultTenant1772329237979 implements MigrationInterface {
|
||||
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||
// 1. Insert "Default" tenant if it doesn't exist.
|
||||
// We use a predefined UUID or let the DB generate it.
|
||||
// Assuming Postgres/MySQL compatible uuid generation isn't strictly standard here,
|
||||
// we'll insert and get the ID back, or use a workaround for SQLite if it's SQLite.
|
||||
// Actually, since this is a TypeORM setup, we can use standard SQL.
|
||||
|
||||
// This is a bit tricky to write purely in SQL that works across all DBs (SQLite/Postgres/MySQL)
|
||||
// without knowing the exact DB. AuraK seems to use SQLite locally by default.
|
||||
// First, check if there's any record in the tenant table.
|
||||
const tenants = await queryRunner.query(
|
||||
`SELECT id FROM "tenant" WHERE "name" = 'Default'`,
|
||||
);
|
||||
let defaultTenantId;
|
||||
|
||||
if (tenants && tenants.length > 0) {
|
||||
defaultTenantId = tenants[0].id;
|
||||
} else {
|
||||
// Create it with a JS generated UUID to be database agnostic
|
||||
const crypto = require('crypto');
|
||||
defaultTenantId = crypto.randomUUID();
|
||||
const now = new Date().toISOString();
|
||||
await queryRunner.query(
|
||||
`INSERT INTO "tenant" (id, name, description, "createdAt", "updatedAt") VALUES (?, ?, ?, ?, ?)`,
|
||||
[
|
||||
defaultTenantId,
|
||||
'Default',
|
||||
'Default tenant created by migration',
|
||||
now,
|
||||
now,
|
||||
],
|
||||
);
|
||||
|
||||
// Create tenant settings
|
||||
const settingsId = crypto.randomUUID();
|
||||
await queryRunner.query(
|
||||
`INSERT INTO "tenant_setting" (id, "tenantId", "createdAt", "updatedAt") VALUES (?, ?, ?, ?)`,
|
||||
[settingsId, defaultTenantId, now, now],
|
||||
);
|
||||
}
|
||||
|
||||
// 2. Assign the Default tenant to all relevant existing records that have no tenantId
|
||||
const tablesToUpdate = [
|
||||
'user',
|
||||
'knowledge_base',
|
||||
'knowledge_group',
|
||||
'search_history',
|
||||
'note',
|
||||
'model_config',
|
||||
];
|
||||
|
||||
for (const table of tablesToUpdate) {
|
||||
// Check if table exists first (some might be missing if DB is fresh)
|
||||
try {
|
||||
await queryRunner.query(
|
||||
`UPDATE "${table}" SET "tenantId" = ? WHERE "tenantId" IS NULL`,
|
||||
[defaultTenantId],
|
||||
);
|
||||
} catch (e) {
|
||||
console.warn(
|
||||
`Could not update table ${table}, it might not exist or the tenantId column might not exist yet.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public async down(queryRunner: QueryRunner): Promise<void> {
|
||||
// We don't necessarily want to delete the tenant data in a down migration
|
||||
// as it would orphan all the records or require setting them to NULL.
|
||||
// But for completeness, we can set them back to NULL.
|
||||
|
||||
const tablesToUpdate = [
|
||||
'user',
|
||||
'knowledge_base',
|
||||
'knowledge_group',
|
||||
'search_history',
|
||||
'note',
|
||||
'model_config',
|
||||
];
|
||||
for (const table of tablesToUpdate) {
|
||||
try {
|
||||
await queryRunner.query(`UPDATE "${table}" SET "tenantId" = NULL`);
|
||||
} catch (e) {
|
||||
// Ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
import { MigrationInterface, QueryRunner } from 'typeorm';
|
||||
|
||||
export class AddTenantModule1772334811108 implements MigrationInterface {
|
||||
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||
// 1. Ensure 'tenants' table exists before we insert into it.
|
||||
// This assumes the schema definition migrations have run or TypeORM synchronize handled the tables.
|
||||
// We will insert a system default tenant.
|
||||
|
||||
await queryRunner.query(`
|
||||
INSERT INTO "tenants" ("id", "name", "description", "isActive", "created_at", "updated_at")
|
||||
SELECT '00000000-0000-0000-0000-000000000000', 'Default Tenant', 'System Default Organization', 1, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP
|
||||
WHERE NOT EXISTS (SELECT 1 FROM "tenants" WHERE "id" = '00000000-0000-0000-0000-000000000000');
|
||||
`);
|
||||
|
||||
// 2. Link existing users to the default tenant
|
||||
await queryRunner.query(
|
||||
`UPDATE "users" SET "tenant_id" = '00000000-0000-0000-0000-000000000000' WHERE "tenant_id" IS NULL;`,
|
||||
);
|
||||
|
||||
// 3. Link existing knowledge bases to the default tenant
|
||||
await queryRunner.query(
|
||||
`UPDATE "knowledge_bases" SET "tenant_id" = '00000000-0000-0000-0000-000000000000' WHERE "tenant_id" IS NULL;`,
|
||||
);
|
||||
|
||||
// 4. Link existing knowledge groups to the default tenant
|
||||
await queryRunner.query(
|
||||
`UPDATE "knowledge_groups" SET "tenant_id" = '00000000-0000-0000-0000-000000000000' WHERE "tenant_id" IS NULL;`,
|
||||
);
|
||||
|
||||
// 5. Link existing search histories to the default tenant
|
||||
await queryRunner.query(
|
||||
`UPDATE "search_history" SET "tenant_id" = '00000000-0000-0000-0000-000000000000' WHERE "tenant_id" IS NULL;`,
|
||||
);
|
||||
|
||||
// 6. Link existing notes to the default tenant
|
||||
await queryRunner.query(
|
||||
`UPDATE "notes" SET "tenant_id" = '00000000-0000-0000-0000-000000000000' WHERE "tenant_id" IS NULL;`,
|
||||
);
|
||||
|
||||
// 7. Make the existing admin users SUPER_ADMIN
|
||||
await queryRunner.query(
|
||||
`UPDATE "users" SET "role" = 'SUPER_ADMIN' WHERE "isAdmin" = 1;`,
|
||||
);
|
||||
}
|
||||
|
||||
public async down(queryRunner: QueryRunner): Promise<void> {
|
||||
// Reverse operations if needed. Note: We do not delete the Default tenant here because it might be in active use.
|
||||
// But we could nullify the links if we were strictly rolling back to a state without tenant links.
|
||||
/*
|
||||
await queryRunner.query(`UPDATE "users" SET "tenant_id" = NULL WHERE "tenant_id" = '00000000-0000-0000-0000-000000000000';`);
|
||||
await queryRunner.query(`UPDATE "knowledge_bases" SET "tenant_id" = NULL WHERE "tenant_id" = '00000000-0000-0000-0000-000000000000';`);
|
||||
await queryRunner.query(`UPDATE "knowledge_groups" SET "tenant_id" = NULL WHERE "tenant_id" = '00000000-0000-0000-0000-000000000000';`);
|
||||
await queryRunner.query(`UPDATE "search_history" SET "tenant_id" = NULL WHERE "tenant_id" = '00000000-0000-0000-0000-000000000000';`);
|
||||
await queryRunner.query(`UPDATE "notes" SET "tenant_id" = NULL WHERE "tenant_id" = '00000000-0000-0000-0000-000000000000';`);
|
||||
*/
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import { MigrationInterface, QueryRunner } from 'typeorm';
|
||||
|
||||
export class AddParentIdToKnowledgeGroups1772340000000 implements MigrationInterface {
|
||||
name = 'AddParentIdToKnowledgeGroups1772340000000';
|
||||
|
||||
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||
// Add parent_id column to knowledge_groups table
|
||||
await queryRunner.query(
|
||||
`ALTER TABLE "knowledge_groups" ADD COLUMN "parent_id" text REFERENCES "knowledge_groups"("id") ON DELETE SET NULL`,
|
||||
);
|
||||
}
|
||||
|
||||
public async down(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.query(
|
||||
`ALTER TABLE "knowledge_groups" DROP COLUMN "parent_id"`,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
import { MigrationInterface, QueryRunner } from 'typeorm';
|
||||
|
||||
export class AddAssessmentTablesManual1773198650000 implements MigrationInterface {
|
||||
name = 'AddAssessmentTablesManual1773198650000';
|
||||
|
||||
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.query(
|
||||
`CREATE TABLE IF NOT EXISTS "assessment_sessions" ("id" varchar PRIMARY KEY NOT NULL, "user_id" varchar NOT NULL, "knowledge_base_id" varchar NOT NULL, "thread_id" varchar, "status" varchar CHECK( "status" IN ('IN_PROGRESS','COMPLETED') ) NOT NULL DEFAULT ('IN_PROGRESS'), "final_score" float, "final_report" text, "created_at" datetime NOT NULL DEFAULT (datetime('now')), "updated_at" datetime NOT NULL DEFAULT (datetime('now')))`,
|
||||
);
|
||||
await queryRunner.query(
|
||||
`CREATE TABLE IF NOT EXISTS "assessment_questions" ("id" varchar PRIMARY KEY NOT NULL, "session_id" varchar NOT NULL, "question_text" text NOT NULL, "key_points" text, "difficulty" varchar, "created_at" datetime NOT NULL DEFAULT (datetime('now')), "updated_at" datetime NOT NULL DEFAULT (datetime('now')))`,
|
||||
);
|
||||
await queryRunner.query(
|
||||
`CREATE TABLE IF NOT EXISTS "assessment_answers" ("id" varchar PRIMARY KEY NOT NULL, "question_id" varchar NOT NULL, "user_answer" text NOT NULL, "score" float, "feedback" text, "is_follow_up" boolean NOT NULL DEFAULT (0), "created_at" datetime NOT NULL DEFAULT (datetime('now')), "updated_at" datetime NOT NULL DEFAULT (datetime('now')))`,
|
||||
);
|
||||
}
|
||||
|
||||
public async down(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.query(`DROP TABLE "assessment_answers"`);
|
||||
await queryRunner.query(`DROP TABLE "assessment_questions"`);
|
||||
await queryRunner.query(`DROP TABLE "assessment_sessions"`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import { MigrationInterface, QueryRunner } from 'typeorm';
|
||||
|
||||
export class AddFeishuBotKnowledgeFields1773200000000 implements MigrationInterface {
|
||||
name = 'AddFeishuBotKnowledgeFields1773200000000';
|
||||
|
||||
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||
// 添加知识库ID字段
|
||||
await queryRunner.query(`
|
||||
ALTER TABLE feishu_bots
|
||||
ADD COLUMN knowledge_base_id VARCHAR(36) NULL;
|
||||
`);
|
||||
|
||||
// 添加知识组ID字段
|
||||
await queryRunner.query(`
|
||||
ALTER TABLE feishu_bots
|
||||
ADD COLUMN knowledge_group_id VARCHAR(36) NULL;
|
||||
`);
|
||||
|
||||
// 添加外键约束（可选，如果需要引用完整性）
|
||||
// 注意：这里假设 knowledge_bases 和 knowledge_groups 表存在
|
||||
// 如果表不存在，可以先创建或移除外键约束
|
||||
}
|
||||
|
||||
public async down(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.query(`
|
||||
ALTER TABLE feishu_bots
|
||||
DROP COLUMN knowledge_base_id;
|
||||
`);
|
||||
|
||||
await queryRunner.query(`
|
||||
ALTER TABLE feishu_bots
|
||||
DROP COLUMN knowledge_group_id;
|
||||
`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import { MigrationInterface, QueryRunner } from 'typeorm';
|
||||
|
||||
export class CreateFeishuAssessmentSessionTable1773200000001 implements MigrationInterface {
|
||||
name = 'CreateFeishuAssessmentSessionTable1773200000001';
|
||||
|
||||
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.query(`
|
||||
CREATE TABLE feishu_assessment_sessions (
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
bot_id VARCHAR(36) NOT NULL,
|
||||
open_id VARCHAR(255) NOT NULL,
|
||||
assessment_session_id VARCHAR(36) NOT NULL,
|
||||
status ENUM('active', 'completed', 'cancelled') DEFAULT 'active',
|
||||
current_question_index INT DEFAULT 0,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
INDEX idx_bot_open (bot_id, open_id),
|
||||
INDEX idx_assessment_session (assessment_session_id),
|
||||
CONSTRAINT fk_feishu_assessment_bot
|
||||
FOREIGN KEY (bot_id)
|
||||
REFERENCES feishu_bots(id)
|
||||
ON DELETE CASCADE
|
||||
);
|
||||
`);
|
||||
}
|
||||
|
||||
public async down(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.query(`
|
||||
DROP TABLE feishu_assessment_sessions;
|
||||
`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import { MigrationInterface, QueryRunner } from 'typeorm';
|
||||
|
||||
export class DropTenantFromNotes1773210000000 implements MigrationInterface {
|
||||
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||
// Log tables to help debug
|
||||
const tables = await queryRunner.getTables();
|
||||
console.log('Database tables found:', tables.map(t => t.name).join(', '));
|
||||
|
||||
const noteTables = ['note', 'notes'];
|
||||
const noteCategoryTables = ['note_category', 'note_categories'];
|
||||
const tenantColumns = ['tenant_id', 'tenantId', 'tenantid'];
|
||||
|
||||
// 1. Drop from notes tables
|
||||
for (const table of noteTables) {
|
||||
for (const col of tenantColumns) {
|
||||
try {
|
||||
await queryRunner.query(`ALTER TABLE "${table}" DROP COLUMN "${col}"`);
|
||||
console.log(`Successfully dropped ${col} from ${table}`);
|
||||
} catch (e) {
|
||||
// Ignore - column or table might not exist
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Drop from note_categories tables
|
||||
for (const table of noteCategoryTables) {
|
||||
for (const col of tenantColumns) {
|
||||
try {
|
||||
await queryRunner.query(`ALTER TABLE "${table}" DROP COLUMN "${col}"`);
|
||||
console.log(`Successfully dropped ${col} from ${table}`);
|
||||
} catch (e) {
|
||||
// Ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public async down(queryRunner: QueryRunner): Promise<void> {
|
||||
// To reverse, we would add the column back.
|
||||
try {
|
||||
await queryRunner.query('ALTER TABLE "notes" ADD COLUMN "tenant_id" varchar');
|
||||
await queryRunner.query('ALTER TABLE "note_categories" ADD COLUMN "tenant_id" varchar');
|
||||
} catch (e) {
|
||||
// Ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
import { MigrationInterface, QueryRunner, TableColumn } from 'typeorm';
|
||||
|
||||
export class AddTemplateExtensions1773210000002 implements MigrationInterface {
|
||||
name = 'AddTemplateExtensions1773210000002';
|
||||
|
||||
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||
// Add linked_group_ids column
|
||||
await queryRunner.addColumn(
|
||||
'assessment_templates',
|
||||
new TableColumn({
|
||||
name: 'linked_group_ids',
|
||||
type: 'simple-json',
|
||||
isNullable: true,
|
||||
}),
|
||||
);
|
||||
|
||||
// Add weight_config column
|
||||
await queryRunner.addColumn(
|
||||
'assessment_templates',
|
||||
new TableColumn({
|
||||
name: 'weight_config',
|
||||
type: 'simple-json',
|
||||
isNullable: true,
|
||||
}),
|
||||
);
|
||||
|
||||
// Add difficulty_config column
|
||||
await queryRunner.addColumn(
|
||||
'assessment_templates',
|
||||
new TableColumn({
|
||||
name: 'difficulty_config',
|
||||
type: 'simple-json',
|
||||
isNullable: true,
|
||||
}),
|
||||
);
|
||||
|
||||
// Add question_count_min column
|
||||
await queryRunner.addColumn(
|
||||
'assessment_templates',
|
||||
new TableColumn({
|
||||
name: 'question_count_min',
|
||||
type: 'int',
|
||||
default: 8,
|
||||
}),
|
||||
);
|
||||
|
||||
// Add question_count_max column
|
||||
await queryRunner.addColumn(
|
||||
'assessment_templates',
|
||||
new TableColumn({
|
||||
name: 'question_count_max',
|
||||
type: 'int',
|
||||
default: 10,
|
||||
}),
|
||||
);
|
||||
|
||||
// Add passing_score column
|
||||
await queryRunner.addColumn(
|
||||
'assessment_templates',
|
||||
new TableColumn({
|
||||
name: 'passing_score',
|
||||
type: 'int',
|
||||
default: 90,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
public async down(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.dropColumn('assessment_templates', 'linked_group_ids');
|
||||
await queryRunner.dropColumn('assessment_templates', 'weight_config');
|
||||
await queryRunner.dropColumn('assessment_templates', 'difficulty_config');
|
||||
await queryRunner.dropColumn('assessment_templates', 'question_count_min');
|
||||
await queryRunner.dropColumn('assessment_templates', 'question_count_max');
|
||||
await queryRunner.dropColumn('assessment_templates', 'passing_score');
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
import { MigrationInterface, QueryRunner, Table } from 'typeorm';
|
||||
|
||||
export class CreateAssessmentCertificateTable1773210000003 implements MigrationInterface {
|
||||
name = 'CreateAssessmentCertificateTable1773210000003';
|
||||
|
||||
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.createTable(
|
||||
new Table({
|
||||
name: 'assessment_certificates',
|
||||
columns: [
|
||||
{
|
||||
name: 'id',
|
||||
type: 'uuid',
|
||||
isPrimary: true,
|
||||
generationStrategy: 'uuid',
|
||||
isGenerated: true,
|
||||
},
|
||||
{
|
||||
name: 'user_id',
|
||||
type: 'uuid',
|
||||
isNullable: false,
|
||||
},
|
||||
{
|
||||
name: 'session_id',
|
||||
type: 'uuid',
|
||||
isNullable: false,
|
||||
},
|
||||
{
|
||||
name: 'template_id',
|
||||
type: 'uuid',
|
||||
isNullable: false,
|
||||
},
|
||||
{
|
||||
name: 'level',
|
||||
type: 'varchar',
|
||||
length: '50',
|
||||
isNullable: false,
|
||||
},
|
||||
{
|
||||
name: 'total_score',
|
||||
type: 'float',
|
||||
isNullable: false,
|
||||
},
|
||||
{
|
||||
name: 'qr_code',
|
||||
type: 'varchar',
|
||||
length: '255',
|
||||
isNullable: true,
|
||||
},
|
||||
{
|
||||
name: 'dimension_scores',
|
||||
type: 'simple-json',
|
||||
isNullable: true,
|
||||
},
|
||||
{
|
||||
name: 'radar_data',
|
||||
type: 'simple-json',
|
||||
isNullable: true,
|
||||
},
|
||||
{
|
||||
name: 'passed',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
},
|
||||
{
|
||||
name: 'issued_at',
|
||||
type: 'timestamp',
|
||||
default: 'now()',
|
||||
},
|
||||
],
|
||||
}),
|
||||
true,
|
||||
);
|
||||
}
|
||||
|
||||
public async down(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.dropTable('assessment_certificates');
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
import { MigrationInterface, QueryRunner, Table, TableForeignKey, TableIndex } from 'typeorm';
|
||||
|
||||
export class CreateQuestionBankTables1773220000000 implements MigrationInterface {
|
||||
name = 'CreateQuestionBankTables1773220000000';
|
||||
|
||||
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.createTable(
|
||||
new Table({
|
||||
name: 'question_banks',
|
||||
columns: [
|
||||
{
|
||||
name: 'id',
|
||||
type: 'uuid',
|
||||
isPrimary: true,
|
||||
generationStrategy: 'uuid',
|
||||
isGenerated: true,
|
||||
},
|
||||
{
|
||||
name: 'tenant_id',
|
||||
type: 'uuid',
|
||||
isNullable: true,
|
||||
},
|
||||
{
|
||||
name: 'template_id',
|
||||
type: 'uuid',
|
||||
isNullable: true,
|
||||
},
|
||||
{
|
||||
name: 'name',
|
||||
type: 'varchar',
|
||||
length: '255',
|
||||
isNullable: false,
|
||||
},
|
||||
{
|
||||
name: 'description',
|
||||
type: 'text',
|
||||
isNullable: true,
|
||||
},
|
||||
{
|
||||
name: 'status',
|
||||
type: 'enum',
|
||||
enum: ['DRAFT', 'PENDING_REVIEW', 'PUBLISHED', 'REJECTED'],
|
||||
default: "'DRAFT'",
|
||||
},
|
||||
{
|
||||
name: 'created_by',
|
||||
type: 'varchar',
|
||||
length: '255',
|
||||
isNullable: true,
|
||||
},
|
||||
{
|
||||
name: 'reviewed_by',
|
||||
type: 'varchar',
|
||||
length: '255',
|
||||
isNullable: true,
|
||||
},
|
||||
{
|
||||
name: 'reviewed_at',
|
||||
type: 'timestamp',
|
||||
isNullable: true,
|
||||
},
|
||||
{
|
||||
name: 'review_comment',
|
||||
type: 'text',
|
||||
isNullable: true,
|
||||
},
|
||||
{
|
||||
name: 'created_at',
|
||||
type: 'timestamp',
|
||||
default: 'now()',
|
||||
},
|
||||
{
|
||||
name: 'updated_at',
|
||||
type: 'timestamp',
|
||||
default: 'now()',
|
||||
},
|
||||
],
|
||||
}),
|
||||
true,
|
||||
);
|
||||
|
||||
await queryRunner.createIndex('question_banks', new TableIndex({
|
||||
name: 'IDX_QUESTION_BANK_TEMPLATE',
|
||||
columnNames: ['template_id'],
|
||||
isUnique: true,
|
||||
}));
|
||||
|
||||
await queryRunner.createForeignKey('question_banks', new TableForeignKey({
|
||||
columnNames: ['template_id'],
|
||||
referencedColumnNames: ['id'],
|
||||
referencedTableName: 'assessment_templates',
|
||||
onDelete: 'SET NULL',
|
||||
}));
|
||||
|
||||
await queryRunner.createForeignKey('question_banks', new TableForeignKey({
|
||||
columnNames: ['tenant_id'],
|
||||
referencedColumnNames: ['id'],
|
||||
referencedTableName: 'tenants',
|
||||
onDelete: 'CASCADE',
|
||||
}));
|
||||
|
||||
await queryRunner.createTable(
|
||||
new Table({
|
||||
name: 'question_bank_items',
|
||||
columns: [
|
||||
{
|
||||
name: 'id',
|
||||
type: 'uuid',
|
||||
isPrimary: true,
|
||||
generationStrategy: 'uuid',
|
||||
isGenerated: true,
|
||||
},
|
||||
{
|
||||
name: 'bank_id',
|
||||
type: 'uuid',
|
||||
isNullable: false,
|
||||
},
|
||||
{
|
||||
name: 'question_text',
|
||||
type: 'text',
|
||||
isNullable: false,
|
||||
},
|
||||
{
|
||||
name: 'question_type',
|
||||
type: 'enum',
|
||||
enum: ['SHORT_ANSWER', 'MULTIPLE_CHOICE', 'TRUE_FALSE'],
|
||||
default: "'SHORT_ANSWER'",
|
||||
},
|
||||
{
|
||||
name: 'options',
|
||||
type: 'simple-json',
|
||||
isNullable: true,
|
||||
},
|
||||
{
|
||||
name: 'correct_answer',
|
||||
type: 'text',
|
||||
isNullable: true,
|
||||
},
|
||||
{
|
||||
name: 'key_points',
|
||||
type: 'simple-json',
|
||||
isNullable: false,
|
||||
},
|
||||
{
|
||||
name: 'difficulty',
|
||||
type: 'enum',
|
||||
enum: ['STANDARD', 'ADVANCED', 'SPECIALIST'],
|
||||
default: "'STANDARD'",
|
||||
},
|
||||
{
|
||||
name: 'dimension',
|
||||
type: 'enum',
|
||||
enum: ['PROMPT', 'LLM', 'IDE', 'DEV_PATTERN', 'WORK_CAPABILITY'],
|
||||
default: "'PROMPT'",
|
||||
},
|
||||
{
|
||||
name: 'basis',
|
||||
type: 'text',
|
||||
isNullable: true,
|
||||
},
|
||||
{
|
||||
name: 'created_by',
|
||||
type: 'varchar',
|
||||
length: '255',
|
||||
isNullable: true,
|
||||
},
|
||||
{
|
||||
name: 'status',
|
||||
type: 'enum',
|
||||
enum: ['PENDING_REVIEW', 'PUBLISHED'],
|
||||
default: "'PENDING_REVIEW'",
|
||||
},
|
||||
{
|
||||
name: 'created_at',
|
||||
type: 'timestamp',
|
||||
default: 'now()',
|
||||
},
|
||||
{
|
||||
name: 'updated_at',
|
||||
type: 'timestamp',
|
||||
default: 'now()',
|
||||
},
|
||||
],
|
||||
}),
|
||||
true,
|
||||
);
|
||||
|
||||
await queryRunner.createForeignKey('question_bank_items', new TableForeignKey({
|
||||
columnNames: ['bank_id'],
|
||||
referencedColumnNames: ['id'],
|
||||
referencedTableName: 'question_banks',
|
||||
onDelete: 'CASCADE',
|
||||
}));
|
||||
}
|
||||
|
||||
public async down(queryRunner: QueryRunner): Promise<void> {
|
||||
const questionBankItemsTable = await queryRunner.getTable('question_bank_items');
|
||||
if (questionBankItemsTable) {
|
||||
const foreignKey = questionBankItemsTable.foreignKeys.find(
|
||||
(fk) => fk.columnNames.indexOf('bank_id') !== -1,
|
||||
);
|
||||
if (foreignKey) {
|
||||
await queryRunner.dropForeignKey('question_bank_items', foreignKey);
|
||||
}
|
||||
await queryRunner.dropTable('question_bank_items');
|
||||
}
|
||||
|
||||
const questionBanksTable = await queryRunner.getTable('question_banks');
|
||||
if (questionBanksTable) {
|
||||
const foreignKeys = questionBanksTable.foreignKeys;
|
||||
for (const fk of foreignKeys) {
|
||||
await queryRunner.dropForeignKey('question_banks', fk);
|
||||
}
|
||||
const index = questionBanksTable.indices.find(
|
||||
(idx) => idx.name === 'IDX_QUESTION_BANK_TEMPLATE',
|
||||
);
|
||||
if (index) {
|
||||
await queryRunner.dropIndex('question_banks', index);
|
||||
}
|
||||
await queryRunner.dropTable('question_banks');
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
-- Add language column to user_settings table
|
||||
ALTER TABLE user_settings
|
||||
ADD COLUMN language TEXT DEFAULT 'zh';
|
||||
@@ -0,0 +1,29 @@
|
||||
-- cleanup-settings-tables.sql
|
||||
-- Drop unnecessary columns from settings tables to align with the refined architecture.
|
||||
|
||||
-- 1. Prune user_settings table
|
||||
-- Keeps only id, userId, and language.
|
||||
ALTER TABLE user_settings DROP COLUMN selectedLLMId;
|
||||
ALTER TABLE user_settings DROP COLUMN selectedEmbeddingId;
|
||||
ALTER TABLE user_settings DROP COLUMN selectedRerankId;
|
||||
ALTER TABLE user_settings DROP COLUMN temperature;
|
||||
ALTER TABLE user_settings DROP COLUMN maxTokens;
|
||||
ALTER TABLE user_settings DROP COLUMN enableRerank;
|
||||
ALTER TABLE user_settings DROP COLUMN topK;
|
||||
ALTER TABLE user_settings DROP COLUMN similarityThreshold;
|
||||
ALTER TABLE user_settings DROP COLUMN enableFullTextSearch;
|
||||
ALTER TABLE user_settings DROP COLUMN defaultVisionModelId;
|
||||
ALTER TABLE user_settings DROP COLUMN coachKbId;
|
||||
ALTER TABLE user_settings DROP COLUMN created_at;
|
||||
ALTER TABLE user_settings DROP COLUMN updated_at;
|
||||
ALTER TABLE user_settings DROP COLUMN rerankSimilarityThreshold;
|
||||
ALTER TABLE user_settings DROP COLUMN hybridVectorWeight;
|
||||
ALTER TABLE user_settings DROP COLUMN isGlobal;
|
||||
ALTER TABLE user_settings DROP COLUMN enableQueryExpansion;
|
||||
ALTER TABLE user_settings DROP COLUMN enableHyDE;
|
||||
ALTER TABLE user_settings DROP COLUMN chunkSize;
|
||||
ALTER TABLE user_settings DROP COLUMN chunkOverlap;
|
||||
|
||||
-- 2. Prune tenant_settings table
|
||||
-- Language is now strictly a user-level setting.
|
||||
ALTER TABLE tenant_settings DROP COLUMN language;
|
||||
@@ -0,0 +1,8 @@
|
||||
-- restore-timestamps.sql
|
||||
-- Restore created_at and updated_at columns to user_settings table.
|
||||
|
||||
ALTER TABLE user_settings ADD COLUMN created_at datetime;
|
||||
ALTER TABLE user_settings ADD COLUMN updated_at datetime;
|
||||
|
||||
UPDATE user_settings SET created_at = datetime('now') WHERE created_at IS NULL;
|
||||
UPDATE user_settings SET updated_at = datetime('now') WHERE updated_at IS NULL;
|
||||
@@ -0,0 +1,92 @@
|
||||
// server/src/model-config/dto/create-model-config.dto.ts
|
||||
import {
|
||||
IsBoolean,
|
||||
IsEnum,
|
||||
IsNotEmpty,
|
||||
IsNumber,
|
||||
IsOptional,
|
||||
IsString,
|
||||
IsUrl,
|
||||
Max,
|
||||
Min,
|
||||
MinLength,
|
||||
} from 'class-validator';
|
||||
import { ModelType } from '../../types';
|
||||
|
||||
export class CreateModelConfigDto {
|
||||
@IsString()
|
||||
@IsNotEmpty()
|
||||
name: string;
|
||||
|
||||
@IsString()
|
||||
@IsNotEmpty()
|
||||
modelId: string;
|
||||
|
||||
@IsUrl({ require_tld: false }, { message: 'Base URL must be a valid URL' })
|
||||
@IsOptional()
|
||||
baseUrl?: string;
|
||||
|
||||
@IsString()
|
||||
@IsOptional()
|
||||
apiKey?: string; // API key is optional - allows local models
|
||||
|
||||
@IsEnum(ModelType)
|
||||
@IsNotEmpty()
|
||||
type: ModelType;
|
||||
|
||||
@IsNumber()
|
||||
@Min(1, { message: 'Minimum vector dimension is 1' })
|
||||
@Max(4096, {
|
||||
message: 'Maximum vector dimension is 4096 (Elasticsearch limit)',
|
||||
})
|
||||
@IsOptional()
|
||||
dimensions?: number;
|
||||
|
||||
// ==================== Additional Fields ====================
|
||||
|
||||
/**
|
||||
* Model input token limit (only valid for embedding/rerank)
|
||||
*/
|
||||
@IsNumber()
|
||||
@Min(1)
|
||||
@Max(100000)
|
||||
@IsOptional()
|
||||
maxInputTokens?: number;
|
||||
|
||||
/**
|
||||
* Batch processing limit (only valid for embedding/rerank)
|
||||
*/
|
||||
@IsNumber()
|
||||
@Min(1)
|
||||
@Max(10000)
|
||||
@IsOptional()
|
||||
maxBatchSize?: number;
|
||||
|
||||
/**
|
||||
* Whether this is a vector model
|
||||
*/
|
||||
@IsBoolean()
|
||||
@IsOptional()
|
||||
isVectorModel?: boolean;
|
||||
|
||||
/**
|
||||
* Model provider name
|
||||
*/
|
||||
@IsString()
|
||||
@IsOptional()
|
||||
providerName?: string;
|
||||
|
||||
/**
|
||||
* Whether to enable this model
|
||||
*/
|
||||
@IsBoolean()
|
||||
@IsOptional()
|
||||
isEnabled?: boolean;
|
||||
|
||||
/**
|
||||
* Whether to use this model as default
|
||||
*/
|
||||
@IsBoolean()
|
||||
@IsOptional()
|
||||
isDefault?: boolean;
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
// server/src/model-config/dto/model-config-response.dto.ts
|
||||
import { Exclude, Expose, Transform } from 'class-transformer';
|
||||
import { ModelConfig } from '../model-config.entity';
|
||||
|
||||
export class ModelConfigResponseDto {
|
||||
id: string;
|
||||
name: string;
|
||||
provider: string;
|
||||
modelId: string;
|
||||
baseUrl?: string;
|
||||
|
||||
@Transform(({ value }) => (value ? '********' : undefined))
|
||||
apiKey?: string;
|
||||
|
||||
type: string;
|
||||
isEnabled?: boolean;
|
||||
isDefault?: boolean;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
|
||||
constructor(partial: Partial<ModelConfig>) {
|
||||
Object.assign(this, partial);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
// server/src/model-config/dto/update-model-config.dto.ts
|
||||
import { PartialType } from '@nestjs/mapped-types';
|
||||
import { CreateModelConfigDto } from './create-model-config.dto';
|
||||
|
||||
export class UpdateModelConfigDto extends PartialType(CreateModelConfigDto) {}
|
||||
@@ -0,0 +1,80 @@
|
||||
// server/src/model-config/model-config.controller.ts
|
||||
import {
|
||||
Body,
|
||||
Controller,
|
||||
Delete,
|
||||
Get,
|
||||
HttpCode,
|
||||
HttpStatus,
|
||||
Param,
|
||||
Patch,
|
||||
Post,
|
||||
Put,
|
||||
Req,
|
||||
UseGuards,
|
||||
} from '@nestjs/common';
|
||||
import { ModelConfigService } from './model-config.service';
|
||||
import { CreateModelConfigDto } from './dto/create-model-config.dto';
|
||||
import { UpdateModelConfigDto } from './dto/update-model-config.dto';
|
||||
import { CombinedAuthGuard } from '../auth/combined-auth.guard';
|
||||
import { RolesGuard } from '../auth/roles.guard';
|
||||
import { Roles } from '../auth/roles.decorator';
|
||||
import { UserRole } from '../user/user-role.enum';
|
||||
import { ModelConfigResponseDto } from './dto/model-config-response.dto';
|
||||
import { plainToClass } from 'class-transformer';
|
||||
|
||||
@UseGuards(CombinedAuthGuard)
|
||||
@Controller('models') // Global prefix /api/models
|
||||
export class ModelConfigController {
|
||||
constructor(private readonly modelConfigService: ModelConfigService) {}
|
||||
|
||||
@Roles(UserRole.TENANT_ADMIN, UserRole.SUPER_ADMIN)
|
||||
@Post()
|
||||
@HttpCode(HttpStatus.CREATED)
|
||||
async create(
|
||||
@Body() createModelConfigDto: CreateModelConfigDto,
|
||||
): Promise<ModelConfigResponseDto> {
|
||||
const modelConfig =
|
||||
await this.modelConfigService.create(createModelConfigDto);
|
||||
return plainToClass(ModelConfigResponseDto, modelConfig);
|
||||
}
|
||||
|
||||
@Get()
|
||||
async findAll(): Promise<ModelConfigResponseDto[]> {
|
||||
const modelConfigs = await this.modelConfigService.findAll();
|
||||
return modelConfigs.map((mc) => plainToClass(ModelConfigResponseDto, mc));
|
||||
}
|
||||
|
||||
@Get(':id')
|
||||
async findOne(@Param('id') id: string): Promise<ModelConfigResponseDto> {
|
||||
const modelConfig = await this.modelConfigService.findOne(id);
|
||||
return plainToClass(ModelConfigResponseDto, modelConfig);
|
||||
}
|
||||
|
||||
@Roles(UserRole.TENANT_ADMIN, UserRole.SUPER_ADMIN)
|
||||
@Put(':id')
|
||||
async update(
|
||||
@Param('id') id: string,
|
||||
@Body() updateModelConfigDto: UpdateModelConfigDto,
|
||||
): Promise<ModelConfigResponseDto> {
|
||||
const modelConfig = await this.modelConfigService.update(
|
||||
id,
|
||||
updateModelConfigDto,
|
||||
);
|
||||
return plainToClass(ModelConfigResponseDto, modelConfig);
|
||||
}
|
||||
|
||||
@Roles(UserRole.TENANT_ADMIN, UserRole.SUPER_ADMIN)
|
||||
@Delete(':id')
|
||||
@HttpCode(HttpStatus.NO_CONTENT)
|
||||
async remove(@Param('id') id: string): Promise<void> {
|
||||
await this.modelConfigService.remove(id);
|
||||
}
|
||||
|
||||
@Roles(UserRole.TENANT_ADMIN, UserRole.SUPER_ADMIN)
|
||||
@Patch(':id/set-default')
|
||||
async setDefault(@Param('id') id: string): Promise<ModelConfigResponseDto> {
|
||||
const modelConfig = await this.modelConfigService.setDefault(id);
|
||||
return plainToClass(ModelConfigResponseDto, modelConfig);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
// server/src/model-config/model-config.entity.ts
|
||||
import {
|
||||
Column,
|
||||
CreateDateColumn,
|
||||
Entity,
|
||||
PrimaryGeneratedColumn,
|
||||
UpdateDateColumn,
|
||||
} from 'typeorm';
|
||||
|
||||
@Entity('model_configs')
|
||||
export class ModelConfig {
|
||||
@PrimaryGeneratedColumn('uuid')
|
||||
id: string;
|
||||
|
||||
@Column({ type: 'text' })
|
||||
name: string;
|
||||
|
||||
@Column({ type: 'text' })
|
||||
modelId: string;
|
||||
|
||||
@Column({ type: 'text', nullable: true })
|
||||
baseUrl?: string;
|
||||
|
||||
@Column({ type: 'text', nullable: true })
|
||||
apiKey?: string; // Should be encrypted in production
|
||||
|
||||
@Column({ type: 'text' })
|
||||
type: string; // ModelType enum values
|
||||
|
||||
@Column({ type: 'integer', nullable: true })
|
||||
dimensions?: number; // Embedding model dimensions, auto-detected and saved by system
|
||||
|
||||
// ==================== Additional Fields ====================
|
||||
// The following fields are only meaningful for embedding/rerank models
|
||||
|
||||
/**
|
||||
* Model input token limit
|
||||
* Example: OpenAI=8191, Gemini=2048
|
||||
*/
|
||||
@Column({ type: 'integer', nullable: true, default: 8191 })
|
||||
maxInputTokens?: number;
|
||||
|
||||
/**
|
||||
* Batch processing limit (max inputs per request)
|
||||
* Example: OpenAI=2048, Gemini=100
|
||||
*/
|
||||
@Column({ type: 'integer', nullable: true, default: 2048 })
|
||||
maxBatchSize?: number;
|
||||
|
||||
/**
|
||||
* Whether this is a vector model (for system identification)
|
||||
*/
|
||||
@Column({ type: 'boolean', default: false })
|
||||
isVectorModel?: boolean;
|
||||
|
||||
/**
|
||||
* Whether to enable this model
|
||||
* Users can disable models they don't use to prevent accidental selection
|
||||
*/
|
||||
@Column({ type: 'boolean', default: true })
|
||||
isEnabled?: boolean;
|
||||
|
||||
/**
|
||||
* Whether to use this model as default
|
||||
* Only one default allowed per type (llm, embedding, rerank)
|
||||
*/
|
||||
@Column({ type: 'boolean', default: false })
|
||||
isDefault?: boolean;
|
||||
|
||||
/**
|
||||
* Model provider name (for display and identification)
|
||||
* Example: "OpenAI", "Google Gemini", "Custom"
|
||||
*/
|
||||
@Column({ type: 'text', nullable: true })
|
||||
providerName?: string;
|
||||
|
||||
@CreateDateColumn({ name: 'created_at' })
|
||||
createdAt: Date;
|
||||
|
||||
@UpdateDateColumn({ name: 'updated_at' })
|
||||
updatedAt: Date;
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import { Module, forwardRef } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
import { ModelConfig } from './model-config.entity';
|
||||
import { ModelConfigService } from './model-config.service';
|
||||
import { ModelConfigController } from './model-config.controller';
|
||||
import { TenantModule } from '../tenant/tenant.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
TypeOrmModule.forFeature([ModelConfig]),
|
||||
forwardRef(() => TenantModule),
|
||||
],
|
||||
providers: [ModelConfigService],
|
||||
controllers: [ModelConfigController],
|
||||
exports: [ModelConfigService],
|
||||
})
|
||||
export class ModelConfigModule {}
|
||||
@@ -0,0 +1,142 @@
|
||||
import {
|
||||
Injectable,
|
||||
NotFoundException,
|
||||
ForbiddenException,
|
||||
BadRequestException,
|
||||
forwardRef,
|
||||
Inject,
|
||||
} from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Repository } from 'typeorm';
|
||||
import { ModelConfig } from './model-config.entity';
|
||||
import { CreateModelConfigDto } from './dto/create-model-config.dto';
|
||||
import { UpdateModelConfigDto } from './dto/update-model-config.dto';
|
||||
import { GLOBAL_TENANT_ID } from '../common/constants';
|
||||
import { TenantService } from '../tenant/tenant.service';
|
||||
import { ModelType } from '../types';
|
||||
import { I18nService } from '../i18n/i18n.service';
|
||||
|
||||
@Injectable()
|
||||
export class ModelConfigService {
|
||||
constructor(
|
||||
@InjectRepository(ModelConfig)
|
||||
private modelConfigRepository: Repository<ModelConfig>,
|
||||
@Inject(forwardRef(() => TenantService))
|
||||
private readonly tenantService: TenantService,
|
||||
private i18nService: I18nService,
|
||||
) {}
|
||||
|
||||
async create(
|
||||
createModelConfigDto: CreateModelConfigDto,
|
||||
): Promise<ModelConfig> {
|
||||
const modelConfig = this.modelConfigRepository.create({
|
||||
...createModelConfigDto,
|
||||
});
|
||||
return this.modelConfigRepository.save(modelConfig);
|
||||
}
|
||||
|
||||
async findAll(): Promise<ModelConfig[]> {
|
||||
return this.modelConfigRepository.find();
|
||||
}
|
||||
|
||||
async findOne(id: string): Promise<ModelConfig> {
|
||||
const modelConfig = await this.modelConfigRepository.findOne({
|
||||
where: { id },
|
||||
});
|
||||
|
||||
if (!modelConfig) {
|
||||
throw new NotFoundException(
|
||||
this.i18nService.formatMessage('modelConfigNotFound', { id }),
|
||||
);
|
||||
}
|
||||
return modelConfig;
|
||||
}
|
||||
|
||||
async findByType(type: string): Promise<ModelConfig[]> {
|
||||
return this.modelConfigRepository.find({ where: { type } });
|
||||
}
|
||||
|
||||
async update(
|
||||
id: string,
|
||||
updateModelConfigDto: UpdateModelConfigDto,
|
||||
): Promise<ModelConfig> {
|
||||
const modelConfig = await this.findOne(id);
|
||||
|
||||
// Update the model
|
||||
const updated = this.modelConfigRepository.merge(
|
||||
modelConfig,
|
||||
updateModelConfigDto,
|
||||
);
|
||||
return this.modelConfigRepository.save(updated);
|
||||
}
|
||||
|
||||
async remove(id: string): Promise<void> {
|
||||
const result = await this.modelConfigRepository.delete({ id });
|
||||
if (result.affected === 0) {
|
||||
throw new NotFoundException(
|
||||
this.i18nService.formatMessage('modelConfigNotFound', { id }),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the specified model as default
|
||||
*/
|
||||
async setDefault(id: string): Promise<ModelConfig> {
|
||||
const modelConfig = await this.findOne(id);
|
||||
|
||||
// Clear default flag for other models of the same type
|
||||
await this.modelConfigRepository
|
||||
.createQueryBuilder()
|
||||
.update(ModelConfig)
|
||||
.set({ isDefault: false })
|
||||
.where('type = :type', { type: modelConfig.type })
|
||||
.execute();
|
||||
|
||||
modelConfig.isDefault = true;
|
||||
return this.modelConfigRepository.save(modelConfig);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get default model for specified type
|
||||
* Strict rule: Only return models specified in Index Chat Config, throw error if not found
|
||||
*/
|
||||
async findDefaultByType(
|
||||
tenantId: string,
|
||||
type: ModelType,
|
||||
): Promise<ModelConfig> {
|
||||
const settings = await this.tenantService.getSettings(tenantId);
|
||||
if (!settings) {
|
||||
throw new BadRequestException(
|
||||
`Organization settings not found for tenant: ${tenantId}`,
|
||||
);
|
||||
}
|
||||
|
||||
let modelId: string | undefined;
|
||||
if (type === ModelType.LLM) {
|
||||
modelId = settings.selectedLLMId;
|
||||
} else if (type === ModelType.EMBEDDING) {
|
||||
modelId = settings.selectedEmbeddingId;
|
||||
} else if (type === ModelType.RERANK) {
|
||||
modelId = settings.selectedRerankId;
|
||||
}
|
||||
|
||||
if (!modelId) {
|
||||
throw new BadRequestException(
|
||||
`Model of type "${type}" is not configured in Index Chat Config for this organization.`,
|
||||
);
|
||||
}
|
||||
|
||||
const model = await this.modelConfigRepository.findOne({
|
||||
where: { id: modelId, isEnabled: true },
|
||||
});
|
||||
|
||||
if (!model) {
|
||||
throw new BadRequestException(
|
||||
`The configured model for "${type}" (ID: ${modelId}) is either missing or disabled in model management.`,
|
||||
);
|
||||
}
|
||||
|
||||
return model;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
import {
|
||||
Controller,
|
||||
Get,
|
||||
Post,
|
||||
Put,
|
||||
Delete,
|
||||
Body,
|
||||
Param,
|
||||
UseGuards,
|
||||
Request,
|
||||
} from '@nestjs/common';
|
||||
import { NoteCategoryService } from './note-category.service';
|
||||
import { CombinedAuthGuard } from '../auth/combined-auth.guard';
|
||||
|
||||
@Controller('v1/note-categories')
|
||||
@UseGuards(CombinedAuthGuard)
|
||||
export class NoteCategoryController {
|
||||
constructor(private readonly categoryService: NoteCategoryService) {}
|
||||
|
||||
@Get()
|
||||
async findAll(@Request() req: any) {
|
||||
return this.categoryService.findAll(req.user.id);
|
||||
}
|
||||
|
||||
@Post()
|
||||
async create(
|
||||
@Request() req: any,
|
||||
@Body('name') name: string,
|
||||
@Body('parentId') parentId?: string,
|
||||
) {
|
||||
return this.categoryService.create(
|
||||
req.user.id,
|
||||
name,
|
||||
parentId,
|
||||
);
|
||||
}
|
||||
|
||||
@Put(':id')
|
||||
async update(
|
||||
@Request() req: any,
|
||||
@Param('id') id: string,
|
||||
@Body('name') name?: string,
|
||||
@Body('parentId') parentId?: string,
|
||||
) {
|
||||
return this.categoryService.update(
|
||||
req.user.id,
|
||||
id,
|
||||
name,
|
||||
parentId,
|
||||
);
|
||||
}
|
||||
|
||||
@Delete(':id')
|
||||
async remove(@Request() req: any, @Param('id') id: string) {
|
||||
return this.categoryService.remove(req.user.id, id);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
import {
|
||||
Entity,
|
||||
PrimaryGeneratedColumn,
|
||||
Column,
|
||||
CreateDateColumn,
|
||||
UpdateDateColumn,
|
||||
ManyToOne,
|
||||
JoinColumn,
|
||||
OneToMany,
|
||||
} from 'typeorm';
|
||||
import { User } from '../user/user.entity';
|
||||
import { Note } from './note.entity';
|
||||
|
||||
@Entity('note_categories')
|
||||
export class NoteCategory {
|
||||
@PrimaryGeneratedColumn('uuid')
|
||||
id: string;
|
||||
|
||||
@Column()
|
||||
name: string;
|
||||
|
||||
@Column({ name: 'user_id' })
|
||||
userId: string;
|
||||
|
||||
@Column({ name: 'parent_id', nullable: true, type: 'text' })
|
||||
parentId: string;
|
||||
|
||||
@Column({ default: 1 })
|
||||
level: number;
|
||||
|
||||
@ManyToOne(() => NoteCategory, (category) => category.children, {
|
||||
onDelete: 'CASCADE',
|
||||
})
|
||||
@JoinColumn({ name: 'parent_id' })
|
||||
parent: NoteCategory;
|
||||
|
||||
@OneToMany(() => NoteCategory, (category) => category.parent)
|
||||
children: NoteCategory[];
|
||||
|
||||
@ManyToOne(() => User)
|
||||
@JoinColumn({ name: 'user_id' })
|
||||
user: User;
|
||||
|
||||
@OneToMany(() => Note, (note) => note.category)
|
||||
notes: Note[];
|
||||
|
||||
@CreateDateColumn({ name: 'created_at' })
|
||||
createdAt: Date;
|
||||
|
||||
@UpdateDateColumn({ name: 'updated_at' })
|
||||
updatedAt: Date;
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
import { Injectable, NotFoundException } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Repository } from 'typeorm';
|
||||
import { NoteCategory } from './note-category.entity';
|
||||
import { I18nService } from '../i18n/i18n.service';
|
||||
|
||||
@Injectable()
|
||||
export class NoteCategoryService {
|
||||
constructor(
|
||||
@InjectRepository(NoteCategory)
|
||||
private readonly categoryRepository: Repository<NoteCategory>,
|
||||
private readonly i18nService: I18nService,
|
||||
) {}
|
||||
|
||||
async findAll(userId: string): Promise<NoteCategory[]> {
|
||||
return this.categoryRepository.find({
|
||||
where: { userId },
|
||||
order: { level: 'ASC', name: 'ASC' },
|
||||
});
|
||||
}
|
||||
|
||||
async create(
|
||||
userId: string,
|
||||
name: string,
|
||||
parentId?: string,
|
||||
): Promise<NoteCategory> {
|
||||
let level = 1;
|
||||
if (parentId) {
|
||||
const parent = await this.categoryRepository.findOne({
|
||||
where: { id: parentId, userId },
|
||||
});
|
||||
if (!parent) {
|
||||
throw new NotFoundException(
|
||||
this.i18nService.getMessage('parentCategoryNotFound'),
|
||||
);
|
||||
}
|
||||
if (parent.level >= 3) {
|
||||
throw new Error(
|
||||
this.i18nService.getMessage('maxCategoryDepthExceeded'),
|
||||
);
|
||||
}
|
||||
level = parent.level + 1;
|
||||
}
|
||||
|
||||
const category = this.categoryRepository.create({
|
||||
name,
|
||||
userId,
|
||||
parentId,
|
||||
level,
|
||||
});
|
||||
return this.categoryRepository.save(category);
|
||||
}
|
||||
|
||||
async update(
|
||||
userId: string,
|
||||
id: string,
|
||||
name?: string,
|
||||
parentId?: string,
|
||||
): Promise<NoteCategory> {
|
||||
const category = await this.categoryRepository.findOne({
|
||||
where: { id, userId },
|
||||
});
|
||||
if (!category) {
|
||||
throw new NotFoundException(
|
||||
this.i18nService.getMessage('categoryNotFound'),
|
||||
);
|
||||
}
|
||||
|
||||
if (name !== undefined) {
|
||||
category.name = name;
|
||||
}
|
||||
|
||||
if (parentId !== undefined && parentId !== category.parentId) {
|
||||
if (parentId === null) {
|
||||
category.parentId = null as any;
|
||||
category.level = 1;
|
||||
} else {
|
||||
const parent = await this.categoryRepository.findOne({
|
||||
where: { id: parentId, userId },
|
||||
});
|
||||
if (!parent)
|
||||
throw new NotFoundException(
|
||||
this.i18nService.getMessage('parentCategoryNotFound'),
|
||||
);
|
||||
if (parent.level >= 3)
|
||||
throw new Error(
|
||||
this.i18nService.getMessage('maxCategoryDepthExceeded'),
|
||||
);
|
||||
|
||||
category.parentId = parentId;
|
||||
category.level = parent.level + 1;
|
||||
}
|
||||
}
|
||||
|
||||
return this.categoryRepository.save(category);
|
||||
}
|
||||
|
||||
async remove(userId: string, id: string): Promise<void> {
|
||||
const result = await this.categoryRepository.delete({
|
||||
id,
|
||||
userId,
|
||||
});
|
||||
if (result.affected === 0) {
|
||||
throw new NotFoundException(
|
||||
this.i18nService.getMessage('categoryNotFound'),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
import {
|
||||
Controller,
|
||||
Get,
|
||||
Post,
|
||||
Body,
|
||||
Patch,
|
||||
Param,
|
||||
Delete,
|
||||
UseGuards,
|
||||
Req,
|
||||
Query,
|
||||
UseInterceptors,
|
||||
UploadedFile,
|
||||
} from '@nestjs/common';
|
||||
import { FileInterceptor } from '@nestjs/platform-express';
|
||||
import { CombinedAuthGuard } from '../auth/combined-auth.guard';
|
||||
import { NoteService } from './note.service';
|
||||
import { Note } from './note.entity';
|
||||
|
||||
@Controller('notes')
|
||||
@UseGuards(CombinedAuthGuard)
|
||||
export class NoteController {
|
||||
constructor(private readonly noteService: NoteService) {}
|
||||
|
||||
@Post()
|
||||
create(@Req() req, @Body() createNoteDto: Partial<Note>) {
|
||||
return this.noteService.create(
|
||||
req.user.id,
|
||||
createNoteDto,
|
||||
);
|
||||
}
|
||||
|
||||
@Get()
|
||||
findAll(
|
||||
@Req() req,
|
||||
@Query('groupId') groupId?: string,
|
||||
@Query('categoryId') categoryId?: string,
|
||||
) {
|
||||
return this.noteService.findAll(
|
||||
req.user.id,
|
||||
req.user.isAdmin,
|
||||
groupId,
|
||||
categoryId,
|
||||
);
|
||||
}
|
||||
|
||||
@Get(':id')
|
||||
findOne(@Req() req, @Param('id') id: string) {
|
||||
return this.noteService.findOne(
|
||||
req.user.id,
|
||||
id,
|
||||
req.user.isAdmin,
|
||||
);
|
||||
}
|
||||
|
||||
@Patch(':id')
|
||||
update(
|
||||
@Req() req,
|
||||
@Param('id') id: string,
|
||||
@Body() updateNoteDto: Partial<Note>,
|
||||
) {
|
||||
return this.noteService.update(
|
||||
req.user.id,
|
||||
id,
|
||||
updateNoteDto,
|
||||
req.user.isAdmin,
|
||||
);
|
||||
}
|
||||
|
||||
@Delete(':id')
|
||||
remove(@Req() req, @Param('id') id: string) {
|
||||
return this.noteService.remove(
|
||||
req.user.id,
|
||||
id,
|
||||
req.user.isAdmin,
|
||||
);
|
||||
}
|
||||
|
||||
@Post('from-pdf-selection')
|
||||
@UseInterceptors(FileInterceptor('screenshot'))
|
||||
createFromPDFSelection(
|
||||
@Req() req,
|
||||
@UploadedFile() screenshot: Express.Multer.File,
|
||||
@Body('fileId') fileId: string,
|
||||
@Body('groupId') groupId?: string,
|
||||
@Body('categoryId') categoryId?: string,
|
||||
@Body('pageNumber') pageNumber?: string,
|
||||
) {
|
||||
return this.noteService.createFromPDFSelection(
|
||||
req.user.id,
|
||||
fileId,
|
||||
screenshot,
|
||||
groupId,
|
||||
categoryId,
|
||||
pageNumber ? parseInt(pageNumber, 10) : undefined,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
import {
|
||||
Entity,
|
||||
PrimaryGeneratedColumn,
|
||||
Column,
|
||||
CreateDateColumn,
|
||||
UpdateDateColumn,
|
||||
ManyToOne,
|
||||
JoinColumn,
|
||||
} from 'typeorm';
|
||||
import { User } from '../user/user.entity';
|
||||
import { KnowledgeGroup } from '../knowledge-group/knowledge-group.entity';
|
||||
|
||||
@Entity('notes')
|
||||
export class Note {
|
||||
@PrimaryGeneratedColumn('uuid')
|
||||
id: string;
|
||||
|
||||
@Column()
|
||||
title: string;
|
||||
|
||||
@Column('text')
|
||||
content: string;
|
||||
|
||||
@Column({ name: 'user_id' })
|
||||
userId: string;
|
||||
|
||||
@Column({ name: 'group_id', nullable: true })
|
||||
groupId: string; // Corresponds to Notebook/KnowledgeGroup ID
|
||||
|
||||
@Column({
|
||||
type: 'simple-enum',
|
||||
enum: ['PRIVATE', 'TENANT', 'GLOBAL_PENDING', 'GLOBAL_APPROVED'],
|
||||
default: 'PRIVATE',
|
||||
name: 'sharing_status',
|
||||
})
|
||||
sharingStatus: string;
|
||||
|
||||
@Column({ name: 'screenshot_path', nullable: true })
|
||||
screenshotPath: string; // Path to screenshot image for PDF selections
|
||||
|
||||
@Column({ name: 'source_file_id', nullable: true })
|
||||
sourceFileId: string; // ID of the source PDF file
|
||||
|
||||
@Column({ name: 'source_page_number', nullable: true })
|
||||
sourcePageNumber: number; // Page number in the source PDF
|
||||
|
||||
@CreateDateColumn({ name: 'created_at' })
|
||||
createdAt: Date;
|
||||
|
||||
@UpdateDateColumn({ name: 'updated_at' })
|
||||
updatedAt: Date;
|
||||
|
||||
@ManyToOne(() => User)
|
||||
@JoinColumn({ name: 'user_id' })
|
||||
user: User;
|
||||
|
||||
@ManyToOne(() => KnowledgeGroup)
|
||||
@JoinColumn({ name: 'group_id' })
|
||||
group: KnowledgeGroup;
|
||||
|
||||
@Column({ name: 'category_id', nullable: true })
|
||||
categoryId: string;
|
||||
|
||||
@ManyToOne('NoteCategory', 'notes', { nullable: true, onDelete: 'SET NULL' })
|
||||
@JoinColumn({ name: 'category_id' })
|
||||
category: any;
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import { Module, forwardRef } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
import { Note } from './note.entity';
|
||||
import { NoteCategory } from './note-category.entity';
|
||||
import { NoteService } from './note.service';
|
||||
import { NoteController } from './note.controller';
|
||||
import { NoteCategoryService } from './note-category.service';
|
||||
import { NoteCategoryController } from './note-category.controller';
|
||||
import { KnowledgeGroupModule } from '../knowledge-group/knowledge-group.module';
|
||||
import { OcrModule } from '../ocr/ocr.module';
|
||||
import { I18nModule } from '../i18n/i18n.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
TypeOrmModule.forFeature([Note, NoteCategory]),
|
||||
forwardRef(() => KnowledgeGroupModule),
|
||||
OcrModule,
|
||||
I18nModule,
|
||||
],
|
||||
providers: [NoteService, NoteCategoryService],
|
||||
controllers: [NoteController, NoteCategoryController],
|
||||
exports: [NoteService, NoteCategoryService],
|
||||
})
|
||||
export class NoteModule {}
|
||||
@@ -0,0 +1,218 @@
|
||||
import { Injectable, NotFoundException } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Repository } from 'typeorm';
|
||||
import { Note } from './note.entity';
|
||||
import { KnowledgeGroup } from '../knowledge-group/knowledge-group.entity';
|
||||
import { OcrService } from '../ocr/ocr.service';
|
||||
import * as fs from 'fs/promises';
|
||||
import * as path from 'path';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import { I18nService } from '../i18n/i18n.service';
|
||||
|
||||
@Injectable()
|
||||
export class NoteService {
|
||||
// Directory will be created dynamically per user
|
||||
private getScreenshotsDir(userId: string) {
|
||||
return path.join(process.cwd(), 'uploads', 'notes-screenshots', userId);
|
||||
}
|
||||
|
||||
constructor(
|
||||
@InjectRepository(Note)
|
||||
private readonly noteRepository: Repository<Note>,
|
||||
private readonly ocrService: OcrService,
|
||||
private readonly i18nService: I18nService,
|
||||
) {}
|
||||
|
||||
private async ensureScreenshotsDir(userId: string) {
|
||||
const dir = this.getScreenshotsDir(userId);
|
||||
try {
|
||||
await fs.access(dir);
|
||||
} catch {
|
||||
await fs.mkdir(dir, { recursive: true });
|
||||
}
|
||||
}
|
||||
|
||||
async create(
|
||||
userId: string,
|
||||
data: Partial<Note>,
|
||||
): Promise<Note> {
|
||||
// Handle empty strings for foreign keys
|
||||
if (data.groupId === '') {
|
||||
data.groupId = null as any;
|
||||
}
|
||||
if (data.categoryId === '') {
|
||||
data.categoryId = null as any;
|
||||
}
|
||||
|
||||
const note = this.noteRepository.create({
|
||||
...data,
|
||||
userId,
|
||||
});
|
||||
return this.noteRepository.save(note);
|
||||
}
|
||||
|
||||
async findAll(
|
||||
userId: string,
|
||||
isAdmin: boolean,
|
||||
groupId?: string,
|
||||
categoryId?: string,
|
||||
): Promise<Note[]> {
|
||||
const query = this.noteRepository
|
||||
.createQueryBuilder('note')
|
||||
.leftJoinAndSelect('note.user', 'user');
|
||||
|
||||
if (!isAdmin) {
|
||||
query.where('note.userId = :userId', { userId });
|
||||
}
|
||||
|
||||
if (groupId) {
|
||||
query.andWhere('note.groupId = :groupId', { groupId });
|
||||
}
|
||||
|
||||
if (categoryId) {
|
||||
query.andWhere('note.categoryId = :categoryId', { categoryId });
|
||||
}
|
||||
|
||||
return query.getMany();
|
||||
}
|
||||
|
||||
async findOne(
|
||||
userId: string,
|
||||
id: string,
|
||||
isAdmin: boolean,
|
||||
): Promise<Note> {
|
||||
let note;
|
||||
if (isAdmin) {
|
||||
note = await this.noteRepository.findOne({
|
||||
where: { id },
|
||||
relations: ['user'],
|
||||
});
|
||||
} else {
|
||||
note = await this.noteRepository.findOne({
|
||||
where: { id, userId },
|
||||
relations: ['user'],
|
||||
});
|
||||
}
|
||||
|
||||
if (!note) {
|
||||
throw new NotFoundException(
|
||||
this.i18nService.formatMessage('noteNotFound', { id }),
|
||||
);
|
||||
}
|
||||
return note;
|
||||
}
|
||||
|
||||
async update(
|
||||
userId: string,
|
||||
id: string,
|
||||
data: Partial<Note>,
|
||||
isAdmin: boolean,
|
||||
): Promise<Note> {
|
||||
const note = await this.findOne(userId, id, isAdmin);
|
||||
// Remove protected fields
|
||||
delete (data as any).id;
|
||||
delete (data as any).userId;
|
||||
delete (data as any).createdAt;
|
||||
|
||||
// Handle empty strings for foreign keys
|
||||
if (data.groupId === '') {
|
||||
data.groupId = null as any;
|
||||
}
|
||||
if (data.categoryId === '') {
|
||||
data.categoryId = null as any;
|
||||
}
|
||||
|
||||
Object.assign(note, data);
|
||||
return this.noteRepository.save(note);
|
||||
}
|
||||
|
||||
async createFromPDFSelection(
|
||||
userId: string,
|
||||
fileId: string,
|
||||
screenshot: Express.Multer.File,
|
||||
groupId?: string,
|
||||
categoryId?: string,
|
||||
pageNumber?: number,
|
||||
): Promise<Note> {
|
||||
// If groupId is provided, verify that the group exists
|
||||
// We'll directly query the group to ensure it exists, regardless of user permissions
|
||||
// Since all groups are accessible to all users anyway
|
||||
if (groupId) {
|
||||
const groupRepo =
|
||||
this.noteRepository.manager.getRepository(KnowledgeGroup);
|
||||
const group = await groupRepo.findOne({
|
||||
where: { id: groupId },
|
||||
});
|
||||
|
||||
if (!group) {
|
||||
throw new NotFoundException(
|
||||
this.i18nService.formatMessage('knowledgeGroupNotFound', {
|
||||
id: groupId,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
// Optional: Add logging to help debug permission issues
|
||||
console.log(`User ${userId} attempting to add note to group ${groupId}`);
|
||||
}
|
||||
|
||||
if (categoryId === '') {
|
||||
categoryId = null as any;
|
||||
}
|
||||
|
||||
// Save screenshot to disk
|
||||
await this.ensureScreenshotsDir(userId);
|
||||
const filename = `${uuidv4()}-${Date.now()}.png`;
|
||||
const screenshotPath = path.join(
|
||||
this.getScreenshotsDir(userId),
|
||||
filename,
|
||||
);
|
||||
await fs.writeFile(screenshotPath, screenshot.buffer);
|
||||
|
||||
// Extract text using OCR
|
||||
let extractedText = '';
|
||||
try {
|
||||
extractedText = await this.ocrService.extractTextFromImage(
|
||||
screenshot.buffer,
|
||||
);
|
||||
} catch (error) {
|
||||
console.error('OCR extraction failed:', error);
|
||||
// Continue without OCR text if extraction fails
|
||||
}
|
||||
|
||||
// Create note with screenshot and extracted text
|
||||
const note = this.noteRepository.create({
|
||||
userId,
|
||||
groupId: groupId || (null as any),
|
||||
categoryId: categoryId || (null as any),
|
||||
title: this.i18nService.formatMessage('pdfNoteTitle', {
|
||||
date: new Date().toLocaleString(),
|
||||
}),
|
||||
content: extractedText || this.i18nService.getMessage('noTextExtracted'),
|
||||
screenshotPath: `notes-screenshots/${userId}/${filename}`,
|
||||
sourceFileId: fileId,
|
||||
sourcePageNumber: pageNumber,
|
||||
});
|
||||
|
||||
return this.noteRepository.save(note);
|
||||
}
|
||||
|
||||
async remove(
|
||||
userId: string,
|
||||
id: string,
|
||||
isAdmin: boolean,
|
||||
): Promise<void> {
|
||||
let result;
|
||||
if (isAdmin) {
|
||||
result = await this.noteRepository.delete({ id });
|
||||
} else {
|
||||
result = await this.noteRepository.delete({ id, userId });
|
||||
}
|
||||
|
||||
if (result.affected === 0) {
|
||||
throw new NotFoundException(
|
||||
this.i18nService.formatMessage('noteNotFound', { id }),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import {
|
||||
Controller,
|
||||
Post,
|
||||
UseGuards,
|
||||
UseInterceptors,
|
||||
UploadedFile,
|
||||
} from '@nestjs/common';
|
||||
import { FileInterceptor } from '@nestjs/platform-express';
|
||||
import { CombinedAuthGuard } from '../auth/combined-auth.guard';
|
||||
import { OcrService } from './ocr.service';
|
||||
import { I18nService } from '../i18n/i18n.service';
|
||||
|
||||
@Controller('ocr')
|
||||
@UseGuards(CombinedAuthGuard)
|
||||
@UseGuards(CombinedAuthGuard)
|
||||
export class OcrController {
|
||||
constructor(
|
||||
private readonly ocrService: OcrService,
|
||||
private readonly i18n: I18nService,
|
||||
) {}
|
||||
|
||||
@Post('recognize')
|
||||
@UseInterceptors(FileInterceptor('image'))
|
||||
async recognizeText(@UploadedFile() image: Express.Multer.File) {
|
||||
console.log('OCR recognition endpoint called');
|
||||
if (!image) {
|
||||
console.error('No image uploaded');
|
||||
throw new Error(this.i18n.getMessage('noImageUploaded'));
|
||||
}
|
||||
console.log(`Received image. Size: ${image.size} bytes`);
|
||||
const text = await this.ocrService.extractTextFromImage(image.buffer);
|
||||
console.log(`OCR extraction completed. Text length: ${text.length}`);
|
||||
return { text };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { OcrService } from './ocr.service';
|
||||
import { OcrController } from './ocr.controller';
|
||||
|
||||
@Module({
|
||||
controllers: [OcrController],
|
||||
providers: [OcrService],
|
||||
exports: [OcrService],
|
||||
})
|
||||
export class OcrModule {}
|
||||
@@ -0,0 +1,80 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { createWorker, Worker } from 'tesseract.js';
|
||||
import { I18nService } from '../i18n/i18n.service';
|
||||
|
||||
@Injectable()
|
||||
export class OcrService {
|
||||
private readonly logger = new Logger(OcrService.name);
|
||||
|
||||
constructor(private readonly i18n: I18nService) {}
|
||||
|
||||
async extractTextFromImage(imageBuffer: Buffer): Promise<string> {
|
||||
this.logger.log(
|
||||
`Starting OCR extraction from image (${imageBuffer.length} bytes)...`,
|
||||
);
|
||||
|
||||
// Create worker for this request to ensure stability
|
||||
let worker: any = null;
|
||||
try {
|
||||
worker = await createWorker('chi_sim+eng+jpn');
|
||||
|
||||
const {
|
||||
data: { text },
|
||||
} = await worker.recognize(imageBuffer);
|
||||
this.logger.log(
|
||||
`OCR extraction completed. ${text.length} characters extracted.`,
|
||||
);
|
||||
|
||||
if (text.length === 0) {
|
||||
this.logger.warn('OCR returned empty text.');
|
||||
}
|
||||
|
||||
await worker.terminate();
|
||||
return text.trim();
|
||||
} catch (error) {
|
||||
this.logger.error(`OCR text extraction failed: ${error.message}`);
|
||||
if (worker) {
|
||||
try {
|
||||
await worker.terminate();
|
||||
} catch (e) {}
|
||||
}
|
||||
throw new Error(
|
||||
this.i18n.formatMessage('ocrFailed', { message: error.message }),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async extractTextWithConfidence(imageBuffer: Buffer): Promise<{
|
||||
text: string;
|
||||
confidence: number;
|
||||
}> {
|
||||
this.logger.log(
|
||||
`Starting OCR extraction with confidence (${imageBuffer.length} bytes)...`,
|
||||
);
|
||||
|
||||
let worker: any = null;
|
||||
try {
|
||||
worker = await createWorker('chi_sim+eng+jpn');
|
||||
const { data } = await worker.recognize(imageBuffer);
|
||||
this.logger.log(
|
||||
`OCR extraction completed. Confidence: ${data.confidence}%`,
|
||||
);
|
||||
|
||||
await worker.terminate();
|
||||
return {
|
||||
text: data.text.trim(),
|
||||
confidence: data.confidence,
|
||||
};
|
||||
} catch (error) {
|
||||
this.logger.error(`OCR text extraction failed: ${error.message}`);
|
||||
if (worker) {
|
||||
try {
|
||||
await worker.terminate();
|
||||
} catch (e) {}
|
||||
}
|
||||
throw new Error(
|
||||
this.i18n.formatMessage('ocrFailed', { message: error.message }),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
/**
|
||||
* PDF to Image Interface Definitions
|
||||
*/
|
||||
|
||||
export interface Pdf2ImageOptions {
|
||||
density?: number; // DPI resolution, default 300
|
||||
quality?: number; // JPEG quality (1-100), default 85
|
||||
format?: 'jpeg' | 'png'; // output format, default jpeg
|
||||
outDir?: string; // Output directory, default ./temp
|
||||
}
|
||||
|
||||
export interface ImageInfo {
|
||||
path: string; // Image file path
|
||||
pageIndex: number; // Page number (starting from 1)
|
||||
size: number; // File size (bytes)
|
||||
width?: number; // Image width
|
||||
height?: number; // Image height
|
||||
}
|
||||
|
||||
export interface ConversionResult {
|
||||
images: ImageInfo[];
|
||||
totalPages: number;
|
||||
successCount: number;
|
||||
failedCount: number;
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { Pdf2ImageService } from './pdf2image.service';
|
||||
|
||||
@Module({
|
||||
providers: [Pdf2ImageService],
|
||||
exports: [Pdf2ImageService],
|
||||
})
|
||||
export class Pdf2ImageModule {}
|
||||
@@ -0,0 +1,181 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import * as fs from 'fs/promises';
|
||||
import * as path from 'path';
|
||||
import { PDFDocument } from 'pdf-lib';
|
||||
import { exec } from 'child_process';
|
||||
import { promisify } from 'util';
|
||||
import {
|
||||
Pdf2ImageOptions,
|
||||
ImageInfo,
|
||||
ConversionResult,
|
||||
} from './pdf2image.interface';
|
||||
import { I18nService } from '../i18n/i18n.service';
|
||||
|
||||
const execAsync = promisify(exec);
|
||||
|
||||
@Injectable()
|
||||
export class Pdf2ImageService {
|
||||
private readonly logger = new Logger(Pdf2ImageService.name);
|
||||
private tempDir: string;
|
||||
|
||||
constructor(
|
||||
private configService: ConfigService,
|
||||
private i18nService: I18nService,
|
||||
) {
|
||||
this.tempDir = this.configService.get<string>('TEMP_DIR', './temp');
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert PDF to list of images
|
||||
* Uses ImageMagick's convert command
|
||||
*/
|
||||
async convertToImages(
|
||||
pdfPath: string,
|
||||
options: Pdf2ImageOptions = {},
|
||||
): Promise<ConversionResult> {
|
||||
const {
|
||||
density = 300,
|
||||
quality = 85,
|
||||
format = 'jpeg',
|
||||
outDir = this.tempDir,
|
||||
} = options;
|
||||
|
||||
// Validate PDF file
|
||||
try {
|
||||
await fs.access(pdfPath);
|
||||
} catch {
|
||||
throw new Error(`PDF file not found: ${pdfPath}`);
|
||||
}
|
||||
|
||||
// Create output directory
|
||||
const timestamp = Date.now();
|
||||
const outputDir = path.join(outDir, `pdf2img_${timestamp}`);
|
||||
await fs.mkdir(outputDir, { recursive: true });
|
||||
|
||||
this.logger.log(`Converting PDF to images: ${pdfPath}`);
|
||||
this.logger.log(`Output directory: ${outputDir}`);
|
||||
|
||||
try {
|
||||
// Get total page count using pdf-lib instead of pdfinfo
|
||||
const pdfBytes = await fs.readFile(pdfPath);
|
||||
const pdfDoc = await PDFDocument.load(pdfBytes, {
|
||||
ignoreEncryption: true,
|
||||
});
|
||||
const totalPages = pdfDoc.getPageCount();
|
||||
|
||||
if (totalPages === 0) {
|
||||
throw new Error(this.i18nService.getMessage('pdfPageCountError'));
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`Starting PDF conversion: ${path.basename(pdfPath)} (${totalPages} pages)`,
|
||||
);
|
||||
this.logger.log(`Output directory: ${outputDir}`);
|
||||
this.logger.log(
|
||||
`Conversion parameters: density=${density}dpi, quality=${quality}%, format=${format}`,
|
||||
);
|
||||
|
||||
// Convert using Python script
|
||||
const zoom = (density / 72).toFixed(2);
|
||||
const pythonScript = path.join(process.cwd(), 'pdf_to_images.py');
|
||||
const cmd = `python "${pythonScript}" "${pdfPath}" "${outputDir}" ${zoom} ${quality}`;
|
||||
|
||||
this.logger.log(`Executing conversion command: ${cmd}`);
|
||||
const { stdout } = await execAsync(cmd);
|
||||
const result = JSON.parse(stdout);
|
||||
|
||||
if (!result.success) {
|
||||
throw new Error(`Python conversion failed: ${result.error}`);
|
||||
}
|
||||
|
||||
const images: ImageInfo[] = result.images;
|
||||
const successCount = result.images.length;
|
||||
const failedCount = totalPages - successCount;
|
||||
|
||||
this.logger.log(
|
||||
`🎉 PDF conversion completed! ✅ Success: ${successCount} pages, ❌ Failed: ${failedCount} pages, 📊 Total pages: ${totalPages}`,
|
||||
);
|
||||
|
||||
return {
|
||||
images,
|
||||
totalPages,
|
||||
successCount,
|
||||
failedCount,
|
||||
};
|
||||
} catch (error) {
|
||||
// Cleanup temp directory
|
||||
await this.cleanupDirectory(outputDir);
|
||||
throw new Error(`PDF to image conversion failed: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch convert multiple PDFs
|
||||
*/
|
||||
async batchConvert(
|
||||
pdfPaths: string[],
|
||||
options?: Pdf2ImageOptions,
|
||||
): Promise<ConversionResult[]> {
|
||||
const results: ConversionResult[] = [];
|
||||
for (const pdfPath of pdfPaths) {
|
||||
try {
|
||||
const result = await this.convertToImages(pdfPath, options);
|
||||
results.push(result);
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to convert ${pdfPath}: ${error.message}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleanup image files
|
||||
*/
|
||||
async cleanupImages(images: ImageInfo[]): Promise<void> {
|
||||
for (const image of images) {
|
||||
try {
|
||||
await fs.unlink(image.path);
|
||||
this.logger.log(`Deleted: ${image.path}`);
|
||||
} catch (error) {
|
||||
this.logger.warn(`Failed to delete ${image.path}: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Try to cleanup empty directory
|
||||
if (images.length > 0) {
|
||||
const dir = path.dirname(images[0].path);
|
||||
await this.cleanupDirectory(dir);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleanup directory
|
||||
*/
|
||||
async cleanupDirectory(dir: string): Promise<void> {
|
||||
try {
|
||||
const files = await fs.readdir(dir);
|
||||
if (files.length === 0) {
|
||||
await fs.rmdir(dir);
|
||||
this.logger.log(`Cleaned up empty directory: ${dir}`);
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.warn(`Failed to cleanup directory ${dir}: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if image quality is acceptable
|
||||
*/
|
||||
isImageQualityGood(imageInfo: ImageInfo, minSizeKB: number = 10): boolean {
|
||||
const sizeKB = imageInfo.size / 1024;
|
||||
if (sizeKB < minSizeKB) {
|
||||
this.logger.warn(
|
||||
`Image too small: ${sizeKB.toFixed(2)}KB < ${minSizeKB}KB`,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
import {
|
||||
Entity,
|
||||
PrimaryGeneratedColumn,
|
||||
Column,
|
||||
CreateDateColumn,
|
||||
UpdateDateColumn,
|
||||
ManyToOne,
|
||||
JoinColumn,
|
||||
} from 'typeorm';
|
||||
import { User } from '../../user/user.entity';
|
||||
import { KnowledgeGroup } from '../../knowledge-group/knowledge-group.entity';
|
||||
import { Tenant } from '../../tenant/tenant.entity';
|
||||
|
||||
export enum PodcastStatus {
|
||||
PENDING = 'pending',
|
||||
PROCESSING = 'processing',
|
||||
COMPLETED = 'completed',
|
||||
FAILED = 'failed',
|
||||
}
|
||||
|
||||
@Entity('podcast_episodes')
|
||||
export class PodcastEpisode {
|
||||
@PrimaryGeneratedColumn('uuid')
|
||||
id: string;
|
||||
|
||||
@Column()
|
||||
title: string;
|
||||
|
||||
@Column('text', { nullable: true })
|
||||
briefing: string; // The instruction/prompt used
|
||||
|
||||
@Column({ nullable: true })
|
||||
audioUrl: string;
|
||||
|
||||
@Column('simple-json', { nullable: true })
|
||||
transcript: any; // The dialogue script
|
||||
|
||||
@Column({
|
||||
type: 'simple-enum',
|
||||
enum: PodcastStatus,
|
||||
default: PodcastStatus.PENDING,
|
||||
})
|
||||
status: PodcastStatus;
|
||||
|
||||
@Column({ name: 'user_id' })
|
||||
userId: string;
|
||||
|
||||
@Column({ name: 'group_id', nullable: true })
|
||||
groupId: string;
|
||||
|
||||
@Column({ name: 'tenant_id', nullable: true, type: 'text' })
|
||||
tenantId: string;
|
||||
|
||||
@CreateDateColumn({ name: 'created_at' })
|
||||
createdAt: Date;
|
||||
|
||||
@UpdateDateColumn({ name: 'updated_at' })
|
||||
updatedAt: Date;
|
||||
|
||||
@ManyToOne(() => User)
|
||||
@JoinColumn({ name: 'user_id' })
|
||||
user: User;
|
||||
|
||||
@ManyToOne(() => KnowledgeGroup)
|
||||
@JoinColumn({ name: 'group_id' })
|
||||
group: KnowledgeGroup;
|
||||
|
||||
@ManyToOne(() => Tenant)
|
||||
@JoinColumn({ name: 'tenant_id' })
|
||||
tenant: Tenant;
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
import {
|
||||
Controller,
|
||||
Get,
|
||||
Post,
|
||||
Body,
|
||||
Param,
|
||||
Delete,
|
||||
UseGuards,
|
||||
Req,
|
||||
Query,
|
||||
Res,
|
||||
} from '@nestjs/common';
|
||||
import { CombinedAuthGuard } from '../auth/combined-auth.guard';
|
||||
import { PodcastService } from './podcast.service';
|
||||
import { Response } from 'express';
|
||||
import * as path from 'path';
|
||||
|
||||
@Controller('podcasts')
|
||||
export class PodcastController {
|
||||
constructor(private readonly podcastService: PodcastService) {}
|
||||
|
||||
@Post()
|
||||
@UseGuards(CombinedAuthGuard)
|
||||
create(@Req() req, @Body() createDto: any) {
|
||||
return this.podcastService.create(req.user.id, createDto);
|
||||
}
|
||||
|
||||
@Get()
|
||||
@UseGuards(CombinedAuthGuard)
|
||||
findAll(@Req() req, @Query('groupId') groupId?: string) {
|
||||
return this.podcastService.findAll(req.user.id, groupId);
|
||||
}
|
||||
|
||||
@Get(':id')
|
||||
@UseGuards(CombinedAuthGuard)
|
||||
findOne(@Req() req, @Param('id') id: string) {
|
||||
return this.podcastService.findOne(req.user.id, id);
|
||||
}
|
||||
|
||||
@Delete(':id')
|
||||
@UseGuards(CombinedAuthGuard)
|
||||
remove(@Req() req, @Param('id') id: string) {
|
||||
return this.podcastService.delete(req.user.id, id);
|
||||
}
|
||||
|
||||
// Public route for audio streaming (or protected if preferred)
|
||||
@Get('audio/:filename')
|
||||
async streamAudio(@Param('filename') filename: string, @Res() res: Response) {
|
||||
const filePath = path.join(process.cwd(), 'uploads', 'podcasts', filename);
|
||||
res.sendFile(filePath);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
import { PodcastEpisode } from './entities/podcast-episode.entity';
|
||||
import { PodcastService } from './podcast.service';
|
||||
import { PodcastController } from './podcast.controller';
|
||||
import { KnowledgeGroup } from '../knowledge-group/knowledge-group.entity';
|
||||
import { ChatModule } from '../chat/chat.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
TypeOrmModule.forFeature([PodcastEpisode, KnowledgeGroup]),
|
||||
ChatModule, // Import ChatModule to use ChatService
|
||||
],
|
||||
controllers: [PodcastController],
|
||||
providers: [PodcastService],
|
||||
exports: [PodcastService],
|
||||
})
|
||||
export class PodcastModule {}
|
||||
@@ -0,0 +1,306 @@
|
||||
import { Injectable, Logger, NotFoundException } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Repository } from 'typeorm';
|
||||
import {
|
||||
PodcastEpisode,
|
||||
PodcastStatus,
|
||||
} from './entities/podcast-episode.entity';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
// import { EdgeTTS } from 'node-edge-tts'; // Deprecated due to 403 errors
|
||||
import * as fs from 'fs-extra';
|
||||
import * as path from 'path';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import { KnowledgeGroup } from '../knowledge-group/knowledge-group.entity';
|
||||
import { ChatService } from '../chat/chat.service';
|
||||
import { I18nService } from '../i18n/i18n.service';
|
||||
|
||||
@Injectable()
|
||||
export class PodcastService {
|
||||
private readonly logger = new Logger(PodcastService.name);
|
||||
private readonly outputDir: string;
|
||||
private readonly pythonPath = 'python'; // Or from config
|
||||
private readonly scriptPath = path.join(process.cwd(), 'text_to_speech.py');
|
||||
|
||||
constructor(
|
||||
@InjectRepository(PodcastEpisode)
|
||||
private podcastRepository: Repository<PodcastEpisode>,
|
||||
@InjectRepository(KnowledgeGroup)
|
||||
private groupRepository: Repository<KnowledgeGroup>,
|
||||
private configService: ConfigService,
|
||||
private chatService: ChatService, // Reusing ChatService to generate script
|
||||
private i18nService: I18nService,
|
||||
) {
|
||||
// this.tts = new EdgeTTS();
|
||||
this.outputDir = path.join(process.cwd(), 'uploads', 'podcasts');
|
||||
fs.ensureDirSync(this.outputDir);
|
||||
}
|
||||
|
||||
async create(userId: string, createDto: any): Promise<PodcastEpisode> {
|
||||
this.logger.log(`Creating podcast with DTO: ${JSON.stringify(createDto)}`);
|
||||
if (!userId) {
|
||||
throw new Error(this.i18nService.getMessage('userIdRequired'));
|
||||
}
|
||||
|
||||
const episode = this.podcastRepository.create({
|
||||
...createDto,
|
||||
briefing: createDto.content || createDto.briefing, // Map content to briefing if needed
|
||||
user: { id: userId },
|
||||
group: createDto.groupId ? { id: createDto.groupId } : undefined,
|
||||
status: PodcastStatus.PENDING,
|
||||
}) as unknown as PodcastEpisode; // Restore cast to fix TS inference issue
|
||||
|
||||
const saved = await this.podcastRepository.save(episode);
|
||||
|
||||
// Start background processing
|
||||
this.processPodcast(saved.id, userId, createDto);
|
||||
|
||||
return saved;
|
||||
}
|
||||
|
||||
async findAll(userId: string, groupId?: string): Promise<PodcastEpisode[]> {
|
||||
const query = this.podcastRepository
|
||||
.createQueryBuilder('podcast')
|
||||
.where('podcast.userId = :userId', { userId })
|
||||
.orderBy('podcast.createdAt', 'DESC');
|
||||
|
||||
if (groupId) {
|
||||
query.andWhere('podcast.groupId = :groupId', { groupId });
|
||||
}
|
||||
|
||||
return query.getMany();
|
||||
}
|
||||
|
||||
async findOne(userId: string, id: string): Promise<PodcastEpisode> {
|
||||
const episode = await this.podcastRepository.findOne({
|
||||
where: { id, userId },
|
||||
});
|
||||
if (!episode)
|
||||
throw new NotFoundException(
|
||||
this.i18nService.formatMessage('podcastNotFound', { id }),
|
||||
);
|
||||
return episode;
|
||||
}
|
||||
|
||||
async delete(userId: string, id: string): Promise<void> {
|
||||
const episode = await this.findOne(userId, id);
|
||||
|
||||
// Delete audio file if exists
|
||||
if (episode.audioUrl) {
|
||||
const filename = path.basename(episode.audioUrl);
|
||||
const filePath = path.join(this.outputDir, filename);
|
||||
if (fs.existsSync(filePath)) {
|
||||
fs.unlinkSync(filePath);
|
||||
}
|
||||
}
|
||||
|
||||
await this.podcastRepository.remove(episode);
|
||||
}
|
||||
|
||||
async processPodcast(episodeId: string, userId: string, dto: any) {
|
||||
try {
|
||||
this.logger.log(`Starting processing for podcast ${episodeId}`);
|
||||
await this.updateStatus(episodeId, PodcastStatus.PROCESSING);
|
||||
|
||||
// 1. Gather Context
|
||||
const context = dto.content || '';
|
||||
if (dto.groupId) {
|
||||
// TODO: Fetch context from group (files) if content is empty
|
||||
// For now assuming content is passed or we just use what we have
|
||||
}
|
||||
|
||||
// 2. Generate Script using ChatService (LLM)
|
||||
const fileIds = dto.fileId ? [dto.fileId] : undefined;
|
||||
const language = dto.language || 'zh';
|
||||
this.logger.log(`Generating script for language: ${language}`);
|
||||
const script = await this.generateScript(
|
||||
context,
|
||||
dto.topic || 'General Discussion',
|
||||
userId,
|
||||
language,
|
||||
dto.groupId,
|
||||
fileIds,
|
||||
);
|
||||
await this.podcastRepository.update(episodeId, { transcript: script });
|
||||
|
||||
// 3. Generate Audio using Edge TTS
|
||||
const audioFileName = `${episodeId}.mp3`;
|
||||
const audioFilePath = path.join(this.outputDir, audioFileName);
|
||||
|
||||
await this.generateAudioInternal(script, audioFilePath, language);
|
||||
|
||||
// 4. Update Episode
|
||||
await this.podcastRepository.update(episodeId, {
|
||||
status: PodcastStatus.COMPLETED,
|
||||
audioUrl: `/api/podcasts/audio/${audioFileName}`,
|
||||
});
|
||||
|
||||
this.logger.log(`Podcast ${episodeId} completed (Language: ${language})`);
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to process podcast ${episodeId}`, error);
|
||||
await this.updateStatus(episodeId, PodcastStatus.FAILED);
|
||||
}
|
||||
}
|
||||
|
||||
private async updateStatus(id: string, status: PodcastStatus) {
|
||||
await this.podcastRepository.update(id, { status });
|
||||
}
|
||||
|
||||
async generateScript(
|
||||
context: string,
|
||||
topic: string,
|
||||
userId: string,
|
||||
language: string = 'zh',
|
||||
groupId?: string,
|
||||
fileIds?: string[],
|
||||
): Promise<any[]> {
|
||||
// ... (RAG context logic omitted for brevity, logic remains same)
|
||||
// If groupId or fileIds are provided, try to enhance context with RAG
|
||||
if (
|
||||
(groupId || (fileIds && fileIds.length > 0)) &&
|
||||
(!context || context.length < 100)
|
||||
) {
|
||||
try {
|
||||
// tenantId is optional, we pass undefined here, groupId is string, fileIds is string[]
|
||||
const ragContext = await this.chatService.getContextForTopic(
|
||||
topic,
|
||||
userId,
|
||||
undefined,
|
||||
groupId,
|
||||
fileIds,
|
||||
);
|
||||
if (ragContext) {
|
||||
context = `Manual Context: ${context}\n\nSearch Results:\n${ragContext}`;
|
||||
}
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
`Failed to fetch RAG context for podcast: ${err.message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let targetLang = 'Chinese (Simplified)';
|
||||
if (language === 'en') targetLang = 'English';
|
||||
if (language === 'ja') targetLang = 'Japanese';
|
||||
|
||||
const prompt = `
|
||||
You are an expert podcast producer. Create a podcast script about the following topic: "${topic}".
|
||||
|
||||
Context information (use this to inform the discussion):
|
||||
${context ? context.substring(0, 5000) : 'No specific context provided, use general knowledge.'}
|
||||
|
||||
The podcast should be a dialogue between a Host and a Guest.
|
||||
- Host: Ask insightful questions and guide the conversation.
|
||||
- Guest: Provide expert answers and insights based on the context.
|
||||
- Tone: Professional yet conversational.
|
||||
- Length: Approximately 8-12 exchanges.
|
||||
- Language: ${targetLang}.
|
||||
|
||||
IMPORTANT: The dialogue MUST be spoken in separate ${targetLang} sentences. Even if the context is valid in another language, translate the concepts and discuss them in ${targetLang}.
|
||||
|
||||
Output the script as a valid JSON array of objects, where each object has "speaker" (Host/Guest) and "text" (the spoken content).
|
||||
Example:
|
||||
[
|
||||
// Example structure, ensure actual content is in ${targetLang}
|
||||
{"speaker": "Host", "text": "..."}
|
||||
]
|
||||
|
||||
Do not include markdown formatting like \`\`\`json. Just the raw JSON.
|
||||
`;
|
||||
|
||||
try {
|
||||
const rawContent = await this.chatService.generateSimpleChat(
|
||||
[{ role: 'user', content: prompt }],
|
||||
userId,
|
||||
);
|
||||
|
||||
// Clean up code blocks if present
|
||||
const jsonString = rawContent
|
||||
.replace(/```json/g, '')
|
||||
.replace(/```/g, '')
|
||||
.trim();
|
||||
|
||||
try {
|
||||
return JSON.parse(jsonString);
|
||||
} catch (e) {
|
||||
this.logger.error('Failed to parse podcast script JSON:', rawContent);
|
||||
throw new Error(this.i18nService.getMessage('scriptGenerationFailed'));
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to generate script:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async generateAudioInternal(
|
||||
script: any[],
|
||||
outputPath: string,
|
||||
language: string = 'zh',
|
||||
) {
|
||||
const { spawn } = await import('child_process');
|
||||
const writeStream = fs.createWriteStream(outputPath);
|
||||
|
||||
// Voice map
|
||||
const voices = {
|
||||
zh: { host: 'zh-CN-YunxiNeural', guest: 'zh-CN-XiaoxiaoNeural' },
|
||||
en: { host: 'en-US-AndrewNeural', guest: 'en-US-AvaNeural' },
|
||||
ja: { host: 'ja-JP-KeitaNeural', guest: 'ja-JP-NanamiNeural' },
|
||||
};
|
||||
|
||||
const voiceConfig = voices[language] || voices['zh'];
|
||||
|
||||
for (const line of script) {
|
||||
if (!line.text) continue;
|
||||
|
||||
// Voice selection
|
||||
const voice =
|
||||
line.speaker === 'Host' ? voiceConfig.host : voiceConfig.guest;
|
||||
const tempPath = path.join(this.outputDir, `temp_${uuidv4()}.mp3`);
|
||||
|
||||
try {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const process = spawn(this.pythonPath, [
|
||||
this.scriptPath,
|
||||
'--text',
|
||||
line.text,
|
||||
'--voice',
|
||||
voice,
|
||||
'--output',
|
||||
tempPath,
|
||||
]);
|
||||
|
||||
let errorOutput = '';
|
||||
|
||||
process.stderr.on('data', (data) => {
|
||||
errorOutput += data.toString();
|
||||
});
|
||||
|
||||
process.on('close', (code) => {
|
||||
if (code !== 0) {
|
||||
reject(
|
||||
new Error(
|
||||
`Python script exited with code ${code}: ${errorOutput}`,
|
||||
),
|
||||
);
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
if (fs.existsSync(tempPath)) {
|
||||
const buffer = fs.readFileSync(tempPath);
|
||||
writeStream.write(buffer);
|
||||
fs.unlinkSync(tempPath);
|
||||
}
|
||||
} catch (e) {
|
||||
this.logger.error(`TTS Error for line: ${line.text}`, e);
|
||||
}
|
||||
}
|
||||
|
||||
writeStream.end();
|
||||
await new Promise((resolve, reject) => {
|
||||
writeStream.on('finish', resolve);
|
||||
writeStream.on('error', reject);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
import { Module, forwardRef } from '@nestjs/common';
|
||||
import { RagService } from './rag.service';
|
||||
import { ElasticsearchModule } from '../elasticsearch/elasticsearch.module';
|
||||
import { EmbeddingService } from '../knowledge-base/embedding.service';
|
||||
import { ModelConfigModule } from '../model-config/model-config.module';
|
||||
import { TenantModule } from '../tenant/tenant.module';
|
||||
import { UserModule } from '../user/user.module';
|
||||
|
||||
import { RerankService } from './rerank.service';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
forwardRef(() => ElasticsearchModule),
|
||||
ModelConfigModule,
|
||||
TenantModule,
|
||||
UserModule,
|
||||
],
|
||||
providers: [RagService, EmbeddingService, RerankService],
|
||||
exports: [RagService],
|
||||
})
|
||||
export class RagModule {}
|
||||
@@ -0,0 +1,173 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { ElasticsearchService } from '../elasticsearch/elasticsearch.service';
|
||||
import { EmbeddingService } from '../knowledge-base/embedding.service';
|
||||
import { RerankService } from './rerank.service';
|
||||
import { I18nService } from '../i18n/i18n.service';
|
||||
import { DEFAULT_LANGUAGE } from '../common/constants';
|
||||
|
||||
export interface RagSearchResult {
|
||||
id: string;
|
||||
score: number;
|
||||
content: string;
|
||||
fileId: string;
|
||||
fileName: string;
|
||||
title: string;
|
||||
chunkIndex: number;
|
||||
startPosition?: number;
|
||||
endPosition?: number;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class RagService {
|
||||
private readonly logger = new Logger(RagService.name);
|
||||
|
||||
constructor(
|
||||
private readonly elasticsearchService: ElasticsearchService,
|
||||
private readonly embeddingService: EmbeddingService,
|
||||
private readonly rerankService: RerankService,
|
||||
private readonly i18nService: I18nService,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Main research method for RAG.
|
||||
* Supports hybrid search, reranking, query expansion, and HyDE.
|
||||
*/
|
||||
async searchKnowledge(
|
||||
query: string,
|
||||
userId: string,
|
||||
topK: number = 5,
|
||||
similarityThreshold: number = 0.0,
|
||||
embeddingModelId?: string,
|
||||
enableFullTextSearch: boolean = true,
|
||||
enableRerank: boolean = false,
|
||||
rerankModelId?: string,
|
||||
selectedGroups?: string[],
|
||||
selectedFiles?: string[],
|
||||
rerankThreshold: number = 0.0,
|
||||
tenantId?: string,
|
||||
enableQueryExpansion: boolean = false,
|
||||
enableHyDE: boolean = false,
|
||||
language: string = DEFAULT_LANGUAGE,
|
||||
): Promise<RagSearchResult[]> {
|
||||
this.logger.log(
|
||||
`RAG Search: query="${query}", rerank=${enableRerank}, tenantId=${tenantId}`,
|
||||
);
|
||||
|
||||
// 1. Get embedding for the query if needed
|
||||
let queryVector: number[] = [];
|
||||
if (embeddingModelId) {
|
||||
try {
|
||||
const vectors = await this.embeddingService.getEmbeddings(
|
||||
[query],
|
||||
embeddingModelId,
|
||||
);
|
||||
queryVector = vectors[0];
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to generate query embedding', error);
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Perform search via Elasticsearch
|
||||
// Note: ElasticsearchService.hybridSearch supports explicitFileIds
|
||||
const searchResults = await this.elasticsearchService.hybridSearch(
|
||||
queryVector,
|
||||
query,
|
||||
userId,
|
||||
topK * 2, // Get more results for reranking
|
||||
0.7, // Default vector weight
|
||||
selectedGroups,
|
||||
selectedFiles,
|
||||
tenantId,
|
||||
);
|
||||
|
||||
let finalResults = searchResults;
|
||||
|
||||
// 3. Apply Threshold
|
||||
finalResults = finalResults.filter((r) => r.score >= similarityThreshold);
|
||||
|
||||
// 4. Perform Reranking if enabled
|
||||
if (enableRerank && rerankModelId && finalResults.length > 0) {
|
||||
try {
|
||||
// Map search results to string array for reranking
|
||||
const documentTexts = finalResults.map((r) => r.content);
|
||||
|
||||
const rerankedPairs = await this.rerankService.rerank(
|
||||
query,
|
||||
documentTexts,
|
||||
userId,
|
||||
rerankModelId,
|
||||
topK,
|
||||
tenantId,
|
||||
);
|
||||
|
||||
// Map reranked results back to RagSearchResult
|
||||
finalResults = rerankedPairs
|
||||
.filter((pair) => pair.score >= rerankThreshold)
|
||||
.map((pair) => {
|
||||
const originalResult = finalResults[pair.index];
|
||||
return {
|
||||
...originalResult,
|
||||
score: pair.score, // Update with rerank score
|
||||
};
|
||||
});
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
'Reranking failed, falling back to original results',
|
||||
error,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// 5. Final Slice
|
||||
return finalResults.slice(0, topK);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract unique document names as sources
|
||||
*/
|
||||
extractSources(results: RagSearchResult[]): string[] {
|
||||
const sources = new Set<string>();
|
||||
results.forEach((r) => {
|
||||
if (r.fileName) sources.add(r.fileName);
|
||||
});
|
||||
return Array.from(sources);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the RAG prompt with instructions in the correct language
|
||||
*/
|
||||
buildRagPrompt(
|
||||
query: string,
|
||||
searchResults: RagSearchResult[],
|
||||
language: string = DEFAULT_LANGUAGE,
|
||||
): string {
|
||||
const normalizedLang = this.i18nService.normalizeLanguage(language);
|
||||
|
||||
// Build context
|
||||
let context = '';
|
||||
if (searchResults.length === 0) {
|
||||
context =
|
||||
normalizedLang === 'zh'
|
||||
? '未找到相关信息。'
|
||||
: normalizedLang === 'ja'
|
||||
? '関連情報が見つかりませんでした。'
|
||||
: 'No relevant information found.';
|
||||
} else {
|
||||
searchResults.forEach((result, index) => {
|
||||
context += `[${index + 1}] File: ${result.fileName} (Score: ${result.score.toFixed(4)})\nContent: ${result.content}\n\n`;
|
||||
});
|
||||
}
|
||||
|
||||
// Get localized prompt template from I18nService
|
||||
const promptTemplate = this.i18nService.getPrompt(
|
||||
language,
|
||||
'withContext',
|
||||
false,
|
||||
);
|
||||
|
||||
return promptTemplate
|
||||
.replace('{context}', context)
|
||||
.replace('{history}', '') // History placeholders are usually handled in ChatService
|
||||
.replace('{question}', query);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { ModelConfigService } from '../model-config/model-config.service';
|
||||
import { ModelType } from '../types';
|
||||
import axios from 'axios';
|
||||
|
||||
export interface RerankResult {
|
||||
index: number;
|
||||
relevance_score: number;
|
||||
document?: string; // Optional, some APIs return it
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class RerankService {
|
||||
private readonly logger = new Logger(RerankService.name);
|
||||
|
||||
constructor(
|
||||
private modelConfigService: ModelConfigService,
|
||||
private configService: ConfigService,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Execute reranking
|
||||
* @param query User query
|
||||
* @param documents Candidate document list
|
||||
* @param userId User ID
|
||||
* @param rerankModelId Selected rerank model config ID
|
||||
* @param topN Number of results to return (top N)
|
||||
*/
|
||||
async rerank(
|
||||
query: string,
|
||||
documents: string[],
|
||||
userId: string,
|
||||
rerankModelId: string,
|
||||
topN: number = 5,
|
||||
tenantId?: string,
|
||||
): Promise<{ index: number; score: number }[]> {
|
||||
if (!documents || documents.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
let modelConfig;
|
||||
try {
|
||||
// 1. Get model config
|
||||
modelConfig = await this.modelConfigService.findOne(rerankModelId);
|
||||
|
||||
if (!modelConfig || modelConfig.type !== ModelType.RERANK) {
|
||||
this.logger.warn(`Invalid rerank model config: ${rerankModelId}`);
|
||||
// Fallback: return original order with dummy scores
|
||||
return documents.map((_, index) => ({
|
||||
index,
|
||||
score: 0.99 - index * 0.01,
|
||||
}));
|
||||
}
|
||||
|
||||
const apiKey = modelConfig.apiKey;
|
||||
const baseUrl = modelConfig.baseUrl || ''; // e.g. https://api.siliconflow.cn/v1
|
||||
const modelName = modelConfig.modelId; // e.g. BAAI/bge-reranker-v2-m3
|
||||
|
||||
this.logger.log(
|
||||
`Reranking ${documents.length} docs with model ${modelName} at ${baseUrl}`,
|
||||
);
|
||||
|
||||
// 2. Build API request (OpenAI/SiliconFlow compatible Rerank API)
|
||||
// Note: Standard OpenAI API does not have /rerank, but SiliconFlow/Jina/Cohere use similar structure
|
||||
// SiliconFlow format: POST /v1/rerank { model, query, documents, top_n }
|
||||
|
||||
const endpoint = baseUrl.replace(/\/+$/, '');
|
||||
|
||||
// Log the exact endpoint being called
|
||||
this.logger.log(`Calling Rerank API: ${endpoint} (Model: ${modelName})`);
|
||||
|
||||
const response = await axios.post(
|
||||
endpoint,
|
||||
{
|
||||
model: modelName,
|
||||
query: query,
|
||||
documents: documents,
|
||||
top_n: topN,
|
||||
return_documents: false, // We only need indices and scores
|
||||
},
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
timeout: 10000,
|
||||
},
|
||||
);
|
||||
|
||||
// 3. Parse response
|
||||
// Expected response format (SiliconFlow/Cohere):
|
||||
// { results: [ { index: 0, relevance_score: 0.98 }, ... ] }
|
||||
|
||||
if (response.data && response.data.results) {
|
||||
const results = response.data.results as RerankResult[];
|
||||
return results
|
||||
.map((r) => ({
|
||||
index: r.index,
|
||||
score: r.relevance_score,
|
||||
}))
|
||||
.sort((a, b) => b.score - a.score); // Ensure sorted
|
||||
} else {
|
||||
this.logger.error(
|
||||
'Unexpected rerank API response structure',
|
||||
response.data,
|
||||
);
|
||||
return documents.map((_, index) => ({ index, score: 0 }));
|
||||
}
|
||||
} catch (error) {
|
||||
let errorMessage = error.message;
|
||||
if (
|
||||
error.code === 'EPROTO' ||
|
||||
error.message.includes('wrong version number')
|
||||
) {
|
||||
errorMessage = `${error.message}. This often happens when using HTTPS to connect to an HTTP server. Please check your model Base URL protocol (http vs https).`;
|
||||
} else if (error.response?.status === 404) {
|
||||
const endpoint = modelConfig?.baseUrl?.replace(/\/+$/, '');
|
||||
errorMessage = `Endpoint not found (404). Tried: ${endpoint}. Please check if your Base URL is correct. (Note: We use the Base URL exactly as provided for Rerank models).`;
|
||||
}
|
||||
this.logger.error(`Rerank failed: ${errorMessage}`, error.response?.data);
|
||||
|
||||
// Fallback on error: return original order
|
||||
return documents.map((_, index) => ({ index, score: 0 }));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
import {
|
||||
Entity,
|
||||
PrimaryGeneratedColumn,
|
||||
Column,
|
||||
CreateDateColumn,
|
||||
ManyToOne,
|
||||
JoinColumn,
|
||||
} from 'typeorm';
|
||||
import { SearchHistory } from './search-history.entity';
|
||||
import { Tenant } from '../tenant/tenant.entity';
|
||||
|
||||
@Entity('chat_messages')
|
||||
export class ChatMessage {
|
||||
@PrimaryGeneratedColumn('uuid')
|
||||
id: string;
|
||||
|
||||
@Column({ name: 'search_history_id' })
|
||||
searchHistoryId: string;
|
||||
|
||||
@Column({ name: 'tenant_id', nullable: true, type: 'text' })
|
||||
tenantId: string;
|
||||
|
||||
@Column()
|
||||
role: 'user' | 'assistant';
|
||||
|
||||
@Column('text')
|
||||
content: string;
|
||||
|
||||
@Column({ nullable: true })
|
||||
sources?: string; // JSON string
|
||||
|
||||
@CreateDateColumn({ name: 'created_at' })
|
||||
createdAt: Date;
|
||||
|
||||
@ManyToOne(() => SearchHistory, (history) => history.messages, {
|
||||
onDelete: 'CASCADE',
|
||||
})
|
||||
@JoinColumn({ name: 'search_history_id' })
|
||||
searchHistory: SearchHistory;
|
||||
|
||||
@ManyToOne(() => Tenant)
|
||||
@JoinColumn({ name: 'tenant_id' })
|
||||
tenant: Tenant;
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
import {
|
||||
Controller,
|
||||
Get,
|
||||
Post,
|
||||
Delete,
|
||||
Body,
|
||||
Param,
|
||||
Query,
|
||||
UseGuards,
|
||||
Request,
|
||||
} from '@nestjs/common';
|
||||
import { CombinedAuthGuard } from '../auth/combined-auth.guard';
|
||||
import { SearchHistoryService } from './search-history.service';
|
||||
import { I18nService } from '../i18n/i18n.service';
|
||||
|
||||
@Controller('search-history')
|
||||
@UseGuards(CombinedAuthGuard)
|
||||
export class SearchHistoryController {
|
||||
constructor(
|
||||
private readonly searchHistoryService: SearchHistoryService,
|
||||
private readonly i18nService: I18nService,
|
||||
) {}
|
||||
|
||||
@Get()
|
||||
async findAll(
|
||||
@Request() req,
|
||||
@Query('page') page: string = '1',
|
||||
@Query('limit') limit: string = '20',
|
||||
) {
|
||||
const pageNum = parseInt(page, 10) || 1;
|
||||
const limitNum = parseInt(limit, 10) || 20;
|
||||
|
||||
return await this.searchHistoryService.findAll(
|
||||
req.user.id,
|
||||
req.user.tenantId,
|
||||
pageNum,
|
||||
limitNum,
|
||||
);
|
||||
}
|
||||
|
||||
@Get(':id')
|
||||
async findOne(@Param('id') id: string, @Request() req) {
|
||||
return await this.searchHistoryService.findOne(
|
||||
id,
|
||||
req.user.id,
|
||||
req.user.tenantId,
|
||||
);
|
||||
}
|
||||
|
||||
@Post()
|
||||
async create(
|
||||
@Body() body: { title: string; selectedGroups?: string[] },
|
||||
@Request() req,
|
||||
) {
|
||||
const history = await this.searchHistoryService.create(
|
||||
req.user.id,
|
||||
req.user.tenantId,
|
||||
body.title,
|
||||
body.selectedGroups,
|
||||
);
|
||||
return { id: history.id };
|
||||
}
|
||||
|
||||
@Delete(':id')
|
||||
async remove(@Param('id') id: string, @Request() req) {
|
||||
await this.searchHistoryService.remove(id, req.user.id, req.user.tenantId);
|
||||
return { message: this.i18nService.getMessage('searchHistoryDeleted') };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
import {
|
||||
Entity,
|
||||
PrimaryGeneratedColumn,
|
||||
Column,
|
||||
CreateDateColumn,
|
||||
UpdateDateColumn,
|
||||
OneToMany,
|
||||
} from 'typeorm';
|
||||
import { ChatMessage } from './chat-message.entity';
|
||||
|
||||
@Entity('search_history')
|
||||
export class SearchHistory {
|
||||
@PrimaryGeneratedColumn('uuid')
|
||||
id: string;
|
||||
|
||||
@Column({ name: 'user_id' })
|
||||
userId: string;
|
||||
|
||||
@Column({ name: 'tenant_id', nullable: true })
|
||||
tenantId: string;
|
||||
|
||||
@Column()
|
||||
title: string;
|
||||
|
||||
@Column({ name: 'selected_groups', nullable: true })
|
||||
selectedGroups?: string; // JSON string
|
||||
|
||||
@CreateDateColumn({ name: 'created_at' })
|
||||
createdAt: Date;
|
||||
|
||||
@UpdateDateColumn({ name: 'updated_at' })
|
||||
updatedAt: Date;
|
||||
|
||||
@OneToMany(() => ChatMessage, (message) => message.searchHistory)
|
||||
messages: ChatMessage[];
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
import { SearchHistory } from './search-history.entity';
|
||||
import { ChatMessage } from './chat-message.entity';
|
||||
import { SearchHistoryService } from './search-history.service';
|
||||
import { SearchHistoryController } from './search-history.controller';
|
||||
|
||||
@Module({
|
||||
imports: [TypeOrmModule.forFeature([SearchHistory, ChatMessage])],
|
||||
controllers: [SearchHistoryController],
|
||||
providers: [SearchHistoryService],
|
||||
exports: [SearchHistoryService],
|
||||
})
|
||||
export class SearchHistoryModule {}
|
||||
@@ -0,0 +1,197 @@
|
||||
import { Injectable, NotFoundException } from '@nestjs/common';
|
||||
import { I18nService } from '../i18n/i18n.service';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Repository } from 'typeorm';
|
||||
import { SearchHistory } from './search-history.entity';
|
||||
import { ChatMessage } from './chat-message.entity';
|
||||
|
||||
export interface SearchHistoryItem {
|
||||
id: string;
|
||||
title: string;
|
||||
selectedGroups: string[] | null;
|
||||
messageCount: number;
|
||||
lastMessageAt: string;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
export interface SearchHistoryDetail {
|
||||
id: string;
|
||||
title: string;
|
||||
selectedGroups: string[] | null;
|
||||
messages: Array<{
|
||||
id: string;
|
||||
role: 'user' | 'assistant';
|
||||
content: string;
|
||||
sources?: any[];
|
||||
createdAt: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
export interface PaginatedSearchHistory {
|
||||
histories: SearchHistoryItem[];
|
||||
total: number;
|
||||
page: number;
|
||||
limit: number;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class SearchHistoryService {
|
||||
constructor(
|
||||
@InjectRepository(SearchHistory)
|
||||
private searchHistoryRepository: Repository<SearchHistory>,
|
||||
@InjectRepository(ChatMessage)
|
||||
private chatMessageRepository: Repository<ChatMessage>,
|
||||
private i18nService: I18nService,
|
||||
) {}
|
||||
|
||||
async findAll(
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
page: number = 1,
|
||||
limit: number = 20,
|
||||
): Promise<PaginatedSearchHistory> {
|
||||
const [histories, total] = await this.searchHistoryRepository.findAndCount({
|
||||
where: { userId, tenantId },
|
||||
order: { updatedAt: 'DESC' },
|
||||
skip: (page - 1) * limit,
|
||||
take: limit,
|
||||
});
|
||||
|
||||
const items: SearchHistoryItem[] = await Promise.all(
|
||||
histories.map(async (history) => {
|
||||
const messageCount = await this.chatMessageRepository.count({
|
||||
where: { searchHistoryId: history.id },
|
||||
});
|
||||
|
||||
const lastMessage = await this.chatMessageRepository.findOne({
|
||||
where: { searchHistoryId: history.id },
|
||||
order: { createdAt: 'DESC' },
|
||||
});
|
||||
|
||||
return {
|
||||
id: history.id,
|
||||
title: history.title,
|
||||
selectedGroups: history.selectedGroups
|
||||
? JSON.parse(history.selectedGroups)
|
||||
: null,
|
||||
messageCount,
|
||||
lastMessageAt:
|
||||
lastMessage?.createdAt.toISOString() ||
|
||||
history.createdAt.toISOString(),
|
||||
createdAt: history.createdAt.toISOString(),
|
||||
};
|
||||
}),
|
||||
);
|
||||
|
||||
return {
|
||||
histories: items,
|
||||
total,
|
||||
page,
|
||||
limit,
|
||||
};
|
||||
}
|
||||
|
||||
async findOne(
|
||||
id: string,
|
||||
userId: string,
|
||||
tenantId?: string,
|
||||
): Promise<SearchHistoryDetail> {
|
||||
const whereClause: any = { id, userId };
|
||||
if (tenantId) {
|
||||
whereClause.tenantId = tenantId;
|
||||
}
|
||||
const history = await this.searchHistoryRepository.findOne({
|
||||
where: whereClause,
|
||||
relations: ['messages'],
|
||||
order: { messages: { createdAt: 'ASC' } },
|
||||
});
|
||||
|
||||
if (!history) {
|
||||
throw new NotFoundException(
|
||||
this.i18nService.getMessage('conversationHistoryNotFound'),
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
id: history.id,
|
||||
title: history.title,
|
||||
selectedGroups: history.selectedGroups
|
||||
? JSON.parse(history.selectedGroups)
|
||||
: null,
|
||||
messages: history.messages.map((message) => ({
|
||||
id: message.id,
|
||||
role: message.role,
|
||||
content: message.content,
|
||||
sources: message.sources ? JSON.parse(message.sources) : undefined,
|
||||
createdAt: message.createdAt.toISOString(),
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
async create(
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
title: string,
|
||||
selectedGroups?: string[],
|
||||
): Promise<SearchHistory> {
|
||||
const history = this.searchHistoryRepository.create({
|
||||
userId,
|
||||
tenantId,
|
||||
title: title.length > 50 ? title.substring(0, 50) + '...' : title,
|
||||
selectedGroups: selectedGroups
|
||||
? JSON.stringify(selectedGroups)
|
||||
: undefined,
|
||||
});
|
||||
|
||||
return await this.searchHistoryRepository.save(history);
|
||||
}
|
||||
|
||||
async addMessage(
|
||||
historyId: string,
|
||||
role: 'user' | 'assistant',
|
||||
content: string,
|
||||
sources?: any[],
|
||||
): Promise<ChatMessage> {
|
||||
const message = this.chatMessageRepository.create({
|
||||
searchHistoryId: historyId,
|
||||
role,
|
||||
content,
|
||||
sources: sources ? JSON.stringify(sources) : undefined,
|
||||
});
|
||||
|
||||
const savedMessage = await this.chatMessageRepository.save(message);
|
||||
|
||||
// Update history record update time
|
||||
await this.searchHistoryRepository.update(historyId, {
|
||||
updatedAt: new Date(),
|
||||
});
|
||||
|
||||
return savedMessage;
|
||||
}
|
||||
|
||||
async remove(id: string, userId: string, tenantId: string): Promise<void> {
|
||||
const history = await this.searchHistoryRepository.findOne({
|
||||
where: { id, userId, tenantId },
|
||||
});
|
||||
|
||||
if (!history) {
|
||||
throw new NotFoundException(
|
||||
this.i18nService.getMessage('conversationHistoryNotFound'),
|
||||
);
|
||||
}
|
||||
|
||||
await this.searchHistoryRepository.remove(history);
|
||||
}
|
||||
|
||||
async updateTitle(
|
||||
id: string,
|
||||
title: string,
|
||||
tenantId?: string,
|
||||
): Promise<void> {
|
||||
const whereClause: any = { id };
|
||||
if (tenantId) {
|
||||
whereClause.tenantId = tenantId;
|
||||
}
|
||||
await this.searchHistoryRepository.update(whereClause, { title });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
import {
|
||||
Controller,
|
||||
Get,
|
||||
Post,
|
||||
Put,
|
||||
Body,
|
||||
UseGuards,
|
||||
Param,
|
||||
Delete,
|
||||
HttpCode,
|
||||
Patch,
|
||||
ForbiddenException,
|
||||
Query,
|
||||
} from '@nestjs/common';
|
||||
import { SuperAdminService } from './super-admin.service';
|
||||
import { TenantService } from '../tenant/tenant.service';
|
||||
import { CombinedAuthGuard } from '../auth/combined-auth.guard';
|
||||
import { RolesGuard } from '../auth/roles.guard';
|
||||
import { Roles } from '../auth/roles.decorator';
|
||||
import { UserRole } from '../user/user-role.enum';
|
||||
import { I18nService } from '../i18n/i18n.service';
|
||||
|
||||
@Controller('v1/tenants')
|
||||
@UseGuards(CombinedAuthGuard, RolesGuard)
|
||||
@Roles(UserRole.SUPER_ADMIN)
|
||||
export class SuperAdminController {
|
||||
constructor(
|
||||
private readonly superAdminService: SuperAdminService,
|
||||
private readonly tenantService: TenantService,
|
||||
private readonly i18nService: I18nService,
|
||||
) {}
|
||||
|
||||
@Get()
|
||||
async getTenants(
|
||||
@Query('page') page?: string,
|
||||
@Query('limit') limit?: string,
|
||||
) {
|
||||
return this.superAdminService.getAllTenants(
|
||||
page ? parseInt(page) : undefined,
|
||||
limit ? parseInt(limit) : undefined,
|
||||
);
|
||||
}
|
||||
|
||||
@Post()
|
||||
async createTenant(
|
||||
@Body()
|
||||
body: {
|
||||
name: string;
|
||||
domain?: string;
|
||||
adminUserId?: string;
|
||||
parentId?: string;
|
||||
},
|
||||
) {
|
||||
return this.superAdminService.createTenant(
|
||||
body.name,
|
||||
body.domain,
|
||||
body.adminUserId,
|
||||
body.parentId,
|
||||
);
|
||||
}
|
||||
|
||||
@Put(':tenantId/admin')
|
||||
async bindTenantAdmin(
|
||||
@Param('tenantId') tenantId: string,
|
||||
@Body() body: { userId: string },
|
||||
) {
|
||||
return this.superAdminService.assignUserToTenant(body.userId, tenantId);
|
||||
}
|
||||
|
||||
@Post(':tenantId/admin/new')
|
||||
async createTenantAdmin(
|
||||
@Param('tenantId') tenantId: string,
|
||||
@Body() body: { username: string; password?: string },
|
||||
) {
|
||||
return this.superAdminService.createTenantAdmin(
|
||||
tenantId,
|
||||
body.username,
|
||||
body.password,
|
||||
);
|
||||
}
|
||||
|
||||
@Put(':tenantId')
|
||||
async updateTenant(
|
||||
@Param('tenantId') tenantId: string,
|
||||
@Body() body: { name?: string; domain?: string; parentId?: string },
|
||||
) {
|
||||
return this.superAdminService.updateTenant(tenantId, body);
|
||||
}
|
||||
|
||||
@Delete(':tenantId')
|
||||
async deleteTenant(@Param('tenantId') tenantId: string) {
|
||||
return this.superAdminService.deleteTenant(tenantId);
|
||||
}
|
||||
|
||||
// --- Member Management ---
|
||||
|
||||
@Get(':tenantId/members')
|
||||
async getMembers(
|
||||
@Param('tenantId') tenantId: string,
|
||||
@Query('page') page?: string,
|
||||
@Query('limit') limit?: string,
|
||||
) {
|
||||
const p = page ? parseInt(page) : undefined;
|
||||
const l = limit ? parseInt(limit) : undefined;
|
||||
return this.tenantService.getMembers(tenantId, p, l);
|
||||
}
|
||||
|
||||
@Post(':tenantId/members')
|
||||
async addMember(
|
||||
@Param('tenantId') tenantId: string,
|
||||
@Body() body: { userId: string; role?: string },
|
||||
) {
|
||||
return this.tenantService.addMember(tenantId, body.userId, body.role);
|
||||
}
|
||||
|
||||
@Delete(':tenantId/members/:userId')
|
||||
@HttpCode(204)
|
||||
async removeMember(
|
||||
@Param('tenantId') tenantId: string,
|
||||
@Param('userId') userId: string,
|
||||
) {
|
||||
await this.tenantService.removeMember(tenantId, userId);
|
||||
}
|
||||
|
||||
@Patch(':tenantId/members/:userId')
|
||||
async updateMemberRole(
|
||||
@Param('tenantId') tenantId: string,
|
||||
@Param('userId') userId: string,
|
||||
@Body() body: { role: string },
|
||||
) {
|
||||
if (body.role !== UserRole.USER && body.role !== UserRole.TENANT_ADMIN) {
|
||||
throw new ForbiddenException(
|
||||
this.i18nService.getErrorMessage('invalidMemberRole'),
|
||||
);
|
||||
}
|
||||
return this.tenantService.updateMemberRole(tenantId, userId, body.role);
|
||||
}
|
||||
|
||||
@Get(':tenantId/members/ids')
|
||||
async getMemberIds(@Param('tenantId') tenantId: string) {
|
||||
return this.tenantService.getMemberIds(tenantId);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { SuperAdminController } from './super-admin.controller';
|
||||
import { SuperAdminService } from './super-admin.service';
|
||||
import { TenantModule } from '../tenant/tenant.module';
|
||||
import { UserModule } from '../user/user.module';
|
||||
|
||||
@Module({
|
||||
imports: [TenantModule, UserModule],
|
||||
controllers: [SuperAdminController],
|
||||
providers: [SuperAdminService],
|
||||
})
|
||||
export class SuperAdminModule {}
|
||||
@@ -0,0 +1,88 @@
|
||||
import { Injectable, NotFoundException } from '@nestjs/common';
|
||||
import { TenantService } from '../tenant/tenant.service';
|
||||
import { UserService } from '../user/user.service';
|
||||
import { User } from '../user/user.entity';
|
||||
import { UserRole } from '../user/user-role.enum';
|
||||
|
||||
@Injectable()
|
||||
export class SuperAdminService {
|
||||
constructor(
|
||||
private readonly tenantService: TenantService,
|
||||
private readonly userService: UserService,
|
||||
) {}
|
||||
|
||||
async getAllTenants(page?: number, limit?: number) {
|
||||
return this.tenantService.findAll(page, limit);
|
||||
}
|
||||
|
||||
async createTenant(
|
||||
name: string,
|
||||
domain?: string,
|
||||
adminUserId?: string,
|
||||
parentId?: string,
|
||||
) {
|
||||
const tenant = await this.tenantService.create(name, domain, parentId);
|
||||
if (adminUserId) {
|
||||
await this.tenantService.addMember(
|
||||
tenant.id,
|
||||
adminUserId,
|
||||
UserRole.TENANT_ADMIN,
|
||||
);
|
||||
}
|
||||
return tenant;
|
||||
}
|
||||
|
||||
async assignUserToTenant(
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
role: UserRole = UserRole.TENANT_ADMIN,
|
||||
) {
|
||||
// Find existing members of this tenant
|
||||
const members = await this.tenantService.getMembers(tenantId);
|
||||
|
||||
// Remove existing admins from this tenant (unlinking them, not changing their role)
|
||||
for (const member of members.data) {
|
||||
if (
|
||||
member.role === UserRole.TENANT_ADMIN ||
|
||||
member.role === UserRole.SUPER_ADMIN
|
||||
) {
|
||||
await this.tenantService.removeMember(tenantId, member.userId);
|
||||
}
|
||||
}
|
||||
|
||||
// Add the new admin association for this tenant
|
||||
return this.tenantService.addMember(tenantId, userId, role);
|
||||
}
|
||||
|
||||
async createTenantAdmin(
|
||||
tenantId: string,
|
||||
username: string,
|
||||
password?: string,
|
||||
) {
|
||||
const defaultPassword = password || Math.random().toString(36).slice(-8);
|
||||
const result = await this.userService.createUser(
|
||||
username,
|
||||
defaultPassword,
|
||||
false, // isAdmin
|
||||
tenantId,
|
||||
username, // displayName
|
||||
);
|
||||
return {
|
||||
user: result.user,
|
||||
defaultPassword: defaultPassword,
|
||||
};
|
||||
}
|
||||
|
||||
async updateTenant(
|
||||
tenantId: string,
|
||||
data: { name?: string; domain?: string; parentId?: string },
|
||||
) {
|
||||
return this.tenantService.update(tenantId, data);
|
||||
}
|
||||
|
||||
async deleteTenant(tenantId: string) {
|
||||
return this.tenantService.remove(tenantId);
|
||||
}
|
||||
|
||||
// NOTE: Model Management would be added here depending on ModelService functionality
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
import {
|
||||
EntitySubscriberInterface,
|
||||
EventSubscriber,
|
||||
InsertEvent,
|
||||
UpdateEvent,
|
||||
} from 'typeorm';
|
||||
import { tenantStore } from './tenant.store';
|
||||
|
||||
@EventSubscriber()
|
||||
export class TenantEntitySubscriber implements EntitySubscriberInterface {
|
||||
/**
|
||||
* Called before entity insertion.
|
||||
*/
|
||||
beforeInsert(event: InsertEvent<any>) {
|
||||
this.injectTenantId(event.entity);
|
||||
}
|
||||
|
||||
/**
|
||||
* Called before entity update.
|
||||
*/
|
||||
beforeUpdate(event: UpdateEvent<any>) {
|
||||
this.injectTenantId(event.entity);
|
||||
}
|
||||
|
||||
private injectTenantId(entity: any) {
|
||||
if (!entity) return;
|
||||
|
||||
// Check if the entity has a tenantId property
|
||||
if ('tenantId' in entity) {
|
||||
const store = tenantStore.getStore();
|
||||
const currentTenantId = store?.tenantId;
|
||||
|
||||
// Only set if it's not already set and we have a tenantId in context
|
||||
if (!entity.tenantId && currentTenantId) {
|
||||
entity.tenantId = currentTenantId;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
import {
|
||||
Column,
|
||||
CreateDateColumn,
|
||||
Entity,
|
||||
JoinColumn,
|
||||
ManyToOne,
|
||||
PrimaryGeneratedColumn,
|
||||
UpdateDateColumn,
|
||||
} from 'typeorm';
|
||||
import { User } from '../user/user.entity';
|
||||
import { UserRole } from '../user/user-role.enum';
|
||||
import { Tenant } from './tenant.entity';
|
||||
|
||||
/**
|
||||
* Join table for User and Tenant to support Many-to-Many relationship.
|
||||
* A user can belong to multiple tenants with different roles in each.
|
||||
*/
|
||||
@Entity('tenant_members')
|
||||
export class TenantMember {
|
||||
@PrimaryGeneratedColumn('uuid')
|
||||
id: string;
|
||||
|
||||
@Column({ name: 'user_id' })
|
||||
userId: string;
|
||||
|
||||
@ManyToOne(() => User, (user) => user.tenantMembers, { onDelete: 'CASCADE' })
|
||||
@JoinColumn({ name: 'user_id' })
|
||||
user: User;
|
||||
|
||||
@Column({ name: 'tenant_id' })
|
||||
tenantId: string;
|
||||
|
||||
@ManyToOne(() => Tenant, (tenant) => tenant.members, { onDelete: 'CASCADE' })
|
||||
@JoinColumn({ name: 'tenant_id' })
|
||||
tenant: Tenant;
|
||||
|
||||
@Column({
|
||||
type: 'simple-enum',
|
||||
enum: UserRole,
|
||||
default: UserRole.USER,
|
||||
})
|
||||
role: UserRole;
|
||||
|
||||
@CreateDateColumn({ name: 'created_at' })
|
||||
createdAt: Date;
|
||||
|
||||
@UpdateDateColumn({ name: 'updated_at' })
|
||||
updatedAt: Date;
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
import {
|
||||
Column,
|
||||
CreateDateColumn,
|
||||
Entity,
|
||||
JoinColumn,
|
||||
OneToOne,
|
||||
PrimaryGeneratedColumn,
|
||||
UpdateDateColumn,
|
||||
} from 'typeorm';
|
||||
import { Tenant } from './tenant.entity';
|
||||
|
||||
/**
|
||||
* Organization-wide default settings.
|
||||
* UserSetting can still override these on a per-user basis.
|
||||
*/
|
||||
@Entity('tenant_settings')
|
||||
export class TenantSetting {
|
||||
@PrimaryGeneratedColumn('uuid')
|
||||
id: string;
|
||||
|
||||
@Column({ type: 'text' })
|
||||
tenantId: string;
|
||||
|
||||
@OneToOne(() => Tenant, { onDelete: 'CASCADE' })
|
||||
@JoinColumn({ name: 'tenantId' })
|
||||
tenant: Tenant;
|
||||
|
||||
// Default LLM model (override per user in UserSetting)
|
||||
@Column({ type: 'text', nullable: true })
|
||||
selectedLLMId: string;
|
||||
|
||||
// Default embedding model
|
||||
@Column({ type: 'text', nullable: true })
|
||||
selectedEmbeddingId: string;
|
||||
|
||||
// Default rerank model
|
||||
@Column({ type: 'text', nullable: true })
|
||||
selectedRerankId: string;
|
||||
|
||||
@Column({ type: 'text', nullable: true })
|
||||
selectedVisionId: string;
|
||||
|
||||
// Search configuration defaults
|
||||
@Column({ type: 'real', default: 0.3 })
|
||||
similarityThreshold: number;
|
||||
|
||||
@Column({ type: 'real', default: 0.5 })
|
||||
rerankSimilarityThreshold: number;
|
||||
|
||||
@Column({ type: 'integer', default: 5 })
|
||||
topK: number;
|
||||
|
||||
@Column({ type: 'boolean', default: false })
|
||||
enableFullTextSearch: boolean;
|
||||
|
||||
@Column({ type: 'boolean', default: false })
|
||||
enableRerank: boolean;
|
||||
|
||||
@Column({ type: 'real', default: 0.7 })
|
||||
hybridVectorWeight: number;
|
||||
|
||||
@Column({ type: 'boolean', default: false })
|
||||
enableQueryExpansion: boolean;
|
||||
|
||||
@Column({ type: 'boolean', default: false })
|
||||
enableHyDE: boolean;
|
||||
|
||||
@Column({ type: 'boolean', default: true })
|
||||
isNotebookEnabled: boolean;
|
||||
|
||||
@Column({ type: 'integer', default: 1000 })
|
||||
chunkSize: number;
|
||||
|
||||
@Column({ type: 'integer', default: 100 })
|
||||
chunkOverlap: number;
|
||||
|
||||
// LLM generation defaults
|
||||
@Column({ type: 'real', default: 0.7 })
|
||||
temperature: number;
|
||||
|
||||
@Column({ type: 'integer', default: 2048 })
|
||||
maxTokens: number;
|
||||
|
||||
// The model IDs that the Tenant Admin has enabled for this tenant
|
||||
@Column({ type: 'simple-array', nullable: true })
|
||||
enabledModelIds: string[];
|
||||
|
||||
@CreateDateColumn({ name: 'created_at' })
|
||||
createdAt: Date;
|
||||
|
||||
@UpdateDateColumn({ name: 'updated_at' })
|
||||
updatedAt: Date;
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
import {
|
||||
Body,
|
||||
Controller,
|
||||
Delete,
|
||||
ForbiddenException,
|
||||
Get,
|
||||
Param,
|
||||
Patch,
|
||||
Post,
|
||||
Put,
|
||||
Request,
|
||||
UseGuards,
|
||||
HttpCode,
|
||||
Query,
|
||||
} from '@nestjs/common';
|
||||
import { TenantService } from './tenant.service';
|
||||
import { CombinedAuthGuard } from '../auth/combined-auth.guard';
|
||||
import { SuperAdminGuard } from '../auth/super-admin.guard';
|
||||
|
||||
@Controller('tenants')
|
||||
@UseGuards(CombinedAuthGuard, SuperAdminGuard)
|
||||
export class TenantController {
|
||||
constructor(private readonly tenantService: TenantService) {}
|
||||
|
||||
@Get()
|
||||
findAll(@Query('page') page?: string, @Query('limit') limit?: string) {
|
||||
const p = page ? parseInt(page) : undefined;
|
||||
const l = limit ? parseInt(limit) : undefined;
|
||||
return this.tenantService.findAll(p, l);
|
||||
}
|
||||
|
||||
@Get(':id')
|
||||
findOne(@Param('id') id: string) {
|
||||
return this.tenantService.findById(id);
|
||||
}
|
||||
|
||||
@Post()
|
||||
create(@Body() body: { name: string; domain?: string; parentId?: string }) {
|
||||
return this.tenantService.create(body.name, body.domain, body.parentId);
|
||||
}
|
||||
|
||||
@Put(':id')
|
||||
update(
|
||||
@Param('id') id: string,
|
||||
@Body()
|
||||
body: {
|
||||
name?: string;
|
||||
domain?: string;
|
||||
parentId?: string;
|
||||
isActive?: boolean;
|
||||
},
|
||||
) {
|
||||
return this.tenantService.update(id, body);
|
||||
}
|
||||
|
||||
@Delete(':id')
|
||||
remove(@Param('id') id: string) {
|
||||
return this.tenantService.remove(id);
|
||||
}
|
||||
|
||||
@Get(':id/settings')
|
||||
getSettings(@Param('id') id: string) {
|
||||
return this.tenantService.getSettings(id);
|
||||
}
|
||||
|
||||
@Put(':id/settings')
|
||||
updateSettings(@Param('id') id: string, @Body() body: any) {
|
||||
return this.tenantService.updateSettings(id, body);
|
||||
}
|
||||
|
||||
@Get(':id/members')
|
||||
getMembers(
|
||||
@Param('id') id: string,
|
||||
@Query('page') page?: string,
|
||||
@Query('limit') limit?: string,
|
||||
) {
|
||||
const p = page ? parseInt(page) : undefined;
|
||||
const l = limit ? parseInt(limit) : undefined;
|
||||
return this.tenantService.getMembers(id, p, l);
|
||||
}
|
||||
|
||||
@Post(':id/members')
|
||||
addMember(
|
||||
@Param('id') id: string,
|
||||
@Body() body: { userId: string; role?: string },
|
||||
) {
|
||||
return this.tenantService.addMember(id, body.userId, body.role);
|
||||
}
|
||||
|
||||
@Patch(':id/members/:userId')
|
||||
async updateMemberRole(
|
||||
@Param('id') id: string,
|
||||
@Param('userId') userId: string,
|
||||
@Body() body: { role: string },
|
||||
) {
|
||||
return this.tenantService.updateMemberRole(id, userId, body.role);
|
||||
}
|
||||
|
||||
@Delete(':id/members/:userId')
|
||||
@HttpCode(204)
|
||||
async removeMember(@Param('id') id: string, @Param('userId') userId: string) {
|
||||
await this.tenantService.removeMember(id, userId);
|
||||
}
|
||||
|
||||
@Get(':id/members/ids')
|
||||
getMemberIds(@Param('id') id: string) {
|
||||
return this.tenantService.getMemberIds(id);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import {
|
||||
Column,
|
||||
CreateDateColumn,
|
||||
Entity,
|
||||
OneToMany,
|
||||
PrimaryGeneratedColumn,
|
||||
UpdateDateColumn,
|
||||
} from 'typeorm';
|
||||
import { User } from '../user/user.entity';
|
||||
import { TenantMember } from './tenant-member.entity';
|
||||
import { JoinColumn, ManyToOne } from 'typeorm';
|
||||
|
||||
@Entity('tenants')
|
||||
export class Tenant {
|
||||
@PrimaryGeneratedColumn('uuid')
|
||||
id: string;
|
||||
|
||||
@Column({ type: 'text', unique: true })
|
||||
name: string;
|
||||
|
||||
@Column({ type: 'text', unique: true, nullable: true })
|
||||
domain: string;
|
||||
|
||||
@Column({ type: 'boolean', default: false })
|
||||
isSystem: boolean;
|
||||
|
||||
@Column({ name: 'parent_id', type: 'text', nullable: true })
|
||||
parentId: string;
|
||||
|
||||
@ManyToOne(() => Tenant, (tenant) => tenant.children, {
|
||||
onDelete: 'SET NULL',
|
||||
})
|
||||
@JoinColumn({ name: 'parent_id' })
|
||||
parent: Tenant;
|
||||
|
||||
@OneToMany(() => Tenant, (tenant) => tenant.parent)
|
||||
children: Tenant[];
|
||||
|
||||
@OneToMany(() => TenantMember, (member) => member.tenant)
|
||||
members: TenantMember[];
|
||||
|
||||
@CreateDateColumn({ name: 'created_at' })
|
||||
createdAt: Date;
|
||||
|
||||
@UpdateDateColumn({ name: 'updated_at' })
|
||||
updatedAt: Date;
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import { Injectable, NestMiddleware } from '@nestjs/common';
|
||||
import { Request, Response, NextFunction } from 'express';
|
||||
import { tenantStore } from './tenant.store';
|
||||
|
||||
@Injectable()
|
||||
export class TenantMiddleware implements NestMiddleware {
|
||||
use(req: Request, res: Response, next: NextFunction) {
|
||||
const tenantId = req.headers['x-tenant-id'] as string;
|
||||
|
||||
// Wrap the execution in the tenant store context
|
||||
// Initial values might be empty or partial until the AuthGuard validates them
|
||||
tenantStore.run({ tenantId: tenantId || '' }, () => {
|
||||
next();
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import { Global, Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
import { Tenant } from './tenant.entity';
|
||||
import { TenantSetting } from './tenant-setting.entity';
|
||||
import { TenantMember } from './tenant-member.entity';
|
||||
import { TenantController } from './tenant.controller';
|
||||
import { TenantService } from './tenant.service';
|
||||
import { TenantEntitySubscriber } from './tenant-entity.subscriber';
|
||||
|
||||
@Global()
|
||||
@Module({
|
||||
imports: [TypeOrmModule.forFeature([Tenant, TenantSetting, TenantMember])],
|
||||
providers: [TenantService, TenantEntitySubscriber],
|
||||
controllers: [TenantController],
|
||||
exports: [TenantService],
|
||||
})
|
||||
export class TenantModule {}
|
||||
@@ -0,0 +1,321 @@
|
||||
import {
|
||||
BadRequestException,
|
||||
ForbiddenException,
|
||||
Injectable,
|
||||
NotFoundException,
|
||||
} from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Repository } from 'typeorm';
|
||||
import { Tenant } from './tenant.entity';
|
||||
import { TenantSetting } from './tenant-setting.entity';
|
||||
import { TenantMember } from './tenant-member.entity';
|
||||
import { I18nService } from '../i18n/i18n.service';
|
||||
import { UserRole } from '../user/user-role.enum';
|
||||
|
||||
@Injectable()
|
||||
export class TenantService {
|
||||
public static readonly DEFAULT_TENANT_NAME = 'Default';
|
||||
|
||||
constructor(
|
||||
@InjectRepository(Tenant)
|
||||
private readonly tenantRepository: Repository<Tenant>,
|
||||
@InjectRepository(TenantSetting)
|
||||
private readonly tenantSettingRepository: Repository<TenantSetting>,
|
||||
@InjectRepository(TenantMember)
|
||||
private readonly tenantMemberRepository: Repository<TenantMember>,
|
||||
private readonly i18nService: I18nService,
|
||||
) {}
|
||||
|
||||
async findAll(
|
||||
page?: number,
|
||||
limit?: number,
|
||||
): Promise<{ data: Tenant[]; total: number } | Tenant[]> {
|
||||
const queryBuilder = this.tenantRepository
|
||||
.createQueryBuilder('tenant')
|
||||
.leftJoinAndSelect('tenant.members', 'members')
|
||||
.leftJoinAndSelect('members.user', 'user')
|
||||
.orderBy('tenant.createdAt', 'ASC');
|
||||
|
||||
if (page !== undefined && limit !== undefined) {
|
||||
const [data, total] = await queryBuilder
|
||||
.skip((page - 1) * limit)
|
||||
.take(limit)
|
||||
.getManyAndCount();
|
||||
return { data, total };
|
||||
}
|
||||
|
||||
return queryBuilder.getMany();
|
||||
}
|
||||
|
||||
async findById(id: string): Promise<Tenant> {
|
||||
const tenant = await this.tenantRepository.findOneBy({ id });
|
||||
if (!tenant)
|
||||
throw new NotFoundException(
|
||||
this.i18nService.getMessage('tenantNotFound'),
|
||||
);
|
||||
return tenant;
|
||||
}
|
||||
|
||||
async findByName(name: string): Promise<Tenant | null> {
|
||||
return this.tenantRepository.findOneBy({ name });
|
||||
}
|
||||
|
||||
async create(
|
||||
name: string,
|
||||
domain?: string,
|
||||
parentId?: string,
|
||||
isSystem: boolean = false,
|
||||
): Promise<Tenant> {
|
||||
const tenant = this.tenantRepository.create({
|
||||
name,
|
||||
domain,
|
||||
parentId,
|
||||
isSystem,
|
||||
});
|
||||
return this.tenantRepository.save(tenant);
|
||||
}
|
||||
|
||||
async update(id: string, data: Partial<Tenant>): Promise<Tenant> {
|
||||
await this.tenantRepository.update(id, data);
|
||||
return this.findById(id);
|
||||
}
|
||||
|
||||
async remove(id: string): Promise<void> {
|
||||
await this.tenantRepository.delete(id);
|
||||
}
|
||||
|
||||
async getSettings(tenantId: string): Promise<TenantSetting | null> {
|
||||
return this.tenantSettingRepository.findOneBy({ tenantId });
|
||||
}
|
||||
|
||||
async updateSettings(
|
||||
tenantId: string,
|
||||
data: Partial<TenantSetting>,
|
||||
): Promise<TenantSetting> {
|
||||
let setting = await this.tenantSettingRepository.findOneBy({ tenantId });
|
||||
if (!setting) {
|
||||
setting = this.tenantSettingRepository.create({ tenantId, ...data });
|
||||
} else {
|
||||
if (data.enabledModelIds) {
|
||||
if (
|
||||
setting.selectedLLMId &&
|
||||
!data.enabledModelIds.includes(setting.selectedLLMId)
|
||||
) {
|
||||
data.selectedLLMId = null as any;
|
||||
}
|
||||
if (
|
||||
setting.selectedEmbeddingId &&
|
||||
!data.enabledModelIds.includes(setting.selectedEmbeddingId)
|
||||
) {
|
||||
data.selectedEmbeddingId = null as any;
|
||||
}
|
||||
if (
|
||||
setting.selectedRerankId &&
|
||||
!data.enabledModelIds.includes(setting.selectedRerankId)
|
||||
) {
|
||||
data.selectedRerankId = null as any;
|
||||
}
|
||||
if (
|
||||
setting.selectedVisionId &&
|
||||
!data.enabledModelIds.includes(setting.selectedVisionId)
|
||||
) {
|
||||
data.selectedVisionId = null as any;
|
||||
}
|
||||
}
|
||||
Object.assign(setting, data);
|
||||
}
|
||||
return this.tenantSettingRepository.save(setting);
|
||||
}
|
||||
|
||||
async updateMemberRole(
|
||||
tenantId: string,
|
||||
userId: string,
|
||||
role: string,
|
||||
): Promise<TenantMember> {
|
||||
const existing = await this.tenantMemberRepository.findOneBy({
|
||||
tenantId,
|
||||
userId,
|
||||
});
|
||||
if (!existing) {
|
||||
throw new ForbiddenException(`Member not found in this organization`);
|
||||
}
|
||||
existing.role = role as any;
|
||||
return this.tenantMemberRepository.save(existing);
|
||||
}
|
||||
|
||||
async addMember(
|
||||
tenantId: string,
|
||||
userId: string,
|
||||
role: string = 'USER',
|
||||
): Promise<TenantMember> {
|
||||
const existing = await this.tenantMemberRepository.findOneBy({
|
||||
tenantId,
|
||||
userId,
|
||||
});
|
||||
if (existing) {
|
||||
existing.role = role as any;
|
||||
return this.tenantMemberRepository.save(existing);
|
||||
}
|
||||
const member = this.tenantMemberRepository.create({
|
||||
tenantId,
|
||||
userId,
|
||||
role: role as any,
|
||||
});
|
||||
return this.tenantMemberRepository.save(member);
|
||||
}
|
||||
|
||||
async removeMember(tenantId: string, userId: string): Promise<void> {
|
||||
await this.tenantMemberRepository.delete({ tenantId, userId });
|
||||
}
|
||||
|
||||
async getMembers(
|
||||
tenantId: string,
|
||||
page?: number,
|
||||
limit?: number,
|
||||
): Promise<{ data: TenantMember[]; total: number }> {
|
||||
const queryBuilder = this.tenantMemberRepository
|
||||
.createQueryBuilder('member')
|
||||
.leftJoinAndSelect('member.user', 'user')
|
||||
.where('member.tenantId = :tenantId', { tenantId })
|
||||
.select([
|
||||
'member',
|
||||
'user.id',
|
||||
'user.username',
|
||||
'user.displayName',
|
||||
'user.isAdmin',
|
||||
])
|
||||
.orderBy('member.createdAt', 'DESC');
|
||||
|
||||
if (page !== undefined && limit !== undefined) {
|
||||
const [data, total] = await queryBuilder
|
||||
.skip((page - 1) * limit)
|
||||
.take(limit)
|
||||
.getManyAndCount();
|
||||
return { data, total };
|
||||
}
|
||||
|
||||
const [data, total] = await queryBuilder.getManyAndCount();
|
||||
return { data, total };
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure a "Default" tenant exists for data migration purposes.
|
||||
* Called during app bootstrap.
|
||||
*/
|
||||
async ensureDefaultTenant(): Promise<Tenant> {
|
||||
let defaultTenant = await this.findByName(
|
||||
TenantService.DEFAULT_TENANT_NAME,
|
||||
);
|
||||
if (!defaultTenant) {
|
||||
defaultTenant = await this.create(
|
||||
TenantService.DEFAULT_TENANT_NAME,
|
||||
'default.localhost',
|
||||
undefined,
|
||||
true,
|
||||
);
|
||||
} else if (!defaultTenant.isSystem) {
|
||||
defaultTenant.isSystem = true;
|
||||
await this.tenantRepository.save(defaultTenant);
|
||||
}
|
||||
return defaultTenant;
|
||||
}
|
||||
|
||||
async getMemberIds(tenantId: string): Promise<string[]> {
|
||||
const members = await this.tenantMemberRepository.find({
|
||||
where: { tenantId },
|
||||
select: ['userId'],
|
||||
});
|
||||
return members.map((m) => m.userId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a user can access a tenant's resources.
|
||||
* Returns true if:
|
||||
* 1. Target tenant matches user's primary/current tenantId
|
||||
* 2. User is a member of the target tenant (any role)
|
||||
* 3. (Optional/Future) User is a global SUPER_ADMIN
|
||||
*/
|
||||
async canAccessTenant(
|
||||
userId: string,
|
||||
targetTenantId: string,
|
||||
currentTenantId?: string,
|
||||
): Promise<boolean> {
|
||||
// Case 1: Direct match with current/active tenant
|
||||
if (currentTenantId && currentTenantId === targetTenantId) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Case 2: Global Super Admin check
|
||||
const user = await this.tenantMemberRepository.query(
|
||||
'SELECT isAdmin FROM users WHERE id = ?',
|
||||
[userId],
|
||||
);
|
||||
if (user && user[0] && user[0].isAdmin) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Case 3: Hierarchical check (Direct membership or Parent Admin)
|
||||
let checkTenantId = targetTenantId;
|
||||
const seenTenantIds = new Set<string>();
|
||||
|
||||
while (checkTenantId && !seenTenantIds.has(checkTenantId)) {
|
||||
seenTenantIds.add(checkTenantId);
|
||||
|
||||
const membership = await this.tenantMemberRepository.findOneBy({
|
||||
userId,
|
||||
tenantId: checkTenantId,
|
||||
});
|
||||
|
||||
if (membership) {
|
||||
// If it's the exact target tenant, any membership role works
|
||||
if (checkTenantId === targetTenantId) {
|
||||
return true;
|
||||
}
|
||||
// If it's an ancestor, user must be a TENANT_ADMIN or SUPER_ADMIN of that ancestor
|
||||
if (
|
||||
membership.role === UserRole.TENANT_ADMIN ||
|
||||
membership.role === UserRole.SUPER_ADMIN
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
// Move up to parent
|
||||
const tenant = await this.tenantRepository.findOneBy({
|
||||
id: checkTenantId,
|
||||
});
|
||||
checkTenantId = tenant?.parentId as string;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine the user's role in a specific tenant.
|
||||
* Checks for global SUPER_ADMIN status first.
|
||||
*/
|
||||
async getUserRole(userId: string, tenantId: string): Promise<UserRole> {
|
||||
// 1. Check if user is a global SUPER_ADMIN
|
||||
const user = await this.tenantRepository.query(
|
||||
'SELECT isAdmin FROM users WHERE id = ?',
|
||||
[userId],
|
||||
);
|
||||
|
||||
if (user && user[0] && user[0].isAdmin) {
|
||||
return UserRole.SUPER_ADMIN;
|
||||
}
|
||||
|
||||
// 2. Check for tenant-specific role
|
||||
const membership = await this.tenantMemberRepository.findOneBy({
|
||||
userId,
|
||||
tenantId,
|
||||
});
|
||||
|
||||
if (membership) {
|
||||
return membership.role;
|
||||
}
|
||||
|
||||
// Default to USER
|
||||
return UserRole.USER;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
import { AsyncLocalStorage } from 'async_hooks';
|
||||
|
||||
export interface TenantContext {
|
||||
tenantId: string;
|
||||
userId?: string;
|
||||
}
|
||||
|
||||
export const tenantStore = new AsyncLocalStorage<TenantContext>();
|
||||
@@ -0,0 +1,10 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { TikaService } from './tika.service';
|
||||
import { ConfigModule } from '@nestjs/config';
|
||||
|
||||
@Module({
|
||||
imports: [ConfigModule],
|
||||
providers: [TikaService],
|
||||
exports: [TikaService],
|
||||
})
|
||||
export class TikaModule {}
|
||||
@@ -0,0 +1,57 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import * as fs from 'fs';
|
||||
import { I18nService } from '../i18n/i18n.service';
|
||||
|
||||
@Injectable()
|
||||
export class TikaService {
|
||||
private readonly logger = new Logger(TikaService.name);
|
||||
private readonly tikaHost: string;
|
||||
|
||||
constructor(
|
||||
private configService: ConfigService,
|
||||
private i18nService: I18nService,
|
||||
) {
|
||||
const tikaHost = this.configService.get<string>('TIKA_HOST');
|
||||
if (!tikaHost) {
|
||||
throw new Error(this.i18nService.getMessage('tikaHostRequired'));
|
||||
}
|
||||
this.tikaHost = tikaHost;
|
||||
}
|
||||
|
||||
async extractText(filePath: string): Promise<string> {
|
||||
try {
|
||||
// Use stream instead of reading entire file into memory
|
||||
const fileStream = fs.createReadStream(filePath);
|
||||
|
||||
// Node.js native fetch supports passing a stream as body, but often requires 'duplex: "half"'
|
||||
// and checking if we need to convert to Web Stream (Readable.toWeb) depends on Node version,
|
||||
// but usually standard stream works or fs.openAsBlob (Node 20).
|
||||
// Let's try passing the stream directly with duplex option.
|
||||
|
||||
const response = await fetch(`${this.tikaHost}/tika`, {
|
||||
method: 'PUT',
|
||||
headers: {
|
||||
Accept: 'text/plain',
|
||||
},
|
||||
// @ts-expect-error - duplex is present in Node's RequestInit but strictly typed definitions might miss it
|
||||
duplex: 'half',
|
||||
body: fileStream as any,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const statusText = response.statusText;
|
||||
// Ensure stream is closed if request failed
|
||||
fileStream.destroy();
|
||||
throw new Error(
|
||||
`Tika extraction failed: ${response.status} ${statusText}`,
|
||||
);
|
||||
}
|
||||
|
||||
return await response.text();
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to extract text from ${filePath}`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
// server/src/types.ts
|
||||
|
||||
export enum ModelType {
|
||||
LLM = 'llm',
|
||||
EMBEDDING = 'embedding',
|
||||
RERANK = 'rerank',
|
||||
VISION = 'vision',
|
||||
}
|
||||
|
||||
// 1. Model Definition (The "Provider" setup)
|
||||
export interface ModelConfig {
|
||||
id: string;
|
||||
name: string; // Display name, e.g. "My DeepSeek"
|
||||
modelId: string; // The actual string ID sent to API, e.g., "deepseek-chat"
|
||||
baseUrl?: string; // Base URL for OpenAI compatible API
|
||||
apiKey?: string; // API key for the service
|
||||
type: ModelType;
|
||||
dimensions?: number;
|
||||
supportsVision?: boolean;
|
||||
maxInputTokens?: number;
|
||||
maxBatchSize?: number;
|
||||
isVectorModel?: boolean;
|
||||
providerName?: string;
|
||||
isEnabled?: boolean;
|
||||
isDefault?: boolean;
|
||||
}
|
||||
|
||||
// 2. Application Logic Settings (The "App" setup)
|
||||
export interface AppSettings {
|
||||
// References to ModelConfig IDs
|
||||
selectedLLMId: string;
|
||||
selectedEmbeddingId: string; // Default for new uploads, and used for query encoding
|
||||
selectedRerankId: string;
|
||||
|
||||
// Model Hyperparameters
|
||||
temperature: number;
|
||||
maxTokens: number;
|
||||
|
||||
// Retrieval
|
||||
enableRerank: boolean;
|
||||
topK: number;
|
||||
similarityThreshold: number;
|
||||
rerankSimilarityThreshold: number;
|
||||
enableFullTextSearch: boolean;
|
||||
hybridVectorWeight: number;
|
||||
|
||||
// Search Enhancement
|
||||
enableQueryExpansion: boolean;
|
||||
enableHyDE: boolean;
|
||||
|
||||
chunkSize: number;
|
||||
chunkOverlap: number;
|
||||
|
||||
// Language
|
||||
language: string;
|
||||
}
|
||||
@@ -0,0 +1,320 @@
|
||||
import {
|
||||
BadRequestException,
|
||||
Body,
|
||||
Controller,
|
||||
Post,
|
||||
Request,
|
||||
UploadedFile,
|
||||
UseGuards,
|
||||
UseInterceptors,
|
||||
Logger,
|
||||
} from '@nestjs/common';
|
||||
import { FileInterceptor } from '@nestjs/platform-express';
|
||||
import { UploadService } from './upload.service';
|
||||
import { KnowledgeBaseService } from '../knowledge-base/knowledge-base.service';
|
||||
import { CombinedAuthGuard } from '../auth/combined-auth.guard';
|
||||
import { RolesGuard } from '../auth/roles.guard';
|
||||
import { Roles } from '../auth/roles.decorator';
|
||||
import { UserRole } from '../user/user-role.enum';
|
||||
import { errorMessages } from '../i18n/messages';
|
||||
|
||||
import {
|
||||
DEFAULT_CHUNK_SIZE,
|
||||
MIN_CHUNK_SIZE,
|
||||
MAX_CHUNK_SIZE,
|
||||
DEFAULT_CHUNK_OVERLAP,
|
||||
DEFAULT_MAX_OVERLAP_RATIO,
|
||||
MAX_FILE_SIZE,
|
||||
DEFAULT_LANGUAGE,
|
||||
} from '../common/constants';
|
||||
import { I18nService } from '../i18n/i18n.service';
|
||||
import {
|
||||
isAllowedByExtension,
|
||||
IMAGE_MIME_TYPES,
|
||||
} from '../common/file-support.constants';
|
||||
|
||||
export interface UploadConfigDto {
|
||||
chunkSize?: string;
|
||||
chunkOverlap?: string;
|
||||
embeddingModelId?: string;
|
||||
mode?: 'fast' | 'precise'; // Processing mode
|
||||
groupIds?: string; // JSON string of group IDs
|
||||
}
|
||||
|
||||
@Controller('upload')
|
||||
@UseGuards(CombinedAuthGuard, RolesGuard)
|
||||
export class UploadController {
|
||||
private readonly logger = new Logger(UploadController.name);
|
||||
|
||||
constructor(
|
||||
private readonly uploadService: UploadService,
|
||||
private readonly knowledgeBaseService: KnowledgeBaseService,
|
||||
private readonly i18nService: I18nService,
|
||||
) {}
|
||||
|
||||
@Post('local-folder')
|
||||
@Roles(UserRole.TENANT_ADMIN, UserRole.SUPER_ADMIN)
|
||||
async importLocalFolder(
|
||||
@Request() req,
|
||||
@Body()
|
||||
body: {
|
||||
sourcePath: string;
|
||||
embeddingModelId: string;
|
||||
chunkSize?: string;
|
||||
chunkOverlap?: string;
|
||||
mode?: 'fast' | 'precise';
|
||||
useHierarchy?: boolean;
|
||||
groupIds?: string[];
|
||||
},
|
||||
) {
|
||||
if (!body.sourcePath) {
|
||||
throw new BadRequestException(
|
||||
this.i18nService.getMessage('sourcePathRequired' as any) ||
|
||||
'Source path is required',
|
||||
);
|
||||
}
|
||||
|
||||
if (!body.embeddingModelId) {
|
||||
throw new BadRequestException(
|
||||
this.i18nService.getMessage('uploadModelRequired'),
|
||||
);
|
||||
}
|
||||
|
||||
const indexingConfig = {
|
||||
chunkSize: body.chunkSize ? parseInt(body.chunkSize) : DEFAULT_CHUNK_SIZE,
|
||||
chunkOverlap: body.chunkOverlap
|
||||
? parseInt(body.chunkOverlap)
|
||||
: DEFAULT_CHUNK_OVERLAP,
|
||||
embeddingModelId: body.embeddingModelId,
|
||||
mode: body.mode || 'fast',
|
||||
useHierarchy: body.useHierarchy ?? false,
|
||||
groupIds: body.groupIds || [],
|
||||
};
|
||||
|
||||
const result = await this.uploadService.importLocalFolder(
|
||||
body.sourcePath,
|
||||
req.user.id,
|
||||
req.user.tenantId,
|
||||
indexingConfig,
|
||||
);
|
||||
|
||||
return {
|
||||
message:
|
||||
this.i18nService.getMessage('importStarted' as any) || 'Import started',
|
||||
...result,
|
||||
};
|
||||
}
|
||||
|
||||
@Post()
|
||||
@Roles(UserRole.TENANT_ADMIN, UserRole.SUPER_ADMIN)
|
||||
@UseInterceptors(
|
||||
FileInterceptor('file', {
|
||||
fileFilter: (req, file, cb) => {
|
||||
// Check by image MIME type or extension
|
||||
const isAllowed =
|
||||
IMAGE_MIME_TYPES.includes(file.mimetype) ||
|
||||
isAllowedByExtension(file.originalname);
|
||||
|
||||
if (isAllowed) {
|
||||
cb(null, true);
|
||||
} else {
|
||||
cb(
|
||||
new BadRequestException(
|
||||
(
|
||||
errorMessages[DEFAULT_LANGUAGE]?.uploadTypeUnsupported ||
|
||||
errorMessages['ja'].uploadTypeUnsupported
|
||||
).replace('{type}', file?.mimetype ?? 'unknown'),
|
||||
),
|
||||
false,
|
||||
);
|
||||
}
|
||||
},
|
||||
}),
|
||||
)
|
||||
async uploadFile(
|
||||
@UploadedFile() file: Express.Multer.File,
|
||||
@Request() req,
|
||||
@Body() config: UploadConfigDto,
|
||||
) {
|
||||
if (!file) {
|
||||
throw new BadRequestException(
|
||||
this.i18nService.getMessage('uploadNoFile'),
|
||||
);
|
||||
}
|
||||
|
||||
// Validate file size（frontend limit + backend validation）
|
||||
const maxFileSize = parseInt(
|
||||
process.env.MAX_FILE_SIZE || String(MAX_FILE_SIZE),
|
||||
); // 100MB
|
||||
if (file.size > maxFileSize) {
|
||||
throw new BadRequestException(
|
||||
this.i18nService.formatMessage('uploadSizeExceeded', {
|
||||
size: this.formatBytes(file.size),
|
||||
max: this.formatBytes(maxFileSize),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
// Validate embedding model config
|
||||
if (!config.embeddingModelId) {
|
||||
throw new BadRequestException(
|
||||
this.i18nService.getMessage('uploadModelRequired'),
|
||||
);
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`User ${req.user.id} uploaded file: ${file.originalname} (${this.formatBytes(file.size)})`,
|
||||
);
|
||||
|
||||
const fileInfo = await this.uploadService.processUploadedFile(file);
|
||||
|
||||
// Parse config parameters and set safe default values
|
||||
const indexingConfig = {
|
||||
chunkSize: config.chunkSize
|
||||
? parseInt(config.chunkSize)
|
||||
: DEFAULT_CHUNK_SIZE,
|
||||
chunkOverlap: config.chunkOverlap
|
||||
? parseInt(config.chunkOverlap)
|
||||
: DEFAULT_CHUNK_OVERLAP,
|
||||
embeddingModelId: config.embeddingModelId || null,
|
||||
groupIds: config.groupIds ? JSON.parse(config.groupIds) : [],
|
||||
};
|
||||
|
||||
// Ensure overlap <= 50% of chunk size
|
||||
if (
|
||||
indexingConfig.chunkOverlap >
|
||||
indexingConfig.chunkSize * DEFAULT_MAX_OVERLAP_RATIO
|
||||
) {
|
||||
indexingConfig.chunkOverlap = Math.floor(
|
||||
indexingConfig.chunkSize * DEFAULT_MAX_OVERLAP_RATIO,
|
||||
);
|
||||
this.logger.warn(
|
||||
this.i18nService.formatMessage('overlapAdjusted', {
|
||||
newSize: indexingConfig.chunkOverlap,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
// Save to database and trigger indexing（async）
|
||||
const kb = await this.knowledgeBaseService.createAndIndex(
|
||||
fileInfo,
|
||||
req.user.id,
|
||||
req.user.tenantId, // Ensure tenantId is passed down
|
||||
{
|
||||
...indexingConfig,
|
||||
mode: config.mode || 'fast',
|
||||
} as any, // Bypass strict type check for now or cast to correct type
|
||||
);
|
||||
|
||||
return {
|
||||
message: this.i18nService.getMessage('uploadSuccess'),
|
||||
id: kb.id,
|
||||
filename: fileInfo.filename,
|
||||
originalname: fileInfo.originalname,
|
||||
status: kb.status,
|
||||
mode: config.mode || 'fast',
|
||||
config: indexingConfig,
|
||||
estimatedChunks: Math.ceil(file.size / (indexingConfig.chunkSize * 4)), // Estimated chunk count
|
||||
};
|
||||
}
|
||||
|
||||
@Post('text')
|
||||
@Roles(UserRole.TENANT_ADMIN, UserRole.SUPER_ADMIN)
|
||||
async uploadText(
|
||||
@Request() req,
|
||||
@Body()
|
||||
body: {
|
||||
content: string;
|
||||
title: string;
|
||||
chunkSize?: string;
|
||||
chunkOverlap?: string;
|
||||
embeddingModelId?: string;
|
||||
mode?: 'fast' | 'precise';
|
||||
},
|
||||
) {
|
||||
if (!body.content || !body.title) {
|
||||
throw new BadRequestException(
|
||||
this.i18nService.getMessage('contentAndTitleRequired'),
|
||||
);
|
||||
}
|
||||
|
||||
const fs = await import('fs');
|
||||
const path = await import('path');
|
||||
const uploadPath = process.env.UPLOAD_FILE_PATH || './uploads';
|
||||
|
||||
if (!fs.existsSync(uploadPath)) {
|
||||
fs.mkdirSync(uploadPath, { recursive: true });
|
||||
}
|
||||
|
||||
const uniqueSuffix = Date.now() + '-' + Math.round(Math.random() * 1e9);
|
||||
const filename = `note-${uniqueSuffix}.md`;
|
||||
const filePath = path.join(uploadPath, filename);
|
||||
|
||||
// Save content to file (mocking a file upload)
|
||||
fs.writeFileSync(filePath, body.content, 'utf8');
|
||||
|
||||
const fileSize = Buffer.byteLength(body.content, 'utf8');
|
||||
|
||||
// Mimic Multer file info
|
||||
const fileInfo = {
|
||||
filename: filename,
|
||||
originalname: body.title.endsWith('.md')
|
||||
? body.title
|
||||
: `${body.title}.md`,
|
||||
size: fileSize,
|
||||
mimetype: 'text/markdown',
|
||||
path: filePath,
|
||||
};
|
||||
|
||||
// Validating Config
|
||||
if (!body.embeddingModelId) {
|
||||
throw new BadRequestException(
|
||||
this.i18nService.getMessage('uploadModelRequired'),
|
||||
);
|
||||
}
|
||||
|
||||
const indexingConfig = {
|
||||
chunkSize: body.chunkSize ? parseInt(body.chunkSize) : DEFAULT_CHUNK_SIZE,
|
||||
chunkOverlap: body.chunkOverlap
|
||||
? parseInt(body.chunkOverlap)
|
||||
: DEFAULT_CHUNK_OVERLAP,
|
||||
embeddingModelId: body.embeddingModelId || null,
|
||||
};
|
||||
|
||||
if (
|
||||
indexingConfig.chunkOverlap >
|
||||
indexingConfig.chunkSize * DEFAULT_MAX_OVERLAP_RATIO
|
||||
) {
|
||||
indexingConfig.chunkOverlap = Math.floor(
|
||||
indexingConfig.chunkSize * DEFAULT_MAX_OVERLAP_RATIO,
|
||||
);
|
||||
}
|
||||
|
||||
const kb = await this.knowledgeBaseService.createAndIndex(
|
||||
fileInfo,
|
||||
req.user.id,
|
||||
req.user.tenantId,
|
||||
{
|
||||
...indexingConfig,
|
||||
mode: body.mode || 'fast',
|
||||
} as any,
|
||||
);
|
||||
|
||||
return {
|
||||
message: this.i18nService.getMessage('uploadTextSuccess'),
|
||||
id: kb.id,
|
||||
filename: fileInfo.filename,
|
||||
originalname: fileInfo.originalname,
|
||||
status: kb.status,
|
||||
config: indexingConfig,
|
||||
};
|
||||
}
|
||||
|
||||
private formatBytes(bytes: number): string {
|
||||
if (bytes === 0) return '0 B';
|
||||
const k = 1024;
|
||||
const sizes = ['B', 'KB', 'MB', 'GB'];
|
||||
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
||||
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { UploadService } from './upload.service';
|
||||
import { UploadController } from './upload.controller';
|
||||
import { MulterModule } from '@nestjs/platform-express';
|
||||
import { ConfigModule, ConfigService } from '@nestjs/config';
|
||||
import { KnowledgeBaseModule } from '../knowledge-base/knowledge-base.module'; // Import KnowledgeBaseModule
|
||||
import { KnowledgeGroupModule } from '../knowledge-group/knowledge-group.module';
|
||||
import * as multer from 'multer';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { UserModule } from '../user/user.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
KnowledgeBaseModule,
|
||||
KnowledgeGroupModule,
|
||||
UserModule,
|
||||
MulterModule.registerAsync({
|
||||
imports: [ConfigModule],
|
||||
useFactory: (configService: ConfigService) => {
|
||||
const uploadPath = configService.get<string>(
|
||||
'UPLOAD_FILE_PATH',
|
||||
'./uploads',
|
||||
); // Get upload path from env varor use default './uploads' use
|
||||
|
||||
// Ensure upload directory exists
|
||||
if (!fs.existsSync(uploadPath)) {
|
||||
fs.mkdirSync(uploadPath, { recursive: true });
|
||||
}
|
||||
|
||||
// Get max file size from env var, default 100MB
|
||||
const maxFileSize = parseInt(
|
||||
configService.get<string>('MAX_FILE_SIZE', '104857600'), // 100MB in bytes
|
||||
);
|
||||
|
||||
return {
|
||||
storage: multer.diskStorage({
|
||||
destination: (req: any, file, cb) => {
|
||||
const tenantId = req.user?.tenantId || 'default';
|
||||
const fullPath = path.join(uploadPath, tenantId);
|
||||
|
||||
if (!fs.existsSync(fullPath)) {
|
||||
fs.mkdirSync(fullPath, { recursive: true });
|
||||
}
|
||||
cb(null, fullPath);
|
||||
},
|
||||
filename: (req, file, cb) => {
|
||||
// Fix Chinese filename garbling
|
||||
file.originalname = Buffer.from(
|
||||
file.originalname,
|
||||
'latin1',
|
||||
).toString('utf8');
|
||||
const uniqueSuffix =
|
||||
Date.now() + '-' + Math.round(Math.random() * 1e9);
|
||||
cb(
|
||||
null,
|
||||
`${file.fieldname}-${uniqueSuffix}${path.extname(file.originalname)}`,
|
||||
);
|
||||
},
|
||||
}),
|
||||
limits: {
|
||||
fileSize: maxFileSize, // File size limit
|
||||
},
|
||||
};
|
||||
},
|
||||
inject: [ConfigService],
|
||||
}),
|
||||
],
|
||||
controllers: [UploadController],
|
||||
providers: [UploadService],
|
||||
})
|
||||
export class UploadModule {}
|
||||
@@ -0,0 +1,246 @@
|
||||
import { Injectable, Logger, BadRequestException } from '@nestjs/common';
|
||||
import { KnowledgeBaseService } from '../knowledge-base/knowledge-base.service';
|
||||
import { KnowledgeGroupService } from '../knowledge-group/knowledge-group.service';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
@Injectable()
|
||||
export class UploadService {
|
||||
private readonly logger = new Logger(UploadService.name);
|
||||
|
||||
constructor(
|
||||
private kbService: KnowledgeBaseService,
|
||||
private groupService: KnowledgeGroupService,
|
||||
) {}
|
||||
|
||||
async processUploadedFile(file: Express.Multer.File) {
|
||||
// Add more business logic here. Example:
|
||||
// - Save file info to database
|
||||
// - Call other services to process file (Tika text extraction, ES indexing etc.)
|
||||
// - Validate file format or analyze content
|
||||
|
||||
// Currently only return basic file info
|
||||
return {
|
||||
filename: file.filename,
|
||||
originalname: file.originalname,
|
||||
size: file.size,
|
||||
mimetype: file.mimetype,
|
||||
path: file.path, // After Multer saves file, full path is in file.path
|
||||
};
|
||||
}
|
||||
|
||||
async importLocalFolder(
|
||||
sourcePath: string,
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
config: any,
|
||||
) {
|
||||
if (!fs.existsSync(sourcePath)) {
|
||||
throw new BadRequestException(`Directory not found: ${sourcePath}`);
|
||||
}
|
||||
|
||||
const stat = fs.statSync(sourcePath);
|
||||
if (!stat.isDirectory()) {
|
||||
throw new BadRequestException(`Path is not a directory: ${sourcePath}`);
|
||||
}
|
||||
|
||||
// Determine root group for hierarchy or single group
|
||||
let rootGroupId: string | null = null;
|
||||
if (config.groupIds && config.groupIds.length > 0) {
|
||||
rootGroupId = config.groupIds[0];
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`Starting local folder import: ${sourcePath} for user ${userId}, tenant ${tenantId}`,
|
||||
);
|
||||
|
||||
// Trigger scanning and processing asynchronously to not block the request
|
||||
this.executeLocalImport(
|
||||
sourcePath,
|
||||
userId,
|
||||
tenantId,
|
||||
config,
|
||||
rootGroupId,
|
||||
).catch((err) => {
|
||||
this.logger.error(`Local folder import failed for ${sourcePath}`, err);
|
||||
});
|
||||
|
||||
return {
|
||||
sourcePath,
|
||||
status: 'PROCESSING',
|
||||
};
|
||||
}
|
||||
|
||||
private async executeLocalImport(
|
||||
sourcePath: string,
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
config: any,
|
||||
rootGroupId: string | null,
|
||||
) {
|
||||
const files = this.scanDir(sourcePath);
|
||||
this.logger.log(`Found ${files.length} files in ${sourcePath}`);
|
||||
|
||||
const dirToGroupId = new Map<string, string>();
|
||||
if (rootGroupId) {
|
||||
dirToGroupId.set('.', rootGroupId);
|
||||
} else {
|
||||
// Create a root group based on folder name if none provided
|
||||
const rootName = path.basename(sourcePath);
|
||||
const rootGroup = await this.groupService.create(userId, tenantId, {
|
||||
name: rootName,
|
||||
description: `Imported from local path: ${sourcePath}`,
|
||||
});
|
||||
rootGroupId = rootGroup.id;
|
||||
dirToGroupId.set('.', rootGroupId);
|
||||
}
|
||||
|
||||
const uploadBaseDir = process.env.UPLOAD_FILE_PATH || './uploads';
|
||||
|
||||
for (const filePath of files) {
|
||||
try {
|
||||
const relativeDir = path.relative(sourcePath, path.dirname(filePath));
|
||||
const normalizedDir = relativeDir || '.';
|
||||
|
||||
let targetGroupId = rootGroupId;
|
||||
if (config.useHierarchy) {
|
||||
targetGroupId = await this.ensureHierarchy(
|
||||
userId,
|
||||
tenantId,
|
||||
normalizedDir,
|
||||
dirToGroupId,
|
||||
rootGroupId,
|
||||
);
|
||||
}
|
||||
|
||||
const filename = path.basename(filePath);
|
||||
const uniqueSuffix = Date.now() + '-' + Math.round(Math.random() * 1e9);
|
||||
const storedFilename = `local-${uniqueSuffix}-${filename}`;
|
||||
|
||||
// Ensure tenant directory exists
|
||||
const tenantDir = path.join(uploadBaseDir, tenantId);
|
||||
if (!fs.existsSync(tenantDir)) {
|
||||
fs.mkdirSync(tenantDir, { recursive: true });
|
||||
}
|
||||
|
||||
const targetPath = path.join(tenantDir, storedFilename);
|
||||
fs.copyFileSync(filePath, targetPath);
|
||||
|
||||
const stats = fs.statSync(targetPath);
|
||||
const fileInfo = {
|
||||
filename: storedFilename,
|
||||
originalname: filename,
|
||||
path: targetPath,
|
||||
size: stats.size,
|
||||
mimetype: this.getMimeType(filename),
|
||||
};
|
||||
|
||||
await this.kbService.createAndIndex(fileInfo, userId, tenantId, {
|
||||
...config,
|
||||
groupIds: [targetGroupId],
|
||||
});
|
||||
} catch (err) {
|
||||
this.logger.error(`Failed to process local file: ${filePath}`, err);
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.log(`Local folder import completed: ${sourcePath}`);
|
||||
}
|
||||
|
||||
private async ensureHierarchy(
|
||||
userId: string,
|
||||
tenantId: string,
|
||||
relativeDir: string,
|
||||
dirToGroupId: Map<string, string>,
|
||||
rootGroupId: string,
|
||||
): Promise<string> {
|
||||
if (dirToGroupId.has(relativeDir)) {
|
||||
return dirToGroupId.get(relativeDir)!;
|
||||
}
|
||||
|
||||
const segments = relativeDir.split(path.sep);
|
||||
let currentPath = '';
|
||||
let parentId = rootGroupId;
|
||||
|
||||
for (const segment of segments) {
|
||||
if (!segment || segment === '.') continue;
|
||||
currentPath = currentPath ? path.join(currentPath, segment) : segment;
|
||||
|
||||
if (dirToGroupId.has(currentPath)) {
|
||||
parentId = dirToGroupId.get(currentPath)!;
|
||||
continue;
|
||||
}
|
||||
|
||||
const group = await this.groupService.findOrCreate(
|
||||
userId,
|
||||
tenantId,
|
||||
segment,
|
||||
parentId,
|
||||
`Sub-folder from local import: ${currentPath}`,
|
||||
);
|
||||
dirToGroupId.set(currentPath, group.id);
|
||||
parentId = group.id;
|
||||
}
|
||||
|
||||
return parentId;
|
||||
}
|
||||
|
||||
private scanDir(directory: string): string[] {
|
||||
let results: string[] = [];
|
||||
if (!fs.existsSync(directory)) return results;
|
||||
|
||||
const items = fs.readdirSync(directory);
|
||||
for (const item of items) {
|
||||
const fullPath = path.join(directory, item);
|
||||
const stat = fs.statSync(fullPath);
|
||||
if (stat.isDirectory()) {
|
||||
results = results.concat(this.scanDir(fullPath));
|
||||
} else {
|
||||
// Only include supported document and code extensions
|
||||
const ext = path.extname(item).toLowerCase().slice(1);
|
||||
if (
|
||||
[
|
||||
'pdf',
|
||||
'doc',
|
||||
'docx',
|
||||
'xls',
|
||||
'xlsx',
|
||||
'ppt',
|
||||
'pptx',
|
||||
'rtf',
|
||||
'csv',
|
||||
'txt',
|
||||
'md',
|
||||
'html',
|
||||
'json',
|
||||
'xml',
|
||||
'js',
|
||||
'ts',
|
||||
'py',
|
||||
'java',
|
||||
'sql',
|
||||
].includes(ext)
|
||||
) {
|
||||
results.push(fullPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
private getMimeType(filename: string): string {
|
||||
const ext = path.extname(filename).toLowerCase();
|
||||
const mimeMap: Record<string, string> = {
|
||||
'.pdf': 'application/pdf',
|
||||
'.doc': 'application/msword',
|
||||
'.docx':
|
||||
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
||||
'.md': 'text/markdown',
|
||||
'.txt': 'text/plain',
|
||||
'.json': 'application/json',
|
||||
'.html': 'text/html',
|
||||
'.csv': 'text/csv',
|
||||
};
|
||||
return mimeMap[ext] || 'application/octet-stream';
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import {
|
||||
IsNotEmpty,
|
||||
IsString,
|
||||
MinLength,
|
||||
IsOptional,
|
||||
IsEnum,
|
||||
} from 'class-validator';
|
||||
import { ApiPropertyOptional } from '@nestjs/swagger';
|
||||
import { UserRole } from '../user-role.enum';
|
||||
|
||||
export class CreateUserDto {
|
||||
@IsString()
|
||||
@IsNotEmpty()
|
||||
username: string;
|
||||
|
||||
@IsString()
|
||||
@IsNotEmpty()
|
||||
password: string;
|
||||
|
||||
@ApiPropertyOptional()
|
||||
@IsString()
|
||||
@IsOptional()
|
||||
displayName?: string;
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
import { IsBoolean, IsOptional, IsString, IsEnum } from 'class-validator';
|
||||
import { ApiPropertyOptional } from '@nestjs/swagger';
|
||||
import { UserRole } from '../user-role.enum';
|
||||
|
||||
export class UpdateUserDto {
|
||||
@IsOptional()
|
||||
@IsString()
|
||||
displayName?: string;
|
||||
|
||||
@IsOptional()
|
||||
@IsBoolean()
|
||||
isAdmin?: boolean;
|
||||
|
||||
@IsOptional()
|
||||
@IsString()
|
||||
tenantId?: string;
|
||||
|
||||
@IsOptional()
|
||||
@IsString()
|
||||
password?: string;
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
// server/src/user/dto/user-safe.dto.ts
|
||||
|
||||
import { UserRole } from '../user-role.enum';
|
||||
|
||||
export type SafeUser = {
|
||||
id: string;
|
||||
username: string;
|
||||
displayName?: string;
|
||||
isAdmin: boolean;
|
||||
role: UserRole; // Computed property
|
||||
tenantId: string;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
};
|
||||
@@ -0,0 +1,5 @@
|
||||
export enum UserRole {
|
||||
SUPER_ADMIN = 'SUPER_ADMIN',
|
||||
TENANT_ADMIN = 'TENANT_ADMIN',
|
||||
USER = 'USER',
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import {
|
||||
Column,
|
||||
CreateDateColumn,
|
||||
Entity,
|
||||
JoinColumn,
|
||||
OneToOne,
|
||||
PrimaryGeneratedColumn,
|
||||
UpdateDateColumn,
|
||||
} from 'typeorm';
|
||||
import { User } from './user.entity';
|
||||
|
||||
@Entity('user_settings')
|
||||
export class UserSetting {
|
||||
@PrimaryGeneratedColumn('uuid')
|
||||
id: string;
|
||||
|
||||
@Column({ type: 'text' })
|
||||
userId: string;
|
||||
|
||||
@OneToOne(() => User, { onDelete: 'CASCADE' })
|
||||
@JoinColumn({ name: 'userId' })
|
||||
user: User;
|
||||
|
||||
@Column({ type: 'text', default: 'zh' })
|
||||
language: string;
|
||||
|
||||
@CreateDateColumn({ name: 'created_at' })
|
||||
createdAt: Date;
|
||||
|
||||
@UpdateDateColumn({ name: 'updated_at' })
|
||||
updatedAt: Date;
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Repository } from 'typeorm';
|
||||
import { UserSetting } from './user-setting.entity';
|
||||
|
||||
@Injectable()
|
||||
export class UserSettingService {
|
||||
constructor(
|
||||
@InjectRepository(UserSetting)
|
||||
private userSettingRepository: Repository<UserSetting>,
|
||||
) {}
|
||||
|
||||
async getByUser(userId: string): Promise<UserSetting> {
|
||||
let setting = await this.userSettingRepository.findOne({
|
||||
where: { userId },
|
||||
});
|
||||
if (!setting) {
|
||||
setting = this.userSettingRepository.create({ userId, language: 'zh' });
|
||||
await this.userSettingRepository.save(setting);
|
||||
}
|
||||
return setting;
|
||||
}
|
||||
|
||||
async update(userId: string, language: string): Promise<UserSetting> {
|
||||
const setting = await this.getByUser(userId);
|
||||
setting.language = language;
|
||||
return this.userSettingRepository.save(setting);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,293 @@
|
||||
import {
|
||||
BadRequestException,
|
||||
Body,
|
||||
Controller,
|
||||
Delete,
|
||||
ForbiddenException,
|
||||
NotFoundException,
|
||||
Get,
|
||||
Param,
|
||||
Post,
|
||||
Put,
|
||||
Request,
|
||||
UseGuards,
|
||||
Query,
|
||||
} from '@nestjs/common';
|
||||
import { UserService } from './user.service';
|
||||
import { CombinedAuthGuard } from '../auth/combined-auth.guard';
|
||||
import { CreateUserDto } from './dto/create-user.dto';
|
||||
import { UpdateUserDto } from './dto/update-user.dto';
|
||||
import { I18nService } from '../i18n/i18n.service';
|
||||
import { UserRole } from './user-role.enum';
|
||||
import { UserSettingService } from './user-setting.service';
|
||||
|
||||
@Controller('users')
|
||||
@UseGuards(CombinedAuthGuard)
|
||||
export class UserController {
|
||||
constructor(
|
||||
private readonly userService: UserService,
|
||||
private readonly i18nService: I18nService,
|
||||
private readonly userSettingService: UserSettingService,
|
||||
) {}
|
||||
|
||||
// --- API Key Management ---
|
||||
@Get('api-key')
|
||||
async getApiKey(@Request() req) {
|
||||
const apiKey = await this.userService.getOrCreateApiKey(req.user.id);
|
||||
return { apiKey };
|
||||
}
|
||||
|
||||
@Post('api-key/rotate')
|
||||
async rotateApiKey(@Request() req) {
|
||||
const apiKey = await this.userService.regenerateApiKey(req.user.id);
|
||||
return { apiKey };
|
||||
}
|
||||
|
||||
// --- Personal Settings ---
|
||||
@Get('settings')
|
||||
async getSettings(@Request() req) {
|
||||
return this.userSettingService.getByUser(req.user.id);
|
||||
}
|
||||
|
||||
@Put('settings/language')
|
||||
async updateLanguage(@Request() req, @Body() body: { language: string }) {
|
||||
if (!body.language) throw new BadRequestException('language is required');
|
||||
return this.userSettingService.update(req.user.id, body.language);
|
||||
}
|
||||
|
||||
// --- Profile ---
|
||||
@Get('profile')
|
||||
async getProfile(@Request() req: any) {
|
||||
return this.userService.findOneById(req.user.id);
|
||||
}
|
||||
|
||||
@Get('tenants')
|
||||
async getMyTenants(@Request() req: any) {
|
||||
return this.userService.getUserTenants(req.user.id);
|
||||
}
|
||||
|
||||
@Get('me')
|
||||
async getMe(@Request() req) {
|
||||
const user = await this.userService.findOneById(req.user.id);
|
||||
if (!user) throw new NotFoundException();
|
||||
|
||||
let isNotebookEnabled = true;
|
||||
if (user.tenantId) {
|
||||
const settings = await this.userService.getTenantSettings(user.tenantId);
|
||||
isNotebookEnabled = settings?.isNotebookEnabled ?? true;
|
||||
}
|
||||
|
||||
const tenantName = user.tenantMembers?.[0]?.tenant?.name || 'Default';
|
||||
|
||||
return {
|
||||
id: user.id,
|
||||
username: user.username,
|
||||
displayName: user.displayName,
|
||||
role: user.isAdmin ? UserRole.SUPER_ADMIN : UserRole.USER,
|
||||
tenantId: user.tenantId,
|
||||
tenantName,
|
||||
isAdmin: user.isAdmin,
|
||||
isNotebookEnabled,
|
||||
};
|
||||
}
|
||||
|
||||
@Get()
|
||||
async findAll(
|
||||
@Request() req,
|
||||
@Query('page') page?: string,
|
||||
@Query('limit') limit?: string,
|
||||
) {
|
||||
const callerRole = req.user.role;
|
||||
if (
|
||||
callerRole !== UserRole.SUPER_ADMIN &&
|
||||
callerRole !== UserRole.TENANT_ADMIN
|
||||
) {
|
||||
throw new ForbiddenException(
|
||||
this.i18nService.getErrorMessage('adminOnlyViewList'),
|
||||
);
|
||||
}
|
||||
|
||||
const p = page ? parseInt(page) : undefined;
|
||||
const l = limit ? parseInt(limit) : undefined;
|
||||
|
||||
if (callerRole === UserRole.SUPER_ADMIN) {
|
||||
return this.userService.findAll(p, l);
|
||||
} else {
|
||||
return this.userService.findByTenantId(req.user.tenantId, p, l);
|
||||
}
|
||||
}
|
||||
|
||||
@Put('password')
|
||||
async changePassword(
|
||||
@Request() req,
|
||||
@Body() body: { currentPassword: string; newPassword: string },
|
||||
) {
|
||||
const { currentPassword, newPassword } = body;
|
||||
|
||||
if (!currentPassword || !newPassword) {
|
||||
throw new BadRequestException(
|
||||
this.i18nService.getErrorMessage('passwordsRequired'),
|
||||
);
|
||||
}
|
||||
|
||||
if (newPassword.length < 6) {
|
||||
throw new BadRequestException(
|
||||
this.i18nService.getErrorMessage('newPasswordMinLength'),
|
||||
);
|
||||
}
|
||||
|
||||
return this.userService.changePassword(
|
||||
req.user.id,
|
||||
currentPassword,
|
||||
newPassword,
|
||||
);
|
||||
}
|
||||
|
||||
@Post()
|
||||
async createUser(@Request() req, @Body() body: CreateUserDto) {
|
||||
const callerRole = req.user.role;
|
||||
if (
|
||||
callerRole !== UserRole.SUPER_ADMIN &&
|
||||
callerRole !== UserRole.TENANT_ADMIN
|
||||
) {
|
||||
throw new ForbiddenException(
|
||||
this.i18nService.getErrorMessage('adminOnlyCreateUser'),
|
||||
);
|
||||
}
|
||||
|
||||
const { username, password } = body;
|
||||
|
||||
if (!username || !password) {
|
||||
throw new BadRequestException(
|
||||
this.i18nService.getErrorMessage('usernamePasswordRequired'),
|
||||
);
|
||||
}
|
||||
|
||||
if (password.length < 6) {
|
||||
throw new BadRequestException(
|
||||
this.i18nService.getErrorMessage('passwordMinLength'),
|
||||
);
|
||||
}
|
||||
|
||||
// All new global users default to non-admin.
|
||||
// Elevation to Super Admin status is handled separately.
|
||||
let isAdmin = false;
|
||||
|
||||
if (callerRole === UserRole.SUPER_ADMIN) {
|
||||
isAdmin = false;
|
||||
} else if (callerRole === UserRole.TENANT_ADMIN) {
|
||||
isAdmin = false;
|
||||
}
|
||||
|
||||
// Pass the calculated params to the service
|
||||
return this.userService.createUser(
|
||||
username,
|
||||
password,
|
||||
isAdmin,
|
||||
req.user.tenantId,
|
||||
body.displayName,
|
||||
);
|
||||
}
|
||||
|
||||
@Put(':id')
|
||||
async updateUser(
|
||||
@Request() req,
|
||||
@Body() body: UpdateUserDto,
|
||||
@Param('id') id: string,
|
||||
) {
|
||||
const callerRole = req.user.role;
|
||||
if (
|
||||
callerRole !== UserRole.SUPER_ADMIN &&
|
||||
callerRole !== UserRole.TENANT_ADMIN
|
||||
) {
|
||||
throw new ForbiddenException(
|
||||
this.i18nService.getErrorMessage('adminOnlyUpdateUser'),
|
||||
);
|
||||
}
|
||||
|
||||
// Get user info to update
|
||||
const userToUpdate = await this.userService.findOneById(id);
|
||||
if (!userToUpdate) {
|
||||
throw new NotFoundException(
|
||||
this.i18nService.getErrorMessage('userNotFound'),
|
||||
);
|
||||
}
|
||||
|
||||
if (
|
||||
callerRole === 'TENANT_ADMIN' &&
|
||||
userToUpdate.tenantId !== req.user.tenantId
|
||||
) {
|
||||
throw new ForbiddenException('Cannot modify users outside your tenant');
|
||||
}
|
||||
|
||||
// Prevent modifying the builtin admin account
|
||||
if (userToUpdate.username === 'admin') {
|
||||
throw new ForbiddenException(
|
||||
this.i18nService.getErrorMessage('cannotModifyBuiltinAdmin'),
|
||||
);
|
||||
}
|
||||
|
||||
// Role modification is now obsolete on global level.
|
||||
// If Admin wants to elevate, they set isAdmin property directly.
|
||||
if (body.isAdmin !== undefined && userToUpdate.isAdmin !== body.isAdmin) {
|
||||
if (callerRole !== UserRole.SUPER_ADMIN) {
|
||||
throw new ForbiddenException(
|
||||
'Only Super Admins can change user admin status.',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Validate password length if provided
|
||||
if (body.password && body.password.length < 6) {
|
||||
throw new BadRequestException(
|
||||
this.i18nService.getErrorMessage('passwordMinLength'),
|
||||
);
|
||||
}
|
||||
|
||||
return this.userService.updateUser(id, body);
|
||||
}
|
||||
|
||||
@Delete(':id')
|
||||
async deleteUser(@Request() req, @Param('id') id: string) {
|
||||
const callerRole = req.user.role;
|
||||
if (
|
||||
callerRole !== UserRole.SUPER_ADMIN &&
|
||||
callerRole !== UserRole.TENANT_ADMIN
|
||||
) {
|
||||
throw new ForbiddenException(
|
||||
this.i18nService.getErrorMessage('adminOnlyDeleteUser'),
|
||||
);
|
||||
}
|
||||
|
||||
// Prevent admin from deleting themselves
|
||||
if (req.user.id === id) {
|
||||
throw new BadRequestException(
|
||||
this.i18nService.getErrorMessage('cannotDeleteSelf'),
|
||||
);
|
||||
}
|
||||
|
||||
// Get user info to delete
|
||||
const userToDelete = await this.userService.findOneById(id);
|
||||
if (!userToDelete) {
|
||||
throw new NotFoundException(
|
||||
this.i18nService.getErrorMessage('userNotFound'),
|
||||
);
|
||||
}
|
||||
|
||||
if (
|
||||
callerRole === 'TENANT_ADMIN' &&
|
||||
userToDelete.tenantId !== req.user.tenantId
|
||||
) {
|
||||
throw new ForbiddenException('Cannot delete users outside your tenant');
|
||||
}
|
||||
|
||||
// Block deletion of built-in admin account
|
||||
if (userToDelete.username === 'admin') {
|
||||
throw new ForbiddenException(
|
||||
this.i18nService.getErrorMessage('cannotDeleteBuiltinAdmin'),
|
||||
);
|
||||
}
|
||||
|
||||
return this.userService.deleteUser(id);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
import {
|
||||
BeforeInsert,
|
||||
BeforeUpdate,
|
||||
Column,
|
||||
CreateDateColumn,
|
||||
Entity,
|
||||
JoinColumn,
|
||||
ManyToOne,
|
||||
OneToMany,
|
||||
OneToOne,
|
||||
PrimaryGeneratedColumn,
|
||||
UpdateDateColumn,
|
||||
} from 'typeorm';
|
||||
import * as bcrypt from 'bcrypt';
|
||||
import { ModelConfig } from '../model-config/model-config.entity';
|
||||
import { Tenant } from '../tenant/tenant.entity';
|
||||
import { TenantMember } from '../tenant/tenant-member.entity';
|
||||
import { ApiKey } from '../auth/entities/api-key.entity';
|
||||
|
||||
import { UserRole } from './user-role.enum';
|
||||
import { UserSetting } from './user-setting.entity';
|
||||
|
||||
@Entity('users')
|
||||
export class User {
|
||||
@PrimaryGeneratedColumn('uuid')
|
||||
id: string;
|
||||
|
||||
@Column({ type: 'text', unique: true })
|
||||
username: string;
|
||||
|
||||
@Column({ type: 'text' })
|
||||
password: string;
|
||||
|
||||
// Legacy field - kept for backward compatibility, use `role` for new logic
|
||||
@Column({ type: 'boolean', default: false })
|
||||
isAdmin: boolean;
|
||||
|
||||
@Column({ type: 'text', nullable: false })
|
||||
displayName: string;
|
||||
|
||||
// Multi-tenancy: A user can belong to multiple tenants via TenantMember
|
||||
@OneToMany(() => TenantMember, (member) => member.user)
|
||||
tenantMembers: TenantMember[];
|
||||
|
||||
// Legacy field - kept for backward compatibility if needed, but primary tenant is now determined by context
|
||||
@Column({ type: 'text', nullable: true, name: 'tenant_id' })
|
||||
tenantId: string;
|
||||
|
||||
// API Keys for external API access
|
||||
@OneToMany(() => ApiKey, (apiKey) => apiKey.user)
|
||||
apiKeys: ApiKey[];
|
||||
|
||||
// Quota management field
|
||||
@Column({ type: 'decimal', precision: 10, scale: 2, default: 0 })
|
||||
monthlyCost: number;
|
||||
|
||||
@Column({ type: 'decimal', precision: 10, scale: 2, default: 100 })
|
||||
maxCost: number;
|
||||
|
||||
@Column({ type: 'datetime', nullable: true })
|
||||
lastQuotaReset: Date;
|
||||
|
||||
@CreateDateColumn({ name: 'created_at' })
|
||||
createdAt: Date;
|
||||
|
||||
@UpdateDateColumn({ name: 'updated_at' })
|
||||
updatedAt: Date;
|
||||
|
||||
@OneToOne(() => UserSetting, (setting) => setting.user)
|
||||
userSetting: UserSetting;
|
||||
|
||||
@BeforeInsert()
|
||||
@BeforeUpdate()
|
||||
async hashPassword() {
|
||||
// Only hash if the password is not yet hashed (BCrypt hash length is 60)
|
||||
if (this.password && this.password.length < 60) {
|
||||
this.password = await bcrypt.hash(this.password, 10);
|
||||
}
|
||||
}
|
||||
|
||||
async validatePassword(password: string): Promise<boolean> {
|
||||
return bcrypt.compare(password, this.password);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import { Module, Global } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
import { User } from './user.entity';
|
||||
import { UserSetting } from './user-setting.entity';
|
||||
import { UserSettingService } from './user-setting.service';
|
||||
import { TenantMember } from '../tenant/tenant-member.entity';
|
||||
import { ApiKey } from '../auth/entities/api-key.entity';
|
||||
import { UserService } from './user.service';
|
||||
import { UserController } from './user.controller';
|
||||
import { TenantModule } from '../tenant/tenant.module';
|
||||
|
||||
@Global()
|
||||
@Module({
|
||||
imports: [
|
||||
TypeOrmModule.forFeature([User, ApiKey, TenantMember, UserSetting]),
|
||||
TenantModule,
|
||||
],
|
||||
controllers: [UserController],
|
||||
providers: [UserService, UserSettingService],
|
||||
exports: [UserService, UserSettingService],
|
||||
})
|
||||
export class UserModule {}
|
||||
@@ -0,0 +1,412 @@
|
||||
import {
|
||||
BadRequestException,
|
||||
ConflictException,
|
||||
Injectable,
|
||||
NotFoundException,
|
||||
OnModuleInit,
|
||||
ForbiddenException,
|
||||
Logger,
|
||||
} from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Repository } from 'typeorm';
|
||||
import { User } from './user.entity';
|
||||
import { UserRole } from './user-role.enum';
|
||||
import { TenantMember } from '../tenant/tenant-member.entity';
|
||||
import { ApiKey } from '../auth/entities/api-key.entity';
|
||||
import * as bcrypt from 'bcrypt';
|
||||
import { CreateUserDto } from './dto/create-user.dto';
|
||||
import * as crypto from 'crypto';
|
||||
import { I18nService } from '../i18n/i18n.service';
|
||||
import { TenantService } from '../tenant/tenant.service';
|
||||
|
||||
@Injectable()
|
||||
export class UserService implements OnModuleInit {
|
||||
private readonly logger = new Logger(UserService.name);
|
||||
|
||||
constructor(
|
||||
@InjectRepository(User)
|
||||
private usersRepository: Repository<User>,
|
||||
@InjectRepository(ApiKey)
|
||||
private apiKeyRepository: Repository<ApiKey>,
|
||||
@InjectRepository(TenantMember)
|
||||
private tenantMemberRepository: Repository<TenantMember>,
|
||||
private i18nService: I18nService,
|
||||
private tenantService: TenantService,
|
||||
) {}
|
||||
|
||||
async findOneByUsername(username: string): Promise<User | null> {
|
||||
return this.usersRepository.findOne({ where: { username } });
|
||||
}
|
||||
|
||||
async create(createUserDto: CreateUserDto): Promise<User> {
|
||||
const user = this.usersRepository.create(createUserDto as any);
|
||||
return this.usersRepository.save(user as any);
|
||||
}
|
||||
|
||||
async onModuleInit() {
|
||||
await this.createAdminIfNotExists();
|
||||
}
|
||||
|
||||
async findAll(
|
||||
page?: number,
|
||||
limit?: number,
|
||||
): Promise<{ data: User[]; total: number }> {
|
||||
const queryBuilder = this.usersRepository
|
||||
.createQueryBuilder('user')
|
||||
.leftJoinAndSelect('user.tenantMembers', 'tenantMember')
|
||||
.leftJoinAndSelect('tenantMember.tenant', 'tenant')
|
||||
.select([
|
||||
'user.id',
|
||||
'user.username',
|
||||
'user.displayName',
|
||||
'user.isAdmin',
|
||||
'user.createdAt',
|
||||
'user.tenantId',
|
||||
'tenantMember',
|
||||
'tenant',
|
||||
])
|
||||
.orderBy('user.createdAt', 'DESC');
|
||||
|
||||
if (page && limit) {
|
||||
const [data, total] = await queryBuilder
|
||||
.skip((page - 1) * limit)
|
||||
.take(limit)
|
||||
.getManyAndCount();
|
||||
return { data, total };
|
||||
}
|
||||
|
||||
const [data, total] = await queryBuilder.getManyAndCount();
|
||||
return { data, total };
|
||||
}
|
||||
|
||||
async findByTenantId(
|
||||
tenantId: string,
|
||||
page?: number,
|
||||
limit?: number,
|
||||
): Promise<{ data: User[]; total: number }> {
|
||||
const queryBuilder = this.usersRepository
|
||||
.createQueryBuilder('user')
|
||||
.innerJoin(
|
||||
'user.tenantMembers',
|
||||
'member',
|
||||
'member.tenantId = :tenantId',
|
||||
{ tenantId },
|
||||
)
|
||||
.select([
|
||||
'user.id',
|
||||
'user.username',
|
||||
'user.displayName',
|
||||
'user.isAdmin',
|
||||
'user.createdAt',
|
||||
'user.tenantId',
|
||||
])
|
||||
.orderBy('user.createdAt', 'DESC');
|
||||
|
||||
if (page && limit) {
|
||||
const [data, total] = await queryBuilder
|
||||
.skip((page - 1) * limit)
|
||||
.take(limit)
|
||||
.getManyAndCount();
|
||||
return { data, total };
|
||||
}
|
||||
|
||||
const [data, total] = await queryBuilder.getManyAndCount();
|
||||
return { data, total };
|
||||
}
|
||||
|
||||
async isAdmin(userId: string): Promise<boolean> {
|
||||
const user = await this.usersRepository.findOne({
|
||||
where: { id: userId },
|
||||
select: ['isAdmin'],
|
||||
});
|
||||
return user?.isAdmin || false;
|
||||
}
|
||||
|
||||
async changePassword(
|
||||
userId: string,
|
||||
currentPassword: string,
|
||||
newPassword: string,
|
||||
): Promise<{ message: string }> {
|
||||
const user = await this.usersRepository.findOne({ where: { id: userId } });
|
||||
if (!user) {
|
||||
throw new NotFoundException(this.i18nService.getMessage('userNotFound'));
|
||||
}
|
||||
|
||||
const isCurrentPasswordValid = await bcrypt.compare(
|
||||
currentPassword,
|
||||
user.password,
|
||||
);
|
||||
if (!isCurrentPasswordValid) {
|
||||
throw new BadRequestException(
|
||||
this.i18nService.getMessage('incorrectCurrentPassword'),
|
||||
);
|
||||
}
|
||||
|
||||
const hashedNewPassword = await bcrypt.hash(newPassword, 10);
|
||||
await this.usersRepository.update(userId, { password: hashedNewPassword });
|
||||
|
||||
return { message: this.i18nService.getMessage('passwordChanged') };
|
||||
}
|
||||
|
||||
async createUser(
|
||||
username: string,
|
||||
password: string,
|
||||
isAdmin: boolean = false,
|
||||
tenantId?: string,
|
||||
displayName?: string,
|
||||
): Promise<{
|
||||
message: string;
|
||||
user: {
|
||||
id: string;
|
||||
username: string;
|
||||
displayName: string;
|
||||
isAdmin: boolean;
|
||||
};
|
||||
}> {
|
||||
const existingUser = await this.findOneByUsername(username);
|
||||
if (existingUser) {
|
||||
throw new ConflictException(
|
||||
this.i18nService.getMessage('usernameExists'),
|
||||
);
|
||||
}
|
||||
|
||||
const hashedPassword = await bcrypt.hash(password, 10);
|
||||
console.log(
|
||||
`[UserService] Creating user: ${username}, isAdmin: ${isAdmin}`,
|
||||
);
|
||||
const user = await this.usersRepository.save({
|
||||
username,
|
||||
password: hashedPassword,
|
||||
displayName: displayName || username,
|
||||
isAdmin,
|
||||
tenantId: tenantId ?? undefined,
|
||||
} as any);
|
||||
|
||||
return {
|
||||
message: this.i18nService.getMessage('userCreated'),
|
||||
user: {
|
||||
id: user.id,
|
||||
username: user.username,
|
||||
displayName: user.displayName,
|
||||
isAdmin: user.isAdmin,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async findOneById(userId: string): Promise<User | null> {
|
||||
return this.usersRepository.findOne({
|
||||
where: { id: userId },
|
||||
relations: ['tenantMembers', 'tenantMembers.tenant'],
|
||||
});
|
||||
}
|
||||
|
||||
async findByApiKey(apiKeyValue: string): Promise<User | null> {
|
||||
const apiKey = await this.apiKeyRepository.findOne({
|
||||
where: { key: apiKeyValue },
|
||||
relations: ['user'],
|
||||
});
|
||||
return apiKey ? apiKey.user : null;
|
||||
}
|
||||
|
||||
async getUserTenants(
|
||||
userId: string,
|
||||
): Promise<(TenantMember & { features?: { isNotebookEnabled: boolean } })[]> {
|
||||
const user = await this.usersRepository.findOne({
|
||||
where: { id: userId },
|
||||
select: ['isAdmin'],
|
||||
});
|
||||
|
||||
if (user?.isAdmin) {
|
||||
const tenantsData = await this.tenantService.findAll();
|
||||
const allTenants = Array.isArray(tenantsData)
|
||||
? tenantsData
|
||||
: tenantsData.data;
|
||||
const results = await Promise.all(
|
||||
allTenants.map(async (t) => {
|
||||
const settings = await this.tenantService.getSettings(t.id);
|
||||
return {
|
||||
tenantId: t.id,
|
||||
tenant: t,
|
||||
role: UserRole.SUPER_ADMIN,
|
||||
userId: userId,
|
||||
features: {
|
||||
isNotebookEnabled: settings?.isNotebookEnabled ?? true,
|
||||
},
|
||||
} as TenantMember & { features: { isNotebookEnabled: boolean } };
|
||||
}),
|
||||
);
|
||||
return results;
|
||||
}
|
||||
|
||||
const members = await this.tenantMemberRepository.find({
|
||||
where: { userId },
|
||||
relations: ['tenant'],
|
||||
});
|
||||
|
||||
// Filter out the "Default" tenant for non-super admins
|
||||
const filtered = members.filter(
|
||||
(m) => m.tenant?.name !== TenantService.DEFAULT_TENANT_NAME,
|
||||
);
|
||||
|
||||
// Attach per-tenant feature flags
|
||||
return Promise.all(
|
||||
filtered.map(async (m) => {
|
||||
const settings = await this.tenantService.getSettings(m.tenantId);
|
||||
return {
|
||||
...m,
|
||||
features: {
|
||||
isNotebookEnabled: settings?.isNotebookEnabled ?? true,
|
||||
},
|
||||
};
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates a new API key for the user, or returns the existing one (first one).
|
||||
*/
|
||||
async getOrCreateApiKey(userId: string): Promise<string> {
|
||||
const user = await this.usersRepository.findOne({
|
||||
where: { id: userId },
|
||||
relations: ['apiKeys'],
|
||||
});
|
||||
|
||||
if (!user)
|
||||
throw new NotFoundException(this.i18nService.getMessage('userNotFound'));
|
||||
|
||||
if (user.apiKeys && user.apiKeys.length > 0) {
|
||||
return user.apiKeys[0].key;
|
||||
}
|
||||
|
||||
const keyString = 'kb_' + crypto.randomBytes(32).toString('hex');
|
||||
const newApiKey = this.apiKeyRepository.create({
|
||||
userId: user.id,
|
||||
key: keyString,
|
||||
});
|
||||
await this.apiKeyRepository.save(newApiKey);
|
||||
|
||||
return keyString;
|
||||
}
|
||||
|
||||
/**
|
||||
* Regenerates (rotates) the API key for the user.
|
||||
* This clears existing keys and creates a new one.
|
||||
*/
|
||||
async regenerateApiKey(userId: string): Promise<string> {
|
||||
const user = await this.usersRepository.findOne({ where: { id: userId } });
|
||||
if (!user)
|
||||
throw new NotFoundException(this.i18nService.getMessage('userNotFound'));
|
||||
|
||||
// Delete existing keys
|
||||
await this.apiKeyRepository.delete({ userId: user.id });
|
||||
|
||||
// Create new key
|
||||
const keyString = 'kb_' + crypto.randomBytes(32).toString('hex');
|
||||
const newApiKey = this.apiKeyRepository.create({
|
||||
userId: user.id,
|
||||
key: keyString,
|
||||
});
|
||||
await this.apiKeyRepository.save(newApiKey);
|
||||
|
||||
return keyString;
|
||||
}
|
||||
|
||||
async updateUser(
|
||||
userId: string,
|
||||
updateData: {
|
||||
username?: string;
|
||||
isAdmin?: boolean;
|
||||
password?: string;
|
||||
tenantId?: string;
|
||||
displayName?: string;
|
||||
},
|
||||
): Promise<{
|
||||
message: string;
|
||||
user: {
|
||||
id: string;
|
||||
username: string;
|
||||
displayName: string;
|
||||
isAdmin: boolean;
|
||||
};
|
||||
}> {
|
||||
const user = await this.usersRepository.findOne({ where: { id: userId } });
|
||||
if (!user) {
|
||||
throw new NotFoundException(this.i18nService.getMessage('userNotFound'));
|
||||
}
|
||||
|
||||
// Hash password first if update needed
|
||||
if (updateData.password) {
|
||||
const hashedPassword = await bcrypt.hash(updateData.password, 10);
|
||||
updateData.password = hashedPassword;
|
||||
}
|
||||
|
||||
// Block any changes to user "admin"
|
||||
if (user.username === 'admin') {
|
||||
throw new ForbiddenException(
|
||||
this.i18nService.getMessage('cannotModifyBuiltinAdmin'),
|
||||
);
|
||||
}
|
||||
|
||||
await this.usersRepository.update(userId, updateData as any);
|
||||
|
||||
const updatedUser = await this.usersRepository.findOne({
|
||||
where: { id: userId },
|
||||
select: ['id', 'username', 'displayName', 'isAdmin'],
|
||||
});
|
||||
|
||||
return {
|
||||
message: this.i18nService.getMessage('userInfoUpdated'),
|
||||
user: {
|
||||
id: updatedUser!.id,
|
||||
username: updatedUser!.username,
|
||||
displayName: updatedUser!.displayName,
|
||||
isAdmin: updatedUser!.isAdmin,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async deleteUser(userId: string): Promise<{ message: string }> {
|
||||
const user = await this.usersRepository.findOne({ where: { id: userId } });
|
||||
if (!user) {
|
||||
throw new NotFoundException(this.i18nService.getMessage('userNotFound'));
|
||||
}
|
||||
|
||||
// Block deletion of user "admin"
|
||||
if (user.username === 'admin') {
|
||||
throw new ForbiddenException(
|
||||
this.i18nService.getMessage('cannotDeleteBuiltinAdmin'),
|
||||
);
|
||||
}
|
||||
|
||||
await this.usersRepository.delete(userId);
|
||||
|
||||
return {
|
||||
message: this.i18nService.getMessage('userDeleted'),
|
||||
};
|
||||
}
|
||||
|
||||
async getTenantSettings(tenantId: string) {
|
||||
return this.tenantService.getSettings(tenantId);
|
||||
}
|
||||
|
||||
private async createAdminIfNotExists() {
|
||||
const adminUser = await this.findOneByUsername('admin');
|
||||
if (!adminUser) {
|
||||
const randomPassword = Math.random().toString(36).slice(-8);
|
||||
const hashedPassword = await bcrypt.hash(randomPassword, 10);
|
||||
|
||||
await this.usersRepository.save({
|
||||
username: 'admin',
|
||||
password: hashedPassword,
|
||||
displayName: 'Admin',
|
||||
isAdmin: true,
|
||||
role: UserRole.SUPER_ADMIN,
|
||||
});
|
||||
|
||||
console.log('\n=== Admin account created ===');
|
||||
console.log('Username: admin');
|
||||
console.log('Password:', randomPassword);
|
||||
console.log('========================================\n');
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
import { CostControlService } from './cost-control.service';
|
||||
import { User } from '../user/user.entity';
|
||||
|
||||
@Module({
|
||||
imports: [TypeOrmModule.forFeature([User])],
|
||||
providers: [CostControlService],
|
||||
exports: [CostControlService],
|
||||
})
|
||||
export class CostControlModule {}
|
||||
@@ -0,0 +1,261 @@
|
||||
/**
|
||||
* Cost control and quota management service
|
||||
* Used to manage API call costs for Vision Pipeline
|
||||
*/
|
||||
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Repository } from 'typeorm';
|
||||
import { User } from '../user/user.entity';
|
||||
|
||||
export interface UserQuota {
|
||||
userId: string;
|
||||
monthlyCost: number; // Current month used cost
|
||||
maxCost: number; // Monthly max cost
|
||||
remaining: number; // Remaining cost
|
||||
lastReset: Date; // Last reset time
|
||||
}
|
||||
|
||||
export interface CostEstimate {
|
||||
estimatedCost: number; // Estimated cost
|
||||
estimatedTime: number; // Estimated time(seconds)
|
||||
pageBreakdown: {
|
||||
// Per-page breakdown
|
||||
pageIndex: number;
|
||||
cost: number;
|
||||
}[];
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class CostControlService {
|
||||
private readonly logger = new Logger(CostControlService.name);
|
||||
private readonly COST_PER_PAGE = 0.01; // Cost per page(USD)
|
||||
private readonly DEFAULT_MONTHLY_LIMIT = 100; // Default monthly limit(USD)
|
||||
|
||||
constructor(
|
||||
private configService: ConfigService,
|
||||
@InjectRepository(User)
|
||||
private userRepository: Repository<User>,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Estimate processing cost
|
||||
*/
|
||||
estimateCost(
|
||||
pageCount: number,
|
||||
quality: 'low' | 'medium' | 'high' = 'medium',
|
||||
): CostEstimate {
|
||||
// Adjust cost coefficient based on quality
|
||||
const qualityMultiplier = {
|
||||
low: 0.5,
|
||||
medium: 1.0,
|
||||
high: 1.5,
|
||||
};
|
||||
|
||||
const baseCost =
|
||||
pageCount * this.COST_PER_PAGE * qualityMultiplier[quality];
|
||||
const estimatedTime = pageCount * 3; // // Approximately 3 seconds
|
||||
|
||||
const pageBreakdown = Array.from({ length: pageCount }, (_, i) => ({
|
||||
pageIndex: i + 1,
|
||||
cost: this.COST_PER_PAGE * qualityMultiplier[quality],
|
||||
}));
|
||||
|
||||
return {
|
||||
estimatedCost: baseCost,
|
||||
estimatedTime,
|
||||
pageBreakdown,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Check user quota
|
||||
*/
|
||||
async checkQuota(
|
||||
userId: string,
|
||||
estimatedCost: number,
|
||||
): Promise<{
|
||||
allowed: boolean;
|
||||
quota: UserQuota;
|
||||
reason?: string;
|
||||
}> {
|
||||
const quota = await this.getUserQuota(userId);
|
||||
|
||||
// Check monthly reset
|
||||
this.checkAndResetMonthlyQuota(quota);
|
||||
|
||||
if (quota.remaining < estimatedCost) {
|
||||
this.logger.warn(
|
||||
`User ${userId} quota insufficient: remaining $${quota.remaining.toFixed(2)}, required $${estimatedCost.toFixed(2)}`,
|
||||
);
|
||||
return {
|
||||
allowed: false,
|
||||
quota,
|
||||
reason: `Insufficient quota: remaining $${quota.remaining.toFixed(2)}, required $${estimatedCost.toFixed(2)}`,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
allowed: true,
|
||||
quota,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Deduct from quota
|
||||
*/
|
||||
async deductQuota(userId: string, actualCost: number): Promise<void> {
|
||||
const quota = await this.getUserQuota(userId);
|
||||
quota.monthlyCost += actualCost;
|
||||
quota.remaining = quota.maxCost - quota.monthlyCost;
|
||||
|
||||
await this.userRepository.update(userId, {
|
||||
monthlyCost: quota.monthlyCost,
|
||||
});
|
||||
|
||||
this.logger.log(
|
||||
`Deducted $${actualCost.toFixed(2)} from user ${userId} quota. Remaining: $${quota.remaining.toFixed(2)}`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get user quota
|
||||
*/
|
||||
async getUserQuota(userId: string): Promise<UserQuota> {
|
||||
const user = await this.userRepository.findOne({ where: { id: userId } });
|
||||
|
||||
if (!user) {
|
||||
throw new Error(`User ${userId} does not exist`);
|
||||
}
|
||||
|
||||
// Use default if user has no quota info
|
||||
const monthlyCost = user.monthlyCost || 0;
|
||||
const maxCost = user.maxCost || this.DEFAULT_MONTHLY_LIMIT;
|
||||
const lastReset = user.lastQuotaReset || new Date();
|
||||
|
||||
return {
|
||||
userId,
|
||||
monthlyCost,
|
||||
maxCost,
|
||||
remaining: maxCost - monthlyCost,
|
||||
lastReset,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Check and reset monthly quota
|
||||
*/
|
||||
private checkAndResetMonthlyQuota(quota: UserQuota): void {
|
||||
const now = new Date();
|
||||
const lastReset = quota.lastReset;
|
||||
|
||||
// Check if crossed month
|
||||
if (
|
||||
now.getMonth() !== lastReset.getMonth() ||
|
||||
now.getFullYear() !== lastReset.getFullYear()
|
||||
) {
|
||||
this.logger.log(`Reset monthly quota for user ${quota.userId}`);
|
||||
|
||||
// Reset quota
|
||||
quota.monthlyCost = 0;
|
||||
quota.remaining = quota.maxCost;
|
||||
quota.lastReset = now;
|
||||
|
||||
// Update database
|
||||
this.userRepository.update(quota.userId, {
|
||||
monthlyCost: 0,
|
||||
lastQuotaReset: now,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Set user quota limit
|
||||
*/
|
||||
async setQuotaLimit(userId: string, maxCost: number): Promise<void> {
|
||||
await this.userRepository.update(userId, { maxCost });
|
||||
this.logger.log(`Set quota limit to $${maxCost} for user ${userId}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cost report
|
||||
*/
|
||||
async getCostReport(
|
||||
userId: string,
|
||||
days: number = 30,
|
||||
): Promise<{
|
||||
totalCost: number;
|
||||
dailyAverage: number;
|
||||
pageStats: {
|
||||
totalPages: number;
|
||||
avgCostPerPage: number;
|
||||
};
|
||||
quotaUsage: number; // Percentage
|
||||
}> {
|
||||
const quota = await this.getUserQuota(userId);
|
||||
const usagePercent = (quota.monthlyCost / quota.maxCost) * 100;
|
||||
|
||||
// Query history records here(if implemented)
|
||||
// Return current quota info temporarily
|
||||
|
||||
return {
|
||||
totalCost: quota.monthlyCost,
|
||||
dailyAverage: quota.monthlyCost / Math.max(days, 1),
|
||||
pageStats: {
|
||||
totalPages: Math.floor(quota.monthlyCost / this.COST_PER_PAGE),
|
||||
avgCostPerPage: this.COST_PER_PAGE,
|
||||
},
|
||||
quotaUsage: usagePercent,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Check cost warning threshold
|
||||
*/
|
||||
async checkWarningThreshold(userId: string): Promise<{
|
||||
shouldWarn: boolean;
|
||||
message: string;
|
||||
}> {
|
||||
const quota = await this.getUserQuota(userId);
|
||||
const usagePercent = (quota.monthlyCost / quota.maxCost) * 100;
|
||||
|
||||
if (usagePercent >= 90) {
|
||||
return {
|
||||
shouldWarn: true,
|
||||
message: `⚠️ Quota usage reached ${usagePercent.toFixed(1)}%. Remaining: $${quota.remaining.toFixed(2)}`,
|
||||
};
|
||||
}
|
||||
|
||||
if (usagePercent >= 75) {
|
||||
return {
|
||||
shouldWarn: true,
|
||||
message: `💡 Quota usage at ${usagePercent.toFixed(1)}%. Please monitor your costs carefully`,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
shouldWarn: false,
|
||||
message: '',
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Format cost display
|
||||
*/
|
||||
formatCost(cost: number): string {
|
||||
return `$${cost.toFixed(2)}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format time display
|
||||
*/
|
||||
formatTime(seconds: number): string {
|
||||
if (seconds < 60) {
|
||||
return `${seconds.toFixed(0)}s`;
|
||||
}
|
||||
const minutes = Math.floor(seconds / 60);
|
||||
const remainingSeconds = seconds % 60;
|
||||
return `${minutes}m ${remainingSeconds.toFixed(0)}s`;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,341 @@
|
||||
/**
|
||||
* Vision Pipeline Service (with cost control)
|
||||
* This is an extended version of vision-pipeline.service.ts with integrated cost control
|
||||
*/
|
||||
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import * as fs from 'fs/promises';
|
||||
import * as path from 'path';
|
||||
import { LibreOfficeService } from '../libreoffice/libreoffice.service';
|
||||
import { Pdf2ImageService } from '../pdf2image/pdf2image.service';
|
||||
import { VisionService } from '../vision/vision.service';
|
||||
import { ElasticsearchService } from '../elasticsearch/elasticsearch.service';
|
||||
import { ModelConfigService } from '../model-config/model-config.service';
|
||||
import {
|
||||
PreciseModeOptions,
|
||||
PipelineResult,
|
||||
ProcessingStatus,
|
||||
ModeRecommendation,
|
||||
} from './vision-pipeline.interface';
|
||||
import {
|
||||
VisionModelConfig,
|
||||
VisionAnalysisResult,
|
||||
} from '../vision/vision.interface';
|
||||
import { CostControlService } from './cost-control.service';
|
||||
import { I18nService } from '../i18n/i18n.service';
|
||||
|
||||
@Injectable()
|
||||
export class VisionPipelineCostAwareService {
|
||||
private readonly logger = new Logger(VisionPipelineCostAwareService.name);
|
||||
|
||||
constructor(
|
||||
private libreOffice: LibreOfficeService,
|
||||
private pdf2Image: Pdf2ImageService,
|
||||
private vision: VisionService,
|
||||
private elasticsearch: ElasticsearchService,
|
||||
private modelConfigService: ModelConfigService,
|
||||
private configService: ConfigService,
|
||||
private costControl: CostControlService,
|
||||
private i18nService: I18nService,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Main processing flow: Precise mode (with cost control)
|
||||
*/
|
||||
async processPreciseMode(
|
||||
filePath: string,
|
||||
options: PreciseModeOptions,
|
||||
): Promise<PipelineResult> {
|
||||
const startTime = Date.now();
|
||||
const results: VisionAnalysisResult[] = [];
|
||||
let processedPages = 0;
|
||||
let failedPages = 0;
|
||||
let totalCost = 0;
|
||||
let pdfPath = filePath;
|
||||
let imagesToProcess: any[] = [];
|
||||
|
||||
this.logger.log(
|
||||
`Starting precise mode processing for ${options.fileName} (user: ${options.userId})`,
|
||||
);
|
||||
|
||||
try {
|
||||
// Step 1: Convert format
|
||||
this.updateStatus('converting', 10, 'Converting document format...');
|
||||
pdfPath = await this.convertToPDF(filePath);
|
||||
|
||||
// Step 2: Convert PDF to images
|
||||
this.updateStatus('splitting', 30, 'Converting PDF to images...');
|
||||
const conversionResult = await this.pdf2Image.convertToImages(pdfPath, {
|
||||
density: 300,
|
||||
quality: 85,
|
||||
format: 'jpeg',
|
||||
});
|
||||
|
||||
if (conversionResult.images.length === 0) {
|
||||
throw new Error(
|
||||
this.i18nService.getMessage('pdfToImageConversionFailed'),
|
||||
);
|
||||
}
|
||||
|
||||
// Limit processing pages
|
||||
imagesToProcess = options.maxPages
|
||||
? conversionResult.images.slice(0, options.maxPages)
|
||||
: conversionResult.images;
|
||||
|
||||
const pageCount = imagesToProcess.length;
|
||||
|
||||
// Step 3: Cost estimation and quota check
|
||||
this.updateStatus(
|
||||
'checking',
|
||||
40,
|
||||
'Checking quota and estimating cost...',
|
||||
);
|
||||
const costEstimate = this.costControl.estimateCost(pageCount);
|
||||
this.logger.log(
|
||||
`Estimated cost: $${costEstimate.estimatedCost.toFixed(2)}, Estimated time: ${this.costControl.formatTime(costEstimate.estimatedTime)}`,
|
||||
);
|
||||
|
||||
// Quota check
|
||||
const quotaCheck = await this.costControl.checkQuota(
|
||||
options.userId,
|
||||
costEstimate.estimatedCost,
|
||||
);
|
||||
|
||||
if (!quotaCheck.allowed) {
|
||||
throw new Error(quotaCheck.reason);
|
||||
}
|
||||
|
||||
// Cost warning check
|
||||
const warning = await this.costControl.checkWarningThreshold(
|
||||
options.userId,
|
||||
);
|
||||
if (warning.shouldWarn) {
|
||||
this.logger.warn(warning.message);
|
||||
}
|
||||
|
||||
// Step 4: Get Vision model config
|
||||
const modelConfig = await this.getVisionModelConfig(
|
||||
options.userId,
|
||||
options.modelId,
|
||||
options.tenantId,
|
||||
);
|
||||
|
||||
// Step 5: VL model analysis
|
||||
this.updateStatus(
|
||||
'analyzing',
|
||||
50,
|
||||
'Analyzing pages with Vision model...',
|
||||
);
|
||||
const batchResult = await this.vision.batchAnalyze(
|
||||
imagesToProcess.map((img) => img.path),
|
||||
modelConfig,
|
||||
{
|
||||
startIndex: 1,
|
||||
skipQualityCheck: options.skipQualityCheck,
|
||||
},
|
||||
);
|
||||
|
||||
totalCost = batchResult.estimatedCost;
|
||||
processedPages = batchResult.successCount;
|
||||
failedPages = batchResult.failedCount;
|
||||
results.push(...batchResult.results);
|
||||
|
||||
// Step 6: Subtract actual cost
|
||||
if (totalCost > 0) {
|
||||
await this.costControl.deductQuota(options.userId, totalCost);
|
||||
this.logger.log(`Actual cost deducted: $${totalCost.toFixed(2)}`);
|
||||
}
|
||||
|
||||
// Step 7: Cleanup temp files
|
||||
this.updateStatus(
|
||||
'completed',
|
||||
100,
|
||||
'Processing completed. Cleaning up temp files...',
|
||||
);
|
||||
await this.pdf2Image.cleanupImages(imagesToProcess);
|
||||
|
||||
// Cleanup converted PDF file if converted
|
||||
if (pdfPath !== filePath) {
|
||||
try {
|
||||
await fs.unlink(pdfPath);
|
||||
} catch (error) {
|
||||
this.logger.warn(`Failed to cleanup converted PDF: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
const duration = (Date.now() - startTime) / 1000;
|
||||
|
||||
this.logger.log(
|
||||
`Precise mode completed: ${processedPages} pages processed, ` +
|
||||
`cost: $${totalCost.toFixed(2)}, duration: ${duration.toFixed(1)}s`,
|
||||
);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
fileId: options.fileId,
|
||||
fileName: options.fileName,
|
||||
totalPages: conversionResult.totalPages,
|
||||
processedPages,
|
||||
failedPages,
|
||||
results,
|
||||
cost: totalCost,
|
||||
duration,
|
||||
mode: 'precise',
|
||||
};
|
||||
} catch (error) {
|
||||
this.logger.error(`Precise mode failed: ${error.message}`);
|
||||
|
||||
// Try to clean up temp files
|
||||
try {
|
||||
if (pdfPath !== filePath && pdfPath !== filePath) {
|
||||
await fs.unlink(pdfPath);
|
||||
}
|
||||
if (imagesToProcess.length > 0) {
|
||||
await this.pdf2Image.cleanupImages(imagesToProcess);
|
||||
}
|
||||
} catch {}
|
||||
|
||||
return {
|
||||
success: false,
|
||||
fileId: options.fileId,
|
||||
fileName: options.fileName,
|
||||
totalPages: 0,
|
||||
processedPages,
|
||||
failedPages,
|
||||
results: [],
|
||||
cost: totalCost,
|
||||
duration: (Date.now() - startTime) / 1000,
|
||||
mode: 'precise',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get Vision model configuration
|
||||
*/
|
||||
private async getVisionModelConfig(
|
||||
userId: string,
|
||||
modelId: string,
|
||||
tenantId?: string,
|
||||
): Promise<VisionModelConfig> {
|
||||
const config = await this.modelConfigService.findOne(modelId);
|
||||
|
||||
if (!config) {
|
||||
throw new Error(`Model config not found: ${modelId}`);
|
||||
}
|
||||
|
||||
// API key is optional - allows local models
|
||||
|
||||
return {
|
||||
baseUrl: config.baseUrl || '',
|
||||
apiKey: config.apiKey || '',
|
||||
modelId: config.modelId,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert to PDF
|
||||
*/
|
||||
private async convertToPDF(filePath: string): Promise<string> {
|
||||
const ext = path.extname(filePath).toLowerCase();
|
||||
|
||||
// Return as-is if already PDF
|
||||
if (ext === '.pdf') {
|
||||
return filePath;
|
||||
}
|
||||
|
||||
// Call LibreOffice to convert
|
||||
return await this.libreOffice.convertToPDF(filePath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Format detection and mode recommendation (with cost estimation)
|
||||
*/
|
||||
async recommendMode(filePath: string): Promise<ModeRecommendation> {
|
||||
const ext = path.extname(filePath).toLowerCase();
|
||||
const stats = await fs.stat(filePath);
|
||||
const sizeMB = stats.size / (1024 * 1024);
|
||||
|
||||
const supportedFormats = [
|
||||
'.pdf',
|
||||
'.doc',
|
||||
'.docx',
|
||||
'.ppt',
|
||||
'.pptx',
|
||||
'.xls',
|
||||
'.xlsx',
|
||||
];
|
||||
const preciseFormats = ['.pdf', '.doc', '.docx', '.ppt', '.pptx'];
|
||||
|
||||
if (!supportedFormats.includes(ext)) {
|
||||
return {
|
||||
recommendedMode: 'fast',
|
||||
reason: `Unsupported file format: ${ext}`,
|
||||
warnings: ['Using fast mode (text extraction only)'],
|
||||
};
|
||||
}
|
||||
|
||||
if (!preciseFormats.includes(ext)) {
|
||||
return {
|
||||
recommendedMode: 'fast',
|
||||
reason: `Format ${ext} does not support precise mode`,
|
||||
warnings: ['Using fast mode (text extraction only)'],
|
||||
};
|
||||
}
|
||||
|
||||
// Estimate page count（based on file size）
|
||||
const estimatedPages = Math.max(1, Math.ceil(sizeMB * 2));
|
||||
const costEstimate = this.costControl.estimateCost(estimatedPages);
|
||||
|
||||
// Recommend precise mode for large files
|
||||
if (sizeMB > 50) {
|
||||
return {
|
||||
recommendedMode: 'precise',
|
||||
reason:
|
||||
'File is large, recommend precise mode to preserve full content',
|
||||
estimatedCost: costEstimate.estimatedCost,
|
||||
estimatedTime: costEstimate.estimatedTime,
|
||||
warnings: [
|
||||
'Processing time may be longer',
|
||||
'API costs will be incurred',
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
// Recommend precise mode
|
||||
return {
|
||||
recommendedMode: 'precise',
|
||||
reason:
|
||||
'Precise mode available. Can preserve mixed text and image content',
|
||||
estimatedCost: costEstimate.estimatedCost,
|
||||
estimatedTime: costEstimate.estimatedTime,
|
||||
warnings: ['API costs will be incurred'],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get user quota information
|
||||
*/
|
||||
async getUserQuotaInfo(userId: string) {
|
||||
const quota = await this.costControl.getUserQuota(userId);
|
||||
const report = await this.costControl.getCostReport(userId);
|
||||
|
||||
return {
|
||||
...quota,
|
||||
report,
|
||||
warnings: await this.costControl.checkWarningThreshold(userId),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Update processing status (for real-time feedback)
|
||||
*/
|
||||
private updateStatus(
|
||||
status: ProcessingStatus['status'],
|
||||
progress: number,
|
||||
message: string,
|
||||
): void {
|
||||
this.logger.log(`[${status}] ${progress}% - ${message}`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
/**
|
||||
* Vision Pipeline Interface Definitions
|
||||
*/
|
||||
|
||||
import { VisionAnalysisResult } from '../vision/vision.interface';
|
||||
|
||||
export interface PreciseModeOptions {
|
||||
userId: string;
|
||||
tenantId: string;
|
||||
modelId: string;
|
||||
fileId: string;
|
||||
fileName: string;
|
||||
skipQualityCheck?: boolean;
|
||||
maxPages?: number;
|
||||
}
|
||||
|
||||
export interface PipelineResult {
|
||||
success: boolean;
|
||||
fileId: string;
|
||||
fileName: string;
|
||||
totalPages: number;
|
||||
processedPages: number;
|
||||
failedPages: number;
|
||||
results: VisionAnalysisResult[];
|
||||
cost: number;
|
||||
duration: number; // seconds
|
||||
mode: 'precise';
|
||||
}
|
||||
|
||||
export interface ProcessingStatus {
|
||||
status:
|
||||
| 'converting'
|
||||
| 'splitting'
|
||||
| 'checking'
|
||||
| 'analyzing'
|
||||
| 'indexing'
|
||||
| 'completed'
|
||||
| 'failed';
|
||||
progress: number;
|
||||
message: string;
|
||||
cost?: number;
|
||||
}
|
||||
|
||||
export interface FileFormat {
|
||||
extension: string;
|
||||
needsConversion: boolean;
|
||||
supported: boolean;
|
||||
}
|
||||
|
||||
export interface ModeRecommendation {
|
||||
recommendedMode: 'precise' | 'fast';
|
||||
reason: string;
|
||||
estimatedCost?: number;
|
||||
estimatedTime?: number; // seconds
|
||||
warnings?: string[];
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { VisionPipelineService } from './vision-pipeline.service';
|
||||
import { LibreOfficeModule } from '../libreoffice/libreoffice.module';
|
||||
import { Pdf2ImageModule } from '../pdf2image/pdf2image.module';
|
||||
import { VisionModule } from '../vision/vision.module';
|
||||
import { ModelConfigModule } from '../model-config/model-config.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
LibreOfficeModule,
|
||||
Pdf2ImageModule,
|
||||
VisionModule,
|
||||
ModelConfigModule,
|
||||
],
|
||||
providers: [VisionPipelineService],
|
||||
exports: [VisionPipelineService],
|
||||
})
|
||||
export class VisionPipelineModule {}
|
||||
@@ -0,0 +1,364 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import * as fs from 'fs/promises';
|
||||
import * as path from 'path';
|
||||
import { LibreOfficeService } from '../libreoffice/libreoffice.service';
|
||||
import { Pdf2ImageService } from '../pdf2image/pdf2image.service';
|
||||
import { VisionService } from '../vision/vision.service';
|
||||
import { ModelConfigService } from '../model-config/model-config.service';
|
||||
import {
|
||||
ModeRecommendation,
|
||||
PipelineResult,
|
||||
PreciseModeOptions,
|
||||
ProcessingStatus,
|
||||
} from './vision-pipeline.interface';
|
||||
import {
|
||||
VisionAnalysisResult,
|
||||
VisionModelConfig,
|
||||
} from '../vision/vision.interface';
|
||||
import { I18nService } from '../i18n/i18n.service';
|
||||
|
||||
@Injectable()
|
||||
export class VisionPipelineService {
|
||||
private readonly logger = new Logger(VisionPipelineService.name);
|
||||
|
||||
constructor(
|
||||
private libreOffice: LibreOfficeService,
|
||||
private pdf2Image: Pdf2ImageService,
|
||||
private vision: VisionService,
|
||||
private modelConfigService: ModelConfigService,
|
||||
private configService: ConfigService,
|
||||
private i18nService: I18nService,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Main processing flow: Precise mode
|
||||
* Returns the processing result, and the caller is responsible for vectorization and indexing.
|
||||
*/
|
||||
async processPreciseMode(
|
||||
filePath: string,
|
||||
options: PreciseModeOptions,
|
||||
): Promise<PipelineResult> {
|
||||
const startTime = Date.now();
|
||||
const results: VisionAnalysisResult[] = [];
|
||||
let processedPages = 0;
|
||||
let failedPages = 0;
|
||||
let totalCost = 0;
|
||||
let pdfPath = filePath;
|
||||
let imagesToProcess: any[] = [];
|
||||
|
||||
this.logger.log(
|
||||
`🚀 Starting precise mode processing: ${options.fileName} (User: ${options.userId})`,
|
||||
);
|
||||
|
||||
try {
|
||||
// Step 1: Unification of formats
|
||||
this.logger.log('📄 Step 1/4: Unification of formats');
|
||||
this.updateStatus('converting', 10, 'Converting document format...');
|
||||
try {
|
||||
pdfPath = await this.convertToPDF(filePath);
|
||||
this.logger.log(`✅ Format conversion completed: ${pdfPath}`);
|
||||
} catch (convertError) {
|
||||
this.logger.error(
|
||||
`❌ Format conversion failed: ${convertError.message}`,
|
||||
);
|
||||
throw convertError;
|
||||
}
|
||||
|
||||
// Step 2: Conversion from PDF to images
|
||||
this.logger.log('🖼️ Step 2/4: Conversion from PDF to images');
|
||||
this.updateStatus('splitting', 30, 'Converting PDF to images...');
|
||||
let conversionResult;
|
||||
try {
|
||||
conversionResult = await this.pdf2Image.convertToImages(pdfPath, {
|
||||
density: 300,
|
||||
quality: 85,
|
||||
format: 'jpeg',
|
||||
});
|
||||
} catch (imageError) {
|
||||
this.logger.error(
|
||||
`❌ PDF to image conversion failed: ${imageError.message}`,
|
||||
);
|
||||
throw imageError;
|
||||
}
|
||||
|
||||
if (conversionResult.images.length === 0) {
|
||||
throw new Error(
|
||||
this.i18nService.getMessage('pdfToImageConversionFailed'),
|
||||
);
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`✅ PDF to image conversion completed: Total ${conversionResult.totalPages} pages, ${conversionResult.images.length} images generated`,
|
||||
);
|
||||
|
||||
// Limit the number of pages to process
|
||||
imagesToProcess = options.maxPages
|
||||
? conversionResult.images.slice(0, options.maxPages)
|
||||
: conversionResult.images;
|
||||
|
||||
this.logger.log(
|
||||
`📊 Processing ${imagesToProcess.length} pages (${options.maxPages ? 'limited' : 'all'})`,
|
||||
);
|
||||
|
||||
// Step 3: Get Vision model configuration
|
||||
this.logger.log('🤖 Step 3/4: Preparation of Vision model');
|
||||
const modelConfig = await this.getVisionModelConfig(
|
||||
options.modelId,
|
||||
);
|
||||
this.logger.log(
|
||||
`✅ Vision model configuration completed: ${modelConfig.modelId}`,
|
||||
);
|
||||
|
||||
// Step 4: VL model analysis
|
||||
this.logger.log('🔍 Step 4/4: Vision model analysis');
|
||||
this.updateStatus(
|
||||
'analyzing',
|
||||
50,
|
||||
'Analyzing pages using Vision model...',
|
||||
);
|
||||
|
||||
// Display processing progress of each page
|
||||
this.logger.log(
|
||||
`Starting analysis of ${imagesToProcess.length} page contents...`,
|
||||
);
|
||||
const batchResult = await this.vision.batchAnalyze(
|
||||
imagesToProcess.map((img) => img.path),
|
||||
modelConfig,
|
||||
{
|
||||
startIndex: 1,
|
||||
skipQualityCheck: options.skipQualityCheck,
|
||||
onProgress: (current: number, total: number, pageResult?: any) => {
|
||||
const progress = Math.round((current / total) * 100);
|
||||
this.logger.log(
|
||||
`📄 Processing progress: ${current}/${total} (${progress}%) ${pageResult ? `- Page ${pageResult.pageIndex} completed` : ''}`,
|
||||
);
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
totalCost = batchResult.estimatedCost;
|
||||
processedPages = batchResult.successCount;
|
||||
failedPages = batchResult.failedCount;
|
||||
results.push(...batchResult.results);
|
||||
|
||||
this.logger.log(
|
||||
`✅ Vision analysis completed: Success ${processedPages} pages, Fail ${failedPages} pages, Cost $${totalCost.toFixed(2)}`,
|
||||
);
|
||||
|
||||
// Step 5: Cleanup of temporary files (images)
|
||||
this.logger.log('🧹 Cleaning up temporary files...');
|
||||
this.updateStatus(
|
||||
'completed',
|
||||
100,
|
||||
'Processing completed. Cleaning up temporary files...',
|
||||
);
|
||||
await this.pdf2Image.cleanupImages(imagesToProcess);
|
||||
|
||||
// If converted to PDF, clean up the converted file
|
||||
if (pdfPath !== filePath) {
|
||||
try {
|
||||
await fs.unlink(pdfPath);
|
||||
this.logger.log('🗑️ Cleaned up converted PDF file');
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
`⚠️ Failed to clean up converted PDF: ${error.message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const duration = (Date.now() - startTime) / 1000;
|
||||
|
||||
this.logger.log(
|
||||
`🎉 Precise mode processing completed! ` +
|
||||
`📊 Statistics: ${processedPages}/${imagesToProcess.length} pages success, ` +
|
||||
`💰 Cost: $${totalCost.toFixed(2)}, ` +
|
||||
`⏱️ Duration: ${duration.toFixed(1)}s`,
|
||||
);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
fileId: options.fileId,
|
||||
fileName: options.fileName,
|
||||
totalPages: conversionResult.totalPages,
|
||||
processedPages,
|
||||
failedPages,
|
||||
results,
|
||||
cost: totalCost,
|
||||
duration,
|
||||
mode: 'precise',
|
||||
};
|
||||
} catch (error) {
|
||||
this.logger.error(`❌ Precise mode processing failed: ${error.message}`);
|
||||
|
||||
// Attempting cleanup of temporary files
|
||||
try {
|
||||
if (pdfPath !== filePath && pdfPath !== filePath) {
|
||||
await fs.unlink(pdfPath);
|
||||
}
|
||||
if (imagesToProcess.length > 0) {
|
||||
await this.pdf2Image.cleanupImages(imagesToProcess);
|
||||
}
|
||||
this.logger.log('🧹 Cleaned up temporary files');
|
||||
} catch {
|
||||
this.logger.warn('⚠️ Failed to clean up temporary files');
|
||||
}
|
||||
|
||||
return {
|
||||
success: false,
|
||||
fileId: options.fileId,
|
||||
fileName: options.fileName,
|
||||
totalPages: 0,
|
||||
processedPages,
|
||||
failedPages,
|
||||
results: [],
|
||||
cost: totalCost,
|
||||
duration: (Date.now() - startTime) / 1000,
|
||||
mode: 'precise',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get Vision model configuration
|
||||
*/
|
||||
private async getVisionModelConfig(
|
||||
modelId: string,
|
||||
): Promise<VisionModelConfig> {
|
||||
const config = await this.modelConfigService.findOne(
|
||||
modelId,
|
||||
);
|
||||
|
||||
if (!config) {
|
||||
throw new Error(`Model configuration not found: ${modelId}`);
|
||||
}
|
||||
|
||||
// API key is optional - Allows local models
|
||||
|
||||
return {
|
||||
baseUrl: config.baseUrl || '',
|
||||
apiKey: config.apiKey || '',
|
||||
modelId: config.modelId,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Conversion to PDF
|
||||
*/
|
||||
private async convertToPDF(filePath: string): Promise<string> {
|
||||
const ext = path.extname(filePath).toLowerCase();
|
||||
|
||||
// If already PDF, return as is
|
||||
if (ext === '.pdf') {
|
||||
return filePath;
|
||||
}
|
||||
|
||||
// Call LibreOffice to convert
|
||||
const containerPdfPath = await this.libreOffice.convertToPDF(filePath);
|
||||
|
||||
// The path returned from the LibreOffice container is already correct. All point to the same uploads directory.
|
||||
// Inside container: /uploads/xxx.pdf -> Host machine: ../uploads/xxx.pdf
|
||||
const hostPdfPath = containerPdfPath.startsWith('/uploads/')
|
||||
? path.join('..', containerPdfPath) // ../uploads/xxx.pdf
|
||||
: containerPdfPath;
|
||||
|
||||
this.logger.log(`Path conversion: ${containerPdfPath} -> ${hostPdfPath}`);
|
||||
|
||||
// Check existence of file
|
||||
try {
|
||||
await fs.access(hostPdfPath);
|
||||
return hostPdfPath;
|
||||
} catch (error) {
|
||||
this.logger.error(`PDF file does not exist: ${hostPdfPath}`);
|
||||
throw new Error(`PDF file does not exist: ${hostPdfPath}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Index results to Elasticsearch (using knowledge-base embedding service)
|
||||
* Note: This method requires the embedding service, so it should be called within knowledge-base.service.
|
||||
*/
|
||||
private async indexResults(
|
||||
results: any[],
|
||||
options: PreciseModeOptions,
|
||||
): Promise<void> {
|
||||
// This method is currently called from knowledge-base.service
|
||||
// vision-pipeline is only responsible for processing and returning results.
|
||||
this.logger.log(
|
||||
`indexResults called with ${results.length} results - should be handled by knowledge-base service`,
|
||||
);
|
||||
throw new Error(
|
||||
'VisionPipelineService.indexResults should not be called directly. Use knowledge-base service instead.',
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Format detection and mode recommendation
|
||||
*/
|
||||
async recommendMode(filePath: string): Promise<ModeRecommendation> {
|
||||
const ext = path.extname(filePath).toLowerCase();
|
||||
const stats = await fs.stat(filePath);
|
||||
const sizeMB = stats.size / (1024 * 1024);
|
||||
|
||||
const supportedFormats = [
|
||||
'.pdf',
|
||||
'.doc',
|
||||
'.docx',
|
||||
'.ppt',
|
||||
'.pptx',
|
||||
'.xls',
|
||||
'.xlsx',
|
||||
];
|
||||
const preciseFormats = ['.pdf', '.doc', '.docx', '.ppt', '.pptx'];
|
||||
|
||||
if (!supportedFormats.includes(ext)) {
|
||||
return {
|
||||
recommendedMode: 'fast',
|
||||
reason: `Unsupported file format: ${ext}`,
|
||||
warnings: ['Using fast mode (text extraction only)'],
|
||||
};
|
||||
}
|
||||
|
||||
if (!preciseFormats.includes(ext)) {
|
||||
return {
|
||||
recommendedMode: 'fast',
|
||||
reason: `Format ${ext} does not support precise mode`,
|
||||
warnings: ['Using fast mode (text extraction only)'],
|
||||
};
|
||||
}
|
||||
|
||||
// File size check
|
||||
if (sizeMB > 50) {
|
||||
return {
|
||||
recommendedMode: 'precise',
|
||||
reason:
|
||||
'The file is large, so precise mode is recommended to retain full information.',
|
||||
estimatedCost: sizeMB * 0.01, // Rough estimate
|
||||
estimatedTime: sizeMB * 12, // Approx. 12 seconds per 1MB
|
||||
warnings: ['Processing time may be long', 'API costs will occur'],
|
||||
};
|
||||
}
|
||||
|
||||
// Precise mode recommended
|
||||
return {
|
||||
recommendedMode: 'precise',
|
||||
reason:
|
||||
'Precise mode is available. Can retain mixed content of text and images.',
|
||||
estimatedCost: sizeMB * 0.01,
|
||||
estimatedTime: sizeMB * 10,
|
||||
warnings: ['API costs will occur'],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Update processing status (for real-time feedback)
|
||||
*/
|
||||
private updateStatus(
|
||||
status: ProcessingStatus['status'],
|
||||
progress: number,
|
||||
message: string,
|
||||
): void {
|
||||
// You can send WebSocket messages or update the database here.
|
||||
this.logger.log(`[${status}] ${progress}% - ${message}`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
/**
|
||||
* Vision Service Interface Definitions
|
||||
*/
|
||||
|
||||
export interface VisionAnalysisResult {
|
||||
text: string; // Extracted text content
|
||||
images: ImageDescription[]; // Image description
|
||||
layout: string; // Layout type
|
||||
confidence: number; // Confidence (0-1)
|
||||
pageIndex?: number; // Page number
|
||||
}
|
||||
|
||||
export interface ImageDescription {
|
||||
type: string; // Image type (chart/diagram/flowchart etc.)
|
||||
description: string; // Detailed description
|
||||
position?: number; // Position in page
|
||||
}
|
||||
|
||||
export interface VisionModelConfig {
|
||||
baseUrl: string;
|
||||
apiKey: string;
|
||||
modelId: string;
|
||||
}
|
||||
|
||||
export interface BatchAnalysisResult {
|
||||
results: VisionAnalysisResult[];
|
||||
totalPages: number;
|
||||
successCount: number;
|
||||
failedCount: number;
|
||||
estimatedCost: number; // Estimated cost(USD)
|
||||
}
|
||||
|
||||
export interface PageQuality {
|
||||
isGood: boolean;
|
||||
reason?: string;
|
||||
score?: number;
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { VisionService } from './vision.service';
|
||||
|
||||
@Module({
|
||||
providers: [VisionService],
|
||||
exports: [VisionService],
|
||||
})
|
||||
export class VisionModule {}
|
||||
@@ -0,0 +1,410 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { I18nService } from '../i18n/i18n.service';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { ChatOpenAI } from '@langchain/openai';
|
||||
import { HumanMessage } from '@langchain/core/messages';
|
||||
import * as fs from 'fs/promises';
|
||||
import {
|
||||
VisionAnalysisResult,
|
||||
VisionModelConfig,
|
||||
BatchAnalysisResult,
|
||||
ImageDescription,
|
||||
} from './vision.interface';
|
||||
|
||||
@Injectable()
|
||||
export class VisionService {
|
||||
private readonly logger = new Logger(VisionService.name);
|
||||
|
||||
constructor(
|
||||
private configService: ConfigService,
|
||||
private i18nService: I18nService,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Analyze single image (document page)
|
||||
*/
|
||||
async analyzeImage(
|
||||
imagePath: string,
|
||||
modelConfig: VisionModelConfig,
|
||||
pageIndex?: number,
|
||||
): Promise<VisionAnalysisResult> {
|
||||
const maxRetries = 3;
|
||||
const baseDelay = 3000; // 3 second base delay
|
||||
|
||||
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
||||
try {
|
||||
return await this.performAnalysis(imagePath, modelConfig, pageIndex);
|
||||
} catch (error) {
|
||||
const isRetryableError = this.isRetryableError(error);
|
||||
|
||||
if (attempt === maxRetries || !isRetryableError) {
|
||||
throw new Error(
|
||||
this.i18nService.formatMessage('visionAnalysisFailed', {
|
||||
message: error.message,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
const delay = baseDelay + Math.random() * 2000; // 3-5 second random delay
|
||||
this.logger.warn(
|
||||
`⚠️ Failed to analyze page ${pageIndex || '?'} (${attempt}/${maxRetries}), retrying in ${delay.toFixed(0)}ms: ${error.message}`,
|
||||
);
|
||||
|
||||
await this.sleep(delay);
|
||||
}
|
||||
}
|
||||
|
||||
// This line theoretically should not execute, but included to satisfy TypeScript
|
||||
throw new Error(this.i18nService.getMessage('retryMechanismError'));
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform actual image analysis
|
||||
*/
|
||||
private async performAnalysis(
|
||||
imagePath: string,
|
||||
modelConfig: VisionModelConfig,
|
||||
pageIndex?: number,
|
||||
): Promise<VisionAnalysisResult> {
|
||||
try {
|
||||
// Load image and convert to base64
|
||||
const imageBuffer = await fs.readFile(imagePath);
|
||||
const base64Image = imageBuffer.toString('base64');
|
||||
const mimeType = this.getMimeType(imagePath);
|
||||
|
||||
// Create vision model instance
|
||||
const model = new ChatOpenAI({
|
||||
apiKey: modelConfig.apiKey,
|
||||
model: modelConfig.modelId,
|
||||
configuration: {
|
||||
baseURL: modelConfig.baseUrl,
|
||||
},
|
||||
temperature: 0.1, // Reduce randomness, increase consistency
|
||||
});
|
||||
|
||||
// Build professional document analysis prompt
|
||||
const systemPrompt = this.i18nService.getMessage('visionSystemPrompt');
|
||||
|
||||
const message = new HumanMessage({
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: systemPrompt,
|
||||
},
|
||||
{
|
||||
type: 'image_url',
|
||||
image_url: {
|
||||
url: `data:${mimeType};base64,${base64Image}`,
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
// Call model
|
||||
this.logger.log(
|
||||
this.i18nService.formatMessage('visionModelCall', {
|
||||
model: modelConfig.modelId,
|
||||
page: pageIndex || 'single',
|
||||
}),
|
||||
);
|
||||
const response = await model.invoke([message]);
|
||||
let content = response.content as string;
|
||||
|
||||
// Try to parse JSON
|
||||
let result: VisionAnalysisResult;
|
||||
try {
|
||||
// Clean up markdown code block tags
|
||||
content = content
|
||||
.replace(/```json/g, '')
|
||||
.replace(/```/g, '')
|
||||
.trim();
|
||||
const parsed = JSON.parse(content);
|
||||
|
||||
result = {
|
||||
text: parsed.text || '',
|
||||
images: parsed.images || [],
|
||||
layout: parsed.layout || 'unknown',
|
||||
confidence: parsed.confidence ?? 0.8,
|
||||
pageIndex,
|
||||
};
|
||||
} catch (parseError) {
|
||||
// If parsing fails, treat entire content as text
|
||||
this.logger.warn(
|
||||
`Failed to parse JSON response for ${imagePath}, using raw text`,
|
||||
);
|
||||
result = {
|
||||
text: content,
|
||||
images: [],
|
||||
layout: 'unknown',
|
||||
confidence: 0.5,
|
||||
pageIndex,
|
||||
};
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
this.i18nService.formatMessage('visionAnalysisSuccess', {
|
||||
path: imagePath,
|
||||
page: pageIndex ? ` (page ${pageIndex})` : '',
|
||||
textLen: result.text.length,
|
||||
imgCount: result.images.length,
|
||||
layout: result.layout,
|
||||
confidence: (result.confidence * 100).toFixed(1),
|
||||
}),
|
||||
);
|
||||
|
||||
return result;
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
this.i18nService.formatMessage('visionAnalysisFailed', {
|
||||
message: error.message,
|
||||
}),
|
||||
);
|
||||
this.logger.error(`Vision analysis error details: ${error.stack}`);
|
||||
throw error; // Re-throw error for retry mechanism
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine if error is retryable
|
||||
*/
|
||||
private isRetryableError(error: any): boolean {
|
||||
const errorMessage = error.message?.toLowerCase() || '';
|
||||
const errorCode = error.status || error.code;
|
||||
|
||||
// 429 rate limit error
|
||||
if (
|
||||
errorCode === 429 ||
|
||||
errorMessage.includes('rate limit') ||
|
||||
errorMessage.includes('too many requests')
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// 5xx server error
|
||||
if (errorCode >= 500 && errorCode < 600) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Network related error
|
||||
if (
|
||||
errorMessage.includes('timeout') ||
|
||||
errorMessage.includes('network') ||
|
||||
errorMessage.includes('connection')
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sleep function
|
||||
*/
|
||||
private sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch analyze multiple images
|
||||
*/
|
||||
async batchAnalyze(
|
||||
imagePaths: string[],
|
||||
modelConfig: VisionModelConfig,
|
||||
options: {
|
||||
startIndex?: number;
|
||||
skipQualityCheck?: boolean;
|
||||
onProgress?: (
|
||||
current: number,
|
||||
total: number,
|
||||
pageResult?: VisionAnalysisResult,
|
||||
) => void;
|
||||
} = {},
|
||||
): Promise<BatchAnalysisResult> {
|
||||
const { startIndex = 1, skipQualityCheck = false, onProgress } = options;
|
||||
const results: VisionAnalysisResult[] = [];
|
||||
let successCount = 0;
|
||||
let failedCount = 0;
|
||||
|
||||
this.logger.log(
|
||||
this.i18nService.formatMessage('batchAnalysisStarted', {
|
||||
count: imagePaths.length,
|
||||
}),
|
||||
);
|
||||
this.logger.log(
|
||||
`🔧 Model config: ${modelConfig.modelId} (${modelConfig.baseUrl || 'OpenAI'})`,
|
||||
);
|
||||
|
||||
for (let i = 0; i < imagePaths.length; i++) {
|
||||
const imagePath = imagePaths[i];
|
||||
const pageIndex = startIndex + i;
|
||||
const progress = Math.round(((i + 1) / imagePaths.length) * 100);
|
||||
|
||||
this.logger.log(
|
||||
`🖼️ Analyzing page ${pageIndex} (${i + 1}/${imagePaths.length}, ${progress}%)`,
|
||||
);
|
||||
|
||||
// Call progress callback
|
||||
if (onProgress) {
|
||||
onProgress(i + 1, imagePaths.length);
|
||||
}
|
||||
|
||||
// Quality check(skip analysis if skipped)
|
||||
if (!skipQualityCheck) {
|
||||
const quality = await this.checkImageQuality(imagePath);
|
||||
if (!quality.isGood) {
|
||||
this.logger.warn(
|
||||
`⚠️ Skipped page ${pageIndex} (poor quality): ${quality.reason}`,
|
||||
);
|
||||
failedCount++;
|
||||
continue;
|
||||
} else {
|
||||
this.logger.log(
|
||||
`✅ Page ${pageIndex} quality check passed (score: ${(quality.score || 0).toFixed(2)})`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
this.logger.log(`🔍 Analyzing page ${pageIndex} with Vision model...`);
|
||||
const startTime = Date.now();
|
||||
const result = await this.analyzeImage(
|
||||
imagePath,
|
||||
modelConfig,
|
||||
pageIndex,
|
||||
);
|
||||
const duration = ((Date.now() - startTime) / 1000).toFixed(1);
|
||||
|
||||
results.push(result);
|
||||
successCount++;
|
||||
|
||||
this.logger.log(
|
||||
`✅ Page ${pageIndex} analysis completed (time: ${duration}s, ` +
|
||||
`text: ${result.text.length} chars, ` +
|
||||
`images: ${result.images.length}, ` +
|
||||
`confidence: ${(result.confidence * 100).toFixed(1)}%)`,
|
||||
);
|
||||
|
||||
// Call progress callback with result
|
||||
if (onProgress) {
|
||||
onProgress(i + 1, imagePaths.length, result);
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
this.i18nService.formatMessage('pageAnalysisFailed', {
|
||||
page: pageIndex,
|
||||
}) + `: ${error.message}`,
|
||||
);
|
||||
failedCount++;
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate estimated cost (assuming $0.01 per image)
|
||||
const estimatedCost = successCount * 0.01;
|
||||
|
||||
this.logger.log(
|
||||
`🎉 Vision batch analysis completed! ` +
|
||||
`✅ Success: ${successCount} pages, ❌ Failed: ${failedCount} pages, ` +
|
||||
`💰 Estimated cost: $${estimatedCost.toFixed(2)}`,
|
||||
);
|
||||
|
||||
return {
|
||||
results,
|
||||
totalPages: imagePaths.length,
|
||||
successCount,
|
||||
failedCount,
|
||||
estimatedCost,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Check image quality
|
||||
*/
|
||||
async checkImageQuality(
|
||||
imagePath: string,
|
||||
): Promise<{ isGood: boolean; reason?: string; score?: number }> {
|
||||
try {
|
||||
const stats = await fs.stat(imagePath);
|
||||
const sizeKB = stats.size / 1024;
|
||||
|
||||
// Check file size(5KB+)
|
||||
if (sizeKB < 5) {
|
||||
return {
|
||||
isGood: false,
|
||||
reason: `File too small (${sizeKB.toFixed(2)}KB)`,
|
||||
score: 0,
|
||||
};
|
||||
}
|
||||
|
||||
// Check file size limit(10MB)
|
||||
if (sizeKB > 10240) {
|
||||
return {
|
||||
isGood: false,
|
||||
reason: `File too large (${sizeKB.toFixed(2)}KB)`,
|
||||
score: 0,
|
||||
};
|
||||
}
|
||||
|
||||
// Simple quality scoring
|
||||
let score = 0.5;
|
||||
if (sizeKB > 50) score += 0.2;
|
||||
if (sizeKB > 100) score += 0.2;
|
||||
if (sizeKB > 500) score += 0.1;
|
||||
|
||||
score = Math.min(score, 1.0);
|
||||
|
||||
return { isGood: true, score };
|
||||
} catch (error) {
|
||||
return {
|
||||
isGood: false,
|
||||
reason: this.i18nService.formatMessage('imageLoadError', {
|
||||
message: error.message,
|
||||
}),
|
||||
score: 0,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if file is a supported image format
|
||||
*/
|
||||
isImageFile(mimetype: string): boolean {
|
||||
const imageMimeTypes = [
|
||||
'image/jpeg',
|
||||
'image/jpg',
|
||||
'image/png',
|
||||
'image/gif',
|
||||
'image/bmp',
|
||||
'image/webp',
|
||||
];
|
||||
return imageMimeTypes.includes(mimetype);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get MIME type
|
||||
*/
|
||||
private getMimeType(filePath: string): string {
|
||||
const ext = filePath.toLowerCase().split('.').pop();
|
||||
if (!ext) return 'image/jpeg';
|
||||
|
||||
const mimeTypes: Record<string, string> = {
|
||||
jpg: 'image/jpeg',
|
||||
jpeg: 'image/jpeg',
|
||||
png: 'image/png',
|
||||
gif: 'image/gif',
|
||||
bmp: 'image/bmp',
|
||||
webp: 'image/webp',
|
||||
};
|
||||
return mimeTypes[ext] || 'image/jpeg';
|
||||
}
|
||||
|
||||
/**
|
||||
* Legacy interface compatibility: extract content from single image
|
||||
*/
|
||||
async extractImageContent(
|
||||
imagePath: string,
|
||||
modelConfig: { baseUrl: string; apiKey: string; modelId: string },
|
||||
): Promise<string> {
|
||||
const result = await this.analyzeImage(imagePath, modelConfig);
|
||||
return result.text;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user