F1-F10: audit fixes (dimension normalize, passingScore scale, DB defaults, onDelete, item status filter, timeout event type, userId privacy) + generator.node.ts strict prompt rules (anti-hallucination)

This commit is contained in:
Developer
2026-05-20 11:13:37 +08:00
parent 29bac74b58
commit 83483d8117
13 changed files with 205 additions and 202 deletions
+11 -5
View File
@@ -137,6 +137,13 @@ private async getModel(tenantId: string): Promise<ChatOpenAI> {
return result;
}
private normalizeDimension(dim: string): string {
const lower = dim.toLowerCase();
if (lower === 'dev_pattern') return 'devPattern';
if (lower === 'work_capability') return 'workCapability';
return lower;
}
private calculateScores(
questions: any[],
scores: Record<string, number>,
@@ -157,7 +164,7 @@ private async getModel(tenantId: string): Promise<ChatOpenAI> {
};
questions.forEach((q: any, idx: number) => {
const dimension = q.dimension || 'workCapability';
const dimension = this.normalizeDimension(q.dimension || 'workCapability');
const score = scores[q.id || idx.toString()] || 0;
if (dimensionScoresMap[dimension]) {
dimensionScoresMap[dimension].push(score);
@@ -729,7 +736,7 @@ const initialState: Partial<EvaluationState> = {
const scores = finalData.scores;
const questions = finalData.questions || [];
const weightConfig = session.templateJson?.weightConfig || { prompt: 50, other: 50 };
const passingScore = session.templateJson?.passingScore || 90;
const passingScore = (session.templateJson?.passingScore ?? 90) / 10;
if (questions.length > 0 && Object.keys(scores).length > 0) {
const { finalScore, dimensionScores, radarData } = this.calculateScores(
@@ -820,7 +827,7 @@ const initialState: Partial<EvaluationState> = {
let finalResult: any = null;
const weightConfig = session.templateJson?.weightConfig || { prompt: 50, other: 50 };
const passingScore = session.templateJson?.passingScore || 90;
const passingScore = (session.templateJson?.passingScore ?? 90) / 10;
// Resume from the last interrupt (typically after interviewer)
const stream = await this.graph.stream(null, {
@@ -965,6 +972,7 @@ const initialState: Partial<EvaluationState> = {
await this.sessionRepository.save(session);
this.logger.log(`[submitAnswerStream] Session ${sessionId} auto-ended due to timeout`);
observer.next({
type: 'final',
assessmentSessionId: sessionId,
status: 'COMPLETED',
timeout: true,
@@ -1713,7 +1721,6 @@ const initialState: Partial<EvaluationState> = {
totalScore: number;
passed: boolean;
issuedAt: Date;
userId: string;
};
message?: string;
}> {
@@ -1734,7 +1741,6 @@ const initialState: Partial<EvaluationState> = {
totalScore: certificate.totalScore,
passed: certificate.passed,
issuedAt: certificate.issuedAt,
userId: certificate.userId,
},
};
}
@@ -22,6 +22,7 @@ import {
ReviewDto,
} from '../services/question-bank.service';
import { CombinedAuthGuard } from '../../auth/combined-auth.guard';
import { KnowledgeGroupService } from '../../knowledge-group/knowledge-group.service';
@Controller('question-banks')
@UseGuards(CombinedAuthGuard)
@@ -29,12 +30,20 @@ import { CombinedAuthGuard } from '../../auth/combined-auth.guard';
export class QuestionBankController {
private readonly logger = new Logger(QuestionBankController.name);
constructor(private readonly questionBankService: QuestionBankService) {}
constructor(
private readonly questionBankService: QuestionBankService,
private readonly groupService: KnowledgeGroupService,
) {}
@Post()
create(@Body() createDto: CreateQuestionBankDto, @Req() req: any) {
this.logger.log(`Creating question bank: ${createDto.name}`);
return this.questionBankService.create(createDto, req.user.id, req.user.tenantId);
async create(@Body() createDto: CreateQuestionBankDto, @Req() req: any) {
try {
this.logger.log(`Creating question bank: ${createDto.name}, user: ${req.user?.id}, tenant: ${req.user?.tenantId}`);
return await this.questionBankService.create(createDto, req.user.id, req.user.tenantId);
} catch (err: any) {
this.logger.error(`[create] Failed: ${err.message}`, err.stack);
throw err;
}
}
@Get()
@@ -125,11 +134,32 @@ export class QuestionBankController {
@Body() body: { count: number; knowledgeBaseContent?: string },
@Req() req: any,
) {
this.logger.log(`[generate] Generating ${body.count} questions for bank ${bankId}`);
let content = body.knowledgeBaseContent || '';
if (!content || content.trim().length < 10) {
try {
const bank = await this.questionBankService.findOne(bankId);
if (bank?.template?.knowledgeGroupId) {
const files = await this.groupService.getGroupFiles(
bank.template.knowledgeGroupId,
req.user.id,
req.user.tenantId,
);
content = files
.filter((f: any) => f.content && f.content.trim().length > 0)
.map((f: any) => `--- ${f.title || f.originalName || 'Document'} ---\n${f.content}`)
.join('\n\n');
this.logger.log(`[generate] Auto-loaded ${files.length} files, content length: ${content.length}`);
}
} catch (err: any) {
this.logger.warn(`[generate] Auto-load failed: ${err.message}`);
}
}
this.logger.log(`[generate] Generating ${body.count} questions for bank ${bankId}, content length: ${content.length}`);
return this.questionBankService.generateQuestions(
bankId,
body.count,
body.knowledgeBaseContent || '',
content,
req.user.tenantId,
);
}
@@ -56,6 +56,7 @@ export class QuestionBankItem {
@Column({
type: 'simple-enum',
enum: QuestionType,
default: QuestionType.SHORT_ANSWER,
})
questionType: QuestionType;
@@ -71,12 +72,14 @@ export class QuestionBankItem {
@Column({
type: 'simple-enum',
enum: QuestionDifficulty,
default: QuestionDifficulty.STANDARD,
})
difficulty: QuestionDifficulty;
@Column({
type: 'simple-enum',
enum: QuestionDimension,
default: QuestionDimension.PROMPT,
})
dimension: QuestionDimension;
@@ -89,6 +92,7 @@ export class QuestionBankItem {
@Column({
type: 'simple-enum',
enum: QuestionBankItemStatus,
default: QuestionBankItemStatus.PENDING_REVIEW,
})
status: QuestionBankItemStatus;
@@ -37,7 +37,7 @@ export class QuestionBank {
@Column({ name: 'template_id', nullable: true })
templateId: string | null;
@OneToOne(() => AssessmentTemplate, { nullable: true })
@OneToOne(() => AssessmentTemplate, { nullable: true, onDelete: 'SET NULL' })
@JoinColumn({ name: 'template_id' })
template: AssessmentTemplate;
@@ -89,12 +89,18 @@ export const questionGeneratorNode = async (
.map((q, i) => `Q${i + 1}: ${q.questionText}`)
.join('\n');
const systemPromptZh = `你是一位专业的知识评估专家。请根据提供的知识库片段生成 1 个唯一的测试题目。
const systemPromptZh = `你是一位严格的知识评估专家。你必须**仅基于**下方提供的知识库内容来生成测试题目。
### 核心铁律(违反将导致题目无效):
1. **所有题目必须直接来源于提供的知识库内容**,每个题目必须能找到对应的原文依据
2. **绝对禁止**编造知识库内容中未提及的概念、术语、流程或数据
3. **绝对禁止**使用你自身知识库中的内容来编造题目
4. 如果知识库内容不足以出题,诚实地报告而不是编造
### 强制性语言规则:
**必须使用中文 (Simplified Chinese) 进行回复**。即使知识库内容是英文或其他语言,问题(question_text)和关键点(key_points)也必须使用中文。
**必须使用中文 (Simplified Chinese) 进行回复**。
### 强制性多样性规则:
### 多样性规则:
${rulesZh}
### 禁止重复列表(已出过):
@@ -111,15 +117,21 @@ ${hasKeywords ? `目标关键词:${keywordText}\n` : ''}出题风格:${style
"key_points": ["点1", "点2"],
"difficulty": "...",
"dimension": "prompt/llm/ide/devPattern/workCapability",
"basis": "[n] 引用原文..."
"basis": "【必须填写】从知识库中引用与此题相关的原文内容,用引号标注来源段落"
}
]`;
// dimension取值:prompt=提示词, llm=LLM原理, ide=IDE协作, devPattern=开发范式, workCapability=工作能力
const systemPromptJa = `あなたは専門的なアセスメントエキスパートです。提供されたナレッジベースに基づいて、ユニークな問題を 1 つ作成してください。
const systemPromptJa = `あなたは厳格な知識評価の専門家です。提供されたナレッジベースの内容**のみ**に基づいて問題を作成してください。
### 核心鉄則(違反した問題は無効):
1. **すべての問題は提供されたナレッジベースから直接導出**し、各問題に原文の根拠が必要
2. **絶対禁止**:ナレッジベースに記載されていない概念、用語、プロセス、データを作り出すこと
3. **絶対禁止**:自身の知識ベースの内容を問題として使用すること
4. 内容が不十分な場合は、正直に報告し、捏造しないこと
### 言語ルール(最重要):
**必ず日本語で作成してください**。提供されたナレッジベースが英語や中国語、その他の言語であっても、質問文(question_text)およびキーポイント(key_points)は必ず日本語で回答してください。中国語が混ざらないように厳格に注意してください。
**必ず日本語で作成してください**。中国語が混ざらないように厳格に注意してください。
### 多様性ルール:
${rulesJa}
@@ -138,11 +150,17 @@ ${hasKeywords ? `目標キーワード:${keywordText}\n` : ''}出題スタイ
"key_points": ["ポイント1", "ポイント2"],
"difficulty": "...",
"dimension": "prompt/llm/ide/devPattern/workCapability",
"basis": "[n] 引用箇所..."
"basis": "【必須】ナレッジベースから関連する原文を引用し、出典段落を明記"
}
]`;
]`;
const systemPromptEn = `You are an expert examiner. Generate 1 UNIQUE question based on the provided context.
const systemPromptEn = `You are a strict knowledge assessment expert. You MUST generate questions **ONLY** from the provided knowledge base content below.
### Core Rules (violations invalidate the question):
1. **All questions MUST directly derive from the provided content**, each question requires a verifiable source reference
2. **ABSOLUTELY FORBIDDEN**: inventing concepts, terminology, processes, or data not in the provided content
3. **ABSOLUTELY FORBIDDEN**: using your own knowledge to fabricate questions
4. If content is insufficient, honestly report rather than fabricate
### Language Rule:
**You MUST generate the question and key points in English.**
@@ -160,7 +178,7 @@ Return 1 question as a JSON array with format:
"key_points": ["point1", "point2"],
"difficulty": "...",
"dimension": "prompt/llm/ide/devPattern/workCapability",
"basis": "[n] citation..."
"basis": "【REQUIRED】Cite the specific source text from the knowledge base, noting the source paragraph"
}
]`;
@@ -172,10 +190,10 @@ Return 1 question as a JSON array with format:
? systemPromptJa
: systemPromptEn;
const humanMsg = isZh
? `请使用中文基于以下内容生成题目:\n\n${knowledgeBaseContent}`
? `【知识库内容 - 以下是你出题的唯一依据】\n\n--- 知识库开始 ---\n${knowledgeBaseContent}\n--- 知识库结束 ---\n\n请严格基于以上内容生成题目。`
: isJa
? `以下の内容に基づいて、必ず日本語でアセスメント問題を作成してください:\n\n${knowledgeBaseContent}`
: `Generate evaluation question in English based on:\n\n${knowledgeBaseContent}`;
? `【ナレッジベース内容 - 以下は出題の唯一の根拠です】\n\n--- ナレッジベース開始 ---\n${knowledgeBaseContent}\n--- ナレッジベース終了 ---\n\n上記の内容のみに基づいて問題を作成してください。`
: `【Knowledge Base Content - Your ONLY source for questions】\n\n--- KB START ---\n${knowledgeBaseContent}\n--- KB END ---\n\nGenerate questions strictly from the above content only.`;
try {
const response = await model.invoke([
@@ -92,7 +92,7 @@ export class QuestionBankService {
}
if (createDto.templateId) {
const existing = await this.bankRepository.findOne({
where: { templateId: createDto.templateId, tenantId: tenantId as any },
where: { templateId: createDto.templateId },
});
if (existing) {
if (existing.status === QuestionBankStatus.DRAFT || existing.status === QuestionBankStatus.REJECTED) {
@@ -295,35 +295,45 @@ export class QuestionBankService {
const model = new ChatOpenAI({
apiKey: modelConfig.apiKey || 'ollama',
modelName: modelConfig.modelId,
temperature: 0.7,
temperature: 0.3,
configuration: {
baseURL: modelConfig.baseUrl || 'https://api.deepseek.com/v1',
},
});
const systemPrompt = `你是一位专业的知识评估专家。请根据提供的知识库片段生成 ${count} 个唯一的测试题目。
const systemPrompt = `你是一位严格的知识评估专家。你必须**仅基于**下方 Human 消息中提供的知识库内容】来生成题目。
### 强制性语言规则
**必须使用中文 (Simplified Chinese) 进行回复**。即使知识库内容是英文或其他语言,问题(question_text)和关键点(key_points)也必须使用中文。
### 核心铁律(违反将导致题目无效)
1. **所有题目必须直接来源于提供的知识库内容**,每个题目必须能找到对应的原文依据
2. **绝对禁止**编造知识库内容中未提及的概念、术语、流程或数据
3. **绝对禁止**使用你自身知识库中的内容来编造题目
4. 如果知识库内容不足以生成 ${count} 道有意义的题目,可以生成少于 ${count} 道,但题目质量优先
### 多样性规则
1. 禁止重复:绝对禁止生成相似的题目
2. 深度挖掘:从不同的角度出题,如流程、限制、优缺点、具体参数等
3. 随机扰动:从不同的逻辑链条出发
### 任务:
请以 JSON 数组格式返回 ${count} 个问题:
### 格式要求
请以 JSON 数组格式返回题目
[
{
"question_text": "问题内容",
"key_points": ["要点1", "要点2"],
"question_text": "基于知识库内容的实际问题",
"key_points": ["评分要点1", "评分要点2"],
"difficulty": "STANDARD|ADVANCED|SPECIALIST",
"dimension": "prompt|llm|ide|devPattern|workCapability",
"basis": "[n] 引用原文..."
"basis": "【必须填写】从知识库中引用与此题相关的原文内容,用引号标注来源段落"
}
]`;
]
const humanMsg = `请使用中文基于以下内容生成题目:\n\n${knowledgeBaseContent}`;
### 维度说明(根据题目内容归类):
- prompt: 关于提示词设计、AI交互优化
- llm: 关于大语言模型原理、架构、参数
- ide: 关于开发工具使用、协作效率
- devPattern: 关于开发方法论、工程范式
- workCapability: 关于工作能力、综合素养
### 出题规范:
1. 每个题目必须标注 basis,引用知识库中的具体原文作为依据
2. 题目难度分布合理,覆盖 STANDARD/ADVANCED/SPECIALIST
3. 不同维度各出一部分,不要集中在一个维度`;
const humanMsg = `【知识库内容 - 以下是你出题的唯一依据】\n\n--- 知识库开始 ---\n${knowledgeBaseContent}\n--- 知识库结束 ---\n\n请严格基于以上知识库内容生成题目。`;
try {
const response = await model.invoke([
@@ -395,7 +405,7 @@ export class QuestionBankService {
}
const allItems = await this.itemRepository.find({
where: { bankId },
where: { bankId, status: QuestionBankItemStatus.PUBLISHED },
});
if (allItems.length === 0) {
@@ -260,7 +260,6 @@ export class KnowledgeGroupService {
throw new NotFoundException(this.i18nService.getMessage('groupNotFound'));
}
// Check permission using TenantService
const hasAccess = await this.tenantService.canAccessTenant(
userId,
group.tenantId,
@@ -272,7 +271,31 @@ export class KnowledgeGroupService {
);
}
return group.knowledgeBases;
const allGroups = await this.groupRepository.find({
where: tenantId === null ? {} : { tenantId },
relations: ['knowledgeBases'],
});
const childIds = new Set<string>();
const collectDescendantIds = (parentId: string) => {
for (const g of allGroups) {
if (g.parentId === parentId) {
childIds.add(g.id);
collectDescendantIds(g.id);
}
}
};
collectDescendantIds(groupId);
const result = [...(group.knowledgeBases || [])];
for (const childId of childIds) {
const childGroup = allGroups.find(g => g.id === childId);
if (childGroup?.knowledgeBases) {
result.push(...childGroup.knowledgeBases);
}
}
return result;
}
async addFilesToGroup(