forked from hangshuo652/aurak
F1-F10: audit fixes (dimension normalize, passingScore scale, DB defaults, onDelete, item status filter, timeout event type, userId privacy) + generator.node.ts strict prompt rules (anti-hallucination)
This commit is contained in:
@@ -137,6 +137,13 @@ private async getModel(tenantId: string): Promise<ChatOpenAI> {
|
||||
return result;
|
||||
}
|
||||
|
||||
private normalizeDimension(dim: string): string {
|
||||
const lower = dim.toLowerCase();
|
||||
if (lower === 'dev_pattern') return 'devPattern';
|
||||
if (lower === 'work_capability') return 'workCapability';
|
||||
return lower;
|
||||
}
|
||||
|
||||
private calculateScores(
|
||||
questions: any[],
|
||||
scores: Record<string, number>,
|
||||
@@ -157,7 +164,7 @@ private async getModel(tenantId: string): Promise<ChatOpenAI> {
|
||||
};
|
||||
|
||||
questions.forEach((q: any, idx: number) => {
|
||||
const dimension = q.dimension || 'workCapability';
|
||||
const dimension = this.normalizeDimension(q.dimension || 'workCapability');
|
||||
const score = scores[q.id || idx.toString()] || 0;
|
||||
if (dimensionScoresMap[dimension]) {
|
||||
dimensionScoresMap[dimension].push(score);
|
||||
@@ -729,7 +736,7 @@ const initialState: Partial<EvaluationState> = {
|
||||
const scores = finalData.scores;
|
||||
const questions = finalData.questions || [];
|
||||
const weightConfig = session.templateJson?.weightConfig || { prompt: 50, other: 50 };
|
||||
const passingScore = session.templateJson?.passingScore || 90;
|
||||
const passingScore = (session.templateJson?.passingScore ?? 90) / 10;
|
||||
|
||||
if (questions.length > 0 && Object.keys(scores).length > 0) {
|
||||
const { finalScore, dimensionScores, radarData } = this.calculateScores(
|
||||
@@ -820,7 +827,7 @@ const initialState: Partial<EvaluationState> = {
|
||||
|
||||
let finalResult: any = null;
|
||||
const weightConfig = session.templateJson?.weightConfig || { prompt: 50, other: 50 };
|
||||
const passingScore = session.templateJson?.passingScore || 90;
|
||||
const passingScore = (session.templateJson?.passingScore ?? 90) / 10;
|
||||
|
||||
// Resume from the last interrupt (typically after interviewer)
|
||||
const stream = await this.graph.stream(null, {
|
||||
@@ -965,6 +972,7 @@ const initialState: Partial<EvaluationState> = {
|
||||
await this.sessionRepository.save(session);
|
||||
this.logger.log(`[submitAnswerStream] Session ${sessionId} auto-ended due to timeout`);
|
||||
observer.next({
|
||||
type: 'final',
|
||||
assessmentSessionId: sessionId,
|
||||
status: 'COMPLETED',
|
||||
timeout: true,
|
||||
@@ -1713,7 +1721,6 @@ const initialState: Partial<EvaluationState> = {
|
||||
totalScore: number;
|
||||
passed: boolean;
|
||||
issuedAt: Date;
|
||||
userId: string;
|
||||
};
|
||||
message?: string;
|
||||
}> {
|
||||
@@ -1734,7 +1741,6 @@ const initialState: Partial<EvaluationState> = {
|
||||
totalScore: certificate.totalScore,
|
||||
passed: certificate.passed,
|
||||
issuedAt: certificate.issuedAt,
|
||||
userId: certificate.userId,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@ import {
|
||||
ReviewDto,
|
||||
} from '../services/question-bank.service';
|
||||
import { CombinedAuthGuard } from '../../auth/combined-auth.guard';
|
||||
import { KnowledgeGroupService } from '../../knowledge-group/knowledge-group.service';
|
||||
|
||||
@Controller('question-banks')
|
||||
@UseGuards(CombinedAuthGuard)
|
||||
@@ -29,12 +30,20 @@ import { CombinedAuthGuard } from '../../auth/combined-auth.guard';
|
||||
export class QuestionBankController {
|
||||
private readonly logger = new Logger(QuestionBankController.name);
|
||||
|
||||
constructor(private readonly questionBankService: QuestionBankService) {}
|
||||
constructor(
|
||||
private readonly questionBankService: QuestionBankService,
|
||||
private readonly groupService: KnowledgeGroupService,
|
||||
) {}
|
||||
|
||||
@Post()
|
||||
create(@Body() createDto: CreateQuestionBankDto, @Req() req: any) {
|
||||
this.logger.log(`Creating question bank: ${createDto.name}`);
|
||||
return this.questionBankService.create(createDto, req.user.id, req.user.tenantId);
|
||||
async create(@Body() createDto: CreateQuestionBankDto, @Req() req: any) {
|
||||
try {
|
||||
this.logger.log(`Creating question bank: ${createDto.name}, user: ${req.user?.id}, tenant: ${req.user?.tenantId}`);
|
||||
return await this.questionBankService.create(createDto, req.user.id, req.user.tenantId);
|
||||
} catch (err: any) {
|
||||
this.logger.error(`[create] Failed: ${err.message}`, err.stack);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
@Get()
|
||||
@@ -125,11 +134,32 @@ export class QuestionBankController {
|
||||
@Body() body: { count: number; knowledgeBaseContent?: string },
|
||||
@Req() req: any,
|
||||
) {
|
||||
this.logger.log(`[generate] Generating ${body.count} questions for bank ${bankId}`);
|
||||
let content = body.knowledgeBaseContent || '';
|
||||
if (!content || content.trim().length < 10) {
|
||||
try {
|
||||
const bank = await this.questionBankService.findOne(bankId);
|
||||
if (bank?.template?.knowledgeGroupId) {
|
||||
const files = await this.groupService.getGroupFiles(
|
||||
bank.template.knowledgeGroupId,
|
||||
req.user.id,
|
||||
req.user.tenantId,
|
||||
);
|
||||
content = files
|
||||
.filter((f: any) => f.content && f.content.trim().length > 0)
|
||||
.map((f: any) => `--- ${f.title || f.originalName || 'Document'} ---\n${f.content}`)
|
||||
.join('\n\n');
|
||||
this.logger.log(`[generate] Auto-loaded ${files.length} files, content length: ${content.length}`);
|
||||
}
|
||||
} catch (err: any) {
|
||||
this.logger.warn(`[generate] Auto-load failed: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.log(`[generate] Generating ${body.count} questions for bank ${bankId}, content length: ${content.length}`);
|
||||
return this.questionBankService.generateQuestions(
|
||||
bankId,
|
||||
body.count,
|
||||
body.knowledgeBaseContent || '',
|
||||
content,
|
||||
req.user.tenantId,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -56,6 +56,7 @@ export class QuestionBankItem {
|
||||
@Column({
|
||||
type: 'simple-enum',
|
||||
enum: QuestionType,
|
||||
default: QuestionType.SHORT_ANSWER,
|
||||
})
|
||||
questionType: QuestionType;
|
||||
|
||||
@@ -71,12 +72,14 @@ export class QuestionBankItem {
|
||||
@Column({
|
||||
type: 'simple-enum',
|
||||
enum: QuestionDifficulty,
|
||||
default: QuestionDifficulty.STANDARD,
|
||||
})
|
||||
difficulty: QuestionDifficulty;
|
||||
|
||||
@Column({
|
||||
type: 'simple-enum',
|
||||
enum: QuestionDimension,
|
||||
default: QuestionDimension.PROMPT,
|
||||
})
|
||||
dimension: QuestionDimension;
|
||||
|
||||
@@ -89,6 +92,7 @@ export class QuestionBankItem {
|
||||
@Column({
|
||||
type: 'simple-enum',
|
||||
enum: QuestionBankItemStatus,
|
||||
default: QuestionBankItemStatus.PENDING_REVIEW,
|
||||
})
|
||||
status: QuestionBankItemStatus;
|
||||
|
||||
|
||||
@@ -37,7 +37,7 @@ export class QuestionBank {
|
||||
@Column({ name: 'template_id', nullable: true })
|
||||
templateId: string | null;
|
||||
|
||||
@OneToOne(() => AssessmentTemplate, { nullable: true })
|
||||
@OneToOne(() => AssessmentTemplate, { nullable: true, onDelete: 'SET NULL' })
|
||||
@JoinColumn({ name: 'template_id' })
|
||||
template: AssessmentTemplate;
|
||||
|
||||
|
||||
@@ -89,12 +89,18 @@ export const questionGeneratorNode = async (
|
||||
.map((q, i) => `Q${i + 1}: ${q.questionText}`)
|
||||
.join('\n');
|
||||
|
||||
const systemPromptZh = `你是一位专业的知识评估专家。请根据提供的知识库片段生成 1 个唯一的测试题目。
|
||||
const systemPromptZh = `你是一位严格的知识评估专家。你必须**仅基于**下方提供的知识库内容来生成测试题目。
|
||||
|
||||
### 核心铁律(违反将导致题目无效):
|
||||
1. **所有题目必须直接来源于提供的知识库内容**,每个题目必须能找到对应的原文依据
|
||||
2. **绝对禁止**编造知识库内容中未提及的概念、术语、流程或数据
|
||||
3. **绝对禁止**使用你自身知识库中的内容来编造题目
|
||||
4. 如果知识库内容不足以出题,诚实地报告而不是编造
|
||||
|
||||
### 强制性语言规则:
|
||||
**必须使用中文 (Simplified Chinese) 进行回复**。即使知识库内容是英文或其他语言,问题(question_text)和关键点(key_points)也必须使用中文。
|
||||
**必须使用中文 (Simplified Chinese) 进行回复**。
|
||||
|
||||
### 强制性多样性规则:
|
||||
### 多样性规则:
|
||||
${rulesZh}
|
||||
|
||||
### 禁止重复列表(已出过):
|
||||
@@ -111,15 +117,21 @@ ${hasKeywords ? `目标关键词:${keywordText}\n` : ''}出题风格:${style
|
||||
"key_points": ["点1", "点2"],
|
||||
"difficulty": "...",
|
||||
"dimension": "prompt/llm/ide/devPattern/workCapability",
|
||||
"basis": "[n] 引用原文..."
|
||||
"basis": "【必须填写】从知识库中引用与此题相关的原文内容,用引号标注来源段落"
|
||||
}
|
||||
]`;
|
||||
// dimension取值:prompt=提示词, llm=LLM原理, ide=IDE协作, devPattern=开发范式, workCapability=工作能力
|
||||
|
||||
const systemPromptJa = `あなたは専門的なアセスメントエキスパートです。提供されたナレッジベースに基づいて、ユニークな問題を 1 つ作成してください。
|
||||
const systemPromptJa = `あなたは厳格な知識評価の専門家です。提供されたナレッジベースの内容**のみ**に基づいて問題を作成してください。
|
||||
|
||||
### 核心鉄則(違反した問題は無効):
|
||||
1. **すべての問題は提供されたナレッジベースから直接導出**し、各問題に原文の根拠が必要
|
||||
2. **絶対禁止**:ナレッジベースに記載されていない概念、用語、プロセス、データを作り出すこと
|
||||
3. **絶対禁止**:自身の知識ベースの内容を問題として使用すること
|
||||
4. 内容が不十分な場合は、正直に報告し、捏造しないこと
|
||||
|
||||
### 言語ルール(最重要):
|
||||
**必ず日本語で作成してください**。提供されたナレッジベースが英語や中国語、その他の言語であっても、質問文(question_text)およびキーポイント(key_points)は必ず日本語で回答してください。中国語が混ざらないように厳格に注意してください。
|
||||
**必ず日本語で作成してください**。中国語が混ざらないように厳格に注意してください。
|
||||
|
||||
### 多様性ルール:
|
||||
${rulesJa}
|
||||
@@ -138,11 +150,17 @@ ${hasKeywords ? `目標キーワード:${keywordText}\n` : ''}出題スタイ
|
||||
"key_points": ["ポイント1", "ポイント2"],
|
||||
"difficulty": "...",
|
||||
"dimension": "prompt/llm/ide/devPattern/workCapability",
|
||||
"basis": "[n] 引用箇所..."
|
||||
"basis": "【必須】ナレッジベースから関連する原文を引用し、出典段落を明記"
|
||||
}
|
||||
]`;
|
||||
]`;
|
||||
|
||||
const systemPromptEn = `You are an expert examiner. Generate 1 UNIQUE question based on the provided context.
|
||||
const systemPromptEn = `You are a strict knowledge assessment expert. You MUST generate questions **ONLY** from the provided knowledge base content below.
|
||||
|
||||
### Core Rules (violations invalidate the question):
|
||||
1. **All questions MUST directly derive from the provided content**, each question requires a verifiable source reference
|
||||
2. **ABSOLUTELY FORBIDDEN**: inventing concepts, terminology, processes, or data not in the provided content
|
||||
3. **ABSOLUTELY FORBIDDEN**: using your own knowledge to fabricate questions
|
||||
4. If content is insufficient, honestly report rather than fabricate
|
||||
|
||||
### Language Rule:
|
||||
**You MUST generate the question and key points in English.**
|
||||
@@ -160,7 +178,7 @@ Return 1 question as a JSON array with format:
|
||||
"key_points": ["point1", "point2"],
|
||||
"difficulty": "...",
|
||||
"dimension": "prompt/llm/ide/devPattern/workCapability",
|
||||
"basis": "[n] citation..."
|
||||
"basis": "【REQUIRED】Cite the specific source text from the knowledge base, noting the source paragraph"
|
||||
}
|
||||
]`;
|
||||
|
||||
@@ -172,10 +190,10 @@ Return 1 question as a JSON array with format:
|
||||
? systemPromptJa
|
||||
: systemPromptEn;
|
||||
const humanMsg = isZh
|
||||
? `请使用中文基于以下内容生成题目:\n\n${knowledgeBaseContent}`
|
||||
? `【知识库内容 - 以下是你出题的唯一依据】\n\n--- 知识库开始 ---\n${knowledgeBaseContent}\n--- 知识库结束 ---\n\n请严格基于以上内容生成题目。`
|
||||
: isJa
|
||||
? `以下の内容に基づいて、必ず日本語でアセスメント問題を作成してください:\n\n${knowledgeBaseContent}`
|
||||
: `Generate evaluation question in English based on:\n\n${knowledgeBaseContent}`;
|
||||
? `【ナレッジベース内容 - 以下は出題の唯一の根拠です】\n\n--- ナレッジベース開始 ---\n${knowledgeBaseContent}\n--- ナレッジベース終了 ---\n\n上記の内容のみに基づいて問題を作成してください。`
|
||||
: `【Knowledge Base Content - Your ONLY source for questions】\n\n--- KB START ---\n${knowledgeBaseContent}\n--- KB END ---\n\nGenerate questions strictly from the above content only.`;
|
||||
|
||||
try {
|
||||
const response = await model.invoke([
|
||||
|
||||
@@ -92,7 +92,7 @@ export class QuestionBankService {
|
||||
}
|
||||
if (createDto.templateId) {
|
||||
const existing = await this.bankRepository.findOne({
|
||||
where: { templateId: createDto.templateId, tenantId: tenantId as any },
|
||||
where: { templateId: createDto.templateId },
|
||||
});
|
||||
if (existing) {
|
||||
if (existing.status === QuestionBankStatus.DRAFT || existing.status === QuestionBankStatus.REJECTED) {
|
||||
@@ -295,35 +295,45 @@ export class QuestionBankService {
|
||||
const model = new ChatOpenAI({
|
||||
apiKey: modelConfig.apiKey || 'ollama',
|
||||
modelName: modelConfig.modelId,
|
||||
temperature: 0.7,
|
||||
temperature: 0.3,
|
||||
configuration: {
|
||||
baseURL: modelConfig.baseUrl || 'https://api.deepseek.com/v1',
|
||||
},
|
||||
});
|
||||
|
||||
const systemPrompt = `你是一位专业的知识评估专家。请根据提供的知识库片段生成 ${count} 个唯一的测试题目。
|
||||
const systemPrompt = `你是一位严格的知识评估专家。你必须**仅基于**下方 Human 消息中提供的【知识库内容】来生成题目。
|
||||
|
||||
### 强制性语言规则:
|
||||
**必须使用中文 (Simplified Chinese) 进行回复**。即使知识库内容是英文或其他语言,问题(question_text)和关键点(key_points)也必须使用中文。
|
||||
### 核心铁律(违反将导致题目无效):
|
||||
1. **所有题目必须直接来源于提供的知识库内容**,每个题目必须能找到对应的原文依据
|
||||
2. **绝对禁止**编造知识库内容中未提及的概念、术语、流程或数据
|
||||
3. **绝对禁止**使用你自身知识库中的内容来编造题目
|
||||
4. 如果知识库内容不足以生成 ${count} 道有意义的题目,可以生成少于 ${count} 道,但题目质量优先
|
||||
|
||||
### 多样性规则:
|
||||
1. 禁止重复:绝对禁止生成相似的题目
|
||||
2. 深度挖掘:从不同的角度出题,如流程、限制、优缺点、具体参数等
|
||||
3. 随机扰动:从不同的逻辑链条出发
|
||||
|
||||
### 任务:
|
||||
请以 JSON 数组格式返回 ${count} 个问题:
|
||||
### 格式要求:
|
||||
请以 JSON 数组格式返回题目:
|
||||
[
|
||||
{
|
||||
"question_text": "问题内容",
|
||||
"key_points": ["要点1", "要点2"],
|
||||
"question_text": "基于知识库内容的实际问题",
|
||||
"key_points": ["评分要点1", "评分要点2"],
|
||||
"difficulty": "STANDARD|ADVANCED|SPECIALIST",
|
||||
"dimension": "prompt|llm|ide|devPattern|workCapability",
|
||||
"basis": "[n] 引用原文..."
|
||||
"basis": "【必须填写】从知识库中引用与此题相关的原文内容,用引号标注来源段落"
|
||||
}
|
||||
]`;
|
||||
]
|
||||
|
||||
const humanMsg = `请使用中文基于以下内容生成题目:\n\n${knowledgeBaseContent}`;
|
||||
### 维度说明(根据题目内容归类):
|
||||
- prompt: 关于提示词设计、AI交互优化
|
||||
- llm: 关于大语言模型原理、架构、参数
|
||||
- ide: 关于开发工具使用、协作效率
|
||||
- devPattern: 关于开发方法论、工程范式
|
||||
- workCapability: 关于工作能力、综合素养
|
||||
|
||||
### 出题规范:
|
||||
1. 每个题目必须标注 basis,引用知识库中的具体原文作为依据
|
||||
2. 题目难度分布合理,覆盖 STANDARD/ADVANCED/SPECIALIST
|
||||
3. 不同维度各出一部分,不要集中在一个维度`;
|
||||
|
||||
const humanMsg = `【知识库内容 - 以下是你出题的唯一依据】\n\n--- 知识库开始 ---\n${knowledgeBaseContent}\n--- 知识库结束 ---\n\n请严格基于以上知识库内容生成题目。`;
|
||||
|
||||
try {
|
||||
const response = await model.invoke([
|
||||
@@ -395,7 +405,7 @@ export class QuestionBankService {
|
||||
}
|
||||
|
||||
const allItems = await this.itemRepository.find({
|
||||
where: { bankId },
|
||||
where: { bankId, status: QuestionBankItemStatus.PUBLISHED },
|
||||
});
|
||||
|
||||
if (allItems.length === 0) {
|
||||
|
||||
@@ -260,7 +260,6 @@ export class KnowledgeGroupService {
|
||||
throw new NotFoundException(this.i18nService.getMessage('groupNotFound'));
|
||||
}
|
||||
|
||||
// Check permission using TenantService
|
||||
const hasAccess = await this.tenantService.canAccessTenant(
|
||||
userId,
|
||||
group.tenantId,
|
||||
@@ -272,7 +271,31 @@ export class KnowledgeGroupService {
|
||||
);
|
||||
}
|
||||
|
||||
return group.knowledgeBases;
|
||||
const allGroups = await this.groupRepository.find({
|
||||
where: tenantId === null ? {} : { tenantId },
|
||||
relations: ['knowledgeBases'],
|
||||
});
|
||||
|
||||
const childIds = new Set<string>();
|
||||
const collectDescendantIds = (parentId: string) => {
|
||||
for (const g of allGroups) {
|
||||
if (g.parentId === parentId) {
|
||||
childIds.add(g.id);
|
||||
collectDescendantIds(g.id);
|
||||
}
|
||||
}
|
||||
};
|
||||
collectDescendantIds(groupId);
|
||||
|
||||
const result = [...(group.knowledgeBases || [])];
|
||||
for (const childId of childIds) {
|
||||
const childGroup = allGroups.find(g => g.id === childId);
|
||||
if (childGroup?.knowledgeBases) {
|
||||
result.push(...childGroup.knowledgeBases);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
async addFilesToGroup(
|
||||
|
||||
Reference in New Issue
Block a user