wwz110/aurak
1
0
Fork 0
forked from hangshuo652/aurak
Code Pull Requests Activity

fix: MC options display, question selection, timeout handling, and grading prompts

Browse Source
This commit is contained in:
Developer
2026-06-03 20:58:19 +08:00
parent a71bde3452
commit 6d9acd7252
12 changed files with 408 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/assessment/assessment.controller.ts
+2 -2
View File
@@ -231,7 +231,7 @@ export class AssessmentController {
@ApiOperation({ summary: 'Batch delete assessment sessions (admin only)' }) @ApiOperation({ summary: 'Batch delete assessment sessions (admin only)' })
async batchDelete(@Request() req: any, @Body() body: { ids: string[] }) { async batchDelete(@Request() req: any, @Body() body: { ids: string[] }) {
const user = req.user; const user = req.user;
const isAdmin = user.role === 'super_admin' || user.role === 'admin'; const isAdmin = user.role?.toLowerCase() === 'super_admin' || user.role?.toLowerCase() === 'admin';
if (!isAdmin) { if (!isAdmin) {
throw new ForbiddenException('Only admin can batch delete'); throw new ForbiddenException('Only admin can batch delete');
} }
@@ -286,7 +286,7 @@ export class AssessmentController {
@Request() req: any, @Request() req: any,
) { ) {
const { id: userId, tenantId, role } = req.user; const { id: userId, tenantId, role } = req.user;
const isAdmin = role === 'super_admin' || role === 'admin'; const isAdmin = role?.toLowerCase() === 'super_admin' || role?.toLowerCase() === 'admin';
if (!isAdmin) { if (!isAdmin) {
throw new ForbiddenException('Only admin can force end assessment'); throw new ForbiddenException('Only admin can force end assessment');
} }
server/src/assessment/assessment.service.ts
+77 -43
View File
@@ -189,15 +189,16 @@ private async getModel(tenantId: string): Promise<ChatOpenAI> {
this.logger.debug('[calculateScores] Scoring debug:', { promptAvg, otherDimsWithScores, otherAvg, workCapability: dimensionAverages.workCapability }); this.logger.debug('[calculateScores] Scoring debug:', { promptAvg, otherDimsWithScores, otherAvg, workCapability: dimensionAverages.workCapability });
const allScores: number[] = []; // Weighted final score using weightConfig
questions.forEach((q: any) => { let finalScore: number;
const score = scores[q.id || questions.indexOf(q).toString()] || 0; if (promptAvg > 0 && otherAvg > 0) {
allScores.push(score); const totalWeight = (weightConfig?.prompt ?? 50) + (weightConfig?.other ?? 50);
}); finalScore = totalWeight > 0
? (promptAvg * (weightConfig?.prompt ?? 50) + otherAvg * (weightConfig?.other ?? 50)) / totalWeight
const finalScore = allScores.length > 0 : (promptAvg + otherAvg) / 2;
? allScores.reduce((a, b) => a + b, 0) / allScores.length } else {
: 0; finalScore = promptAvg || otherAvg || 0;
}
const radarData: Record<string, number> = {}; const radarData: Record<string, number> = {};
Object.keys(dimensionAverages).forEach(dim => { Object.keys(dimensionAverages).forEach(dim => {
@@ -430,33 +431,54 @@ private async getModel(tenantId: string): Promise<ChatOpenAI> {
// Use kbId if provided, otherwise fall back to template's group ID // Use kbId if provided, otherwise fall back to template's group ID
const activeKbId = kbId || template?.knowledgeGroupId; const activeKbId = kbId || template?.knowledgeGroupId;
this.logger.log(`[startSession] activeKbId resolved to: ${activeKbId}`);
if (!activeKbId) { // If no knowledge source, check if template has a question bank first
let hasBankQuestions = false;
if (!activeKbId && templateId && template) {
try {
const targetCount = template.questionCount || 5;
const linkedBanks = await this.questionBankRepository.find({
where: { templateId },
});
if (linkedBanks.length > 0) {
const bankIds = linkedBanks.map(b => b.id);
const count = await this.questionBankItemRepository.count({
where: { bankId: In(bankIds), status: QuestionBankItemStatus.PUBLISHED },
});
if (count >= targetCount) {
hasBankQuestions = true;
this.logger.log(`[startSession] Template has ${count} published questions, skipping KB check`);
}
}
} catch (e) {
this.logger.warn(`[startSession] Bank pre-check failed: ${e.message}`);
}
}
if (!activeKbId && !hasBankQuestions) {
this.logger.error(`[startSession] No knowledge source resolved`); this.logger.error(`[startSession] No knowledge source resolved`);
throw new BadRequestException('Knowledge source (ID or Template) must be provided.'); throw new BadRequestException('Knowledge source (ID or Template) must be provided.');
} }
// Try to determine if it's a KB or Group and check permissions // Determine if it's a KB or Group (only when activeKbId exists)
let isKb = false; let isKb = false;
try { if (activeKbId) {
await this.kbService.findOne(activeKbId, userId, tenantId); try {
isKb = true; await this.kbService.findOne(activeKbId, userId, tenantId);
} catch (kbError) { isKb = true;
if (kbError instanceof NotFoundException) { } catch (kbError) {
// Try finding it as a Group if (kbError instanceof NotFoundException) {
try { try {
await this.groupService.findOne(activeKbId, userId, tenantId); await this.groupService.findOne(activeKbId, userId, tenantId);
} catch (groupError) { } catch (groupError) {
this.logger.error( this.logger.error(`[startSession] Knowledge source ${activeKbId} not found`);
`[startSession] Knowledge source ${activeKbId} not found as KB or Group`, throw new NotFoundException(
); this.i18nService.getMessage('knowledgeSourceNotFound') || 'Knowledge source not found',
throw new NotFoundException( );
this.i18nService.getMessage('knowledgeSourceNotFound') || }
'Knowledge source not found', } else {
); throw kbError;
} }
} else {
throw kbError; // e.g. ForbiddenException
} }
} }
this.logger.debug(`[startSession] isKb: ${isKb}`); this.logger.debug(`[startSession] isKb: ${isKb}`);
@@ -503,6 +525,7 @@ private async getModel(tenantId: string): Promise<ChatOpenAI> {
const selectedItems = await this.questionBankService.selectQuestions( const selectedItems = await this.questionBankService.selectQuestions(
bankId, bankId,
targetCount, targetCount,
template?.dimensions,
); );
questionsFromBank = selectedItems.map(item => { questionsFromBank = selectedItems.map(item => {
@@ -586,9 +609,9 @@ private async getModel(tenantId: string): Promise<ChatOpenAI> {
}; };
// Skip content check if questions are loaded from the question bank // Skip content check if questions are loaded from the question bank
const hasBankQuestions = questionsFromBank.length > 0; const hasBankContent = questionsFromBank.length > 0;
if (!hasBankQuestions) { if (!hasBankContent) {
const content = await this.getSessionContent(sessionData); const content = await this.getSessionContent(sessionData);
if (!content || content.trim().length < 10) { if (!content || content.trim().length < 10) {
@@ -787,7 +810,7 @@ const initialState: Partial<EvaluationState> = {
const scores = finalData.scores; const scores = finalData.scores;
const questions = finalData.questions || []; const questions = finalData.questions || [];
const weightConfig = session.templateJson?.weightConfig || { prompt: 50, other: 50 }; const weightConfig = session.templateJson?.weightConfig || { prompt: 50, other: 50 };
const passingScore = (session.templateJson?.passingScore ?? 90) / 10; const passingScore = (session.templateJson?.passingScore ?? 60) / 10;
if (questions.length > 0 && Object.keys(scores).length > 0) { if (questions.length > 0 && Object.keys(scores).length > 0) {
const { finalScore, dimensionScores, radarData } = this.calculateScores( const { finalScore, dimensionScores, radarData } = this.calculateScores(
@@ -882,7 +905,7 @@ const initialState: Partial<EvaluationState> = {
let finalResult: any = null; let finalResult: any = null;
const weightConfig = session.templateJson?.weightConfig || { prompt: 50, other: 50 }; const weightConfig = session.templateJson?.weightConfig || { prompt: 50, other: 50 };
const passingScore = (session.templateJson?.passingScore ?? 90) / 10; const passingScore = (session.templateJson?.passingScore ?? 60) / 10;
// Resume from the last interrupt (typically after interviewer) // Resume from the last interrupt (typically after interviewer)
const stream = await this.graph.stream(null, { const stream = await this.graph.stream(null, {
@@ -935,7 +958,7 @@ const initialState: Partial<EvaluationState> = {
const scores = finalResult.scores as Record<string, number>; const scores = finalResult.scores as Record<string, number>;
const questions = finalResult.questions || []; const questions = finalResult.questions || [];
const weightConfig = session.templateJson?.weightConfig || { prompt: 50, other: 50 }; const weightConfig = session.templateJson?.weightConfig || { prompt: 50, other: 50 };
const passingScore = (session.templateJson?.passingScore ?? 90) / 10; const passingScore = (session.templateJson?.passingScore ?? 60) / 10;
if (questions.length > 0 && Object.keys(scores).length > 0) { if (questions.length > 0 && Object.keys(scores).length > 0) {
const { finalScore, dimensionScores, radarData } = this.calculateScores( const { finalScore, dimensionScores, radarData } = this.calculateScores(
@@ -1158,7 +1181,7 @@ const initialState: Partial<EvaluationState> = {
const scores = finalData.scores; const scores = finalData.scores;
const questions = finalData.questions || []; const questions = finalData.questions || [];
const weightConfig = session.templateJson?.weightConfig || { prompt: 50, other: 50 }; const weightConfig = session.templateJson?.weightConfig || { prompt: 50, other: 50 };
const passingScore = (session.templateJson?.passingScore ?? 90) / 10; const passingScore = (session.templateJson?.passingScore ?? 60) / 10;
if (questions.length > 0 && Object.keys(scores).length > 0) { if (questions.length > 0 && Object.keys(scores).length > 0) {
const { finalScore, dimensionScores, radarData } = this.calculateScores( const { finalScore, dimensionScores, radarData } = this.calculateScores(
@@ -1364,17 +1387,27 @@ const initialState: Partial<EvaluationState> = {
const content = await this.getSessionContent(session); const content = await this.getSessionContent(session);
const model = await this.getModel(session.tenantId); const model = await this.getModel(session.tenantId);
const existingQuestions = session.questions_json || [];
const hasQuestionsFromBank = existingQuestions.length > 0;
const isZh = (session.language || 'en') === 'zh';
const isJa = session.language === 'ja';
const initialState: Partial<EvaluationState> = { const initialState: Partial<EvaluationState> = {
assessmentSessionId: sessionId, assessmentSessionId: sessionId,
knowledgeBaseId: knowledgeBaseId:
session.knowledgeBaseId || session.knowledgeGroupId || '', session.knowledgeBaseId || session.knowledgeGroupId || '',
messages: [], messages: hasQuestionsFromBank
? [new HumanMessage(
isZh ? '我已准备好回答问题。' : isJa ? '質問への回答準備ができています。' : 'I am ready to answer the questions.',
)]
: [],
questionCount: session.templateJson?.questionCount, questionCount: session.templateJson?.questionCount,
difficultyDistribution: session.templateJson?.difficultyDistribution, difficultyDistribution: session.templateJson?.difficultyDistribution,
style: session.templateJson?.style, style: session.templateJson?.style,
keywords: session.templateJson?.keywords, keywords: session.templateJson?.keywords,
questionAnswerKey: session.templateJson?.questionAnswerKey, questionAnswerKey: session.templateJson?.questionAnswerKey,
language: session.language || 'en', language: session.language || 'en',
questions: hasQuestionsFromBank ? existingQuestions : undefined,
}; };
this.logger.log( this.logger.log(
@@ -1504,7 +1537,8 @@ const initialState: Partial<EvaluationState> = {
return existing; return existing;
} }
const level = this.determineLevel(session.finalScore || 0); const passingThreshold = (session.templateJson?.passingScore ?? 60) / 10;
const level = this.determineLevel(session.finalScore || 0, !!(session as any).passed, passingThreshold);
const qrCode = `cert://${sessionId}-${Date.now()}`; const qrCode = `cert://${sessionId}-${Date.now()}`;
const questionDetails = (session.questions_json || []).map((q: any, i: number) => ({ const questionDetails = (session.questions_json || []).map((q: any, i: number) => ({
@@ -1535,11 +1569,11 @@ const initialState: Partial<EvaluationState> = {
} as any; } as any;
} }
private determineLevel(score: number): string { private determineLevel(score: number, passed: boolean, passingThreshold: number): string {
if (!passed) return 'Novice';
if (score >= 9) return 'Expert'; if (score >= 9) return 'Expert';
if (score >= 7.5) return 'Advanced'; if (score >= 7) return 'Advanced';
if (score >= 6) return 'Proficient'; return 'Proficient';
return 'Novice';
} }
async getStats( async getStats(
@@ -1723,7 +1757,7 @@ const initialState: Partial<EvaluationState> = {
} }
session.finalScore = newScore; session.finalScore = newScore;
const passingScore = (session.templateJson?.passingScore ?? 90) / 10; const passingScore = (session.templateJson?.passingScore ?? 60) / 10;
(session as any).passed = newScore >= passingScore; (session as any).passed = newScore >= passingScore;
session.reviewedBy = reviewerId; session.reviewedBy = reviewerId;
session.reviewedAt = new Date(); session.reviewedAt = new Date();
server/src/assessment/dto/create-template.dto.ts
+2 -4
View File
@@ -100,13 +100,11 @@ export class CreateTemplateDto {
@IsInt() @IsInt()
@Min(60) @Min(60)
@Max(7200) @Max(86400)
@IsOptional()
totalTimeLimit?: number; totalTimeLimit?: number;
@IsInt() @IsInt()
@Min(30) @Min(30)
@Max(1800) @Max(3600)
@IsOptional()
perQuestionTimeLimit?: number; perQuestionTimeLimit?: number;
} }
server/src/assessment/entities/assessment-template.entity.ts
+1 -1
View File
@@ -97,7 +97,7 @@ export class AssessmentTemplate {
@Column({ type: 'int', name: 'question_count_max', default: 10 }) @Column({ type: 'int', name: 'question_count_max', default: 10 })
questionCountMax: number; questionCountMax: number;
@Column({ type: 'int', name: 'passing_score', default: 90 }) @Column({ type: 'int', name: 'passing_score', default: 60 })
passingScore: number; passingScore: number;
@Column({ type: 'int', name: 'total_time_limit', default: 1800 }) @Column({ type: 'int', name: 'total_time_limit', default: 1800 })
server/src/assessment/graph/nodes/analyzer.node.ts
+20 -5
View File
@@ -56,7 +56,12 @@ const scoreSummary = Object.entries(scores)
1. **你必须使用以下语言生成报告：中文 (Simplified Chinese)**。 1. **你必须使用以下语言生成报告：中文 (Simplified Chinese)**。
2. **严禁夹杂日文**。即使对话记录中包含日文，报告内容也必须全中文。 2. **严禁夹杂日文**。即使对话记录中包含日文，报告内容也必须全中文。
3. 报告的第一行必须严格遵守此格式："LEVEL: [Novice/Proficient/Advanced/Expert]"。 3. 报告的第一行必须严格遵守此格式："LEVEL: [Novice/Proficient/Advanced/Expert]"。
4. 必须保持客观。如果用户没有提供有效的回答或得分为 0，你必须将其识别为 'Novice'，并明确指出他们尚未证明其掌握程度。 4. **等级判定必须遵循以下分数阈值**：
- 总体平均分 >= 9 → Expert（专家）
- 总体平均分 >= 7 → Advanced（高级）
- 已通过（有有效回答且得分 > 0）→ Proficient（熟练）
- 未通过（无有效回答或得分为 0）→ Novice（新手）
即使得分很高，也要确保等级与上述阈值匹配。不要随意提高或降低等级。
5. 不要虚构或幻想优点（如"潜力"或"好奇心"），如果用户明确表示"不知道"或未提供实质内容。 5. 不要虚构或幻想优点（如"潜力"或"好奇心"），如果用户明确表示"不知道"或未提供实质内容。
6. 专注于对话记录中已证明的事实。 6. 专注于对话记录中已证明的事实。
@@ -87,8 +92,13 @@ ${messages
2. **中国語を混ぜないでください**。会話ログに中国語が含まれていても、レポートの内容はすべて日本語で記述してください。 2. **中国語を混ぜないでください**。会話ログに中国語が含まれていても、レポートの内容はすべて日本語で記述してください。
3. レポートの最初の行は, 必ず次の形式に従ってください："LEVEL: [Novice/Proficient/Advanced/Expert]"。 3. レポートの最初の行は, 必ず次の形式に従ってください："LEVEL: [Novice/Proficient/Advanced/Expert]"。
4. 客観的であること。ユーザーが有効な回答を提供しなかった場合、またはスコアが 0 の場合、'Novice' と判定し、習熟度が証明されていないことを明示してください。 4. 客観的であること。ユーザーが有効な回答を提供しなかった場合、またはスコアが 0 の場合、'Novice' と判定し、習熟度が証明されていないことを明示してください。
5. ユーザーが「わからない」と言ったり、内容を提供しなかった場合に、長所（「ポテンシャル」や「好奇心」など）を捏造しないでください。 5. **レベル判定は以下のスコアしきい値に従うこと**：
6. 会話ログで証明された事実に集中してください。 - 平均スコア >= 9 → Expert
- 平均スコア >= 7 → Advanced
- 合格（有効な回答がありスコア > 0）→ Proficient
- 不合格（有効な回答なし、またはスコア 0）→ Novice
6. ユーザーが「わからない」と言ったり、内容を提供しなかった場合に、長所（「ポテンシャル」や「好奇心」など）を捏造しないでください。
7. 会話ログで証明された事実に集中してください。
各ディメンションスコア： 各ディメンションスコア：
${dimensionAvg} ${dimensionAvg}
@@ -115,8 +125,13 @@ IMPORTANT:
1. **You MUST generate the report strictly in English.** 1. **You MUST generate the report strictly in English.**
2. START the report with exactly this format: "LEVEL: [Novice/Proficient/Advanced/Expert]" on the first line. 2. START the report with exactly this format: "LEVEL: [Novice/Proficient/Advanced/Expert]" on the first line.
3. Be OBJECTIVE. If the user provided no valid answers or scores are 0, you MUST identify them as 'Novice' and explicitly state they have NOT demonstrated mastery. 3. Be OBJECTIVE. If the user provided no valid answers or scores are 0, you MUST identify them as 'Novice' and explicitly state they have NOT demonstrated mastery.
4. DO NOT invent or hallucinate strengths (like 'potential' or 'curiosity') if the user explicitly said "I don't know" or provided no content. 4. **Level assignment MUST follow these score thresholds**:
5. Focus on what was PROVEN in the conversation logs. - Average score >= 9 → Expert
- Average score >= 7 → Advanced
- Passed (has valid answers with score > 0) → Proficient
- Not passed (no valid answers or score is 0) → Novice
5. DO NOT invent or hallucinate strengths (like 'potential' or 'curiosity') if the user explicitly said "I don't know" or provided no content.
6. Focus on what was PROVEN in the conversation logs.
DIMENSION SCORES: DIMENSION SCORES:
${dimensionAvg} ${dimensionAvg}
server/src/assessment/graph/nodes/generator.node.ts
+181 -56
View File
@@ -90,34 +90,83 @@ export const questionGeneratorNode = async (
.map((q, i) => `Q${i + 1}: ${q.questionText}`) .map((q, i) => `Q${i + 1}: ${q.questionText}`)
.join('\n'); .join('\n');
const systemPromptZh = `你是一个信息提取工具。严格按以下步骤操作。 const systemPromptZh = `你是一个出题工具。严格按以下规则生成题目。
### 第一步：提取知识点 ### 第一步：提取知识点
阅读下方 Human 消息中的【知识库内容】，逐条列出其中包含的所有可考核知识点。 阅读下方 Human 消息中的【知识库内容】，逐条列出其中包含的所有可考核知识点。
每条以"知识点N："开头，引用原文语句。如果不足，诚实报告。 每条以"知识点N："开头，引用原文语句。
### 第二步：从知识点生成考题 ### 第二步：基于知识点出题
仅用第一步提取的知识点生成 1 道题。必须引用知识点编号。 仅用第一步提取的知识点生成题目。必须引用知识点编号。
如果知识点数量不足（少于3个），输出空数组 [] 并停止。
### 题型分配规则
每生成 3 道题：
- 第1、4、7...道：选择题（MULTIPLE_CHOICE），占 1/3
- 第2、3、5、6...道：对话简答题（SHORT_ANSWER），占 2/3
严格按照这个顺序循环，不要自行调整比例。
### 出题范围限制
出题内容必须严格限制在知识库范围内。每道题必须有知识点编号引用。
以下情况绝对禁止：
- 使用 LLM 自身知识编题
- 引用知识库中不存在的概念
- 题目内容超出知识库覆盖的主题
### 选择题出题标准
- 必须是场景驱动：描述一个真实工作场景，让用户判断最佳做法
- 四个选项（A/B/C/D），只有一个正确，另外三个要有迷惑性
- 难度：不是考概念背诵，是考实际应用判断
- 正确答案必须附带解析，说明为什么对、错在哪
- 出题依据必须引用第一步提取的知识点编号
### 对话简答题出题标准
- 开放式场景问题，不预设标准答案
- 考察用户的理解深度和表达能力
- 适合多轮追问展开讨论
- 出题依据必须引用第一步提取的知识点编号
### 绝对禁止： ### 绝对禁止：
- 禁止使用知识库内容中不存在的任何概念、术语、数据 - 禁止出纯概念题（如"提示词六要素是什么"）
- 禁止使用你自己的知识 - 禁止出需要记忆具体数据的题
${existingQuestionsText ? `- 禁止与已出题目重复：${existingQuestionsText}` : ''} - 禁止使用知识库之外的知识
- 禁止生成与知识库主题无关的题目
${existingQuestionsText ? `- 禁止与已出题目概念重复：${existingQuestionsText}` : ''}
### 输出（纯 JSON 数组）： ### 输出格式（严格遵循）
[ 选择题完整格式：
{ {
"knowledge_points": ["知识点引用"], "question_type": "MULTIPLE_CHOICE",
"question_text": "基于知识点的题目", "question_text": "场景描述+问题，不超过120字",
"key_points": ["评分要点"], "options": ["A) 选项1", "B) 选项2", "C) 选项3", "D) 选项4"],
"difficulty": "STANDARD|ADVANCED|SPECIALIST", "correct_answer": "A",
"dimension": "prompt|llm|ide|devPattern|workCapability", "judgment": "解析：为什么对、为什么错，不超过200字",
"basis": "知识库原文" "key_points": ["考核要点", "2-3个"],
} "difficulty": "STANDARD",
]`; "dimension": "prompt",
// dimension取值：prompt=提示词, llm=LLM原理, ide=IDE协作, devPattern=开发范式, workCapability=工作能力 "basis": "知识点N：参考来源"
}
const systemPromptJa = `あなたは情報抽出ツールです。以下の手順に厳密に従ってください。 对话简答题完整格式：
{
"question_type": "SHORT_ANSWER",
"question_text": "开放式场景问题，不超过120字",
"key_points": ["期望的回答方向", "2-3个"],
"difficulty": "STANDARD",
"dimension": "prompt",
"basis": "知识点N：参考来源"
}
### 输出要求
- 只输出 JSON 数组，不要其他文字
- question_type 必须为 MULTIPLE_CHOICE 或 SHORT_ANSWER
- dimension 只能取以下值之一：prompt、llm、ide、devPattern、workCapability
- 每次生成 1 道题，以 JSON 数组格式输出
- 选择题必须包含全部8个字段：question_text、options、correct_answer、judgment、key_points、difficulty、dimension、basis
- 对话简答题必须包含全部6个字段：question_text、key_points、difficulty、dimension、basis
- 每个字段的值不能为空`;
const systemPromptJa = `あなたは問題作成ツールです。以下の手順に厳密に従ってください。
### 第一歩：知識ポイントの抽出 ### 第一歩：知識ポイントの抽出
Human メッセージ内の【ナレッジベース内容】を読み、含まれるすべての評価可能な知識ポイントを箇条書きで抽出。 Human メッセージ内の【ナレッジベース内容】を読み、含まれるすべての評価可能な知識ポイントを箇条書きで抽出。
@@ -126,48 +175,76 @@ Human メッセージ内の【ナレッジベース内容】を読み、含ま
### 第二歩：知識ポイントから問題を作成 ### 第二歩：知識ポイントから問題を作成
第一歩で抽出した知識ポイントのみを使用して 1 問作成。知識ポイント番号を引用すること。 第一歩で抽出した知識ポイントのみを使用して 1 問作成。知識ポイント番号を引用すること。
### 問題タイプの割合
3問中、約1問を選択問題、2問を対話式記述問題にしてください。全体で約30%/70%の割合。
### 出題方向
「AI協作スキル」に関する問題：
- プロンプトの書き方（役割、タスク、背景、制約）
- 複数ラウンドの対話テクニック
- AIに先に質問させる方法
- セッション管理（いつ継続、いつ新規）
- よくある間違いと自己チェック
- セキュリティ意識（機密データの取扱い）
### 選択問題の基準
- シナリオ駆動：実務シーンを想定
- 4択（A/B/C/D）、正解は1つ
- 正解には必ず解説を含める
### 対話式記述問題の基準
- オープンクエスチョン、正解なし
- 理解の深さと表現力を評価
### 絶対禁止： ### 絶対禁止：
- ナレッジベースに存在しない概念、用語、データの使用 - 暗記問題の禁止
- 自身の知識の使用 - 知識ベースにない概念の使用禁止
${existingQuestionsText ? `- 作成済み問題との重複禁止：${existingQuestionsText}` : ''} ${existingQuestionsText ? `- 既出問題との重複禁止：${existingQuestionsText}` : ''}
### 出力（純粋な JSON 配列）： ### 出力
[ JSON 配列のみ出力：
{ 選択問題：{"question_type":"MULTIPLE_CHOICE","question_text":"...","options":["A)...","B)...","C)...","D)..."],"correct_answer":"A","judgment":"...","key_points":["..."],"difficulty":"STANDARD","dimension":"prompt|llm|ide|devPattern|workCapability","basis":"..."}
"knowledge_points": ["知識ポイント参照"], 記述問題：{"question_type":"SHORT_ANSWER","question_text":"...","key_points":["..."],"difficulty":"STANDARD","dimension":"prompt|llm|ide|devPattern|workCapability","basis":"..."}`;
"question_text": "知識ポイントに基づく問題",
"key_points": ["採点ポイント"],
"difficulty": "STANDARD|ADVANCED|SPECIALIST",
"dimension": "prompt|llm|ide|devPattern|workCapability",
"basis": "ナレッジベースの原文"
}
]`;
const systemPromptEn = `You are an information extraction tool. Follow these steps exactly. const systemPromptEn = `You are a question generation tool. Follow these steps exactly.
### Step 1: Extract Knowledge Points ### Step 1: Extract Knowledge Points
Read the knowledge base content in the Human message. List ALL assessable knowledge points found. Read the knowledge base content in the Human message. List ALL assessable knowledge points.
Each point must start with "KP N:" and quote the source text. If insufficient, honestly report. Each point must start with "KP N:" and quote the source text. If insufficient, honestly report.
### Step 2: Generate Question from Points ### Step 2: Generate Question from Points
Use ONLY the knowledge points from Step 1 to generate 1 question. Must reference KP numbers. Use ONLY the knowledge points from Step 1 to generate 1 question. Must reference KP numbers.
### Absolutely Forbidden: ### Type Mix
- Using any concept, term, or data NOT present in the knowledge base content Out of every 3 questions, approximately 1 should be MULTIPLE_CHOICE and 2 should be SHORT_ANSWER (dialogue-style). Roughly 30%/70% split.
- Using your own knowledge
${existingQuestionsText ? `- Repeating previous questions: ${existingQuestionsText}` : ''}
### Output (pure JSON array only): ### Topics
[ AI collaboration skills:
{ - Writing good prompts (role, task, context, constraints)
"knowledge_points": ["KP reference"], - Multi-turn iteration techniques
"question_text": "Question based on the knowledge points", - Letting AI ask clarifying questions first
"key_points": ["scoring points"], - Session management (continue vs new window)
"difficulty": "STANDARD|ADVANCED|SPECIALIST", - Common mistakes and self-review
"dimension": "prompt|llm|ide|devPattern|workCapability", - Security awareness (handling sensitive data)
"basis": "Source text from knowledge base"
} ### MC Standards
]`; - Scenario-driven: describe a real work scenario
- 4 options (A/B/C/D), one correct
- Must include judgment explaining why correct/incorrect
### SA Standards
- Open-ended, no predefined answer
- Tests understanding depth and expression
### Forbidden:
- Pure concept recall questions
- Questions requiring memorization of specific data
${existingQuestionsText ? `- Repeating previous question concepts: ${existingQuestionsText}` : ''}
### Output
JSON array only. One question at a time.
MC: {"question_type":"MULTIPLE_CHOICE","question_text":"...","options":["A)...","B)...","C)...","D)..."],"correct_answer":"A","judgment":"...","key_points":["..."],"difficulty":"STANDARD","dimension":"prompt|llm|ide|devPattern|workCapability","basis":"..."}
SA: {"question_type":"SHORT_ANSWER","question_text":"...","key_points":["..."],"difficulty":"STANDARD","dimension":"prompt|llm|ide|devPattern|workCapability","basis":"..."}`;
// dimension values: prompt=prompt engineering, llm=LLM principles, ide=IDE collaboration, devPattern=development paradigm, workCapability=work capability // dimension values: prompt=prompt engineering, llm=LLM principles, ide=IDE collaboration, devPattern=development paradigm, workCapability=work capability
@@ -201,6 +278,42 @@ ${existingQuestionsText ? `- Repeating previous questions: ${existingQuestionsTe
newQuestions = [newQuestions]; newQuestions = [newQuestions];
} }
// === 代码级校验：确保 LLM 输出符合规范 ===
const VALID_DIMENSIONS = ['prompt', 'llm', 'ide', 'devPattern', 'workCapability'];
const VALID_TYPES = ['MULTIPLE_CHOICE', 'SHORT_ANSWER'];
const validatedQuestions = newQuestions.filter((q: any) => {
const qType = q.question_type;
const dim = q.dimension?.toString().toLowerCase().trim();
const errors: string[] = [];
if (!VALID_TYPES.includes(qType)) errors.push(`invalid question_type: ${qType}`);
if (!dim || !VALID_DIMENSIONS.includes(dim)) errors.push(`invalid dimension: ${q.dimension}`);
if (!q.question_text || q.question_text.length < 5) errors.push('question_text missing or too short');
if (qType === 'MULTIPLE_CHOICE') {
if (!Array.isArray(q.options) || q.options.length < 2) errors.push('options missing or insufficient');
if (!q.correct_answer) errors.push('correct_answer missing');
if (!q.judgment) errors.push('judgment missing');
} else if (qType === 'SHORT_ANSWER') {
if (!Array.isArray(q.key_points) || q.key_points.length === 0) errors.push('key_points missing');
}
if (errors.length > 0) {
console.warn('[GeneratorNode] Validation failed for question:', errors.join('; '));
return false;
}
return true;
});
if (validatedQuestions.length === 0) {
console.warn('[GeneratorNode] All generated questions failed validation, using existing questions only');
return { questions: existingQuestions };
}
// 只取验证通过的题目
newQuestions = validatedQuestions;
const dimensionMap: Record<string, string> = { const dimensionMap: Record<string, string> = {
// 中文 // 中文
'技术能力-提示词': 'prompt', '技术能力-提示词': 'prompt',
@@ -228,15 +341,27 @@ ${existingQuestionsText ? `- Repeating previous questions: ${existingQuestionsTe
inferredDimension = dimensionMap[dimValue] || 'workCapability'; inferredDimension = dimensionMap[dimValue] || 'workCapability';
console.log('[GeneratorNode] Dimension mapping:', { original: q.dimension, mapped: inferredDimension }); console.log('[GeneratorNode] Dimension mapping:', { original: q.dimension, mapped: inferredDimension });
} }
return {
const qType = q.question_type === 'MULTIPLE_CHOICE' ? 'MULTIPLE_CHOICE' : 'SHORT_ANSWER';
const base = {
id: (existingQuestions.length + 1).toString(), id: (existingQuestions.length + 1).toString(),
questionText: q.question_text, questionText: q.question_text,
questionType: 'SHORT_ANSWER', questionType: qType,
keyPoints: q.key_points, keyPoints: q.key_points || [],
difficulty: q.difficulty, difficulty: q.difficulty || 'STANDARD',
basis: q.basis, basis: q.basis || '',
dimension: inferredDimension, dimension: inferredDimension,
}; };
if (qType === 'MULTIPLE_CHOICE') {
return {
...base,
options: q.options || [],
correctAnswer: q.correct_answer || '',
judgment: q.judgment || '',
};
}
return base;
}); });
const questionsToGenerate = Math.max(1, limitCount - existingQuestions.length); const questionsToGenerate = Math.max(1, limitCount - existingQuestions.length);
server/src/assessment/graph/nodes/grader.node.ts
+70 -2
View File
@@ -91,6 +91,72 @@ export const graderNode = async (
}; };
} }
// ── Rule-based grading: use structured followupMapping if available ──
if (currentQuestion.followupHints) {
let mapping: any = null;
if (typeof currentQuestion.followupHints === 'string') {
try { mapping = JSON.parse(currentQuestion.followupHints); } catch {}
} else if (typeof currentQuestion.followupHints === 'object') {
mapping = currentQuestion.followupHints;
}
if (mapping && Array.isArray(mapping.branches)) {
const userAnswerText = typeof lastUserMessage.content === 'string'
? lastUserMessage.content : JSON.stringify(lastUserMessage.content);
// Score based on keyword coverage
let bestScore = mapping.defaultScore ?? 5;
let matchedFollowup = mapping.defaultFollowup || '';
let matchedAll = true;
const maxFollowUps = mapping.maxFollowups ?? 2;
for (const branch of mapping.branches) {
const kws = branch.keywords || [];
const matchCount = kws.filter((kw: string) => userAnswerText.toLowerCase().includes(kw.toLowerCase())).length;
if (kws.length > 0 && matchCount >= kws.length * 0.5) {
const branchScore = branch.score ?? 7;
if (branchScore > bestScore) bestScore = branchScore;
if (branch.followup) matchedFollowup = branch.followup;
} else if (kws.length > 0 && matchCount === 0) {
matchedAll = false;
}
}
const completionThreshold = mapping.completionThreshold ?? 80;
const tooShort = userAnswerText.trim().length < 8;
const saysIDontKnow = userAnswerText.trim().length < 10 && (
userAnswerText.includes('不知道') || userAnswerText.includes("don't know") || userAnswerText.includes('わかりません')
);
let shouldFollowUp: boolean;
if (saysIDontKnow || tooShort) {
shouldFollowUp = false;
bestScore = Math.min(bestScore, 2);
} else if (bestScore >= completionThreshold / 10) {
shouldFollowUp = false;
} else if (currentFollowUpCount >= maxFollowUps) {
shouldFollowUp = false;
} else {
shouldFollowUp = true;
}
const feedbackMessage = new AIMessage(`Score: ${bestScore}/10\n\nFeedback: ${shouldFollowUp ? matchedFollowup : '回答已覆盖关键点。'}`);
const feedbackHistoryMessages = shouldFollowUp && matchedFollowup
? [feedbackMessage, new AIMessage(matchedFollowup)]
: [feedbackMessage];
console.log('[GraderNode] Rule grading:', { score: bestScore, shouldFollowUp, matchedAll, followup: matchedFollowup?.substring(0, 60) });
return {
feedbackHistory: feedbackHistoryMessages,
scores: { [currentQuestion.id || currentQuestionIndex.toString()]: bestScore },
shouldFollowUp,
followUpCount: shouldFollowUp ? currentFollowUpCount + 1 : 0,
currentQuestionIndex: shouldFollowUp ? currentQuestionIndex : currentQuestionIndex + 1,
} as any;
}
}
const systemPromptZh = `你是一位考官。请评分并给出反馈。 const systemPromptZh = `你是一位考官。请评分并给出反馈。
规则： 规则：
@@ -100,8 +166,10 @@ export const graderNode = async (
问题：${currentQuestion.questionText} 问题：${currentQuestion.questionText}
关键点：${currentQuestion.keyPoints.join(', ')} 关键点：${currentQuestion.keyPoints.join(', ')}
评分标准：准确性、完整性、深度。 评分标准：不要求深度，不要求使用特定术语，只看用户是否理解了概念。
部分正确也给分（5-7分），完全不沾边才0-2分。 用户理解核心概念就给分。即使没有使用关键点中的原词，只要意思到位就算覆盖。
例如关键点是"上下文窗口有限"，用户说"信息太多超过AI处理长度"也是覆盖。
评分原则：往宽了给分，不确定时就给高分。明显正确就给8-10分，部分正确5-7分，完全不沾边才0-2分。
返回JSON： 返回JSON：
- score: 0-10 - score: 0-10
server/src/assessment/services/question-bank.service.ts
+40 -32
View File
@@ -456,10 +456,6 @@ export class QuestionBankService {
): Promise<QuestionBankItem[]> { ): Promise<QuestionBankItem[]> {
const bank = await this.findOne(bankId); const bank = await this.findOne(bankId);
if (bank.status !== QuestionBankStatus.DRAFT) {
throw new ForbiddenException('仅草稿状态的题库可生成题目');
}
if (count <= 0 || count > 50) { if (count <= 0 || count > 50) {
throw new BadRequestException('生成数量必须在 1-50 之间'); throw new BadRequestException('生成数量必须在 1-50 之间');
} }
@@ -523,6 +519,7 @@ export class QuestionBankService {
async selectQuestions( async selectQuestions(
bankId: string, bankId: string,
count: number, count: number,
dimensionWeights?: Array<{ name: string; weight: number }>,
): Promise<QuestionBankItem[]> { ): Promise<QuestionBankItem[]> {
const bank = await this.findOne(bankId); const bank = await this.findOne(bankId);
@@ -537,40 +534,51 @@ export class QuestionBankService {
const usedIds = new Set<string>(); const usedIds = new Set<string>();
const selected: QuestionBankItem[] = []; const selected: QuestionBankItem[] = [];
const availableItems = [...allItems]; let availableItems = [...allItems];
let dimIdx = 0; if (dimensionWeights && dimensionWeights.length > 0) {
while (selected.length < count && availableItems.length > 0) { const totalWeight = dimensionWeights.reduce((s, d) => s + d.weight, 0);
const dim = DIMENSIONS[dimIdx % DIMENSIONS.length]; for (const dw of dimensionWeights) {
dimIdx++; const dimName = dw.name as QuestionDimension;
const targetForDim = Math.round(count * dw.weight / totalWeight);
const available = availableItems.filter( const pool = availableItems.filter(i => i.dimension === dimName && !usedIds.has(i.id));
(i) => i.dimension === dim && !usedIds.has(i.id), this.shuffleArray(pool);
); const take = Math.min(targetForDim, pool.length);
for (let i = 0; i < take; i++) {
if (available.length > 0) { selected.push(pool[i]);
const idx = Math.floor(Math.random() * available.length); usedIds.add(pool[i].id);
const item = available[idx];
selected.push(item);
usedIds.add(item.id);
const actualIdx = availableItems.findIndex(i => i.id === item.id);
if (actualIdx > -1) {
availableItems.splice(actualIdx, 1);
} }
} }
availableItems = availableItems.filter(i => !usedIds.has(i.id));
if (dimIdx >= DIMENSIONS.length * 3) { this.shuffleArray(availableItems);
break; while (selected.length < count && availableItems.length > 0) {
const item = availableItems.pop()!;
selected.push(item);
usedIds.add(item.id);
} }
} } else {
let dimIdx = 0;
if (selected.length < count && availableItems.length > 0) { while (selected.length < count && availableItems.length > 0) {
const shuffled = this.shuffleArray([...availableItems]); const dim = DIMENSIONS[dimIdx % DIMENSIONS.length];
for (const item of shuffled) { dimIdx++;
if (selected.length >= count) break; const pool = availableItems.filter(i => i.dimension === dim && !usedIds.has(i.id));
if (!usedIds.has(item.id)) { if (pool.length > 0) {
const idx = Math.floor(Math.random() * pool.length);
const item = pool[idx];
selected.push(item); selected.push(item);
usedIds.add(item.id); usedIds.add(item.id);
availableItems = availableItems.filter(i => i.id !== item.id);
}
if (dimIdx >= DIMENSIONS.length * 3) break;
}
if (selected.length < count && availableItems.length > 0) {
this.shuffleArray(availableItems);
for (const item of availableItems) {
if (selected.length >= count) break;
if (!usedIds.has(item.id)) {
selected.push(item);
usedIds.add(item.id);
}
} }
} }
} }
server/src/assessment/services/template.service.ts
+1
View File
@@ -49,6 +49,7 @@ export class TemplateService {
const { ...data } = createDto; const { ...data } = createDto;
const template = this.templateRepository.create({ const template = this.templateRepository.create({
...data, ...data,
isActive: data.isActive !== undefined ? data.isActive : true,
createdBy: userId, createdBy: userId,
tenantId, tenantId,
}); });
web/components/views/AssessmentTemplateManager.tsx
+3 -3
View File
@@ -76,7 +76,7 @@ export const AssessmentTemplateManager: React.FC = () => {
: (template.difficultyDistribution || ''), : (template.difficultyDistribution || ''),
style: template.style || 'Professional', style: template.style || 'Professional',
knowledgeGroupId: template.knowledgeGroupId || '', knowledgeGroupId: template.knowledgeGroupId || '',
passingScore: template.passingScore ? template.passingScore / 10 : 6, passingScore: template.passingScore !== null && template.passingScore !== undefined ? template.passingScore / 10 : 6,
totalTimeLimit: template.totalTimeLimit ?? 1800, totalTimeLimit: template.totalTimeLimit ?? 1800,
perQuestionTimeLimit: template.perQuestionTimeLimit ?? 300, perQuestionTimeLimit: template.perQuestionTimeLimit ?? 300,
}); });
@@ -436,7 +436,7 @@ export const AssessmentTemplateManager: React.FC = () => {
<label className="text-[10px] font-black text-slate-400 uppercase tracking-widest px-1 ml-1 flex items-center gap-2"> <label className="text-[10px] font-black text-slate-400 uppercase tracking-widest px-1 ml-1 flex items-center gap-2">
<Hash size={12} className="text-indigo-500" />总时间限制 (秒) <Hash size={12} className="text-indigo-500" />总时间限制 (秒)
</label> </label>
<input type="number" min="60" max="7200" step="60" <input type="number" min="60" max="86400" step="60"
className="w-full px-5 py-4 bg-slate-50 border border-slate-200 rounded-[1.25rem] text-sm font-bold focus:ring-4 focus:ring-indigo-500/10 focus:border-indigo-500/50 outline-none transition-all" className="w-full px-5 py-4 bg-slate-50 border border-slate-200 rounded-[1.25rem] text-sm font-bold focus:ring-4 focus:ring-indigo-500/10 focus:border-indigo-500/50 outline-none transition-all"
value={formData.totalTimeLimit} value={formData.totalTimeLimit}
onChange={e => setFormData({ ...formData, totalTimeLimit: parseInt(e.target.value) || 1800 })} onChange={e => setFormData({ ...formData, totalTimeLimit: parseInt(e.target.value) || 1800 })}
@@ -446,7 +446,7 @@ export const AssessmentTemplateManager: React.FC = () => {
<label className="text-[10px] font-black text-slate-400 uppercase tracking-widest px-1 ml-1 flex items-center gap-2"> <label className="text-[10px] font-black text-slate-400 uppercase tracking-widest px-1 ml-1 flex items-center gap-2">
<Hash size={12} className="text-indigo-500" />单题时间限制 (秒) <Hash size={12} className="text-indigo-500" />单题时间限制 (秒)
</label> </label>
<input type="number" min="30" max="1800" step="30" <input type="number" min="30" max="3600" step="30"
className="w-full px-5 py-4 bg-slate-50 border border-slate-200 rounded-[1.25rem] text-sm font-bold focus:ring-4 focus:ring-indigo-500/10 focus:border-indigo-500/50 outline-none transition-all" className="w-full px-5 py-4 bg-slate-50 border border-slate-200 rounded-[1.25rem] text-sm font-bold focus:ring-4 focus:ring-indigo-500/10 focus:border-indigo-500/50 outline-none transition-all"
value={formData.perQuestionTimeLimit} value={formData.perQuestionTimeLimit}
onChange={e => setFormData({ ...formData, perQuestionTimeLimit: parseInt(e.target.value) || 300 })} onChange={e => setFormData({ ...formData, perQuestionTimeLimit: parseInt(e.target.value) || 300 })}
web/components/views/AssessmentView.tsx
+6 -9
View File
@@ -296,6 +296,8 @@ export const AssessmentView: React.FC<AssessmentViewProps> = ({
if (event.data.status === 'COMPLETED') { if (event.data.status === 'COMPLETED') {
setSession(prev => prev ? { ...prev, status: 'COMPLETED' } : null); setSession(prev => prev ? { ...prev, status: 'COMPLETED' } : null);
fetchHistory(); fetchHistory();
} else if (event.data.currentQuestionIndex !== undefined) {
assessmentService.nextQuestion(session.id).catch(() => {});
} }
} }
} }
@@ -620,7 +622,6 @@ export const AssessmentView: React.FC<AssessmentViewProps> = ({
{currentQuestion.options.map((opt: string, i: number) => { {currentQuestion.options.map((opt: string, i: number) => {
const letter = optionLabels[i]; const letter = optionLabels[i];
const isSelected = selectedChoice === letter; const isSelected = selectedChoice === letter;
const displayText = opt.slice(1);
return ( return (
<button <button
key={letter} key={letter}
@@ -634,10 +635,7 @@ export const AssessmentView: React.FC<AssessmentViewProps> = ({
isTimedOut && "opacity-50 cursor-not-allowed" isTimedOut && "opacity-50 cursor-not-allowed"
)} )}
> >
<span className="inline-flex items-center justify-center w-7 h-7 rounded-xl text-xs font-black mr-3 shrink-0 border-2 border-current"> {opt}
{letter}
</span>
{displayText}
</button> </button>
); );
})} })}
@@ -662,7 +660,7 @@ export const AssessmentView: React.FC<AssessmentViewProps> = ({
value={inputValue} value={inputValue}
onChange={(e) => setInputValue(e.target.value)} onChange={(e) => setInputValue(e.target.value)}
onKeyDown={(e) => { onKeyDown={(e) => {
if (e.key === 'Enter' && !e.shiftKey && !isTimedOut) { if (e.key === 'Enter' && (e.ctrlKey || e.metaKey) && !isTimedOut) {
e.preventDefault(); e.preventDefault();
handleSubmitAnswer(); handleSubmitAnswer();
} }
@@ -867,16 +865,15 @@ export const AssessmentView: React.FC<AssessmentViewProps> = ({
{q.options?.map((opt: string, oi: number) => { {q.options?.map((opt: string, oi: number) => {
const letter = String.fromCharCode(65 + oi); const letter = String.fromCharCode(65 + oi);
const isAnswer = letter === q.correctAnswer; const isAnswer = letter === q.correctAnswer;
const displayText = opt.slice(1);
return ( return (
<span key={oi} className={cn( <span key={oi} className={cn(
"px-3 py-1 rounded-lg font-medium", "px-3 py-1 rounded-lg font-medium",
isAnswer ? "bg-emerald-100 text-emerald-700 border border-emerald-200" : "bg-slate-50 text-slate-500" isAnswer ? "bg-emerald-100 text-emerald-700 border border-emerald-200" : "bg-slate-50 text-slate-500"
)}> )}>
{letter}. {displayText} {opt}
</span> </span>
); );
})} })}
</div> </div>
)} )}
{q.judgment && ( {q.judgment && (
web/services/assessmentService.ts
+5
View File
@@ -147,6 +147,11 @@ export class AssessmentService {
return data; return data;
} }
async nextQuestion(sessionId: string): Promise<{ success: boolean }> {
const { data } = await apiClient.post<{ success: boolean }>(`/assessment/${sessionId}/next-question`, {});
return data;
}
async forceEnd(sessionId: string): Promise<AssessmentSession> { async forceEnd(sessionId: string): Promise<AssessmentSession> {
const { data } = await apiClient.post<AssessmentSession>(`/assessment/${sessionId}/force-end`, {}); const { data } = await apiClient.post<AssessmentSession>(`/assessment/${sessionId}/force-end`, {});
return data; return data;