feat: implement QuestionBank CRUD with pagination and template query
- Add pagination support to findAll (page, limit query params) - Add findByTemplateId method to service - Add GET /by-template/:templateId endpoint to controller - Service already includes CRUD for QuestionBank and QuestionBankItem
This commit is contained in:
@@ -0,0 +1,77 @@
|
||||
import { StateGraph, MemorySaver } from '@langchain/langgraph';
|
||||
import { EvaluationAnnotation } from './state';
|
||||
import { questionGeneratorNode } from './nodes/generator.node';
|
||||
import { interviewerNode } from './nodes/interviewer.node';
|
||||
import { graderNode } from './nodes/grader.node';
|
||||
import { reportAnalyzerNode } from './nodes/analyzer.node';
|
||||
|
||||
/**
|
||||
* Conditional routing logic for the Grader node.
|
||||
*/
|
||||
const routeAfterGrading = (state: typeof EvaluationAnnotation.State) => {
|
||||
const targetCount = state.questionCount || 5;
|
||||
const questionsLen = state.questions?.length || 0;
|
||||
|
||||
console.log('[Router] Evaluation Result:', {
|
||||
currentIndex: state.currentQuestionIndex,
|
||||
shouldFollowUp: state.shouldFollowUp,
|
||||
numQuestions: questionsLen,
|
||||
targetCount,
|
||||
});
|
||||
|
||||
if (state.shouldFollowUp) {
|
||||
console.log('[Router] Routing to follow-up interviewer');
|
||||
return 'interviewer';
|
||||
}
|
||||
|
||||
if (state.currentQuestionIndex < targetCount) {
|
||||
// If the next question isn't generated yet, go back to generator
|
||||
if (state.currentQuestionIndex >= questionsLen) {
|
||||
console.log('[Router] Index >= Questions, routing to generator');
|
||||
return 'generator';
|
||||
}
|
||||
// If it is generated, go to interviewer
|
||||
console.log('[Router] Index < Questions, routing to interviewer');
|
||||
return 'interviewer';
|
||||
}
|
||||
|
||||
console.log('[Router] Assessment complete, routing to analyzer');
|
||||
return 'analyzer';
|
||||
};
|
||||
|
||||
/**
|
||||
* Builds and compiles the Evaluation Graph.
|
||||
*/
|
||||
export const createEvaluationGraph = () => {
|
||||
const workflow = new StateGraph(EvaluationAnnotation)
|
||||
.addNode('generator', questionGeneratorNode)
|
||||
.addNode('interviewer', interviewerNode)
|
||||
.addNode('grader', graderNode)
|
||||
.addNode('analyzer', reportAnalyzerNode)
|
||||
|
||||
// Flow definition
|
||||
.addEdge('__start__', 'generator')
|
||||
.addEdge('generator', 'interviewer')
|
||||
|
||||
// After interviewer, the graph will naturally pause for user input
|
||||
// if we use it in a thread-safe way with interrupts or simple invocation.
|
||||
.addEdge('interviewer', 'grader')
|
||||
|
||||
// After grading, decide where to go
|
||||
.addConditionalEdges('grader', routeAfterGrading, {
|
||||
interviewer: 'interviewer',
|
||||
generator: 'generator',
|
||||
analyzer: 'analyzer',
|
||||
})
|
||||
|
||||
.addEdge('analyzer', '__end__');
|
||||
|
||||
// Using MemorySaver for thread-based persistence
|
||||
const checkpointer = new MemorySaver();
|
||||
|
||||
return workflow.compile({
|
||||
checkpointer,
|
||||
// We want the graph to stop after the interviewer presents the question
|
||||
interruptAfter: ['interviewer'],
|
||||
});
|
||||
};
|
||||
@@ -0,0 +1,162 @@
|
||||
import { ChatOpenAI } from '@langchain/openai';
|
||||
import { SystemMessage, HumanMessage } from '@langchain/core/messages';
|
||||
import { RunnableConfig } from '@langchain/core/runnables';
|
||||
import { EvaluationState } from '../state';
|
||||
|
||||
/**
|
||||
* Node responsible for generating the final mastery report at the end of the session.
|
||||
*/
|
||||
export const reportAnalyzerNode = async (
|
||||
state: EvaluationState,
|
||||
config?: RunnableConfig,
|
||||
): Promise<Partial<EvaluationState>> => {
|
||||
const { model } = (config?.configurable as any) || {};
|
||||
const { scores, messages } = state;
|
||||
const questionList = state.questions || [];
|
||||
|
||||
console.log('[AnalyzerNode] Entering node...', {
|
||||
numScores: Object.keys(scores || {}).length,
|
||||
numMessages: messages?.length,
|
||||
scores,
|
||||
});
|
||||
|
||||
if (!model) {
|
||||
throw new Error('Missing model in node configuration');
|
||||
}
|
||||
|
||||
const scoreSummary = Object.entries(scores)
|
||||
.map(([qId, score]) => {
|
||||
const displayId = isNaN(parseInt(qId))
|
||||
? qId
|
||||
: (parseInt(qId) + 1).toString();
|
||||
return `Question ${displayId}: Score ${score}/10`;
|
||||
})
|
||||
.join('\n');
|
||||
|
||||
const dimensionSummary = questionList.reduce((acc: Record<string, number[]>, q: any) => {
|
||||
const dim = q.dimension || 'workCapability';
|
||||
const score = scores[q.id] || 0;
|
||||
if (!acc[dim]) acc[dim] = [];
|
||||
acc[dim].push(score);
|
||||
return acc;
|
||||
}, {});
|
||||
|
||||
const dimensionAvg = Object.entries(dimensionSummary).map(([dim, arr]: [string, any]) => {
|
||||
const avg = arr.reduce((a: number, b: number) => a + b, 0) / arr.length;
|
||||
return `${dim}: ${avg.toFixed(1)}/10`;
|
||||
}).join('\n');
|
||||
|
||||
const isZh = state.language === 'zh';
|
||||
const isJa = state.language === 'ja';
|
||||
|
||||
const systemPromptZh = `你是一位客观且严谨的高级教育顾问。
|
||||
请审查以下评估结果,并为员工提供一份严谨的掌握程度报告。
|
||||
|
||||
重要提示:
|
||||
1. **你必须使用以下语言生成报告:中文 (Simplified Chinese)**。
|
||||
2. **严禁夹杂日文**。即使对话记录中包含日文,报告内容也必须全中文。
|
||||
3. 报告的第一行必须严格遵守此格式:"LEVEL: [Novice/Proficient/Advanced/Expert]"。
|
||||
4. 必须保持客观。如果用户没有提供有效的回答或得分为 0,你必须将其识别为 'Novice',并明确指出他们尚未证明其掌握程度。
|
||||
5. 不要虚构或幻想优点(如"潜力"或"好奇心"),如果用户明确表示"不知道"或未提供实质内容。
|
||||
6. 专注于对话记录中已证明的事实。
|
||||
|
||||
各维度得分:
|
||||
${dimensionAvg}
|
||||
|
||||
问题与得分:
|
||||
${scoreSummary}
|
||||
|
||||
对话记录:
|
||||
${messages
|
||||
.filter((m: any) => m._getType() !== 'system')
|
||||
.map((m: any) => `${m.role || m._getType()}: ${m.content}`)
|
||||
.join('\n')}
|
||||
|
||||
报告结构:
|
||||
1. 总体级别(已在顶部指定)
|
||||
2. 各维度得分分析(提示词、LLM、IDE、开发范式、工作能力)
|
||||
3. 薄弱环节识别
|
||||
4. 针对性改进建议
|
||||
5. 推荐的学习路径。`;
|
||||
|
||||
const systemPromptJa = `あなたは客観的で厳格なシニア教育コンサルタントです。
|
||||
以下の評価結果をレビューし、従業員に対して厳格な習熟度レポートを提供してください。
|
||||
|
||||
重要事項:
|
||||
1. **レポートは必ず次の言語で生成してください:日本語**。
|
||||
2. **中国語を混ぜないでください**。会話ログに中国語が含まれていても、レポートの内容はすべて日本語で記述してください。
|
||||
3. レポートの最初の行は, 必ず次の形式に従ってください:"LEVEL: [Novice/Proficient/Advanced/Expert]"。
|
||||
4. 客観的であること。ユーザーが有効な回答を提供しなかった場合、またはスコアが 0 の場合、'Novice' と判定し、習熟度が証明されていないことを明示してください。
|
||||
5. ユーザーが「わからない」と言ったり、内容を提供しなかった場合に、長所(「ポテンシャル」や「好奇心」など)を捏造しないでください。
|
||||
6. 会話ログで証明された事実に集中してください。
|
||||
|
||||
各ディメンションスコア:
|
||||
${dimensionAvg}
|
||||
|
||||
質問とスコア:
|
||||
${scoreSummary}
|
||||
|
||||
会話ログ:
|
||||
${messages
|
||||
.filter((m: any) => m._getType() !== 'system')
|
||||
.map((m: any) => `${m.role || m._getType()}: ${m.content}`)
|
||||
.join('\n')}
|
||||
|
||||
レポート構成:
|
||||
1. 総合レベル(一番上に指定済み)
|
||||
2. 各ディメンション分析(提示詞、LLM、IDE、開発範式、工作能力)
|
||||
3. 薄弱环节识别
|
||||
4. 推奨される学習パス。`;
|
||||
|
||||
const systemPromptEn = `You are an objective and critical seniority education consultant.
|
||||
Review the following assessment results and provide a rigorous mastery report for the employee.
|
||||
|
||||
IMPORTANT:
|
||||
1. **You MUST generate the report strictly in English.**
|
||||
2. START the report with exactly this format: "LEVEL: [Novice/Proficient/Advanced/Expert]" on the first line.
|
||||
3. Be OBJECTIVE. If the user provided no valid answers or scores are 0, you MUST identify them as 'Novice' and explicitly state they have NOT demonstrated mastery.
|
||||
4. DO NOT invent or hallucinate strengths (like 'potential' or 'curiosity') if the user explicitly said "I don't know" or provided no content.
|
||||
5. Focus on what was PROVEN in the conversation logs.
|
||||
|
||||
DIMENSION SCORES:
|
||||
${dimensionAvg}
|
||||
|
||||
QUESTIONS AND SCORES:
|
||||
${scoreSummary}
|
||||
|
||||
CONVERSATION LOGS:
|
||||
${messages
|
||||
.filter((m: any) => m._getType() !== 'system')
|
||||
.map((m: any) => `${m.role || m._getType()}: ${m.content}`)
|
||||
.join('\n')}
|
||||
|
||||
REPORT STRUCTURE:
|
||||
1. Overall Level (Already specified at top)
|
||||
2. Dimension Analysis (Prompt, LLM, IDE, DevPattern, WorkCapability)
|
||||
3. Weak Areas Identification
|
||||
4. Targeted Learning Recommendations.`;
|
||||
|
||||
const systemPrompt = isZh
|
||||
? systemPromptZh
|
||||
: isJa
|
||||
? systemPromptJa
|
||||
: systemPromptEn;
|
||||
const humanMsg = isZh
|
||||
? '生成最终掌握程度报告。'
|
||||
: isJa
|
||||
? '最終的な習熟度レポートを生成してください。'
|
||||
: 'Generate the final mastery report.';
|
||||
|
||||
const response = await model.invoke([
|
||||
new SystemMessage(systemPrompt),
|
||||
new HumanMessage(humanMsg),
|
||||
]);
|
||||
|
||||
console.log(
|
||||
'[AnalyzerNode] Report generated successfully. Length:',
|
||||
response.content?.toString().length,
|
||||
);
|
||||
return {
|
||||
report: response.content as string,
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,246 @@
|
||||
import { ChatOpenAI } from '@langchain/openai';
|
||||
import { SystemMessage, HumanMessage } from '@langchain/core/messages';
|
||||
import { RunnableConfig } from '@langchain/core/runnables';
|
||||
import { EvaluationState } from '../state';
|
||||
import { safeParseJson } from '../../../common/json-utils';
|
||||
|
||||
/**
|
||||
* Node responsible for generating assessment questions based on the knowledge base content.
|
||||
*/
|
||||
export const questionGeneratorNode = async (
|
||||
state: EvaluationState,
|
||||
config?: RunnableConfig,
|
||||
): Promise<Partial<EvaluationState>> => {
|
||||
const { model, knowledgeBaseContent, targetCount } = (config?.configurable as any) || {};
|
||||
const limitCount = targetCount || 5;
|
||||
|
||||
console.log('[GeneratorNode] Starting generation...', {
|
||||
language: state.language,
|
||||
hasModel: !!model,
|
||||
contentLength: knowledgeBaseContent?.length,
|
||||
keywords: state.keywords || [],
|
||||
targetCount: limitCount,
|
||||
});
|
||||
|
||||
if (!model || !knowledgeBaseContent) {
|
||||
console.error('[GeneratorNode] Missing model or knowledgeBaseContent');
|
||||
throw new Error(
|
||||
'Missing model or knowledgeBaseContent in node configuration',
|
||||
);
|
||||
}
|
||||
|
||||
const isZh = state.language === 'zh';
|
||||
const isJa = state.language === 'ja';
|
||||
|
||||
const style = state.style || 'technical';
|
||||
const difficultyText = state.difficultyDistribution
|
||||
? JSON.stringify(state.difficultyDistribution)
|
||||
: isZh
|
||||
? '随机分布'
|
||||
: isJa
|
||||
? 'ランダム分布'
|
||||
: 'Random distribution';
|
||||
const keywords = state.keywords || [];
|
||||
const hasKeywords = keywords.length > 0;
|
||||
const keywordText = hasKeywords ? keywords.join(', ') : '';
|
||||
|
||||
const rulesZh = [
|
||||
`**禁止重复**:绝对禁止生成与下方“禁止重复列表”中相似的题目。`,
|
||||
`**深度挖掘**:如果之前的题目考查了核心定义,新题目必须考查具体的应用案例、对比分析或隐藏的细节。`,
|
||||
hasKeywords
|
||||
? `**关键词权重**:必须围绕关键词 (${keywordText}) 展开,但要从关键词的不同侧面(如流程、限制、优缺点、具体参数等)进行挖掘。`
|
||||
: null,
|
||||
`**随机扰动**:即使对于相同的主题或关键词,也要尝试从不同的逻辑链条(如“因为...所以...” vs “如果没有...会怎样”)出发。`,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.map((r, i) => `${i + 1}. ${r}`)
|
||||
.join('\n');
|
||||
|
||||
const rulesJa = [
|
||||
`**重複禁止**:下記の「作成済み問題リスト」と類似した内容は絶対に避けてください。`,
|
||||
`**多角的アプローチ**:前回が定義だった場合は、今回は応用方法、制限事項、具体的な数値などに焦点を当ててください。`,
|
||||
hasKeywords
|
||||
? `**キーワードの深掘り**:キーワード (${keywordText}) の異なる側面から出題してください。`
|
||||
: null,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.map((r, i) => `${i + 1}. ${r}`)
|
||||
.join('\n');
|
||||
|
||||
const rulesEn = [
|
||||
`**NO REPETITION**: Strictly avoid any conceptual overlap with the "Previous Questions" list below.`,
|
||||
`**New Facets**: If previous questions were about definitions, focus on applications, edge cases, or specific details.`,
|
||||
hasKeywords
|
||||
? `**Keyword Variety**: Center on (${keywordText}), but explore different aspects (process, pros/cons, requirements).`
|
||||
: null,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.map((r, i) => `${i + 1}. ${r}`)
|
||||
.join('\n');
|
||||
|
||||
const existingQuestions = state.questions || [];
|
||||
const existingQuestionsText = existingQuestions
|
||||
.map((q, i) => `Q${i + 1}: ${q.questionText}`)
|
||||
.join('\n');
|
||||
|
||||
const systemPromptZh = `你是一位专业的知识评估专家。请根据提供的知识库片段生成 1 个唯一的测试题目。
|
||||
|
||||
### 强制性语言规则:
|
||||
**必须使用中文 (Simplified Chinese) 进行回复**。即使知识库内容是英文或其他语言,问题(question_text)和关键点(key_points)也必须使用中文。
|
||||
|
||||
### 强制性多样性规则:
|
||||
${rulesZh}
|
||||
|
||||
### 禁止重复列表(已出过):
|
||||
${existingQuestionsText || '无'}
|
||||
|
||||
### 任务:
|
||||
${hasKeywords ? `目标关键词:${keywordText}\n` : ''}出题风格:${style}
|
||||
难度:${difficultyText}
|
||||
|
||||
请以 JSON 数组格式返回 1 个问题:
|
||||
[
|
||||
{
|
||||
"question_text": "...",
|
||||
"key_points": ["点1", "点2"],
|
||||
"difficulty": "...",
|
||||
"dimension": "prompt/llm/ide/devPattern/workCapability",
|
||||
"basis": "[n] 引用原文..."
|
||||
}
|
||||
]`;
|
||||
// dimension取值:prompt=提示词, llm=LLM原理, ide=IDE协作, devPattern=开发范式, workCapability=工作能力
|
||||
|
||||
const systemPromptJa = `あなたは専門的なアセスメントエキスパートです。提供されたナレッジベースに基づいて、ユニークな問題を 1 つ作成してください。
|
||||
|
||||
### 言語ルール(最重要):
|
||||
**必ず日本語で作成してください**。提供されたナレッジベースが英語や中国語、その他の言語であっても、質問文(question_text)およびキーポイント(key_points)は必ず日本語で回答してください。中国語が混ざらないように厳格に注意してください。
|
||||
|
||||
### 多様性ルール:
|
||||
${rulesJa}
|
||||
|
||||
### 作成済み問題リスト:
|
||||
${existingQuestionsText || 'なし'}
|
||||
|
||||
### 任務:
|
||||
${hasKeywords ? `目標キーワード:${keywordText}\n` : ''}出題スタイル:${style}
|
||||
難易度:${difficultyText}
|
||||
|
||||
以下のJSON配列形式で問題を1つ返してください:
|
||||
[
|
||||
{
|
||||
"question_text": "...",
|
||||
"key_points": ["ポイント1", "ポイント2"],
|
||||
"difficulty": "...",
|
||||
"dimension": "prompt/llm/ide/devPattern/workCapability",
|
||||
"basis": "[n] 引用箇所..."
|
||||
}
|
||||
]`;
|
||||
|
||||
const systemPromptEn = `You are an expert examiner. Generate 1 UNIQUE question based on the provided context.
|
||||
|
||||
### Language Rule:
|
||||
**You MUST generate the question and key points in English.**
|
||||
|
||||
### Diversity Rules:
|
||||
${rulesEn}
|
||||
|
||||
### Previous Questions (DO NOT REPEAT):
|
||||
${existingQuestionsText || 'None'}
|
||||
|
||||
Return 1 question as a JSON array with format:
|
||||
[
|
||||
{
|
||||
"question_text": "...",
|
||||
"key_points": ["point1", "point2"],
|
||||
"difficulty": "...",
|
||||
"dimension": "prompt/llm/ide/devPattern/workCapability",
|
||||
"basis": "[n] citation..."
|
||||
}
|
||||
]`;
|
||||
|
||||
// dimension values: prompt=prompt engineering, llm=LLM principles, ide=IDE collaboration, devPattern=development paradigm, workCapability=work capability
|
||||
|
||||
const systemPrompt = isZh
|
||||
? systemPromptZh
|
||||
: isJa
|
||||
? systemPromptJa
|
||||
: systemPromptEn;
|
||||
const humanMsg = isZh
|
||||
? `请使用中文基于以下内容生成题目:\n\n${knowledgeBaseContent}`
|
||||
: isJa
|
||||
? `以下の内容に基づいて、必ず日本語でアセスメント問題を作成してください:\n\n${knowledgeBaseContent}`
|
||||
: `Generate evaluation question in English based on:\n\n${knowledgeBaseContent}`;
|
||||
|
||||
try {
|
||||
const response = await model.invoke([
|
||||
new SystemMessage(systemPrompt),
|
||||
new HumanMessage(humanMsg),
|
||||
]);
|
||||
|
||||
try {
|
||||
let newQuestions = safeParseJson<any>(response.content as string);
|
||||
|
||||
if (!newQuestions) {
|
||||
console.error('[GeneratorNode] Failed to parse JSON. Raw content:', response.content);
|
||||
throw new Error('Invalid JSON format from AI');
|
||||
}
|
||||
|
||||
// Handle both array and single object
|
||||
if (!Array.isArray(newQuestions)) {
|
||||
newQuestions = [newQuestions];
|
||||
}
|
||||
|
||||
const dimensionMap: Record<string, string> = {
|
||||
// 中文
|
||||
'技术能力-提示词': 'prompt',
|
||||
'提示词': 'prompt',
|
||||
'技术能力-LLM': 'llm',
|
||||
'LLM': 'llm',
|
||||
'IDE协作能力': 'ide',
|
||||
'IDE': 'ide',
|
||||
'AI开发范式': 'devPattern',
|
||||
'开发范式': 'devPattern',
|
||||
'工作能力-安全': 'workCapability',
|
||||
'工作能力': 'workCapability',
|
||||
// 英文直接映射
|
||||
'prompt': 'prompt',
|
||||
'llm': 'llm',
|
||||
'ide': 'ide',
|
||||
'devPattern': 'devPattern',
|
||||
'workCapability': 'workCapability',
|
||||
};
|
||||
|
||||
const mappedNewQuestions = newQuestions.map((q: any) => {
|
||||
let inferredDimension = 'workCapability';
|
||||
const dimValue = q.dimension?.toString().toLowerCase().trim();
|
||||
if (dimValue) {
|
||||
inferredDimension = dimensionMap[dimValue] || 'workCapability';
|
||||
console.log('[GeneratorNode] Dimension mapping:', { original: q.dimension, mapped: inferredDimension });
|
||||
}
|
||||
return {
|
||||
id: (existingQuestions.length + 1).toString(),
|
||||
questionText: q.question_text,
|
||||
keyPoints: q.key_points,
|
||||
difficulty: q.difficulty,
|
||||
basis: q.basis,
|
||||
dimension: inferredDimension,
|
||||
};
|
||||
});
|
||||
|
||||
const questionsToGenerate = Math.max(1, limitCount - existingQuestions.length);
|
||||
const limitedNewQuestions = mappedNewQuestions.slice(0, questionsToGenerate);
|
||||
|
||||
console.log('[GeneratorNode] Generated questions:', mappedNewQuestions.length, 'Limit:', questionsToGenerate);
|
||||
|
||||
return {
|
||||
questions: [...existingQuestions, ...limitedNewQuestions],
|
||||
};
|
||||
} catch (error) {
|
||||
console.error('[GeneratorNode] Parse error:', error);
|
||||
return { questions: existingQuestions };
|
||||
}
|
||||
} catch (invokeError) {
|
||||
console.error('[GeneratorNode] Invoke error:', invokeError);
|
||||
throw invokeError;
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,252 @@
|
||||
import { ChatOpenAI } from '@langchain/openai';
|
||||
import {
|
||||
SystemMessage,
|
||||
HumanMessage,
|
||||
AIMessage,
|
||||
} from '@langchain/core/messages';
|
||||
import { RunnableConfig } from '@langchain/core/runnables';
|
||||
import { EvaluationState } from '../state';
|
||||
import { safeParseJson } from '../../../common/json-utils';
|
||||
|
||||
/**
|
||||
* Node responsible for grading the user's answer and deciding if a follow-up is needed.
|
||||
*/
|
||||
export const graderNode = async (
|
||||
state: EvaluationState,
|
||||
config?: RunnableConfig,
|
||||
): Promise<Partial<EvaluationState>> => {
|
||||
const { model } = (config?.configurable as any) || {};
|
||||
const { questions, currentQuestionIndex, messages } = state;
|
||||
const currentFollowUpCount = state.followUpCount || 0;
|
||||
|
||||
console.log('[GraderNode] Entering node...', {
|
||||
currentIndex: currentQuestionIndex,
|
||||
numMessages: messages?.length,
|
||||
questionCount: state.questionCount,
|
||||
hasQuestions: !!questions?.length,
|
||||
});
|
||||
|
||||
if (!model) {
|
||||
throw new Error('Missing model in node configuration');
|
||||
}
|
||||
|
||||
const lastUserMessage = messages[messages.length - 1];
|
||||
|
||||
console.log('[GraderNode] Incoming Messages Count:', messages.length);
|
||||
if (lastUserMessage) {
|
||||
console.log(
|
||||
'[GraderNode] Last Message Type:',
|
||||
lastUserMessage.constructor.name,
|
||||
);
|
||||
// Safely extract content for logging
|
||||
const logContent =
|
||||
typeof lastUserMessage.content === 'string'
|
||||
? lastUserMessage.content
|
||||
: JSON.stringify(lastUserMessage.content);
|
||||
console.log(
|
||||
'[GraderNode] Last Message Content:',
|
||||
logContent.substring(0, 50),
|
||||
);
|
||||
}
|
||||
|
||||
if (!(lastUserMessage instanceof HumanMessage)) {
|
||||
console.log(
|
||||
'[GraderNode] Last message is not HumanMessage, skipping grading.',
|
||||
);
|
||||
return {};
|
||||
}
|
||||
|
||||
const isZh = state.language === 'zh';
|
||||
const isJa = state.language === 'ja';
|
||||
|
||||
const currentQuestion = questions[currentQuestionIndex];
|
||||
if (!currentQuestion) {
|
||||
console.error(
|
||||
`[GraderNode] Question at index ${currentQuestionIndex} not found!`,
|
||||
);
|
||||
return { currentQuestionIndex: currentQuestionIndex + 1 };
|
||||
}
|
||||
|
||||
const systemPromptZh = `你是一位专业的考官。
|
||||
请根据以下问题和关键点对用户的回答进行评分。
|
||||
|
||||
重要提示:
|
||||
1. **你必须使用以下语言提供反馈:中文 (Simplified Chinese)**。
|
||||
2. 即使用户的回答或知识库内容涉及其他语言,请确保你的反馈和解释依然严格使用中文。不要夹杂日文。
|
||||
|
||||
问题:${currentQuestion.questionText}
|
||||
预期的关键点:${currentQuestion.keyPoints.join(', ')}
|
||||
|
||||
评估标准:
|
||||
1. 准确性:他们是否正确覆盖了关键点?
|
||||
2. 完整性:他们是否遗漏了任何重要内容?
|
||||
3. 深度:解释是否充分?
|
||||
|
||||
请提供:
|
||||
1. 0 到 10 的评分。
|
||||
2. 建设性的反馈。
|
||||
3. 如果回答不完整或不清晰,需要进一步解释,请将 'should_follow_up' 标志设为 true。
|
||||
|
||||
请以 JSON 格式返回响应:
|
||||
{
|
||||
"score": 8,
|
||||
"feedback": "...",
|
||||
"should_follow_up": false
|
||||
}`;
|
||||
|
||||
const systemPromptJa = `あなたは専門的な試験官です。
|
||||
以下の質問とキーポイントに基づいて、ユーザーの回答を採点してください。
|
||||
|
||||
重要事項:
|
||||
1. **フィードバックは必ず次の言語で提供してください:日本語**。
|
||||
2. ユーザーの回答やナレッジベースの内容に他の言語(中国語や英語など)が含まれている場合でも、フィードバックと説明は必ず日本語のみで行ってください。中国語が混ざらないよう厳格に注意してください。
|
||||
|
||||
質問:${currentQuestion.questionText}
|
||||
期待されるキーポイント:${currentQuestion.keyPoints.join(', ')}
|
||||
|
||||
評価基準:
|
||||
1. 正確性:キーポイントを正確に網羅していますか?
|
||||
2. 網羅性:重要な内容が欠落していませんか?
|
||||
3. 深さ:説明は十分ですか?
|
||||
|
||||
以下を提供してください:
|
||||
1. 0 から 10 までのスコア。
|
||||
2. 建設的なフィードバック。
|
||||
3. 回答が不完全または不明確で、さらなる説明が必要な場合は、'should_follow_up' フラグを true に設定してください。
|
||||
|
||||
JSON 形式で回答してください:
|
||||
{
|
||||
"score": 8,
|
||||
"feedback": "...",
|
||||
"should_follow_up": false
|
||||
}`;
|
||||
|
||||
const systemPromptEn = `You are an expert examiner.
|
||||
Grade the user's answer based on the following question and key points.
|
||||
|
||||
IMPORTANT:
|
||||
1. **You MUST provide the feedback in English.**
|
||||
2. If the user's answer or knowledge base content references other languages, ensure your feedback and explanation remain strictly in English.
|
||||
|
||||
QUESTION: ${currentQuestion.questionText}
|
||||
EXPECTED KEY POINTS: ${currentQuestion.keyPoints.join(', ')}
|
||||
|
||||
Evaluate:
|
||||
1. Accuracy: Did they cover the key points correctly?
|
||||
2. Completeness: Did they miss anything important?
|
||||
3. Depth: Is the explanation sufficient?
|
||||
|
||||
Provide:
|
||||
1. A score from 0 to 10.
|
||||
2. Constructive feedback.
|
||||
3. A boolean flag 'should_follow_up' if the answer is incomplete or unclear and needs further clarification.
|
||||
|
||||
Format your response as JSON:
|
||||
{
|
||||
"score": 8,
|
||||
"feedback": "...",
|
||||
"should_follow_up": false
|
||||
}`;
|
||||
|
||||
const systemPrompt = isZh
|
||||
? systemPromptZh
|
||||
: isJa
|
||||
? systemPromptJa
|
||||
: systemPromptEn;
|
||||
|
||||
const userContentText =
|
||||
typeof lastUserMessage.content === 'string'
|
||||
? lastUserMessage.content
|
||||
: JSON.stringify(lastUserMessage.content);
|
||||
|
||||
console.log('[GraderNode] === START GRADING ===');
|
||||
console.log('[GraderNode] User answer length:', userContentText.length);
|
||||
console.log('[GraderNode] Question:', currentQuestion?.questionText?.substring(0, 100));
|
||||
console.log('[GraderNode] Target dimension:', currentQuestion?.dimension);
|
||||
|
||||
const response = await model.invoke([
|
||||
new SystemMessage(systemPrompt),
|
||||
new HumanMessage(userContentText),
|
||||
]);
|
||||
|
||||
console.log('[GraderNode] LLM invoke completed');
|
||||
try {
|
||||
const rawContent = response.content as string;
|
||||
console.log('[GraderNode] Raw AI response length:', rawContent.length);
|
||||
console.log('[GraderNode] Raw AI response:', rawContent.substring(0, 800));
|
||||
|
||||
const result = safeParseJson<any>(rawContent);
|
||||
if (!result) {
|
||||
console.error('[GraderNode] Failed to parse JSON. Raw content:', rawContent);
|
||||
throw new Error('Invalid JSON format from AI');
|
||||
}
|
||||
console.log('[GraderNode] === GRADING RESULT ===');
|
||||
console.log('[GraderNode] Parsed result:', JSON.stringify(result, null, 2));
|
||||
console.log('[GraderNode] Score value:', result.score);
|
||||
console.log('[GraderNode] Feedback value:', result.feedback?.substring(0, 200));
|
||||
|
||||
const scoreLabel = isZh ? '得分' : isJa ? 'スコア' : 'Score';
|
||||
const feedbackLabel = isZh ? '反馈' : isJa ? 'フィードバック' : 'Feedback';
|
||||
|
||||
const feedbackMessage = new AIMessage(
|
||||
`${scoreLabel}: ${result.score}/10\n\n${feedbackLabel}: ${result.feedback}`,
|
||||
);
|
||||
|
||||
const newScores = {
|
||||
...state.scores,
|
||||
[currentQuestion.id || currentQuestionIndex.toString()]: result.score,
|
||||
};
|
||||
|
||||
let shouldFollowUp = result.should_follow_up === true;
|
||||
|
||||
// Breakout logic:
|
||||
// 1. Max 1 follow-up per question
|
||||
// 2. If score is decent (>= 8), don't follow up
|
||||
// 3. If answer is short "don't know", don't follow up
|
||||
const normalizedContent = userContentText.trim().toLowerCase();
|
||||
const saysIDontKnow =
|
||||
normalizedContent.length < 10 &&
|
||||
(normalizedContent.includes('不知道') ||
|
||||
normalizedContent.includes('不会') ||
|
||||
normalizedContent.includes("don't know") ||
|
||||
normalizedContent.includes('no idea') ||
|
||||
normalizedContent.includes('不知') ||
|
||||
normalizedContent.includes('わかりません') ||
|
||||
normalizedContent.includes('わからん') ||
|
||||
normalizedContent.includes('知らない') ||
|
||||
normalizedContent.includes('不明') ||
|
||||
normalizedContent.includes('わからない'));
|
||||
|
||||
if (currentFollowUpCount >= 2 || result.score >= 8 || saysIDontKnow) {
|
||||
shouldFollowUp = false;
|
||||
}
|
||||
|
||||
console.log('[GraderNode] Final State decision:', {
|
||||
shouldFollowUp,
|
||||
nextIndex: shouldFollowUp
|
||||
? currentQuestionIndex
|
||||
: currentQuestionIndex + 1,
|
||||
score: result.score,
|
||||
saysIDontKnow,
|
||||
});
|
||||
|
||||
return {
|
||||
feedbackHistory: [feedbackMessage],
|
||||
scores: newScores,
|
||||
shouldFollowUp: shouldFollowUp,
|
||||
followUpCount: shouldFollowUp ? currentFollowUpCount + 1 : 0,
|
||||
currentQuestionIndex: shouldFollowUp
|
||||
? currentQuestionIndex
|
||||
: currentQuestionIndex + 1,
|
||||
} as any;
|
||||
} catch (error) {
|
||||
console.error('Failed to parse grade from AI response:', error);
|
||||
return {
|
||||
feedbackHistory: [
|
||||
new AIMessage("I had some trouble grading that, but let's move on."),
|
||||
],
|
||||
currentQuestionIndex: currentQuestionIndex + 1,
|
||||
shouldFollowUp: false,
|
||||
} as any;
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,99 @@
|
||||
import { AIMessage } from '@langchain/core/messages';
|
||||
import { RunnableConfig } from '@langchain/core/runnables';
|
||||
import { EvaluationState } from '../state';
|
||||
|
||||
/**
|
||||
* Node responsible for presenting the current question or follow-up to the user.
|
||||
*/
|
||||
export const interviewerNode = async (
|
||||
state: EvaluationState,
|
||||
config?: RunnableConfig,
|
||||
): Promise<Partial<EvaluationState>> => {
|
||||
const { questions, currentQuestionIndex, shouldFollowUp, messages } = state;
|
||||
|
||||
console.log('[InterviewerNode] Entering node...', {
|
||||
numQuestions: questions?.length,
|
||||
currentIndex: currentQuestionIndex,
|
||||
shouldFollowUp,
|
||||
numMessages: messages?.length,
|
||||
});
|
||||
|
||||
if (!questions || questions.length === 0) {
|
||||
const isZh = state.language === 'zh';
|
||||
const isJa = state.language === 'ja';
|
||||
const msg = isZh
|
||||
? '很抱歉,我无法为此会话生成任何问题。'
|
||||
: isJa
|
||||
? '申し訳ありませんが、このセッションの問題を生成できませんでした。'
|
||||
: "I'm sorry, I couldn't generate any questions for this session.";
|
||||
return {
|
||||
messages: [new AIMessage(msg)],
|
||||
};
|
||||
}
|
||||
|
||||
const currentQuestion = questions[currentQuestionIndex];
|
||||
|
||||
// If it's a follow-up, we add a prefix to the label later.
|
||||
// If we've run out of questions and no follow-up requested, we shouldn't be here, but let's be safe.
|
||||
if (currentQuestionIndex >= questions.length) {
|
||||
return { shouldFollowUp: false };
|
||||
}
|
||||
|
||||
const isZh = state.language === 'zh';
|
||||
const isJa = state.language === 'ja';
|
||||
|
||||
let prompt = '';
|
||||
|
||||
if (
|
||||
shouldFollowUp &&
|
||||
state.feedbackHistory &&
|
||||
state.feedbackHistory.length > 0
|
||||
) {
|
||||
// Construct a follow-up prompt based on last feedback
|
||||
const lastFeedbackMsg =
|
||||
state.feedbackHistory[state.feedbackHistory.length - 1];
|
||||
const feedbackText = lastFeedbackMsg.content.toString();
|
||||
|
||||
// Extract the "Feedback: ..." part if possible, otherwise use whole text
|
||||
const feedbackMatch = feedbackText.match(
|
||||
/(?:Feedback|反馈|フィードバック): ([\s\S]*)/i,
|
||||
);
|
||||
const specificFeedback = feedbackMatch
|
||||
? feedbackMatch[1].trim()
|
||||
: feedbackText;
|
||||
|
||||
const followUpLabel = isZh
|
||||
? '补充追问'
|
||||
: isJa
|
||||
? '追加の質問'
|
||||
: 'Follow-up Clarification';
|
||||
const followUpInstruction = isZh
|
||||
? '根据以上反馈,请补充更具体的信息:'
|
||||
: isJa
|
||||
? '上記のフィードバックに基づき、より具体的な情報を追加してください:'
|
||||
: 'Based on the feedback above, please provide more specific details:';
|
||||
|
||||
prompt = `${followUpLabel}\n\n${specificFeedback}\n\n${followUpInstruction}`;
|
||||
} else {
|
||||
// Standard question presentation
|
||||
const label = isZh
|
||||
? `问题 ${currentQuestionIndex + 1}`
|
||||
: isJa
|
||||
? `質問 ${currentQuestionIndex + 1}`
|
||||
: `Question ${currentQuestionIndex + 1}`;
|
||||
|
||||
const instruction = isZh
|
||||
? '请提供您的回答。'
|
||||
: isJa
|
||||
? '回答を入力してください。'
|
||||
: 'Please provide your answer.';
|
||||
|
||||
prompt = `${label}: ${currentQuestion.questionText}\n\n${instruction}`;
|
||||
}
|
||||
|
||||
console.log('[InterviewerNode] Returning question:', { currentQuestionIndex, questionText: currentQuestion?.questionText });
|
||||
return {
|
||||
messages: [new AIMessage(prompt)],
|
||||
shouldFollowUp: false,
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,124 @@
|
||||
import { Annotation, MessagesAnnotation } from '@langchain/langgraph';
|
||||
import { BaseMessage } from '@langchain/core/messages';
|
||||
|
||||
/**
|
||||
* State representing the evaluation session using LangGraph Annotation.
|
||||
*/
|
||||
export const EvaluationAnnotation = Annotation.Root({
|
||||
/**
|
||||
* The message history of the conversation.
|
||||
* Inherits from MessagesAnnotation to handle message merging.
|
||||
*/
|
||||
...MessagesAnnotation.spec,
|
||||
|
||||
/**
|
||||
* Historical evaluation feedback from the grader.
|
||||
* Separated from main messages to keep conversation context clean.
|
||||
*/
|
||||
feedbackHistory: Annotation<BaseMessage[]>({
|
||||
reducer: (prev, next) => [...(prev || []), ...(next || [])],
|
||||
default: () => [],
|
||||
}),
|
||||
|
||||
/**
|
||||
* The database ID of the current assessment session.
|
||||
*/
|
||||
assessmentSessionId: Annotation<string>(),
|
||||
|
||||
/**
|
||||
* The knowledge base ID used as ground truth for this evaluation.
|
||||
*/
|
||||
knowledgeBaseId: Annotation<string>(),
|
||||
|
||||
/**
|
||||
* List of questions generated for this session.
|
||||
*/
|
||||
questions: Annotation<any[]>({
|
||||
reducer: (prev, next) => next ?? prev,
|
||||
default: () => [],
|
||||
}),
|
||||
|
||||
/**
|
||||
* Index of the current question being discussed.
|
||||
*/
|
||||
currentQuestionIndex: Annotation<number>({
|
||||
reducer: (prev, next) => next ?? prev,
|
||||
default: () => 0,
|
||||
}),
|
||||
|
||||
/**
|
||||
* Flag indicating if the Grader believes a follow-up question is needed for clarity.
|
||||
*/
|
||||
shouldFollowUp: Annotation<boolean>({
|
||||
reducer: (prev, next) => next ?? prev,
|
||||
default: () => false,
|
||||
}),
|
||||
|
||||
/**
|
||||
* Map of scores for each question.
|
||||
*/
|
||||
scores: Annotation<Record<string, number>>({
|
||||
reducer: (prev, next) => ({ ...prev, ...next }),
|
||||
default: () => ({}),
|
||||
}),
|
||||
|
||||
/**
|
||||
* Final report generated by the ReportAnalyzer.
|
||||
*/
|
||||
report: Annotation<string | undefined>({
|
||||
reducer: (prev, next) => next ?? prev,
|
||||
}),
|
||||
|
||||
/**
|
||||
* Context chunks retrieved from the knowledge base for grounding.
|
||||
*/
|
||||
context: Annotation<string[] | undefined>({
|
||||
reducer: (prev, next) => next ?? prev,
|
||||
}),
|
||||
|
||||
/**
|
||||
* Preferred language for the assessment (zh, en, ja).
|
||||
*/
|
||||
language: Annotation<string>({
|
||||
reducer: (prev, next) => next ?? prev,
|
||||
default: () => 'zh',
|
||||
}),
|
||||
|
||||
/**
|
||||
* Number of times we have followed up on the current question.
|
||||
*/
|
||||
followUpCount: Annotation<number>({
|
||||
reducer: (prev, next) => next ?? prev,
|
||||
default: () => 0,
|
||||
}),
|
||||
|
||||
/**
|
||||
* Number of questions to generate.
|
||||
*/
|
||||
questionCount: Annotation<number | undefined>({
|
||||
reducer: (prev, next) => next ?? prev,
|
||||
}),
|
||||
|
||||
/**
|
||||
* Desired difficulty distribution.
|
||||
*/
|
||||
difficultyDistribution: Annotation<any | undefined>({
|
||||
reducer: (prev, next) => next ?? prev,
|
||||
}),
|
||||
|
||||
/**
|
||||
* Desired question style.
|
||||
*/
|
||||
style: Annotation<string | undefined>({
|
||||
reducer: (prev, next) => next ?? prev,
|
||||
}),
|
||||
|
||||
/**
|
||||
* Target keywords for question generation.
|
||||
*/
|
||||
keywords: Annotation<string[] | undefined>({
|
||||
reducer: (prev, next) => next ?? prev,
|
||||
}),
|
||||
});
|
||||
|
||||
export type EvaluationState = typeof EvaluationAnnotation.State;
|
||||
Reference in New Issue
Block a user