forked from hangshuo652/aurak
feat: end-to-end choice question support in assessment pipeline
- Data pathway: flow options through questions, answerKey in graph state - Interviewer: format MULTIPLE_CHOICE with A/B/C/D options - Grader: instant choice scoring (zero LLM), compare correctAnswer - AssessmentView: render choice buttons vs textarea based on questionType - Security: sanitizeStateForClient strips correctAnswer/judgment/answerKey - Bank detection: check PUBLISHED items (not PUBLISHED bank status) - Batch UI: select all / batch approve / batch reject on detail view
This commit is contained in:
@@ -27,7 +27,7 @@ import { AssessmentAnswer } from './entities/assessment-answer.entity';
|
||||
import { AssessmentTemplate } from './entities/assessment-template.entity';
|
||||
import { AssessmentCertificate } from './entities/assessment-certificate.entity';
|
||||
import { QuestionBank, QuestionBankStatus } from './entities/question-bank.entity';
|
||||
import { QuestionBankItem } from './entities/question-bank-item.entity';
|
||||
import { QuestionBankItem, QuestionBankItemStatus } from './entities/question-bank-item.entity';
|
||||
import { KnowledgeBaseService } from '../knowledge-base/knowledge-base.service';
|
||||
import { KnowledgeGroupService } from '../knowledge-group/knowledge-group.service';
|
||||
import { ModelConfigService } from '../model-config/model-config.service';
|
||||
@@ -453,7 +453,7 @@ private async getModel(tenantId: string): Promise<ChatOpenAI> {
|
||||
}
|
||||
this.logger.debug(`[startSession] isKb: ${isKb}`);
|
||||
|
||||
const templateData = template
|
||||
const templateData: any = template
|
||||
? {
|
||||
name: template.name,
|
||||
keywords: template.keywords,
|
||||
@@ -475,22 +475,22 @@ private async getModel(tenantId: string): Promise<ChatOpenAI> {
|
||||
if (templateId) {
|
||||
try {
|
||||
const targetCount = template?.questionCount || 5;
|
||||
const publishedBanks = await this.questionBankRepository.find({
|
||||
where: { templateId, status: QuestionBankStatus.PUBLISHED },
|
||||
const linkedBanks = await this.questionBankRepository.find({
|
||||
where: { templateId },
|
||||
});
|
||||
|
||||
if (publishedBanks.length > 0) {
|
||||
const bankIds = publishedBanks.map(b => b.id);
|
||||
if (linkedBanks.length > 0) {
|
||||
const bankIds = linkedBanks.map(b => b.id);
|
||||
const questionCount = await this.questionBankItemRepository.count({
|
||||
where: { bankId: In(bankIds) },
|
||||
where: { bankId: In(bankIds), status: QuestionBankItemStatus.PUBLISHED },
|
||||
});
|
||||
|
||||
this.logger.log(
|
||||
`[startSession] Found ${publishedBanks.length} published banks with ${questionCount} questions, target: ${targetCount}`,
|
||||
`[startSession] Found ${linkedBanks.length} banks with ${questionCount} published questions, target: ${targetCount}`,
|
||||
);
|
||||
|
||||
if (questionCount >= targetCount) {
|
||||
const bankId = publishedBanks[0].id;
|
||||
const bankId = linkedBanks[0].id;
|
||||
const selectedItems = await this.questionBankService.selectQuestions(
|
||||
bankId,
|
||||
targetCount,
|
||||
@@ -500,12 +500,27 @@ private async getModel(tenantId: string): Promise<ChatOpenAI> {
|
||||
id: item.id,
|
||||
questionText: item.questionText,
|
||||
questionType: item.questionType,
|
||||
options: item.options,
|
||||
keyPoints: item.keyPoints,
|
||||
difficulty: item.difficulty,
|
||||
dimension: item.dimension,
|
||||
basis: item.basis,
|
||||
}));
|
||||
|
||||
const answerKey: Record<string, { correctAnswer?: string | null; judgment?: string | null; followupHints?: string[] | null }> = {};
|
||||
selectedItems.forEach(item => {
|
||||
if (item.correctAnswer || item.judgment || item.followupHints) {
|
||||
answerKey[item.id] = {
|
||||
correctAnswer: item.correctAnswer,
|
||||
judgment: item.judgment,
|
||||
followupHints: item.followupHints,
|
||||
};
|
||||
}
|
||||
});
|
||||
if (Object.keys(answerKey).length > 0 && templateData) {
|
||||
templateData.questionAnswerKey = answerKey;
|
||||
}
|
||||
|
||||
questionSource = 'bank';
|
||||
this.logger.log(
|
||||
`[startSession] Selected ${questionsFromBank.length} questions from question bank`,
|
||||
@@ -609,7 +624,7 @@ private async getModel(tenantId: string): Promise<ChatOpenAI> {
|
||||
this.logger.log(
|
||||
`Session ${sessionId} already has state, skipping generation.`,
|
||||
);
|
||||
const mappedData = { ...existingState.values };
|
||||
const mappedData = this.sanitizeStateForClient({ ...existingState.values });
|
||||
mappedData.messages = this.mapMessages(mappedData.messages || []);
|
||||
mappedData.feedbackHistory = this.mapMessages(
|
||||
mappedData.feedbackHistory || [],
|
||||
@@ -631,6 +646,7 @@ const initialState: Partial<EvaluationState> = {
|
||||
|
||||
style: session.templateJson?.style,
|
||||
keywords: session.templateJson?.keywords,
|
||||
questionAnswerKey: session.templateJson?.questionAnswerKey,
|
||||
currentQuestionIndex: 0,
|
||||
};
|
||||
|
||||
@@ -752,7 +768,7 @@ const initialState: Partial<EvaluationState> = {
|
||||
}
|
||||
await this.sessionRepository.save(session);
|
||||
|
||||
const mappedData: any = { ...finalData };
|
||||
const mappedData: any = this.sanitizeStateForClient({ ...finalData });
|
||||
mappedData.messages = this.mapMessages(finalData.messages);
|
||||
mappedData.feedbackHistory = this.mapMessages(
|
||||
finalData.feedbackHistory || [],
|
||||
@@ -1123,7 +1139,7 @@ const initialState: Partial<EvaluationState> = {
|
||||
}
|
||||
await this.sessionRepository.save(session);
|
||||
|
||||
const mappedData: any = { ...finalData };
|
||||
const mappedData: any = this.sanitizeStateForClient({ ...finalData });
|
||||
mappedData.messages = this.mapMessages(finalData.messages);
|
||||
mappedData.feedbackHistory = this.mapMessages(
|
||||
finalData.feedbackHistory || [],
|
||||
@@ -1169,7 +1185,7 @@ const initialState: Partial<EvaluationState> = {
|
||||
values.feedbackHistory = this.mapMessages(values.feedbackHistory);
|
||||
}
|
||||
|
||||
return values;
|
||||
return this.sanitizeStateForClient(values);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1280,6 +1296,7 @@ const initialState: Partial<EvaluationState> = {
|
||||
session.templateJson?.difficultyDistribution,
|
||||
style: session.templateJson?.style,
|
||||
keywords: session.templateJson?.keywords,
|
||||
questionAnswerKey: session.templateJson?.questionAnswerKey,
|
||||
language: session.language || 'zh',
|
||||
report: session.finalReport || undefined,
|
||||
};
|
||||
@@ -1309,6 +1326,7 @@ const initialState: Partial<EvaluationState> = {
|
||||
difficultyDistribution: session.templateJson?.difficultyDistribution,
|
||||
style: session.templateJson?.style,
|
||||
keywords: session.templateJson?.keywords,
|
||||
questionAnswerKey: session.templateJson?.questionAnswerKey,
|
||||
language: session.language || 'en',
|
||||
};
|
||||
|
||||
@@ -1373,6 +1391,22 @@ const initialState: Partial<EvaluationState> = {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Strips sensitive fields before sending state to frontend.
|
||||
*/
|
||||
private sanitizeStateForClient(data: any): any {
|
||||
if (!data) return data;
|
||||
const sanitized = { ...data };
|
||||
delete sanitized.questionAnswerKey;
|
||||
if (Array.isArray(sanitized.questions)) {
|
||||
sanitized.questions = sanitized.questions.map((q: any) => {
|
||||
const { correctAnswer, judgment, followupHints, ...rest } = q;
|
||||
return rest;
|
||||
});
|
||||
}
|
||||
return sanitized;
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps LangChain messages to a simple format for the frontend and storage.
|
||||
*/
|
||||
|
||||
@@ -230,6 +230,7 @@ ${existingQuestionsText ? `- Repeating previous questions: ${existingQuestionsTe
|
||||
return {
|
||||
id: (existingQuestions.length + 1).toString(),
|
||||
questionText: q.question_text,
|
||||
questionType: 'SHORT_ANSWER',
|
||||
keyPoints: q.key_points,
|
||||
difficulty: q.difficulty,
|
||||
basis: q.basis,
|
||||
|
||||
@@ -67,6 +67,31 @@ export const graderNode = async (
|
||||
return { currentQuestionIndex: currentQuestionIndex + 1 };
|
||||
}
|
||||
|
||||
const isChoice = currentQuestion.questionType === 'MULTIPLE_CHOICE';
|
||||
const answerKey = (state.questionAnswerKey as any)?.[currentQuestion.id];
|
||||
|
||||
if (isChoice || answerKey?.correctAnswer) {
|
||||
const expectedAnswer = answerKey?.correctAnswer || currentQuestion.correctAnswer;
|
||||
const userAnswer = (lastUserMessage.content as string).trim();
|
||||
const isCorrect = userAnswer.toUpperCase() === expectedAnswer?.toUpperCase();
|
||||
|
||||
console.log('[GraderNode] Choice grading:', { userAnswer, expectedAnswer, isCorrect });
|
||||
|
||||
const feedback = isCorrect ? '✅ 正确' : `❌ 错误,正确答案是 ${expectedAnswer}`;
|
||||
const feedbackMessage = new AIMessage(
|
||||
{ content: `Score: ${isCorrect ? 10 : 0}\nFeedback: ${feedback}` } as any,
|
||||
);
|
||||
|
||||
return {
|
||||
messages: [feedbackMessage],
|
||||
feedbackHistory: [feedbackMessage],
|
||||
scores: { [currentQuestion.id || currentQuestionIndex.toString()]: isCorrect ? 10 : 0 },
|
||||
shouldFollowUp: false,
|
||||
followUpCount: 0,
|
||||
currentQuestionIndex: currentQuestionIndex + 1,
|
||||
};
|
||||
}
|
||||
|
||||
const systemPromptZh = `你是一位专业的考官。
|
||||
请根据以下问题和关键点对用户的回答进行评分。
|
||||
|
||||
|
||||
@@ -33,8 +33,6 @@ export const interviewerNode = async (
|
||||
|
||||
const currentQuestion = questions[currentQuestionIndex];
|
||||
|
||||
// If it's a follow-up, we add a prefix to the label later.
|
||||
// If we've run out of questions and no follow-up requested, we shouldn't be here, but let's be safe.
|
||||
if (currentQuestionIndex >= questions.length) {
|
||||
return { shouldFollowUp: false };
|
||||
}
|
||||
@@ -49,12 +47,10 @@ export const interviewerNode = async (
|
||||
state.feedbackHistory &&
|
||||
state.feedbackHistory.length > 0
|
||||
) {
|
||||
// Construct a follow-up prompt based on last feedback
|
||||
const lastFeedbackMsg =
|
||||
state.feedbackHistory[state.feedbackHistory.length - 1];
|
||||
const feedbackText = lastFeedbackMsg.content.toString();
|
||||
|
||||
// Extract the "Feedback: ..." part if possible, otherwise use whole text
|
||||
const feedbackMatch = feedbackText.match(
|
||||
/(?:Feedback|反馈|フィードバック): ([\s\S]*)/i,
|
||||
);
|
||||
@@ -74,8 +70,22 @@ export const interviewerNode = async (
|
||||
: 'Based on the feedback above, please provide more specific details:';
|
||||
|
||||
prompt = `${followUpLabel}\n\n${specificFeedback}\n\n${followUpInstruction}`;
|
||||
} else if (currentQuestion.questionType === 'MULTIPLE_CHOICE' && currentQuestion.options?.length > 0) {
|
||||
const label = isZh
|
||||
? `问题 ${currentQuestionIndex + 1}`
|
||||
: isJa
|
||||
? `質問 ${currentQuestionIndex + 1}`
|
||||
: `Question ${currentQuestionIndex + 1}`;
|
||||
|
||||
const optionsText = currentQuestion.options.join('\n');
|
||||
const instruction = isZh
|
||||
? '请选择一个选项(输入字母 A/B/C/D)'
|
||||
: isJa
|
||||
? '選択肢から1つ選んでください(A/B/C/Dを入力)'
|
||||
: 'Please select one option (enter A, B, C, or D)';
|
||||
|
||||
prompt = `${label}: ${currentQuestion.questionText}\n\n${optionsText}\n\n${instruction}`;
|
||||
} else {
|
||||
// Standard question presentation
|
||||
const label = isZh
|
||||
? `问题 ${currentQuestionIndex + 1}`
|
||||
: isJa
|
||||
|
||||
@@ -119,6 +119,15 @@ export const EvaluationAnnotation = Annotation.Root({
|
||||
keywords: Annotation<string[] | undefined>({
|
||||
reducer: (prev, next) => next ?? prev,
|
||||
}),
|
||||
|
||||
/**
|
||||
* Answer key for bank questions: id → { correctAnswer, judgment, followupHints }.
|
||||
* Used by grader for instant choice scoring and open-question anchoring.
|
||||
* NOT sent to frontend.
|
||||
*/
|
||||
questionAnswerKey: Annotation<Record<string, any> | undefined>({
|
||||
reducer: (prev, next) => next ?? prev,
|
||||
}),
|
||||
});
|
||||
|
||||
export type EvaluationState = typeof EvaluationAnnotation.State;
|
||||
|
||||
Reference in New Issue
Block a user