P2-1: remove dead cost-control module (3 files)

P2-2: switch TypeORM to autoLoadEntities: true

Remove unused vision-pipeline-cost-aware.service.ts,
cost-control.service.ts and its orphan module.
Switch explicit entities[] list to autoLoadEntities.
This commit is contained in:
Developer
2026-05-19 09:39:41 +08:00
parent 33e48f6d4e
commit eb0798de5b
4 changed files with 3 additions and 664 deletions
+3 -51
View File
@@ -31,34 +31,11 @@ import { ImportTaskModule } from './import-task/import-task.module';
import { AssessmentModule } from './assessment/assessment.module';
import { I18nMiddleware } from './i18n/i18n.middleware';
import { TenantMiddleware } from './tenant/tenant.middleware';
import { User } from './user/user.entity';
import { UserSetting } from './user/user-setting.entity';
import { ModelConfig } from './model-config/model-config.entity';
import { KnowledgeBase } from './knowledge-base/knowledge-base.entity';
import { KnowledgeGroup } from './knowledge-group/knowledge-group.entity';
import { SearchHistory } from './search-history/search-history.entity';
import { ChatMessage } from './search-history/chat-message.entity';
import { Note } from './note/note.entity';
import { NoteCategory } from './note/note-category.entity';
import { PodcastEpisode } from './podcasts/entities/podcast-episode.entity';
import { ImportTask } from './import-task/import-task.entity';
import { AssessmentSession } from './assessment/entities/assessment-session.entity';
import { AssessmentQuestion } from './assessment/entities/assessment-question.entity';
import { AssessmentAnswer } from './assessment/entities/assessment-answer.entity';
import { AssessmentTemplate } from './assessment/entities/assessment-template.entity';
import { QuestionBank } from './assessment/entities/question-bank.entity';
import { QuestionBankItem } from './assessment/entities/question-bank-item.entity';
import { Tenant } from './tenant/tenant.entity';
import { TenantSetting } from './tenant/tenant-setting.entity';
import { ApiKey } from './auth/entities/api-key.entity';
import { TenantMember } from './tenant/tenant-member.entity';
import { TenantModule } from './tenant/tenant.module';
import { SuperAdminModule } from './super-admin/super-admin.module';
import { AdminModule } from './admin/admin.module';
import { FeishuModule } from './feishu/feishu.module';
import { FeishuBot } from './feishu/entities/feishu-bot.entity';
import { FeishuAssessmentSession } from './feishu/entities/feishu-assessment-session.entity';
import { AssessmentCertificate } from './assessment/entities/assessment-certificate.entity';
@Module({
imports: [
@@ -77,33 +54,8 @@ import { AssessmentCertificate } from './assessment/entities/assessment-certific
useFactory: (configService: ConfigService) => ({
type: 'better-sqlite3',
database: configService.get<string>('DATABASE_PATH'),
entities: [
User,
UserSetting,
ModelConfig,
KnowledgeBase,
KnowledgeGroup,
SearchHistory,
ChatMessage,
Note,
NoteCategory,
PodcastEpisode,
ImportTask,
AssessmentSession,
AssessmentQuestion,
AssessmentAnswer,
AssessmentTemplate,
QuestionBank,
QuestionBankItem,
Tenant,
TenantSetting,
TenantMember,
ApiKey,
FeishuBot,
FeishuAssessmentSession,
AssessmentCertificate,
],
synchronize: true, // Auto-create database schema. Disable in production.
autoLoadEntities: true,
synchronize: true,
}),
}),
AuthModule,
@@ -1,11 +0,0 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { CostControlService } from './cost-control.service';
import { User } from '../user/user.entity';
@Module({
imports: [TypeOrmModule.forFeature([User])],
providers: [CostControlService],
exports: [CostControlService],
})
export class CostControlModule {}
@@ -1,261 +0,0 @@
/**
* Cost control and quota management service
* Used to manage API call costs for Vision Pipeline
*/
import { Injectable, Logger } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { User } from '../user/user.entity';
export interface UserQuota {
userId: string;
monthlyCost: number; // Current month used cost
maxCost: number; // Monthly max cost
remaining: number; // Remaining cost
lastReset: Date; // Last reset time
}
export interface CostEstimate {
estimatedCost: number; // Estimated cost
estimatedTime: number; // Estimated time(seconds)
pageBreakdown: {
// Per-page breakdown
pageIndex: number;
cost: number;
}[];
}
@Injectable()
export class CostControlService {
private readonly logger = new Logger(CostControlService.name);
private readonly COST_PER_PAGE = 0.01; // Cost per page(USD)
private readonly DEFAULT_MONTHLY_LIMIT = 100; // Default monthly limit(USD)
constructor(
private configService: ConfigService,
@InjectRepository(User)
private userRepository: Repository<User>,
) {}
/**
* Estimate processing cost
*/
estimateCost(
pageCount: number,
quality: 'low' | 'medium' | 'high' = 'medium',
): CostEstimate {
// Adjust cost coefficient based on quality
const qualityMultiplier = {
low: 0.5,
medium: 1.0,
high: 1.5,
};
const baseCost =
pageCount * this.COST_PER_PAGE * qualityMultiplier[quality];
const estimatedTime = pageCount * 3; // // Approximately 3 seconds
const pageBreakdown = Array.from({ length: pageCount }, (_, i) => ({
pageIndex: i + 1,
cost: this.COST_PER_PAGE * qualityMultiplier[quality],
}));
return {
estimatedCost: baseCost,
estimatedTime,
pageBreakdown,
};
}
/**
* Check user quota
*/
async checkQuota(
userId: string,
estimatedCost: number,
): Promise<{
allowed: boolean;
quota: UserQuota;
reason?: string;
}> {
const quota = await this.getUserQuota(userId);
// Check monthly reset
this.checkAndResetMonthlyQuota(quota);
if (quota.remaining < estimatedCost) {
this.logger.warn(
`User ${userId} quota insufficient: remaining $${quota.remaining.toFixed(2)}, required $${estimatedCost.toFixed(2)}`,
);
return {
allowed: false,
quota,
reason: `Insufficient quota: remaining $${quota.remaining.toFixed(2)}, required $${estimatedCost.toFixed(2)}`,
};
}
return {
allowed: true,
quota,
};
}
/**
* Deduct from quota
*/
async deductQuota(userId: string, actualCost: number): Promise<void> {
const quota = await this.getUserQuota(userId);
quota.monthlyCost += actualCost;
quota.remaining = quota.maxCost - quota.monthlyCost;
await this.userRepository.update(userId, {
monthlyCost: quota.monthlyCost,
});
this.logger.log(
`Deducted $${actualCost.toFixed(2)} from user ${userId} quota. Remaining: $${quota.remaining.toFixed(2)}`,
);
}
/**
* Get user quota
*/
async getUserQuota(userId: string): Promise<UserQuota> {
const user = await this.userRepository.findOne({ where: { id: userId } });
if (!user) {
throw new Error(`User ${userId} does not exist`);
}
// Use default if user has no quota info
const monthlyCost = user.monthlyCost || 0;
const maxCost = user.maxCost || this.DEFAULT_MONTHLY_LIMIT;
const lastReset = user.lastQuotaReset || new Date();
return {
userId,
monthlyCost,
maxCost,
remaining: maxCost - monthlyCost,
lastReset,
};
}
/**
* Check and reset monthly quota
*/
private checkAndResetMonthlyQuota(quota: UserQuota): void {
const now = new Date();
const lastReset = quota.lastReset;
// Check if crossed month
if (
now.getMonth() !== lastReset.getMonth() ||
now.getFullYear() !== lastReset.getFullYear()
) {
this.logger.log(`Reset monthly quota for user ${quota.userId}`);
// Reset quota
quota.monthlyCost = 0;
quota.remaining = quota.maxCost;
quota.lastReset = now;
// Update database
this.userRepository.update(quota.userId, {
monthlyCost: 0,
lastQuotaReset: now,
});
}
}
/**
* Set user quota limit
*/
async setQuotaLimit(userId: string, maxCost: number): Promise<void> {
await this.userRepository.update(userId, { maxCost });
this.logger.log(`Set quota limit to $${maxCost} for user ${userId}`);
}
/**
* Get cost report
*/
async getCostReport(
userId: string,
days: number = 30,
): Promise<{
totalCost: number;
dailyAverage: number;
pageStats: {
totalPages: number;
avgCostPerPage: number;
};
quotaUsage: number; // Percentage
}> {
const quota = await this.getUserQuota(userId);
const usagePercent = (quota.monthlyCost / quota.maxCost) * 100;
// Query history records here(if implemented)
// Return current quota info temporarily
return {
totalCost: quota.monthlyCost,
dailyAverage: quota.monthlyCost / Math.max(days, 1),
pageStats: {
totalPages: Math.floor(quota.monthlyCost / this.COST_PER_PAGE),
avgCostPerPage: this.COST_PER_PAGE,
},
quotaUsage: usagePercent,
};
}
/**
* Check cost warning threshold
*/
async checkWarningThreshold(userId: string): Promise<{
shouldWarn: boolean;
message: string;
}> {
const quota = await this.getUserQuota(userId);
const usagePercent = (quota.monthlyCost / quota.maxCost) * 100;
if (usagePercent >= 90) {
return {
shouldWarn: true,
message: `⚠️ Quota usage reached ${usagePercent.toFixed(1)}%. Remaining: $${quota.remaining.toFixed(2)}`,
};
}
if (usagePercent >= 75) {
return {
shouldWarn: true,
message: `💡 Quota usage at ${usagePercent.toFixed(1)}%. Please monitor your costs carefully`,
};
}
return {
shouldWarn: false,
message: '',
};
}
/**
* Format cost display
*/
formatCost(cost: number): string {
return `$${cost.toFixed(2)}`;
}
/**
* Format time display
*/
formatTime(seconds: number): string {
if (seconds < 60) {
return `${seconds.toFixed(0)}s`;
}
const minutes = Math.floor(seconds / 60);
const remainingSeconds = seconds % 60;
return `${minutes}m ${remainingSeconds.toFixed(0)}s`;
}
}
@@ -1,341 +0,0 @@
/**
* Vision Pipeline Service (with cost control)
* This is an extended version of vision-pipeline.service.ts with integrated cost control
*/
import { Injectable, Logger } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import * as fs from 'fs/promises';
import * as path from 'path';
import { LibreOfficeService } from '../libreoffice/libreoffice.service';
import { Pdf2ImageService } from '../pdf2image/pdf2image.service';
import { VisionService } from '../vision/vision.service';
import { ElasticsearchService } from '../elasticsearch/elasticsearch.service';
import { ModelConfigService } from '../model-config/model-config.service';
import {
PreciseModeOptions,
PipelineResult,
ProcessingStatus,
ModeRecommendation,
} from './vision-pipeline.interface';
import {
VisionModelConfig,
VisionAnalysisResult,
} from '../vision/vision.interface';
import { CostControlService } from './cost-control.service';
import { I18nService } from '../i18n/i18n.service';
@Injectable()
export class VisionPipelineCostAwareService {
private readonly logger = new Logger(VisionPipelineCostAwareService.name);
constructor(
private libreOffice: LibreOfficeService,
private pdf2Image: Pdf2ImageService,
private vision: VisionService,
private elasticsearch: ElasticsearchService,
private modelConfigService: ModelConfigService,
private configService: ConfigService,
private costControl: CostControlService,
private i18nService: I18nService,
) {}
/**
* Main processing flow: Precise mode (with cost control)
*/
async processPreciseMode(
filePath: string,
options: PreciseModeOptions,
): Promise<PipelineResult> {
const startTime = Date.now();
const results: VisionAnalysisResult[] = [];
let processedPages = 0;
let failedPages = 0;
let totalCost = 0;
let pdfPath = filePath;
let imagesToProcess: any[] = [];
this.logger.log(
`Starting precise mode processing for ${options.fileName} (user: ${options.userId})`,
);
try {
// Step 1: Convert format
this.updateStatus('converting', 10, 'Converting document format...');
pdfPath = await this.convertToPDF(filePath);
// Step 2: Convert PDF to images
this.updateStatus('splitting', 30, 'Converting PDF to images...');
const conversionResult = await this.pdf2Image.convertToImages(pdfPath, {
density: 300,
quality: 85,
format: 'jpeg',
});
if (conversionResult.images.length === 0) {
throw new Error(
this.i18nService.getMessage('pdfToImageConversionFailed'),
);
}
// Limit processing pages
imagesToProcess = options.maxPages
? conversionResult.images.slice(0, options.maxPages)
: conversionResult.images;
const pageCount = imagesToProcess.length;
// Step 3: Cost estimation and quota check
this.updateStatus(
'checking',
40,
'Checking quota and estimating cost...',
);
const costEstimate = this.costControl.estimateCost(pageCount);
this.logger.log(
`Estimated cost: $${costEstimate.estimatedCost.toFixed(2)}, Estimated time: ${this.costControl.formatTime(costEstimate.estimatedTime)}`,
);
// Quota check
const quotaCheck = await this.costControl.checkQuota(
options.userId,
costEstimate.estimatedCost,
);
if (!quotaCheck.allowed) {
throw new Error(quotaCheck.reason);
}
// Cost warning check
const warning = await this.costControl.checkWarningThreshold(
options.userId,
);
if (warning.shouldWarn) {
this.logger.warn(warning.message);
}
// Step 4: Get Vision model config
const modelConfig = await this.getVisionModelConfig(
options.userId,
options.modelId,
options.tenantId,
);
// Step 5: VL model analysis
this.updateStatus(
'analyzing',
50,
'Analyzing pages with Vision model...',
);
const batchResult = await this.vision.batchAnalyze(
imagesToProcess.map((img) => img.path),
modelConfig,
{
startIndex: 1,
skipQualityCheck: options.skipQualityCheck,
},
);
totalCost = batchResult.estimatedCost;
processedPages = batchResult.successCount;
failedPages = batchResult.failedCount;
results.push(...batchResult.results);
// Step 6: Subtract actual cost
if (totalCost > 0) {
await this.costControl.deductQuota(options.userId, totalCost);
this.logger.log(`Actual cost deducted: $${totalCost.toFixed(2)}`);
}
// Step 7: Cleanup temp files
this.updateStatus(
'completed',
100,
'Processing completed. Cleaning up temp files...',
);
await this.pdf2Image.cleanupImages(imagesToProcess);
// Cleanup converted PDF file if converted
if (pdfPath !== filePath) {
try {
await fs.unlink(pdfPath);
} catch (error) {
this.logger.warn(`Failed to cleanup converted PDF: ${error.message}`);
}
}
const duration = (Date.now() - startTime) / 1000;
this.logger.log(
`Precise mode completed: ${processedPages} pages processed, ` +
`cost: $${totalCost.toFixed(2)}, duration: ${duration.toFixed(1)}s`,
);
return {
success: true,
fileId: options.fileId,
fileName: options.fileName,
totalPages: conversionResult.totalPages,
processedPages,
failedPages,
results,
cost: totalCost,
duration,
mode: 'precise',
};
} catch (error) {
this.logger.error(`Precise mode failed: ${error.message}`);
// Try to clean up temp files
try {
if (pdfPath !== filePath && pdfPath !== filePath) {
await fs.unlink(pdfPath);
}
if (imagesToProcess.length > 0) {
await this.pdf2Image.cleanupImages(imagesToProcess);
}
} catch {}
return {
success: false,
fileId: options.fileId,
fileName: options.fileName,
totalPages: 0,
processedPages,
failedPages,
results: [],
cost: totalCost,
duration: (Date.now() - startTime) / 1000,
mode: 'precise',
};
}
}
/**
* Get Vision model configuration
*/
private async getVisionModelConfig(
userId: string,
modelId: string,
tenantId?: string,
): Promise<VisionModelConfig> {
const config = await this.modelConfigService.findOne(modelId);
if (!config) {
throw new Error(`Model config not found: ${modelId}`);
}
// API key is optional - allows local models
return {
baseUrl: config.baseUrl || '',
apiKey: config.apiKey || '',
modelId: config.modelId,
};
}
/**
* Convert to PDF
*/
private async convertToPDF(filePath: string): Promise<string> {
const ext = path.extname(filePath).toLowerCase();
// Return as-is if already PDF
if (ext === '.pdf') {
return filePath;
}
// Call LibreOffice to convert
return await this.libreOffice.convertToPDF(filePath);
}
/**
* Format detection and mode recommendation (with cost estimation)
*/
async recommendMode(filePath: string): Promise<ModeRecommendation> {
const ext = path.extname(filePath).toLowerCase();
const stats = await fs.stat(filePath);
const sizeMB = stats.size / (1024 * 1024);
const supportedFormats = [
'.pdf',
'.doc',
'.docx',
'.ppt',
'.pptx',
'.xls',
'.xlsx',
];
const preciseFormats = ['.pdf', '.doc', '.docx', '.ppt', '.pptx'];
if (!supportedFormats.includes(ext)) {
return {
recommendedMode: 'fast',
reason: `Unsupported file format: ${ext}`,
warnings: ['Using fast mode (text extraction only)'],
};
}
if (!preciseFormats.includes(ext)) {
return {
recommendedMode: 'fast',
reason: `Format ${ext} does not support precise mode`,
warnings: ['Using fast mode (text extraction only)'],
};
}
// Estimate page countbased on file size
const estimatedPages = Math.max(1, Math.ceil(sizeMB * 2));
const costEstimate = this.costControl.estimateCost(estimatedPages);
// Recommend precise mode for large files
if (sizeMB > 50) {
return {
recommendedMode: 'precise',
reason:
'File is large, recommend precise mode to preserve full content',
estimatedCost: costEstimate.estimatedCost,
estimatedTime: costEstimate.estimatedTime,
warnings: [
'Processing time may be longer',
'API costs will be incurred',
],
};
}
// Recommend precise mode
return {
recommendedMode: 'precise',
reason:
'Precise mode available. Can preserve mixed text and image content',
estimatedCost: costEstimate.estimatedCost,
estimatedTime: costEstimate.estimatedTime,
warnings: ['API costs will be incurred'],
};
}
/**
* Get user quota information
*/
async getUserQuotaInfo(userId: string) {
const quota = await this.costControl.getUserQuota(userId);
const report = await this.costControl.getCostReport(userId);
return {
...quota,
report,
warnings: await this.costControl.checkWarningThreshold(userId),
};
}
/**
* Update processing status (for real-time feedback)
*/
private updateStatus(
status: ProcessingStatus['status'],
progress: number,
message: string,
): void {
this.logger.log(`[${status}] ${progress}% - ${message}`);
}
}