lhl
d9aa3a3325
- Writer: 表格表头行/Table Grid 边框、列表 List Bullet/Number 样式、行内字符格式不再塌缩（外视 #6 反转） - Writer: 打通章间引用（WriterState 摘要 → 后章 prompt prior_summaries） - Writer: 删除 _chunk_source 死代码，章节数据经 DataGate 控 token 预算 - Writer: 章节结果落盘快照，命中即跳过 LLM（录制重拍可续跑，损坏快照自动忽略） - Parser: 按 body 顺序遍历正文+单元格（含嵌套表、合并单元格去重），修复表格内锚点漏检导致的静默丢章 - Parser/服务层: .xls 显式拒绝（可操作提示），上传即校验扩展名，rules 对齐 docx-only - 测试: 全量 680 通过，覆盖率 99.37%（红线 99%）
223 lines
7.1 KiB
Python
223 lines
7.1 KiB
Python
from __future__ import annotations
|
||
|
||
import os
|
||
from pathlib import Path
|
||
from typing import Any, Literal
|
||
|
||
import yaml
|
||
from pydantic import BaseModel, Field
|
||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||
|
||
SECRET_KEYWORDS = ("key", "secret", "token")
|
||
ENV_PREFIX = "GENESIS_"
|
||
|
||
|
||
# ---------- 各 yaml 对应的 pydantic 模型 ----------
|
||
|
||
class ServerConfig(BaseModel):
|
||
max_upload_mb: int = 100
|
||
allowed_extensions: list[str] = Field(
|
||
default_factory=lambda: [".xlsx", ".docx", ".pptx", ".java", ".xml", ".yml",
|
||
".py", ".ts", ".go", ".cs"]
|
||
)
|
||
|
||
|
||
class AppConfig(BaseModel):
|
||
name: str = "genesis"
|
||
version: str = "0.1.0"
|
||
timezone: str = "Asia/Tokyo"
|
||
server: ServerConfig = Field(default_factory=ServerConfig)
|
||
session: dict[str, Any] = Field(default_factory=lambda: {
|
||
"sqlite_path": "/data/db/genesis.db",
|
||
"snapshot_dir": "/data/db/snapshots",
|
||
})
|
||
paths: dict[str, Any] = Field(default_factory=lambda: {
|
||
"user_root": "/data/users",
|
||
"shared_root": "/data/shared",
|
||
})
|
||
task_queue: dict[str, Any] = Field(default_factory=lambda: {
|
||
"backend": "memory",
|
||
"timeout_sec": 600,
|
||
"retry_default": 2,
|
||
})
|
||
|
||
|
||
class ModelSpec(BaseModel):
|
||
provider: str = "deepseek"
|
||
name: str = "deepseek-chat"
|
||
temperature: float = 0.2
|
||
max_tokens: int = 4096
|
||
timeout_sec: int = 60
|
||
retry_backoff: list[float] = Field(default_factory=lambda: [1.0, 3.0, 7.0])
|
||
|
||
|
||
class InferenceModels(BaseModel):
|
||
primary: ModelSpec = Field(default_factory=ModelSpec)
|
||
fallback: ModelSpec = Field(default_factory=lambda: ModelSpec(provider="qwen", name="qwen-max"))
|
||
vision: ModelSpec = Field(default_factory=lambda: ModelSpec(name="deepseek-vl", timeout_sec=90))
|
||
|
||
|
||
class LlmCallsConfig(BaseModel):
|
||
token_estimation: str = "tiktoken"
|
||
max_context_tokens: int = 32000
|
||
truncation_policy: dict[str, Any] = Field(default_factory=lambda: {
|
||
"priority": ["shrink_rule_chunks", "summarize_history", "truncate_data"],
|
||
})
|
||
|
||
|
||
class StructuredOutputConfig(BaseModel):
|
||
max_parse_retry: int = 2
|
||
|
||
|
||
class PromptRegistryConfig(BaseModel):
|
||
prompts_dir: str = "./prompts"
|
||
default_version: str = "latest"
|
||
|
||
|
||
class InferenceConfig(BaseModel):
|
||
models: InferenceModels = Field(default_factory=InferenceModels)
|
||
llm_calls: LlmCallsConfig = Field(default_factory=LlmCallsConfig)
|
||
structured_output: StructuredOutputConfig = Field(default_factory=StructuredOutputConfig)
|
||
prompt_registry: PromptRegistryConfig = Field(default_factory=PromptRegistryConfig)
|
||
|
||
|
||
class EmbeddingConfig(BaseModel):
|
||
# OV2/T11：实际语料为日文，bge-small-zh 面向中文 → 默认多语言 bge-m3（中/日/英）
|
||
model: str = "BAAI/bge-m3"
|
||
device: str = "cpu"
|
||
max_batch_size: int = 32
|
||
cache_dir: str = "/data/shared/models"
|
||
|
||
|
||
class ChromaStoreConfig(BaseModel):
|
||
persist_dir: str = "/data/shared/rules-handbook/chroma"
|
||
|
||
|
||
class VectorStoreConfig(BaseModel):
|
||
adapter: str = "chroma"
|
||
chroma: ChromaStoreConfig = Field(default_factory=ChromaStoreConfig)
|
||
|
||
|
||
class ChunkingConfig(BaseModel):
|
||
word_max_tokens: int = 512
|
||
excel_rule_block_rows: int = 10
|
||
ppt_pages_per_chunk: int = 2
|
||
min_tokens: int = 30
|
||
|
||
|
||
class RetrievalConfig(BaseModel):
|
||
channel_top_k: int = 10
|
||
rrf_k: int = 60
|
||
default_top_k: int = 5
|
||
contextual_enrichment: bool = True
|
||
|
||
|
||
class RerankConfig(BaseModel):
|
||
# I6/T6：v1 引入 rerank 精排（2026 主流实践：向量→rerank→精排）
|
||
enabled: bool = True
|
||
model: str = "BAAI/bge-reranker-v2-m3"
|
||
device: str = "cpu"
|
||
|
||
|
||
class RagConfig(BaseModel):
|
||
embedding: EmbeddingConfig = Field(default_factory=EmbeddingConfig)
|
||
vector_store: VectorStoreConfig = Field(default_factory=VectorStoreConfig)
|
||
chunking: ChunkingConfig = Field(default_factory=ChunkingConfig)
|
||
retrieval: RetrievalConfig = Field(default_factory=RetrievalConfig)
|
||
rerank: RerankConfig = Field(default_factory=RerankConfig)
|
||
|
||
|
||
class WriterConfig(BaseModel):
|
||
"""Writer 子系统配置（步骤 1：输出语言参数）。
|
||
|
||
output_language: 生成概要设计书正文的自然语言
|
||
- "auto"：与章节标题所用语言保持一致（默认，向后兼容既有日文文档）
|
||
- "zh"：强制简体中文
|
||
- "ja"：强制日文
|
||
表格数据始终照抄源 Excel 原文（不翻译），见 design.md §7.2。
|
||
"""
|
||
|
||
output_language: Literal["auto", "zh", "ja"] = "auto"
|
||
|
||
|
||
# ---------- 加载辅助 ----------
|
||
|
||
def _expand_env(data: Any) -> Any:
|
||
"""递归展开 ${VAR} 占位（读环境变量，缺失→空串）"""
|
||
if isinstance(data, dict):
|
||
return {k: _expand_env(v) for k, v in data.items()}
|
||
if isinstance(data, list):
|
||
return [_expand_env(v) for v in data]
|
||
if isinstance(data, str) and data.startswith("${") and data.endswith("}"):
|
||
return os.environ.get(data[2:-1], "")
|
||
return data
|
||
|
||
|
||
def _deep_merge(base: dict, override: dict) -> dict:
|
||
"""递归合并：override 覆盖 base；非 dict 值直接取 override 存在者"""
|
||
out = dict(base)
|
||
for k, v in override.items():
|
||
if isinstance(v, dict) and isinstance(out.get(k), dict):
|
||
out[k] = _deep_merge(out[k], v)
|
||
else:
|
||
out[k] = v
|
||
return out
|
||
|
||
|
||
def _env_overrides() -> dict:
|
||
"""收集 GENESIS_ 前缀的条目为嵌套 dict，__ 为嵌套分隔（键统一小写以匹配 yaml）"""
|
||
result: dict[str, Any] = {}
|
||
for key, value in os.environ.items():
|
||
if key.startswith(ENV_PREFIX):
|
||
parts = key[len(ENV_PREFIX):].split("__")
|
||
node = result
|
||
for part in parts[:-1]:
|
||
node = node.setdefault(part.lower(), {})
|
||
node[parts[-1].lower()] = value
|
||
return result
|
||
|
||
|
||
def _load_yaml(config_dir: Path, name: str) -> dict:
|
||
path = config_dir / f"{name}.yaml"
|
||
if not path.exists():
|
||
return {}
|
||
with path.open("r", encoding="utf-8") as f:
|
||
return yaml.safe_load(f) or {}
|
||
|
||
|
||
def _redact(data: dict) -> dict:
|
||
out = {}
|
||
for k, v in data.items():
|
||
if any(kw in str(k).lower() for kw in SECRET_KEYWORDS):
|
||
out[k] = "***"
|
||
elif isinstance(v, dict):
|
||
out[k] = _redact(v)
|
||
else:
|
||
out[k] = v
|
||
return out
|
||
|
||
|
||
# ---------- 根 Settings ----------
|
||
|
||
class Settings(BaseSettings):
|
||
model_config = SettingsConfigDict(env_prefix=ENV_PREFIX, env_file=".env", extra="ignore")
|
||
|
||
app: AppConfig = Field(default_factory=AppConfig)
|
||
inference: InferenceConfig = Field(default_factory=InferenceConfig)
|
||
rag: RagConfig = Field(default_factory=RagConfig)
|
||
writer: WriterConfig = Field(default_factory=WriterConfig)
|
||
|
||
@classmethod
|
||
def from_dir(cls, config_dir: Path | str) -> "Settings":
|
||
config_dir = Path(config_dir)
|
||
raw = {
|
||
"app": _load_yaml(config_dir, "app"),
|
||
"inference": _load_yaml(config_dir, "inference"),
|
||
"rag": _load_yaml(config_dir, "rag"),
|
||
}
|
||
env = _env_overrides()
|
||
merged = {k: _deep_merge(raw[k], env.get(k, {})) for k in raw}
|
||
return cls(**{k: _expand_env(v) for k, v in merged.items()})
|
||
|
||
def get_redacted(self) -> dict:
|
||
return _redact(self.model_dump(mode="json")) |