diff --git a/_AI_USAGE_LOG.md b/_AI_USAGE_LOG.md index 7fc1a65..0f86b5b 100644 --- a/_AI_USAGE_LOG.md +++ b/_AI_USAGE_LOG.md @@ -39,3 +39,4 @@ | 2026-08-09 04:40 | 反馈迭代 | 里程碑2.5 Task13 遗留清理：① extraction_method 枚举同源（free_text_extractor→ExtractionMethod.LLM_FROM_FREE_TEXT.value、table_extractor 空/非空两分支→ExtractionMethod.OPENPYXL.value，值不变既有用例兼容）；② table_extractor header_row 越界/负值防御（<0 或 >=len(matrix) 回落 0，keep if not matrix 早返回）；③ merge_fill.forward_fill 完整替换（min_row<1/min_col<1 非法值跳过 + 越界防御）；④ data_models.Provenance.row 语义注释；⑤ 补测试 6 个（sheet_nature MIXED ・/■ 正向 x2、formatting_detector 字体色正向、table_extractor 枚举锁 + forward_fill 防御、free_text_extractor 枚举锁）。TDD：先加测试确认 RED（仅 forward_fill 防御用例失败，其余为已通过回归锁）→ 实现 → GREEN；pytest 全量 58 passed（52 baseline + 6） | src/genesis/parsers/free_text_extractor.py, src/genesis/parsers/table_extractor.py, src/genesis/parsers/merge_fill.py, src/genesis/data_models.py, tests/test_sheet_nature.py, tests/test_formatting_detector.py, tests/test_free_text_extractor.py, tests/test_table_extractor.py, _AI_USAGE_LOG.md | deepseek-v4-flash-free | | 2026-08-09 10:05 | 反馈迭代 | 里程碑2.5 最终评审与交付：整体评审通过（base 81c6b6c..d93cc77，58 passed）；清理 progress.md Task11 重复/编码行；修正 _AI_USAGE_LOG.md Task13 范式步骤用词；产出交付报告 docs/milestone2.5-mixed-parser-review.md（MIXED 遗留#1 闭环） | docs/milestone2.5-mixed-parser-review.md, .superpowers/sdd/progress.md, _AI_USAGE_LOG.md | deepseek-v4-flash-free | | 2026-08-09 10:12 | 测试验证 | 补齐覆盖率工具：安装 pytest-cov 7.1.0/coverage 7.15.4，pyproject.toml 固化 dev 依赖+addopts（--cov=genesis --cov-report=term-missing）+[tool.coverage.run/report]（source=genesis、branch=true、fail_under=90、skip_empty、sort=miss）；验证 fail_under=99 拦截 exit 1；产出 docs/coverage-report.md 与 htmlcov（已 ignore）；基线覆盖 96.67%（571 stmts，12 miss） | pyproject.toml, docs/coverage-report.md | deepseek-v4-flash-free | +| 2026-08-09 10:20 | 测试验证 | 覆盖率提升至 100%：新增 13 个防御/边界用例（空矩阵/稀疏多列/>表头越界/非法色/RGB访问异常/fill=None/list分值/空sh山 skipped/MXED 段外格式隔离）；pragma 标注 3 处不可达防御分支（classify 恒有表头）；fail_under 90→99；全量 71 passed，覆盖率 100.00%（563 stmts/140 br 全绿）；更新 docs/coverage-report.md | tests/test_sheet_nature.py, tests/test_table_extractor.py, tests/test_formatting_detector.py, tests/test_config.py, tests/test_excel_parser.py, src/genesis/parsers/excel_parser.py, src/genesis/parsers/sheet_nature.py, pyproject.toml, docs/coverage-report.md | deepseek-v4-flash-free | diff --git a/docs/coverage-report.md b/docs/coverage-report.md index 09902b6..62883c1 100644 --- a/docs/coverage-report.md +++ b/docs/coverage-report.md @@ -3,58 +3,57 @@ - 日期：2026-08-09 - 工具：`pytest-cov` 7.1.0 / `coverage` 7.15.4 - 范围：`src/genesis`（排除 tests 自身） -- 阈值：`fail_under = 90`（`pyproject.toml` `[tool.coverage.report]`），未达标时 pytest 退出码非 0 -- 全量用例：**58 passed，无 skip** +- 阈值：`fail_under = 99`（`pyproject.toml` `[tool.coverage.report]`），未达标时 pytest 退出码非 0 +- 全量用例：**71 passed，无 skip** - 输出：终端摘要（`--cov-report=term-missing`）、HTML（`htmlcov/index.html`，已 .gitignore） ## 总览 | 指标 | 值 | |------|-----| -| 语句总数 | 571 | -| 未覆盖语句 | 12 | -| 分支总数 | 150 | -| 未覆盖分支 | 12 | -| **总覆盖率** | **96.67%** | -| 最低模块 | `table_extractor.py` 88% | -| 最高模块 | 9 个文件 100% | +| 语句总数 | 563（含 pragma 排除的不可达防御分支） | +| 未覆盖语句 | 0 | +| 分支总数 | 140 | +| 未覆盖分支 | 0 | +| **总覆盖率** | **100.00%** | -## 分模块 +## 分模块（全部 100%） -| 模块 | Stmts | Miss | Branch | BrPart | Cover | 未覆盖行 | -|------|------:|-----:|-------:|-------:|------:|----------| -| `__init__.py` | 1 | 0 | — | — | 100% | — | -| `data_models.py` | 180 | 0 | — | — | 100% | — | -| `excel_reader.py` | 12 | 0 | 2 | 0 | 100% | — | -| `free_text_extractor.py` | 22 | 0 | 10 | 0 | 100% | — | -| `merge_fill.py` | 15 | 0 | 10 | 0 | 100% | — | -| `paragraph_splitter.py` | 17 | 0 | 10 | 0 | 100% | — | -| `provenance.py` | 2 | 0 | — | — | 100% | — | -| `sheet_detector.py` | 20 | 0 | 14 | 0 | 100% | — | -| `config.py` | 128 | 1 | 24 | 1 | 99% | 134 | -| `sheet_nature.py` | 33 | 2 | 18 | 3 | 90% | 26, 32, 40→44 | -| `table_extractor.py` | 24 | 2 | 10 | 2 | 88% | 32, 37 | -| `formatting_detector.py` | 40 | 3 | 20 | 2 | 92% | 15–16, 22, 31→33 | -| `excel_parser.py` | 77 | 4 | 32 | 4 | 93% | 36–37, 61, 66→65, 100 | +| 模块 | Stmts | Miss | Branch | BrPart | Cover | +|------|------:|-----:|-------:|-------:|------:| +| `__init__.py` | 1 | 0 | — | — | 100% | +| `config.py` | 128 | 0 | 24 | 0 | 100% | +| `data_models.py` | 180 | 0 | — | — | 100% | +| `parsers/excel_parser.py` | 73 | 0 | 28 | 0 | 100% | +| `parsers/excel_reader.py` | 12 | 0 | 2 | 0 | 100% | +| `parsers/formatting_detector.py` | 40 | 0 | 20 | 0 | 100% | +| `parsers/free_text_extractor.py` | 22 | 0 | 10 | 0 | 100% | +| `parsers/merge_fill.py` | 15 | 0 | 10 | 0 | 100% | +| `parsers/paragraph_splitter.py` | 17 | 0 | 10 | 0 | 100% | +| `parsers/provenance.py` | 2 | 0 | — | — | 100% | +| `parsers/sheet_detector.py` | 20 | 0 | 14 | 0 | 100% | +| `parsers/sheet_nature.py` | 29 | 0 | 12 | 0 | 100% | +| `parsers/table_extractor.py` | 24 | 0 | 10 | 0 | 100% | -## 未覆盖点说明（12 语句 + 12 分支） +## 达到 100% 的补充（相对首次基线 96.67%） -主要未覆盖区域属于防御分支 / 异常路径： +新增 13 个防御/边界用例覆盖了此前未触达的分支： -- `config.py:134` — `unset(_env)` 分支（枚举加载外的环境变量消除路径） -- `sheet_nature.py:26/32` — 空白行修正 / `value` 缺失兜底 -- `table_extractor.py:32/37` — `column_letter` 越界兜底 / 空列头兜底 -- `formatting_detector.py:15-22` — 非法 RGB 字符串剔除分支 -- `excel_parser.py:36-37,61,100` — 空表跳过、`header_row<0` 回落、无表头 free_text 直接文本化 - -均为「非法输入 / 防御分支」，非主路径；当前 96.67% 已满足红线 90%。 +- `test_sheet_nature.py`：空矩阵→FREE_TEXT、稀疏多列（非空占比<0.7）→FREE_TEXT、有表头无 ・/■ 行→TABLE +- `test_table_extractor.py`：空矩阵→空表、`header_row` 越界/负值回落 0 +- `test_formatting_detector.py`：6/8 位十六进制接受、纯黑/主题色/非法 hex/长度不足/访问异常→None、仅背景色命中、`fill=None` 防御分支 +- `test_config.py`：`_expand_env` 的 list 分值（列表内递归展开占位） +- `test_excel_parser.py`：空 sheet→skipped、碎片段格式不污染表格段 `seg_fmt_map`（66→65） +- pragma 标注两处不可达防御分支：`excel_parser` 两处 `header_row<0` 回落（`classify_sheet==TABLE` 恒有表头）、`sheet_nature` 的 `header_row>=0` 检测（`_free_text_like()==False` 恒含表头） ## 门槛策略 -- `pyproject.toml` 已固化 `fail_under = 90`（含 12 个 `branch` 分支统计） -- 每次 `python -m pytest` 默认附带覆盖率检查；未来若引入重业务逻辑（LLM/Writer 里程碑），可将 `fail_under` 抬至 95~98 并补充新用例 +- `fail_under = 99`（余量 1%）：未来若引入未测的新模块/分支（LLM 结构化、Writer 等），需为新代码补测试或显式 `# pragma: no cover` 标注不可达防御分支，否则 CI 拦截 +- 每次 `python -m pytest` 默认附带覆盖率检查 +- 报告归档：本文件（docs/）+ HTML（htmlcov/，不纳入 git） ## 后续可选增强（非当前阻塞） -- 将 `fail_under` 提升至 95% 时需补上述 12 行（多为防御分支，成本低） -- 可在 CI 中将 `--cov-report=html` 或 junit/xml 纳入流水线归档 \ No newline at end of file +- 将 `fail_under` 保持 99 并作为 CI 红线 +- 对 LLM 相关（InferenceEngine）等后续模块按同样标准配置覆盖率 +- 可将 HTML 报告接入 CI artifact 归档 \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 1719377..34480b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ branch = true [tool.coverage.report] # 覆盖率门槛：低于该值 pytest 失败（红线，防止回归） -fail_under = 90 +fail_under = 99 # 跳过 __init__.py 等空文件干扰 skip_empty = true # 按语句总数降序展示 diff --git a/src/genesis/parsers/excel_parser.py b/src/genesis/parsers/excel_parser.py index 9b653ac..d7e8c2c 100644 --- a/src/genesis/parsers/excel_parser.py +++ b/src/genesis/parsers/excel_parser.py @@ -57,7 +57,7 @@ class ExcelParser: seg_nature = classify_sheet(seg) if seg_nature == SheetNature.TABLE: header_row = find_header_row(seg) - if header_row < 0: + if header_row < 0: # pragma: no cover — classify==TABLE 时 find_header_row 恒 ≥0 header_row = 0 # extract_table 的 formatting_map 用段内矩阵坐标 (r,c)； # 从整 sheet 物理 map 抽出段内子 map（物理行 pr → 段内 pr-s） @@ -96,7 +96,7 @@ class ExcelParser: ] filled = forward_fill(matrix, merged) if merged else matrix header_row = find_header_row(filled) - if header_row < 0: + if header_row < 0: # pragma: no cover — classify==TABLE 时 find_header_row 恒 ≥0 header_row = 0 fmt_map = {} for row in ws.iter_rows(): diff --git a/src/genesis/parsers/sheet_nature.py b/src/genesis/parsers/sheet_nature.py index d77c825..a1137fa 100644 --- a/src/genesis/parsers/sheet_nature.py +++ b/src/genesis/parsers/sheet_nature.py @@ -37,7 +37,7 @@ def classify_sheet(matrix: list[list[Any]]) -> SheetNature: if _free_text_like(matrix): return SheetNature.FREE_TEXT header_row = find_header_row(matrix) - if header_row >= 0: + if header_row >= 0: # pragma: no cover — _free_text_like()==False 时 find_header_row 恒 ≥0 for row in matrix[header_row + 1:]: if any(str(c).strip().startswith(("・", "■")) for c in _non_empty(row)): return SheetNature.MIXED diff --git a/tests/test_config.py b/tests/test_config.py index 3f28bc7..cbc5fd2 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -48,4 +48,14 @@ def test_env_placeholder_expansion(monkeypatch): def test_redacted_hides_secrets(): s = Settings.from_dir(FIXTURES) red = s.get_redacted() - assert red["rag"]["vector_store"]["qdrant"]["api_key"] == "***" \ No newline at end of file + assert red["rag"]["vector_store"]["qdrant"]["api_key"] == "***" + + +def test_expand_env_list_branch(monkeypatch): + # _expand_env 的 list 分支（L134）：列表项递归展开环境占位 + from genesis.config import _expand_env + monkeypatch.setenv("QDRANT_API_KEY", "sk-list-xyz") + data = {"models": [{"name": "a", "key": "${QDRANT_API_KEY}"}, "plain"]} + out = _expand_env(data) + assert out["models"][0]["key"] == "sk-list-xyz" + assert out["models"][1] == "plain" \ No newline at end of file diff --git a/tests/test_excel_parser.py b/tests/test_excel_parser.py index e1af748..e0bb076 100644 --- a/tests/test_excel_parser.py +++ b/tests/test_excel_parser.py @@ -4,6 +4,43 @@ from genesis.parsers.excel_parser import ExcelParseResult, ExcelParser from tests.excel_helpers import new_workbook, save_workbook +def test_parse_skips_empty_sheet(tmp_path): + # 空 sheet → skipped 不进入（L skips 分支） + wb = new_workbook({"空白": [[]]}) + path = save_workbook(tmp_path, wb) + result = ExcelParser().parse(path) + assert "空白" in result.skipped + assert result.tables == [] + + +def test_parse_mixed_formatting_outside_table_segment(tmp_path): + # 碎片段（free_text 段）存在取消线格式 → seg_fmt_map 构建时该行不在表格段 [s,e] 内，被跳过（66-65 分支） + from copy import copy + from openpyxl import Workbook + wb = Workbook() + ws = wb.active + ws.title = "混合" + data = [ + ["機能ID", "機能名"], ["F101", "社員登録"], + [], + ["・改述ポイント"], + ] + for r, row in enumerate(data, start=1): + for c, v in enumerate(row, start=1): + if v: + ws.cell(row=r, column=c, value=v) + font = copy(ws.cell(row=4, column=1).font) # 碎片段行（物理行 4） + font.strike = True + ws.cell(row=4, column=1).font = font + path = save_workbook(tmp_path, wb) + result = ExcelParser().parse(path) + ms = result.mixed[0] + tbl = [p.table for p in ms.paragraphs if p.kind == "table"][0] + # 表格段数据行不应带上碎片段的取消线格式 + assert tbl.rows[0]["機能ID"].formatting is None + assert len(tbl.rows) == 1 + + def test_parse_table_sheets(tmp_path): wb = new_workbook({ "機能一覧": [["機能ID", "機能名"], ["A001", "社員登録"]], diff --git a/tests/test_formatting_detector.py b/tests/test_formatting_detector.py index 5a214c7..2da1ce5 100644 --- a/tests/test_formatting_detector.py +++ b/tests/test_formatting_detector.py @@ -59,4 +59,58 @@ def test_cell_comment_returns_obj(): def test_collect_comments(): comments = collect_comments(make_wb().active, "f.xlsx") - assert len(comments) == 1 \ No newline at end of file + assert len(comments) == 1 + + +def test_to_rgb_hex_accepts_6_digit_hex(): + from types import SimpleNamespace + from genesis.parsers.formatting_detector import _to_rgb_hex + assert _to_rgb_hex(SimpleNamespace(rgb="FF3333")) == "FF3333" + assert _to_rgb_hex(SimpleNamespace(rgb="00FF00FF")) == "00FF00FF" + + +def test_to_rgb_hex_rejects_black_and_theme(): + from types import SimpleNamespace + from genesis.parsers.formatting_detector import _to_rgb_hex + # 纯黑（默认色）与主题色（rgb 为空/异常/非 hex）→ None + assert _to_rgb_hex(SimpleNamespace(rgb="00000000")) is None + assert _to_rgb_hex(SimpleNamespace(rgb="FF000000")) is None + assert _to_rgb_hex(SimpleNamespace(rgb="")) is None + assert _to_rgb_hex(SimpleNamespace(rgb="ZZZZZZ")) is None + assert _to_rgb_hex(SimpleNamespace(rgb="FFFFF")) is None # 长度不足 + + +class _ExplodingRgb: + @property + def rgb(self): + raise Exception("boom") + + +def test_to_rgb_hex_rgb_access_exception_returns_none(): + from genesis.parsers.formatting_detector import _to_rgb_hex + # L15-16：color.rgb 访问抛异常 → 返回 None（防御分支） + assert _to_rgb_hex(_ExplodingRgb()) is None + + +def test_cell_formatting_fills_only_color(): + from copy import copy + from openpyxl.styles import PatternFill + from genesis.parsers.formatting_detector import cell_formatting + wb = Workbook() + ws = wb.active + ws["A1"] = "x" + # 仅背景色（无 strikethrough / 无字体色）→ 返回 CellFormatting 含 bg_color + f = copy(ws["A1"].fill) + f.fgColor = PatternFill(start_color="00CC00").fgColor + ws["A1"].fill = f + fmt = cell_formatting(ws["A1"]) + assert fmt is not None + assert fmt.bg_color is not None + + +def test_cell_formatting_fill_none_branch(): + # 覆盖 L31 防御分支：fill 为 None（无样式对象）时直接返回 None + from types import SimpleNamespace + from genesis.parsers.formatting_detector import cell_formatting + fake = SimpleNamespace(font=SimpleNamespace(strike=False), fill=None) + assert cell_formatting(fake) is None \ No newline at end of file diff --git a/tests/test_sheet_nature.py b/tests/test_sheet_nature.py index e5a2b35..38f6138 100644 --- a/tests/test_sheet_nature.py +++ b/tests/test_sheet_nature.py @@ -29,4 +29,21 @@ def test_classify_sheet_mixed_with_bullet_line(): def test_classify_sheet_mixed_with_square_line(): m = [["ID", "名前"], ["1", "田中"], ["■備考行"]] - assert classify_sheet(m) == SheetNature.MIXED \ No newline at end of file + assert classify_sheet(m) == SheetNature.MIXED + + +def test_classify_empty_matrix_is_free_text(): + # 空矩阵：_free_text_like 对空矩阵返回 True（L26 分支） + assert classify_sheet([]) == SheetNature.FREE_TEXT + + +def test_classify_sparse_multicol_is_free_text(): + # 多列但非空行占比 < 0.7 → FREE_TEXT（L32 稀疏分支） + m = [["ID", "名前"], [], [], [], [], ["1", "田中"]] + assert classify_sheet(m) == SheetNature.FREE_TEXT + + +def test_classify_header_no_bullet_is_table(): + # 表头有效且无 ・/■ 行 → 落入 TABLE（L40→44 无匹配循环路径） + m = [["ID", "名前"], ["1", "田中"], ["2", "佐藤"]] + assert classify_sheet(m) == SheetNature.TABLE \ No newline at end of file diff --git a/tests/test_table_extractor.py b/tests/test_table_extractor.py index e80e083..09e78ac 100644 --- a/tests/test_table_extractor.py +++ b/tests/test_table_extractor.py @@ -49,4 +49,20 @@ def test_extract_table_uses_enum_value(): def test_forward_fill_ignores_invalid_range(): matrix = [["A"], ["B"]] assert forward_fill(matrix, [(0, 1, 2, 1)]) == matrix # min_row=0 非法 → 不崩溃不改数据 - assert forward_fill(matrix, [(9, 9, 9, 9)]) == matrix # 越界 → 不崩溃 \ No newline at end of file + assert forward_fill(matrix, [(9, 9, 9, 9)]) == matrix # 越界 → 不崩溃 + + +def test_extract_table_empty_matrix(): + # 空矩阵 → 返回空表（headers/rows 均为空），不崩溃（L 空矩阵分支） + table = extract_table("空シート", [], "f.xlsx", SheetType.FUNCTION) + assert table.headers == [] + assert table.rows == [] + + +def test_extract_table_header_row_out_of_range(): + # header_row 越界/负值 → 回落 0（防御分支） + mat = [["ID", "名前"], ["1", "田中"]] + t_high = extract_table("社員", mat, "f.xlsx", SheetType.FUNCTION, header_row=9) + t_neg = extract_table("社員", mat, "f.xlsx", SheetType.FUNCTION, header_row=-1) + assert t_high.headers == ["ID", "名前"] + assert t_neg.headers == ["ID", "名前"] \ No newline at end of file