feat: 段落分割 split_paragraphs(通用空行分词)

This commit is contained in:
lhl
2026-08-09 03:59:24 +08:00
parent 81c6b6c522
commit 34ef28be45
2 changed files with 49 additions and 0 deletions
+24
View File
@@ -0,0 +1,24 @@
from __future__ import annotations
from typing import Any
def _is_blank_row(row: list[Any]) -> bool:
return all(c is None or str(c).strip() == "" for c in row)
def split_paragraphs(matrix: list[list[Any]]) -> list[tuple[int, int]]:
"""以全空行为界的通用段落分割;返回 (start_row, end_row)(含端,0-based)。"""
paragraphs: list[tuple[int, int]] = []
start: int | None = None
for i, row in enumerate(matrix):
if not _is_blank_row(row):
if start is None:
start = i
else:
if start is not None:
paragraphs.append((start, i - 1))
start = None
if start is not None:
paragraphs.append((start, len(matrix) - 1))
return paragraphs
+25
View File
@@ -0,0 +1,25 @@
from genesis.parsers.paragraph_splitter import split_paragraphs
def test_empty_matrix():
assert split_paragraphs([]) == []
def test_single_paragraph_no_empty_rows():
m = [["a", "b"], ["c", "d"]]
assert split_paragraphs(m) == [(0, 1)]
def test_split_on_middle_empty_row():
m = [["a"], [], ["b"], ["c"], []]
assert split_paragraphs(m) == [(0, 0), (2, 3)]
def test_trailing_empty_rows_no_extra_paragraph():
m = [["a"], [], [], []]
assert split_paragraphs(m) == [(0, 0)]
def test_leading_empty_rows_start_at_first_nonempty():
m = [[], ["a"], [], ["b"]]
assert split_paragraphs(m) == [(1, 1), (3, 3)]