feat: 段落分割 split_paragraphs(通用空行分词)
This commit is contained in:
@@ -0,0 +1,24 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
|
def _is_blank_row(row: list[Any]) -> bool:
|
||||||
|
return all(c is None or str(c).strip() == "" for c in row)
|
||||||
|
|
||||||
|
|
||||||
|
def split_paragraphs(matrix: list[list[Any]]) -> list[tuple[int, int]]:
|
||||||
|
"""以全空行为界的通用段落分割;返回 (start_row, end_row)(含端,0-based)。"""
|
||||||
|
paragraphs: list[tuple[int, int]] = []
|
||||||
|
start: int | None = None
|
||||||
|
for i, row in enumerate(matrix):
|
||||||
|
if not _is_blank_row(row):
|
||||||
|
if start is None:
|
||||||
|
start = i
|
||||||
|
else:
|
||||||
|
if start is not None:
|
||||||
|
paragraphs.append((start, i - 1))
|
||||||
|
start = None
|
||||||
|
if start is not None:
|
||||||
|
paragraphs.append((start, len(matrix) - 1))
|
||||||
|
return paragraphs
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
from genesis.parsers.paragraph_splitter import split_paragraphs
|
||||||
|
|
||||||
|
|
||||||
|
def test_empty_matrix():
|
||||||
|
assert split_paragraphs([]) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_single_paragraph_no_empty_rows():
|
||||||
|
m = [["a", "b"], ["c", "d"]]
|
||||||
|
assert split_paragraphs(m) == [(0, 1)]
|
||||||
|
|
||||||
|
|
||||||
|
def test_split_on_middle_empty_row():
|
||||||
|
m = [["a"], [], ["b"], ["c"], []]
|
||||||
|
assert split_paragraphs(m) == [(0, 0), (2, 3)]
|
||||||
|
|
||||||
|
|
||||||
|
def test_trailing_empty_rows_no_extra_paragraph():
|
||||||
|
m = [["a"], [], [], []]
|
||||||
|
assert split_paragraphs(m) == [(0, 0)]
|
||||||
|
|
||||||
|
|
||||||
|
def test_leading_empty_rows_start_at_first_nonempty():
|
||||||
|
m = [[], ["a"], [], ["b"]]
|
||||||
|
assert split_paragraphs(m) == [(1, 1), (3, 3)]
|
||||||
Reference in New Issue
Block a user