Use PDF outlines for configurable chapter splitting
Add outline-level chapter selection with a numbered-heading fallback and document the new CLI option. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Сергей Маринкевич
@@ -36,13 +36,22 @@ out/RealTek-r8169/
|
||||
| Флаг | Описание |
|
||||
|------|----------|
|
||||
| `--single-file` | Один `{stem}.md` вместо `INDEX.md` + `chapters/` |
|
||||
| `--chapter-level N` | Уровень встроенного оглавления PDF для границ глав: `1`, `2` или `3` (default `2`) |
|
||||
| `--tables` | Включить извлечение таблиц pdfplumber (по умолчанию выключено) |
|
||||
| `--page-dpi N` | DPI для PNG-рендера страниц с битым текстом (default 150) |
|
||||
| `-o DIR` | Каталог вывода |
|
||||
|
||||
## Grep для агента
|
||||
|
||||
Главы разбиты по крупным разделам документа. Каждая страница помечена HTML-комментарием:
|
||||
Главы разбиты по встроенному оглавлению PDF. Уровень нарезки можно выбрать:
|
||||
|
||||
```bash
|
||||
./run.sh doc.pdf --chapter-level 1 # крупные разделы
|
||||
./run.sh doc.pdf --chapter-level 2 # подразделы, default
|
||||
./run.sh doc.pdf --chapter-level 3 # самые мелкие разделы
|
||||
```
|
||||
|
||||
Каждая страница помечена HTML-комментарием:
|
||||
|
||||
```bash
|
||||
# найти упоминание регистра в главе 6
|
||||
|
||||
+86
-49
@@ -16,13 +16,14 @@ HEADER_FOOTER = re.compile(
|
||||
r"^(RTL8169|2002/\d{2}/\d{2}|Rev\.\d+\.\d+|\d{1,3})$",
|
||||
re.MULTILINE,
|
||||
)
|
||||
MAJOR_SECTION = re.compile(r"^(\d{1,2})\.\s+([A-Z][^\n]{2,})$")
|
||||
TOC_LINE = re.compile(r"\.{4,}\s*\d+\s*$")
|
||||
NUMBERED_HEADING = re.compile(r"^(\d{1,2}(?:\.\d{1,2})*)\s+(.+)$")
|
||||
MIN_TABLE_ROWS = 2
|
||||
MIN_TABLE_CELLS = 6
|
||||
MIN_CELL_LEN_FOR_DEDUP = 4
|
||||
SPACED_LETTERS = re.compile(r"^([A-Z](?: [A-Z]){1,}|[A-Z])$")
|
||||
DEFAULT_PAGE_DPI = 150
|
||||
DEFAULT_CHAPTER_LEVEL = 2
|
||||
|
||||
|
||||
def slugify(title: str) -> str:
|
||||
@@ -190,29 +191,45 @@ def render_page_png(
|
||||
return f"assets/{name}"
|
||||
|
||||
|
||||
def find_major_section(line: str) -> tuple[str, str] | None:
|
||||
m = MAJOR_SECTION.match(line.strip())
|
||||
if not m:
|
||||
return None
|
||||
num, title = m.group(1), m.group(2).strip()
|
||||
if TOC_LINE.search(title) or "." * 3 in title:
|
||||
return None
|
||||
if title.endswith(":"):
|
||||
return None
|
||||
if len(title) > 70:
|
||||
return None
|
||||
return num, title
|
||||
def toc_sections(doc: fitz.Document, level: int) -> list[dict]:
|
||||
"""Return unique page boundaries from the PDF outline at one level."""
|
||||
sections: list[dict] = []
|
||||
seen_pages: set[int] = set()
|
||||
for toc_level, title, page in doc.get_toc():
|
||||
if toc_level != level or not title.strip():
|
||||
continue
|
||||
page = max(1, min(page, doc.page_count))
|
||||
# Several outline entries can point to the same page. A page cannot
|
||||
# be split without guessing from the extracted text, so keep its first
|
||||
# entry and let the following headings remain in that page's content.
|
||||
if page in seen_pages:
|
||||
continue
|
||||
seen_pages.add(page)
|
||||
sections.append({"title": title.strip(), "start_page": page})
|
||||
if sections:
|
||||
return sections
|
||||
|
||||
|
||||
def should_start_chapter(num: str, current: dict | None) -> bool:
|
||||
if current is None:
|
||||
return True
|
||||
if current.get("num") == "00":
|
||||
return True
|
||||
try:
|
||||
return int(num) > int(current["num"])
|
||||
except ValueError:
|
||||
return False
|
||||
# Keep PDFs without bookmarks usable. This fallback only recognizes
|
||||
# numbered headings and is deliberately limited to one boundary per page.
|
||||
for page_index, page in enumerate(doc, start=1):
|
||||
text = clean_page_text(page.get_text())
|
||||
if is_toc_page(text):
|
||||
continue
|
||||
for line in text.splitlines():
|
||||
match = NUMBERED_HEADING.match(line.strip())
|
||||
if not match:
|
||||
continue
|
||||
number, title = match.groups()
|
||||
if number.count(".") + 1 != level:
|
||||
continue
|
||||
title = title.strip()
|
||||
if not title or not title[0].isalpha() or TOC_LINE.search(title):
|
||||
continue
|
||||
if page_index not in seen_pages:
|
||||
seen_pages.add(page_index)
|
||||
sections.append({"title": title, "start_page": page_index})
|
||||
break
|
||||
return sections
|
||||
|
||||
|
||||
def convert(
|
||||
@@ -222,6 +239,7 @@ def convert(
|
||||
extract_tables: bool = False,
|
||||
page_dpi: int = DEFAULT_PAGE_DPI,
|
||||
single_file: bool = False,
|
||||
chapter_level: int = DEFAULT_CHAPTER_LEVEL,
|
||||
) -> None:
|
||||
if out_dir.exists():
|
||||
shutil.rmtree(out_dir)
|
||||
@@ -234,9 +252,8 @@ def convert(
|
||||
doc = fitz.open(pdf_path)
|
||||
stem = pdf_path.stem
|
||||
|
||||
sections: list[dict] = []
|
||||
current: dict | None = None
|
||||
page_png_count = 0
|
||||
page_contents: dict[int, str] = {}
|
||||
|
||||
with pdfplumber.open(pdf_path) as plumber:
|
||||
for page_index in range(doc.page_count):
|
||||
@@ -271,35 +288,44 @@ def convert(
|
||||
page_block.append("\n".join(f"" for p in img_refs))
|
||||
page_content = "\n\n".join(x for x in page_block if x)
|
||||
|
||||
section_hit = None
|
||||
source = text or raw
|
||||
for line in source.splitlines():
|
||||
hit = find_major_section(line)
|
||||
if hit:
|
||||
section_hit = hit
|
||||
page_contents[page_index + 1] = page_content
|
||||
|
||||
if section_hit and should_start_chapter(section_hit[0], current):
|
||||
num, title = section_hit
|
||||
if current:
|
||||
sections.append(current)
|
||||
current = {
|
||||
"num": num,
|
||||
"title": title,
|
||||
"slug": f"{int(num):02d}-{slugify(title)}",
|
||||
"pages": [],
|
||||
}
|
||||
|
||||
if current is None:
|
||||
current = {
|
||||
toc = toc_sections(doc, chapter_level)
|
||||
sections: list[dict] = []
|
||||
first_start = toc[0]["start_page"] if toc else doc.page_count + 1
|
||||
preamble_pages = [
|
||||
page_contents[page]
|
||||
for page in sorted(page_contents)
|
||||
if page < first_start
|
||||
]
|
||||
if preamble_pages:
|
||||
sections.append(
|
||||
{
|
||||
"num": "00",
|
||||
"title": "front-matter",
|
||||
"slug": "00-front-matter",
|
||||
"pages": [],
|
||||
"pages": preamble_pages,
|
||||
}
|
||||
current["pages"].append(page_content)
|
||||
|
||||
if current:
|
||||
sections.append(current)
|
||||
)
|
||||
for index, entry in enumerate(toc, start=1):
|
||||
next_start = (
|
||||
toc[index]["start_page"] if index < len(toc) else doc.page_count + 1
|
||||
)
|
||||
pages = [
|
||||
page_contents[page]
|
||||
for page in sorted(page_contents)
|
||||
if entry["start_page"] <= page < next_start
|
||||
]
|
||||
if not pages:
|
||||
continue
|
||||
sections.append(
|
||||
{
|
||||
"num": f"{index:02d}",
|
||||
"title": entry["title"],
|
||||
"slug": f"{index:02d}-{slugify(entry['title'])}",
|
||||
"pages": pages,
|
||||
}
|
||||
)
|
||||
doc.close()
|
||||
|
||||
if single_file:
|
||||
@@ -351,6 +377,16 @@ def main() -> None:
|
||||
action="store_true",
|
||||
help="write one {stem}.md instead of INDEX.md + chapters/",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--chapter-level",
|
||||
type=int,
|
||||
choices=(1, 2, 3),
|
||||
default=DEFAULT_CHAPTER_LEVEL,
|
||||
help=(
|
||||
"PDF outline level used for chapter boundaries "
|
||||
f"(default: {DEFAULT_CHAPTER_LEVEL})"
|
||||
),
|
||||
)
|
||||
args = parser.parse_args()
|
||||
out = args.output or Path("out") / args.pdf.stem
|
||||
convert(
|
||||
@@ -359,6 +395,7 @@ def main() -> None:
|
||||
extract_tables=args.tables,
|
||||
page_dpi=args.page_dpi,
|
||||
single_file=args.single_file,
|
||||
chapter_level=args.chapter_level,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user