Use five-digit chapter indices

Keep generated chapter filenames and displayed section numbers lexicographically ordered beyond 99 sections.

Co-authored-by: Cursor <cursoragent@cursor.com>

README.md

@@ -24,7 +24,7 @@ chmod +x run.sh
 out/RealTek-r8169/
 ├── INDEX.md              # оглавление со ссылками на главы
 ├── chapters/             # одна глава = один файл
-│   ├── 01-features.md
+│   ├── 00001-features.md
 │   └── ...
 └── assets/               # embedded PNG + page PNG для сломанных layout-страниц
 ```
@@ -36,13 +36,22 @@ out/RealTek-r8169/
 | Флаг | Описание |
 |------|----------|
 | `--single-file` | Один `{stem}.md` вместо `INDEX.md` + `chapters/` |
+| `--chapter-level N` | Уровень встроенного оглавления PDF для границ глав: `1`, `2` или `3` (default `2`) |
 | `--tables` | Включить извлечение таблиц pdfplumber (по умолчанию выключено) |
 | `--page-dpi N` | DPI для PNG-рендера страниц с битым текстом (default 150) |
 | `-o DIR` | Каталог вывода |
 
 ## Grep для агента
 
-Главы разбиты по крупным разделам документа. Каждая страница помечена HTML-комментарием:
+Главы разбиты по встроенному оглавлению PDF. Уровень нарезки можно выбрать:
+
+```bash
+./run.sh doc.pdf --chapter-level 1  # крупные разделы
+./run.sh doc.pdf --chapter-level 2  # подразделы, default
+./run.sh doc.pdf --chapter-level 3  # самые мелкие разделы
+```
+
+Каждая страница помечена HTML-комментарием:
 
 ```bash
 # найти упоминание регистра в главе 6

extract.py

@@ -16,13 +16,15 @@ HEADER_FOOTER = re.compile(
     r"^(RTL8169|2002/\d{2}/\d{2}|Rev\.\d+\.\d+|\d{1,3})$",
     re.MULTILINE,
 )
-MAJOR_SECTION = re.compile(r"^(\d{1,2})\.\s+([A-Z][^\n]{2,})$")
 TOC_LINE = re.compile(r"\.{4,}\s*\d+\s*$")
+NUMBERED_HEADING = re.compile(r"^(\d{1,2}(?:\.\d{1,2})*)\s+(.+)$")
 MIN_TABLE_ROWS = 2
 MIN_TABLE_CELLS = 6
 MIN_CELL_LEN_FOR_DEDUP = 4
 SPACED_LETTERS = re.compile(r"^([A-Z](?: [A-Z]){1,}|[A-Z])$")
 DEFAULT_PAGE_DPI = 150
+DEFAULT_CHAPTER_LEVEL = 2
+SECTION_INDEX_WIDTH = 5
 
 
 def slugify(title: str) -> str:
@@ -190,29 +192,64 @@ def render_page_png(
     return f"assets/{name}"
 
 
-def find_major_section(line: str) -> tuple[str, str] | None:
-    m = MAJOR_SECTION.match(line.strip())
-    if not m:
-        return None
-    num, title = m.group(1), m.group(2).strip()
-    if TOC_LINE.search(title) or "." * 3 in title:
-        return None
-    if title.endswith(":"):
-        return None
-    if len(title) > 70:
-        return None
-    return num, title
+def toc_sections(doc: fitz.Document, level: int) -> list[dict]:
+    """Return unique page boundaries and ancestor titles from the PDF outline."""
+    sections: list[dict] = []
+    seen_pages: set[int] = set()
+    parents: list[str] = []
+    for toc_level, title, page in doc.get_toc():
+        title = title.strip()
+        if not title:
+            continue
+        parents = parents[: toc_level - 1]
+        ancestors = parents.copy()
+        parents.append(title)
+        if toc_level != level or not title.strip():
+            continue
+        page = max(1, min(page, doc.page_count))
+        # Several outline entries can point to the same page.  A page cannot
+        # be split without guessing from the extracted text, so keep its first
+        # entry and let the following headings remain in that page's content.
+        if page in seen_pages:
+            continue
+        seen_pages.add(page)
+        sections.append(
+            {"title": title, "parents": ancestors, "start_page": page}
+        )
+    if sections:
+        return sections
 
-
-def should_start_chapter(num: str, current: dict | None) -> bool:
-    if current is None:
-        return True
-    if current.get("num") == "00":
-        return True
-    try:
-        return int(num) > int(current["num"])
-    except ValueError:
-        return False
+    # Keep PDFs without bookmarks usable.  This fallback only recognizes
+    # numbered headings and is deliberately limited to one boundary per page.
+    heading_path: list[str] = []
+    for page_index, page in enumerate(doc, start=1):
+        text = clean_page_text(page.get_text())
+        if is_toc_page(text):
+            continue
+        for line in text.splitlines():
+            match = NUMBERED_HEADING.match(line.strip())
+            if not match:
+                continue
+            number, title = match.groups()
+            heading_level = number.count(".") + 1
+            heading_path = heading_path[: heading_level - 1]
+            heading_path.append(title.strip())
+            if heading_level != level:
+                continue
+            title = title.strip()
+            if not title or not title[0].isalpha() or TOC_LINE.search(title):
+                continue
+            if page_index not in seen_pages:
+                seen_pages.add(page_index)
+                sections.append(
+                    {
+                        "title": title,
+                        "parents": heading_path[:-1],
+                        "start_page": page_index,
+                    }
+                )
+            break
+    return sections
 
 
 def convert(
@@ -222,6 +259,7 @@ def convert(
     extract_tables: bool = False,
     page_dpi: int = DEFAULT_PAGE_DPI,
     single_file: bool = False,
+    chapter_level: int = DEFAULT_CHAPTER_LEVEL,
 ) -> None:
     if out_dir.exists():
         shutil.rmtree(out_dir)
@@ -234,9 +272,8 @@ def convert(
     doc = fitz.open(pdf_path)
     stem = pdf_path.stem
 
-    sections: list[dict] = []
-    current: dict | None = None
     page_png_count = 0
+    page_contents: dict[int, str] = {}
 
     with pdfplumber.open(pdf_path) as plumber:
         for page_index in range(doc.page_count):
@@ -271,35 +308,47 @@ def convert(
                 page_block.append("\n".join(f"![figure]({p})" for p in img_refs))
             page_content = "\n\n".join(x for x in page_block if x)
 
-            section_hit = None
-            source = text or raw
-            for line in source.splitlines():
-                hit = find_major_section(line)
-                if hit:
-                    section_hit = hit
+            page_contents[page_index + 1] = page_content
 
-            if section_hit and should_start_chapter(section_hit[0], current):
-                num, title = section_hit
-                if current:
-                    sections.append(current)
-                current = {
-                    "num": num,
-                    "title": title,
-                    "slug": f"{int(num):02d}-{slugify(title)}",
-                    "pages": [],
-                }
-
-            if current is None:
-                current = {
-                    "num": "00",
-                    "title": "front-matter",
-                    "slug": "00-front-matter",
-                    "pages": [],
-                }
-            current["pages"].append(page_content)
-
-    if current:
-        sections.append(current)
+    toc = toc_sections(doc, chapter_level)
+    sections: list[dict] = []
+    first_start = toc[0]["start_page"] if toc else doc.page_count + 1
+    preamble_pages = [
+        page_contents[page]
+        for page in sorted(page_contents)
+        if page < first_start
+    ]
+    if preamble_pages:
+        sections.append(
+            {
+                "num": f"{0:0{SECTION_INDEX_WIDTH}d}",
+                "title": "front-matter",
+                "slug": f"{0:0{SECTION_INDEX_WIDTH}d}-front-matter",
+                "pages": preamble_pages,
+            }
+        )
+    for index, entry in enumerate(toc, start=1):
+        next_start = (
+            toc[index]["start_page"] if index < len(toc) else doc.page_count + 1
+        )
+        pages = [
+            page_contents[page]
+            for page in sorted(page_contents)
+            if entry["start_page"] <= page < next_start
+        ]
+        if not pages:
+            continue
+        sections.append(
+            {
+                "num": f"{index:0{SECTION_INDEX_WIDTH}d}",
+                "title": entry["title"],
+                "slug": (
+                    f"{index:0{SECTION_INDEX_WIDTH}d}-"
+                    f"{slugify(' - '.join((*entry['parents'], entry['title'])))}"
+                ),
+                "pages": pages,
+            }
+        )
     doc.close()
 
     if single_file:
@@ -351,6 +400,16 @@ def main() -> None:
         action="store_true",
         help="write one {stem}.md instead of INDEX.md + chapters/",
     )
+    parser.add_argument(
+        "--chapter-level",
+        type=int,
+        choices=(1, 2, 3),
+        default=DEFAULT_CHAPTER_LEVEL,
+        help=(
+            "PDF outline level used for chapter boundaries "
+            f"(default: {DEFAULT_CHAPTER_LEVEL})"
+        ),
+    )
     args = parser.parse_args()
     out = args.output or Path("out") / args.pdf.stem
     convert(
@@ -359,6 +418,7 @@ def main() -> None:
         extract_tables=args.tables,
         page_dpi=args.page_dpi,
         single_file=args.single_file,
+        chapter_level=args.chapter_level,
     )