diff --git a/README.md b/README.md index d37b029..463f521 100644 --- a/README.md +++ b/README.md @@ -36,13 +36,22 @@ out/RealTek-r8169/ | Флаг | Описание | |------|----------| | `--single-file` | Один `{stem}.md` вместо `INDEX.md` + `chapters/` | +| `--chapter-level N` | Уровень встроенного оглавления PDF для границ глав: `1`, `2` или `3` (default `2`) | | `--tables` | Включить извлечение таблиц pdfplumber (по умолчанию выключено) | | `--page-dpi N` | DPI для PNG-рендера страниц с битым текстом (default 150) | | `-o DIR` | Каталог вывода | ## Grep для агента -Главы разбиты по крупным разделам документа. Каждая страница помечена HTML-комментарием: +Главы разбиты по встроенному оглавлению PDF. Уровень нарезки можно выбрать: + +```bash +./run.sh doc.pdf --chapter-level 1 # крупные разделы +./run.sh doc.pdf --chapter-level 2 # подразделы, default +./run.sh doc.pdf --chapter-level 3 # самые мелкие разделы +``` + +Каждая страница помечена HTML-комментарием: ```bash # найти упоминание регистра в главе 6 diff --git a/extract.py b/extract.py index 417b45c..e76137f 100644 --- a/extract.py +++ b/extract.py @@ -16,13 +16,14 @@ HEADER_FOOTER = re.compile( r"^(RTL8169|2002/\d{2}/\d{2}|Rev\.\d+\.\d+|\d{1,3})$", re.MULTILINE, ) -MAJOR_SECTION = re.compile(r"^(\d{1,2})\.\s+([A-Z][^\n]{2,})$") TOC_LINE = re.compile(r"\.{4,}\s*\d+\s*$") +NUMBERED_HEADING = re.compile(r"^(\d{1,2}(?:\.\d{1,2})*)\s+(.+)$") MIN_TABLE_ROWS = 2 MIN_TABLE_CELLS = 6 MIN_CELL_LEN_FOR_DEDUP = 4 SPACED_LETTERS = re.compile(r"^([A-Z](?: [A-Z]){1,}|[A-Z])$") DEFAULT_PAGE_DPI = 150 +DEFAULT_CHAPTER_LEVEL = 2 def slugify(title: str) -> str: @@ -190,29 +191,45 @@ def render_page_png( return f"assets/{name}" -def find_major_section(line: str) -> tuple[str, str] | None: - m = MAJOR_SECTION.match(line.strip()) - if not m: - return None - num, title = m.group(1), m.group(2).strip() - if TOC_LINE.search(title) or "." * 3 in title: - return None - if title.endswith(":"): - return None - if len(title) > 70: - return None - return num, title +def toc_sections(doc: fitz.Document, level: int) -> list[dict]: + """Return unique page boundaries from the PDF outline at one level.""" + sections: list[dict] = [] + seen_pages: set[int] = set() + for toc_level, title, page in doc.get_toc(): + if toc_level != level or not title.strip(): + continue + page = max(1, min(page, doc.page_count)) + # Several outline entries can point to the same page. A page cannot + # be split without guessing from the extracted text, so keep its first + # entry and let the following headings remain in that page's content. + if page in seen_pages: + continue + seen_pages.add(page) + sections.append({"title": title.strip(), "start_page": page}) + if sections: + return sections - -def should_start_chapter(num: str, current: dict | None) -> bool: - if current is None: - return True - if current.get("num") == "00": - return True - try: - return int(num) > int(current["num"]) - except ValueError: - return False + # Keep PDFs without bookmarks usable. This fallback only recognizes + # numbered headings and is deliberately limited to one boundary per page. + for page_index, page in enumerate(doc, start=1): + text = clean_page_text(page.get_text()) + if is_toc_page(text): + continue + for line in text.splitlines(): + match = NUMBERED_HEADING.match(line.strip()) + if not match: + continue + number, title = match.groups() + if number.count(".") + 1 != level: + continue + title = title.strip() + if not title or not title[0].isalpha() or TOC_LINE.search(title): + continue + if page_index not in seen_pages: + seen_pages.add(page_index) + sections.append({"title": title, "start_page": page_index}) + break + return sections def convert( @@ -222,6 +239,7 @@ def convert( extract_tables: bool = False, page_dpi: int = DEFAULT_PAGE_DPI, single_file: bool = False, + chapter_level: int = DEFAULT_CHAPTER_LEVEL, ) -> None: if out_dir.exists(): shutil.rmtree(out_dir) @@ -234,9 +252,8 @@ def convert( doc = fitz.open(pdf_path) stem = pdf_path.stem - sections: list[dict] = [] - current: dict | None = None page_png_count = 0 + page_contents: dict[int, str] = {} with pdfplumber.open(pdf_path) as plumber: for page_index in range(doc.page_count): @@ -271,35 +288,44 @@ def convert( page_block.append("\n".join(f"![figure]({p})" for p in img_refs)) page_content = "\n\n".join(x for x in page_block if x) - section_hit = None - source = text or raw - for line in source.splitlines(): - hit = find_major_section(line) - if hit: - section_hit = hit + page_contents[page_index + 1] = page_content - if section_hit and should_start_chapter(section_hit[0], current): - num, title = section_hit - if current: - sections.append(current) - current = { - "num": num, - "title": title, - "slug": f"{int(num):02d}-{slugify(title)}", - "pages": [], - } - - if current is None: - current = { - "num": "00", - "title": "front-matter", - "slug": "00-front-matter", - "pages": [], - } - current["pages"].append(page_content) - - if current: - sections.append(current) + toc = toc_sections(doc, chapter_level) + sections: list[dict] = [] + first_start = toc[0]["start_page"] if toc else doc.page_count + 1 + preamble_pages = [ + page_contents[page] + for page in sorted(page_contents) + if page < first_start + ] + if preamble_pages: + sections.append( + { + "num": "00", + "title": "front-matter", + "slug": "00-front-matter", + "pages": preamble_pages, + } + ) + for index, entry in enumerate(toc, start=1): + next_start = ( + toc[index]["start_page"] if index < len(toc) else doc.page_count + 1 + ) + pages = [ + page_contents[page] + for page in sorted(page_contents) + if entry["start_page"] <= page < next_start + ] + if not pages: + continue + sections.append( + { + "num": f"{index:02d}", + "title": entry["title"], + "slug": f"{index:02d}-{slugify(entry['title'])}", + "pages": pages, + } + ) doc.close() if single_file: @@ -351,6 +377,16 @@ def main() -> None: action="store_true", help="write one {stem}.md instead of INDEX.md + chapters/", ) + parser.add_argument( + "--chapter-level", + type=int, + choices=(1, 2, 3), + default=DEFAULT_CHAPTER_LEVEL, + help=( + "PDF outline level used for chapter boundaries " + f"(default: {DEFAULT_CHAPTER_LEVEL})" + ), + ) args = parser.parse_args() out = args.output or Path("out") / args.pdf.stem convert( @@ -359,6 +395,7 @@ def main() -> None: extract_tables=args.tables, page_dpi=args.page_dpi, single_file=args.single_file, + chapter_level=args.chapter_level, )