From 5975d2b6e3eef0e6f1137259e3d1723e9e0afa35 Mon Sep 17 00:00:00 2001 From: Deep Kumar Singh Kushwah Date: Sun, 24 May 2026 23:26:10 +0530 Subject: [PATCH] fix(parsers): normalize CRLF/CR to LF in text & markdown parsers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #57 (Step 1.3) merged with a failing windows-latest job: test_parse_plain_text expected 2 paragraphs but saw 1 because the PlainTextParser's blank-line splitter (\n[ \t]*\n+) can't see across the \r between two CRLF-terminated lines, and the test wrote the sample with Path.write_text which emits CRLF on Windows. Root-cause fix in the parsers (not just the test) so any CRLF or bare-CR input — including content fetched from connectors on a Windows host or downloaded from Windows-authored sources — produces the same blocks on every platform. - PlainTextParser._decode now normalises \r\n and bare \r to \n after UTF-8/latin-1 decode, before the paragraph regex runs. - MarkdownParser.parse does the same after decode so markdown-it's line-based Token.map offsets stay in sync with the text we hand back. - Two new tests in tests/parsers/test_text_family.py exercise CRLF + bare-CR inputs via write_bytes so they're platform-independent. - packages/ragctl/tests/test_parse.py switched to write_bytes(b"...\n...") for the three text fixtures — defence in depth against platform newline translation in the integration-style CLI tests. Other parsers (HTML / JSON / YAML / CSV / PDF / DOCX / PPTX / XLSX) use libraries that handle CRLF natively, so no changes there. Co-Authored-By: Claude Opus 4.7 (1M context) --- packages/parsers/src/rag_parsers/markdown.py | 2 + packages/parsers/src/rag_parsers/text.py | 10 ++++- packages/ragctl/tests/test_parse.py | 7 +-- tests/parsers/test_text_family.py | 46 ++++++++++++++++++++ 4 files changed, 60 insertions(+), 5 deletions(-) diff --git a/packages/parsers/src/rag_parsers/markdown.py b/packages/parsers/src/rag_parsers/markdown.py index ab66e72..c3fbfab 100644 --- a/packages/parsers/src/rag_parsers/markdown.py +++ b/packages/parsers/src/rag_parsers/markdown.py @@ -51,6 +51,8 @@ async def parse( except UnicodeDecodeError as exc: raise ParseError(f"Markdown bytes are not valid UTF-8: {exc}") from exc + text = text.replace("\r\n", "\n").replace("\r", "\n") + if not text.strip(): return empty_parsed(ctx, data, mime_type, document_id, corpus_id, source_uri) diff --git a/packages/parsers/src/rag_parsers/text.py b/packages/parsers/src/rag_parsers/text.py index dfa0e0d..9dfc3b5 100644 --- a/packages/parsers/src/rag_parsers/text.py +++ b/packages/parsers/src/rag_parsers/text.py @@ -65,12 +65,18 @@ def _decode(data: bytes) -> str: if data.startswith(b"\xef\xbb\xbf"): data = data[3:] try: - return data.decode("utf-8") + text = data.decode("utf-8") except UnicodeDecodeError as exc: try: - return data.decode("latin-1") + text = data.decode("latin-1") except UnicodeDecodeError: raise ParseError(f"Unable to decode text bytes: {exc}") from exc + return _normalize_newlines(text) + + +def _normalize_newlines(text: str) -> str: + """Collapse CRLF and bare CR to LF so paragraph splitters are platform-stable.""" + return text.replace("\r\n", "\n").replace("\r", "\n") def _split_paragraphs(text: str) -> list[Block]: diff --git a/packages/ragctl/tests/test_parse.py b/packages/ragctl/tests/test_parse.py index be2ac35..77d18e0 100644 --- a/packages/ragctl/tests/test_parse.py +++ b/packages/ragctl/tests/test_parse.py @@ -12,7 +12,8 @@ def test_parse_plain_text(tmp_path: Path) -> None: sample = tmp_path / "hello.txt" - sample.write_text("first paragraph.\n\nsecond paragraph.\n") + # write_bytes (not write_text) so line endings stay LF on every platform. + sample.write_bytes(b"first paragraph.\n\nsecond paragraph.\n") result = runner.invoke(app, ["parse", str(sample)]) assert result.exit_code == 0, result.output assert "PlainTextParser" in result.output @@ -22,7 +23,7 @@ def test_parse_plain_text(tmp_path: Path) -> None: def test_parse_markdown_blocks_preview(tmp_path: Path) -> None: sample = tmp_path / "doc.md" - sample.write_text("# Heading\n\nA paragraph.\n\n- item\n") + sample.write_bytes(b"# Heading\n\nA paragraph.\n\n- item\n") result = runner.invoke(app, ["parse", str(sample), "-n", "3"]) assert result.exit_code == 0, result.output assert "MarkdownParser" in result.output @@ -33,7 +34,7 @@ def test_parse_markdown_blocks_preview(tmp_path: Path) -> None: def test_parse_mime_override(tmp_path: Path) -> None: # File extension says .txt; --mime forces JSON parsing of a JSON body. sample = tmp_path / "data.txt" - sample.write_text('{"a": 1}') + sample.write_bytes(b'{"a": 1}') result = runner.invoke(app, ["parse", str(sample), "--mime", "application/json"]) assert result.exit_code == 0, result.output assert "JsonParser" in result.output diff --git a/tests/parsers/test_text_family.py b/tests/parsers/test_text_family.py index da7a2cd..9341b06 100644 --- a/tests/parsers/test_text_family.py +++ b/tests/parsers/test_text_family.py @@ -67,6 +67,36 @@ async def test_plain_text_empty( assert parsed.blocks == [] +async def test_plain_text_crlf_paragraph_split( + ctx: RequestContext, corpus_id: CorpusId, document_id: DocumentId +) -> None: + # CRLF-encoded input (as produced by Windows text-mode writes) must split on + # blank lines the same way LF input does — otherwise paragraph detection is + # platform-dependent. + body = b"first paragraph.\r\n\r\nsecond paragraph.\r\n" + parsed = await _parse(PlainTextParser(), ctx, "text/plain", body, document_id, corpus_id) + assert "\r" not in parsed.text + assert len(parsed.blocks) == 2 + assert parsed.blocks[0].text.startswith("first paragraph") + assert parsed.blocks[1].text.startswith("second paragraph") + + +async def test_plain_text_bare_cr_normalized( + ctx: RequestContext, corpus_id: CorpusId, document_id: DocumentId +) -> None: + # Legacy classic-Mac files use bare CR. Treat them as line separators too. + parsed = await _parse( + PlainTextParser(), + ctx, + "text/plain", + b"a\r\rb", + document_id, + corpus_id, + ) + assert "\r" not in parsed.text + assert len(parsed.blocks) == 2 + + # --------------------------------------------------------------------------- # Markdown # --------------------------------------------------------------------------- @@ -104,6 +134,22 @@ async def test_markdown_invalid_utf8( await _parse(MarkdownParser(), ctx, "text/markdown", b"\xff\xfe", document_id, corpus_id) +async def test_markdown_crlf_input( + ctx: RequestContext, corpus_id: CorpusId, document_id: DocumentId +) -> None: + # Markdown sources saved on Windows arrive with CRLF; ensure heading/paragraph + # detection still works after normalization. + src = b"# Heading\r\n\r\nA paragraph.\r\n\r\n- item\r\n" + parsed = await _parse(MarkdownParser(), ctx, "text/markdown", src, document_id, corpus_id) + assert "\r" not in parsed.text + by_type: dict[BlockType, list] = {} + for block in parsed.blocks: + by_type.setdefault(block.type, []).append(block) + assert by_type[BlockType.heading][0].text == "Heading" + assert any(b.type == BlockType.paragraph for b in parsed.blocks) + assert any(b.type == BlockType.list_item for b in parsed.blocks) + + # --------------------------------------------------------------------------- # JSON # ---------------------------------------------------------------------------