diff --git a/packages/parsers/src/rag_parsers/markdown.py b/packages/parsers/src/rag_parsers/markdown.py index ab66e72..c3fbfab 100644 --- a/packages/parsers/src/rag_parsers/markdown.py +++ b/packages/parsers/src/rag_parsers/markdown.py @@ -51,6 +51,8 @@ async def parse( except UnicodeDecodeError as exc: raise ParseError(f"Markdown bytes are not valid UTF-8: {exc}") from exc + text = text.replace("\r\n", "\n").replace("\r", "\n") + if not text.strip(): return empty_parsed(ctx, data, mime_type, document_id, corpus_id, source_uri) diff --git a/packages/parsers/src/rag_parsers/text.py b/packages/parsers/src/rag_parsers/text.py index dfa0e0d..9dfc3b5 100644 --- a/packages/parsers/src/rag_parsers/text.py +++ b/packages/parsers/src/rag_parsers/text.py @@ -65,12 +65,18 @@ def _decode(data: bytes) -> str: if data.startswith(b"\xef\xbb\xbf"): data = data[3:] try: - return data.decode("utf-8") + text = data.decode("utf-8") except UnicodeDecodeError as exc: try: - return data.decode("latin-1") + text = data.decode("latin-1") except UnicodeDecodeError: raise ParseError(f"Unable to decode text bytes: {exc}") from exc + return _normalize_newlines(text) + + +def _normalize_newlines(text: str) -> str: + """Collapse CRLF and bare CR to LF so paragraph splitters are platform-stable.""" + return text.replace("\r\n", "\n").replace("\r", "\n") def _split_paragraphs(text: str) -> list[Block]: diff --git a/packages/ragctl/tests/test_parse.py b/packages/ragctl/tests/test_parse.py index be2ac35..77d18e0 100644 --- a/packages/ragctl/tests/test_parse.py +++ b/packages/ragctl/tests/test_parse.py @@ -12,7 +12,8 @@ def test_parse_plain_text(tmp_path: Path) -> None: sample = tmp_path / "hello.txt" - sample.write_text("first paragraph.\n\nsecond paragraph.\n") + # write_bytes (not write_text) so line endings stay LF on every platform. + sample.write_bytes(b"first paragraph.\n\nsecond paragraph.\n") result = runner.invoke(app, ["parse", str(sample)]) assert result.exit_code == 0, result.output assert "PlainTextParser" in result.output @@ -22,7 +23,7 @@ def test_parse_plain_text(tmp_path: Path) -> None: def test_parse_markdown_blocks_preview(tmp_path: Path) -> None: sample = tmp_path / "doc.md" - sample.write_text("# Heading\n\nA paragraph.\n\n- item\n") + sample.write_bytes(b"# Heading\n\nA paragraph.\n\n- item\n") result = runner.invoke(app, ["parse", str(sample), "-n", "3"]) assert result.exit_code == 0, result.output assert "MarkdownParser" in result.output @@ -33,7 +34,7 @@ def test_parse_markdown_blocks_preview(tmp_path: Path) -> None: def test_parse_mime_override(tmp_path: Path) -> None: # File extension says .txt; --mime forces JSON parsing of a JSON body. sample = tmp_path / "data.txt" - sample.write_text('{"a": 1}') + sample.write_bytes(b'{"a": 1}') result = runner.invoke(app, ["parse", str(sample), "--mime", "application/json"]) assert result.exit_code == 0, result.output assert "JsonParser" in result.output diff --git a/tests/parsers/test_text_family.py b/tests/parsers/test_text_family.py index da7a2cd..9341b06 100644 --- a/tests/parsers/test_text_family.py +++ b/tests/parsers/test_text_family.py @@ -67,6 +67,36 @@ async def test_plain_text_empty( assert parsed.blocks == [] +async def test_plain_text_crlf_paragraph_split( + ctx: RequestContext, corpus_id: CorpusId, document_id: DocumentId +) -> None: + # CRLF-encoded input (as produced by Windows text-mode writes) must split on + # blank lines the same way LF input does — otherwise paragraph detection is + # platform-dependent. + body = b"first paragraph.\r\n\r\nsecond paragraph.\r\n" + parsed = await _parse(PlainTextParser(), ctx, "text/plain", body, document_id, corpus_id) + assert "\r" not in parsed.text + assert len(parsed.blocks) == 2 + assert parsed.blocks[0].text.startswith("first paragraph") + assert parsed.blocks[1].text.startswith("second paragraph") + + +async def test_plain_text_bare_cr_normalized( + ctx: RequestContext, corpus_id: CorpusId, document_id: DocumentId +) -> None: + # Legacy classic-Mac files use bare CR. Treat them as line separators too. + parsed = await _parse( + PlainTextParser(), + ctx, + "text/plain", + b"a\r\rb", + document_id, + corpus_id, + ) + assert "\r" not in parsed.text + assert len(parsed.blocks) == 2 + + # --------------------------------------------------------------------------- # Markdown # --------------------------------------------------------------------------- @@ -104,6 +134,22 @@ async def test_markdown_invalid_utf8( await _parse(MarkdownParser(), ctx, "text/markdown", b"\xff\xfe", document_id, corpus_id) +async def test_markdown_crlf_input( + ctx: RequestContext, corpus_id: CorpusId, document_id: DocumentId +) -> None: + # Markdown sources saved on Windows arrive with CRLF; ensure heading/paragraph + # detection still works after normalization. + src = b"# Heading\r\n\r\nA paragraph.\r\n\r\n- item\r\n" + parsed = await _parse(MarkdownParser(), ctx, "text/markdown", src, document_id, corpus_id) + assert "\r" not in parsed.text + by_type: dict[BlockType, list] = {} + for block in parsed.blocks: + by_type.setdefault(block.type, []).append(block) + assert by_type[BlockType.heading][0].text == "Heading" + assert any(b.type == BlockType.paragraph for b in parsed.blocks) + assert any(b.type == BlockType.list_item for b in parsed.blocks) + + # --------------------------------------------------------------------------- # JSON # ---------------------------------------------------------------------------