From 5fe3e451af44cc5660c2920e1dab35c5a1477264 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dapeng=20Sun=28=E5=AD=99=E5=A4=A7=E9=B9=8F=29?= Date: Sat, 1 Aug 2026 02:16:25 +0800 Subject: [PATCH] fix(table): keep every split of a format table under a positive LIMIT A format table has no manifest, so scan planning does not know how many rows a data file holds. apply_limit_pushdown kept only the first `limit` splits anyway, which reads the number of files as if it were a number of rows. An empty data file is enough to break it. An engine that writes one file per task leaves one behind whenever a task matched nothing, so a table of three rows whose first file is empty answers `SELECT id FROM t LIMIT 1` with zero rows, and reports success. Keep every split for a positive limit and let execution stop when it has enough rows. A limit of zero still needs no split. This is what Java does: FormatTableScan.FormatTableScanPlan.splits returns an empty list only for a non-positive limit, with the same reason in a comment. --- .../datafusion/tests/format_table_statistics.rs | 14 ++++++++++++++ crates/paimon/src/table/format_table_scan.rs | 6 +++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/crates/integrations/datafusion/tests/format_table_statistics.rs b/crates/integrations/datafusion/tests/format_table_statistics.rs index c5505fb12..b29ae0f1e 100644 --- a/crates/integrations/datafusion/tests/format_table_statistics.rs +++ b/crates/integrations/datafusion/tests/format_table_statistics.rs @@ -197,3 +197,17 @@ async fn test_format_table_with_empty_file_counts_zero() { 0 ); } + +/// A positive `LIMIT` must not be answered from a file count. A format table has no +/// row counts, so keeping only the first `limit` files is a guess, and an empty data +/// file — ordinary output from an engine that writes one file per task — makes the +/// guess wrong. +#[tokio::test] +async fn test_format_table_limit_is_not_answered_from_a_file_count() { + let (_tmp, context, table_dir) = setup_format_table().await; + write_parquet(&table_dir.join("part-0.parquet"), &[]); + write_parquet(&table_dir.join("part-1.parquet"), &[1, 2, 3]); + + let scanned = scanned_rows(&context, "SELECT id FROM paimon.test_db.events LIMIT 1").await; + assert_eq!(scanned, 1, "LIMIT 1 over 3 rows must return a row"); +} diff --git a/crates/paimon/src/table/format_table_scan.rs b/crates/paimon/src/table/format_table_scan.rs index df2a48e02..4190526aa 100644 --- a/crates/paimon/src/table/format_table_scan.rs +++ b/crates/paimon/src/table/format_table_scan.rs @@ -272,13 +272,17 @@ impl<'a> FormatTableScan<'a> { } } + /// A limit of zero needs no split at all. A positive one keeps every split: a format + /// table has no row counts, so the number of files says nothing about the number of + /// rows, and dropping files would answer the query from a guess. + /// + /// Mirrors Java `FormatTableScan.FormatTableScanPlan.splits`. pub(crate) fn apply_limit_pushdown( &self, splits: Vec, ) -> Vec { match self.limit { Some(0) => Vec::new(), - Some(limit) if splits.len() > limit => splits.into_iter().take(limit).collect(), _ => splits, } }