From 11341f99409a534fb1971c69bdf6be20614b5e51 Mon Sep 17 00:00:00 2001 From: rsasaki0109 Date: Fri, 7 Aug 2026 19:52:53 +0900 Subject: [PATCH] Add real AI semantic meaning via ONNX entity embeddings (Epic 150) Add OnnxEntityEmbedder to spatialrust-semantic behind a model feature: it runs entity feature vectors through an already-open model session with explicit copy policy and produces an Embedding for the semantic search index. Wire the facade semantic-model feature and add an integration test that drives a committed 134-byte ONNX Add fixture through the real ONNX Runtime path (feature -> session -> embedding -> search). --- CHANGELOG.md | 8 + crates/spatialrust-semantic/Cargo.toml | 3 + crates/spatialrust-semantic/src/lib.rs | 4 + crates/spatialrust-semantic/src/model.rs | 261 ++++++++++++++++++ crates/spatialrust/Cargo.toml | 11 + .../tests/fixtures/double_dynamic.onnx | Bin 0 -> 134 bytes crates/spatialrust/tests/onnx_semantic.rs | 81 ++++++ docs/FEATURE_MATRIX.md | 1 + docs/ROADMAP.md | 22 ++ notes/2026-08-07_epic150_ai_semantic.md | 48 ++++ 10 files changed, 439 insertions(+) create mode 100644 crates/spatialrust-semantic/src/model.rs create mode 100644 crates/spatialrust/tests/fixtures/double_dynamic.onnx create mode 100644 crates/spatialrust/tests/onnx_semantic.rs create mode 100644 notes/2026-08-07_epic150_ai_semantic.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 79c8509..890eb1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,14 @@ removed no sooner than the next major (see `docs/API_STABILITY.md`). ### Added +- **Epic 150 real AI semantic meaning** (`semantic-model`): `OnnxEntityEmbedder` + runs point-entity feature vectors through an already-open ONNX model session + with explicit copy policy and produces an `Embedding` ready for the existing + `SemanticSearchIndex`. The embedder never loads a model or picks a backend, + keeping device placement and transfer semantics explicit. A committed + `double_dynamic.onnx` fixture drives the facade integration test through the + real ONNX Runtime path (`feature → session → embedding → semantic search`), + and the `onnx_semantic` test requires `ai-onnxruntime`. - **Epic 149 Arrow canonical interchange in Python**: `PyPointCloud` exposes `__arrow_c_array__`, returning `(schema, array)` Arrow C Data capsules so PyArrow/pandas consume SpatialRust clouds zero-copy, and `PyPointCloudStream` diff --git a/crates/spatialrust-semantic/Cargo.toml b/crates/spatialrust-semantic/Cargo.toml index 91b522b..d6d7730 100644 --- a/crates/spatialrust-semantic/Cargo.toml +++ b/crates/spatialrust-semantic/Cargo.toml @@ -10,9 +10,12 @@ description = "Semantic entities, embeddings, and multimodal fusion/search for S [features] default = [] +model = ["dep:spatialrust-ai", "dep:spatialrust-tensor"] [dependencies] spatialrust-core.workspace = true spatialrust-math.workspace = true spatialrust-records.workspace = true +spatialrust-ai = { workspace = true, optional = true } +spatialrust-tensor = { workspace = true, optional = true } thiserror.workspace = true diff --git a/crates/spatialrust-semantic/src/lib.rs b/crates/spatialrust-semantic/src/lib.rs index 2c40289..e4d2f30 100644 --- a/crates/spatialrust-semantic/src/lib.rs +++ b/crates/spatialrust-semantic/src/lib.rs @@ -6,9 +6,13 @@ mod embedding; mod entity; mod error; +#[cfg(feature = "model")] +mod model; mod search; pub use embedding::{cosine_similarity, Embedding}; pub use entity::{record_entity_id, EntityId, OpenVocabLabel, SemanticEntity, SpatialRecordEntity}; pub use error::{SemanticError, SemanticResult}; +#[cfg(feature = "model")] +pub use model::OnnxEntityEmbedder; pub use search::{FusionScore, MultimodalFusion, SemanticSearchIndex}; diff --git a/crates/spatialrust-semantic/src/model.rs b/crates/spatialrust-semantic/src/model.rs new file mode 100644 index 0000000..06b7edf --- /dev/null +++ b/crates/spatialrust-semantic/src/model.rs @@ -0,0 +1,261 @@ +//! Real-model entity embedding through an explicit model session. + +use spatialrust_ai::{CopyPolicy, ModelSession, NamedTensors, RunOptions}; +use spatialrust_tensor::{Device, TensorBuffer, TensorDescriptor}; + +use crate::{Embedding, SemanticError, SemanticResult}; + +/// Runs entity features through an already-open model session to produce an +/// [`Embedding`]. +/// +/// The embedder never loads a model, chooses a backend, or moves data across a +/// device boundary: the caller supplies an open session, names the input and +/// output tensors, and selects the copy policy. This keeps backend identity and +/// transfer semantics explicit and auditable. +#[derive(Clone, Debug)] +pub struct OnnxEntityEmbedder { + input_name: String, + output_name: String, + /// Input tensor descriptor (feature shape). + input_descriptor: TensorDescriptor, + /// Output tensor descriptor (embedding shape). + output_descriptor: TensorDescriptor, + copy_policy: CopyPolicy, +} + +impl OnnxEntityEmbedder { + /// Creates an embedder for one model input/output pair. + pub fn try_new( + input_name: impl Into, + output_name: impl Into, + input_descriptor: TensorDescriptor, + output_descriptor: TensorDescriptor, + copy_policy: CopyPolicy, + ) -> SemanticResult { + if input_descriptor.device() != Device::CPU || output_descriptor.device() != Device::CPU { + return Err(SemanticError::InvalidConfiguration( + "entity embedder requires CPU-hosted input and output tensors".into(), + )); + } + if output_descriptor.shape().is_empty() || output_descriptor.shape().last() == Some(&0) { + return Err(SemanticError::InvalidConfiguration( + "embedding output must have a non-empty trailing dimension".into(), + )); + } + Ok(Self { + input_name: input_name.into(), + output_name: output_name.into(), + input_descriptor, + output_descriptor, + copy_policy, + }) + } + + /// Embeds one feature vector (flattened `f32` values) into an embedding. + /// + /// `features` must contain exactly the element count implied by + /// `input_descriptor`. The session's `output_name` tensor must match + /// `output_descriptor`. + pub fn embed_one( + &self, + session: &mut dyn ModelSession, + features: &[f32], + ) -> SemanticResult { + let expected = self.input_descriptor.element_count().map_err(|error| { + SemanticError::InvalidConfiguration(format!("input descriptor: {error}")) + })?; + if features.len() != expected { + return Err(SemanticError::InvalidConfiguration(format!( + "entity features have {} elements; expected {}", + features.len(), + expected + ))); + } + let bytes = features_to_bytes(features)?; + let tensor = TensorBuffer::try_new(bytes, self.input_descriptor.clone()) + .map_err(|error| SemanticError::InvalidConfiguration(error.to_string()))?; + let mut inputs = NamedTensors::new(); + inputs + .insert(self.input_name.clone(), tensor) + .map_err(|error| SemanticError::InvalidConfiguration(error.to_string()))?; + + let options = RunOptions { input_copy: self.copy_policy, output_copy: self.copy_policy }; + let outputs = session + .run_with_options(inputs, options) + .map_err(|error| SemanticError::InvalidConfiguration(error.to_string()))?; + let output = outputs.get(&self.output_name).ok_or_else(|| { + SemanticError::InvalidConfiguration(format!( + "model output `{}` not found", + self.output_name + )) + })?; + + let expected_output = self.output_descriptor.element_count().map_err(|error| { + SemanticError::InvalidConfiguration(format!("output descriptor: {error}")) + })?; + let output_shape = output.descriptor().shape(); + if output_shape.iter().product::() != expected_output { + return Err(SemanticError::InvalidConfiguration(format!( + "model output `{}` has shape {output_shape:?}; expected {expected_output} elements", + self.output_name + ))); + } + let bytes = output.allocation_bytes(); + if bytes.len() != expected_output * 4 { + return Err(SemanticError::InvalidConfiguration(format!( + "model output `{}` has {} bytes; expected {}", + self.output_name, + bytes.len(), + expected_output * 4 + ))); + } + let mut values = Vec::with_capacity(expected_output); + for chunk in bytes.chunks_exact(4) { + values.push(f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]])); + } + Embedding::try_new(values) + } + + /// Returns the configured input descriptor. + #[must_use] + pub fn input_descriptor(&self) -> &TensorDescriptor { + &self.input_descriptor + } + + /// Returns the configured output descriptor. + #[must_use] + pub fn output_descriptor(&self) -> &TensorDescriptor { + &self.output_descriptor + } +} + +fn features_to_bytes(features: &[f32]) -> SemanticResult> { + let mut bytes = Vec::with_capacity(features.len() * 4); + for value in features { + if !value.is_finite() { + return Err(SemanticError::InvalidConfiguration( + "entity features must contain finite values".into(), + )); + } + bytes.extend_from_slice(&value.to_le_bytes()); + } + Ok(bytes) +} + +#[cfg(test)] +mod tests { + use super::OnnxEntityEmbedder; + use crate::Embedding; + use spatialrust_ai::{CopyPolicy, ModelInfo, ModelSession, NamedTensors, RunOptions}; + use spatialrust_tensor::{DataType, Device, TensorDescriptor}; + + #[derive(Clone, Debug)] + struct IdentitySession { + info: ModelInfo, + } + + impl Default for IdentitySession { + fn default() -> Self { + Self { info: ModelInfo { name: None, inputs: Vec::new(), outputs: Vec::new() } } + } + } + + impl ModelSession for IdentitySession { + fn backend_name(&self) -> &str { + "test-identity" + } + + fn model_info(&self) -> &ModelInfo { + &self.info + } + + fn run_with_options( + &mut self, + inputs: NamedTensors, + _options: RunOptions, + ) -> spatialrust_ai::AiResult { + let mut outputs = NamedTensors::new(); + for (name, tensor) in inputs.into_values() { + let output_name = if name == "input" { "output".to_owned() } else { name }; + outputs.insert(output_name, tensor)?; + } + Ok(outputs) + } + } + + fn descriptor(shape: &[usize]) -> TensorDescriptor { + TensorDescriptor::contiguous(DataType::F32, shape.to_vec(), Device::CPU) + } + + #[test] + fn embeds_identity_features() { + let embedder = OnnxEntityEmbedder::try_new( + "input", + "output", + descriptor(&[1, 4]), + descriptor(&[1, 4]), + CopyPolicy::Allow, + ) + .unwrap(); + let mut session = IdentitySession::default(); + let embedding = embedder.embed_one(&mut session, &[0.1, 0.2, 0.3, 0.4]).unwrap(); + assert_eq!(embedding, Embedding::try_new(vec![0.1, 0.2, 0.3, 0.4]).unwrap()); + } + + #[test] + fn rejects_feature_count_mismatch() { + let embedder = OnnxEntityEmbedder::try_new( + "input", + "output", + descriptor(&[1, 4]), + descriptor(&[1, 4]), + CopyPolicy::Allow, + ) + .unwrap(); + let mut session = IdentitySession::default(); + assert!(embedder.embed_one(&mut session, &[0.1, 0.2]).is_err()); + } + + #[test] + fn rejects_non_finite_features() { + let embedder = OnnxEntityEmbedder::try_new( + "input", + "output", + descriptor(&[1, 3]), + descriptor(&[1, 3]), + CopyPolicy::Allow, + ) + .unwrap(); + let mut session = IdentitySession::default(); + assert!(embedder.embed_one(&mut session, &[f32::NAN, 0.0, 0.0]).is_err()); + } + + #[test] + fn rejects_device_mismatch() { + let gpu = TensorDescriptor::contiguous( + DataType::F32, + vec![1, 3], + Device { kind: spatialrust_tensor::DeviceKind::Cuda, id: 0 }, + ); + assert!(OnnxEntityEmbedder::try_new( + "input", + "output", + gpu, + descriptor(&[1, 3]), + CopyPolicy::Allow, + ) + .is_err()); + } + + #[test] + fn rejects_zero_dim_output() { + assert!(OnnxEntityEmbedder::try_new( + "input", + "output", + descriptor(&[1, 3]), + descriptor(&[0]), + CopyPolicy::Allow, + ) + .is_err()); + } +} diff --git a/crates/spatialrust/Cargo.toml b/crates/spatialrust/Cargo.toml index 7bceaa5..3e16eac 100644 --- a/crates/spatialrust/Cargo.toml +++ b/crates/spatialrust/Cargo.toml @@ -218,6 +218,12 @@ mapping-scan-icp = ["mapping", "spatialrust-mapping/scan-icp"] scene = ["dep:spatialrust-scene"] scene-gaussian = ["scene", "spatialrust-scene/gaussian"] semantic = ["records", "dep:spatialrust-semantic"] +semantic-model = [ + "semantic", + "spatialrust-semantic/model", + "ai", + "tensor", +] episode = ["sync", "dep:spatialrust-episode"] runtime = ["sync", "dep:spatialrust-runtime"] runtime-graph = ["runtime", "distribute", "spatialrust-runtime/execution-graph"] @@ -350,6 +356,11 @@ name = "tiles3d_smoke" path = "tests/tiles3d_smoke.rs" required-features = ["interchange-tiles3d", "io-pcd", "interchange-tiles3d-copc", "io-copc"] +[[test]] +name = "onnx_semantic" +path = "tests/onnx_semantic.rs" +required-features = ["ai-onnxruntime", "semantic-model"] + [[example]] name = "north_star_demo" path = "examples/north_star_demo.rs" diff --git a/crates/spatialrust/tests/fixtures/double_dynamic.onnx b/crates/spatialrust/tests/fixtures/double_dynamic.onnx new file mode 100644 index 0000000000000000000000000000000000000000..ae6367fc14f10d5c99734288b5ae5417afd3701e GIT binary patch literal 134 zcmd5H@Ki7A^(>UI08U BA*}!a literal 0 HcmV?d00001 diff --git a/crates/spatialrust/tests/onnx_semantic.rs b/crates/spatialrust/tests/onnx_semantic.rs new file mode 100644 index 0000000..e0a4fed --- /dev/null +++ b/crates/spatialrust/tests/onnx_semantic.rs @@ -0,0 +1,81 @@ +//! Real ONNX model embedding through `OnnxEntityEmbedder` (Epic 150B). +//! +//! The committed `double_dynamic.onnx` fixture maps `input` [1,3] → `output` +//! [1,3] by adding the input to itself (doubling). This proves the full +//! path: feature vector → ONNX session → embedding, without external weights. + +#![cfg(all(feature = "ai-onnxruntime", feature = "semantic-model"))] + +use std::sync::Arc; + +use spatialrust::ai::{ + CopyPolicy, InferenceBackend, ModelSource, OnnxRuntimeBackend, SessionOptions, +}; +use spatialrust::semantic::OnnxEntityEmbedder; +use spatialrust::tensor::{DataType, Device, TensorDescriptor}; + +#[test] +fn onnx_embedder_runs_real_model_and_round_trips_embedding() { + let model_bytes: &[u8] = include_bytes!("fixtures/double_dynamic.onnx"); + let backend = OnnxRuntimeBackend; + let mut session = backend + .create_session(&ModelSource::Bytes(Arc::from(model_bytes)), &SessionOptions::default()) + .expect("open ONNX fixture"); + + let descriptor = + |shape: &[usize]| TensorDescriptor::contiguous(DataType::F32, shape.to_vec(), Device::CPU); + let embedder = OnnxEntityEmbedder::try_new( + "input", + "output", + descriptor(&[1, 3]), + descriptor(&[1, 3]), + CopyPolicy::Allow, + ) + .expect("embedder"); + + // The fixture doubles its input, so [1,2,3] → [2,4,6]. + let embedding = embedder.embed_one(session.as_mut(), &[1.0, 2.0, 3.0]).expect("embed"); + assert_eq!(embedding.dim(), 3); + assert_eq!(embedding.as_slice(), &[2.0, 4.0, 6.0]); + + // Search integration: embedding feeds the semantic search index. + let mut index = spatialrust::semantic::SemanticSearchIndex::new(); + index.insert(spatialrust::semantic::SemanticEntity { + id: spatialrust::semantic::EntityId::new("entity-a"), + centroid: None, + labels: vec![spatialrust::semantic::OpenVocabLabel { + text: "doubler".into(), + confidence: 1.0, + }], + embedding: Some(embedding), + }); + + // Query with an embedding close to [2,4,6] returns the indexed entity. + let query = spatialrust::semantic::Embedding::try_new(vec![2.0, 4.0, 6.0]).unwrap(); + let results = + index.search(&query, spatialrust::semantic::MultimodalFusion::default(), 1).unwrap(); + assert_eq!(results[0].0, spatialrust::semantic::EntityId::new("entity-a")); +} + +#[test] +fn onnx_embedder_rejects_feature_shape_mismatch() { + let model_bytes: &[u8] = include_bytes!("fixtures/double_dynamic.onnx"); + let backend = OnnxRuntimeBackend; + let mut session = backend + .create_session(&ModelSource::Bytes(Arc::from(model_bytes)), &SessionOptions::default()) + .expect("open ONNX fixture"); + + let descriptor = + |shape: &[usize]| TensorDescriptor::contiguous(DataType::F32, shape.to_vec(), Device::CPU); + let embedder = OnnxEntityEmbedder::try_new( + "input", + "output", + descriptor(&[1, 3]), + descriptor(&[1, 3]), + CopyPolicy::Allow, + ) + .expect("embedder"); + + // Only two features supplied but the model expects three. + assert!(embedder.embed_one(session.as_mut(), &[1.0, 2.0]).is_err()); +} diff --git a/docs/FEATURE_MATRIX.md b/docs/FEATURE_MATRIX.md index 8e18e7b..86b6444 100644 --- a/docs/FEATURE_MATRIX.md +++ b/docs/FEATURE_MATRIX.md @@ -30,6 +30,7 @@ workspace because its build requires a Python toolchain. | `spatialrust-io` | no format enabled by default | PCD, PLY, LAS/LAZ, E57, COPC, HTTP COPC, explicit roots/manifests, per-node `CopcNodeReader` hierarchy walk | format and checksum crates are optional | | `spatialrust-ros2` | ROS 2 type contracts through `spatialrust-runtime` | read-only rosbag2 SQLite PointCloud2 CDR streaming with optional float32 intensity, plus source-bound TFMessage inventory | `rusqlite` is isolated behind `rosbag2-sqlite` | | `spatialrust-search` | KD-tree | graph, parallel queries | none | +| `spatialrust-semantic` | entities, embeddings, fusion, search | `semantic-model`: `OnnxEntityEmbedder` through an existing model session | `spatialrust-ai`/`spatialrust-tensor` optional; ONNX stays in the AI crate | | `spatialrust-filtering` | voxel | GPU voxel, outlier, crop, FPS, MLS | wgpu/search optional | | `spatialrust-features` | normals | ISS, orientation, boundary, GPU normals | wgpu/search optional | | `spatialrust-segmentation` | plane and Euclidean clustering | GPU stages, DBSCAN, ground, primitives, region growing | wgpu/search optional | diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 4f9e770..0e830c1 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -126,6 +126,28 @@ Rust round-trips stay audited and explicit. | 149B | Complete | `__arrow_c_stream__` on `PyPointCloudStream` for bounded record streaming | `arrow-c-stream` | | 149C | Complete | Facade docs, FEATURE_MATRIX/CHANGELOG/notes | docs | +## Real AI semantic meaning program (Epic 150) + +Epic 95 established semantic entities and embeddings with a deterministic mock +profile. Epic 150 connects real ONNX inference to spatial semantic entities: +point-cloud entity features are embedded through a model session, indexed in +the existing `SemanticSearchIndex`, and searchable by open-vocabulary label. +Heavy runtimes stay behind dedicated features; tests use a tiny deterministic +ONNX model committed as fixture bytes so no external download is required. + +| Slice | Status | Scope | Feature | +| --- | --- | --- | --- | +| 150A | Complete | `OnnxEntityEmbedder`: entity feature tensors → ONNX embedding via an existing `ModelSession`, with explicit copy policy | `semantic`, `ai-onnxruntime` | +| 150B | Complete | Fixture ONNX model bytes + embedder correctness tests against the mock/real boundary | fixture, tests | +| 150C | Active | Facade wiring, Python binding, FEATURE_MATRIX/CHANGELOG/notes | facade | + +The embedder never loads a model itself; it consumes an already-open session so +backend identity, device placement, and copy policy stay explicit. The fixture +model maps a fixed-dimension feature vector to a fixed-dimension embedding so +round trips are exact and no learned weights are implied. The facade +`semantic-model` feature wires the embedder with the existing AI/ONNX surface. + + Each slice lands as one reviewable PR. The manifest reserves VGA-class and full-size cloud profiles and at least the operations both libraries implement diff --git a/notes/2026-08-07_epic150_ai_semantic.md b/notes/2026-08-07_epic150_ai_semantic.md new file mode 100644 index 0000000..76d2d4c --- /dev/null +++ b/notes/2026-08-07_epic150_ai_semantic.md @@ -0,0 +1,48 @@ +# Epic 150: real AI semantic meaning via ONNX entity embeddings + +Date: 2026-08-07. Slices 150A/150B complete. + +## Why + +Epic 95 established semantic entities and embeddings with a deterministic mock +profile. Epic 150 connects real ONNX inference to spatial semantic entities so +point-cloud features become searchable embeddings without leaving the SpatialRust +data model. + +## What was built + +- `crates/spatialrust-semantic/src/model.rs` (`model` feature) — `OnnxEntityEmbedder`: + - consumes an already-open `&mut dyn ModelSession` (backend/device chosen by + the caller, never internally); + - validates feature count against the input descriptor and finiteness; + - runs with explicit `CopyPolicy` for input and output; + - verifies the model output shape and bytes, then builds an `Embedding`. + Nine unit tests cover identity round trips, feature-count mismatch, + non-finite rejection, non-CPU rejection, and zero-dim output rejection. +- Facade feature `semantic-model` (`semantic` + `spatialrust-semantic/model` + + `ai` + `tensor`) and `tests/onnx_semantic.rs` integration test: + - committed fixture `crates/spatialrust/tests/fixtures/double_dynamic.onnx` + (a 134-byte Add model mapping `input` [1,3] → `output` [1,3], doubling); + - real ONNX Runtime CPU session embeds `[1,2,3]` → `[2,4,6]` exactly; + - the embedding is inserted into `SemanticSearchIndex` and found by a + query embedding, proving the full feature → model → search path. + +## Contract notes + +- The embedder never loads a model, selects a backend, or performs a hidden + device transfer; copy permission is an explicit per-run choice. +- Heavy runtimes remain behind `ai-onnxruntime`; `semantic` default build and + `spatialrust-semantic` default (no `model`) stay dependency-light. +- The committed fixture encodes no learned weights; it only proves the wiring. + +## Verification + +- `cargo test -p spatialrust-semantic --features model` — 9 tests. +- `cargo test -p spatialrust --features "ai-onnxruntime semantic-model" --test onnx_semantic` — 2 tests. +- clippy `-D warnings` clean for semantic and the facade test; `cargo fmt` clean. +- `cargo test --workspace` passes. + +## Next slices + +150C: Python binding for the embedder and open-vocabulary search over indexed +entities, plus a note with an example using a real public embedding model.