Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion src/bin/test_numbers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,9 @@ pub fn main() {
// println!("Queries: {:?}", random_queries);

for &n in &random_numbers {
features.push(SimpleNumberRecord::create(n.to_string()))
features.push(
SimpleNumberRecord::create(n.to_string()).expect("internally generated, always valid"),
)
}

let creation_start = Instant::now();
Expand Down
4 changes: 3 additions & 1 deletion src/bin/test_tlsh.rs
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,9 @@ pub fn main() {
);
println!("Starting insertion into HNSW model...");
for f in dataset_copy {
apotheosis.insert(SimpleTlshRecord::create(f));
apotheosis.insert(
SimpleTlshRecord::create(f).expect("hash from output_hashes.json, always valid"),
);
}

let query_hash = TlshDefault::from_str(
Expand Down
2 changes: 1 addition & 1 deletion src/controllers/apotheosis.rs
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ where
if let Some(ref key) = radix_key
&& self.radix.find(key).is_some()
{
println!("Key already exists in radix tree: {:?}", key);
tracing::warn!("Key already exists in radix tree: {:?}", key);
return false;
}

Expand Down
1 change: 1 addition & 0 deletions src/datalayer.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
pub mod algorithms;
pub mod error;
pub mod nodes;
pub mod record;
39 changes: 39 additions & 0 deletions src/datalayer/error.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
use std::fmt;
use std::num::ParseIntError;

/// Error returned when constructing a record from an untrusted string, e.g. a
/// line read from a dataset file. Record constructors return this instead of
/// panicking so a malformed entry can be skipped without aborting the whole
/// ingestion.
#[derive(Debug)]
pub enum RecordError {
/// The input was expected to be a `u32` but did not parse as one.
InvalidNumber {
input: String,
source: ParseIntError,
},
/// The input was expected to be a TLSH digest but did not parse as one.
InvalidTlshHash { input: String },
}

impl fmt::Display for RecordError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
RecordError::InvalidNumber { input, source } => {
write!(f, "invalid number record {input:?}: {source}")
}
RecordError::InvalidTlshHash { input } => {
write!(f, "invalid TLSH hash {input:?}")
}
}
}
}

impl std::error::Error for RecordError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match self {
RecordError::InvalidNumber { source, .. } => Some(source),
RecordError::InvalidTlshHash { .. } => None,
}
}
}
37 changes: 27 additions & 10 deletions src/datalayer/record.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
use crate::datalayer::error::RecordError;
use std::str::FromStr;
use tlsh2::TlshDefault;

Expand Down Expand Up @@ -53,16 +54,28 @@ pub type SimpleNumberRecord = SimpleRecord<u32>;
pub type SimpleTlshRecord = SimpleRecord<TlshDefault>;

impl SimpleNumberRecord {
pub fn create(s: String) -> Self {
let id = s.parse::<u32>().unwrap();
Self { id, radix_key: s }
/// Builds a record from an untrusted string (e.g. a line read from a
/// dataset file). Returns `Err` instead of panicking so a malformed entry
/// can be skipped without aborting the whole ingestion.
pub fn create(s: String) -> Result<Self, RecordError> {
let id = s
.parse::<u32>()
.map_err(|source| RecordError::InvalidNumber {
input: s.clone(),
source,
})?;
Ok(Self { id, radix_key: s })
}
}

impl SimpleTlshRecord {
pub fn create(s: String) -> Self {
let id = TlshDefault::from_str(&s).unwrap();
Self { id, radix_key: s }
/// Builds a record from an untrusted TLSH hash string. Returns `Err`
/// instead of panicking so a malformed entry can be skipped without
/// aborting the whole ingestion.
pub fn create(s: String) -> Result<Self, RecordError> {
let id = TlshDefault::from_str(&s)
.map_err(|_| RecordError::InvalidTlshHash { input: s.clone() })?;
Ok(Self { id, radix_key: s })
}
}

Expand Down Expand Up @@ -99,13 +112,17 @@ impl ApotheosisRecord for GenericJsonRecord {
}

impl GenericJsonRecord {
pub fn create(s: String, metadata: serde_json::Value) -> Self {
let id = TlshDefault::from_str(&s).unwrap();
Self {
/// Builds a record from an untrusted TLSH hash string. Returns `Err`
/// instead of panicking so a malformed entry can be skipped without
/// aborting the whole ingestion.
pub fn create(s: String, metadata: serde_json::Value) -> Result<Self, RecordError> {
let id = TlshDefault::from_str(&s)
.map_err(|_| RecordError::InvalidTlshHash { input: s.clone() })?;
Ok(Self {
id,
radix_key: s,
metadata,
}
})
}
}

Expand Down
45 changes: 45 additions & 0 deletions tests/record_construction.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
// create() constructors build records from untrusted strings (e.g. a line
// read from a dataset file). They must return Err on malformed input instead
// of panicking, so a single bad entry does not abort a whole ingestion run.

use apotheosis2::datalayer::record::{GenericJsonRecord, SimpleNumberRecord, SimpleTlshRecord};

#[test]
fn number_record_rejects_non_numeric_input() {
let result = SimpleNumberRecord::create("not-a-number".to_string());
assert!(result.is_err(), "malformed number input must return Err");

let msg = result.err().unwrap().to_string();
assert!(
msg.contains("not-a-number"),
"error message must include the offending input, got: {msg}"
);
}

#[test]
fn number_record_accepts_valid_input() {
let result = SimpleNumberRecord::create("42".to_string());
assert!(result.is_ok());
assert_eq!(result.unwrap().id, 42);
}

#[test]
fn tlsh_record_rejects_malformed_hash() {
let result = SimpleTlshRecord::create("not-a-tlsh-hash".to_string());
assert!(result.is_err(), "malformed TLSH hash must return Err");

let msg = result.err().unwrap().to_string();
assert!(
msg.contains("not-a-tlsh-hash"),
"error message must include the offending input, got: {msg}"
);
}

#[test]
fn generic_json_record_rejects_malformed_hash() {
let result = GenericJsonRecord::create(
"not-a-tlsh-hash".to_string(),
serde_json::json!({"key": "value"}),
);
assert!(result.is_err(), "malformed TLSH hash must return Err");
}
Loading