Calibrex/examples/public_datasets/solid_state_cross_dataset_benchmark_v04.yaml at main · rsasaki0109/Calibrex · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
schema_version: slac.solid_state_cross_dataset_benchmark_config/v0.2

protocol:
  name: bounded_continuous_time_uniform_vs_adaptive_mad25_solid_state_lidar
  solver: continuous_time_lidar_pair
  variants:
    - uniform_none
    - adaptive_mad
  same_capture_windows: true
  same_temporal_holdout: true
  same_solver_budget: true
  bounded_replay_allowed: true
  bounded_replay_note: >
    Each dataset uses the same bounded replay, temporal split, sampling seed,
    and solver budget for both variants. The v0.4 adaptive candidate fixes
    continuous_time_outlier_mad_scale=2.5 before holdout scoring.
  primary_metric: final_holdout_rmse_m
  lower_is_better: true
  minimum_holdout_correspondences: 6
  split_ids:
    - early_holdout
    - middle_holdout
    - late_holdout
  holdout_start_fractions:
    early_holdout: 0.60
    middle_holdout: 0.70
    late_holdout: 0.80
  seed_values: [0, 17, 42]
  confidence_level: 0.95
  bootstrap_samples: 2000
  bootstrap_seed: 20260804
  minimum_scored_replicates: 4
  require_converged: false

datasets:
  - id: agrob_modular_e
    name: AgRob Modular-e Livox MID-70 plus RS-LiDAR
    family: agrob_scorpion
    dataset_manifest_path: examples/public_datasets/agrob_modular_e/manifest.yaml
    config_path: examples/public_datasets/agrob_modular_e/online_joint_config.yaml
    continuous_time_options:
      continuous_time_outlier_mad_scale: 2.5
    ablation_manifest_path: outputs/solid_state_benchmark_v04_replicates/agrob_modular_e/ablation_manifest.yaml
    reference_mode: trajectory_only
    absolute_extrinsic_ground_truth: false
    identity_control: false
    independent_temporal_holdout: true
    notes:
      - Candidate fixed from train-only AgRob diagnostics on early, middle, and late seed-0 probes.
      - This is a real sensor pair with no independently surveyed extrinsic ground truth.

  - id: tiers_livox_lidars_cali
    name: TIERS LidarsCali Velodyne VLP-16 plus Livox Horizon
    family: tiers_lidars
    dataset_manifest_path: examples/public_datasets/tiers_livox_lidars_cali/manifest.yaml
    config_path: examples/public_datasets/tiers_livox_lidars_cali/online_continuous_time_config.yaml
    continuous_time_options:
      continuous_time_outlier_mad_scale: 2.5
    ablation_manifest_path: outputs/solid_state_benchmark_v04_replicates/tiers_livox_lidars_cali/ablation_manifest.yaml
    reference_mode: trajectory_only
    absolute_extrinsic_ground_truth: false
    identity_control: false
    independent_temporal_holdout: true
    notes:
      - The same candidate is stress-tested on this distinct public multimodal bag.
      - ROS1 replay uses bag-record clock for temporal partitioning.

  - id: glim_versatile
    name: AIST GLIM Livox Avia duplicate-topic identity control
    family: glim_versatile
    dataset_manifest_path: examples/public_datasets/glim_versatile/manifest.yaml
    config_path: examples/public_datasets/glim_versatile/online_continuous_time_benchmark_config.yaml
    continuous_time_options:
      continuous_time_outlier_mad_scale: 2.5
    ablation_manifest_path: outputs/solid_state_benchmark_v04_replicates/glim_versatile/ablation_manifest.yaml
    reference_mode: identity_control
    absolute_extrinsic_ground_truth: false
    identity_control: true
    independent_temporal_holdout: false
    notes:
      - This duplicate-topic identity control is not a surveyed second-sensor transform.
      - It tests whether the candidate preserves the existing public control behavior.