matrixfurry.com
The open source OpenXR runtime
1// Copyright 2023-2024, Collabora, Ltd.
2// SPDX-License-Identifier: BSL-1.0
3/*!
4 * @file
5 * @brief Visual-Intertial Tracking interface header.
6 * @author Mateo de Mayo <mateo.demayo@collabora.com>
7 * @author Simon Zeni <simon.zeni@collabora.com>
8 * @author Jakob Bornecrantz <jakob@collabora.com>
9 *
10 * This file contains the declaration of the @ref vit_tracker struct. This
11 * header is intended to appear in both consumers (e.g., Monado) and external
12 * visual-inertial tracking (VIT) systems (e.g., Basalt). The implementation of
13 * `vit_interface` is provided by the external system. Additional data types are
14 * declared for the communication between consumers and the system.
15 */
16
17#pragma once
18
19#if defined(__cplusplus)
20extern "C" {
21#endif
22
23#include <stdbool.h>
24#include <stdint.h>
25
26//! Compatibility with these values should be checked against @ref vit_api_get_version.
27#define VIT_HEADER_VERSION_MAJOR 2 //!< API Breakages
28#define VIT_HEADER_VERSION_MINOR 0 //!< Backwards compatible API changes
29#define VIT_HEADER_VERSION_PATCH 1 //!< Backw. comp. .h-implemented changes
30
31#define VIT_CAMERA_CALIBRATION_DISTORTION_MAX_COUNT 32
32
33/*!
34 * Result type used by the VIT system
35 *
36 * 0 is @ref VIT_SUCCESS, positive values are non fatal results and negative values are errors.
37 */
38typedef enum vit_result {
39 /*!
40 * Operation suceeded
41 */
42 VIT_SUCCESS = 0,
43
44 /*!
45 * The operation is not available on the current version
46 */
47 VIT_ERROR_INVALID_VERSION = -1,
48
49 /*!
50 * The operation received an invalid value.
51 */
52 VIT_ERROR_INVALID_VALUE = -2,
53
54 /*!
55 * The operation was not able to allocate memory to pursue the operation.
56 */
57 VIT_ERROR_ALLOCATION_FAILURE = -3,
58
59 /*!
60 * The operation requires an extension that is not supported.
61 */
62 VIT_ERROR_NOT_SUPPORTED = -4,
63
64 /*!
65 * The operation requires an extension that is not enabled.
66 */
67 VIT_ERROR_NOT_ENABLED = -5,
68} vit_result_t;
69
70/*!
71 * Image formats.
72 */
73typedef enum vit_image_format {
74 //! 8-bit luminance
75 VIT_IMAGE_FORMAT_L8 = 1,
76 //! 16-bit luminance
77 VIT_IMAGE_FORMAT_L16 = 2,
78 //! 24-bit rgb, tightly packed.
79 VIT_IMAGE_FORMAT_R8G8B8 = 3,
80} vit_image_format_t;
81
82/*!
83 * Camera calibration types.
84 */
85typedef enum vit_camera_distortion {
86 //! No distortion (pre-disotorted).
87 VIT_CAMERA_DISTORTION_NONE,
88 //! Distortion radial-tangential (OpenCV), 4 parameters.
89 VIT_CAMERA_DISTORTION_RT4,
90 //! Distortion radial-tangential (OpenCV), 5 parameters.
91 VIT_CAMERA_DISTORTION_RT5,
92 //! Distortion radial-tangential (OpenCV), 8 parameters.
93 VIT_CAMERA_DISTORTION_RT8,
94 //! Distortion Kannala-Brandt (OpenCV fisheye), 4 parameters.
95 VIT_CAMERA_DISTORTION_KB4,
96} vit_camera_distortion_t;
97
98/*!
99 * Optional functionality a tracker can provide.
100 */
101typedef enum vit_tracker_extension {
102 //! Allows to specify camera calibration data programatically
103 VIT_TRACKER_EXTENSION_ADD_CAMERA_CALIBRATION,
104 //! Allows to specify IMU calibration data programatically
105 VIT_TRACKER_EXTENSION_ADD_IMU_CALIBRATION,
106 //! Optionally provide timing information with the returned poses
107 VIT_TRACKER_EXTENSION_POSE_TIMING,
108 //! Optionally provide feature count information with the returned poses
109 VIT_TRACKER_EXTENSION_POSE_FEATURES,
110 //! Number of extensions
111 VIT_TRACKER_EXTENSION_COUNT,
112} vit_tracker_extension_t;
113
114/*!
115 * Set of extensions. Field order matches enumerators in @ref vit_tracker_extension.
116 */
117typedef struct vit_tracker_extension_set {
118 bool has_add_camera_calibration;
119 bool has_add_imu_calibration;
120 bool has_pose_timing;
121 bool has_pose_features;
122} vit_tracker_extension_set_t;
123
124/*!
125 * Visual-Inertial Tracking interface, opaque type.
126 */
127struct vit_tracker;
128typedef struct vit_tracker vit_tracker_t;
129
130/*!
131 * Pose interface, opaque type.
132 */
133struct vit_pose;
134typedef struct vit_pose vit_pose_t;
135
136/*!
137 * Names of the timestamps returned by `vit_pose_get_timings`
138 */
139typedef struct vit_tracker_timing_titles {
140 uint32_t count; //! Number of titles
141 const char **titles; //! Names of the measures timestamps
142} vit_tracker_timing_titles;
143
144/*!
145 * Parameters for creating the system pipeline.
146 */
147typedef struct vit_config {
148 //! Path to a implementation-specific config file. If null, use defaults.
149 const char *file;
150
151 //! Number of cameras to use. Required.
152 uint32_t cam_count;
153
154 //! Number of IMU to use. Required.
155 uint32_t imu_count;
156
157 //! If supported, whether to open the system's UI.
158 bool show_ui;
159} vit_config_t;
160
161/*!
162 * IMU sample type feed into VIT tracker
163 */
164typedef struct vit_imu_sample {
165 //! In nanoseconds
166 int64_t timestamp;
167
168 //! Acceleration in meters per second squared (m/s²)
169 float ax, ay, az;
170
171 //! Gyro in radians per second (rad/s)
172 float wx, wy, wz;
173} vit_imu_sample_t;
174
175/*!
176 * Region in image space that this mask covers.
177 */
178typedef struct vit_mask {
179 //! In pixels.
180 float x, y, w, h;
181} vit_mask_t;
182
183/*!
184 * Image sample type feed into VIT tracker
185 *
186 * Can easily be converted into an OpenCV Matrix for processing.
187 */
188typedef struct vit_img_sample {
189 uint32_t cam_index;
190
191 //! In nanoseconds, must increase monotonically.
192 int64_t timestamp;
193
194 // !Image data
195 uint8_t *data;
196 uint32_t width, height;
197 uint32_t stride, size;
198 vit_image_format_t format;
199
200 //! Regions to ignore
201 uint32_t mask_count;
202 vit_mask_t *masks;
203} vit_img_sample_t;
204
205/*!
206 * Data that is always returned from tracker.
207 */
208typedef struct vit_pose_data {
209 //! In nanoseconds, must increase monotonically.
210 int64_t timestamp;
211
212 //! Position vector.
213 float px, py, pz;
214
215 //! Orientation quaternion.
216 float ox, oy, oz, ow;
217
218 //! Linear velocity.
219 float vx, vy, vz;
220} vit_pose_data_t;
221
222/*!
223 * Result of pose timing request function.
224 */
225typedef struct vit_pose_timing {
226 uint32_t count;
227 const int64_t *timestamps;
228} vit_pose_timing_t;
229
230/*!
231 * One single feature, element of @ref vit_pose_features result.
232 */
233typedef struct vit_pose_feature {
234 int64_t id;
235 float u, v, depth;
236} vit_pose_feature_t;
237
238/*!
239 * Result of pose feature request function.
240 */
241typedef struct vit_pose_features {
242 uint32_t count;
243 const struct vit_pose_feature *features;
244} vit_pose_features_t;
245
246/*!
247 * Container of parameters for a pinhole camera calibration (fx, fy, cx, cy)
248 * with an optional distortion.
249 *
250 *`distortion_model` and its corresponding `distortion` parameters are not
251 * standardized in this struct to facilitate implementation prototyping.
252 */
253typedef struct vit_camera_calibration {
254 uint32_t camera_index; // <! For multi-camera setup. For stereo 0 ~ left, 1 ~ right.
255
256 int width, height; //<! Resolution
257 double frequency; //<! Frames per second
258 double fx, fy; //<! Focal point
259 double cx, cy; //<! Principal point
260 enum vit_camera_distortion model;
261 uint32_t distortion_count;
262 double distortion[VIT_CAMERA_CALIBRATION_DISTORTION_MAX_COUNT]; //!< Parameters for the distortion model
263 double transform[4 * 4]; //!< Row-major 4x4 camera transform w.r.t. the IMU (i.e., T_imu_cam)
264} vit_camera_calibration_t;
265
266typedef struct vit_inertial_calibration {
267 // Calibration intrinsics to apply to each raw measurement.
268
269 //! Row major 3x3 linear transformation for raw measurements alignment and scaling.
270 double transform[3 * 3];
271
272 //! Offset to add to raw measurements; called bias in other contexts.
273 double offset[3];
274
275 // Parameters for the random processes that model this IMU. See section "2.1
276 // Gyro Noise Model" of N. Trawny and S. I. Roumeliotis, "Indirect Kalman
277 // Filter for 3D Attitude Estimation". Analogous for accelerometers.
278 // http://mars.cs.umn.edu/tr/reports/Trawny05b.pdf#page=15
279
280 //! IMU internal bias ~ wiener process with steps N(0, σ²); this field is σ;
281 //! [σ] = U / sqrt(sec³) with U = rad if gyroscope, U = m/s if accelerometer.
282 double bias_std[3];
283
284 //! IMU measurement noise ~ N(0, σ²); this field is σ.
285 //! [σ] = U / sqrt(sec) with U = rad if gyroscope, U = m/s if accelerometer.
286 double noise_std[3];
287} vit_inertial_calibration_t;
288
289/*!
290 * Calibration for one IMU.
291 */
292typedef struct vit_imu_calibration {
293 uint32_t imu_index; //!< For multi-imu setup, usually just 0.
294
295 double frequency; //!< Samples per second
296 struct vit_inertial_calibration accel;
297 struct vit_inertial_calibration gyro;
298} vit_imu_calibration_t;
299
300/*
301 *
302 * Function prototypes.
303 *
304 */
305
306typedef vit_result_t (*PFN_vit_api_get_version)(uint32_t *out_major, uint32_t *out_minor, uint32_t *out_patch);
307typedef vit_result_t (*PFN_vit_tracker_create)(const vit_config_t *config, vit_tracker_t **out_tracker);
308typedef void (*PFN_vit_tracker_destroy)(vit_tracker_t *tracker);
309typedef vit_result_t (*PFN_vit_tracker_has_image_format)(const vit_tracker_t *tracker, vit_image_format_t image_format,
310 bool *out_supported);
311typedef vit_result_t (*PFN_vit_tracker_get_supported_extensions)(const vit_tracker_t *tracker,
312 vit_tracker_extension_set_t *out_exts);
313typedef vit_result_t (*PFN_vit_tracker_get_enabled_extensions)(const vit_tracker_t *tracker,
314 vit_tracker_extension_set_t *out_exts);
315typedef vit_result_t (*PFN_vit_tracker_enable_extension)(vit_tracker_t *tracker, vit_tracker_extension_t ext,
316 bool enable);
317typedef vit_result_t (*PFN_vit_tracker_start)(vit_tracker_t *tracker);
318typedef vit_result_t (*PFN_vit_tracker_stop)(vit_tracker_t *tracker);
319typedef vit_result_t (*PFN_vit_tracker_reset)(vit_tracker_t *tracker);
320typedef vit_result_t (*PFN_vit_tracker_is_running)(const vit_tracker_t *tracker, bool *out_bool);
321typedef vit_result_t (*PFN_vit_tracker_push_imu_sample)(vit_tracker_t *tracker, const vit_imu_sample_t *sample);
322typedef vit_result_t (*PFN_vit_tracker_push_img_sample)(vit_tracker_t *tracker, const vit_img_sample_t *sample);
323typedef vit_result_t (*PFN_vit_tracker_add_imu_calibration)(vit_tracker_t *tracker,
324 const vit_imu_calibration_t *calibration);
325typedef vit_result_t (*PFN_vit_tracker_add_camera_calibration)(vit_tracker_t *tracker,
326 const vit_camera_calibration_t *calibration);
327typedef vit_result_t (*PFN_vit_tracker_pop_pose)(vit_tracker_t *tracker, vit_pose_t **out_pose);
328typedef vit_result_t (*PFN_vit_tracker_get_timing_titles)(const vit_tracker_t *tracker,
329 vit_tracker_timing_titles *out_titles);
330typedef void (*PFN_vit_pose_destroy)(vit_pose_t *pose);
331typedef vit_result_t (*PFN_vit_pose_get_data)(const vit_pose_t *pose, vit_pose_data_t *out_data);
332typedef vit_result_t (*PFN_vit_pose_get_timing)(const vit_pose_t *pose, vit_pose_timing_t *out_timing);
333typedef vit_result_t (*PFN_vit_pose_get_features)(const vit_pose_t *pose, uint32_t camera_index,
334 vit_pose_features_t *out_features);
335
336/*
337 *
338 * Functions.
339 *
340 */
341
342#ifdef VIT_INTERFACE_IMPLEMENTATION
343
344/*!
345 * Returns the API version implemented by the VIT system.
346 */
347vit_result_t vit_api_get_version(uint32_t *out_major, uint32_t *out_minor, uint32_t *out_patch);
348
349/*!
350 * Creates a new VIT tracker. The caller is responsible of destroying it when done.
351 */
352vit_result_t vit_tracker_create(const vit_config_t *config, vit_tracker_t **out_tracker);
353
354/*!
355 * Destroys the VIT tracker and free all resources allocated.
356 */
357void vit_tracker_destroy(vit_tracker_t *tracker);
358
359/*!
360 * Verifies if the tracker supports a given `vit_image_format_t`.
361 */
362vit_result_t vit_tracker_has_image_format(const vit_tracker_t *tracker, vit_image_format_t image_format,
363 bool *out_supported);
364
365/*!
366 * Returns a set with the extensions supported by this tracker.
367 *
368 * @see vit_tracker_extension_t
369 */
370
371vit_result_t vit_tracker_get_supported_extensions(const vit_tracker_t *tracker, vit_tracker_extension_set_t *out_exts);
372
373/*!
374 * Returns a set with the extensions enabled in this tracker.
375 *
376 * @see vit_tracker_extension_t
377 */
378
379vit_result_t vit_tracker_get_enabled_extensions(const vit_tracker_t *tracker, vit_tracker_extension_set_t *out_exts);
380
381/*!
382 * Enables or disables a tracker extension.
383 *
384 * @see vit_tracker_extension_t
385 */
386vit_result_t vit_tracker_enable_extension(vit_tracker_t *tracker, vit_tracker_extension_t ext, bool value);
387
388/*!
389 * Starts the VIT tracker. Image and IMU samples can be pushed and pose can be retrieved.
390 *
391 * This function must be non blocking. The VIT system implementing it is expected to start its own event loop.
392 */
393vit_result_t vit_tracker_start(vit_tracker_t *tracker);
394
395/*!
396 * Stops the VIT tracker. The tracker wont accept image and IMU samples, and will not return poses.
397 */
398vit_result_t vit_tracker_stop(vit_tracker_t *tracker);
399
400/*!
401 * Resets the VIT tracker. The tracker internal state will be set to its original state.
402 */
403vit_result_t vit_tracker_reset(vit_tracker_t *tracker);
404
405/*!
406 * Verifies if the tracker is running.
407 */
408vit_result_t vit_tracker_is_running(const vit_tracker_t *tracker, bool *out_bool);
409
410/*!
411 * Push an IMU sample into the tracker.
412 *
413 * There must be a single producer thread pushing samples.
414 * Samples must have monotonically increasing timestamps.
415 * The implementation must be non-blocking.
416 * Thus, a separate consumer thread should process the samples.
417 */
418vit_result_t vit_tracker_push_imu_sample(vit_tracker_t *tracker, const vit_imu_sample_t *sample);
419
420/*!
421 * Push an image sample into the tracker.
422 *
423 * Same conditions as @ref push_imu_sample apply.
424 * When using N>1 cameras, the N frames must be pushed following @ref cam_index order.
425 * The bundle of N frames must have the same timestamps.
426 */
427vit_result_t vit_tracker_push_img_sample(vit_tracker_t *tracker, const vit_img_sample_t *sample);
428
429/*!
430 * Adds an inertial measurement unit calibration to the tracker. The tracker must not be started.
431 *
432 * Returns `VIT_ERROR_NOT_SUPPORTED` if the tracker doesn't offer the extension.
433 *
434 * @see vit_tracker_get_supported_extensions
435 * @see vit_tracker_extension_t
436 */
437vit_result_t vit_tracker_add_imu_calibration(vit_tracker_t *tracker, const vit_imu_calibration_t *calibration);
438
439/*!
440 * Adds a camera calibration to the tracker. The tracker must not be started.
441 *
442 * Returns `VIT_ERROR_NOT_SUPPORTED` if the tracker doesn't offer the extension.
443 *
444 * @see vit_tracker_get_supported_extensions
445 * @see vit_tracker_extension_t
446 */
447vit_result_t vit_tracker_add_camera_calibration(vit_tracker_t *tracker, const vit_camera_calibration_t *calibration);
448
449/*!
450 * Get the pose from the front of the tracking queue from the VIT tracker
451 *
452 * This function must be non-blocking and consumed by a single consummer.
453 *
454 * If @p out_pose is NULL the pose will be immediately destroyed.
455 *
456 * @param[out] out_pose Pose returned to the caller, NULL if there is not pose available or on error.
457 */
458vit_result_t vit_tracker_pop_pose(vit_tracker_t *tracker, vit_pose_t **out_pose);
459
460/*!
461 * Get the titles of the timestamps measured by the pose timings.
462 *
463 * Returns `VIT_ERROR_NOT_SUPPORTED` if the tracker doesn't offer the pose timing extension.
464 */
465vit_result_t vit_tracker_get_timing_titles(const vit_tracker_t *tracker, vit_tracker_timing_titles *out_titles);
466
467/*!
468 * Destroys a pose. All of the data, timing and features associated to it will be invalidated.
469 */
470void vit_pose_destroy(vit_pose_t *pose);
471
472/*!
473 * Gets the data form a given `vit_pose_t`.
474 *
475 * The data becomes invalid when the associated pose gets destroyed.
476 */
477vit_result_t vit_pose_get_data(const vit_pose_t *pose, vit_pose_data_t *out_data);
478
479/*!
480 * Gets the timing form a given `vit_pose_t`.
481 *
482 * The timing data becomes invalid when the associated pose gets destroyed.
483 */
484vit_result_t vit_pose_get_timing(const vit_pose_t *pose, vit_pose_timing_t *out_timing);
485
486/*!
487 * Gets the features form a given `vit_pose_t`.
488 *
489 * The features data becomes invalid when the associated pose gets destroyed.
490 */
491vit_result_t vit_pose_get_features(const vit_pose_t *pose, uint32_t camera_index, vit_pose_features_t *out_features);
492
493#endif
494
495#if defined(__cplusplus)
496}
497#endif