qualia_vision/spatial/
image_to_3d.rs1use crate::semantic::{media_digest, q_hash, MediaDigest};
8use crate::types::{ImageView, PixelFormat, VisionError};
9use qualia_core_db::specialized_libs::computer_vision::spatial::{
10 validate_mesh_ir, MeshIR, MeshValidationReport, MAX_VERTICES,
11};
12
13pub const RECON_MODEL_ID: &str = "qualia-vision-heightfield-recon-v1";
14
15#[derive(Debug, Clone, Copy, PartialEq, Eq)]
16pub struct ImageTo3dReceipt {
17 pub model_hash: u64,
18 pub source_digest: MediaDigest,
19 pub mesh_hash: u64,
20 pub grid_w: u32,
21 pub grid_h: u32,
22 pub is_reference_recon: bool,
24}
25
26fn sample_luma(img: ImageView<'_>, x: u32, y: u32) -> f32 {
27 let bpp = img.bytes_per_pixel() as usize;
28 let off = (y as usize)
29 .saturating_mul(img.row_stride as usize)
30 .saturating_add((x as usize).saturating_mul(bpp));
31 if off + bpp > img.bytes.len() {
32 return 0.0;
33 }
34 match img.format {
35 PixelFormat::Gray8 => img.bytes[off] as f32 / 255.0,
36 PixelFormat::Rgb8 | PixelFormat::Rgba8 => {
37 let r = img.bytes[off] as f32;
38 let g = img.bytes[off + 1] as f32;
39 let b = img.bytes[off + 2] as f32;
40 (0.299 * r + 0.587 * g + 0.114 * b) / 255.0
41 }
42 PixelFormat::Bgr8 => {
43 let b = img.bytes[off] as f32;
44 let g = img.bytes[off + 1] as f32;
45 let r = img.bytes[off + 2] as f32;
46 (0.299 * r + 0.587 * g + 0.114 * b) / 255.0
47 }
48 PixelFormat::RgbF32 => 0.0,
49 }
50}
51
52pub fn image_to_heightfield_mesh(
54 image: ImageView<'_>,
55 grid: u32,
56) -> Result<(MeshIR, ImageTo3dReceipt, MeshValidationReport), VisionError> {
57 if !image.is_well_formed() {
58 return Err(VisionError::MalformedImage);
59 }
60 let g = grid.clamp(2, 64);
61 let vc = (g * g) as usize;
62 if vc > MAX_VERTICES {
63 return Err(VisionError::OutputBufferTooSmall);
64 }
65
66 let mut mesh = MeshIR::empty();
67 mesh.positions.reserve(vc);
68 mesh.normals.reserve(vc);
69 mesh.uvs.reserve(vc);
70 mesh.indices
71 .reserve((g as usize - 1) * (g as usize - 1) * 6);
72
73 for j in 0..g {
74 for i in 0..g {
75 let u = i as f32 / (g - 1) as f32;
76 let v = j as f32 / (g - 1) as f32;
77 let px = (u * (image.width.saturating_sub(1)) as f32) as u32;
78 let py = (v * (image.height.saturating_sub(1)) as f32) as u32;
79 let z = sample_luma(image, px, py) * 0.35;
80 let x = u * 2.0 - 1.0;
81 let y = 1.0 - v * 2.0;
82 mesh.positions.push([x, y, z]);
83 mesh.normals.push([0.0, 0.0, 1.0]);
84 mesh.uvs.push([u, v]);
85 }
86 }
87
88 for j in 0..g - 1 {
89 for i in 0..g - 1 {
90 let i0 = j * g + i;
91 let i1 = i0 + 1;
92 let i2 = i0 + g;
93 let i3 = i2 + 1;
94 mesh.indices.extend_from_slice(&[i0, i2, i1, i1, i2, i3]);
95 }
96 }
97
98 for n in &mut mesh.normals {
100 *n = [0.0, 0.0, 0.0];
101 }
102 let mut t = 0usize;
103 while t + 2 < mesh.indices.len() {
104 let a = mesh.indices[t] as usize;
105 let b = mesh.indices[t + 1] as usize;
106 let c = mesh.indices[t + 2] as usize;
107 let pa = mesh.positions[a];
108 let pb = mesh.positions[b];
109 let pc = mesh.positions[c];
110 let ab = [pb[0] - pa[0], pb[1] - pa[1], pb[2] - pa[2]];
111 let ac = [pc[0] - pa[0], pc[1] - pa[1], pc[2] - pa[2]];
112 let nx = ab[1] * ac[2] - ab[2] * ac[1];
113 let ny = ab[2] * ac[0] - ab[0] * ac[2];
114 let nz = ab[0] * ac[1] - ab[1] * ac[0];
115 for idx in [a, b, c] {
116 mesh.normals[idx][0] += nx;
117 mesh.normals[idx][1] += ny;
118 mesh.normals[idx][2] += nz;
119 }
120 t += 3;
121 }
122 for n in &mut mesh.normals {
123 let len = (n[0] * n[0] + n[1] * n[1] + n[2] * n[2]).sqrt().max(1e-8);
124 n[0] /= len;
125 n[1] /= len;
126 n[2] /= len;
127 }
128
129 mesh.recompute_bounds_and_hash();
130 let report = validate_mesh_ir(&mesh);
131 let src = media_digest(image.bytes);
132 let receipt = ImageTo3dReceipt {
133 model_hash: q_hash(RECON_MODEL_ID),
134 source_digest: src,
135 mesh_hash: mesh.content_hash,
136 grid_w: g,
137 grid_h: g,
138 is_reference_recon: true,
139 };
140 Ok((mesh, receipt, report))
141}
142
143#[cfg(test)]
144mod tests {
145 use super::*;
146 use crate::types::PixelFormat;
147
148 #[test]
149 fn heightfield_validates() {
150 let mut rgb = vec![0u8; 16 * 16 * 3];
151 for y in 0..16 {
152 for x in 0..16 {
153 let i = (y * 16 + x) * 3;
154 rgb[i] = (x * 16) as u8;
155 rgb[i + 1] = (y * 16) as u8;
156 rgb[i + 2] = 128;
157 }
158 }
159 let img = ImageView {
160 bytes: &rgb,
161 width: 16,
162 height: 16,
163 row_stride: 48,
164 format: PixelFormat::Rgb8,
165 };
166 let (mesh, rec, rep) = image_to_heightfield_mesh(img, 8).unwrap();
167 assert!(rep.ok(), "{rep:?}");
168 assert!(mesh.triangle_count() > 0);
169 assert!(rec.is_reference_recon);
170 }
171}