Skip to main content

qualia_vision/spatial/
image_to_3d.rs

1//! Swarm S-V10 — image → heightfield mesh (reference reconstruction).
2//!
3//! Honest: monocular height from luminance gradient — not multi-view SfM or a
4//! foundation image-to-3D network. Output is an **epistemic proposed mesh**;
5//! must pass [`validate_mesh_ir`] before any Q42 commit.
6
7use crate::semantic::{media_digest, q_hash, MediaDigest};
8use crate::types::{ImageView, PixelFormat, VisionError};
9use qualia_core_db::specialized_libs::computer_vision::spatial::{
10    validate_mesh_ir, MeshIR, MeshValidationReport, MAX_VERTICES,
11};
12
13pub const RECON_MODEL_ID: &str = "qualia-vision-heightfield-recon-v1";
14
15#[derive(Debug, Clone, Copy, PartialEq, Eq)]
16pub struct ImageTo3dReceipt {
17    pub model_hash: u64,
18    pub source_digest: MediaDigest,
19    pub mesh_hash: u64,
20    pub grid_w: u32,
21    pub grid_h: u32,
22    /// Always true for this path until a licensed recon model is loaded.
23    pub is_reference_recon: bool,
24}
25
26fn sample_luma(img: ImageView<'_>, x: u32, y: u32) -> f32 {
27    let bpp = img.bytes_per_pixel() as usize;
28    let off = (y as usize)
29        .saturating_mul(img.row_stride as usize)
30        .saturating_add((x as usize).saturating_mul(bpp));
31    if off + bpp > img.bytes.len() {
32        return 0.0;
33    }
34    match img.format {
35        PixelFormat::Gray8 => img.bytes[off] as f32 / 255.0,
36        PixelFormat::Rgb8 | PixelFormat::Rgba8 => {
37            let r = img.bytes[off] as f32;
38            let g = img.bytes[off + 1] as f32;
39            let b = img.bytes[off + 2] as f32;
40            (0.299 * r + 0.587 * g + 0.114 * b) / 255.0
41        }
42        PixelFormat::Bgr8 => {
43            let b = img.bytes[off] as f32;
44            let g = img.bytes[off + 1] as f32;
45            let r = img.bytes[off + 2] as f32;
46            (0.299 * r + 0.587 * g + 0.114 * b) / 255.0
47        }
48        PixelFormat::RgbF32 => 0.0,
49    }
50}
51
52/// Build a heightfield mesh from image luminance. `grid` is samples per side (2..=64).
53pub fn image_to_heightfield_mesh(
54    image: ImageView<'_>,
55    grid: u32,
56) -> Result<(MeshIR, ImageTo3dReceipt, MeshValidationReport), VisionError> {
57    if !image.is_well_formed() {
58        return Err(VisionError::MalformedImage);
59    }
60    let g = grid.clamp(2, 64);
61    let vc = (g * g) as usize;
62    if vc > MAX_VERTICES {
63        return Err(VisionError::OutputBufferTooSmall);
64    }
65
66    let mut mesh = MeshIR::empty();
67    mesh.positions.reserve(vc);
68    mesh.normals.reserve(vc);
69    mesh.uvs.reserve(vc);
70    mesh.indices
71        .reserve((g as usize - 1) * (g as usize - 1) * 6);
72
73    for j in 0..g {
74        for i in 0..g {
75            let u = i as f32 / (g - 1) as f32;
76            let v = j as f32 / (g - 1) as f32;
77            let px = (u * (image.width.saturating_sub(1)) as f32) as u32;
78            let py = (v * (image.height.saturating_sub(1)) as f32) as u32;
79            let z = sample_luma(image, px, py) * 0.35;
80            let x = u * 2.0 - 1.0;
81            let y = 1.0 - v * 2.0;
82            mesh.positions.push([x, y, z]);
83            mesh.normals.push([0.0, 0.0, 1.0]);
84            mesh.uvs.push([u, v]);
85        }
86    }
87
88    for j in 0..g - 1 {
89        for i in 0..g - 1 {
90            let i0 = j * g + i;
91            let i1 = i0 + 1;
92            let i2 = i0 + g;
93            let i3 = i2 + 1;
94            mesh.indices.extend_from_slice(&[i0, i2, i1, i1, i2, i3]);
95        }
96    }
97
98    // Recompute flat normals per vertex (average of faces — simple).
99    for n in &mut mesh.normals {
100        *n = [0.0, 0.0, 0.0];
101    }
102    let mut t = 0usize;
103    while t + 2 < mesh.indices.len() {
104        let a = mesh.indices[t] as usize;
105        let b = mesh.indices[t + 1] as usize;
106        let c = mesh.indices[t + 2] as usize;
107        let pa = mesh.positions[a];
108        let pb = mesh.positions[b];
109        let pc = mesh.positions[c];
110        let ab = [pb[0] - pa[0], pb[1] - pa[1], pb[2] - pa[2]];
111        let ac = [pc[0] - pa[0], pc[1] - pa[1], pc[2] - pa[2]];
112        let nx = ab[1] * ac[2] - ab[2] * ac[1];
113        let ny = ab[2] * ac[0] - ab[0] * ac[2];
114        let nz = ab[0] * ac[1] - ab[1] * ac[0];
115        for idx in [a, b, c] {
116            mesh.normals[idx][0] += nx;
117            mesh.normals[idx][1] += ny;
118            mesh.normals[idx][2] += nz;
119        }
120        t += 3;
121    }
122    for n in &mut mesh.normals {
123        let len = (n[0] * n[0] + n[1] * n[1] + n[2] * n[2]).sqrt().max(1e-8);
124        n[0] /= len;
125        n[1] /= len;
126        n[2] /= len;
127    }
128
129    mesh.recompute_bounds_and_hash();
130    let report = validate_mesh_ir(&mesh);
131    let src = media_digest(image.bytes);
132    let receipt = ImageTo3dReceipt {
133        model_hash: q_hash(RECON_MODEL_ID),
134        source_digest: src,
135        mesh_hash: mesh.content_hash,
136        grid_w: g,
137        grid_h: g,
138        is_reference_recon: true,
139    };
140    Ok((mesh, receipt, report))
141}
142
143#[cfg(test)]
144mod tests {
145    use super::*;
146    use crate::types::PixelFormat;
147
148    #[test]
149    fn heightfield_validates() {
150        let mut rgb = vec![0u8; 16 * 16 * 3];
151        for y in 0..16 {
152            for x in 0..16 {
153                let i = (y * 16 + x) * 3;
154                rgb[i] = (x * 16) as u8;
155                rgb[i + 1] = (y * 16) as u8;
156                rgb[i + 2] = 128;
157            }
158        }
159        let img = ImageView {
160            bytes: &rgb,
161            width: 16,
162            height: 16,
163            row_stride: 48,
164            format: PixelFormat::Rgb8,
165        };
166        let (mesh, rec, rep) = image_to_heightfield_mesh(img, 8).unwrap();
167        assert!(rep.ok(), "{rep:?}");
168        assert!(mesh.triangle_count() > 0);
169        assert!(rec.is_reference_recon);
170    }
171}