Skip to main content

qualia_extensions/
lib.rs

1//! QualiaDB Advanced Extensions
2//!
3//! This crate provides the extension interface for heavy computational workloads
4//! that cannot run in the zero-allocation core QualiaDB engine.
5//!
6//! # Architecture
7//!
8//! - Core QualiaDB: Semantic orchestrator (48-byte Super-Quin logic)
9//! - Extensions: Heavy computational work (std, GPU, external APIs)
10//! - Communication: FFI bridge with strict memory boundaries
11//!
12//! # Extension Types
13//!
14//! - QPU Extension: Quantum computing via remote APIs
15//! - PINN Extension: Physics-Informed Neural Networks (uses native Qualia LLM pipeline with wgpu + WGSL)
16//! - SNN Extension: Spiking Neural Networks with CRDT synchronization
17//! - Fluid Extension: WebGPU-based fluid dynamics
18//! - Math Extension: Advanced mathematical solvers
19//!
20//! # Native Pipeline Integration
21//!
22//! The PINN extension now uses the native Qualia LLM pipeline:
23//! - wgpu for GPU compute (not Candle)
24//! - Custom WGSL compute shaders
25//! - memmap2 for GGUF model loading
26//! - Same infrastructure as the core LLM agent
27
28#![allow(dead_code)]
29
30use serde::{Deserialize, Serialize};
31use std::collections::HashMap;
32use std::sync::{Arc, Mutex};
33
34/// Extension capability descriptor
35#[derive(Debug, Clone, Serialize, Deserialize)]
36pub struct ExtensionCapability {
37    pub name: String,
38    pub version: String,
39    pub description: String,
40    pub required_resources: ResourceRequirements,
41    pub supported_operations: Vec<String>,
42}
43
44/// Resource requirements for an extension
45#[derive(Debug, Clone, Serialize, Deserialize)]
46pub struct ResourceRequirements {
47    pub min_memory_mb: u64,
48    pub min_vram_mb: Option<u64>,
49    pub requires_gpu: bool,
50    pub requires_network: bool,
51    pub max_concurrent_jobs: u32,
52}
53
54/// Extension job request from core QualiaDB
55#[derive(Debug, Clone, Serialize, Deserialize)]
56pub struct ExtensionJob {
57    pub job_id: String,
58    pub extension_name: String,
59    pub operation: String,
60    pub parameters: HashMap<String, serde_json::Value>,
61    pub boundary_conditions: Vec<NQuin>,
62}
63
64/// Extension job result for core QualiaDB
65#[derive(Debug, Clone, Serialize, Deserialize)]
66pub struct ExtensionResult {
67    pub job_id: String,
68    pub success: bool,
69    pub result_quins: Vec<NQuin>,
70    pub metadata: HashMap<String, String>,
71    pub execution_time_ms: u64,
72}
73
74/// NQuin representation for extension communication
75#[derive(Debug, Clone, Serialize, Deserialize)]
76pub struct NQuin {
77    pub subject: u64,
78    pub predicate: u64,
79    pub object: u64,
80    pub context: u64,
81    pub metadata: u64,
82    pub parity: u64,
83}
84
85/// Extension registry and manager
86pub struct ExtensionManager {
87    extensions: HashMap<String, Box<dyn Extension>>,
88    active_jobs: Arc<Mutex<HashMap<String, ExtensionJob>>>,
89}
90
91impl ExtensionManager {
92    /// Creates a new instance.
93    pub fn new() -> Self {
94        Self {
95            extensions: HashMap::new(),
96            active_jobs: Arc::new(Mutex::new(HashMap::new())),
97        }
98    }
99
100    /// Registers extension.
101    pub fn register_extension(&mut self, extension: Box<dyn Extension>) {
102        let capability = extension.capability();
103        self.extensions.insert(capability.name.clone(), extension);
104    }
105
106    /// Executes a job.
107    pub async fn execute_job(&self, job: ExtensionJob) -> Result<ExtensionResult, ExtensionError> {
108        let extension =
109            self.extensions
110                .get(&job.extension_name)
111                .ok_or(ExtensionError::ExtensionNotFound(
112                    job.extension_name.clone(),
113                ))?;
114
115        // Track active job
116        {
117            let mut active = self.active_jobs.lock().unwrap();
118            active.insert(job.job_id.clone(), job.clone());
119        }
120
121        // Execute the job
122        let result = extension.execute(job).await;
123
124        // Remove from active jobs
125        {
126            let mut active = self.active_jobs.lock().unwrap();
127            active.remove(
128                &result
129                    .as_ref()
130                    .map(|r| r.job_id.clone())
131                    .unwrap_or_default(),
132            );
133        }
134
135        result
136    }
137
138    /// Lists capabilities.
139    pub fn list_capabilities(&self) -> Vec<ExtensionCapability> {
140        self.extensions
141            .values()
142            .map(|ext| ext.capability())
143            .collect()
144    }
145}
146
147/// Extension trait for all computational extensions
148#[async_trait::async_trait]
149pub trait Extension: Send + Sync {
150    fn capability(&self) -> ExtensionCapability;
151    async fn execute(&self, job: ExtensionJob) -> Result<ExtensionResult, ExtensionError>;
152    fn shutdown(&self) -> Result<(), ExtensionError>;
153}
154
155/// Extension error types
156#[derive(Debug, thiserror::Error)]
157pub enum ExtensionError {
158    #[error("Extension '{0}' not found")]
159    ExtensionNotFound(String),
160
161    #[error("Operation '{0}' not supported by extension")]
162    OperationNotSupported(String),
163
164    #[error("Insufficient resources: {0}")]
165    InsufficientResources(String),
166
167    #[error("Job execution failed: {0}")]
168    ExecutionFailed(String),
169
170    #[error("Network error: {0}")]
171    NetworkError(String),
172
173    #[error("GPU error: {0}")]
174    GpuError(String),
175}
176
177/// FFI bridge for core QualiaDB communication
178#[repr(C)]
179pub struct ExtensionBridge {
180    manager: *mut ExtensionManager,
181}
182
183#[repr(C)]
184pub struct CExtensionJob {
185    pub job_id: *const u8,
186    pub job_id_len: usize,
187    pub extension_name: *const u8,
188    pub extension_name_len: usize,
189    pub operation: *const u8,
190    pub operation_len: usize,
191    /// UTF-8 JSON object of `parameters` (may be null/0 ⇒ no parameters).
192    pub parameters: *const u8,
193    pub parameters_len: usize,
194}
195
196#[repr(C)]
197pub struct CExtensionResult {
198    pub success: bool,
199    /// UTF-8 JSON of the [`ExtensionResult`] on success (else null).
200    /// Free with [`extension_result_free`].
201    pub result_data: *const u8,
202    pub result_len: usize,
203    /// UTF-8 error message on failure (else null). Free with
204    /// [`extension_result_free`].
205    pub error_msg: *const u8,
206    pub error_len: usize,
207}
208
209/// FFI: allocates an `ExtensionManager` and returns a raw pointer (caller frees via `extension_manager_free`).
210#[no_mangle]
211pub extern "C" fn extension_manager_new() -> *mut ExtensionManager {
212    Box::into_raw(Box::new(ExtensionManager::new()))
213}
214
215/// Process-wide runtime used to drive async extension jobs across the (sync)
216/// FFI boundary. Built once; `None` only if Tokio fails to initialise.
217fn ffi_runtime() -> Option<&'static tokio::runtime::Runtime> {
218    use std::sync::OnceLock;
219    static RT: OnceLock<Option<tokio::runtime::Runtime>> = OnceLock::new();
220    RT.get_or_init(|| {
221        tokio::runtime::Builder::new_multi_thread()
222            .enable_all()
223            .build()
224            .ok()
225    })
226    .as_ref()
227}
228
229/// SAFETY: `ptr`/`len` must describe a valid byte range for `'a`, or `ptr` may
230/// be null with `len == 0`.
231unsafe fn ffi_str<'a>(ptr: *const u8, len: usize) -> Option<&'a str> {
232    if ptr.is_null() {
233        return if len == 0 { Some("") } else { None };
234    }
235    std::str::from_utf8(std::slice::from_raw_parts(ptr, len)).ok()
236}
237
238fn heap_bytes(bytes: Vec<u8>) -> (*const u8, usize) {
239    if bytes.is_empty() {
240        return (std::ptr::null(), 0);
241    }
242    let boxed = bytes.into_boxed_slice();
243    let len = boxed.len();
244    (Box::into_raw(boxed) as *const u8, len)
245}
246
247fn ffi_ok(json: Vec<u8>) -> CExtensionResult {
248    let (ptr, len) = heap_bytes(json);
249    CExtensionResult {
250        success: true,
251        result_data: ptr,
252        result_len: len,
253        error_msg: std::ptr::null(),
254        error_len: 0,
255    }
256}
257
258fn ffi_err(msg: impl Into<String>) -> CExtensionResult {
259    let (ptr, len) = heap_bytes(msg.into().into_bytes());
260    CExtensionResult {
261        success: false,
262        result_data: std::ptr::null(),
263        result_len: 0,
264        error_msg: ptr,
265        error_len: len,
266    }
267}
268
269/// Execute an extension job submitted over the C ABI.
270///
271/// Marshals the [`CExtensionJob`] (UTF-8 fields + JSON `parameters`) into an
272/// [`ExtensionJob`], runs it on the shared runtime, and returns the
273/// JSON-serialised [`ExtensionResult`] (on success) or an error message. The
274/// returned buffers are heap-owned and must be released with
275/// [`extension_result_free`].
276///
277/// # Safety
278/// `manager` must be a pointer returned by [`extension_manager_new`] (and not
279/// yet freed); `job` must point to a valid `CExtensionJob` whose byte ranges
280/// are valid for the duration of the call. Must not be called from within an
281/// existing Tokio runtime on the same thread.
282#[no_mangle]
283pub extern "C" fn extension_manager_execute_job(
284    manager: *mut ExtensionManager,
285    job: *const CExtensionJob,
286) -> CExtensionResult {
287    if manager.is_null() {
288        return ffi_err("null manager pointer");
289    }
290    if job.is_null() {
291        return ffi_err("null job pointer");
292    }
293
294    let mgr = unsafe { &*manager };
295    let cjob = unsafe { &*job };
296
297    let job_id = match unsafe { ffi_str(cjob.job_id, cjob.job_id_len) } {
298        Some(s) => s.to_string(),
299        None => return ffi_err("job_id is not valid UTF-8"),
300    };
301    let extension_name = match unsafe { ffi_str(cjob.extension_name, cjob.extension_name_len) } {
302        Some(s) => s.to_string(),
303        None => return ffi_err("extension_name is not valid UTF-8"),
304    };
305    let operation = match unsafe { ffi_str(cjob.operation, cjob.operation_len) } {
306        Some(s) => s.to_string(),
307        None => return ffi_err("operation is not valid UTF-8"),
308    };
309    let parameters: HashMap<String, serde_json::Value> =
310        if cjob.parameters.is_null() || cjob.parameters_len == 0 {
311            HashMap::new()
312        } else {
313            let raw = unsafe { std::slice::from_raw_parts(cjob.parameters, cjob.parameters_len) };
314            match serde_json::from_slice(raw) {
315                Ok(v) => v,
316                Err(e) => return ffi_err(format!("invalid parameters JSON: {e}")),
317            }
318        };
319
320    let ext_job = ExtensionJob {
321        job_id,
322        extension_name,
323        operation,
324        parameters,
325        boundary_conditions: Vec::new(),
326    };
327
328    let rt = match ffi_runtime() {
329        Some(rt) => rt,
330        None => return ffi_err("failed to initialise async runtime"),
331    };
332
333    match rt.block_on(mgr.execute_job(ext_job)) {
334        Ok(result) => match serde_json::to_vec(&result) {
335            Ok(json) => ffi_ok(json),
336            Err(e) => ffi_err(format!("failed to serialise result: {e}")),
337        },
338        Err(e) => ffi_err(e.to_string()),
339    }
340}
341
342/// Release the heap buffers owned by a [`CExtensionResult`].
343///
344/// # Safety
345/// `result` must be a value previously returned by
346/// [`extension_manager_execute_job`] and not already freed.
347#[no_mangle]
348pub extern "C" fn extension_result_free(result: CExtensionResult) {
349    unsafe {
350        if !result.result_data.is_null() && result.result_len > 0 {
351            let _ = Box::from_raw(std::slice::from_raw_parts_mut(
352                result.result_data as *mut u8,
353                result.result_len,
354            ));
355        }
356        if !result.error_msg.is_null() && result.error_len > 0 {
357            let _ = Box::from_raw(std::slice::from_raw_parts_mut(
358                result.error_msg as *mut u8,
359                result.error_len,
360            ));
361        }
362    }
363}
364
365/// FFI: frees an `ExtensionManager` allocated by `extension_manager_new`; `NULL` is a no-op.
366#[no_mangle]
367pub extern "C" fn extension_manager_free(manager: *mut ExtensionManager) {
368    if !manager.is_null() {
369        unsafe {
370            let _ = Box::from_raw(manager);
371        }
372    }
373}
374
375// Module declarations
376pub mod pinn_extension;
377pub mod qpu_extension;
378pub mod snn_extension;
379pub mod webgpu_extension;
380
381/// Helper function for hashing strings to 64-bit integers
382pub fn q_hash(input: &str) -> u64 {
383    use std::collections::hash_map::DefaultHasher;
384    use std::hash::{Hash, Hasher};
385    let mut hasher = DefaultHasher::new();
386    input.hash(&mut hasher);
387    hasher.finish()
388}
389
390#[cfg(test)]
391mod tests {
392    use super::*;
393
394    #[test]
395    fn test_quin_conversion() {
396        let core_quin = crate::NQuin {
397            subject: 1,
398            predicate: 2,
399            object: 3,
400            context: 4,
401            metadata: 5,
402            parity: 6,
403        };
404
405        let ext_quin = NQuin::from(core_quin.clone());
406        let converted_back = crate::NQuin::from(ext_quin);
407
408        assert_eq!(core_quin.subject, converted_back.subject);
409        assert_eq!(core_quin.predicate, converted_back.predicate);
410        assert_eq!(core_quin.object, converted_back.object);
411        assert_eq!(core_quin.context, converted_back.context);
412        assert_eq!(core_quin.metadata, converted_back.metadata);
413        assert_eq!(core_quin.parity, converted_back.parity);
414    }
415
416    /// Build a manager with the (real) webgpu extension registered and leak it
417    /// to a raw pointer for the FFI calls.
418    fn manager_with_webgpu() -> *mut ExtensionManager {
419        let mut manager = ExtensionManager::new();
420        manager.register_extension(Box::new(crate::webgpu_extension::WebGpuExtension::new()));
421        Box::into_raw(Box::new(manager))
422    }
423
424    fn cjob(job_id: &[u8], ext: &[u8], op: &[u8], params: &[u8]) -> CExtensionJob {
425        CExtensionJob {
426            job_id: job_id.as_ptr(),
427            job_id_len: job_id.len(),
428            extension_name: ext.as_ptr(),
429            extension_name_len: ext.len(),
430            operation: op.as_ptr(),
431            operation_len: op.len(),
432            parameters: params.as_ptr(),
433            parameters_len: params.len(),
434        }
435    }
436
437    #[test]
438    fn ffi_bridge_executes_a_real_job() {
439        let mgr = manager_with_webgpu();
440        let job = cjob(b"ffi-1", b"webgpu", b"tensor_operations", b"{}");
441        let res = extension_manager_execute_job(mgr, &job);
442
443        assert!(res.success, "FFI job did not succeed");
444        assert!(!res.result_data.is_null());
445        let bytes = unsafe { std::slice::from_raw_parts(res.result_data, res.result_len) };
446        let parsed: ExtensionResult = serde_json::from_slice(bytes).expect("result JSON");
447        assert_eq!(parsed.job_id, "ffi-1");
448        assert!(parsed.success);
449        assert!(!parsed.result_quins.is_empty());
450
451        extension_result_free(res);
452        extension_manager_free(mgr);
453    }
454
455    #[test]
456    fn ffi_bridge_reports_unknown_extension() {
457        let mgr = manager_with_webgpu();
458        let job = cjob(b"ffi-2", b"nonexistent", b"tensor_operations", b"{}");
459        let res = extension_manager_execute_job(mgr, &job);
460
461        assert!(!res.success);
462        assert!(res.result_data.is_null());
463        let msg = unsafe { std::slice::from_raw_parts(res.error_msg, res.error_len) };
464        assert!(std::str::from_utf8(msg).unwrap().contains("nonexistent"));
465
466        extension_result_free(res);
467        extension_manager_free(mgr);
468    }
469
470    #[test]
471    fn ffi_bridge_rejects_null_manager() {
472        let job = cjob(b"ffi-3", b"webgpu", b"tensor_operations", b"{}");
473        let res = extension_manager_execute_job(std::ptr::null_mut(), &job);
474        assert!(!res.success);
475        assert!(res.error_len > 0);
476        extension_result_free(res);
477    }
478
479    #[test]
480    fn ffi_bridge_rejects_bad_parameters_json() {
481        let mgr = manager_with_webgpu();
482        let job = cjob(b"ffi-4", b"webgpu", b"tensor_operations", b"{not json");
483        let res = extension_manager_execute_job(mgr, &job);
484        assert!(!res.success);
485        let msg = unsafe { std::slice::from_raw_parts(res.error_msg, res.error_len) };
486        assert!(std::str::from_utf8(msg)
487            .unwrap()
488            .contains("parameters JSON"));
489        extension_result_free(res);
490        extension_manager_free(mgr);
491    }
492}