Skip to main content

qualia_core_db/
csd_storage.rs

1//! NVMe Computational Storage (CSD) Implementation
2//!
3//! This module provides computational storage pushdown using NVMe CSD (Computational Storage Device).
4//! Designed for hardware-accelerated mathematical computations and query processing.
5
6use crate::q_hash;
7use std::collections::HashMap;
8use std::fs::OpenOptions;
9
10use serde::{Deserialize, Serialize};
11use std::path::Path;
12
13/// CSD Manager for computational storage operations
14pub struct CsdManager {
15    devices: HashMap<String, CsdDevice>,
16    functions: HashMap<String, CsdFunction>,
17    scheduler: CsdScheduler,
18    performance_monitor: CsdPerformanceMonitor,
19}
20
21/// Compact, zero-heap view of a discovered CSD device.
22#[derive(Debug, Clone, Copy, PartialEq)]
23pub struct CsdDeviceHandle {
24    pub device_id_hash: u64,
25    pub max_concurrent_operations: u32,
26    pub max_data_size: u64,
27    pub memory_size: u64,
28    pub compute_units: u32,
29}
30
31/// Compact, zero-heap view of a registered computational function.
32#[derive(Debug, Clone, Copy, PartialEq)]
33pub struct CsdFunctionHandle {
34    pub function_id_hash: u64,
35    pub operation_tag: u8,
36    pub parameter_count: u16,
37    pub bytecode_len: u32,
38}
39
40/// CSD device information
41#[derive(Debug, Clone)]
42pub struct CsdDevice {
43    pub device_id: String,
44    pub device_path: String,
45    pub capabilities: CsdCapabilities,
46    pub supported_functions: Vec<String>,
47    pub device_stats: CsdDeviceStats,
48}
49
50/// CSD device capabilities
51#[derive(Debug, Clone, Serialize, Deserialize)]
52pub struct CsdCapabilities {
53    pub max_concurrent_operations: u32,
54    pub max_data_size: u64,
55    pub supported_operations: Vec<CsdOperationType>,
56    pub memory_size: u64,
57    pub compute_units: u32,
58    pub clock_speed: f64,
59}
60
61/// CSD operations supported by device
62#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
63pub enum CsdOperationType {
64    MatrixMultiply,
65    VectorDotProduct,
66    Convolution,
67    Filter,
68    Aggregate,
69    Sort,
70    Search,
71    Custom(String),
72}
73
74/// CSD function for computational operations
75#[derive(Debug, Clone)]
76pub struct CsdFunction {
77    pub function_id: String,
78    pub operation: CsdOperationType,
79    pub parameters: Vec<FunctionParameter>,
80    pub bytecode: Vec<u8>,
81    pub performance_profile: PerformanceProfile,
82}
83
84/// Function parameter definition
85#[derive(Debug, Clone, Serialize, Deserialize)]
86pub struct FunctionParameter {
87    pub name: String,
88    pub param_type: ParameterType,
89    pub size: u64,
90    pub is_input: bool,
91    pub is_output: bool,
92}
93
94/// Parameter types
95#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
96pub enum ParameterType {
97    Matrix,
98    Vector,
99    Scalar,
100    Tensor,
101    Buffer,
102}
103
104/// Performance profile for functions
105#[derive(Debug, Clone)]
106pub struct PerformanceProfile {
107    pub expected_execution_time: f64,
108    pub memory_usage: u64,
109    pub compute_intensity: f64,
110    pub data_intensity: f64,
111}
112
113/// CSD device statistics
114#[derive(Debug, Clone)]
115pub struct CsdDeviceStats {
116    pub operations_completed: u64,
117    pub total_execution_time: u64,
118    pub average_execution_time: f64,
119    pub data_processed: u64,
120    pub error_count: u64,
121    pub utilization: f64,
122}
123
124/// CSD scheduler for operation management
125pub struct CsdScheduler {
126    pending_operations: Vec<CsdOperationRequest>,
127    running_operations: HashMap<u64, CsdRunningOperation>,
128    completion_queue: Vec<CsdCompletion>,
129    scheduling_policy: SchedulingPolicy,
130}
131
132/// CSD operation request to be executed
133#[derive(Debug, Clone, Serialize, Deserialize)]
134pub struct CsdOperationRequest {
135    pub operation_id: u64,
136    pub function_id: String,
137    pub device_id: String,
138    pub inputs: Vec<OperationInput>,
139    pub outputs: Vec<OperationOutput>,
140    pub priority: OperationPriority,
141    pub deadline: Option<u64>,
142}
143
144/// Operation input
145#[derive(Debug, Clone, Serialize, Deserialize)]
146pub struct OperationInput {
147    pub name: String,
148    pub data: Vec<u8>,
149    pub location: DataLocation,
150}
151
152/// Operation output
153#[derive(Debug, Clone, Serialize, Deserialize)]
154pub struct OperationOutput {
155    pub name: String,
156    pub size: u64,
157    pub location: DataLocation,
158}
159
160/// Data location for operations
161#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
162pub enum DataLocation {
163    HostMemory,
164    DeviceMemory,
165    PersistentStorage,
166}
167
168/// Operation priority
169#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
170pub enum OperationPriority {
171    Low,
172    Normal,
173    High,
174    Critical,
175}
176
177/// Running operation tracking
178#[derive(Debug, Clone)]
179pub struct CsdRunningOperation {
180    pub operation_id: u64,
181    pub device_id: String,
182    pub start_time: u64,
183    pub progress: f64,
184}
185
186/// Operation completion result
187#[derive(Debug, Clone)]
188pub struct CsdCompletion {
189    pub operation_id: u64,
190    pub status: CompletionStatus,
191    pub execution_time: u64,
192    pub outputs: Vec<OperationOutput>,
193    pub error_message: Option<String>,
194}
195
196/// Completion status
197#[derive(Debug, Clone, PartialEq)]
198pub enum CompletionStatus {
199    Success,
200    Error,
201    Timeout,
202    Cancelled,
203}
204
205/// Scheduling policies
206#[derive(Debug, Clone)]
207pub enum SchedulingPolicy {
208    /// First-In-First-Out
209    Fifo,
210    /// Priority-based scheduling
211    Priority,
212    /// Shortest-Job-First
213    ShortestJobFirst,
214    /// Deadline-Driven Scheduling
215    Deadline,
216    /// Load-Balanced Scheduling
217    LoadBalanced,
218}
219
220/// CSD performance monitor
221pub struct CsdPerformanceMonitor {
222    device_metrics: HashMap<String, CsdDeviceMetrics>,
223    function_metrics: HashMap<String, CsdFunctionMetrics>,
224    global_metrics: CsdGlobalMetrics,
225}
226
227/// Device performance metrics
228#[derive(Debug, Clone)]
229pub struct CsdDeviceMetrics {
230    pub device_id: String,
231    pub utilization: f64,
232    pub throughput: f64,
233    pub latency: f64,
234    pub error_rate: f64,
235    pub power_consumption: f64,
236}
237
238/// Function performance metrics
239#[derive(Debug, Clone)]
240pub struct CsdFunctionMetrics {
241    pub function_id: String,
242    pub execution_count: u64,
243    pub total_execution_time: u64,
244    pub average_execution_time: f64,
245    pub success_rate: f64,
246    pub data_throughput: f64,
247}
248
249/// Global performance metrics
250#[derive(Debug, Clone)]
251pub struct CsdGlobalMetrics {
252    pub total_operations: u64,
253    pub total_execution_time: u64,
254    pub average_execution_time: f64,
255    pub total_data_processed: u64,
256    pub overall_throughput: f64,
257    pub system_utilization: f64,
258}
259
260/// Mathematical computation builder
261pub struct MathComputationBuilder {
262    operations: Vec<CsdOperationRequest>,
263    data_dependencies: HashMap<String, Vec<String>>,
264    execution_plan: ExecutionPlan,
265}
266
267/// Execution plan for computations
268#[derive(Debug, Clone)]
269pub struct ExecutionPlan {
270    pub stages: Vec<ExecutionStage>,
271    pub parallel_groups: Vec<Vec<String>>,
272    pub estimated_time: f64,
273    pub resource_requirements: ResourceRequirements,
274}
275
276/// Execution stage
277#[derive(Debug, Clone)]
278pub struct ExecutionStage {
279    pub stage_id: u32,
280    pub operations: Vec<String>,
281    pub dependencies: Vec<String>,
282    pub estimated_time: f64,
283}
284
285/// Resource requirements
286#[derive(Debug, Clone)]
287pub struct ResourceRequirements {
288    pub memory_usage: u64,
289    pub compute_units: u32,
290    pub bandwidth: f64,
291}
292
293impl CsdManager {
294    /// Create new CSD manager
295    pub fn new() -> Self {
296        Self {
297            devices: HashMap::new(),
298            functions: HashMap::new(),
299            scheduler: CsdScheduler::new(),
300            performance_monitor: CsdPerformanceMonitor::new(),
301        }
302    }
303
304    /// Registers `device`.
305    pub fn register_device(&mut self, device: CsdDevice) -> Result<(), CsdError> {
306        self.devices.insert(device.device_id.clone(), device);
307        Ok(())
308    }
309
310    /// Discover CSD devices
311    pub fn discover_devices(&mut self) -> Result<Vec<String>, CsdError> {
312        let mut discovered_devices = Vec::new();
313
314        // Scan for CSD devices
315        for i in 0..16 {
316            let device_path = format!("/dev/nvme{}", i);
317            if Path::new(&device_path).exists() {
318                if let Ok(device) = self.probe_device(&device_path) {
319                    discovered_devices.push(device.device_id.clone());
320                    self.devices.insert(device.device_id.clone(), device);
321                }
322            }
323        }
324
325        Ok(discovered_devices)
326    }
327
328    /// Discover devices into a caller-owned fixed buffer.
329    pub fn discover_devices_into(
330        &mut self,
331        out: &mut [CsdDeviceHandle],
332    ) -> Result<usize, CsdError> {
333        let mut written = 0;
334
335        for i in 0..16 {
336            let device_path = format!("/dev/nvme{}", i);
337            if Path::new(&device_path).exists() {
338                if let Ok(device) = self.probe_device(&device_path) {
339                    if written >= out.len() {
340                        return Err(CsdError::BufferTooSmall(
341                            "device discovery output buffer exhausted".to_string(),
342                        ));
343                    }
344                    out[written] = Self::device_handle(&device);
345                    written += 1;
346                    self.devices.insert(device.device_id.clone(), device);
347                }
348            }
349        }
350
351        Ok(written)
352    }
353
354    /// Probe CSD device
355    fn probe_device(&self, device_path: &str) -> Result<CsdDevice, CsdError> {
356        let _device_file = OpenOptions::new()
357            .read(true)
358            .write(true)
359            .open(device_path)
360            .map_err(|e| CsdError::DeviceOpen(e.to_string()))?;
361
362        // Get device information using ioctl
363        let device_id = format!("csd-{}", device_path);
364
365        // For now, use reasonable defaults
366        let capabilities = CsdCapabilities {
367            max_concurrent_operations: 16,
368            max_data_size: 1024 * 1024 * 1024, // 1GB
369            supported_operations: vec![
370                CsdOperationType::MatrixMultiply,
371                CsdOperationType::VectorDotProduct,
372                CsdOperationType::Convolution,
373                CsdOperationType::Filter,
374                CsdOperationType::Aggregate,
375            ],
376            memory_size: 8 * 1024 * 1024 * 1024, // 8GB
377            compute_units: 64,
378            clock_speed: 1.5, // 1.5 GHz
379        };
380
381        let device = CsdDevice {
382            device_id: device_id.clone(),
383            device_path: device_path.to_string(),
384            capabilities,
385            supported_functions: vec![], // Will be populated later
386            device_stats: CsdDeviceStats {
387                operations_completed: 0,
388                total_execution_time: 0,
389                average_execution_time: 0.0,
390                data_processed: 0,
391                error_count: 0,
392                utilization: 0.0,
393            },
394        };
395
396        Ok(device)
397    }
398
399    /// Register computational function
400    pub fn register_function(&mut self, function: CsdFunction) -> Result<(), CsdError> {
401        // Validate function
402        self.validate_function(&function)?;
403
404        // Store function
405        self.functions
406            .insert(function.function_id.clone(), function);
407
408        Ok(())
409    }
410
411    /// Execute computational operation
412    pub fn execute_operation(&mut self, operation: CsdOperationRequest) -> Result<u64, CsdError> {
413        // Validate operation
414        self.validate_operation(&operation)?;
415
416        // Schedule operation
417        self.scheduler.schedule_operation(operation.clone())?;
418
419        Ok(operation.operation_id)
420    }
421
422    /// Execute matrix multiplication
423    pub fn matrix_multiply(
424        &mut self,
425        device_id: &str,
426        a: &[f32],
427        b: &[f32],
428        dimensions: (usize, usize, usize),
429    ) -> Result<Vec<f32>, CsdError> {
430        let mut result = vec![0.0f32; dimensions.0 * dimensions.2];
431        let written = self.matrix_multiply_into(device_id, a, b, dimensions, &mut result)?;
432        result.truncate(written);
433        Ok(result)
434    }
435
436    /// Execute matrix multiplication into a caller-owned output buffer.
437    pub fn matrix_multiply_into(
438        &mut self,
439        device_id: &str,
440        a: &[f32],
441        b: &[f32],
442        dimensions: (usize, usize, usize),
443        out: &mut [f32],
444    ) -> Result<usize, CsdError> {
445        self.ensure_device_exists(device_id)?;
446        let (rows_a, shared, cols_b) = dimensions;
447        if a.len() != rows_a * shared {
448            return Err(CsdError::InvalidOperation(
449                "matrix A dimensions do not match input length".to_string(),
450            ));
451        }
452        if b.len() != shared * cols_b {
453            return Err(CsdError::InvalidOperation(
454                "matrix B dimensions do not match input length".to_string(),
455            ));
456        }
457        let required = rows_a * cols_b;
458        if out.len() < required {
459            return Err(CsdError::BufferTooSmall(
460                "matrix multiply output buffer too small".to_string(),
461            ));
462        }
463
464        let operation_id = self.generate_operation_id();
465        let operation = CsdOperationRequest {
466            operation_id,
467            function_id: "matrix_multiply".to_string(),
468            device_id: device_id.to_string(),
469            inputs: vec![
470                OperationInput {
471                    name: "matrix_a".to_string(),
472                    data: self.f32_slice_to_bytes(a),
473                    location: DataLocation::HostMemory,
474                },
475                OperationInput {
476                    name: "matrix_b".to_string(),
477                    data: self.f32_slice_to_bytes(b),
478                    location: DataLocation::HostMemory,
479                },
480                OperationInput {
481                    name: "dimensions".to_string(),
482                    data: self.serialize_dimensions(dimensions),
483                    location: DataLocation::HostMemory,
484                },
485            ],
486            outputs: vec![OperationOutput {
487                name: "result".to_string(),
488                size: (required * 4) as u64,
489                location: DataLocation::HostMemory,
490            }],
491            priority: OperationPriority::Normal,
492            deadline: None,
493        };
494        if self.execute_operation(operation).is_ok() {
495            if let Ok(completion) = self.wait_for_completion(operation_id) {
496                if let Some(output) = completion.outputs.first() {
497                    let floats = self.bytes_to_f32_slice(output);
498                    let n = floats.len().min(required);
499                    out[..n].copy_from_slice(&floats[..n]);
500                    return Ok(n);
501                }
502            }
503        }
504
505        for row in 0..rows_a {
506            for col in 0..cols_b {
507                let mut sum = 0.0f32;
508                for inner in 0..shared {
509                    sum += a[row * shared + inner] * b[inner * cols_b + col];
510                }
511                out[row * cols_b + col] = sum;
512            }
513        }
514
515        Ok(required)
516    }
517
518    /// Execute vector dot product
519    pub fn vector_dot_product(
520        &mut self,
521        device_id: &str,
522        a: &[f32],
523        b: &[f32],
524    ) -> Result<f32, CsdError> {
525        let operation_id = self.generate_operation_id();
526
527        let operation = CsdOperationRequest {
528            operation_id,
529            function_id: "vector_dot_product".to_string(),
530            device_id: device_id.to_string(),
531            inputs: vec![
532                OperationInput {
533                    name: "vector_a".to_string(),
534                    data: self.f32_slice_to_bytes(a),
535                    location: DataLocation::HostMemory,
536                },
537                OperationInput {
538                    name: "vector_b".to_string(),
539                    data: self.f32_slice_to_bytes(b),
540                    location: DataLocation::HostMemory,
541                },
542            ],
543            outputs: vec![OperationOutput {
544                name: "result".to_string(),
545                size: 4, // 4 bytes for f32
546                location: DataLocation::HostMemory,
547            }],
548            priority: OperationPriority::Normal,
549            deadline: None,
550        };
551
552        self.execute_operation(operation)?;
553        let completion = self.wait_for_completion(operation_id)?;
554
555        if let Some(output) = completion.outputs.first() {
556            let result = self.bytes_to_f32_value(&output);
557            Ok(result)
558        } else {
559            Err(CsdError::NoOutput("No output generated".to_string()))
560        }
561    }
562
563    /// Execute convolution operation
564    pub fn convolution(
565        &mut self,
566        device_id: &str,
567        input: &[f32],
568        kernel: &[f32],
569        dimensions: (usize, usize, usize, usize),
570    ) -> Result<Vec<f32>, CsdError> {
571        let mut result = vec![0.0f32; dimensions.0 * dimensions.1];
572        let written = self.convolution_into(device_id, input, kernel, dimensions, &mut result)?;
573        result.truncate(written);
574        Ok(result)
575    }
576
577    /// Execute a same-shape 2D convolution into a caller-owned output buffer.
578    pub fn convolution_into(
579        &mut self,
580        device_id: &str,
581        input: &[f32],
582        kernel: &[f32],
583        dimensions: (usize, usize, usize, usize),
584        out: &mut [f32],
585    ) -> Result<usize, CsdError> {
586        self.ensure_device_exists(device_id)?;
587        let _dim_wire = self.serialize_convolution_dimensions(dimensions);
588        let (width, height, kernel_width, kernel_height) = dimensions;
589        if input.len() != width * height {
590            return Err(CsdError::InvalidOperation(
591                "convolution input dimensions do not match input length".to_string(),
592            ));
593        }
594        if kernel.len() != kernel_width * kernel_height {
595            return Err(CsdError::InvalidOperation(
596                "convolution kernel dimensions do not match kernel length".to_string(),
597            ));
598        }
599        let required = width * height;
600        if out.len() < required {
601            return Err(CsdError::BufferTooSmall(
602                "convolution output buffer too small".to_string(),
603            ));
604        }
605
606        let kernel_x_radius = kernel_width / 2;
607        let kernel_y_radius = kernel_height / 2;
608
609        for y in 0..height {
610            for x in 0..width {
611                let mut acc = 0.0f32;
612                for ky in 0..kernel_height {
613                    let Some(input_y) = y
614                        .checked_add(ky)
615                        .and_then(|value| value.checked_sub(kernel_y_radius))
616                    else {
617                        continue;
618                    };
619                    if input_y >= height {
620                        continue;
621                    }
622                    for kx in 0..kernel_width {
623                        let Some(input_x) = x
624                            .checked_add(kx)
625                            .and_then(|value| value.checked_sub(kernel_x_radius))
626                        else {
627                            continue;
628                        };
629                        if input_x >= width {
630                            continue;
631                        }
632                        let input_index = input_y * width + input_x;
633                        let kernel_index = ky * kernel_width + kx;
634                        acc += input[input_index] * kernel[kernel_index];
635                    }
636                }
637                out[y * width + x] = acc;
638            }
639        }
640
641        Ok(required)
642    }
643
644    /// Get device statistics
645    pub fn get_device_stats(&self, device_id: &str) -> Option<CsdDeviceStats> {
646        self.devices
647            .get(device_id)
648            .map(|device| device.device_stats.clone())
649    }
650
651    /// Get performance statistics
652    pub fn get_performance_stats(&self) -> CsdGlobalMetrics {
653        self.performance_monitor.get_global_stats()
654    }
655
656    /// List available devices
657    pub fn list_devices(&self) -> Vec<String> {
658        self.devices.keys().cloned().collect()
659    }
660
661    /// List available devices into a compact caller-owned buffer.
662    pub fn list_devices_into(&self, out: &mut [CsdDeviceHandle]) -> Result<usize, CsdError> {
663        if out.len() < self.devices.len() {
664            return Err(CsdError::BufferTooSmall(
665                "device listing output buffer too small".to_string(),
666            ));
667        }
668
669        let mut written = 0;
670        for device in self.devices.values() {
671            out[written] = Self::device_handle(device);
672            written += 1;
673        }
674        Ok(written)
675    }
676
677    /// List available functions
678    pub fn list_functions(&self) -> Vec<String> {
679        self.functions.keys().cloned().collect()
680    }
681
682    /// List available functions into a compact caller-owned buffer.
683    pub fn list_functions_into(&self, out: &mut [CsdFunctionHandle]) -> Result<usize, CsdError> {
684        if out.len() < self.functions.len() {
685            return Err(CsdError::BufferTooSmall(
686                "function listing output buffer too small".to_string(),
687            ));
688        }
689
690        let mut written = 0;
691        for function in self.functions.values() {
692            out[written] = Self::function_handle(function);
693            written += 1;
694        }
695        Ok(written)
696    }
697
698    // Internal methods
699
700    /// Validate function
701    fn validate_function(&self, function: &CsdFunction) -> Result<(), CsdError> {
702        if function.function_id.is_empty() {
703            return Err(CsdError::InvalidFunction(
704                "Function ID cannot be empty".to_string(),
705            ));
706        }
707
708        if function.bytecode.is_empty() {
709            return Err(CsdError::InvalidFunction(
710                "Function bytecode cannot be empty".to_string(),
711            ));
712        }
713
714        Ok(())
715    }
716
717    /// Validate operation
718    fn validate_operation(&self, operation: &CsdOperationRequest) -> Result<(), CsdError> {
719        // Check if device exists
720        if !self.devices.contains_key(&operation.device_id) {
721            return Err(CsdError::DeviceNotFound(operation.device_id.clone()));
722        }
723
724        // Check if function exists
725        if !self.functions.contains_key(&operation.function_id) {
726            return Err(CsdError::FunctionNotFound(operation.function_id.clone()));
727        }
728
729        // Check inputs
730        if operation.inputs.is_empty() {
731            return Err(CsdError::InvalidOperation(
732                "Operation must have inputs".to_string(),
733            ));
734        }
735
736        Ok(())
737    }
738
739    fn ensure_device_exists(&self, device_id: &str) -> Result<(), CsdError> {
740        if self.devices.contains_key(device_id) {
741            Ok(())
742        } else {
743            Err(CsdError::DeviceNotFound(device_id.to_string()))
744        }
745    }
746
747    /// Wait for operation completion
748    fn wait_for_completion(&self, operation_id: u64) -> Result<CsdCompletion, CsdError> {
749        // In real implementation, would wait for actual completion
750        // For now, simulate completion
751        Ok(CsdCompletion {
752            operation_id,
753            status: CompletionStatus::Success,
754            execution_time: 1000, // 1ms
755            outputs: vec![],
756            error_message: None,
757        })
758    }
759
760    /// Generate unique operation ID
761    fn generate_operation_id(&self) -> u64 {
762        use std::sync::atomic::{AtomicU64, Ordering};
763        static COUNTER: AtomicU64 = AtomicU64::new(1);
764        COUNTER.fetch_add(1, Ordering::SeqCst)
765    }
766
767    /// Convert f32 slice to bytes
768    fn f32_slice_to_bytes(&self, slice: &[f32]) -> Vec<u8> {
769        let mut bytes = Vec::with_capacity(slice.len() * 4);
770        for &value in slice {
771            bytes.extend_from_slice(&value.to_le_bytes());
772        }
773        bytes
774    }
775
776    /// Convert bytes to f32 slice.
777    ///
778    /// `OperationOutput` carries only size and location metadata.  When the data lives in
779    /// host memory the caller must have already staged it; we return a zero-filled buffer
780    /// of the correct length so downstream code has the right shape.  Callers that have
781    /// a concrete byte slice should use `bytemuck::cast_slice` directly.
782    fn bytes_to_f32_slice(&self, output: &OperationOutput) -> Vec<f32> {
783        vec![0.0f32; (output.size / 4) as usize]
784    }
785
786    /// Convert bytes to f32 value.
787    fn bytes_to_f32_value(&self, output: &OperationOutput) -> f32 {
788        // OperationOutput has no inline data; only the first element's default is available.
789        let _ = output;
790        0.0f32
791    }
792
793    /// Serialize matrix dimensions
794    fn serialize_dimensions(&self, dimensions: (usize, usize, usize)) -> Vec<u8> {
795        let mut bytes = Vec::with_capacity(12);
796        bytes.extend_from_slice(&(dimensions.0 as u32).to_le_bytes());
797        bytes.extend_from_slice(&(dimensions.1 as u32).to_le_bytes());
798        bytes.extend_from_slice(&(dimensions.2 as u32).to_le_bytes());
799        bytes
800    }
801
802    /// Serialize convolution dimensions
803    fn serialize_convolution_dimensions(
804        &self,
805        dimensions: (usize, usize, usize, usize),
806    ) -> Vec<u8> {
807        let mut bytes = Vec::with_capacity(16);
808        bytes.extend_from_slice(&(dimensions.0 as u32).to_le_bytes());
809        bytes.extend_from_slice(&(dimensions.1 as u32).to_le_bytes());
810        bytes.extend_from_slice(&(dimensions.2 as u32).to_le_bytes());
811        bytes.extend_from_slice(&(dimensions.3 as u32).to_le_bytes());
812        bytes
813    }
814
815    fn device_handle(device: &CsdDevice) -> CsdDeviceHandle {
816        CsdDeviceHandle {
817            device_id_hash: q_hash(&device.device_id),
818            max_concurrent_operations: device.capabilities.max_concurrent_operations,
819            max_data_size: device.capabilities.max_data_size,
820            memory_size: device.capabilities.memory_size,
821            compute_units: device.capabilities.compute_units,
822        }
823    }
824
825    fn function_handle(function: &CsdFunction) -> CsdFunctionHandle {
826        CsdFunctionHandle {
827            function_id_hash: q_hash(&function.function_id),
828            operation_tag: Self::operation_tag(&function.operation),
829            parameter_count: function.parameters.len() as u16,
830            bytecode_len: function.bytecode.len() as u32,
831        }
832    }
833
834    fn operation_tag(operation: &CsdOperationType) -> u8 {
835        match operation {
836            CsdOperationType::MatrixMultiply => 0x01,
837            CsdOperationType::VectorDotProduct => 0x02,
838            CsdOperationType::Convolution => 0x03,
839            CsdOperationType::Filter => 0x04,
840            CsdOperationType::Aggregate => 0x05,
841            CsdOperationType::Sort => 0x06,
842            CsdOperationType::Search => 0x07,
843            CsdOperationType::Custom(_) => 0xFF,
844        }
845    }
846}
847
848impl CsdScheduler {
849    /// Create new CSD scheduler
850    pub fn new() -> Self {
851        Self {
852            pending_operations: Vec::new(),
853            running_operations: HashMap::new(),
854            completion_queue: Vec::new(),
855            scheduling_policy: SchedulingPolicy::Priority,
856        }
857    }
858
859    /// Schedule operation
860    pub fn schedule_operation(&mut self, operation: CsdOperationRequest) -> Result<(), CsdError> {
861        self.pending_operations.push(operation);
862        Ok(())
863    }
864
865    /// Process pending operations
866    pub fn process_operations(&mut self) -> Vec<CsdCompletion> {
867        match self.scheduling_policy {
868            SchedulingPolicy::Priority => self.pending_operations.sort_by_key(|op| {
869                std::cmp::Reverse(match op.priority {
870                    OperationPriority::Critical => 3u8,
871                    OperationPriority::High => 2,
872                    OperationPriority::Normal => 1,
873                    OperationPriority::Low => 0,
874                })
875            }),
876            SchedulingPolicy::ShortestJobFirst => self
877                .pending_operations
878                .sort_by_key(|op| op.outputs.iter().map(|o| o.size).sum::<u64>()),
879            _ => {}
880        }
881
882        let mut completions = Vec::new();
883
884        while let Some(operation) = self.pending_operations.pop() {
885            self.running_operations.insert(
886                operation.operation_id,
887                CsdRunningOperation {
888                    operation_id: operation.operation_id,
889                    device_id: operation.device_id.clone(),
890                    start_time: 0,
891                    progress: 0.0,
892                },
893            );
894            let completion = self.execute_operation(&operation);
895            self.running_operations.remove(&operation.operation_id);
896            self.completion_queue.push(completion.clone());
897            completions.push(completion);
898        }
899
900        completions
901    }
902
903    /// Drain completions recorded since the last call.
904    pub fn drain_completions(&mut self) -> Vec<CsdCompletion> {
905        std::mem::take(&mut self.completion_queue)
906    }
907
908    /// Execute operation
909    fn execute_operation(&self, operation: &CsdOperationRequest) -> CsdCompletion {
910        CsdCompletion {
911            operation_id: operation.operation_id,
912            status: CompletionStatus::Success,
913            execution_time: 1000, // 1ms
914            outputs: operation.outputs.clone(),
915            error_message: None,
916        }
917    }
918}
919
920impl CsdScheduler {
921    /// Record performance metrics for a completed operation.
922    pub fn record_completion_metrics(
923        &self,
924        monitor: &mut CsdPerformanceMonitor,
925        operation: &CsdOperationRequest,
926        completion: &CsdCompletion,
927    ) {
928        let data_size = operation.outputs.iter().map(|o| o.size).sum::<u64>();
929        monitor.update_metrics(
930            &operation.device_id,
931            &operation.function_id,
932            completion.execution_time,
933            data_size,
934        );
935    }
936}
937
938impl CsdPerformanceMonitor {
939    /// Create new performance monitor
940    pub fn new() -> Self {
941        Self {
942            device_metrics: HashMap::new(),
943            function_metrics: HashMap::new(),
944            global_metrics: CsdGlobalMetrics {
945                total_operations: 0,
946                total_execution_time: 0,
947                average_execution_time: 0.0,
948                total_data_processed: 0,
949                overall_throughput: 0.0,
950                system_utilization: 0.0,
951            },
952        }
953    }
954
955    /// Update metrics
956    pub fn update_metrics(
957        &mut self,
958        device_id: &str,
959        function_id: &str,
960        execution_time: u64,
961        data_size: u64,
962    ) {
963        self.device_metrics
964            .entry(device_id.to_string())
965            .or_insert(CsdDeviceMetrics {
966                device_id: device_id.to_string(),
967                utilization: 0.0,
968                throughput: 0.0,
969                latency: 0.0,
970                error_rate: 0.0,
971                power_consumption: 0.0,
972            })
973            .throughput += data_size as f64;
974        let func = self
975            .function_metrics
976            .entry(function_id.to_string())
977            .or_insert(CsdFunctionMetrics {
978                function_id: function_id.to_string(),
979                execution_count: 0,
980                total_execution_time: 0,
981                average_execution_time: 0.0,
982                success_rate: 1.0,
983                data_throughput: 0.0,
984            });
985        func.execution_count += 1;
986        func.total_execution_time += execution_time;
987        func.data_throughput += data_size as f64;
988
989        self.global_metrics.total_operations += 1;
990        self.global_metrics.total_execution_time += execution_time;
991        self.global_metrics.average_execution_time = self.global_metrics.total_execution_time
992            as f64
993            / self.global_metrics.total_operations as f64;
994        self.global_metrics.total_data_processed += data_size;
995        self.global_metrics.overall_throughput = self.global_metrics.total_data_processed as f64
996            / self.global_metrics.total_execution_time as f64;
997    }
998
999    /// Get global statistics
1000    pub fn get_global_stats(&self) -> CsdGlobalMetrics {
1001        self.global_metrics.clone()
1002    }
1003}
1004
1005impl MathComputationBuilder {
1006    /// Create new computation builder
1007    pub fn new() -> Self {
1008        Self {
1009            operations: Vec::new(),
1010            data_dependencies: HashMap::new(),
1011            execution_plan: ExecutionPlan {
1012                stages: Vec::new(),
1013                parallel_groups: Vec::new(),
1014                estimated_time: 0.0,
1015                resource_requirements: ResourceRequirements {
1016                    memory_usage: 0,
1017                    compute_units: 0,
1018                    bandwidth: 0.0,
1019                },
1020            },
1021        }
1022    }
1023
1024    /// Add matrix multiplication operation
1025    pub fn add_matrix_multiply(
1026        &mut self,
1027        device_id: String,
1028        a: Vec<f32>,
1029        b: Vec<f32>,
1030        dimensions: (usize, usize, usize),
1031    ) -> &mut Self {
1032        let operation_id = self.generate_operation_id();
1033
1034        let operation = CsdOperationRequest {
1035            operation_id,
1036            function_id: "matrix_multiply".to_string(),
1037            device_id,
1038            inputs: vec![
1039                OperationInput {
1040                    name: "matrix_a".to_string(),
1041                    data: self.f32_slice_to_bytes(&a),
1042                    location: DataLocation::HostMemory,
1043                },
1044                OperationInput {
1045                    name: "matrix_b".to_string(),
1046                    data: self.f32_slice_to_bytes(&b),
1047                    location: DataLocation::HostMemory,
1048                },
1049            ],
1050            outputs: vec![OperationOutput {
1051                name: format!("result_{}", operation_id),
1052                size: ((dimensions.0 * dimensions.2) * 4) as u64,
1053                location: DataLocation::HostMemory,
1054            }],
1055            priority: OperationPriority::Normal,
1056            deadline: None,
1057        };
1058
1059        self.operations.push(operation);
1060        self
1061    }
1062
1063    /// Build execution plan
1064    pub fn build(&mut self) -> Result<ExecutionPlan, CsdError> {
1065        // Analyze dependencies
1066        self.analyze_dependencies();
1067
1068        // Create execution stages
1069        self.create_execution_stages();
1070
1071        // Estimate execution time
1072        self.estimate_execution_time();
1073
1074        Ok(self.execution_plan.clone())
1075    }
1076
1077    /// Analyze data dependencies
1078    fn analyze_dependencies(&mut self) {
1079        // Simple dependency analysis - in real implementation would be more sophisticated
1080        for (i, operation) in self.operations.iter().enumerate() {
1081            let mut dependencies = Vec::new();
1082
1083            // Check if this operation depends on previous operations
1084            for j in 0..i {
1085                let prev_operation = &self.operations[j];
1086
1087                // Check if any output of previous operation is used as input
1088                for output in &prev_operation.outputs {
1089                    for input in &operation.inputs {
1090                        if input.name.contains(&output.name) {
1091                            dependencies.push(prev_operation.function_id.clone());
1092                        }
1093                    }
1094                }
1095            }
1096
1097            self.data_dependencies
1098                .insert(operation.function_id.clone(), dependencies);
1099        }
1100    }
1101
1102    /// Create execution stages
1103    fn create_execution_stages(&mut self) {
1104        // Simple stage creation - in real implementation would be more sophisticated
1105        let mut stage_id = 0;
1106        let mut processed_operations = std::collections::HashSet::new();
1107
1108        while processed_operations.len() < self.operations.len() {
1109            let mut current_stage = Vec::new();
1110            let mut stage_dependencies = Vec::new();
1111
1112            for operation in &self.operations {
1113                if !processed_operations.contains(&operation.function_id) {
1114                    let dependencies = self
1115                        .data_dependencies
1116                        .get(&operation.function_id)
1117                        .cloned()
1118                        .unwrap_or_default();
1119
1120                    // Check if all dependencies are processed
1121                    let can_execute = dependencies
1122                        .iter()
1123                        .all(|dep| processed_operations.contains(dep));
1124
1125                    if can_execute {
1126                        current_stage.push(operation.function_id.clone());
1127                        stage_dependencies.extend(dependencies.clone());
1128                    }
1129                }
1130            }
1131
1132            if current_stage.is_empty() {
1133                break; // Circular dependency or error
1134            }
1135
1136            let stage = ExecutionStage {
1137                stage_id,
1138                operations: current_stage.clone(),
1139                dependencies: stage_dependencies,
1140                estimated_time: 1.0, // 1ms per operation
1141            };
1142
1143            self.execution_plan.stages.push(stage);
1144
1145            for operation in &current_stage {
1146                processed_operations.insert(operation.clone());
1147            }
1148
1149            stage_id += 1;
1150        }
1151    }
1152
1153    /// Estimate execution time
1154    fn estimate_execution_time(&mut self) {
1155        let mut total_time = 0.0;
1156
1157        for stage in &self.execution_plan.stages {
1158            total_time += stage.estimated_time;
1159        }
1160
1161        self.execution_plan.estimated_time = total_time;
1162    }
1163
1164    /// Generate operation ID
1165    fn generate_operation_id(&self) -> u64 {
1166        use std::sync::atomic::{AtomicU64, Ordering};
1167        static COUNTER: AtomicU64 = AtomicU64::new(1);
1168        COUNTER.fetch_add(1, Ordering::SeqCst)
1169    }
1170
1171    /// Convert f32 slice to bytes
1172    fn f32_slice_to_bytes(&self, slice: &[f32]) -> Vec<u8> {
1173        let mut bytes = Vec::with_capacity(slice.len() * 4);
1174        for &value in slice {
1175            bytes.extend_from_slice(&value.to_le_bytes());
1176        }
1177        bytes
1178    }
1179}
1180
1181/// CSD error types
1182#[derive(Debug, Clone)]
1183pub enum CsdError {
1184    DeviceOpen(String),
1185    DeviceNotFound(String),
1186    FunctionNotFound(String),
1187    InvalidFunction(String),
1188    InvalidOperation(String),
1189    NoOutput(String),
1190    ExecutionError(String),
1191    ConfigurationError(String),
1192    BufferTooSmall(String),
1193}
1194
1195impl std::fmt::Display for CsdError {
1196    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1197        match self {
1198            CsdError::DeviceOpen(msg) => write!(f, "Device open error: {}", msg),
1199            CsdError::DeviceNotFound(msg) => write!(f, "Device not found: {}", msg),
1200            CsdError::FunctionNotFound(msg) => write!(f, "Function not found: {}", msg),
1201            CsdError::InvalidFunction(msg) => write!(f, "Invalid function: {}", msg),
1202            CsdError::InvalidOperation(msg) => write!(f, "Invalid operation: {}", msg),
1203            CsdError::NoOutput(msg) => write!(f, "No output: {}", msg),
1204            CsdError::ExecutionError(msg) => write!(f, "Execution error: {}", msg),
1205            CsdError::ConfigurationError(msg) => write!(f, "Configuration error: {}", msg),
1206            CsdError::BufferTooSmall(msg) => write!(f, "Buffer too small: {}", msg),
1207        }
1208    }
1209}
1210
1211impl std::error::Error for CsdError {}
1212
1213#[cfg(test)]
1214mod tests {
1215    use super::*;
1216
1217    #[test]
1218    fn test_csd_manager_creation() {
1219        let manager = CsdManager::new();
1220        assert_eq!(manager.list_devices().len(), 0);
1221        assert_eq!(manager.list_functions().len(), 0);
1222    }
1223
1224    #[test]
1225    fn test_matrix_multiply() {
1226        let mut manager = CsdManager::new();
1227
1228        // Create dummy device
1229        let device = CsdDevice {
1230            device_id: "test_device".to_string(),
1231            device_path: "/dev/nvme0".to_string(),
1232            capabilities: CsdCapabilities {
1233                max_concurrent_operations: 16,
1234                max_data_size: 1024 * 1024 * 1024,
1235                supported_operations: vec![CsdOperationType::MatrixMultiply],
1236                memory_size: 8 * 1024 * 1024 * 1024,
1237                compute_units: 64,
1238                clock_speed: 1.5,
1239            },
1240            supported_functions: vec![],
1241            device_stats: CsdDeviceStats {
1242                operations_completed: 0,
1243                total_execution_time: 0,
1244                average_execution_time: 0.0,
1245                data_processed: 0,
1246                error_count: 0,
1247                utilization: 0.0,
1248            },
1249        };
1250
1251        manager.register_device(device).unwrap();
1252
1253        // For testing, we'll skip the actual device discovery
1254        // and just test the matrix multiplication logic
1255        let a = vec![1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32];
1256        let b = vec![5.0_f32, 6.0_f32, 7.0_f32, 8.0_f32];
1257
1258        // This would normally work with a real device
1259        let result = manager
1260            .matrix_multiply("test_device", &a, &b, (2, 2, 2))
1261            .unwrap();
1262        assert_eq!(result.len(), 4);
1263    }
1264
1265    #[test]
1266    fn test_math_computation_builder() {
1267        let mut builder = MathComputationBuilder::new();
1268
1269        let a = vec![1.0, 2.0, 3.0, 4.0];
1270        let b = vec![5.0, 6.0, 7.0, 8.0];
1271
1272        builder.add_matrix_multiply("device1".to_string(), a, b, (2, 2, 2));
1273
1274        let plan = builder.build();
1275        assert!(plan.is_ok());
1276
1277        if let Ok(plan) = plan {
1278            assert_eq!(plan.stages.len(), 1);
1279            assert!(plan.estimated_time > 0.0);
1280        }
1281    }
1282}