Skip to main content

qualia_core_db/platform/
local_scheduler.rs

1//! Local Scheduler — Thread-per-Core Production Queue
2//!
3//! Implements multi-core parallelism for calculus operations by pinning isolated
4//! SLG VM loops to each logical processor. Each worker operates within a 512MB
5//! memory boundary and pulls jobs from a central lock-free ring buffer.
6//!
7//! Architecture:
8//! - Supervisor thread: Manages job queue and coordinates workers
9//! - Worker threads: Pinned to specific cores, process NQuin jobs
10//! - SPSC channels: Lock-free communication between supervisor and workers
11//! - NVMe WAL: Persistent backing store for job state (future)
12//!
13//! Environmental yielding:
14//! - Thermal governor integration for power-aware execution
15//! - Pause/resume capability via Quin state persistence
16//! - Solar/battery power constraint handling
17
18#![cfg(not(target_arch = "wasm32"))]
19
20use crate::wal::WriteAheadLog;
21use crate::NQuin;
22use core_affinity::CoreId;
23use crossbeam_channel::{bounded, Receiver, Sender};
24use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
25use std::sync::{Arc, Mutex};
26use std::thread;
27use std::time::{Duration, Instant};
28
29/// Memory boundary for each worker cell (512MB)
30const WORKER_MEMORY_BOUNDARY: usize = 512 * 1024 * 1024;
31
32/// Ring buffer capacity for job distribution
33const JOB_QUEUE_CAPACITY: usize = 4096;
34
35/// Job status tracking
36#[derive(Debug, Clone, Copy, PartialEq, Eq)]
37pub enum JobStatus {
38    Pending,
39    InProgress,
40    Completed,
41    Paused,
42    Failed,
43}
44
45/// Production queue job wrapper
46#[derive(Debug, Clone)]
47pub struct Job {
48    pub quin: NQuin,
49    pub status: JobStatus,
50    pub total_bytes: u64,
51    pub dispatched_at: Option<Instant>,
52    pub completed_at: Option<Instant>,
53    pub compute_target: ComputeTarget,
54}
55
56impl Job {
57    /// Creates a job from a quin with `total_bytes` of work.
58    pub fn new(quin: NQuin, total_bytes: u64) -> Self {
59        Self {
60            quin,
61            status: JobStatus::Pending,
62            total_bytes,
63            dispatched_at: None,
64            completed_at: None,
65            compute_target: ComputeTarget::Cpu, // Default to CPU
66        }
67    }
68
69    /// Creates a job pinned to `compute_target`.
70    pub fn with_target(quin: NQuin, total_bytes: u64, compute_target: ComputeTarget) -> Self {
71        Self {
72            quin,
73            status: JobStatus::Pending,
74            total_bytes,
75            dispatched_at: None,
76            completed_at: None,
77            compute_target,
78        }
79    }
80
81    /// Calculate progress percentage from Quin object field (byte offset) or completion state for calculus jobs
82    pub fn progress(&self) -> f64 {
83        if self.total_bytes == 0 {
84            return 0.0;
85        }
86        let opcode = (self.quin.predicate & 0xFF) as u8;
87        if opcode != 0 {
88            return match self.status {
89                JobStatus::Completed => 100.0,
90                JobStatus::InProgress => 50.0,
91                JobStatus::Paused => 25.0,
92                JobStatus::Pending | JobStatus::Failed => 0.0,
93            };
94        }
95        let current_offset = self.quin.object;
96        ((current_offset as f64 / self.total_bytes as f64) * 100.0).min(100.0)
97    }
98
99    /// Calculate processing velocity (bytes per second)
100    pub fn velocity(&self) -> f64 {
101        match (self.dispatched_at, self.completed_at) {
102            (Some(dispatched), Some(completed)) => {
103                let duration = completed.duration_since(dispatched).as_secs_f64();
104                if duration > 0.0 {
105                    let opcode = (self.quin.predicate & 0xFF) as u8;
106                    if opcode != 0 {
107                        (self.total_bytes as f64).max(4096.0) / duration
108                    } else {
109                        self.quin.object as f64 / duration
110                    }
111                } else {
112                    0.0
113                }
114            }
115            _ => 0.0,
116        }
117    }
118}
119
120/// Worker cell - isolated execution context pinned to a specific core
121pub struct WorkerCell {
122    pub cell_id: usize,
123    pub core_id: CoreId,
124    pub memory_boundary: usize,
125    pub job_receiver: Receiver<Job>,
126    pub result_sender: Sender<Job>,
127    pub shutdown_signal: Arc<AtomicBool>,
128    pub pause_signal: Arc<AtomicBool>,
129}
130
131impl WorkerCell {
132    /// Creates a worker cell bound to its channels/target.
133    pub fn new(
134        cell_id: usize,
135        core_id: CoreId,
136        job_receiver: Receiver<Job>,
137        result_sender: Sender<Job>,
138        shutdown_signal: Arc<AtomicBool>,
139        pause_signal: Arc<AtomicBool>,
140    ) -> Self {
141        Self {
142            cell_id,
143            core_id,
144            memory_boundary: WORKER_MEMORY_BOUNDARY,
145            job_receiver,
146            result_sender,
147            shutdown_signal,
148            pause_signal,
149        }
150    }
151
152    /// Main worker loop - processes jobs from the receiver channel
153    pub fn run(self) {
154        // Pin this thread to the assigned core
155        let pinned = core_affinity::set_for_current(self.core_id);
156        if !pinned {
157            log::error!(
158                "Failed to pin worker {} to core {:?}",
159                self.cell_id,
160                self.core_id
161            );
162        } else {
163            log::info!("Worker {} pinned to core {:?}", self.cell_id, self.core_id);
164        }
165
166        log::info!("Worker {} starting execution loop", self.cell_id);
167
168        while !self.shutdown_signal.load(Ordering::Relaxed) {
169            // Check for pause signal (environmental yielding)
170            if self.pause_signal.load(Ordering::Relaxed) {
171                log::info!("Worker {} paused (environmental yielding)", self.cell_id);
172                std::thread::sleep(Duration::from_millis(100));
173                continue;
174            }
175
176            // Poll for new job with timeout to allow shutdown check
177            match self.job_receiver.recv_timeout(Duration::from_millis(100)) {
178                Ok(mut job) => {
179                    log::debug!(
180                        "Worker {} received job at offset {}",
181                        self.cell_id,
182                        job.quin.object
183                    );
184
185                    job.status = JobStatus::InProgress;
186                    job.dispatched_at = Some(Instant::now());
187
188                    // Process the job (executing opcode-driven calculus or chunk streaming)
189                    self.process_job(&mut job);
190
191                    job.status = JobStatus::Completed;
192                    job.completed_at = Some(Instant::now());
193
194                    // Send result back to supervisor
195                    if let Err(e) = self.result_sender.send(job) {
196                        log::error!("Worker {} failed to send result: {}", self.cell_id, e);
197                    }
198                }
199                Err(crossbeam_channel::RecvTimeoutError::Timeout) => {
200                    // No job available, continue loop
201                    continue;
202                }
203                Err(crossbeam_channel::RecvTimeoutError::Disconnected) => {
204                    log::info!(
205                        "Worker {} job channel disconnected, shutting down",
206                        self.cell_id
207                    );
208                    break;
209                }
210            }
211        }
212
213        log::info!("Worker {} shutting down", self.cell_id);
214    }
215
216    /// Process a single job executing genuine opcode-driven calculus operations
217    /// (RK4, Simpson's quadrature, Trapezoidal integration, Adaptive step) or advancing
218    /// chunk state for data streaming jobs.
219    fn process_job(&self, job: &mut Job) {
220        let opcode = (job.quin.predicate & 0xFF) as u8;
221        let chunk_size = 4096.min(job.total_bytes.saturating_sub(job.quin.object)).max(1);
222
223        // Execute calculus operation if an opcode is specified
224        if opcode != 0 {
225            // Unpack step size from context or default to 0.01
226            let h = if job.quin.context != 0 {
227                let f32_cand = f32::from_bits((job.quin.context & 0xFFFFFFFF) as u32) as f64;
228                if f32_cand > 0.0 && f32_cand.is_finite() && f32_cand <= 10.0 {
229                    f32_cand
230                } else {
231                    let f64_cand = f64::from_bits(job.quin.context);
232                    if f64_cand > 0.0 && f64_cand.is_finite() && f64_cand <= 10.0 {
233                        f64_cand
234                    } else {
235                        0.01
236                    }
237                }
238            } else {
239                0.01
240            };
241
242            // Unpack state (t, y) from metadata and object fields
243            let (t_raw, y_raw) = crate::solvers::calculus::ode_solver::extract_ode_state(&job.quin);
244            let (t, y) = if t_raw.is_finite() && y_raw.is_finite() {
245                (t_raw, y_raw)
246            } else {
247                (0.0, (job.quin.object as f64) * 0.001)
248            };
249
250            // System parameter (e.g. omega or lambda) encoded in predicate bits [16..31]
251            let param_raw = ((job.quin.predicate >> 16) & 0xFFFF) as f64 / 1000.0;
252            let param = if param_raw > 0.0 { param_raw } else { 1.0 };
253            let system_type = ((job.quin.predicate >> 8) & 0xFF) as u8;
254
255            match opcode {
256                // OP_RK4_STEP (0x52) or general ODE integration step
257                crate::modalities::calculus::OP_RK4_STEP | 0x01 => {
258                    let (t_next, y_next) = match system_type {
259                        1 => {
260                            // Exponential decay: dy/dt = -λy
261                            let mut solver = crate::solvers::calculus::ode_solver::Rk4Solver::new(
262                                crate::solvers::calculus::ode_solver::ExponentialDecay::new(param),
263                                h,
264                            );
265                            (t + h, solver.step(t, y, h))
266                        }
267                        2 => {
268                            // Coupled 2D harmonic oscillator: [x, v]
269                            let mut solver = crate::solvers::calculus::ode_solver::Rk4Solver::new(
270                                crate::solvers::calculus::ode_solver::CoupledHarmonicOscillator::new(param),
271                                h,
272                            );
273                            (t + h, solver.step(t, y, h))
274                        }
275                        _ => {
276                            // Harmonic oscillator: d²x/dt² + ω²x = 0
277                            let mut solver = crate::solvers::calculus::ode_solver::Rk4Solver::new(
278                                crate::solvers::calculus::ode_solver::HarmonicOscillator::new(param),
279                                h,
280                            );
281                            (t + h, solver.step(t, y, h))
282                        }
283                    };
284                    crate::solvers::calculus::ode_solver::pack_ode_state(&mut job.quin, t_next, y_next);
285                }
286                // OP_SIMPSONS_INTEGRATION (0x50)
287                crate::modalities::calculus::OP_SIMPSONS_INTEGRATION => {
288                    let f_t = -param * param * y;
289                    let f_mid = -param * param * (y + 0.5 * h * f_t);
290                    let f_end = -param * param * (y + h * f_mid);
291                    let delta_y = (h / 6.0) * (f_t + 4.0 * f_mid + f_end);
292                    let (t_next, y_next) = (t + h, y + delta_y);
293                    crate::solvers::calculus::ode_solver::pack_ode_state(&mut job.quin, t_next, y_next);
294                }
295                // OP_TRAPEZOIDAL_INTEGRATION (0x51)
296                crate::modalities::calculus::OP_TRAPEZOIDAL_INTEGRATION => {
297                    let f_t = -param * y;
298                    let f_end = -param * (y + h * f_t);
299                    let delta_y = 0.5 * h * (f_t + f_end);
300                    let (t_next, y_next) = (t + h, y + delta_y);
301                    crate::solvers::calculus::ode_solver::pack_ode_state(&mut job.quin, t_next, y_next);
302                }
303                // OP_ADAPTIVE_STEP (0x53)
304                crate::modalities::calculus::OP_ADAPTIVE_STEP => {
305                    let mut rk4 = crate::solvers::calculus::ode_solver::Rk4Solver::new(
306                        crate::solvers::calculus::ode_solver::HarmonicOscillator::new(param),
307                        h,
308                    );
309                    let half_h = h * 0.5;
310                    let y_half = rk4.step(t, y, half_h);
311                    let y_full_from_half = rk4.step(t + half_h, y_half, half_h);
312                    let y_single = rk4.step(t, y, h);
313                    let err = (y_full_from_half - y_single).abs();
314                    let adapted_h = if err < 1e-6 { (h * 1.5).min(0.1) } else { (h * 0.5).max(0.0001) };
315                    crate::solvers::calculus::ode_solver::pack_ode_state(&mut job.quin, t + h, y_full_from_half);
316                    job.quin.context = (adapted_h as f32).to_bits() as u64;
317                }
318                _ => {
319                    // General fallback ODE step preserving calculated state
320                    let mut rk4 = crate::solvers::calculus::ode_solver::Rk4Solver::new(
321                        crate::solvers::calculus::ode_solver::HarmonicOscillator::new(param),
322                        h,
323                    );
324                    let y_next = rk4.step(t, y, h);
325                    crate::solvers::calculus::ode_solver::pack_ode_state(&mut job.quin, t + h, y_next);
326                }
327            }
328        } else {
329            // Standard data streaming job (opcode == 0): advance chunk byte offset
330            job.quin.object = job.quin.object.saturating_add(chunk_size).min(job.total_bytes.max(chunk_size));
331        }
332
333        // Recompute parity fold
334        job.quin.parity = job.quin.subject ^ job.quin.predicate ^ job.quin.object ^ job.quin.context;
335
336        log::trace!(
337            "Worker {} processed job (opcode=0x{:02x}, offset/state={})",
338            self.cell_id,
339            opcode,
340            job.quin.object
341        );
342    }
343}
344
345/// Production queue supervisor - coordinates worker threads
346pub struct ProductionQueue {
347    num_workers: usize,
348    job_sender: Sender<Job>,
349    result_receiver: Receiver<Job>,
350    shutdown_signal: Arc<AtomicBool>,
351    pause_signal: Arc<AtomicBool>,
352    active_jobs: Arc<AtomicU64>,
353    completed_jobs: Arc<AtomicU64>,
354    wal: Arc<Mutex<Option<WriteAheadLog>>>,
355    wal_path: String,
356    estimator: Arc<Mutex<JobEstimator>>,
357}
358
359impl ProductionQueue {
360    /// Create a new production queue with one worker per logical processor
361    pub fn new() -> Self {
362        Self::with_wal_path("jobs.wal")
363    }
364
365    /// Create a new production queue with a specific WAL path
366    pub fn with_wal_path(wal_path: &str) -> Self {
367        let num_logical_cores = num_cpus::get();
368        log::info!(
369            "Initializing production queue with {} workers",
370            num_logical_cores
371        );
372
373        let (job_sender, job_receiver) = bounded(JOB_QUEUE_CAPACITY);
374        let (result_sender, result_receiver) = bounded(JOB_QUEUE_CAPACITY);
375        let shutdown_signal = Arc::new(AtomicBool::new(false));
376        let pause_signal = Arc::new(AtomicBool::new(false));
377        let active_jobs = Arc::new(AtomicU64::new(0));
378        let completed_jobs = Arc::new(AtomicU64::new(0));
379
380        // Initialize WAL
381        let wal = Arc::new(Mutex::new(WriteAheadLog::open(wal_path).ok()));
382        if wal.lock().unwrap().is_some() {
383            log::info!("Production queue WAL initialized at {}", wal_path);
384        } else {
385            log::warn!(
386                "Failed to initialize WAL at {}, job persistence disabled",
387                wal_path
388            );
389        }
390
391        // Initialize job estimator
392        let estimator = Arc::new(Mutex::new(JobEstimator::new()));
393        log::info!("Job estimator initialized");
394
395        let mut worker_senders = Vec::new();
396        let mut worker_receivers = Vec::new();
397        for _ in 0..num_logical_cores {
398            let (worker_job_sender, worker_job_receiver) = bounded(16);
399            worker_senders.push(worker_job_sender);
400            worker_receivers.push(worker_job_receiver);
401        }
402
403        // Ingress dispatcher: shared queue → per-worker mailboxes.
404        let dispatch_targets = worker_senders.clone();
405        let dispatcher_active = active_jobs.clone();
406        let dispatcher_shutdown = shutdown_signal.clone();
407        thread::spawn(move || {
408            while !dispatcher_shutdown.load(Ordering::Relaxed) {
409                match job_receiver.recv_timeout(Duration::from_millis(100)) {
410                    Ok(job) => {
411                        let worker_id = dispatcher_active.fetch_add(1, Ordering::Relaxed) as usize
412                            % num_logical_cores;
413                        if dispatch_targets[worker_id].send(job).is_err() {
414                            dispatcher_active.fetch_sub(1, Ordering::Relaxed);
415                        }
416                    }
417                    Err(crossbeam_channel::RecvTimeoutError::Timeout) => continue,
418                    Err(crossbeam_channel::RecvTimeoutError::Disconnected) => break,
419                }
420            }
421        });
422
423        for (worker_id, worker_job_receiver) in worker_receivers.into_iter().enumerate() {
424            let core_id = CoreId {
425                id: worker_id % num_logical_cores,
426            };
427            let worker_result_sender = result_sender.clone();
428            let worker_shutdown = shutdown_signal.clone();
429            let worker_pause = pause_signal.clone();
430
431            let worker = WorkerCell::new(
432                worker_id,
433                core_id,
434                worker_job_receiver,
435                worker_result_sender,
436                worker_shutdown,
437                worker_pause,
438            );
439
440            thread::spawn(move || {
441                worker.run();
442            });
443        }
444
445        Self {
446            num_workers: num_logical_cores,
447            job_sender,
448            result_receiver,
449            shutdown_signal,
450            pause_signal,
451            active_jobs,
452            completed_jobs,
453            wal,
454            wal_path: wal_path.to_string(),
455            estimator,
456        }
457    }
458
459    /// Submit a job to the production queue
460    pub fn submit_job(&self, job: Job) -> Result<(), String> {
461        // Persist job to WAL if available
462        if let Ok(mut wal_guard) = self.wal.lock() {
463            if let Some(ref mut wal) = wal_guard.as_mut() {
464                if let Err(e) = wal.append_mutation(&job.quin) {
465                    log::warn!("Failed to persist job to WAL: {}", e);
466                }
467            }
468        }
469
470        if let Err(e) = self.job_sender.send(job) {
471            return Err(format!("Failed to enqueue job: {}", e));
472        }
473
474        log::debug!("Job enqueued for dispatcher");
475        Ok(())
476    }
477
478    /// WAL path used for job persistence and recovery.
479    pub fn wal_path(&self) -> &str {
480        &self.wal_path
481    }
482
483    /// Recover pending jobs from WAL on startup
484    pub fn recover_jobs(&mut self) -> Result<Vec<Job>, String> {
485        if let Ok(mut wal_guard) = self.wal.lock() {
486            if let Some(ref mut wal) = wal_guard.as_mut() {
487                match wal.recover() {
488                    Ok(quins) => {
489                        let jobs: Vec<Job> = quins
490                            .into_iter()
491                            .map(|quin| Job::new(quin, 10000)) // Default total_bytes, should be stored in metadata
492                            .collect();
493                        log::info!(
494                            "Recovered {} jobs from WAL at {}",
495                            jobs.len(),
496                            self.wal_path()
497                        );
498                        return Ok(jobs);
499                    }
500                    Err(e) => return Err(format!("Failed to recover jobs from WAL: {}", e)),
501                }
502            }
503        }
504        Ok(Vec::new())
505    }
506
507    /// Persist completed job state to WAL
508    pub fn persist_job_completion(&self, job: &Job) -> Result<(), String> {
509        if let Ok(mut wal_guard) = self.wal.lock() {
510            if let Some(ref mut wal) = wal_guard.as_mut() {
511                if let Err(e) = wal.append_mutation(&job.quin) {
512                    return Err(format!("Failed to persist job completion to WAL: {}", e));
513                }
514            }
515        }
516        Ok(())
517    }
518
519    /// Collect completed jobs from workers
520    pub fn collect_results(&self) -> Vec<Job> {
521        let mut results = Vec::new();
522
523        while let Ok(job) = self.result_receiver.try_recv() {
524            self.completed_jobs.fetch_add(1, Ordering::Relaxed);
525            self.active_jobs.fetch_sub(1, Ordering::Relaxed);
526
527            // Persist completed job state to WAL
528            if let Err(e) = self.persist_job_completion(&job) {
529                log::warn!("Failed to persist job completion: {}", e);
530            }
531
532            results.push(job);
533        }
534
535        results
536    }
537
538    /// Get current queue statistics
539    pub fn stats(&self) -> QueueStats {
540        QueueStats {
541            num_workers: self.num_workers,
542            active_jobs: self.active_jobs.load(Ordering::Relaxed),
543            completed_jobs: self.completed_jobs.load(Ordering::Relaxed),
544        }
545    }
546
547    /// Gracefully shutdown all workers
548    pub fn shutdown(&self) {
549        log::info!("Shutting down production queue");
550        self.shutdown_signal.store(true, Ordering::Relaxed);
551
552        // Give workers time to finish current jobs
553        std::thread::sleep(Duration::from_secs(2));
554
555        log::info!("Production queue shutdown complete");
556    }
557
558    /// Pause all workers (environmental yielding)
559    pub fn pause(&self) {
560        log::info!("Pausing production queue (environmental yielding)");
561        self.pause_signal.store(true, Ordering::Relaxed);
562    }
563
564    /// Resume all workers after environmental yielding
565    pub fn resume(&self) {
566        log::info!("Resuming production queue");
567        self.pause_signal.store(false, Ordering::Relaxed);
568    }
569
570    /// Check if the queue is currently paused
571    pub fn is_paused(&self) -> bool {
572        self.pause_signal.load(Ordering::Relaxed)
573    }
574
575    /// Get pre-execution job estimate
576    pub fn estimate_job(&self, params: &JobEstimateParams) -> JobEstimate {
577        if let Ok(estimator) = self.estimator.lock() {
578            estimator.estimate_job_duration(params)
579        } else {
580            // Fallback if estimator is locked
581            JobEstimate {
582                estimated_duration: Duration::from_secs(0),
583                confidence: 0.0,
584                compute_target: params.compute_target,
585                workload_bytes: params.workload_bytes,
586            }
587        }
588    }
589
590    /// Record job performance for velocity learning
591    pub fn record_performance(
592        &self,
593        target: ComputeTarget,
594        bytes_processed: u64,
595        duration: Duration,
596    ) {
597        if let Ok(mut estimator) = self.estimator.lock() {
598            estimator.record_performance(target, bytes_processed, duration);
599        }
600    }
601
602    /// Set power throttling factor (called by power daemon)
603    pub fn set_power_throttle_factor(&self, factor: f64) {
604        if let Ok(mut estimator) = self.estimator.lock() {
605            estimator.set_power_throttle_factor(factor);
606            log::info!("Power throttle factor set to {}", factor);
607        }
608    }
609
610    /// Get current power throttling factor
611    pub fn get_power_throttle_factor(&self) -> f64 {
612        if let Ok(estimator) = self.estimator.lock() {
613            estimator.get_power_throttle_factor()
614        } else {
615            1.0
616        }
617    }
618
619    /// Calculate progress and telemetry from Quin fields
620    pub fn calculate_telemetry(&self, total_job_bytes: u64) -> QueueTelemetry {
621        let active = self.active_jobs.load(Ordering::Relaxed);
622        let completed = self.completed_jobs.load(Ordering::Relaxed);
623        let total = active + completed;
624
625        // Calculate overall progress from completed jobs
626        let overall_progress = if total > 0 {
627            (completed as f64 / total as f64) * 100.0
628        } else {
629            0.0
630        };
631
632        // Calculate bytes per second from job velocities
633        let results = self.collect_results();
634        let total_velocity: f64 = results.iter().map(|j| j.velocity()).sum();
635        let avg_velocity = if !results.is_empty() {
636            total_velocity / results.len() as f64
637        } else {
638            0.0
639        };
640
641        // Estimate time remaining based on velocity
642        let estimated_time_remaining = if avg_velocity > 0.0 && total_job_bytes > 0 {
643            let remaining_bytes =
644                total_job_bytes.saturating_sub(results.iter().map(|j| j.quin.object).sum::<u64>());
645            Some(Duration::from_secs_f64(
646                remaining_bytes as f64 / avg_velocity,
647            ))
648        } else {
649            None
650        };
651
652        QueueTelemetry {
653            total_jobs: total,
654            active_jobs: active,
655            completed_jobs: completed,
656            overall_progress,
657            estimated_time_remaining,
658            bytes_per_second: avg_velocity,
659        }
660    }
661}
662
663impl Drop for ProductionQueue {
664    fn drop(&mut self) {
665        self.shutdown();
666    }
667}
668
669/// Queue statistics
670#[derive(Debug, Clone, Copy)]
671pub struct QueueStats {
672    pub num_workers: usize,
673    pub active_jobs: u64,
674    pub completed_jobs: u64,
675}
676
677/// Progress and telemetry data for the production queue
678#[derive(Debug, Clone)]
679pub struct QueueTelemetry {
680    pub total_jobs: u64,
681    pub active_jobs: u64,
682    pub completed_jobs: u64,
683    pub overall_progress: f64, // Percentage (0-100)
684    pub estimated_time_remaining: Option<Duration>,
685    pub bytes_per_second: f64,
686}
687
688/// Compute target for calculus operations
689#[derive(Debug, Clone, Copy, PartialEq, Eq)]
690pub enum ComputeTarget {
691    Cpu,
692    WebGpu,
693    DirectMl,
694}
695
696/// Hardware velocity metrics for a specific compute target
697#[derive(Debug, Clone)]
698pub struct HardwareVelocity {
699    pub target: ComputeTarget,
700    pub bytes_per_second: f64,
701    pub sample_count: u64,
702    pub last_updated: Option<Instant>,
703}
704
705impl HardwareVelocity {
706    /// Creates a velocity record for `target`.
707    pub fn new(target: ComputeTarget) -> Self {
708        Self {
709            target,
710            bytes_per_second: 0.0,
711            sample_count: 0,
712            last_updated: None,
713        }
714    }
715
716    /// Update velocity with a new sample using exponential moving average
717    pub fn update(&mut self, bytes_processed: u64, duration: Duration) {
718        let duration_secs = duration.as_secs_f64();
719        if duration_secs > 0.0 {
720            let new_velocity = bytes_processed as f64 / duration_secs;
721
722            // Exponential moving average with alpha=0.2 (weights recent samples more)
723            let alpha = 0.2;
724            if self.sample_count == 0 {
725                self.bytes_per_second = new_velocity;
726            } else {
727                self.bytes_per_second =
728                    alpha * new_velocity + (1.0 - alpha) * self.bytes_per_second;
729            }
730
731            self.sample_count += 1;
732            self.last_updated = Some(Instant::now());
733        }
734    }
735
736    /// Get current velocity with default fallback if no samples
737    pub fn get_velocity(&self) -> f64 {
738        if self.sample_count == 0 {
739            // Default velocities based on target (conservative estimates)
740            match self.target {
741                ComputeTarget::Cpu => 10_000_000.0,       // 10 MB/s CPU
742                ComputeTarget::WebGpu => 100_000_000.0,   // 100 MB/s WebGPU
743                ComputeTarget::DirectMl => 500_000_000.0, // 500 MB/s DirectML
744            }
745        } else {
746            self.bytes_per_second
747        }
748    }
749}
750
751/// Job estimation parameters
752#[derive(Debug, Clone)]
753pub struct JobEstimateParams {
754    pub workload_bytes: u64,
755    pub step_size: f32,
756    pub compute_target: ComputeTarget,
757    pub contention_factor: f64, // Multiplier for parallel execution overhead
758}
759
760/// Pre-execution job estimate
761#[derive(Debug, Clone)]
762pub struct JobEstimate {
763    pub estimated_duration: Duration,
764    pub confidence: f64, // 0-1 based on sample count
765    pub compute_target: ComputeTarget,
766    pub workload_bytes: u64,
767}
768
769/// Job estimator for pre-execution time prediction
770pub struct JobEstimator {
771    cpu_velocity: HardwareVelocity,
772    webgpu_velocity: HardwareVelocity,
773    directml_velocity: HardwareVelocity,
774    power_throttle_factor: f64, // Multiplier when power-constrained
775}
776
777impl JobEstimator {
778    /// Creates an empty job estimator.
779    pub fn new() -> Self {
780        Self {
781            cpu_velocity: HardwareVelocity::new(ComputeTarget::Cpu),
782            webgpu_velocity: HardwareVelocity::new(ComputeTarget::WebGpu),
783            directml_velocity: HardwareVelocity::new(ComputeTarget::DirectMl),
784            power_throttle_factor: 1.0, // No throttling by default
785        }
786    }
787
788    /// Estimate job duration before execution
789    pub fn estimate_job_duration(&self, params: &JobEstimateParams) -> JobEstimate {
790        // Get velocity for the target compute path
791        let velocity = match params.compute_target {
792            ComputeTarget::Cpu => self.cpu_velocity.get_velocity(),
793            ComputeTarget::WebGpu => self.webgpu_velocity.get_velocity(),
794            ComputeTarget::DirectMl => self.directml_velocity.get_velocity(),
795        };
796
797        // Apply power throttling factor
798        let adjusted_velocity = velocity / self.power_throttle_factor;
799
800        // Apply contention factor for parallel execution
801        let effective_velocity = adjusted_velocity / params.contention_factor;
802
803        // Calculate workload (number of integration points)
804        let workload_points = if params.step_size > 0.0 {
805            params.workload_bytes as f64 / params.step_size as f64
806        } else {
807            params.workload_bytes as f64
808        };
809
810        // Get overhead for target
811        let overhead_secs = match params.compute_target {
812            ComputeTarget::Cpu => 0.005,      // 5ms loop startup
813            ComputeTarget::WebGpu => 0.250,   // 250ms shader dispatch
814            ComputeTarget::DirectMl => 0.100, // 100ms DirectML setup
815        };
816
817        // Calculate estimated duration
818        let duration_secs = (workload_points / effective_velocity) + overhead_secs;
819        let estimated_duration = Duration::from_secs_f64(duration_secs.max(0.0));
820
821        // Calculate confidence based on sample count
822        let sample_count = match params.compute_target {
823            ComputeTarget::Cpu => self.cpu_velocity.sample_count,
824            ComputeTarget::WebGpu => self.webgpu_velocity.sample_count,
825            ComputeTarget::DirectMl => self.directml_velocity.sample_count,
826        };
827
828        // Confidence increases with sample count, caps at 0.95
829        let confidence = (sample_count as f64 / (sample_count as f64 + 10.0)).min(0.95);
830
831        JobEstimate {
832            estimated_duration,
833            confidence,
834            compute_target: params.compute_target,
835            workload_bytes: params.workload_bytes,
836        }
837    }
838
839    /// Update velocity metrics after job completion
840    pub fn record_performance(
841        &mut self,
842        target: ComputeTarget,
843        bytes_processed: u64,
844        duration: Duration,
845    ) {
846        match target {
847            ComputeTarget::Cpu => self.cpu_velocity.update(bytes_processed, duration),
848            ComputeTarget::WebGpu => self.webgpu_velocity.update(bytes_processed, duration),
849            ComputeTarget::DirectMl => self.directml_velocity.update(bytes_processed, duration),
850        }
851    }
852
853    /// Set power throttling factor (called by power daemon)
854    pub fn set_power_throttle_factor(&mut self, factor: f64) {
855        self.power_throttle_factor = factor.max(0.5).min(2.0); // Clamp between 0.5x and 2x
856    }
857
858    /// Get current power throttling factor
859    pub fn get_power_throttle_factor(&self) -> f64 {
860        self.power_throttle_factor
861    }
862
863    /// Get velocity for a specific target
864    pub fn get_velocity(&self, target: ComputeTarget) -> f64 {
865        match target {
866            ComputeTarget::Cpu => self.cpu_velocity.get_velocity(),
867            ComputeTarget::WebGpu => self.webgpu_velocity.get_velocity(),
868            ComputeTarget::DirectMl => self.directml_velocity.get_velocity(),
869        }
870    }
871}
872
873impl Default for JobEstimator {
874    fn default() -> Self {
875        Self::new()
876    }
877}
878
879#[cfg(test)]
880mod tests {
881    use super::*;
882
883    #[test]
884    fn test_job_progress_calculation() {
885        let mut quin = NQuin::default();
886        quin.object = 5000; // 5000 bytes processed
887
888        let job = Job::new(quin, 10000); // Total 10000 bytes
889
890        assert_eq!(job.progress(), 50.0);
891        assert_eq!(job.compute_target, ComputeTarget::Cpu);
892    }
893
894    #[test]
895    fn test_job_velocity_calculation() {
896        let mut quin = NQuin::default();
897        quin.object = 1000;
898
899        let mut job = Job::new(quin, 10000);
900        job.dispatched_at = Some(Instant::now() - Duration::from_secs(1));
901        job.completed_at = Some(Instant::now());
902
903        // 1000 bytes in 1 second ≈ 1000 bytes/sec (allow for timing precision)
904        assert!((job.velocity() - 1000.0).abs() < 10.0);
905    }
906
907    #[test]
908    fn test_production_queue_creation() {
909        let queue = ProductionQueue::new();
910        let stats = queue.stats();
911
912        assert!(stats.num_workers > 0);
913        assert_eq!(stats.active_jobs, 0);
914        assert_eq!(stats.completed_jobs, 0);
915
916        queue.shutdown();
917    }
918
919    #[test]
920    fn test_worker_thread_job_processing() {
921        let queue = ProductionQueue::new();
922        let num_jobs = 10;
923
924        // Submit dummy jobs
925        for i in 0..num_jobs {
926            let mut quin = NQuin::default();
927            quin.object = i * 1000; // Different starting offsets
928            let job = Job::new(quin, 10000);
929            queue.submit_job(job).unwrap();
930        }
931
932        // Wait for jobs to complete
933        std::thread::sleep(Duration::from_secs(2));
934
935        // Collect results
936        let results = queue.collect_results();
937        assert!(results.len() > 0, "At least one job should be completed");
938
939        // Verify progress was made
940        for result in &results {
941            assert!(result.quin.object > 0, "Job should have progressed");
942            assert_eq!(result.status, JobStatus::Completed);
943        }
944
945        queue.shutdown();
946    }
947
948    #[test]
949    fn test_environmental_yielding() {
950        let queue = ProductionQueue::new();
951
952        // Submit a job
953        let mut quin = NQuin::default();
954        quin.object = 1000;
955        let job = Job::new(quin, 10000);
956        queue.submit_job(job).unwrap();
957
958        // Wait a bit for job to start
959        std::thread::sleep(Duration::from_millis(100));
960
961        // Pause the queue (environmental yielding)
962        queue.pause();
963        assert!(queue.is_paused(), "Queue should be paused");
964
965        // Wait to ensure workers are paused
966        std::thread::sleep(Duration::from_millis(200));
967
968        // Resume the queue
969        queue.resume();
970        assert!(!queue.is_paused(), "Queue should not be paused");
971
972        // Wait for job to complete
973        std::thread::sleep(Duration::from_secs(1));
974
975        queue.shutdown();
976    }
977
978    #[test]
979    fn test_telemetry_calculation() {
980        let queue = ProductionQueue::new();
981        let total_bytes = 100000;
982
983        // Submit jobs
984        for i in 0..5 {
985            let mut quin = NQuin::default();
986            quin.object = i * 1000;
987            let job = Job::new(quin, 20000);
988            queue.submit_job(job).unwrap();
989        }
990
991        // Wait for some jobs to complete
992        std::thread::sleep(Duration::from_secs(1));
993
994        // Calculate telemetry
995        let telemetry = queue.calculate_telemetry(total_bytes);
996
997        assert!(telemetry.total_jobs > 0);
998        assert!(telemetry.overall_progress >= 0.0 && telemetry.overall_progress <= 100.0);
999
1000        queue.shutdown();
1001    }
1002
1003    #[test]
1004    fn test_hardware_velocity_tracking() {
1005        let mut velocity = HardwareVelocity::new(ComputeTarget::Cpu);
1006
1007        // Update with some samples
1008        velocity.update(1_000_000, Duration::from_millis(100)); // 10 MB/s
1009        velocity.update(2_000_000, Duration::from_millis(200)); // 10 MB/s
1010
1011        assert_eq!(velocity.sample_count, 2);
1012        assert!(velocity.get_velocity() > 0.0);
1013    }
1014
1015    #[test]
1016    fn test_job_estimator() {
1017        let estimator = JobEstimator::new();
1018
1019        let params = JobEstimateParams {
1020            workload_bytes: 100_000_000, // 100 MB
1021            step_size: 0.01,
1022            compute_target: ComputeTarget::Cpu,
1023            contention_factor: 1.0,
1024        };
1025
1026        let estimate = estimator.estimate_job_duration(&params);
1027
1028        assert!(estimate.estimated_duration.as_secs() > 0);
1029        assert!(estimate.confidence >= 0.0 && estimate.confidence <= 1.0);
1030        assert_eq!(estimate.compute_target, ComputeTarget::Cpu);
1031    }
1032
1033    #[test]
1034    fn test_job_estimator_with_throttling() {
1035        let mut estimator = JobEstimator::new();
1036
1037        // Set power throttling factor
1038        estimator.set_power_throttle_factor(1.5); // 1.5x slower due to power constraints
1039
1040        let params = JobEstimateParams {
1041            workload_bytes: 100_000_000,
1042            step_size: 0.01,
1043            compute_target: ComputeTarget::Cpu,
1044            contention_factor: 1.0,
1045        };
1046
1047        let _estimate = estimator.estimate_job_duration(&params);
1048
1049        // Verify throttling is applied
1050        assert_eq!(estimator.get_power_throttle_factor(), 1.5);
1051    }
1052
1053    #[test]
1054    fn test_job_estimator_velocity_learning() {
1055        let mut estimator = JobEstimator::new();
1056
1057        // Record some performance data
1058        estimator.record_performance(ComputeTarget::Cpu, 10_000_000, Duration::from_secs(1));
1059        estimator.record_performance(ComputeTarget::Cpu, 20_000_000, Duration::from_secs(2));
1060
1061        // Now estimates should be more confident
1062        let params = JobEstimateParams {
1063            workload_bytes: 100_000_000,
1064            step_size: 0.01,
1065            compute_target: ComputeTarget::Cpu,
1066            contention_factor: 1.0,
1067        };
1068
1069        let estimate = estimator.estimate_job_duration(&params);
1070
1071        // Confidence should be higher after recording performance
1072        assert!(estimate.confidence > 0.0);
1073    }
1074
1075    #[test]
1076    fn test_production_queue_estimator_integration() {
1077        let queue = ProductionQueue::new();
1078
1079        // Estimate a job before submission
1080        let params = JobEstimateParams {
1081            workload_bytes: 50_000_000,
1082            step_size: 0.01,
1083            compute_target: ComputeTarget::Cpu,
1084            contention_factor: 1.0,
1085        };
1086
1087        let estimate = queue.estimate_job(&params);
1088        assert!(estimate.estimated_duration.as_secs() > 0);
1089
1090        // Test power throttling
1091        queue.set_power_throttle_factor(1.5);
1092        assert_eq!(queue.get_power_throttle_factor(), 1.5);
1093
1094        // Estimate with throttling should be longer
1095        let throttled_estimate = queue.estimate_job(&params);
1096        assert!(throttled_estimate.estimated_duration >= estimate.estimated_duration);
1097
1098        // Record performance
1099        queue.record_performance(ComputeTarget::Cpu, 10_000_000, Duration::from_secs(1));
1100
1101        queue.shutdown();
1102    }
1103
1104    #[test]
1105    fn test_iterative_ode_workload() {
1106        use crate::modalities::calculus::ode_solver::{ExponentialDecay, Rk4Solver};
1107
1108        let queue = ProductionQueue::new();
1109
1110        // Create ODE solver
1111        let decay = ExponentialDecay::new(0.5);
1112        let mut solver = Rk4Solver::new(decay, 0.01);
1113
1114        // Simulate iterative workload by submitting multiple jobs
1115        let num_steps = 10;
1116        let mut y: f64 = 1.0;
1117        let mut t: f64 = 0.0;
1118
1119        for step in 0..num_steps {
1120            let mut quin = NQuin::default();
1121            quin.object = y.to_bits() as u64;
1122            quin.metadata = t.to_bits();
1123
1124            let job = Job::with_target(quin, 1000, ComputeTarget::Cpu);
1125            queue.submit_job(job).unwrap();
1126
1127            // Advance solver state
1128            y = solver.step(t, y, 0.01);
1129            t += 0.01;
1130
1131            log::debug!("Step {}: t={}, y={}", step, t, y);
1132        }
1133
1134        // Wait for jobs to complete
1135        std::thread::sleep(Duration::from_secs(2));
1136
1137        // Collect results
1138        let results = queue.collect_results();
1139        assert!(results.len() > 0, "At least one job should complete");
1140
1141        // Verify results were processed
1142        for result in &results {
1143            assert_eq!(result.status, JobStatus::Completed);
1144        }
1145
1146        queue.shutdown();
1147    }
1148
1149    #[test]
1150    fn test_process_job_rk4_harmonic_oscillator() {
1151        use crate::modalities::calculus::OP_RK4_STEP;
1152        use crate::solvers::calculus::ode_solver::{extract_ode_state, pack_ode_state};
1153
1154        let (_job_tx, job_rx) = crossbeam_channel::bounded(1);
1155        let (res_tx, _res_rx) = crossbeam_channel::bounded(1);
1156        let worker = WorkerCell::new(
1157            0,
1158            CoreId { id: 0 },
1159            job_rx,
1160            res_tx,
1161            Arc::new(AtomicBool::new(false)),
1162            Arc::new(AtomicBool::new(false)),
1163        );
1164
1165        let mut quin = NQuin::default();
1166        // Opcode OP_RK4_STEP (0x52), system_type = 0 (HarmonicOscillator), param = 2.0 (omega = 2000 in bits 16..31)
1167        let param_u16: u64 = 2000;
1168        quin.predicate = (OP_RK4_STEP as u64) | (0 << 8) | (param_u16 << 16);
1169        let step_size: f32 = 0.05;
1170        quin.context = step_size.to_bits() as u64;
1171        let t0: f64 = 0.0;
1172        let y0: f64 = 1.0;
1173        pack_ode_state(&mut quin, t0, y0);
1174
1175        let mut job = Job::new(quin, 1000);
1176        worker.process_job(&mut job);
1177
1178        let (t1, y1) = extract_ode_state(&job.quin);
1179        assert!((t1 - 0.05).abs() < 1e-6);
1180        assert!(y1.is_finite());
1181        assert_ne!(y0, y1);
1182        // Parity must be valid
1183        assert_eq!(
1184            job.quin.parity,
1185            job.quin.subject ^ job.quin.predicate ^ job.quin.object ^ job.quin.context
1186        );
1187    }
1188
1189    #[test]
1190    fn test_process_job_rk4_exponential_decay() {
1191        use crate::modalities::calculus::OP_RK4_STEP;
1192        use crate::solvers::calculus::ode_solver::{extract_ode_state, pack_ode_state};
1193
1194        let (_job_tx, job_rx) = crossbeam_channel::bounded(1);
1195        let (res_tx, _res_rx) = crossbeam_channel::bounded(1);
1196        let worker = WorkerCell::new(
1197            0,
1198            CoreId { id: 0 },
1199            job_rx,
1200            res_tx,
1201            Arc::new(AtomicBool::new(false)),
1202            Arc::new(AtomicBool::new(false)),
1203        );
1204
1205        let mut quin = NQuin::default();
1206        // system_type = 1 (ExponentialDecay: dy/dt = -lambda * y), lambda = 0.5 (param = 500)
1207        let param_u16: u64 = 500;
1208        quin.predicate = (OP_RK4_STEP as u64) | (1 << 8) | (param_u16 << 16);
1209        let step_size: f32 = 0.1;
1210        quin.context = step_size.to_bits() as u64;
1211        let t0: f64 = 0.0;
1212        let y0: f64 = 1.0;
1213        pack_ode_state(&mut quin, t0, y0);
1214
1215        let mut job = Job::new(quin, 1000);
1216        worker.process_job(&mut job);
1217
1218        let (t1, y1) = extract_ode_state(&job.quin);
1219        assert!((t1 - 0.1).abs() < 1e-6);
1220        assert!(y1 < y0);
1221        assert!((y1 - (-0.05_f64).exp()).abs() < 1e-4);
1222    }
1223
1224    #[test]
1225    fn test_process_job_adaptive_step() {
1226        use crate::modalities::calculus::OP_ADAPTIVE_STEP;
1227        use crate::solvers::calculus::ode_solver::{extract_ode_state, pack_ode_state};
1228
1229        let (_job_tx, job_rx) = crossbeam_channel::bounded(1);
1230        let (res_tx, _res_rx) = crossbeam_channel::bounded(1);
1231        let worker = WorkerCell::new(
1232            0,
1233            CoreId { id: 0 },
1234            job_rx,
1235            res_tx,
1236            Arc::new(AtomicBool::new(false)),
1237            Arc::new(AtomicBool::new(false)),
1238        );
1239
1240        let mut quin = NQuin::default();
1241        quin.predicate = OP_ADAPTIVE_STEP as u64;
1242        let initial_h: f32 = 0.01;
1243        quin.context = initial_h.to_bits() as u64;
1244        pack_ode_state(&mut quin, 0.0, 1.0);
1245
1246        let mut job = Job::new(quin, 1000);
1247        worker.process_job(&mut job);
1248
1249        let (t1, y1) = extract_ode_state(&job.quin);
1250        assert!((t1 - 0.01).abs() < 1e-6);
1251        assert!(y1.is_finite());
1252        let adapted_h = f32::from_bits((job.quin.context & 0xFFFFFFFF) as u32);
1253        assert!(adapted_h > 0.0 && adapted_h <= 1.0);
1254    }
1255
1256    #[test]
1257    fn test_process_job_quadrature_opcodes() {
1258        use crate::modalities::calculus::{OP_SIMPSONS_INTEGRATION, OP_TRAPEZOIDAL_INTEGRATION};
1259        use crate::solvers::calculus::ode_solver::{extract_ode_state, pack_ode_state};
1260
1261        let (_job_tx, job_rx) = crossbeam_channel::bounded(1);
1262        let (res_tx, _res_rx) = crossbeam_channel::bounded(1);
1263        let worker = WorkerCell::new(
1264            0,
1265            CoreId { id: 0 },
1266            job_rx,
1267            res_tx,
1268            Arc::new(AtomicBool::new(false)),
1269            Arc::new(AtomicBool::new(false)),
1270        );
1271
1272        // Test Simpson's
1273        let mut quin = NQuin::default();
1274        quin.predicate = OP_SIMPSONS_INTEGRATION as u64 | (1000 << 16);
1275        quin.context = (0.01_f32).to_bits() as u64;
1276        pack_ode_state(&mut quin, 0.0, 1.0);
1277
1278        let mut job = Job::new(quin, 1000);
1279        worker.process_job(&mut job);
1280        let (t_simp, y_simp) = extract_ode_state(&job.quin);
1281        assert!((t_simp - 0.01).abs() < 1e-6);
1282        assert!(y_simp.is_finite());
1283
1284        // Test Trapezoidal
1285        let mut quin = NQuin::default();
1286        quin.predicate = OP_TRAPEZOIDAL_INTEGRATION as u64 | (1000 << 16);
1287        quin.context = (0.01_f32).to_bits() as u64;
1288        pack_ode_state(&mut quin, 0.0, 1.0);
1289
1290        let mut job = Job::new(quin, 1000);
1291        worker.process_job(&mut job);
1292        let (t_trap, y_trap) = extract_ode_state(&job.quin);
1293        assert!((t_trap - 0.01).abs() < 1e-6);
1294        assert!(y_trap.is_finite());
1295    }
1296}