1use std::fs::{File, OpenOptions};
11use std::io::{BufReader, Read, Seek, Write};
12use std::path::Path;
13
14use serde::{Deserialize, Serialize};
15use sha2::{Digest, Sha256};
16
17use super::graph_contract::{validate_qwen4exp_graph, Qwen4ExpGraphReport, Qwen4ExpLayerContract};
18use crate::bundle::{BundleReader, BundleWriter};
19use crate::ggml_quants::tensor_byte_len;
20use crate::gguf_sharder::{GgufTensorIndex, GgufTensorInfo, Qwen4ExpPleConfig, ARCH_QWEN4EXP};
21
22pub const QWEN4EXP_NATIVE_FORMAT: &str = "qualia.qwen4exp.external-ple.v1";
23const HMC_DESCRIPTOR_KEY: &str = "qwen4exp/placement.cbor";
24const HMC_REQUIREMENTS_KEY: &str = "qwen4exp/runtime-requirements.txt";
25
26#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
29pub struct ExternalGgufAsset {
30 pub file_name: String,
31 #[serde(default)]
35 pub source_path: String,
36 pub byte_len: u64,
37 #[serde(default)]
40 pub sha256: Vec<u8>,
41}
42
43#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
45pub struct Qwen4ExpPleSpan {
46 pub tensor_data_start: u64,
47 pub tensor_byte_offset: u64,
48 pub absolute_offset: u64,
49 pub storage_bytes: u64,
50 pub ggml_type: u32,
51 pub row_width: u64,
52 pub rows: u64,
53 pub row_bytes: u64,
54}
55
56#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
60pub struct Qwen4ExpPleStorage {
61 pub file_path: String,
62 pub byte_len: u64,
63}
64
65#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
67pub struct Qwen4ExpNativeDescriptor {
68 pub format: String,
69 pub architecture: String,
70 pub source: ExternalGgufAsset,
71 pub ple: Qwen4ExpPleSpan,
72 pub ple_storage: Qwen4ExpPleStorage,
73 #[serde(default)]
75 pub ple_config: Qwen4ExpPleConfig,
76 pub trunk_gguf_bytes: u64,
79 pub n_layer: u32,
80 pub n_embd: u32,
81 pub n_head: u32,
82 pub n_kv_head: u32,
83 pub full_attention_interval: u32,
84 pub gated_deltanet_layers: u32,
85 pub qsa_layers: u32,
86 pub ple_layers: u32,
87 pub required_graph_features: Vec<String>,
88}
89
90#[derive(Clone, Debug, PartialEq, Eq)]
91pub enum Qwen4ExpNativeAssetError {
92 InputMissing,
93 OutputExists,
94 Open,
95 Map,
96 NotQwen4Exp,
97 MissingPle,
98 InvalidPle {
99 dims: [u64; 4],
100 n_dims: u32,
101 ggml_type: u32,
102 },
103 SpanOverflow,
104 SourceTruncated,
105 PackageOpen,
106 PackageInvalid,
107 SourceNameMismatch,
108 SourceLengthMismatch,
109 SourceHashUnavailable,
110 PleOutputExists,
111 PleOutputLengthMismatch {
112 expected: u64,
113 actual: u64,
114 },
115 PleOutputPath,
116 PleCopy,
117 SourceHashMismatch,
118 GraphContract,
119 Serialize,
120 Bundle,
121 Write,
122}
123
124impl core::fmt::Display for Qwen4ExpNativeAssetError {
125 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
126 use Qwen4ExpNativeAssetError::*;
127 match self {
128 InputMissing => write!(f, "Qwen4Exp source GGUF was not found"),
129 OutputExists => write!(f, "refusing to overwrite an existing native HMC"),
130 Open => write!(f, "could not open Qwen4Exp source GGUF"),
131 Map => write!(f, "could not map Qwen4Exp source GGUF"),
132 NotQwen4Exp => write!(f, "GGUF is not a Qwen4Exp checkpoint with a PLE table"),
133 MissingPle => write!(f, "Qwen4Exp GGUF is missing its PLE n-gram tensor"),
134 InvalidPle {
135 dims,
136 n_dims,
137 ggml_type,
138 } => write!(
139 f,
140 "Qwen4Exp PLE tensor is not a supported row-addressable table (dims={dims:?}, n_dims={n_dims}, ggml_type={ggml_type})"
141 ),
142 SpanOverflow => write!(f, "Qwen4Exp PLE byte span overflowed"),
143 SourceTruncated => write!(f, "Qwen4Exp PLE span extends beyond source GGUF"),
144 PackageOpen => write!(f, "could not open Qwen4Exp native HMC package"),
145 PackageInvalid => write!(f, "Qwen4Exp native HMC package is invalid"),
146 SourceNameMismatch => write!(f, "Qwen4Exp source name does not match HMC contract"),
147 SourceLengthMismatch => write!(f, "Qwen4Exp source length does not match HMC contract"),
148 SourceHashUnavailable => write!(f, "Qwen4Exp HMC has no completed SHA-256 audit"),
149 PleOutputExists => write!(f, "refusing to overwrite an existing extracted PLE file"),
150 PleOutputLengthMismatch { expected, actual } => write!(
151 f,
152 "existing extracted PLE has {actual} bytes; source contract requires {expected}"
153 ),
154 PleOutputPath => write!(f, "could not create the extracted PLE output path"),
155 PleCopy => write!(f, "could not extract the Qwen4Exp PLE payload"),
156 SourceHashMismatch => write!(f, "Qwen4Exp source SHA-256 does not match HMC contract"),
157 GraphContract => write!(
158 f,
159 "Qwen4Exp GGUF does not satisfy its required execution graph"
160 ),
161 Serialize => write!(f, "could not encode Qwen4Exp HMC descriptor"),
162 Bundle => write!(f, "could not build Qwen4Exp HMC package"),
163 Write => write!(f, "could not write Qwen4Exp HMC package"),
164 }
165 }
166}
167
168impl std::error::Error for Qwen4ExpNativeAssetError {}
169
170impl Qwen4ExpNativeDescriptor {
171 pub fn verify_source_metadata(&self, source: &Path) -> Result<(), Qwen4ExpNativeAssetError> {
175 let name = source
176 .file_name()
177 .and_then(|value| value.to_str())
178 .ok_or(Qwen4ExpNativeAssetError::SourceNameMismatch)?;
179 if name != self.source.file_name {
180 return Err(Qwen4ExpNativeAssetError::SourceNameMismatch);
181 }
182 let length = std::fs::metadata(source)
183 .map_err(|_| Qwen4ExpNativeAssetError::InputMissing)?
184 .len();
185 if length != self.source.byte_len {
186 return Err(Qwen4ExpNativeAssetError::SourceLengthMismatch);
187 }
188 Ok(())
189 }
190
191 pub fn verify_source_sha256(&self, source: &Path) -> Result<(), Qwen4ExpNativeAssetError> {
194 self.verify_source_metadata(source)?;
195 if self.source.sha256.len() != 32 {
196 return Err(Qwen4ExpNativeAssetError::SourceHashUnavailable);
197 }
198 if sha256_file(source)? != self.source.sha256 {
199 return Err(Qwen4ExpNativeAssetError::SourceHashMismatch);
200 }
201 Ok(())
202 }
203
204 pub fn source_beside(
207 &self,
208 package: &Path,
209 ) -> Result<std::path::PathBuf, Qwen4ExpNativeAssetError> {
210 if !self.source.source_path.is_empty() {
211 return Ok(std::path::PathBuf::from(&self.source.source_path));
212 }
213 let parent = package
214 .parent()
215 .ok_or(Qwen4ExpNativeAssetError::PackageOpen)?;
216 Ok(parent.join(&self.source.file_name))
217 }
218}
219
220fn sha256_file(path: &Path) -> Result<Vec<u8>, Qwen4ExpNativeAssetError> {
221 let file = File::open(path).map_err(|_| Qwen4ExpNativeAssetError::Open)?;
222 let mut reader = BufReader::with_capacity(1024 * 1024, file);
223 let mut block = [0u8; 1024 * 1024];
224 let mut hasher = Sha256::new();
225 loop {
226 let n = reader
227 .read(&mut block)
228 .map_err(|_| Qwen4ExpNativeAssetError::Open)?;
229 if n == 0 {
230 break;
231 }
232 hasher.update(&block[..n]);
233 }
234 Ok(hasher.finalize().to_vec())
235}
236
237fn ple_span(
238 index: &GgufTensorIndex,
239 source_bytes: u64,
240) -> Result<Qwen4ExpPleSpan, Qwen4ExpNativeAssetError> {
241 let info: GgufTensorInfo = *index
242 .ple_ngram_embedding_info()
243 .ok_or(Qwen4ExpNativeAssetError::MissingPle)?;
244 if info.n_dims != 2 || info.dims[0] == 0 || info.dims[1] == 0 {
245 return Err(Qwen4ExpNativeAssetError::InvalidPle {
246 dims: info.dims,
247 n_dims: info.n_dims,
248 ggml_type: info.ggml_type,
249 });
250 }
251 let storage_bytes = tensor_byte_len(&info).ok_or(Qwen4ExpNativeAssetError::InvalidPle {
252 dims: info.dims,
253 n_dims: info.n_dims,
254 ggml_type: info.ggml_type,
255 })? as u64;
256 let row_width = info.dims[0];
257 let rows = info.dims[1];
258 let row_bytes = storage_bytes
259 .checked_div(rows)
260 .filter(|bytes| *bytes != 0)
261 .ok_or(Qwen4ExpNativeAssetError::InvalidPle {
262 dims: info.dims,
263 n_dims: info.n_dims,
264 ggml_type: info.ggml_type,
265 })?;
266 let absolute_offset = index
267 .tensor_data_start
268 .checked_add(info.byte_offset)
269 .ok_or(Qwen4ExpNativeAssetError::SpanOverflow)?;
270 let end = absolute_offset
271 .checked_add(storage_bytes)
272 .ok_or(Qwen4ExpNativeAssetError::SpanOverflow)?;
273 if end > source_bytes {
274 return Err(Qwen4ExpNativeAssetError::SourceTruncated);
275 }
276 Ok(Qwen4ExpPleSpan {
277 tensor_data_start: index.tensor_data_start,
278 tensor_byte_offset: info.byte_offset,
279 absolute_offset,
280 storage_bytes,
281 ggml_type: info.ggml_type,
282 row_width,
283 rows,
284 row_bytes,
285 })
286}
287
288fn extract_ple_to_nvme(
293 source: &Path,
294 ple: Qwen4ExpPleSpan,
295 destination: &Path,
296) -> Result<(), Qwen4ExpNativeAssetError> {
297 if destination.exists() {
298 return Err(Qwen4ExpNativeAssetError::PleOutputExists);
299 }
300 let parent = destination
301 .parent()
302 .ok_or(Qwen4ExpNativeAssetError::PleOutputPath)?;
303 if !parent.is_dir() {
304 return Err(Qwen4ExpNativeAssetError::PleOutputPath);
305 }
306 let mut input = File::open(source).map_err(|_| Qwen4ExpNativeAssetError::Open)?;
307 input
308 .seek(std::io::SeekFrom::Start(ple.absolute_offset))
309 .map_err(|_| Qwen4ExpNativeAssetError::PleCopy)?;
310 let mut output = OpenOptions::new()
311 .write(true)
312 .create_new(true)
313 .open(destination)
314 .map_err(|_| Qwen4ExpNativeAssetError::PleOutputExists)?;
315 let mut remaining = ple.storage_bytes;
316 let mut buffer = [0u8; 1024 * 1024];
317 while remaining != 0 {
318 let take = (remaining as usize).min(buffer.len());
319 input
320 .read_exact(&mut buffer[..take])
321 .map_err(|_| Qwen4ExpNativeAssetError::PleCopy)?;
322 if output.write_all(&buffer[..take]).is_err() {
323 let _ = std::fs::remove_file(destination);
324 return Err(Qwen4ExpNativeAssetError::PleCopy);
325 }
326 remaining -= take as u64;
327 }
328 if output.flush().is_err() {
329 let _ = std::fs::remove_file(destination);
330 return Err(Qwen4ExpNativeAssetError::PleCopy);
331 }
332 Ok(())
333}
334
335pub fn prepare_qwen4exp_native_hmc(
341 source: &Path,
342 output: &Path,
343 ple_output: &Path,
344) -> Result<Qwen4ExpNativeDescriptor, Qwen4ExpNativeAssetError> {
345 if !source.is_file() {
346 return Err(Qwen4ExpNativeAssetError::InputMissing);
347 }
348 if output.exists() {
349 return Err(Qwen4ExpNativeAssetError::OutputExists);
350 }
351 let source_bytes = std::fs::metadata(source)
352 .map_err(|_| Qwen4ExpNativeAssetError::Open)?
353 .len();
354 let file = File::open(source).map_err(|_| Qwen4ExpNativeAssetError::Open)?;
355 let mmap = unsafe { memmap2::Mmap::map(&file).map_err(|_| Qwen4ExpNativeAssetError::Map)? };
357 let index = GgufTensorIndex::from_gguf(&mmap);
358 if index.hyperparams.architecture != ARCH_QWEN4EXP {
359 return Err(Qwen4ExpNativeAssetError::NotQwen4Exp);
360 }
361 let ple = ple_span(&index, source_bytes)?;
362 let ple_config = index
363 .ple_config
364 .filter(Qwen4ExpPleConfig::is_complete)
365 .ok_or(Qwen4ExpNativeAssetError::MissingPle)?;
366 let mut layer_contracts = [Qwen4ExpLayerContract::default(); 128];
367 let graph: Qwen4ExpGraphReport = validate_qwen4exp_graph(&index, &mut layer_contracts)
368 .map_err(|_| Qwen4ExpNativeAssetError::GraphContract)?;
369 let descriptor = Qwen4ExpNativeDescriptor {
370 format: QWEN4EXP_NATIVE_FORMAT.to_string(),
371 architecture: index.hyperparams.architecture_name().to_string(),
372 source: ExternalGgufAsset {
373 file_name: source
374 .file_name()
375 .and_then(|name| name.to_str())
376 .ok_or(Qwen4ExpNativeAssetError::InputMissing)?
377 .to_string(),
378 source_path: source
379 .canonicalize()
380 .map_err(|_| Qwen4ExpNativeAssetError::InputMissing)?
381 .display()
382 .to_string(),
383 byte_len: source_bytes,
384 sha256: Vec::new(),
385 },
386 ple,
387 ple_storage: Qwen4ExpPleStorage {
388 file_path: ple_output.display().to_string(),
389 byte_len: ple.storage_bytes,
390 },
391 ple_config,
392 trunk_gguf_bytes: source_bytes.saturating_sub(ple.storage_bytes),
393 n_layer: index.hyperparams.n_layer,
394 n_embd: index.hyperparams.n_embd,
395 n_head: index.hyperparams.n_head,
396 n_kv_head: index.hyperparams.effective_n_kv_head(),
397 full_attention_interval: index.hyperparams.full_attention_interval,
398 gated_deltanet_layers: graph.gated_deltanet_layers,
399 qsa_layers: graph.qsa_layers,
400 ple_layers: graph.ple_layers,
401 required_graph_features: vec![
402 "PLE direct NVMe row gather".to_string(),
403 "Hyper-Connection".to_string(),
404 "GatedDeltaNet recurrent state".to_string(),
405 "Qwen Sparse Attention".to_string(),
406 "top-10-of-512 MoE plus shared expert".to_string(),
407 ],
408 };
409 let mut descriptor_cbor = Vec::new();
410 ciborium::into_writer(&descriptor, &mut descriptor_cbor)
411 .map_err(|_| Qwen4ExpNativeAssetError::Serialize)?;
412 let requirements = b"Qwen4Exp native package. The PLE table is external by design: resolve the source_path in placement.cbor and read selected rows at the recorded spans. Preparation attests file name, length, architecture and graph; a full SHA-256 audit is optional and must not delay first activation. The runtime must not require whole-table residency, staging, or conversion. P64 v4 is limited to one <4 GiB payload; add trunk shards only after each shard has a coherent Qwen4Exp graph consumer.\n".to_vec();
413 let mut bundle = BundleWriter::new();
414 bundle
415 .add_file(
416 HMC_DESCRIPTOR_KEY,
417 "qwen4exp-placement",
418 descriptor_cbor,
419 None,
420 )
421 .map_err(|_| Qwen4ExpNativeAssetError::Bundle)?;
422 bundle
423 .add_file(HMC_REQUIREMENTS_KEY, "text", requirements, None)
424 .map_err(|_| Qwen4ExpNativeAssetError::Bundle)?;
425 let package = bundle
426 .build()
427 .map_err(|_| Qwen4ExpNativeAssetError::Bundle)?;
428 let created_ple = if ple_output.exists() {
429 let length = std::fs::metadata(ple_output)
430 .map_err(|_| Qwen4ExpNativeAssetError::PleOutputPath)?
431 .len();
432 if length != ple.storage_bytes {
433 return Err(Qwen4ExpNativeAssetError::PleOutputLengthMismatch {
434 expected: ple.storage_bytes,
435 actual: length,
436 });
437 }
438 false
439 } else {
440 extract_ple_to_nvme(source, ple, ple_output)?;
441 true
442 };
443 let mut target = OpenOptions::new()
444 .write(true)
445 .create_new(true)
446 .open(output)
447 .map_err(|_| {
448 if created_ple {
449 let _ = std::fs::remove_file(ple_output);
450 }
451 Qwen4ExpNativeAssetError::Write
452 })?;
453 target
454 .write_all(&package)
455 .and_then(|_| target.flush())
456 .map_err(|_| {
457 if created_ple {
458 let _ = std::fs::remove_file(ple_output);
459 }
460 let _ = std::fs::remove_file(output);
461 Qwen4ExpNativeAssetError::Write
462 })?;
463 Ok(descriptor)
464}
465
466pub fn load_qwen4exp_native_hmc(
471 package: &Path,
472) -> Result<Qwen4ExpNativeDescriptor, Qwen4ExpNativeAssetError> {
473 let bytes = std::fs::read(package).map_err(|_| Qwen4ExpNativeAssetError::PackageOpen)?;
474 parse_qwen4exp_native_hmc(&bytes)
475}
476
477pub(super) fn parse_qwen4exp_native_hmc(bytes: &[u8]) -> Result<Qwen4ExpNativeDescriptor, Qwen4ExpNativeAssetError> {
478 let reader =
479 BundleReader::parse(bytes).map_err(|_| Qwen4ExpNativeAssetError::PackageInvalid)?;
480 let descriptor_bytes = reader
481 .get(HMC_DESCRIPTOR_KEY)
482 .ok_or(Qwen4ExpNativeAssetError::PackageInvalid)?;
483 let descriptor: Qwen4ExpNativeDescriptor = ciborium::from_reader(descriptor_bytes)
484 .map_err(|_| Qwen4ExpNativeAssetError::PackageInvalid)?;
485 validate_qwen4exp_descriptor(descriptor)
486}
487
488pub(super) fn validate_qwen4exp_descriptor(descriptor: Qwen4ExpNativeDescriptor)
489 -> Result<Qwen4ExpNativeDescriptor, Qwen4ExpNativeAssetError>
490{
491 if descriptor.format != QWEN4EXP_NATIVE_FORMAT
492 || descriptor.architecture != "qwen4exp"
493 || descriptor.ple.storage_bytes == 0
494 || descriptor.ple.row_width == 0
495 || descriptor.ple.rows == 0
496 || descriptor.ple.row_bytes == 0
497 || descriptor.ple_storage.file_path.is_empty()
498 || descriptor.ple_storage.byte_len != descriptor.ple.storage_bytes
499 || !descriptor.ple_config.is_complete()
500 || !(descriptor.source.sha256.is_empty() || descriptor.source.sha256.len() == 32)
501 || descriptor.gated_deltanet_layers == 0
502 || descriptor.qsa_layers == 0
503 || descriptor.ple_layers == 0
504 {
505 return Err(Qwen4ExpNativeAssetError::PackageInvalid);
506 }
507 let span_end = descriptor
508 .ple
509 .absolute_offset
510 .checked_add(descriptor.ple.storage_bytes)
511 .ok_or(Qwen4ExpNativeAssetError::PackageInvalid)?;
512 if span_end > descriptor.source.byte_len {
513 return Err(Qwen4ExpNativeAssetError::PackageInvalid);
514 }
515 Ok(descriptor)
516}
517
518#[cfg(test)]
519mod tests {
520 use super::*;
521
522 #[test]
523 fn ple_span_rejects_truncated_external_source() {
524 let ple = GgufTensorInfo {
525 dims: [32, 16, 1, 1],
526 n_dims: 2,
527 ggml_type: crate::ggml_quants::GGML_TYPE_F32,
528 byte_offset: 512,
529 };
530 let mut hp = crate::gguf_sharder::GgufHyperparams::default();
531 hp.architecture = ARCH_QWEN4EXP;
532 let index =
533 GgufTensorIndex::from_components(&[(b"per_layer_token_embd.weight", ple)], hp, 128);
534 assert_eq!(
535 ple_span(&index, 1024).unwrap_err(),
536 Qwen4ExpNativeAssetError::SourceTruncated
537 );
538 }
539
540 #[test]
541 fn descriptor_rejects_wrong_external_identity_before_opening_ple() {
542 let descriptor = Qwen4ExpNativeDescriptor {
543 format: QWEN4EXP_NATIVE_FORMAT.to_string(),
544 architecture: "qwen4exp".to_string(),
545 source: ExternalGgufAsset {
546 file_name: "expected.gguf".to_string(),
547 source_path: String::new(),
548 byte_len: 1,
549 sha256: vec![0; 32],
550 },
551 ple: Qwen4ExpPleSpan {
552 tensor_data_start: 0,
553 tensor_byte_offset: 0,
554 absolute_offset: 0,
555 storage_bytes: 1,
556 ggml_type: 0,
557 row_width: 1,
558 rows: 1,
559 row_bytes: 1,
560 },
561 ple_storage: Qwen4ExpPleStorage {
562 file_path: "C:\\ple.bin".to_string(),
563 byte_len: 1,
564 },
565 ple_config: Qwen4ExpPleConfig::default(),
566 trunk_gguf_bytes: 0,
567 n_layer: 48,
568 n_embd: 2560,
569 n_head: 24,
570 n_kv_head: 2,
571 full_attention_interval: 4,
572 gated_deltanet_layers: 36,
573 qsa_layers: 12,
574 ple_layers: 1,
575 required_graph_features: vec![],
576 };
577 let dir = tempfile::tempdir().unwrap();
578 let wrong = dir.path().join("wrong.gguf");
579 std::fs::write(&wrong, [0u8]).unwrap();
580 assert_eq!(
581 descriptor.verify_source_metadata(&wrong).unwrap_err(),
582 Qwen4ExpNativeAssetError::SourceNameMismatch
583 );
584 }
585}