Skip to main content

qualia_core_db/q42/
yaml_ld_q42.rs

1use crate::{q_hash, NQuin};
2use serde::{Deserialize, Serialize};
3
4/// Represents a simple event from the streaming YAML lexer.
5/// In a zero-alloc edge system, these would stream directly from a `&[u8]` buffer.
6#[derive(Debug)]
7pub enum YamlToken<'a> {
8    MapStart,
9    MapEnd,
10    ListStart,
11    ListEnd,
12    Key(&'a str),
13    ValueString(&'a str),
14    ValueInt(i64),
15}
16
17/// A lightweight, state-machine based lexer for yaml-ld-q42.
18/// Designed to parse without full document materialisation, avoiding `Vec` or `String` allocs
19/// during the hot-path traversal of pane manifests.
20pub struct YamlStreamingLexer<'a> {
21    buffer: &'a [u8],
22    cursor: usize,
23}
24
25impl<'a> YamlStreamingLexer<'a> {
26    /// Creates a default `YamlStreamingLexer`.
27    pub fn new(buffer: &'a [u8]) -> Self {
28        Self { buffer, cursor: 0 }
29    }
30
31    fn skip_ws_and_comments(&mut self) {
32        while self.cursor < self.buffer.len() {
33            let b = self.buffer[self.cursor];
34            if b == b' ' || b == b'\t' || b == b'\r' || b == b'\n' {
35                self.cursor += 1;
36                continue;
37            }
38            if b == b'#' {
39                while self.cursor < self.buffer.len() && self.buffer[self.cursor] != b'\n' {
40                    self.cursor += 1;
41                }
42                continue;
43            }
44            break;
45        }
46    }
47
48    /// Advance the lexer to the next semantic token.
49    pub fn next_token(&mut self) -> Option<YamlToken<'a>> {
50        self.skip_ws_and_comments();
51        if self.cursor >= self.buffer.len() {
52            return None;
53        }
54        if self.buffer[self.cursor] == b'-' {
55            self.cursor += 1;
56            return Some(YamlToken::ListStart);
57        }
58        let start = self.cursor;
59        while self.cursor < self.buffer.len() {
60            let b = self.buffer[self.cursor];
61            if b == b':' || b.is_ascii_whitespace() {
62                break;
63            }
64            self.cursor += 1;
65        }
66        if self.cursor >= self.buffer.len() || self.buffer[self.cursor] != b':' {
67            self.cursor = self.buffer.len();
68            return None;
69        }
70        let key = std::str::from_utf8(&self.buffer[start..self.cursor]).ok()?;
71        self.cursor += 1;
72        Some(YamlToken::Key(key))
73    }
74}
75
76/// A fully structured Webizen Studio workspace, typically deserialized from yaml-ld-q42.
77#[derive(Serialize, Deserialize, Debug, Clone)]
78pub struct WebizenWorkspace {
79    pub pages: Vec<Page>,
80}
81
82#[derive(Serialize, Deserialize, Debug, Clone)]
83pub struct Page {
84    pub url_path: String,
85    pub name: String,
86    pub panes: Vec<Pane>,
87}
88
89#[derive(Serialize, Deserialize, Debug, Clone)]
90pub struct Pane {
91    pub component_id: String,
92    pub x: u8,
93    pub y: u8,
94    pub w: u8,
95    pub h: u8,
96}
97
98/// Parses a yaml-ld-q42 byte stream, packs the layout metadata into CBOR-LD (if needed),
99/// and compiles the results down into a list of 48-byte NQuins.
100pub fn compile_yaml_ld_to_quins(
101    yaml_bytes: &[u8],
102    namespace: u64,
103    lamport_clock: u64,
104) -> Result<Vec<NQuin>, &'static str> {
105    // 1. Validate the byte stream with the streaming lexer, then parse structurally.
106    let mut lexer = YamlStreamingLexer::new(yaml_bytes);
107    while lexer.next_token().is_some() {}
108
109    let workspace: WebizenWorkspace =
110        serde_yaml::from_slice(yaml_bytes).map_err(|_| "Failed to parse yaml-ld-q42 payload")?;
111
112    let mut quins = Vec::new();
113    let pred_pane = q_hash("q42:SystemPaneState");
114    let pred_page = q_hash("q42:SystemPageDef");
115
116    // 2. Iterate through pages and panes
117    for page in workspace.pages {
118        let page_hash = q_hash(&page.url_path);
119
120        // Emit page definition Quin
121        let page_name_hash = q_hash(&page.name);
122        quins.push(NQuin {
123            subject: page_hash,
124            predicate: pred_page,
125            object: page_name_hash,
126            context: namespace,
127            metadata: lamport_clock << 32,
128            parity: page_hash ^ pred_page ^ page_name_hash ^ namespace,
129        });
130
131        // 3. For each pane, pack the mathematical bounding box into the NQuin metadata
132        for pane in page.panes {
133            // Encode the 4-byte bounding box (x, y, w, h) into the lower 32 bits.
134            let packed_layout: u64 = ((pane.x as u64) << 24)
135                | ((pane.y as u64) << 16)
136                | ((pane.w as u64) << 8)
137                | (pane.h as u64);
138
139            let metadata = packed_layout | (lamport_clock << 32);
140            let subject = q_hash(&pane.component_id);
141
142            // Note: If we needed to store complex configurations beyond the bounding box,
143            // we would CBOR-encode them here:
144            // let mut cbor_buf = [0u8; 128];
145            // let cbor_len = ciborium::into_writer(&pane_config, &mut cbor_buf[..]).unwrap();
146            // Then we would store the cbor_buf in a dedicated payload store and put the
147            // payload hash in the NQuin `object` field.
148
149            quins.push(NQuin {
150                subject,
151                predicate: pred_pane,
152                object: page_hash, // Panes belong to a page
153                context: namespace,
154                metadata,
155                parity: subject ^ pred_pane ^ page_hash ^ namespace,
156            });
157        }
158    }
159
160    Ok(quins)
161}
162
163use serde_yaml::Value as YamlValue;
164use std::collections::HashMap;
165
166/// Result of compiling an HCF (HypermediaDocument) yaml-ld-q42 authoring document.
167pub type HcfCompileResult = (Vec<NQuin>, HashMap<u64, String>);
168
169/// Detect `@type: HypermediaDocument` (string or list containing it).
170fn yaml_type_is_hypermedia_document(value: &YamlValue) -> bool {
171    match value.get("@type") {
172        Some(YamlValue::String(s)) => type_str_is_hypermedia(s),
173        Some(YamlValue::Sequence(seq)) => seq
174            .iter()
175            .any(|v| v.as_str().map(type_str_is_hypermedia).unwrap_or(false)),
176        _ => false,
177    }
178}
179
180fn type_str_is_hypermedia(s: &str) -> bool {
181    s == "HypermediaDocument"
182        || s.ends_with(":HypermediaDocument")
183        || s.ends_with("/HypermediaDocument")
184        || s.ends_with("#HypermediaDocument")
185}
186
187/// True when `content` is a non-empty sequence (HCF sections/blocks).
188fn yaml_has_content_sections(value: &YamlValue) -> bool {
189    value
190        .get("content")
191        .and_then(YamlValue::as_sequence)
192        .map(|s| !s.is_empty())
193        .unwrap_or(false)
194}
195
196/// True when top-level `pages` is a non-empty sequence (WebizenWorkspace).
197fn yaml_has_workspace_pages(value: &YamlValue) -> bool {
198    value
199        .get("pages")
200        .and_then(YamlValue::as_sequence)
201        .map(|s| !s.is_empty())
202        .unwrap_or(false)
203}
204
205fn lex_insert(lexicon: &mut HashMap<u64, String>, surface: &str) -> u64 {
206    let h = q_hash(surface);
207    lexicon.entry(h).or_insert_with(|| surface.to_string());
208    h
209}
210
211fn emit_spo(
212    quins: &mut Vec<NQuin>,
213    lexicon: &mut HashMap<u64, String>,
214    subject: u64,
215    pred_iri: &str,
216    object_surface: &str,
217    namespace: u64,
218    lamport: u64,
219) {
220    let predicate = lex_insert(lexicon, pred_iri);
221    let object = lex_insert(lexicon, object_surface);
222    quins.push(NQuin {
223        subject,
224        predicate,
225        object,
226        context: namespace,
227        metadata: lamport << 32,
228        parity: subject ^ predicate ^ object ^ namespace,
229    });
230}
231
232fn yaml_string<'a>(v: &'a YamlValue) -> Option<&'a str> {
233    v.as_str()
234}
235
236fn yaml_string_owned(v: &YamlValue) -> Option<String> {
237    match v {
238        YamlValue::String(s) => Some(s.clone()),
239        YamlValue::Number(n) => Some(n.to_string()),
240        YamlValue::Bool(b) => Some(b.to_string()),
241        _ => None,
242    }
243}
244
245/// Compile an HCF HypermediaDocument (yaml-ld-q42 authoring layer) into Quins + lexicon.
246///
247/// Accepts documents where `@type` is `HypermediaDocument` (string or list) **or**
248/// a non-empty `content:` sequence of sections/blocks exists.
249///
250/// Emits Quins (with lexicon surfaces) for `@id`, `title`, optional
251/// `date`/`author`/`summary`/`tags`, section `heading`s, paragraph `text`, and
252/// optional `markdown` projector fields (as text).
253pub fn compile_hcf_yaml_ld_to_quins(
254    yaml_bytes: &[u8],
255    namespace: u64,
256    lamport: u64,
257) -> Result<HcfCompileResult, String> {
258    let root: YamlValue = serde_yaml::from_slice(yaml_bytes)
259        .map_err(|e| format!("Failed to parse HCF yaml-ld-q42: {e}"))?;
260
261    if !yaml_type_is_hypermedia_document(&root) && !yaml_has_content_sections(&root) {
262        return Err(
263            "not an HCF HypermediaDocument: need @type HypermediaDocument or content: sections"
264                .into(),
265        );
266    }
267
268    let mut quins = Vec::new();
269    let mut lexicon: HashMap<u64, String> = HashMap::new();
270
271    let doc_id_surface = root
272        .get("@id")
273        .and_then(yaml_string)
274        .unwrap_or("hcf:anonymous");
275    let doc_subject = lex_insert(&mut lexicon, doc_id_surface);
276
277    // rdf:type
278    emit_spo(
279        &mut quins,
280        &mut lexicon,
281        doc_subject,
282        "rdf:type",
283        "HypermediaDocument",
284        namespace,
285        lamport,
286    );
287
288    // Scalar document fields
289    for (key, pred) in [
290        ("title", "hcf:title"),
291        ("date", "hcf:date"),
292        ("author", "hcf:author"),
293        ("summary", "hcf:summary"),
294        ("markdown", "hcf:markdown"),
295    ] {
296        if let Some(v) = root.get(key).and_then(yaml_string_owned) {
297            emit_spo(
298                &mut quins,
299                &mut lexicon,
300                doc_subject,
301                pred,
302                &v,
303                namespace,
304                lamport,
305            );
306        }
307    }
308
309    // tags: string or sequence of strings
310    if let Some(tags) = root.get("tags") {
311        match tags {
312            YamlValue::String(s) => {
313                emit_spo(
314                    &mut quins,
315                    &mut lexicon,
316                    doc_subject,
317                    "hcf:tag",
318                    s,
319                    namespace,
320                    lamport,
321                );
322            }
323            YamlValue::Sequence(seq) => {
324                for t in seq {
325                    if let Some(s) = yaml_string_owned(t) {
326                        emit_spo(
327                            &mut quins,
328                            &mut lexicon,
329                            doc_subject,
330                            "hcf:tag",
331                            &s,
332                            namespace,
333                            lamport,
334                        );
335                    }
336                }
337            }
338            _ => {}
339        }
340    }
341
342    // Walk content sections / blocks
343    if let Some(YamlValue::Sequence(content)) = root.get("content") {
344        for (si, section) in content.iter().enumerate() {
345            let section_id = section
346                .get("@id")
347                .and_then(yaml_string)
348                .map(|s| s.to_string())
349                .unwrap_or_else(|| format!("{doc_id_surface}#section-{si}"));
350            let section_subject = lex_insert(&mut lexicon, &section_id);
351
352            emit_spo(
353                &mut quins,
354                &mut lexicon,
355                doc_subject,
356                "hcf:hasSection",
357                &section_id,
358                namespace,
359                lamport,
360            );
361
362            if let Some(heading) = section.get("heading").and_then(yaml_string_owned) {
363                emit_spo(
364                    &mut quins,
365                    &mut lexicon,
366                    section_subject,
367                    "hcf:heading",
368                    &heading,
369                    namespace,
370                    lamport,
371                );
372            }
373            if let Some(md) = section.get("markdown").and_then(yaml_string_owned) {
374                emit_spo(
375                    &mut quins,
376                    &mut lexicon,
377                    section_subject,
378                    "hcf:markdown",
379                    &md,
380                    namespace,
381                    lamport,
382                );
383            }
384            // Section-level text (rare but allowed)
385            if let Some(text) = section.get("text").and_then(yaml_string_owned) {
386                emit_spo(
387                    &mut quins,
388                    &mut lexicon,
389                    section_subject,
390                    "hcf:text",
391                    &text,
392                    namespace,
393                    lamport,
394                );
395            }
396
397            // blocks under section
398            if let Some(YamlValue::Sequence(blocks)) = section.get("blocks") {
399                for (bi, block) in blocks.iter().enumerate() {
400                    compile_hcf_block(
401                        &mut quins,
402                        &mut lexicon,
403                        section_subject,
404                        &section_id,
405                        bi,
406                        block,
407                        namespace,
408                        lamport,
409                    );
410                }
411            }
412
413            // Some HCF docs put paragraphs directly under content without blocks
414            if section.get("text").is_none() && section.get("blocks").is_none() {
415                if let Some(md) = section.get("markdown").and_then(yaml_string_owned) {
416                    let _ = md; // already handled above
417                }
418            }
419        }
420    }
421
422    Ok((quins, lexicon))
423}
424
425fn compile_hcf_block(
426    quins: &mut Vec<NQuin>,
427    lexicon: &mut HashMap<u64, String>,
428    section_subject: u64,
429    section_id: &str,
430    bi: usize,
431    block: &YamlValue,
432    namespace: u64,
433    lamport: u64,
434) {
435    let block_id = block
436        .get("@id")
437        .and_then(yaml_string)
438        .map(|s| s.to_string())
439        .unwrap_or_else(|| format!("{section_id}#block-{bi}"));
440    let block_subject = lex_insert(lexicon, &block_id);
441
442    emit_spo(
443        quins,
444        lexicon,
445        section_subject,
446        "hcf:hasBlock",
447        &block_id,
448        namespace,
449        lamport,
450    );
451
452    if let Some(text) = block.get("text").and_then(yaml_string_owned) {
453        emit_spo(
454            quins,
455            lexicon,
456            block_subject,
457            "hcf:text",
458            &text,
459            namespace,
460            lamport,
461        );
462    }
463    if let Some(md) = block.get("markdown").and_then(yaml_string_owned) {
464        emit_spo(
465            quins,
466            lexicon,
467            block_subject,
468            "hcf:markdown",
469            &md,
470            namespace,
471            lamport,
472        );
473    }
474}
475
476/// Auto-dispatch yaml-ld-q42 compilation:
477/// 1. workspace `pages` present → [`compile_yaml_ld_to_quins`] (lexicon empty/minimal)
478/// 2. else HCF HypermediaDocument / `content:` → [`compile_hcf_yaml_ld_to_quins`]
479/// 3. else error with a clear message
480///
481/// Returns `(quins, lexicon, source_format)` where `source_format` is
482/// `"yaml-ld-q42/workspace"` or `"yaml-ld-q42/hcf"`.
483pub fn compile_yaml_ld_q42_auto(
484    yaml_bytes: &[u8],
485    namespace: u64,
486    lamport: u64,
487) -> Result<(Vec<NQuin>, HashMap<u64, String>, &'static str), String> {
488    let root: YamlValue = serde_yaml::from_slice(yaml_bytes)
489        .map_err(|e| format!("Failed to parse yaml-ld-q42: {e}"))?;
490
491    if yaml_has_workspace_pages(&root) {
492        let quins =
493            compile_yaml_ld_to_quins(yaml_bytes, namespace, lamport).map_err(|e| e.to_string())?;
494        // Minimal lexicon: page/pane surface labels from the typed compile path are hashed
495        // only; keep lexicon empty/minimal as documented for workspace shape.
496        Ok((quins, HashMap::new(), "yaml-ld-q42/workspace"))
497    } else if yaml_type_is_hypermedia_document(&root) || yaml_has_content_sections(&root) {
498        let (quins, lexicon) = compile_hcf_yaml_ld_to_quins(yaml_bytes, namespace, lamport)?;
499        Ok((quins, lexicon, "yaml-ld-q42/hcf"))
500    } else {
501        Err(
502            "yaml-ld-q42: expected WebizenWorkspace (pages:) or HCF HypermediaDocument \
503             (@type HypermediaDocument or content: sections)"
504                .into(),
505        )
506    }
507}
508
509#[cfg(test)]
510mod tests {
511    use super::*;
512
513    const WORKSPACE_YAML: &str = r#"
514pages:
515  - url_path: "/health"
516    name: "Health Dashboard"
517    panes:
518      - component_id: "sensor-data"
519        x: 1
520        y: 1
521        w: 6
522        h: 4
523"#;
524
525    const HCF_YAML: &str = r#"
526"@context":
527  "@vocab": "https://qualiadb.org/schema/hcf#"
528"@id": "doc:civics_writing_sample"
529"@type": "HypermediaDocument"
530"title": "Civics Writing Sample"
531"author": "Timothy Holborn"
532"date": "2026-09-26"
533"summary": "Minimal HCF for wasm ingest"
534"tags":
535  - civics
536  - yaml-ld-q42
537"content":
538  - "@type": "Section"
539    "heading": "Introduction"
540    "blocks":
541      - "@type": "Paragraph"
542        "text": "HypermediaDocument is the SoT; Markdown is an optional projector."
543        "markdown": "**Bold** projector field"
544"#;
545
546    #[test]
547    fn compile_workspace_pages_emits_page_and_pane_quins() {
548        let quins = compile_yaml_ld_to_quins(WORKSPACE_YAML.as_bytes(), 42, 7).unwrap();
549        assert!(
550            quins.len() >= 2,
551            "expected page + pane quins, got {}",
552            quins.len()
553        );
554        let pred_page = q_hash("q42:SystemPageDef");
555        let pred_pane = q_hash("q42:SystemPaneState");
556        assert!(quins.iter().any(|q| q.predicate == pred_page));
557        assert!(quins.iter().any(|q| q.predicate == pred_pane));
558        assert_eq!(quins[0].context, 42);
559    }
560
561    #[test]
562    fn compile_hcf_emits_title_heading_text_and_lexicon() {
563        let (quins, lexicon) = compile_hcf_yaml_ld_to_quins(HCF_YAML.as_bytes(), 1, 9).unwrap();
564        assert!(!quins.is_empty());
565        assert!(lexicon.values().any(|s| s == "Civics Writing Sample"));
566        assert!(lexicon.values().any(|s| s == "Introduction"));
567        assert!(lexicon
568            .values()
569            .any(|s| s.contains("HypermediaDocument is the SoT")));
570        assert!(lexicon.values().any(|s| s.contains("Bold")));
571        assert!(lexicon.values().any(|s| s == "civics"));
572        // title quin present
573        let pred_title = q_hash("hcf:title");
574        assert!(quins.iter().any(|q| q.predicate == pred_title));
575        let pred_heading = q_hash("hcf:heading");
576        assert!(quins.iter().any(|q| q.predicate == pred_heading));
577        let pred_text = q_hash("hcf:text");
578        assert!(quins.iter().any(|q| q.predicate == pred_text));
579        let pred_md = q_hash("hcf:markdown");
580        assert!(quins.iter().any(|q| q.predicate == pred_md));
581    }
582
583    #[test]
584    fn auto_dispatches_workspace_and_hcf() {
585        let (wq, wlex, wfmt) = compile_yaml_ld_q42_auto(WORKSPACE_YAML.as_bytes(), 0, 1).unwrap();
586        assert_eq!(wfmt, "yaml-ld-q42/workspace");
587        assert!(wq.len() >= 2);
588        assert!(wlex.is_empty());
589
590        let (hq, hlex, hfmt) = compile_yaml_ld_q42_auto(HCF_YAML.as_bytes(), 0, 1).unwrap();
591        assert_eq!(hfmt, "yaml-ld-q42/hcf");
592        assert!(!hq.is_empty());
593        assert!(!hlex.is_empty());
594    }
595
596    #[test]
597    fn auto_rejects_unknown_shape() {
598        let err = compile_yaml_ld_q42_auto(b"foo: bar\n", 0, 0).unwrap_err();
599        assert!(
600            err.contains("WebizenWorkspace") || err.contains("HypermediaDocument"),
601            "unexpected error: {err}"
602        );
603    }
604
605    #[test]
606    fn hcf_content_without_type_still_compiles() {
607        let yaml = r#"
608"@id": "doc:bare"
609"content":
610  - heading: "Only section"
611    blocks:
612      - text: "hello"
613"#;
614        let (quins, lexicon) = compile_hcf_yaml_ld_to_quins(yaml.as_bytes(), 0, 0).unwrap();
615        assert!(!quins.is_empty());
616        assert!(lexicon.values().any(|s| s == "Only section"));
617        assert!(lexicon.values().any(|s| s == "hello"));
618    }
619}