Skip to main content

qualia_semantic_library/ingest/
chunk.rs

1//! Stage 3 — chunk: split the extracted text into structural chunks suitable
2//! for embedding and for targeted LLM extraction. Chunks follow paragraph and
3//! heading boundaries (not arbitrary character windows) and carry the running
4//! heading path so a retrieved chunk keeps its context.
5
6use serde::{Deserialize, Serialize};
7
8/// One retrievable unit of a document.
9#[derive(Debug, Clone, Serialize, Deserialize)]
10pub struct Chunk {
11    /// Zero-based index within the document.
12    pub idx: u32,
13    /// Heading path leading to this chunk (e.g. ["3 Methods", "3.2 Defeat"]).
14    #[serde(default)]
15    pub heading_path: Vec<String>,
16    /// Character offset of the chunk start within the source text.
17    pub char_start: usize,
18    pub text: String,
19}
20
21/// Target chunk size in characters (~roughly 250–400 tokens).
22const TARGET_CHARS: usize = 1400;
23/// Hard ceiling so a single huge paragraph still gets split.
24const MAX_CHARS: usize = 2400;
25
26/// Split text into structural chunks. Heading-looking lines start a new chunk
27/// and update the heading path; paragraphs accumulate up to the target size.
28pub fn chunk_text(text: &str) -> Vec<Chunk> {
29    let mut chunks = Vec::new();
30    let mut heading_path: Vec<String> = Vec::new();
31    let mut buf = String::new();
32    let mut buf_start = 0usize;
33    let mut offset = 0usize;
34    let mut idx = 0u32;
35
36    let flush =
37        |buf: &mut String, start: usize, path: &[String], idx: &mut u32, out: &mut Vec<Chunk>| {
38            let t = buf.trim();
39            if !t.is_empty() {
40                out.push(Chunk {
41                    idx: *idx,
42                    heading_path: path.to_vec(),
43                    char_start: start,
44                    text: t.to_string(),
45                });
46                *idx += 1;
47            }
48            buf.clear();
49        };
50
51    for block in split_blocks(text) {
52        let block_len = block.len();
53        let trimmed = block.trim();
54        if trimmed.is_empty() {
55            offset += block_len;
56            continue;
57        }
58
59        if let Some(level) = heading_level(trimmed) {
60            // boundary: finish current chunk, then update heading path
61            flush(&mut buf, buf_start, &heading_path, &mut idx, &mut chunks);
62            buf_start = offset;
63            update_heading_path(&mut heading_path, level, trimmed);
64            offset += block_len;
65            continue;
66        }
67
68        if buf.is_empty() {
69            buf_start = offset;
70        }
71        if !buf.is_empty() {
72            buf.push_str("\n\n");
73        }
74        buf.push_str(trimmed);
75        offset += block_len;
76
77        if buf.len() >= TARGET_CHARS {
78            // If we blew past the ceiling on one block, hard-split it.
79            while buf.len() > MAX_CHARS {
80                let split_at = floor_char_boundary(&buf, MAX_CHARS);
81                let rest = buf.split_off(split_at);
82                flush(&mut buf, buf_start, &heading_path, &mut idx, &mut chunks);
83                buf = rest;
84                buf_start += split_at;
85            }
86            flush(&mut buf, buf_start, &heading_path, &mut idx, &mut chunks);
87        }
88    }
89    flush(&mut buf, buf_start, &heading_path, &mut idx, &mut chunks);
90    chunks
91}
92
93/// Serialize chunks to JSON-Lines (one object per line).
94pub fn chunks_to_jsonl(chunks: &[Chunk]) -> String {
95    let mut out = String::new();
96    for c in chunks {
97        if let Ok(line) = serde_json::to_string(c) {
98            out.push_str(&line);
99            out.push('\n');
100        }
101    }
102    out
103}
104
105fn split_blocks(text: &str) -> impl Iterator<Item = &str> {
106    // Keep separators' length accounted for by re-deriving from inclusive split.
107    text.split_inclusive("\n\n")
108}
109
110/// Returns a heading "level" (1 = top) for heading-looking lines, else None.
111fn heading_level(s: &str) -> Option<u8> {
112    if s.len() > 120 || s.contains('\n') {
113        return None;
114    }
115    // Markdown ATX
116    if let Some(hashes) = s
117        .strip_suffix(|_: char| false)
118        .and_then(|_| s.find(|c| c != '#'))
119    {
120        if s.starts_with('#') && hashes >= 1 {
121            return Some(hashes.min(6) as u8);
122        }
123    }
124    // Numbered sections: depth = number of dotted components.
125    let first = s.chars().next().unwrap_or(' ');
126    if first.is_ascii_digit() && s.split_whitespace().count() <= 12 {
127        let head = s.split_whitespace().next().unwrap_or("");
128        let depth = head.split('.').filter(|p| !p.is_empty()).count();
129        return Some(depth.clamp(1, 6) as u8);
130    }
131    // ALL CAPS short line = top-level heading.
132    let alpha = s.chars().filter(|c| c.is_alphabetic()).count();
133    if alpha > 2
134        && s.chars()
135            .filter(|c| c.is_alphabetic())
136            .all(|c| c.is_uppercase())
137    {
138        return Some(1);
139    }
140    None
141}
142
143fn update_heading_path(path: &mut Vec<String>, level: u8, heading: &str) {
144    let level = level as usize;
145    path.truncate(level.saturating_sub(1));
146    while path.len() < level.saturating_sub(1) {
147        path.push(String::new());
148    }
149    path.push(heading.to_string());
150}
151
152fn floor_char_boundary(s: &str, mut i: usize) -> usize {
153    if i >= s.len() {
154        return s.len();
155    }
156    while i > 0 && !s.is_char_boundary(i) {
157        i -= 1;
158    }
159    i
160}
161
162#[cfg(test)]
163mod tests {
164    use super::*;
165
166    #[test]
167    fn chunks_respect_headings_and_size() {
168        let text = "1 Introduction\n\nSome intro text here.\n\n2 Methods\n\nA method paragraph.\n\nAnother method paragraph.";
169        let chunks = chunk_text(text);
170        assert!(!chunks.is_empty());
171        // the methods chunk should carry the "2 Methods" heading in its path
172        let methods = chunks
173            .iter()
174            .find(|c| c.text.contains("method paragraph"))
175            .unwrap();
176        assert!(methods.heading_path.iter().any(|h| h.contains("Methods")));
177    }
178
179    #[test]
180    fn jsonl_round_trips() {
181        let chunks = chunk_text("hello world\n\nsecond para");
182        let jsonl = chunks_to_jsonl(&chunks);
183        let n = jsonl.lines().count();
184        assert_eq!(n, chunks.len());
185    }
186}