qualia_semantic_library/ingest/
chunk.rs1use serde::{Deserialize, Serialize};
7
8#[derive(Debug, Clone, Serialize, Deserialize)]
10pub struct Chunk {
11 pub idx: u32,
13 #[serde(default)]
15 pub heading_path: Vec<String>,
16 pub char_start: usize,
18 pub text: String,
19}
20
21const TARGET_CHARS: usize = 1400;
23const MAX_CHARS: usize = 2400;
25
26pub fn chunk_text(text: &str) -> Vec<Chunk> {
29 let mut chunks = Vec::new();
30 let mut heading_path: Vec<String> = Vec::new();
31 let mut buf = String::new();
32 let mut buf_start = 0usize;
33 let mut offset = 0usize;
34 let mut idx = 0u32;
35
36 let flush =
37 |buf: &mut String, start: usize, path: &[String], idx: &mut u32, out: &mut Vec<Chunk>| {
38 let t = buf.trim();
39 if !t.is_empty() {
40 out.push(Chunk {
41 idx: *idx,
42 heading_path: path.to_vec(),
43 char_start: start,
44 text: t.to_string(),
45 });
46 *idx += 1;
47 }
48 buf.clear();
49 };
50
51 for block in split_blocks(text) {
52 let block_len = block.len();
53 let trimmed = block.trim();
54 if trimmed.is_empty() {
55 offset += block_len;
56 continue;
57 }
58
59 if let Some(level) = heading_level(trimmed) {
60 flush(&mut buf, buf_start, &heading_path, &mut idx, &mut chunks);
62 buf_start = offset;
63 update_heading_path(&mut heading_path, level, trimmed);
64 offset += block_len;
65 continue;
66 }
67
68 if buf.is_empty() {
69 buf_start = offset;
70 }
71 if !buf.is_empty() {
72 buf.push_str("\n\n");
73 }
74 buf.push_str(trimmed);
75 offset += block_len;
76
77 if buf.len() >= TARGET_CHARS {
78 while buf.len() > MAX_CHARS {
80 let split_at = floor_char_boundary(&buf, MAX_CHARS);
81 let rest = buf.split_off(split_at);
82 flush(&mut buf, buf_start, &heading_path, &mut idx, &mut chunks);
83 buf = rest;
84 buf_start += split_at;
85 }
86 flush(&mut buf, buf_start, &heading_path, &mut idx, &mut chunks);
87 }
88 }
89 flush(&mut buf, buf_start, &heading_path, &mut idx, &mut chunks);
90 chunks
91}
92
93pub fn chunks_to_jsonl(chunks: &[Chunk]) -> String {
95 let mut out = String::new();
96 for c in chunks {
97 if let Ok(line) = serde_json::to_string(c) {
98 out.push_str(&line);
99 out.push('\n');
100 }
101 }
102 out
103}
104
105fn split_blocks(text: &str) -> impl Iterator<Item = &str> {
106 text.split_inclusive("\n\n")
108}
109
110fn heading_level(s: &str) -> Option<u8> {
112 if s.len() > 120 || s.contains('\n') {
113 return None;
114 }
115 if let Some(hashes) = s
117 .strip_suffix(|_: char| false)
118 .and_then(|_| s.find(|c| c != '#'))
119 {
120 if s.starts_with('#') && hashes >= 1 {
121 return Some(hashes.min(6) as u8);
122 }
123 }
124 let first = s.chars().next().unwrap_or(' ');
126 if first.is_ascii_digit() && s.split_whitespace().count() <= 12 {
127 let head = s.split_whitespace().next().unwrap_or("");
128 let depth = head.split('.').filter(|p| !p.is_empty()).count();
129 return Some(depth.clamp(1, 6) as u8);
130 }
131 let alpha = s.chars().filter(|c| c.is_alphabetic()).count();
133 if alpha > 2
134 && s.chars()
135 .filter(|c| c.is_alphabetic())
136 .all(|c| c.is_uppercase())
137 {
138 return Some(1);
139 }
140 None
141}
142
143fn update_heading_path(path: &mut Vec<String>, level: u8, heading: &str) {
144 let level = level as usize;
145 path.truncate(level.saturating_sub(1));
146 while path.len() < level.saturating_sub(1) {
147 path.push(String::new());
148 }
149 path.push(heading.to_string());
150}
151
152fn floor_char_boundary(s: &str, mut i: usize) -> usize {
153 if i >= s.len() {
154 return s.len();
155 }
156 while i > 0 && !s.is_char_boundary(i) {
157 i -= 1;
158 }
159 i
160}
161
162#[cfg(test)]
163mod tests {
164 use super::*;
165
166 #[test]
167 fn chunks_respect_headings_and_size() {
168 let text = "1 Introduction\n\nSome intro text here.\n\n2 Methods\n\nA method paragraph.\n\nAnother method paragraph.";
169 let chunks = chunk_text(text);
170 assert!(!chunks.is_empty());
171 let methods = chunks
173 .iter()
174 .find(|c| c.text.contains("method paragraph"))
175 .unwrap();
176 assert!(methods.heading_path.iter().any(|h| h.contains("Methods")));
177 }
178
179 #[test]
180 fn jsonl_round_trips() {
181 let chunks = chunk_text("hello world\n\nsecond para");
182 let jsonl = chunks_to_jsonl(&chunks);
183 let n = jsonl.lines().count();
184 assert_eq!(n, chunks.len());
185 }
186}