use std::fs; use std::path::Path; use std::time::SystemTime; use chrono::{DateTime, SecondsFormat, Utc}; pub const MAX_BODY_BYTES: usize = 1_000_000; pub struct Extracted { pub title: String, pub body: String, pub published: String, } pub fn supported(path: &Path) -> bool { match path.extension().and_then(|e| e.to_str()) { Some(ext) => matches!( ext.to_ascii_lowercase().as_str(), "txt" | "md" | "markdown" | "html" | "htm" | "rst" | "log" ), None => false, } } pub fn extract_file(path: &Path) -> Option { if !supported(path) { return None; } let raw = fs::read_to_string(path).ok()?; let ext = path .extension() .and_then(|e| e.to_str()) .unwrap_or("") .to_ascii_lowercase(); let body = if ext == "html" || ext == "htm" { strip_html(&raw) } else { collapse_blank_lines(&raw) }; let body = truncate_chars(&body, MAX_BODY_BYTES); let title = if ext == "html" || ext == "htm" { html_title(&raw).unwrap_or_else(|| fallback_title(path, &body)) } else { markdown_title(&raw).unwrap_or_else(|| fallback_title(path, &body)) }; let published = fs::metadata(path) .and_then(|m| m.modified()) .ok() .map(format_time) .unwrap_or_default(); Some(Extracted { title: truncate_chars(title.trim(), 200), body, published, }) } pub fn summary_of(body: &str) -> String { truncate_chars(body.trim(), 300).replace('\n', " ") } fn fallback_title(path: &Path, body: &str) -> String { if let Some(line) = body.lines().map(str::trim).find(|l| !l.is_empty()) { if line.chars().count() > 3 { return truncate_chars(line, 120); } } path.file_stem() .and_then(|s| s.to_str()) .unwrap_or("untitled") .to_string() } fn markdown_title(raw: &str) -> Option { raw.lines() .map(str::trim) .find(|l| l.starts_with("# ")) .map(|l| l.trim_start_matches('#').trim().to_string()) } fn html_title(raw: &str) -> Option { let lower = raw.to_ascii_lowercase(); let start = lower.find("')? + start + 1; let end = lower[open_end..].find("")? + open_end; let title = decode_entities(raw[open_end..end].trim()); if title.is_empty() { None } else { Some(title) } } fn strip_html(raw: &str) -> String { let mut out = String::with_capacity(raw.len() / 2); let mut chars = raw.chars().peekable(); let mut skipping: Option = None; while let Some(c) = chars.next() { if c == '<' { let mut tag = String::new(); for t in chars.by_ref() { if t == '>' { break; } tag.push(t); } let name = tag .trim_start_matches('/') .trim() .split(|c: char| c.is_whitespace() || c == '/') .next() .unwrap_or("") .to_ascii_lowercase(); if skipping.is_none() && (name == "script" || name == "style") { skipping = Some(name.clone()); } else if skipping.as_deref() == Some(name.as_str()) && tag.trim_start().starts_with('/') { skipping = None; } out.push(' '); continue; } if skipping.is_none() { out.push(c); } } collapse_whitespace(&decode_entities(&out)) } fn collapse_whitespace(s: &str) -> String { let mut out = String::with_capacity(s.len()); let mut last_space = false; for c in s.chars() { if c.is_whitespace() { if !last_space { out.push(' '); } last_space = true; } else { out.push(c); last_space = false; } } out.trim().to_string() } fn collapse_blank_lines(s: &str) -> String { let mut out = String::with_capacity(s.len()); let mut blank_run = 0; for line in s.lines() { if line.trim().is_empty() { blank_run += 1; if blank_run > 1 { continue; } } else { blank_run = 0; } out.push_str(line); out.push('\n'); } out.trim().to_string() } fn decode_entities(s: &str) -> String { s.replace(" ", " ") .replace("&", "&") .replace("<", "<") .replace(">", ">") .replace(""", "\"") .replace("'", "'") .replace("'", "'") } fn truncate_chars(s: &str, max: usize) -> String { if s.len() <= max { return s.to_string(); } let end = s .char_indices() .take_while(|(i, _)| *i < max) .last() .map(|(i, c)| i + c.len_utf8()) .unwrap_or(0); s[..end].to_string() } fn format_time(t: SystemTime) -> String { let dt: DateTime = t.into(); dt.to_rfc3339_opts(SecondsFormat::Secs, true) } #[cfg(test)] mod tests { use super::*; use std::io::Write; #[test] fn html_is_stripped_and_title_extracted() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("page.html"); let mut f = fs::File::create(&path).unwrap(); write!( f, "Rust & Ownership

Hello world

" ) .unwrap(); let extracted = extract_file(&path).unwrap(); assert_eq!(extracted.title, "Rust & Ownership"); assert!(extracted.body.contains("Hello world")); assert!(!extracted.body.contains("alert")); assert!(!extracted.body.contains("color:red")); } #[test] fn unsupported_extensions_are_ignored() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("archive.bin"); fs::write(&path, "binary-ish").unwrap(); assert!(!supported(&path)); assert!(extract_file(&path).is_none()); } #[test] fn oversized_bodies_are_truncated_on_char_boundaries() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("big.txt"); let mut body = "é".repeat(MAX_BODY_BYTES / 2 + 50); body.push_str(" tail"); fs::write(&path, &body).unwrap(); let extracted = extract_file(&path).unwrap(); assert!(extracted.body.len() <= MAX_BODY_BYTES); assert!(extracted.body.chars().all(|c| c == 'é')); } #[test] fn markdown_heading_becomes_title() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("note.md"); fs::write(&path, "# Borrowing\n\nRules of borrowing.\n").unwrap(); let extracted = extract_file(&path).unwrap(); assert_eq!(extracted.title, "Borrowing"); assert!(extracted.body.contains("Rules of borrowing.")); } }