Add Phase 1 frxd implementation with conformance test suite
This commit is contained in:
George Coles
+242
@@ -0,0 +1,242 @@
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
use std::time::SystemTime;
|
||||
|
||||
use chrono::{DateTime, SecondsFormat, Utc};
|
||||
|
||||
pub const MAX_BODY_BYTES: usize = 1_000_000;
|
||||
|
||||
pub struct Extracted {
|
||||
pub title: String,
|
||||
pub body: String,
|
||||
pub published: String,
|
||||
}
|
||||
|
||||
pub fn supported(path: &Path) -> bool {
|
||||
match path.extension().and_then(|e| e.to_str()) {
|
||||
Some(ext) => matches!(
|
||||
ext.to_ascii_lowercase().as_str(),
|
||||
"txt" | "md" | "markdown" | "html" | "htm" | "rst" | "log"
|
||||
),
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn extract_file(path: &Path) -> Option<Extracted> {
|
||||
if !supported(path) {
|
||||
return None;
|
||||
}
|
||||
let raw = fs::read_to_string(path).ok()?;
|
||||
let ext = path
|
||||
.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.unwrap_or("")
|
||||
.to_ascii_lowercase();
|
||||
let body = if ext == "html" || ext == "htm" {
|
||||
strip_html(&raw)
|
||||
} else {
|
||||
collapse_blank_lines(&raw)
|
||||
};
|
||||
let body = truncate_chars(&body, MAX_BODY_BYTES);
|
||||
let title = if ext == "html" || ext == "htm" {
|
||||
html_title(&raw).unwrap_or_else(|| fallback_title(path, &body))
|
||||
} else {
|
||||
markdown_title(&raw).unwrap_or_else(|| fallback_title(path, &body))
|
||||
};
|
||||
let published = fs::metadata(path)
|
||||
.and_then(|m| m.modified())
|
||||
.ok()
|
||||
.map(format_time)
|
||||
.unwrap_or_default();
|
||||
Some(Extracted {
|
||||
title: truncate_chars(title.trim(), 200),
|
||||
body,
|
||||
published,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn summary_of(body: &str) -> String {
|
||||
truncate_chars(body.trim(), 300).replace('\n', " ")
|
||||
}
|
||||
|
||||
fn fallback_title(path: &Path, body: &str) -> String {
|
||||
if let Some(line) = body.lines().map(str::trim).find(|l| !l.is_empty()) {
|
||||
if line.chars().count() > 3 {
|
||||
return truncate_chars(line, 120);
|
||||
}
|
||||
}
|
||||
path.file_stem()
|
||||
.and_then(|s| s.to_str())
|
||||
.unwrap_or("untitled")
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn markdown_title(raw: &str) -> Option<String> {
|
||||
raw.lines()
|
||||
.map(str::trim)
|
||||
.find(|l| l.starts_with("# "))
|
||||
.map(|l| l.trim_start_matches('#').trim().to_string())
|
||||
}
|
||||
|
||||
fn html_title(raw: &str) -> Option<String> {
|
||||
let lower = raw.to_ascii_lowercase();
|
||||
let start = lower.find("<title")?;
|
||||
let open_end = lower[start..].find('>')? + start + 1;
|
||||
let end = lower[open_end..].find("</title>")? + open_end;
|
||||
let title = decode_entities(raw[open_end..end].trim());
|
||||
if title.is_empty() { None } else { Some(title) }
|
||||
}
|
||||
|
||||
fn strip_html(raw: &str) -> String {
|
||||
let mut out = String::with_capacity(raw.len() / 2);
|
||||
let mut chars = raw.chars().peekable();
|
||||
let mut skipping: Option<String> = None;
|
||||
while let Some(c) = chars.next() {
|
||||
if c == '<' {
|
||||
let mut tag = String::new();
|
||||
for t in chars.by_ref() {
|
||||
if t == '>' {
|
||||
break;
|
||||
}
|
||||
tag.push(t);
|
||||
}
|
||||
let name = tag
|
||||
.trim_start_matches('/')
|
||||
.trim()
|
||||
.split(|c: char| c.is_whitespace() || c == '/')
|
||||
.next()
|
||||
.unwrap_or("")
|
||||
.to_ascii_lowercase();
|
||||
if skipping.is_none() && (name == "script" || name == "style") {
|
||||
skipping = Some(name.clone());
|
||||
} else if skipping.as_deref() == Some(name.as_str())
|
||||
&& tag.trim_start().starts_with('/')
|
||||
{
|
||||
skipping = None;
|
||||
}
|
||||
out.push(' ');
|
||||
continue;
|
||||
}
|
||||
if skipping.is_none() {
|
||||
out.push(c);
|
||||
}
|
||||
}
|
||||
collapse_whitespace(&decode_entities(&out))
|
||||
}
|
||||
|
||||
fn collapse_whitespace(s: &str) -> String {
|
||||
let mut out = String::with_capacity(s.len());
|
||||
let mut last_space = false;
|
||||
for c in s.chars() {
|
||||
if c.is_whitespace() {
|
||||
if !last_space {
|
||||
out.push(' ');
|
||||
}
|
||||
last_space = true;
|
||||
} else {
|
||||
out.push(c);
|
||||
last_space = false;
|
||||
}
|
||||
}
|
||||
out.trim().to_string()
|
||||
}
|
||||
|
||||
fn collapse_blank_lines(s: &str) -> String {
|
||||
let mut out = String::with_capacity(s.len());
|
||||
let mut blank_run = 0;
|
||||
for line in s.lines() {
|
||||
if line.trim().is_empty() {
|
||||
blank_run += 1;
|
||||
if blank_run > 1 {
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
blank_run = 0;
|
||||
}
|
||||
out.push_str(line);
|
||||
out.push('\n');
|
||||
}
|
||||
out.trim().to_string()
|
||||
}
|
||||
|
||||
fn decode_entities(s: &str) -> String {
|
||||
s.replace(" ", " ")
|
||||
.replace("&", "&")
|
||||
.replace("<", "<")
|
||||
.replace(">", ">")
|
||||
.replace(""", "\"")
|
||||
.replace("'", "'")
|
||||
.replace("'", "'")
|
||||
}
|
||||
|
||||
fn truncate_chars(s: &str, max: usize) -> String {
|
||||
if s.len() <= max {
|
||||
return s.to_string();
|
||||
}
|
||||
let end = s
|
||||
.char_indices()
|
||||
.take_while(|(i, _)| *i < max)
|
||||
.last()
|
||||
.map(|(i, c)| i + c.len_utf8())
|
||||
.unwrap_or(0);
|
||||
s[..end].to_string()
|
||||
}
|
||||
|
||||
fn format_time(t: SystemTime) -> String {
|
||||
let dt: DateTime<Utc> = t.into();
|
||||
dt.to_rfc3339_opts(SecondsFormat::Secs, true)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Write;
|
||||
|
||||
#[test]
|
||||
fn html_is_stripped_and_title_extracted() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("page.html");
|
||||
let mut f = fs::File::create(&path).unwrap();
|
||||
write!(
|
||||
f,
|
||||
"<html><head><title>Rust & Ownership</title><style>p{{color:red}}</style></head><body><p>Hello <b>world</b></p><script>alert(1)</script></body></html>"
|
||||
)
|
||||
.unwrap();
|
||||
let extracted = extract_file(&path).unwrap();
|
||||
assert_eq!(extracted.title, "Rust & Ownership");
|
||||
assert!(extracted.body.contains("Hello world"));
|
||||
assert!(!extracted.body.contains("alert"));
|
||||
assert!(!extracted.body.contains("color:red"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unsupported_extensions_are_ignored() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("archive.bin");
|
||||
fs::write(&path, "binary-ish").unwrap();
|
||||
assert!(!supported(&path));
|
||||
assert!(extract_file(&path).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oversized_bodies_are_truncated_on_char_boundaries() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("big.txt");
|
||||
let mut body = "é".repeat(MAX_BODY_BYTES / 2 + 50);
|
||||
body.push_str(" tail");
|
||||
fs::write(&path, &body).unwrap();
|
||||
let extracted = extract_file(&path).unwrap();
|
||||
assert!(extracted.body.len() <= MAX_BODY_BYTES);
|
||||
assert!(extracted.body.chars().all(|c| c == 'é'));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn markdown_heading_becomes_title() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("note.md");
|
||||
fs::write(&path, "# Borrowing\n\nRules of borrowing.\n").unwrap();
|
||||
let extracted = extract_file(&path).unwrap();
|
||||
assert_eq!(extracted.title, "Borrowing");
|
||||
assert!(extracted.body.contains("Rules of borrowing."));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user