diff --git a/.cloudflare/docs-proxy/src/worker.js b/.cloudflare/docs-proxy/src/worker.js index 08b0265fafb..99fca0c7116 100644 --- a/.cloudflare/docs-proxy/src/worker.js +++ b/.cloudflare/docs-proxy/src/worker.js @@ -1,6 +1,11 @@ export default { async fetch(request, _env, _ctx) { const url = new URL(request.url); + const acceptHeader = request.headers.get("Accept") || ""; + const wantsMarkdown = acceptHeader + .split(",") + .map((mediaType) => mediaType.split(";")[0].trim().toLowerCase()) + .includes("text/markdown"); if (url.pathname === "/docs/nightly") { url.hostname = "docs-nightly.pages.dev"; @@ -18,6 +23,14 @@ export default { url.hostname = "docs-anw.pages.dev"; } + if (url.pathname === "/docs.md") { + url.pathname = "/docs/getting-started.md"; + } + + if (wantsMarkdown) { + url.pathname = markdownPathFor(url.pathname); + } + let res = await fetch(url, request); if (res.status === 404) { @@ -27,3 +40,31 @@ export default { return res; }, }; + +function markdownPathFor(pathname) { + if (pathname === "/docs" || pathname === "/docs/") { + return "/docs/getting-started.md"; + } + + if (pathname.endsWith("/index.md")) { + return pathname.replace(/\/index\.md$/, "/getting-started.md"); + } + + if (pathname.endsWith(".md")) { + return pathname; + } + + if (pathname.endsWith(".html")) { + return pathname.replace(/\.html$/, ".md"); + } + + if (pathname.split("/").pop().includes(".")) { + return pathname; + } + + if (pathname.endsWith("/")) { + return `${pathname}getting-started.md`; + } + + return `${pathname}.md`; +} diff --git a/crates/docs_preprocessor/Cargo.toml b/crates/docs_preprocessor/Cargo.toml index 87e5110ff52..a422d2811db 100644 --- a/crates/docs_preprocessor/Cargo.toml +++ b/crates/docs_preprocessor/Cargo.toml @@ -27,4 +27,4 @@ workspace = true [[bin]] name = "docs_preprocessor" -path = "src/main.rs" \ No newline at end of file +path = "src/main.rs" diff --git a/crates/docs_preprocessor/src/ai_discovery.rs b/crates/docs_preprocessor/src/ai_discovery.rs new file mode 100644 index 00000000000..4f3f9195827 --- /dev/null +++ b/crates/docs_preprocessor/src/ai_discovery.rs @@ -0,0 +1,549 @@ +use anyhow::{Context, Result}; +use mdbook::BookItem; +use mdbook::book::Book; +use regex::Regex; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::OnceLock; + +use crate::FRONT_MATTER_COMMENT; + +#[derive(Debug)] +pub(crate) struct DocsPage { + section: String, + title: String, + description: Option, + pub(crate) source_path: PathBuf, + content: String, +} + +pub(crate) fn write_ai_discovery_artifacts( + pages: &[DocsPage], + destination: &Path, + site_url: &str, +) -> Result<()> { + copy_markdown_sources(destination, site_url, pages)?; + write_llms_txt(destination, site_url, pages)?; + write_sitemap_xml(destination, site_url, pages)?; + Ok(()) +} + +pub(crate) fn docs_pages(book: &Book) -> Result> { + let mut pages = Vec::new(); + let mut section = "Docs".to_string(); + for item in book.iter() { + let BookItem::Chapter(chapter) = item else { + if let BookItem::PartTitle(part_title) = item { + section.clone_from(part_title); + } + continue; + }; + let Some(source_path) = chapter.source_path.as_ref() else { + continue; + }; + if source_path == Path::new("SUMMARY.md") { + continue; + } + pages.push(DocsPage { + section: section.clone(), + title: chapter.name.clone(), + description: docs_page_description(&chapter.content), + source_path: source_path.clone(), + content: chapter.content.clone(), + }); + } + Ok(pages) +} + +fn copy_markdown_sources(destination: &Path, site_url: &str, pages: &[DocsPage]) -> Result<()> { + for page in pages { + let destination = destination.join(&page.source_path); + if let Some(parent) = destination.parent() { + std::fs::create_dir_all(parent).with_context(|| { + format!("failed to create markdown destination {}", parent.display()) + })?; + } + let contents = rewrite_docs_links(&markdown_source_contents(&page.content), site_url); + std::fs::write( + &destination, + add_llms_markdown_directive(&contents, site_url), + ) + .with_context(|| { + format!( + "failed to write markdown page {} to {}", + page.source_path.display(), + destination.display() + ) + })?; + } + let getting_started = destination.join("getting-started.md"); + if getting_started.exists() { + std::fs::copy(&getting_started, destination.join("index.md")) + .context("failed to write index.md markdown alias")?; + } + Ok(()) +} + +fn markdown_source_contents(contents: &str) -> String { + front_matter_comment_regex() + .replace(contents, "") + .trim_start() + .to_string() +} + +fn docs_page_description(contents: &str) -> Option { + docs_page_metadata(contents).and_then(|metadata| { + metadata + .get("description") + .map(|description| { + description + .trim() + .trim_matches('"') + .split_whitespace() + .collect::>() + .join(" ") + }) + .filter(|description| !description.is_empty()) + }) +} + +fn docs_page_metadata(contents: &str) -> Option> { + let captures = front_matter_comment_regex().captures(contents)?; + serde_json::from_str(&captures[1]).ok() +} + +fn front_matter_comment_regex() -> &'static Regex { + static FRONT_MATTER_COMMENT_REGEX: OnceLock = OnceLock::new(); + FRONT_MATTER_COMMENT_REGEX + .get_or_init(|| Regex::new(&FRONT_MATTER_COMMENT.replace("{}", "([^\\n]*)")).unwrap()) +} + +fn write_llms_txt(destination: &Path, site_url: &str, pages: &[DocsPage]) -> Result<()> { + let mut contents = String::new(); + contents.push_str("# Zed Docs\n\n"); + contents.push_str( + "> Official Zed documentation index with links to Markdown versions of each docs page.\n\n", + ); + contents.push_str( + "Use these links for concise Markdown copies of Zed documentation pages. Each linked page mirrors the corresponding `/docs/*.html` page without site navigation or styling.\n\n", + ); + let mut current_section = None; + for page in pages { + if current_section != Some(page.section.as_str()) { + if current_section.is_some() { + contents.push('\n'); + } + contents.push_str("## "); + contents.push_str(&markdown_text(&page.section)); + contents.push_str("\n\n"); + current_section = Some(page.section.as_str()); + } + contents.push_str("- ["); + contents.push_str(&markdown_text(&page.title)); + contents.push_str("]("); + contents.push_str(&absolute_docs_url(site_url, &page.source_path)); + contents.push(')'); + if let Some(description) = &page.description { + contents.push_str(": "); + contents.push_str(&markdown_text(description)); + } + contents.push('\n'); + } + std::fs::write(destination.join("llms.txt"), contents).context("failed to write llms.txt")?; + Ok(()) +} + +fn markdown_text(text: &str) -> String { + text.replace('\\', "\\\\") + .replace('[', "\\[") + .replace(']', "\\]") +} + +fn write_sitemap_xml(destination: &Path, site_url: &str, pages: &[DocsPage]) -> Result<()> { + let mut contents = String::new(); + contents.push_str("\n"); + contents.push_str("\n"); + for page in pages { + contents.push_str(" "); + contents.push_str(&xml_escape(&absolute_docs_url( + site_url, + &page.source_path.with_extension("html"), + ))); + contents.push_str(""); + contents.push_str("\n"); + } + contents.push_str("\n"); + std::fs::write(destination.join("sitemap.xml"), contents) + .context("failed to write sitemap.xml")?; + Ok(()) +} + +pub(crate) fn write_pages_redirects( + destination: &Path, + redirects: &[(String, String)], + site_url: &str, +) -> Result<()> { + let Some(deploy_root) = destination.parent() else { + return Ok(()); + }; + let mut contents = String::new(); + for (source, destination) in redirects { + write_redirect_line( + &mut contents, + &docs_path("/docs/", source), + &redirect_destination(site_url, destination), + ); + if let Some(extensionless_source) = strip_html_suffix(source) { + write_redirect_line( + &mut contents, + &docs_path("/docs/", &extensionless_source), + &redirect_destination( + site_url, + &strip_html_suffix(destination).unwrap_or_else(|| destination.to_string()), + ), + ); + } + if let Some(markdown_source) = html_path_to_markdown(source) { + if let Some(markdown_destination) = html_path_to_markdown(destination) { + write_redirect_line( + &mut contents, + &docs_path("/docs/", &markdown_source), + &redirect_destination(site_url, &markdown_destination), + ); + } + } + } + std::fs::write(deploy_root.join("_redirects"), contents) + .context("failed to write Cloudflare Pages _redirects")?; + Ok(()) +} + +pub(crate) fn write_markdown_redirect_aliases( + destination: &Path, + redirects: &[(String, String)], + site_url: &str, +) -> Result<()> { + for (source, redirect_destination_path) in redirects { + let Some(source_markdown) = html_path_to_markdown(source) else { + continue; + }; + let Some(destination_markdown) = html_path_to_markdown(redirect_destination_path) else { + continue; + }; + let source_markdown = destination.join(source_markdown.trim_start_matches('/')); + let destination_markdown = + destination.join(destination_markdown.trim_start_matches("/docs/")); + if !destination_markdown.exists() { + continue; + } + if let Some(parent) = source_markdown.parent() { + std::fs::create_dir_all(parent).with_context(|| { + format!( + "failed to create markdown alias directory {}", + parent.display() + ) + })?; + } + let contents = format!( + "# Moved\n\n> For the complete documentation index and Markdown links, see [llms.txt]({}).\n\nThis page moved to [the current docs page]({}).\n", + docs_url(site_url, Path::new("llms.txt")), + html_path_to_markdown(redirect_destination_path) + .map(|path| redirect_destination(site_url, &path)) + .unwrap_or_else(|| redirect_destination(site_url, redirect_destination_path)) + ); + std::fs::write(&source_markdown, contents).with_context(|| { + format!( + "failed to write markdown redirect alias from {} to {}", + redirect_destination_path, + source_markdown.display() + ) + })?; + } + Ok(()) +} + +fn write_redirect_line(contents: &mut String, source: &str, destination: &str) { + contents.push_str(source); + contents.push(' '); + contents.push_str(destination); + contents.push_str(" 301\n"); +} + +fn docs_path(site_url: &str, path: &str) -> String { + docs_url(site_url, Path::new(path.trim_start_matches('/'))) +} + +fn redirect_destination(site_url: &str, destination: &str) -> String { + if let Some(path) = destination.strip_prefix("/docs/") { + docs_url(site_url, Path::new(path)) + } else if destination == "/docs" { + docs_url(site_url, Path::new("")) + } else { + destination.to_string() + } +} + +fn strip_html_suffix(path: &str) -> Option { + let (path, fragment) = split_fragment(path); + let path = path.strip_suffix(".html")?; + Some(format!("{path}{fragment}")) +} + +fn html_path_to_markdown(path: &str) -> Option { + let (path, fragment) = split_fragment(path); + if !path.starts_with("/docs/") && path != "/docs" && !path.ends_with(".html") { + return None; + } + let markdown_path = path.strip_suffix(".html").unwrap_or(path); + Some(format!("{markdown_path}.md{fragment}")) +} + +fn split_fragment(path: &str) -> (&str, &str) { + match path.find('#') { + Some(index) => (&path[..index], &path[index..]), + None => (path, ""), + } +} + +pub(crate) fn rewrite_docs_links(contents: &str, site_url: &str) -> String { + const STABLE_DOCS_PREFIX: &str = "https://zed.dev/docs/"; + let channel_docs_prefix = absolute_docs_url(site_url, Path::new("")); + if channel_docs_prefix == STABLE_DOCS_PREFIX { + return contents.to_string(); + } + + let mut output = String::with_capacity(contents.len()); + let mut remaining = contents; + while let Some(index) = remaining.find(STABLE_DOCS_PREFIX) { + output.push_str(&remaining[..index]); + let after_prefix = &remaining[index + STABLE_DOCS_PREFIX.len()..]; + if after_prefix.starts_with("preview/") || after_prefix.starts_with("nightly/") { + output.push_str(STABLE_DOCS_PREFIX); + } else { + output.push_str(&channel_docs_prefix); + } + remaining = after_prefix; + } + output.push_str(remaining); + output +} + +pub(crate) fn add_markdown_alternate_link( + contents: &str, + html_file: &Path, + root_dir: &Path, + site_url: &str, +) -> String { + let Ok(relative_path) = html_file.strip_prefix(root_dir) else { + return contents.to_string(); + }; + let markdown_path = relative_path.with_extension("md"); + if !root_dir.join(&markdown_path).exists() { + return contents.to_string(); + } + let markdown_url = docs_url(site_url, &markdown_path); + let link = format!( + " \n", + markdown_url + ); + contents.replacen("", &(link + " "), 1) +} + +fn add_llms_markdown_directive(contents: &str, site_url: &str) -> String { + let directive = format!( + "> For the complete documentation index and Markdown links, see [llms.txt]({}).\n\n", + docs_url(site_url, Path::new("llms.txt")), + ); + if let Some(rest) = contents.strip_prefix("---\n") { + if let Some(frontmatter_end) = rest.find("\n---\n") { + let split_at = "---\n".len() + frontmatter_end + "\n---\n".len(); + let mut output = String::with_capacity(contents.len() + directive.len()); + output.push_str(&contents[..split_at]); + output.push('\n'); + output.push_str(&directive); + output.push_str(&contents[split_at..]); + return output; + } + } + + let mut output = String::with_capacity(contents.len() + directive.len()); + output.push_str(&directive); + output.push_str(contents); + output +} + +fn docs_url(site_url: &str, path: &Path) -> String { + let mut url = site_url.to_string(); + if !url.ends_with('/') { + url.push('/'); + } + url.push_str(&path.to_string_lossy().replace('\\', "/")); + url +} + +fn absolute_docs_url(site_url: &str, path: &Path) -> String { + let url = docs_url(site_url, path); + if url.starts_with("http://") || url.starts_with("https://") { + url + } else { + format!("https://zed.dev{}", url) + } +} + +fn xml_escape(value: &str) -> String { + value + .replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('"', """) + .replace('\'', "'") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_add_llms_markdown_directive_inserts_after_frontmatter() { + let contents = "---\ntitle: Example\n---\n# Example\n"; + let output = add_llms_markdown_directive(contents, "/docs/"); + + assert!(output.starts_with("---\ntitle: Example\n---\n\n")); + assert!(output.contains( + "> For the complete documentation index and Markdown links, see [llms.txt](/docs/llms.txt)." + )); + } + + #[test] + fn test_redirect_destination_uses_channel_site_url_for_docs_paths() { + assert_eq!( + redirect_destination("/docs/preview/", "/docs/ai/overview.html"), + "/docs/preview/ai/overview.html" + ); + assert_eq!( + redirect_destination("/docs/preview/", "/community-links"), + "/community-links" + ); + } + + #[test] + fn test_rewrite_docs_links_uses_channel_site_url() { + assert_eq!( + rewrite_docs_links( + "See [Code Actions](https://zed.dev/docs/configuring-languages#code-actions) and [Preview](https://zed.dev/docs/preview/ai/overview.html).", + "/docs/preview/" + ), + "See [Code Actions](https://zed.dev/docs/preview/configuring-languages#code-actions) and [Preview](https://zed.dev/docs/preview/ai/overview.html)." + ); + } + + #[test] + fn test_docs_path_uses_channel_site_url() { + assert_eq!( + docs_path("/docs/preview/", "/assistant.md"), + "/docs/preview/assistant.md" + ); + } + + #[test] + fn test_write_pages_redirects_keeps_sources_on_internal_pages_path() -> Result<()> { + let deploy_root = std::env::temp_dir().join(format!( + "docs_preprocessor_pages_redirects_test_{}_{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH)? + .as_nanos() + )); + let destination = deploy_root.join("docs"); + std::fs::create_dir_all(&destination)?; + let redirects = vec![ + ( + "/assistant.html".to_string(), + "/docs/ai/overview.html".to_string(), + ), + ( + "/community/feedback.html".to_string(), + "/community-links".to_string(), + ), + ]; + + write_pages_redirects(&destination, &redirects, "/docs/preview/")?; + + assert_eq!( + std::fs::read_to_string(deploy_root.join("_redirects"))?, + "/docs/assistant.html /docs/preview/ai/overview.html 301\n\ +/docs/assistant /docs/preview/ai/overview 301\n\ +/docs/assistant.md /docs/preview/ai/overview.md 301\n\ +/docs/community/feedback.html /community-links 301\n\ +/docs/community/feedback /community-links 301\n" + ); + std::fs::remove_dir_all(&deploy_root)?; + Ok(()) + } + + #[test] + fn test_write_ai_discovery_artifacts_generates_agent_facing_metadata() -> Result<()> { + let destination = std::env::temp_dir().join(format!( + "docs_preprocessor_ai_discovery_test_{}_{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH)? + .as_nanos() + )); + std::fs::create_dir_all(&destination)?; + + let pages = vec![ + DocsPage { + section: "Docs".to_string(), + title: "Getting Started".to_string(), + description: Some("Start using Zed.".to_string()), + source_path: PathBuf::from("getting-started.md"), + content: format!( + "{}\n# Getting Started\n", + FRONT_MATTER_COMMENT.replace("{}", r#"{"description":"Start using Zed."}"#) + ), + }, + DocsPage { + section: "AI".to_string(), + title: "MCP".to_string(), + description: Some("Connect model context servers.".to_string()), + source_path: PathBuf::from("ai/mcp.md"), + content: format!( + "{}\n# MCP\n", + FRONT_MATTER_COMMENT + .replace("{}", r#"{"description":"Connect model context servers."}"#) + ), + }, + ]; + + write_ai_discovery_artifacts(&pages, &destination, "/docs/")?; + + let llms_txt = std::fs::read_to_string(destination.join("llms.txt"))?; + assert!(llms_txt.contains("## Docs")); + assert!(llms_txt.contains( + "- [Getting Started](https://zed.dev/docs/getting-started.md): Start using Zed." + )); + assert!(llms_txt.contains("## AI")); + assert!( + llms_txt.contains( + "- [MCP](https://zed.dev/docs/ai/mcp.md): Connect model context servers." + ) + ); + + let sitemap_xml = std::fs::read_to_string(destination.join("sitemap.xml"))?; + assert!(sitemap_xml.contains("https://zed.dev/docs/getting-started.html")); + assert!(sitemap_xml.contains("https://zed.dev/docs/ai/mcp.html")); + + let mcp_markdown = std::fs::read_to_string(destination.join("ai/mcp.md"))?; + assert!(mcp_markdown.starts_with( + "> For the complete documentation index and Markdown links, see [llms.txt](/docs/llms.txt).\n\n# MCP" + )); + assert!(!mcp_markdown.contains("ZED_META")); + + let index_markdown = std::fs::read_to_string(destination.join("index.md"))?; + assert!(index_markdown.contains("# Getting Started")); + + std::fs::remove_dir_all(&destination)?; + Ok(()) + } +} diff --git a/crates/docs_preprocessor/src/main.rs b/crates/docs_preprocessor/src/main.rs index df3b556d6a4..4b0c2ca04bc 100644 --- a/crates/docs_preprocessor/src/main.rs +++ b/crates/docs_preprocessor/src/main.rs @@ -1,4 +1,10 @@ use anyhow::{Context, Result}; +mod ai_discovery; + +use ai_discovery::{ + add_markdown_alternate_link, docs_pages, rewrite_docs_links, write_ai_discovery_artifacts, + write_markdown_redirect_aliases, write_pages_redirects, +}; use mdbook::BookItem; use mdbook::book::{Book, Chapter}; use mdbook::preprocess::CmdPreprocessor; @@ -188,7 +194,7 @@ fn handle_preprocessing() -> Result<()> { template_big_table_of_actions(&mut book); template_and_validate_keybindings(&mut book, &mut errors); template_and_validate_actions(&mut book, &mut errors); - template_and_validate_json_snippets(&mut book, &mut errors); + template_and_validate_json_snippets(&mut book, &mut errors)?; if !errors.is_empty() { const ANSI_RED: &str = "\x1b[31m"; @@ -252,7 +258,7 @@ fn format_binding(binding: String) -> String { } fn template_and_validate_keybindings(book: &mut Book, errors: &mut HashSet) { - let regex = Regex::new(r"\{#kb(?::(\w+))?\s+(.*?)\}").unwrap(); + let regex = Regex::new(r"(?s)\{#kb(?::(\w+))?\s+(.*?)\}").unwrap(); for_each_chapter_mut(book, |chapter| { chapter.content = regex @@ -300,7 +306,7 @@ fn template_and_validate_keybindings(book: &mut Book, errors: &mut HashSet) { - let regex = Regex::new(r"\{#action (.*?)\}").unwrap(); + let regex = Regex::new(r"(?s)\{#action\s+(.*?)\}").unwrap(); for_each_chapter_mut(book, |chapter| { chapter.content = regex @@ -379,7 +385,10 @@ fn find_binding_with_overlay( .or_else(|| find_binding(os, action)) } -fn template_and_validate_json_snippets(book: &mut Book, errors: &mut HashSet) { +fn template_and_validate_json_snippets( + book: &mut Book, + errors: &mut HashSet, +) -> Result<()> { let params = SettingsJsonSchemaParams { language_names: &[], font_names: &[], @@ -393,7 +402,7 @@ fn template_and_validate_json_snippets(book: &mut Book, errors: &mut HashSet
 Result<()> {
         .as_table_mut()
         .expect("output is table");
     let zed_html = output.remove("zed-html").expect("zed-html output defined");
+    let redirects = zed_html
+        .get("redirect")
+        .and_then(|redirects| redirects.as_table())
+        .map(|redirects| {
+            redirects
+                .iter()
+                .filter_map(|(source, destination)| {
+                    destination
+                        .as_str()
+                        .map(|destination| (source.clone(), destination.to_string()))
+                })
+                .collect::>()
+        });
     let default_description = zed_html
         .get("default-description")
         .expect("Default description not found")
@@ -694,6 +718,17 @@ fn handle_postprocessing() -> Result<()> {
     let amplitude_key = std::env::var("DOCS_AMPLITUDE_API_KEY").unwrap_or_default();
     let consent_io_instance = std::env::var("DOCS_CONSENT_IO_INSTANCE").unwrap_or_default();
     let docs_channel = std::env::var("DOCS_CHANNEL").unwrap_or_else(|_| "stable".to_string());
+    let site_url = std::env::var("MDBOOK_BOOK__SITE_URL")
+        .ok()
+        .filter(|site_url| !site_url.trim().is_empty())
+        .unwrap_or_else(|| {
+            match docs_channel.as_str() {
+                "nightly" => "/docs/nightly/",
+                "preview" => "/docs/preview/",
+                _ => "/docs/",
+            }
+            .to_string()
+        });
     let noindex = if docs_channel == "nightly" || docs_channel == "preview" {
         ""
     } else {
@@ -733,8 +768,10 @@ fn handle_postprocessing() -> Result<()> {
     }
 
     zlog::info!(logger => "Processing {} `.html` files", files.len());
+    let pages = docs_pages(&ctx.book)?;
+    write_ai_discovery_artifacts(&pages, &root_dir, &site_url)?;
     let meta_regex = Regex::new(&FRONT_MATTER_COMMENT.replace("{}", "(.*)")).unwrap();
-    for file in files {
+    for file in &files {
         let contents = std::fs::read_to_string(&file)?;
         let mut meta_description = None;
         let mut meta_title = None;
@@ -770,14 +807,19 @@ fn handle_postprocessing() -> Result<()> {
         let contents = contents.replace("#amplitude_key#", &litude_key);
         let contents = contents.replace("#consent_io_instance#", &consent_io_instance);
         let contents = contents.replace("#noindex#", noindex);
+        let contents = rewrite_docs_links(&contents, &site_url);
+        let contents = add_markdown_alternate_link(&contents, file, &root_dir, &site_url);
         let contents = title_regex()
             .replace(&contents, |_: ®ex::Captures| {
                 format!("{}", meta_title)
             })
             .to_string();
-        // let contents = contents.replace("#title#", &meta_title);
         std::fs::write(file, contents)?;
     }
+    if let Some(redirects) = redirects {
+        write_markdown_redirect_aliases(&root_dir, &redirects, &site_url)?;
+        write_pages_redirects(&root_dir, &redirects, &site_url)?;
+    }
     return Ok(());
 
     fn pretty_path<'a>(
@@ -906,66 +948,4 @@ fn keymap_schema_for_actions(
 }
 
 #[cfg(test)]
-mod tests {
-    use super::*;
-    use serde_json::json;
-
-    #[test]
-    fn test_find_binding_prefers_exact_match_over_parameterized() {
-        let keymap: KeymapFile = serde_json::from_value(json!([
-            {
-                "bindings": {
-                    "ctrl-tab": "agents_sidebar::ToggleThreadSwitcher",
-                    "ctrl-shift-tab": ["agents_sidebar::ToggleThreadSwitcher", { "select_last": true }]
-                }
-            }
-        ]))
-        .unwrap();
-
-        let binding = find_binding_in_keymap(&keymap, "agents_sidebar::ToggleThreadSwitcher");
-        assert_eq!(binding.as_deref(), Some("ctrl-tab"));
-    }
-
-    #[test]
-    fn test_find_binding_falls_back_to_parameterized_match() {
-        let keymap: KeymapFile = serde_json::from_value(json!([
-            {
-                "bindings": {
-                    "ctrl-shift-tab": ["agents_sidebar::ToggleThreadSwitcher", { "select_last": true }]
-                }
-            }
-        ]))
-        .unwrap();
-
-        let binding = find_binding_in_keymap(&keymap, "agents_sidebar::ToggleThreadSwitcher");
-        assert_eq!(binding.as_deref(), Some("ctrl-shift-tab"));
-    }
-
-    #[test]
-    fn test_find_binding_prefers_exact_match_regardless_of_order() {
-        let keymap: KeymapFile = serde_json::from_value(json!([
-            {
-                "bindings": {
-                    "ctrl-shift-tab": ["agents_sidebar::ToggleThreadSwitcher", { "select_last": true }],
-                    "ctrl-tab": "agents_sidebar::ToggleThreadSwitcher"
-                }
-            }
-        ]))
-        .unwrap();
-
-        let binding = find_binding_in_keymap(&keymap, "agents_sidebar::ToggleThreadSwitcher");
-        assert_eq!(binding.as_deref(), Some("ctrl-tab"));
-    }
-
-    #[test]
-    fn test_find_binding_later_section_overrides_earlier() {
-        let keymap: KeymapFile = serde_json::from_value(json!([
-            { "bindings": { "ctrl-a": "some::Action" } },
-            { "bindings": { "ctrl-b": "some::Action" } }
-        ]))
-        .unwrap();
-
-        let binding = find_binding_in_keymap(&keymap, "some::Action");
-        assert_eq!(binding.as_deref(), Some("ctrl-b"));
-    }
-}
+mod tests;
diff --git a/crates/docs_preprocessor/src/tests.rs b/crates/docs_preprocessor/src/tests.rs
new file mode 100644
index 00000000000..43438207d8c
--- /dev/null
+++ b/crates/docs_preprocessor/src/tests.rs
@@ -0,0 +1,61 @@
+use super::*;
+use serde_json::json;
+
+#[test]
+fn test_find_binding_prefers_exact_match_over_parameterized() {
+    let keymap: KeymapFile = serde_json::from_value(json!([
+        {
+            "bindings": {
+                "ctrl-tab": "agents_sidebar::ToggleThreadSwitcher",
+                "ctrl-shift-tab": ["agents_sidebar::ToggleThreadSwitcher", { "select_last": true }]
+            }
+        }
+    ]))
+    .unwrap();
+
+    let binding = find_binding_in_keymap(&keymap, "agents_sidebar::ToggleThreadSwitcher");
+    assert_eq!(binding.as_deref(), Some("ctrl-tab"));
+}
+
+#[test]
+fn test_find_binding_falls_back_to_parameterized_match() {
+    let keymap: KeymapFile = serde_json::from_value(json!([
+        {
+            "bindings": {
+                "ctrl-shift-tab": ["agents_sidebar::ToggleThreadSwitcher", { "select_last": true }]
+            }
+        }
+    ]))
+    .unwrap();
+
+    let binding = find_binding_in_keymap(&keymap, "agents_sidebar::ToggleThreadSwitcher");
+    assert_eq!(binding.as_deref(), Some("ctrl-shift-tab"));
+}
+
+#[test]
+fn test_find_binding_prefers_exact_match_regardless_of_order() {
+    let keymap: KeymapFile = serde_json::from_value(json!([
+        {
+            "bindings": {
+                "ctrl-shift-tab": ["agents_sidebar::ToggleThreadSwitcher", { "select_last": true }],
+                "ctrl-tab": "agents_sidebar::ToggleThreadSwitcher"
+            }
+        }
+    ]))
+    .unwrap();
+
+    let binding = find_binding_in_keymap(&keymap, "agents_sidebar::ToggleThreadSwitcher");
+    assert_eq!(binding.as_deref(), Some("ctrl-tab"));
+}
+
+#[test]
+fn test_find_binding_later_section_overrides_earlier() {
+    let keymap: KeymapFile = serde_json::from_value(json!([
+        { "bindings": { "ctrl-a": "some::Action" } },
+        { "bindings": { "ctrl-b": "some::Action" } }
+    ]))
+    .unwrap();
+
+    let binding = find_binding_in_keymap(&keymap, "some::Action");
+    assert_eq!(binding.as_deref(), Some("ctrl-b"));
+}
diff --git a/docs/book.toml b/docs/book.toml
index 2eed5f7907a..e1a48d1f7f0 100644
--- a/docs/book.toml
+++ b/docs/book.toml
@@ -45,10 +45,10 @@ enable = false
 "/assistant.html" = "/docs/assistant/assistant.html"
 "/assistant/assistant-panel.html" = "/docs/ai/agent-panel.html"
 "/assistant/assistant.html" = "/docs/ai/overview.html"
-"/assistant/commands.html" = "/docs/ai/text-threads.html"
+"/assistant/commands.html" = "/docs/ai/agent-panel.html"
 "/assistant/configuration.html" = "/docs/ai/quick-start.html"
 "/assistant/context-servers.html" = "/docs/ai/mcp.html"
-"/assistant/contexts.html" = "/docs/ai/text-threads.html"
+"/assistant/contexts.html" = "/docs/ai/agent-panel.html"
 "/assistant/inline-assistant.html" = "/docs/ai/inline-assistant.html"
 "/assistant/model-context-protocol.html" = "/docs/ai/mcp.html"
 "/assistant/prompting.html" = "/docs/ai/skills.html"
diff --git a/docs/theme/css/chrome.css b/docs/theme/css/chrome.css
index 8f5b40cc19e..f0169fd7c87 100644
--- a/docs/theme/css/chrome.css
+++ b/docs/theme/css/chrome.css
@@ -434,6 +434,7 @@ ul#searchresults span.teaser em {
 
 .sidebar {
   position: relative;
+  order: 0;
   width: var(--sidebar-width);
   flex-shrink: 0;
   display: flex;
diff --git a/docs/theme/css/general.css b/docs/theme/css/general.css
index 9c8077bad52..d32edaa8d61 100644
--- a/docs/theme/css/general.css
+++ b/docs/theme/css/general.css
@@ -201,6 +201,7 @@ hr {
 
 .page {
   outline: 0;
+  order: 1;
   flex: 1;
   display: flex;
   flex-direction: column;
diff --git a/docs/theme/index.hbs b/docs/theme/index.hbs
index 2c7786817aa..d5e0b620051 100644
--- a/docs/theme/index.hbs
+++ b/docs/theme/index.hbs
@@ -27,6 +27,7 @@
             })();
         
         {{ title }}
+        
         {{#if is_print }}
         
         {{/if}}
@@ -86,6 +87,9 @@
             document.body.classList.remove('no-js');
             document.body.classList.add('js');
         
+        

+            Agent documentation index: llms.txt. Markdown versions are available for docs pages.
+        

 
         

             

@@ -155,6 +159,95 @@
             

             {{/if}}
 
+            

+                

+                    

+                      {{{ content }}}
+                        

+                            {{#previous}}
+                            
+                            {{/previous}}
+                            {{#next}}
+                            
+                            {{/next}}
+                        

+                        
+                    

+                    

+                        
+                    

+                    
+                    
+                

+            

+