mirror of
https://github.com/rust-lang/mdBook.git
synced 2025-12-27 09:05:40 -05:00
2b242494b0
This rewrites the HTML rendering pipeline to use a tree data structure,
and implements a custom HTML serializer. The intent is to make it easier
to make changes and to manipulate the output. This should make some
future changes much easier.
This is a large change, but I'll try to briefly summarize what's
changing:
- All of the HTML rendering support has been moved out of
mdbook-markdown into mdbook-html. For now, all of the API surface is
private, though we may consider ways to safely expose it in the
future.
- Instead of using pulldown-cmark's html serializer, this takes the
pulldown-cmark events and translates them into a tree data structure
(using the ego-tree crate to define the tree). See `tree.rs`.
- HTML in the markdown document is parsed using html5ever, and then
lives inside the same tree data structure. See `tokenizer.rs`.
- Transformations are then applied to the tree data structure. For
example, adding header links or hiding code lines.
- Serialization is a simple process of writing out the nodes to a
string. See `serialize.rs`.
- The search indexer works on the tree structure instead of re-rendering
every chapter twice. See `html_handlebars/search.rs`.
- The print page now takes a very different approach of taking the
same tree structure built for rendering the chapters, and applies
transformations to it. This avoid re-parsing everything again. See
`print.rs`.
- I changed the linking behavior so that links on the print page
link to items on the print page instead of outside the print page.
- There are a variety of small changes to how it serializes as can be
seen in the changes to the tests. Some highlights:
- Code blocks no longer have a second layer of `<pre>` tags wrapping
it.
- Fixed a minor issue where a rust code block with a specific
edition was having the wrong classes when there was a default
edition.
- Drops the ammonia dependency, which significantly reduces the number
of dependencies. It was only being used for a very minor task, and
we can handle it much more easily now.
- Drops `pretty_assertions`, they are no longer used (mostly being
migrated to the testsuite).
There's obviously a lot of risk trying to parse everything to such a low
level, but I think the benefits are worth it. Also, the API isn't super
ergonomic compared to say javascript (there are no selectors), but it
works well enough so far.
I have not run this through rigorous benchmarking, but it does have a
very noticeable performance improvement, especially in a debug build.
I expect in the future that we'll want to expose some kind of
integration with extensions so they have access to this tree structure
(or some kind of tree structure).
Closes https://github.com/rust-lang/mdBook/issues/1736
137 lines
5.0 KiB
Rust
137 lines
5.0 KiB
Rust
//! Tests for search support.
|
|
|
|
use crate::prelude::*;
|
|
use mdbook_core::book::{BookItem, Chapter};
|
|
use snapbox::file;
|
|
use std::path::Path;
|
|
|
|
fn read_book_index(root: &Path) -> serde_json::Value {
|
|
let index_path = glob_one(root, "book/searchindex*.js");
|
|
let index = read_to_string(&index_path);
|
|
let index =
|
|
index.trim_start_matches("window.search = Object.assign(window.search, JSON.parse('");
|
|
let index = index.trim_end_matches("'));");
|
|
// We need unescape the string as it's supposed to be an escaped JS string.
|
|
serde_json::from_str(&index.replace("\\'", "'").replace("\\\\", "\\")).unwrap()
|
|
}
|
|
|
|
// Some spot checks for the generation of the search index.
|
|
#[test]
|
|
fn reasonable_search_index() {
|
|
let mut test = BookTest::from_dir("search/reasonable_search_index");
|
|
test.build();
|
|
let index = read_book_index(&test.dir);
|
|
|
|
let doc_urls = index["doc_urls"].as_array().unwrap();
|
|
eprintln!("doc_urls={doc_urls:#?}",);
|
|
let get_doc_ref = |url: &str| -> String {
|
|
doc_urls
|
|
.iter()
|
|
.position(|s| s == url)
|
|
.unwrap_or_else(|| panic!("failed to find {url}"))
|
|
.to_string()
|
|
};
|
|
|
|
let first_chapter = get_doc_ref("first/index.html#first-chapter");
|
|
let introduction = get_doc_ref("intro.html#introduction");
|
|
let some_section = get_doc_ref("first/index.html#some-section");
|
|
let summary = get_doc_ref("first/includes.html#summary");
|
|
let no_headers = get_doc_ref("first/no-headers.html");
|
|
let duplicate_headers_1 = get_doc_ref("first/duplicate-headers.html#header-text-1");
|
|
let heading_attrs = get_doc_ref("first/heading-attributes.html#both");
|
|
let sneaky = get_doc_ref("intro.html#sneaky");
|
|
|
|
let bodyidx = &index["index"]["index"]["body"]["root"];
|
|
let textidx = &bodyidx["t"]["e"]["x"]["t"];
|
|
assert_eq!(textidx["df"], 5);
|
|
assert_eq!(textidx["docs"][&first_chapter]["tf"], 1.0);
|
|
assert_eq!(textidx["docs"][&introduction]["tf"], 1.0);
|
|
|
|
let docs = &index["index"]["documentStore"]["docs"];
|
|
assert_eq!(docs[&first_chapter]["body"], "more text.");
|
|
assert_eq!(docs[&some_section]["body"], "");
|
|
assert_eq!(
|
|
docs[&summary]["body"],
|
|
"Introduction First Chapter Includes Unicode No Headers Duplicate Headers Heading Attributes"
|
|
);
|
|
assert_eq!(
|
|
docs[&summary]["breadcrumbs"],
|
|
"First Chapter » Includes » Summary"
|
|
);
|
|
// See note about InlineHtml in search.rs. Ideally the `alert()` part
|
|
// should not be in the index, but we don't have a way to scrub inline
|
|
// html.
|
|
assert_eq!(
|
|
docs[&sneaky]["body"],
|
|
"I put <HTML> in here! Sneaky inline event . But regular inline is indexed."
|
|
);
|
|
assert_eq!(
|
|
docs[&no_headers]["breadcrumbs"],
|
|
"First Chapter » No Headers"
|
|
);
|
|
assert_eq!(
|
|
docs[&duplicate_headers_1]["breadcrumbs"],
|
|
"First Chapter » Duplicate Headers » Header Text"
|
|
);
|
|
assert_eq!(
|
|
docs[&no_headers]["body"],
|
|
"Capybara capybara capybara. Capybara capybara capybara. ThisLongWordIsIncludedSoWeCanCheckThatSufficientlyLongWordsAreOmittedFromTheSearchIndex."
|
|
);
|
|
assert_eq!(
|
|
docs[&heading_attrs]["breadcrumbs"],
|
|
"First Chapter » Heading Attributes » Heading with id and classes"
|
|
);
|
|
}
|
|
|
|
// This test is here to catch any unexpected changes to the search index.
|
|
#[test]
|
|
fn search_index_hasnt_changed_accidentally() {
|
|
BookTest::from_dir("search/reasonable_search_index").check_file(
|
|
"book/searchindex*.js",
|
|
file!["search/reasonable_search_index/expected_index.js"],
|
|
);
|
|
}
|
|
|
|
// Ability to disable search chapters.
|
|
#[test]
|
|
fn can_disable_individual_chapters() {
|
|
let mut test = BookTest::from_dir("search/disable_search_chapter");
|
|
test.build();
|
|
let index = read_book_index(&test.dir);
|
|
let doc_urls = index["doc_urls"].as_array().unwrap();
|
|
let contains = |path| {
|
|
doc_urls
|
|
.iter()
|
|
.any(|p| p.as_str().unwrap().starts_with(path))
|
|
};
|
|
assert!(contains("second.html"));
|
|
assert!(!contains("second/"));
|
|
assert!(!contains("first/disable_me.html"));
|
|
assert!(contains("first/keep_me.html"));
|
|
}
|
|
|
|
// Test for a regression where search would fail if source_path is None.
|
|
// https://github.com/rust-lang/mdBook/pull/2550
|
|
#[test]
|
|
fn with_no_source_path() {
|
|
let test = BookTest::from_dir("search/reasonable_search_index");
|
|
let mut book = test.load_book();
|
|
let chapter = Chapter::new("Sample chapter", String::new(), "sample.html", vec![]);
|
|
book.book.items.push(BookItem::Chapter(chapter));
|
|
book.build().unwrap();
|
|
}
|
|
|
|
// Checks that invalid settings in search chapter is rejected.
|
|
#[test]
|
|
fn chapter_settings_validation_error() {
|
|
BookTest::from_dir("search/chapter_settings_validation_error").run("build", |cmd| {
|
|
cmd.expect_failure().expect_stderr(str![[r#"
|
|
INFO Book building has started
|
|
INFO Running the html backend
|
|
ERROR Rendering failed
|
|
[TAB]Caused by: [output.html.search.chapter] key `does-not-exist` does not match any chapter paths
|
|
|
|
"#]]);
|
|
});
|
|
}
|