diff --git a/Cargo.lock b/Cargo.lock index b7bdb47c5..f4cafeb50 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -396,6 +396,7 @@ dependencies = [ "iron 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "kuchiki 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "lol_html 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "mime_guess 2.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "notify 4.0.15 (registry+https://github.com/rust-lang/crates.io-index)", "once_cell 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -538,6 +539,24 @@ dependencies = [ "subtle 2.2.3 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "cssparser" +version = "0.25.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "cssparser-macros 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "dtoa-short 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "itoa 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", + "matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "phf 0.7.24 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 1.0.13 (registry+https://github.com/rust-lang/crates.io-index)", + "procedural-masquerade 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.22 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "cssparser" version = "0.27.2" @@ -554,6 +573,18 @@ dependencies = [ "syn 1.0.22 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "cssparser-macros" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "phf_codegen 0.7.24 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 1.0.13 (registry+https://github.com/rust-lang/crates.io-index)", + "procedural-masquerade 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.22 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "cssparser-macros" version = "0.6.0" @@ -1402,6 +1433,23 @@ dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "lol_html" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "cssparser 0.25.9 (registry+https://github.com/rust-lang/crates.io-index)", + "encoding_rs 0.8.23 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lazycell 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "safemem 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "selectors 0.21.0 (registry+https://github.com/rust-lang/crates.io-index)", + "thiserror 1.0.20 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "mac" version = "0.1.1" @@ -2185,6 +2233,11 @@ dependencies = [ "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "procedural-masquerade" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "procfs" version = "0.7.9" @@ -2794,6 +2847,24 @@ dependencies = [ "libc 0.2.70 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "selectors" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "cssparser 0.25.9 (registry+https://github.com/rust-lang/crates.io-index)", + "fxhash 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "phf 0.7.24 (registry+https://github.com/rust-lang/crates.io-index)", + "phf_codegen 0.7.24 (registry+https://github.com/rust-lang/crates.io-index)", + "precomputed-hash 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "servo_arc 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", + "thin-slice 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "selectors" version = "0.22.0" @@ -2965,6 +3036,14 @@ dependencies = [ "unidecode 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "smallvec" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "smallvec" version = "1.4.0" @@ -3284,6 +3363,24 @@ name = "thin-slice" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "thiserror" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "thiserror-impl 1.0.20 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.13 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.22 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "thread_local" version = "1.0.1" @@ -3893,7 +3990,9 @@ dependencies = [ "checksum crossbeam-utils 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" "checksum crypto-mac 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0999b4ff4d3446d4ddb19a63e9e00c1876e75cd7000d20e57a693b4b3f08d958" "checksum crypto-mac 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" +"checksum cssparser 0.25.9 (registry+https://github.com/rust-lang/crates.io-index)" = "fbe18ca4efb9ba3716c6da66cc3d7e673bf59fa576353011f48c4cfddbdd740e" "checksum cssparser 0.27.2 (registry+https://github.com/rust-lang/crates.io-index)" = "754b69d351cdc2d8ee09ae203db831e005560fc6030da058f86ad60c92a9cb0a" +"checksum cssparser-macros 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "5bb1c84e87c717666564ec056105052331431803d606bd45529b28547b611eef" "checksum cssparser-macros 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "dfae75de57f2b2e85e8768c3ea840fd159c8f33e2b6522c7835b7abac81be16e" "checksum csv 1.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "00affe7f6ab566df61b4be3ce8cf16bc2576bca0963ceb0955e45d514bf9a279" "checksum csv-core 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" @@ -3991,6 +4090,7 @@ dependencies = [ "checksum lock_api 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" "checksum log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" "checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" +"checksum lol_html 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "169299b3b58aa5cd8ad25fd8fe984e93748046d24c80f05aaadd9022f95423ec" "checksum mac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" "checksum maplit 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" "checksum markup5ever 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "aae38d669396ca9b707bfc3db254bc382ddb94f57cc5c235f34623a669a01dab" @@ -4079,6 +4179,7 @@ dependencies = [ "checksum proc-macro-hack 0.5.15 (registry+https://github.com/rust-lang/crates.io-index)" = "0d659fe7c6d27f25e9d80a1a094c223f5246f6a6596453e09d7229bf42750b63" "checksum proc-macro-nested 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8e946095f9d3ed29ec38de908c22f95d9ac008e424c7bcae54c75a79c527c694" "checksum proc-macro2 1.0.13 (registry+https://github.com/rust-lang/crates.io-index)" = "53f5ffe53a6b28e37c9c1ce74893477864d64f74778a93a4beb43c8fa167f639" +"checksum procedural-masquerade 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "8f1383dff4092fe903ac180e391a8d4121cc48f08ccf850614b0290c6673b69d" "checksum procfs 0.7.9 (registry+https://github.com/rust-lang/crates.io-index)" = "c434e93ef69c216e68e4f417c927b4f31502c3560b72cfdb6827e2321c5c6b3e" "checksum prometheus 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5567486d5778e2c6455b1b90ff1c558f29e751fc018130fa182e15828e728af1" "checksum quick-error 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" @@ -4139,6 +4240,7 @@ dependencies = [ "checksum scopeguard 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" "checksum security-framework 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "64808902d7d99f78eaddd2b4e2509713babc3dc3c85ad6f4c447680f3c01e535" "checksum security-framework-sys 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "17bf11d99252f512695eb468de5516e5cf75455521e69dfe343f3b74e4748405" +"checksum selectors 0.21.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1b86b100bede4f651059740afc3b6cb83458d7401cb7c1ad96d8a11e91742c86" "checksum selectors 0.22.0 (registry+https://github.com/rust-lang/crates.io-index)" = "df320f1889ac4ba6bc0cdc9c9af7af4bd64bb927bccdf32d81140dc1f9be12fe" "checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" "checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" @@ -4159,6 +4261,7 @@ dependencies = [ "checksum siphasher 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "fa8f3741c7372e75519bd9346068370c9cdaabcc1f9599cbcf2a2719352286b7" "checksum slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" "checksum slug 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "39af1ce888a1253c8b9fcfa36626557650fb487c013620a743262d2769a3e9f3" +"checksum smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "f7b0758c52e15a8b5e3691eae6cc559f08eee9406e548a4477ba4e67770a82b6" "checksum smallvec 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c7cb5678e1615754284ec264d9bb5b4c27d2018577fd90ac0ceb578591ed5ee4" "checksum socket2 0.3.12 (registry+https://github.com/rust-lang/crates.io-index)" = "03088793f677dce356f3ccc2edb1b314ad191ab702a5de3faf49304f7e104918" "checksum spin 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" @@ -4192,6 +4295,8 @@ dependencies = [ "checksum termcolor 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb6bfa289a4d7c5766392812c0a1f4c1ba45afa1ad47803c11e1f407d846d75f" "checksum textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" "checksum thin-slice 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c" +"checksum thiserror 1.0.20 (registry+https://github.com/rust-lang/crates.io-index)" = "7dfdd070ccd8ccb78f4ad66bf1982dc37f620ef696c6b5028fe2ed83dd3d0d08" +"checksum thiserror-impl 1.0.20 (registry+https://github.com/rust-lang/crates.io-index)" = "bd80fc12f73063ac132ac92aceea36734f04a1d93c1240c6944e23a3b8841793" "checksum thread_local 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" "checksum time 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)" = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" "checksum time 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)" = "3a51cadc5b1eec673a685ff7c33192ff7b7603d0b75446fb354939ee615acb15" diff --git a/Cargo.toml b/Cargo.toml index ba3d01646..6e658d7e0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,6 @@ backtrace = "0.3" failure = { version = "0.1.3", features = ["backtrace"] } comrak = { version = "0.3", default-features = false } toml = "0.5" -kuchiki = "0.8" schemamama = "0.3" schemamama_postgres = "0.2" systemstat = "0.1.4" @@ -41,6 +40,7 @@ path-slash = "0.1.3" once_cell = { version = "1.4.0", features = ["parking_lot"] } base64 = "0.12.1" strum = { version = "0.18.0", features = ["derive"] } +lol_html = "0.2" # Async tokio = { version = "0.2.22", features = ["rt-threaded"] } @@ -82,6 +82,7 @@ procfs = "0.7" [dev-dependencies] criterion = "0.3" +kuchiki = "0.8" rand = "0.7.3" [[bench]] diff --git a/src/config.rs b/src/config.rs index 003f3e9d4..936018b6a 100644 --- a/src/config.rs +++ b/src/config.rs @@ -28,6 +28,8 @@ pub struct Config { // Max size of the files served by the docs.rs frontend pub(crate) max_file_size: usize, pub(crate) max_file_size_html: usize, + // The most memory that can be used to parse an HTML file + pub(crate) max_parse_memory: usize, } impl Config { @@ -55,7 +57,10 @@ impl Config { github_accesstoken: maybe_env("CRATESFYI_GITHUB_ACCESSTOKEN")?, max_file_size: env("DOCSRS_MAX_FILE_SIZE", 50 * 1024 * 1024)?, - max_file_size_html: env("DOCSRS_MAX_FILE_SIZE_HTML", 5 * 1024 * 1024)?, + max_file_size_html: env("DOCSRS_MAX_FILE_SIZE_HTML", 50 * 1024 * 1024)?, + // LOL HTML only uses as much memory as the size of the start tag! + // https://github.com/rust-lang/docs.rs/pull/930#issuecomment-667729380 + max_parse_memory: env("DOCSRS_MAX_PARSE_MEMORY", 5 * 1024 * 1024)?, }) } diff --git a/src/test/fakes.rs b/src/test/fakes.rs index c9ecd9f7a..c4d88f024 100644 --- a/src/test/fakes.rs +++ b/src/test/fakes.rs @@ -27,6 +27,9 @@ pub(crate) struct FakeRelease<'a> { readme: Option<&'a str>, } +const DEFAULT_CONTENT: &[u8] = + b"default content for test/fakes"; + impl<'a> FakeRelease<'a> { pub(super) fn new(db: &'a TestDatabase, storage: Arc) -> Self { FakeRelease { @@ -121,7 +124,14 @@ impl<'a> FakeRelease<'a> { self } - pub(crate) fn rustdoc_file(mut self, path: &'a str, data: &'a [u8]) -> Self { + /// Since we switched to LOL HTML, all data must have a valid and . + /// To avoid duplicating them in every test, this just makes up some content. + pub(crate) fn rustdoc_file(mut self, path: &'a str) -> Self { + self.rustdoc_files.push((path, DEFAULT_CONTENT)); + self + } + + pub(crate) fn rustdoc_file_with(mut self, path: &'a str, data: &'a [u8]) -> Self { self.rustdoc_files.push((path, data)); self } @@ -217,7 +227,7 @@ impl<'a> FakeRelease<'a> { let index = [&package.name, "index.html"].join("/"); let mut rustdoc_files = self.rustdoc_files; if package.is_library() && !rustdoc_files.iter().any(|(path, _)| path == &index) { - rustdoc_files.push((&index, b"default index content")); + rustdoc_files.push((&index, DEFAULT_CONTENT)); } for (source_path, data) in &self.source_files { if source_path.starts_with("src/") { diff --git a/src/utils/html.rs b/src/utils/html.rs index 9a24b4561..b194a9cc0 100644 --- a/src/utils/html.rs +++ b/src/utils/html.rs @@ -1,62 +1,87 @@ -use crate::error::Result; -use failure::err_msg; -use kuchiki::traits::TendrilSink; -use kuchiki::NodeRef; +use crate::web::page::TemplateData; +use lol_html::errors::RewritingError; +use tera::Context; -/// Extracts the contents of the `` and `` tags from an HTML document, as well as the -/// classes on the `` tag, if any. -pub fn extract_head_and_body(html: &str) -> Result<(String, String, String)> { - let dom = kuchiki::parse_html().one(html); +/// Rewrite a rustdoc page to have the docs.rs header +/// +/// Given a rustdoc HTML page and a context to serialize it with, +/// render the `rustdoc/` templates with the `html`. +/// The output is an HTML page which has not yet been UTF-8 validated. +/// In practice, the output should always be valid UTF-8. +pub(crate) fn rewrite_lol( + html: &[u8], + max_allowed_memory_usage: usize, + ctx: Context, + templates: &TemplateData, +) -> Result, RewritingError> { + use lol_html::html_content::{ContentType, Element}; + use lol_html::{ElementContentHandlers, HtmlRewriter, MemorySettings, Settings}; - let head = dom - .select_first("head") - .map_err(|_| err_msg("couldn't find tag in rustdoc output"))?; - let body = dom - .select_first("body") - .map_err(|_| err_msg("couldn't find tag in rustdoc output"))?; + let templates = templates.templates.load(); + let tera_head = templates.render("rustdoc/head.html", &ctx).unwrap(); + let tera_body = templates.render("rustdoc/body.html", &ctx).unwrap(); - let class = body - .attributes - .borrow() - .get("class") - .map(|v| v.to_owned()) - .unwrap_or_default(); + let head_handler = |head: &mut Element| { + head.append(&tera_head, ContentType::Html); + Ok(()) + }; - Ok((serialize(head.as_node()), serialize(body.as_node()), class)) -} - -fn serialize(v: &NodeRef) -> String { - let mut contents = Vec::new(); - for child in v.children() { - child - .serialize(&mut contents) - .expect("serialization failed"); - } - String::from_utf8(contents).expect("non utf-8 html") -} + // Before: ... rustdoc content ... + // After: + // ```html + //
+ // ... rustdoc content ... + //
+ // ``` + let body_handler = |rustdoc_body_class: &mut Element| { + // Add the `rustdoc` classes to the html body + let mut tmp; + let klass = if let Some(classes) = rustdoc_body_class.get_attribute("class") { + tmp = classes; + tmp.push_str(" container-rustdoc"); + &tmp + } else { + "container-rustdoc" + }; + rustdoc_body_class.set_attribute("class", klass)?; + rustdoc_body_class.set_attribute("id", "rustdoc_body_wrapper")?; + rustdoc_body_class.set_attribute("tabindex", "-1")?; + // Change the `body` to a `div` + rustdoc_body_class.set_tag_name("div")?; + // Prepend the tera content + rustdoc_body_class.prepend(&tera_body, ContentType::Html); + // Now, make this a full tag + rustdoc_body_class.before("", ContentType::Html); + rustdoc_body_class.after("", ContentType::Html); -#[cfg(test)] -mod test { - #[test] - fn small_html() { - let (head, body, class) = super::extract_head_and_body( - r#"

hello

"# - ).unwrap(); - assert_eq!(head, r#""#); - assert_eq!(body, "

hello

"); - assert_eq!(class, "rustdoc struct"); - } + Ok(()) + }; - // more of an integration test - #[test] - fn parse_regex_html() { - let original = std::fs::read_to_string("benches/struct.CaptureMatches.html").unwrap(); - let expected_head = std::fs::read_to_string("tests/regex/head.html").unwrap(); - let expected_body = std::fs::read_to_string("tests/regex/body.html").unwrap(); - let (head, body, class) = super::extract_head_and_body(&original).unwrap(); + let (head_selector, body_selector) = ("head".parse().unwrap(), "body".parse().unwrap()); + let head = ( + &head_selector, + ElementContentHandlers::default().element(head_handler), + ); + let body = ( + &body_selector, + ElementContentHandlers::default().element(body_handler), + ); + let settings = Settings { + element_content_handlers: vec![head, body], + memory_settings: MemorySettings { + max_allowed_memory_usage, + ..MemorySettings::default() + }, + ..Settings::default() + }; - assert_eq!(head, expected_head.trim()); - assert_eq!(&body, &expected_body.trim()); - assert_eq!(class, "rustdoc struct"); - } + // The input and output are always strings, we just use `&[u8]` so we only have to validate once. + let mut buffer = Vec::new(); + let mut writer = HtmlRewriter::try_new(settings, |bytes: &[u8]| { + buffer.extend_from_slice(bytes); + }) + .expect("utf8 is a valid encoding"); + writer.write(html)?; + writer.end()?; + Ok(buffer) } diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 101b3930d..69935dac4 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -4,7 +4,7 @@ pub(crate) use self::cargo_metadata::{CargoMetadata, Package as MetadataPackage} pub(crate) use self::copy::copy_doc_dir; pub use self::daemon::start_daemon; pub use self::github_updater::GithubUpdater; -pub use self::html::extract_head_and_body; +pub(crate) use self::html::rewrite_lol; pub use self::queue::{get_crate_priority, remove_crate_priority, set_crate_priority}; pub use self::queue_builder::queue_builder; pub use self::release_activity_updater::update_release_activity; diff --git a/src/web/file.rs b/src/web/file.rs index 985ab5679..96b4ce8f4 100644 --- a/src/web/file.rs +++ b/src/web/file.rs @@ -115,12 +115,12 @@ mod tests { env.fake_release() .name("dummy") .version("0.1.0") - .rustdoc_file("small.html", &[b'A'; MAX_HTML_SIZE / 2] as &[u8]) - .rustdoc_file("exact.html", &[b'A'; MAX_HTML_SIZE] as &[u8]) - .rustdoc_file("big.html", &[b'A'; MAX_HTML_SIZE * 2] as &[u8]) - .rustdoc_file("small.js", &[b'A'; MAX_SIZE / 2] as &[u8]) - .rustdoc_file("exact.js", &[b'A'; MAX_SIZE] as &[u8]) - .rustdoc_file("big.js", &[b'A'; MAX_SIZE * 2] as &[u8]) + .rustdoc_file_with("small.html", &[b'A'; MAX_HTML_SIZE / 2] as &[u8]) + .rustdoc_file_with("exact.html", &[b'A'; MAX_HTML_SIZE] as &[u8]) + .rustdoc_file_with("big.html", &[b'A'; MAX_HTML_SIZE * 2] as &[u8]) + .rustdoc_file_with("small.js", &[b'A'; MAX_SIZE / 2] as &[u8]) + .rustdoc_file_with("exact.js", &[b'A'; MAX_SIZE] as &[u8]) + .rustdoc_file_with("big.js", &[b'A'; MAX_SIZE * 2] as &[u8]) .create()?; let file = |path| { diff --git a/src/web/rustdoc.rs b/src/web/rustdoc.rs index f4c3f2ae0..4d756a750 100644 --- a/src/web/rustdoc.rs +++ b/src/web/rustdoc.rs @@ -2,10 +2,10 @@ use crate::{ db::Pool, - impl_webpage, utils, + utils, web::{ crate_details::CrateDetails, error::Nope, file::File, match_version, metrics, - page::WebPage, redirect_base, MatchSemver, + redirect_base, MatchSemver, }, Config, Storage, }; @@ -188,14 +188,36 @@ struct RustdocPage { latest_version: String, inner_path: String, is_latest_version: bool, - rustdoc_head: String, - rustdoc_body: String, - rustdoc_body_class: String, krate: CrateDetails, } -impl_webpage! { - RustdocPage = "rustdoc/page.html", +impl RustdocPage { + fn into_response( + self, + rustdoc_html: &[u8], + max_parse_memory: usize, + req: &mut Request, + ) -> IronResult { + use iron::{headers::ContentType, status::Status}; + + let templates = req + .extensions + .get::() + .expect("missing TemplateData from the request extensions"); + + // Build the page of documentation + let ctx = ctry!(req, tera::Context::from_serialize(self),); + // Extract the head and body of the rustdoc file so that we can insert it into our own html + let html = ctry!( + req, + utils::rewrite_lol(rustdoc_html, max_parse_memory, ctx, templates) + ); + + let mut response = Response::with((Status::Ok, html)); + response.headers.set(ContentType::html()); + + Ok(response) + } } /// Serves documentation generated by rustdoc. @@ -318,21 +340,6 @@ pub fn rustdoc_html_server_handler(req: &mut Request) -> IronResult { return Ok(file.serve()); } - rendering_time.step("parse html"); - - let file_content = ctry!(req, String::from_utf8(file.0.content)); - // Extract the head and body of the rustdoc file so that we can insert it into our own html - let (rustdoc_head, rustdoc_body, mut rustdoc_body_class) = - ctry!(req, utils::extract_head_and_body(&file_content)); - - // Add the `rustdoc` classes to the html body - if rustdoc_body_class.is_empty() { - rustdoc_body_class = "rustdoc container-rustdoc".to_string(); - } else { - // rustdoc adds its own "rustdoc" class to the body - rustdoc_body_class.push_str(" container-rustdoc"); - } - rendering_time.step("find latest path"); let latest_release = krate.latest_release(); @@ -361,8 +368,6 @@ pub fn rustdoc_html_server_handler(req: &mut Request) -> IronResult { format!("/crate/{}/{}", name, latest_version) }; - rendering_time.step("serve html"); - // The path within this crate version's rustdoc output let inner_path = { let mut inner_path = req_path.clone(); @@ -377,18 +382,15 @@ pub fn rustdoc_html_server_handler(req: &mut Request) -> IronResult { inner_path.join("/") }; - // Build the page of documentation + rendering_time.step("rewrite html"); RustdocPage { latest_path, latest_version, inner_path, is_latest_version, - rustdoc_head, - rustdoc_body, - rustdoc_body_class, krate, } - .into_response(req) + .into_response(&file.0.content, config.max_parse_memory, req) } /// Checks whether the given path exists. @@ -598,7 +600,7 @@ mod test { ) -> Result, failure::Error> { assert_success(path, web)?; let data = web.get(path).send()?.text()?; - println!("{}", data); + log::info!("fetched path {} and got content {}", path, data); let dom = kuchiki::parse_html().one(data); if let Some(elem) = dom @@ -615,8 +617,8 @@ mod test { } fn latest_version_redirect(path: &str, web: &TestFrontend) -> Result { - try_latest_version_redirect(path, web) - .and_then(|v| v.ok_or_else(|| failure::format_err!("no redirect found for {}", path))) + try_latest_version_redirect(path, web)? + .ok_or_else(|| failure::format_err!("no redirect found for {}", path)) } #[test] @@ -628,12 +630,12 @@ mod test { .name("buggy") .version("0.1.0") .build_result_successful(true) - .rustdoc_file("settings.html", b"some data") - .rustdoc_file("directory_1/index.html", b"some data 1") - .rustdoc_file("directory_2.html/index.html", b"some data 1") - .rustdoc_file("all.html", b"some data 2") - .rustdoc_file("directory_3/.gitignore", b"*.ext") - .rustdoc_file("directory_4/empty_file_no_ext", b"") + .rustdoc_file("settings.html") + .rustdoc_file("directory_1/index.html") + .rustdoc_file("directory_2.html/index.html") + .rustdoc_file("all.html") + .rustdoc_file("directory_3/.gitignore") + .rustdoc_file("directory_4/empty_file_no_ext") .create()?; env.fake_release() .name("buggy") @@ -659,7 +661,7 @@ mod test { env.fake_release() .name("dummy") .version("0.1.0") - .rustdoc_file("dummy/index.html", b"some content") + .rustdoc_file("dummy/index.html") .create()?; let web = env.frontend(); @@ -673,7 +675,7 @@ mod test { env.fake_release() .name("dummy") .version("0.2.0") - .rustdoc_file("dummy/index.html", b"some content") + .rustdoc_file("dummy/index.html") .default_target(target) .create()?; let base = "/dummy/0.2.0/dummy/"; @@ -686,8 +688,8 @@ mod test { env.fake_release() .name("dummy") .version("0.3.0") - .rustdoc_file("dummy/index.html", b"some content") - .rustdoc_file("all.html", b"html") + .rustdoc_file("dummy/index.html") + .rustdoc_file("all.html") .default_target(target) .create()?; let base = "/dummy/0.3.0/dummy/"; @@ -710,15 +712,15 @@ mod test { env.fake_release() .name("dummy") .version("0.1.0") - .rustdoc_file("dummy/blah/index.html", b"lah") - .rustdoc_file("dummy/blah/blah.html", b"lah") - .rustdoc_file("dummy/struct.will-be-deleted.html", b"lah") + .rustdoc_file("dummy/blah/index.html") + .rustdoc_file("dummy/blah/blah.html") + .rustdoc_file("dummy/struct.will-be-deleted.html") .create()?; env.fake_release() .name("dummy") .version("0.2.0") - .rustdoc_file("dummy/blah/index.html", b"lah") - .rustdoc_file("dummy/blah/blah.html", b"lah") + .rustdoc_file("dummy/blah/index.html") + .rustdoc_file("dummy/blah/blah.html") .create()?; let web = env.frontend(); @@ -756,7 +758,7 @@ mod test { .name("dummy") .version("0.1.0") .add_platform("x86_64-pc-windows-msvc") - .rustdoc_file("dummy/struct.Blah.html", b"lah") + .rustdoc_file("dummy/struct.Blah.html") .create()?; env.fake_release() .name("dummy") @@ -802,7 +804,7 @@ mod test { env.fake_release() .name("dummy") .version("0.1.0") - .rustdoc_file("dummy/index.html", b"lah") + .rustdoc_file("dummy/index.html") .create()?; env.fake_release() .name("dummy") @@ -824,17 +826,17 @@ mod test { env.fake_release() .name("dummy") .version("0.1.0") - .rustdoc_file("dummy/index.html", b"lah") + .rustdoc_file("dummy/index.html") .create()?; env.fake_release() .name("dummy") .version("0.2.0") - .rustdoc_file("dummy/index.html", b"lah") + .rustdoc_file("dummy/index.html") .create()?; env.fake_release() .name("dummy") .version("0.2.1") - .rustdoc_file("dummy/index.html", b"lah") + .rustdoc_file("dummy/index.html") .yanked(true) .create()?; @@ -855,19 +857,19 @@ mod test { env.fake_release() .name("dummy") .version("0.1.0") - .rustdoc_file("dummy/index.html", b"lah") + .rustdoc_file("dummy/index.html") .yanked(true) .create()?; env.fake_release() .name("dummy") .version("0.2.0") - .rustdoc_file("dummy/index.html", b"lah") + .rustdoc_file("dummy/index.html") .yanked(true) .create()?; env.fake_release() .name("dummy") .version("0.2.1") - .rustdoc_file("dummy/index.html", b"lah") + .rustdoc_file("dummy/index.html") .yanked(true) .create()?; @@ -900,7 +902,7 @@ mod test { env.fake_release() .name("dummy") .version("0.1.0") - .rustdoc_file("dummy/index.html", b"lah") + .rustdoc_file("dummy/index.html") .yanked(true) .create()?; @@ -909,7 +911,7 @@ mod test { env.fake_release() .name("dummy") .version("0.2.0") - .rustdoc_file("dummy/index.html", b"lah") + .rustdoc_file("dummy/index.html") .yanked(true) .create()?; @@ -943,7 +945,7 @@ mod test { env.fake_release() .name("fake-crate") .version("0.0.1") - .rustdoc_file("fake_crate/index.html", b"some content") + .rustdoc_file("fake_crate/index.html") .create()?; let web = env.frontend(); @@ -969,7 +971,7 @@ mod test { env.fake_release() .name(name) .version(version) - .rustdoc_file(&(name.replace("-", "_") + "/index.html"), b"") + .rustdoc_file(&(name.replace("-", "_") + "/index.html")) .create()?; } @@ -1019,13 +1021,13 @@ mod test { env.fake_release() .name("dummy-dash") .version("0.1.0") - .rustdoc_file("dummy_dash/index.html", b"") + .rustdoc_file("dummy_dash/index.html") .create()?; env.fake_release() .name("dummy_mixed-separators") .version("0.1.0") - .rustdoc_file("dummy_mixed_separators/index.html", b"") + .rustdoc_file("dummy_mixed_separators/index.html") .create()?; let web = env.frontend(); @@ -1131,8 +1133,8 @@ mod test { env.fake_release() .name("dummy") .version("0.1.0") - .rustdoc_file("dummy/index.html", b"some content") - .rustdoc_file("dummy/struct.Dummy.html", b"some other content") + .rustdoc_file("dummy/index.html") + .rustdoc_file("dummy/struct.Dummy.html") .add_target("x86_64-unknown-linux-gnu") .create()?; @@ -1161,8 +1163,8 @@ mod test { env.fake_release() .name("dummy") .version("0.2.0") - .rustdoc_file("dummy/index.html", b"some content") - .rustdoc_file("dummy/struct.Dummy.html", b"some other content") + .rustdoc_file("dummy/index.html") + .rustdoc_file("dummy/struct.Dummy.html") .default_target("x86_64-pc-windows-msvc") .create()?; @@ -1191,8 +1193,8 @@ mod test { env.fake_release() .name("dummy") .version("0.3.0") - .rustdoc_file("dummy/index.html", b"some content") - .rustdoc_file("dummy/struct.Dummy.html", b"some other content") + .rustdoc_file("dummy/index.html") + .rustdoc_file("dummy/struct.Dummy.html") .default_target("x86_64-unknown-linux-gnu") .create()?; @@ -1221,20 +1223,14 @@ mod test { env.fake_release() .name("dummy") .version("0.4.0") - .rustdoc_file("settings.html", b"top-level items") - .rustdoc_file("dummy/index.html", b"some content") - .rustdoc_file("dummy/struct.Dummy.html", b"some other content") - .rustdoc_file("dummy/struct.DefaultOnly.html", b"some otter content") - .rustdoc_file("x86_64-pc-windows-msvc/settings.html", b"top-level items") - .rustdoc_file("x86_64-pc-windows-msvc/dummy/index.html", b"some content") - .rustdoc_file( - "x86_64-pc-windows-msvc/dummy/struct.Dummy.html", - b"some other content", - ) - .rustdoc_file( - "x86_64-pc-windows-msvc/dummy/struct.WindowsOnly.html", - b"some otter content", - ) + .rustdoc_file("settings.html") + .rustdoc_file("dummy/index.html") + .rustdoc_file("dummy/struct.Dummy.html") + .rustdoc_file("dummy/struct.DefaultOnly.html") + .rustdoc_file("x86_64-pc-windows-msvc/settings.html") + .rustdoc_file("x86_64-pc-windows-msvc/dummy/index.html") + .rustdoc_file("x86_64-pc-windows-msvc/dummy/struct.Dummy.html") + .rustdoc_file("x86_64-pc-windows-msvc/dummy/struct.WindowsOnly.html") .default_target("x86_64-unknown-linux-gnu") .add_target("x86_64-pc-windows-msvc") .create()?; @@ -1503,7 +1499,7 @@ mod test { env.fake_release() .name("tokio") .version("0.2.21") - .rustdoc_file("tokio/time/index.html", b"content") + .rustdoc_file("tokio/time/index.html") .create()?; assert_redirect( "/tokio/0.2.21/tokio/time", diff --git a/templates/rustdoc/navigation.html b/templates/rustdoc/body.html similarity index 88% rename from templates/rustdoc/navigation.html rename to templates/rustdoc/body.html index 690ee27ec..e74067451 100644 --- a/templates/rustdoc/navigation.html +++ b/templates/rustdoc/body.html @@ -140,7 +140,7 @@
  • Versions
  • - +
+ + + diff --git a/templates/rustdoc/head.html b/templates/rustdoc/head.html new file mode 100644 index 000000000..3fd4aaa6e --- /dev/null +++ b/templates/rustdoc/head.html @@ -0,0 +1,11 @@ +{%- import "macros.html" as macros -%} + + + + + + + {{ macros::doc_title(name=krate.name, version=krate.version) }} diff --git a/templates/rustdoc/page.html b/templates/rustdoc/page.html deleted file mode 100644 index a0e52177b..000000000 --- a/templates/rustdoc/page.html +++ /dev/null @@ -1,64 +0,0 @@ -{%- import "macros.html" as macros -%} - - - - - - {# Include any head that rustdoc requires #} - {{ rustdoc_head | safe }} - - - - - - - - {{ macros::doc_title(name=krate.name, version=krate.version) }} - - - - {%- include "rustdoc/navigation.html" -%} - - {# Make the body of the page the documentation generated by rustdoc #} -
- {{ rustdoc_body | safe }} -
- - - - - - -