Update to pulldown-cmark 0.5. (rust-lang#898)

* Update to pulldown-cmark 0.4.1.

* Update to pulldown-cmark 0.5.2.

* Remove pulldown-cmark-to-cmark dependency.

Since it is not compatible with the new pulldown-cmark. This example isn't
directly usable, anyways, and I think the no-op example sufficiently shows how
to make a preprocessor.

* cargo fmt

* Fix example link.

Cargo.toml

@@ -26,7 +26,7 @@ lazy_static = "1.0"
 log = "0.4"
 memchr = "2.0"
 open = "1.1"
-pulldown-cmark = "0.1.2"
+pulldown-cmark = "0.5"
 regex = "1.0.0"
 serde = "1.0"
 serde_derive = "1.0"
@@ -52,8 +52,6 @@ ammonia = { version = "2.1", optional = true }
 select = "0.4"
 pretty_assertions = "0.6"
 walkdir = "2.0"
-# FIXME(issue #898)
-pulldown-cmark-to-cmark = "=1.1.0"
 
 [features]
 default = ["output", "watch", "serve", "search"]

book-example/src/for_developers/preprocessors.md

@@ -109,7 +109,7 @@ For everything else, have a look [at the complete example][example].
 [preprocessor-docs]: https://docs.rs/mdbook/latest/mdbook/preprocess/trait.Preprocessor.html
 [pc]: https://crates.io/crates/pulldown-cmark
 [pctc]: https://crates.io/crates/pulldown-cmark-to-cmark
-[example]: https://github.com/rust-lang-nursery/mdBook/blob/master/examples/de-emphasize.rs
+[example]: https://github.com/rust-lang-nursery/mdBook/blob/master/examples/nop-preprocessor.rs
 [an example no-op preprocessor]: https://github.com/rust-lang-nursery/mdBook/blob/master/examples/nop-preprocessor.rs
 [`CmdPreprocessor::parse_input()`]: https://docs.rs/mdbook/latest/mdbook/preprocess/trait.Preprocessor.html#method.parse_input
 [`Book::for_each_mut()`]: https://docs.rs/mdbook/latest/mdbook/book/struct.Book.html#method.for_each_mut

src/book/summary.rs

@@ -259,7 +259,7 @@ impl<'a> SummaryParser<'a> {
                         bail!(self.parse_error("Suffix chapters cannot be followed by a list"));
                     }
                 }
-                Some(Event::Start(Tag::Link(href, _))) => {
+                Some(Event::Start(Tag::Link(_type, href, _title))) => {
                     let link = self.parse_link(href.to_string())?;
                     items.push(SummaryItem::Link(link));
                 }
@@ -397,7 +397,7 @@ impl<'a> SummaryParser<'a> {
         loop {
             match self.next_event() {
                 Some(Event::Start(Tag::Paragraph)) => continue,
-                Some(Event::Start(Tag::Link(href, _))) => {
+                Some(Event::Start(Tag::Link(_type, href, _title))) => {
                     let mut link = self.parse_link(href.to_string())?;
 
                     let mut number = parent.clone();
@@ -475,7 +475,7 @@ fn stringify_events(events: Vec<Event<'_>>) -> String {
     events
         .into_iter()
         .filter_map(|t| match t {
-            Event::Text(text) => Some(text.into_owned()),
+            Event::Text(text) | Event::Code(text) => Some(text.into_string()),
             _ => None,
         })
         .collect()
@@ -629,7 +629,7 @@ mod tests {
         let _ = parser.stream.next(); // skip past start of paragraph
 
         let href = match parser.stream.next() {
-            Some(Event::Start(Tag::Link(href, _))) => href.to_string(),
+            Some(Event::Start(Tag::Link(_type, href, _title))) => href.to_string(),
             other => panic!("Unreachable, {:?}", other),
         };
 

src/renderer/html_handlebars/helpers/toc.rs

@@ -4,7 +4,7 @@ use std::path::Path;
 use crate::utils;
 
 use handlebars::{Context, Handlebars, Helper, HelperDef, Output, RenderContext, RenderError};
-use pulldown_cmark::{html, Event, Parser, Tag};
+use pulldown_cmark::{html, Event, Parser};
 
 // Handlebars helper to construct TOC
 #[derive(Clone, Copy)]
@@ -117,10 +117,7 @@ impl HelperDef for RenderToc {
 
                 // filter all events that are not inline code blocks
                 let parser = Parser::new(name).filter(|event| match *event {
-                    Event::Start(Tag::Code)
-                    | Event::End(Tag::Code)
-                    | Event::InlineHtml(_)
-                    | Event::Text(_) => true,
+                    Event::Code(_) | Event::InlineHtml(_) | Event::Text(_) => true,
                     _ => false,
                 });
 

src/renderer/html_handlebars/search.rs

@@ -82,15 +82,16 @@ fn render_item(
     let anchor_base = utils::fs::normalize_path(filepath);
 
     let mut opts = Options::empty();
-    opts.insert(OPTION_ENABLE_TABLES);
-    opts.insert(OPTION_ENABLE_FOOTNOTES);
+    opts.insert(Options::ENABLE_TABLES);
+    opts.insert(Options::ENABLE_FOOTNOTES);
     let p = Parser::new_ext(&chapter.content, opts);
 
     let mut in_header = false;
     let max_section_depth = i32::from(search_config.heading_split_level);
     let mut section_id = None;
     let mut heading = String::new();
     let mut body = String::new();
+    let mut html_block = String::new();
     let mut breadcrumbs = chapter.parent_names.clone();
     let mut footnote_numbers = HashMap::new();
 
@@ -124,6 +125,13 @@ fn render_item(
                 let number = footnote_numbers.len() + 1;
                 footnote_numbers.entry(name).or_insert(number);
             }
+            Event::Html(html) => {
+                html_block.push_str(&html);
+            }
+            Event::End(Tag::HtmlBlock) => {
+                body.push_str(&clean_html(&html_block));
+                html_block.clear();
+            }
             Event::Start(_) | Event::End(_) | Event::SoftBreak | Event::HardBreak => {
                 // Insert spaces where HTML output would usually seperate text
                 // to ensure words don't get merged together
@@ -133,21 +141,22 @@ fn render_item(
                     body.push(' ');
                 }
             }
-            Event::Text(text) => {
+            Event::Text(text) | Event::Code(text) => {
                 if in_header {
                     heading.push_str(&text);
                 } else {
                     body.push_str(&text);
                 }
             }
-            Event::Html(html) | Event::InlineHtml(html) => {
+            Event::InlineHtml(html) => {
                 body.push_str(&clean_html(&html));
             }
             Event::FootnoteReference(name) => {
                 let len = footnote_numbers.len() + 1;
                 let number = footnote_numbers.entry(name).or_insert(len);
                 body.push_str(&format!(" [{}] ", number));
             }
+            Event::TaskListMarker(_checked) => {}
         }
     }
 

src/utils/mod.rs

@@ -5,9 +5,7 @@ mod string;
 use crate::errors::Error;
 use regex::Regex;
 
-use pulldown_cmark::{
-    html, Event, Options, Parser, Tag, OPTION_ENABLE_FOOTNOTES, OPTION_ENABLE_TABLES,
-};
+use pulldown_cmark::{html, CowStr, Event, Options, Parser, Tag};
 
 use std::borrow::Cow;
 
@@ -73,7 +71,7 @@ fn adjust_links<'a>(event: Event<'a>, with_base: &str) -> Event<'a> {
         static ref MD_LINK: Regex = Regex::new(r"(?P<link>.*)\.md(?P<anchor>#.*)?").unwrap();
     }
 
-    fn fix<'a>(dest: Cow<'a, str>, base: &str) -> Cow<'a, str> {
+    fn fix<'a>(dest: CowStr<'a>, base: &str) -> CowStr<'a> {
         // Don't modify links with schemes like `https`.
         if !SCHEME_LINK.is_match(&dest) {
             // This is a relative link, adjust it as necessary.
@@ -92,17 +90,17 @@ fn adjust_links<'a>(event: Event<'a>, with_base: &str) -> Event<'a> {
             } else {
                 fixed_link.push_str(&dest);
             };
-            return Cow::from(fixed_link);
+            return CowStr::from(fixed_link);
         }
         dest
     }
 
     match event {
-        Event::Start(Tag::Link(dest, title)) => {
-            Event::Start(Tag::Link(fix(dest, with_base), title))
+        Event::Start(Tag::Link(link_type, dest, title)) => {
+            Event::Start(Tag::Link(link_type, fix(dest, with_base), title))
         }
-        Event::Start(Tag::Image(dest, title)) => {
-            Event::Start(Tag::Image(fix(dest, with_base), title))
+        Event::Start(Tag::Image(link_type, dest, title)) => {
+            Event::Start(Tag::Image(link_type, fix(dest, with_base), title))
         }
         _ => event,
     }
@@ -117,8 +115,8 @@ pub fn render_markdown_with_base(text: &str, curly_quotes: bool, base: &str) ->
     let mut s = String::with_capacity(text.len() * 3 / 2);
 
     let mut opts = Options::empty();
-    opts.insert(OPTION_ENABLE_TABLES);
-    opts.insert(OPTION_ENABLE_FOOTNOTES);
+    opts.insert(Options::ENABLE_TABLES);
+    opts.insert(Options::ENABLE_FOOTNOTES);
 
     let p = Parser::new_ext(text, opts);
     let mut converter = EventQuoteConverter::new(curly_quotes);
@@ -150,16 +148,16 @@ impl EventQuoteConverter {
         }
 
         match event {
-            Event::Start(Tag::CodeBlock(_)) | Event::Start(Tag::Code) => {
+            Event::Start(Tag::CodeBlock(_)) => {
                 self.convert_text = false;
                 event
             }
-            Event::End(Tag::CodeBlock(_)) | Event::End(Tag::Code) => {
+            Event::End(Tag::CodeBlock(_)) => {
                 self.convert_text = true;
                 event
             }
             Event::Text(ref text) if self.convert_text => {
-                Event::Text(Cow::from(convert_quotes_to_curly(text)))
+                Event::Text(CowStr::from(convert_quotes_to_curly(text)))
             }
             _ => event,
         }
@@ -171,7 +169,7 @@ fn clean_codeblock_headers(event: Event<'_>) -> Event<'_> {
         Event::Start(Tag::CodeBlock(ref info)) => {
             let info: String = info.chars().filter(|ch| !ch.is_whitespace()).collect();
 
-            Event::Start(Tag::CodeBlock(Cow::from(info)))
+            Event::Start(Tag::CodeBlock(CowStr::from(info)))
         }
         _ => event,
     }