From 61a24e824f5dc290529246fa16efa11a57c4a1c8 Mon Sep 17 00:00:00 2001 From: Mike Innes Date: Sun, 5 Oct 2014 19:02:03 +0100 Subject: [PATCH 01/15] add markdown.jl --- base/markdown/Common/Common.jl | 5 + base/markdown/Common/block.jl | 166 +++++++++++++++++++ base/markdown/Common/inline.jl | 97 +++++++++++ base/markdown/GitHub/GitHub.jl | 39 +++++ base/markdown/Julia/Julia.jl | 11 ++ base/markdown/Julia/interp.jl | 50 ++++++ base/markdown/Markdown.jl | 61 +++++++ base/markdown/parse/config.jl | 95 +++++++++++ base/markdown/parse/parse.jl | 84 ++++++++++ base/markdown/parse/util.jl | 171 ++++++++++++++++++++ base/markdown/render/html.jl | 113 +++++++++++++ base/markdown/render/latex.jl | 112 +++++++++++++ base/markdown/render/plain.jl | 64 ++++++++ base/markdown/render/rich.jl | 28 ++++ base/markdown/render/terminal/formatting.jl | 96 +++++++++++ base/markdown/render/terminal/render.jl | 103 ++++++++++++ base/sysimg.jl | 1 + 17 files changed, 1296 insertions(+) create mode 100644 base/markdown/Common/Common.jl create mode 100644 base/markdown/Common/block.jl create mode 100644 base/markdown/Common/inline.jl create mode 100644 base/markdown/GitHub/GitHub.jl create mode 100644 base/markdown/Julia/Julia.jl create mode 100644 base/markdown/Julia/interp.jl create mode 100644 base/markdown/Markdown.jl create mode 100644 base/markdown/parse/config.jl create mode 100644 base/markdown/parse/parse.jl create mode 100644 base/markdown/parse/util.jl create mode 100644 base/markdown/render/html.jl create mode 100644 base/markdown/render/latex.jl create mode 100644 base/markdown/render/plain.jl create mode 100644 base/markdown/render/rich.jl create mode 100644 base/markdown/render/terminal/formatting.jl create mode 100644 base/markdown/render/terminal/render.jl diff --git a/base/markdown/Common/Common.jl b/base/markdown/Common/Common.jl new file mode 100644 index 0000000000000..68ab0cb265596 --- /dev/null +++ b/base/markdown/Common/Common.jl @@ -0,0 +1,5 @@ +include("block.jl") +include("inline.jl") + +@flavour common [list, indentcode, blockquote, hashheader, paragraph, + escapes, en_dash, inline_code, asterisk_bold, asterisk_italic, image, link] diff --git a/base/markdown/Common/block.jl b/base/markdown/Common/block.jl new file mode 100644 index 0000000000000..2b225c1e65293 --- /dev/null +++ b/base/markdown/Common/block.jl @@ -0,0 +1,166 @@ +# –––––––––– +# Paragraphs +# –––––––––– + +type Paragraph + content +end + +Paragraph() = Paragraph({}) + +function paragraph(stream::IO, md::MD, config::Config) + buffer = IOBuffer() + p = Paragraph() + push!(md, p) + skipwhitespace(stream) + while !eof(stream) + char = read(stream, Char) + if char == '\n' || char == '\r' + if blankline(stream) || parse(stream, md, config, breaking = true) + break + else + write(buffer, ' ') + end + else + write(buffer, char) + end + end + p.content = parseinline(seek(buffer, 0), config) + return true +end + +# ––––––– +# Headers +# ––––––– + +type Header{level} + text +end + +Header(s, level::Int) = Header{level}(s) +Header(s) = Header(s, 1) + +@breaking true -> +function hashheader(stream::IO, md::MD, config::Config) + startswith(stream, "#") || return false + level = 1 + while startswith(stream, "#") + level += 1 + end + h = readline(stream) |> chomp + h = match(r"\s*(.*)(? +function blockquote(stream::IO, block::MD, config::Config) + withstream(stream) do + buffer = IOBuffer() + while startswith(stream, ">") + startswith(stream, " ") + write(buffer, readline(stream)) + end + md = takebuf_string(buffer) + if !isempty(md) + push!(block, BlockQuote(parse(md, flavour = config).content)) + return true + else + return false + end + end +end + +# ––––– +# Lists +# ––––– + +type List + items::Vector{Any} + ordered::Bool + + List(x::AbstractVector) = new(x) +end + +List(xs...) = List([xs...]) + +const bullets = ["* ", "• ", "+ ", "- "] + +# Todo: ordered lists, inline formatting +function list(stream::IO, block::MD, config::Config) + withstream(stream) do + skipwhitespace(stream) + startswith(stream, bullets) || return false + the_list = List() + buffer = IOBuffer() + fresh_line = false + while !eof(stream) + if fresh_line + skipwhitespace(stream) + if startswith(stream, bullets) + push!(the_list.items, takebuf_string(buffer)) + buffer = IOBuffer() + else + write(buffer, ' ') + end + fresh_line = false + else + c = read(stream, Char) + if c == '\n' + eof(stream) && break + next = peek(stream) + if next == '\n' + break + else + fresh_line = true + end + else + write(buffer, c) + end + end + end + push!(the_list.items, takebuf_string(buffer)) + push!(block, the_list) + return true + end +end diff --git a/base/markdown/Common/inline.jl b/base/markdown/Common/inline.jl new file mode 100644 index 0000000000000..199257146e273 --- /dev/null +++ b/base/markdown/Common/inline.jl @@ -0,0 +1,97 @@ +# –––––––– +# Emphasis +# –––––––– + +type Italic + text +end + +@trigger '*' -> +function asterisk_italic(stream::IO) + result = parse_inline_wrapper(stream, "*") + return result == nothing ? nothing : Italic(result) +end + +type Bold + text +end + +@trigger '*' -> +function asterisk_bold(stream::IO) + result = parse_inline_wrapper(stream, "**") + return result == nothing ? nothing : Bold(result) +end + +# –––– +# Code +# –––– + +@trigger '`' -> +function inline_code(stream::IO) + result = parse_inline_wrapper(stream, "`") + return result == nothing ? nothing : Code(result) +end + +# –––––––––––––– +# Images & Links +# –––––––––––––– + +type Image + url::UTF8String + alt::UTF8String +end + +@trigger '!' -> +function image(stream::IO) + withstream(stream) do + startswith(stream, "![") || return + alt = readuntil(stream, ']', match = '[') + alt ≡ nothing && return + skipwhitespace(stream) + startswith(stream, '(') || return + url = readuntil(stream, ')', match = '(') + url ≡ nothing && return + return Image(url, alt) + end +end + +type Link + text + url::UTF8String +end + +@trigger '[' -> +function link(stream::IO) + withstream(stream) do + startswith(stream, '[') || return + text = readuntil(stream, ']', match = '[') + text ≡ nothing && return + skipwhitespace(stream) + startswith(stream, '(') || return + url = readuntil(stream, ')', match = '(') + url ≡ nothing && return + return Link(parseinline(text), url) + end +end + +# ––––––––––– +# Punctuation +# ––––––––––– + +@trigger '-' -> +function en_dash(stream::IO) + if startswith(stream, "--") + return "–" + end +end + +const escape_chars = "\\`*_#+-.!{[(" + +@trigger '\\' -> +function escapes(stream::IO) + withstream(stream) do + if startswith(stream, "\\") && !eof(stream) && (c = read(stream, Char)) in escape_chars + return string(c) + end + end +end diff --git a/base/markdown/GitHub/GitHub.jl b/base/markdown/GitHub/GitHub.jl new file mode 100644 index 0000000000000..c60a9d68502dd --- /dev/null +++ b/base/markdown/GitHub/GitHub.jl @@ -0,0 +1,39 @@ +@breaking true -> +function fencedcode(stream::IO, block::MD, config::Config) + startswith(stream, "```", padding = true) || return false + readline(stream) + buffer = IOBuffer() + while !eof(stream) + startswith(stream, "```") && break + write(buffer, readline(stream)) + end + push!(block, Code(takebuf_string(buffer) |> chomp)) + return true +end + +function github_paragraph(stream::IO, md::MD, config::Config) + skipwhitespace(stream) + buffer = IOBuffer() + p = Paragraph() + push!(md, p) + while !eof(stream) + char = read(stream, Char) + if char == '\n' + eof(stream) && break + if blankline(stream) || parse(stream, md, config, breaking = true) + break + else + write(buffer, '\n') + end + else + write(buffer, char) + end + end + p.content = parseinline(seek(buffer, 0), config) + return true +end + +# TODO: tables + +@flavour github [list, indentcode, blockquote, fencedcode, hashheader, github_paragraph, + en_dash, inline_code, asterisk_bold, asterisk_italic, image, link] diff --git a/base/markdown/Julia/Julia.jl b/base/markdown/Julia/Julia.jl new file mode 100644 index 0000000000000..dedc15686c7eb --- /dev/null +++ b/base/markdown/Julia/Julia.jl @@ -0,0 +1,11 @@ +""" +This file contains markdown extensions designed to make documenting +Julia easy peasy. + +We start by borrowing GitHub's `fencedcode` extension – more to follow. +""" + +include("interp.jl") + +@flavour julia [blockinterp, hashheader, list, indentcode, fencedcode, blockquote, paragraph, + escapes, interp, en_dash, inline_code, asterisk_bold, asterisk_italic, image, link] diff --git a/base/markdown/Julia/interp.jl b/base/markdown/Julia/interp.jl new file mode 100644 index 0000000000000..735477221ffb8 --- /dev/null +++ b/base/markdown/Julia/interp.jl @@ -0,0 +1,50 @@ +function Base.parse(stream::IOBuffer; greedy::Bool = true, raise::Bool = true) + pos = position(stream) + ex, Δ = Base.parse(readall(stream), 1, greedy = greedy, raise = raise) + seek(stream, pos + Δ - 1) + return ex +end + +function interpinner(stream::IO, greedy = false) + startswith(stream, '$') || return + (eof(stream) || peek(stream) in whitespace) && return + try + return Base.parse(stream::IOBuffer, greedy = greedy) + catch e + return + end +end + +@trigger '$' -> +function interp(stream::IO) + withstream(stream) do + ex = interpinner(stream) + return ex + end +end + +function blockinterp(stream::IO, md::MD, config::Config) + r=withstream(stream) do + ex = interpinner(stream, true) + if ex ≡ nothing + return false + else + push!(md, ex) + return true + end + end + return r +end + +toexpr(x) = x + +toexpr(xs::Vector{Any}) = Expr(:cell1d, map(toexpr, xs)...) + +function deftoexpr(T) + @eval function toexpr(md::$T) + Expr(:call, $T, $(map(x->:(toexpr(md.$x)), names(T))...)) + end +end + +map(deftoexpr, [MD, Paragraph, Header, + Link]) diff --git a/base/markdown/Markdown.jl b/base/markdown/Markdown.jl new file mode 100644 index 0000000000000..5f30154f2f5f9 --- /dev/null +++ b/base/markdown/Markdown.jl @@ -0,0 +1,61 @@ +module Markdown + +include("parse/config.jl") +include("parse/util.jl") +include("parse/parse.jl") + +include("Common/Common.jl") +include("GitHub/GitHub.jl") +include("Julia/Julia.jl") + +include("render/plain.jl") +include("render/html.jl") +# include("render/latex.jl") + +include("render/terminal/render.jl") + +export readme, license, @md_str, @md_mstr, @doc_str, @doc_mstr + +parse(markdown::String; flavour = julia) = parse(IOBuffer(markdown), flavour = flavour) +parse_file(file::String; flavour = julia) = parse(readall(file), flavour = flavour) + +readme(pkg::String; flavour = github) = parse_file(Pkg.dir(pkg, "README.md"), flavour = flavour) +readme(pkg::Module; flavour = github) = readme(string(pkg), flavour = flavour) + +license(pkg::String; flavour = github) = parse_file(Pkg.dir(pkg, "LICENSE.md"), flavour = flavour) +license(pkg::Module; flavour = github) = license(string(pkg), flavour = flavour) + +function mdexpr(s, flavour = :julia) + md = parse(s, flavour = symbol(flavour)) + esc(toexpr(md)) +end + +function docexpr(s, flavour = :julia) + quote + let md = $(mdexpr(s, flavour)) + md.meta[:path] = @__FILE__ + md.meta[:module] = current_module() + md + end + end +end + +macro md_str(s, t...) + mdexpr(s, t...) +end + +macro md_mstr(s, t...) + s = Base.triplequoted(s) + mdexpr(s, t...) +end + +macro doc_str(s, t...) + docexpr(s, t...) +end + +macro doc_mstr(s, t...) + s = Base.triplequoted(s) + docexpr(s, t...) +end + +end diff --git a/base/markdown/parse/config.jl b/base/markdown/parse/config.jl new file mode 100644 index 0000000000000..552d91e1d3101 --- /dev/null +++ b/base/markdown/parse/config.jl @@ -0,0 +1,95 @@ +typealias InnerConfig Dict{Char, Vector{Function}} + +type Config + breaking::Vector{Function} + regular::Vector{Function} + inner::InnerConfig +end + +Config() = Config(Function[], Function[], InnerConfig()) + +const META = Dict{Function, Dict{Symbol, Any}}() + +getset(coll, key, default) = coll[key] = get(coll, key, default) + +meta(f) = getset(META, f, Dict{Symbol, Any}()) + +breaking!(f) = meta(f)[:breaking] = true +breaking(f) = get(meta(f), :breaking, false) + +triggers!(f, ts) = meta(f)[:triggers] = Set{Char}(ts) +triggers(f) = get(meta(f), :triggers, Set{Char}()) + +# Macros + +isexpr(x::Expr, ts...) = x.head in ts +isexpr{T}(x::T, ts...) = T in ts + +macro breaking (ex) + isexpr(ex, :->) || error("invalid @breaking form, use ->") + b, def = ex.args + if b + quote + f = $(esc(def)) + breaking!(f) + f + end + else + esc(def) + end +end + +macro trigger (ex) + isexpr(ex, :->) || error("invalid @triggers form, use ->") + ts, def = ex.args + quote + f = $(esc(def)) + triggers!(f, $ts) + f + end +end + +# Construction + +function config(parsers::Function...) + c = Config() + for parser in parsers + ts = triggers(parser) + if breaking(parser) + push!(c.breaking, parser) + elseif !isempty(ts) + for t in ts + push!(getset(c.inner, t, Function[]), parser) + end + else + push!(c.regular, parser) + end + end + return c +end + +# Flavour definitions + +const flavours = Dict{Symbol, Config}() + +macro flavour (name, features) + quote + const $(esc(name)) = config($(map(esc,features.args)...)) + flavours[$(Expr(:quote, name))] = $(esc(name)) + end +end + +# Dynamic scoping of current config + +_config_ = nothing + +function withconfig(f, config) + global _config_ + old = _config_ + _config_ = config + try + f() + finally + _config_ = old + end +end diff --git a/base/markdown/parse/parse.jl b/base/markdown/parse/parse.jl new file mode 100644 index 0000000000000..f6354590c2e8a --- /dev/null +++ b/base/markdown/parse/parse.jl @@ -0,0 +1,84 @@ +type MD + content::Vector{Any} + meta::Dict{Any, Any} + + MD(content::AbstractVector, meta::Dict = Dict()) = + new(content, meta) +end + +MD(xs...) = MD([xs...]) + +# Forward some array methods + +Base.push!(md::MD, x) = push!(md.content, x) +Base.getindex(md::MD, args...) = md.content[args...] +Base.setindex!(md::MD, args...) = setindex!(md.content, args...) +Base.endof(md::MD) = endof(md.content) +Base.length(md::MD) = length(md.content) +Base.isempty(md::MD) = isempty(md.content) + +# Parser functions: +#  md – should be modified appropriately +#  return – basically, true if parse was successful +#  false uses the next parser in the queue, true +#  goes back to the beginning +#  +# Inner parsers: +#  return – element to use or nothing + +# Inner parsing + +function innerparse(stream::IO, parsers::Vector{Function}) + for parser in parsers + inner = parser(stream) + inner ≡ nothing || return inner + end +end + +innerparse(stream::IO, config::Config) = + innerparse(stream, config.inner.parsers) + +function parseinline(stream::IO, config::Config) + content = {} + buffer = IOBuffer() + while !eof(stream) + char = peek(stream) + if haskey(config.inner, char) && + (inner = innerparse(stream, config.inner[char])) != nothing + c = takebuf_string(buffer) + !isempty(c) && push!(content, c) + buffer = IOBuffer() + push!(content, inner) + else + write(buffer, read(stream, Char)) + end + end + c = takebuf_string(buffer) + !isempty(c) && push!(content, c) + return content +end + +parseinline(s::String, c::Config) = + parseinline(IOBuffer(s), c) + +parseinline(s) = parseinline(s, _config_) + +# Block parsing + +function parse(stream::IO, block::MD, config::Config; breaking = false) + skipblank(stream) + eof(stream) && return false + for parser in (breaking ? config.breaking : [config.breaking, config.regular]) + parser(stream, block, config) && return true + end + return false +end + +function parse(stream::IO; flavour = julia) + isa(flavour, Symbol) && (flavour = flavours[flavour]) + markdown = MD() + withconfig(flavour) do + while parse(stream, markdown, flavour) end + end + return markdown +end diff --git a/base/markdown/parse/util.jl b/base/markdown/parse/util.jl new file mode 100644 index 0000000000000..0962bcee7ba41 --- /dev/null +++ b/base/markdown/parse/util.jl @@ -0,0 +1,171 @@ +import Base: peek + +macro dotimes(n, body) + quote + for i = 1:$(esc(n)) + $(esc(body)) + end + end +end + +const whitespace = " \t\r" + +""" +Skip any leading whitespace. Returns io. +""" +function skipwhitespace(io::IO; newlines = true) + while !eof(io) && (peek(io) in whitespace || (newlines && peek(io) == '\n')) + read(io, Char) + end + return io +end + +""" +Skip any leading blank lines. Returns the number skipped. +""" +function skipblank(io::IO) + start = position(io) + i = 0 + while !eof(io) + c = read(io, Char) + c == '\n' && (start = position(io); i+=1; continue) + c in whitespace || break + end + seek(io, start) + return i +end + +""" +Returns true if the line contains only (and +at least one of) the characters given. +""" +function linecontains(io::IO, chars; allow_whitespace = true, + eat = true, + allowempty = false) + start = position(io) + l = readline(io) |> chomp + length(l) == 0 && return allowempty + + result = false + for c in l + c in whitespace && (allow_whitespace ? continue : (result = false; break)) + c in chars && (result = true; continue) + result = false; break + end + !(result && eat) && seek(io, start) + return result +end + +blankline(io::IO; eat = true) = + linecontains(io, "", + allow_whitespace = true, + allowempty = true, + eat = eat) + +""" +Test if the stream starts with the given string. +`eat` specifies whether to advance on success (true by default). +`padding` specifies whether leading whitespace should be ignored. +""" +function startswith(stream::IO, s::String; eat = true, padding = false, newlines = true) + start = position(stream) + padding && skipwhitespace(stream, newlines = newlines) + result = true + for char in s + !eof(stream) && read(stream, Char) == char || + (result = false; break) + end + !(result && eat) && seek(stream, start) + return result +end + +function startswith(stream::IO, c::Char; eat = true) + if peek(stream) == c + eat && read(stream, Char) + return true + else + return false + end +end + +function startswith{T<:String}(stream::IO, ss::Vector{T}; kws...) + any(s->startswith(stream, s; kws...), ss) +end + +function startswith(stream::IO, r::Regex; eat = true, padding = false) + @assert beginswith(r.pattern, "^") + start = position(stream) + padding && skipwhitespace(stream) + line = chomp(readline(stream)) + seek(stream, start) + m = match(r, line) + m == nothing && return "" + eat && @dotimes length(m.match) read(stream, Char) + return m.match +end + +""" +Executes the block of code, and if the return value is `nothing`, +returns the stream to its initial position. +""" +function withstream(f, stream) + pos = position(stream) + result = f() + (result ≡ nothing || result ≡ false) && seek(stream, pos) + return result +end + +""" +Read the stream until startswith(stream, delim) +The delimiter is consumed but not included. +Returns nothing and resets the stream if delim is +not found. +""" +function readuntil(stream::IO, delimiter; newlines = false, match = nothing) + withstream(stream) do + buffer = IOBuffer() + count = 0 + while !eof(stream) + if startswith(stream, delimiter) + if count == 0 + return takebuf_string(buffer) + else + count -= 1 + write(buffer, delimiter) + end + end + char = read(stream, Char) + char == match && (count += 1) + !newlines && char == '\n' && break + write(buffer, char) + end + end +end + +""" +Parse a symmetrical delimiter which wraps words. +i.e. `*word word*` but not `*word * word` +""" +function parse_inline_wrapper(stream::IO, delimiter::String, no_newlines = true) + withstream(stream) do + startswith(stream, delimiter) || return nothing + + buffer = IOBuffer() + while !eof(stream) + char = read(stream, Char) + no_newlines && char == '\n' && break + if !(char in whitespace) && startswith(stream, delimiter) + write(buffer, char) + return takebuf_string(buffer) + end + write(buffer, char) + end + end +end + +function showrest(io::IO) + start = position(io) + show(readall(io)) + println() + seek(io, start) +end diff --git a/base/markdown/render/html.jl b/base/markdown/render/html.jl new file mode 100644 index 0000000000000..fcf26b4f8c3ee --- /dev/null +++ b/base/markdown/render/html.jl @@ -0,0 +1,113 @@ +include("rich.jl") + +# Utils + +function withtag(f, io, tag) + print(io, "<$tag>") + f() + print(io, "") +end + +# Block elements + +function html(io::IO, content::Vector) + for md in content + html(io, md) + println(io) + end +end + +html(io::IO, md::MD) = html(io, md.content) + +function html{l}(io::IO, header::Header{l}) + withtag(io, "h$l") do + htmlinline(io, header.text) + end +end + +function html(io::IO, code::Code) + withtag(io, :pre) do + withtag(io, :code) do + print(io, code.code) + end + end +end + +function html(io::IO, md::Paragraph) + withtag(io, :p) do + htmlinline(io, md.content) + end +end + +function html(io::IO, md::BlockQuote) + withtag(io, :blockquote) do + html(io, block.content) + end +end + +function html(io::IO, md::List) + withtag(io, :ul) do + for item in md.items + withtag(io, :li) do + htmlinline(io, item) + println(io) + end + end + end +end + +html(io::IO, x) = tohtml(io, x) + +# Inline elements + +function htmlinline(io::IO, content::Vector) + for x in content + htmlinline(io, x) + end +end + +function htmlinline(io::IO, code::Code) + withtag(io, :code) do + print(io, code.code) + end +end + +function htmlinline(io::IO, md::String) + print(io, md) +end + +function htmlinline(io::IO, md::Bold) + withtag(io, :strong) do + print(io, md.text) + end +end + +function htmlinline(io::IO, md::Italic) + withtag(io, :em) do + print(io, md.text) + end +end + +function htmlinline(io::IO, md::Image) + print(io, """$(md.alt)""") +end + +function htmlinline(io::IO, link::Link) + print(io, """""") + htmlinline(io, link.text) + print(io,"""""") +end + +htmlinline(io::IO, x) = tohtml(io, x) + +# API + +export html + +html(md) = sprint(html, md) + +function Base.writemime(io::IO, ::MIME"text/html", md::MD) + println(io, """
""") + html(io, md) + println(io, """
""") +end diff --git a/base/markdown/render/latex.jl b/base/markdown/render/latex.jl new file mode 100644 index 0000000000000..e9034efb76ad2 --- /dev/null +++ b/base/markdown/render/latex.jl @@ -0,0 +1,112 @@ +import Base.writemime + +export latex + +function wrapblock(f, io, env) + println(io, "\\begin{", env, "}") + f() + println(io, "\\end{", env, "}") +end + +function wrapinline(f, io, cmd) + print(io, "\\", cmd, "{") + f() + print(io, "}") +end + +writemime(io::IO, ::MIME"text/latex", md::Content) = + writemime(io, "text/plain", md) + +function writemime(io::IO, mime::MIME"text/latex", block::Block) + for md in block.content[1:end-1] + writemime(io::IO, mime, md) + println(io) + end + writemime(io::IO, mime, block.content[end]) +end + +function writemime{l}(io::IO, mime::MIME"text/latex", header::Header{l}) + tag = l < 4 ? "sub"^(l-1) * "section" : "sub"^(l-4) * "paragraph" + wrapinline(io, tag) do + print(io, header.text) + end + println(io) +end + +function writemime(io::IO, ::MIME"text/latex", code::BlockCode) + wrapblock(io, "verbatim") do + println(io, code.code) + end +end + +function writemime(io::IO, ::MIME"text/latex", code::InlineCode) + wrapinline(io, "texttt") do + print(io, code.code) + end +end + +function writemime(io::IO, ::MIME"text/latex", md::Paragraph) + for md in md.content + latex_inline(io, md) + end + println(io) +end + +function writemime(io::IO, ::MIME"text/latex", md::BlockQuote) + wrapblock(io, "quote") do + writemime(io, "text/latex", Block(md.content)) + end +end + +function writemime(io::IO, ::MIME"text/latex", md::List) + wrapblock(io, "itemize") do + for item in md.content + print(io, "\\item ") + latex_inline(io, item) + println(io) + end + end +end + +# Inline elements + +function writemime(io::IO, ::MIME"text/latex", md::Plain) + print(io, md.text) +end + +function writemime(io::IO, ::MIME"text/latex", md::Bold) + wrapinline(io, "textbf") do + print(io, md.text) + end +end + +function writemime(io::IO, ::MIME"text/latex", md::Italic) + wrapinline(io, "emph") do + print(io, md.text) + end +end + +function writemime(io::IO, ::MIME"text/latex", md::Image) + wrapblock(io, "figure") do + println(io, "\\centering") + wrapinline(io, "includegraphics") do + print(io, md.url) + end + println(io) + wrapinline(io, "caption") do + print(io, md.alt) + end + println(io) + end +end + +function writemime(io::IO, ::MIME"text/latex", md::Link) + wrapinline(io, "href") do + print(io, md.url) + end + print(io, "{", md.text, "}") +end + +latex_inline(io::IO, el::Content) = writemime(io, "text/latex", el) + +latex(md::Content) = stringmime("text/latex", md) diff --git a/base/markdown/render/plain.jl b/base/markdown/render/plain.jl new file mode 100644 index 0000000000000..eadfa87c8ad68 --- /dev/null +++ b/base/markdown/render/plain.jl @@ -0,0 +1,64 @@ +plain(x) = sprint(plain, x) + +function plain(io::IO, content::Vector) + for md in content[1:end-1] + plain(io, md) + println(io) + end + plain(io, content[end]) +end + +plain(io::IO, md::MD) = plain(io, md.content) + +function plain{l}(io::IO, header::Header{l}) + print(io, "#"^l*" ") + plaininline(io, header.text) + println(io) +end + +function plain(io::IO, code::Code) + println(io, "```", code.language) + println(io, code.code) + println(io, "```") +end + +function plain(io::IO, p::Paragraph) + for md in p.content + plaininline(io, md) + end + println(io) +end + +function plain(io::IO, list::List) + for item in list.items + print(io, " * ") + plaininline(io, item) + println(io) + end +end + +plain(io::IO, x) = tohtml(io, x) + +# Inline elements + +function plaininline(io::IO, md::Vector) + for el in md + plaininline(io, el) + end +end + +plaininline(io::IO, md::Image) = print(io, "![$(md.alt)]($(md.url))") + +plaininline(io::IO, s::String) = print(io, s) + +plaininline(io::IO, md::Bold) = print(io, "**", md.text, "**") + +plaininline(io::IO, md::Italic) = print(io, "*", md.text, "*") + +plaininline(io::IO, md::Code) = print(io, "`", md.code, "`") + +plaininline(io::IO, x) = writemime(io, MIME"text/plain"(), x) + +# writemime + +Base.writemime(io::IO, ::MIME"text/plain", md::MD) = plain(io, md) diff --git a/base/markdown/render/rich.jl b/base/markdown/render/rich.jl new file mode 100644 index 0000000000000..89a70662d850b --- /dev/null +++ b/base/markdown/render/rich.jl @@ -0,0 +1,28 @@ +function tohtml(io::IO, m::MIME"text/html", x) + writemime(io, m, x) +end + +function tohtml(io::IO, m::MIME"text/plain", x) + writemime(io, m, x) +end + +function tohtml(io::IO, m::MIME"image/png", img) + print(io, """") +end + +function tohtml(m::MIME"image/svg+xml", img) + writemime(io, m, img) +end + +# Display infrastructure + +function bestmime(val) + for mime in ("text/html", "image/svg+xml", "image/png", "text/plain") + mimewritable(mime, val) && return MIME(symbol(mime)) + end + error("Cannot render $val to Markdown.") +end + +tohtml(io::IO, x) = tohtml(io, bestmime(x), x) diff --git a/base/markdown/render/terminal/formatting.jl b/base/markdown/render/terminal/formatting.jl new file mode 100644 index 0000000000000..59f2817ed32f2 --- /dev/null +++ b/base/markdown/render/terminal/formatting.jl @@ -0,0 +1,96 @@ +# Styles + +const text_formats = [ + :black => "\e[30m", + :red => "\e[31m", + :green => "\e[32m", + :yellow => "\e[33m", + :blue => "\e[34m", + :magenta => "\e[35m", + :cyan => "\e[36m", + :white => "\e[37m", + :reset => "\e[0m", + :bold => "\e[1m", + :underline => "\e[4m", + :blink => "\e[5m", + :negative => "\e[7m"] + +function with_output_format(f::Function, formats::Vector{Symbol}, io::IO, args...) + Base.have_color && for format in formats + print(io, get(text_formats, format, "")) + end + try f(io, args...) + finally + Base.have_color && print(io, text_formats[:reset]) + end +end + +with_output_format(f::Function, format::Symbol, args...) = + with_output_format(f, [format], args...) + +function print_with_format(format, io::IO, x) + with_output_format(format, io) do io + print(io, x) + end +end + +function println_with_format(format, io::IO, x) + print_with_format(format, io, x) + println(io) +end + +# Wrapping + +function ansi_length(s) + replace(s, r"\e\[[0-9]+m", "") |> length +end + +words(s) = split(s, " ") +lines(s) = split(s, "\n") + +# This could really be more efficient +function wrapped_lines(s::String; width = 80, i = 0) + if ismatch(r"\n", s) + return [map(s->wrapped_lines(s, width = width, i = i), split(s, "\n"))...] + end + ws = words(s) + lines = String[ws[1]] + i += ws[1] |> ansi_length + for word in ws[2:end] + word_length = ansi_length(word) + if i + word_length + 1 > width + i = word_length + push!(lines, word) + else + i += word_length + 1 + lines[end] *= " " * word + end + end + return lines +end + +wrapped_lines(f::Function, args...; width = 80, i = 0) = + wrapped_lines(sprint(f, args...), width = width, i = 0) + +function print_wrapped(io::IO, s...; width = 80, pre = "", i = 0) + lines = wrapped_lines(s..., width = width, i = i) + println(io, lines[1]) + for line in lines[2:end] + println(io, pre, line) + end +end + +print_wrapped(f::Function, io::IO, args...; kws...) = print_wrapped(io, f, args...; kws...) + +function print_centred(io::IO, s...; columns = 80, width = columns) + lines = wrapped_lines(s..., width = width) + for line in lines + print(io, " "^(div(columns-ansi_length(line), 2))) + println(io, line) + end +end + +function centred(s, columns) + pad = div(columns - ansi_length(s), 2) + " "^pad * s +end diff --git a/base/markdown/render/terminal/render.jl b/base/markdown/render/terminal/render.jl new file mode 100644 index 0000000000000..38dfcb0f61776 --- /dev/null +++ b/base/markdown/render/terminal/render.jl @@ -0,0 +1,103 @@ +include("formatting.jl") + +const margin = 2 +cols() = Base.tty_size()[2] + +function term(io::IO, content::Vector, cols) + for md in content[1:end-1] + term(io, md, cols) + println(io) + end + term(io, content[end], cols) +end + +term(io::IO, md::MD, columns = cols()) = term(io, md.content, columns) + +function term(io::IO, md::Paragraph, columns) + print(io, " "^margin) + print_wrapped(io, width = columns-2margin, pre = " "^margin) do io + terminline(io, md.content) + end +end + +function term(io::IO, md::BlockQuote, columns) + s = sprint(io->term(io, Block(md.content), columns - 10)) + for line in split(rstrip(s), "\n") + println(io, " "^margin, "|", line) + end + println(io) +end + +function term(io::IO, md::List, columns) + for point in md.items + print(io, " "^2margin, "• ") + print_wrapped(io, width = columns-(4margin+2), pre = " "^(2margin+2), i = 2margin+2) do io + terminline(io, point) + end + end +end + +function term(io::IO, md::Header{1}, columns) + text = terminline(md.text) + with_output_format(:bold, io) do io + print_centred(io, text, width = columns - 4margin, columns = columns) + end + print_centred(io, "-"*"–"^min(length(text), div(columns, 2))*"-", columns = columns) +end + +function term{l}(io::IO, md::Header{l}, columns) + print(io, "#"^l, " ") + terminline(io, md.text) + println(io) +end + +function term(io::IO, md::Code, columns) + with_output_format(:cyan, io) do io + for line in lines(md.code) + print(io, " "^margin) + println(io, line) + end + end +end + +term(io::IO, x, _) = writemime(io, MIME"text/plain"(), x) + +# Inline Content + +terminline(md) = sprint(terminline, md) + +function terminline(io::IO, content::Vector) + for md in content + terminline(io, md) + end +end + +function terminline(io::IO, md::String) + print_with_format(:normal, io, md) +end + +function terminline(io::IO, md::Bold) + print_with_format(:bold, io, md.text) +end + +function terminline(io::IO, md::Italic) + print_with_format(:underline, io, md.text) +end + +function terminline(io::IO, md::Image) + print(io, "(Image: $(md.alt))") +end + +function terminline(io::IO, md::Link) + terminline(io, md.text) +end + +function terminline(io::IO, code::Code) + print_with_format(:cyan, io, code.code) +end + +terminline(io::IO, x) = writemime(io, MIME"text/plain"(), x) + +# Show in terminal + +Base.display(d::Base.REPL.REPLDisplay, md::MD) = term(Base.REPL.outstream(d.repl), md) diff --git a/base/sysimg.jl b/base/sysimg.jl index 8c4e5e99f5752..7821e94c466a6 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -218,6 +218,7 @@ include("LineEdit.jl") include("REPLCompletions.jl") include("REPL.jl") include("client.jl") +include("markdown/Markdown.jl") # (s)printf macros include("printf.jl") From d145bceba1a5b52e552d41f028b23f4da86b1093 Mon Sep 17 00:00:00 2001 From: Mike Innes Date: Tue, 7 Oct 2014 21:57:40 +0100 Subject: [PATCH 02/15] make "flavour" slightly less readable --- base/markdown/Common/Common.jl | 2 +- base/markdown/Common/block.jl | 2 +- base/markdown/GitHub/GitHub.jl | 2 +- base/markdown/Julia/Julia.jl | 2 +- base/markdown/Markdown.jl | 20 ++++++++++---------- base/markdown/parse/config.jl | 6 +++--- base/markdown/parse/parse.jl | 8 ++++---- 7 files changed, 21 insertions(+), 21 deletions(-) diff --git a/base/markdown/Common/Common.jl b/base/markdown/Common/Common.jl index 68ab0cb265596..b98cb68d97cf7 100644 --- a/base/markdown/Common/Common.jl +++ b/base/markdown/Common/Common.jl @@ -1,5 +1,5 @@ include("block.jl") include("inline.jl") -@flavour common [list, indentcode, blockquote, hashheader, paragraph, +@flavor common [list, indentcode, blockquote, hashheader, paragraph, escapes, en_dash, inline_code, asterisk_bold, asterisk_italic, image, link] diff --git a/base/markdown/Common/block.jl b/base/markdown/Common/block.jl index 2b225c1e65293..f6b80a8f68554 100644 --- a/base/markdown/Common/block.jl +++ b/base/markdown/Common/block.jl @@ -103,7 +103,7 @@ function blockquote(stream::IO, block::MD, config::Config) end md = takebuf_string(buffer) if !isempty(md) - push!(block, BlockQuote(parse(md, flavour = config).content)) + push!(block, BlockQuote(parse(md, flavor = config).content)) return true else return false diff --git a/base/markdown/GitHub/GitHub.jl b/base/markdown/GitHub/GitHub.jl index c60a9d68502dd..74cc9566691f2 100644 --- a/base/markdown/GitHub/GitHub.jl +++ b/base/markdown/GitHub/GitHub.jl @@ -35,5 +35,5 @@ end # TODO: tables -@flavour github [list, indentcode, blockquote, fencedcode, hashheader, github_paragraph, +@flavor github [list, indentcode, blockquote, fencedcode, hashheader, github_paragraph, en_dash, inline_code, asterisk_bold, asterisk_italic, image, link] diff --git a/base/markdown/Julia/Julia.jl b/base/markdown/Julia/Julia.jl index dedc15686c7eb..500420d486579 100644 --- a/base/markdown/Julia/Julia.jl +++ b/base/markdown/Julia/Julia.jl @@ -7,5 +7,5 @@ We start by borrowing GitHub's `fencedcode` extension – more to follow. include("interp.jl") -@flavour julia [blockinterp, hashheader, list, indentcode, fencedcode, blockquote, paragraph, +@flavor julia [blockinterp, hashheader, list, indentcode, fencedcode, blockquote, paragraph, escapes, interp, en_dash, inline_code, asterisk_bold, asterisk_italic, image, link] diff --git a/base/markdown/Markdown.jl b/base/markdown/Markdown.jl index 5f30154f2f5f9..92724cf96fed3 100644 --- a/base/markdown/Markdown.jl +++ b/base/markdown/Markdown.jl @@ -16,23 +16,23 @@ include("render/terminal/render.jl") export readme, license, @md_str, @md_mstr, @doc_str, @doc_mstr -parse(markdown::String; flavour = julia) = parse(IOBuffer(markdown), flavour = flavour) -parse_file(file::String; flavour = julia) = parse(readall(file), flavour = flavour) +parse(markdown::String; flavor = julia) = parse(IOBuffer(markdown), flavor = flavor) +parse_file(file::String; flavor = julia) = parse(readall(file), flavor = flavor) -readme(pkg::String; flavour = github) = parse_file(Pkg.dir(pkg, "README.md"), flavour = flavour) -readme(pkg::Module; flavour = github) = readme(string(pkg), flavour = flavour) +readme(pkg::String; flavor = github) = parse_file(Pkg.dir(pkg, "README.md"), flavor = flavor) +readme(pkg::Module; flavor = github) = readme(string(pkg), flavor = flavor) -license(pkg::String; flavour = github) = parse_file(Pkg.dir(pkg, "LICENSE.md"), flavour = flavour) -license(pkg::Module; flavour = github) = license(string(pkg), flavour = flavour) +license(pkg::String; flavor = github) = parse_file(Pkg.dir(pkg, "LICENSE.md"), flavor = flavor) +license(pkg::Module; flavor = github) = license(string(pkg), flavor = flavor) -function mdexpr(s, flavour = :julia) - md = parse(s, flavour = symbol(flavour)) +function mdexpr(s, flavor = :julia) + md = parse(s, flavor = symbol(flavor)) esc(toexpr(md)) end -function docexpr(s, flavour = :julia) +function docexpr(s, flavor = :julia) quote - let md = $(mdexpr(s, flavour)) + let md = $(mdexpr(s, flavor)) md.meta[:path] = @__FILE__ md.meta[:module] = current_module() md diff --git a/base/markdown/parse/config.jl b/base/markdown/parse/config.jl index 552d91e1d3101..822b0d6013f89 100644 --- a/base/markdown/parse/config.jl +++ b/base/markdown/parse/config.jl @@ -70,12 +70,12 @@ end # Flavour definitions -const flavours = Dict{Symbol, Config}() +const flavors = Dict{Symbol, Config}() -macro flavour (name, features) +macro flavor (name, features) quote const $(esc(name)) = config($(map(esc,features.args)...)) - flavours[$(Expr(:quote, name))] = $(esc(name)) + flavors[$(Expr(:quote, name))] = $(esc(name)) end end diff --git a/base/markdown/parse/parse.jl b/base/markdown/parse/parse.jl index f6354590c2e8a..2605630601ed6 100644 --- a/base/markdown/parse/parse.jl +++ b/base/markdown/parse/parse.jl @@ -74,11 +74,11 @@ function parse(stream::IO, block::MD, config::Config; breaking = false) return false end -function parse(stream::IO; flavour = julia) - isa(flavour, Symbol) && (flavour = flavours[flavour]) +function parse(stream::IO; flavor = julia) + isa(flavor, Symbol) && (flavor = flavors[flavor]) markdown = MD() - withconfig(flavour) do - while parse(stream, markdown, flavour) end + withconfig(flavor) do + while parse(stream, markdown, flavor) end end return markdown end From 8b717f2a08ebdb0f7c75d62622d939bf4771fc8c Mon Sep 17 00:00:00 2001 From: Mike Innes Date: Sun, 9 Nov 2014 14:57:01 +0000 Subject: [PATCH 03/15] update to track Markdown.jl/julia-breaking branch --- base/markdown/Common/block.jl | 4 ++-- base/markdown/Markdown.jl | 2 ++ base/markdown/parse/parse.jl | 2 +- base/markdown/render/terminal/formatting.jl | 4 ++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/base/markdown/Common/block.jl b/base/markdown/Common/block.jl index f6b80a8f68554..ae1e982705e66 100644 --- a/base/markdown/Common/block.jl +++ b/base/markdown/Common/block.jl @@ -6,7 +6,7 @@ type Paragraph content end -Paragraph() = Paragraph({}) +Paragraph() = Paragraph([]) function paragraph(stream::IO, md::MD, config::Config) buffer = IOBuffer() @@ -90,7 +90,7 @@ type BlockQuote content end -BlockQuote() = BlockQuote({}) +BlockQuote() = BlockQuote([]) # TODO: Laziness @breaking true -> diff --git a/base/markdown/Markdown.jl b/base/markdown/Markdown.jl index 92724cf96fed3..2fcb24d08d655 100644 --- a/base/markdown/Markdown.jl +++ b/base/markdown/Markdown.jl @@ -1,5 +1,7 @@ module Markdown +typealias String AbstractString + include("parse/config.jl") include("parse/util.jl") include("parse/parse.jl") diff --git a/base/markdown/parse/parse.jl b/base/markdown/parse/parse.jl index 2605630601ed6..a13571b2cef9a 100644 --- a/base/markdown/parse/parse.jl +++ b/base/markdown/parse/parse.jl @@ -39,7 +39,7 @@ innerparse(stream::IO, config::Config) = innerparse(stream, config.inner.parsers) function parseinline(stream::IO, config::Config) - content = {} + content = [] buffer = IOBuffer() while !eof(stream) char = peek(stream) diff --git a/base/markdown/render/terminal/formatting.jl b/base/markdown/render/terminal/formatting.jl index 59f2817ed32f2..323b99297a973 100644 --- a/base/markdown/render/terminal/formatting.jl +++ b/base/markdown/render/terminal/formatting.jl @@ -1,6 +1,6 @@ # Styles -const text_formats = [ +const text_formats = Dict( :black => "\e[30m", :red => "\e[31m", :green => "\e[32m", @@ -13,7 +13,7 @@ const text_formats = [ :bold => "\e[1m", :underline => "\e[4m", :blink => "\e[5m", - :negative => "\e[7m"] + :negative => "\e[7m") function with_output_format(f::Function, formats::Vector{Symbol}, io::IO, args...) Base.have_color && for format in formats From 9ccfa0e2f17490ef7a244ad9a22b7abcfa55ac87 Mon Sep 17 00:00:00 2001 From: Mike Innes Date: Sun, 5 Oct 2014 20:00:56 +0100 Subject: [PATCH 04/15] implement Docs module --- base/docs.jl | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++ base/sysimg.jl | 4 ++++ 2 files changed, 56 insertions(+) create mode 100644 base/docs.jl diff --git a/base/docs.jl b/base/docs.jl new file mode 100644 index 0000000000000..32148fe99186e --- /dev/null +++ b/base/docs.jl @@ -0,0 +1,52 @@ +module Docs + +import Base.Markdown: @doc_str, @doc_mstr + +export doc, @doc + +# Basic API + +const META = Dict() + +function doc(obj, meta) + META[obj] = meta +end + +doc(obj) = get(META, obj, nothing) + +doc(obj::Union(Symbol, String)) = get(META, current_module().(symbol(obj)), nothing) + +# Macros + +isexpr(x::Expr, ts...) = x.head in ts +isexpr{T}(x::T, ts...) = T in ts + +function mdify(ex) + if isexpr(ex, String) + :(@doc_str $(esc(ex))) + elseif isexpr(ex, :macrocall) && ex.args[1] == symbol("@mstr") + :(@doc_mstr $(esc(ex.args[2]))) + else + esc(ex) + end +end + +function getdoc(ex) + if isexpr(ex, :macrocall) + :(doc($(esc(ex.args[1])))) + else + :(doc($(esc(ex)))) + end +end + +macro doc (ex) + isexpr(ex, :(->)) || return getdoc(ex) + meta, def = ex.args + quote + f = $(esc(def)) + doc(f, $(mdify(meta))) + f + end +end + +end diff --git a/base/sysimg.jl b/base/sysimg.jl index 7821e94c466a6..2bc2a86b181b0 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -218,7 +218,11 @@ include("LineEdit.jl") include("REPLCompletions.jl") include("REPL.jl") include("client.jl") + +# Documentation + include("markdown/Markdown.jl") +include("docs.jl") # (s)printf macros include("printf.jl") From 12613a84f620497d051cdff968055c6d6e35417b Mon Sep 17 00:00:00 2001 From: Mike Innes Date: Mon, 6 Oct 2014 14:43:32 +0100 Subject: [PATCH 05/15] support for macros --- base/docs.jl | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/base/docs.jl b/base/docs.jl index 32148fe99186e..2335f5ee1ecc9 100644 --- a/base/docs.jl +++ b/base/docs.jl @@ -21,8 +21,15 @@ doc(obj::Union(Symbol, String)) = get(META, current_module().(symbol(obj)), noth isexpr(x::Expr, ts...) = x.head in ts isexpr{T}(x::T, ts...) = T in ts +function unblock (ex) + isexpr(ex, :block) || return ex + exs = filter(ex->!isexpr(ex, :line), ex.args) + length(exs) > 1 && return ex + return exs[1] +end + function mdify(ex) - if isexpr(ex, String) + if isa(ex, String) :(@doc_str $(esc(ex))) elseif isexpr(ex, :macrocall) && ex.args[1] == symbol("@mstr") :(@doc_mstr $(esc(ex.args[2]))) @@ -39,9 +46,16 @@ function getdoc(ex) end end -macro doc (ex) - isexpr(ex, :(->)) || return getdoc(ex) - meta, def = ex.args +function macrodoc(meta, def) + name = esc(symbol(string("@", unblock(def).args[1].args[1]))) + quote + $(esc(def)) + doc($name, $(mdify(meta))) + nothing + end +end + +function objdoc(meta, def) quote f = $(esc(def)) doc(f, $(mdify(meta))) @@ -49,4 +63,11 @@ macro doc (ex) end end +macro doc (ex) + isexpr(ex, :(->)) || return getdoc(ex) + meta, def = ex.args + isexpr(unblock(def), :macro) && return macrodoc(meta, def) + return objdoc(meta, def) +end + end From b642bc7aff75edb2da1e2ccb6f7d91124418fcb0 Mon Sep 17 00:00:00 2001 From: Mike Innes Date: Mon, 6 Oct 2014 16:13:18 +0100 Subject: [PATCH 06/15] support for methods --- base/docs.jl | 71 +++++++++++++++++++++++++++++++++++++-- base/markdown/Markdown.jl | 12 +++++++ 2 files changed, 81 insertions(+), 2 deletions(-) diff --git a/base/docs.jl b/base/docs.jl index 2335f5ee1ecc9..bffc93a733ede 100644 --- a/base/docs.jl +++ b/base/docs.jl @@ -16,12 +16,70 @@ doc(obj) = get(META, obj, nothing) doc(obj::Union(Symbol, String)) = get(META, current_module().(symbol(obj)), nothing) +# Function / Method support + +function newmethod(defs) + keylen = -1 + key = nothing + for def in defs + length(def.sig) > keylen && (keylen = length(def.sig); key = def) + end + return key +end + +function newmethod(funcs, f) + applicable = Method[] + for def in methods(f) + (!haskey(funcs, def) || funcs[def] != def.func) && push!(applicable, def) + end + return newmethod(applicable) +end + +function trackmethod (def) + name = unblock(def).args[1].args[1] + f = esc(name) + quote + if isdefined($(Expr(:quote, name))) + funcs = [def => def.func for def in methods($f)] + $(esc(def)) + $f, newmethod(funcs, $f) + else + $(esc(def)) + $f, newmethod(methods($f)) + end + end +end + +type FuncDoc + order::Vector{Method} + meta::Dict{Method, Any} + source::Dict{Method, Any} +end + +FuncDoc() = FuncDoc(Method[], Dict(), Dict()) + +getset(coll, key, default) = coll[key] = get(coll, key, default) + +function doc(f::Function, m::Method, meta) + fd = getset(META, f, FuncDoc()) + !haskey(fd.meta, m) && push!(fd.order, m) + fd.meta[m] = meta +end + +function doc(f::Function) + fd = get(META, f, nothing) + fd == nothing && return + catdoc([fd.meta[m] for m in fd.order]...) +end + +catdoc(xs...) = [xs...] + # Macros isexpr(x::Expr, ts...) = x.head in ts isexpr{T}(x::T, ts...) = T in ts -function unblock (ex) +function unblock(ex) isexpr(ex, :block) || return ex exs = filter(ex->!isexpr(ex, :line), ex.args) length(exs) > 1 && return ex @@ -55,6 +113,14 @@ function macrodoc(meta, def) end end +function funcdoc(meta, def) + quote + f, m = $(trackmethod(def)) + doc(f, m, $(mdify(meta)), $(esc(Expr(:quote, def)))) + f + end +end + function objdoc(meta, def) quote f = $(esc(def)) @@ -64,9 +130,10 @@ function objdoc(meta, def) end macro doc (ex) - isexpr(ex, :(->)) || return getdoc(ex) + isexpr(ex, :->) || return getdoc(ex) meta, def = ex.args isexpr(unblock(def), :macro) && return macrodoc(meta, def) + isexpr(unblock(def), :function, :(=)) && return funcdoc(meta, def) return objdoc(meta, def) end diff --git a/base/markdown/Markdown.jl b/base/markdown/Markdown.jl index 2fcb24d08d655..56339ef88f5e7 100644 --- a/base/markdown/Markdown.jl +++ b/base/markdown/Markdown.jl @@ -60,4 +60,16 @@ macro doc_mstr(s, t...) docexpr(s, t...) end +function writemime(io::IO, m, md::Vector{MD}) + for md in md + writemime(io, m, md) + end +end + +function Base.display(d::Base.REPL.REPLDisplay, md::Vector{MD}) + for md in md + display(d, md) + end +end + end From 3fd4bfb6b8ed36596be9cfd7a03a2aac83731be6 Mon Sep 17 00:00:00 2001 From: Mike Innes Date: Thu, 9 Oct 2014 16:24:49 +0100 Subject: [PATCH 07/15] implement Text and HTML types --- base/docs.jl | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/base/docs.jl b/base/docs.jl index bffc93a733ede..ace8f3a98457a 100644 --- a/base/docs.jl +++ b/base/docs.jl @@ -72,6 +72,7 @@ function doc(f::Function) catdoc([fd.meta[m] for m in fd.order]...) end +catdoc() = nothing catdoc(xs...) = [xs...] # Macros @@ -137,4 +138,69 @@ macro doc (ex) return objdoc(meta, def) end +# Text / HTML objects + +import Base: print, writemime + +export HTML, @html_str, @html_mstr + +export HTML, Text + +type HTML{T} + content::T +end + +function HTML(xs...) + HTML() do io + for x in xs + writemime(io, MIME"text/html"(), x) + end + end +end + +writemime(io::IO, ::MIME"text/html", h::HTML) = print(io, h.content) +writemime(io::IO, ::MIME"text/html", h::HTML{Function}) = h.content(io) + +macro html_str (s) + :(HTML($s)) +end + +macro html_mstr (s) + :(HTML($(Base.triplequoted(s)))) +end + +function catdoc(xs::HTML...) + HTML() do io + for x in xs + writemime(io, MIME"text/html"(), x) + end + end +end + +export Text, @text_str, @text_mstr + +type Text{T} + content::T +end + +print(io::IO, t::Text) = print(io, t.content) +print(io::IO, t::Text{Function}) = t.content(io) +writemime(io::IO, ::MIME"text/plain", t::Text) = print(io, t) + +macro text_str (s) + :(Text($s)) +end + +macro text_mstr (s) + :(Text($(Base.triplequoted(s)))) +end + +function catdoc(xs::Text...) + Text() do io + for x in xs + writemime(io, MIME"text/plain"(), x) + end + end +end + end From ee423a80da12d5a3e0d591a18caf9a95289fe09c Mon Sep 17 00:00:00 2001 From: Mike Innes Date: Thu, 9 Oct 2014 16:53:26 +0100 Subject: [PATCH 08/15] improve error messages --- base/docs.jl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/base/docs.jl b/base/docs.jl index ace8f3a98457a..1db9a9b0817e7 100644 --- a/base/docs.jl +++ b/base/docs.jl @@ -39,7 +39,7 @@ function trackmethod (def) name = unblock(def).args[1].args[1] f = esc(name) quote - if isdefined($(Expr(:quote, name))) + if isdefined($(Expr(:quote, name))) && isgeneric($f) funcs = [def => def.func for def in methods($f)] $(esc(def)) $f, newmethod(funcs, $f) @@ -130,11 +130,13 @@ function objdoc(meta, def) end end +fexpr(ex) = isexpr(ex, :function) || (isexpr(ex, :(=)) && isexpr(ex.args[1], :call)) + macro doc (ex) isexpr(ex, :->) || return getdoc(ex) meta, def = ex.args isexpr(unblock(def), :macro) && return macrodoc(meta, def) - isexpr(unblock(def), :function, :(=)) && return funcdoc(meta, def) + fexpr(unblock(def)) && return funcdoc(meta, def) return objdoc(meta, def) end From 5fbedc75cfecfcf42e01f288f0e2d96d4fc0ee4b Mon Sep 17 00:00:00 2001 From: Mike Innes Date: Fri, 10 Oct 2014 23:35:34 +0100 Subject: [PATCH 09/15] give each module its own META --- base/docs.jl | 61 +++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 48 insertions(+), 13 deletions(-) diff --git a/base/docs.jl b/base/docs.jl index 1db9a9b0817e7..eb1cbf0fcbcb5 100644 --- a/base/docs.jl +++ b/base/docs.jl @@ -4,17 +4,36 @@ import Base.Markdown: @doc_str, @doc_mstr export doc, @doc -# Basic API +# Basic API / Storage -const META = Dict() +const modules = Module[] -function doc(obj, meta) - META[obj] = meta +meta() = current_module().META + +macro init () + META = esc(:META) + quote + if !isdefined(:META) + const $META = ObjectIdDict() + push!(modules, current_module()) + nothing + end + end end -doc(obj) = get(META, obj, nothing) +function doc(obj, data) + meta()[obj] = data +end -doc(obj::Union(Symbol, String)) = get(META, current_module().(symbol(obj)), nothing) +function doc(obj) + for mod in modules + haskey(mod.META, obj) && return mod.META[obj] + end +end + +function doc(obj::Union(Symbol, String)) + doc(current_module().(symbol(obj))) +end # Function / Method support @@ -35,7 +54,7 @@ function newmethod(funcs, f) return newmethod(applicable) end -function trackmethod (def) +function trackmethod(def) name = unblock(def).args[1].args[1] f = esc(name) quote @@ -60,16 +79,29 @@ FuncDoc() = FuncDoc(Method[], Dict(), Dict()) getset(coll, key, default) = coll[key] = get(coll, key, default) -function doc(f::Function, m::Method, meta) - fd = getset(META, f, FuncDoc()) +function doc(f::Function, m::Method, data, source) + fd = getset(meta(), f, FuncDoc()) + isa(fd, FuncDoc) || error("Can't document a method when the function already has metadata") !haskey(fd.meta, m) && push!(fd.order, m) - fd.meta[m] = meta + fd.meta[m] = data + fd.source[m] = source end function doc(f::Function) - fd = get(META, f, nothing) - fd == nothing && return - catdoc([fd.meta[m] for m in fd.order]...) + docs = {} + for mod in modules + if haskey(mod.META, f) + fd = mod.META[f] + if isa(fd, FuncDoc) + for m in fd.order + push!(docs, fd.meta[m]) + end + elseif length(docs) == 0 + return fd + end + end + end + return catdoc(docs...) end catdoc() = nothing @@ -108,6 +140,7 @@ end function macrodoc(meta, def) name = esc(symbol(string("@", unblock(def).args[1].args[1]))) quote + @init $(esc(def)) doc($name, $(mdify(meta))) nothing @@ -116,6 +149,7 @@ end function funcdoc(meta, def) quote + @init f, m = $(trackmethod(def)) doc(f, m, $(mdify(meta)), $(esc(Expr(:quote, def)))) f @@ -124,6 +158,7 @@ end function objdoc(meta, def) quote + @init f = $(esc(def)) doc(f, $(mdify(meta))) f From c2caf0f0c315489d78ddd2424991d64bb258375c Mon Sep 17 00:00:00 2001 From: Mike Innes Date: Sat, 11 Oct 2014 20:47:40 +0100 Subject: [PATCH 10/15] support one-line syntax --- base/docs.jl | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/base/docs.jl b/base/docs.jl index eb1cbf0fcbcb5..6c57ec0d43c52 100644 --- a/base/docs.jl +++ b/base/docs.jl @@ -115,7 +115,7 @@ isexpr{T}(x::T, ts...) = T in ts function unblock(ex) isexpr(ex, :block) || return ex exs = filter(ex->!isexpr(ex, :line), ex.args) - length(exs) > 1 && return ex + length(exs) == 1 || return ex return exs[1] end @@ -129,14 +129,6 @@ function mdify(ex) end end -function getdoc(ex) - if isexpr(ex, :macrocall) - :(doc($(esc(ex.args[1])))) - else - :(doc($(esc(ex)))) - end -end - function macrodoc(meta, def) name = esc(symbol(string("@", unblock(def).args[1].args[1]))) quote @@ -167,14 +159,23 @@ end fexpr(ex) = isexpr(ex, :function) || (isexpr(ex, :(=)) && isexpr(ex.args[1], :call)) -macro doc (ex) - isexpr(ex, :->) || return getdoc(ex) - meta, def = ex.args +function docm(meta, def) isexpr(unblock(def), :macro) && return macrodoc(meta, def) fexpr(unblock(def)) && return funcdoc(meta, def) + isexpr(def, :macrocall) && (def = def.args[1]) return objdoc(meta, def) end +function docm(ex) + isexpr(ex, :->) && return docm(ex.args...) + isexpr(ex, :macrocall) && (ex = ex.args[1]) + :(doc($(esc(ex)))) +end + +macro doc (args...) + docm(args...) +end + # Text / HTML objects import Base: print, writemime From 8103dff4b558ac21e2ce5218988f0aafcae54763 Mon Sep 17 00:00:00 2001 From: Mike Innes Date: Mon, 13 Oct 2014 10:41:02 +0100 Subject: [PATCH 11/15] support for types --- base/docs.jl | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/base/docs.jl b/base/docs.jl index 6c57ec0d43c52..5db09630073ea 100644 --- a/base/docs.jl +++ b/base/docs.jl @@ -55,7 +55,7 @@ function newmethod(funcs, f) end function trackmethod(def) - name = unblock(def).args[1].args[1] + name = namify(unblock(def)) f = esc(name) quote if isdefined($(Expr(:quote, name))) && isgeneric($f) @@ -119,22 +119,24 @@ function unblock(ex) return exs[1] end +namify(ex::Expr) = namify(ex.args[1]) +namify(sy::Symbol) = sy + function mdify(ex) if isa(ex, String) :(@doc_str $(esc(ex))) - elseif isexpr(ex, :macrocall) && ex.args[1] == symbol("@mstr") + elseif isexpr(ex, :macrocall) && namify(ex) == symbol("@mstr") :(@doc_mstr $(esc(ex.args[2]))) else esc(ex) end end -function macrodoc(meta, def) - name = esc(symbol(string("@", unblock(def).args[1].args[1]))) +function namedoc(meta, def, name) quote @init $(esc(def)) - doc($name, $(mdify(meta))) + doc($(esc(name)), $(mdify(meta))) nothing end end @@ -160,15 +162,18 @@ end fexpr(ex) = isexpr(ex, :function) || (isexpr(ex, :(=)) && isexpr(ex.args[1], :call)) function docm(meta, def) - isexpr(unblock(def), :macro) && return macrodoc(meta, def) - fexpr(unblock(def)) && return funcdoc(meta, def) - isexpr(def, :macrocall) && (def = def.args[1]) + def′ = unblock(def) + isexpr(def′, :macro) && return namedoc(meta, def, + symbol(string("@", namify(def′)))) + isexpr(def′, :type) && return namedoc(meta, def, namify(def′.args[2])) + fexpr(def′) && return funcdoc(meta, def) + isexpr(def, :macrocall) && (def = namify(def)) return objdoc(meta, def) end function docm(ex) isexpr(ex, :->) && return docm(ex.args...) - isexpr(ex, :macrocall) && (ex = ex.args[1]) + isexpr(ex, :macrocall) && (ex = namify(ex)) :(doc($(esc(ex)))) end From 18f88f3483e0a91bbc05a1fb4c393221660cb829 Mon Sep 17 00:00:00 2001 From: Mike Innes Date: Fri, 24 Oct 2014 07:54:16 +0100 Subject: [PATCH 12/15] fixes for deprecations in Base --- base/docs.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/base/docs.jl b/base/docs.jl index 5db09630073ea..5d197dbec4d22 100644 --- a/base/docs.jl +++ b/base/docs.jl @@ -31,7 +31,7 @@ function doc(obj) end end -function doc(obj::Union(Symbol, String)) +function doc(obj::Union(Symbol, AbstractString)) doc(current_module().(symbol(obj))) end @@ -88,7 +88,7 @@ function doc(f::Function, m::Method, data, source) end function doc(f::Function) - docs = {} + docs = [] for mod in modules if haskey(mod.META, f) fd = mod.META[f] @@ -123,7 +123,7 @@ namify(ex::Expr) = namify(ex.args[1]) namify(sy::Symbol) = sy function mdify(ex) - if isa(ex, String) + if isa(ex, AbstractString) :(@doc_str $(esc(ex))) elseif isexpr(ex, :macrocall) && namify(ex) == symbol("@mstr") :(@doc_mstr $(esc(ex.args[2]))) From 1ed2327dc32716fd794ba0f47dd6022280fafa77 Mon Sep 17 00:00:00 2001 From: Mike Innes Date: Sun, 9 Nov 2014 15:41:27 +0000 Subject: [PATCH 13/15] Get the repl to use Docs primarily --- base/help.jl | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/base/help.jl b/base/help.jl index 164f63eb51b34..3da19c9757dbd 100644 --- a/base/help.jl +++ b/base/help.jl @@ -186,7 +186,7 @@ isname(n::Symbol) = true isname(ex::Expr) = ((ex.head == :. && isname(ex.args[1]) && isname(ex.args[2])) || (ex.head == :quote && isname(ex.args[1]))) -macro help(ex) +macro help_(ex) if ex === :? || ex === :help return Expr(:call, :help) elseif !isa(ex, Expr) || isname(ex) @@ -199,4 +199,16 @@ macro help(ex) end end +macro help (ex) + if ex === :? || ex === :help + return :(@help_ $(esc(ex))) + else + quote + let doc = @doc $(esc(ex)) + doc ≠ nothing ? doc : @help_ $(esc(ex)) + end + end + end +end + end # module From 65175ae8530534f800528ee6803b78a657f8890f Mon Sep 17 00:00:00 2001 From: Mike Innes Date: Mon, 13 Oct 2014 11:37:24 +0100 Subject: [PATCH 14/15] add metametadata --- base/docs.jl | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/base/docs.jl b/base/docs.jl index 5d197dbec4d22..2280a2687475d 100644 --- a/base/docs.jl +++ b/base/docs.jl @@ -15,6 +15,7 @@ macro init () quote if !isdefined(:META) const $META = ObjectIdDict() + doc($META, doc"Documentation metadata for $(string(current_module())).") push!(modules, current_module()) nothing end @@ -181,6 +182,46 @@ macro doc (args...) docm(args...) end +# Metametadata + +@doc """ + # Documentation + The `@doc` macro can be used to both set and retrieve documentation / + metadata. By default, documentation is written as Markdown, but any + object can be placed before the arrow. For example: + + @doc \""" + # The Foo Function + `foo(x)`: Foo the living hell out of `x`. + \""" -> + function foo() ... + + The `->` is not required if the object is on the same line, e.g. + + @doc "foo" foo + + # Retrieving Documentation + You can retrieve docs for functions, macros and other objects as + follows: + + @doc foo + @doc @time + @doc md"" + + # Functions & Methods + Placing documentation before a method definition (e.g. `function foo() + ...` or `foo() = ...`) will cause that specific method to be + documented, as opposed to the whole function. Method docs are + concatenated together in the order they were defined to provide docs + for the function. + """ @doc + +@doc "`doc(obj)`: Get the doc metadata for `obj`." doc + +@doc """ + `catdoc(xs...)`: Combine the documentation metadata `xs` into a single meta object. + """ catdoc + # Text / HTML objects import Base: print, writemime @@ -189,6 +230,17 @@ export HTML, @html_str, @html_mstr export HTML, Text +@doc """ +`HTML(s)`: Create an object that renders `s` as html. + + HTML("
foo
") + +You can also use a stream for large amounts of data: + + HTML() do io + println(io, "
foo
") + end +""" -> type HTML{T} content::T end @@ -204,10 +256,12 @@ end writemime(io::IO, ::MIME"text/html", h::HTML) = print(io, h.content) writemime(io::IO, ::MIME"text/html", h::HTML{Function}) = h.content(io) +@doc "Create an `HTML` object from a literal string." -> macro html_str (s) :(HTML($s)) end +@doc (@doc html"") -> macro html_mstr (s) :(HTML($(Base.triplequoted(s)))) end @@ -222,6 +276,17 @@ end export Text, @text_str, @text_mstr +# @doc """ +# `Text(s)`: Create an object that renders `s` as plain text. + +# HTML("foo") + +# You can also use a stream for large amounts of data: + +# Text() do io +# println(io, "foo") +# end +# """ -> type Text{T} content::T end @@ -230,10 +295,12 @@ print(io::IO, t::Text) = print(io, t.content) print(io::IO, t::Text{Function}) = t.content(io) writemime(io::IO, ::MIME"text/plain", t::Text) = print(io, t) +@doc "Create a `Text` object from a literal string." -> macro text_str (s) :(Text($s)) end +@doc (@doc text"") -> macro text_mstr (s) :(Text($(Base.triplequoted(s)))) end From 6ecd0cdedea7cf7d6b8cac35b10fa221856f2943 Mon Sep 17 00:00:00 2001 From: Mike Innes Date: Thu, 9 Oct 2014 16:25:04 +0100 Subject: [PATCH 15/15] add exports --- base/exports.jl | 13 ++++++++++++- base/sysimg.jl | 2 ++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/base/exports.jl b/base/exports.jl index 5ace9b39d0856..ea14ae36e8d98 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -12,6 +12,8 @@ export Test, BLAS, LAPACK, + Docs, + Markdown, # Types AbstractMatrix, @@ -1221,6 +1223,8 @@ export popdisplay, pushdisplay, redisplay, + HTML, + Text, # distributed arrays dfill, @@ -1339,6 +1343,10 @@ export @r_str, @r_mstr, @v_str, + @text_str, + @text_mstr, + @html_str, + @html_mstr, @int128_str, @uint128_str, @bigint_str, @@ -1392,4 +1400,7 @@ export @simd, @label, @goto, - @inline + @inline, + @doc, + @doc_str, + @doc_mstr diff --git a/base/sysimg.jl b/base/sysimg.jl index 2bc2a86b181b0..0857ff93dfe5f 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -223,6 +223,8 @@ include("client.jl") include("markdown/Markdown.jl") include("docs.jl") +using .Docs +using .Markdown # (s)printf macros include("printf.jl")