From 1dd67d5750898e7ccd6ef9ce9addc5c93a79c45a Mon Sep 17 00:00:00 2001 From: yuin Date: Sun, 25 Sep 2022 03:31:14 +0900 Subject: [PATCH] Add CJK extension --- README.md | 13 ++++ ast/inline.go | 2 +- extension/cjk.go | 49 ++++++++++++++ extension/cjk_test.go | 154 ++++++++++++++++++++++++++++++++++++++++++ renderer/html/html.go | 99 +++++++++++++++++++++++---- util/util.go | 87 +++++++++++++----------- 6 files changed, 352 insertions(+), 52 deletions(-) create mode 100644 extension/cjk.go create mode 100644 extension/cjk_test.go diff --git a/README.md b/README.md index cb8cb77..dfcb6da 100644 --- a/README.md +++ b/README.md @@ -180,6 +180,8 @@ Parser and Renderer options - [PHP Markdown Extra: Footnotes](https://michelf.ca/projects/php-markdown/extra/#footnotes) - `extension.Typographer` - This extension substitutes punctuations with typographic entities like [smartypants](https://daringfireball.net/projects/smartypants/). +- `extension.CJK` + - This extension is a shortcut for CJK related functionalities. ### Attributes The `parser.WithAttribute` option allows you to define attributes on some elements. @@ -369,6 +371,17 @@ footnote-prefix: article1 # My article ``` + +### CJK extension +CommonMark gives compatibilities a high priority and original markdown was designed by westerners. So CommonMark lacks considerations for languages like CJK. + +This extension provides additional options for CJK users. + +| Functional option | Type | Description | +| ----------------- | ---- | ----------- | +| `extension.WithEastAsianLineBreaks` | `-` | Soft line breaks are rendered as a newline. Some asian users will see it as an unnecessary space. With this option, soft line breaks between east asian wide characters will be ignored. | +| `extension.WithEscapedSpace` | `-` | Without spaces around an emphasis started with east asian punctuations, it is not interpreted as an emphasis(as defined in CommonMark spec). With this option, you can avoid this inconvenient behavior by putting 'not rendered' spaces around an emphasis like `太郎は\ **「こんにちわ」**\ といった`. | + Security -------------------- diff --git a/ast/inline.go b/ast/inline.go index fa6fc34..7da098f 100644 --- a/ast/inline.go +++ b/ast/inline.go @@ -91,7 +91,7 @@ func (n *Text) SetSoftLineBreak(v bool) { if v { n.flags |= textSoftLineBreak } else { - n.flags = n.flags &^ textHardLineBreak + n.flags = n.flags &^ textSoftLineBreak } } diff --git a/extension/cjk.go b/extension/cjk.go new file mode 100644 index 0000000..60e07ff --- /dev/null +++ b/extension/cjk.go @@ -0,0 +1,49 @@ +package extension + +import ( + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/renderer/html" +) + +// A CJKOption sets options for CJK support mostly for HTML based renderers. +type CJKOption func(*cjk) + +// WithEastAsianLineBreaks is a functional option that indicates whether softline breaks +// between east asian wide characters should be ignored. +func WithEastAsianLineBreaks() CJKOption { + return func(c *cjk) { + c.EastAsianLineBreaks = true + } +} + +// WithEscapedSpace is a functional option that indicates that a '\' escaped half-space(0x20) should not be rendered. +func WithEscapedSpace() CJKOption { + return func(c *cjk) { + c.EscapedSpace = true + } +} + +type cjk struct { + EastAsianLineBreaks bool + EscapedSpace bool +} + +var CJK = NewCJK(WithEastAsianLineBreaks(), WithEscapedSpace()) + +// NewCJK returns a new extension with given options. +func NewCJK(opts ...CJKOption) goldmark.Extender { + e := &cjk{} + for _, opt := range opts { + opt(e) + } + return e +} + +func (e *cjk) Extend(m goldmark.Markdown) { + if e.EastAsianLineBreaks { + m.Renderer().AddOptions(html.WithEastAsianLineBreaks()) + } + if e.EscapedSpace { + m.Renderer().AddOptions(html.WithWriter(html.NewWriter(html.WithEscapedSpace()))) + } +} diff --git a/extension/cjk_test.go b/extension/cjk_test.go new file mode 100644 index 0000000..8863a10 --- /dev/null +++ b/extension/cjk_test.go @@ -0,0 +1,154 @@ +package extension + +import ( + "testing" + + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/renderer/html" + "github.com/yuin/goldmark/testutil" +) + +func TestEscapedSpace(t *testing.T) { + markdown := goldmark.New(goldmark.WithRendererOptions( + html.WithXHTML(), + html.WithUnsafe(), + )) + no := 1 + testutil.DoTestCase( + markdown, + testutil.MarkdownTestCase{ + No: no, + Description: "Without spaces around an emphasis started with east asian punctuations, it is not interpreted as an emphasis(as defined in CommonMark spec)", + Markdown: "太郎は**「こんにちわ」**と言った\nんです", + Expected: "

太郎は**「こんにちわ」**と言った\nんです

", + }, + t, + ) + + no = 2 + testutil.DoTestCase( + markdown, + testutil.MarkdownTestCase{ + No: no, + Description: "With spaces around an emphasis started with east asian punctuations, it is interpreted as an emphasis(but remains unnecessary spaces)", + Markdown: "太郎は **「こんにちわ」** と言った\nんです", + Expected: "

太郎は 「こんにちわ」 と言った\nんです

", + }, + t, + ) + + // Enables EscapedSpace + markdown = goldmark.New(goldmark.WithRendererOptions( + html.WithXHTML(), + html.WithUnsafe(), + ), + goldmark.WithExtensions(NewCJK(WithEscapedSpace())), + ) + + no = 3 + testutil.DoTestCase( + markdown, + testutil.MarkdownTestCase{ + No: no, + Description: "With spaces around an emphasis started with east asian punctuations,it is interpreted as an emphasis", + Markdown: "太郎は\\ **「こんにちわ」**\\ と言った\nんです", + Expected: "

太郎は「こんにちわ」と言った\nんです

", + }, + t, + ) +} + +func TestEastAsianLineBreaks(t *testing.T) { + markdown := goldmark.New(goldmark.WithRendererOptions( + html.WithXHTML(), + html.WithUnsafe(), + )) + no := 1 + testutil.DoTestCase( + markdown, + testutil.MarkdownTestCase{ + No: no, + Description: "Soft line breaks are rendered as a newline, so some asian users will see it as an unnecessary space", + Markdown: "太郎は\\ **「こんにちわ」**\\ と言った\nんです", + Expected: "

太郎は\\ 「こんにちわ」\\ と言った\nんです

", + }, + t, + ) + + // Enables EastAsianLineBreaks + + markdown = goldmark.New(goldmark.WithRendererOptions( + html.WithXHTML(), + html.WithUnsafe(), + ), + goldmark.WithExtensions(NewCJK(WithEastAsianLineBreaks())), + ) + + no = 2 + testutil.DoTestCase( + markdown, + testutil.MarkdownTestCase{ + No: no, + Description: "Soft line breaks between east asian wide characters are ignored", + Markdown: "太郎は\\ **「こんにちわ」**\\ と言った\nんです", + Expected: "

太郎は\\ 「こんにちわ」\\ と言ったんです

", + }, + t, + ) + + no = 3 + testutil.DoTestCase( + markdown, + testutil.MarkdownTestCase{ + No: no, + Description: "Soft line breaks between western characters are rendered as a newline", + Markdown: "太郎は\\ **「こんにちわ」**\\ と言ったa\nbんです", + Expected: "

太郎は\\ 「こんにちわ」\\ と言ったa\nbんです

", + }, + t, + ) + + no = 4 + testutil.DoTestCase( + markdown, + testutil.MarkdownTestCase{ + No: no, + Description: "Soft line breaks between a western character and an east asian wide character are rendered as a newline", + Markdown: "太郎は\\ **「こんにちわ」**\\ と言ったa\nんです", + Expected: "

太郎は\\ 「こんにちわ」\\ と言ったa\nんです

", + }, + t, + ) + + no = 5 + testutil.DoTestCase( + markdown, + testutil.MarkdownTestCase{ + No: no, + Description: "Soft line breaks between an east asian wide character and a western character are rendered as a newline", + Markdown: "太郎は\\ **「こんにちわ」**\\ と言った\nbんです", + Expected: "

太郎は\\ 「こんにちわ」\\ と言った\nbんです

", + }, + t, + ) + + // WithHardWraps take precedence over WithEastAsianLineBreaks + markdown = goldmark.New(goldmark.WithRendererOptions( + html.WithHardWraps(), + html.WithXHTML(), + html.WithUnsafe(), + ), + goldmark.WithExtensions(NewCJK(WithEastAsianLineBreaks())), + ) + no = 6 + testutil.DoTestCase( + markdown, + testutil.MarkdownTestCase{ + No: no, + Description: "WithHardWraps take precedence over WithEastAsianLineBreaks", + Markdown: "太郎は\\ **「こんにちわ」**\\ と言った\nんです", + Expected: "

太郎は\\ 「こんにちわ」\\ と言った
\nんです

", + }, + t, + ) +} diff --git a/renderer/html/html.go b/renderer/html/html.go index 65ffc72..a3d1fe2 100644 --- a/renderer/html/html.go +++ b/renderer/html/html.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" "strconv" + "unicode/utf8" "github.com/yuin/goldmark/ast" "github.com/yuin/goldmark/renderer" @@ -12,19 +13,21 @@ import ( // A Config struct has configurations for the HTML based renderers. type Config struct { - Writer Writer - HardWraps bool - XHTML bool - Unsafe bool + Writer Writer + HardWraps bool + EastAsianLineBreaks bool + XHTML bool + Unsafe bool } // NewConfig returns a new Config with defaults. func NewConfig() Config { return Config{ - Writer: DefaultWriter, - HardWraps: false, - XHTML: false, - Unsafe: false, + Writer: DefaultWriter, + HardWraps: false, + EastAsianLineBreaks: false, + XHTML: false, + Unsafe: false, } } @@ -33,6 +36,8 @@ func (c *Config) SetOption(name renderer.OptionName, value interface{}) { switch name { case optHardWraps: c.HardWraps = value.(bool) + case optEastAsianLineBreaks: + c.EastAsianLineBreaks = value.(bool) case optXHTML: c.XHTML = value.(bool) case optUnsafe: @@ -94,6 +99,29 @@ func WithHardWraps() interface { return &withHardWraps{} } +// EastAsianLineBreaks is an option name used in WithEastAsianLineBreaks. +const optEastAsianLineBreaks renderer.OptionName = "EastAsianLineBreaks" + +type withEastAsianLineBreaks struct { +} + +func (o *withEastAsianLineBreaks) SetConfig(c *renderer.Config) { + c.Options[optEastAsianLineBreaks] = true +} + +func (o *withEastAsianLineBreaks) SetHTMLOption(c *Config) { + c.EastAsianLineBreaks = true +} + +// WithEastAsianLineBreaks is a functional option that indicates whether softline breaks +// between east asian wide characters should be ignored. +func WithEastAsianLineBreaks() interface { + renderer.Option + Option +} { + return &withEastAsianLineBreaks{} +} + // XHTML is an option name used in WithXHTML. const optXHTML renderer.OptionName = "XHTML" @@ -615,7 +643,8 @@ func (r *Renderer) renderText(w util.BufWriter, source []byte, node ast.Node, en if n.IsRaw() { r.Writer.RawWrite(w, segment.Value(source)) } else { - r.Writer.Write(w, segment.Value(source)) + value := segment.Value(source) + r.Writer.Write(w, value) if n.HardLineBreak() || (n.SoftLineBreak() && r.HardWraps) { if r.XHTML { _, _ = w.WriteString("
\n") @@ -623,7 +652,21 @@ func (r *Renderer) renderText(w util.BufWriter, source []byte, node ast.Node, en _, _ = w.WriteString("
\n") } } else if n.SoftLineBreak() { - _ = w.WriteByte('\n') + if r.EastAsianLineBreaks && len(value) != 0 { + sibling := node.NextSibling() + if sibling != nil && sibling.Kind() == ast.KindText { + if siblingText := sibling.(*ast.Text).Text(source); len(siblingText) != 0 { + thisLastRune := util.ToRune(value, len(value)-1) + siblingFirstRune, _ := utf8.DecodeRune(siblingText) + if !(util.IsEastAsianWideRune(thisLastRune) && + util.IsEastAsianWideRune(siblingFirstRune)) { + _ = w.WriteByte('\n') + } + } + } + } else { + _ = w.WriteByte('\n') + } } } return ast.WalkContinue, nil @@ -683,7 +726,33 @@ type Writer interface { var replacementCharacter = []byte("\ufffd") +// A WriterConfig struct has configurations for the HTML based writers. +type WriterConfig struct { + // EscapedSpace is an option that indicates that a '\' escaped half-space(0x20) should not be rendered. + EscapedSpace bool +} + +// A WriterOption interface sets options for HTML based writers. +type WriterOption func(*WriterConfig) + +// WithEscapedSpace is a WriterOption indicates that a '\' escaped half-space(0x20) should not be rendered. +func WithEscapedSpace() WriterOption { + return func(c *WriterConfig) { + c.EscapedSpace = true + } +} + type defaultWriter struct { + WriterConfig +} + +// NewWriter returns a new Writer. +func NewWriter(opts ...WriterOption) Writer { + w := &defaultWriter{} + for _, opt := range opts { + opt(&w.WriterConfig) + } + return w } func escapeRune(writer util.BufWriter, r rune) { @@ -746,6 +815,12 @@ func (d *defaultWriter) Write(writer util.BufWriter, source []byte) { escaped = false continue } + if d.EscapedSpace && c == ' ' { + d.RawWrite(writer, source[n:i-1]) + n = i + 1 + escaped = false + continue + } } if c == '\x00' { d.RawWrite(writer, source[n:i]) @@ -811,8 +886,8 @@ func (d *defaultWriter) Write(writer util.BufWriter, source []byte) { d.RawWrite(writer, source[n:]) } -// DefaultWriter is a default implementation of the Writer. -var DefaultWriter = &defaultWriter{} +// DefaultWriter is a default instance of the Writer. +var DefaultWriter = NewWriter() var bDataImage = []byte("data:image/") var bPng = []byte("png;") diff --git a/util/util.go b/util/util.go index a817ec6..88d2538 100644 --- a/util/util.go +++ b/util/util.go @@ -184,25 +184,25 @@ func IndentPositionPadding(bs []byte, currentPos, paddingv, width int) (pos, pad // // Deprecated: This function has bugs. Use util.IndentPositionPadding and util.FirstNonSpacePosition. func DedentPosition(bs []byte, currentPos, width int) (pos, padding int) { - if width == 0 { - return 0, 0 - } - w := 0 - l := len(bs) - i := 0 - for ; i < l; i++ { - if bs[i] == '\t' { - w += TabWidth(currentPos + w) - } else if bs[i] == ' ' { - w++ - } else { - break - } - } - if w >= width { - return i, w - width - } - return i, 0 + if width == 0 { + return 0, 0 + } + w := 0 + l := len(bs) + i := 0 + for ; i < l; i++ { + if bs[i] == '\t' { + w += TabWidth(currentPos + w) + } else if bs[i] == ' ' { + w++ + } else { + break + } + } + if w >= width { + return i, w - width + } + return i, 0 } // DedentPositionPadding dedents lines by the given width. @@ -211,26 +211,26 @@ func DedentPosition(bs []byte, currentPos, width int) (pos, padding int) { // // Deprecated: This function has bugs. Use util.IndentPositionPadding and util.FirstNonSpacePosition. func DedentPositionPadding(bs []byte, currentPos, paddingv, width int) (pos, padding int) { - if width == 0 { - return 0, paddingv - } - - w := 0 - i := 0 - l := len(bs) - for ; i < l; i++ { - if bs[i] == '\t' { - w += TabWidth(currentPos + w) - } else if bs[i] == ' ' { - w++ - } else { - break - } - } - if w >= width { - return i - paddingv, w - width - } - return i - paddingv, 0 + if width == 0 { + return 0, paddingv + } + + w := 0 + i := 0 + l := len(bs) + for ; i < l; i++ { + if bs[i] == '\t' { + w += TabWidth(currentPos + w) + } else if bs[i] == ' ' { + w++ + } else { + break + } + } + if w >= width { + return i - paddingv, w - width + } + return i - paddingv, 0 } // IndentWidth calculate an indent width for the given line. @@ -834,6 +834,15 @@ func IsAlphaNumeric(c byte) bool { return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' } +// IsEastAsianWideRune returns trhe if the given rune is an east asian wide character, otherwise false. +func IsEastAsianWideRune(r rune) bool { + return unicode.Is(unicode.Hiragana, r) || + unicode.Is(unicode.Katakana, r) || + unicode.Is(unicode.Han, r) || + unicode.Is(unicode.Lm, r) || + unicode.Is(unicode.Hangul, r) +} + // A BufWriter is a subset of the bufio.Writer . type BufWriter interface { io.Writer