diff --git a/go.mod b/go.mod
index 89b7e44..6b9ce25 100644
--- a/go.mod
+++ b/go.mod
@@ -7,6 +7,7 @@ require (
github.com/Masterminds/sprig/v3 v3.2.2
github.com/gomarkdown/markdown v0.0.0-20210915032930-fe0e174ee09a
github.com/google/uuid v1.3.0 // indirect
+ github.com/grokify/html-strip-tags-go v0.0.1
github.com/hexops/gotextdiff v1.0.3
github.com/huandu/xstrings v1.3.2 // indirect
github.com/imdario/mergo v0.3.12 // indirect
diff --git a/go.sum b/go.sum
index d3ede79..894b2f0 100644
--- a/go.sum
+++ b/go.sum
@@ -21,6 +21,8 @@ github.com/gomarkdown/markdown v0.0.0-20210915032930-fe0e174ee09a/go.mod h1:JDGc
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/grokify/html-strip-tags-go v0.0.1 h1:0fThFwLbW7P/kOiTBs03FsJSV9RM2M/Q/MOnCQxKMo0=
+github.com/grokify/html-strip-tags-go v0.0.1/go.mod h1:2Su6romC5/1VXOQMaWL2yb618ARB8iVo6/DR99A6d78=
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
diff --git a/internal/renderer/renderer.go b/internal/renderer/renderer.go
index 9f35bf7..c864976 100644
--- a/internal/renderer/renderer.go
+++ b/internal/renderer/renderer.go
@@ -20,11 +20,12 @@ package renderer
import (
"bytes"
"fmt"
+ "html"
"io"
"regexp"
- "strings"
"github.com/gomarkdown/markdown/ast"
+ "github.com/grokify/html-strip-tags-go"
"github.com/olekukonko/tablewriter"
)
@@ -51,6 +52,25 @@ var (
// matches a FULL string that contains no non-whitespace characters
var emptyLineRegex = regexp.MustCompile(`\A[\s]*\z`)
+// fairly tolerant to handle weird HTML
+var tagPairRegexString = `<[\n\f ]*%s([\n\f ]+[^\n\f \/>"'=]+[\n\f ]*(=[\n\f ]*([a-zA-Z1-9\-]+|"[^\n\f"]+"|'[^\n\f']+'))?)*[\n\f ]*>.*?<[\n\f ]*/[\n\f ]*%s[\n\f ]*>`
+
+// HTML block tags whose contents should not be rendered
+var htmlNoRenderRegex = []*regexp.Regexp{
+ regexp.MustCompile(fmt.Sprintf(tagPairRegexString, "fieldset", "fieldset")),
+ regexp.MustCompile(fmt.Sprintf(tagPairRegexString, "form", "form")),
+ regexp.MustCompile(fmt.Sprintf(tagPairRegexString, "iframe", "iframe")),
+ regexp.MustCompile(fmt.Sprintf(tagPairRegexString, "script", "script")),
+ regexp.MustCompile(fmt.Sprintf(tagPairRegexString, "style", "style")),
+ regexp.MustCompile(fmt.Sprintf(tagPairRegexString, "canvas", "canvas")),
+ regexp.MustCompile(fmt.Sprintf(tagPairRegexString, "dialog", "dialog")),
+ regexp.MustCompile(fmt.Sprintf(tagPairRegexString, "progress", "progress")),
+}
+
+var lineBreakCharacters = regexp.MustCompile(`[\n\r]+`)
+var hardBreakTag = regexp.MustCompile(`< *br */? *>`)
+var escapedHtmlChar = regexp.MustCompile(`(?:^|[^\\\\])&[[:alnum:]]+;`)
+
// Renderer implements markdown.Renderer.
type Renderer struct{}
@@ -82,7 +102,7 @@ func (r Renderer) link(w io.Writer, node *ast.Link, entering bool) {
w.Write(linkPrefix)
w.Write(node.Destination)
w.Write(space)
- r.text(w, node)
+ r.text(w, node, true)
}
}
}
@@ -92,7 +112,7 @@ func (r Renderer) image(w io.Writer, node *ast.Image, entering bool) {
w.Write(linkPrefix)
w.Write(node.Destination)
w.Write(space)
- r.text(w, node)
+ r.text(w, node, true)
}
}
@@ -125,7 +145,7 @@ func (r Renderer) subscript(w io.Writer, node *ast.Subscript, entering bool) {
if entering {
if node := node.AsLeaf(); node != nil {
w.Write(subOpen)
- w.Write([]byte(strings.ReplaceAll(string(node.Literal), "\n", " ")))
+ w.Write(bytes.ReplaceAll(node.Literal, lineBreak, space))
w.Write(subClose)
}
}
@@ -134,7 +154,7 @@ func (r Renderer) superscript(w io.Writer, node *ast.Superscript, entering bool)
if entering {
if node := node.AsLeaf(); node != nil {
w.Write(supOpen)
- w.Write([]byte(strings.ReplaceAll(string(node.Literal), "\n", " ")))
+ w.Write(bytes.ReplaceAll(node.Literal, lineBreak, space))
w.Write(supClose)
}
}
@@ -151,7 +171,7 @@ func (r Renderer) heading(w io.Writer, node *ast.Heading, entering bool) {
heading[i] = '#'
}
w.Write(heading)
- r.text(w, node)
+ r.text(w, node, true)
} else {
w.Write(lineBreak)
}
@@ -277,8 +297,16 @@ func (r Renderer) paragraph(w io.Writer, node *ast.Paragraph, entering bool) (no
// only render links text in the paragraph if they're
// combined with some other text on page
switch child := child.(type) {
- case *ast.Text, *ast.Code, *ast.Emph, *ast.Strong, *ast.Del, *ast.Link, *ast.Image:
- r.text(w, child)
+ case *ast.Text, *ast.Emph, *ast.Strong, *ast.Del, *ast.Link, *ast.Image:
+ r.text(w, child, true)
+ case *ast.Code:
+ r.text(w, child, false)
+ case *ast.Hardbreak:
+ w.Write(lineBreak)
+ case *ast.HTMLSpan:
+ if hardBreakTag.Match(child.AsLeaf().Literal) {
+ w.Write(lineBreak)
+ }
case *ast.Subscript:
r.subscript(w, child, true)
case *ast.Superscript:
@@ -326,7 +354,7 @@ func (r Renderer) list(w io.Writer, node *ast.List, level int) {
} else if !isTerm {
w.Write(itemPrefix)
}
- r.text(w, item)
+ r.text(w, item, true)
w.Write(lineBreak)
if l >= 2 {
if list, ok := item.Children[1].(*ast.List); ok {
@@ -337,22 +365,43 @@ func (r Renderer) list(w io.Writer, node *ast.List, level int) {
}
}
-var lineBreakCharacters = regexp.MustCompile(`[\n\r]+`)
-
-func textWithNewlineReplacement(node ast.Node, replacement []byte) []byte {
+func textWithNewlineReplacement(node ast.Node, replacement []byte, unescapeHtml bool) []byte {
buf := bytes.Buffer{}
delimiter := getNodeDelimiter(node)
// special case for footnotes: we want them in the text
if node, ok := node.(*ast.Link); ok && node.Footnote != nil {
fmt.Fprintf(&buf, "[^%d]", node.NoteID)
}
- if node := node.AsLeaf(); node != nil {
+ if leaf := node.AsLeaf(); leaf != nil {
// replace all newlines in text with preferred symbols; this may
// be spaces for general text, allowing for soft wrapping, which
// is recommended as per Gemini spec p. 5.4.1, or line breaks
// with a blockquote symbols for blockquotes, or just nothing
buf.Write(delimiter)
- buf.Write(lineBreakCharacters.ReplaceAll(node.Literal, replacement))
+ switch node := node.(type) {
+ case *ast.Hardbreak:
+ buf.Write(lineBreak)
+ // If the blockquote ends with a double space, the parser will
+ // not create a Hardbreak at the end, so this works.
+ if _, ok := leaf.Parent.(*ast.BlockQuote); !ok {
+ buf.Write(quotePrefix)
+ }
+ case *ast.HTMLSpan:
+ if hardBreakTag.Match(leaf.Literal) {
+ buf.Write(lineBreak)
+ }
+ buf.Write(leaf.Content)
+ case *ast.HTMLBlock:
+ buf.Write([]byte(extractHtml(node, quotePrefix)))
+ default:
+ textWithoutBreaks := lineBreakCharacters.ReplaceAll(leaf.Literal, replacement)
+ if unescapeHtml {
+ unescapedText := escapedHtmlChar.ReplaceAll(textWithoutBreaks, []byte(html.UnescapeString(string(textWithoutBreaks))))
+ buf.Write(unescapedText)
+ } else {
+ buf.Write(textWithoutBreaks)
+ }
+ }
buf.Write(delimiter)
}
if node := node.AsContainer(); node != nil {
@@ -362,7 +411,7 @@ func textWithNewlineReplacement(node ast.Node, replacement []byte) []byte {
switch child := child.(type) {
case *ast.List:
default:
- buf.Write(textWithNewlineReplacement(child, replacement))
+ buf.Write(textWithNewlineReplacement(child, replacement, unescapeHtml))
}
}
buf.Write(delimiter)
@@ -370,16 +419,30 @@ func textWithNewlineReplacement(node ast.Node, replacement []byte) []byte {
return buf.Bytes()
}
-func (r Renderer) text(w io.Writer, node ast.Node) {
- w.Write(textWithNewlineReplacement(node, space))
+func (r Renderer) text(w io.Writer, node ast.Node, unescapeHtml bool) {
+ w.Write(textWithNewlineReplacement(node, space, unescapeHtml))
}
func (r Renderer) blockquoteText(w io.Writer, node ast.Node) {
- w.Write(textWithNewlineReplacement(node, quoteBrPrefix))
+ w.Write(textWithNewlineReplacement(node, quoteBrPrefix, true))
}
func extractText(node ast.Node) string {
- return string(textWithNewlineReplacement(node, space))
+ return string(textWithNewlineReplacement(node, space, true))
+}
+
+func extractHtml(node *ast.HTMLBlock, linePrefix []byte) string {
+ // Only render contents of allowed tags
+ literal := node.Literal
+ for _, re := range htmlNoRenderRegex {
+ literal = re.ReplaceAllLiteral(literal, []byte{})
+ }
+ if len(literal) > 0 {
+ literalWithBreaks := hardBreakTag.ReplaceAll(lineBreakCharacters.ReplaceAll(literal, space), append([]byte(lineBreak), linePrefix...))
+ literalStripped := strip.StripTags(string(literalWithBreaks))
+ return html.UnescapeString(literalStripped)
+ }
+ return ""
}
func (r Renderer) tableHead(t *tablewriter.Table, node *ast.TableHeader) {
@@ -440,6 +503,17 @@ func (r Renderer) table(w io.Writer, node *ast.Table, entering bool) {
}
}
+func (r Renderer) htmlBlock(w io.Writer, node *ast.HTMLBlock, entering bool) {
+ if entering {
+ htmlString := extractHtml(node, []byte{})
+ if len(htmlString) > 0 {
+ w.Write([]byte(htmlString))
+ w.Write(lineBreak)
+ w.Write(lineBreak)
+ }
+ }
+}
+
// RenderNode implements Renderer.RenderNode().
func (r Renderer) RenderNode(w io.Writer, node ast.Node, entering bool) ast.WalkStatus {
// entering in gomarkdown was made to have elements of type switch
@@ -487,6 +561,11 @@ func (r Renderer) RenderNode(w io.Writer, node ast.Node, entering bool) ast.Walk
r.table(w, node, entering)
noNewLine = false
fetchLinks = true
+ case *ast.HTMLBlock:
+ // Do not render if already rendered as part of a blockquote
+ if _, ok := node.Parent.(*ast.BlockQuote); !ok {
+ r.htmlBlock(w, node, entering)
+ }
}
if !noNewLine && !entering {
w.Write(lineBreak)
diff --git a/testdata/general_text.gmi b/testdata/general_text.gmi
index 326f879..d578655 100644
--- a/testdata/general_text.gmi
+++ b/testdata/general_text.gmi
@@ -6,6 +6,10 @@ Single newlines (like in this multi-line paragraph) will get replaced by a space
Inline formatting bits (like this **bold** text, *emphasized* text, ~~strikethrough~~ text, `preformatted text`) are kept to make sure Gemini readers still have the stylistic context of your text.
+Adding two spaces at the end of a line will insert a hard
+break. You can also create a hard break using a backslash at the end
+of a line. Hard breaks at the end of a paragraph are ignored.
+
## Blockquotes
Newlines in blockquote paragraphs, unlike usual paragraphs, aren't replaced with a space. This facilitates appending authorship information to the quote, or using blockquotes to write poems.
@@ -22,6 +26,9 @@ Newlines in blockquote paragraphs, unlike usual paragraphs, aren't replaced with
> — also Timur Demin, in the process of writing this test file
+> Hard breaks are also supported in blockquotes,
+> for compatibility. Hard breaks at the end of a blockquote are ignored.
+
## Code
gmnhg will use Gemtext preformatted blocks for that. Markdown alt-text for preformatted blocks is supported, and is used to render alt-text as specified by Gemini spec p. 5.4.3.
@@ -74,16 +81,46 @@ Since clients like Lagrange treat the fourth and the rest of #-s as heading cont
###### Heading 6
+## HTML
+
+Inline HTML is currently stripped, but HTML contents remain on-screen. This may change in the future. HTML tags can be escaped with \ as in or enclosed with ``.
+
+HTML tags are stripped from HTML blocks. (Note that HTML blocks must begin and end with a supported HTML block tag, and must have blank lines before and after the block.)
+
+### Break tags
+
+Hard breaks
+using are supported.
+
+Hard breaks using are supported
+inside HTML blocks.
+
+### HTML entities
+
+HTML escaped entities like & and < are unescaped, even when they show up inside an inline HTML section. Escaping them with a leading backslash is possible outside of HTML blocks: &, <. Any escaped characters inside a code span (such as `< or >`) will not be unescaped.
+
+HTML escaped entities like < and > are also unescaped inside HTML blocks. Backslash escapes have no effect: \&.
+
+### Forbidden tags
+
+Tags that are unable to output Gemini-compatible text are completely removed from the output.
+
+Note that the contents of "forbidden" tags will be rendered if they are placed inline, although the tags themselves will be stripped. Placing HTML block elements inline in this manner violates the spec of common Markdown flavors, but gmnhg handles it the best it can.
+
+### HTML in blockquotes
+
+> HTML spans are stripped from
+> inside blockquotes.
+
+> Non HTML block text before the block.
+> HTML blocks are stripped from inside blockquotes.
+> Non HTML block text after the block.
+
+> Standalone blockquoted HTML blocks
+> are also stripped of their tags.
+
## Misc
-Inline HTML is currently stripped, but HTML contents remain on-screen. This may change in the future.
-
-> There's currently a bug in gmnhg which prevents it from
-> stripping HTML in certain scenarios. HTML is noticeably still present
-> inside blockquotes.
-
-=> https://github.com/tdemin/gmnhg/issues/6 bug in gmnhg
-
---
The Markdown horizontal line above is rendered as triple dashes.
diff --git a/testdata/general_text.md b/testdata/general_text.md
index f9af9b9..1f978ad 100644
--- a/testdata/general_text.md
+++ b/testdata/general_text.md
@@ -10,6 +10,10 @@ Inline formatting bits (like this **bold** text, _emphasized_ text,
~~strikethrough~~ text, `preformatted text`) are kept to make sure
Gemini readers still have the stylistic context of your text.
+Adding two spaces at the end of a line will insert a hard
+break. You can also create a hard break using a backslash at the end\
+of a line. Hard breaks at the end of a paragraph are ignored.
+
## Blockquotes
Newlines in blockquote paragraphs, unlike usual paragraphs, aren't
@@ -28,6 +32,9 @@ to the quote, or using blockquotes to write poems.
>
> — also Timur Demin, in the process of writing this test file
+> Hard breaks are also supported in blockquotes,
+> for compatibility. Hard breaks at the end of a blockquote are ignored.
+
## Code
gmnhg will use Gemtext preformatted blocks for that. Markdown alt-text
@@ -86,15 +93,73 @@ your client handles that.
###### Heading 6
-## Misc
+## HTML
Inline HTML is currently stripped, but HTML
-contents remain on-screen. This may change in the future.
+contents remain on-screen. This may change in the future. HTML tags
+can be escaped with \ as in \\ or enclosed with \`\`.
-> There's currently a [bug in gmnhg][bug] which prevents it from
-> stripping HTML in certain scenarios. HTML is noticeably still present
+
HTML tags are stripped from HTML blocks. (Note that HTML blocks
+must begin and end with a supported HTML block tag, and must
+have blank lines before and after the block.)
+
+### Break tags
+
+Hard breaks using \ are supported.
+
+
Hard breaks using <br> are supported inside HTML blocks.
+
+### HTML entities
+
+HTML escaped entities like & and < are unescaped,
+even when they show up inside an inline HTML section. Escaping
+them with a leading backslash is possible outside of HTML blocks:
+\&, \<. Any escaped characters inside a code span (such as `<
+or >`) will not be unescaped.
+
+
HTML escaped entities like < and > are also unescaped
+inside HTML blocks. Backslash escapes have no effect: \&.
+
+### Forbidden tags
+
+Tags that are unable to output Gemini-compatible text are completely
+removed from the output.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Note that the contents of "forbidden" tags will be rendered if they are
+placed , although the tags themselves will be
+stripped. Placing HTML block elements inline in this manner violates
+the spec of common Markdown flavors, but gmnhg handles it the best it
+can.
+
+### HTML in blockquotes
+
+> HTML spans are stripped from
> inside blockquotes.
+> Non HTML block text before the block.
+>
HTML blocks are stripped from inside blockquotes.
+> Non HTML block text after the block.
+
+>
Standalone blockquoted HTML blocks are also stripped of their tags.
+
+## Misc
+
***
The Markdown horizontal line above is rendered as triple dashes.