123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229 | package highlight
import (
"os"
"path/filepath"
"regexp"
"runtime"
"slices"
"strings"
"sync"
"testing"
"github.com/alecthomas/chroma/v2"
"github.com/alecthomas/chroma/v2/lexers"
)
// sampleNames builds filenames that exercise every glob chroma knows:
// each glob instantiated, with and without backup suffixes and prefixes.
func sampleNames() []string {
class := regexp.MustCompile(`\[(\^?)([^\]]*)\]`)
names := []string{"", ".", "x", "noext", "a.b.c", "foo.tar.gz", "Makefile.am.in", "README", ".bashrc~"}
for _, lx := range lexers.GlobalLexerRegistry.Lexers {
cfg := lx.Config()
for _, g := range append(cfg.Filenames, cfg.AliasFilenames...) {
base := class.ReplaceAllStringFunc(g, func(m string) string {
sub := class.FindStringSubmatch(m)
if sub[1] != "" || sub[2] == "" {
return "q"
}
return sub[2][:1]
})
for _, star := range []string{"", "a.b"} {
n := strings.NewReplacer("*", star, "?", "z", `\`, "").Replace(base)
names = append(names, n, n+"~", n+".in", "x"+n, strings.ToUpper(n))
}
}
}
slices.Sort(names)
return slices.Compact(names)
}
func TestIndexMatchesChroma(t *testing.T) {
ix := builtins()
// chroma takes ~8ms a lookup, so spread the comparison over the CPUs.
names := sampleNames()
var wg sync.WaitGroup
for w := range runtime.NumCPU() {
wg.Go(func() {
for i := w; i < len(names); i += runtime.NumCPU() {
n := names[i]
if got, want := Name(ix.match(n)), Name(lexers.Match(n)); got != want {
t.Errorf("match(%q) = %s, chroma says %s", n, got, want)
}
}
})
}
wg.Wait()
gets := append(lexers.Names(true), "", "tomo", "py", "PY", "rs", "h", "m", "pl", "makefile", "Dockerfile", "nonsense", "php5", "1")
for _, name := range append(gets, ix.names()...) {
if got, want := Name(ix.get(name)), Name(lexers.Get(name)); got != want {
t.Errorf("get(%q) = %s, chroma says %s", name, got, want)
}
}
}
func TestMarkdownFences(t *testing.T) {
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, "toy.xml"), []byte(toyLexer), 0o644); err != nil {
t.Fatal(err)
}
r, err := New([]string{dir}, []Custom{{Name: "toy", Aliases: []string{"tm"}}}, "tokyonight-night")
if err != nil {
t.Fatal(err)
}
md := r.ByPath("README.md")
if md != r.ByName("markdown") || md != r.markdown {
t.Fatal("markdown files do not use the fence-aware lexer")
}
out := string(r.HTML([]byte("# Hi\n\n```tm\nzap 1\n```\n\n```py\ndef f(): pass\n```\n\n```\nzap\n```\n\n```nonsense\nzap\n```\n"), md))
if !strings.Contains(out, `<span class="k">zap</span> <span class="m">1</span>`) {
t.Errorf("custom lexer not used in fence:\n%s", out)
}
if !strings.Contains(out, `<span class="kd">def</span>`) && !strings.Contains(out, `<span class="k">def</span>`) {
t.Errorf("builtin lexer not used in fence:\n%s", out)
}
if strings.Count(out, `<span class="k">zap</span>`) != 1 {
t.Errorf("unlabelled or unknown fence was highlighted:\n%s", out)
}
// The shared chroma lexer must still use chroma's own registry.
global := string(r.HTML([]byte("```tm\nzap 1\n```\n"), lexers.Markdown))
if strings.Contains(global, `<span class="k">zap</span>`) {
t.Error("chroma's global markdown lexer was rewired")
}
if !strings.Contains(string(r.HTML([]byte("---\ntitle: x\n---\n# Hi\n"), md)), `<span class="nt">title</span>`) {
t.Error("frontmatter no longer highlighted")
}
}
// markdownSample exercises every rule of chroma's markdown lexer, and the
// places a run of prose stops: at markup, mid-word and at line ends.
const markdownSample = "---\ntitle: x\n---\n" +
"# Heading with *em*\n## Sub_heading\n\n" +
"Plain prose, naïve café — ✓ snake_case and a*b and 2*3 = 6.\n" +
"Some *em*, _em_, **strong**, __strong__, ~~gone~~ and `code`.\n" +
"Mentions @user, #tag, issue#12, a/b@c:d and a lone # or @.\n" +
"Links [text](http://x.y/z),  and !bang.\n" +
"<!-- a\ncomment --> and <b>html</b>, word<!-- mid -->word, a and x[l](u).\n" +
"Escapes: \\*not em\\*, \\`not code`, \\#nottag, \\@nouser, \\[not](link), a\\ *em* and \\\\.\n" +
"- item with *em* and `code`\n* [x] done\n- [ ] todo\n - nested **b**\n" +
"**Lead** strong, _lead_ em\n- **item** lead ~~x~~\n* * *\n- **`PI`**: code\n" +
"1. first\n10. tenth [l](u)\n> quote with *em*\n>not a quote\n\n" +
"```\nplain fence *x*\n```\n\n```go\nfunc f() {}\n```\n\n" +
"tab\tseparated\r\nwindows line\r\n" +
// Last: chroma's link rule runs to the next "](" anywhere after.
"[unclosed and no final newline"
func markdownTokens(t *testing.T, lx chroma.Lexer, src string) []chroma.Token {
t.Helper()
it, err := lx.Tokenise(nil, src)
if err != nil {
t.Fatal(err)
}
return it.Tokens()
}
// TestMarkdownProseRun checks proseRun changes how fast markdown lexes and
// nothing else: the same tokens, once merged, as the rules without it,
// over a sample of every rule and this repository's README.
func TestMarkdownProseRun(t *testing.T) {
r, err := New(nil, nil, "tokyonight-night")
if err != nil {
t.Fatal(err)
}
readme, err := os.ReadFile(filepath.Join("..", "..", "README.md"))
if err != nil {
t.Fatal(err)
}
stock, err := fenceMarkdown(r.fences, lineStart...)
if err != nil {
t.Fatal(err)
}
fast, err := fenceMarkdown(r.fences, markdownInline()...)
if err != nil {
t.Fatal(err)
}
for name, src := range map[string]string{"sample": markdownSample, "README.md": string(readme)} {
want := markdownTokens(t, chroma.Coalesce(stock), src)
got := markdownTokens(t, chroma.Coalesce(fast), src)
if !slices.Equal(got, want) {
for i := range min(len(got), len(want)) {
if got[i] != want[i] {
t.Errorf("%s: token %d is %v, without proseRun %v", name, i, got[i], want[i])
break
}
}
t.Errorf("%s: %d tokens, without proseRun %d", name, len(got), len(want))
}
// Unmerged, the run must have taken prose in bigger pieces, or
// it is no longer reached.
if n, m := len(markdownTokens(t, fast, src)), len(markdownTokens(t, stock, src)); n >= m {
t.Errorf("%s: %d raw tokens with proseRun, %d without; the rule is not doing its job", name, n, m)
}
}
}
// TestMarkdownLineStart checks emphasis at the start of a line and of a
// list item is highlighted as it is mid-line, and that what is not
// emphasis there stays plain.
func TestMarkdownLineStart(t *testing.T) {
r, err := New(nil, nil, "tokyonight-night")
if err != nil {
t.Fatal(err)
}
strong, emph, del := chroma.GenericStrong, chroma.GenericEmph, chroma.GenericDeleted
type tok struct {
typ chroma.TokenType
val string
}
cases := []struct {
src string
want []tok // every token not plain text or a list marker
}{
{"**Bold** at start\n", []tok{{strong, "**Bold**"}}},
{"__init__ method\n", []tok{{strong, "__init__"}}},
{"*em* first and _u_\n", []tok{{emph, "*em*"}, {emph, "_u_"}}},
{"~~gone~~ now\n", []tok{{del, "~~gone~~"}}},
{"- **Bold** lead\n", []tok{{strong, "**Bold**"}}},
{"* **Bold** lead\n", []tok{{strong, "**Bold**"}}},
{" - **nested** two\n", []tok{{strong, "**nested**"}}},
{"-\t**tab** sep\n", []tok{{strong, "**tab**"}}},
{"- *em* item\n", []tok{{emph, "*em*"}}},
{"- _u_ item\n", []tok{{emph, "_u_"}}},
{"- ~~gone~~ item\n", []tok{{del, "~~gone~~"}}},
{"- **a** and **b**\n", []tok{{strong, "**a**"}, {strong, "**b**"}}},
{"- *a* **b** ~~c~~\n", []tok{{emph, "*a*"}, {strong, "**b**"}, {del, "~~c~~"}}},
{"- **Search.** The renderer\n continued **line**\n", []tok{{strong, "**Search.**"}, {strong, "**line**"}}},
// Already highlighted before: a space precedes the marker.
{"- [x] **done** task\n", []tok{{strong, "**done**"}}},
{"1. **num** item\n", []tok{{strong, "**num**"}}},
{"* **wide** gap\n", []tok{{strong, "**wide**"}}},
// Not emphasis: rules, unclosed or spaced markers, no list.
{"* * *\n", nil},
{"- - -\n", nil},
{"***\n", nil},
{"- **unclosed bold\n", nil},
{"- * not em *\n", nil},
{"__ spaced__ x\n", nil},
{"~~ spaced~~ x\n", nil},
{"`c`~~mid~~ line\n", []tok{{chroma.LiteralStringBacktick, "`c`"}}},
{"-**nolist** x\n", nil},
{"- \\*\\*escaped\\*\\*\n", nil},
// Code inside keeps its highlighting, as before.
{"- `**code**` first\n", []tok{{chroma.LiteralStringBacktick, "`**code**`"}}},
{"- **`PI`**: the constant\n", []tok{{chroma.LiteralStringBacktick, "`PI`"}}},
{"*`c` em* x\n", []tok{{chroma.LiteralStringBacktick, "`c`"}}},
{"~~`c` gone~~ x\n", []tok{{chroma.LiteralStringBacktick, "`c`"}}},
}
for _, c := range cases {
var got []tok
for _, tk := range markdownTokens(t, chroma.Coalesce(r.markdown), c.src) {
if tk.Type != chroma.Text && tk.Type != chroma.Keyword {
got = append(got, tok{tk.Type, tk.Value})
}
}
if !slices.Equal(got, c.want) {
t.Errorf("%q: got %v, want %v", c.src, got, c.want)
}
}
}
|