Download
229 lines 8.6 KiBGoModified 2026-10-18 raw
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229
package highlight

import (
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"slices"
	"strings"
	"sync"
	"testing"

	"github.com/alecthomas/chroma/v2"
	"github.com/alecthomas/chroma/v2/lexers"
)

// sampleNames builds filenames that exercise every glob chroma knows:
// each glob instantiated, with and without backup suffixes and prefixes.
func sampleNames() []string {
	class := regexp.MustCompile(`\[(\^?)([^\]]*)\]`)
	names := []string{"", ".", "x", "noext", "a.b.c", "foo.tar.gz", "Makefile.am.in", "README", ".bashrc~"}
	for _, lx := range lexers.GlobalLexerRegistry.Lexers {
		cfg := lx.Config()
		for _, g := range append(cfg.Filenames, cfg.AliasFilenames...) {
			base := class.ReplaceAllStringFunc(g, func(m string) string {
				sub := class.FindStringSubmatch(m)
				if sub[1] != "" || sub[2] == "" {
					return "q"
				}
				return sub[2][:1]
			})
			for _, star := range []string{"", "a.b"} {
				n := strings.NewReplacer("*", star, "?", "z", `\`, "").Replace(base)
				names = append(names, n, n+"~", n+".in", "x"+n, strings.ToUpper(n))
			}
		}
	}
	slices.Sort(names)
	return slices.Compact(names)
}

func TestIndexMatchesChroma(t *testing.T) {
	ix := builtins()
	// chroma takes ~8ms a lookup, so spread the comparison over the CPUs.
	names := sampleNames()
	var wg sync.WaitGroup
	for w := range runtime.NumCPU() {
		wg.Go(func() {
			for i := w; i < len(names); i += runtime.NumCPU() {
				n := names[i]
				if got, want := Name(ix.match(n)), Name(lexers.Match(n)); got != want {
					t.Errorf("match(%q) = %s, chroma says %s", n, got, want)
				}
			}
		})
	}
	wg.Wait()
	gets := append(lexers.Names(true), "", "tomo", "py", "PY", "rs", "h", "m", "pl", "makefile", "Dockerfile", "nonsense", "php5", "1")
	for _, name := range append(gets, ix.names()...) {
		if got, want := Name(ix.get(name)), Name(lexers.Get(name)); got != want {
			t.Errorf("get(%q) = %s, chroma says %s", name, got, want)
		}
	}
}

func TestMarkdownFences(t *testing.T) {
	dir := t.TempDir()
	if err := os.WriteFile(filepath.Join(dir, "toy.xml"), []byte(toyLexer), 0o644); err != nil {
		t.Fatal(err)
	}
	r, err := New([]string{dir}, []Custom{{Name: "toy", Aliases: []string{"tm"}}}, "tokyonight-night")
	if err != nil {
		t.Fatal(err)
	}
	md := r.ByPath("README.md")
	if md != r.ByName("markdown") || md != r.markdown {
		t.Fatal("markdown files do not use the fence-aware lexer")
	}
	out := string(r.HTML([]byte("# Hi\n\n```tm\nzap 1\n```\n\n```py\ndef f(): pass\n```\n\n```\nzap\n```\n\n```nonsense\nzap\n```\n"), md))
	if !strings.Contains(out, `<span class="k">zap</span> <span class="m">1</span>`) {
		t.Errorf("custom lexer not used in fence:\n%s", out)
	}
	if !strings.Contains(out, `<span class="kd">def</span>`) && !strings.Contains(out, `<span class="k">def</span>`) {
		t.Errorf("builtin lexer not used in fence:\n%s", out)
	}
	if strings.Count(out, `<span class="k">zap</span>`) != 1 {
		t.Errorf("unlabelled or unknown fence was highlighted:\n%s", out)
	}
	// The shared chroma lexer must still use chroma's own registry.
	global := string(r.HTML([]byte("```tm\nzap 1\n```\n"), lexers.Markdown))
	if strings.Contains(global, `<span class="k">zap</span>`) {
		t.Error("chroma's global markdown lexer was rewired")
	}
	if !strings.Contains(string(r.HTML([]byte("---\ntitle: x\n---\n# Hi\n"), md)), `<span class="nt">title</span>`) {
		t.Error("frontmatter no longer highlighted")
	}
}

// markdownSample exercises every rule of chroma's markdown lexer, and the
// places a run of prose stops: at markup, mid-word and at line ends.
const markdownSample = "---\ntitle: x\n---\n" +
	"# Heading with *em*\n## Sub_heading\n\n" +
	"Plain prose, naïve café — ✓ snake_case and a*b and 2*3 = 6.\n" +
	"Some *em*, _em_, **strong**, __strong__, ~~gone~~ and `code`.\n" +
	"Mentions @user, #tag, issue#12, a/b@c:d and a lone # or @.\n" +
	"Links [text](http://x.y/z), ![img](a.png) and !bang.\n" +
	"<!-- a\ncomment --> and <b>html</b>, word<!-- mid -->word, a![img](b) and x[l](u).\n" +
	"Escapes: \\*not em\\*, \\`not code`, \\#nottag, \\@nouser, \\[not](link), a\\ *em* and \\\\.\n" +
	"- item with *em* and `code`\n* [x] done\n- [ ] todo\n  - nested **b**\n" +
	"**Lead** strong, _lead_ em\n- **item** lead ~~x~~\n* * *\n- **`PI`**: code\n" +
	"1. first\n10. tenth [l](u)\n> quote with *em*\n>not a quote\n\n" +
	"```\nplain fence *x*\n```\n\n```go\nfunc f() {}\n```\n\n" +
	"tab\tseparated\r\nwindows line\r\n" +
	// Last: chroma's link rule runs to the next "](" anywhere after.
	"[unclosed and no final newline"

func markdownTokens(t *testing.T, lx chroma.Lexer, src string) []chroma.Token {
	t.Helper()
	it, err := lx.Tokenise(nil, src)
	if err != nil {
		t.Fatal(err)
	}
	return it.Tokens()
}

// TestMarkdownProseRun checks proseRun changes how fast markdown lexes and
// nothing else: the same tokens, once merged, as the rules without it,
// over a sample of every rule and this repository's README.
func TestMarkdownProseRun(t *testing.T) {
	r, err := New(nil, nil, "tokyonight-night")
	if err != nil {
		t.Fatal(err)
	}
	readme, err := os.ReadFile(filepath.Join("..", "..", "README.md"))
	if err != nil {
		t.Fatal(err)
	}
	stock, err := fenceMarkdown(r.fences, lineStart...)
	if err != nil {
		t.Fatal(err)
	}
	fast, err := fenceMarkdown(r.fences, markdownInline()...)
	if err != nil {
		t.Fatal(err)
	}
	for name, src := range map[string]string{"sample": markdownSample, "README.md": string(readme)} {
		want := markdownTokens(t, chroma.Coalesce(stock), src)
		got := markdownTokens(t, chroma.Coalesce(fast), src)
		if !slices.Equal(got, want) {
			for i := range min(len(got), len(want)) {
				if got[i] != want[i] {
					t.Errorf("%s: token %d is %v, without proseRun %v", name, i, got[i], want[i])
					break
				}
			}
			t.Errorf("%s: %d tokens, without proseRun %d", name, len(got), len(want))
		}
		// Unmerged, the run must have taken prose in bigger pieces, or
		// it is no longer reached.
		if n, m := len(markdownTokens(t, fast, src)), len(markdownTokens(t, stock, src)); n >= m {
			t.Errorf("%s: %d raw tokens with proseRun, %d without; the rule is not doing its job", name, n, m)
		}
	}
}

// TestMarkdownLineStart checks emphasis at the start of a line and of a
// list item is highlighted as it is mid-line, and that what is not
// emphasis there stays plain.
func TestMarkdownLineStart(t *testing.T) {
	r, err := New(nil, nil, "tokyonight-night")
	if err != nil {
		t.Fatal(err)
	}
	strong, emph, del := chroma.GenericStrong, chroma.GenericEmph, chroma.GenericDeleted
	type tok struct {
		typ chroma.TokenType
		val string
	}
	cases := []struct {
		src  string
		want []tok // every token not plain text or a list marker
	}{
		{"**Bold** at start\n", []tok{{strong, "**Bold**"}}},
		{"__init__ method\n", []tok{{strong, "__init__"}}},
		{"*em* first and _u_\n", []tok{{emph, "*em*"}, {emph, "_u_"}}},
		{"~~gone~~ now\n", []tok{{del, "~~gone~~"}}},
		{"- **Bold** lead\n", []tok{{strong, "**Bold**"}}},
		{"* **Bold** lead\n", []tok{{strong, "**Bold**"}}},
		{"  - **nested** two\n", []tok{{strong, "**nested**"}}},
		{"-\t**tab** sep\n", []tok{{strong, "**tab**"}}},
		{"- *em* item\n", []tok{{emph, "*em*"}}},
		{"- _u_ item\n", []tok{{emph, "_u_"}}},
		{"- ~~gone~~ item\n", []tok{{del, "~~gone~~"}}},
		{"- **a** and **b**\n", []tok{{strong, "**a**"}, {strong, "**b**"}}},
		{"- *a* **b** ~~c~~\n", []tok{{emph, "*a*"}, {strong, "**b**"}, {del, "~~c~~"}}},
		{"- **Search.** The renderer\n  continued **line**\n", []tok{{strong, "**Search.**"}, {strong, "**line**"}}},
		// Already highlighted before: a space precedes the marker.
		{"- [x] **done** task\n", []tok{{strong, "**done**"}}},
		{"1. **num** item\n", []tok{{strong, "**num**"}}},
		{"*   **wide** gap\n", []tok{{strong, "**wide**"}}},
		// Not emphasis: rules, unclosed or spaced markers, no list.
		{"* * *\n", nil},
		{"- - -\n", nil},
		{"***\n", nil},
		{"- **unclosed bold\n", nil},
		{"- * not em *\n", nil},
		{"__ spaced__ x\n", nil},
		{"~~ spaced~~ x\n", nil},
		{"`c`~~mid~~ line\n", []tok{{chroma.LiteralStringBacktick, "`c`"}}},
		{"-**nolist** x\n", nil},
		{"- \\*\\*escaped\\*\\*\n", nil},
		// Code inside keeps its highlighting, as before.
		{"- `**code**` first\n", []tok{{chroma.LiteralStringBacktick, "`**code**`"}}},
		{"- **`PI`**: the constant\n", []tok{{chroma.LiteralStringBacktick, "`PI`"}}},
		{"*`c` em* x\n", []tok{{chroma.LiteralStringBacktick, "`c`"}}},
		{"~~`c` gone~~ x\n", []tok{{chroma.LiteralStringBacktick, "`c`"}}},
	}
	for _, c := range cases {
		var got []tok
		for _, tk := range markdownTokens(t, chroma.Coalesce(r.markdown), c.src) {
			if tk.Type != chroma.Text && tk.Type != chroma.Keyword {
				got = append(got, tok{tk.Type, tk.Value})
			}
		}
		if !slices.Equal(got, c.want) {
			t.Errorf("%q: got %v, want %v", c.src, got, c.want)
		}
	}
}