123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574 | // Package highlight is the single place that turns source code into HTML.
// File pages, markdown fenced blocks and README rendering all call HTML, so
// identical input always yields identical markup regardless of caller.
package highlight
import (
"bytes"
"embed"
"errors"
"fmt"
"html"
"html/template"
"io"
"io/fs"
"maps"
"os"
"path/filepath"
"reflect"
"slices"
"strings"
"time"
"github.com/alecthomas/chroma/v2"
chromahtml "github.com/alecthomas/chroma/v2/formatters/html"
"github.com/alecthomas/chroma/v2/lexers"
"github.com/alecthomas/chroma/v2/styles"
)
// Custom describes a lexer configured in repo-render.yaml.
type Custom struct {
Name string
Globs []string
Aliases []string
}
type globEntry struct {
glob string
lexer chroma.Lexer
}
// Registry resolves lexers by path or name and renders code.
type Registry struct {
style *chroma.Style
light *chroma.Style
globs []globEntry
aliases map[string]chroma.Lexer
plain chroma.Lexer
// markdown is chroma's markdown lexer, but looking up fenced-block
// languages in fences rather than chroma's global registry.
markdown chroma.Lexer
fences *chroma.LexerRegistry
}
// New builds a registry. Custom lexers are looked up as <dir>/<name>.xml in
// each of lexerDirs in order, then as chroma builtins; a name matching
// neither is an error. Missing directories are skipped.
func New(lexerDirs []string, custom []Custom, styleName string) (*Registry, error) {
st := styles.Fallback
if styleName != "" {
var err error
if st, err = findStyle(styleName); err != nil {
return nil, err
}
}
r := &Registry{
style: st,
aliases: map[string]chroma.Lexer{},
plain: lexers.Get("plaintext"),
}
r.fences = chroma.NewLexerRegistry()
md, err := fenceMarkdown(r.fences, markdownInline()...)
if err != nil {
return nil, err
}
r.markdown = md
// Globs are tried in name order whatever order the caller has, and
// the caller's slice is left as it is.
custom = slices.SortedFunc(slices.Values(custom), func(a, b Custom) int { return strings.Compare(a.Name, b.Name) })
for _, c := range custom {
lx, err := loadLexer(lexerDirs, c.Name)
if err != nil {
return nil, err
}
for _, g := range c.Globs {
r.globs = append(r.globs, globEntry{glob: g, lexer: lx})
}
r.aliases[strings.ToLower(c.Name)] = lx
for _, a := range c.Aliases {
r.aliases[strings.ToLower(a)] = lx
}
}
r.fillFences()
return r, nil
}
// Bundled holds the lexers that ship with the renderer (lexers/*.xml),
// used for a name no lexer directory has.
//
//go:embed lexers/*.xml
var Bundled embed.FS
// loadLexer returns the lexer called name: from name.xml in the first of
// lexerDirs that has one, or else the bundled one, or else chroma's
// builtin of that name.
func loadLexer(lexerDirs []string, name string) (chroma.Lexer, error) {
for _, dir := range lexerDirs {
if _, err := os.Stat(filepath.Join(dir, name+".xml")); err != nil {
continue
}
lx, err := chroma.NewXMLLexer(os.DirFS(dir), name+".xml")
if err != nil {
return nil, fmt.Errorf("lexer %s: %w", name, err)
}
return lx, nil
}
if sub, err := fs.Sub(Bundled, "lexers"); err == nil {
if _, err := fs.Stat(sub, name+".xml"); err == nil {
lx, err := chroma.NewXMLLexer(sub, name+".xml")
if err != nil {
return nil, fmt.Errorf("bundled lexer %s: %w", name, err)
}
return lx, nil
}
}
if lx := lexers.Get(name); lx != nil {
return lx, nil
}
where := "the bundled lexers"
if len(lexerDirs) > 0 {
where = strings.Join(lexerDirs, ", ") + " or " + where
}
return nil, fmt.Errorf("lexer %q: no %s.xml in %s and no chroma builtin by that name", name, name, where)
}
// proseRun lexes a run of plain markdown text in one match. chroma's inline
// state ends in `.|\n`, so prose otherwise goes a character at a time, each
// only after every other rule has failed there: three times slower. The
// run stops at each character that begins another inline rule (whitespace
// for emphasis, < for comments, \ for escapes, ` for code, @ and # for
// mentions, [ for links), so it never covers a position where one could
// match. Inline is tried only after root's rules, which need the start of
// a line or, for comments, a <.
var proseRun = chroma.Rule{Pattern: "[^\\s<\\\\`@#\\[]+", Type: chroma.Text}
// lineStart has chroma's emphasis, strong and strikethrough rules again
// for the start of a line. Theirs need a space before the marker, which a
// paragraph's first word lacks, and so does a "- " or "* " list item's,
// since chroma lexes the item's text on its own. The marked text must not
// start with a space, so "* * *" stays a rule, nor hold a backtick, so
// code inside keeps its own highlighting. Mid-line, proseRun never covers
// a line start, so these only have to come before it.
var lineStart = []chroma.Rule{
{Pattern: "^(\\*|_)(?!\\s)((?:(?!\\1)[^`\\n])+)(\\1)((?=\\W|\\n))", Type: chroma.ByGroups(chroma.GenericEmph, chroma.GenericEmph, chroma.GenericEmph, chroma.Text)},
{Pattern: "^((\\*\\*|__)(?!\\s)[^`\\n]+?)\\2((?=\\W|\\n))", Type: chroma.ByGroups(chroma.GenericStrong, chroma.GenericStrong, chroma.Text)},
{Pattern: "^(~~(?!\\s)[^~`]+~~)((?=\\W|\\n))", Type: chroma.ByGroups(chroma.GenericDeleted, chroma.Text)},
}
// markdownInline is what the markdown lexer tries first in its inline
// state, in order.
func markdownInline() []chroma.Rule { return append(slices.Clone(lineStart), proseRun) }
// fenceMarkdown returns a copy of chroma's markdown lexer that resolves
// fenced-block languages in reg. The original uses chroma's global
// registry, which knows nothing of custom lexers and, for a fence that
// names no known lexer (or none at all), scans every filename glob twice.
// Rules in inline are tried first in the copy's inline state. chroma's
// lexer is a frontmatter-handling wrapper around a RegexLexer; if a chroma
// upgrade changes that shape, this fails rather than quietly losing custom
// fence languages.
func fenceMarkdown(reg *chroma.LexerRegistry, inline ...chroma.Rule) (chroma.Lexer, error) {
unexpected := errors.New("chroma's markdown lexer is not the wrapped RegexLexer fenceMarkdown expects")
md := reflect.ValueOf(lexers.Markdown)
if md.Kind() != reflect.Pointer || md.Elem().Kind() != reflect.Struct {
return nil, unexpected
}
inner := md.Elem().FieldByName("Lexer")
if !inner.IsValid() {
return nil, unexpected
}
base, ok := reflect.TypeAssert[*chroma.RegexLexer](inner)
if !ok {
return nil, unexpected
}
rules, err := base.Rules()
if err != nil {
return nil, err
}
if len(inline) > 0 {
own := maps.Clone(rules)
own["inline"] = append(slices.Clone(inline), rules["inline"]...)
rules = own
}
clone := chroma.MustNewLexer(base.Config(), func() chroma.Rules { return rules })
clone.SetRegistry(reg)
out := reflect.New(md.Elem().Type())
out.Elem().Set(md.Elem())
out.Elem().FieldByName("Lexer").Set(reflect.ValueOf(clone))
lx, ok := reflect.TypeAssert[chroma.Lexer](out)
if !ok {
return nil, unexpected
}
return lx, nil
}
// fenceAlias registers a lexer in the fence registry under extra names.
// Its SetRegistry does nothing, so registering it leaves chroma's shared
// lexers pointing at chroma's registry.
type fenceAlias struct {
chroma.Lexer
cfg *chroma.Config
}
func (a *fenceAlias) Config() *chroma.Config { return a.cfg }
func (a *fenceAlias) SetRegistry(*chroma.LexerRegistry) chroma.Lexer { return a }
// fillFences registers every name ByName knows other than through a
// wildcard glob, then empties the registry's lexer list: chroma's Get
// falls back to Match over that list, so a miss now costs nothing and
// leaves the block unhighlighted, as it always did.
func (r *Registry) fillFences() {
custom := map[chroma.Lexer]bool{}
for _, lx := range r.aliases {
custom[lx] = true
}
groups := map[chroma.Lexer]*fenceAlias{}
var order []chroma.Lexer
add := func(name string, lx chroma.Lexer) {
a, ok := groups[lx]
if !ok {
final := lx
if !custom[lx] {
final = r.builtin(lx)
}
a = &fenceAlias{Lexer: final, cfg: &chroma.Config{Name: fmt.Sprintf("\x00fence%d", len(order))}}
groups[lx] = a
order = append(order, lx)
}
a.cfg.Aliases = append(a.cfg.Aliases, name)
}
for name, lx := range r.aliases {
add(name, lx)
}
for _, name := range builtins().names() {
if _, ok := r.aliases[name]; ok {
continue
}
if lx := builtins().get(name); lx != nil {
add(name, lx)
}
}
for _, lx := range order {
r.fences.Register(groups[lx])
}
r.fences.Lexers = nil
}
// builtin readies a lexer from chroma's registry for use, or returns nil.
func (r *Registry) builtin(lx chroma.Lexer) chroma.Lexer {
switch lx {
case nil:
return nil
case lexers.Markdown:
return r.markdown
}
return lx
}
// ByPath picks a lexer for a file path: repo-render.yaml globs, then
// chroma's filename registry, then plain text. Never nil.
func (r *Registry) ByPath(path string) chroma.Lexer {
base := filepath.Base(path)
for _, g := range r.globs {
if ok, _ := filepath.Match(g.glob, base); ok {
return g.lexer
}
}
if lx := r.builtin(builtins().match(base)); lx != nil {
return lx
}
return r.plain
}
// ByFile is ByPath, except that a file its name leaves as plain text is
// looked up by the interpreter on its "#!" line ("#!/bin/sh",
// "#!/usr/bin/env python3"), so extensionless scripts are still code.
func (r *Registry) ByFile(path string, data []byte) chroma.Lexer {
lx := r.ByPath(path)
if !r.IsPlain(lx) {
return lx
}
if sb := r.byShebang(data); sb != nil {
return sb
}
return lx
}
// byShebang returns the lexer named by data's "#!" line, or nil.
func (r *Registry) byShebang(data []byte) chroma.Lexer {
if !bytes.HasPrefix(data, []byte("#!")) {
return nil
}
line := data[2:]
if i := bytes.IndexByte(line, '\n'); i >= 0 {
line = line[:i]
}
if len(line) > 256 {
return nil
}
fields := strings.Fields(string(line))
if len(fields) == 0 {
return nil
}
name := filepath.Base(fields[0])
if name == "env" {
// Skip env's options and VAR=value assignments.
name = ""
for _, f := range fields[1:] {
if !strings.HasPrefix(f, "-") && !strings.Contains(f, "=") {
name = filepath.Base(f)
break
}
}
}
// "python3.12" → "python3" → "python".
for _, n := range []string{name, strings.TrimRight(name, "0123456789.")} {
if n == "" {
continue
}
if lx, ok := r.aliases[strings.ToLower(n)]; ok {
return lx
}
if lx := r.builtin(builtins().get(n)); lx != nil && !r.IsPlain(lx) {
return lx
}
}
return nil
}
// ByName picks a lexer for a fence info string or alias. Never nil.
func (r *Registry) ByName(name string) chroma.Lexer {
name = strings.ToLower(strings.TrimSpace(name))
if name == "" {
return r.plain
}
if lx, ok := r.aliases[name]; ok {
return lx
}
if lx := r.builtin(builtins().get(name)); lx != nil {
return lx
}
return r.plain
}
// Name returns the display name of a lexer.
func Name(lx chroma.Lexer) string {
if lx == nil {
return "Text"
}
return lx.Config().Name
}
// IsPlain reports whether lx is the plain text fallback.
func (r *Registry) IsPlain(lx chroma.Lexer) bool {
return lx == nil || lx.Config().Name == r.plain.Config().Name
}
// Limits on highlighting one input, past which HTML gives up and shows it
// plain. Some of chroma's lexers can loop forever without consuming input
// (Jungle on an unclosed '"', JSONata on a control character, chroma issue
// #1377), emitting only empty tokens; others could just be very slow.
// Either would stall the render while it holds the render lock.
const (
maxEmptyTokens = 1000
highlightTime = 5 * time.Second
)
// HTML renders src as a <code class="chroma"> element with one span per
// token, classes only (no inline styles), tabs preserved. Line numbers and
// anchors are the caller's job. Input a lexer cannot finish within the
// limits above, or panics on (YANG on "/*/*"), is shown unhighlighted.
func (r *Registry) HTML(src []byte, lx chroma.Lexer) template.HTML {
if lx == nil {
lx = r.plain
}
if out, ok := r.tokensHTML(src, lx); ok {
return out
}
return template.HTML(`<code class="chroma">` + html.EscapeString(string(src)) + `</code>`)
}
// tokensHTML is HTML's highlighted rendering, or false when lexing fails,
// panics or passes a limit.
func (r *Registry) tokensHTML(src []byte, lx chroma.Lexer) (out template.HTML, ok bool) {
defer func() {
if recover() != nil {
out, ok = "", false
}
}()
it, err := lx.Tokenise(nil, string(src))
if err != nil {
return "", false
}
var b strings.Builder
b.Grow(len(src) * 2)
b.WriteString(`<code class="chroma">`)
write := func(tok chroma.Token) {
cls := class(tok.Type)
if cls == "" {
b.WriteString(html.EscapeString(tok.Value))
return
}
b.WriteString(`<span class="`)
b.WriteString(cls)
b.WriteString(`">`)
b.WriteString(html.EscapeString(tok.Value))
b.WriteString(`</span>`)
}
// Tokens are merged into runs of one type as chroma.Coalesce would,
// but here, where the limits can see every token: Coalesce drops the
// empty ones inside its own loop, so a lexer looping on them would
// never return from it.
deadline := time.Now().Add(highlightTime)
empty := 0
var run chroma.Token
for tok := it(); tok != chroma.EOF; tok = it() {
if time.Now().After(deadline) {
return "", false
}
if tok.Value == "" {
if empty++; empty > maxEmptyTokens {
return "", false
}
continue
}
empty = 0
if run.Value != "" && run.Type == tok.Type && len(run.Value) < 8192 {
run.Value += tok.Value
continue
}
if run.Value != "" {
write(run)
}
run = tok
}
if run.Value != "" {
write(run)
}
b.WriteString(`</code>`)
return template.HTML(b.String()), true
}
// class mirrors chroma's HTML formatter class naming so WriteCSS output
// matches: walk up the token hierarchy to the first standard class.
func class(t chroma.TokenType) string {
for t != 0 {
if cls, ok := chroma.StandardTypes[t]; ok {
return cls
}
t = t.Parent()
}
return chroma.StandardTypes[t]
}
// LineCount returns the number of lines src occupies on a file page.
func LineCount(src []byte) int {
if len(src) == 0 {
return 0
}
n := bytes.Count(src, []byte{'\n'})
if src[len(src)-1] != '\n' {
n++
}
return n
}
// findStyle looks up a chroma style by name. chroma's Get answers an
// unknown name with its fallback style, so that is an error here unless it
// was the one asked for.
func findStyle(name string) (*chroma.Style, error) {
st := styles.Get(name)
if st == nil || (st == styles.Fallback && st.Name != name) {
return nil, fmt.Errorf("unknown highlight style %q", name)
}
return st, nil
}
// SetLight selects the style used in light mode.
func (r *Registry) SetLight(name string) error {
st, err := findStyle(name)
if err != nil {
return err
}
r.light = st
return nil
}
// WriteCSS writes the syntax stylesheet. With no light style it is the dark
// style as is. With one, the two are mutually exclusive: layering light
// rules over dark ones would leave any property the light style does not
// set (token backgrounds, bold) leaking through. Each style is emitted for
// the user's explicit choice (:root[data-theme]) and, when they have made
// none, for the system preference.
func (r *Registry) WriteCSS(w io.Writer) error {
f := chromahtml.New(chromahtml.WithClasses(true), chromahtml.WithAllClasses(true))
var dark bytes.Buffer
if err := f.WriteCSS(&dark, r.style); err != nil {
return err
}
if r.light == nil {
_, err := w.Write(dark.Bytes())
return err
}
var light bytes.Buffer
if err := f.WriteCSS(&light, r.light); err != nil {
return err
}
_, err := fmt.Fprintf(w, "/* dark */\n%s@media (prefers-color-scheme: dark) {\n%s}\n/* light */\n%s@media (prefers-color-scheme: light) {\n%s}\n",
scopeCSS(dark.String(), `:root[data-theme="dark"] `, false),
scopeCSS(dark.String(), `:root:not([data-theme]) `, true),
scopeCSS(light.String(), `:root[data-theme="light"] `, false),
scopeCSS(light.String(), `:root:not([data-theme]) `, true))
return err
}
// scopeCSS prefixes the selector of every one-line rule chroma emits
// ("/* Name */ .sel { ... }") with prefix.
func scopeCSS(css, prefix string, indent bool) string {
var b strings.Builder
for line := range strings.SplitSeq(css, "\n") {
_, rule, ok := strings.Cut(line, "*/ ")
if !ok || !strings.Contains(line, "{") {
continue
}
if indent {
b.WriteString(" ")
}
b.WriteString(prefix)
b.WriteString(rule)
b.WriteString("\n")
}
return b.String()
}
// StyleName returns the configured chroma style name.
func (r *Registry) StyleName() string { return r.style.Name }
// TextColour returns the style's foreground colour as "#rrggbb", or "" if
// the style does not set one.
func (r *Registry) TextColour() string { return textColour(r.style) }
// LightTextColour is TextColour for the light style ("" when none is set).
func (r *Registry) LightTextColour() string { return textColour(r.light) }
func textColour(st *chroma.Style) string {
if st == nil {
return ""
}
if e := st.Get(chroma.Text); e.Colour.IsSet() {
return e.Colour.String()
}
return ""
}
// BinaryProbe is how much of the start of a file IsBinary looks at.
const BinaryProbe = 8192
// IsBinary applies git's heuristic: a NUL in the first 8 KiB means binary.
func IsBinary(data []byte) bool {
if len(data) > BinaryProbe {
data = data[:BinaryProbe]
}
return bytes.IndexByte(data, 0) >= 0
}
|