Download
206 lines 4.9 KiBGoModified 2026-10-27 raw
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206
package render

import (
	"bufio"
	"compress/gzip"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/fs"
	"os"
	"path"
	"strings"
	"time"
)

// ---- search corpus ----

type searchView struct {
	DataURL    string
	MaxResults int
}

func (c *repoCtx) excluded(p string) bool {
	base := path.Base(p)
	for _, g := range c.s.Cfg.SearchExclude {
		if ok, _ := path.Match(g, base); ok {
			return true
		}
		if ok, _ := path.Match(g, p); ok {
			return true
		}
	}
	return false
}

func splitLines(data []byte) []string {
	s := string(data)
	lines := strings.Split(s, "\n")
	if len(lines) > 0 && lines[len(lines)-1] == "" {
		lines = lines[:len(lines)-1]
	}
	for i, l := range lines {
		lines[i] = strings.TrimSuffix(l, "\r")
	}
	return lines
}

// writeSearchData streams the whole-repo text corpus to corpus.json and
// its .gz twin, and computes the per-language line counts on the way.
func (c *repoCtx) writeSearchData() (pending, error) {
	lay := c.s.Lay
	cfg := c.s.Cfg
	batch := c.batch

	target := lay.SearchData(c.key)
	jsonTmp, err := createTemp(target)
	if err != nil {
		return pending{}, err
	}
	gzTmp, err := createTemp(target + ".gz")
	if err != nil {
		jsonTmp.discard()
		return pending{}, err
	}
	discard := func() {
		jsonTmp.discard()
		gzTmp.discard()
	}
	fail := func(err error) (pending, error) {
		discard()
		return pending{}, err
	}

	zw := gzipWriters.Get().(*gzip.Writer)
	defer gzipWriters.Put(zw)
	zw.Reset(gzTmp)
	// The page fetches the corpus by its hash, so it can be cached for
	// good: a config change can rewrite it without a new commit.
	sum := sha256.New()
	w := bufio.NewWriterSize(io.MultiWriter(jsonTmp, zw, sum), 256<<10)
	// The corpus is fetched as data, never inlined into a page, so <, >
	// and & need no \u escapes; source code is full of them.
	enc := json.NewEncoder(w)
	enc.SetEscapeHTML(false)

	header, err := json.Marshal(struct {
		Repo      string `json:"repo"`
		SHA       string `json:"sha"`
		Generated string `json:"generated"`
	}{c.repo, c.sha, time.Now().UTC().Format(time.RFC3339)})
	if err != nil {
		return fail(err)
	}
	// Reopen the header object so "files" can stream in.
	header = header[:len(header)-1]
	if _, err := fmt.Fprintf(w, `%s,"files":[`, header); err != nil {
		return fail(err)
	}

	c.typed = map[string][]typedFile{}
	first := true
	type fileObj struct {
		P string   `json:"p"`
		L []string `json:"l"`
	}
	c.lines, c.prose = map[string]int{}, map[string]bool{}
	cache := map[string]int{}
	for _, p := range c.tree.sortedPaths() {
		e := c.tree.byPath[p]
		if e.IsSymlink() || e.IsSubmodule() {
			continue
		}
		inCorpus := e.Size <= cfg.SearchMaxBytes && !c.excluded(p)
		n, known := c.lineCache[e.SHA]
		if !inCorpus {
			// Listings still want its line count: from the cache, or by
			// streaming it through once, however big it is.
			if !known {
				var lc lineCounter
				if err := batch.Copy(e.SHA, &lc); err != nil {
					return fail(err)
				}
				n = lc.count()
			}
			c.lines[p], cache[e.SHA] = n, n
			c.prose[p] = proseLangs[c.lang(p, c.s.HL.ByPath(p))]
			continue
		}
		data, err := batch.Get(e.SHA)
		if err != nil {
			return fail(err)
		}
		n = countLines(data)
		c.lines[p], cache[e.SHA] = n, n
		if n < 0 {
			continue
		}
		lines := splitLines(data)
		// Files that count for nothing are left out, here and on the
		// type pages.
		if lang := c.lang(p, c.s.HL.ByFile(p, data)); proseLangs[lang] {
			c.prose[p] = true
			if n := countWords(data); n > 0 {
				c.typed[rowDocs] = append(c.typed[rowDocs], typedFile{p, lang, n})
			}
		} else if n := countNonBlank(lines); n > 0 {
			c.typed[rowCode] = append(c.typed[rowCode], typedFile{p, lang, n})
		}
		if !first {
			if err := w.WriteByte(','); err != nil {
				return fail(err)
			}
		}
		first = false
		if err := enc.Encode(fileObj{P: p, L: lines}); err != nil {
			return fail(err)
		}
	}
	if _, err := io.WriteString(w, "]}\n"); err != nil {
		return fail(err)
	}
	if err := w.Flush(); err != nil {
		return fail(err)
	}
	if err := zw.Close(); err != nil {
		return fail(err)
	}
	for _, t := range []*tempFile{jsonTmp, gzTmp} {
		if err := t.finish(); err != nil {
			return fail(err)
		}
	}

	c.corpusHash = hex.EncodeToString(sum.Sum(nil))[:16]
	c.typed[rowBinary] = c.binaryFiles()
	c.buildStats()
	c.lineCache = cache
	c.sumSizes()
	commit := func() error {
		if err := os.Remove(target + ".gz"); err != nil && !errors.Is(err, fs.ErrNotExist) {
			return err
		}
		if err := jsonTmp.commit(); err != nil {
			return err
		}
		return gzTmp.commit()
	}
	return pending{commit: commit, discard: discard}, nil
}

func (c *repoCtx) renderSearchPage() error {
	page := c.page()
	page.Title = "Search " + c.repo + c.branchSuffix()
	page.URL = c.s.absURL(c.s.SearchURL(c.key))
	page.BodyClass = "search"
	page.Drawer = c.drawer("")
	page.Search = &searchView{DataURL: c.s.SearchDataURL(c.key) + "?v=" + c.corpusHash, MaxResults: c.s.Cfg.MaxResults}
	html, err := c.s.render("search", page)
	if err != nil {
		return err
	}
	return writePair(c.s.Lay.SearchPage(c.key), html, true)
}