123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206 | package render
import (
"bufio"
"compress/gzip"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"os"
"path"
"strings"
"time"
)
// ---- search corpus ----
type searchView struct {
DataURL string
MaxResults int
}
func (c *repoCtx) excluded(p string) bool {
base := path.Base(p)
for _, g := range c.s.Cfg.SearchExclude {
if ok, _ := path.Match(g, base); ok {
return true
}
if ok, _ := path.Match(g, p); ok {
return true
}
}
return false
}
func splitLines(data []byte) []string {
s := string(data)
lines := strings.Split(s, "\n")
if len(lines) > 0 && lines[len(lines)-1] == "" {
lines = lines[:len(lines)-1]
}
for i, l := range lines {
lines[i] = strings.TrimSuffix(l, "\r")
}
return lines
}
// writeSearchData streams the whole-repo text corpus to corpus.json and
// its .gz twin, and computes the per-language line counts on the way.
func (c *repoCtx) writeSearchData() (pending, error) {
lay := c.s.Lay
cfg := c.s.Cfg
batch := c.batch
target := lay.SearchData(c.key)
jsonTmp, err := createTemp(target)
if err != nil {
return pending{}, err
}
gzTmp, err := createTemp(target + ".gz")
if err != nil {
jsonTmp.discard()
return pending{}, err
}
discard := func() {
jsonTmp.discard()
gzTmp.discard()
}
fail := func(err error) (pending, error) {
discard()
return pending{}, err
}
zw := gzipWriters.Get().(*gzip.Writer)
defer gzipWriters.Put(zw)
zw.Reset(gzTmp)
// The page fetches the corpus by its hash, so it can be cached for
// good: a config change can rewrite it without a new commit.
sum := sha256.New()
w := bufio.NewWriterSize(io.MultiWriter(jsonTmp, zw, sum), 256<<10)
// The corpus is fetched as data, never inlined into a page, so <, >
// and & need no \u escapes; source code is full of them.
enc := json.NewEncoder(w)
enc.SetEscapeHTML(false)
header, err := json.Marshal(struct {
Repo string `json:"repo"`
SHA string `json:"sha"`
Generated string `json:"generated"`
}{c.repo, c.sha, time.Now().UTC().Format(time.RFC3339)})
if err != nil {
return fail(err)
}
// Reopen the header object so "files" can stream in.
header = header[:len(header)-1]
if _, err := fmt.Fprintf(w, `%s,"files":[`, header); err != nil {
return fail(err)
}
c.typed = map[string][]typedFile{}
first := true
type fileObj struct {
P string `json:"p"`
L []string `json:"l"`
}
c.lines, c.prose = map[string]int{}, map[string]bool{}
cache := map[string]int{}
for _, p := range c.tree.sortedPaths() {
e := c.tree.byPath[p]
if e.IsSymlink() || e.IsSubmodule() {
continue
}
inCorpus := e.Size <= cfg.SearchMaxBytes && !c.excluded(p)
n, known := c.lineCache[e.SHA]
if !inCorpus {
// Listings still want its line count: from the cache, or by
// streaming it through once, however big it is.
if !known {
var lc lineCounter
if err := batch.Copy(e.SHA, &lc); err != nil {
return fail(err)
}
n = lc.count()
}
c.lines[p], cache[e.SHA] = n, n
c.prose[p] = proseLangs[c.lang(p, c.s.HL.ByPath(p))]
continue
}
data, err := batch.Get(e.SHA)
if err != nil {
return fail(err)
}
n = countLines(data)
c.lines[p], cache[e.SHA] = n, n
if n < 0 {
continue
}
lines := splitLines(data)
// Files that count for nothing are left out, here and on the
// type pages.
if lang := c.lang(p, c.s.HL.ByFile(p, data)); proseLangs[lang] {
c.prose[p] = true
if n := countWords(data); n > 0 {
c.typed[rowDocs] = append(c.typed[rowDocs], typedFile{p, lang, n})
}
} else if n := countNonBlank(lines); n > 0 {
c.typed[rowCode] = append(c.typed[rowCode], typedFile{p, lang, n})
}
if !first {
if err := w.WriteByte(','); err != nil {
return fail(err)
}
}
first = false
if err := enc.Encode(fileObj{P: p, L: lines}); err != nil {
return fail(err)
}
}
if _, err := io.WriteString(w, "]}\n"); err != nil {
return fail(err)
}
if err := w.Flush(); err != nil {
return fail(err)
}
if err := zw.Close(); err != nil {
return fail(err)
}
for _, t := range []*tempFile{jsonTmp, gzTmp} {
if err := t.finish(); err != nil {
return fail(err)
}
}
c.corpusHash = hex.EncodeToString(sum.Sum(nil))[:16]
c.typed[rowBinary] = c.binaryFiles()
c.buildStats()
c.lineCache = cache
c.sumSizes()
commit := func() error {
if err := os.Remove(target + ".gz"); err != nil && !errors.Is(err, fs.ErrNotExist) {
return err
}
if err := jsonTmp.commit(); err != nil {
return err
}
return gzTmp.commit()
}
return pending{commit: commit, discard: discard}, nil
}
func (c *repoCtx) renderSearchPage() error {
page := c.page()
page.Title = "Search " + c.repo + c.branchSuffix()
page.URL = c.s.absURL(c.s.SearchURL(c.key))
page.BodyClass = "search"
page.Drawer = c.drawer("")
page.Search = &searchView{DataURL: c.s.SearchDataURL(c.key) + "?v=" + c.corpusHash, MaxResults: c.s.Cfg.MaxResults}
html, err := c.s.render("search", page)
if err != nil {
return err
}
return writePair(c.s.Lay.SearchPage(c.key), html, true)
}
|