123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569 | package render
import (
"bytes"
"cmp"
"errors"
"fmt"
"io/fs"
"os"
"path"
"path/filepath"
"slices"
"strconv"
"strings"
"unicode"
"github.com/alecthomas/chroma/v2"
"code.bruce-hill.com/repo-renderer/internal/highlight"
)
// ---- files by type ----
// The rows of the root page's size summary, each measured in its own unit.
const (
rowCode = "code" // non-blank lines
rowDocs = "docs" // words
rowBinary = "binary" // bytes
)
// typedFile is one file's share of a summary row: its language or format,
// before display naming, and its size in the row's unit.
type typedFile struct {
Path string
Type string
Count int
}
// LangStat is a count for one language or format: non-blank lines of code,
// words of documentation, or bytes of binary data.
type LangStat struct {
Name string
Count int
URL string // the page listing its files
Title string // tooltip: "5 lines of C across 2 files"
slug string
files []typedFile
}
// statsView is the root page's size summary: code in non-blank lines, docs
// in words, binary data in bytes.
type statsView struct {
Code []LangStat
CodeLines int
CodeTitle string
Docs []LangStat
DocWords int
DocsTitle string
Binary []LangStat
BinaryBytes int
BinaryTitle string
}
// typeView is a page listing the files of one type.
type typeView struct {
Name string // "C", "PNG", or "Other" for the folded tail
Row string
Summary string // "2 files · 5 lines of code"
Entries []entryView
}
// buildStats folds c.typed into the root page's summary, each pill linking
// to its type's page and holding the files that page lists.
func (c *repoCtx) buildStats() {
row := func(name string) ([]LangStat, int, string) {
m := map[string]int{}
for _, f := range c.typed[name] {
m[f.Type] += f.Count
}
stats := foldStats(m, 8)
shown := map[string]int{}
used := map[string]bool{}
total := 0
for i := range stats {
shown[stats[i].Name] = i
total += stats[i].Count
slug := typeSlug(stats[i].Name)
for n := 2; used[slug]; n++ {
slug = typeSlug(stats[i].Name) + "-" + strconv.Itoa(n)
}
used[slug] = true
stats[i].slug, stats[i].URL = slug, c.s.TypeURL(c.key, name, slug)
}
for _, f := range c.typed[name] {
i, ok := shown[displayLang(f.Type)]
if !ok {
i = shown[otherName]
}
stats[i].files = append(stats[i].files, f)
}
for i := range stats {
what := stats[i].Name
if what == otherName {
what = otherTypes[name]
}
stats[i].Title = rowTitle(name, stats[i].Count, what, len(stats[i].files))
}
return stats, total, rowTitle(name, total, rowWhat[name], len(c.typed[name]))
}
c.stats = &statsView{}
c.stats.Code, c.stats.CodeLines, c.stats.CodeTitle = row(rowCode)
c.stats.Docs, c.stats.DocWords, c.stats.DocsTitle = row(rowDocs)
c.stats.Binary, c.stats.BinaryBytes, c.stats.BinaryTitle = row(rowBinary)
}
// rowWhat and otherTypes name what a row's total and its "Other" pill
// measure, for tooltips.
var (
rowWhat = map[string]string{rowCode: "code", rowDocs: "documentation", rowBinary: "binary data"}
otherTypes = map[string]string{rowCode: "other languages", rowDocs: "other formats", rowBinary: "other formats"}
)
// rowTitle is a pill's tooltip: "4.4K lines of C across 31 files", with a
// rough page count for docs.
func rowTitle(row string, n int, what string, files int) string {
noun := " files"
if files == 1 {
noun = " file"
}
t := rowAmount(row, n) + " of " + what + " across " + humanCount(files) + noun
if row == rowDocs {
t += " (≈ " + pageCount(n) + ")"
}
return t
}
// typeSlug makes a type's name into a file name: "C++" → "cpp",
// "Objective-C" → "objective-c".
func typeSlug(name string) string {
var b strings.Builder
dash := false
for _, r := range strings.ToLower(name) {
switch {
case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
b.WriteRune(r)
case r == '+':
b.WriteByte('p')
case r == '#':
b.WriteString("sharp")
default:
if b.Len() > 0 && !dash {
b.WriteByte('-')
dash = true
}
continue
}
dash = false
}
if s := strings.TrimSuffix(b.String(), "-"); s != "" {
return s
}
return "other"
}
// renderTypePages writes a page for every pill in the size summary and
// removes the pages of types that are gone.
func (c *repoCtx) renderTypePages() error {
lay := c.s.Lay
written := map[string]bool{}
for _, row := range []struct {
name string
stats []LangStat
}{{rowCode, c.stats.Code}, {rowDocs, c.stats.Docs}, {rowBinary, c.stats.Binary}} {
for _, s := range row.stats {
page := lay.TypePage(c.key, row.name, s.slug)
if err := c.renderTypePage(row.name, s, page); err != nil {
return fmt.Errorf("%s %s: %w", row.name, s.Name, err)
}
written[page] = true
}
}
base := lay.TypesDir(c.key)
err := filepath.WalkDir(base, func(p string, d fs.DirEntry, err error) error {
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
return nil
}
return err
}
if d.IsDir() || written[strings.TrimSuffix(p, ".gz")] {
return nil
}
if err := os.Remove(p); err != nil {
return err
}
pruneEmptyDirs(filepath.Dir(p), base)
return nil
})
return err
}
func (c *repoCtx) renderTypePage(row string, s LangStat, out string) error {
tv := &typeView{Name: s.Name, Row: row}
files := s.files
for _, f := range files {
ev := entryView{Name: path.Base(f.Path), URL: c.s.CodeURL(c.key, f.Path), Kind: "file", Size: rowAmount(row, f.Count)}
if crumbs := c.s.crumbsFor(c.key, f.Path); len(crumbs) > 1 {
ev.Dirs = crumbs[:len(crumbs)-1]
}
if e := c.tree.byPath[f.Path]; e != nil {
ev.Exec = e.IsExecutable()
}
ev.setModified(c.mtime[f.Path])
if s.Name == otherName {
ev.Note = displayLang(f.Type)
}
tv.Entries = append(tv.Entries, ev)
}
noun := "files"
if len(files) == 1 {
noun = "file"
}
tv.Summary = fmt.Sprintf("%s %s · %s", humanCount(len(files)), noun, rowTotal(row, s.Count))
page := c.page()
page.BodyClass = "listing"
page.Drawer = c.drawer("")
page.Type = tv
page.Title = c.repo + c.branchSuffix() + " · " + s.Name + " files"
page.URL = c.s.absURL(s.URL)
html, err := c.s.render("types", page)
if err != nil {
return err
}
return writePair(out, html, true)
}
// rowAmount is one file's size in its row's unit, for the Size column.
func rowAmount(row string, n int) string {
switch row {
case rowCode:
return lineCount(n)
case rowDocs:
if n == 1 {
return "1 word"
}
return humanCount(n) + " words"
}
return humanSize(int64(n))
}
// rowTotal is a type's total, worded for the top of its page.
func rowTotal(row string, n int) string {
switch row {
case rowCode:
return rowAmount(row, n) + " of code"
case rowDocs:
return pagesText(n)
}
return humanSize(int64(n))
}
// ---- languages and counts ----
// proseLangs are chroma lexers for documentation rather than code; the root
// page counts their words, not lines, since prose may or may not be wrapped.
var proseLangs = map[string]bool{
"markdown": true, "plaintext": true, "Groff": true, "reStructuredText": true,
"Org Mode": true, "TeX": true, "BibTeX": true,
}
// dotfileLang labels plain-text dotfiles (".clang-format", ".gitignore").
const dotfileLang = "Dotfile"
// lang is the language the size summary files p under, lexed as lx. Of the
// files no lexer knows, only those named like prose ("notes.txt",
// "LICENSE") count as plain text, and so as docs; the rest count as code:
// dotfiles, which configure tools, as "Dotfile", and anything else by its
// extension ("foo.qwerxxx" → "QWERXXX").
func (c *repoCtx) lang(p string, lx chroma.Lexer) string {
if !c.s.HL.IsPlain(lx) {
return highlight.Name(lx)
}
base := path.Base(p)
switch ext := strings.ToLower(path.Ext(base)); {
case strings.HasPrefix(base, "."):
return dotfileLang
case ext == "" || ext == ".txt":
return highlight.Name(lx)
}
return binaryFormat(p)
}
// binaryFiles are the binary files, which the size summary measures in
// bytes, in path order.
func (c *repoCtx) binaryFiles() []typedFile {
var out []typedFile
for _, p := range c.tree.sortedPaths() {
e := c.tree.byPath[p]
if e.IsSymlink() || e.IsSubmodule() {
continue
}
if c.lines[p] < 0 && e.Size > 0 {
out = append(out, typedFile{p, binaryFormat(p), int(e.Size)})
}
}
return out
}
// binaryFormats names extensions whose label is not just the extension in
// capitals.
var binaryFormats = map[string]string{"jpg": "JPEG", "jpe": "JPEG", "tif": "TIFF", "htm": "HTML"}
// binaryFormat labels a file by its extension: "logo.png" → "PNG". Files
// without one come out as "", which foldStats shows as "Other". Text files
// no lexer knows are labelled the same way.
func binaryFormat(p string) string {
ext := strings.ToLower(strings.TrimPrefix(path.Ext(p), "."))
if f, ok := binaryFormats[ext]; ok {
return f
}
return strings.ToUpper(ext)
}
// countNonBlank counts the lines with something other than whitespace.
func countNonBlank(lines []string) int {
n := 0
for _, l := range lines {
if strings.TrimSpace(l) != "" {
n++
}
}
return n
}
// countWords counts whitespace-separated words, skipping tokens without a
// letter or digit so markup like "#", "-" or "|" is not counted.
func countWords(data []byte) int {
n := 0
for w := range bytes.FieldsSeq(data) {
if bytes.ContainsFunc(w, func(r rune) bool { return unicode.IsLetter(r) || unicode.IsDigit(r) }) {
n++
}
}
return n
}
// countLines is the line count shown in listings, as wc -l counts them
// (newlines), or -1 for binary data.
func countLines(data []byte) int {
if highlight.IsBinary(data) {
return -1
}
return bytes.Count(data, []byte{'\n'})
}
// lineCounter is countLines for a blob streamed through it as a Writer.
type lineCounter struct {
probed int // how much of the start has been looked at for binary data
binary bool // a NUL there, as highlight.IsBinary looks for
lines int
}
func (lc *lineCounter) Write(p []byte) (int, error) {
if room := highlight.BinaryProbe - lc.probed; room > 0 {
h := p[:min(room, len(p))]
lc.probed += len(h)
lc.binary = lc.binary || bytes.IndexByte(h, 0) >= 0
}
lc.lines += bytes.Count(p, []byte{'\n'})
return len(p), nil
}
func (lc *lineCounter) count() int {
if lc.binary {
return -1
}
return lc.lines
}
// otherName names the pill that foldStats folds the tail of a row into,
// and the types without a name of their own.
const otherName = "Other"
// displayLang turns chroma's lexer ids into labels ("plaintext" → "Text",
// "markdown" → "Markdown").
func displayLang(name string) string {
switch name {
case "plaintext":
return "Text"
case "":
return otherName
}
if strings.ToLower(name) == name {
return strings.ToUpper(name[:1]) + name[1:]
}
return name
}
// foldStats sorts languages by count and folds the tail into "Other",
// along with anything already named "Other" (types with no name), so
// there is only ever one "Other", and it comes last.
func foldStats(m map[string]int, keep int) []LangStat {
merged := map[string]int{}
for n, l := range m {
merged[displayLang(n)] += l
}
other := merged[otherName]
delete(merged, otherName)
var out []LangStat
for n, l := range merged {
if l > 0 {
out = append(out, LangStat{Name: n, Count: l})
}
}
slices.SortFunc(out, func(a, b LangStat) int {
return cmp.Or(cmp.Compare(b.Count, a.Count), strings.Compare(a.Name, b.Name))
})
if len(out) > keep+1 || (other > 0 && len(out) > keep) {
for _, s := range out[keep:] {
other += s.Count
}
out = out[:keep]
}
if other > 0 {
out = append(out, LangStat{Name: otherName, Count: other})
}
return out
}
// ---- size bars ----
// sizes is what a file or directory holds: lines of code and of docs,
// counted as wc -l does, and bytes of everything.
type sizes struct{ code, docs, bytes int }
// sumSizes fills c.sizes from the tree: each file's lines (text only, as
// docs when its language is prose, else as code) and bytes, added into
// every directory above it. Symlinks and submodules hold nothing.
func (c *repoCtx) sumSizes() {
c.sizes = map[string]*sizes{}
for p, e := range c.tree.byPath {
if e.IsSymlink() || e.IsSubmodule() {
continue
}
var add sizes
add.bytes = int(e.Size)
if n := c.lines[p]; n > 0 { // binary is -1, unknown absent
if c.prose[p] {
add.docs = n
} else {
add.code = n
}
}
for d := p; ; {
s := c.sizes[d]
if s == nil {
s = &sizes{}
c.sizes[d] = s
}
s.code += add.code
s.docs += add.docs
s.bytes += add.bytes
if d == "" {
break
}
if j := strings.LastIndexByte(d, '/'); j >= 0 {
d = d[:j]
} else {
d = ""
}
}
}
}
// sizeBars is the pair of Lines and Size cells for a listing's entry.
type sizeBars struct{ Lines, Size bar }
// bar is an entry's amount and its share of the folder, drawn as a ring
// filled that far round, an arc per part.
type bar struct {
Col string // its column, "lines" or "size"
Text string // "4.4K lines", "40 KiB"
Arcs []arc
Title string // the share, for the tooltip
URL string // the entry's page, if any, which the bar links to as its row does
}
// arc is one part of a ring, in percent of the way round, starting where
// the part before it ends.
type arc struct {
Class string // its color: "code", "docs" or "size"
Len, Off string // its length, and minus where it starts ("" at the top): "12.50", "-25.00"
}
// part is one colored share of a bar.
type part struct {
class string
n int
}
// sizeBars is the Lines and Size cells of entry p in folder dir, whose
// page is url. An entry holding nothing, like a symlink, has "0 lines" and
// "0 B" by empty rings.
func (c *repoCtx) sizeBars(dir, p, url string) *sizeBars {
total, own := c.sizes[dir], c.sizes[p]
if total == nil {
total = &sizes{}
}
if own == nil {
own = &sizes{}
}
lines := newBar("lines", []part{{"code", own.code}, {"docs", own.docs}},
total.code+total.docs, lineCount(own.code+own.docs), "lines", url)
if own.code > 0 && own.docs > 0 {
lines.Title += fmt.Sprintf(": %s of code, %s of docs", lineCount(own.code), lineCount(own.docs))
}
return &sizeBars{
Lines: lines,
Size: newBar("size", []part{{"size", own.bytes}}, total.bytes, humanSize(int64(own.bytes)), "bytes", url),
}
}
// minShare is the least of the way round a ring is filled, in percent,
// when an entry holds anything at all.
const minShare = 2
// newBar is column col's bar for parts of a folder's total, labelled text.
func newBar(col string, parts []part, total int, text, what, url string) bar {
n := 0
for _, pt := range parts {
n += pt.n
}
b := bar{Col: col, Text: text, Title: "0% of this folder's " + what, URL: url}
if n == 0 {
return b
}
share := 100 * float64(n) / float64(total)
b.Title = percent(share) + " of this folder's " + what
// A sliver at least, so a tiny share still shows; its parts keep their
// proportions.
scale := max(share, minShare) / share
pcts := func(f float64) string { return strconv.FormatFloat(f, 'f', 2, 64) }
at := 0.0
for _, pt := range parts {
if pt.n == 0 {
continue
}
l := scale * 100 * float64(pt.n) / float64(total)
a := arc{Class: pt.class, Len: pcts(l)}
if at > 0 {
a.Off = pcts(-at)
}
b.Arcs = append(b.Arcs, a)
at += l
}
return b
}
// percent rounds a share for reading, never claiming none or all of it
// when that is not exactly so: "<1%", "40%", ">99%".
func percent(p float64) string {
switch {
case p > 0 && p < 1:
return "<1%"
case p > 99 && p < 100:
return ">99%"
}
return strconv.Itoa(int(p+0.5)) + "%"
}
|