Download
718 lines 19.9 KiBGoModified 2026-10-27 raw
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718
// Package render turns bare repositories into the static site. Site holds
// everything shared across repositories: config, templates, the highlighter
// and the markdown renderer.
package render

import (
	"bytes"
	"crypto/sha256"
	"embed"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"html/template"
	"io"
	"io/fs"
	"log"
	"maps"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"slices"
	"strings"
	"time"

	"code.bruce-hill.com/repo-renderer/internal/config"
	"code.bruce-hill.com/repo-renderer/internal/gitx"
	"code.bruce-hill.com/repo-renderer/internal/highlight"
	"code.bruce-hill.com/repo-renderer/internal/markdown"
)

//go:embed templates/*.html static/*
var assets embed.FS

// version is folded into the template hash so upgrading the binary forces a
// full re-render even when the embedded assets are unchanged. Bump it when
// output changes without them, or when countLines counts differently: a
// new hash also drops the State's cached line counts.
const version = "repo-render 2"

// Site is the shared rendering context.
type Site struct {
	Cfg      *config.Config
	Lay      Layout
	HL       *highlight.Registry
	MD       *markdown.Renderer
	TmplHash string
	Log      *log.Logger

	tmpls map[string]*template.Template
	// prefix is where the site is served: "" for the public site,
	// config.PrivatePrefix for the private one.
	prefix string
	// priv is the public site's private one, nil when there is none; the
	// private site's is nil too.
	priv *Site
}

// New builds a Site from config. It fails on bad lexers, styles or templates,
// which is what `repo-render check` relies on.
func New(cfg *config.Config, logger *log.Logger) (*Site, error) {
	if logger == nil {
		logger = log.New(os.Stderr, "", log.LstdFlags)
	}
	customs := make([]highlight.Custom, 0, len(cfg.Lexers))
	for _, name := range cfg.LexerNames() {
		lx := cfg.Lexers[name]
		customs = append(customs, highlight.Custom{Name: name, Globs: lx.Globs, Aliases: lx.Aliases})
	}
	hl, err := highlight.New(cfg.LexerDirs, customs, cfg.Style)
	if err != nil {
		return nil, err
	}
	if cfg.StyleLight != "" {
		if err := hl.SetLight(cfg.StyleLight); err != nil {
			return nil, err
		}
	}
	s := &Site{
		Cfg: cfg,
		Lay: LayoutOf(cfg),
		HL:  hl,
		MD:  markdown.New(hl),
		Log: logger,
	}
	if err := s.parseTemplates(); err != nil {
		return nil, err
	}
	h, err := s.computeHash()
	if err != nil {
		return nil, err
	}
	s.TmplHash = h
	if cfg.PrivateHtdocs != "" {
		p := *s
		p.Lay = s.Lay.private(cfg.PrivateHtdocs)
		p.prefix = config.PrivatePrefix
		s.priv = &p
	}
	return s, nil
}

// siteFor is the site a repository is rendered to: the public one, or
// for a private repository the private one, nil when there is none.
func (s *Site) siteFor(repo string) *Site {
	if s.Cfg.Private(repo) {
		return s.priv
	}
	return s
}

// sites are the public site and the private one, when there is one.
func (s *Site) sites() []*Site {
	if s.priv == nil {
		return []*Site{s}
	}
	return []*Site{s, s.priv}
}

// Icons are empty elements masked by /static/icon-*.svg and painted in
// currentColor (see .icon in style.css). The CSP allows no data: URLs, and
// inline <svg> markup would cost ~230 bytes per use, which the drawer
// repeats for every entry it lists. Each SVG is 16×16; a folder in the
// drawer switches to the open glyph in CSS while it is open.
const (
	iconDir    = `<i class="icon i-dir"></i>`
	iconModule = `<i class="icon i-submodule"></i>`
	iconFile   = `<i class="icon i-file"></i>`
	iconLink   = `<i class="icon i-symlink"></i>`
)

var icons = map[string]template.HTML{
	"dir":       iconDir,
	"file":      iconFile,
	"symlink":   iconLink,
	"submodule": iconModule,
}

func icon(kind string) template.HTML { return icons[kind] }

var funcs = template.FuncMap{
	"count": humanCount,
	"icon":  icon,
	"bytes": func(n int) string { return humanSize(int64(n)) },
}

// asset is the URL of a file in static/, versioned by the template hash so
// that a page never pairs with a cached copy of an older stylesheet or
// script. Any change to the assets changes the hash and re-renders every
// page.
func (s *Site) asset(name string) string { return "/static/" + name + "?v=" + s.TmplHash }

func (s *Site) parseTemplates() error {
	fm := maps.Clone(funcs)
	fm["asset"] = s.asset
	base, err := template.New("base").Funcs(fm).ParseFS(assets, "templates/base.html")
	if err != nil {
		return err
	}
	s.tmpls = map[string]*template.Template{}
	for _, name := range []string{"index", "notfound", "forbidden", "listing", "file", "search", "types"} {
		t, err := base.Clone()
		if err == nil {
			t, err = t.ParseFS(assets, "templates/"+name+".html")
		}
		if err != nil {
			return fmt.Errorf("template %s: %w", name, err)
		}
		s.tmpls[name] = t
	}
	// Documents shown in frames stand alone, without the base page.
	for _, name := range []string{"drawer", "tags"} {
		t, err := template.New(name).Funcs(fm).ParseFS(assets, "templates/"+name+".html")
		if err != nil {
			return fmt.Errorf("template %s: %w", name, err)
		}
		s.tmpls[name] = t
	}
	return nil
}

// computeHash hashes everything that affects output besides repo content.
func (s *Site) computeHash() (string, error) {
	// A hash.Hash never returns a write error, so none is checked.
	h := sha256.New()
	_, _ = io.WriteString(h, version)
	// WalkDir goes in lexical order, so the hash is stable.
	for _, fsys := range []embed.FS{assets, highlight.Bundled} {
		if err := fs.WalkDir(fsys, ".", func(p string, d fs.DirEntry, err error) error {
			if err != nil || d.IsDir() {
				return err
			}
			b, err := fsys.ReadFile(p)
			if err != nil {
				return err
			}
			_, _ = fmt.Fprintf(h, "%s\x00%d\x00", p, len(b))
			_, _ = h.Write(b)
			return nil
		}); err != nil {
			return "", err
		}
	}
	for _, dir := range s.Cfg.LexerDirs {
		xmls, err := listDir(dir, func(n string) bool { return strings.HasSuffix(n, ".xml") })
		if err != nil {
			return "", err
		}
		for _, x := range xmls {
			b, err := os.ReadFile(x)
			if err != nil {
				return "", err
			}
			_, _ = fmt.Fprintf(h, "%s\x00%d\x00", filepath.Base(x), len(b))
			_, _ = h.Write(b)
		}
	}
	_, _ = io.WriteString(h, "\x00config\x00")
	_, _ = h.Write(s.Cfg.Source)
	// The favicon's URL is versioned too, and the config names only its
	// path.
	if f := s.Cfg.Site.Favicon; f != "" {
		b, err := os.ReadFile(f)
		if err != nil {
			return "", fmt.Errorf("site.favicon: %w", err)
		}
		_, _ = io.WriteString(h, "\x00favicon\x00")
		_, _ = h.Write(b)
	}
	return hex.EncodeToString(h.Sum(nil))[:16], nil
}

// ---- state ----

// State is what the renderer remembers per repository between runs.
type State struct {
	SHA      string   `json:"sha"`
	TmplHash string   `json:"tmpl"`
	Branch   string   `json:"branch"`
	Views    []string `json:"views,omitempty"` // branches rendered for the repo, default first
	// CommitTime is when SHA was committed, shown on the index.
	CommitTime time.Time `json:"commit_time"`
	// Subject is SHA's subject line, shown in the download list.
	Subject string `json:"subject,omitempty"`
	// Description is the repository's description when SHA was rendered.
	// Every page carries it, and it lives outside the commit, so a change
	// calls for a full render.
	Description string `json:"description,omitempty"`
	// Tip names SHA's tarball in the view's archive directory, "" when
	// git could not make one.
	Tip string `json:"tip,omitempty"`
	// Lines caches line counts by blob id (-1 for binary) so unchanged
	// blobs outside the search corpus need not be re-read on every push.
	// They are kept only while TmplHash is unchanged.
	Lines map[string]int `json:"lines,omitempty"`
	// Unfinished is set, on the state of the last finished render, while
	// a render is writing pages. A render that fails or dies part way
	// leaves pages and raw files of its commit that no later diff from
	// SHA knows about, so the next render must be a full one, which
	// sweeps them away. A revert or a force push back to SHA would
	// otherwise keep them on the site.
	Unfinished bool `json:"unfinished,omitempty"`
}

// LoadState returns the saved state, or nil when the repo was never rendered.
func (s *Site) LoadState(repo string) (*State, error) {
	b, err := os.ReadFile(s.Lay.StatePath(repo))
	if err != nil {
		if errors.Is(err, fs.ErrNotExist) {
			return nil, nil
		}
		return nil, err
	}
	var st State
	if err := json.Unmarshal(b, &st); err != nil {
		// Corrupt state → full re-render.
		s.Log.Printf("%s: ignoring corrupt state file: %v", repo, err)
		return nil, nil
	}
	if !gitx.IsSHA(st.SHA) {
		s.Log.Printf("%s: ignoring state file with bad commit %q", repo, st.SHA)
		return nil, nil
	}
	return &st, nil
}

func (s *Site) saveState(repo string, st *State) error {
	b, err := json.MarshalIndent(st, "", "  ")
	if err != nil {
		return err
	}
	return writeFileAtomic(s.Lay.StatePath(repo), b)
}

// Repos lists the bare repositories under git/ by name.
func (s *Site) Repos() ([]string, error) {
	ents, err := os.ReadDir(s.Lay.GitBase)
	if err != nil {
		if errors.Is(err, fs.ErrNotExist) {
			return nil, nil
		}
		return nil, err
	}
	var names []string
	for _, e := range ents {
		n := e.Name()
		if !strings.HasSuffix(n, ".git") {
			continue
		}
		n = strings.TrimSuffix(n, ".git")
		if !config.ValidRepoName(n) {
			s.Log.Printf("skipping repository with unsafe name %q", e.Name())
			continue
		}
		if st, err := os.Stat(filepath.Join(s.Lay.GitBase, e.Name())); err != nil || !st.IsDir() {
			continue
		}
		names = append(names, n)
	}
	slices.Sort(names)
	return names, nil
}

// Description returns the repo description, from the bare repo's
// description file.
func (s *Site) Description(repo string) string {
	return gitx.Repo{Dir: s.Lay.GitDir(repo)}.Description()
}

// ---- pages ----

type crumb struct {
	Name string
	URL  string
}

type pageData struct {
	SiteTitle    string
	PrivateURL   string // the private site's index, on its pages
	Tagline      string
	Footer       template.HTML
	Title        string
	Description  string
	Tags         []string // the repository's tags, on its root listing
	URL          string   // absolute URL of the page, for link previews
	BodyClass    string
	Repo         string
	RepoURL      string
	SearchURL    string
	CloneURL     string
	Crumbs       []crumb
	Generated    string
	Drawer       *drawerView
	Stats        *statsView
	DownloadsURL string // the download list document, framed by the Download menu
	FaviconURL   string
	FaviconType  string
	Branch       string     // branch being viewed (repo pages)
	Views        []viewLink // branch picker; nil when there is one branch
	NoIndex      bool       // non-default branch: keep search engines to the default

	Index   *indexView
	Listing *listingView
	File    *fileView
	Search  *searchView
	Type    *typeView
}

func (s *Site) basePage(repo string) pageData {
	p := pageData{
		SiteTitle:   s.Cfg.Site.Title,
		Tagline:     s.Cfg.Site.Tagline,
		Footer:      template.HTML(s.Cfg.Site.Footer),
		FaviconURL:  s.faviconURL(),
		FaviconType: faviconType(s.faviconName()),
		Generated:   time.Now().UTC().Format("2006-01-02 15:04 UTC"),
	}
	if s.prefix != "" {
		p.PrivateURL = s.prefix + "/"
		p.NoIndex = true
	}
	if repo != "" {
		p.Repo = repo
		p.RepoURL = s.CodeURL(repo, "")
		p.SearchURL = s.SearchURL(repo)
		p.CloneURL = s.Cfg.CloneURL(repo)
		if s.prefix != "" {
			p.CloneURL = s.Cfg.PrivateCloneURL(repo)
		}
	}
	return p
}

// crumbsFor is the path breadcrumb shown above a file or directory: one
// link per directory, then the current name (no URL) last. The repository
// itself is in the header, so it is not repeated here.
func (s *Site) crumbsFor(repo, p string) []crumb {
	if p == "" {
		return nil
	}
	parts := strings.Split(p, "/")
	crumbs := make([]crumb, 0, len(parts))
	for i := range parts {
		c := crumb{Name: parts[i]}
		if i < len(parts)-1 {
			c.URL = s.CodeURL(repo, strings.Join(parts[:i+1], "/"))
		}
		crumbs = append(crumbs, c)
	}
	return crumbs
}

// absURL is the public URL of a site path.
func (s *Site) absURL(p string) string { return s.Cfg.BaseURL() + p }

func (s *Site) render(name string, data pageData) ([]byte, error) {
	t, ok := s.tmpls[name]
	if !ok {
		return nil, fmt.Errorf("no template %q", name)
	}
	var buf bytes.Buffer
	if err := t.ExecuteTemplate(&buf, "base", data); err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

// renderDoc renders a document that is shown in a frame, not as a page:
// the file drawer or the tag list.
func (s *Site) renderDoc(name string, data any) ([]byte, error) {
	var buf bytes.Buffer
	if err := s.tmpls[name].ExecuteTemplate(&buf, name, data); err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

// faviconURL is where the site icon is served from: the configured file,
// copied into static/, or the built-in SVG.
func (s *Site) faviconURL() string {
	return s.asset(s.faviconName())
}

func (s *Site) faviconName() string {
	if f := s.Cfg.Site.Favicon; f != "" {
		return "favicon" + strings.ToLower(filepath.Ext(f))
	}
	return "favicon.svg"
}

func faviconType(name string) string {
	switch filepath.Ext(name) {
	case ".svg":
		return "image/svg+xml"
	case ".png":
		return "image/png"
	case ".ico":
		return "image/x-icon"
	case ".gif":
		return "image/gif"
	case ".webp":
		return "image/webp"
	}
	return ""
}

var cssURL = regexp.MustCompile(`url\(([\w.-]+)\)`)

// EnsureStatic writes the stylesheet, the generated chroma stylesheet and
// the search script into htdocs/static.
func (s *Site) EnsureStatic() error {
	ents, err := fs.ReadDir(assets, "static")
	if err != nil {
		return err
	}
	for _, e := range ents {
		b, err := assets.ReadFile("static/" + e.Name())
		if err != nil {
			return err
		}
		if e.Name() == "style.css" {
			// A relative url() drops the stylesheet's ?v=, so the icons
			// get their own, or /static/ could not be cached for good.
			b = cssURL.ReplaceAll(b, []byte("url($1?v="+s.TmplHash+")"))
		}
		if err := writePair(filepath.Join(s.Lay.StaticDir(), e.Name()), b, true); err != nil {
			return err
		}
	}
	if f := s.Cfg.Site.Favicon; f != "" {
		b, err := os.ReadFile(f)
		if err != nil {
			return fmt.Errorf("site.favicon: %w", err)
		}
		if err := writePair(filepath.Join(s.Lay.StaticDir(), s.faviconName()), b, false); err != nil {
			return err
		}
	}
	var css bytes.Buffer
	if err := s.HL.WriteCSS(&css); err != nil {
		return err
	}
	return writePair(filepath.Join(s.Lay.StaticDir(), "chroma.css"), css.Bytes(), true)
}

// ---- index ----

type indexView struct {
	Sections []sectionView
}

type sectionView struct {
	Title       template.HTML
	Description template.HTML
	Repos       []repoRow
}

type repoRow struct {
	Name        string
	URL         string
	Description string
	Tags        []string
	Updated     string // RFC 3339 for <time datetime>
	UpdatedText string
	UpdatedFull string // date and time, for the tooltip
}

// RenderIndex writes the top-level repository index, organised by the
// configured sections, and the private site's index of the private
// repositories.
func (s *Site) RenderIndex() error {
	if err := s.EnsureStatic(); err != nil {
		return err
	}
	names, err := s.Repos()
	if err != nil {
		return err
	}
	if s.priv != nil {
		if err := s.priv.renderPrivateIndex(names); err != nil {
			return err
		}
	}
	rows, err := s.repoRows(names)
	if err != nil {
		return err
	}
	var view indexView
	listed := map[string]bool{}
	for _, sec := range s.Cfg.Sections {
		var sv sectionView
		if sv.Title, err = s.MD.RenderInline(sec.Title); err != nil {
			return err
		}
		if sec.Description != "" {
			if sv.Description, err = s.MD.Render([]byte(sec.Description), markdown.Context{}); err != nil {
				return err
			}
		}
		for _, r := range sec.Repos {
			listed[r] = true
			row, ok := rows[r]
			if !ok {
				continue
			}
			sv.Repos = append(sv.Repos, row)
		}
		slices.SortFunc(sv.Repos, func(a, b repoRow) int { return strings.Compare(a.Name, b.Name) })
		if len(sv.Repos) > 0 {
			view.Sections = append(view.Sections, sv)
		}
	}
	if s.Cfg.Site.Unlisted == "show" {
		var rest sectionView
		if len(s.Cfg.Sections) > 0 {
			if rest.Title, err = s.MD.RenderInline(s.Cfg.Site.UnlistedTitle); err != nil {
				return err
			}
		}
		for _, name := range names {
			if row, ok := rows[name]; ok && !listed[name] {
				rest.Repos = append(rest.Repos, row)
			}
		}
		if len(rest.Repos) > 0 {
			view.Sections = append(view.Sections, rest)
		}
	}
	page := s.basePage("")
	page.BodyClass = "index"
	page.Description = s.Cfg.Site.Tagline
	page.URL = s.absURL("/")
	page.Index = &view
	html, err := s.render("index", page)
	if err != nil {
		return err
	}
	if err := writePair(s.Lay.IndexPage(), html, true); err != nil {
		return err
	}
	if err := s.writeSiteSitemap(view); err != nil {
		return err
	}
	if err := writePair(s.Lay.RobotsTxt(), []byte(robotsTxt+"\nSitemap: "+s.absURL("/sitemap.xml")+"\n"), false); err != nil {
		return err
	}
	if err := writePair(s.Lay.AdsTxt(), []byte(adsTxt), false); err != nil {
		return err
	}
	return s.renderErrorPages()
}

// repoRows are the index rows of the repositories among names that are
// on this site and have been rendered, by name.
func (s *Site) repoRows(names []string) (map[string]repoRow, error) {
	rows := map[string]repoRow{}
	for _, name := range names {
		if s.Cfg.Private(name) != (s.prefix != "") {
			continue // the other site's; RenderRepo removes it from this one
		}
		st, err := s.LoadState(name)
		if err != nil {
			return nil, err
		}
		if st == nil {
			continue // never rendered: no pages to link to yet
		}
		rows[name] = repoRow{
			Name:        name,
			URL:         s.CodeURL(name, ""),
			Description: s.Description(name),
			Tags:        s.Cfg.Repo(name).Tags,
			Updated:     st.CommitTime.UTC().Format(time.RFC3339),
			UpdatedText: st.CommitTime.UTC().Format("2006-01-02"),
			UpdatedFull: st.CommitTime.UTC().Format(fullTime),
		}
	}
	return rows, nil
}

// renderPrivateIndex writes the private site's index: its repositories in
// one section. The public site has the sitemap, robots.txt and the error
// pages; this one is for those who log in, and search engines cannot.
func (s *Site) renderPrivateIndex(names []string) error {
	rows, err := s.repoRows(names)
	if err != nil {
		return err
	}
	var sec sectionView
	for _, name := range names {
		if row, ok := rows[name]; ok {
			sec.Repos = append(sec.Repos, row)
		}
	}
	var view indexView
	if len(sec.Repos) > 0 {
		view.Sections = append(view.Sections, sec)
	}
	page := s.basePage("")
	page.BodyClass = "index"
	page.Title = "Private repositories"
	page.Index = &view
	html, err := s.render("index", page)
	if err != nil {
		return err
	}
	return writePair(s.Lay.IndexPage(), html, true)
}

// adsTxt says no one is authorized to sell ads on the site, in the form
// the ads.txt 1.1 spec gives for that: one placeholder record.
const adsTxt = "placeholder.example.com, placeholder, DIRECT, placeholder\n"

// robotsTxt keeps crawlers to the pages. Tarballs, the clone export and
// raw copies cost the server most to send and are not worth indexing;
// search needs script, and the frames are parts of pages. Branch views
// stay open: crawlers must fetch them to see that they ask not to be
// indexed. Crawl-delay asks for a pause between fetches; Bing and Yandex
// honour it, Google ignores it. RenderIndex adds the sitemap's address,
// which robots.txt must give in full.
const robotsTxt = `User-agent: *
Crawl-delay: 10
Disallow: /archive/
Disallow: /snapshot/
Disallow: /git/
Disallow: /raw/
Disallow: /svg/
Disallow: /search/
Disallow: /iframes/
`

// renderErrorPages writes the pages httpd shows for a missing path and for
// a directory with no index.html. They live with the index because they
// are the site's own, not any repository's.
func (s *Site) renderErrorPages() error {
	for _, e := range []struct {
		code        int
		name, title string
	}{
		{http.StatusNotFound, "notfound", "Page not found"},
		{http.StatusForbidden, "forbidden", "Nothing to show here"},
	} {
		page := s.basePage("")
		page.Title = e.title
		page.BodyClass = "error"
		page.NoIndex = true
		html, err := s.render(e.name, page)
		if err != nil {
			return err
		}
		// httpd reads error pages as they are, never a gzip twin.
		if err := writePair(s.Lay.ErrorPage(e.code), html, false); err != nil {
			return err
		}
	}
	return nil
}