123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718 | // Package render turns bare repositories into the static site. Site holds
// everything shared across repositories: config, templates, the highlighter
// and the markdown renderer.
package render
import (
"bytes"
"crypto/sha256"
"embed"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"html/template"
"io"
"io/fs"
"log"
"maps"
"net/http"
"os"
"path/filepath"
"regexp"
"slices"
"strings"
"time"
"code.bruce-hill.com/repo-renderer/internal/config"
"code.bruce-hill.com/repo-renderer/internal/gitx"
"code.bruce-hill.com/repo-renderer/internal/highlight"
"code.bruce-hill.com/repo-renderer/internal/markdown"
)
//go:embed templates/*.html static/*
var assets embed.FS
// version is folded into the template hash so upgrading the binary forces a
// full re-render even when the embedded assets are unchanged. Bump it when
// output changes without them, or when countLines counts differently: a
// new hash also drops the State's cached line counts.
const version = "repo-render 2"
// Site is the shared rendering context.
type Site struct {
Cfg *config.Config
Lay Layout
HL *highlight.Registry
MD *markdown.Renderer
TmplHash string
Log *log.Logger
tmpls map[string]*template.Template
// prefix is where the site is served: "" for the public site,
// config.PrivatePrefix for the private one.
prefix string
// priv is the public site's private one, nil when there is none; the
// private site's is nil too.
priv *Site
}
// New builds a Site from config. It fails on bad lexers, styles or templates,
// which is what `repo-render check` relies on.
func New(cfg *config.Config, logger *log.Logger) (*Site, error) {
if logger == nil {
logger = log.New(os.Stderr, "", log.LstdFlags)
}
customs := make([]highlight.Custom, 0, len(cfg.Lexers))
for _, name := range cfg.LexerNames() {
lx := cfg.Lexers[name]
customs = append(customs, highlight.Custom{Name: name, Globs: lx.Globs, Aliases: lx.Aliases})
}
hl, err := highlight.New(cfg.LexerDirs, customs, cfg.Style)
if err != nil {
return nil, err
}
if cfg.StyleLight != "" {
if err := hl.SetLight(cfg.StyleLight); err != nil {
return nil, err
}
}
s := &Site{
Cfg: cfg,
Lay: LayoutOf(cfg),
HL: hl,
MD: markdown.New(hl),
Log: logger,
}
if err := s.parseTemplates(); err != nil {
return nil, err
}
h, err := s.computeHash()
if err != nil {
return nil, err
}
s.TmplHash = h
if cfg.PrivateHtdocs != "" {
p := *s
p.Lay = s.Lay.private(cfg.PrivateHtdocs)
p.prefix = config.PrivatePrefix
s.priv = &p
}
return s, nil
}
// siteFor is the site a repository is rendered to: the public one, or
// for a private repository the private one, nil when there is none.
func (s *Site) siteFor(repo string) *Site {
if s.Cfg.Private(repo) {
return s.priv
}
return s
}
// sites are the public site and the private one, when there is one.
func (s *Site) sites() []*Site {
if s.priv == nil {
return []*Site{s}
}
return []*Site{s, s.priv}
}
// Icons are empty elements masked by /static/icon-*.svg and painted in
// currentColor (see .icon in style.css). The CSP allows no data: URLs, and
// inline <svg> markup would cost ~230 bytes per use, which the drawer
// repeats for every entry it lists. Each SVG is 16×16; a folder in the
// drawer switches to the open glyph in CSS while it is open.
const (
iconDir = `<i class="icon i-dir"></i>`
iconModule = `<i class="icon i-submodule"></i>`
iconFile = `<i class="icon i-file"></i>`
iconLink = `<i class="icon i-symlink"></i>`
)
var icons = map[string]template.HTML{
"dir": iconDir,
"file": iconFile,
"symlink": iconLink,
"submodule": iconModule,
}
func icon(kind string) template.HTML { return icons[kind] }
var funcs = template.FuncMap{
"count": humanCount,
"icon": icon,
"bytes": func(n int) string { return humanSize(int64(n)) },
}
// asset is the URL of a file in static/, versioned by the template hash so
// that a page never pairs with a cached copy of an older stylesheet or
// script. Any change to the assets changes the hash and re-renders every
// page.
func (s *Site) asset(name string) string { return "/static/" + name + "?v=" + s.TmplHash }
func (s *Site) parseTemplates() error {
fm := maps.Clone(funcs)
fm["asset"] = s.asset
base, err := template.New("base").Funcs(fm).ParseFS(assets, "templates/base.html")
if err != nil {
return err
}
s.tmpls = map[string]*template.Template{}
for _, name := range []string{"index", "notfound", "forbidden", "listing", "file", "search", "types"} {
t, err := base.Clone()
if err == nil {
t, err = t.ParseFS(assets, "templates/"+name+".html")
}
if err != nil {
return fmt.Errorf("template %s: %w", name, err)
}
s.tmpls[name] = t
}
// Documents shown in frames stand alone, without the base page.
for _, name := range []string{"drawer", "tags"} {
t, err := template.New(name).Funcs(fm).ParseFS(assets, "templates/"+name+".html")
if err != nil {
return fmt.Errorf("template %s: %w", name, err)
}
s.tmpls[name] = t
}
return nil
}
// computeHash hashes everything that affects output besides repo content.
func (s *Site) computeHash() (string, error) {
// A hash.Hash never returns a write error, so none is checked.
h := sha256.New()
_, _ = io.WriteString(h, version)
// WalkDir goes in lexical order, so the hash is stable.
for _, fsys := range []embed.FS{assets, highlight.Bundled} {
if err := fs.WalkDir(fsys, ".", func(p string, d fs.DirEntry, err error) error {
if err != nil || d.IsDir() {
return err
}
b, err := fsys.ReadFile(p)
if err != nil {
return err
}
_, _ = fmt.Fprintf(h, "%s\x00%d\x00", p, len(b))
_, _ = h.Write(b)
return nil
}); err != nil {
return "", err
}
}
for _, dir := range s.Cfg.LexerDirs {
xmls, err := listDir(dir, func(n string) bool { return strings.HasSuffix(n, ".xml") })
if err != nil {
return "", err
}
for _, x := range xmls {
b, err := os.ReadFile(x)
if err != nil {
return "", err
}
_, _ = fmt.Fprintf(h, "%s\x00%d\x00", filepath.Base(x), len(b))
_, _ = h.Write(b)
}
}
_, _ = io.WriteString(h, "\x00config\x00")
_, _ = h.Write(s.Cfg.Source)
// The favicon's URL is versioned too, and the config names only its
// path.
if f := s.Cfg.Site.Favicon; f != "" {
b, err := os.ReadFile(f)
if err != nil {
return "", fmt.Errorf("site.favicon: %w", err)
}
_, _ = io.WriteString(h, "\x00favicon\x00")
_, _ = h.Write(b)
}
return hex.EncodeToString(h.Sum(nil))[:16], nil
}
// ---- state ----
// State is what the renderer remembers per repository between runs.
type State struct {
SHA string `json:"sha"`
TmplHash string `json:"tmpl"`
Branch string `json:"branch"`
Views []string `json:"views,omitempty"` // branches rendered for the repo, default first
// CommitTime is when SHA was committed, shown on the index.
CommitTime time.Time `json:"commit_time"`
// Subject is SHA's subject line, shown in the download list.
Subject string `json:"subject,omitempty"`
// Description is the repository's description when SHA was rendered.
// Every page carries it, and it lives outside the commit, so a change
// calls for a full render.
Description string `json:"description,omitempty"`
// Tip names SHA's tarball in the view's archive directory, "" when
// git could not make one.
Tip string `json:"tip,omitempty"`
// Lines caches line counts by blob id (-1 for binary) so unchanged
// blobs outside the search corpus need not be re-read on every push.
// They are kept only while TmplHash is unchanged.
Lines map[string]int `json:"lines,omitempty"`
// Unfinished is set, on the state of the last finished render, while
// a render is writing pages. A render that fails or dies part way
// leaves pages and raw files of its commit that no later diff from
// SHA knows about, so the next render must be a full one, which
// sweeps them away. A revert or a force push back to SHA would
// otherwise keep them on the site.
Unfinished bool `json:"unfinished,omitempty"`
}
// LoadState returns the saved state, or nil when the repo was never rendered.
func (s *Site) LoadState(repo string) (*State, error) {
b, err := os.ReadFile(s.Lay.StatePath(repo))
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
return nil, nil
}
return nil, err
}
var st State
if err := json.Unmarshal(b, &st); err != nil {
// Corrupt state → full re-render.
s.Log.Printf("%s: ignoring corrupt state file: %v", repo, err)
return nil, nil
}
if !gitx.IsSHA(st.SHA) {
s.Log.Printf("%s: ignoring state file with bad commit %q", repo, st.SHA)
return nil, nil
}
return &st, nil
}
func (s *Site) saveState(repo string, st *State) error {
b, err := json.MarshalIndent(st, "", " ")
if err != nil {
return err
}
return writeFileAtomic(s.Lay.StatePath(repo), b)
}
// Repos lists the bare repositories under git/ by name.
func (s *Site) Repos() ([]string, error) {
ents, err := os.ReadDir(s.Lay.GitBase)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
return nil, nil
}
return nil, err
}
var names []string
for _, e := range ents {
n := e.Name()
if !strings.HasSuffix(n, ".git") {
continue
}
n = strings.TrimSuffix(n, ".git")
if !config.ValidRepoName(n) {
s.Log.Printf("skipping repository with unsafe name %q", e.Name())
continue
}
if st, err := os.Stat(filepath.Join(s.Lay.GitBase, e.Name())); err != nil || !st.IsDir() {
continue
}
names = append(names, n)
}
slices.Sort(names)
return names, nil
}
// Description returns the repo description, from the bare repo's
// description file.
func (s *Site) Description(repo string) string {
return gitx.Repo{Dir: s.Lay.GitDir(repo)}.Description()
}
// ---- pages ----
type crumb struct {
Name string
URL string
}
type pageData struct {
SiteTitle string
PrivateURL string // the private site's index, on its pages
Tagline string
Footer template.HTML
Title string
Description string
Tags []string // the repository's tags, on its root listing
URL string // absolute URL of the page, for link previews
BodyClass string
Repo string
RepoURL string
SearchURL string
CloneURL string
Crumbs []crumb
Generated string
Drawer *drawerView
Stats *statsView
DownloadsURL string // the download list document, framed by the Download menu
FaviconURL string
FaviconType string
Branch string // branch being viewed (repo pages)
Views []viewLink // branch picker; nil when there is one branch
NoIndex bool // non-default branch: keep search engines to the default
Index *indexView
Listing *listingView
File *fileView
Search *searchView
Type *typeView
}
func (s *Site) basePage(repo string) pageData {
p := pageData{
SiteTitle: s.Cfg.Site.Title,
Tagline: s.Cfg.Site.Tagline,
Footer: template.HTML(s.Cfg.Site.Footer),
FaviconURL: s.faviconURL(),
FaviconType: faviconType(s.faviconName()),
Generated: time.Now().UTC().Format("2006-01-02 15:04 UTC"),
}
if s.prefix != "" {
p.PrivateURL = s.prefix + "/"
p.NoIndex = true
}
if repo != "" {
p.Repo = repo
p.RepoURL = s.CodeURL(repo, "")
p.SearchURL = s.SearchURL(repo)
p.CloneURL = s.Cfg.CloneURL(repo)
if s.prefix != "" {
p.CloneURL = s.Cfg.PrivateCloneURL(repo)
}
}
return p
}
// crumbsFor is the path breadcrumb shown above a file or directory: one
// link per directory, then the current name (no URL) last. The repository
// itself is in the header, so it is not repeated here.
func (s *Site) crumbsFor(repo, p string) []crumb {
if p == "" {
return nil
}
parts := strings.Split(p, "/")
crumbs := make([]crumb, 0, len(parts))
for i := range parts {
c := crumb{Name: parts[i]}
if i < len(parts)-1 {
c.URL = s.CodeURL(repo, strings.Join(parts[:i+1], "/"))
}
crumbs = append(crumbs, c)
}
return crumbs
}
// absURL is the public URL of a site path.
func (s *Site) absURL(p string) string { return s.Cfg.BaseURL() + p }
func (s *Site) render(name string, data pageData) ([]byte, error) {
t, ok := s.tmpls[name]
if !ok {
return nil, fmt.Errorf("no template %q", name)
}
var buf bytes.Buffer
if err := t.ExecuteTemplate(&buf, "base", data); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// renderDoc renders a document that is shown in a frame, not as a page:
// the file drawer or the tag list.
func (s *Site) renderDoc(name string, data any) ([]byte, error) {
var buf bytes.Buffer
if err := s.tmpls[name].ExecuteTemplate(&buf, name, data); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// faviconURL is where the site icon is served from: the configured file,
// copied into static/, or the built-in SVG.
func (s *Site) faviconURL() string {
return s.asset(s.faviconName())
}
func (s *Site) faviconName() string {
if f := s.Cfg.Site.Favicon; f != "" {
return "favicon" + strings.ToLower(filepath.Ext(f))
}
return "favicon.svg"
}
func faviconType(name string) string {
switch filepath.Ext(name) {
case ".svg":
return "image/svg+xml"
case ".png":
return "image/png"
case ".ico":
return "image/x-icon"
case ".gif":
return "image/gif"
case ".webp":
return "image/webp"
}
return ""
}
var cssURL = regexp.MustCompile(`url\(([\w.-]+)\)`)
// EnsureStatic writes the stylesheet, the generated chroma stylesheet and
// the search script into htdocs/static.
func (s *Site) EnsureStatic() error {
ents, err := fs.ReadDir(assets, "static")
if err != nil {
return err
}
for _, e := range ents {
b, err := assets.ReadFile("static/" + e.Name())
if err != nil {
return err
}
if e.Name() == "style.css" {
// A relative url() drops the stylesheet's ?v=, so the icons
// get their own, or /static/ could not be cached for good.
b = cssURL.ReplaceAll(b, []byte("url($1?v="+s.TmplHash+")"))
}
if err := writePair(filepath.Join(s.Lay.StaticDir(), e.Name()), b, true); err != nil {
return err
}
}
if f := s.Cfg.Site.Favicon; f != "" {
b, err := os.ReadFile(f)
if err != nil {
return fmt.Errorf("site.favicon: %w", err)
}
if err := writePair(filepath.Join(s.Lay.StaticDir(), s.faviconName()), b, false); err != nil {
return err
}
}
var css bytes.Buffer
if err := s.HL.WriteCSS(&css); err != nil {
return err
}
return writePair(filepath.Join(s.Lay.StaticDir(), "chroma.css"), css.Bytes(), true)
}
// ---- index ----
type indexView struct {
Sections []sectionView
}
type sectionView struct {
Title template.HTML
Description template.HTML
Repos []repoRow
}
type repoRow struct {
Name string
URL string
Description string
Tags []string
Updated string // RFC 3339 for <time datetime>
UpdatedText string
UpdatedFull string // date and time, for the tooltip
}
// RenderIndex writes the top-level repository index, organised by the
// configured sections, and the private site's index of the private
// repositories.
func (s *Site) RenderIndex() error {
if err := s.EnsureStatic(); err != nil {
return err
}
names, err := s.Repos()
if err != nil {
return err
}
if s.priv != nil {
if err := s.priv.renderPrivateIndex(names); err != nil {
return err
}
}
rows, err := s.repoRows(names)
if err != nil {
return err
}
var view indexView
listed := map[string]bool{}
for _, sec := range s.Cfg.Sections {
var sv sectionView
if sv.Title, err = s.MD.RenderInline(sec.Title); err != nil {
return err
}
if sec.Description != "" {
if sv.Description, err = s.MD.Render([]byte(sec.Description), markdown.Context{}); err != nil {
return err
}
}
for _, r := range sec.Repos {
listed[r] = true
row, ok := rows[r]
if !ok {
continue
}
sv.Repos = append(sv.Repos, row)
}
slices.SortFunc(sv.Repos, func(a, b repoRow) int { return strings.Compare(a.Name, b.Name) })
if len(sv.Repos) > 0 {
view.Sections = append(view.Sections, sv)
}
}
if s.Cfg.Site.Unlisted == "show" {
var rest sectionView
if len(s.Cfg.Sections) > 0 {
if rest.Title, err = s.MD.RenderInline(s.Cfg.Site.UnlistedTitle); err != nil {
return err
}
}
for _, name := range names {
if row, ok := rows[name]; ok && !listed[name] {
rest.Repos = append(rest.Repos, row)
}
}
if len(rest.Repos) > 0 {
view.Sections = append(view.Sections, rest)
}
}
page := s.basePage("")
page.BodyClass = "index"
page.Description = s.Cfg.Site.Tagline
page.URL = s.absURL("/")
page.Index = &view
html, err := s.render("index", page)
if err != nil {
return err
}
if err := writePair(s.Lay.IndexPage(), html, true); err != nil {
return err
}
if err := s.writeSiteSitemap(view); err != nil {
return err
}
if err := writePair(s.Lay.RobotsTxt(), []byte(robotsTxt+"\nSitemap: "+s.absURL("/sitemap.xml")+"\n"), false); err != nil {
return err
}
if err := writePair(s.Lay.AdsTxt(), []byte(adsTxt), false); err != nil {
return err
}
return s.renderErrorPages()
}
// repoRows are the index rows of the repositories among names that are
// on this site and have been rendered, by name.
func (s *Site) repoRows(names []string) (map[string]repoRow, error) {
rows := map[string]repoRow{}
for _, name := range names {
if s.Cfg.Private(name) != (s.prefix != "") {
continue // the other site's; RenderRepo removes it from this one
}
st, err := s.LoadState(name)
if err != nil {
return nil, err
}
if st == nil {
continue // never rendered: no pages to link to yet
}
rows[name] = repoRow{
Name: name,
URL: s.CodeURL(name, ""),
Description: s.Description(name),
Tags: s.Cfg.Repo(name).Tags,
Updated: st.CommitTime.UTC().Format(time.RFC3339),
UpdatedText: st.CommitTime.UTC().Format("2006-01-02"),
UpdatedFull: st.CommitTime.UTC().Format(fullTime),
}
}
return rows, nil
}
// renderPrivateIndex writes the private site's index: its repositories in
// one section. The public site has the sitemap, robots.txt and the error
// pages; this one is for those who log in, and search engines cannot.
func (s *Site) renderPrivateIndex(names []string) error {
rows, err := s.repoRows(names)
if err != nil {
return err
}
var sec sectionView
for _, name := range names {
if row, ok := rows[name]; ok {
sec.Repos = append(sec.Repos, row)
}
}
var view indexView
if len(sec.Repos) > 0 {
view.Sections = append(view.Sections, sec)
}
page := s.basePage("")
page.BodyClass = "index"
page.Title = "Private repositories"
page.Index = &view
html, err := s.render("index", page)
if err != nil {
return err
}
return writePair(s.Lay.IndexPage(), html, true)
}
// adsTxt says no one is authorized to sell ads on the site, in the form
// the ads.txt 1.1 spec gives for that: one placeholder record.
const adsTxt = "placeholder.example.com, placeholder, DIRECT, placeholder\n"
// robotsTxt keeps crawlers to the pages. Tarballs, the clone export and
// raw copies cost the server most to send and are not worth indexing;
// search needs script, and the frames are parts of pages. Branch views
// stay open: crawlers must fetch them to see that they ask not to be
// indexed. Crawl-delay asks for a pause between fetches; Bing and Yandex
// honour it, Google ignores it. RenderIndex adds the sitemap's address,
// which robots.txt must give in full.
const robotsTxt = `User-agent: *
Crawl-delay: 10
Disallow: /archive/
Disallow: /snapshot/
Disallow: /git/
Disallow: /raw/
Disallow: /svg/
Disallow: /search/
Disallow: /iframes/
`
// renderErrorPages writes the pages httpd shows for a missing path and for
// a directory with no index.html. They live with the index because they
// are the site's own, not any repository's.
func (s *Site) renderErrorPages() error {
for _, e := range []struct {
code int
name, title string
}{
{http.StatusNotFound, "notfound", "Page not found"},
{http.StatusForbidden, "forbidden", "Nothing to show here"},
} {
page := s.basePage("")
page.Title = e.title
page.BodyClass = "error"
page.NoIndex = true
html, err := s.render(e.name, page)
if err != nil {
return err
}
// httpd reads error pages as they are, never a gzip twin.
if err := writePair(s.Lay.ErrorPage(e.code), html, false); err != nil {
return err
}
}
return nil
}
|