123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315 | package render
import (
"errors"
"fmt"
"os"
"slices"
"strings"
"code.bruce-hill.com/repo-renderer/internal/config"
"code.bruce-hill.com/repo-renderer/internal/gitx"
"code.bruce-hill.com/repo-renderer/internal/queue"
"code.bruce-hill.com/repo-renderer/internal/sandbox"
)
// ---- the job queue and render runs ----
// Enqueue appends a job to the queue. It needs only the config: the
// post-receive hook runs it as the git user on every push, so it must be
// fast.
func Enqueue(cfg *config.Config, j queue.Job) error {
if err := checkJob(j); err != nil {
return err
}
lay := LayoutOf(cfg)
if err := os.MkdirAll(lay.QueueDir, 0o755); err != nil {
return err
}
return queue.Append(lay.QueuePath(), j)
}
// checkJob rejects a job that names no valid repository, carries a ref
// that would break the queue's line format, or object ids that are not
// SHAs: a ref expression there would reach git archive as a tree-ish.
func checkJob(j queue.Job) error {
switch {
case !config.ValidRepoName(j.Repo):
return fmt.Errorf("invalid repo name %q", j.Repo)
case strings.ContainsAny(j.Ref, "\t\n"):
return fmt.Errorf("malformed ref %q", j.Ref)
case !gitx.IsSHA(j.Old) || !gitx.IsSHA(j.New):
return fmt.Errorf("malformed object id in %q", []string{j.Old, j.New})
}
return nil
}
// Kick drains the job queue under the render lock.
func (s *Site) Kick() error {
if err := sandbox.Nice(10); err != nil {
s.Log.Printf("setpriority: %v", err)
}
if err := s.restrict(); err != nil {
return err
}
return s.withLock(func() error {
// Claims left behind by a crashed run; under the lock, nothing
// else leaves one. They stay on disk until their jobs are done,
// so another crash recovers them again.
jobs, done, err := queue.Recover(s.Lay.QueuePath())
if err != nil {
return err
}
for {
fresh, claimed, err := queue.Take(s.Lay.QueuePath())
if err != nil {
return err
}
jobs = append(jobs, fresh...)
if claimed != "" {
done = append(done, claimed)
}
if len(jobs) > 0 {
s.processJobs(jobs)
}
// Claimed files holding only torn lines go too.
for _, f := range done {
if err := os.Remove(f); err != nil {
return err
}
}
// Pushes may have queued more while these ran.
if len(jobs) == 0 && claimed == "" {
return nil
}
jobs, done = nil, nil
}
})
}
// processJobs groups queue entries by repository: one branch render per
// repo (the tip is rendered, so the newest push wins) plus tag events in
// order. Errors are logged per repo so one bad repo never blocks the rest.
func (s *Site) processJobs(jobs []queue.Job) {
type work struct {
main bool
tags []queue.Job
}
var order []string
byRepo := map[string]*work{}
for _, j := range jobs {
// Only the git user writes the queue, but check again anyway.
if err := checkJob(j); err != nil {
s.Log.Printf("queue: ignoring job: %v", err)
continue
}
w := byRepo[j.Repo]
if w == nil {
w = &work{}
byRepo[j.Repo] = w
order = append(order, j.Repo)
}
switch {
case strings.HasPrefix(j.Ref, "refs/tags/"):
w.tags = append(w.tags, j)
case strings.HasPrefix(j.Ref, "refs/heads/"):
branch, err := s.DefaultBranch(j.Repo)
if err != nil {
s.Log.Printf("%s: %v", j.Repo, err)
continue
}
name := strings.TrimPrefix(j.Ref, "refs/heads/")
if name != branch && !slices.Contains(s.Cfg.Repo(j.Repo).Branches, name) {
s.Log.Printf("%s: push to %s not shown; the site shows %s (set repos.%s.branch or .branches to change that)", j.Repo, name, branch, j.Repo)
continue
}
w.main = true
}
}
var touched []string
for _, repo := range order {
w := byRepo[repo]
// What it has on the site it is not rendered to goes first, so
// that a tag push alone moves it too.
changed, err := s.removeElsewhere(repo)
if err != nil {
s.Log.Printf("%s: %v", repo, err)
}
site := s.siteFor(repo)
if site == nil {
// Private with no private site: nothing of it is published,
// tags included, and it is only mirrored.
if changed || w.main || len(w.tags) > 0 {
touched = append(touched, repo)
}
continue
}
if !w.main && len(w.tags) > 0 {
// A tag push to a repository its site has not rendered, as
// when it has just moved between the sites, renders it all.
if st, err := site.LoadState(repo); err == nil && st == nil {
w.main = true
}
}
// Tags first: the branch render rewrites the download lists from
// the archives that exist.
for _, t := range w.tags {
if err := site.HandleTag(repo, strings.TrimPrefix(t.Ref, "refs/tags/"), t.New); err != nil {
s.Log.Printf("%s: tag %s: %v", repo, t.Ref, err)
}
changed = true
}
if w.main {
c, err := s.RenderRepo(repo, false)
if err != nil {
s.Log.Printf("%s: render failed: %v", repo, err)
}
changed = changed || c
} else if len(w.tags) > 0 {
// Pages frame the download lists, so they are the only pages
// to rewrite. RenderRepo would bring every tag's tarball in
// line with its tag first; this push needs that as much.
if err := site.EnsureArchives(repo); err != nil {
s.Log.Printf("%s: archives: %v", repo, err)
}
if err := site.WriteDownloads(repo); err != nil {
s.Log.Printf("%s: download lists: %v", repo, err)
}
}
if changed {
touched = append(touched, repo)
s.exportRepo(repo)
}
}
if len(touched) > 0 {
if err := s.RenderIndex(); err != nil {
s.Log.Printf("index: %v", err)
}
}
s.mirrorRepos(touched)
}
// Render renders repos at their branch tips, then the index. full
// re-renders everything, ignoring saved state.
func (s *Site) Render(repos []string, full bool) error {
return s.renderRepos(repos, full, false)
}
// RenderAll renders every repository under git_dir, and removes the
// output of any other: the site then publishes exactly what is there.
func (s *Site) RenderAll(full bool) error {
repos, err := s.Repos()
if err != nil {
return err
}
return s.renderRepos(repos, full, true)
}
func (s *Site) renderRepos(repos []string, full, removeOthers bool) error {
if err := s.restrict(); err != nil {
return err
}
return s.withLock(func() error {
var errs []error
var touched []string
for _, repo := range repos {
changed, err := s.RenderRepo(repo, full)
if err != nil {
errs = append(errs, fmt.Errorf("%s: %w", repo, err))
continue
}
if changed || full {
touched = append(touched, repo)
s.exportRepo(repo)
}
}
if removeOthers {
if err := s.RemoveOthers(repos); err != nil {
errs = append(errs, err)
}
}
if err := s.RenderIndex(); err != nil {
errs = append(errs, err)
}
s.mirrorRepos(touched)
return errors.Join(errs...)
})
}
// restrict sandboxes the renderer to the site's own files. A log file is
// reopened for git's output when mirroring, so it needs an entry too, as
// does syslog's socket, which the logger reconnects to if it must.
func (s *Site) restrict() error {
paths := []sandbox.Path{
{Path: s.Lay.GitBase, Perms: "r"}, // read only: pushes are the only way in
{Path: s.Lay.Htdocs, Perms: "rwc"},
{Path: s.Lay.QueueDir, Perms: "rwc"},
{Path: s.Lay.State, Perms: "rwc"},
}
logPath, logPerms := s.Cfg.Log, "rwc"
if logPath == config.LogSyslog {
logPath, logPerms = "/dev/log", "rw"
}
for _, p := range []sandbox.Path{
{Path: logPath, Perms: logPerms},
{Path: s.Cfg.PrivateHtdocs, Perms: "rwc"},
{Path: s.Cfg.SSHKey, Perms: "r"},
{Path: s.Cfg.Site.Favicon, Perms: "r"}, // copied into static/ by index
} {
if p.Path != "" {
paths = append(paths, p)
}
}
for _, d := range s.Cfg.LexerDirs {
paths = append(paths, sandbox.Path{Path: d, Perms: "r"})
}
return sandbox.Restrict(paths)
}
// withLock runs fn holding the render lock.
func (s *Site) withLock(fn func() error) error {
if err := os.MkdirAll(s.Lay.State, 0o755); err != nil {
return err
}
lock, err := queue.Lock(s.Lay.LockPath())
if err != nil {
return fmt.Errorf("lock: %w", err)
}
defer func() { _ = lock.Close() }() // releases the lock; exit would too
return fn()
}
// exportRepo refreshes a changed repository's clone export, on the site
// it is rendered to. A private repository with no private site has none.
func (s *Site) exportRepo(repo string) {
site := s.siteFor(repo)
if site == nil {
return
}
branch, err := s.DefaultBranch(repo)
if err != nil {
s.Log.Printf("%s: %v", repo, err)
return
}
if err := site.ExportPub(repo, branch); err != nil {
s.Log.Printf("%s: clone export failed: %v", repo, err)
}
}
// mirrorRepos pushes changed repositories to their mirrors, unless
// NoMirror. Callers run it last, once every page and the index are
// written: a push can take a minute, and a slow mirror should not hold
// back the site.
func (s *Site) mirrorRepos(repos []string) {
if s.NoMirror {
return
}
for _, repo := range repos {
branch, err := s.DefaultBranch(repo)
if err != nil {
s.Log.Printf("%s: %v", repo, err)
continue
}
s.Mirror(repo, branch)
}
}
|