Download
315 lines 8.8 KiBGoModified 2026-10-27 raw
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315
package render

import (
	"errors"
	"fmt"
	"os"
	"slices"
	"strings"

	"code.bruce-hill.com/repo-renderer/internal/config"
	"code.bruce-hill.com/repo-renderer/internal/gitx"
	"code.bruce-hill.com/repo-renderer/internal/queue"
	"code.bruce-hill.com/repo-renderer/internal/sandbox"
)

// ---- the job queue and render runs ----

// Enqueue appends a job to the queue. It needs only the config: the
// post-receive hook runs it as the git user on every push, so it must be
// fast.
func Enqueue(cfg *config.Config, j queue.Job) error {
	if err := checkJob(j); err != nil {
		return err
	}
	lay := LayoutOf(cfg)
	if err := os.MkdirAll(lay.QueueDir, 0o755); err != nil {
		return err
	}
	return queue.Append(lay.QueuePath(), j)
}

// checkJob rejects a job that names no valid repository, carries a ref
// that would break the queue's line format, or object ids that are not
// SHAs: a ref expression there would reach git archive as a tree-ish.
func checkJob(j queue.Job) error {
	switch {
	case !config.ValidRepoName(j.Repo):
		return fmt.Errorf("invalid repo name %q", j.Repo)
	case strings.ContainsAny(j.Ref, "\t\n"):
		return fmt.Errorf("malformed ref %q", j.Ref)
	case !gitx.IsSHA(j.Old) || !gitx.IsSHA(j.New):
		return fmt.Errorf("malformed object id in %q", []string{j.Old, j.New})
	}
	return nil
}

// Kick drains the job queue under the render lock.
func (s *Site) Kick() error {
	if err := sandbox.Nice(10); err != nil {
		s.Log.Printf("setpriority: %v", err)
	}
	if err := s.restrict(); err != nil {
		return err
	}
	return s.withLock(func() error {
		// Claims left behind by a crashed run; under the lock, nothing
		// else leaves one. They stay on disk until their jobs are done,
		// so another crash recovers them again.
		jobs, done, err := queue.Recover(s.Lay.QueuePath())
		if err != nil {
			return err
		}
		for {
			fresh, claimed, err := queue.Take(s.Lay.QueuePath())
			if err != nil {
				return err
			}
			jobs = append(jobs, fresh...)
			if claimed != "" {
				done = append(done, claimed)
			}
			if len(jobs) > 0 {
				s.processJobs(jobs)
			}
			// Claimed files holding only torn lines go too.
			for _, f := range done {
				if err := os.Remove(f); err != nil {
					return err
				}
			}
			// Pushes may have queued more while these ran.
			if len(jobs) == 0 && claimed == "" {
				return nil
			}
			jobs, done = nil, nil
		}
	})
}

// processJobs groups queue entries by repository: one branch render per
// repo (the tip is rendered, so the newest push wins) plus tag events in
// order. Errors are logged per repo so one bad repo never blocks the rest.
func (s *Site) processJobs(jobs []queue.Job) {
	type work struct {
		main bool
		tags []queue.Job
	}
	var order []string
	byRepo := map[string]*work{}
	for _, j := range jobs {
		// Only the git user writes the queue, but check again anyway.
		if err := checkJob(j); err != nil {
			s.Log.Printf("queue: ignoring job: %v", err)
			continue
		}
		w := byRepo[j.Repo]
		if w == nil {
			w = &work{}
			byRepo[j.Repo] = w
			order = append(order, j.Repo)
		}
		switch {
		case strings.HasPrefix(j.Ref, "refs/tags/"):
			w.tags = append(w.tags, j)
		case strings.HasPrefix(j.Ref, "refs/heads/"):
			branch, err := s.DefaultBranch(j.Repo)
			if err != nil {
				s.Log.Printf("%s: %v", j.Repo, err)
				continue
			}
			name := strings.TrimPrefix(j.Ref, "refs/heads/")
			if name != branch && !slices.Contains(s.Cfg.Repo(j.Repo).Branches, name) {
				s.Log.Printf("%s: push to %s not shown; the site shows %s (set repos.%s.branch or .branches to change that)", j.Repo, name, branch, j.Repo)
				continue
			}
			w.main = true
		}
	}
	var touched []string
	for _, repo := range order {
		w := byRepo[repo]
		// What it has on the site it is not rendered to goes first, so
		// that a tag push alone moves it too.
		changed, err := s.removeElsewhere(repo)
		if err != nil {
			s.Log.Printf("%s: %v", repo, err)
		}
		site := s.siteFor(repo)
		if site == nil {
			// Private with no private site: nothing of it is published,
			// tags included, and it is only mirrored.
			if changed || w.main || len(w.tags) > 0 {
				touched = append(touched, repo)
			}
			continue
		}
		if !w.main && len(w.tags) > 0 {
			// A tag push to a repository its site has not rendered, as
			// when it has just moved between the sites, renders it all.
			if st, err := site.LoadState(repo); err == nil && st == nil {
				w.main = true
			}
		}
		// Tags first: the branch render rewrites the download lists from
		// the archives that exist.
		for _, t := range w.tags {
			if err := site.HandleTag(repo, strings.TrimPrefix(t.Ref, "refs/tags/"), t.New); err != nil {
				s.Log.Printf("%s: tag %s: %v", repo, t.Ref, err)
			}
			changed = true
		}
		if w.main {
			c, err := s.RenderRepo(repo, false)
			if err != nil {
				s.Log.Printf("%s: render failed: %v", repo, err)
			}
			changed = changed || c
		} else if len(w.tags) > 0 {
			// Pages frame the download lists, so they are the only pages
			// to rewrite. RenderRepo would bring every tag's tarball in
			// line with its tag first; this push needs that as much.
			if err := site.EnsureArchives(repo); err != nil {
				s.Log.Printf("%s: archives: %v", repo, err)
			}
			if err := site.WriteDownloads(repo); err != nil {
				s.Log.Printf("%s: download lists: %v", repo, err)
			}
		}
		if changed {
			touched = append(touched, repo)
			s.exportRepo(repo)
		}
	}
	if len(touched) > 0 {
		if err := s.RenderIndex(); err != nil {
			s.Log.Printf("index: %v", err)
		}
	}
	s.mirrorRepos(touched)
}

// Render renders repos at their branch tips, then the index. full
// re-renders everything, ignoring saved state.
func (s *Site) Render(repos []string, full bool) error {
	return s.renderRepos(repos, full, false)
}

// RenderAll renders every repository under git_dir, and removes the
// output of any other: the site then publishes exactly what is there.
func (s *Site) RenderAll(full bool) error {
	repos, err := s.Repos()
	if err != nil {
		return err
	}
	return s.renderRepos(repos, full, true)
}

func (s *Site) renderRepos(repos []string, full, removeOthers bool) error {
	if err := s.restrict(); err != nil {
		return err
	}
	return s.withLock(func() error {
		var errs []error
		var touched []string
		for _, repo := range repos {
			changed, err := s.RenderRepo(repo, full)
			if err != nil {
				errs = append(errs, fmt.Errorf("%s: %w", repo, err))
				continue
			}
			if changed || full {
				touched = append(touched, repo)
				s.exportRepo(repo)
			}
		}
		if removeOthers {
			if err := s.RemoveOthers(repos); err != nil {
				errs = append(errs, err)
			}
		}
		if err := s.RenderIndex(); err != nil {
			errs = append(errs, err)
		}
		s.mirrorRepos(touched)
		return errors.Join(errs...)
	})
}

// restrict sandboxes the renderer to the site's own files. A log file is
// reopened for git's output when mirroring, so it needs an entry too, as
// does syslog's socket, which the logger reconnects to if it must.
func (s *Site) restrict() error {
	paths := []sandbox.Path{
		{Path: s.Lay.GitBase, Perms: "r"}, // read only: pushes are the only way in
		{Path: s.Lay.Htdocs, Perms: "rwc"},
		{Path: s.Lay.QueueDir, Perms: "rwc"},
		{Path: s.Lay.State, Perms: "rwc"},
	}
	logPath, logPerms := s.Cfg.Log, "rwc"
	if logPath == config.LogSyslog {
		logPath, logPerms = "/dev/log", "rw"
	}
	for _, p := range []sandbox.Path{
		{Path: logPath, Perms: logPerms},
		{Path: s.Cfg.PrivateHtdocs, Perms: "rwc"},
		{Path: s.Cfg.SSHKey, Perms: "r"},
		{Path: s.Cfg.Site.Favicon, Perms: "r"}, // copied into static/ by index
	} {
		if p.Path != "" {
			paths = append(paths, p)
		}
	}
	for _, d := range s.Cfg.LexerDirs {
		paths = append(paths, sandbox.Path{Path: d, Perms: "r"})
	}
	return sandbox.Restrict(paths)
}

// withLock runs fn holding the render lock.
func (s *Site) withLock(fn func() error) error {
	if err := os.MkdirAll(s.Lay.State, 0o755); err != nil {
		return err
	}
	lock, err := queue.Lock(s.Lay.LockPath())
	if err != nil {
		return fmt.Errorf("lock: %w", err)
	}
	defer func() { _ = lock.Close() }() // releases the lock; exit would too
	return fn()
}

// exportRepo refreshes a changed repository's clone export, on the site
// it is rendered to. A private repository with no private site has none.
func (s *Site) exportRepo(repo string) {
	site := s.siteFor(repo)
	if site == nil {
		return
	}
	branch, err := s.DefaultBranch(repo)
	if err != nil {
		s.Log.Printf("%s: %v", repo, err)
		return
	}
	if err := site.ExportPub(repo, branch); err != nil {
		s.Log.Printf("%s: clone export failed: %v", repo, err)
	}
}

// mirrorRepos pushes changed repositories to their mirrors, unless
// NoMirror. Callers run it last, once every page and the index are
// written: a push can take a minute, and a slow mirror should not hold
// back the site.
func (s *Site) mirrorRepos(repos []string) {
	if s.NoMirror {
		return
	}
	for _, repo := range repos {
		branch, err := s.DefaultBranch(repo)
		if err != nil {
			s.Log.Printf("%s: %v", repo, err)
			continue
		}
		s.Mirror(repo, branch)
	}
}