123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328 | package render
import (
"archive/tar"
"compress/gzip"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"regexp"
"strings"
"time"
"code.bruce-hill.com/repo-renderer/internal/gitx"
)
// ---- tags & archives ----
var tagNameRE = regexp.MustCompile(`^[A-Za-z0-9._-]+$`)
// SanitizeTag maps a tag name to an archive-safe name, or false.
func SanitizeTag(name string) (string, bool) {
s := strings.ReplaceAll(name, "/", "-")
if !tagNameRE.MatchString(s) || strings.HasPrefix(s, ".") {
return "", false
}
return s, true
}
// tagsDoc is a view's download list: a document of its own that the
// Download menu of every page of the view frames. It names the rendered
// commit, so every push rewrites it, and the pages stay as they are.
type tagsDoc struct {
Title string // the view: repo, or repo@branch
Latest latestView
Tags []tagView
}
// latestView is the rendered branch tip.
type latestView struct {
URL string // its tarball; "" when there is none
SHA string // abbreviated
Subject string
When string // RFC 3339, for script to restate as "2 days ago"
WhenText string
WhenFull string
}
type tagView struct {
Name string
URL string
}
// WriteDownloads writes the download list of each rendered view of the
// repository, and removes the view's superseded tip tarballs once they
// are older than archive_retention. Tags are those whose archive exists,
// newest first.
func (s *Site) WriteDownloads(repo string) error {
tags, err := gitx.Repo{Dir: s.Lay.GitDir(repo)}.Tags()
if err != nil {
return err
}
var tagViews []tagView
for _, t := range tags {
name, ok := SanitizeTag(t.Name)
if !ok {
continue
}
if _, err := os.Stat(s.Lay.Archive(repo, name)); err == nil {
tagViews = append(tagViews, tagView{Name: t.Name, URL: s.ArchiveURL(repo, name)})
}
}
branches, err := listDir(s.Lay.StateDir(), func(n string) bool { return strings.HasPrefix(n, repo+"@") })
if err != nil {
return err
}
keys := []string{repo}
for _, b := range branches {
keys = append(keys, filepath.Base(b))
}
var errs []error
for _, key := range keys {
st, err := s.LoadState(key)
if err != nil {
errs = append(errs, err)
continue
}
if st == nil {
continue // never rendered: no page frames its list
}
if err := s.writeDownloads(repo, key, st, tagViews); err != nil {
errs = append(errs, fmt.Errorf("%s: %w", key, err))
}
}
return errors.Join(errs...)
}
func (s *Site) writeDownloads(repo, key string, st *State, tags []tagView) error {
at := st.CommitTime.UTC()
doc := tagsDoc{Title: key, Tags: tags, Latest: latestView{
SHA: st.SHA[:min(12, len(st.SHA))],
Subject: st.Subject,
When: at.Format(time.RFC3339),
WhenText: at.Format("2006-01-02"),
WhenFull: at.Format(fullTime),
}}
if st.Tip != "" {
doc.Latest.URL = s.TipArchiveURL(key, st.Tip)
}
html, err := s.renderDoc("tags", doc)
if err != nil {
return err
}
if err := writePair(s.Lay.Downloads(key), html, true); err != nil {
return err
}
return s.sweepTips(repo, key, st.Tip, st.SHA)
}
// supersedeTip marks a view's previous tip tarball as replaced now, so
// archive_retention counts from the push that replaced it, not from when
// it was made.
func (s *Site) supersedeTip(key, name string) error {
now := time.Now()
err := os.Chtimes(s.Lay.TipArchive(key, name), now, now)
if errors.Is(err, fs.ErrNotExist) {
return nil
}
return err
}
// sweepTips removes a view's tip tarballs other than keep, the tarball of
// tip: those superseded more than archive_retention ago, and straight away
// those of commits no longer in tip's history, which a force push took
// off the branch (a page that links to one is a page of history the
// maintainer withdrew), and temporary files left by writes that died.
func (s *Site) sweepTips(repo, key, keep, tip string) error {
old, err := listDir(s.Lay.SnapshotDir(key), func(n string) bool { return n != keep })
if err != nil {
return err
}
g := gitx.Repo{Dir: s.Lay.GitDir(repo)}
cutoff := time.Now().Add(-s.Cfg.ArchiveRetention)
for _, p := range old {
abbrev, ok := strings.CutPrefix(filepath.Base(p), repo+"@")
abbrev, ok2 := strings.CutSuffix(abbrev, ".tar.gz")
if ok && ok2 && tipAbbrevRE.MatchString(abbrev) {
if in, err := g.IsAncestor(abbrev, tip); err == nil && in {
fi, err := os.Stat(p)
if errors.Is(err, fs.ErrNotExist) {
continue
}
if err != nil {
return err
}
if !fi.ModTime().Before(cutoff) {
continue
}
}
}
if err := removeOutputs(p); err != nil {
return err
}
}
return nil
}
// tipAbbrevRE matches the abbreviated commit in a tip tarball's name.
var tipAbbrevRE = regexp.MustCompile(`^[0-9a-f]{12}$`)
// tipArchiveLevel is the gzip level of the branch-tip tarball. Every push
// rebuilds it, and compressing it is often the push's largest single cost;
// level 2 takes up to a third less time than git's default 6 for files a
// few percent larger. Tag tarballs are made once and keep the default.
const tipArchiveLevel = 2
// prepareArchive writes a tarball of sha with every path under prefix to a
// temporary file next to dst, gzipped at level (0 for git's default);
// commit moves it into place.
func prepareArchive(g gitx.Repo, dst, prefix, sha string, level int) (pending, error) {
t, err := createTemp(dst)
if err != nil {
return pending{}, err
}
// The *os.File itself, so git writes to it directly, not through a pipe.
if err := g.Archive(t.File, sha, prefix, level); err != nil {
t.discard()
return pending{}, err
}
if err := t.finish(); err != nil {
t.discard()
return pending{}, err
}
return pending{commit: t.commit, discard: t.discard}, nil
}
// writeArchive is prepareArchive followed by commit.
func writeArchive(g gitx.Repo, dst, prefix, sha string) error {
p, err := prepareArchive(g, dst, prefix, sha, 0)
if err != nil {
return err
}
defer p.discard()
return p.commit()
}
func (s *Site) writeTagArchive(g gitx.Repo, repo, name, sha string) error {
entries, err := g.LsTree(sha)
if err != nil {
return err
}
if err := s.checkTreeLimits(entries); err != nil {
return err
}
return writeArchive(g, s.Lay.Archive(repo, name), repo+"-"+name+"/", sha)
}
// EnsureArchives makes the repository's tag tarballs match its tags: it
// creates those missing, remakes those of another commit, and removes
// those of tags that are gone. A tag moved or deleted on the server, where
// no hook runs, or a tag event whose job failed, must not leave a tarball
// of what the tag used to name.
func (s *Site) EnsureArchives(repo string) error {
g := gitx.Repo{Dir: s.Lay.GitDir(repo)}
tags, err := g.Tags()
if err != nil {
return err
}
keep := map[string]bool{}
var errs []error
for _, t := range tags {
name, ok := SanitizeTag(t.Name)
if !ok {
s.Log.Printf("%s: skipping tag %q: unsafe name", repo, t.Name)
continue
}
dst := s.Lay.Archive(repo, name)
if keep[dst] {
// Tags list newest first, and the newest keeps the name.
s.Log.Printf("%s: skipping tag %q: its tarball's name is taken by a newer tag", repo, t.Name)
continue
}
keep[dst] = true
if archiveCommit(dst) == t.SHA {
continue
}
if err := s.writeTagArchive(g, repo, name, t.SHA); err != nil {
// A tarball of what the tag no longer names must not stay.
errs = append(errs, fmt.Errorf("tag %s: %w", t.Name, err), removeOutputs(dst))
}
}
// Anything else here is a tarball of a tag that is gone, or a
// temporary file left by a write that died.
old, err := listDir(s.Lay.ArchiveDir(repo), func(string) bool { return true })
if err != nil {
return errors.Join(append(errs, err)...)
}
for _, p := range old {
if !keep[p] {
s.Log.Printf("%s: removing %s: no tag names it", repo, filepath.Base(p))
errs = append(errs, removeOutputs(p))
}
}
return errors.Join(errs...)
}
// archiveCommit is the commit a tarball git archive made holds, from the
// comment git writes in its pax global header, or "" when there is none:
// the file is missing or unreadable, or git archived a tree, not a commit.
func archiveCommit(p string) string {
f, err := os.Open(p)
if err != nil {
return ""
}
defer func() { _ = f.Close() }()
zr, err := gzip.NewReader(f)
if err != nil {
return ""
}
h, err := tar.NewReader(zr).Next()
if err != nil || h.Typeflag != tar.TypeXGlobalHeader {
return ""
}
return h.PAXRecords["comment"]
}
// HandleTag processes one refs/tags event: create/overwrite the archive, or
// delete it when the tag was removed.
func (s *Site) HandleTag(repo, tag, newSHA string) error {
name, ok := SanitizeTag(tag)
if !ok {
s.Log.Printf("%s: skipping tag %q: unsafe name", repo, tag)
return nil
}
if gitx.IsZeroSHA(newSHA) {
s.Log.Printf("%s: removing archive for tag %s", repo, tag)
return removeOutputs(s.Lay.Archive(repo, name))
}
g := gitx.Repo{Dir: s.Lay.GitDir(repo)}
s.Log.Printf("%s: archiving tag %s", repo, tag)
if err := s.writeTagArchive(g, repo, name, newSHA); err != nil {
// A tarball of what the tag named before must not stay.
return errors.Join(err, removeOutputs(s.Lay.Archive(repo, name)))
}
return nil
}
// checkTreeLimits fails when a tree has more than repo_max_files entries
// or its files add up to more than repo_max_bytes, before anything of it
// is written.
func (s *Site) checkTreeLimits(entries []gitx.Entry) error {
if n := s.Cfg.RepoMaxFiles; n > 0 && len(entries) > n {
return fmt.Errorf("it has %d files, more than repo_max_files (%d)", len(entries), n)
}
limit := s.Cfg.RepoMaxBytes
if limit == 0 {
return nil
}
var total int64
for _, e := range entries {
total += max(e.Size, 0)
}
if total > limit {
return fmt.Errorf("its files add up to %s, more than repo_max_bytes (%s)", humanSize(total), humanSize(limit))
}
return nil
}
|