123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206 | package queue
import (
"fmt"
"os"
"path/filepath"
"strings"
"testing"
)
func TestAppendTakeRecover(t *testing.T) {
q := filepath.Join(t.TempDir(), "queue")
jobs, claimed, err := Take(q)
if err != nil || len(jobs) != 0 || claimed != "" {
t.Fatalf("empty take: %v %v %q", jobs, err, claimed)
}
for _, j := range []Job{
{"repo", "refs/heads/main", "0", "1"},
{"repo", "refs/tags/v1", "0", "2"},
} {
if err := Append(q, j); err != nil {
t.Fatal(err)
}
}
if err := Append(q, Job{"repo", "refs/heads/main", "0", "1"}); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(q, append(mustRead(t, q), []byte("torn line\n")...), 0o644); err != nil {
t.Fatal(err)
}
jobs, claimed, err = Take(q)
if err != nil || len(jobs) != 3 || jobs[1].Ref != "refs/tags/v1" {
t.Fatalf("take: %+v %v", jobs, err)
}
if _, err := os.Stat(q); err == nil {
t.Error("queue should have been renamed aside")
}
// Simulate a crash before the claimed file was deleted.
rec, files, err := Recover(q)
if err != nil || len(rec) != 3 || len(files) != 1 || files[0] != claimed {
t.Fatalf("recover: %+v %q %v", rec, files, err)
}
// Recovering again, as after a second crash, finds the same jobs.
if rec, _, err := Recover(q); err != nil || len(rec) != 3 {
t.Fatalf("second recover: %+v %v", rec, err)
}
if err := os.Remove(claimed); err != nil {
t.Fatal(err)
}
if rec, files, err := Recover(q); err != nil || len(rec) != 0 || len(files) != 0 {
t.Errorf("recover after removal should be empty: %v %q %v", rec, files, err)
}
}
// A claim left by a crashed run under this process's pid is not
// overwritten by the next Take.
func TestTakeKeepsLeftoverClaim(t *testing.T) {
q := filepath.Join(t.TempDir(), "queue")
leftover := fmt.Sprintf("%s.%d", q, os.Getpid())
if err := os.WriteFile(leftover, []byte("old\trefs/heads/main\t0\t1\n"), 0o644); err != nil {
t.Fatal(err)
}
if err := Append(q, Job{"new", "refs/heads/main", "0", "2"}); err != nil {
t.Fatal(err)
}
jobs, claimed, err := Take(q)
if err != nil || len(jobs) != 1 || jobs[0].Repo != "new" {
t.Fatalf("take: %+v %v", jobs, err)
}
if claimed == leftover {
t.Fatalf("take claimed the leftover's name %q", claimed)
}
if rec, _, err := Recover(q); err != nil || len(rec) != 2 {
t.Errorf("both claims should be recoverable: %+v %v", rec, err)
}
}
// A line too long for any buffer must be skipped like any other bad line,
// not fail the take: the claimed file would then fail every Recover after.
func TestTakeSkipsHugeLine(t *testing.T) {
q := filepath.Join(t.TempDir(), "queue")
huge := strings.Repeat("x", 2<<20) + "\n"
if err := os.WriteFile(q, []byte(huge), 0o644); err != nil {
t.Fatal(err)
}
if err := Append(q, Job{"repo", "refs/heads/main", "0", "1"}); err != nil {
t.Fatal(err)
}
jobs, _, err := Take(q)
if err != nil || len(jobs) != 1 || jobs[0].Repo != "repo" {
t.Fatalf("take: %+v %v", jobs, err)
}
}
// A line ending in CRLF (say, after a hand edit) keeps its job, as it did
// when lines came from a bufio.Scanner, which drops the CR.
func TestTakeCRLF(t *testing.T) {
q := filepath.Join(t.TempDir(), "queue")
if err := os.WriteFile(q, []byte("repo\trefs/heads/main\t0\t1\r\n"), 0o644); err != nil {
t.Fatal(err)
}
jobs, _, err := Take(q)
if err != nil || len(jobs) != 1 || jobs[0].New != "1" {
t.Fatalf("take: %q %v", jobs, err)
}
}
// Claimed files are found by name, not by globbing the queue's path, so
// a spool directory whose name holds glob characters still recovers.
func TestRecoverGlobCharsInDir(t *testing.T) {
dir := filepath.Join(t.TempDir(), "spool[1]")
if err := os.Mkdir(dir, 0o755); err != nil {
t.Fatal(err)
}
q := filepath.Join(dir, "queue")
if err := Append(q, Job{"repo", "refs/heads/main", "0", "1"}); err != nil {
t.Fatal(err)
}
if err := os.Rename(q, q+".123"); err != nil { // a claim left by a crash
t.Fatal(err)
}
jobs, files, err := Recover(q)
if err != nil || len(jobs) != 1 || len(files) != 1 || files[0] != q+".123" {
t.Fatalf("recover: %+v %q %v", jobs, files, err)
}
}
func mustRead(t *testing.T, p string) []byte {
t.Helper()
b, err := os.ReadFile(p)
if err != nil {
t.Fatal(err)
}
return b
}
func TestLock(t *testing.T) {
p := filepath.Join(t.TempDir(), "lock")
f, err := Lock(p)
if err != nil {
t.Fatal(err)
}
if err := f.Close(); err != nil {
t.Fatal(err)
}
f2, err := Lock(p)
if err != nil {
t.Fatal(err)
}
if err := f2.Close(); err != nil {
t.Fatal(err)
}
}
// Appends racing with Take must all land in some claimed batch: none may be
// written into a file Take has already read.
func TestAppendRacesTake(t *testing.T) {
q := filepath.Join(t.TempDir(), "queue")
const writers, each = 8, 200
done := make(chan struct{})
errs := make(chan error, writers)
for range writers {
go func() {
for range each {
if err := Append(q, Job{"repo", "refs/heads/main", "0", "1"}); err != nil {
errs <- err
return
}
}
errs <- nil
}()
}
go func() {
for range writers {
if err := <-errs; err != nil {
t.Error(err)
}
}
close(done)
}()
got := 0
take := func() {
jobs, claimed, err := Take(q)
if err != nil {
t.Fatal(err)
}
got += len(jobs)
if claimed != "" {
if err := os.Remove(claimed); err != nil {
t.Fatal(err)
}
}
}
for running := true; running; {
select {
case <-done:
running = false
default:
}
take()
}
take()
if got != writers*each {
t.Fatalf("took %d jobs, appended %d", got, writers*each)
}
}
|