llm
Download
535 lines 22 KiBCModified 2026-08-31 raw
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535
// The runtime test harness used by `tomo test`. See test_harness.h.
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/ioctl.h>
#include <sys/wait.h>
#include <time.h>
#include <unistd.h>

#include "files.h" // highlight_error_to, load_file
#include "report.h" // shared with `tomo format --check`
#include "stdlib.h" // USE_COLOR
#include "test_harness.h"

#define DEFAULT_TIMEOUT_SECS 10

// ---- child-output capture -------------------------------------------------

// Read everything from `fd` until EOF into a NUL-terminated, malloc'd buffer.
static char *drain_fd(int fd) {
    size_t cap = 1024, len = 0;
    char *buf = malloc(cap);
    ssize_t n;
    while ((n = read(fd, buf + len, cap - len - 1)) > 0) {
        len += (size_t)n;
        if (len + 1 >= cap) {
            cap *= 2;
            buf = realloc(buf, cap);
        }
    }
    buf[len] = '\0';
    return buf;
}

// Strip the TOMO_FAIL_SPAN_TAG trailer (if any) off the end of `output`, reporting the span it named. The trailer is
// removed *before* classification so `fails "..."` substring matching never sees it.
void tomo_test_take_span(char *output, const char **file, int *start, int *end) {
    if (!output) return;
    char *tag = NULL;
    for (char *p = output; (p = strstr(p, TOMO_FAIL_SPAN_TAG)); p += 1)
        tag = p; // the last one wins: an earlier one could only be program output faking the tag
    if (!tag) return;
    char *fields = tag + sizeof(TOMO_FAIL_SPAN_TAG) - 1;
    char *sep1 = strchr(fields, '\x1e');
    if (!sep1) return;
    char *sep2 = strchr(sep1 + 1, '\x1e');
    if (!sep2) return;
    *sep1 = '\0';
    *file = fields;
    *start = atoi(sep1 + 1);
    *end = atoi(sep2 + 1);
    // Drop the trailer, plus the newline that separated it from the message:
    while (tag > output && tag[-1] == '\n')
        tag -= 1;
    *tag = '\0';
}

static test_outcome_t classify(int status, const char *output, const tomo_test_t *t) {
    if (WIFSIGNALED(status) && WTERMSIG(status) == SIGALRM) return TEST_RESULT_TIMEOUT;
    bool failed = !WIFEXITED(status) || WEXITSTATUS(status) != 0;
    if (!t->expect_failure) return failed ? TEST_RESULT_UNEXPECTED_FAILURE : TEST_RESULT_PASS;
    if (!failed) return TEST_RESULT_UNEXPECTED_SUCCESS;
    if (t->expected_msg && t->expected_msg[0] && !strstr(output, t->expected_msg)) return TEST_RESULT_WRONG_MESSAGE;
    return TEST_RESULT_PASS;
}

// ---- record protocol ------------------------------------------------------

static void write_all(int fd, const void *buf, size_t n) {
    const char *p = buf;
    while (n > 0) {
        ssize_t w = write(fd, p, n);
        if (w <= 0) return;
        p += w;
        n -= (size_t)w;
    }
}

// Returns the number of bytes actually read (short read only at EOF).
static size_t read_all(int fd, void *buf, size_t n) {
    char *p = buf;
    size_t got = 0;
    while (got < n) {
        ssize_t r = read(fd, p + got, n - got);
        if (r <= 0) break;
        got += (size_t)r;
    }
    return got;
}

void tomo_test_write_record(int fd, const test_result_t *r) {
    int32_t outcome = (int32_t)r->outcome;
    uint8_t ef = r->expect_failure ? 1 : 0;
    int32_t llen = (int32_t)strlen(r->label);
    int32_t elen = r->expected_msg ? (int32_t)strlen(r->expected_msg) : 0;
    int32_t olen = r->output ? (int32_t)strlen(r->output) : 0;
    write_all(fd, &outcome, sizeof(outcome));
    write_all(fd, &ef, sizeof(ef));
    write_all(fd, &llen, sizeof(llen));
    write_all(fd, r->label, (size_t)llen);
    write_all(fd, &elen, sizeof(elen));
    if (elen) write_all(fd, r->expected_msg, (size_t)elen);
    write_all(fd, &olen, sizeof(olen));
    if (olen) write_all(fd, r->output, (size_t)olen);
    int32_t first = (int32_t)r->first_line, last = (int32_t)r->last_line;
    write_all(fd, &first, sizeof(first));
    write_all(fd, &last, sizeof(last));
    int32_t flen = r->fail_file ? (int32_t)strlen(r->fail_file) : 0;
    write_all(fd, &flen, sizeof(flen));
    if (flen) write_all(fd, r->fail_file, (size_t)flen);
    int32_t fstart = (int32_t)r->fail_start, fend = (int32_t)r->fail_end;
    write_all(fd, &fstart, sizeof(fstart));
    write_all(fd, &fend, sizeof(fend));
    write_all(fd, &r->seconds, sizeof(r->seconds));
}

static char *read_str(int fd, int32_t len) {
    char *s = malloc((size_t)len + 1);
    read_all(fd, s, (size_t)len);
    s[len] = '\0';
    return s;
}

bool tomo_test_read_record(int fd, test_result_t *out) {
    int32_t outcome;
    if (read_all(fd, &outcome, sizeof(outcome)) != sizeof(outcome)) return false; // EOF at boundary
    uint8_t ef = 0;
    read_all(fd, &ef, sizeof(ef));
    int32_t llen = 0;
    read_all(fd, &llen, sizeof(llen));
    out->label = read_str(fd, llen);
    int32_t elen = 0;
    read_all(fd, &elen, sizeof(elen));
    out->expected_msg = elen ? read_str(fd, elen) : NULL;
    int32_t olen = 0;
    read_all(fd, &olen, sizeof(olen));
    out->output = read_str(fd, olen);
    int32_t first = 0, last = 0;
    read_all(fd, &first, sizeof(first));
    read_all(fd, &last, sizeof(last));
    int32_t flen = 0;
    read_all(fd, &flen, sizeof(flen));
    out->fail_file = flen ? read_str(fd, flen) : NULL;
    int32_t fstart = 0, fend = 0;
    read_all(fd, &fstart, sizeof(fstart));
    read_all(fd, &fend, sizeof(fend));
    double seconds = 0.0;
    read_all(fd, &seconds, sizeof(seconds));
    out->outcome = (test_outcome_t)outcome;
    out->expect_failure = ef != 0;
    out->first_line = first;
    out->last_line = last;
    out->fail_start = fstart;
    out->fail_end = fend;
    out->seconds = seconds;
    out->file = NULL;
    return true;
}

// ---- rendering ------------------------------------------------------------
//
// Two looks, chosen by USE_COLOR (a tty, or COLOR=1): a rich report with color,
// box drawing and source excerpts; and a plain ASCII one that
// stays greppable and log-friendly when piped. Both carry the same
// information: the plain version drops only decoration, never a detail you'd
// need to fix a failure.

// ---- source excerpts ------------------------------------------------------

// Show the source around a failure. This hands off to the same renderer the
// compiler and runtime error paths use, so a failing test's excerpt is shaped
// exactly like every other diagnostic Tomo prints: the file name in bold
// yellow, dim line numbers behind a box-drawing gutter, the offending span
// highlighted inline (or underlined with carets when there's no color).
static void print_excerpt(FILE *f, const test_result_t *r, int indent) {
    file_t *file = NULL;
    const char *start = NULL, *end = NULL;

    // The span comes from a trailer the test's *own* output carries, so it is
    // only as trustworthy as the program under test: bound both ends against
    // the file before turning them into pointers.
    if (r->fail_file && r->fail_start >= 0 && r->fail_end > r->fail_start) {
        file = load_file(r->fail_file);
        if (file && (int64_t)r->fail_end <= file->len) {
            start = file->text + r->fail_start;
            end = file->text + r->fail_end;
        } else file = NULL;
    }
    if (!file && r->file && r->first_line > 0) {
        // No precise span (a timeout, or a test that should have failed and didn't): point at the `test` line so
        // there's still somewhere to look.
        file = load_file(r->file);
        if (file) {
            start = get_line(file, r->first_line);
            if (start) end = start + strcspn(start, "\r\n");
            else file = NULL;
        }
    }
    if (!file || !start) return;

    highlight_error_to(f, indent, file, start, end, "\x1b[91;7;1m", 2, USE_COLOR);
    fprintf(f, "\n");
}

// ---- failure details ------------------------------------------------------

// The test's own stdout/stderr, behind a gutter so it can't be mistaken for
// the report's own text.
static void print_output(FILE *f, const char *output, int indent, style_t s, const char *color) {
    if (!output) return;
    size_t end = strlen(output);
    while (end > 0 && output[end - 1] == '\n')
        end -= 1; // no dangling blank line
    if (end == 0) return;
    for (const char *line = output; line < output + end;) {
        const char *nl = memchr(line, '\n', (size_t)(output + end - line));
        size_t len = nl ? (size_t)(nl - line) : (size_t)(output + end - line);
        fprintf(f, "%*s%s%s%s %s%.*s%s\n", indent, "", s.dim, USE_COLOR ? "┆" : "|", s.reset, color, (int)len, line,
                s.reset);
        if (!nl) break;
        line = nl + 1;
    }
}

// The "-- expected / ++ got" pair used whenever a `fails` message didn't match.
static void print_mismatch(FILE *f, const test_result_t *r, int indent, style_t s) {
    fprintf(f, "%*s%s-%s %sexpected the failure to contain%s\n", indent, "", s.green, s.reset, s.dim, s.reset);
    fprintf(f, "%*s  %s%s%s\n", indent, "", s.green, r->expected_msg ? r->expected_msg : "", s.reset);
    fprintf(f, "%*s%s+%s %sbut it actually said%s\n", indent, "", s.red, s.reset, s.dim, s.reset);
    print_output(f, r->output, indent + 2, s, s.red);
}

static const char *outcome_headline(const test_result_t *r) {
    switch (r->outcome) {
    case TEST_RESULT_UNEXPECTED_FAILURE: return "this test was expected to pass, but it failed";
    case TEST_RESULT_UNEXPECTED_SUCCESS: return "this test was expected to fail, but it passed";
    case TEST_RESULT_WRONG_MESSAGE: return "this test failed with the wrong message";
    case TEST_RESULT_TIMEOUT: return "this test never finished";
    default: return "";
    }
}

// Laid out like every other Tomo diagnostic: what went wrong, the source it went
// wrong in, then the message, the same order `parser_err` and `compiler_err`
// use, with the test's name standing in for the error-kind badge.
static void print_failure(FILE *f, const test_result_t *r, int indent, style_t s) {
    int sub = indent + 2;
    fprintf(f, "%*s%s%s%s %s%s%s\n", indent, "", s.red, s.fail_mark, s.reset, s.bold, r->label, s.reset);
    print_excerpt(f, r, sub);
    fprintf(f, "%*s%s%s%s%s\n", sub, "", s.red, s.bold, outcome_headline(r), s.reset);

    switch (r->outcome) {
    case TEST_RESULT_UNEXPECTED_FAILURE: print_output(f, r->output, sub, s, ""); break;
    case TEST_RESULT_UNEXPECTED_SUCCESS:
        if (r->expected_msg)
            fprintf(f, "%*s%sit should have failed with a message containing%s\n", sub, "", s.dim, s.reset);
        if (r->expected_msg) fprintf(f, "%*s  %s%s%s\n", sub, "", s.green, r->expected_msg, s.reset);
        break;
    case TEST_RESULT_WRONG_MESSAGE: print_mismatch(f, r, sub, s); break;
    case TEST_RESULT_TIMEOUT:
        fprintf(f, "%*s%sit hit the time limit and was killed; raise it with TOMO_TEST_TIMEOUT=<seconds>%s\n", sub, "",
                s.dim, s.reset);
        print_output(f, r->output, sub, s, "");
        break;
    case TEST_RESULT_PASS:
    default: break;
    }
    fprintf(f, "\n");
}

// ---- per-file roll-up -----------------------------------------------------

// Results are tagged with a per-file label; compare by text so the grouping
// doesn't depend on the caller interning those strings.
static bool same_file(const char *a, const char *b) {
    if (a == b) return true;
    return a && b && strcmp(a, b) == 0;
}

// Results arrive grouped by file (the driver appends a file at a time), so a
// single pass over consecutive runs gives the per-file tallies.
// One file's tallies, plus where the next group starts.
typedef struct {
    const char *file;
    int64_t passed, failed;
    double seconds;
    int64_t next;
} file_group_t;

static file_group_t file_group_at(test_result_t *results, int64_t n, int64_t i) {
    file_group_t g = {.file = results[i].file, .next = i};
    for (; g.next < n && same_file(results[g.next].file, g.file); g.next++) {
        if (results[g.next].outcome == TEST_RESULT_PASS) g.passed += 1;
        else g.failed += 1;
        g.seconds += results[g.next].seconds;
    }
    return g;
}

static int digits_of(int64_t v) {
    int d = 1;
    for (; v >= 10; v /= 10)
        d += 1;
    return d;
}

// The counts and the elapsed time are laid out as fixed-width columns, sized in
// a first pass over the groups. Letting the dot leader absorb their variation
// instead would slide "N passed" left and right from row to row, because both
// the digit count and the time ("9ms" vs "513ms" vs "1.20s") vary.
static void print_file_lines(FILE *f, test_result_t *results, int64_t n, int indent, style_t s) {
    int pass_digits = 1, fail_digits = 1, time_w = 0;
    bool any_failed = false;
    for (int64_t i = 0; i < n;) {
        file_group_t g = file_group_at(results, n, i);
        if (digits_of(g.passed) > pass_digits) pass_digits = digits_of(g.passed);
        if (g.failed > 0) {
            any_failed = true;
            if (digits_of(g.failed) > fail_digits) fail_digits = digits_of(g.failed);
        }
        char time_str[32];
        report_duration(time_str, sizeof(time_str), g.seconds);
        if ((int)strlen(time_str) > time_w) time_w = (int)strlen(time_str);
        i = g.next;
    }

    int pass_w = pass_digits + (int)strlen(" passed");
    int fail_w = fail_digits + (int)strlen(" failed");
    int counts_w = pass_w + (any_failed ? 2 + fail_w : 0);

    for (int64_t i = 0; i < n;) {
        file_group_t g = file_group_at(results, n, i);
        char time_str[32];
        report_duration(time_str, sizeof(time_str), g.seconds);

        const char *name = g.file ? g.file : "(unknown file)";
        report_leader(f, indent, g.failed == 0, name, counts_w + 2 + time_w);

        // The passed column is always the same width, so it lines up whether or not this file had failures:
        fprintf(f, "%s%*lld passed%s", g.failed == 0 ? s.green : s.dim, pass_digits, (long long)g.passed, s.reset);
        int used = pass_w;
        if (g.failed > 0) {
            fprintf(f, "  %s%*lld failed%s", s.red, fail_digits, (long long)g.failed, s.reset);
            used += 2 + fail_w;
        }
        report_repeat(f, " ", counts_w - used);
        fprintf(f, "  %s%*s%s\n", s.dim, time_w, time_str, s.reset);
        i = g.next;
    }
}

// ---- the whole report -----------------------------------------------------

void tomo_test_render(test_result_t *results, int64_t n, bool verbose) {
    // Results arrive grouped by file, but within a file the compile-failure tests are checked before the runtime
    // ones. Put each file's tests back in source order so the report reads top-to-bottom like the file does:
    for (int64_t i = 0; i < n;) {
        int64_t j = i;
        while (j < n && same_file(results[j].file, results[i].file))
            j += 1;
        for (int64_t a = i + 1; a < j; a++) // insertion sort: stable, and these runs are short
            for (int64_t b = a; b > i && results[b].first_line < results[b - 1].first_line; b--) {
                test_result_t tmp = results[b];
                results[b] = results[b - 1];
                results[b - 1] = tmp;
            }
        i = j;
    }

    style_t s = report_style();
    int indent = USE_COLOR ? 2 : 0;
    FILE *f = stdout;

    int64_t passed = 0, failed = 0, files = 0;
    double seconds = 0.0;
    for (int64_t i = 0; i < n; i++) {
        if (results[i].outcome == TEST_RESULT_PASS) passed += 1;
        else failed += 1;
        seconds += results[i].seconds;
        if (i == 0 || !same_file(results[i].file, results[i - 1].file)) files += 1;
    }

    // Verbose: the full roster, so you can see what actually ran (and what each
    // test printed). Failures are only marked here, and their details come
    // below, right above the summary, so you never scroll back up through the
    // logs.
    if (verbose) {
        const char *cur = NULL;
        for (int64_t i = 0; i < n; i++) {
            test_result_t *r = &results[i];
            if (i == 0 || !same_file(r->file, cur)) {
                cur = r->file;
                fprintf(f, "%s%*s%s%s%s\n", i > 0 ? "\n" : "", indent, "", s.hdr, cur ? cur : "(unknown file)",
                        s.reset);
            }
            bool ok = r->outcome == TEST_RESULT_PASS;
            // Most tests finish in well under a millisecond; a column of "0ms" is just noise, so only the ones
            // slow enough to be worth noticing get a time:
            char time_str[32] = "";
            if (r->seconds >= 0.001) {
                char d[24];
                report_duration(d, sizeof(d), r->seconds);
                snprintf(time_str, sizeof(time_str), " %s%s%s", s.dim, d, s.reset);
            }
            fprintf(f, "%*s%s%s%s %s%s%s%s\n", indent + 2, "", ok ? s.green : s.red, ok ? s.pass_mark : s.fail_mark,
                    s.reset, ok ? s.dim : s.bold, r->label, s.reset, time_str);
            if (ok) print_output(f, r->output, indent + 4, s, s.dim);
        }
        fprintf(f, "\n");
    }

    if (failed > 0) {
        // A banner, so the eye lands on the failures even in a wall of build output:
        if (USE_COLOR)
            fprintf(f, "\n\x1b[91;7;1m %lld test%s failed \x1b[m\n\n", (long long)failed, failed == 1 ? "" : "s");
        else fprintf(f, "\n%lld test%s failed\n\n", (long long)failed, failed == 1 ? "" : "s");
        for (int64_t i = 0; i < n; i++)
            if (results[i].outcome != TEST_RESULT_PASS) print_failure(f, &results[i], indent, s);
    }

    if (n > 0 && (verbose || failed > 0 || files > 1)) {
        print_file_lines(f, results, n, indent, s);
        fprintf(f, "\n");
    }

    if (n == 0) {
        // Not a pass and not a failure, so say so plainly rather than reporting a triumphant "0 passed":
        fprintf(f, "%*s%sno tests found%s\n", indent, "", s.dim, s.reset);
        fflush(f);
        return;
    }

    char time_str[32];
    report_duration(time_str, sizeof(time_str), seconds);
    if (failed == 0) {
        // The payoff line. Everything above it is green, and so is this.
        fprintf(f, "%*s%s%s%s %lld passed%s %s%s %lld %s %s %s%s\n", indent, "", s.green, s.bold,
                USE_COLOR ? s.pass_mark : "PASS:", (long long)passed, s.reset, s.dim, s.sep, (long long)files,
                files == 1 ? "file" : "files", s.sep, time_str, s.reset);
    } else {
        fprintf(f, "%*s%s%s%s %lld failed%s %s%s %lld passed %s %s%s\n", indent, "", s.red, s.bold,
                USE_COLOR ? s.fail_mark : "FAIL:", (long long)failed, s.reset, s.dim, s.sep, (long long)passed, s.sep,
                time_str, s.reset);
        // The single most useful next command, defined so it can be copied:
        for (int64_t i = 0; i < n; i++) {
            if (results[i].outcome == TEST_RESULT_PASS) continue;
            if (!results[i].file) break;
            fprintf(f, "%*s%srerun just this one:%s tomo test %s --filter \"%s\"%s\n", indent, "", s.dim, s.reset,
                    results[i].file, results[i].label, s.reset);
            break;
        }
    }
    fflush(f);
}

// ---- runner ---------------------------------------------------------------

int _tomo_run_tests(tomo_test_t *tests, int64_t n) {
    const char *fd_env = getenv("TOMO_TEST_RESULT_FD");
    int result_fd = fd_env ? atoi(fd_env) : -1;
    bool verbose = getenv("TOMO_TEST_VERBOSE") != NULL;
    const char *timeout_env = getenv("TOMO_TEST_TIMEOUT");
    unsigned int timeout = timeout_env ? (unsigned int)atoi(timeout_env) : DEFAULT_TIMEOUT_SECS;
    const char *filter = getenv("TOMO_TEST_FILTER"); // only run tests whose label contains this substring

    test_result_t *results = result_fd < 0 ? calloc((size_t)n, sizeof(test_result_t)) : NULL;
    int64_t failures = 0, ran = 0;

    for (int64_t i = 0; i < n; i++) {
        if (filter && !strstr(tests[i].label, filter)) continue;
        int pipes[2];
        if (pipe(pipes) != 0) {
            perror("pipe");
            return 2;
        }
        fflush(NULL);
        struct timespec t0;
        clock_gettime(CLOCK_MONOTONIC, &t0);
        pid_t pid = fork();
        if (pid == 0) { // ---- child ----
            dup2(pipes[1], STDOUT_FILENO);
            dup2(pipes[1], STDERR_FILENO);
            close(pipes[0]);
            close(pipes[1]);
            if (result_fd >= 0) close(result_fd);
            // Keep the compact failure message rather than the full runtime-error display: the report draws its
            // own source excerpt, and a test's stacktrace is mostly test-runner scaffolding. This also keeps the
            // echoed source (which contains the test's own text) from producing false substring matches below.
            setenv("TOMO_PLAIN_ERRORS", "1", 1);
            // Only `fails "..."` matches its expected message against this output, and ANSI codes in the middle
            // of the message would break that match. Every other test's output is shown to the user verbatim, so
            // leave its syntax highlighting (colorized `>>` values and types) intact.
            if (tests[i].expect_failure) USE_COLOR = false;
            alarm(timeout);
            tests[i].fn();
            fflush(NULL);
            _exit(0); // reached only if the body ran without failing
        }
        // ---- parent ----
        close(pipes[1]);
        char *output = drain_fd(pipes[0]); // drain before waitpid to avoid deadlock
        close(pipes[0]);
        int status = 0;
        waitpid(pid, &status, 0);
        struct timespec t1;
        clock_gettime(CLOCK_MONOTONIC, &t1);

        // Peel the span trailer off before classifying, so `fails "..."` matching never sees it:
        const char *fail_file = NULL;
        int fail_start = 0, fail_end = 0;
        tomo_test_take_span(output, &fail_file, &fail_start, &fail_end);

        test_result_t r = {
            .outcome = classify(status, output, &tests[i]),
            .fail_file = fail_file,
            .fail_start = fail_start,
            .fail_end = fail_end,
            .seconds = (double)(t1.tv_sec - t0.tv_sec) + 1e-9 * (double)(t1.tv_nsec - t0.tv_nsec),
            .label = tests[i].label,
            .expected_msg = tests[i].expected_msg,
            .expect_failure = tests[i].expect_failure,
            .output = output,
            .first_line = tests[i].first_line,
            .last_line = tests[i].last_line,
        };
        if (r.outcome != TEST_RESULT_PASS) failures += 1;
        if (result_fd >= 0) tomo_test_write_record(result_fd, &r);
        else results[ran] = r;
        ran += 1;
    }

    if (result_fd < 0) tomo_test_render(results, ran, verbose);
    return failures > 0 ? 1 : 0;
}