Major overhaul of some of the memory tracking code, as well as some

cleanup of the compilation code.

9 files changed, 194 insertions, 157 deletions

diff --git a/bp.c b/bp.c
index 1db9bf4..309e187 100644
--- a/bp.c
+++ b/bp.c
@@ -120,6 +120,7 @@ static int process_file(def_t *defs, const char *filename, vm_op_t *pattern, uns
 
     if (m != NULL)
         destroy_match(&m);
+
     destroy_file(&f);
 
     return success;
@@ -286,6 +287,8 @@ int main(int argc, char *argv[])
     }
     if (flags & BP_JSON) printf("]\n");
 
+    free_defs(&defs, NULL);
+
     return (found > 0) ? 0 : 1;
 }
 
diff --git a/compiler.c b/compiler.c
index aa82942..87b92c6 100644
--- a/compiler.c
+++ b/compiler.c
@@ -16,16 +16,34 @@ __attribute__((nonnull))
 static vm_op_t *expand_chain(file_t *f, vm_op_t *first);
 __attribute__((nonnull))
 static vm_op_t *expand_choices(file_t *f, vm_op_t *first);
-static vm_op_t *chain_together(vm_op_t *first, vm_op_t *second);
-__attribute__((nonnull(1,4)))
-static void set_range(vm_op_t *op, ssize_t min, ssize_t max, vm_op_t *pat, vm_op_t *sep);
+__attribute__((nonnull))
+static vm_op_t *_bp_simplepattern(file_t *f, const char *str);
+__attribute__((nonnull(1)))
+static vm_op_t *chain_together(file_t *f,vm_op_t *first, vm_op_t *second);
+__attribute__((nonnull(1,2,3,6)))
+static vm_op_t *new_range(file_t *f, const char *start, const char *end, ssize_t min, ssize_t max, vm_op_t *pat, vm_op_t *sep);
+
+//
+// Allocate a new opcode for this file (ensuring it will be automatically freed
+// when the file is freed)
+//
+vm_op_t *new_op(file_t *f, const char *start, enum VMOpcode type)
+{
+    allocated_op_t *tracker = new(allocated_op_t);
+    tracker->next = f->ops;
+    f->ops = tracker;
+    tracker->op.type = type;
+    tracker->op.start = start;
+    tracker->op.len = -1;
+    return &tracker->op;
+}
 
 //
 // Helper function to initialize a range object.
 //
-static void set_range(vm_op_t *op, ssize_t min, ssize_t max, vm_op_t *pat, vm_op_t *sep)
+static vm_op_t *new_range(file_t *f, const char *start, const char *end, ssize_t min, ssize_t max, vm_op_t *pat, vm_op_t *sep)
 {
-    op->type = VM_REPEAT;
+    vm_op_t *op = new_op(f, start, VM_REPEAT);
     if (pat->len >= 0 && (sep == NULL || sep->len >= 0) && min == max && min >= 0)
         op->len = pat->len * min + (sep == NULL || min == 0 ? 0 : sep->len * (min-1));
     else
@@ -40,6 +58,8 @@ static void set_range(vm_op_t *op, ssize_t min, ssize_t max, vm_op_t *pat, vm_op
         if (sep->start < op->start) op->start = sep->start;
         if (sep->end > op->end) op->end = sep->end;
     }
+    op->end = end;
+    return op;
 }
 
 //
@@ -54,7 +74,7 @@ static vm_op_t *expand_chain(file_t *f, vm_op_t *first)
     if (second->end <= first->end)
         file_err(f, second->end, second->end,
                  "This chain is not parsing properly");
-    return chain_together(first, second);
+    return chain_together(f, first, second);
 }
 
 //
@@ -89,13 +109,11 @@ static vm_op_t *expand_choices(file_t *f, vm_op_t *first)
         memcpy((void*)replacement, repstr, replace_len);
         
         vm_op_t *pat = first;
-        first = new(vm_op_t);
-        first->type = VM_REPLACE;
+        first = new_op(f, pat->start, VM_REPLACE);
         first->args.replace.pat = pat;
         first->args.replace.text = replacement;
         first->args.replace.len = replace_len;
         first->len = pat->len;
-        first->start = pat->start;
         first->end = str;
     }
 
@@ -104,9 +122,7 @@ static vm_op_t *expand_choices(file_t *f, vm_op_t *first)
     if (!second)
         file_err(f, str, str, "There should be a pattern here after a '/'");
     second = expand_choices(f, second);
-    vm_op_t *choice = new(vm_op_t);
-    choice->type = VM_OTHERWISE;
-    choice->start = first->start;
+    vm_op_t *choice = new_op(f, first->start, VM_OTHERWISE);
     if (first->len == second->len)
         choice->len = first->len;
     else choice->len = -1;
@@ -120,13 +136,12 @@ static vm_op_t *expand_choices(file_t *f, vm_op_t *first)
 // Given two patterns, return a new opcode for the first pattern followed by
 // the second. If either pattern is NULL, return the other.
 //
-static vm_op_t *chain_together(vm_op_t *first, vm_op_t *second)
+static vm_op_t *chain_together(file_t *f, vm_op_t *first, vm_op_t *second)
 {
     if (first == NULL) return second;
     if (second == NULL) return first;
     check(first->type != VM_CHAIN, "A chain should not be the first item in a chain.\n");
-    vm_op_t *chain = new(vm_op_t);
-    chain->type = VM_CHAIN;
+    vm_op_t *chain = new_op(f, first->start, VM_CHAIN);
     chain->start = first->start;
     if (first->len >= 0 && second->len >= 0)
         chain->len = first->len + second->len;
@@ -138,26 +153,58 @@ static vm_op_t *chain_together(vm_op_t *first, vm_op_t *second)
 }
 
 //
-// Compile a string of BP code into virtual machine opcodes
+// Wrapper for _bp_simplepattern() that expands any postfix operators
 //
 vm_op_t *bp_simplepattern(file_t *f, const char *str)
 {
+    vm_op_t *op = _bp_simplepattern(f, str);
+    if (op == NULL) return op;
+
+    check(op->end != NULL, "op->end is uninitialized!\n");
+
+    // Expand postfix operators (if any)
+    str = after_spaces(op->end);
+    while (str+2 < f->end && (matchstr(&str, "!=") || matchstr(&str, "=="))) { // Equality <pat1>==<pat2> and inequality <pat1>!=<pat2>
+        int equal = str[-2] == '=';
+        vm_op_t *first = op;
+        vm_op_t *second = bp_simplepattern(f, str);
+        if (!second)
+            file_err(f, str, str, "The '%c=' operator expects a pattern before and after.", equal?'=':'!');
+        if (equal) {
+            if (!(first->len == -1 || second->len == -1 || first->len == second->len))
+                file_err(f, op->start, second->end,
+                  "These two patterns cannot possibly give the same result (different lengths: %ld != %ld)",
+                  first->len, second->len);
+        }
+        op = new_op(f, str, equal ? VM_EQUAL : VM_NOT_EQUAL);
+        op->end = second->end;
+        op->len = first->len != -1 ? first->len : second->len;
+        op->args.multiple.first = first;
+        op->args.multiple.second = second;
+        str = op->end;
+        str = after_spaces(str);
+    }
+
+    return op;
+}
+
+//
+// Compile a string of BP code into virtual machine opcodes
+//
+static vm_op_t *_bp_simplepattern(file_t *f, const char *str)
+{
     str = after_spaces(str);
     if (!*str) return NULL;
-    vm_op_t *op = new(vm_op_t);
-    op->start = str;
-    op->len = -1;
+    const char *start = str;
     char c = *str;
-    const char *origin = str;
     ++str;
     switch (c) {
         // Any char (dot)
         case '.': {
             if (*str == '.') { // ".."
+                vm_op_t *op = new_op(f, start, VM_UPTO_AND);
                 ++str;
                 vm_op_t *till = bp_simplepattern(f, str);
-                op->type = VM_UPTO_AND;
-                op->len = -1;
                 op->args.multiple.first = till;
                 if (till)
                     str = till->end;
@@ -168,11 +215,13 @@ vm_op_t *bp_simplepattern(file_t *f, const char *str)
                     op->args.multiple.second = skip;
                     str = skip->end;
                 }
-                break;
+                op->end = str;
+                return op;
             } else {
-                op->type = VM_ANYCHAR;
+                vm_op_t *op = new_op(f, start, VM_ANYCHAR);
                 op->len = 1;
-                break;
+                op->end = str;
+                return op;
             }
         }
         // Char literals
@@ -182,18 +231,15 @@ vm_op_t *bp_simplepattern(file_t *f, const char *str)
                 char c = *str;
                 if (!c || c == '\n')
                     file_err(f, str, str, "There should be a character here after the '`'");
+                const char *opstart = str-1;
 
-                if (op == NULL)
-                    op = new(vm_op_t);
-
-                op->start = str-1;
-                op->len = 1;
+                vm_op_t *op;
                 ++str;
                 if (matchchar(&str, '-')) { // Range
                     char c2 = *str;
                     if (!c2 || c2 == '\n')
                         file_err(f, str, str, "There should be a character here to complete the character range.");
-                    op->type = VM_RANGE;
+                    op = new_op(f, opstart, VM_RANGE);
                     if (c < c2) {
                         op->args.range.low = (unsigned char)c;
                         op->args.range.high = (unsigned char)c2;
@@ -203,20 +249,19 @@ vm_op_t *bp_simplepattern(file_t *f, const char *str)
                     }
                     ++str;
                 } else {
-                    op->type = VM_STRING;
+                    op = new_op(f, opstart, VM_STRING);
                     char *s = xcalloc(sizeof(char), 2);
                     s[0] = c;
                     op->args.s = s;
                 }
 
+                op->len = 1;
                 op->end = str;
 
                 if (all == NULL) {
                     all = op;
                 } else {
-                    vm_op_t *either = new(vm_op_t);
-                    either->type = VM_OTHERWISE;
-                    either->start = all->start;
+                    vm_op_t *either = new_op(f, all->start, VM_OTHERWISE);
                     either->end = op->end;
                     either->args.multiple.first = all;
                     either->args.multiple.second = op;
@@ -226,8 +271,7 @@ vm_op_t *bp_simplepattern(file_t *f, const char *str)
                 op = NULL;
             } while (matchchar(&str, ','));
 
-            op = all;
-            break;
+            return all;
         }
         // Escapes
         case '\\': {
@@ -235,11 +279,13 @@ vm_op_t *bp_simplepattern(file_t *f, const char *str)
                 file_err(f, str, str, "There should be an escape sequence here after this backslash.");
 
             if (matchchar(&str, 'N')) { // \N (nodent)
-                op->type = VM_NODENT;
-                break;
+                vm_op_t *op = new_op(f, start, VM_NODENT);
+                op->end = str;
+                return op;
             }
 
-            op->len = 1;
+            vm_op_t *op;
+            const char *opstart = str;
             unsigned char e = unescapechar(str, &str);
             if (*str == '-') { // Escape range (e.g. \x00-\xFF)
                 ++str;
@@ -248,22 +294,24 @@ vm_op_t *bp_simplepattern(file_t *f, const char *str)
                 if (str == seqstart)
                     file_err(f, seqstart, str+1, "This value isn't a valid escape sequence");
                 if (e2 < e)
-                    file_err(f, origin, str, "Escape ranges should be low-to-high, but this is high-to-low.");
-                op->type = VM_RANGE;
+                    file_err(f, start, str, "Escape ranges should be low-to-high, but this is high-to-low.");
+                op = new_op(f, opstart, VM_RANGE);
                 op->args.range.low = e;
                 op->args.range.high = e2;
             } else {
-                op->type = VM_STRING;
+                op = new_op(f, opstart, VM_STRING);
                 char *s = xcalloc(sizeof(char), 2);
                 s[0] = (char)e;
                 op->args.s = s;
             }
-            break;
+            op->len = 1;
+            op->end = str;
+            return op;
         }
         // String literal
         case '"': case '\'': case '\002': {
             char endquote = c == '\002' ? '\003' : c;
-            char *start = (char*)str;
+            char *litstart = (char*)str;
             for (; *str && *str != endquote; str++) {
                 if (*str == '\\') {
                     if (!str[1] || str[1] == '\n')
@@ -272,30 +320,32 @@ vm_op_t *bp_simplepattern(file_t *f, const char *str)
                     ++str;
                 }
             }
-            size_t len = (size_t)(str - start);
+            size_t len = (size_t)(str - litstart);
             char *literal = xcalloc(sizeof(char), len+1);
-            memcpy(literal, start, len);
+            memcpy(literal, litstart, len);
             // Note: an unescaped string is guaranteed to be no longer than the
             // escaped string, so this is safe to do inplace.
             len = unescape_string(literal, literal, len);
 
-            op->type = VM_STRING;
+            vm_op_t *op = new_op(f, start, VM_STRING);
             op->len = (ssize_t)len;
             op->args.s = literal;
 
             if (!matchchar(&str, endquote))
-                file_err(f, origin, str, "This string doesn't have a closing quote.");
-            break;
+                file_err(f, start, str, "This string doesn't have a closing quote.");
+
+            op->end = str;
+            return op;
         }
         // Not <pat>
         case '!': {
             vm_op_t *p = bp_simplepattern(f, str);
             if (!p) file_err(f, str, str, "There should be a pattern after this '!'");
-            str = p->end;
-            op->type = VM_NOT;
+            vm_op_t *op = new_op(f, start, VM_NOT);
             op->len = 0;
             op->args.pat = p;
-            break;
+            op->end = p->end;
+            return op;
         }
         // Number of repetitions: <N>(-<N> / - / + / "")
         case '0': case '1': case '2': case '3': case '4': case '5':
@@ -305,9 +355,9 @@ vm_op_t *bp_simplepattern(file_t *f, const char *str)
             long n1 = strtol(str, (char**)&str, 10);
             if (matchchar(&str, '-')) {
                 str = after_spaces(str);
-                const char *start = str;
+                const char *numstart = str;
                 long n2 = strtol(str, (char**)&str, 10);
-                if (str == start) min = 0, max = n1;
+                if (str == numstart) min = 0, max = n1;
                 else min = n1, max = n2;
             } else if (matchchar(&str, '+')) {
                 min = n1, max = -1;
@@ -327,8 +377,7 @@ vm_op_t *bp_simplepattern(file_t *f, const char *str)
             } else {
                 str = pat->end;
             }
-            set_range(op, min, max, pat, sep);
-            break;
+            return new_range(f, start, str, min, max, pat, sep);
         }
         // Lookbehind
         case '<': {
@@ -337,14 +386,15 @@ vm_op_t *bp_simplepattern(file_t *f, const char *str)
                 file_err(f, str, str, "There should be a pattern after this '<'");
             str = pat->end;
             if (pat->len == -1)
-                file_err(f, origin, pat->end,
+                file_err(f, start, pat->end,
                          "Sorry, variable-length lookbehind patterns like this are not supported.\n"
                          "Please use a fixed-length lookbehind pattern instead.");
             str = pat->end;
-            op->type = VM_AFTER;
+            vm_op_t *op = new_op(f, start, VM_AFTER);
             op->len = 0;
             op->args.pat = pat;
-            break;
+            op->end = str;
+            return op;
         }
         // Lookahead
         case '>': {
@@ -352,25 +402,25 @@ vm_op_t *bp_simplepattern(file_t *f, const char *str)
             if (!pat)
                 file_err(f, str, str, "There should be a pattern after this '>'");
             str = pat->end;
-            op->type = VM_BEFORE;
+            vm_op_t *op = new_op(f, start, VM_BEFORE);
             op->len = 0;
             op->args.pat = pat;
-            break;
+            op->end = str;
+            return op;
         }
         // Parentheses
         case '(': case '{': {
             char closing = c == '(' ? ')' : '}';
-            xfree(&op);
-            op = bp_simplepattern(f, str);
+            vm_op_t *op = bp_simplepattern(f, str);
             if (!op)
                 file_err(f, str, str, "There should be a valid pattern after this parenthesis.");
             op = expand_choices(f, op);
             str = op->end;
             if (!matchchar(&str, closing))
-                file_err(f, origin, str, "This parenthesis group isn't properly closed.");
-            op->start = origin;
+                file_err(f, start, str, "This parenthesis group isn't properly closed.");
+            op->start = start;
             op->end = str;
-            break;
+            return op;
         }
         // Square brackets
         case '[': {
@@ -380,9 +430,8 @@ vm_op_t *bp_simplepattern(file_t *f, const char *str)
             pat = expand_choices(f, pat);
             str = pat->end;
             if (!matchchar(&str, ']'))
-                file_err(f, origin, str, "This square bracket group isn't properly closed.");
-            set_range(op, 0, 1, pat, NULL);
-            break;
+                file_err(f, start, str, "This square bracket group isn't properly closed.");
+            return new_range(f, start, str, 0, 1, pat, NULL);
         }
         // Repeating
         case '*': case '+': {
@@ -398,12 +447,11 @@ vm_op_t *bp_simplepattern(file_t *f, const char *str)
                     file_err(f, str, str, "There should be a separator pattern after the '%%' here.");
                 str = sep->end;
             }
-            set_range(op, min, -1, pat, sep);
-            break;
+            return new_range(f, start, str, min, -1, pat, sep);
         }
         // Capture
         case '@': {
-            op->type = VM_CAPTURE;
+            vm_op_t *op = new_op(f, start, VM_CAPTURE);
             const char *a = *str == '!' ? &str[1] : after_name(str);
             if (a > str && after_spaces(a)[0] == '=' && after_spaces(a)[1] != '>') {
                 op->args.capture.name = strndup(str, (size_t)(a-str));
@@ -412,86 +460,53 @@ vm_op_t *bp_simplepattern(file_t *f, const char *str)
             vm_op_t *pat = bp_simplepattern(f, str);
             if (!pat)
                 file_err(f, str, str, "There should be a valid pattern here to capture after the '@'");
-            str = pat->end;
             op->args.capture.capture_pat = pat;
             op->len = pat->len;
-            break;
+            op->end = pat->end;
+            return op;
         }
         // Hide
         case '~': {
             vm_op_t *pat = bp_simplepattern(f, str);
             if (!pat)
                 file_err(f, str, str, "There should be a pattern after this '~'");
-            str = pat->end;
-            op->type = VM_HIDE;
+            vm_op_t *op = new_op(f, start, VM_HIDE);
             op->len = 0;
             op->args.pat = pat;
-            break;
+            op->end = pat->end;
+            return op;
         }
         // Special rules:
         case '_': case '^': case '$': case '|': {
+            const char *name = NULL;
             if (matchchar(&str, c)) { // double __, ^^, $$
+                if (matchchar(&str, ':')) return NULL; // Don't match definitions
                 char tmp[3] = {c, c, '\0'};
-                op->args.s = strdup(tmp);
+                name = strdup(tmp);
             } else {
-                op->args.s = strndup(&c, 1);
-            }
-            if (matchchar(&str, ':')) { // Don't match definitions
-                xfree(&op->args.s);
-                xfree(&op);
-                return NULL;
+                if (matchchar(&str, ':')) return NULL; // Don't match definitions
+                name = strndup(&c, 1);
             }
-            op->type = VM_REF;
-            break;
+            vm_op_t *op = new_op(f, start, VM_REF);
+            op->args.s = name;
+            op->end = str;
+            return op;
         }
         default: {
             // Reference
-            if (isalpha(c)) {
-                --str;
-                const char *refname = str;
-                str = after_name(str);
-                if (matchchar(&str, ':')) { // Don't match definitions
-                    xfree(&op);
-                    return NULL;
-                }
-                op->type = VM_REF;
-                op->args.s = strndup(refname, (size_t)(str - refname));
-                break;
-            } else {
-                xfree(&op);
+            if (!isalpha(c)) return NULL;
+            --str;
+            const char *refname = str;
+            str = after_name(str);
+            if (matchchar(&str, ':')) // Don't match definitions
                 return NULL;
-            }
-        }
-    }
-    op->end = str;
-
-    // Postfix operators:
-  postfix:
-    str = after_spaces(str);
-    if (str+2 < f->end && (matchstr(&str, "!=") || matchstr(&str, "=="))) { // Equality <pat1>==<pat2> and inequality <pat1>!=<pat2>
-        int equal = str[-2] == '=';
-        vm_op_t *first = op;
-        vm_op_t *second = bp_simplepattern(f, str);
-        if (!second)
-            file_err(f, str, str, "The '%c=' operator expects a pattern before and after.", equal?'=':'!');
-        if (equal) {
-            if (!(first->len == -1 || second->len == -1 || first->len == second->len))
-                file_err(f, origin, second->end,
-                  "These two patterns cannot possibly give the same result (different lengths: %ld != %ld)",
-                  first->len, second->len);
+            vm_op_t *op = new_op(f, start, VM_REF);
+            op->args.s = strndup(refname, (size_t)(str - refname));
+            op->end = str;
+            return op;
         }
-        op = new(vm_op_t);
-        op->type = equal ? VM_EQUAL : VM_NOT_EQUAL;
-        op->start = str;
-        op->end = second->end;
-        op->len = first->len != -1 ? first->len : second->len;
-        op->args.multiple.first = first;
-        op->args.multiple.second = second;
-        str = op->end;
-        goto postfix;
     }
-
-    return op;
+    return NULL;
 }
 
 //
@@ -501,10 +516,8 @@ vm_op_t *bp_stringpattern(file_t *f, const char *str)
 {
     vm_op_t *ret = NULL;
     while (*str) {
-        vm_op_t *strop = new(vm_op_t);
-        strop->start = str;
+        vm_op_t *strop = new_op(f, str, VM_STRING);
         strop->len = 0;
-        strop->type = VM_STRING;
         char *start = (char*)str;
         vm_op_t *interp = NULL;
         for (; *str; str++) {
@@ -513,8 +526,7 @@ vm_op_t *bp_stringpattern(file_t *f, const char *str)
                     file_err(f, str, str, "There should be an escape sequence or pattern here after this backslash.");
                 
                 if (matchchar(&str, 'N')) { // \N (nodent)
-                    interp = new(vm_op_t);
-                    interp->type = VM_NODENT;
+                    interp = new_op(f, str-2, VM_NODENT);
                     break;
                 }
 
@@ -548,10 +560,10 @@ vm_op_t *bp_stringpattern(file_t *f, const char *str)
         if (strop->len == 0) {
             xfree(&strop);
         } else {
-            ret = chain_together(ret, strop);
+            ret = chain_together(f, ret, strop);
         }
         if (interp) {
-            ret = chain_together(ret, interp);
+            ret = chain_together(f, ret, interp);
             str = interp->end;
             // allow terminating seq
             matchchar(&str, ';');
@@ -566,9 +578,7 @@ vm_op_t *bp_stringpattern(file_t *f, const char *str)
 //
 vm_op_t *bp_replacement(file_t *f, vm_op_t *pat, const char *replacement)
 {
-    vm_op_t *op = new(vm_op_t);
-    op->type = VM_REPLACE;
-    op->start = pat->start;
+    vm_op_t *op = new_op(f, pat->start, VM_REPLACE);
     op->end = pat->end;
     op->len = pat->len;
     op->args.replace.pat = pat;
diff --git a/compiler.h b/compiler.h
index 66af44f..3b61c17 100644
--- a/compiler.h
+++ b/compiler.h
@@ -7,6 +7,8 @@
 #include "file_loader.h"
 #include "types.h"
 
+__attribute__((nonnull))
+vm_op_t *new_op(file_t *f, const char *start, enum VMOpcode type);
 __attribute__((nonnull(1,2)))
 vm_op_t *bp_simplepattern(file_t *f, const char *str);
 __attribute__((nonnull(1,2)))
diff --git a/file_loader.c b/file_loader.c
index 749e985..27f871f 100644
--- a/file_loader.c
+++ b/file_loader.c
@@ -124,10 +124,12 @@ void destroy_file(file_t **f)
         xfree(&((*f)->filename));
         (*f)->filename = NULL;
     }
+
     if ((*f)->lines) {
         xfree(&((*f)->lines));
         (*f)->lines = NULL;
     }
+
     if ((*f)->contents) {
         if ((*f)->mmapped) {
             munmap((*f)->contents, (size_t)((*f)->end - (*f)->contents));
@@ -136,6 +138,14 @@ void destroy_file(file_t **f)
         }
         (*f)->contents = NULL;
     }
+
+    while ((*f)->ops) {
+        allocated_op_t *tofree = (*f)->ops;
+        (*f)->ops = tofree->next;
+        memset(tofree, 'A', sizeof(allocated_op_t)); // Sentinel
+        xfree(&tofree);
+    }
+
     xfree(f);
 }
 
diff --git a/file_loader.h b/file_loader.h
index 0094732..2a47d40 100644
--- a/file_loader.h
+++ b/file_loader.h
@@ -6,10 +6,13 @@
 
 #include <stdio.h>
 
+struct allocated_op_s; // declared in types.h
+
 typedef struct {
     const char *filename;
     char *contents, **lines, *end;
     size_t nlines;
+    struct allocated_op_s *ops;
     unsigned int mmapped:1;
 } file_t;
 
diff --git a/grammar.c b/grammar.c
index cdc8f51..08e7083 100644
--- a/grammar.c
+++ b/grammar.c
@@ -72,9 +72,7 @@ vm_op_t *lookup(def_t *defs, const char *name)
 //
 static def_t *with_backref(def_t *defs, file_t *f, const char *name, match_t *m)
 {
-    vm_op_t *op = new(vm_op_t);
-    op->type = VM_BACKREF;
-    op->start = m->start;
+    vm_op_t *op = new_op(f, m->start, VM_BACKREF);
     op->end = m->end;
     op->len = -1; // TODO: maybe calculate this? (nontrivial because of replacements)
     op->args.backref = m;
@@ -95,4 +93,17 @@ def_t *with_backrefs(def_t *defs, file_t *f, match_t *m)
     return defs;
 }
 
+//
+// Free all the given definitions up till (but not including) `stop`
+//
+void free_defs(def_t **defs, def_t *stop)
+{
+    while (*defs != stop && *defs != NULL) {
+        def_t *next = (*defs)->next;
+        (*defs)->next = NULL;
+        free(*defs);
+        (*defs) = next;
+    }
+}
+
 // vim: ts=4 sw=0 et cino=L2,l1,(0,W4,m1
diff --git a/grammar.h b/grammar.h
index a7eedc4..6822c24 100644
--- a/grammar.h
+++ b/grammar.h
@@ -15,6 +15,8 @@ __attribute__((nonnull(2)))
 def_t *load_grammar(def_t *defs, file_t *f);
 __attribute__((pure, nonnull(2)))
 vm_op_t *lookup(def_t *defs, const char *name);
+__attribute__((nonnull(1)))
+void free_defs(def_t **defs, def_t *stop);
 
 #endif
 // vim: ts=4 sw=0 et cino=L2,l1,(0,W4,m1
diff --git a/types.h b/types.h
index 52b17e5..ad7781b 100644
--- a/types.h
+++ b/types.h
@@ -98,5 +98,14 @@ typedef struct def_s {
     struct def_s *next;
 } def_t;
 
+//
+// Structure used for tracking allocated ops, which must be freed when the file
+// is freed.
+//
+typedef struct allocated_op_s {
+    struct allocated_op_s *next;
+    vm_op_t op;
+} allocated_op_t;
+
 #endif
 // vim: ts=4 sw=0 et cino=L2,l1,(0,W4,m1
diff --git a/vm.c b/vm.c
index a936f7a..0e5293f 100644
--- a/vm.c
+++ b/vm.c
@@ -324,20 +324,7 @@ static match_t *_match(def_t *defs, file_t *f, const char *str, vm_op_t *op, uns
             { // Push backrefs and run matching, then cleanup
                 def_t *defs2 = with_backrefs(defs, f, m1);
                 m2 = _match(defs2, f, m1->end, op->args.multiple.second, flags, rec);
-                while (defs2 != defs) {
-                    def_t *next = defs2->next;
-                    defs2->next = NULL;
-                    // Deliberate memory leak, if there is a match, then the op
-                    // will be stored on the match and can't be freed here.
-                    // There's currently no refcounting on ops but that should
-                    // be how to prevent a memory leak from this.
-                    // TODO: add refcounting to ops?
-                    if (m2 == NULL) {
-                        xfree(&defs2->op);
-                    }
-                    xfree(&defs2);
-                    defs2 = next;
-                }
+                free_defs(&defs2, defs);
             }
 
             if (m2 == NULL) {