Bruce/bp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Major overhaul of some of the memory tracking code, as well as some

Browse Source 
cleanup of the compilation code.
This commit is contained in:
Bruce Hill 2021-01-13 01:48:36 -08:00
parent 7e4b287d9b
commit 45d7aff475
9 changed files with 194 additions and 157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
bp.c
View File

@ -120,6 +120,7 @@ static int process_file(def_t *defs, const char *filename, vm_op_t *pattern, uns
if (m != NULL)
destroy_match(&m);
destroy_file(&f);
return success;
@ -286,6 +287,8 @@ int main(int argc, char *argv[])
}
if (flags & BP_JSON) printf("]\n");
free_defs(&defs, NULL);
return (found > 0) ? 0 : 1;
}

290
compiler.c
View File

@ -16,16 +16,34 @@ __attribute__((nonnull))
static vm_op_t *expand_chain(file_t *f, vm_op_t *first);
__attribute__((nonnull))
static vm_op_t *expand_choices(file_t *f, vm_op_t *first);
static vm_op_t *chain_together(vm_op_t *first, vm_op_t *second);
__attribute__((nonnull(1,4)))
static void set_range(vm_op_t *op, ssize_t min, ssize_t max, vm_op_t *pat, vm_op_t *sep);
__attribute__((nonnull))
static vm_op_t *_bp_simplepattern(file_t *f, const char *str);
__attribute__((nonnull(1)))
static vm_op_t *chain_together(file_t *f,vm_op_t *first, vm_op_t *second);
__attribute__((nonnull(1,2,3,6)))
static vm_op_t *new_range(file_t *f, const char *start, const char *end, ssize_t min, ssize_t max, vm_op_t *pat, vm_op_t *sep);
//
// Allocate a new opcode for this file (ensuring it will be automatically freed
// when the file is freed)
//
vm_op_t *new_op(file_t *f, const char *start, enum VMOpcode type)
{
allocated_op_t *tracker = new(allocated_op_t);
tracker->next = f->ops;
f->ops = tracker;
tracker->op.type = type;
tracker->op.start = start;
tracker->op.len = -1;
return &tracker->op;
}
//
// Helper function to initialize a range object.
//
static void set_range(vm_op_t *op, ssize_t min, ssize_t max, vm_op_t *pat, vm_op_t *sep)
static vm_op_t *new_range(file_t *f, const char *start, const char *end, ssize_t min, ssize_t max, vm_op_t *pat, vm_op_t *sep)
{
op->type = VM_REPEAT;
vm_op_t *op = new_op(f, start, VM_REPEAT);
if (pat->len >= 0 && (sep == NULL || sep->len >= 0) && min == max && min >= 0)
op->len = pat->len * min + (sep == NULL || min == 0 ? 0 : sep->len * (min-1));
else
@ -40,6 +58,8 @@ static void set_range(vm_op_t *op, ssize_t min, ssize_t max, vm_op_t *pat, vm_op
if (sep->start < op->start) op->start = sep->start;
if (sep->end > op->end) op->end = sep->end;
}
op->end = end;
return op;
}
//
@ -54,7 +74,7 @@ static vm_op_t *expand_chain(file_t *f, vm_op_t *first)
if (second->end <= first->end)
file_err(f, second->end, second->end,
"This chain is not parsing properly");
return chain_together(first, second);
return chain_together(f, first, second);
}
//
@ -89,13 +109,11 @@ static vm_op_t *expand_choices(file_t *f, vm_op_t *first)
memcpy((void*)replacement, repstr, replace_len);
vm_op_t *pat = first;
first = new(vm_op_t);
first->type = VM_REPLACE;
first = new_op(f, pat->start, VM_REPLACE);
first->args.replace.pat = pat;
first->args.replace.text = replacement;
first->args.replace.len = replace_len;
first->len = pat->len;
first->start = pat->start;
first->end = str;
}
@ -104,9 +122,7 @@ static vm_op_t *expand_choices(file_t *f, vm_op_t *first)
if (!second)
file_err(f, str, str, "There should be a pattern here after a '/'");
second = expand_choices(f, second);
vm_op_t *choice = new(vm_op_t);
choice->type = VM_OTHERWISE;
choice->start = first->start;
vm_op_t *choice = new_op(f, first->start, VM_OTHERWISE);
if (first->len == second->len)
choice->len = first->len;
else choice->len = -1;
@ -120,13 +136,12 @@ static vm_op_t *expand_choices(file_t *f, vm_op_t *first)
// Given two patterns, return a new opcode for the first pattern followed by
// the second. If either pattern is NULL, return the other.
//
static vm_op_t *chain_together(vm_op_t *first, vm_op_t *second)
static vm_op_t *chain_together(file_t *f, vm_op_t *first, vm_op_t *second)
{
if (first == NULL) return second;
if (second == NULL) return first;
check(first->type != VM_CHAIN, "A chain should not be the first item in a chain.\n");
vm_op_t *chain = new(vm_op_t);
chain->type = VM_CHAIN;
vm_op_t *chain = new_op(f, first->start, VM_CHAIN);
chain->start = first->start;
if (first->len >= 0 && second->len >= 0)
chain->len = first->len + second->len;
@ -138,26 +153,58 @@ static vm_op_t *chain_together(vm_op_t *first, vm_op_t *second)
}
//
// Compile a string of BP code into virtual machine opcodes
// Wrapper for _bp_simplepattern() that expands any postfix operators
//
vm_op_t *bp_simplepattern(file_t *f, const char *str)
{
vm_op_t *op = _bp_simplepattern(f, str);
if (op == NULL) return op;
check(op->end != NULL, "op->end is uninitialized!\n");
// Expand postfix operators (if any)
str = after_spaces(op->end);
while (str+2 < f->end && (matchstr(&str, "!=") || matchstr(&str, "=="))) { // Equality <pat1>==<pat2> and inequality <pat1>!=<pat2>
int equal = str[-2] == '=';
vm_op_t *first = op;
vm_op_t *second = bp_simplepattern(f, str);
if (!second)
file_err(f, str, str, "The '%c=' operator expects a pattern before and after.", equal?'=':'!');
if (equal) {
if (!(first->len == -1 || second->len == -1 || first->len == second->len))
file_err(f, op->start, second->end,
"These two patterns cannot possibly give the same result (different lengths: %ld != %ld)",
first->len, second->len);
}
op = new_op(f, str, equal ? VM_EQUAL : VM_NOT_EQUAL);
op->end = second->end;
op->len = first->len != -1 ? first->len : second->len;
op->args.multiple.first = first;
op->args.multiple.second = second;
str = op->end;
str = after_spaces(str);
}
return op;
}
//
// Compile a string of BP code into virtual machine opcodes
//
static vm_op_t *_bp_simplepattern(file_t *f, const char *str)
{
str = after_spaces(str);
if (!*str) return NULL;
vm_op_t *op = new(vm_op_t);
op->start = str;
op->len = -1;
const char *start = str;
char c = *str;
const char *origin = str;
++str;
switch (c) {
// Any char (dot)
case '.': {
if (*str == '.') { // ".."
vm_op_t *op = new_op(f, start, VM_UPTO_AND);
++str;
vm_op_t *till = bp_simplepattern(f, str);
op->type = VM_UPTO_AND;
op->len = -1;
op->args.multiple.first = till;
if (till)
str = till->end;
@ -168,11 +215,13 @@ vm_op_t *bp_simplepattern(file_t *f, const char *str)
op->args.multiple.second = skip;
str = skip->end;
}
break;
op->end = str;
return op;
} else {
op->type = VM_ANYCHAR;
vm_op_t *op = new_op(f, start, VM_ANYCHAR);
op->len = 1;
break;
op->end = str;
return op;
}
}
// Char literals
@ -182,18 +231,15 @@ vm_op_t *bp_simplepattern(file_t *f, const char *str)
char c = *str;
if (!c || c == '\n')
file_err(f, str, str, "There should be a character here after the '`'");
const char *opstart = str-1;
if (op == NULL)
op = new(vm_op_t);
op->start = str-1;
op->len = 1;
vm_op_t *op;
++str;
if (matchchar(&str, '-')) { // Range
char c2 = *str;
if (!c2 || c2 == '\n')
file_err(f, str, str, "There should be a character here to complete the character range.");
op->type = VM_RANGE;
op = new_op(f, opstart, VM_RANGE);
if (c < c2) {
op->args.range.low = (unsigned char)c;
op->args.range.high = (unsigned char)c2;
@ -203,20 +249,19 @@ vm_op_t *bp_simplepattern(file_t *f, const char *str)
}
++str;
} else {
op->type = VM_STRING;
op = new_op(f, opstart, VM_STRING);
char *s = xcalloc(sizeof(char), 2);
s[0] = c;
op->args.s = s;
}
op->len = 1;
op->end = str;
if (all == NULL) {
all = op;
} else {
vm_op_t *either = new(vm_op_t);
either->type = VM_OTHERWISE;
either->start = all->start;
vm_op_t *either = new_op(f, all->start, VM_OTHERWISE);
either->end = op->end;
either->args.multiple.first = all;
either->args.multiple.second = op;
@ -226,8 +271,7 @@ vm_op_t *bp_simplepattern(file_t *f, const char *str)
op = NULL;
} while (matchchar(&str, ','));
op = all;
break;
return all;
}
// Escapes
case '\\': {
@ -235,11 +279,13 @@ vm_op_t *bp_simplepattern(file_t *f, const char *str)
file_err(f, str, str, "There should be an escape sequence here after this backslash.");
if (matchchar(&str, 'N')) { // \N (nodent)
op->type = VM_NODENT;
break;
vm_op_t *op = new_op(f, start, VM_NODENT);
op->end = str;
return op;
}
op->len = 1;
vm_op_t *op;
const char *opstart = str;
unsigned char e = unescapechar(str, &str);
if (*str == '-') { // Escape range (e.g. \x00-\xFF)
++str;
@ -248,22 +294,24 @@ vm_op_t *bp_simplepattern(file_t *f, const char *str)
if (str == seqstart)
file_err(f, seqstart, str+1, "This value isn't a valid escape sequence");
if (e2 < e)
file_err(f, origin, str, "Escape ranges should be low-to-high, but this is high-to-low.");
op->type = VM_RANGE;
file_err(f, start, str, "Escape ranges should be low-to-high, but this is high-to-low.");
op = new_op(f, opstart, VM_RANGE);
op->args.range.low = e;
op->args.range.high = e2;
} else {
op->type = VM_STRING;
op = new_op(f, opstart, VM_STRING);
char *s = xcalloc(sizeof(char), 2);
s[0] = (char)e;
op->args.s = s;
}
break;
op->len = 1;
op->end = str;
return op;
}
// String literal
case '"': case '\'': case '\002': {
char endquote = c == '\002' ? '\003' : c;
char *start = (char*)str;
char *litstart = (char*)str;
for (; *str && *str != endquote; str++) {
if (*str == '\\') {
if (!str[1] || str[1] == '\n')
@ -272,30 +320,32 @@ vm_op_t *bp_simplepattern(file_t *f, const char *str)
++str;
}
}
size_t len = (size_t)(str - start);
size_t len = (size_t)(str - litstart);
char *literal = xcalloc(sizeof(char), len+1);
memcpy(literal, start, len);
memcpy(literal, litstart, len);
// Note: an unescaped string is guaranteed to be no longer than the
// escaped string, so this is safe to do inplace.
len = unescape_string(literal, literal, len);
op->type = VM_STRING;
vm_op_t *op = new_op(f, start, VM_STRING);
op->len = (ssize_t)len;
op->args.s = literal;
if (!matchchar(&str, endquote))
file_err(f, origin, str, "This string doesn't have a closing quote.");
break;
file_err(f, start, str, "This string doesn't have a closing quote.");
op->end = str;
return op;
}
// Not <pat>
case '!': {
vm_op_t *p = bp_simplepattern(f, str);
if (!p) file_err(f, str, str, "There should be a pattern after this '!'");
str = p->end;
op->type = VM_NOT;
vm_op_t *op = new_op(f, start, VM_NOT);
op->len = 0;
op->args.pat = p;
break;
op->end = p->end;
return op;
}
// Number of repetitions: <N>(-<N> / - / + / "")
case '0': case '1': case '2': case '3': case '4': case '5':
@ -305,9 +355,9 @@ vm_op_t *bp_simplepattern(file_t *f, const char *str)
long n1 = strtol(str, (char**)&str, 10);
if (matchchar(&str, '-')) {
str = after_spaces(str);
const char *start = str;
const char *numstart = str;
long n2 = strtol(str, (char**)&str, 10);
if (str == start) min = 0, max = n1;
if (str == numstart) min = 0, max = n1;
else min = n1, max = n2;
} else if (matchchar(&str, '+')) {
min = n1, max = -1;
@ -327,8 +377,7 @@ vm_op_t *bp_simplepattern(file_t *f, const char *str)
} else {
str = pat->end;
}
set_range(op, min, max, pat, sep);
break;
return new_range(f, start, str, min, max, pat, sep);
}
// Lookbehind
case '<': {
@ -337,14 +386,15 @@ vm_op_t *bp_simplepattern(file_t *f, const char *str)
file_err(f, str, str, "There should be a pattern after this '<'");
str = pat->end;
if (pat->len == -1)
file_err(f, origin, pat->end,
file_err(f, start, pat->end,
"Sorry, variable-length lookbehind patterns like this are not supported.\n"
"Please use a fixed-length lookbehind pattern instead.");
str = pat->end;
op->type = VM_AFTER;
vm_op_t *op = new_op(f, start, VM_AFTER);
op->len = 0;
op->args.pat = pat;
break;
op->end = str;
return op;
}
// Lookahead
case '>': {
@ -352,25 +402,25 @@ vm_op_t *bp_simplepattern(file_t *f, const char *str)
if (!pat)
file_err(f, str, str, "There should be a pattern after this '>'");
str = pat->end;
op->type = VM_BEFORE;
vm_op_t *op = new_op(f, start, VM_BEFORE);
op->len = 0;
op->args.pat = pat;
break;
op->end = str;
return op;
}
// Parentheses
case '(': case '{': {
char closing = c == '(' ? ')' : '}';
xfree(&op);
op = bp_simplepattern(f, str);
vm_op_t *op = bp_simplepattern(f, str);
if (!op)
file_err(f, str, str, "There should be a valid pattern after this parenthesis.");
op = expand_choices(f, op);
str = op->end;
if (!matchchar(&str, closing))
file_err(f, origin, str, "This parenthesis group isn't properly closed.");
op->start = origin;
file_err(f, start, str, "This parenthesis group isn't properly closed.");
op->start = start;
op->end = str;
break;
return op;
}
// Square brackets
case '[': {
@ -380,9 +430,8 @@ vm_op_t *bp_simplepattern(file_t *f, const char *str)
pat = expand_choices(f, pat);
str = pat->end;
if (!matchchar(&str, ']'))
file_err(f, origin, str, "This square bracket group isn't properly closed.");
set_range(op, 0, 1, pat, NULL);
break;
file_err(f, start, str, "This square bracket group isn't properly closed.");
return new_range(f, start, str, 0, 1, pat, NULL);
}
// Repeating
case '*': case '+': {
@ -398,12 +447,11 @@ vm_op_t *bp_simplepattern(file_t *f, const char *str)
file_err(f, str, str, "There should be a separator pattern after the '%%' here.");
str = sep->end;
}
set_range(op, min, -1, pat, sep);
break;
return new_range(f, start, str, min, -1, pat, sep);
}
// Capture
case '@': {
op->type = VM_CAPTURE;
vm_op_t *op = new_op(f, start, VM_CAPTURE);
const char *a = *str == '!' ? &str[1] : after_name(str);
if (a > str && after_spaces(a)[0] == '=' && after_spaces(a)[1] != '>') {
op->args.capture.name = strndup(str, (size_t)(a-str));
@ -412,86 +460,53 @@ vm_op_t *bp_simplepattern(file_t *f, const char *str)
vm_op_t *pat = bp_simplepattern(f, str);
if (!pat)
file_err(f, str, str, "There should be a valid pattern here to capture after the '@'");
str = pat->end;
op->args.capture.capture_pat = pat;
op->len = pat->len;
break;
op->end = pat->end;
return op;
}
// Hide
case '~': {
vm_op_t *pat = bp_simplepattern(f, str);
if (!pat)
file_err(f, str, str, "There should be a pattern after this '~'");
str = pat->end;
op->type = VM_HIDE;
vm_op_t *op = new_op(f, start, VM_HIDE);
op->len = 0;
op->args.pat = pat;
break;
op->end = pat->end;
return op;
}
// Special rules:
case '_': case '^': case '$': case '|': {
const char *name = NULL;
if (matchchar(&str, c)) { // double __, ^^, $$
if (matchchar(&str, ':')) return NULL; // Don't match definitions
char tmp[3] = {c, c, '\0'};
op->args.s = strdup(tmp);
name = strdup(tmp);
} else {
op->args.s = strndup(&c, 1);
if (matchchar(&str, ':')) return NULL; // Don't match definitions
name = strndup(&c, 1);
}
if (matchchar(&str, ':')) { // Don't match definitions
xfree(&op->args.s);
xfree(&op);
return NULL;
}
op->type = VM_REF;
break;
vm_op_t *op = new_op(f, start, VM_REF);
op->args.s = name;
op->end = str;
return op;
}
default: {
// Reference
if (isalpha(c)) {
--str;
const char *refname = str;
str = after_name(str);
if (matchchar(&str, ':')) { // Don't match definitions
xfree(&op);
return NULL;
}
op->type = VM_REF;
op->args.s = strndup(refname, (size_t)(str - refname));
break;
} else {
xfree(&op);
if (!isalpha(c)) return NULL;
--str;
const char *refname = str;
str = after_name(str);
if (matchchar(&str, ':')) // Don't match definitions
return NULL;
}
vm_op_t *op = new_op(f, start, VM_REF);
op->args.s = strndup(refname, (size_t)(str - refname));
op->end = str;
return op;
}
}
op->end = str;
// Postfix operators:
postfix:
str = after_spaces(str);
if (str+2 < f->end && (matchstr(&str, "!=") || matchstr(&str, "=="))) { // Equality <pat1>==<pat2> and inequality <pat1>!=<pat2>
int equal = str[-2] == '=';
vm_op_t *first = op;
vm_op_t *second = bp_simplepattern(f, str);
if (!second)
file_err(f, str, str, "The '%c=' operator expects a pattern before and after.", equal?'=':'!');
if (equal) {
if (!(first->len == -1 || second->len == -1 || first->len == second->len))
file_err(f, origin, second->end,
"These two patterns cannot possibly give the same result (different lengths: %ld != %ld)",
first->len, second->len);
}
op = new(vm_op_t);
op->type = equal ? VM_EQUAL : VM_NOT_EQUAL;
op->start = str;
op->end = second->end;
op->len = first->len != -1 ? first->len : second->len;
op->args.multiple.first = first;
op->args.multiple.second = second;
str = op->end;
goto postfix;
}
return op;
return NULL;
}
//
@ -501,10 +516,8 @@ vm_op_t *bp_stringpattern(file_t *f, const char *str)
{
vm_op_t *ret = NULL;
while (*str) {
vm_op_t *strop = new(vm_op_t);
strop->start = str;
vm_op_t *strop = new_op(f, str, VM_STRING);
strop->len = 0;
strop->type = VM_STRING;
char *start = (char*)str;
vm_op_t *interp = NULL;
for (; *str; str++) {
@ -513,8 +526,7 @@ vm_op_t *bp_stringpattern(file_t *f, const char *str)
file_err(f, str, str, "There should be an escape sequence or pattern here after this backslash.");
if (matchchar(&str, 'N')) { // \N (nodent)
interp = new(vm_op_t);
interp->type = VM_NODENT;
interp = new_op(f, str-2, VM_NODENT);
break;
}
@ -548,10 +560,10 @@ vm_op_t *bp_stringpattern(file_t *f, const char *str)
if (strop->len == 0) {
xfree(&strop);
} else {
ret = chain_together(ret, strop);
ret = chain_together(f, ret, strop);
}
if (interp) {
ret = chain_together(ret, interp);
ret = chain_together(f, ret, interp);
str = interp->end;
// allow terminating seq
matchchar(&str, ';');
@ -566,9 +578,7 @@ vm_op_t *bp_stringpattern(file_t *f, const char *str)
//
vm_op_t *bp_replacement(file_t *f, vm_op_t *pat, const char *replacement)
{
vm_op_t *op = new(vm_op_t);
op->type = VM_REPLACE;
op->start = pat->start;
vm_op_t *op = new_op(f, pat->start, VM_REPLACE);
op->end = pat->end;
op->len = pat->len;
op->args.replace.pat = pat;

2
compiler.h
View File

@ -7,6 +7,8 @@
#include "file_loader.h"
#include "types.h"
__attribute__((nonnull))
vm_op_t *new_op(file_t *f, const char *start, enum VMOpcode type);
__attribute__((nonnull(1,2)))
vm_op_t *bp_simplepattern(file_t *f, const char *str);
__attribute__((nonnull(1,2)))

10
file_loader.c
View File

@ -124,10 +124,12 @@ void destroy_file(file_t **f)
xfree(&((*f)->filename));
(*f)->filename = NULL;
}
if ((*f)->lines) {
xfree(&((*f)->lines));
(*f)->lines = NULL;
}
if ((*f)->contents) {
if ((*f)->mmapped) {
munmap((*f)->contents, (size_t)((*f)->end - (*f)->contents));
@ -136,6 +138,14 @@ void destroy_file(file_t **f)
}
(*f)->contents = NULL;
}
while ((*f)->ops) {
allocated_op_t *tofree = (*f)->ops;
(*f)->ops = tofree->next;
memset(tofree, 'A', sizeof(allocated_op_t)); // Sentinel
xfree(&tofree);
}
xfree(f);
}

3
file_loader.h
View File

@ -6,10 +6,13 @@
#include <stdio.h>
struct allocated_op_s; // declared in types.h
typedef struct {
const char *filename;
char *contents, **lines, *end;
size_t nlines;
struct allocated_op_s *ops;
unsigned int mmapped:1;
} file_t;

17
grammar.c
View File

@ -72,9 +72,7 @@ vm_op_t *lookup(def_t *defs, const char *name)
//
static def_t *with_backref(def_t *defs, file_t *f, const char *name, match_t *m)
{
vm_op_t *op = new(vm_op_t);
op->type = VM_BACKREF;
op->start = m->start;
vm_op_t *op = new_op(f, m->start, VM_BACKREF);
op->end = m->end;
op->len = -1; // TODO: maybe calculate this? (nontrivial because of replacements)
op->args.backref = m;
@ -95,4 +93,17 @@ def_t *with_backrefs(def_t *defs, file_t *f, match_t *m)
return defs;
}
//
// Free all the given definitions up till (but not including) `stop`
//
void free_defs(def_t **defs, def_t *stop)
{
while (*defs != stop && *defs != NULL) {
def_t *next = (*defs)->next;
(*defs)->next = NULL;
free(*defs);
(*defs) = next;
}
}
// vim: ts=4 sw=0 et cino=L2,l1,(0,W4,m1

2
grammar.h
View File

@ -15,6 +15,8 @@ __attribute__((nonnull(2)))
def_t *load_grammar(def_t *defs, file_t *f);
__attribute__((pure, nonnull(2)))
vm_op_t *lookup(def_t *defs, const char *name);
__attribute__((nonnull(1)))
void free_defs(def_t **defs, def_t *stop);
#endif
// vim: ts=4 sw=0 et cino=L2,l1,(0,W4,m1

9
types.h
View File

@ -98,5 +98,14 @@ typedef struct def_s {
struct def_s *next;
} def_t;
//
// Structure used for tracking allocated ops, which must be freed when the file
// is freed.
//
typedef struct allocated_op_s {
struct allocated_op_s *next;
vm_op_t op;
} allocated_op_t;
#endif
// vim: ts=4 sw=0 et cino=L2,l1,(0,W4,m1

15
vm.c
View File

@ -324,20 +324,7 @@ static match_t *_match(def_t *defs, file_t *f, const char *str, vm_op_t *op, uns
{ // Push backrefs and run matching, then cleanup
def_t *defs2 = with_backrefs(defs, f, m1);
m2 = _match(defs2, f, m1->end, op->args.multiple.second, flags, rec);
while (defs2 != defs) {
def_t *next = defs2->next;
defs2->next = NULL;
// Deliberate memory leak, if there is a match, then the op
// will be stored on the match and can't be freed here.
// There's currently no refcounting on ops but that should
// be how to prevent a memory leak from this.
// TODO: add refcounting to ops?
if (m2 == NULL) {
xfree(&defs2->op);
}
xfree(&defs2);
defs2 = next;
}
free_defs(&defs2, defs);
}
if (m2 == NULL) {