From 2e9a3cdf009e80caced01f81dc34d2968346d429 Mon Sep 17 00:00:00 2001 From: Bruce Hill Date: Tue, 5 Sep 2017 21:20:11 -0700 Subject: Added longstrings and removed indentification step. --- nomic.moon | 340 ++++++++++++++++++++++++++++++++----------------------------- 1 file changed, 176 insertions(+), 164 deletions(-) (limited to 'nomic.moon') diff --git a/nomic.moon b/nomic.moon index 4a7ecf3..3c14f7d 100644 --- a/nomic.moon +++ b/nomic.moon @@ -4,8 +4,10 @@ utils = require 'utils' moon = require 'moon' lpeg.setmaxstack 10000 -- whoa +{:P,:V,:S,:Cg,:C,:Cp,:B,:Cmt} = lpeg -linebreak = lpeg.P("\r")^-1 * lpeg.P("\n") +wordchar = P(1)-S(' \t\n\r%:;,.{}[]()"') +spaces = S(" \t")^1 get_line_indentation = (line)-> indent_amounts = {[" "]:1, ["\t"]:4} @@ -14,158 +16,27 @@ get_line_indentation = (line)-> for c in leading_space\gmatch "[\t ]" sum += indent_amounts[c] -pos_to_line = (str,pos)-> - line_no = 1 - for line in str\gmatch("[^%n]+") - if #line >= pos then return line, line_no - pos -= (#line + 1) - line_no += 1 - error "Failed to find position #{pos} in str" - -pos_to_line_no = (str,pos)-> - with line = 1 - for _ in str\sub(1, pos)\gmatch("\n") - line += 1 - -add_indent_tokens = (str)-> - indent_stack = {0} - result = {} - -- TODO: Store mapping from new line numbers to old ones - defs = - linebreak: linebreak - process_line: (line)-> - -- Remove blank lines - unless line\match"[^ \t\n]" - return - indent = get_line_indentation(line) - if indent > indent_stack[#indent_stack] - table.insert result, "{\n " - table.insert indent_stack, indent - elseif indent < indent_stack[#indent_stack] - dedents = 0 - tokens = {} - while indent < indent_stack[#indent_stack] - table.remove indent_stack - table.insert tokens, "}" - table.insert tokens, " " - table.insert result, table.concat(tokens, "\n") - else - table.insert result, " " - -- Delete leading whitespace - --line = line\gsub("[ \t]*", "", 1) - -- Delete trailing whitespace and carriage returns - line = line\gsub("[ \t\r]*\n", "\n", 1) - table.insert result, line - - indentflagger = [=[ - file <- line* - line <- ((string / [^%linebreak])* %linebreak) -> process_line - string <- '"' (("\" .) / [^"])* '"' - ]=] - indentflagger = re.compile indentflagger, defs - indentflagger\match(str.."\n") - while #indent_stack > 1 - table.remove indent_stack - table.insert result, "}\n" - return (table.concat result)\sub(1,-2) - -lingo = [=[ - file <- ({ {| {:body: (" " block) :} ({:errors: errors :})? |} }) -> File - errors <- ({ {.+} }) -> Errors - block <- ({ {| statement (%nodent statement)* |} }) -> Block - statement <- ({ (functioncall / expression) }) -> Statement - one_liner <- ({ {| - (({ - (({ {| - (expression (%word_boundary fn_bit)+) / (word (%word_boundary fn_bit)*) - |} }) -> FunctionCall) - / (expression) - }) -> Statement) - |} }) -> Block - - functioncall <- ({ {| (expression %word_boundary fn_bits) / (word (%word_boundary fn_bits)?) |} }) -> FunctionCall - fn_bit <- (expression / word) - fn_bits <- - ((".." %ws? (%indent %nodent indented_fn_bits %dedent) (%nodent ".." %ws? fn_bits)?) - / (%nodent ".." fn_bit fn_bits) - / (fn_bit (%word_boundary fn_bits)?)) - indented_fn_bits <- - fn_bit ((%nodent / %word_boundary) indented_fn_bits)? - - thunk <- - ({ ":" %ws? - ((%indent %nodent block %dedent (%nodent "..")?) - / (one_liner (%ws? ((%nodent? "..")))?)) }) -> Thunk - - word <- ({ !number {%wordchar+} }) -> Word - expression <- ({ (string / number / variable / list / thunk / subexpression) }) -> Expression - - string <- ({ '"' {(("\" .) / [^"])*} '"' }) -> String - number <- ({ {'-'? [0-9]+ ("." [0-9]+)?} }) -> Number - variable <- ({ ("%" {%wordchar+}) }) -> Var - - subexpression <- - ("(" %ws? (functioncall / expression) %ws? ")") - / ("(..)" %ws? %indent %nodent (expression / (({ {| indented_fn_bits |} }) -> FunctionCall)) %dedent (%nodent "..")?) - - list <- ({ {| - ("[..]" %ws? %indent %nodent indented_list ","? %dedent (%nodent "..")?) - / ("[" %ws? (list_items ","?)? %ws?"]") - |} }) -> List - list_items <- (expression (list_sep list_items)?) - list_sep <- %ws? "," %ws? - indented_list <- - expression (((list_sep %nodent?) / %nodent) indented_list)? -]=] - -wordchar = lpeg.P(1)-lpeg.S(' \t\n\r%:;,.{}[]()"') -defs = - eol: #(linebreak) + (lpeg.P("")-lpeg.P(1)) - ws: lpeg.S(" \t")^1 - wordchar: wordchar - indent: linebreak * lpeg.P("{") * lpeg.S(" \t")^0 - nodent: linebreak * lpeg.P(" ") * lpeg.S(" \t")^0 - dedent: linebreak * lpeg.P("}") * lpeg.S(" \t")^0 - word_boundary: lpeg.S(" \t")^1 + lpeg.B(lpeg.P("..")) + lpeg.B(lpeg.S("\";)]")) + #lpeg.S("\":([") + #lpeg.P("..") - -setmetatable(defs, { - __index: (t,key)-> - --print("WORKING for #{key}") - fn = (src, value, ...)-> - token = {type: key, src:src, value: value} - return token - t[key] = fn - return fn -}) -lingo = re.compile lingo, defs - class Game new:(parent)=> @defs = setmetatable({}, {__index:parent and parent.defs}) - @macros = setmetatable({}, {__index: parent and parent.macros}) @debug = false call: (fn_name,...)=> - if @defs[fn_name] == nil + fn_info = @defs[fn_name] + if fn_info == nil error "Attempt to call undefined function: #{fn_name}" - {fn, arg_names} = @defs[fn_name] - if @debug - print("Calling #{fn_name}...") - args = {} - for i,name in ipairs(arg_names) - args[name] = select(i,...) - if @debug - print("arg #{utils.repr(name,true)} = #{utils.repr(select(i,...), true)}") - ret = fn(self, args) + {:fn, :arg_names} = fn_info + args = {name, select(i,...) for i,name in ipairs(arg_names)} if @debug - print "returned #{utils.repr(ret,true)}" - return ret + print "Calling #{fn_name} with args: #{utils.repr(args)}" + return fn(self, args) def: (spec, fn)=> invocations,arg_names = self\get_invocations spec + fn_info = {:fn, :arg_names, :invocations, is_macro:false} for invocation in *invocations - @defs[invocation] = {fn, arg_names} + @defs[invocation] = fn_info get_invocations:(text)=> if type(text) == 'string' then text = {text} @@ -190,21 +61,55 @@ class Game defmacro: (spec, fn)=> assert fn, "No function supplied" invocations,arg_names = self\get_invocations spec + fn_info = {:fn, :arg_names, :invocations, is_macro:true} for invocation in *invocations - @macros[invocation] = {fn, arg_names} + @defs[invocation] = fn_info simplemacro: (spec, replacement)=> + spec = spec\gsub("\r", "") + replacement = replacement\gsub("\r", "") + indent_stack = {0} + push = (n)-> table.insert indent_stack, n + pop = ()-> table.remove indent_stack + check_indent = (subject,end_pos,spaces)-> + num_spaces = get_line_indentation(spaces) + if num_spaces <= indent_stack[#indent_stack] then return nil + push num_spaces + return end_pos + check_dedent = (subject,end_pos,spaces)-> + num_spaces = get_line_indentation(spaces) + if num_spaces >= indent_stack[#indent_stack] then return nil + pop! + return end_pos + check_nodent = (subject,end_pos,spaces)-> + num_spaces = get_line_indentation(spaces) + if num_spaces != indent_stack[#indent_stack] then return nil + return end_pos + + nl = P("\n") + blank_line = spaces^-1 * nl + defs = + eol: #(nl) + (P("")-P(1)) + ws: S(" \t")^1 + :wordchar + :replacer + :nl, :spaces + word_boundary: S(" \t")^1 + B(P("..")) + B(S("\";)]")) + #S("\":([") + #P("..") + indent: #(nl * blank_line^0 * Cmt(spaces^-1, check_indent)) + dedent: #(nl * blank_line^0 * Cmt(spaces^-1, check_dedent)) + new_line: nl * blank_line^0 * Cmt(spaces^-1, check_nodent) + replace_grammar = [[ - stuff <- {~ (var / string / .)+ ~} + stuff <- {~ (var / longstring / string / .)+ ~} var <- ("%" {%wordchar+}) -> replacer string <- '"' (("\" .) / [^"])* '"' + longstring <- ('"..' %indent %nl {(!%dedent .)*} %new_line '.."') ]] - replacement = add_indent_tokens replacement fn = (vars, helpers, ftype)=> replacer = (varname)-> ret = vars[varname].src return ret - replacement_grammar = re.compile(replace_grammar, {:wordchar, :replacer}) + replacement_grammar = re.compile(replace_grammar, defs) replaced = replacement_grammar\match(replacement) tree = lingo\match replaced if tree.value.errors and #tree.value.errors.value > 0 @@ -242,10 +147,98 @@ class Game parse: (str)=> if @debug print("PARSING:\n#{str}") - indentified = add_indent_tokens str - if @debug - print("\nINDENTIFIED:\n#{indentified}") - tree = lingo\match indentified + lingo = [=[ + file <- ({ {| %new_line? {:body: block :} %new_line? ({:errors: errors :})? |} }) -> File + errors <- ({ {.+} }) -> Errors + block <- ({ {| statement (%new_line statement)* |} }) -> Block + statement <- ({ (functioncall / expression) }) -> Statement + one_liner <- ({ {| + (({ + (({ {| + (expression (%word_boundary fn_bit)+) / (word (%word_boundary fn_bit)*) + |} }) -> FunctionCall) + / (expression) + }) -> Statement) + |} }) -> Block + + functioncall <- ({ {| (expression %word_boundary fn_bits) / (word (%word_boundary fn_bits)?) |} }) -> FunctionCall + fn_bit <- (expression / word) + fn_bits <- + ((".." %ws? (%indent %new_line indented_fn_bits %dedent) (%new_line ".." %ws? fn_bits)?) + / (%new_line ".." fn_bit fn_bits) + / (fn_bit (%word_boundary fn_bits)?)) + indented_fn_bits <- + fn_bit ((%new_line / %word_boundary) indented_fn_bits)? + + thunk <- + ({ ":" %ws? + ((%indent %new_line block %dedent (%new_line "..")?) + / (one_liner (%ws? (%new_line? ".."))?)) }) -> Thunk + + word <- ({ !number {%wordchar+} }) -> Word + expression <- ({ (longstring / string / number / variable / list / thunk / subexpression) }) -> Expression + + string <- ({ (!('"..' %ws? %nl)) '"' {(("\" .) / [^"])*} '"' }) -> String + longstring <- ({ '"..' %ws? %indent %nl {(!%dedent .)* (%nl %ws? %eol)*} %new_line '.."' }) -> Longstring + number <- ({ {'-'? [0-9]+ ("." [0-9]+)?} }) -> Number + variable <- ({ ("%" {%wordchar+}) }) -> Var + + subexpression <- + ("(" %ws? (functioncall / expression) %ws? ")") + / ("(..)" %ws? %indent %new_line ((({ {| indented_fn_bits |} }) -> FunctionCall) / expression) %dedent (%new_line "..")?) + + list <- ({ {| + ("[..]" %ws? %indent %new_line indented_list ","? %dedent (%new_line "..")?) + / ("[" %ws? (list_items ","?)? %ws?"]") + |} }) -> List + list_items <- (expression (list_sep list_items)?) + list_sep <- %ws? "," %ws? + indented_list <- + expression (((list_sep %new_line?) / %new_line) indented_list)? + ]=] + + str = str\gsub("\r", "") + indent_stack = {0} + push = (n)-> table.insert indent_stack, n + pop = ()-> table.remove indent_stack + check_indent = (subject,end_pos,spaces)-> + num_spaces = get_line_indentation(spaces) + if num_spaces <= indent_stack[#indent_stack] then return nil + push num_spaces + return end_pos + check_dedent = (subject,end_pos,spaces)-> + num_spaces = get_line_indentation(spaces) + if num_spaces >= indent_stack[#indent_stack] then return nil + pop! + return end_pos + check_nodent = (subject,end_pos,spaces)-> + num_spaces = get_line_indentation(spaces) + if num_spaces != indent_stack[#indent_stack] then return nil + return end_pos + + nl = P("\n") + blank_line = spaces^-1 * nl + defs = + eol: #(nl) + (P("")-P(1)) + ws: S(" \t")^1 + :wordchar + :nl, :spaces + word_boundary: S(" \t")^1 + B(P("..")) + B(S("\";)]")) + #S("\":([") + #P("..") + indent: #(nl * blank_line^0 * Cmt(spaces^-1, check_indent)) + dedent: #(nl * blank_line^0 * Cmt(spaces^-1, check_dedent)) + new_line: nl * blank_line^0 * Cmt(spaces^-1, check_nodent) + + setmetatable(defs, { + __index: (t,key)-> + --print("WORKING for #{key}") + fn = (src, value, ...)-> + token = {type: key, src:src, value: value} + return token + t[key] = fn + return fn + }) + lingo = re.compile lingo, defs + tree = lingo\match(str\gsub("\r","").."\n") if @debug print("\nPARSE TREE:") self\print_tree(tree) @@ -326,7 +319,8 @@ class Game for token in *tree.value.value table.insert name_bits, if token.type == "Word" then token.value else "%" name = table.concat(name_bits, " ") - if @macros[name] + if @defs[name] and @defs[name].is_macro + -- This case here is to prevent "ret =" from getting prepended when the macro might not want it lua transform(tree.value) ret = table.concat ret_lines, "\n" return ret @@ -340,8 +334,8 @@ class Game for token in *tree.value table.insert name_bits, if token.type == "Word" then token.value else "%" name = table.concat(name_bits, " ") - if @macros[name] - {fn, arg_names} = @macros[name] + if @defs[name] and @defs[name].is_macro + {:fn, :arg_names} = @defs[name] helpers = {:indented, :transform, :ind, :ded, :lua, :comma_separated_items} args = [a for a in *tree.value when a.type != "Word"] args = {name,args[i] for i,name in ipairs(arg_names)} @@ -359,6 +353,15 @@ class Game unescaped = tree.value\gsub("\\(.)", ((c)-> escapes[c] or c)) lua utils.repr(unescaped, true) + when "Longstring" + first_nonblank_line = tree.value\match("[^\n]+") + indent = first_nonblank_line\match("[ \t]*") + result = {} + for line in (tree.value.."\n")\gmatch("(.-)\n") + line = line\gsub("^"..indent, "", 1) + table.insert result, line + lua utils.repr(table.concat(result, "\n"), true) + when "Number" lua tree.value @@ -417,6 +420,11 @@ class Game self\_yield_tree(a, indent_level+1) when "String" + -- TODO: Better implement + coroutine.yield(ind(utils.repr(tree.value, true))) + + when "Longstring" + -- TODO: Better implement coroutine.yield(ind(utils.repr(tree.value, true))) when "Number" @@ -453,20 +461,24 @@ class Game return code test: (src, expected)=> - if expected == nil - start,stop = src\find"===" + i = 1 + while i != nil + start,stop = src\find("\n\n", i) + + test = src\sub(i,start) + i = stop + start,stop = test\find"===" if not start or not stop then - error("WHERE'S THE ===? in:\n#{src}") - src, expected = src\sub(1,start-1), src\sub(stop+1,-1) - expected = expected\match'[\n]*(.*[^\n])' - if not expected then error("WTF???") - tree = self\parse(src) - got = if tree.value.errors and #tree.value.errors.value > 0 - self\stringify_tree(tree.value.errors) - else - self\stringify_tree(tree.value.body) - if got != expected - error"TEST FAILED!\nSource:\n#{src}\nExpected:\n#{expected}\n\nGot:\n#{got}" + error("WHERE'S THE ===? in:\n#{test}") + test_src, expected = test\sub(1,start-1), test\sub(stop+1,-1) + expected = expected\match'[\n]*(.*[^\n])' + tree = self\parse(test_src) + got = if tree.value.errors and #tree.value.errors.value > 0 + self\stringify_tree(tree.value.errors) + else + self\stringify_tree(tree.value.body) + if got != expected + error"TEST FAILED!\nSource:\n#{test_src}\nExpected:\n#{expected}\n\nGot:\n#{got}" return Game -- cgit v1.2.3