From 6e46f042fdc30507b1add6432124d80f20dc86f5 Mon Sep 17 00:00:00 2001 From: Bruce Hill Date: Wed, 20 Sep 2017 03:06:15 -0700 Subject: Mostly working overhaul again that strips out ending lines with "..", and simplifies a lot of the code. Also stripped out Expression, which was just a useless level of indirection. --- nomsu.moon | 309 +++++++++++++++++++++++++++++++------------------------------ 1 file changed, 157 insertions(+), 152 deletions(-) (limited to 'nomsu.moon') diff --git a/nomsu.moon b/nomsu.moon index 03309b1..77f9672 100755 --- a/nomsu.moon +++ b/nomsu.moon @@ -16,88 +16,14 @@ INDENT = " " lpeg.setmaxstack 10000 -- whoa {:P,:V,:S,:Cg,:C,:Cp,:B,:Cmt} = lpeg -get_line_indentation = (line)-> - indent_amounts = {[" "]:1, ["\t"]:4} - with sum = 0 - leading_space = line\match("[\t ]*") - for c in leading_space\gmatch "[\t ]" - sum += indent_amounts[c] - -make_parser = (lingo, extra_definitions)-> - indent_stack = {0} - push = (n)-> table.insert indent_stack, n - pop = ()-> table.remove indent_stack - check_indent = (subject,end_pos,spaces)-> - num_spaces = get_line_indentation(spaces) - if num_spaces <= indent_stack[#indent_stack] then return nil - push num_spaces - return end_pos - check_dedent = (subject,end_pos,spaces)-> - num_spaces = get_line_indentation(spaces) - if num_spaces >= indent_stack[#indent_stack] then return nil - pop! - return end_pos - check_nodent = (subject,end_pos,spaces)-> - num_spaces = get_line_indentation(spaces) - if num_spaces != indent_stack[#indent_stack] then return nil - return end_pos - - wordchar = P(1)-S(' \t\n\r%#:;,.{}[]()"\\') - nl = P("\n") - whitespace = S(" \t")^1 - blank_line = whitespace^-1 * nl - line_comment = re.compile([=[ "#" [^%nl]* ]=], {:nl}) - block_comment = re.compile([=[ - "#.." (!%nl .)* (%indent (!%dedent %nl [^%nl]*)*) - ]=], {:nl, :whitespace, - indent:#(nl * blank_line^0 * Cmt(S(" \t")^0, check_indent)), - dedent:#(nl * blank_line^0 * Cmt(S(" \t")^0, check_dedent)), - new_line:nl * blank_line^0 * Cmt(S(" \t")^0, check_nodent)}) - blank_line = ((Cmt(whitespace^-1, check_nodent) * (block_comment + line_comment))^-1 + whitespace^-1) * nl - defs = - :wordchar, :nl, ws:whitespace, :blank_line, :block_comment, :line_comment - eol: #nl + (P("")-P(1)) - word_boundary: whitespace^-1 + B(P("..")) + B(S("\";)]")) + #S("\":([") + #((whitespace + nl)^0 * P("..")) - indent: #(nl * blank_line^0 * Cmt(whitespace^-1, check_indent)) - dedent: #(nl * blank_line^0 * Cmt(whitespace^-1, check_dedent)) - new_line: nl * blank_line^0 * Cmt(whitespace^-1, check_nodent) - error_handler: (src,pos,errors)-> - line_no = 1 - for _ in src\sub(1,-#errors)\gmatch("\n") do line_no += 1 - err_pos = #src - #errors + 1 - if errors\sub(1,1) == "\n" - -- Indentation error - err_pos += #errors\match("[ \t]*", 2) - start_of_err_line = err_pos - while src\sub(start_of_err_line, start_of_err_line) != "\n" do start_of_err_line -= 1 - start_of_prev_line = start_of_err_line - 1 - while src\sub(start_of_prev_line, start_of_prev_line) != "\n" do start_of_prev_line -= 1 - - prev_line,err_line,next_line = src\match("([^\n]*)\n([^\n]*)\n([^\n]*)", start_of_prev_line+1) - - pointer = ("-")\rep(err_pos - start_of_err_line + 0) .. "^" - error("\nParse error on line #{line_no}:\n\n#{prev_line}\n#{err_line}\n#{pointer}\n#{next_line}\n") - - if extra_definitions - for k,v in pairs(extra_definitions) do defs[k] = v - - setmetatable(defs, { - __index: (t,key)-> - fn = (src, value, errors)-> - token = {type: key, :src, :value, :errors} - return token - t[key] = fn - return fn - }) - return re.compile lingo, defs class NomsuCompiler new:(parent)=> + @write = (...)=> io.write(...) @defs = setmetatable({}, {__index:parent and parent.defs}) @callstack = {} @debug = false @initialize_core! - @write = (...)=> io.write(...) @utils = utils @loaded_files = {} @@ -141,23 +67,28 @@ class NomsuCompiler @defs[invocation] = fn_info get_invocations_from_definition:(def, vars)=> - if def.type == "String" or def.type == "List" + if def.type == "String" return @tree_to_value(def, vars) - if def.type != "Thunk" - @error "Trying to get invocations from #{def.type}, but expected Thunk." + if def.type != "List" + error "DEF IS: #{utils.repr(def)}" + @error "Trying to get invocations from #{def.type}, but expected List or String." + invocations = {} - for statement in *def.value.value - if statement.value.type != "FunctionCall" - @error "Invalid statement type: #{statement.value.type}, expected FunctionCall" + for item in *def.value + if item.type == "String" + table.insert invocations, @tree_to_value(item, vars) + continue + if item.type != "FunctionCall" + @error "Invalid list item: #{item.type}, expected FunctionCall or String" name_bits = {} - for token in *statement.value.value + for token in *item.value if token.type == "Word" table.insert name_bits, token.value - elseif token.value.type == "Var" - table.insert name_bits, token.value.src + elseif token.type == "Var" + table.insert name_bits, token.src else - @error "Unexpected token type in definition: #{token.value.type} (expected Word or Var)" + @error "Unexpected token type in definition: #{token.type} (expected Word or Var)" table.insert invocations, table.concat(name_bits, " ") return invocations @@ -189,11 +120,11 @@ class NomsuCompiler for invocation in *invocations @defs[invocation] = fn_info - run: (text)=> + run: (text, filename)=> if @debug @writeln "RUNNING TEXT:\n#{text}" -- This will execute each chunk as it goes along - code, retval = @compile(text) + code, retval = @compile(text, filename) if @debug @writeln "\nGENERATED LUA CODE:\n#{code}" @writeln "\nPRODUCED RETURN VALUE:\n#{retval}" @@ -224,70 +155,145 @@ class NomsuCompiler error("Failed to compile generated code:\n#{str}\n\n#{err}") return (lua_thunk!)(self, {}) - parse: (str)=> + parse: (str, filename)=> if @debug @writeln("PARSING:\n#{str}") + + get_line_indentation = (line)-> + indent_amounts = {[" "]:1, ["\t"]:4} + with sum = 0 + leading_space = line\match("[\t ]*") + for c in leading_space\gmatch "[\t ]" + sum += indent_amounts[c] + + indent_stack = {0} + check_indent = (subject,end_pos,spaces)-> + num_spaces = get_line_indentation(spaces) + if num_spaces > indent_stack[#indent_stack] + table.insert(indent_stack, num_spaces) + return end_pos + check_dedent = (subject,end_pos,spaces)-> + num_spaces = get_line_indentation(spaces) + if num_spaces < indent_stack[#indent_stack] + table.remove(indent_stack) + return end_pos + check_nodent = (subject,end_pos,spaces)-> + num_spaces = get_line_indentation(spaces) + if num_spaces == indent_stack[#indent_stack] + return end_pos + lingo = [=[ - file <- ({ {| %blank_line* {:body: block :} %blank_line* (errors)? |} }) -> File - errors <- (({.+}) => error_handler) - block <- ({ {| statement (%new_line statement)* |} }) -> Block - statement <- ({ (functioncall / expression) }) -> Statement - one_liner <- ({ {| - (({ - (({ {| - (expression (%word_boundary fn_bit)+) / (word (%word_boundary fn_bit)*) - |} }) -> FunctionCall) - / (expression) - }) -> Statement) - |} }) -> Block - - functioncall <- ({ {| (expression %word_boundary fn_bits) / (word (%word_boundary fn_bits)?) |} }) -> FunctionCall - fn_bit <- (expression / word) - fn_bits <- - ((".." %ws? %line_comment? (%indent %new_line indented_fn_bits %dedent) (%new_line ".." %ws? fn_bits)?) - / (%new_line ".." fn_bit (%word_boundary fn_bits)?) - / (fn_bit (%word_boundary fn_bits)?)) - indented_fn_bits <- - fn_bit ((%new_line / %word_boundary) indented_fn_bits)? - - thunk <- - ({ ":" %ws? %line_comment? - ((%indent %new_line block ((%dedent (%new_line "..")?) / errors)) - / (one_liner (%ws? (%new_line? ".."))?)) }) -> Thunk + file <- ({ {| shebang? {:body: block :} %nl* (({.+} ("" -> "Unexpected end of file")) => error)? |} }) -> File + + shebang <- "#!" [^%nl]* %nl + + block <- ({ {| + (ignored_line %nl)* + line_of_statements (nodent line_of_statements)* + (%nl ignored_line)* |} }) -> Block + inline_block <- ({ {| inline_line_of_statements |} }) -> Block + + line_of_statements <- statement (%ws? ";" %ws? statement)* + inline_line_of_statements <- inline_statement (%ws? ";" %ws? inline_statement)* + + statement <- ({ functioncall / expression }) -> Statement + inline_statement <- ({ inline_functioncall / expression }) -> Statement + + expression <- ( + longstring / string / number / variable / list / thunk / block_functioncall + / ("(" %ws? (inline_thunk / inline_functioncall) %ws? ")")) + + -- Function calls need at least one word in them + functioncall <- ({ {| + (expression (dotdot / tok_gap))* word ((dotdot / tok_gap) (expression / word))* + |} }) -> FunctionCall + inline_functioncall <- ({ {| + (expression tok_gap)* word (tok_gap (expression / word))* + |} }) -> FunctionCall + block_functioncall <- "(..)" indent functioncall (dedent / (({.+} ("" -> "Error while parsing block function call")) => error)) word <- ({ !number {%wordchar (!"'" %wordchar)*} }) -> Word - expression <- ({ (longstring / string / number / variable / list / thunk / subexpression) }) -> Expression + + thunk <- ({ ":" ((indent block (dedent / (({.+} ("" -> "Error while parsing thunk")) => error))) + / (%ws? inline_block)) }) -> Thunk + inline_thunk <- ({ ":" %ws? inline_block }) -> Thunk string <- ({ (!longstring) '"' {(("\" [^%nl]) / [^"%nl])*} '"' }) -> String + longstring <- ({ '".."' %ws? - {| - (("|" {| ({("\\" / (!string_interpolation [^%nl]))+} / string_interpolation)* |}) - / %line_comment)? - (%indent - (%new_line "|" {| - ({("\\" / (!string_interpolation [^%nl]))+} / string_interpolation)* - |})+ - ((%dedent (%new_line '..')?) / errors))? - |}}) -> Longstring - string_interpolation <- "\" %ws? (functioncall / expression) %ws? "\" - number <- ({ {'-'? [0-9]+ ("." [0-9]+)?} }) -> Number + {| (longstring_line (indent + longstring_line (nodent longstring_line)* + (dedent / longstring_error))?) + /(indent + longstring_line (nodent longstring_line)* + (dedent / longstring_error)) |} }) -> Longstring + longstring_line <- "|" {| ({("\\" / (!string_interpolation [^%nl]))+} / string_interpolation)* |} + longstring_error <- (({.+} ("" -> "Error while parsing Longstring")) => error) + string_interpolation <- "\" %ws? (inline_functioncall / expression) %ws? "\" + + number <- ({ {'-'? ([0-9]+ "." [0-9]+) / ("." [0-9]+) / ([0-9]+) } }) -> Number + + -- Hack to allow %foo's to parse as "%foo" and "'s" separately variable <- ({ ("%" {%wordchar (!"'" %wordchar)*}) }) -> Var - subexpression <- - ("(" %ws? (functioncall / expression) %ws? ")") - / ("(..)" %ws? %line_comment? %indent %new_line ((({ {| indented_fn_bits |} }) -> FunctionCall) / expression) %dedent (%new_line "..")?) - list <- ({ {| - ("[..]" %ws? %line_comment? %indent %new_line indented_list ","? ((%dedent (%new_line "..")?) / errors)) - / ("[" %ws? (list_items ","?)? %ws?"]") + ("[..]" indent + list_line (nodent list_line)* + (dedent / (({.+} ("" -> "Error while parsing list")) => error))) + /("[" %ws? (list_line %ws?)? "]") |} }) -> List - list_items <- ((functioncall / expression) (list_sep list_items)?) - list_sep <- %ws? "," %ws? - indented_list <- - (functioncall / expression) (((list_sep (%line_comment? %new_line)?) / (%line_comment? %new_line)) indented_list)? + list_line <- list_bit (%ws? "," tok_gap list_bit)* (%ws? ",")? + list_bit <- inline_functioncall / expression + + block_comment <- "#.." [^%nl]* indent [^%nl]* (%nl ((%ws? (!. / &%nl)) / (!%dedented [^%nl]*)))* + line_comment <- "#" [^%nl]* + + eol <- %ws? line_comment? (!. / &%nl) + ignored_line <- (%nodented (block_comment / line_comment)) / (%ws? (!. / &%nl)) + indent <- eol (%nl ignored_line)* %nl %indented + nodent <- eol (%nl ignored_line)* %nl %nodented + dedent <- eol (%nl ignored_line)* (((!.) &%dedented) / (&(%nl %dedented))) + tok_gap <- %ws / %prev_edge / &("[" / [.,:;{("#%']) + dotdot <- nodent ".." %ws? ]=] - lingo = make_parser lingo + whitespace = S(" \t")^1 + defs = + ws:whitespace, nl: P("\n") + wordchar: P(1)-S(' \t\n\r%#:;,.{}[]()"\\') + indented: Cmt(S(" \t")^0 * (#(P(1)-S(" \t\n") + (-P(1)))), check_indent) + nodented: Cmt(S(" \t")^0 * (#(P(1)-S(" \t\n") + (-P(1)))), check_nodent) + dedented: Cmt(S(" \t")^0 * (#(P(1)-S(" \t\n") + (-P(1)))), check_dedent) + prev_edge: B(S(" \t\n.,:;}])\"")) + error: (src,pos,errors,err_msg)-> + line_no = 1 + for _ in src\sub(1,-#errors)\gmatch("\n") do line_no += 1 + err_pos = #src - #errors + 1 + if errors\sub(1,1) == "\n" + -- Indentation error + err_pos += #errors\match("[ \t]*", 2) + start_of_err_line = err_pos + while src\sub(start_of_err_line, start_of_err_line) != "\n" and start_of_err_line > 1 + start_of_err_line -= 1 + start_of_prev_line = start_of_err_line - 1 + while src\sub(start_of_prev_line, start_of_prev_line) != "\n" and start_of_prev_line > 1 + start_of_prev_line -= 1 + + local prev_line,err_line,next_line + prev_line,err_line,next_line = src\match("([^\n]*)\n([^\n]*)\n([^\n]*)", start_of_prev_line+1) + + pointer = ("-")\rep(err_pos - start_of_err_line + 0) .. "^" + error("\n#{err_msg or "Parse error"} in #{filename} on line #{line_no}:\n\n#{prev_line}\n#{err_line}\n#{pointer}\n#{next_line}\n") + + setmetatable(defs, { + __index: (t,key)-> + fn = (src, value, errors)-> + token = {type: key, :src, :value, :errors} + return token + t[key] = fn + return fn + }) + lingo = re.compile(lingo, defs) tree = lingo\match(str\gsub("\r","").."\n") if @debug @writeln("\nPARSE TREE:") @@ -305,6 +311,8 @@ class NomsuCompiler tree_to_lua: (tree, kind="Expression")=> assert tree, "No tree provided." + if not tree.type + @error "Invalid tree: #{utils.repr(tree)}" indent = "" buffer = {} return_value = nil @@ -361,9 +369,6 @@ class NomsuCompiler else add "ret = "..(to_lua(tree.value)\match("%s*(.*)")) - when "Expression" - add to_lua(tree.value) - when "FunctionCall" name = @fn_name_from_tree(tree) if @defs[name] and @defs[name].is_macro @@ -485,9 +490,6 @@ class NomsuCompiler when "Statement" @_yield_tree(tree.value, indent_level) - when "Expression" - @_yield_tree(tree.value, indent_level) - when "FunctionCall" name = @fn_name_from_tree(tree) args = [a for a in *tree.value when a.type != "Word"] @@ -534,10 +536,10 @@ class NomsuCompiler table.insert(result, line) return table.concat result, "\n" - compile: (src, output_file=nil)=> + compile: (src, filename, output_file=nil)=> if @debug @writeln "COMPILING:\n#{src}" - tree = @parse(src) + tree = @parse(src, filename) assert tree, "Tree failed to compile: #{src}" code, retval = @tree_to_lua(tree) if output_file @@ -555,7 +557,7 @@ class NomsuCompiler @callstack = {} error! - test: (src, expected)=> + test: (src, filename, expected)=> i = 1 while i != nil start,stop = src\find("\n\n", i) @@ -567,7 +569,7 @@ class NomsuCompiler @error("WHERE'S THE ===? in:\n#{test}") test_src, expected = test\sub(1,start-1), test\sub(stop+1,-1) expected = expected\match'[\n]*(.*[^\n])' - tree = @parse(test_src) + tree = @parse(test_src, filename) got = @stringify_tree(tree.value.body) if got != expected @error"TEST FAILED!\nSource:\n#{test_src}\nExpected:\n#{expected}\n\nGot:\n#{got}" @@ -597,26 +599,29 @@ class NomsuCompiler @def "require %filename", (vars)=> if not @loaded_files[vars.filename] file = io.open(vars.filename) - @loaded_files[vars.filename] = @run(file\read('*a')) + if not file + @error "File does not exist: #{vars.filename}" + @loaded_files[vars.filename] = @run(file\read('*a'), vars.filename) return @loaded_files[vars.filename] @def "run file %filename", (vars)=> file = io.open(vars.filename) - return @run(file\read('*a')) + if not file + @error "File does not exist: #{vars.filename}" + return @run(file\read('*a'), vars.filename) -- Run on the command line via "./nomsu.moon input_file.nom" to execute -- and "./nomsu.moon input_file.nom output_file.lua" to compile (use "-" to compile to stdout) if arg and arg[1] c = NomsuCompiler() - --c.debug = true input = io.open(arg[1])\read("*a") -- If run via "./nomsu.moon file.nom -", then silence output and print generated -- source code instead. _write = c.write if arg[2] == "-" c.write = -> - code, retval = c\compile(input) + code, retval = c\compile(input, arg[1]) c.write = _write -- put it back if arg[2] output = if arg[2] == "-" @@ -637,7 +642,7 @@ if arg and arg[1] elseif arg -- REPL: c = NomsuCompiler() - c\run('run file "core.nom"') + c\run('require "lib/core.nom"') while true buff = "" while true -- cgit v1.2.3