From af62e3e8bf2794048e7c20fc16fabd8c4726961e Mon Sep 17 00:00:00 2001 From: Bruce Hill Date: Thu, 12 Jul 2018 21:24:08 -0700 Subject: [PATCH] All tests passing. --- core/control_flow.nom | 2 +- nomsu.lua | 2 +- nomsu.moon | 2 +- nomsu.peg | 116 ++++++++++++++++++++++-------------------- nomsu_compiler.lua | 8 +-- nomsu_compiler.moon | 8 +-- nomsu_tree.lua | 3 +- nomsu_tree.moon | 2 +- parser.lua | 51 ++++++++++++++++--- parser.moon | 25 ++++++--- tests/text.nom | 4 +- 11 files changed, 136 insertions(+), 87 deletions(-) diff --git a/core/control_flow.nom b/core/control_flow.nom index 4733168..5f3bd33 100644 --- a/core/control_flow.nom +++ b/core/control_flow.nom @@ -95,7 +95,7 @@ compile [repeat while %condition %body] to \(%body as lua statements) if %body has subtree % where: (%.type = "Action") and (%.stub is "do next repeat") - ..: to %lua write "\n ::continue_repeat::" + ..to %lua write "\n ::continue_repeat::" to %lua write "\nend --while-loop" if %body has subtree % where: (%.type = "Action") and (%.stub is "stop repeating") diff --git a/nomsu.lua b/nomsu.lua index 6080637..a69c8df 100644 --- a/nomsu.lua +++ b/nomsu.lua @@ -197,7 +197,7 @@ run = function() end local tree = nomsu:parse(file, source) if tree then - if tree.type ~= "FileChunks" then + if tree.type ~= "File" then tree = { tree } diff --git a/nomsu.moon b/nomsu.moon index cba9eee..e2d79bd 100755 --- a/nomsu.moon +++ b/nomsu.moon @@ -130,7 +130,7 @@ run = -> return unless file tree = nomsu\parse(file, source) if tree - if tree.type != "FileChunks" + if tree.type != "File" tree = {tree} -- Each chunk's compilation is affected by the code in the previous chunks -- (typically), so each chunk needs to compile and run before the next one diff --git a/nomsu.peg b/nomsu.peg index 62f1900..38db312 100644 --- a/nomsu.peg +++ b/nomsu.peg @@ -1,38 +1,37 @@ -- Nomsu version 2 file (File): - {:curr_indent: '' :} - blank_line* + %nl* + {:curr_indent: ' '* :} (chunk (nl_nodent chunk_delimeter nl_nodent chunk)*)? - blank_line* - (!. / (!! .* -> "Parse error" !!)) + %nl* + (!! .+ -> "Parse error" !!)? nodent: =curr_indent !(" ") -indent: =curr_indent " " !(" ") -dedent: !(=curr_indent) (" ")* -eol: %ws* (!. / &%nl) +indent: =curr_indent " " +blank_lines: %nl (%ws* %nl)* +eol: (!. / &%nl) comment (Comment): - "#" {~ [^%nl]* (%nl+ ({:curr_indent: indent :} -> '') [^%nl]* (%nl+ (=curr_indent -> '') [^%nl]*)* (!. / nl_dedent))? ~} + !inline_comment + "#" {~ [^%nl]* (blank_lines (indent -> '') [^%nl]*)* ~} inline_comment (Comment): - "(#" {~ (inline_comment / [^%nl])* ~} "#)" + "#(" {~ (inline_comment / [^%nl])* ~} ")#" -blank_line: %nl eol -nl_nodent: blank_line* %nl nodent -nl_indent: blank_line* %nl {:curr_indent: indent :} -nl_dedent: blank_line* %nl &dedent +nl_nodent: blank_lines nodent +nl_indent: blank_lines {:curr_indent: indent :} -chunk: !chunk_delimeter (block/action/expression) +chunk: block / ((comment nl_nodent)* (action / expression)? (nl_nodent comment)*) chunk_delimeter: ("~")^+3 inline_block (Block): - (inline_comment / inline_statement) (%ws* ";" %ws* (inline_comment / inline_statement))+ + inline_statement (%ws* ";" %ws* inline_statement)+ (%ws* inline_comment)* block (Block): - block_line (nl_nodent block_line)+ + block_line (nl_nodent block_line)+ (nl_nodent comment)* block_line: - comment / inline_comment / statement / (!! [^%nl]* -> "Unexpected character while parsing block line" !!) + statement %ws* (!! [^%nl]+ -> "Unexpected character while parsing block line" !!)? -statement: (action / expression) (eol / (!! [^%nl]* -> "Unexpected character while parsing line" !!)) -inline_statement: inline_action / inline_expression +statement: (comment nl_nodent)* (action / expression) %ws* (!! [^%nl]+ -> "Unexpected character while parsing line" !!)? +inline_statement: (inline_comment %ws*)* (inline_action / inline_expression) noindex_inline_expression: number / variable / inline_text / inline_list / inline_dict / inline_nomsu @@ -40,29 +39,32 @@ noindex_inline_expression: %ws* (inline_block / inline_action / inline_expression) %ws* (%ws* ',' %ws* (inline_block / inline_action / inline_expression) %ws*)* (")" - / (!! eol -> 'Line ended without finding a closing )-parenthesis' !!) + / (!! %ws* eol -> 'Line ended without finding a closing )-parenthesis' !!) / (!! [^%nl]+ -> 'Unexpected character while parsing subexpression' !!) ) ) inline_expression: index_chain / noindex_inline_expression indented_expression: - indented_text / indented_nomsu / indented_list / indented_dict - / (("(..)" / ":")? nl_indent - (block / action / expression) - (!. / &nl_dedent / (!! (!nl_dedent .)* -> "Unexpected character while parsing indented expression" !!)) - ) + indented_text / indented_nomsu / indented_list / indented_dict / {| + ("(..)" / ":")? nl_indent (comment nl_nodent)* + (block / action / expression) + (!! [^%nl]+ -> "Unexpected character while parsing indented expression" !!)? + {:is_halfblock: ''->'yep' :} + |} expression: inline_expression - / (":" %ws* ((inline_block / inline_action / inline_expression) eol - / (!! eol -> "Missing expression after the ':'" !!))) + / (":" %ws* + (!! eol -> "Missing expression after the ':'" !!)? + (inline_block / inline_action / inline_expression) %ws* eol) / indented_expression inline_nomsu (EscapedNomsu): "\" inline_expression indented_nomsu (EscapedNomsu): "\" ( noindex_inline_expression - / (":" %ws* ((inline_block / inline_action / inline_expression) eol - / (!! eol -> "Missing expression after the ':'" !!))) + / (":" %ws* + (!! eol -> "Missing expression after the '\:'" !!)? + (inline_block / inline_action / inline_expression) %ws* eol) / indented_expression) index_chain (IndexChain): @@ -70,26 +72,30 @@ index_chain (IndexChain): -- Actions need either at least 1 word, or at least 2 tokens inline_action (Action): - ( (inline_expression (%ws* (inline_expression / word))+) - / (word (%ws* (inline_expression / word))*)) - (%ws* ":" %ws* (inline_block / inline_action / inline_expression + !chunk_delimeter + (inline_comment %ws*)* + ( (inline_expression ((%ws* inline_comment)* %ws* (inline_expression / word))+) + / (word ((%ws* inline_comment)* %ws* (inline_expression / word))*)) + ((%ws* inline_comment)* %ws* ":" %ws* (inline_block / inline_action / inline_expression / (!! '' -> "Missing expression after the ':'" !!)))? action (Action): - (expression ((nl_nodent "..")? %ws* (expression / word))+) - / (word ((nl_nodent "..")? %ws* (expression / word))*) + !chunk_delimeter + (inline_comment %ws*)* + ( (expression (((nl_nodent comment)* nl_nodent "..")? %ws* (inline_comment %ws*)* (expression / word))+) + / (word (((nl_nodent comment)* nl_nodent "..")? %ws* (inline_comment %ws*)* (expression / word))*)) -word: !number ( %operator_char+ / %ident_char+ ) +word: !number { %operator_char+ / %ident_char+ } text_word (Text): word inline_text (Text): - !('".."' eol) + !('".."' %ws* eol) '"' ({~ (('\"' -> '"') / ('\\' -> '\') / %escaped_char / [^%nl\"])+ ~} / inline_text_interpolation)* ('"' / (!! eol -> 'Line ended before finding a closing double quotation mark' !!) - / (!! [^%nl]* -> 'Unexpected character while parsing Text' !!)) + / (!! [^%nl]+ -> 'Unexpected character while parsing Text' !!)) inline_text_interpolation: "\" ( variable / inline_list / inline_dict / inline_text @@ -97,21 +103,19 @@ inline_text_interpolation: %ws* (inline_block / inline_action / inline_expression) %ws* (%ws* ',' %ws* (inline_block / inline_action / inline_expression) %ws*)* (")" - / (!! &%nl -> 'Line ended without finding a closing )-parenthesis' !!) - / (!! [^%nl]* -> 'Unexpected character while parsing Text interpolation' !!))) + / (!! %ws* eol -> 'Line ended without finding a closing )-parenthesis' !!) + / (!! [^%nl]+ -> 'Unexpected character while parsing Text interpolation' !!))) ) --- Have to use "%indent" instead of "indent" etc. to avoid messing up text lines that start with "#" indented_text (Text): - '".."' eol %nl - {~ (%nl*) ({:curr_indent: indent :} -> "") ~} - (indented_plain_text / text_interpolation / {~ %nl+ (nodent -> "") ~})* - (!. / &nl_dedent / (!! (!nl_dedent .)* -> "Unexpected character while parsing Text" !!)) + '".."' %ws* %nl {:curr_indent: indent :} + (indented_plain_text / text_interpolation / {~ blank_lines (=curr_indent -> "") ~})* + (!! [^%nl]+ -> "Unexpected character while parsing Text" !!)? indented_plain_text (Text): - {~ (("\\" -> "\") / (("\" nl_nodent "..") -> "") / (!text_interpolation "\") / [^%nl\]+)+ - (%nl+ (nodent -> ""))* ~} + {~ (("\\" -> "\") / (("\" (blank_lines =curr_indent comment)* blank_lines =curr_indent "..") -> "") / (!text_interpolation "\") / [^%nl\]+)+ + (blank_lines (=curr_indent -> ""))* ~} text_interpolation: - inline_text_interpolation / ("\" indented_expression nl_nodent "..") + inline_text_interpolation / ("\" indented_expression blank_lines =curr_indent "..") number (Number): (("-"? (([0-9]+ "." [0-9]+) / ("." [0-9]+) / ([0-9]+)))-> tonumber) @@ -124,15 +128,15 @@ inline_list (List): "[" %ws* (inline_list_item (%ws* ',' %ws* inline_list_item)* (%ws* ',')?)? %ws* ("]" / (","? ( - (!! eol -> "Line ended before finding a closing ]-bracket" !!) - /(!! [^%nl]* -> "Unexpected character while parsing List" !!) + (!! %ws* eol -> "Line ended before finding a closing ]-bracket" !!) + /(!! [^%nl]+ -> "Unexpected character while parsing List" !!)? ))) indented_list (List): - "[..]" nl_indent + "[..]" nl_indent (comment nl_nodent)* list_line (nl_nodent list_line)* - (&nl_dedent / (","? (!! (!nl_dedent .)* -> "Unexpected character while parsing List" !!))) + (","? (!! [^%nl]+ -> "Unexpected character while parsing List" !!))? list_line: - (inline_list_item %ws* "," %ws*)+ eol + (inline_list_item %ws* "," %ws*)+ %ws* eol / (inline_list_item %ws* "," %ws*)* (action / expression) inline_list_item: inline_block / inline_action / inline_expression @@ -145,11 +149,11 @@ inline_dict (Dict): / (!! [^%nl]* -> "Unexpected character while parsing Dictionary" !!) ))) indented_dict (Dict): - "{..}" nl_indent - dict_line (nl_nodent dict_line)* - (&nl_dedent / (","? (!! (!nl_dedent .)* -> "Unexpected character while parsing Dictionary" !!))) + "{..}" nl_indent (comment nl_nodent)* + dict_line ((nl_nodent comment)* nl_nodent dict_line)* (nl_nodent comment)* + (","? (!! [^%nl]+ -> "Unexpected character while parsing Dictionary" !!))? dict_line: - (inline_dict_entry %ws* "," %ws*)+ eol + (inline_dict_entry %ws* "," %ws*)+ %ws* eol / (inline_dict_entry %ws* "," %ws*)* dict_entry dict_entry(DictEntry): dict_key (%ws* ":" %ws* (action / expression))? diff --git a/nomsu_compiler.lua b/nomsu_compiler.lua index c3e78ef..0074d29 100644 --- a/nomsu_compiler.lua +++ b/nomsu_compiler.lua @@ -367,7 +367,7 @@ do if tree == nil then return nil end - if tree.type == "FileChunks" then + if tree.type == "File" then local ret = nil local all_lua = { } for _index_0 = 1, #tree do @@ -708,8 +708,8 @@ do return LuaCode.Value(tree.source, tostring(tree[1])) elseif "Var" == _exp_0 then return LuaCode.Value(tree.source, string.as_lua_id(tree[1])) - elseif "FileChunks" == _exp_0 then - return error("Cannot convert FileChunks to a single block of lua, since each chunk's " .. "compilation depends on the earlier chunks") + elseif "File" == _exp_0 then + return error("Cannot convert File to a single block of lua, since each chunk's " .. "compilation depends on the earlier chunks") else return error("Unknown type: " .. tostring(tree.type)) end @@ -757,7 +757,7 @@ do end local inline, can_use_colon = options.inline, options.can_use_colon local _exp_0 = tree.type - if "FileChunks" == _exp_0 then + if "File" == _exp_0 then if inline then return nil end diff --git a/nomsu_compiler.moon b/nomsu_compiler.moon index cd35fa4..90ab354 100644 --- a/nomsu_compiler.moon +++ b/nomsu_compiler.moon @@ -235,7 +235,7 @@ with NomsuCompiler tree = if AST.is_syntax_tree(to_run) then to_run else @parse(to_run, source) if tree == nil -- Happens if pattern matches, but there are no captures, e.g. an empty string return nil - if tree.type == "FileChunks" + if tree.type == "File" -- Each chunk's compilation is affected by the code in the previous chunks -- (typically), so each chunk needs to compile and run before the next one -- compiles. @@ -473,8 +473,8 @@ with NomsuCompiler when "Var" return LuaCode.Value(tree.source, string.as_lua_id(tree[1])) - when "FileChunks" - error("Cannot convert FileChunks to a single block of lua, since each chunk's ".. + when "File" + error("Cannot convert File to a single block of lua, since each chunk's ".. "compilation depends on the earlier chunks") else @@ -501,7 +501,7 @@ with NomsuCompiler inline, can_use_colon = options.inline, options.can_use_colon switch tree.type - when "FileChunks" + when "File" return nil if inline nomsu = NomsuCode(tree.source) for i, chunk in ipairs tree diff --git a/nomsu_tree.lua b/nomsu_tree.lua index 227a674..70ee4ee 100644 --- a/nomsu_tree.lua +++ b/nomsu_tree.lua @@ -26,7 +26,8 @@ local types = { "DictEntry", "IndexChain", "Action", - "FileChunks" + "File", + "Comment" } for _index_0 = 1, #types do local name = types[_index_0] diff --git a/nomsu_tree.moon b/nomsu_tree.moon index b754cd8..9f5a3fb 100644 --- a/nomsu_tree.moon +++ b/nomsu_tree.moon @@ -10,7 +10,7 @@ AST.is_syntax_tree = (n, t=nil)-> type(n) == 'table' and getmetatable(n) and AST[n.type] == getmetatable(n) and (t == nil or n.type == t) types = {"Number", "Var", "Block", "EscapedNomsu", "Text", "List", "Dict", "DictEntry", - "IndexChain", "Action", "FileChunks"} + "IndexChain", "Action", "File", "Comment"} for name in *types cls = {} with cls diff --git a/parser.lua b/parser.lua index c21e95d..5f766b9 100644 --- a/parser.lua +++ b/parser.lua @@ -91,19 +91,54 @@ setmetatable(NOMSU_DEFS, { value.source = Source(_with_0.filename, _with_0.start + start - 1, _with_0.start + stop - 1) end end - if key == "Comment" then - value = value[1] - else - local comments = { } + while true do + local found = false for i = #value, 1, -1 do - if type(value[i]) == 'table' and value[i].type == "Comment" then - insert(comments, remove(value, i)) + local _continue_0 = false + repeat + if not (type(value[i]) == 'table') then + _continue_0 = true + break + end + if value[i].is_halfblock then + found = true + local hb = remove(value, i) + for _index_0 = 1, #hb do + local v = hb[_index_0] + insert(value, i, v) + i = i + 1 + end + end + _continue_0 = true + until true + if not _continue_0 then + break end end - if #comments > 0 then - value.comments = comments + if not (found) then + break end end + local comments = { } + for i = #value, 1, -1 do + local _continue_0 = false + repeat + if not (type(value[i]) == 'table') then + _continue_0 = true + break + end + if value[i].type == "Comment" then + insert(comments, remove(value, i)) + end + _continue_0 = true + until true + if not _continue_0 then + break + end + end + if #comments > 0 then + value.comments = comments + end setmetatable(value, AST[key]) if value.__init then value:__init() diff --git a/parser.moon b/parser.moon index 7786650..5e37a59 100644 --- a/parser.moon +++ b/parser.moon @@ -58,15 +58,24 @@ setmetatable(NOMSU_DEFS, {__index:(key)=> if userdata.source with userdata.source value.source = Source(.filename, .start + start-1, .start + stop-1) - if key == "Comment" - value = value[1] - else - comments = {} + while true + found = false for i=#value,1,-1 - if type(value[i]) == 'table' and value[i].type == "Comment" - insert comments, remove(value, i) - if #comments > 0 - value.comments = comments + continue unless type(value[i]) == 'table' + if value[i].is_halfblock + found = true + hb = remove(value, i) + for v in *hb + insert value, i, v + i += 1 + break unless found + comments = {} + for i=#value,1,-1 + continue unless type(value[i]) == 'table' + if value[i].type == "Comment" + insert comments, remove(value, i) + if #comments > 0 + value.comments = comments setmetatable(value, AST[key]) if value.__init then value\__init! return value diff --git a/tests/text.nom b/tests/text.nom index 39ee496..e42fe73 100644 --- a/tests/text.nom +++ b/tests/text.nom @@ -45,9 +45,9 @@ assume assume ".." - no\ # Comment + no\ #comment - #.. + # block comment ..gap ..= "nogap"