Cleanups and codegen improvements/fixes.

4 files changed, 99 insertions, 82 deletions

diff --git a/core/control_flow.nom b/core/control_flow.nom
index ccf11de..eca92ce 100644
--- a/core/control_flow.nom
+++ b/core/control_flow.nom
@@ -174,13 +174,14 @@ test:
     if (%body has subtree \(do next %var)):
         %lua::add "\n    " (\(---next %var ---) as lua)
     
-    %lua::add "\nend --numeric for-loop"
+    %lua::add "\nend -- numeric for " (%var as lua identifier) " loop"
     if (%body has subtree \(stop %var)):
-        %inner_lua = %lua
-        %lua = (Lua "do -- scope for stopping for-loop\n    ")
-        %lua::add %inner_lua "\n    "
-        %lua::add (\(---stop %var ---) as lua)
-        %lua::add "\nend -- end of scope for stopping for-loop"
+        %lua = (..)
+            Lua "\
+                ..do -- scope for (stop \(%var as lua identifier))
+                    \%lua
+                    \(\(---stop %var ---) as lua)
+                end -- scope for (stop \(%var as lua identifier))"
     return %lua
 
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/nomsu.4.peg b/nomsu.4.peg
index d1840f9..95e3ef0 100644
--- a/nomsu.4.peg
+++ b/nomsu.4.peg
@@ -158,7 +158,7 @@ indented_text (Text):
     _inline_text* 
     (('\' %nl+ {:curr_indent: indent :} ('..')?)
      / disallowed_interpolation? {%nl+} {:curr_indent: indent :})
-    (indented_plain_text / text_interpolation / illegal_char / {~ %nl+ (=curr_indent -> "") ~})*
+    (indented_plain_text / text_interpolation / illegal_char / {~ %nl+ (=curr_indent -> "") [ ]* ~})*
     ('"' eol / eof / missing_quote_err)
     {:curr_indent: %nil :}
 -- Tracking text-lines-within-indented-text as separate objects allows for better debugging line info
@@ -167,7 +167,6 @@ indented_plain_text (Text):
         ((("\" blank_lines =curr_indent "..") -> "") / ('\\' -> '\')
         / (!text_interpolation (escaped_char / '\'))
         / (nonterminal_quote / text_char)+)+
-        (%nl+ (=curr_indent -> ""))*
     ~}
 
 text_interpolation:
diff --git a/nomsu_compiler.lua b/nomsu_compiler.lua
index 16f8187..29c1668 100644
--- a/nomsu_compiler.lua
+++ b/nomsu_compiler.lua
@@ -107,7 +107,7 @@ local compile = setmetatable({
           elseif bit.type == "Text" then
             bit_lua = operate_on_text(bit)
           elseif bit.type == "Block" then
-            bit_lua = LuaCode:from(bit.source, "(function()", "\n    local _lua = LuaCode:from(", tostring(bit.source):as_lua(), ")", "\n    local function add(bit) _lua:add(bit) end", "\n    local function join_with(glue)", "\n        local old_bits = _lua.bits", "\n        _lua = LuaCode:from(_lua.source)", "\n        _lua:concat_add(old_bits, glue)", "\n    end", "\n    ", compile(bit), "\n    return _lua", "\nend)()")
+            bit_lua = LuaCode:from(bit.source, "(function()", "\n    local _lua = LuaCode:from(", tostring(bit.source):as_lua(), ")", "\n    local function add(...) _lua:add(...) end", "\n    local function join_with(glue)", "\n        local old_bits = _lua.bits", "\n        _lua = LuaCode:from(_lua.source)", "\n        _lua:concat_add(old_bits, glue)", "\n    end", "\n    ", compile(bit), "\n    return _lua", "\nend)()")
           else
             bit_lua = compile(bit)
           end
@@ -164,7 +164,12 @@ local compile = setmetatable({
           test_nomsu = test_nomsu:gsub("\n" .. indent, "\n")
         end
       end
-      return LuaCode("TESTS[" .. tostring(tostring(body.source):as_lua()) .. "] = ", test_nomsu:as_lua())
+      local test_text = compile(SyntaxTree({
+        type = "Text",
+        source = body.source,
+        test_nomsu
+      }))
+      return LuaCode("TESTS[" .. tostring(tostring(body.source):as_lua()) .. "] = ", test_text)
     end,
     ["is jit"] = function(compile, code)
       return LuaCode("jit")
@@ -240,7 +245,7 @@ local compile = setmetatable({
         end
       end
       lua:add((stub):as_lua_id(), "(")
-      local args = { }
+      local arg_count = 0
       for i, tok in ipairs(tree) do
         local _continue_0 = false
         repeat
@@ -248,18 +253,22 @@ local compile = setmetatable({
             _continue_0 = true
             break
           end
+          arg_count = arg_count + 1
           local arg_lua = compile(tok)
           if tok.type == "Block" then
             arg_lua = LuaCode:from(tok.source, "(function()\n    ", arg_lua, "\nend)()")
           end
-          insert(args, arg_lua)
+          if arg_count > 1 then
+            lua:add(",")
+          end
+          lua:add(lua:trailing_line_len() + #arg_lua:text() > MAX_LINE and "\n   " or " ")
+          lua:add(arg_lua)
           _continue_0 = true
         until true
         if not _continue_0 then
           break
         end
       end
-      lua:concat_add(args, ", ")
       lua:add(")")
       return lua
     elseif "EscapedNomsu" == _exp_0 then
@@ -271,28 +280,32 @@ local compile = setmetatable({
           return tostring(x)
         elseif SyntaxTree:is_instance(x) then
           return compile(x)
+        elseif Source:is_instance(x) then
+          return tostring(x):as_lua()
         else
           return x:as_lua()
         end
       end
       for k, v in pairs((SyntaxTree:is_instance(tree[1]) and tree[1].type == "EscapedNomsu" and tree) or tree[1]) do
-        if needs_comma then
-          lua:add(", ")
-        else
-          needs_comma = true
-        end
+        local entry_lua = LuaCode()
         if k == i then
           i = i + 1
         elseif type(k) == 'string' and match(k, "[_a-zA-Z][_a-zA-Z0-9]*") then
-          lua:add(k, "= ")
+          entry_lua:add(k, "= ")
         else
-          lua:add("[", as_lua(k), "]= ")
+          entry_lua:add("[", as_lua(k), "]= ")
         end
-        if k == "source" then
-          lua:add(tostring(v):as_lua())
-        else
-          lua:add(as_lua(v))
+        entry_lua:add(as_lua(v))
+        if needs_comma then
+          lua:add(",")
         end
+        if lua:trailing_line_len() + #(entry_lua:text():match("^[\n]*")) > MAX_LINE then
+          lua:add("\n")
+        elseif needs_comma then
+          lua:add(" ")
+        end
+        lua:add(entry_lua)
+        needs_comma = true
       end
       lua:add("}")
       return lua
@@ -309,6 +322,17 @@ local compile = setmetatable({
       local lua = LuaCode:from(tree.source)
       local added = 0
       local string_buffer = ""
+      local add_bit
+      add_bit = function(bit)
+        if added > 0 then
+          if lua:trailing_line_len() + #bit > MAX_LINE then
+            lua:add("\n  ")
+          end
+          lua:add("..")
+        end
+        lua:add(bit)
+        added = added + 1
+      end
       for i, bit in ipairs(tree) do
         local _continue_0 = false
         repeat
@@ -318,50 +342,35 @@ local compile = setmetatable({
             break
           end
           if string_buffer ~= "" then
-            string_buffer = string_buffer:as_lua()
-            if lua:trailing_line_len() + #string_buffer > MAX_LINE then
-              lua:add("\n  ")
-            end
-            if added > 0 then
-              lua:add("..")
+            for i = 1, #string_buffer, MAX_LINE do
+              add_bit(string_buffer:sub(i, i + MAX_LINE - 1):as_lua())
             end
-            lua:add(string_buffer)
-            added = added + 1
             string_buffer = ""
           end
           local bit_lua = compile(bit)
           if bit.type == "Block" then
             bit_lua = LuaCode:from(bit.source, "(function()", "\n    local _buffer = List{}", "\n    local function add(bit) _buffer:add(bit) end", "\n    local function join_with(glue) _buffer = _buffer:joined_with(glue) end", "\n    ", bit_lua, "\n    if lua_type_of(_buffer) == 'table' then _buffer = _buffer:joined() end", "\n    return _buffer", "\nend)()")
           end
-          if lua:trailing_line_len() + #bit_lua:text() > MAX_LINE then
-            lua:add("\n  ")
-          end
-          if added > 0 then
-            lua:add("..")
-          end
           if bit.type ~= "Text" then
             bit_lua = LuaCode:from(bit.source, "tostring(", bit_lua, ")")
           end
-          lua:add(bit_lua)
-          added = added + 1
+          add_bit(bit_lua)
           _continue_0 = true
         until true
         if not _continue_0 then
           break
         end
       end
-      if string_buffer ~= "" or #lua.bits == 0 then
-        string_buffer = string_buffer:as_lua()
-        if lua:trailing_line_len() + #string_buffer > MAX_LINE then
-          lua:add("\n  ")
-        end
-        if added > 0 then
-          lua:add("..")
+      if string_buffer ~= "" then
+        for i = 1, #string_buffer, MAX_LINE do
+          add_bit(string_buffer:sub(i, i + MAX_LINE - 1):as_lua())
         end
-        lua:add(string_buffer)
-        added = added + 1
+        string_buffer = ""
+      end
+      if added == 0 then
+        add_bit('""')
       end
-      if #lua.bits > 1 then
+      if added > 1 then
         lua:parenthesize()
       end
       return lua
diff --git a/nomsu_compiler.moon b/nomsu_compiler.moon
index ae2b8f7..1a1c4f9 100644
--- a/nomsu_compiler.moon
+++ b/nomsu_compiler.moon
@@ -80,7 +80,7 @@ compile = setmetatable({
                     elseif bit.type == "Block"
                         bit_lua = LuaCode\from bit.source, "(function()",
                             "\n    local _lua = LuaCode:from(", tostring(bit.source)\as_lua!, ")",
-                            "\n    local function add(bit) _lua:add(bit) end",
+                            "\n    local function add(...) _lua:add(...) end",
                             "\n    local function join_with(glue)",
                             "\n        local old_bits = _lua.bits",
                             "\n        _lua = LuaCode:from(_lua.source)",
@@ -129,7 +129,8 @@ compile = setmetatable({
             test_nomsu = body\get_source_code!\match(":[ ]*(.*)")
             if indent = test_nomsu\match("\n([ ]*)")
                 test_nomsu = test_nomsu\gsub("\n"..indent, "\n")
-            return LuaCode "TESTS[#{tostring(body.source)\as_lua!}] = ", test_nomsu\as_lua!
+            test_text = compile(SyntaxTree{type:"Text", source:body.source, test_nomsu})
+            return LuaCode "TESTS[#{tostring(body.source)\as_lua!}] = ", test_text
 
         ["is jit"]: (compile, code)-> LuaCode("jit")
         ["Lua version"]: (compile, code)-> LuaCode("_VERSION")
@@ -182,14 +183,17 @@ compile = setmetatable({
                     else
                         lua\add "(", target_lua, "):"
                 lua\add((stub)\as_lua_id!,"(")
-                args = {}
+                arg_count = 0
                 for i, tok in ipairs tree
                     if type(tok) == "string" then continue
+                    arg_count += 1
                     arg_lua = compile(tok)
                     if tok.type == "Block"
                         arg_lua = LuaCode\from(tok.source, "(function()\n    ", arg_lua, "\nend)()")
-                    insert args, arg_lua
-                lua\concat_add args, ", "
+                    if arg_count > 1
+                        lua\add ","
+                    lua\add(lua\trailing_line_len! + #arg_lua\text! > MAX_LINE and "\n   " or " ")
+                    lua\add arg_lua
                 lua\add ")"
                 return lua
 
@@ -201,21 +205,26 @@ compile = setmetatable({
                         tostring(x)
                     elseif SyntaxTree\is_instance(x)
                         compile(x)
+                    elseif Source\is_instance(x)
+                        tostring(x)\as_lua!
                     else x\as_lua!
 
                 for k,v in pairs((SyntaxTree\is_instance(tree[1]) and tree[1].type == "EscapedNomsu" and tree) or tree[1])
-                    if needs_comma then lua\add ", "
-                    else needs_comma = true
+                    entry_lua = LuaCode!
                     if k == i
                         i += 1
                     elseif type(k) == 'string' and match(k,"[_a-zA-Z][_a-zA-Z0-9]*")
-                        lua\add(k, "= ")
+                        entry_lua\add(k, "= ")
                     else
-                        lua\add("[", as_lua(k), "]= ")
-                    if k == "source"
-                        lua\add tostring(v)\as_lua!
-                    else
-                        lua\add as_lua(v)
+                        entry_lua\add("[", as_lua(k), "]= ")
+                    entry_lua\add as_lua(v)
+                    if needs_comma then lua\add ","
+                    if lua\trailing_line_len! + #(entry_lua\text!\match("^[\n]*")) > MAX_LINE
+                        lua\add "\n"
+                    elseif needs_comma
+                        lua\add " "
+                    lua\add entry_lua
+                    needs_comma = true
                 lua\add "}"
                 return lua
             
@@ -230,17 +239,21 @@ compile = setmetatable({
                 lua = LuaCode\from(tree.source)
                 added = 0
                 string_buffer = ""
+                add_bit = (bit)->
+                    if added > 0
+                        if lua\trailing_line_len! + #bit > MAX_LINE
+                            lua\add "\n  "
+                        lua\add ".."
+                    lua\add bit
+                    added += 1
+
                 for i, bit in ipairs tree
                     if type(bit) == "string"
                         string_buffer ..= bit
                         continue
                     if string_buffer != ""
-                        string_buffer = string_buffer\as_lua!
-                        if lua\trailing_line_len! + #string_buffer > MAX_LINE
-                            lua\add "\n  "
-                        if added > 0 then lua\add ".."
-                        lua\add string_buffer
-                        added += 1
+                        for i=1,#string_buffer,MAX_LINE
+                            add_bit string_buffer\sub(i, i+MAX_LINE-1)\as_lua!
                         string_buffer = ""
 
                     bit_lua = compile(bit)
@@ -253,23 +266,18 @@ compile = setmetatable({
                             "\n    if lua_type_of(_buffer) == 'table' then _buffer = _buffer:joined() end",
                             "\n    return _buffer",
                             "\nend)()"
-                    if lua\trailing_line_len! + #bit_lua\text! > MAX_LINE
-                        lua\add "\n  "
-                    if added > 0 then lua\add ".."
                     if bit.type != "Text"
                         bit_lua = LuaCode\from(bit.source, "tostring(",bit_lua,")")
-                    lua\add bit_lua
-                    added += 1
+                    add_bit bit_lua
 
-                if string_buffer ~= "" or #lua.bits == 0
-                    string_buffer = string_buffer\as_lua!
-                    if lua\trailing_line_len! + #string_buffer > MAX_LINE
-                        lua\add "\n  "
-                    if added > 0 then lua\add ".."
-                    lua\add string_buffer
-                    added += 1
+                if string_buffer != ""
+                    for i=1,#string_buffer,MAX_LINE
+                        add_bit string_buffer\sub(i, i+MAX_LINE-1)\as_lua!
+                    string_buffer = ""
 
-                if #lua.bits > 1
+                if added == 0
+                    add_bit '""'
+                if added > 1
                     lua\parenthesize!
                 return lua