Improvements to nomsu codegen (comments are now retained) and some

improvements to handling of stdin and fixes for error reporting.

code_obj.lua

@@ -82,8 +82,14 @@ do
       local bits, indents = self.bits, self.indents
       local match = string.match
       for i = 1, n do
+        local _continue_0 = false
+        repeat
           local b = select(i, ...)
           assert(b)
+          if b == '' then
+            _continue_0 = true
+            break
+          end
           bits[#bits + 1] = b
           if type(b) == 'string' then
             do
@@ -95,6 +101,11 @@ do
           elseif self.current_indent ~= 0 then
             indents[#bits] = self.current_indent
           end
+          _continue_0 = true
+        until true
+        if not _continue_0 then
+          break
+        end
       end
       self.__str = nil
     end,

code_obj.moon

@@ -58,6 +58,7 @@ class Code
         for i=1,n
             b = select(i, ...)
             assert(b)
+            if b == '' then continue
             bits[#bits+1] = b
             if type(b) == 'string'
                 if spaces = match(b, "\n([ ]*)[^\n]*$")

files.lua

@@ -12,6 +12,11 @@ files.read = function(filename)
       return file_contents
     end
   end
+  if filename == 'stdin' then
+    local contents = io.read('*a')
+    _FILE_CACHE['stdin'] = contents
+    return contents
+  end
   local file = io.open(filename)
   if package.nomsupath and not file then
     for nomsupath in package.nomsupath:gmatch("[^;]+") do
@@ -52,6 +57,9 @@ end
 local ok, lfs = pcall(require, "lfs")
 if ok then
   files.walk = function(path)
+    if match(path, "%.nom$") or match(path, "%.lua$") or path == 'stdin' then
+      return iterate_single, path
+    end
     local browse
     browse = function(filename)
       local file_type = lfs.attributes(filename, 'mode')
@@ -89,7 +97,7 @@ else
     error("Could not find 'luafilesystem' module and couldn't run system command `find` (this might happen on Windows). Please install `luafilesystem` (which can be found at: http://keplerproject.github.io/luafilesystem/ or `luarocks install luafilesystem`)", 0)
   end
   files.walk = function(path)
-    if match(path, "%.nom$") or match(path, "%.lua$") or match(path, "^/dev/fd/[012]$") then
+    if match(path, "%.nom$") or match(path, "%.lua$") or path == 'stdin' then
       return iterate_single, path
     end
     path = gsub(path, "\\", "\\\\")
@@ -163,7 +171,7 @@ files.get_line_number = function(str, pos)
 end
 files.get_line = function(str, line_no)
   local line_starts = files.get_line_starts(str)
-  return str:sub(line_starts[line_no] or 1, line_starts[line_no + 1] or -1)
+  return str:sub(line_starts[line_no] or 1, (line_starts[line_no + 1] or 1) - 2)
 end
 files.get_lines = function(str)
   return get_lines:match(str)

files.moon

@@ -13,6 +13,10 @@ files.spoof = (filename, contents)->
 files.read = (filename)->
     if file_contents = _FILE_CACHE[filename]
         return file_contents
+    if filename == 'stdin'
+        contents = io.read('*a')
+        _FILE_CACHE['stdin'] = contents
+        return contents
     file = io.open(filename)
     if package.nomsupath and not file
         for nomsupath in package.nomsupath\gmatch("[^;]+")
@@ -32,6 +36,8 @@ iterate_single = (item, prev) -> if item == prev then nil else item
 ok, lfs = pcall(require, "lfs")
 if ok
     files.walk = (path)->
+        if match(path, "%.nom$") or match(path, "%.lua$") or path == 'stdin'
+            return iterate_single, path
         -- Return 'true' if any files or directories are found, otherwise 'false'
         browse = (filename)->
             file_type = lfs.attributes(filename, 'mode')
@@ -59,9 +65,9 @@ else
         error "Could not find 'luafilesystem' module and couldn't run system command `find` (this might happen on Windows). Please install `luafilesystem` (which can be found at: http://keplerproject.github.io/luafilesystem/ or `luarocks install luafilesystem`)", 0
 
     files.walk = (path)->
-        -- Sanitize path
+        if match(path, "%.nom$") or match(path, "%.lua$") or path == 'stdin'
-        if match(path, "%.nom$") or match(path, "%.lua$") or match(path, "^/dev/fd/[012]$")
             return iterate_single, path
+        -- Sanitize path
         -- TODO: improve sanitization
         path = gsub(path,"\\","\\\\")
         path = gsub(path,"`","")
@@ -119,7 +125,7 @@ files.get_line_number = (str, pos)->
 
 files.get_line = (str, line_no)->
     line_starts = files.get_line_starts(str)
-    return str\sub(line_starts[line_no] or 1, line_starts[line_no+1] or -1)
+    return str\sub(line_starts[line_no] or 1, (line_starts[line_no+1] or 1) - 2)
 
 files.get_lines = (str)-> get_lines\match(str)
 

nomsu.lua

@@ -85,7 +85,6 @@ do
 end
 local repr
 repr = require("utils").repr
-local STDIN, STDOUT, STDERR = "/dev/fd/0", "/dev/fd/1", "/dev/fd/2"
 if not arg or debug.getinfo(2).func == require then
   return NomsuCompiler
 end
@@ -137,7 +136,7 @@ local run
 run = function()
   for i, input in ipairs(args.inputs) do
     if input == "-" then
-      args.inputs[i] = STDIN
+      args.inputs[i] = 'stdin'
     end
   end
   if #args.inputs == 0 and not args.interactive then
@@ -173,7 +172,15 @@ run = function()
   local to_run = { }
   local _list_0 = args.inputs
   for _index_0 = 1, #_list_0 do
+    local _continue_0 = false
+    repeat
       local input = _list_0[_index_0]
+      if input == 'stdin' then
+        input_files[#input_files + 1] = 'stdin'
+        to_run['stdin'] = true
+        _continue_0 = true
+        break
+      end
       local found = false
       for f in files.walk(input) do
         input_files[#input_files + 1] = f
@@ -183,6 +190,11 @@ run = function()
       if not found then
         error("Could not find: " .. tostring(input))
       end
+      _continue_0 = true
+    until true
+    if not _continue_0 then
+      break
+    end
   end
   nomsu.can_optimize = function(f)
     if not (args.optimized) then
@@ -201,8 +213,8 @@ run = function()
       local _continue_0 = false
       repeat
         local file, source
-        if filename == STDIN then
+        if filename == 'stdin' then
-          file = io.input():read("*a")
+          file = io.read("*a")
           files.spoof('stdin', file)
           source = Source('stdin', 1, #file)
         elseif filename:match("%.nom$") then

nomsu.moon

@@ -38,7 +38,6 @@ Errhand = require "error_handling"
 NomsuCompiler = require "nomsu_compiler"
 {:NomsuCode, :LuaCode, :Source} = require "code_obj"
 {:repr} = require "utils"
-STDIN, STDOUT, STDERR = "/dev/fd/0", "/dev/fd/1", "/dev/fd/2"
 
 -- If this file was reached via require(), then just return the Nomsu compiler
 if not arg or debug.getinfo(2).func == require
@@ -89,7 +88,7 @@ FILE_CACHE = setmetatable {}, {
 
 run = ->
     for i,input in ipairs args.inputs
-        if input == "-" then args.inputs[i] = STDIN
+        if input == "-" then args.inputs[i] = 'stdin'
 
     if #args.inputs == 0 and not args.interactive
         args.inputs = {"core"}
@@ -114,6 +113,10 @@ run = ->
     input_files = {}
     to_run = {}
     for input in *args.inputs
+        if input == 'stdin'
+            input_files[#input_files+1] = 'stdin'
+            to_run['stdin'] = true
+            continue
         found = false
         for f in files.walk(input)
             input_files[#input_files+1] = f
@@ -131,8 +134,8 @@ run = ->
     for arg in *args.inputs
         for filename in files.walk(arg)
             local file, source
-            if filename == STDIN
+            if filename == 'stdin'
-                file = io.input!\read("*a")
+                file = io.read("*a")
                 files.spoof('stdin', file)
                 source = Source('stdin', 1, #file)
             elseif filename\match("%.nom$")

nomsu.peg

@@ -149,8 +149,8 @@ inline_dict_entry(DictEntry):
 dict_key:
     text_word / inline_expression
 
-comment: "#" [^%nl]* (%nl+ %indent [^%nl]* (%nl+ %nodent [^%nl]*)* %dedent)?
+comment: ("#" {} {~[^%nl]* (%nl+ (%indent -> '') [^%nl]* (%nl+ (%nodent -> '') [^%nl]*)* %dedent)?~} %userdata) => Comment
-eol_comment: "#" [^%nl]*
+eol_comment: ("#" {} {[^%nl]*} %userdata) => Comment
 
 eol: %ws* eol_comment? (!. / &%nl)
 ignored_line: (%nodent comment) / (%ws* (!. / &%nl))

nomsu_compiler.lua

@@ -588,7 +588,7 @@ do
           end
           local bit_lua = self:compile(bit)
           if not (bit_lua.is_value) then
-            local src = '    ' .. gsub(tostring(self:tree_to_nomsu(bit)), '\n', '\n    ')
+            local src = '    ' .. gsub(tostring(recurse(bit)), '\n', '\n    ')
             local line = tostring(bit.source.filename) .. ":" .. tostring(files.get_line_number(files.read(bit.source.filename), bit.source.start))
             self:compile_error(bit, "Cannot use:\n%s\nas a string interpolation value, since it's not an expression.")
           end
@@ -698,29 +698,61 @@ do
       return error("Unknown type: " .. tostring(tree.type))
     end
   end
-  NomsuCompiler.tree_to_nomsu = function(self, tree, inline, can_use_colon)
+  NomsuCompiler.tree_to_nomsu = function(self, tree, options)
-    if inline == nil then
+    options = options or { }
-      inline = false
+    local comments = options.comments
+    if comments == nil and tree.comments then
+      do
+        local _accum_0 = { }
+        local _len_0 = 1
+        for p, c in pairs(tree.comments) do
+          _accum_0[_len_0] = {
+            comment = c,
+            pos = p
+          }
+          _len_0 = _len_0 + 1
         end
-    if can_use_colon == nil then
+        comments = _accum_0
-      can_use_colon = false
       end
+      table.sort(comments, function(a, b)
+        return a.pos > b.pos
+      end)
+    end
+    local recurse
+    recurse = function(t, opts)
+      opts = opts or { }
+      opts.comments = comments
+      return self:tree_to_nomsu(t, opts)
+    end
+    local pop_comments
+    pop_comments = function(pos)
+      if not (comments) then
+        return ''
+      end
+      local nomsu = NomsuCode(tree.source)
+      while #comments > 0 and comments[#comments].pos <= pos do
+        local comment = table.remove(comments)
+        nomsu:append("#" .. (gsub(comment.comment, "\n", "\n    ")) .. "\n")
+      end
+      if #nomsu.bits == 0 then
+        return ''
+      end
+      return nomsu
+    end
+    local inline, can_use_colon = options.inline, options.can_use_colon
     local _exp_0 = tree.type
     if "FileChunks" == _exp_0 then
       if inline then
         return nil
       end
       local nomsu = NomsuCode(tree.source)
-      nomsu:concat_append((function()
+      for i, chunk in ipairs(tree) do
-        local _accum_0 = { }
+        if i > 1 then
-        local _len_0 = 1
+          nomsu:append("\n\n" .. tostring(("~"):rep(80)) .. "\n\n")
-        for _index_0 = 1, #tree do
+        end
-          local c = tree[_index_0]
+        nomsu:append(pop_comments(chunk.source.start))
-          _accum_0[_len_0] = self:tree_to_nomsu(c)
+        nomsu:append(recurse(chunk))
-          _len_0 = _len_0 + 1
       end
-        return _accum_0
-      end)(), "\n" .. tostring(("~"):rep(80)) .. "\n")
       return nomsu
     elseif "Action" == _exp_0 then
       if inline then
@@ -732,7 +764,9 @@ do
             end
             nomsu:append(bit)
           else
-            local arg_nomsu = self:tree_to_nomsu(bit, true)
+            local arg_nomsu = recurse(bit, {
+              inline = true
+            })
             if not (arg_nomsu) then
               return nil
             end
@@ -762,7 +796,9 @@ do
             elseif bit.type == "Block" then
               arg_nomsu = nil
             else
-              arg_nomsu = self:tree_to_nomsu(bit, true)
+              arg_nomsu = recurse(bit, {
+                inline = true
+              })
             end
             if arg_nomsu and line_len + #tostring(arg_nomsu) < MAX_LINE then
               if bit.type == "Action" then
@@ -782,15 +818,17 @@ do
                 next_space = " "
               end
             else
-              arg_nomsu = self:tree_to_nomsu(bit, nil, true)
+              arg_nomsu = recurse(bit, {
-              if not (nomsu) then
+                can_use_colon = true
+              })
+              if not (arg_nomsu) then
                 return nil
               end
               if bit.type ~= "List" and bit.type ~= "Dict" and bit.type ~= "Text" then
                 if i == 1 then
-                  arg_nomsu = NomsuCode(bit.source, "(..)\n    ", arg_nomsu)
+                  arg_nomsu = NomsuCode(bit.source, "(..)\n    ", pop_comments(bit.source.start), arg_nomsu)
                 else
-                  arg_nomsu = NomsuCode(bit.source, "\n    ", arg_nomsu)
+                  arg_nomsu = NomsuCode(bit.source, "\n    ", pop_comments(bit.source.start), arg_nomsu)
                 end
               end
               if last_colon == i - 1 and (bit.type == "Action" or bit.type == "Block") then
@@ -808,10 +846,12 @@ do
         return nomsu
       end
     elseif "EscapedNomsu" == _exp_0 then
-      local nomsu = self:tree_to_nomsu(tree[1], true)
+      local nomsu = recurse(tree[1], {
+        inline = true
+      })
       if nomsu == nil and not inline then
-        nomsu = self:tree_to_nomsu(tree[1])
+        nomsu = recurse(tree[1])
-        return nomsu and NomsuCode(tree.source, "\\:\n    ", nomsu)
+        return nomsu and NomsuCode(tree.source, "\\:\n    ", pop_comments(tree.source.start), nomsu)
       end
       return nomsu and NomsuCode(tree.source, "\\(", nomsu, ")")
     elseif "Block" == _exp_0 then
@@ -821,7 +861,9 @@ do
           if i > 1 then
             nomsu:append("; ")
           end
-          local line_nomsu = self:tree_to_nomsu(line, true)
+          local line_nomsu = recurse(line, {
+            inline = true
+          })
           if not (line_nomsu) then
             return nil
           end
@@ -831,7 +873,10 @@ do
       end
       local nomsu = NomsuCode(tree.source)
       for i, line in ipairs(tree) do
-        line = assert(self:tree_to_nomsu(line, nil, true), "Could not convert line to nomsu")
+        nomsu:append(pop_comments(line.source.start))
+        line = assert(recurse(line, {
+          can_use_colon = true
+        }), "Could not convert line to nomsu")
         nomsu:append(line)
         if i < #tree then
           nomsu:append("\n")
@@ -849,7 +894,9 @@ do
           if type(bit) == 'string' then
             nomsu:append((gsub(gsub(gsub(bit, "\\", "\\\\"), "\n", "\\n"), '"', '\\"')))
           else
-            local interp_nomsu = self:tree_to_nomsu(bit, true)
+            local interp_nomsu = recurse(bit, {
+              inline = true
+            })
             if interp_nomsu then
               if bit.type ~= "Var" and bit.type ~= "List" and bit.type ~= "Dict" and bit.type ~= "Text" then
                 interp_nomsu:parenthesize()
@@ -863,7 +910,9 @@ do
         nomsu:append('"')
         return nomsu
       else
-        local inline_version = self:tree_to_nomsu(tree, true)
+        local inline_version = recurse(tree, {
+          inline = true
+        })
         if inline_version and #inline_version <= MAX_LINE then
           return inline_version
         end
@@ -901,14 +950,16 @@ do
               end
             end
           else
-            local interp_nomsu = self:tree_to_nomsu(bit, true)
+            local interp_nomsu = recurse(bit, {
+              inline = true
+            })
             if interp_nomsu then
               if bit.type ~= "Var" and bit.type ~= "List" and bit.type ~= "Dict" and bit.type ~= "Text" then
                 interp_nomsu:parenthesize()
               end
               nomsu:append("\\", interp_nomsu)
             else
-              interp_nomsu = assert(self:tree_to_nomsu(bit))
+              interp_nomsu = assert(recurse(bit))
               if not (interp_nomsu) then
                 return nil
               end
@@ -925,7 +976,9 @@ do
       if inline then
         local nomsu = NomsuCode(tree.source, "[")
         for i, item in ipairs(tree) do
-          local item_nomsu = self:tree_to_nomsu(item, true)
+          local item_nomsu = recurse(item, {
+            inline = true
+          })
           if not (item_nomsu) then
             return nil
           end
@@ -937,30 +990,47 @@ do
         nomsu:append("]")
         return nomsu
       else
-        local inline_version = self:tree_to_nomsu(tree, true)
+        local inline_version = recurse(tree, {
+          inline = true
+        })
         if inline_version and #inline_version <= MAX_LINE then
           return inline_version
         end
         local nomsu = NomsuCode(tree.source, "[..]")
         local line = NomsuCode(tree.source, "\n    ")
-        for _index_0 = 1, #tree do
+        local line_comments
-          local item = tree[_index_0]
+        if #tree > 0 then
-          local item_nomsu = self:tree_to_nomsu(item, true)
+          line_comments = pop_comments(tree[1].source.start)
-          if item_nomsu and #line + #", " + #item_nomsu <= MAX_LINE then
+        else
+          line_comments = ''
+        end
+        for i, item in ipairs(tree) do
+          local item_nomsu = recurse(item, {
+            inline = true
+          })
+          if item_nomsu and #tostring(line) + #", " + #item_nomsu <= MAX_LINE then
             if #line.bits > 1 then
               line:append(", ")
             end
             line:append(item_nomsu)
           else
             if not (item_nomsu) then
-              item_nomsu = self:tree_to_nomsu(item)
+              item_nomsu = recurse(item)
               if not (item_nomsu) then
                 return nil
               end
             end
             if #line.bits > 1 then
+              if #tostring(line_comments) > 0 then
+                nomsu:append('\n    ', line_comments)
+              end
               nomsu:append(line)
               line = NomsuCode(line.source, "\n    ")
+              if i < #tree then
+                line_comments = pop_comments(tree[i + 1].source.start)
+              else
+                line_comments = ''
+              end
             end
             line:append(item_nomsu)
           end
@@ -974,7 +1044,9 @@ do
       if inline then
         local nomsu = NomsuCode(tree.source, "{")
         for i, entry in ipairs(tree) do
-          local entry_nomsu = self:tree_to_nomsu(entry, true)
+          local entry_nomsu = recurse(entry, {
+            inline = true
+          })
           if not (entry_nomsu) then
             return nil
           end
@@ -986,15 +1058,22 @@ do
         nomsu:append("}")
         return nomsu
       else
-        local inline_version = self:tree_to_nomsu(tree, true)
+        local inline_version = recurse(tree, {
+          inline = true
+        })
         if inline_version then
           return inline_version
         end
         local nomsu = NomsuCode(tree.source, "{..}")
         local line = NomsuCode(tree.source, "\n    ")
-        for _index_0 = 1, #tree do
+        local line_comments
-          local entry = tree[_index_0]
+        if #tree > 0 then
-          local entry_nomsu = self:tree_to_nomsu(entry)
+          line_comments = pop_comments(tree[1].source.start)
+        else
+          line_comments = ''
+        end
+        for i, entry in ipairs(tree) do
+          local entry_nomsu = recurse(entry)
           if not (entry_nomsu) then
             return nil
           end
@@ -1005,8 +1084,16 @@ do
             line:append(entry_nomsu)
           else
             if #line.bits > 1 then
+              if #tostring(line_comments) > 0 then
+                nomsu:append("\n    ", line_comments)
+              end
               nomsu:append(line)
               line = NomsuCode(line.source, "\n    ")
+              if i < #tree then
+                line_comments = pop_comments(tree[1].source.start)
+              else
+                line_comments = ''
+              end
             end
             line:append(entry_nomsu)
           end
@@ -1018,7 +1105,9 @@ do
       end
     elseif "DictEntry" == _exp_0 then
       local key, value = tree[1], tree[2]
-      local key_nomsu = self:tree_to_nomsu(key, true)
+      local key_nomsu = recurse(key, {
+        inline = true
+      })
       if not (key_nomsu) then
         return nil
       end
@@ -1027,7 +1116,9 @@ do
       end
       local value_nomsu
       if value then
-        value_nomsu = self:tree_to_nomsu(value, true)
+        value_nomsu = recurse(value, {
+          inline = true
+        })
       else
         value_nomsu = NomsuCode(tree.source, "")
       end
@@ -1038,7 +1129,7 @@ do
         if inline then
           return nil
         end
-        value_nomsu = self:tree_to_nomsu(value)
+        value_nomsu = recurse(value)
         if not (value_nomsu) then
           return nil
         end
@@ -1057,7 +1148,9 @@ do
           end
         end
         if not (bit_nomsu) then
-          bit_nomsu = self:tree_to_nomsu(bit, true)
+          bit_nomsu = recurse(bit, {
+            inline = true
+          })
         end
         if not (bit_nomsu) then
           return nil

nomsu_compiler.moon

@@ -374,7 +374,7 @@ with NomsuCompiler
                         string_buffer = ""
                     bit_lua = @compile(bit)
                     unless bit_lua.is_value
-                        src = '    '..gsub(tostring(@tree_to_nomsu(bit)), '\n','\n    ')
+                        src = '    '..gsub(tostring(recurse(bit)), '\n','\n    ')
                         line = "#{bit.source.filename}:#{files.get_line_number(files.read(bit.source.filename), bit.source.start)}"
                         @compile_error bit,
                             "Cannot use:\n%s\nas a string interpolation value, since it's not an expression."
@@ -472,12 +472,35 @@ with NomsuCompiler
             else
                 error("Unknown type: #{tree.type}")
 
-    .tree_to_nomsu = (tree, inline=false, can_use_colon=false)=>
+    .tree_to_nomsu = (tree, options)=>
+        options or= {}
+        comments = options.comments
+        if comments == nil and tree.comments
+            comments = [{comment:c, pos:p} for p,c in pairs tree.comments]
+            table.sort comments, (a,b)-> a.pos > b.pos
+        recurse = (t, opts)->
+            opts or= {}
+            opts.comments = comments
+            return @tree_to_nomsu(t, opts)
+        pop_comments = (pos)->
+            return '' unless comments
+            nomsu = NomsuCode(tree.source)
+            while #comments > 0 and comments[#comments].pos <= pos
+                comment = table.remove comments
+                nomsu\append "#"..(gsub(comment.comment, "\n", "\n    ")).."\n"
+            if #nomsu.bits == 0 then return ''
+            return nomsu
+
+        inline, can_use_colon = options.inline, options.can_use_colon
         switch tree.type
             when "FileChunks"
                 return nil if inline
                 nomsu = NomsuCode(tree.source)
-                nomsu\concat_append [@tree_to_nomsu(c) for c in *tree], "\n#{("~")\rep(80)}\n"
+                for i, chunk in ipairs tree
+                    if i > 1
+                        nomsu\append "\n\n#{("~")\rep(80)}\n\n"
+                    nomsu\append pop_comments(chunk.source.start)
+                    nomsu\append recurse(chunk)
                 return nomsu
 
             when "Action"
@@ -489,7 +512,7 @@ with NomsuCompiler
                                 nomsu\append " "
                             nomsu\append bit
                         else
-                            arg_nomsu = @tree_to_nomsu(bit,true)
+                            arg_nomsu = recurse(bit,inline:true)
                             return nil unless arg_nomsu
                             unless i == 1
                                 nomsu\append " "
@@ -509,7 +532,7 @@ with NomsuCompiler
                         else
                             arg_nomsu = if last_colon == i-1 and bit.type == "Action" then nil
                             elseif bit.type == "Block" then nil
-                            else @tree_to_nomsu(bit,true)
+                            else recurse(bit,inline:true)
 
                             if arg_nomsu and line_len + #tostring(arg_nomsu) < MAX_LINE
                                 if bit.type == "Action"
@@ -527,14 +550,14 @@ with NomsuCompiler
                                     line_len += #next_space + #tostring(arg_nomsu)
                                     next_space = " "
                             else
-                                arg_nomsu = @tree_to_nomsu(bit, nil, true)
+                                arg_nomsu = recurse(bit, can_use_colon:true)
-                                return nil unless nomsu
+                                return nil unless arg_nomsu
                                 -- These types carry their own indentation
                                 if bit.type != "List" and bit.type != "Dict" and bit.type != "Text"
                                     if i == 1
-                                        arg_nomsu = NomsuCode(bit.source, "(..)\n    ", arg_nomsu)
+                                        arg_nomsu = NomsuCode(bit.source, "(..)\n    ", pop_comments(bit.source.start), arg_nomsu)
                                     else
-                                        arg_nomsu = NomsuCode(bit.source, "\n    ", arg_nomsu)
+                                        arg_nomsu = NomsuCode(bit.source, "\n    ", pop_comments(bit.source.start), arg_nomsu)
                                 
                                 if last_colon == i-1 and (bit.type == "Action" or bit.type == "Block")
                                     next_space = ""
@@ -547,10 +570,10 @@ with NomsuCompiler
                     return nomsu
 
             when "EscapedNomsu"
-                nomsu = @tree_to_nomsu(tree[1], true)
+                nomsu = recurse(tree[1], inline:true)
                 if nomsu == nil and not inline
-                    nomsu = @tree_to_nomsu(tree[1])
+                    nomsu = recurse(tree[1])
-                    return nomsu and NomsuCode tree.source, "\\:\n    ", nomsu
+                    return nomsu and NomsuCode tree.source, "\\:\n    ", pop_comments(tree.source.start), nomsu
                 return nomsu and NomsuCode tree.source, "\\(", nomsu, ")"
 
             when "Block"
@@ -559,13 +582,14 @@ with NomsuCompiler
                     for i,line in ipairs tree
                         if i > 1
                             nomsu\append "; "
-                        line_nomsu = @tree_to_nomsu(line,true)
+                        line_nomsu = recurse(line,inline:true)
                         return nil unless line_nomsu
                         nomsu\append line_nomsu
                     return nomsu
                 nomsu = NomsuCode(tree.source)
                 for i, line in ipairs tree
-                    line = assert(@tree_to_nomsu(line, nil, true), "Could not convert line to nomsu")
+                    nomsu\append pop_comments(line.source.start)
+                    line = assert(recurse(line, can_use_colon:true), "Could not convert line to nomsu")
                     nomsu\append line
                     if i < #tree
                         nomsu\append "\n"
@@ -581,7 +605,7 @@ with NomsuCompiler
                             -- TODO: unescape better?
                             nomsu\append (gsub(gsub(gsub(bit,"\\","\\\\"),"\n","\\n"),'"','\\"'))
                         else
-                            interp_nomsu = @tree_to_nomsu(bit, true)
+                            interp_nomsu = recurse(bit, inline:true)
                             if interp_nomsu
                                 if bit.type != "Var" and bit.type != "List" and bit.type != "Dict" and bit.type != "Text"
                                     interp_nomsu\parenthesize!
@@ -590,7 +614,7 @@ with NomsuCompiler
                     nomsu\append '"'
                     return nomsu
                 else
-                    inline_version = @tree_to_nomsu(tree, true)
+                    inline_version = recurse(tree, inline:true)
                     if inline_version and #inline_version <= MAX_LINE
                         return inline_version
                     nomsu = NomsuCode(tree.source, '".."\n    ')
@@ -617,13 +641,13 @@ with NomsuCompiler
                                 else
                                     nomsu\append line
                         else
-                            interp_nomsu = @tree_to_nomsu(bit, true)
+                            interp_nomsu = recurse(bit, inline:true)
                             if interp_nomsu
                                 if bit.type != "Var" and bit.type != "List" and bit.type != "Dict" and bit.type != "Text"
                                     interp_nomsu\parenthesize!
                                 nomsu\append "\\", interp_nomsu
                             else
-                                interp_nomsu = assert(@tree_to_nomsu(bit))
+                                interp_nomsu = assert(recurse(bit))
                                 return nil unless interp_nomsu
                                 nomsu\append "\\\n        ", interp_nomsu
                                 if i < #tree
@@ -634,7 +658,7 @@ with NomsuCompiler
                 if inline
                     nomsu = NomsuCode(tree.source, "[")
                     for i, item in ipairs tree
-                        item_nomsu = @tree_to_nomsu(item, true)
+                        item_nomsu = recurse(item, inline:true)
                         return nil unless item_nomsu
                         if i > 1
                             nomsu\append ", "
@@ -642,24 +666,32 @@ with NomsuCompiler
                     nomsu\append "]"
                     return nomsu
                 else
-                    inline_version = @tree_to_nomsu(tree, true)
+                    inline_version = recurse(tree, inline:true)
                     if inline_version and #inline_version <= MAX_LINE
                         return inline_version
                     nomsu = NomsuCode(tree.source, "[..]")
                     line = NomsuCode(tree.source, "\n    ")
-                    for item in *tree
+                    line_comments = if #tree > 0
-                        item_nomsu = @tree_to_nomsu(item, true)
+                        pop_comments(tree[1].source.start)
-                        if item_nomsu and #line + #", " + #item_nomsu <= MAX_LINE
+                    else ''
+                    for i, item in ipairs tree
+                        item_nomsu = recurse(item, inline:true)
+                        if item_nomsu and #tostring(line) + #", " + #item_nomsu <= MAX_LINE
                             if #line.bits > 1
                                 line\append ", "
                             line\append item_nomsu
                         else
                             unless item_nomsu
-                                item_nomsu = @tree_to_nomsu(item)
+                                item_nomsu = recurse(item)
                                 return nil unless item_nomsu
                             if #line.bits > 1
+                                if #tostring(line_comments) > 0
+                                    nomsu\append '\n    ', line_comments
                                 nomsu\append line
                                 line = NomsuCode(line.source, "\n    ")
+                                line_comments = if i < #tree
+                                    pop_comments(tree[i+1].source.start)
+                                else ''
                             line\append item_nomsu
                     if #line.bits > 1
                         nomsu\append line
@@ -669,7 +701,7 @@ with NomsuCompiler
                 if inline
                     nomsu = NomsuCode(tree.source, "{")
                     for i, entry in ipairs tree
-                        entry_nomsu = @tree_to_nomsu(entry, true)
+                        entry_nomsu = recurse(entry, inline:true)
                         return nil unless entry_nomsu
                         if i > 1
                             nomsu\append ", "
@@ -677,12 +709,15 @@ with NomsuCompiler
                     nomsu\append "}"
                     return nomsu
                 else
-                    inline_version = @tree_to_nomsu(tree, true)
+                    inline_version = recurse(tree, inline:true)
                     if inline_version then return inline_version
                     nomsu = NomsuCode(tree.source, "{..}")
                     line = NomsuCode(tree.source, "\n    ")
-                    for entry in *tree
+                    line_comments = if #tree > 0
-                        entry_nomsu = @tree_to_nomsu(entry)
+                        pop_comments(tree[1].source.start)
+                    else ''
+                    for i, entry in ipairs tree
+                        entry_nomsu = recurse(entry)
                         return nil unless entry_nomsu
                         if #line + #tostring(entry_nomsu) <= MAX_LINE
                             if #line.bits > 1
@@ -690,8 +725,13 @@ with NomsuCompiler
                             line\append entry_nomsu
                         else
                             if #line.bits > 1
+                                if #tostring(line_comments) > 0
+                                    nomsu\append "\n    ", line_comments
                                 nomsu\append line
                                 line = NomsuCode(line.source, "\n    ")
+                                line_comments = if i < #tree
+                                    pop_comments(tree[1].source.start)
+                                else ''
                             line\append entry_nomsu
                     if #line.bits > 1
                         nomsu\append line
@@ -699,17 +739,17 @@ with NomsuCompiler
             
             when "DictEntry"
                 key, value = tree[1], tree[2]
-                key_nomsu = @tree_to_nomsu(key, true)
+                key_nomsu = recurse(key, inline:true)
                 return nil unless key_nomsu
                 if key.type == "Action" or key.type == "Block"
                     key_nomsu\parenthesize!
                 value_nomsu = if value
-                    @tree_to_nomsu(value, true)
+                    recurse(value, inline:true)
                 else NomsuCode(tree.source, "")
                 if inline and not value_nomsu then return nil
                 if not value_nomsu
                     return nil if inline
-                    value_nomsu = @tree_to_nomsu(value)
+                    value_nomsu = recurse(value)
                     return nil unless value_nomsu
                 return NomsuCode tree.source, key_nomsu, ":", value_nomsu
             
@@ -723,7 +763,7 @@ with NomsuCompiler
                         -- TODO: support arbitrary words here, including operators and unicode
                         if bit[1]\match("[_a-zA-Z][_a-zA-Z0-9]*")
                             bit_nomsu = bit[1]
-                    unless bit_nomsu then bit_nomsu = @tree_to_nomsu(bit, true)
+                    unless bit_nomsu then bit_nomsu = recurse(bit, inline:true)
                     return nil unless bit_nomsu
                     switch bit.type
                         when "Action", "Block", "IndexChain"

parser.lua

@@ -8,6 +8,7 @@ do
   local _obj_0 = string
   match, sub = _obj_0.match, _obj_0.sub
 end
+local files = require('files')
 local NomsuCode, LuaCode, Source
 do
   local _obj_0 = require("code_obj")
@@ -74,12 +75,11 @@ do
       return #src + 1
     end
     local err_pos = start_pos
-    local line_no = pos_to_line(src, err_pos)
+    local line_no = files.get_line_number(src, err_pos)
-    local line_starts = LINE_STARTS[src]
+    local prev_line = line_no == 1 and "" or files.get_line(src, line_no - 1)
-    local prev_line = line_no == 1 and "" or src:sub(line_starts[line_no - 1] or 1, line_starts[line_no] - 2)
+    local err_line = files.get_line(src, line_no)
-    local err_line = src:sub(line_starts[line_no], (line_starts[line_no + 1] or 0) - 2)
+    local next_line = files.get_line(src, line_no + 1)
-    local next_line = src:sub(line_starts[line_no + 1] or -1, (line_starts[line_no + 2] or 0) - 2)
+    local i = err_pos - files.get_line_starts(src)[line_no]
-    local i = err_pos - line_starts[line_no]
     local pointer = ("-"):rep(i) .. "^"
     err_msg = colored.bright(colored.yellow(colored.onred((err_msg or "Parse error") .. " at " .. tostring(userdata.source.filename) .. ":" .. tostring(line_no) .. ":")))
     if #prev_line > 0 then
@@ -93,6 +93,10 @@ do
     seen_errors[start_pos] = err_msg
     return true
   end
+  _with_0.Comment = function(src, end_pos, start_pos, value, userdata)
+    userdata.comments[start_pos] = value
+    return true
+  end
   NOMSU_DEFS = _with_0
 end
 setmetatable(NOMSU_DEFS, {
@@ -109,9 +113,6 @@ setmetatable(NOMSU_DEFS, {
       if value.__init then
         value:__init()
       end
-      for i = 1, #value do
-        assert(value[i])
-      end
       return value
     end
     self[key] = make_node
@@ -158,14 +159,15 @@ Parser.parse = function(nomsu_code, source)
   local userdata = {
     indent = "",
     errors = { },
-    source = source
+    source = source,
+    comments = { }
   }
   local tree = NOMSU_PATTERN:match(nomsu_code, nil, userdata)
   if not (tree) then
     error("In file " .. tostring(colored.blue(tostring(source or "<unknown>"))) .. " failed to parse:\n" .. tostring(colored.onyellow(colored.black(nomsu_code))))
   end
   if type(tree) == 'number' then
-    tree = nil
+    return nil
   end
   if next(userdata.errors) then
     local keys
@@ -192,6 +194,7 @@ Parser.parse = function(nomsu_code, source)
     end
     error("Errors occurred while parsing:\n\n" .. table.concat(errors, "\n\n"), 0)
   end
+  tree.comments = userdata.comments
   return tree
 end
 return Parser

parser.moon

@@ -4,6 +4,7 @@ re = require 're'
 lpeg.setmaxstack 10000
 {:P,:R,:S,:C,:Cmt,:Carg} = lpeg
 {:match, :sub} = string
+files = require 'files'
 {:NomsuCode, :LuaCode, :Source} = require "code_obj"
 AST = require "nomsu_tree"
 
@@ -56,13 +57,12 @@ NOMSU_DEFS = with {}
             seen_errors[start_pos+1] = colored.bright colored.yellow colored.onred "Too many errors, canceling parsing..."
             return #src+1
         err_pos = start_pos
-        line_no = pos_to_line(src, err_pos)
+        line_no = files.get_line_number(src, err_pos)
         --src = files.read(userdata.source.filename)
-        line_starts = LINE_STARTS[src]
+        prev_line = line_no == 1 and "" or files.get_line(src, line_no-1)
-        prev_line = line_no == 1 and "" or src\sub(line_starts[line_no-1] or 1, line_starts[line_no]-2)
+        err_line = files.get_line(src, line_no)
-        err_line = src\sub(line_starts[line_no], (line_starts[line_no+1] or 0)-2)
+        next_line = files.get_line(src, line_no+1)
-        next_line = src\sub(line_starts[line_no+1] or -1, (line_starts[line_no+2] or 0)-2)
+        i = err_pos-files.get_line_starts(src)[line_no]
-        i = err_pos-line_starts[line_no]
         pointer = ("-")\rep(i) .. "^"
         err_msg = colored.bright colored.yellow colored.onred (err_msg or "Parse error").." at #{userdata.source.filename}:#{line_no}:"
         if #prev_line > 0 then err_msg ..= "\n"..colored.dim(prev_line)
@@ -72,6 +72,10 @@ NOMSU_DEFS = with {}
         seen_errors[start_pos] = err_msg
         return true
 
+    .Comment = (src,end_pos,start_pos,value,userdata)->
+        userdata.comments[start_pos] = value
+        return true
+
 setmetatable(NOMSU_DEFS, {__index:(key)=>
     make_node = (start, value, stop, userdata)->
         if userdata.source
@@ -79,7 +83,6 @@ setmetatable(NOMSU_DEFS, {__index:(key)=>
                 value.source = Source(.filename, .start + start-1, .start + stop-1)
         setmetatable(value, AST[key])
         if value.__init then value\__init!
-        for i=1,#value do assert(value[i])
         return value
 
     self[key] = make_node
@@ -115,13 +118,13 @@ Parser.parse = (nomsu_code, source=nil)->
     source or= nomsu_code.source
     nomsu_code = tostring(nomsu_code)
     userdata = {
-        indent: "", errors: {}, :source
+        indent: "", errors: {}, :source, comments: {},
     }
     tree = NOMSU_PATTERN\match(nomsu_code, nil, userdata)
     unless tree
         error "In file #{colored.blue tostring(source or "<unknown>")} failed to parse:\n#{colored.onyellow colored.black nomsu_code}"
     if type(tree) == 'number'
-        tree = nil
+        return nil
 
     if next(userdata.errors)
         keys = [k for k,v in pairs(userdata.errors)]
@@ -129,6 +132,7 @@ Parser.parse = (nomsu_code, source=nil)->
         errors = [userdata.errors[k] for k in *keys]
         error("Errors occurred while parsing:\n\n"..table.concat(errors, "\n\n"), 0)
     
+    tree.comments = userdata.comments
     return tree
 
 return Parser