From d535dc57940f6c9558d3b588d53b44a87b2a8ed2 Mon Sep 17 00:00:00 2001
From: Bruce Hill <bitbucket@bruce-hill.com>
Date: Sat, 14 Jul 2018 14:41:17 -0700
Subject: [PATCH] Fixed up comment generation to old standards

---
 nomsu.peg           |  87 ++++++++++++++++++--------------------
 nomsu_compiler.lua  | 101 ++++++++++++++++++++++++++------------------
 nomsu_compiler.moon |  79 ++++++++++++++++++----------------
 nomsu_tree.lua      |   3 +-
 nomsu_tree.moon     |   2 +-
 parser.lua          |  32 +++++---------
 parser.moon         |  15 +++----
 7 files changed, 162 insertions(+), 157 deletions(-)

diff --git a/nomsu.peg b/nomsu.peg
index ee0acb0..565e8b4 100644
--- a/nomsu.peg
+++ b/nomsu.peg
@@ -1,37 +1,34 @@
 -- Nomsu version 2
 file (File):
-    %nl*
     {:curr_indent: ' '* :}
+    comment? blank_lines?
     (chunk (nl_nodent chunk_delimeter nl_nodent chunk)*)?
-    %nl*
+    blank_lines?
     (!! .+ -> "Parse error" !!)?
 
 nodent: =curr_indent !(" ")
 indent: =curr_indent "    "
-blank_lines: %nl (%ws* %nl)*
-eol: (!. / &%nl)
-
-comment (Comment):
-    !inline_comment
-    "#" {~ [^%nl]* (blank_lines (indent -> '') [^%nl]*)* ~}
-inline_comment (Comment):
-    "#(" {~ (inline_comment / [^%nl])* ~} ")#"
+blank_lines: %nl ((nodent comment / %ws*) %nl)*
+eol: %ws* eol_comment? (!. / &%nl)
 
 nl_nodent: blank_lines nodent
-nl_indent: blank_lines {:curr_indent: indent :}
+nl_indent: blank_lines {:curr_indent: indent :} (comment nl_nodent)?
 
-chunk: block / ((comment nl_nodent)* (action / expression)? (nl_nodent comment)*)
-chunk_delimeter: ("~")^+3
+comment:
+    "#" (({} {~ [^%nl]* ((%nl (!indent %ws* %nl)*) (indent -> '') [^%nl]*)* ~} %userdata) => add_comment)
+eol_comment:
+    "#" (({} {[^%nl]*} %userdata) => add_comment)
+
+chunk: block / action / expression
+chunk_delimeter: ("~")^+3 eol
 
 inline_block (Block):
-    inline_statement (%ws* ";" %ws* inline_statement)+ (%ws* inline_comment)*
+    inline_statement (%ws* ";" %ws* inline_statement)+
 block (Block):
-    block_line (nl_nodent block_line)+ (nl_nodent comment)*
-block_line:
-    statement %ws* (!! [^%nl]+ -> "Unexpected character while parsing block line" !!)?
+    statement (nl_nodent statement)+
 
-statement: (comment nl_nodent)* (action / expression) %ws* (!! [^%nl]+ -> "Unexpected character while parsing line" !!)?
-inline_statement: (inline_comment %ws*)* (inline_action / inline_expression)
+statement: (action / expression) (eol / (!! [^%nl]+ -> "Unexpected character while parsing line" !!))
+inline_statement: (inline_action / inline_expression)
 
 noindex_inline_expression:
     number / variable / inline_text / inline_list / inline_dict / inline_nomsu
@@ -39,22 +36,21 @@ noindex_inline_expression:
             %ws* (inline_block / inline_action / inline_expression) %ws*
             (%ws* ',' %ws* (inline_block / inline_action / inline_expression) %ws*)*
         (")"
-            / (!! %ws* eol -> 'Line ended without finding a closing )-parenthesis' !!)
+            / (!! eol -> 'Line ended without finding a closing )-parenthesis' !!)
             / (!! [^%nl]+ -> 'Unexpected character while parsing subexpression' !!)
         )
       )
 inline_expression: index_chain / noindex_inline_expression 
 indented_expression:
     indented_text / indented_nomsu / indented_list / indented_dict / ({|
-        ("(..)" / ":")? nl_indent (comment nl_nodent)*
-            (block / action / expression)
+        ("(..)")? nl_indent
+            (block / action / expression) (nl_nodent comment)*
             (!! [^%nl]+ -> "Unexpected character while parsing indented expression" !!)?
     |} -> unpack)
 expression:
     inline_expression
     / (":" %ws*
-        (!! eol -> "Missing expression after the ':'" !!)?
-        (inline_block / inline_action / inline_expression) %ws* eol)
+        (inline_block / inline_action / inline_expression / (!! '' -> "Missing expression after the ':'" !!)))
     / indented_expression
 
 inline_nomsu (EscapedNomsu): "\" inline_expression
@@ -62,8 +58,7 @@ indented_nomsu (EscapedNomsu):
     "\" (
         noindex_inline_expression
         / (":" %ws*
-            (!! eol -> "Missing expression after the '\:'" !!)?
-            (inline_block / inline_action / inline_expression) %ws* eol)
+            (inline_block / inline_action / inline_expression) (!! '' -> "Missing expression after the '\:'" !!))
         / indented_expression)
 
 index_chain (IndexChain):
@@ -72,23 +67,21 @@ index_chain (IndexChain):
 -- Actions need either at least 1 word, or at least 2 tokens
 inline_action (Action):
     !chunk_delimeter
-    (inline_comment %ws*)* 
-    (   (inline_expression ((%ws* inline_comment)* %ws* (inline_expression / word))+)
-      / (word              ((%ws* inline_comment)* %ws* (inline_expression / word))*))
-    ((%ws* inline_comment)* %ws* ":" %ws* (inline_block / inline_action / inline_expression
+    (   (inline_expression (%ws* (inline_expression / word))+)
+      / (word              (%ws* (inline_expression / word))*))
+    (%ws* ":" %ws* (inline_block / inline_action / inline_expression
         / (!! '' -> "Missing expression after the ':'" !!)))?
 action (Action):
     !chunk_delimeter
-    (inline_comment %ws*)* 
-    (  (expression (((nl_nodent comment)* nl_nodent "..")? %ws* (inline_comment %ws*)* (expression / word))+)
-     / (word       (((nl_nodent comment)* nl_nodent "..")? %ws* (inline_comment %ws*)* (expression / word))*))
+    (  (expression ((nl_nodent "..")? %ws* (expression / word))+)
+     / (word       ((nl_nodent "..")? %ws* (expression / word))*))
 
 word: !number { %operator_char+ / %ident_char+ }
 
 text_word (Text): word
 
 inline_text (Text):
-    !('".."' %ws* eol)
+    !('".."' eol)
     '"'
         ({~ (('\"' -> '"') / ('\\' -> '\') / %escaped_char / [^%nl\"])+ ~}
         / inline_text_interpolation)*
@@ -102,16 +95,16 @@ inline_text_interpolation:
             %ws* (inline_block / inline_action / inline_expression) %ws*
             (%ws* ',' %ws* (inline_block / inline_action / inline_expression) %ws*)*
         (")"
-         / (!! %ws* eol -> 'Line ended without finding a closing )-parenthesis' !!)
+         / (!! eol -> 'Line ended without finding a closing )-parenthesis' !!)
          / (!! [^%nl]+ -> 'Unexpected character while parsing Text interpolation' !!)))
     )
 
 indented_text (Text):
-    '".."' %ws* %nl {:curr_indent: indent :}
+    '".."' eol %nl {:curr_indent: indent :}
         (indented_plain_text / text_interpolation / {~ blank_lines (=curr_indent -> "") ~})*
     (!! [^%nl]+ -> "Unexpected character while parsing Text" !!)?
 indented_plain_text (Text):
-    {~ (("\\" -> "\") / (("\" (blank_lines =curr_indent comment)* blank_lines =curr_indent "..") -> "") / (!text_interpolation "\") / [^%nl\]+)+
+    {~ (("\\" -> "\") / (("\" blank_lines =curr_indent "..") -> "") / (!text_interpolation "\") / [^%nl\]+)+
         (blank_lines (=curr_indent -> ""))* ~}
 text_interpolation:
     inline_text_interpolation / ("\" indented_expression blank_lines =curr_indent "..")
@@ -127,16 +120,16 @@ inline_list (List):
     "[" %ws*
         (inline_list_item (%ws* ',' %ws* inline_list_item)* (%ws* ',')?)? %ws*
     ("]" / (","? (
-        (!! %ws* eol -> "Line ended before finding a closing ]-bracket" !!)
+        (!! eol -> "Line ended before finding a closing ]-bracket" !!)
        /(!! [^%nl]+ -> "Unexpected character while parsing List" !!)?
     )))
 indented_list (List):
-    "[..]" nl_indent (comment nl_nodent)*
-        list_line (nl_nodent list_line)*
+    "[..]" eol nl_indent
+        list_line (nl_nodent list_line)* (nl_nodent comment)*
     (","? (!! [^%nl]+ -> "Unexpected character while parsing List" !!))?
 list_line:
-      (inline_list_item %ws* "," %ws*)+ %ws* eol
-    / (inline_list_item %ws* "," %ws*)* (action / expression)
+      (inline_list_item %ws* "," %ws*)+ eol
+    / (inline_list_item %ws* "," %ws*)* (action / expression) eol
 inline_list_item: inline_block / inline_action / inline_expression
 
 inline_dict (Dict):
@@ -144,16 +137,16 @@ inline_dict (Dict):
     "{" %ws*
         (inline_dict_entry (%ws* ',' %ws* inline_dict_entry)*)? %ws*
     ("}" / (","? (
-        (!! %ws* eol -> "Line ended before finding a closing }-brace" !!)
+        (!! eol -> "Line ended before finding a closing }-brace" !!)
       / (!! [^%nl]* -> "Unexpected character while parsing Dictionary" !!)
     )))
 indented_dict (Dict):
-    "{..}" nl_indent (comment nl_nodent)*
-        dict_line ((nl_nodent comment)* nl_nodent dict_line)* (nl_nodent comment)*
+    "{..}" eol nl_indent
+        dict_line (nl_nodent dict_line)* (nl_nodent comment)*
     (","? (!! [^%nl]+ -> "Unexpected character while parsing Dictionary" !!))?
 dict_line:
-      (inline_dict_entry %ws* "," %ws*)+ %ws* eol
-    / (inline_dict_entry %ws* "," %ws*)* dict_entry
+      (inline_dict_entry %ws* "," %ws*)+ eol
+    / (inline_dict_entry %ws* "," %ws*)* dict_entry eol
 dict_entry(DictEntry):
     dict_key (%ws* ":" %ws* (action / expression))?
 inline_dict_entry(DictEntry):
diff --git a/nomsu_compiler.lua b/nomsu_compiler.lua
index ada6c12..6dbe4c4 100644
--- a/nomsu_compiler.lua
+++ b/nomsu_compiler.lua
@@ -607,7 +607,7 @@ do
           end
           local bit_lua = self:compile(bit)
           if not (bit_lua.is_value) then
-            local src = '    ' .. gsub(tostring(self:tree_to_nomsu(bit)), '\n', '\n    ')
+            local src = '    ' .. gsub(tostring(recurse(bit)), '\n', '\n    ')
             local line = tostring(bit.source.filename) .. ":" .. tostring(files.get_line_number(files.read(bit.source.filename), bit.source.start))
             self:compile_error(bit, "Cannot use:\n%s\nas a string interpolation value, since it's not an expression.")
           end
@@ -719,27 +719,45 @@ do
   end
   NomsuCompiler.tree_to_nomsu = function(self, tree, options)
     options = options or { }
-    local comment_i = 0
-    local pop_comments
-    pop_comments = function(pos)
-      if not (tree.comments) then
-        return ''
-      end
-      local nomsu = NomsuCode(tree.source)
-      for i = comment_i + 1, #tree.comments do
-        comment_i = i
-        local comment = tree.comments[i]
-        if comment.source.start > pos then
-          break
+    if not (options.pop_comments) then
+      local comments
+      do
+        local _accum_0 = { }
+        local _len_0 = 1
+        for p, c in pairs(tree.comments or { }) do
+          _accum_0[_len_0] = {
+            comment = c,
+            pos = p
+          }
+          _len_0 = _len_0 + 1
         end
-        nomsu:append("#" .. (gsub(comment[1], "\n", "\n    ")) .. "\n")
+        comments = _accum_0
       end
-      if #nomsu.bits == 0 then
-        return ''
+      table.sort(comments, function(a, b)
+        return a.pos < b.pos
+      end)
+      local comment_i = 1
+      options.pop_comments = function(pos)
+        local nomsu = NomsuCode(tree.source)
+        while comments[comment_i] and comments[comment_i].pos <= pos do
+          local comment = comments[comment_i].comment
+          nomsu:append("#" .. (gsub(comment, "\n", "\n    ")) .. "\n")
+          if comment:match("^\n.") then
+            nomsu:append("\n")
+          end
+          comment_i = comment_i + 1
+        end
+        return #nomsu.bits == 0 and '' or nomsu
       end
-      return nomsu
     end
-    local inline, can_use_colon = options.inline, options.can_use_colon
+    local recurse
+    recurse = function(t, opts)
+      opts = opts or { }
+      opts.pop_comments = options.pop_comments
+      return self:tree_to_nomsu(t, opts)
+    end
+    local inline, can_use_colon, pop_comments
+    inline, can_use_colon, pop_comments = options.inline, options.can_use_colon, options.pop_comments
     local _exp_0 = tree.type
     if "File" == _exp_0 then
       if inline then
@@ -751,8 +769,9 @@ do
           nomsu:append("\n\n" .. tostring(("~"):rep(80)) .. "\n\n")
         end
         nomsu:append(pop_comments(chunk.source.start))
-        nomsu:append(self:tree_to_nomsu(chunk))
+        nomsu:append(recurse(chunk))
       end
+      nomsu:append(pop_comments(tree.source.stop))
       return nomsu
     elseif "Action" == _exp_0 then
       if inline then
@@ -764,7 +783,7 @@ do
             end
             nomsu:append(bit)
           else
-            local arg_nomsu = self:tree_to_nomsu(bit, {
+            local arg_nomsu = recurse(bit, {
               inline = true
             })
             if not (arg_nomsu) then
@@ -800,7 +819,7 @@ do
             elseif bit.type == "Block" then
               arg_nomsu = nil
             else
-              arg_nomsu = self:tree_to_nomsu(bit, {
+              arg_nomsu = recurse(bit, {
                 inline = true
               })
             end
@@ -822,7 +841,7 @@ do
                 next_space = " "
               end
             else
-              arg_nomsu = self:tree_to_nomsu(bit, {
+              arg_nomsu = recurse(bit, {
                 can_use_colon = true
               })
               if not (arg_nomsu) then
@@ -850,11 +869,11 @@ do
         return nomsu
       end
     elseif "EscapedNomsu" == _exp_0 then
-      local nomsu = self:tree_to_nomsu(tree[1], {
+      local nomsu = recurse(tree[1], {
         inline = true
       })
       if nomsu == nil and not inline then
-        nomsu = self:tree_to_nomsu(tree[1])
+        nomsu = recurse(tree[1])
         return nomsu and NomsuCode(tree.source, "\\:\n    ", pop_comments(tree.source.start), nomsu)
       end
       return nomsu and NomsuCode(tree.source, "\\(", nomsu, ")")
@@ -865,7 +884,7 @@ do
           if i > 1 then
             nomsu:append("; ")
           end
-          local line_nomsu = self:tree_to_nomsu(line, {
+          local line_nomsu = recurse(line, {
             inline = true
           })
           if not (line_nomsu) then
@@ -878,7 +897,7 @@ do
       local nomsu = NomsuCode(tree.source)
       for i, line in ipairs(tree) do
         nomsu:append(pop_comments(line.source.start))
-        line = assert(self:tree_to_nomsu(line, {
+        line = assert(recurse(line, {
           can_use_colon = true
         }), "Could not convert line to nomsu")
         nomsu:append(line)
@@ -902,7 +921,7 @@ do
             elseif bit.type == "Text" then
               nomsu:append(make_text(bit))
             else
-              local interp_nomsu = assert(self:tree_to_nomsu(bit, {
+              local interp_nomsu = assert(recurse(bit, {
                 inline = true
               }))
               if bit.type ~= "Var" and bit.type ~= "List" and bit.type ~= "Dict" and bit.type ~= "Text" then
@@ -915,7 +934,7 @@ do
         end
         return NomsuCode(tree.source, '"', make_text(tree), '"')
       else
-        local inline_version = self:tree_to_nomsu(tree, {
+        local inline_version = recurse(tree, {
           inline = true
         })
         if inline_version and #inline_version <= MAX_LINE then
@@ -959,7 +978,7 @@ do
             elseif bit.type == "Text" then
               nomsu:append(make_text(bit))
             else
-              local interp_nomsu = self:tree_to_nomsu(bit, {
+              local interp_nomsu = recurse(bit, {
                 inline = true
               })
               if interp_nomsu then
@@ -968,7 +987,7 @@ do
                 end
                 nomsu:append("\\", interp_nomsu)
               else
-                interp_nomsu = assert(self:tree_to_nomsu(bit))
+                interp_nomsu = assert(recurse(bit))
                 if not (interp_nomsu) then
                   return nil
                 end
@@ -987,7 +1006,7 @@ do
       if inline then
         local nomsu = NomsuCode(tree.source, "[")
         for i, item in ipairs(tree) do
-          local item_nomsu = self:tree_to_nomsu(item, {
+          local item_nomsu = recurse(item, {
             inline = true
           })
           if not (item_nomsu) then
@@ -1001,7 +1020,7 @@ do
         nomsu:append("]")
         return nomsu
       else
-        local inline_version = self:tree_to_nomsu(tree, {
+        local inline_version = recurse(tree, {
           inline = true
         })
         if inline_version and #inline_version <= MAX_LINE then
@@ -1016,7 +1035,7 @@ do
           line_comments = ''
         end
         for i, item in ipairs(tree) do
-          local item_nomsu = self:tree_to_nomsu(item, {
+          local item_nomsu = recurse(item, {
             inline = true
           })
           if item_nomsu and #tostring(line) + #", " + #item_nomsu <= MAX_LINE then
@@ -1026,7 +1045,7 @@ do
             line:append(item_nomsu)
           else
             if not (item_nomsu) then
-              item_nomsu = self:tree_to_nomsu(item)
+              item_nomsu = recurse(item)
               if not (item_nomsu) then
                 return nil
               end
@@ -1055,7 +1074,7 @@ do
       if inline then
         local nomsu = NomsuCode(tree.source, "{")
         for i, entry in ipairs(tree) do
-          local entry_nomsu = self:tree_to_nomsu(entry, {
+          local entry_nomsu = recurse(entry, {
             inline = true
           })
           if not (entry_nomsu) then
@@ -1069,7 +1088,7 @@ do
         nomsu:append("}")
         return nomsu
       else
-        local inline_version = self:tree_to_nomsu(tree, {
+        local inline_version = recurse(tree, {
           inline = true
         })
         if inline_version then
@@ -1084,7 +1103,7 @@ do
           line_comments = ''
         end
         for i, entry in ipairs(tree) do
-          local entry_nomsu = self:tree_to_nomsu(entry)
+          local entry_nomsu = recurse(entry)
           if not (entry_nomsu) then
             return nil
           end
@@ -1116,7 +1135,7 @@ do
       end
     elseif "DictEntry" == _exp_0 then
       local key, value = tree[1], tree[2]
-      local key_nomsu = self:tree_to_nomsu(key, {
+      local key_nomsu = recurse(key, {
         inline = true
       })
       if not (key_nomsu) then
@@ -1127,7 +1146,7 @@ do
       end
       local value_nomsu
       if value then
-        value_nomsu = self:tree_to_nomsu(value, {
+        value_nomsu = recurse(value, {
           inline = true
         })
       else
@@ -1140,7 +1159,7 @@ do
         if inline then
           return nil
         end
-        value_nomsu = self:tree_to_nomsu(value)
+        value_nomsu = recurse(value)
         if not (value_nomsu) then
           return nil
         end
@@ -1159,7 +1178,7 @@ do
           end
         end
         if not (bit_nomsu) then
-          bit_nomsu = self:tree_to_nomsu(bit, {
+          bit_nomsu = recurse(bit, {
             inline = true
           })
         end
diff --git a/nomsu_compiler.moon b/nomsu_compiler.moon
index 283fa52..0b330c8 100644
--- a/nomsu_compiler.moon
+++ b/nomsu_compiler.moon
@@ -383,7 +383,7 @@ with NomsuCompiler
                         string_buffer = ""
                     bit_lua = @compile(bit)
                     unless bit_lua.is_value
-                        src = '    '..gsub(tostring(@tree_to_nomsu(bit)), '\n','\n    ')
+                        src = '    '..gsub(tostring(recurse(bit)), '\n','\n    ')
                         line = "#{bit.source.filename}:#{files.get_line_number(files.read(bit.source.filename), bit.source.start)}"
                         @compile_error bit,
                             "Cannot use:\n%s\nas a string interpolation value, since it's not an expression."
@@ -483,19 +483,25 @@ with NomsuCompiler
 
     .tree_to_nomsu = (tree, options)=>
         options or= {}
-        comment_i = 0
-        pop_comments = (pos)->
-            return '' unless tree.comments
-            nomsu = NomsuCode(tree.source)
-            for i=comment_i+1,#tree.comments
-                comment_i = i
-                comment = tree.comments[i]
-                break if comment.source.start > pos
-                nomsu\append "#"..(gsub(comment[1], "\n", "\n    ")).."\n"
-            if #nomsu.bits == 0 then return ''
-            return nomsu
+        unless options.pop_comments
+            comments = [{comment:c, pos:p} for p,c in pairs(tree.comments or {})]
+            table.sort comments, (a,b)-> a.pos < b.pos
+            comment_i = 1
+            options.pop_comments = (pos)->
+                nomsu = NomsuCode(tree.source)
+                while comments[comment_i] and comments[comment_i].pos <= pos
+                    comment = comments[comment_i].comment
+                    nomsu\append("#"..(gsub(comment, "\n", "\n    ")).."\n")
+                    if comment\match("^\n.") then nomsu\append("\n") -- for aesthetics
+                    comment_i += 1
+                return #nomsu.bits == 0 and '' or nomsu
 
-        inline, can_use_colon = options.inline, options.can_use_colon
+        recurse = (t, opts)->
+            opts or= {}
+            opts.pop_comments = options.pop_comments
+            return @tree_to_nomsu(t, opts)
+
+        {:inline, :can_use_colon, :pop_comments} = options
         switch tree.type
             when "File"
                 return nil if inline
@@ -504,7 +510,8 @@ with NomsuCompiler
                     if i > 1
                         nomsu\append "\n\n#{("~")\rep(80)}\n\n"
                     nomsu\append pop_comments(chunk.source.start)
-                    nomsu\append @tree_to_nomsu(chunk)
+                    nomsu\append recurse(chunk)
+                nomsu\append pop_comments(tree.source.stop)
                 return nomsu
 
             when "Action"
@@ -516,7 +523,7 @@ with NomsuCompiler
                                 nomsu\append " "
                             nomsu\append bit
                         else
-                            arg_nomsu = @tree_to_nomsu(bit,inline:true)
+                            arg_nomsu = recurse(bit,inline:true)
                             return nil unless arg_nomsu
                             if bit.type == "Action" or bit.type == "Block"
                                 if bit.type == "Action" and i == #tree
@@ -539,7 +546,7 @@ with NomsuCompiler
                         else
                             arg_nomsu = if last_colon == i-1 and bit.type == "Action" then nil
                             elseif bit.type == "Block" then nil
-                            else @tree_to_nomsu(bit,inline:true)
+                            else recurse(bit,inline:true)
 
                             if arg_nomsu and line_len + #tostring(arg_nomsu) < MAX_LINE
                                 if bit.type == "Action"
@@ -557,7 +564,7 @@ with NomsuCompiler
                                     line_len += #next_space + #tostring(arg_nomsu)
                                     next_space = " "
                             else
-                                arg_nomsu = @tree_to_nomsu(bit, can_use_colon:true)
+                                arg_nomsu = recurse(bit, can_use_colon:true)
                                 return nil unless arg_nomsu
                                 -- These types carry their own indentation
                                 if bit.type != "List" and bit.type != "Dict" and bit.type != "Text"
@@ -577,9 +584,9 @@ with NomsuCompiler
                     return nomsu
 
             when "EscapedNomsu"
-                nomsu = @tree_to_nomsu(tree[1], inline:true)
+                nomsu = recurse(tree[1], inline:true)
                 if nomsu == nil and not inline
-                    nomsu = @tree_to_nomsu(tree[1])
+                    nomsu = recurse(tree[1])
                     return nomsu and NomsuCode tree.source, "\\:\n    ", pop_comments(tree.source.start), nomsu
                 return nomsu and NomsuCode tree.source, "\\(", nomsu, ")"
 
@@ -589,14 +596,14 @@ with NomsuCompiler
                     for i,line in ipairs tree
                         if i > 1
                             nomsu\append "; "
-                        line_nomsu = @tree_to_nomsu(line,inline:true)
+                        line_nomsu = recurse(line,inline:true)
                         return nil unless line_nomsu
                         nomsu\append line_nomsu
                     return nomsu
                 nomsu = NomsuCode(tree.source)
                 for i, line in ipairs tree
                     nomsu\append pop_comments(line.source.start)
-                    line = assert(@tree_to_nomsu(line, can_use_colon:true), "Could not convert line to nomsu")
+                    line = assert(recurse(line, can_use_colon:true), "Could not convert line to nomsu")
                     nomsu\append line
                     if i < #tree
                         nomsu\append "\n"
@@ -615,14 +622,14 @@ with NomsuCompiler
                             elseif bit.type == "Text"
                                 nomsu\append(make_text(bit))
                             else
-                                interp_nomsu = assert @tree_to_nomsu(bit, inline:true)
+                                interp_nomsu = assert recurse(bit, inline:true)
                                 if bit.type != "Var" and bit.type != "List" and bit.type != "Dict" and bit.type != "Text"
                                     interp_nomsu\parenthesize!
                                 nomsu\append "\\", interp_nomsu
                         return nomsu
                     return NomsuCode(tree.source, '"', make_text(tree), '"')
                 else
-                    inline_version = @tree_to_nomsu(tree, inline:true)
+                    inline_version = recurse(tree, inline:true)
                     if inline_version and #inline_version <= MAX_LINE
                         return inline_version
                     make_text = (tree)->
@@ -652,13 +659,13 @@ with NomsuCompiler
                             elseif bit.type == "Text"
                                 nomsu\append make_text(bit)
                             else
-                                interp_nomsu = @tree_to_nomsu(bit, inline:true)
+                                interp_nomsu = recurse(bit, inline:true)
                                 if interp_nomsu
                                     if bit.type != "Var" and bit.type != "List" and bit.type != "Dict" and bit.type != "Text"
                                         interp_nomsu\parenthesize!
                                     nomsu\append "\\", interp_nomsu
                                 else
-                                    interp_nomsu = assert(@tree_to_nomsu(bit))
+                                    interp_nomsu = assert(recurse(bit))
                                     return nil unless interp_nomsu
                                     nomsu\append "\\\n    ", interp_nomsu
                                     if i < #tree
@@ -670,7 +677,7 @@ with NomsuCompiler
                 if inline
                     nomsu = NomsuCode(tree.source, "[")
                     for i, item in ipairs tree
-                        item_nomsu = @tree_to_nomsu(item, inline:true)
+                        item_nomsu = recurse(item, inline:true)
                         return nil unless item_nomsu
                         if i > 1
                             nomsu\append ", "
@@ -678,7 +685,7 @@ with NomsuCompiler
                     nomsu\append "]"
                     return nomsu
                 else
-                    inline_version = @tree_to_nomsu(tree, inline:true)
+                    inline_version = recurse(tree, inline:true)
                     if inline_version and #inline_version <= MAX_LINE
                         return inline_version
                     nomsu = NomsuCode(tree.source, "[..]")
@@ -687,14 +694,14 @@ with NomsuCompiler
                         pop_comments(tree[1].source.start)
                     else ''
                     for i, item in ipairs tree
-                        item_nomsu = @tree_to_nomsu(item, inline:true)
+                        item_nomsu = recurse(item, inline:true)
                         if item_nomsu and #tostring(line) + #", " + #item_nomsu <= MAX_LINE
                             if #line.bits > 1
                                 line\append ", "
                             line\append item_nomsu
                         else
                             unless item_nomsu
-                                item_nomsu = @tree_to_nomsu(item)
+                                item_nomsu = recurse(item)
                                 return nil unless item_nomsu
                             if #line.bits > 1
                                 if #tostring(line_comments) > 0
@@ -713,7 +720,7 @@ with NomsuCompiler
                 if inline
                     nomsu = NomsuCode(tree.source, "{")
                     for i, entry in ipairs tree
-                        entry_nomsu = @tree_to_nomsu(entry, inline:true)
+                        entry_nomsu = recurse(entry, inline:true)
                         return nil unless entry_nomsu
                         if i > 1
                             nomsu\append ", "
@@ -721,7 +728,7 @@ with NomsuCompiler
                     nomsu\append "}"
                     return nomsu
                 else
-                    inline_version = @tree_to_nomsu(tree, inline:true)
+                    inline_version = recurse(tree, inline:true)
                     if inline_version then return inline_version
                     nomsu = NomsuCode(tree.source, "{..}")
                     line = NomsuCode(tree.source, "\n    ")
@@ -729,7 +736,7 @@ with NomsuCompiler
                         pop_comments(tree[1].source.start)
                     else ''
                     for i, entry in ipairs tree
-                        entry_nomsu = @tree_to_nomsu(entry)
+                        entry_nomsu = recurse(entry)
                         return nil unless entry_nomsu
                         if #line + #tostring(entry_nomsu) <= MAX_LINE
                             if #line.bits > 1
@@ -751,17 +758,17 @@ with NomsuCompiler
             
             when "DictEntry"
                 key, value = tree[1], tree[2]
-                key_nomsu = @tree_to_nomsu(key, inline:true)
+                key_nomsu = recurse(key, inline:true)
                 return nil unless key_nomsu
                 if key.type == "Action" or key.type == "Block"
                     key_nomsu\parenthesize!
                 value_nomsu = if value
-                    @tree_to_nomsu(value, inline:true)
+                    recurse(value, inline:true)
                 else NomsuCode(tree.source, "")
                 if inline and not value_nomsu then return nil
                 if not value_nomsu
                     return nil if inline
-                    value_nomsu = @tree_to_nomsu(value)
+                    value_nomsu = recurse(value)
                     return nil unless value_nomsu
                 return NomsuCode tree.source, key_nomsu, ":", value_nomsu
             
@@ -775,7 +782,7 @@ with NomsuCompiler
                         -- TODO: support arbitrary words here, including operators and unicode
                         if bit[1]\match("[_a-zA-Z][_a-zA-Z0-9]*")
                             bit_nomsu = bit[1]
-                    unless bit_nomsu then bit_nomsu = @tree_to_nomsu(bit, inline:true)
+                    unless bit_nomsu then bit_nomsu = recurse(bit, inline:true)
                     return nil unless bit_nomsu
                     switch bit.type
                         when "Action", "Block", "IndexChain"
diff --git a/nomsu_tree.lua b/nomsu_tree.lua
index 70ee4ee..d5875bc 100644
--- a/nomsu_tree.lua
+++ b/nomsu_tree.lua
@@ -26,8 +26,7 @@ local types = {
   "DictEntry",
   "IndexChain",
   "Action",
-  "File",
-  "Comment"
+  "File"
 }
 for _index_0 = 1, #types do
   local name = types[_index_0]
diff --git a/nomsu_tree.moon b/nomsu_tree.moon
index 9f5a3fb..e387f37 100644
--- a/nomsu_tree.moon
+++ b/nomsu_tree.moon
@@ -10,7 +10,7 @@ AST.is_syntax_tree = (n, t=nil)->
     type(n) == 'table' and getmetatable(n) and AST[n.type] == getmetatable(n) and (t == nil or n.type == t)
 
 types = {"Number", "Var", "Block", "EscapedNomsu", "Text", "List", "Dict", "DictEntry",
-    "IndexChain", "Action", "File", "Comment"}
+    "IndexChain", "Action", "File"}
 for name in *types
     cls = {}
     with cls
diff --git a/parser.lua b/parser.lua
index a4d2a90..d928fc8 100644
--- a/parser.lua
+++ b/parser.lua
@@ -26,6 +26,9 @@ do
   _with_0.nl = P("\r") ^ -1 * P("\n")
   _with_0.ws = S(" \t")
   _with_0.tonumber = tonumber
+  _with_0.table = function()
+    return { }
+  end
   _with_0.unpack = unpack or table.unpack
   local string_escapes = {
     n = "\n",
@@ -48,6 +51,10 @@ do
   _with_0.utf8_char = (R("\194\223") * R("\128\191") + R("\224\239") * R("\128\191") * R("\128\191") + R("\240\244") * R("\128\191") * R("\128\191") * R("\128\191"))
   _with_0.ident_char = R("az", "AZ", "09") + P("_") + _with_0.utf8_char
   _with_0.userdata = Carg(1)
+  _with_0.add_comment = function(src, end_pos, start_pos, comment, userdata)
+    userdata.comments[start_pos] = comment
+    return true
+  end
   _with_0.error = function(src, end_pos, start_pos, err_msg, userdata)
     local seen_errors = userdata.errors
     if seen_errors[start_pos] then
@@ -92,27 +99,8 @@ setmetatable(NOMSU_DEFS, {
           value.source = Source(_with_0.filename, _with_0.start + start - 1, _with_0.start + stop - 1)
         end
       end
-      local comments = { }
-      for i = #value, 1, -1 do
-        local _continue_0 = false
-        repeat
-          if not (type(value[i]) == 'table') then
-            _continue_0 = true
-            break
-          end
-          if value[i].type == "Comment" then
-            insert(comments, remove(value, i))
-          end
-          _continue_0 = true
-        until true
-        if not _continue_0 then
-          break
-        end
-      end
-      if #comments > 0 then
-        value.comments = comments
-      end
       setmetatable(value, AST[key])
+      value.comments = userdata.comments
       if value.__init then
         value:__init()
       end
@@ -164,9 +152,9 @@ Parser.parse = function(nomsu_code, source)
   source = source or nomsu_code.source
   nomsu_code = tostring(nomsu_code)
   local userdata = {
-    indent = "",
     errors = { },
-    source = source
+    source = source,
+    comments = { }
   }
   local tree = NOMSU_PATTERN:match(nomsu_code, nil, userdata)
   if not (tree) then
diff --git a/parser.moon b/parser.moon
index 78aa042..96f6b6c 100644
--- a/parser.moon
+++ b/parser.moon
@@ -14,6 +14,7 @@ NOMSU_DEFS = with {}
     .nl = P("\r")^-1 * P("\n")
     .ws = S(" \t")
     .tonumber = tonumber
+    .table = -> {}
     .unpack = unpack or table.unpack
     string_escapes = n:"\n", t:"\t", b:"\b", a:"\a", v:"\v", f:"\f", r:"\r"
     digit, hex = R('09'), R('09','af','AF')
@@ -29,6 +30,10 @@ NOMSU_DEFS = with {}
 
     .userdata = Carg(1)
 
+    .add_comment = (src,end_pos,start_pos,comment,userdata)->
+        userdata.comments[start_pos] = comment
+        return true
+
     .error = (src,end_pos,start_pos,err_msg,userdata)->
         seen_errors = userdata.errors
         if seen_errors[start_pos]
@@ -59,14 +64,8 @@ setmetatable(NOMSU_DEFS, {__index:(key)=>
         if userdata.source
             with userdata.source
                 value.source = Source(.filename, .start + start-1, .start + stop-1)
-        comments = {}
-        for i=#value,1,-1
-            continue unless type(value[i]) == 'table'
-            if value[i].type == "Comment"
-                insert comments, remove(value, i)
-        if #comments > 0
-            value.comments = comments
         setmetatable(value, AST[key])
+        value.comments = userdata.comments
         if value.__init then value\__init!
         return value
 
@@ -105,7 +104,7 @@ Parser.parse = (nomsu_code, source=nil)->
     source or= nomsu_code.source
     nomsu_code = tostring(nomsu_code)
     userdata = {
-        indent: "", errors: {}, :source
+        errors: {}, :source, comments: {}
     }
     tree = NOMSU_PATTERN\match(nomsu_code, nil, userdata)
     unless tree