Fixing up comments, slowly.

6 files changed, 70 insertions, 93 deletions

diff --git a/code_obj.lua b/code_obj.lua
index a822499..84af36a 100644
--- a/code_obj.lua
+++ b/code_obj.lua
@@ -86,6 +86,9 @@ do
         repeat
           local b = select(i, ...)
           assert(b, "code bit is nil")
+          if Source:is_instance(b) then
+            require('ldt').breakpoint()
+          end
           if b == '' then
             _continue_0 = true
             break
diff --git a/code_obj.moon b/code_obj.moon
index 1586d15..ce8e715 100644
--- a/code_obj.moon
+++ b/code_obj.moon
@@ -60,6 +60,7 @@ class Code
         for i=1,n
             b = select(i, ...)
             assert(b, "code bit is nil")
+            if Source\is_instance(b) then require('ldt').breakpoint!
             if b == '' then continue
             bits[#bits+1] = b
             if type(b) == 'string'
diff --git a/nomsu_compiler.lua b/nomsu_compiler.lua
index 0074d29..1de8fbb 100644
--- a/nomsu_compiler.lua
+++ b/nomsu_compiler.lua
@@ -604,7 +604,7 @@ do
           end
           local bit_lua = self:compile(bit)
           if not (bit_lua.is_value) then
-            local src = '    ' .. gsub(tostring(recurse(bit)), '\n', '\n    ')
+            local src = '    ' .. gsub(tostring(self:tree_to_nomsu(bit)), '\n', '\n    ')
             local line = tostring(bit.source.filename) .. ":" .. tostring(files.get_line_number(files.read(bit.source.filename), bit.source.start))
             self:compile_error(bit, "Cannot use:\n%s\nas a string interpolation value, since it's not an expression.")
           end
@@ -716,39 +716,20 @@ do
   end
   NomsuCompiler.tree_to_nomsu = function(self, tree, options)
     options = options or { }
-    local comments = options.comments
-    if comments == nil and tree.comments then
-      do
-        local _accum_0 = { }
-        local _len_0 = 1
-        for p, c in pairs(tree.comments) do
-          _accum_0[_len_0] = {
-            comment = c,
-            pos = p
-          }
-          _len_0 = _len_0 + 1
-        end
-        comments = _accum_0
-      end
-      table.sort(comments, function(a, b)
-        return a.pos > b.pos
-      end)
-    end
-    local recurse
-    recurse = function(t, opts)
-      opts = opts or { }
-      opts.comments = comments
-      return self:tree_to_nomsu(t, opts)
-    end
+    local comment_i = 0
     local pop_comments
     pop_comments = function(pos)
-      if not (comments) then
+      if not (tree.comments) then
         return ''
       end
       local nomsu = NomsuCode(tree.source)
-      while #comments > 0 and comments[#comments].pos <= pos do
-        local comment = table.remove(comments)
-        nomsu:append("#" .. (gsub(comment.comment, "\n", "\n    ")) .. "\n")
+      for i = comment_i + 1, #tree.comments do
+        comment_i = i
+        local comment = tree.comments[i]
+        if comment.source.start > pos then
+          break
+        end
+        nomsu:append("#" .. (gsub(comment[1], "\n", "\n    ")) .. "\n")
       end
       if #nomsu.bits == 0 then
         return ''
@@ -767,7 +748,7 @@ do
           nomsu:append("\n\n" .. tostring(("~"):rep(80)) .. "\n\n")
         end
         nomsu:append(pop_comments(chunk.source.start))
-        nomsu:append(recurse(chunk))
+        nomsu:append(self:tree_to_nomsu(chunk))
       end
       return nomsu
     elseif "Action" == _exp_0 then
@@ -780,7 +761,7 @@ do
             end
             nomsu:append(bit)
           else
-            local arg_nomsu = recurse(bit, {
+            local arg_nomsu = self:tree_to_nomsu(bit, {
               inline = true
             })
             if not (arg_nomsu) then
@@ -816,7 +797,7 @@ do
             elseif bit.type == "Block" then
               arg_nomsu = nil
             else
-              arg_nomsu = recurse(bit, {
+              arg_nomsu = self:tree_to_nomsu(bit, {
                 inline = true
               })
             end
@@ -838,7 +819,7 @@ do
                 next_space = " "
               end
             else
-              arg_nomsu = recurse(bit, {
+              arg_nomsu = self:tree_to_nomsu(bit, {
                 can_use_colon = true
               })
               if not (arg_nomsu) then
@@ -866,11 +847,11 @@ do
         return nomsu
       end
     elseif "EscapedNomsu" == _exp_0 then
-      local nomsu = recurse(tree[1], {
+      local nomsu = self:tree_to_nomsu(tree[1], {
         inline = true
       })
       if nomsu == nil and not inline then
-        nomsu = recurse(tree[1])
+        nomsu = self:tree_to_nomsu(tree[1])
         return nomsu and NomsuCode(tree.source, "\\:\n    ", pop_comments(tree.source.start), nomsu)
       end
       return nomsu and NomsuCode(tree.source, "\\(", nomsu, ")")
@@ -881,7 +862,7 @@ do
           if i > 1 then
             nomsu:append("; ")
           end
-          local line_nomsu = recurse(line, {
+          local line_nomsu = self:tree_to_nomsu(line, {
             inline = true
           })
           if not (line_nomsu) then
@@ -894,7 +875,7 @@ do
       local nomsu = NomsuCode(tree.source)
       for i, line in ipairs(tree) do
         nomsu:append(pop_comments(line.source.start))
-        line = assert(recurse(line, {
+        line = assert(self:tree_to_nomsu(line, {
           can_use_colon = true
         }), "Could not convert line to nomsu")
         nomsu:append(line)
@@ -918,7 +899,7 @@ do
             elseif bit.type == "Text" then
               nomsu:append(make_text(bit))
             else
-              local interp_nomsu = assert(recurse(bit, {
+              local interp_nomsu = assert(self:tree_to_nomsu(bit, {
                 inline = true
               }))
               if bit.type ~= "Var" and bit.type ~= "List" and bit.type ~= "Dict" and bit.type ~= "Text" then
@@ -931,7 +912,7 @@ do
         end
         return NomsuCode(tree.source, '"', make_text(tree), '"')
       else
-        local inline_version = recurse(tree, {
+        local inline_version = self:tree_to_nomsu(tree, {
           inline = true
         })
         if inline_version and #inline_version <= MAX_LINE then
@@ -975,7 +956,7 @@ do
             elseif bit.type == "Text" then
               nomsu:append(make_text(bit))
             else
-              local interp_nomsu = recurse(bit, {
+              local interp_nomsu = self:tree_to_nomsu(bit, {
                 inline = true
               })
               if interp_nomsu then
@@ -984,7 +965,7 @@ do
                 end
                 nomsu:append("\\", interp_nomsu)
               else
-                interp_nomsu = assert(recurse(bit))
+                interp_nomsu = assert(self:tree_to_nomsu(bit))
                 if not (interp_nomsu) then
                   return nil
                 end
@@ -1003,7 +984,7 @@ do
       if inline then
         local nomsu = NomsuCode(tree.source, "[")
         for i, item in ipairs(tree) do
-          local item_nomsu = recurse(item, {
+          local item_nomsu = self:tree_to_nomsu(item, {
             inline = true
           })
           if not (item_nomsu) then
@@ -1017,7 +998,7 @@ do
         nomsu:append("]")
         return nomsu
       else
-        local inline_version = recurse(tree, {
+        local inline_version = self:tree_to_nomsu(tree, {
           inline = true
         })
         if inline_version and #inline_version <= MAX_LINE then
@@ -1032,7 +1013,7 @@ do
           line_comments = ''
         end
         for i, item in ipairs(tree) do
-          local item_nomsu = recurse(item, {
+          local item_nomsu = self:tree_to_nomsu(item, {
             inline = true
           })
           if item_nomsu and #tostring(line) + #", " + #item_nomsu <= MAX_LINE then
@@ -1042,7 +1023,7 @@ do
             line:append(item_nomsu)
           else
             if not (item_nomsu) then
-              item_nomsu = recurse(item)
+              item_nomsu = self:tree_to_nomsu(item)
               if not (item_nomsu) then
                 return nil
               end
@@ -1071,7 +1052,7 @@ do
       if inline then
         local nomsu = NomsuCode(tree.source, "{")
         for i, entry in ipairs(tree) do
-          local entry_nomsu = recurse(entry, {
+          local entry_nomsu = self:tree_to_nomsu(entry, {
             inline = true
           })
           if not (entry_nomsu) then
@@ -1085,7 +1066,7 @@ do
         nomsu:append("}")
         return nomsu
       else
-        local inline_version = recurse(tree, {
+        local inline_version = self:tree_to_nomsu(tree, {
           inline = true
         })
         if inline_version then
@@ -1100,7 +1081,7 @@ do
           line_comments = ''
         end
         for i, entry in ipairs(tree) do
-          local entry_nomsu = recurse(entry)
+          local entry_nomsu = self:tree_to_nomsu(entry)
           if not (entry_nomsu) then
             return nil
           end
@@ -1132,7 +1113,7 @@ do
       end
     elseif "DictEntry" == _exp_0 then
       local key, value = tree[1], tree[2]
-      local key_nomsu = recurse(key, {
+      local key_nomsu = self:tree_to_nomsu(key, {
         inline = true
       })
       if not (key_nomsu) then
@@ -1143,7 +1124,7 @@ do
       end
       local value_nomsu
       if value then
-        value_nomsu = recurse(value, {
+        value_nomsu = self:tree_to_nomsu(value, {
           inline = true
         })
       else
@@ -1156,7 +1137,7 @@ do
         if inline then
           return nil
         end
-        value_nomsu = recurse(value)
+        value_nomsu = self:tree_to_nomsu(value)
         if not (value_nomsu) then
           return nil
         end
@@ -1175,7 +1156,7 @@ do
           end
         end
         if not (bit_nomsu) then
-          bit_nomsu = recurse(bit, {
+          bit_nomsu = self:tree_to_nomsu(bit, {
             inline = true
           })
         end
diff --git a/nomsu_compiler.moon b/nomsu_compiler.moon
index 90ab354..dd3000e 100644
--- a/nomsu_compiler.moon
+++ b/nomsu_compiler.moon
@@ -382,7 +382,7 @@ with NomsuCompiler
                         string_buffer = ""
                     bit_lua = @compile(bit)
                     unless bit_lua.is_value
-                        src = '    '..gsub(tostring(recurse(bit)), '\n','\n    ')
+                        src = '    '..gsub(tostring(@tree_to_nomsu(bit)), '\n','\n    ')
                         line = "#{bit.source.filename}:#{files.get_line_number(files.read(bit.source.filename), bit.source.start)}"
                         @compile_error bit,
                             "Cannot use:\n%s\nas a string interpolation value, since it's not an expression."
@@ -482,20 +482,15 @@ with NomsuCompiler
 
     .tree_to_nomsu = (tree, options)=>
         options or= {}
-        comments = options.comments
-        if comments == nil and tree.comments
-            comments = [{comment:c, pos:p} for p,c in pairs tree.comments]
-            table.sort comments, (a,b)-> a.pos > b.pos
-        recurse = (t, opts)->
-            opts or= {}
-            opts.comments = comments
-            return @tree_to_nomsu(t, opts)
+        comment_i = 0
         pop_comments = (pos)->
-            return '' unless comments
+            return '' unless tree.comments
             nomsu = NomsuCode(tree.source)
-            while #comments > 0 and comments[#comments].pos <= pos
-                comment = table.remove comments
-                nomsu\append "#"..(gsub(comment.comment, "\n", "\n    ")).."\n"
+            for i=comment_i+1,#tree.comments
+                comment_i = i
+                comment = tree.comments[i]
+                break if comment.source.start > pos
+                nomsu\append "#"..(gsub(comment[1], "\n", "\n    ")).."\n"
             if #nomsu.bits == 0 then return ''
             return nomsu
 
@@ -508,7 +503,7 @@ with NomsuCompiler
                     if i > 1
                         nomsu\append "\n\n#{("~")\rep(80)}\n\n"
                     nomsu\append pop_comments(chunk.source.start)
-                    nomsu\append recurse(chunk)
+                    nomsu\append @tree_to_nomsu(chunk)
                 return nomsu
 
             when "Action"
@@ -520,7 +515,7 @@ with NomsuCompiler
                                 nomsu\append " "
                             nomsu\append bit
                         else
-                            arg_nomsu = recurse(bit,inline:true)
+                            arg_nomsu = @tree_to_nomsu(bit,inline:true)
                             return nil unless arg_nomsu
                             if bit.type == "Action" or bit.type == "Block"
                                 if bit.type == "Action" and i == #tree
@@ -543,7 +538,7 @@ with NomsuCompiler
                         else
                             arg_nomsu = if last_colon == i-1 and bit.type == "Action" then nil
                             elseif bit.type == "Block" then nil
-                            else recurse(bit,inline:true)
+                            else @tree_to_nomsu(bit,inline:true)
 
                             if arg_nomsu and line_len + #tostring(arg_nomsu) < MAX_LINE
                                 if bit.type == "Action"
@@ -561,7 +556,7 @@ with NomsuCompiler
                                     line_len += #next_space + #tostring(arg_nomsu)
                                     next_space = " "
                             else
-                                arg_nomsu = recurse(bit, can_use_colon:true)
+                                arg_nomsu = @tree_to_nomsu(bit, can_use_colon:true)
                                 return nil unless arg_nomsu
                                 -- These types carry their own indentation
                                 if bit.type != "List" and bit.type != "Dict" and bit.type != "Text"
@@ -581,9 +576,9 @@ with NomsuCompiler
                     return nomsu
 
             when "EscapedNomsu"
-                nomsu = recurse(tree[1], inline:true)
+                nomsu = @tree_to_nomsu(tree[1], inline:true)
                 if nomsu == nil and not inline
-                    nomsu = recurse(tree[1])
+                    nomsu = @tree_to_nomsu(tree[1])
                     return nomsu and NomsuCode tree.source, "\\:\n    ", pop_comments(tree.source.start), nomsu
                 return nomsu and NomsuCode tree.source, "\\(", nomsu, ")"
 
@@ -593,14 +588,14 @@ with NomsuCompiler
                     for i,line in ipairs tree
                         if i > 1
                             nomsu\append "; "
-                        line_nomsu = recurse(line,inline:true)
+                        line_nomsu = @tree_to_nomsu(line,inline:true)
                         return nil unless line_nomsu
                         nomsu\append line_nomsu
                     return nomsu
                 nomsu = NomsuCode(tree.source)
                 for i, line in ipairs tree
                     nomsu\append pop_comments(line.source.start)
-                    line = assert(recurse(line, can_use_colon:true), "Could not convert line to nomsu")
+                    line = assert(@tree_to_nomsu(line, can_use_colon:true), "Could not convert line to nomsu")
                     nomsu\append line
                     if i < #tree
                         nomsu\append "\n"
@@ -619,14 +614,14 @@ with NomsuCompiler
                             elseif bit.type == "Text"
                                 nomsu\append(make_text(bit))
                             else
-                                interp_nomsu = assert recurse(bit, inline:true)
+                                interp_nomsu = assert @tree_to_nomsu(bit, inline:true)
                                 if bit.type != "Var" and bit.type != "List" and bit.type != "Dict" and bit.type != "Text"
                                     interp_nomsu\parenthesize!
                                 nomsu\append "\\", interp_nomsu
                         return nomsu
                     return NomsuCode(tree.source, '"', make_text(tree), '"')
                 else
-                    inline_version = recurse(tree, inline:true)
+                    inline_version = @tree_to_nomsu(tree, inline:true)
                     if inline_version and #inline_version <= MAX_LINE
                         return inline_version
                     make_text = (tree)->
@@ -656,13 +651,13 @@ with NomsuCompiler
                             elseif bit.type == "Text"
                                 nomsu\append make_text(bit)
                             else
-                                interp_nomsu = recurse(bit, inline:true)
+                                interp_nomsu = @tree_to_nomsu(bit, inline:true)
                                 if interp_nomsu
                                     if bit.type != "Var" and bit.type != "List" and bit.type != "Dict" and bit.type != "Text"
                                         interp_nomsu\parenthesize!
                                     nomsu\append "\\", interp_nomsu
                                 else
-                                    interp_nomsu = assert(recurse(bit))
+                                    interp_nomsu = assert(@tree_to_nomsu(bit))
                                     return nil unless interp_nomsu
                                     nomsu\append "\\\n    ", interp_nomsu
                                     if i < #tree
@@ -674,7 +669,7 @@ with NomsuCompiler
                 if inline
                     nomsu = NomsuCode(tree.source, "[")
                     for i, item in ipairs tree
-                        item_nomsu = recurse(item, inline:true)
+                        item_nomsu = @tree_to_nomsu(item, inline:true)
                         return nil unless item_nomsu
                         if i > 1
                             nomsu\append ", "
@@ -682,7 +677,7 @@ with NomsuCompiler
                     nomsu\append "]"
                     return nomsu
                 else
-                    inline_version = recurse(tree, inline:true)
+                    inline_version = @tree_to_nomsu(tree, inline:true)
                     if inline_version and #inline_version <= MAX_LINE
                         return inline_version
                     nomsu = NomsuCode(tree.source, "[..]")
@@ -691,14 +686,14 @@ with NomsuCompiler
                         pop_comments(tree[1].source.start)
                     else ''
                     for i, item in ipairs tree
-                        item_nomsu = recurse(item, inline:true)
+                        item_nomsu = @tree_to_nomsu(item, inline:true)
                         if item_nomsu and #tostring(line) + #", " + #item_nomsu <= MAX_LINE
                             if #line.bits > 1
                                 line\append ", "
                             line\append item_nomsu
                         else
                             unless item_nomsu
-                                item_nomsu = recurse(item)
+                                item_nomsu = @tree_to_nomsu(item)
                                 return nil unless item_nomsu
                             if #line.bits > 1
                                 if #tostring(line_comments) > 0
@@ -717,7 +712,7 @@ with NomsuCompiler
                 if inline
                     nomsu = NomsuCode(tree.source, "{")
                     for i, entry in ipairs tree
-                        entry_nomsu = recurse(entry, inline:true)
+                        entry_nomsu = @tree_to_nomsu(entry, inline:true)
                         return nil unless entry_nomsu
                         if i > 1
                             nomsu\append ", "
@@ -725,7 +720,7 @@ with NomsuCompiler
                     nomsu\append "}"
                     return nomsu
                 else
-                    inline_version = recurse(tree, inline:true)
+                    inline_version = @tree_to_nomsu(tree, inline:true)
                     if inline_version then return inline_version
                     nomsu = NomsuCode(tree.source, "{..}")
                     line = NomsuCode(tree.source, "\n    ")
@@ -733,7 +728,7 @@ with NomsuCompiler
                         pop_comments(tree[1].source.start)
                     else ''
                     for i, entry in ipairs tree
-                        entry_nomsu = recurse(entry)
+                        entry_nomsu = @tree_to_nomsu(entry)
                         return nil unless entry_nomsu
                         if #line + #tostring(entry_nomsu) <= MAX_LINE
                             if #line.bits > 1
@@ -755,17 +750,17 @@ with NomsuCompiler
             
             when "DictEntry"
                 key, value = tree[1], tree[2]
-                key_nomsu = recurse(key, inline:true)
+                key_nomsu = @tree_to_nomsu(key, inline:true)
                 return nil unless key_nomsu
                 if key.type == "Action" or key.type == "Block"
                     key_nomsu\parenthesize!
                 value_nomsu = if value
-                    recurse(value, inline:true)
+                    @tree_to_nomsu(value, inline:true)
                 else NomsuCode(tree.source, "")
                 if inline and not value_nomsu then return nil
                 if not value_nomsu
                     return nil if inline
-                    value_nomsu = recurse(value)
+                    value_nomsu = @tree_to_nomsu(value)
                     return nil unless value_nomsu
                 return NomsuCode tree.source, key_nomsu, ":", value_nomsu
             
@@ -779,7 +774,7 @@ with NomsuCompiler
                         -- TODO: support arbitrary words here, including operators and unicode
                         if bit[1]\match("[_a-zA-Z][_a-zA-Z0-9]*")
                             bit_nomsu = bit[1]
-                    unless bit_nomsu then bit_nomsu = recurse(bit, inline:true)
+                    unless bit_nomsu then bit_nomsu = @tree_to_nomsu(bit, inline:true)
                     return nil unless bit_nomsu
                     switch bit.type
                         when "Action", "Block", "IndexChain"
diff --git a/parser.lua b/parser.lua
index 5f766b9..979a29f 100644
--- a/parser.lua
+++ b/parser.lua
@@ -193,8 +193,7 @@ Parser.parse = function(nomsu_code, source)
   local userdata = {
     indent = "",
     errors = { },
-    source = source,
-    comments = { }
+    source = source
   }
   local tree = NOMSU_PATTERN:match(nomsu_code, nil, userdata)
   if not (tree) then
@@ -228,7 +227,6 @@ Parser.parse = function(nomsu_code, source)
     end
     error("Errors occurred while parsing:\n\n" .. table.concat(errors, "\n\n"), 0)
   end
-  tree.comments = userdata.comments
   tree.version = userdata.version
   return tree
 end
diff --git a/parser.moon b/parser.moon
index 5e37a59..1591161 100644
--- a/parser.moon
+++ b/parser.moon
@@ -115,7 +115,7 @@ Parser.parse = (nomsu_code, source=nil)->
     source or= nomsu_code.source
     nomsu_code = tostring(nomsu_code)
     userdata = {
-        indent: "", errors: {}, :source, comments: {},
+        indent: "", errors: {}, :source
     }
     tree = NOMSU_PATTERN\match(nomsu_code, nil, userdata)
     unless tree
@@ -129,7 +129,6 @@ Parser.parse = (nomsu_code, source=nil)->
         errors = [userdata.errors[k] for k in *keys]
         error("Errors occurred while parsing:\n\n"..table.concat(errors, "\n\n"), 0)
     
-    tree.comments = userdata.comments
     tree.version = userdata.version
     return tree