Overhaul of comment handling, plus a few fixes (e.g. a fix for indented

text that begins with a nomsu comment)

code_obj.lua

@@ -105,11 +105,7 @@ do
               self.trailing_line_len = #trailing_text
             end
           else
-            if #b.indents > 1 then
+            self.trailing_line_len = math.min(self.trailing_line_len + #tostring(b), b.trailing_line_len)
-              self.trailing_line_len = b.trailing_line_len
-            else
-              self.trailing_line_len = self.trailing_line_len + #tostring(b)
-            end
             if self.current_indent ~= 0 then
               indents[#bits] = self.current_indent
             end

code_obj.moon

@@ -73,10 +73,7 @@ class Code
                     @current_indent = #spaces
                     @trailing_line_len = #trailing_text
             else
-                if #b.indents > 1
+                @trailing_line_len = math.min(@trailing_line_len + #tostring(b), b.trailing_line_len)
-                    @trailing_line_len = b.trailing_line_len
-                else
-                    @trailing_line_len += #tostring(b)
                 if @current_indent != 0
                     indents[#bits] = @current_indent
         @__str = nil

core/metaprogramming.nom

@@ -164,12 +164,11 @@ compile [%tree with %t -> %replacement] to
             \(%replacement as lua return)
         end)
 
-compile [%tree with vars %v] to
+action [%tree with vars %replacements]
-    Lua value ".."
+    =lua ".."
-        \(%tree as lua expr):map(function(t)
+        \%tree:map(function(\%t)
-            local replacements = \(%v as lua expr)
+            if \%t.type == "Var" then
-            if t.type == "Var" then
+                return \%replacements[\%t[1]]
-                return replacements[t[1]]
             end
         end)
 

nomsu.1.peg

@@ -1,6 +1,7 @@
 -- Nomsu version 2
 file (FileChunks):
     {:curr_indent: ' '* :}
+    ("#!" (!"nomsu" [^%nl])* "nomsu" %ws+ "-V" %ws* {:version: ([0-9.]+ -> tonumber) :} [^%nl]*)?
     comment? blank_lines?
     (chunk (nl_nodent chunk_delimeter nl_nodent chunk)*)?
     blank_lines?
@@ -100,12 +101,12 @@ inline_text_interpolation:
     )
 
 indented_text (Text):
-    '".."' eol %nl {:curr_indent: indent :}
+    '".."' eol %nl {%nl*} {:curr_indent: indent :}
-        (indented_plain_text / text_interpolation / {~ blank_lines (=curr_indent -> "") ~})*
+        (indented_plain_text / text_interpolation / {~ %nl+ (=curr_indent -> "") ~})*
     (!! [^%nl]+ -> "Unexpected character while parsing Text" !!)?
 indented_plain_text (Text):
     {~ (("\\" -> "\") / (("\" blank_lines =curr_indent "..") -> "") / (!text_interpolation "\") / [^%nl\]+)+
-        (blank_lines (=curr_indent -> ""))* ~}
+        (%nl+ (=curr_indent -> ""))* ~}
 text_interpolation:
     inline_text_interpolation / ("\" indented_expression blank_lines =curr_indent "..")
 

nomsu.2.peg

@@ -1,6 +1,7 @@
 -- Nomsu version 2
 file (FileChunks):
     {:curr_indent: ' '* :}
+    ("#!" (!"nomsu" [^%nl])* "nomsu" %ws+ "-V" %ws* {:version: ([0-9.]+ -> tonumber) :} [^%nl]*)?
     comment? blank_lines?
     (chunk (nl_nodent section_division nl_nodent chunk)*)?
     blank_lines?
@@ -61,13 +62,16 @@ index_chain (IndexChain):
 -- Actions need either at least 1 word, or at least 2 tokens
 inline_action (Action):
     !section_division
-    (   (inline_expression (%ws* (inline_expression / word))+)
+    (   ((smushed_action / inline_expression) (%ws* (smushed_action / inline_expression / word))+)
-      / (word              (%ws* (inline_expression / word))*))
+      / (word              (%ws* (smushed_action / inline_expression / word))*))
     (%ws* inline_block)?
 action (Action):
     !section_division
-    (  (expression ((nl_nodent "..")? %ws* (expression / word))+)
+    (  ((smushed_action / expression) ((nl_nodent "..")? %ws* (smushed_action / expression / word))+)
-     / (word       ((nl_nodent "..")? %ws* (expression / word))*))
+     / (word       ((nl_nodent "..")? %ws* (smushed_action / expression / word))*))
+smushed_action (Action):
+    !section_division
+    (index_chain / noindex_inline_expression / word+) (index_chain / noindex_inline_expression / word+ / "(" %ws* ")")+
 
 word: !number { %operator_char+ / %ident_char+ }
 
@@ -93,20 +97,20 @@ inline_text_interpolation:
     )
 
 indented_text (Text):
-    '".."' eol %nl {:curr_indent: indent :}
+    '".."' eol %nl {%nl*} {:curr_indent: indent :}
-        (indented_plain_text / text_interpolation / {~ blank_lines (=curr_indent -> "") ~})*
+        (indented_plain_text / text_interpolation / {~ %nl+ (=curr_indent -> "") ~})*
     (!! [^%nl]+ -> "Unexpected character while parsing Text" !!)?
 indented_plain_text (Text):
     {~ (("\\" -> "\") / (("\" blank_lines =curr_indent "..") -> "") / (!text_interpolation "\") / [^%nl\]+)+
-        (blank_lines (=curr_indent -> ""))* ~}
+        (%nl+ (=curr_indent -> ""))* ~}
 text_interpolation:
     inline_text_interpolation / ("\" indented_expression blank_lines =curr_indent "..")
 
 number (Number): (("-"? (([0-9]+ "." [0-9]+) / ("." [0-9]+) / ([0-9]+)))-> tonumber)
 
--- Variables can be nameless (i.e. just %) and can't contain operators like apostrophe
+-- Variables can be nameless (i.e. just %) and can only contain identifier chars.
--- which is a hack to allow %'s to parse as "%" and "' s" separately
+-- This ensures you don't get weird parsings of `%x+%y` or `%'s thing`.
-variable (Var): "%" {(%ident_char+ ((!"'" %operator_char+) / %ident_char+)*)?}
+variable (Var): "%" {%ident_char*}
 
 inline_list (List):
     !('[..]')

nomsu.lua

@@ -271,10 +271,11 @@ run = function()
     end
   end
   if #file_queue == 0 then
-    nomsu:run([[use "core"
+    nomsu:run([[#!/usr/bin/env nomsu -V2
+use "core"
 use "lib/consolecolor.nom"
 action [quit, exit]: lua> "os.exit(0)"
-action [help]
+action [help]:
     say ".."
         This is the Nomsu v\(Nomsu version) interactive console.
         You can type in Nomsu code here and hit 'enter' twice to run it.

nomsu.moon

@@ -175,10 +175,11 @@ run = ->
     if #file_queue == 0
         -- Run in interactive mode (REPL)
         nomsu\run [[
+#!/usr/bin/env nomsu -V2
 use "core"
 use "lib/consolecolor.nom"
 action [quit, exit]: lua> "os.exit(0)"
-action [help]
+action [help]:
     say ".."
         This is the Nomsu v\(Nomsu version) interactive console.
         You can type in Nomsu code here and hit 'enter' twice to run it.

nomsu_compiler.lua

@@ -722,60 +722,67 @@ do
   local MIN_COLON_LEN = 20
   NomsuCompiler.tree_to_nomsu = function(self, tree, options)
     options = options or { }
-    if not (options.pop_comments) then
+    options.consumed_comments = options.consumed_comments or { }
-      local comments
+    local pop_comments
-      do
+    pop_comments = function(pos, prefix, suffix)
-        local _accum_0 = { }
+      if prefix == nil then
-        local _len_0 = 1
+        prefix = ''
-        for p, c in pairs(tree.comments or { }) do
+      end
-          if tree.source.start <= p and p <= tree.source.stop then
+      if suffix == nil then
-            _accum_0[_len_0] = {
+        suffix = ''
-              comment = c,
+      end
-              pos = p
+      local find_comments
-            }
+      find_comments = function(t)
-            _len_0 = _len_0 + 1
+        if t.comments and t.source.filename == tree.source.filename then
+          local _list_0 = t.comments
+          for _index_0 = 1, #_list_0 do
+            local c = _list_0[_index_0]
+            if not (options.consumed_comments[c]) then
+              coroutine.yield(c)
+            end
           end
         end
-        comments = _accum_0
+        for _index_0 = 1, #t do
-      end
+          local x = t[_index_0]
-      table.sort(comments, function(a, b)
+          if AST.is_syntax_tree(x) then
-        return a.pos < b.pos
+            find_comments(x)
-      end)
-      local comment_i = 1
-      options.pop_comments = function(pos, prefix)
-        if prefix == nil then
-          prefix = ''
-        end
-        local nomsu = NomsuCode(tree.source)
-        while comments[comment_i] and comments[comment_i].pos <= pos do
-          local comment = comments[comment_i].comment
-          nomsu:append("#" .. (gsub(comment, "\n", "\n    ")) .. "\n")
-          if comment:match("^\n.") then
-            nomsu:append("\n")
           end
-          comment_i = comment_i + 1
         end
-        if #nomsu.bits == 0 then
-          return ''
-        end
-        nomsu:prepend(prefix)
-        return nomsu
       end
+      local nomsu = NomsuCode(tree.source)
+      for comment in coroutine.wrap(function()
+        return find_comments(tree)
+      end) do
+        if comment.pos > pos then
+          break
+        end
+        options.consumed_comments[comment] = true
+        nomsu:append("#" .. (gsub(comment.comment, "\n", "\n    ")) .. "\n")
+        if comment.comment:match("^\n.") then
+          nomsu:append("\n")
+        end
+      end
+      if #nomsu.bits == 0 then
+        return ''
+      end
+      nomsu:prepend(prefix)
+      nomsu:append(suffix)
+      return nomsu
     end
     local recurse
     recurse = function(t, opts)
       opts = opts or { }
-      opts.pop_comments = options.pop_comments
+      opts.consumed_comments = options.consumed_comments
       return self:tree_to_nomsu(t, opts)
     end
-    local inline, pop_comments
+    local inline
-    inline, pop_comments = options.inline, options.pop_comments
+    inline = options.inline
     local _exp_0 = tree.type
     if "FileChunks" == _exp_0 then
       if inline then
         error("Cannot inline a FileChunks")
       end
-      local nomsu = NomsuCode(tree.source)
+      local nomsu = NomsuCode(tree.source, pop_comments(tree.source.start))
       for i, chunk in ipairs(tree) do
         if i > 1 then
           nomsu:append("\n\n" .. tostring(("~"):rep(80)) .. "\n\n")
@@ -785,7 +792,7 @@ do
           top = true
         }))
       end
-      nomsu:append(pop_comments(tree.source.stop))
+      nomsu:append(pop_comments(tree.source.stop, '\n'))
       return nomsu
     elseif "Action" == _exp_0 then
       if inline then
@@ -803,7 +810,7 @@ do
             if not (arg_nomsu) then
               return nil
             end
-            if not (i == 1) then
+            if not (i == 1 or (bit.type == "Block" and not (#bit > 1 or i < #tree))) then
               nomsu:append(" ")
             end
             if bit.type == "Action" or (bit.type == "Block" and (#bit > 1 or i < #tree)) then
@@ -814,9 +821,13 @@ do
         end
         return nomsu
       else
-        local nomsu = NomsuCode(tree.source)
+        local pos = tree.source.start
+        local nomsu = NomsuCode(tree.source, pop_comments(pos))
         local next_space = ""
         for i, bit in ipairs(tree) do
+          if match(next_space, '\n') then
+            nomsu:append(pop_comments(pos, '\n'))
+          end
           if type(bit) == "string" then
             nomsu:append(next_space, bit)
             next_space = " "
@@ -853,11 +864,13 @@ do
               end
               next_space = "\n.."
             end
+            pos = bit.source.stop
           end
           if next_space == " " and nomsu.trailing_line_len > MAX_LINE then
             next_space = "\n.."
           end
         end
+        nomsu:append(pop_comments(pos, '\n'))
         return nomsu
       end
     elseif "EscapedNomsu" == _exp_0 then
@@ -885,7 +898,7 @@ do
         end
         return nomsu
       end
-      local nomsu = NomsuCode(tree.source)
+      local nomsu = NomsuCode(tree.source, pop_comments(tree.source.start))
       for i, line in ipairs(tree) do
         nomsu:append(pop_comments(line.source.start))
         line = assert(recurse(line), "Could not convert line to nomsu")
@@ -897,6 +910,7 @@ do
           end
         end
       end
+      nomsu:append(pop_comments(tree.source.stop, '\n'))
       return options.top and nomsu or NomsuCode(tree.source, ":\n    ", nomsu)
     elseif "Text" == _exp_0 then
       if inline then

nomsu_compiler.moon

@@ -481,36 +481,40 @@ with NomsuCompiler
     MIN_COLON_LEN = 20 -- For beautification purposes, don't bother using colon syntax for short bits
     .tree_to_nomsu = (tree, options)=>
         options or= {}
-        unless options.pop_comments
+        options.consumed_comments or= {}
-            comments = [{comment:c, pos:p} for p,c in pairs(tree.comments or {}) when tree.source.start <= p and p <= tree.source.stop]
+        pop_comments = (pos, prefix='', suffix='')->
-            table.sort comments, (a,b)-> a.pos < b.pos
+            find_comments = (t)->
-            comment_i = 1
+                if t.comments and t.source.filename == tree.source.filename
-            options.pop_comments = (pos, prefix='')->
+                    for c in *t.comments
-                nomsu = NomsuCode(tree.source)
+                        coroutine.yield(c) unless options.consumed_comments[c]
-                while comments[comment_i] and comments[comment_i].pos <= pos
+                for x in *t
-                    comment = comments[comment_i].comment
+                    find_comments(x) if AST.is_syntax_tree x
-                    nomsu\append("#"..(gsub(comment, "\n", "\n    ")).."\n")
+            nomsu = NomsuCode(tree.source)
-                    if comment\match("^\n.") then nomsu\append("\n") -- for aesthetics
+            for comment in coroutine.wrap(-> find_comments(tree))
-                    comment_i += 1
+                break if comment.pos > pos
-                return '' if #nomsu.bits == 0
+                options.consumed_comments[comment] = true
-                nomsu\prepend prefix
+                nomsu\append("#"..(gsub(comment.comment, "\n", "\n    ")).."\n")
-                return nomsu
+                if comment.comment\match("^\n.") then nomsu\append("\n") -- for aesthetics
+            return '' if #nomsu.bits == 0
+            nomsu\prepend prefix
+            nomsu\append suffix
+            return nomsu
 
         recurse = (t, opts)->
             opts or= {}
-            opts.pop_comments = options.pop_comments
+            opts.consumed_comments = options.consumed_comments
             return @tree_to_nomsu(t, opts)
 
-        {:inline, :pop_comments} = options
+        {:inline} = options
         switch tree.type
             when "FileChunks"
                 error("Cannot inline a FileChunks") if inline
-                nomsu = NomsuCode(tree.source)
+                nomsu = NomsuCode(tree.source, pop_comments(tree.source.start))
                 for i, chunk in ipairs tree
                     nomsu\append "\n\n#{("~")\rep(80)}\n\n" if i > 1
                     nomsu\append pop_comments(chunk.source.start)
                     nomsu\append recurse(chunk, top:true)
-                nomsu\append pop_comments(tree.source.stop)
+                nomsu\append pop_comments(tree.source.stop, '\n')
                 return nomsu
 
             when "Action"
@@ -523,16 +527,19 @@ with NomsuCompiler
                         else
                             arg_nomsu = recurse(bit,inline:true)
                             return nil unless arg_nomsu
-                            unless i == 1
+                            unless i == 1 or (bit.type == "Block" and not (#bit > 1 or i < #tree))
                                 nomsu\append " "
                             if bit.type == "Action" or (bit.type == "Block" and (#bit > 1 or i < #tree))
                                 arg_nomsu\parenthesize!
                             nomsu\append arg_nomsu
                     return nomsu
                 else
-                    nomsu = NomsuCode(tree.source)
+                    pos = tree.source.start
+                    nomsu = NomsuCode(tree.source, pop_comments(pos))
                     next_space = ""
                     for i,bit in ipairs tree
+                        if match(next_space, '\n')
+                            nomsu\append pop_comments(pos, '\n')
                         if type(bit) == "string"
                             nomsu\append next_space, bit
                             next_space = " "
@@ -557,9 +564,11 @@ with NomsuCompiler
                                 else
                                     nomsu\append arg_nomsu
                                 next_space = "\n.."
+                            pos = bit.source.stop
 
                         if next_space == " " and nomsu.trailing_line_len > MAX_LINE
                             next_space = "\n.."
+                    nomsu\append pop_comments(pos, '\n')
                     return nomsu
 
             when "EscapedNomsu"
@@ -580,7 +589,7 @@ with NomsuCompiler
                         nomsu\append(i == 1 and " " or "; ")
                         nomsu\append assert(recurse(line, inline:true))
                     return nomsu
-                nomsu = NomsuCode(tree.source)
+                nomsu = NomsuCode(tree.source, pop_comments(tree.source.start))
                 for i, line in ipairs tree
                     nomsu\append pop_comments(line.source.start)
                     line = assert(recurse(line), "Could not convert line to nomsu")
@@ -589,6 +598,7 @@ with NomsuCompiler
                         nomsu\append "\n"
                         if match(tostring(line), "\n")
                             nomsu\append "\n"
+                nomsu\append pop_comments(tree.source.stop, '\n')
                 return options.top and nomsu or NomsuCode(tree.source, ":\n    ", nomsu)
 
             when "Text"

nomsu_tree.lua

@@ -53,24 +53,52 @@ for _index_0 = 1, #types do
     end
     cls.map = function(self, fn)
       local replacement = fn(self)
-      if replacement ~= nil then
+      if replacement == false then
-        return replacement or nil
+        return nil
       end
-      local replacements = { }
+      if replacement then
-      local changes = false
+        replacement = (replacement.__class)(self.source, unpack(replacement))
-      for i, v in ipairs(self) do
+      else
-        if AST.is_syntax_tree(v) then
+        local replacements = { }
-          replacement = v:map(fn)
+        local changes = false
-        else
+        for i, v in ipairs(self) do
-          replacement = v
+          local _continue_0 = false
+          repeat
+            replacements[#replacements + 1] = v
+            if AST.is_syntax_tree(v) then
+              local r = v:map(fn)
+              if r == v or r == nil then
+                _continue_0 = true
+                break
+              end
+              changes = true
+              replacements[#replacements] = r
+            end
+            _continue_0 = true
+          until true
+          if not _continue_0 then
+            break
+          end
         end
-        changes = changes or (replacement ~= v)
+        if not (changes) then
-        replacements[#replacements + 1] = replacement
+          return self
+        end
+        replacement = (self.__class)(self.source, unpack(replacements))
       end
-      if not (changes) then
+      if self.comments then
-        return self
+        do
+          local _accum_0 = { }
+          local _len_0 = 1
+          local _list_0 = self.comments
+          for _index_1 = 1, #_list_0 do
+            local c = _list_0[_index_1]
+            _accum_0[_len_0] = c
+            _len_0 = _len_0 + 1
+          end
+          replacement.comments = _accum_0
+        end
       end
-      return (self.__class)(self.source, unpack(replacements))
+      return replacement
     end
   end
   AST[name] = setmetatable(cls, {

nomsu_tree.moon

@@ -22,17 +22,24 @@ for name in *types
         .__tostring = => "#{@type}(#{repr tostring(@source)}, #{concat([repr(v) for v in *@], ', ')})"
         .map = (fn)=>
             replacement = fn(@)
-            if replacement != nil then return replacement or nil
+            if replacement == false then return nil
-            replacements = {}
+            if replacement
-            changes = false
+                -- Clone the replacement, but give it a proper source
-            for i,v in ipairs(@)
+                replacement = (replacement.__class)(@source, unpack(replacement))
-                replacement = if AST.is_syntax_tree(v)
+            else
-                    v\map(fn)
+                replacements = {}
-                else v
+                changes = false
-                changes or= replacement != v
+                for i,v in ipairs(@)
-                replacements[#replacements+1] = replacement
+                    replacements[#replacements+1] = v
-            return @ unless changes
+                    if AST.is_syntax_tree(v)
-            return (@__class)(@source, unpack(replacements))
+                        r = v\map(fn)
+                        continue if r == v or r == nil
+                        changes = true
+                        replacements[#replacements] = r
+                return @ unless changes
+                replacement = (@__class)(@source, unpack(replacements))
+            replacement.comments = [c for c in *@comments] if @comments
+            return replacement
 
     AST[name] = setmetatable cls,
         __tostring: => @name

parser.lua

@@ -100,7 +100,6 @@ setmetatable(NOMSU_DEFS, {
         end
       end
       setmetatable(value, AST[key])
-      value.comments = userdata.comments
       if value.__init then
         value:__init()
       end
@@ -190,6 +189,43 @@ Parser.parse = function(nomsu_code, source, version)
     end
     error("Errors occurred while parsing (v" .. tostring(version) .. "):\n\n" .. table.concat(errors, "\n\n"), 0)
   end
+  local comments
+  do
+    local _accum_0 = { }
+    local _len_0 = 1
+    for p, c in pairs(userdata.comments) do
+      _accum_0[_len_0] = {
+        comment = c,
+        pos = p
+      }
+      _len_0 = _len_0 + 1
+    end
+    comments = _accum_0
+  end
+  table.sort(comments, function(a, b)
+    return a.pos > b.pos
+  end)
+  local comment_i = 1
+  local walk_tree
+  walk_tree = function(t)
+    local comment_buff = { }
+    while comments[#comments] and comments[#comments].pos <= t.source.start do
+      table.insert(comment_buff, table.remove(comments))
+    end
+    for _index_0 = 1, #t do
+      local x = t[_index_0]
+      if AST.is_syntax_tree(x) then
+        walk_tree(x)
+      end
+    end
+    while comments[#comments] and comments[#comments].pos <= t.source.stop do
+      table.insert(comment_buff, table.remove(comments))
+    end
+    if #comment_buff > 0 then
+      t.comments = comment_buff
+    end
+  end
+  walk_tree(tree)
   tree.version = userdata.version
   return tree
 end

parser.moon

@@ -65,7 +65,6 @@ setmetatable(NOMSU_DEFS, {__index:(key)=>
             with userdata.source
                 value.source = Source(.filename, .start + start-1, .start + stop-1)
         setmetatable(value, AST[key])
-        value.comments = userdata.comments
         if value.__init then value\__init!
         return value
 
@@ -119,7 +118,24 @@ Parser.parse = (nomsu_code, source=nil, version=nil)->
         table.sort(keys)
         errors = [userdata.errors[k] for k in *keys]
         error("Errors occurred while parsing (v#{version}):\n\n"..table.concat(errors, "\n\n"), 0)
-    
+
+    comments = [{comment:c, pos:p} for p,c in pairs(userdata.comments)]
+    -- Sort in descending order so we can pop the first comments off the end one at a time
+    table.sort comments, (a,b)-> a.pos > b.pos
+    comment_i = 1
+    walk_tree = (t)->
+        export comment_i
+        comment_buff = {}
+        while comments[#comments] and comments[#comments].pos <= t.source.start
+            table.insert(comment_buff, table.remove(comments))
+        for x in *t
+            if AST.is_syntax_tree x
+                walk_tree x
+        while comments[#comments] and comments[#comments].pos <= t.source.stop
+            table.insert(comment_buff, table.remove(comments))
+        t.comments = comment_buff if #comment_buff > 0
+    walk_tree tree
+
     tree.version = userdata.version
     return tree