Bruce/nomsu
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Overhaul of comment handling, plus a few fixes (e.g. a fix for indented

Browse Source 
text that begins with a nomsu comment)
This commit is contained in:
Bruce Hill 2018-07-17 14:12:11 -07:00
parent be06fc096a
commit 0442c8dd21
13 changed files with 232 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
code_obj.lua
View File

@ -105,11 +105,7 @@ do
self.trailing_line_len = #trailing_text
end
else
if #b.indents > 1 then
self.trailing_line_len = b.trailing_line_len
else
self.trailing_line_len = self.trailing_line_len + #tostring(b)
end
self.trailing_line_len = math.min(self.trailing_line_len + #tostring(b), b.trailing_line_len)
if self.current_indent ~= 0 then
indents[#bits] = self.current_indent
end

5
code_obj.moon
View File

@ -73,10 +73,7 @@ class Code
@current_indent = #spaces
@trailing_line_len = #trailing_text
else
if #b.indents > 1
@trailing_line_len = b.trailing_line_len
else
@trailing_line_len += #tostring(b)
@trailing_line_len = math.min(@trailing_line_len + #tostring(b), b.trailing_line_len)
if @current_indent != 0
indents[#bits] = @current_indent
@__str = nil

11
core/metaprogramming.nom
View File

@ -164,12 +164,11 @@ compile [%tree with %t -> %replacement] to
\(%replacement as lua return)
end)
compile [%tree with vars %v] to
Lua value ".."
\(%tree as lua expr):map(function(t)
local replacements = \(%v as lua expr)
if t.type == "Var" then
return replacements[t[1]]
action [%tree with vars %replacements]
=lua ".."
\%tree:map(function(\%t)
if \%t.type == "Var" then
return \%replacements[\%t[1]]
end
end)

7
nomsu.1.peg
View File

@ -1,6 +1,7 @@
-- Nomsu version 2
file (FileChunks):
{:curr_indent: ' '* :}
("#!" (!"nomsu" [^%nl])* "nomsu" %ws+ "-V" %ws* {:version: ([0-9.]+ -> tonumber) :} [^%nl]*)?
comment? blank_lines?
(chunk (nl_nodent chunk_delimeter nl_nodent chunk)*)?
blank_lines?
@ -100,12 +101,12 @@ inline_text_interpolation:
)
indented_text (Text):
'".."' eol %nl {:curr_indent: indent :}
(indented_plain_text / text_interpolation / {~ blank_lines (=curr_indent -> "") ~})*
'".."' eol %nl {%nl*} {:curr_indent: indent :}
(indented_plain_text / text_interpolation / {~ %nl+ (=curr_indent -> "") ~})*
(!! [^%nl]+ -> "Unexpected character while parsing Text" !!)?
indented_plain_text (Text):
{~ (("\\" -> "\") / (("\" blank_lines =curr_indent "..") -> "") / (!text_interpolation "\") / [^%nl\]+)+
(blank_lines (=curr_indent -> ""))* ~}
(%nl+ (=curr_indent -> ""))* ~}
text_interpolation:
inline_text_interpolation / ("\" indented_expression blank_lines =curr_indent "..")

24
nomsu.2.peg
View File

@ -1,6 +1,7 @@
-- Nomsu version 2
file (FileChunks):
{:curr_indent: ' '* :}
("#!" (!"nomsu" [^%nl])* "nomsu" %ws+ "-V" %ws* {:version: ([0-9.]+ -> tonumber) :} [^%nl]*)?
comment? blank_lines?
(chunk (nl_nodent section_division nl_nodent chunk)*)?
blank_lines?
@ -61,13 +62,16 @@ index_chain (IndexChain):
-- Actions need either at least 1 word, or at least 2 tokens
inline_action (Action):
!section_division
( (inline_expression (%ws* (inline_expression / word))+)
/ (word (%ws* (inline_expression / word))*))
( ((smushed_action / inline_expression) (%ws* (smushed_action / inline_expression / word))+)
/ (word (%ws* (smushed_action / inline_expression / word))*))
(%ws* inline_block)?
action (Action):
!section_division
( (expression ((nl_nodent "..")? %ws* (expression / word))+)
/ (word ((nl_nodent "..")? %ws* (expression / word))*))
( ((smushed_action / expression) ((nl_nodent "..")? %ws* (smushed_action / expression / word))+)
/ (word ((nl_nodent "..")? %ws* (smushed_action / expression / word))*))
smushed_action (Action):
!section_division
(index_chain / noindex_inline_expression / word+) (index_chain / noindex_inline_expression / word+ / "(" %ws* ")")+
word: !number { %operator_char+ / %ident_char+ }
@ -93,20 +97,20 @@ inline_text_interpolation:
)
indented_text (Text):
'".."' eol %nl {:curr_indent: indent :}
(indented_plain_text / text_interpolation / {~ blank_lines (=curr_indent -> "") ~})*
'".."' eol %nl {%nl*} {:curr_indent: indent :}
(indented_plain_text / text_interpolation / {~ %nl+ (=curr_indent -> "") ~})*
(!! [^%nl]+ -> "Unexpected character while parsing Text" !!)?
indented_plain_text (Text):
{~ (("\\" -> "\") / (("\" blank_lines =curr_indent "..") -> "") / (!text_interpolation "\") / [^%nl\]+)+
(blank_lines (=curr_indent -> ""))* ~}
(%nl+ (=curr_indent -> ""))* ~}
text_interpolation:
inline_text_interpolation / ("\" indented_expression blank_lines =curr_indent "..")
number (Number): (("-"? (([0-9]+ "." [0-9]+) / ("." [0-9]+) / ([0-9]+)))-> tonumber)
-- Variables can be nameless (i.e. just %) and can't contain operators like apostrophe
-- which is a hack to allow %'s to parse as "%" and "' s" separately
variable (Var): "%" {(%ident_char+ ((!"'" %operator_char+) / %ident_char+)*)?}
-- Variables can be nameless (i.e. just %) and can only contain identifier chars.
-- This ensures you don't get weird parsings of `%x+%y` or `%'s thing`.
variable (Var): "%" {%ident_char*}
inline_list (List):
!('[..]')

5
nomsu.lua
View File

@ -271,10 +271,11 @@ run = function()
end
end
if #file_queue == 0 then
nomsu:run([[use "core"
nomsu:run([[#!/usr/bin/env nomsu -V2
use "core"
use "lib/consolecolor.nom"
action [quit, exit]: lua> "os.exit(0)"
action [help]
action [help]:
say ".."
This is the Nomsu v\(Nomsu version) interactive console.
You can type in Nomsu code here and hit 'enter' twice to run it.

3
nomsu.moon
View File

@ -175,10 +175,11 @@ run = ->
if #file_queue == 0
-- Run in interactive mode (REPL)
nomsu\run [[
#!/usr/bin/env nomsu -V2
use "core"
use "lib/consolecolor.nom"
action [quit, exit]: lua> "os.exit(0)"
action [help]
action [help]:
say ".."
This is the Nomsu v\(Nomsu version) interactive console.
You can type in Nomsu code here and hit 'enter' twice to run it.

98
nomsu_compiler.lua
View File

@ -722,60 +722,67 @@ do
local MIN_COLON_LEN = 20
NomsuCompiler.tree_to_nomsu = function(self, tree, options)
options = options or { }
if not (options.pop_comments) then
local comments
do
local _accum_0 = { }
local _len_0 = 1
for p, c in pairs(tree.comments or { }) do
if tree.source.start <= p and p <= tree.source.stop then
_accum_0[_len_0] = {
comment = c,
pos = p
}
_len_0 = _len_0 + 1
options.consumed_comments = options.consumed_comments or { }
local pop_comments
pop_comments = function(pos, prefix, suffix)
if prefix == nil then
prefix = ''
end
if suffix == nil then
suffix = ''
end
local find_comments
find_comments = function(t)
if t.comments and t.source.filename == tree.source.filename then
local _list_0 = t.comments
for _index_0 = 1, #_list_0 do
local c = _list_0[_index_0]
if not (options.consumed_comments[c]) then
coroutine.yield(c)
end
end
end
comments = _accum_0
end
table.sort(comments, function(a, b)
return a.pos < b.pos
end)
local comment_i = 1
options.pop_comments = function(pos, prefix)
if prefix == nil then
prefix = ''
end
local nomsu = NomsuCode(tree.source)
while comments[comment_i] and comments[comment_i].pos <= pos do
local comment = comments[comment_i].comment
nomsu:append("#" .. (gsub(comment, "\n", "\n ")) .. "\n")
if comment:match("^\n.") then
nomsu:append("\n")
for _index_0 = 1, #t do
local x = t[_index_0]
if AST.is_syntax_tree(x) then
find_comments(x)
end
comment_i = comment_i + 1
end
if #nomsu.bits == 0 then
return ''
end
nomsu:prepend(prefix)
return nomsu
end
local nomsu = NomsuCode(tree.source)
for comment in coroutine.wrap(function()
return find_comments(tree)
end) do
if comment.pos > pos then
break
end
options.consumed_comments[comment] = true
nomsu:append("#" .. (gsub(comment.comment, "\n", "\n ")) .. "\n")
if comment.comment:match("^\n.") then
nomsu:append("\n")
end
end
if #nomsu.bits == 0 then
return ''
end
nomsu:prepend(prefix)
nomsu:append(suffix)
return nomsu
end
local recurse
recurse = function(t, opts)
opts = opts or { }
opts.pop_comments = options.pop_comments
opts.consumed_comments = options.consumed_comments
return self:tree_to_nomsu(t, opts)
end
local inline, pop_comments
inline, pop_comments = options.inline, options.pop_comments
local inline
inline = options.inline
local _exp_0 = tree.type
if "FileChunks" == _exp_0 then
if inline then
error("Cannot inline a FileChunks")
end
local nomsu = NomsuCode(tree.source)
local nomsu = NomsuCode(tree.source, pop_comments(tree.source.start))
for i, chunk in ipairs(tree) do
if i > 1 then
nomsu:append("\n\n" .. tostring(("~"):rep(80)) .. "\n\n")
@ -785,7 +792,7 @@ do
top = true
}))
end
nomsu:append(pop_comments(tree.source.stop))
nomsu:append(pop_comments(tree.source.stop, '\n'))
return nomsu
elseif "Action" == _exp_0 then
if inline then
@ -803,7 +810,7 @@ do
if not (arg_nomsu) then
return nil
end
if not (i == 1) then
if not (i == 1 or (bit.type == "Block" and not (#bit > 1 or i < #tree))) then
nomsu:append(" ")
end
if bit.type == "Action" or (bit.type == "Block" and (#bit > 1 or i < #tree)) then
@ -814,9 +821,13 @@ do
end
return nomsu
else
local nomsu = NomsuCode(tree.source)
local pos = tree.source.start
local nomsu = NomsuCode(tree.source, pop_comments(pos))
local next_space = ""
for i, bit in ipairs(tree) do
if match(next_space, '\n') then
nomsu:append(pop_comments(pos, '\n'))
end
if type(bit) == "string" then
nomsu:append(next_space, bit)
next_space = " "
@ -853,11 +864,13 @@ do
end
next_space = "\n.."
end
pos = bit.source.stop
end
if next_space == " " and nomsu.trailing_line_len > MAX_LINE then
next_space = "\n.."
end
end
nomsu:append(pop_comments(pos, '\n'))
return nomsu
end
elseif "EscapedNomsu" == _exp_0 then
@ -885,7 +898,7 @@ do
end
return nomsu
end
local nomsu = NomsuCode(tree.source)
local nomsu = NomsuCode(tree.source, pop_comments(tree.source.start))
for i, line in ipairs(tree) do
nomsu:append(pop_comments(line.source.start))
line = assert(recurse(line), "Could not convert line to nomsu")
@ -897,6 +910,7 @@ do
end
end
end
nomsu:append(pop_comments(tree.source.stop, '\n'))
return options.top and nomsu or NomsuCode(tree.source, ":\n ", nomsu)
elseif "Text" == _exp_0 then
if inline then

52
nomsu_compiler.moon
View File

@ -481,36 +481,40 @@ with NomsuCompiler
MIN_COLON_LEN = 20 -- For beautification purposes, don't bother using colon syntax for short bits
.tree_to_nomsu = (tree, options)=>
options or= {}
unless options.pop_comments
comments = [{comment:c, pos:p} for p,c in pairs(tree.comments or {}) when tree.source.start <= p and p <= tree.source.stop]
table.sort comments, (a,b)-> a.pos < b.pos
comment_i = 1
options.pop_comments = (pos, prefix='')->
nomsu = NomsuCode(tree.source)
while comments[comment_i] and comments[comment_i].pos <= pos
comment = comments[comment_i].comment
nomsu\append("#"..(gsub(comment, "\n", "\n ")).."\n")
if comment\match("^\n.") then nomsu\append("\n") -- for aesthetics
comment_i += 1
return '' if #nomsu.bits == 0
nomsu\prepend prefix
return nomsu
options.consumed_comments or= {}
pop_comments = (pos, prefix='', suffix='')->
find_comments = (t)->
if t.comments and t.source.filename == tree.source.filename
for c in *t.comments
coroutine.yield(c) unless options.consumed_comments[c]
for x in *t
find_comments(x) if AST.is_syntax_tree x
nomsu = NomsuCode(tree.source)
for comment in coroutine.wrap(-> find_comments(tree))
break if comment.pos > pos
options.consumed_comments[comment] = true
nomsu\append("#"..(gsub(comment.comment, "\n", "\n ")).."\n")
if comment.comment\match("^\n.") then nomsu\append("\n") -- for aesthetics
return '' if #nomsu.bits == 0
nomsu\prepend prefix
nomsu\append suffix
return nomsu
recurse = (t, opts)->
opts or= {}
opts.pop_comments = options.pop_comments
opts.consumed_comments = options.consumed_comments
return @tree_to_nomsu(t, opts)
{:inline, :pop_comments} = options
{:inline} = options
switch tree.type
when "FileChunks"
error("Cannot inline a FileChunks") if inline
nomsu = NomsuCode(tree.source)
nomsu = NomsuCode(tree.source, pop_comments(tree.source.start))
for i, chunk in ipairs tree
nomsu\append "\n\n#{("~")\rep(80)}\n\n" if i > 1
nomsu\append pop_comments(chunk.source.start)
nomsu\append recurse(chunk, top:true)
nomsu\append pop_comments(tree.source.stop)
nomsu\append pop_comments(tree.source.stop, '\n')
return nomsu
when "Action"
@ -523,16 +527,19 @@ with NomsuCompiler
else
arg_nomsu = recurse(bit,inline:true)
return nil unless arg_nomsu
unless i == 1
unless i == 1 or (bit.type == "Block" and not (#bit > 1 or i < #tree))
nomsu\append " "
if bit.type == "Action" or (bit.type == "Block" and (#bit > 1 or i < #tree))
arg_nomsu\parenthesize!
nomsu\append arg_nomsu
return nomsu
else
nomsu = NomsuCode(tree.source)
pos = tree.source.start
nomsu = NomsuCode(tree.source, pop_comments(pos))
next_space = ""
for i,bit in ipairs tree
if match(next_space, '\n')
nomsu\append pop_comments(pos, '\n')
if type(bit) == "string"
nomsu\append next_space, bit
next_space = " "
@ -557,9 +564,11 @@ with NomsuCompiler
else
nomsu\append arg_nomsu
next_space = "\n.."
pos = bit.source.stop
if next_space == " " and nomsu.trailing_line_len > MAX_LINE
next_space = "\n.."
nomsu\append pop_comments(pos, '\n')
return nomsu
when "EscapedNomsu"
@ -580,7 +589,7 @@ with NomsuCompiler
nomsu\append(i == 1 and " " or "; ")
nomsu\append assert(recurse(line, inline:true))
return nomsu
nomsu = NomsuCode(tree.source)
nomsu = NomsuCode(tree.source, pop_comments(tree.source.start))
for i, line in ipairs tree
nomsu\append pop_comments(line.source.start)
line = assert(recurse(line), "Could not convert line to nomsu")
@ -589,6 +598,7 @@ with NomsuCompiler
nomsu\append "\n"
if match(tostring(line), "\n")
nomsu\append "\n"
nomsu\append pop_comments(tree.source.stop, '\n')
return options.top and nomsu or NomsuCode(tree.source, ":\n ", nomsu)
when "Text"

56
nomsu_tree.lua
View File

@ -53,24 +53,52 @@ for _index_0 = 1, #types do
end
cls.map = function(self, fn)
local replacement = fn(self)
if replacement ~= nil then
return replacement or nil
if replacement == false then
return nil
end
local replacements = { }
local changes = false
for i, v in ipairs(self) do
if AST.is_syntax_tree(v) then
replacement = v:map(fn)
else
replacement = v
if replacement then
replacement = (replacement.__class)(self.source, unpack(replacement))
else
local replacements = { }
local changes = false
for i, v in ipairs(self) do
local _continue_0 = false
repeat
replacements[#replacements + 1] = v
if AST.is_syntax_tree(v) then
local r = v:map(fn)
if r == v or r == nil then
_continue_0 = true
break
end
changes = true
replacements[#replacements] = r
end
_continue_0 = true
until true
if not _continue_0 then
break
end
end
changes = changes or (replacement ~= v)
replacements[#replacements + 1] = replacement
if not (changes) then
return self
end
replacement = (self.__class)(self.source, unpack(replacements))
end
if not (changes) then
return self
if self.comments then
do
local _accum_0 = { }
local _len_0 = 1
local _list_0 = self.comments
for _index_1 = 1, #_list_0 do
local c = _list_0[_index_1]
_accum_0[_len_0] = c
_len_0 = _len_0 + 1
end
replacement.comments = _accum_0
end
end
return (self.__class)(self.source, unpack(replacements))
return replacement
end
end
AST[name] = setmetatable(cls, {

29
nomsu_tree.moon
View File

@ -22,17 +22,24 @@ for name in *types
.__tostring = => "#{@type}(#{repr tostring(@source)}, #{concat([repr(v) for v in *@], ', ')})"
.map = (fn)=>
replacement = fn(@)
if replacement != nil then return replacement or nil
replacements = {}
changes = false
for i,v in ipairs(@)
replacement = if AST.is_syntax_tree(v)
v\map(fn)
else v
changes or= replacement != v
replacements[#replacements+1] = replacement
return @ unless changes
return (@__class)(@source, unpack(replacements))
if replacement == false then return nil
if replacement
-- Clone the replacement, but give it a proper source
replacement = (replacement.__class)(@source, unpack(replacement))
else
replacements = {}
changes = false
for i,v in ipairs(@)
replacements[#replacements+1] = v
if AST.is_syntax_tree(v)
r = v\map(fn)
continue if r == v or r == nil
changes = true
replacements[#replacements] = r
return @ unless changes
replacement = (@__class)(@source, unpack(replacements))
replacement.comments = [c for c in *@comments] if @comments
return replacement
AST[name] = setmetatable cls,
__tostring: => @name

38
parser.lua
View File

@ -100,7 +100,6 @@ setmetatable(NOMSU_DEFS, {
end
end
setmetatable(value, AST[key])
value.comments = userdata.comments
if value.__init then
value:__init()
end
@ -190,6 +189,43 @@ Parser.parse = function(nomsu_code, source, version)
end
error("Errors occurred while parsing (v" .. tostring(version) .. "):\n\n" .. table.concat(errors, "\n\n"), 0)
end
local comments
do
local _accum_0 = { }
local _len_0 = 1
for p, c in pairs(userdata.comments) do
_accum_0[_len_0] = {
comment = c,
pos = p
}
_len_0 = _len_0 + 1
end
comments = _accum_0
end
table.sort(comments, function(a, b)
return a.pos > b.pos
end)
local comment_i = 1
local walk_tree
walk_tree = function(t)
local comment_buff = { }
while comments[#comments] and comments[#comments].pos <= t.source.start do
table.insert(comment_buff, table.remove(comments))
end
for _index_0 = 1, #t do
local x = t[_index_0]
if AST.is_syntax_tree(x) then
walk_tree(x)
end
end
while comments[#comments] and comments[#comments].pos <= t.source.stop do
table.insert(comment_buff, table.remove(comments))
end
if #comment_buff > 0 then
t.comments = comment_buff
end
end
walk_tree(tree)
tree.version = userdata.version
return tree
end

20
parser.moon
View File

@ -65,7 +65,6 @@ setmetatable(NOMSU_DEFS, {__index:(key)=>
with userdata.source
value.source = Source(.filename, .start + start-1, .start + stop-1)
setmetatable(value, AST[key])
value.comments = userdata.comments
if value.__init then value\__init!
return value
@ -119,7 +118,24 @@ Parser.parse = (nomsu_code, source=nil, version=nil)->
table.sort(keys)
errors = [userdata.errors[k] for k in *keys]
error("Errors occurred while parsing (v#{version}):\n\n"..table.concat(errors, "\n\n"), 0)
comments = [{comment:c, pos:p} for p,c in pairs(userdata.comments)]
-- Sort in descending order so we can pop the first comments off the end one at a time
table.sort comments, (a,b)-> a.pos > b.pos
comment_i = 1
walk_tree = (t)->
export comment_i
comment_buff = {}
while comments[#comments] and comments[#comments].pos <= t.source.start
table.insert(comment_buff, table.remove(comments))
for x in *t
if AST.is_syntax_tree x
walk_tree x
while comments[#comments] and comments[#comments].pos <= t.source.stop
table.insert(comment_buff, table.remove(comments))
t.comments = comment_buff if #comment_buff > 0
walk_tree tree
tree.version = userdata.version
return tree