(546 lines)
1 {:NomsuCode} = require "code_obj"2 {:find, :sub, :match} = string3 {:R,:P,:S} = require 'lpeg'4 re = require 're'5 pretty_error = require("pretty_errors")7 MAX_LINE = 808 GOLDEN_RATIO = ((math.sqrt(5)-1)/2)10 -- Parsing helper functions11 utf8_char_patt = (12 R("\194\223")*R("\128\191") +13 R("\224\239")*R("\128\191")*R("\128\191") +14 R("\240\244")*R("\128\191")*R("\128\191")*R("\128\191"))15 operator_char = S("#'`~@^&*+=<>?/%!|\\-") + (P("\xE2") * (R("\x88\x8B")+R("\xA8\xAB")) * R("\128\191"))16 operator_patt = operator_char^1 * -117 identifier_patt = (R("az","AZ","09") + P("_") + (-operator_char*utf8_char_patt))^1 * -119 is_operator = (s)->20 return type(s) == 'string' and not not operator_patt\match(s)22 is_identifier = (s)->23 return type(s) == 'string' and not not identifier_patt\match(s)25 can_be_unary = (t)->26 t.type == "Action" and #t == 2 and is_operator(t[1]) and type(t[2]) != 'string' and t[2].type != "Block" and not (t[2].type == "Number" and t[1] == "-")28 inline_escaper = re.compile("{~ (%utf8_char / ('\"' -> '\\\"') / ('\n' -> '\\n') / ('\t' -> '\\t') / ('\b' -> '\\b') / ('\a' -> '\\a') / ('\v' -> '\\v') / ('\f' -> '\\f') / ('\r' -> '\\r') / ('\\' -> '\\\\') / ([^ -~] -> escape) / .)* ~}", {utf8_char: utf8_char_patt, escape:(=> ("\\%03d")\format(@byte!))})29 inline_escape = (s)->30 return inline_escaper\match(s)32 escaper = re.compile("{~ (%utf8_char / ('\\' -> '\\\\') / [\n\r\t -~] / (. -> escape))* ~}",33 {utf8_char: utf8_char_patt, escape:(=> ("\\%03d")\format(@byte!))})34 escape = (s)->35 return escaper\match(s)37 tree_to_inline_nomsu = (tree)->38 switch tree.type39 when "Action"40 nomsu = NomsuCode\from(tree.source)41 if can_be_unary(tree)42 nomsu\add tree[1]43 arg_nomsu = tree_to_inline_nomsu(tree[2])44 if tree[2].type == "MethodCall" or tree[2].type == "Action"45 arg_nomsu\parenthesize!46 nomsu\add arg_nomsu47 return nomsu49 num_args, num_words = 0, 050 for i,bit in ipairs tree51 if type(bit) == "string"52 num_words += 153 clump_words = if type(tree[i-1]) == 'string'54 is_operator(bit) != is_operator(tree[i-1])55 else bit == "'"56 nomsu\add " " if i > 1 and not clump_words57 nomsu\add bit58 else59 num_args += 160 arg_nomsu = tree_to_inline_nomsu(bit)61 if tree[i+1] == "'" and bit.type == "Action" and can_be_unary(bit)62 arg_nomsu\parenthesize!63 if bit.type == "Block"64 if i != #tree65 nomsu\add " " if i > 166 arg_nomsu\parenthesize!67 else68 nomsu\add " " if i > 169 if bit.type == "MethodCall"70 arg_nomsu\parenthesize!71 elseif bit.type == "Action" and not can_be_unary(bit)72 arg_nomsu\parenthesize!73 nomsu\add arg_nomsu74 if num_args == 1 and num_words == 075 nomsu\add "()"76 return nomsu78 when "MethodCall"79 target_nomsu = tree_to_inline_nomsu(tree[1])80 if tree[1].type == "Block"81 target_nomsu\parenthesize!82 nomsu = NomsuCode\from(tree.source, target_nomsu, ", ")83 if #tree > 2 then nomsu\add "("84 for i=2,#tree85 nomsu\add "; " if i > 286 nomsu\add tree_to_inline_nomsu(tree[i])87 if #tree > 2 then nomsu\add ")"88 return nomsu90 when "EscapedNomsu"91 inner_nomsu = tree_to_inline_nomsu(tree[1])92 unless tree[1].type == "List" or tree[1].type == "Dict" or tree[1].type == "Var"93 inner_nomsu\parenthesize!94 return NomsuCode\from(tree.source, "\\", inner_nomsu)96 when "Block"97 nomsu = NomsuCode\from(tree.source, ":")98 for i,line in ipairs tree99 nomsu\add(i == 1 and " " or "; ")100 nomsu\add tree_to_inline_nomsu(line)101 nomsu\parenthesize! if #tree > 1102 return nomsu104 when "Text"105 add_text = (nomsu, tree)->106 for i, bit in ipairs tree107 if type(bit) == 'string'108 escaped = inline_escape(bit)109 nomsu\add inline_escape(bit)110 elseif bit.type == "Text"111 add_text(nomsu, bit)112 else113 interp_nomsu = tree_to_inline_nomsu(bit)114 if bit.type != "Var" and bit.type != "List" and bit.type != "Dict"115 interp_nomsu\parenthesize!116 elseif bit.type == "Var" and type(bit[1]) == 'string' and117 type(tree[i+1]) == 'string' and not match(tree[i+1], "^[ \n\t,.:;#(){}[%]]")118 interp_nomsu\parenthesize!119 nomsu\add "\\", interp_nomsu120 nomsu = NomsuCode\from(tree.source)121 add_text(nomsu, tree)122 return NomsuCode\from(tree.source, '"', nomsu, '"')124 when "List", "Dict"125 nomsu = NomsuCode\from(tree.source, (tree.type == "List" and "[" or "{"))126 for i, item in ipairs tree127 nomsu\add ", " if i > 1128 item_nomsu = tree_to_inline_nomsu(item, true)129 --if item.type == "Block" or item.type == "Action" or item.type == "MethodCall"130 -- item_nomsu\parenthesize!131 if item.type == "MethodCall" or (item.type == "Block" and i < #tree)132 item_nomsu\parenthesize!133 nomsu\add item_nomsu134 nomsu\add(tree.type == "List" and "]" or "}")135 return nomsu137 when "DictEntry"138 key, value = tree[1], tree[2]139 nomsu = NomsuCode\from(tree.source)140 -- TODO: remove shim141 if key.type != "Index"142 key = {type:"Index", source:key.source, key}143 nomsu\add tree_to_inline_nomsu(key)144 if value145 nomsu\add " = "146 value_nomsu = tree_to_inline_nomsu(value)147 value_nomsu\parenthesize! if value.type == "Block" or value.type == "Action" or value.type == "MethodCall"148 nomsu\add value_nomsu149 return nomsu151 when "Index"152 key = tree[1]153 nomsu = NomsuCode\from(key.source, ".")154 key_nomsu = if key.type == "Text" and #key == 1 and is_identifier(key[1])155 key[1]156 else157 tree_to_inline_nomsu(key)158 switch key.type159 when "Block", "Action", "MethodCall", "IndexChain"160 key_nomsu\parenthesize!161 return NomsuCode\from(key.source, ".", key_nomsu)163 when "IndexChain"164 nomsu = NomsuCode\from(tree.source)165 target = tree[1]166 target_nomsu = tree_to_inline_nomsu(target)167 switch target.type168 when "Action", "MethodCall", "EscapedNomsu"169 target_nomsu\parenthesize!170 when "Number"171 target_nomsu\parenthesize! if target_nomsu\text!\match("%.")172 nomsu\add target_nomsu173 for i=2,#tree174 -- TODO: remove shim?175 if tree[i].type != "Index"176 tree[i] = {type:"Index", source:tree[i].source, tree[i]}177 nomsu\add tree_to_inline_nomsu(tree[i])178 return nomsu180 when "Number"181 -- Preserve original formatting, just make sure 0xdead_beef -> 0xDEAD_BEEF182 n = tostring(tree[1])183 s = if n\match("^-*0x")184 n\upper!\gsub("0X", "0x")185 elseif tree.hex and tonumber((n\gsub("_",""))) < 0186 ("-0x%X")\format(-tree[1])187 elseif tree.hex188 ("0x%X")\format(tree[1])189 else n190 return NomsuCode\from(tree.source, s)192 when "Var"193 varname = tree[1]194 if type(varname) == "string"195 return NomsuCode\from(tree.source, "$", varname)196 else197 return NomsuCode\from(tree.source, "$(", tree_to_inline_nomsu(varname), ")")199 when "FileChunks"200 error("Can't inline a FileChunks")202 when "Comment"203 -- TODO: implement?204 return NomsuCode\from(tree.source)206 when "Error"207 err_msg = pretty_error{208 title:"Parse error"209 error:tree.error, hint:tree.hint, source:tree\get_source_file!210 start:tree.source.start, stop:tree.source.stop, filename:tree.source.filename211 }212 -- Coroutine yield here?213 error(err_msg)215 else216 error("Unknown type: #{tree.type}")218 tree_to_nomsu = (tree)->219 nomsu = NomsuCode\from(tree.source)221 -- For concision:222 recurse = (t, argnum=nil)->223 space = MAX_LINE - nomsu\trailing_line_len!224 try_inline = true225 for subtree in coroutine.wrap(-> (t\with(coroutine.yield) and nil))226 switch subtree.type227 when "Comment"228 try_inline = false229 when "Block"230 if #subtree > 1231 try_inline = false232 when "Text"233 indented = tree_to_nomsu(subtree)234 indented_lines = [line for line in *indented\text!\lines! when line\match("^ +([^ ].*)")]235 for i=#indented_lines,1,-1236 if indented_lines[i]\match("^ *\\;$")237 table.remove(indented_lines, i)238 if #indented_lines > 1 or (#indented_lines == 1 and #indented_lines[1] > MAX_LINE + 8)239 try_inline = false241 local inline_nomsu242 if try_inline243 inline_nomsu = tree_to_inline_nomsu(t)244 if t.type == "MethodCall"245 inline_nomsu\parenthesize!246 elseif t.type == "Action" and not can_be_unary(t)247 inline_nomsu\parenthesize!248 if #inline_nomsu\text! <= space or #inline_nomsu\text! <= 8249 if t.type != "Text"250 return inline_nomsu251 indented = tree_to_nomsu(t)252 if t.type == "Action" or t.type == "MethodCall"253 if indented\is_multiline!254 if argnum == nil or argnum == 1255 return NomsuCode\from(t.source, "(\n ", indented, "\n)")256 else257 return NomsuCode\from(t.source, "\n ", indented)258 elseif argnum and argnum > 1259 return NomsuCode\from(t.source, "\n ", indented)260 else261 indented\parenthesize!262 indented_lines = [line for line in *indented\text!\lines! when line\match("^ +([^ ].*)")]263 if t.type == "Text"264 for i=#indented_lines,1,-1265 if indented_lines[i]\match("^ *\\;$")266 table.remove(indented_lines, i)267 if inline_nomsu and (#inline_nomsu\text! < MAX_LINE or #inline_nomsu\text! <= space) and268 #indented_lines <= 1269 return inline_nomsu270 return indented272 switch tree.type273 when "FileChunks"274 if tree.shebang275 nomsu\add tree.shebang, "\n"277 for chunk_no, chunk in ipairs tree278 nomsu\add "\n\n#{("~")\rep(80)}\n\n" if chunk_no > 1279 if chunk.type == "Block"280 nomsu\add NomsuCode\from(chunk.source, table.unpack(tree_to_nomsu(chunk).bits, 2))281 else282 nomsu\add tree_to_nomsu(chunk)284 return nomsu286 when "Action"287 if can_be_unary(tree) and not can_be_unary(tree[2])288 nomsu\add tree[1]289 nomsu\add recurse(tree[2])290 return nomsu292 next_space = ""293 word_buffer = {}294 num_args, num_words = 0, 0295 for i,bit in ipairs tree296 -- TODO: properly wrap super long chains of words297 if type(bit) == "string"298 num_words += 1299 if #word_buffer > 0 and is_operator(bit) == is_operator(word_buffer[#word_buffer])300 table.insert word_buffer, " "301 table.insert word_buffer, bit302 continue304 if #word_buffer > 0305 words = table.concat(word_buffer)306 if next_space == " "307 if nomsu\trailing_line_len! + #words > MAX_LINE and nomsu\trailing_line_len! > 8308 next_space = "\n.."309 elseif word_buffer[1] == "'"310 next_space = ""311 nomsu\add next_space, words312 word_buffer = {}313 next_space = " "315 num_args += 1316 bit_nomsu = recurse(bit, i)317 if tree[i+1] == "'" and bit.type == "Action" and not bit_nomsu\is_multiline! and can_be_unary(bit)318 bit_nomsu\parenthesize!319 if bit.type == "Block"320 -- Rule of thumb: nontrivial one-liner block arguments should be no more321 -- than golden ratio * the length of the proceeding part of the line322 if not bit_nomsu\is_multiline! and323 (#bit_nomsu\text! > nomsu\trailing_line_len! * GOLDEN_RATIO and #bit_nomsu\text! > 8) or324 #bit_nomsu\text! + nomsu\trailing_line_len! > MAX_LINE325 bit_nomsu = tree_to_nomsu(bit)326 elseif (not bit_nomsu\is_multiline! and327 nomsu\trailing_line_len! + #bit_nomsu\text! > MAX_LINE and328 nomsu\trailing_line_len! > 8)329 if next_space == " " and #bit_nomsu\text! < MAX_LINE330 if i == #tree331 bit_nomsu = tree_to_inline_nomsu(bit)332 next_space = "\n "333 elseif bit.type == "List" or bit.type == "Dict"334 bit_nomsu = tree_to_nomsu(bit)335 else336 next_space = "\n.."337 elseif bit.type == 'Action' or bit.type == "MethodCall"338 bit_nomsu = NomsuCode\from bit.source, "\n ", tree_to_nomsu(bit)339 else340 bit_nomsu = tree_to_nomsu(bit)342 unless next_space == " " and bit_nomsu\text!\match("^[:\n]")343 nomsu\add next_space345 nomsu\add bit_nomsu346 next_space = (bit.type == "Block" or bit_nomsu\text!\matches("\n [^\n]*$")) and "\n.." or " "348 if #word_buffer > 0349 words = table.concat(word_buffer)350 if next_space == " "351 if nomsu\trailing_line_len! + #words > MAX_LINE + 8 and nomsu\trailing_line_len! > 8352 next_space = "\n.."353 elseif word_buffer[1] == "'"354 next_space = ""355 nomsu\add next_space, words356 next_space = " "358 if num_args == 1 and num_words == 0359 if next_space != " "360 nomsu\add next_space361 nomsu\add "()"362 return nomsu364 when "MethodCall"365 target_nomsu = recurse(tree[1])366 if tree[1].type == "Block" and not target_nomsu\is_multiline!367 target_nomsu\parenthesize!368 nomsu\add target_nomsu, ", "369 inner_nomsu = NomsuCode!370 for i=2,#tree371 inner_nomsu\add "\n" if i > 2372 inner_nomsu\add tree_to_nomsu(tree[i])373 if #tree == 2 and nomsu\trailing_line_len! + #inner_nomsu\text!\match("^[^\n]*") < MAX_LINE374 nomsu\add inner_nomsu375 else376 nomsu\add "\n ", inner_nomsu377 return nomsu379 when "EscapedNomsu"380 nomsu = recurse(tree[1])381 if tree[1].type == 'Block' and not nomsu\is_multiline!382 nomsu\parenthesize!383 return NomsuCode\from tree.source, "\\", nomsu385 when "Block"386 prev_line, needs_space = nil, {}387 for i, line in ipairs tree388 line_nomsu = tree_to_nomsu(line)389 if i > 1390 nomsu\add "\n"391 -- Rule of thumb: add a blank line between two lines if both are392 -- multi-line non-comments, or if a comment comes after a non-comment,393 -- or if the last line starts with ".."394 if tree[i-1].type != "Comment"395 needs_space[i] = (line_nomsu\is_multiline! and prev_line\is_multiline!)396 if (tree[i].type == "Comment" or needs_space[i] or needs_space[i-1] or397 prev_line\text!\match("\n [^\n]*$"))398 nomsu\add "\n"399 nomsu\add line_nomsu400 prev_line = line_nomsu401 return NomsuCode\from(tree.source, ":\n ", nomsu)403 when "Text"404 -- Multi-line text has more generous wrap margins405 max_line = MAX_LINE + 8406 add_text = (tree)->407 for i, bit in ipairs tree408 if type(bit) == 'string'409 bit = escape(bit)410 for j, line in ipairs bit\lines!411 if j > 1412 if nomsu\text!\match(" $")413 nomsu\add "\\;"414 nomsu\add "\n"415 elseif #line > 10 and nomsu\trailing_line_len! > max_line416 nomsu\add "\\\n.."418 while #line > 0419 space = max_line - nomsu\trailing_line_len!420 split = find(line, "[%p%s]", space)421 if not split or split > space + 16422 split = space + 16423 if #line - split < 16424 split = #line425 bite, line = sub(line, 1, split), sub(line, split+1, -1)426 nomsu\add bite427 nomsu\add "\\\n.." if #line > 0428 elseif bit.type == "Text"429 add_text(bit)430 else431 nomsu\add "\\"432 interp_nomsu = recurse(bit)433 if interp_nomsu\is_multiline!434 curr_indent = nomsu\text!\match("\n( *)[^\n]*$") or nomsu\text!\match("^( *)")435 interp_nomsu = NomsuCode((interp_nomsu\text!\gsub("\n", "\n"..curr_indent)))436 else437 space = max_line - nomsu\trailing_line_len!438 if bit.type == "Var"439 next_str = tree[i+1]440 while type(next_str) == 'table' and next_str.type == 'Text'441 next_str = next_str[1]442 if type(next_str) == 'string' and not match(next_str, "^[ \n\t,.:;#(){}[%]]")443 interp_nomsu\parenthesize!444 elseif #interp_nomsu\text! > space445 interp_nomsu2 = if bit.type == "Action" or bit.type == "MethodCall"446 NomsuCode\from(bit.source, "(\n ", tree_to_nomsu(bit), "\n)")447 else448 tree_to_nomsu(bit)450 if #interp_nomsu2\text!\lines! > 3 or #interp_nomsu2\text! >= MAX_LINE*GOLDEN_RATIO451 curr_indent = nomsu\text!\match("\n( *)[^\n]*$") or nomsu\text!\match("^( *)")452 interp_nomsu2 = NomsuCode((interp_nomsu2\text!\gsub("\n", "\n"..curr_indent)))453 interp_nomsu = interp_nomsu2454 else455 nomsu\add "\n..\\"456 if bit.type == "EscapedNomsu" or bit.type == "Block" or bit.type == "IndexChain"457 interp_nomsu\parenthesize!458 elseif bit.type == "EscapedNomsu" or bit.type == "Block" or bit.type == "IndexChain"459 interp_nomsu\parenthesize!460 elseif bit.type == "Action" and can_be_unary(bit)461 interp_nomsu\parenthesize!462 nomsu\add interp_nomsu463 if interp_nomsu\is_multiline! and bit.type == "Block"464 nomsu\add "\n.."465 add_text(tree)466 if nomsu\text!\match(" $")467 nomsu\add "\\;"468 return NomsuCode\from(tree.source, '("\n ', nomsu, '\n")')470 when "List", "Dict"471 if #tree == 0472 nomsu\add(tree.type == "List" and "[]" or "{}")473 return nomsu475 if #tree == 1 and tree[1].type == "Block"476 block_lua = recurse(tree[1])477 if block_lua\is_multiline! then block_lua\add "\n"478 return if tree.type == "List" then479 NomsuCode\from(tree.source, "[", block_lua, "]")480 else481 NomsuCode\from(tree.source, "{", block_lua, "}")483 sep = ''484 prev_item, needs_space = nil, {}485 for i, item in ipairs tree486 local item_nomsu487 if item.type == 'MethodCall'488 item_nomsu = recurse(item)489 elseif item.type == 'Comment'490 item_nomsu = tree_to_nomsu(item)491 sep = '\n' if i > 1492 elseif item.type == 'Block' and #item == 1493 -- Comprehensions use the more concise ": for $ in $: ..." form494 item_nomsu = tree_to_nomsu(item[1])495 item_nomsu\prepend ": "496 sep = '\n' if i > 1497 else498 item_nomsu = tree_to_inline_nomsu(item)499 if nomsu\trailing_line_len! + #item_nomsu\text! > MAX_LINE500 sep = '\n' if i > 1501 item_nomsu = item.type == "Action" and tree_to_nomsu(item) or recurse(item)502 nomsu\add sep503 if sep == '\n'504 -- Rule of thumb: add a blank line between two lines if both are505 -- multi-line non-comments, or if a comment comes after a non-comment,506 -- or if the last line starts with ".."507 if i > 1 and tree[i-1].type != "Comment"508 needs_space[i] = (item_nomsu\is_multiline! and prev_item\is_multiline!)509 if (tree[i].type == "Comment" or needs_space[i] or needs_space[i-1] or510 prev_item\text!\match("\n [^\n]*$"))511 nomsu\add "\n"512 nomsu\add item_nomsu513 prev_item = item_nomsu514 if item_nomsu\is_multiline! or item.type == 'Comment' or item.type == "Block" or515 nomsu\trailing_line_len! + #tostring(item_nomsu) >= MAX_LINE516 sep = '\n'517 else518 sep = ', '519 return if tree.type == "List" then520 NomsuCode\from(tree.source, "[\n ", nomsu, "\n]")521 else522 NomsuCode\from(tree.source, "{\n ", nomsu, "\n}")524 when "DictEntry"525 key, value = tree[1], tree[2]526 nomsu = NomsuCode\from(tree.source)527 -- TODO: remove shim528 if key.type != "Index"529 key = {type:"Index", source:key.source, key}530 nomsu\add tree_to_nomsu(key)531 if value532 value_nomsu = recurse(value)533 nomsu\add " = ", value_nomsu534 return nomsu536 when "Comment"537 nomsu\add "###", (tree[1]\gsub("\n", "\n "))538 return nomsu540 when "IndexChain", "Index", "Number", "Var", "Comment", "Error"541 return tree_to_inline_nomsu tree543 else544 error("Unknown type: #{tree.type}")546 return {:tree_to_nomsu, :tree_to_inline_nomsu}