Better parsing of strings and handling of non-printable characters and

tabs.

7 files changed, 35 insertions, 24 deletions

diff --git a/core/collections.nom b/core/collections.nom
index 5daaa09..8dbc79d 100644
--- a/core/collections.nom
+++ b/core/collections.nom
@@ -84,8 +84,7 @@ parse [..]
 # Dict comprehensions
 test:
     assume (((% * %) = % for % in [1, 2, 3]) == {1:1, 4:2, 9:3})
-parse [%key = %value for %item in %iterable, %key %value for %item in %iterable] as
-..(..)
+parse [%key = %value for %item in %iterable, %key %value for %item in %iterable] as (..)
     result of:
         %comprehension = {}
         for %item in %iterable:
diff --git a/core/math.nom b/core/math.nom
index 11282a4..d6ddbce 100644
--- a/core/math.nom
+++ b/core/math.nom
@@ -99,8 +99,8 @@ action [avg of %items, average of %items] (=lua "(utils.sum(\%items)/#\%items)")
 compile [min of %items, smallest of %items, lowest of %items] to (..)
     Lua value "utils.min(\(%items as lua expr))"
 
-compile [max of %items, biggest of %items, largest of %items, highest of %items] to
-..(Lua value "utils.max(\(%items as lua expr))")
+compile [max of %items, biggest of %items, largest of %items, highest of %items] to (..)
+    Lua value "utils.max(\(%items as lua expr))"
 
 test:
     assume ((min of [3, -4, 1, 2] by % = (% * %)) == 1)
diff --git a/nomsu.4.peg b/nomsu.4.peg
index 77ff404..32a170f 100644
--- a/nomsu.4.peg
+++ b/nomsu.4.peg
@@ -27,12 +27,12 @@ empty_block (Block):
     {:curr_indent: %nil :}
 
 nodent: (unexpected_indent [^%nl]* / =curr_indent)
-indent: =curr_indent ("    ")
+indent: {~ =curr_indent (ws / (%tab -> '    '))+ ~}
 blank_lines: %nl ((nodent comment / ws*) %nl)*
 eol: ws* eol_comment? (!. / &%nl)
 
 nl_nodent: blank_lines nodent
-nl_indent: blank_lines {:curr_indent: indent :} (comment nl_nodent)*
+nl_indent: blank_lines tab_error? {:curr_indent: indent :} (comment nl_nodent)*
 
 comment (Comment):
     "#" {~ [^%nl]* (%nl+ (indent -> '') [^%nl]*)* ~}
@@ -60,12 +60,13 @@ missing_brace_error (Error):
     {:error: {~ eol -> "Line ended before finding a closing }-brace" ~} :}
     {:hint: {~ '' -> 'Put a "}" here' ~} :}
 disallowed_interpolation (Error):
-    {:error: {~ ("\" ('\:' / '(..)' / '[..]' / '{..}') (%nl (&(%nl) / =curr_indent '        ' [^%nl]*))*) -> 
+    {:error: {~ ("\" ('\:' / '(..)' / '[..]' / '{..}') (%nl (&(%nl) / =curr_indent ' ' [^%nl]*))*) -> 
         "Sorry, indented text interpolations are not currently supported on the first line of multi-line text." ~} :}
     {:hint: {~ '' -> 'Move the code for the first line of text to the next line by ending this line with "\" and starting the next line indented with "..", followed by the code for the first line.' ~} :}
-expected_dotdot (Error):
-    {:error: {~ '' -> 'Expected to see a ".." here' ~} :}
-    {:hint: {~ '' -> 'Add a ".." here.' ~} :}
+tab_error (Error):
+    &(=curr_indent %tab)
+    {:error: {~ '' -> 'Tabs are not allowed for indentation.' ~} :}
+    {:hint: {~ '' -> 'Use spaces instead of tabs.' ~} :}
 
 section_division: ("~")^+3 eol
 
@@ -93,7 +94,7 @@ noindex_inline_expression:
 inline_expression: index_chain / noindex_inline_expression 
 indented_expression:
     indented_text / indented_nomsu / indented_list / indented_dict / ({|
-        "(..)" nl_indent
+        "(..)" eol nl_indent
             (action / expression) (eol / unexpected_code)
             (%nl (ws* %nl)* nodent (comment / eol / unexpected_code))*
         {:curr_indent: %nil :}
@@ -118,9 +119,9 @@ inline_action (Action):
 inline_arg: inline_expression / inline_block
 action (Action):
     !section_division
-    ({:target: arg :} (nl_nodent "..")? ws* "::" (nl_nodent "..")? ws*)?
-    (  (arg  ((nl_nodent "..")? ws* (arg / word))+)
-     / (word ((nl_nodent "..")? ws* (arg / word))*))
+    ({:target: arg :} (eol nl_nodent "..")? ws* "::" (eol nl_nodent "..")? ws*)?
+    (  (arg  ((eol nl_nodent "..")? ws* (arg / word))+)
+     / (word ((eol nl_nodent "..")? ws* (arg / word))*))
 arg: expression / inline_block / indented_block
 
 word: !number { operator_char+ / ident_char+ }
@@ -131,8 +132,8 @@ inline_text (Text):
     !(indented_text)
     '"' _inline_text* ('"' / missing_quote_err / unexpected_code)
 _inline_text:
-    {~ (('\"' -> '"') / ('\\' -> '\') / escaped_char / [^%nl\"]+)+ ~}
-    / inline_text_interpolation
+    {~ (('\"' -> '"') / ('\\' -> '\') / escaped_char / text_char+)+ ~}
+    / inline_text_interpolation / illegal_char
 inline_text_interpolation:
     "\" (
         variable / inline_list / inline_dict
@@ -142,14 +143,19 @@ inline_text_interpolation:
             (")" / missing_paren_err / unexpected_code))
     )
 
+text_char: %utf8_char / !["\] %print / %tab
+illegal_char (Error):
+    {:error: {~ (!(%nl / %tab / %print) .) -> "Illegal unprintable character here (it may not be visible, but it's there)" ~} :}
+    {:hint: {~ '' -> "This sort of thing can happen when copying and pasting code. Try deleting and retyping the code." ~} :}
+
 nonterminal_quote:
     '"' &([^%nl] / %nl+ =curr_indent)
 indented_text (Text):
     '"'
     _inline_text* 
-    (('\' %nl+ {:curr_indent: indent :} ('..' / expected_dotdot))
+    (('\' %nl+ {:curr_indent: indent :} ('..')?)
      / disallowed_interpolation? {%nl+} {:curr_indent: indent :})
-    (indented_plain_text / text_interpolation / {~ %nl+ (=curr_indent -> "") ~})*
+    (indented_plain_text / text_interpolation / illegal_char / {~ %nl+ (=curr_indent -> "") ~})*
     ('"' eol / missing_quote_err)
     {:curr_indent: %nil :}
 -- Tracking text-lines-within-indented-text as separate objects allows for better debugging line info
@@ -157,7 +163,7 @@ indented_plain_text (Text):
     {~ 
         ((("\" blank_lines =curr_indent "..") -> "") / ('\\' -> '\')
         / (!text_interpolation ((!("\n") escaped_char) / '\'))
-        / (nonterminal_quote / [^%nl"\])+)+
+        / (nonterminal_quote / text_char)+)+
         (%nl+ (=curr_indent -> ""))*
     ~}
 
@@ -207,7 +213,7 @@ dict_key:
 
 operator_char: ['`~!@$^&*+=|<>?/-]
 ident_char: [a-zA-Z0-9_] / %utf8_char
-ws: [ %tab]
+ws: " "
 
 escaped_char:
     ("\"->'') (
diff --git a/nomsu_compiler.lua b/nomsu_compiler.lua
index d5921cf..bdacdfb 100644
--- a/nomsu_compiler.lua
+++ b/nomsu_compiler.lua
@@ -1217,9 +1217,12 @@ do
         end
       end
       for i, bit in ipairs(tree) do
-        if next_space == "\n.." or (next_space == " " and nomsu:trailing_line_len() > MAX_LINE) then
+        if next_space == "\n.." then
           nomsu:append("\n", pop_comments(pos), '..')
           next_space = ""
+        elseif next_space == " " and nomsu:trailing_line_len() > MAX_LINE then
+          nomsu:append(" \\\n", pop_comments(pos), '..')
+          next_space = ""
         end
         if type(bit) == "string" then
           if not (type(tree[i - 1]) == 'string' and is_operator(tree[i - 1]) ~= is_operator(bit)) then
diff --git a/nomsu_compiler.moon b/nomsu_compiler.moon
index 432a195..a86a9e7 100644
--- a/nomsu_compiler.moon
+++ b/nomsu_compiler.moon
@@ -755,9 +755,12 @@ with NomsuCompiler
                         next_space = target_nomsu\is_multiline! and "\n..::" or "::"
 
                 for i,bit in ipairs tree
-                    if next_space == "\n.." or (next_space == " " and nomsu\trailing_line_len! > MAX_LINE)
+                    if next_space == "\n.."
                         nomsu\append "\n", pop_comments(pos), '..'
                         next_space = ""
+                    elseif next_space == " " and nomsu\trailing_line_len! > MAX_LINE
+                        nomsu\append " \\\n", pop_comments(pos), '..'
+                        next_space = ""
 
                     if type(bit) == "string"
                         unless type(tree[i-1]) == 'string' and is_operator(tree[i-1]) != is_operator(bit)
diff --git a/pretty_errors.lua b/pretty_errors.lua
index 9dad1eb..041c3d1 100644
--- a/pretty_errors.lua
+++ b/pretty_errors.lua
@@ -54,7 +54,7 @@ format_error = function(err)
             local during, after = line:sub(1, err_linepos_end - 1), line:sub(err_linepos_end, -1)
             err_msg = err_msg .. "\n\027[2m" .. tostring(fmt_str:format(i)) .. "\027[0;41;30m" .. tostring(during) .. "\027[0m" .. tostring(after)
           else
-            err_msg = err_msg .. "\n\027[2m" .. tostring(fmt_str:format(i)) .. "\027[0m" .. tostring(line) .. "\027[0m"
+            err_msg = err_msg .. "\n\027[2m" .. tostring(fmt_str:format(i)) .. "\027[0;41;30m" .. tostring(line) .. "\027[0m"
           end
         end
       end
diff --git a/pretty_errors.moon b/pretty_errors.moon
index 5dafe99..300e2c7 100644
--- a/pretty_errors.moon
+++ b/pretty_errors.moon
@@ -44,7 +44,7 @@ format_error = (err)->
                     during, after = line\sub(1,err_linepos_end-1), line\sub(err_linepos_end,-1)
                     err_msg ..= "\n\027[2m#{fmt_str\format(i)}\027[0;41;30m#{during}\027[0m#{after}"
                 else
-                    err_msg ..= "\n\027[2m#{fmt_str\format(i)}\027[0m#{line}\027[0m"
+                    err_msg ..= "\n\027[2m#{fmt_str\format(i)}\027[0;41;30m#{line}\027[0m"
 
     box_width = 70
     err_text = "\027[47;31;1m#{(" "..err.error)\wrap_to_1(box_width)\gsub("\n", "\n\027[47;31;1m ")}"