Fixes for: indented strings ending in ", peeking at code object text

before modifying them, and major upgrades to the nomsu codegen,
including support for indented inline arguments (instead of using
"\n..") and just generally better line wrapping, including finally good
rules for when to use indented text (for almost all cases).

7 files changed, 157 insertions, 72 deletions

diff --git a/code_obj.lua b/code_obj.lua
index b6f7394..5afca1d 100644
--- a/code_obj.lua
+++ b/code_obj.lua
@@ -168,9 +168,6 @@ do
             _continue_0 = true
             break
           end
-          if type(b) ~= 'string' then
-            b.dirty = error
-          end
           bits[#bits + 1] = b
           _continue_0 = true
         until true
diff --git a/code_obj.moon b/code_obj.moon
index 433a178..bdce010 100644
--- a/code_obj.moon
+++ b/code_obj.moon
@@ -100,7 +100,6 @@ class Code
             assert(b, "code bit is nil")
             assert(not Source\is_instance(b), "code bit is a Source")
             if b == '' then continue
-            b.dirty = error if type(b) != 'string'
             bits[#bits+1] = b
         @dirty!
     
diff --git a/nomsu.6.peg b/nomsu.6.peg
index b348c6f..6b4686d 100644
--- a/nomsu.6.peg
+++ b/nomsu.6.peg
@@ -148,8 +148,6 @@ inline_text_interpolation <-
 
 text_char <- %utf8_char / !["\] %print / %tab
 
-nonterminal_quote <- '"' &([^%nl] / (%nl (ws* eol)?)+ =curr_indent [^%nl])
-
 indented_text (Text) <-
     '("' %nl {%nl*} ({|
         {:curr_indent: indent :}
@@ -163,7 +161,7 @@ indented_plain_text (Text) <-
     {~ 
         ((("\" blank_lines =curr_indent "..") -> "") / ('\\' -> '\') / ('\"' -> '"') / ('\;' -> '')
         / (!text_interpolation ((!("\n") escaped_char) / '\'))
-        / (nonterminal_quote / text_char)+)+
+        / ('"' / text_char)+)+
         blank_text_lines?
     ~}
 blank_text_lines <- 
@@ -255,7 +253,7 @@ missing_quote_err (Error) <-
     {:error: {~ eol -> "This text is missing a closing quotation mark." ~} :}
     {:hint: {~ "" -> "Put a quotation mark here." ~} :}
 missing_indented_quote_err (Error) <-
-    {:error: {~ &. -> 'This text is missing a closing ")-quotation mark.' ~} :}
+    {:error: {~ '' -> 'This text is missing a closing ")-quotation mark.' ~} :}
     {:hint: {~ "" -> 'Put a ") after this line, at the same level of indentation as the opening (".' ~} :}
 missing_bracket_error (Error) <-
     {:error: {~ eol -> "This list is missing a closing ]-bracket" ~} :}
diff --git a/nomsu_decompiler.lua b/nomsu_decompiler.lua
index 8e94df8..e17f25e 100644
--- a/nomsu_decompiler.lua
+++ b/nomsu_decompiler.lua
@@ -179,32 +179,33 @@ tree_to_inline_nomsu = function(tree)
     else
       key_nomsu = tree_to_inline_nomsu(key)
     end
-    if key.type == "Block" or key.type == "Action" or key.type == "MethodCall" then
+    local _exp_1 = key.type
+    if "Block" == _exp_1 or "Action" == _exp_1 or "MethodCall" == _exp_1 or "IndexChain" == _exp_1 then
       key_nomsu:parenthesize()
     end
     return NomsuCode:from(key.source, ".", key_nomsu)
   elseif "IndexChain" == _exp_0 then
     local nomsu = NomsuCode:from(tree.source)
-    for i, bit in ipairs(tree) do
-      if i > 1 and bit.type ~= "Index" then
-        nomsu:add(".")
-      end
-      local bit_nomsu
-      if i > 1 and bit.type == "Text" and #bit == 1 and type(bit[1]) == 'string' and is_identifier(bit[1]) then
-        bit_nomsu = bit[1]
-      else
-        bit_nomsu = tree_to_inline_nomsu(bit)
+    local target = tree[1]
+    local target_nomsu = tree_to_inline_nomsu(target)
+    local _exp_1 = target.type
+    if "Action" == _exp_1 or "MethodCall" == _exp_1 then
+      target_nomsu:parenthesize()
+    elseif "Number" == _exp_1 then
+      if target_nomsu:text():match("%.") then
+        target_nomsu:parenthesize()
       end
-      assert(bit.type ~= "Block")
-      local _exp_1 = bit.type
-      if "Action" == _exp_1 or "MethodCall" == _exp_1 or "IndexChain" == _exp_1 then
-        bit_nomsu:parenthesize()
-      elseif "Number" == _exp_1 then
-        if bit_nomsu:text():match("%.") then
-          bit_nomsu:parenthesize()
-        end
+    end
+    nomsu:add(target_nomsu)
+    for i = 2, #tree do
+      if tree[i].type ~= "Index" then
+        tree[i] = {
+          type = "Index",
+          source = tree[i].source,
+          tree[i]
+        }
       end
-      nomsu:add(bit_nomsu)
+      nomsu:add(tree_to_inline_nomsu(tree[i]))
     end
     return nomsu
   elseif "Number" == _exp_0 then
@@ -247,22 +248,46 @@ tree_to_nomsu = function(tree)
     for subtree in coroutine.wrap(function()
       return (t:map(coroutine.yield) and nil)
     end) do
-      if subtree.type == "Comment" then
+      local _exp_0 = subtree.type
+      if "Comment" == _exp_0 then
         try_inline = false
-      end
-      if subtree.type == "Block" then
+      elseif "Block" == _exp_0 then
         if #subtree > 1 then
           try_inline = false
         end
+      elseif "Text" == _exp_0 then
+        local indented = tree_to_nomsu(subtree)
+        local indented_lines
+        do
+          local _accum_0 = { }
+          local _len_0 = 1
+          local _list_0 = indented:text():lines()
+          for _index_0 = 1, #_list_0 do
+            local line = _list_0[_index_0]
+            if line:match("^    +([^ ].*)") then
+              _accum_0[_len_0] = line
+              _len_0 = _len_0 + 1
+            end
+          end
+          indented_lines = _accum_0
+        end
+        for i = #indented_lines, 1, -1 do
+          if indented_lines[i]:match("^ *\\;$") then
+            table.remove(indented_lines, i)
+          end
+        end
+        if #indented_lines > 1 or (#indented_lines == 1 and #indented_lines[1] > MAX_LINE + 8) then
+          try_inline = false
+        end
       end
     end
     local inline_nomsu
     if try_inline then
       inline_nomsu = tree_to_inline_nomsu(t)
+      if (t.type == "Action" or t.type == "MethodCall") then
+        inline_nomsu:parenthesize()
+      end
       if #inline_nomsu:text() <= space or #inline_nomsu:text() <= 8 then
-        if (t.type == "Action" or t.type == "MethodCall") then
-          inline_nomsu:parenthesize()
-        end
         if t.type ~= "Text" then
           return inline_nomsu
         end
@@ -289,16 +314,21 @@ tree_to_nomsu = function(tree)
       local _list_0 = indented:text():lines()
       for _index_0 = 1, #_list_0 do
         local line = _list_0[_index_0]
-        if line:match("%S") then
+        if line:match("^    +([^ ].*)") then
           _accum_0[_len_0] = line
           _len_0 = _len_0 + 1
         end
       end
       indented_lines = _accum_0
     end
-    if inline_nomsu and #indented_lines == ((t.type == 'Block' or t.type == 'Action' or t.type == 'MethodCall') and 2 or 3) and nomsu:trailing_line_len() <= 8 then
-      return inline_nomsu
-    elseif inline_nomsu and t.type == "Text" and #indented_lines <= 3 and (#inline_nomsu:text() - 2 < MAX_LINE + 4 or #inline_nomsu:text() <= space or #inline_nomsu:text() <= 8) then
+    if t.type == "Text" then
+      for i = #indented_lines, 1, -1 do
+        if indented_lines[i]:match("^ *\\;$") then
+          table.remove(indented_lines, i)
+        end
+      end
+    end
+    if inline_nomsu and (#inline_nomsu:text() < MAX_LINE or #inline_nomsu:text() <= space) and #indented_lines <= 1 then
       return inline_nomsu
     end
     return indented
@@ -353,12 +383,19 @@ tree_to_nomsu = function(tree)
         num_args = num_args + 1
         local bit_nomsu = recurse(bit, i)
         if bit.type == "Block" then
-          if not bit_nomsu:is_multiline() and #bit_nomsu:text() > nomsu:trailing_line_len() * GOLDEN_RATIO and #bit_nomsu:text() > 8 then
+          if not bit_nomsu:is_multiline() and (#bit_nomsu:text() > nomsu:trailing_line_len() * GOLDEN_RATIO and #bit_nomsu:text() > 8) or #bit_nomsu:text() + nomsu:trailing_line_len() > MAX_LINE then
             bit_nomsu = tree_to_nomsu(bit)
           end
         elseif (not bit_nomsu:is_multiline() and nomsu:trailing_line_len() + #bit_nomsu:text() > MAX_LINE and nomsu:trailing_line_len() > 8) then
           if next_space == " " and #bit_nomsu:text() < MAX_LINE then
-            next_space = "\n.."
+            if i == #tree then
+              bit_nomsu = tree_to_inline_nomsu(bit)
+              next_space = "\n    "
+            elseif bit.type == "List" or bit.type == "Dict" then
+              bit_nomsu = tree_to_nomsu(bit)
+            else
+              next_space = "\n.."
+            end
           elseif bit.type == 'Action' or bit.type == "MethodCall" then
             bit_nomsu = NomsuCode:from(bit.source, "\n    ", tree_to_nomsu(bit))
           else
@@ -369,7 +406,7 @@ tree_to_nomsu = function(tree)
           nomsu:add(next_space)
         end
         nomsu:add(bit_nomsu)
-        next_space = (bit.type == 'Block' or (i > 1 and (bit.type == 'Action' or bit.type == 'MethodCall')) and bit_nomsu:is_multiline()) and "\n.." or " "
+        next_space = (bit.type == "Block" or bit_nomsu:text():matches("\n    [^\n]*$")) and "\n.." or " "
         _continue_0 = true
       until true
       if not _continue_0 then
@@ -470,6 +507,7 @@ tree_to_nomsu = function(tree)
           nomsu:add("\\")
           local interp_nomsu = recurse(bit)
           if not (interp_nomsu:is_multiline()) then
+            local space = max_line - nomsu:trailing_line_len()
             if bit.type == "Var" then
               local next_str = tree[i + 1]
               while type(next_str) == 'table' and next_str.type == 'Text' do
@@ -478,6 +516,21 @@ tree_to_nomsu = function(tree)
               if type(next_str) == 'string' and not match(next_str, "^[ \n\t,.:;#(){}[%]]") then
                 interp_nomsu:parenthesize()
               end
+            elseif #interp_nomsu:text() > space then
+              local interp_nomsu2
+              if bit.type == "Action" or bit.type == "MethodCall" then
+                interp_nomsu2 = NomsuCode:from(bit.source, "(\n    ", tree_to_nomsu(bit), "\n)")
+              else
+                interp_nomsu2 = tree_to_nomsu(bit)
+              end
+              if #interp_nomsu2:text():lines() > 3 or #interp_nomsu2:text() >= MAX_LINE * GOLDEN_RATIO then
+                interp_nomsu = interp_nomsu2
+              else
+                nomsu:add("\n..\\")
+                if bit.type == "EscapedNomsu" or bit.type == "Block" or bit.type == "IndexChain" then
+                  interp_nomsu:parenthesize()
+                end
+              end
             elseif bit.type == "EscapedNomsu" or bit.type == "Block" or bit.type == "IndexChain" then
               interp_nomsu:parenthesize()
             end
@@ -512,7 +565,10 @@ tree_to_nomsu = function(tree)
       else
         item_nomsu = tree_to_inline_nomsu(item)
         if #item_nomsu:text() > MAX_LINE then
-          item_nomsu = recurse(item)
+          if i > 1 then
+            sep = '\n'
+          end
+          item_nomsu = tree_to_nomsu(item)
         end
       end
       nomsu:add(sep)
diff --git a/nomsu_decompiler.moon b/nomsu_decompiler.moon
index 181d73f..3c20f76 100644
--- a/nomsu_decompiler.moon
+++ b/nomsu_decompiler.moon
@@ -133,24 +133,26 @@ tree_to_inline_nomsu = (tree)->
                 key[1]
             else
                 tree_to_inline_nomsu(key)
-            key_nomsu\parenthesize! if key.type == "Block" or key.type == "Action" or key.type == "MethodCall"
+            switch key.type
+                when "Block", "Action", "MethodCall", "IndexChain"
+                    key_nomsu\parenthesize!
             return NomsuCode\from(key.source, ".", key_nomsu)
         
         when "IndexChain"
             nomsu = NomsuCode\from(tree.source)
-            for i, bit in ipairs tree
-                nomsu\add "." if i > 1 and bit.type != "Index"
-                local bit_nomsu
-                bit_nomsu = if i > 1 and bit.type == "Text" and #bit == 1 and type(bit[1]) == 'string' and is_identifier(bit[1])
-                    bit[1]
-                else tree_to_inline_nomsu(bit)
-                assert bit.type != "Block"
-                switch bit.type
-                    when "Action", "MethodCall", "IndexChain"
-                        bit_nomsu\parenthesize!
-                    when "Number"
-                        bit_nomsu\parenthesize! if bit_nomsu\text!\match("%.")
-                nomsu\add bit_nomsu
+            target = tree[1]
+            target_nomsu = tree_to_inline_nomsu(target)
+            switch target.type
+                when "Action", "MethodCall"
+                    target_nomsu\parenthesize!
+                when "Number"
+                    target_nomsu\parenthesize! if target_nomsu\text!\match("%.")
+            nomsu\add target_nomsu
+            for i=2,#tree
+                -- TODO: remove shim?
+                if tree[i].type != "Index"
+                    tree[i] = {type:"Index", source:tree[i].source, tree[i]}
+                nomsu\add tree_to_inline_nomsu(tree[i])
             return nomsu
         
         when "Number"
@@ -189,18 +191,27 @@ tree_to_nomsu = (tree)->
         space = MAX_LINE - nomsu\trailing_line_len!
         try_inline = true
         for subtree in coroutine.wrap(-> (t\map(coroutine.yield) and nil))
-            if subtree.type == "Comment"
-                try_inline = false
-            if subtree.type == "Block"
-                if #subtree > 1
+            switch subtree.type
+                when "Comment"
                     try_inline = false
+                when "Block"
+                    if #subtree > 1
+                        try_inline = false
+                when "Text"
+                    indented = tree_to_nomsu(subtree)
+                    indented_lines = [line for line in *indented\text!\lines! when line\match("^    +([^ ].*)")]
+                    for i=#indented_lines,1,-1
+                        if indented_lines[i]\match("^ *\\;$")
+                            table.remove(indented_lines, i)
+                    if #indented_lines > 1 or (#indented_lines == 1 and #indented_lines[1] > MAX_LINE + 8)
+                        try_inline = false
         
         local inline_nomsu
         if try_inline
             inline_nomsu = tree_to_inline_nomsu(t)
+            if (t.type == "Action" or t.type == "MethodCall")
+                inline_nomsu\parenthesize!
             if #inline_nomsu\text! <= space or #inline_nomsu\text! <= 8
-                if (t.type == "Action" or t.type == "MethodCall")
-                    inline_nomsu\parenthesize!
                 if t.type != "Text"
                     return inline_nomsu
         indented = tree_to_nomsu(t)
@@ -214,11 +225,13 @@ tree_to_nomsu = (tree)->
                 return NomsuCode\from(t.source, "\n    ", indented)
             else
                 indented\parenthesize!
-        indented_lines = [line for line in *indented\text!\lines! when line\match("%S")]
-        if inline_nomsu and #indented_lines == ((t.type == 'Block' or t.type == 'Action' or t.type == 'MethodCall') and 2 or 3) and nomsu\trailing_line_len! <= 8
-            return inline_nomsu
-        elseif inline_nomsu and t.type == "Text" and #indented_lines <= 3 and
-            (#inline_nomsu\text! - 2 < MAX_LINE + 4 or #inline_nomsu\text! <= space or #inline_nomsu\text! <= 8)
+        indented_lines = [line for line in *indented\text!\lines! when line\match("^    +([^ ].*)")]
+        if t.type == "Text"
+            for i=#indented_lines,1,-1
+                if indented_lines[i]\match("^ *\\;$")
+                    table.remove(indented_lines, i)
+        if inline_nomsu and (#inline_nomsu\text! < MAX_LINE or #inline_nomsu\text! <= space) and
+            #indented_lines <= 1
             return inline_nomsu
         return indented
 
@@ -265,13 +278,21 @@ tree_to_nomsu = (tree)->
                 if bit.type == "Block"
                     -- Rule of thumb: nontrivial one-liner block arguments should be no more
                     -- than golden ratio * the length of the proceeding part of the line
-                    if not bit_nomsu\is_multiline! and #bit_nomsu\text! > nomsu\trailing_line_len! * GOLDEN_RATIO and #bit_nomsu\text! > 8
+                    if not bit_nomsu\is_multiline! and
+                        (#bit_nomsu\text! > nomsu\trailing_line_len! * GOLDEN_RATIO and #bit_nomsu\text! > 8) or
+                        #bit_nomsu\text! + nomsu\trailing_line_len! > MAX_LINE
                         bit_nomsu = tree_to_nomsu(bit)
                 elseif (not bit_nomsu\is_multiline! and
                     nomsu\trailing_line_len! + #bit_nomsu\text! > MAX_LINE and
                     nomsu\trailing_line_len! > 8)
                     if next_space == " " and #bit_nomsu\text! < MAX_LINE
-                        next_space = "\n.."
+                        if i == #tree
+                            bit_nomsu = tree_to_inline_nomsu(bit)
+                            next_space = "\n    "
+                        elseif bit.type == "List" or bit.type == "Dict"
+                            bit_nomsu = tree_to_nomsu(bit)
+                        else
+                            next_space = "\n.."
                     elseif bit.type == 'Action' or bit.type == "MethodCall"
                         bit_nomsu = NomsuCode\from bit.source, "\n    ", tree_to_nomsu(bit)
                     else
@@ -281,7 +302,7 @@ tree_to_nomsu = (tree)->
                     nomsu\add next_space
 
                 nomsu\add bit_nomsu
-                next_space = (bit.type == 'Block' or (i > 1 and (bit.type == 'Action' or bit.type == 'MethodCall')) and bit_nomsu\is_multiline!) and "\n.." or " "
+                next_space = (bit.type == "Block" or bit_nomsu\text!\matches("\n    [^\n]*$")) and "\n.." or " "
 
             if #word_buffer > 0
                 words = table.concat(word_buffer)
@@ -365,12 +386,25 @@ tree_to_nomsu = (tree)->
                         nomsu\add "\\"
                         interp_nomsu = recurse(bit)
                         unless interp_nomsu\is_multiline!
+                            space = max_line - nomsu\trailing_line_len!
                             if bit.type == "Var"
                                 next_str = tree[i+1]
                                 while type(next_str) == 'table' and next_str.type == 'Text'
                                     next_str = next_str[1]
                                 if type(next_str) == 'string' and not match(next_str, "^[ \n\t,.:;#(){}[%]]")
                                     interp_nomsu\parenthesize!
+                            elseif #interp_nomsu\text! > space
+                                interp_nomsu2 = if bit.type == "Action" or bit.type == "MethodCall"
+                                    NomsuCode\from(bit.source, "(\n    ", tree_to_nomsu(bit), "\n)")
+                                else
+                                    tree_to_nomsu(bit)
+
+                                if #interp_nomsu2\text!\lines! > 3 or #interp_nomsu2\text! >= MAX_LINE*GOLDEN_RATIO
+                                    interp_nomsu = interp_nomsu2
+                                else
+                                    nomsu\add "\n..\\"
+                                    if bit.type == "EscapedNomsu" or bit.type == "Block" or bit.type == "IndexChain"
+                                        interp_nomsu\parenthesize!
                             elseif bit.type == "EscapedNomsu" or bit.type == "Block" or bit.type == "IndexChain"
                                 interp_nomsu\parenthesize!
                         nomsu\add interp_nomsu
@@ -396,7 +430,8 @@ tree_to_nomsu = (tree)->
                 else
                     item_nomsu = tree_to_inline_nomsu(item)
                     if #item_nomsu\text! > MAX_LINE
-                        item_nomsu = recurse(item)
+                        sep = '\n' if i > 1
+                        item_nomsu = tree_to_nomsu(item)
                 nomsu\add sep
                 nomsu\add item_nomsu
                 if item_nomsu\is_multiline! or item.type == 'Comment' or nomsu\trailing_line_len! + #tostring(item_nomsu) >= MAX_LINE
diff --git a/tools/format.nom b/tools/format.nom
index 76c4981..7d9d1d4 100755
--- a/tools/format.nom
+++ b/tools/format.nom
@@ -19,7 +19,7 @@ for $filename in $filenames:
     $file = (read file $filename)
     unless $file:
         barf "File does not exist: \$filename"
-    $leading_indent = ($file, matching "[\n]*([ ]*)")
+    $leading_indent = ($file, matching "\n*([ ]*)")
     $code = (NomsuCode from ($Source $filename 1 (size of $file)) $file)
     try:
         $tree = ($code parsed)
diff --git a/tools/upgrade.nom b/tools/upgrade.nom
index f335dc0..23716a2 100755
--- a/tools/upgrade.nom
+++ b/tools/upgrade.nom
@@ -19,7 +19,7 @@ for $filename in $(COMMAND LINE ARGS).extras:
     $file = (read file $filename)
     unless $file:
         barf "File does not exist: \$filename"
-    $leading_indent = ($file, matching "[\n]*([ ]*)")
+    $leading_indent = ($file, matching "\n*([ ]*)")
     $code = (NomsuCode from (Source $filename 1 (size of $file)) $file)
     $tree = ($code parsed $start_version)
     $uptree =