Overhaul in progress, mostly working. Moved all the nomsu packages into

lib/, including core/*. Changes to how nomsu environments and importing
work.

7 files changed, 528 insertions, 0 deletions

diff --git a/lib/tools/find.nom b/lib/tools/find.nom
new file mode 100755
index 0000000..aa4bc7b
--- /dev/null
+++ b/lib/tools/find.nom
@@ -0,0 +1,95 @@
+#!/usr/bin/env nomsu -V6.14
+#
+    This is a tool to find syntax trees matching a pattern. "*" is a wildcard
+    that will match any subtree, and "**" is a wildcard that will match any
+    0 or more subtrees. "**" is greedy, so extra arguments after it will
+    not match.
+    
+    nomsu -t find [flags] "* squared" file1 file2...
+    
+    Flags:
+    -l  List only the names of the files with matches
+    --wildcard=<wildcard>  Specify a custom wildcard (in case you need to
+    match an action with a "*" in the name)
+    
+    Output:
+    <filename>:<line number>:
+    <matching lines>
+    
+use "filesystem"
+use "consolecolor"
+use "commandline"
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+command line program with $args:
+    $wildcard = ($args.wildcard or "%*")
+    $pattern = $args.extras.1
+    if (any of [not $pattern, $pattern == "*", $pattern == "**"]):
+        usage ("
+            nomsu -t find [-l] [--wildcard=<wildcard>] <pattern>, where <pattern> is valid Nomsu code
+        ")
+    $pattern = ($pattern, with "\$wildcard\$wildcard" -> "$multi_wildcard")
+    $pattern = ($pattern, with $wildcard -> "$wildcard")
+    $pattern_tree = ($pattern parsed)
+    ($tree matches $patt) means:
+        when:
+            (not ($tree is syntax tree)): return (no)
+            (($patt.type == "Var") and ($patt.1 == "wildcard")): return (yes)
+            ($tree.type != $patt.type): return (no)
+            ($tree.type == "Action"):
+                if (($tree, get stub) != ($patt, get stub)): return (no)
+        
+        for $ in 1 to (#$patt):
+            if ($patt.$ is syntax tree):
+                if ($patt.$ == \$multi_wildcard): return (yes)
+                unless ($tree.$ matches $patt.$): return (no)
+            ..else:
+                unless ($tree.$ == $patt.$): return (no)
+        
+        if ((#$tree) != (#$patt)): return (no)
+        return (yes)
+    $filenames = ($args.extras, from 2 to -1)
+    if ((#$filenames) == 0):
+        say ("
+            Warning: searching stdin (ctrl-d to abort). To avoid this message, use nomsu -t find -
+        ")
+        $filenames = ["stdin"]
+
+    for $filename in $filenames:
+        $file = (read file $filename)
+        unless $file:
+            fail "File does not exist: \$filename"
+        $code = (NomsuCode from ($Source $filename 1 (size of $file)) $file)
+        try:
+            $tree = ($code parsed)
+        ..if it fails $msg:
+            say
+                red ("
+                    \$filename failed to parse:
+                    \$msg
+                ")
+            $tree = (nil)
+        
+        unless $tree:
+            do next $filename
+        
+        $results = []
+        for $t in recursive $tree:
+            if ($t matches $pattern_tree):
+                $line_num = ($file, line number at $t.source.start)
+                $results, add {
+                    .line = $line_num, .text = "\(blue "\$filename:\$line_num:")\n\(source lines of $t)"
+                }
+            
+            for $sub in $t:
+                if ($sub is syntax tree):
+                    recurse $t on $sub
+        
+        if $args.l:
+            if ((#$results) > 0):
+                say $filename
+        ..else:
+            sort $results by $ -> $.line
+            for $ in $results:
+                say $.text
diff --git a/lib/tools/format.nom b/lib/tools/format.nom
new file mode 100755
index 0000000..e60d12e
--- /dev/null
+++ b/lib/tools/format.nom
@@ -0,0 +1,45 @@
+#!/usr/bin/env nomsu -V6.14
+#
+    Auto-format Nomsu code. Usage:
+    nomsu -t format [-i] file1 file2...
+    
+    If the "-i" flag is used, the file will be edited in-place.
+    If the "-q" flag is used and an error occurs, the original file will be printed.
+    If no files are passed in, this will read from stdin.
+    
+use "filesystem"
+use "commandline"
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+command line program with $args:
+    $filenames = $args.extras
+    if ((#$filenames) == 0):
+        say ("
+            Warning: reading from stdin (ctrl-d to abort). To avoid this message, use nomsu -t format -
+        ")
+        $filenames = ["stdin"]
+
+    for $filename in $filenames:
+        $file = (read file $filename)
+        unless $file:
+            fail "File does not exist: \$filename"
+        $leading_indent = ($file, matching "\n*([ ]*)")
+        $code = (NomsuCode from ($Source $filename 1 (size of $file)) $file)
+        try:
+            $tree = ($code parsed)
+        ..if it fails $msg:
+            if $args.q:
+                $formatted = $file
+            ..else:
+                say $msg
+        
+        if ($tree and (not $formatted)):
+            $formatted =
+                "\$leading_indent\($tree as nomsu, text, with "\n" -> "\n\$leading_indent")"
+        
+        if $formatted:
+            if $args.i:
+                write $formatted to file $filename
+            ..else:
+                say $formatted inline
diff --git a/lib/tools/parse.nom b/lib/tools/parse.nom
new file mode 100755
index 0000000..63cc247
--- /dev/null
+++ b/lib/tools/parse.nom
@@ -0,0 +1,47 @@
+#!/usr/bin/env nomsu -V6.14
+#
+    Tool to print out a parse tree of files in an easy-to-read format. Usage:
+    nomsu tools/parse.nom file1 file2 directory1 ...        
+    
+use "filesystem"
+use "commandline"
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+externally (print tree $t at indent $indent) means:
+    if $t.type is:
+        "Action":
+            say "\($indent)Action (\($t.stub)):"
+            for $arg in $t:
+                if ($arg is syntax tree):
+                    print tree $arg at indent "\$indent  "
+        
+        "MethodCall":
+            say "\($indent)MethodCall on:"
+            print tree $t.1 at indent "\$indent    "
+            print tree $t.2 at indent "\$indent    "
+        
+        "Number":
+            say "\$indent\($t.1)"
+        
+        "Var":
+            say "\($indent)$\($t.1)"
+        
+        else:
+            say "\$indent\($t.type):"
+            for $arg in $t:
+                when:
+                    ($arg is syntax tree):
+                        print tree $arg at indent "\$indent  "
+                    
+                    else:
+                        say "\$indent  \(quote $arg)"
+
+command line program with $args:
+    for $filename in $args.extras:
+        $file = (read file $filename)
+        unless $file:
+            fail "File does not exist: \$filename"
+        $nomsu = (NomsuCode from (Source $filename 1 (size of $file)) $file)
+        $tree = ($nomsu parsed)
+        print tree $tree at indent ""
diff --git a/lib/tools/repl.nom b/lib/tools/repl.nom
new file mode 100755
index 0000000..2c0d3df
--- /dev/null
+++ b/lib/tools/repl.nom
@@ -0,0 +1,86 @@
+#!/usr/bin/env nomsu -V6.14
+#
+    This file defines a Read-Evaluate-Print-Loop (REPL) for Nomsu
+    
+use "consolecolor"
+use "filesystem"
+use "commandline"
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+externally (help) means:
+    say ("
+        This is the Nomsu v\(Nomsu version) interactive console.
+        You can type in Nomsu code here and hit 'enter' twice to run it.
+        To exit, type 'exit' or 'quit' and hit enter twice.
+    ")
+
+command line program with $args:
+    say ("
+        
+        \(bright)\(underscore)Welcome to the Nomsu v\(Nomsu version) interactive console!\
+        ..\(reset color)
+               press 'enter' twice to run a command
+        
+    ")
+
+    repeat:
+        say (bright (yellow ">> ")) inline
+        $buff = []
+        repeat:
+            say (bright) inline
+            $line = ($io.read "*L")
+            say (reset color) inline
+            if (($line == "\n") or (not $line)):
+                if ((size of $buff) > 0):
+                    # clear the line
+                    say "\027[1A\027[2K" inline
+                go to (run buffer)
+            $buff, add ($line, with "\t" -> "    ")
+            say (dim (yellow ".. ")) inline
+        
+        --- (run buffer) ---
+        
+        if ((size of $buff) == 0): stop
+        $buff = ($buff, joined)
+        spoof file $buff
+        try:
+            $tree = ($buff parsed)
+        ..if it fails with $err:
+            say $err
+            do next
+        
+        unless $tree:
+            do next
+        
+        for $chunk in $tree:
+            try:
+                $lua = ($chunk as lua)
+            ..if it fails with $err: say $err
+            
+            unless $lua:
+                do next
+            
+            # TODO: this is a bit hacky, it just defaults variables to global
+                so that stuff mostly works across multiple lines. It would be
+                nicer if local variables actually worked.
+            $lua, remove free vars
+            try:
+                $ret = (run $lua)
+            ..if it fails with $err: say $err
+            ..if it succeeds:
+                if (type of $ret) is:
+                    "nil":
+                        do nothing
+                    
+                    "boolean":
+                        say "= \("yes" if $ret else "no")"
+                    
+                    "table":
+                        if $ret.as_nomsu:
+                            say "= \($ret, as nomsu)"
+                        ..else:
+                            say "= \$ret"
+                    
+                    else:
+                        say "= \$ret"
diff --git a/lib/tools/replace.nom b/lib/tools/replace.nom
new file mode 100755
index 0000000..fdb8e38
--- /dev/null
+++ b/lib/tools/replace.nom
@@ -0,0 +1,147 @@
+#!/usr/bin/env nomsu -V6.14
+#
+    This is a tool to replace syntax trees with something new.
+    
+    Usage:
+        nomsu -t replace [-i] [-f] [-q] [--literal="$v1 $v2..."] <pattern> <replacement> file1 file2...
+    
+    Example:
+        nomsu -t replace "($1 and $2) and $3" "all of [$1, $2, $3]" my_file.nom
+    
+    If the "-i" flag is used, the file(s) will be edited in-place.
+    When editing in-place, if the "-f" flag is not used, each change will be
+        run past the user first.
+    If the "-q" flag is used and a file fails to parse, the original file
+        contents will be output.
+    If no files are passed in, this will read from stdin.
+    
+use "filesystem"
+use "consolecolor"
+use "commandline"
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+command line program with $args:
+    if ((#$args.extras) < 2):
+        fail ("
+            Usage: nomsu -t replace [--literal="$v1 $v2..."] <pattern> <replacement> file1 file2...
+        ")
+    $pattern = $args.extras.1
+    $replacement = $args.extras.2
+    $pattern_tree = ($pattern parsed)
+    $replacement_tree = ($replacement parsed)
+    $literal_vars = {}
+    if $args.literal:
+        for $var in ($args.literal, all matches of "$([^ ]*)"):
+            $literal_vars.$var = (yes)
+
+    if (($pattern_tree.type == "Var") and (not $literal_vars.($pattern_tree.1))):
+        fail "Pattern matches every part of the file."
+
+    $pattern_vars = {
+        : for $ in recursive $pattern_tree:
+            if (($.type == "Var") and (not $literal_vars.($.1))): add $.1
+            for $child in $:
+                if ($child is a "Syntax Tree"):
+                    recurse $ on $child
+    }
+
+    # TODO: support wildcards and unpacking
+        e.g. nomsu -t replace "test(: $test; *$more_tests)" "*$more_tests; *$test"
+    ($tree matches $patt with $substitution_values) means:
+        # TODO: optimize
+        $substitution_values = {: for $k = $v in $substitution_values: add $k = $v}
+        when:
+            (not ($tree is syntax tree)): return (no)
+            (($patt.type == "Var") and $pattern_vars.($patt.1)):
+                if $substitution_values.($patt.1):
+                    if ($tree == $substitution_values.($patt.1)):
+                        return $substitution_values
+                    ..else:
+                        return (nil)
+                ..else:
+                    $substitution_values.($patt.1) = $tree
+                    return $substitution_values
+            ($tree.type != $patt.type): return (nil)
+            ($tree.type == "Action"):
+                if (($tree, get stub) != ($patt, get stub)): return (nil)
+        
+        for $ in 1 to (#$patt):
+            if ($patt.$ is syntax tree):
+                $new_values = ($tree.$ matches $patt.$ with $substitution_values)
+                unless $new_values:
+                    return (nil)
+                
+                for $k = $v in $new_values:
+                    $substitution_values.$k = $v
+            ..else:
+                unless ($tree.$ == $patt.$): return (nil)
+        
+        if ((#$tree) != (#$patt)): return (nil)
+        return $substitution_values
+    $filenames = ($args.extras, from 3 to -1)
+    if ((#$filenames) == 0):
+        say ("
+            Warning: searching stdin (ctrl-d to abort). To avoid this message, use nomsu -t find -
+        ")
+        $filenames = ["stdin"]
+
+    for $filename in $filenames:
+        $file = (read file $filename)
+        unless $file:
+            fail "File does not exist: \$filename"
+        $code = (NomsuCode from ($Source $filename 1 (size of $file)) $file)
+        try:
+            $tree = ($code parsed)
+        ..if it fails $msg:
+            if $args.q:
+                unless $args.i: say $code
+            ..else:
+                say $msg
+        
+        unless $tree:
+            do next $filename
+        
+        $replaced = {}
+        $matched = {}
+        $user_answers = {}
+        ($tree with replacements) means
+            $tree, map
+                for $t:
+                    $values = ($t matches $pattern_tree with {})
+                    if $values:
+                        $matched.$t = (yes)
+                        for $k = $v in $values:
+                            $values.$k = ($v with replacements)
+                        $ret = ($replacement_tree with vars $values)
+                        if ($args.i and (not $args.f)):
+                            if ($user_answers.$t == (nil)):
+                                if ((#$user_answers) > 0): say ""
+                                $user_answers.$t = "n"
+                                say "\(bright)Should this:"
+                                say ("
+                                        \(bright)\(yellow)\("\(($t with replacements) as nomsu)", with "\n" -> "\n    ")\
+                                    ..\(reset color)
+                                ")
+                                say "\(bright)..be replaced with:"
+                                
+                                say ("
+                                        \(bright)\(blue)\("\($ret as nomsu)", with "\n" -> "\n    ")\(reset color)
+                                ")
+                                
+                                $user_answers.$t = (ask "\(bright)..? [Y/n]\(reset color) ")
+                            
+                            if ($user_answers.$t == "n"): return (nil)
+                        $replaced.$t = (yes)
+                        return $ret
+        $tree2 = ($tree with replacements)
+        if $args.i:
+            if ((#$user_answers) > 0): say ""
+            say ("
+                \(#$replaced)/\(#$matched) replacement\("" if ((#$replaced) == 1) else "s") in \$filename
+            ")
+            
+            if ((#$replaced) > 0):
+                write "\($tree2 as nomsu)" to file $filename
+        ..else:
+            say ($tree2 as nomsu)
diff --git a/lib/tools/test.nom b/lib/tools/test.nom
new file mode 100755
index 0000000..4663bd4
--- /dev/null
+++ b/lib/tools/test.nom
@@ -0,0 +1,65 @@
+#!/usr/bin/env nomsu -V6.14
+#
+    Tool to run all tests in a file (i.e. the code block inside a call to 'test $'). Usage:
+    nomsu tools/test.nom file1 file2 directory1 ...
+    
+use "filesystem"
+use "consolecolor"
+use "commandline"
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+command line program with $args:
+    for $filename in $args.extras at $i:
+        $file = (read file $filename)
+        unless $file:
+            fail "Couldn't find \$filename"
+
+        $(test environment) = (new environment)
+        $(test environment), export $filename
+        $version =
+            $file, matching ("
+                #![^
+                ]* nomsu %-V[ ]*([^
+                ]*)
+            ")
+        $file_tests = []
+        for $src = $test in $(test environment).TESTS:
+            if $version:
+                $test = ("
+                    #!/usr/bin/env nomsu -V\$version
+                    \$test
+                ")
+            $file_tests, add {.test = $test, .source = $src}
+        
+        unless ($file_tests is empty):
+            sort $file_tests by $ -> $.source
+            say "[ .. ] \$filename" inline
+            $io.flush()
+            
+            if $args.v: say ""
+            
+            $failures = []
+            for $ in $file_tests:
+                if $args.v:
+                    say "    \(yellow ($.test, with "\n" -> "\n    "))"
+                try:
+                    $(test environment), run $.test
+                ..if it fails with $msg:
+                    $src = ($Source, from string $.source)
+                    $l1 = ($file, line number at $src.start)
+                    $l2 = ($file, line number at $src.stop)
+                    $failures, add "\(yellow "\($src.filename):\($l1)-\$l2:")\n\(bright (red ($msg, indented)))"
+            
+            if ($failures is empty):
+                if $args.v:
+                    say (green "PASS")
+                ..else:
+                    say "\r[\(green "PASS")"
+            ..else:
+                if $args.v:
+                    say (red (bright "FAIL"))
+                ..else:
+                    say "\r[\(red (bright "FAIL"))"
+                say "\($failures, joined with "\n", indented)"
+
diff --git a/lib/tools/upgrade.nom b/lib/tools/upgrade.nom
new file mode 100755
index 0000000..1ef91b9
--- /dev/null
+++ b/lib/tools/upgrade.nom
@@ -0,0 +1,43 @@
+#!/usr/bin/env nomsu -V6.14
+#
+    Tool to automatically update code from old versions of Nomsu. Usage:
+    nomsu tools/upgrade.nom [-i] file1 file2 directory1 ...
+    If "-i" is the first argument, upgrades will be performed in-place. Otherwise, the
+    upgraded code will be printed.
+    
+use "compatibility"
+use "filesystem"
+use "consolecolor"
+use "commandline"
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+command line program with $args:
+    $inplace = ($args.i or $args.inplace)
+    $start_version = $args."upgrade-from"
+    $version = ($args."upgrade-to" or (Nomsu version))
+    $test = ($args.t or $args.test)
+    for $filename in $args.extras:
+        $file = (read file $filename)
+        unless $file:
+            fail "File does not exist: \$filename"
+        $leading_indent = ($file, matching "\n*([ ]*)")
+        $code = (NomsuCode from (Source $filename 1 (size of $file)) $file)
+        $tree = ($code parsed $start_version)
+        $uptree =
+            $tree upgraded from ($start_version or ($tree.version or (Nomsu version))) to
+                $version
+        $text = "\$leading_indent\($uptree as nomsu, text, with "\n" -> "\n\$leading_indent")"
+        when:
+            $inplace:
+                say "Upgraded \$filename"
+                write $text to file $filename
+            
+            $test:
+                if ($uptree == $tree):
+                    say (dim "\$filename will not be changed")
+                ..else:
+                    say (bright "\$filename will be changed")
+            
+            else:
+                say $text inline