1 // The main program that runs compilation
12 #if defined(__linux__)
13 #include <sys/random.h>
17 #include "compile/cli.h"
18 #include "compile/files.h"
19 #include "compile/headers.h"
21 #include "formatter/formatter.h"
24 #include "parse/files.h"
25 #include "stdlib/bools.h"
26 #include "stdlib/bytes.h"
27 #include "stdlib/c_strings.h"
28 #include "stdlib/cli.h"
29 #include "stdlib/datatypes.h"
30 #include "stdlib/enums.h"
31 #include "stdlib/lists.h"
32 #include "stdlib/optionals.h"
33 #include "stdlib/paths.h"
34 #include "stdlib/print.h"
35 #include "stdlib/random.h"
36 #include "stdlib/siphash.h"
37 #include "stdlib/tables.h"
38 #include "stdlib/text.h"
39 #include "stdlib/util.h"
42 #define run_cmd(...) \
44 const char *_cmd = String(__VA_ARGS__); \
45 if (verbose) print("\033[34;1m", _cmd, "\033[m"); \
48 #define xsystem(...) \
50 int _status = system(String(__VA_ARGS__)); \
51 if (!WIFEXITED(_status) || WEXITSTATUS(_status) != 0) \
52 errx(1, "Failed to run command: %s", String(__VA_ARGS__)); \
54 #define list_text(list) Text$join(Text(" "), list)
56 #define whisper(...) print("\033[2m", __VA_ARGS__, "\033[m")
59 // Only on Linux is /proc/self/exe available
60 static struct stat compiler_stat;
63 static const char *paths_str(List_t paths) {
64 Text_t result = EMPTY_TEXT;
65 for (int64_t i = 0; i < (int64_t)paths.length; i++) {
66 if (i > 0) result = Texts(result, Text(" "));
67 result = Texts(result, Path$as_text((Path_t *)(paths.data + i * paths.stride), false, &Path$info));
69 return Text$as_c_string(result);
72 static OptionalBool_t verbose = false, quiet = false, show_version = false, show_prefix = false, clean_build = false,
73 source_mapping = true, should_install = false;
74 static bool is_gcc = false, is_clang = false;
76 static List_t format_files = EMPTY_LIST, format_files_inplace = EMPTY_LIST, parse_files = EMPTY_LIST,
77 transpile_files = EMPTY_LIST, compile_objects = EMPTY_LIST, compile_executables = EMPTY_LIST,
78 run_files = EMPTY_LIST, uninstall_packages = EMPTY_LIST, packages = EMPTY_LIST, args = EMPTY_LIST;
80 static OptionalText_t show_codegen = NONE_TEXT,
81 cflags = Text("-Werror -fdollars-in-identifiers -std=c2x -Wno-trigraphs "
82 " -ffunction-sections -fdata-sections"
84 " -D_XOPEN_SOURCE -D_DEFAULT_SOURCE -fPIC -ggdb"
85 #if defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__NetBSD__) || defined(__APPLE__)
89 ldlibs = Text("-lgc -lm -lgmp -lunistring"), ldflags = Text(""), optimization = Text("2"),
90 cc = Text(DEFAULT_C_COMPILER);
92 static Text_t config_summary,
93 // This will be either "" or "sudo -u <user>" or "doas -u <user>"
94 // to allow a command to put stuff into TOMO_PATH as the owner
98 typedef enum { COMPILE_C_FILES, COMPILE_OBJ, COMPILE_EXE } compile_mode_t;
100 static void transpile_header(env_t *base_env, Path_t path);
101 static void transpile_code(env_t *base_env, Path_t path);
102 static void compile_object_file(Path_t path);
103 static Path_t compile_executable(env_t *base_env, Path_t path, Path_t exe_path, List_t object_files,
104 List_t extra_ldlibs);
105 static void build_file_dependency_graph(Path_t path, Table_t *to_compile, Table_t *to_link);
106 static void build_package(Path_t pkg_dir);
107 static void install_package(Path_t pkg_dir);
108 static void compile_files(env_t *env, List_t files, List_t *object_files, List_t *ldlibs, compile_mode_t mode);
109 static bool is_stale(Path_t path, Path_t relative_to, bool ignore_missing);
110 static bool is_stale_for_any(Path_t path, List_t relative_to, bool ignore_missing);
111 static Path_t build_file(Path_t path, const char *extension);
112 static void wait_for_child_success(pid_t child);
113 static bool is_config_outdated(Path_t path);
114 static Path_t get_exe_path(Path_t path);
117 bool h : 1, c : 1, o : 1;
120 static List_t normalize_tm_paths(List_t paths) {
121 List_t result = EMPTY_LIST;
122 for (int64_t i = 0; i < (int64_t)paths.length; i++) {
123 Path_t path = *(Path_t *)(paths.data + i * paths.stride);
124 // Convert `foo` to `foo/foo.tm` and resolve path to absolute path:
125 Path_t cur_dir = Path$current_dir();
126 if (Path$is_directory(path, true)) path = Path$child(path, Texts(Path$base_name(path), Text(".tm")));
128 path = Path$resolved(path, cur_dir);
129 if (!Path$exists(path)) fail("path not found: ", path);
130 List$insert(&result, &path, I(0), sizeof(path));
135 int main(int argc, char *argv[]) {
140 // Get the file modification time of the compiler, so we
141 // can recompile files after changing the compiler:
142 char compiler_path[PATH_MAX];
143 ssize_t count = readlink("/proc/self/exe", compiler_path, PATH_MAX);
144 if (count == -1) err(1, "Could not find age of compiler");
145 compiler_path[count] = '\0';
146 if (stat(compiler_path, &compiler_stat) != 0) err(1, "Could not find age of compiler");
150 ldlibs = Texts(ldlibs, Text(" -lexecinfo"));
153 const char *color_env = getenv("COLOR");
154 USE_COLOR = color_env ? strcmp(color_env, "1") == 0 : isatty(STDOUT_FILENO);
155 const char *no_color_env = getenv("NO_COLOR");
156 if (no_color_env && no_color_env[0] != '\0') USE_COLOR = false;
158 #if defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__NetBSD__) || defined(__APPLE__)
159 arc4random_buf(TOMO_HASH_KEY, sizeof(TOMO_HASH_KEY));
160 #elif defined(__linux__)
161 assert(getrandom(TOMO_HASH_KEY, sizeof(TOMO_HASH_KEY), 0) == sizeof(TOMO_HASH_KEY));
163 #error "Unsupported platform for secure random number generation"
166 if (getenv("TOMO_PATH")) TOMO_PATH = getenv("TOMO_PATH");
168 cflags = Texts("-I'", TOMO_PATH, "/include' -I'", TOMO_PATH, "/lib/tomo@", TOMO_VERSION, "' ", cflags);
170 // Set up environment variables:
171 const char *PATH = getenv("PATH");
172 setenv("PATH", PATH ? String(TOMO_PATH, "/bin:", PATH) : String(TOMO_PATH, "/bin"), 1);
173 const char *LD_LIBRARY_PATH = getenv("LD_LIBRARY_PATH");
174 setenv("LD_LIBRARY_PATH", LD_LIBRARY_PATH ? String(TOMO_PATH, "/lib:", LD_LIBRARY_PATH) : String(TOMO_PATH, "/lib"),
176 const char *LIBRARY_PATH = getenv("LIBRARY_PATH");
177 setenv("LIBRARY_PATH", LIBRARY_PATH ? String(TOMO_PATH, "/lib:", LIBRARY_PATH) : String(TOMO_PATH, "/lib"), 1);
178 const char *C_INCLUDE_PATH = getenv("C_INCLUDE_PATH");
179 setenv("C_INCLUDE_PATH",
180 C_INCLUDE_PATH ? String(TOMO_PATH, "/include:", C_INCLUDE_PATH) : String(TOMO_PATH, "/include"), 1);
181 const char *CPATH = getenv("CPATH");
182 setenv("CPATH", CPATH ? String(TOMO_PATH, "/include:", CPATH) : String(TOMO_PATH, "/include"), 1);
185 if ((streq(argv[1], "-r") || streq(argv[1], "--run")) && argc >= 3) {
186 if (strcspn(argv[2], "/;$") == strlen(argv[2])) {
187 const char *program = String("'", TOMO_PATH, "'/lib/tomo@", TOMO_VERSION, "/", argv[2], "/", argv[2]);
188 execv(program, &argv[2]);
190 print_err("This is not an installed tomo program: ", argv[2]);
193 Text_t usage = Texts("\x1b[33;4;1mUsage:\x1b[m\n"
194 "\x1b[1mRun a program:\x1b[m tomo file.tm [-- args...]\n"
195 "\x1b[1mTranspile files:\x1b[m tomo -t file.tm\n"
196 "\x1b[1mCompile object file:\x1b[m tomo -c file.tm\n"
197 "\x1b[1mCompile executable:\x1b[m tomo -e file.tm\n"
198 "\x1b[1mBuild packages:\x1b[m tomo -p package...\n"
199 "\x1b[1mUninstall packages:\x1b[m tomo -u package...\n"
200 "\x1b[1mOther flags:\x1b[m\n"
201 " --verbose|-v: verbose output\n"
202 " --prefix: print the Tomo prefix directory\n"
203 " --quiet|-q: quiet output\n"
204 " --parse|-P: show parse tree\n"
205 " --transpile|-t: transpile C code without compiling\n"
206 " --show-codegen|-c <pager>: show generated code\n"
207 " --compile-obj|-c: compile C code for object file\n"
208 " --compile-exe|-e: compile to standalone executable without running\n"
209 " --format|F: print formatted code\n"
210 " --format-inplace: format the code in a file (in place)\n"
211 " --package|p: build a folder as a package\n"
212 " --install|-I: install the executable or package\n"
213 " --uninstall|-u: uninstall an executable or package\n"
214 " --optimization|-O <level>: set optimization level\n"
215 " --force-rebuild|-f: force rebuilding\n"
216 " --source-mapping|-m <yes|no>: toggle source mapping in generated code\n"
217 " --changelog: show the Tomo changelog\n"
218 " --run|-r: run a program from ",
219 TOMO_PATH, "/share/tomo@", TOMO_VERSION, "/installed\n");
220 Text_t help = Texts(Text("\x1b[1mtomo\x1b[m: a compiler for the Tomo programming language"), Text("\n\n"), usage);
221 cli_arg_t tomo_args[] = {
222 {"run", &run_files, List$info(&Path$info), .short_flag = 'r'}, //
223 {"args", &args, List$info(&CString$info)}, //
224 {"format", &format_files, List$info(&Path$info), .short_flag = 'F'}, //
225 {"parse", &parse_files, List$info(&Path$info), .short_flag = 'P'}, //
226 {"format-inplace", &format_files_inplace, List$info(&Path$info)}, //
227 {"transpile", &transpile_files, List$info(&Path$info), .short_flag = 't'}, //
228 {"compile-obj", &compile_objects, List$info(&Path$info), .short_flag = 'c'}, //
229 {"compile-exe", &compile_executables, List$info(&Path$info), .short_flag = 'e'}, //
230 {"package", &packages, List$info(&Path$info), .short_flag = 'p'}, //
231 {"uninstall", &uninstall_packages, List$info(&Text$info), .short_flag = 'u'}, //
232 {"verbose", &verbose, &Bool$info, .short_flag = 'v'}, //
233 {"install", &should_install, &Bool$info, .short_flag = 'I'}, //
234 {"prefix", &show_prefix, &Bool$info}, //
235 {"quiet", &quiet, &Bool$info, .short_flag = 'q'}, //
236 {"version", &show_version, &Bool$info, .short_flag = 'V'}, //
237 {"show-codegen", &show_codegen, &Text$info, .short_flag = 'C'}, //
238 {"optimization", &optimization, &Text$info, .short_flag = 'O'}, //
239 {"force-rebuild", &clean_build, &Bool$info, .short_flag = 'f'}, //
240 {"source-mapping", &source_mapping, &Bool$info, .short_flag = 'm'},
243 tomo_parse_args(argc, argv, usage, help, TOMO_VERSION, sizeof(tomo_args) / sizeof(tomo_args[0]), tomo_args);
250 if (verbose) print(TOMO_VERSION, " ", GIT_VERSION);
251 else print(TOMO_VERSION);
255 is_gcc = (system(String(cc, " -v 2>&1 | grep -q 'gcc version'")) == 0);
257 cflags = Texts(cflags, Text(" -fsanitize=signed-integer-overflow -fno-sanitize-recover"
258 " -fno-signaling-nans -fno-trapping-math -fno-finite-math-only"));
261 is_clang = (system(String(cc, " -v 2>&1 | grep -q 'clang version'")) == 0);
263 cflags = Texts(cflags, Text(" -Wno-parentheses-equality"));
266 ldflags = Texts("-Wl,-rpath,'", TOMO_PATH, "/lib' ", ldflags, " -ffunction-sections -fdata-sections");
268 if (is_gcc) ldflags = Texts(ldflags, " -Wl,--gc-sections");
269 else if (is_clang) ldflags = Texts(ldflags, " -Wl,-dead_strip");
271 ldflags = Texts(ldflags, " -Wl,--gc-sections");
275 cflags = Texts(cflags, Text(" -I/opt/homebrew/include"));
276 ldflags = Texts(ldflags, Text(" -L/opt/homebrew/lib -Wl,-rpath,/opt/homebrew/lib"));
279 if (show_codegen.length > 0 && Text$equal_values(show_codegen, Text("pretty")))
280 show_codegen = Text("{ sed '/^#line/d;/^$/d' | clang-format | bat -l c -P; }");
282 config_summary = Texts("TOMO_VERSION=", TOMO_VERSION, "\n", "COMPILER=", cc, " ", cflags, " -O", optimization, "\n",
283 "SOURCE_MAPPING=", source_mapping ? Text("yes") : Text("no"), "\n");
285 Text_t owner = Path$owner(Path$from_str(TOMO_PATH), true);
286 Text_t user = Text$from_str(getenv("USER"));
287 if (!Text$equal_values(user, owner)) {
288 as_owner = Texts(Text(SUDO " -u "), owner, Text(" "));
291 // Uninstall packages:
292 for (int64_t i = 0; i < (int64_t)uninstall_packages.length; i++) {
293 Text_t *u = (Text_t *)(uninstall_packages.data + i * uninstall_packages.stride);
294 xsystem(as_owner, "rm -rvf '", TOMO_PATH, "'/lib/tomo@", TOMO_VERSION, "/", *u, " '", TOMO_PATH, "'/bin/", *u,
295 " '", TOMO_PATH, "'/man/man1/", *u, ".1");
296 print("Uninstalled ", *u);
299 // Build (and install) packages
300 Path_t cwd = Path$current_dir();
301 for (int64_t i = 0; i < (int64_t)packages.length; i++) {
302 Path_t *lib = (Path_t *)(packages.data + i * packages.stride);
303 *lib = Path$resolved(*lib, cwd);
304 // Fork a child process to build the package to prevent cross-contamination
305 // of side effects when building one package from affecting another package.
306 // This *could* be done in parallel, but there may be some dependency issues.
307 pid_t child = fork();
310 if (should_install) install_package(*lib);
313 wait_for_child_success(child);
316 parse_files = normalize_tm_paths(parse_files);
317 for (int64_t i = 0; i < (int64_t)parse_files.length; i++) {
318 Path_t path = *(Path_t *)(parse_files.data + i * parse_files.stride);
319 ast_t *ast = parse_file(Path$as_c_string(path), NULL);
320 print(ast_to_sexp_str(ast));
323 format_files = normalize_tm_paths(format_files);
324 for (int64_t i = 0; i < (int64_t)format_files.length; i++) {
325 Path_t path = *(Path_t *)(format_files.data + i * format_files.stride);
326 Text_t formatted = format_file(Path$as_c_string(path));
330 format_files_inplace = normalize_tm_paths(format_files_inplace);
331 for (int64_t i = 0; i < (int64_t)format_files.length; i++) {
332 Path_t path = *(Path_t *)(format_files_inplace.data + i * format_files_inplace.stride);
333 Text_t formatted = format_file(Path$as_c_string(path));
334 print("Formatted ", path);
335 Path$write(path, formatted, 0644);
338 if (transpile_files.length > 0) {
339 transpile_files = normalize_tm_paths(transpile_files);
340 env_t *env = global_env(source_mapping);
341 List_t object_files = EMPTY_LIST, extra_ldlibs = EMPTY_LIST;
342 compile_files(env, transpile_files, &object_files, &extra_ldlibs, COMPILE_C_FILES);
345 if (compile_objects.length > 0) {
346 compile_objects = normalize_tm_paths(compile_objects);
347 env_t *env = global_env(source_mapping);
348 List_t object_files = EMPTY_LIST, extra_ldlibs = EMPTY_LIST;
349 compile_files(env, transpile_files, &object_files, &extra_ldlibs, COMPILE_OBJ);
353 struct child_s *next;
355 } *child_processes = NULL;
357 if (compile_executables.length > 0) {
358 compile_executables = normalize_tm_paths(compile_executables);
360 // Compile and install in parallel:
361 for (int64_t i = 0; i < (int64_t)compile_executables.length; i++) {
362 Path_t path = *(Path_t *)(compile_executables.data + i * compile_executables.stride);
364 Path_t exe_path = get_exe_path(path);
365 // Put executable as a sibling to the .tm file instead of in the .build directory
366 exe_path = Path$sibling(path, Path$base_name(exe_path));
367 pid_t child = fork();
369 env_t *env = global_env(source_mapping);
370 List_t object_files = EMPTY_LIST, extra_ldlibs = EMPTY_LIST;
371 compile_files(env, List(path), &object_files, &extra_ldlibs, COMPILE_EXE);
372 compile_executable(env, path, exe_path, object_files, extra_ldlibs);
373 if (should_install) {
374 xsystem(as_owner, "mkdir -p '", TOMO_PATH, "/bin' '", TOMO_PATH, "/man/man1'");
375 xsystem(as_owner, "cp -v '", exe_path, "' '", TOMO_PATH, "/bin/'");
376 Path_t manpage_file = build_file(Path$with_extension(path, Text(".1"), true), "");
377 xsystem(as_owner, "cp -v '", manpage_file, "' '", TOMO_PATH, "/man/man1/'");
382 child_processes = new (struct child_s, .next = child_processes, .pid = child);
385 for (; child_processes; child_processes = child_processes->next)
386 wait_for_child_success(child_processes->pid);
389 // When running files, if `--verbose` is not set, then don't print "compiled to ..." messages
390 if (!verbose) quiet = true;
392 run_files = normalize_tm_paths(run_files);
394 if (run_files.length == 0 && format_files.length == 0 && format_files_inplace.length == 0 && parse_files.length == 0
395 && transpile_files.length == 0 && compile_objects.length == 0 && compile_executables.length == 0
396 && run_files.length == 0 && uninstall_packages.length == 0 && packages.length == 0) {
397 Path_t path = Path$from_str(String("~/.local/tomo/state/tomo@", TOMO_VERSION, "/run.tm"));
398 path = Path$expand_home(path);
399 Path$create_directory(Path$parent(path), 0755, true);
400 if (!Path$exists(path)) {
402 Text("# This is a handy Tomo REPL-like runner\n" //
403 "# Normally you would run `tomo ./file.tm` to run a script\n" //
404 "# See `tomo --help` for full usage\n" //
407 " # Put your code here:\n" //
408 " say(\"Hello world!\")\n" //
410 "# Save and exit to run\n"),
413 List$insert(&run_files, &path, I(0), sizeof(path));
414 const char *editor = getenv("EDITOR");
415 if (!editor || editor[0] == '\0') editor = "vim";
416 int status = system(String(editor, " ", path));
417 if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) return 1;
420 // Compile runnable files in parallel, then execute in serial:
421 for (int64_t i = 0; i < (int64_t)run_files.length; i++) {
422 Path_t path = *(Path_t *)(run_files.data + i * run_files.stride);
423 Path_t exe_path = get_exe_path(path);
424 pid_t child = fork();
426 env_t *env = global_env(source_mapping);
427 List_t object_files = EMPTY_LIST, extra_ldlibs = EMPTY_LIST;
428 compile_files(env, List(path), &object_files, &extra_ldlibs, COMPILE_EXE);
429 compile_executable(env, path, exe_path, object_files, extra_ldlibs);
433 child_processes = new (struct child_s, .next = child_processes, .pid = child);
436 for (; child_processes; child_processes = child_processes->next)
437 wait_for_child_success(child_processes->pid);
439 // After parallel compilation, do serial execution:
440 for (int64_t i = 0; i < (int64_t)run_files.length; i++) {
441 Path_t path = *(Path_t *)(run_files.data + i * run_files.stride);
442 Path_t exe_path = get_exe_path(path);
443 // Don't fork for the last program
444 pid_t child = i == (int64_t)run_files.length - 1 ? 0 : fork();
446 const char *prog_args[1 + args.length + 1];
447 Path_t relative_exe = Path$relative_to(exe_path, Path$current_dir());
448 prog_args[0] = (char *)Path$as_c_string(relative_exe);
449 for (int64_t j = 0; j < (int64_t)args.length; j++)
450 prog_args[j + 1] = *(const char **)(args.data + j * args.stride);
451 prog_args[1 + args.length] = NULL;
452 execv(prog_args[0], (char **)prog_args);
453 print_err("Could not execute program: ", prog_args[0]);
455 wait_for_child_success(child);
461 void wait_for_child_success(pid_t child) {
463 while (waitpid(child, &status, 0) < 0 && errno == EINTR) {
464 if (WIFEXITED(status) || WIFSIGNALED(status)) break;
465 else if (WIFSTOPPED(status)) kill(child, SIGCONT);
468 if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
469 _exit(WIFEXITED(status) ? WEXITSTATUS(status) : EXIT_FAILURE);
473 Path_t get_exe_path(Path_t path) {
474 ast_t *ast = parse_file(Path$as_c_string(path), NULL);
475 OptionalText_t exe_name = ast_metadata(ast, "EXECUTABLE");
476 if (exe_name.tag == TEXT_NONE) exe_name = Path$base_name(Path$with_extension(path, Text(""), true));
478 Path_t build_dir = Path$sibling(path, Text(".build"));
479 if (mkdir(Path$as_c_string(build_dir), 0755) != 0) {
480 if (!Path$is_directory(build_dir, true)) err(1, "Could not make (%s) directory", build_dir);
482 return Path$child(build_dir, exe_name);
485 Path_t build_file(Path_t path, const char *extension) {
486 Path_t build_dir = Path$sibling(path, Text(".build"));
487 if (mkdir(Path$as_c_string(build_dir), 0755) != 0) {
488 if (!Path$is_directory(build_dir, true)) err(1, "Could not make (%s) directory", build_dir);
490 return Path$child(build_dir, Texts(Path$base_name(path), Text$from_str(extension)));
493 void build_package(Path_t pkg_dir) {
494 pkg_dir = Path$resolved(pkg_dir, Path$current_dir());
495 if (!Path$is_directory(pkg_dir, true)) print_err("Not a valid directory: ", pkg_dir);
497 List_t tm_files = Path$glob(Path$child(pkg_dir, Text("[!._0-9]*.tm")));
498 env_t *env = fresh_scope(global_env(source_mapping));
499 List_t object_files = EMPTY_LIST, extra_ldlibs = EMPTY_LIST;
501 compile_files(env, tm_files, &object_files, &extra_ldlibs, COMPILE_OBJ);
503 Text_t pkg_name = get_package_name(pkg_dir);
504 Path_t archive = Path$child(pkg_dir, Texts(Text("lib"), pkg_name, ".a"));
505 if (is_stale_for_any(archive, object_files, false)) {
506 FILE *prog = run_cmd("ar -rcs '", archive, "' ", paths_str(object_files));
507 if (!prog) print_err("Failed to run `ar`");
508 int status = pclose(prog);
509 if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) exit(EXIT_FAILURE);
510 if (!quiet) print("Compiled static package:\t", Path$relative_to(archive, Path$current_dir()));
512 if (verbose) whisper("Unchanged: ", archive);
516 void install_package(Path_t pkg_dir) {
517 Text_t pkg_name = get_package_name(pkg_dir);
518 Path_t dest = Path$child(Path$from_str(String(TOMO_PATH, "/lib/tomo@", TOMO_VERSION)), pkg_name);
519 print("Installing ", pkg_dir, " into ", dest);
520 if (!Enum$equal(&pkg_dir, &dest, &Path$info)) {
521 if (verbose) whisper("Clearing out any pre-existing version of ", pkg_name);
522 xsystem(as_owner, "rm -rf '", dest, "'");
523 if (verbose) whisper("Moving files to ", dest);
524 xsystem(as_owner, "mkdir -p '", dest, "'");
525 xsystem(as_owner, "cp -r '", pkg_dir, "'/* '", dest, "/'");
526 xsystem(as_owner, "cp -r '", pkg_dir, "'/.build '", dest, "/'");
528 // If we have `debugedit` on this system, use it to remap the debugging source information
529 // to point to the installed version of the source file. Otherwise, fail silently.
530 if (verbose) whisper("Updating debug symbols for ", dest, "/lib", pkg_name, ".a");
531 int result = system(String(as_owner, "debugedit -b ", pkg_dir, " -d '", dest,
534 dest, "/lib", pkg_name, ".a",
536 ">/dev/null 2>/dev/null"));
538 print("Installed \033[1m", pkg_dir, "\033[m to ", TOMO_PATH, "/lib/tomo@", TOMO_VERSION, "/", pkg_name);
541 void compile_files(env_t *env, List_t to_compile, List_t *object_files, List_t *extra_ldlibs, compile_mode_t mode) {
542 Table_t to_link = EMPTY_TABLE;
543 Table_t dependency_files = EMPTY_TABLE;
544 for (int64_t i = 0; i < (int64_t)to_compile.length; i++) {
546 Path_t filename = *(Path_t *)(to_compile.data + i * to_compile.stride);
547 if (!Path$has_extension(filename, Text("tm")))
548 print_err("Not a valid .tm file: \x1b[31;1m", filename, "\x1b[m");
549 if (!Path$is_file(filename, true)) print_err("Couldn't find file: ", filename);
550 build_file_dependency_graph(filename, &dependency_files, &to_link);
553 // Make sure all files and dependencies have a .id file:
554 for (int64_t i = 0; i < (int64_t)dependency_files.entries.length; i++) {
557 staleness_t staleness;
558 } *entry = (dependency_files.entries.data + i * dependency_files.entries.stride);
560 Path_t id_file = build_file(entry->filename, ".id");
561 if (!Path$exists(id_file)) {
562 static const char id_chars[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
563 int64_t num_id_chars = (int64_t)strlen(id_chars);
565 for (int j = 0; j < (int)sizeof(id_str); j++) {
566 id_str[j] = id_chars[random_range(0, num_id_chars - 1)];
568 Text_t filename_id = Text("");
569 Text_t base = Path$base_name(entry->filename);
570 TextIter_t state = NEW_TEXT_ITER_STATE(base);
571 for (int64_t j = 0; j < (int64_t)base.length; j++) {
572 uint32_t c = Text$get_main_grapheme_fast(&state, j);
574 if (isalpha(c) || isdigit(c) || c == '_')
575 filename_id = Texts(filename_id, Text$from_strn((char[]){(char)c}, 1));
577 Path$write(id_file, Texts(filename_id, Text("_"), Text$from_strn(id_str, sizeof(id_str))), 0644);
581 // (Re)compile header files, eagerly for explicitly passed in files, lazily
582 // for downstream dependencies:
583 for (int64_t i = 0; i < (int64_t)dependency_files.entries.length; i++) {
586 staleness_t staleness;
587 } *entry = (dependency_files.entries.data + i * dependency_files.entries.stride);
589 if (entry->staleness.h || clean_build) {
590 transpile_header(env, entry->filename);
591 entry->staleness.o = true;
593 if (verbose) whisper("Unchanged: ", build_file(entry->filename, ".h"));
594 if (show_codegen.length > 0) xsystem(show_codegen, " <", build_file(entry->filename, ".h"));
598 env->imports = new (Table_t);
601 struct child_s *next;
603 } *child_processes = NULL;
605 // (Re)transpile and compile object files, eagerly for files explicitly
606 // specified and lazily for downstream dependencies:
607 for (int64_t i = 0; i < (int64_t)dependency_files.entries.length; i++) {
610 staleness_t staleness;
611 } *entry = (dependency_files.entries.data + i * dependency_files.entries.stride);
612 if (!clean_build && !entry->staleness.c && !entry->staleness.h && !entry->staleness.o
613 && !is_config_outdated(entry->filename)) {
614 if (verbose) whisper("Unchanged: ", build_file(entry->filename, ".c"));
615 if (show_codegen.length > 0) xsystem(show_codegen, " <", build_file(entry->filename, ".c"));
616 if (verbose) whisper("Unchanged: ", build_file(entry->filename, ".o"));
622 if (clean_build || entry->staleness.c) transpile_code(env, entry->filename);
623 else if (verbose) whisper("Unchanged: ", build_file(entry->filename, ".c"));
624 if (mode != COMPILE_C_FILES) compile_object_file(entry->filename);
627 child_processes = new (struct child_s, .next = child_processes, .pid = pid);
630 for (; child_processes; child_processes = child_processes->next)
631 wait_for_child_success(child_processes->pid);
634 for (int64_t i = 0; i < (int64_t)dependency_files.entries.length; i++) {
637 staleness_t staleness;
638 } *entry = (dependency_files.entries.data + i * dependency_files.entries.stride);
639 Path_t path = entry->filename;
640 path = build_file(path, ".o");
641 List$insert(object_files, &path, I(0), sizeof(Path_t));
645 for (int64_t i = 0; i < (int64_t)to_link.entries.length; i++) {
646 Text_t lib = *(Text_t *)(to_link.entries.data + i * to_link.entries.stride);
647 List$insert(extra_ldlibs, &lib, I(0), sizeof(Text_t));
652 bool is_config_outdated(Path_t path) {
653 OptionalText_t config = Path$read(build_file(path, ".config"));
654 if (config.tag == TEXT_NONE) return true;
655 return !Text$equal_values(config, config_summary);
658 void build_file_dependency_graph(Path_t path, Table_t *to_compile, Table_t *to_link) {
659 if (Table$has_value(*to_compile, path, Table$info(&Path$info, &Byte$info))) return;
661 staleness_t staleness = {
662 .h = is_stale(build_file(path, ".h"), Path$sibling(path, Text("packages.ini")), true)
663 || is_stale(build_file(path, ".h"), build_file(path, ":packages.ini"), true)
664 || is_stale(build_file(path, ".h"), path, false)
665 || is_stale(build_file(path, ".h"), build_file(path, ".id"), false),
666 .c = is_stale(build_file(path, ".c"), Path$sibling(path, Text("packages.ini")), true)
667 || is_stale(build_file(path, ".c"), build_file(path, ":packages.ini"), true)
668 || is_stale(build_file(path, ".c"), path, false)
669 || is_stale(build_file(path, ".c"), build_file(path, ".id"), false),
671 staleness.o = staleness.c || staleness.h || is_stale(build_file(path, ".o"), build_file(path, ".c"), false)
672 || is_stale(build_file(path, ".o"), build_file(path, ".h"), false);
673 Table$set(to_compile, &path, &staleness, Table$info(&Path$info, &Byte$info));
675 assert(Text$equal_values(Path$extension(path, true), Text("tm")));
677 ast_t *ast = parse_file(Path$as_c_string(path), NULL);
678 if (!ast) print_err("Could not parse file: ", path);
680 for (ast_list_t *stmt = Match(ast, Block)->statements; stmt; stmt = stmt->next) {
681 ast_t *stmt_ast = stmt->ast;
682 if (stmt_ast->tag != Use) continue;
683 DeclareMatch(use, stmt_ast, Use);
687 Path_t dep_tm = Path$resolved(Path$from_str(use->path), Path$parent(path));
688 if (!Path$is_file(dep_tm, true)) code_err(stmt_ast, "Not a valid file: ", dep_tm);
689 if (is_stale(build_file(path, ".h"), dep_tm, false)) staleness.h = true;
690 if (is_stale(build_file(path, ".c"), dep_tm, false)) staleness.c = true;
691 if (staleness.c || staleness.h) staleness.o = true;
692 Table$set(to_compile, &path, &staleness, Table$info(&Path$info, &Byte$info));
693 build_file_dependency_graph(dep_tm, to_compile, to_link);
697 OptionalPath_t installed = find_installed_package(stmt_ast);
699 Text_t name = get_package_name(installed);
700 Text_t lib = Texts(installed, "/lib", name, ".a");
701 Table$set(to_link, &lib, NULL, Table$info(&Text$info, &Void$info));
703 List_t children = Path$glob(Path$child(installed, Text("/[!._0-9]*.tm")));
704 for (int64_t i = 0; i < (int64_t)children.length; i++) {
705 Path_t *child = (Path_t *)(children.data + i * children.stride);
706 Table_t discarded = {.entries = EMPTY_LIST, .fallback = to_compile};
707 build_file_dependency_graph(*child, &discarded, to_link);
711 case USE_SHARED_OBJECT: {
712 Text_t lib = Text$from_str(use->path);
713 Table$set(to_link, &lib, NULL, Table$info(&Text$info, &Void$info));
717 Path_t asm_path = Path$from_str(use->path);
718 asm_path = Path$concat(Path$parent(path), asm_path);
719 Text_t linker_text = Path$as_text(&asm_path, NULL, &Path$info);
720 Table$set(to_link, &linker_text, NULL, Table$info(&Text$info, &Void$info));
721 if (is_stale(build_file(path, ".o"), asm_path, false)) {
723 Table$set(to_compile, &path, &staleness, Table$info(&Path$info, &Byte$info));
729 if (use->path[0] == '<') break;
731 Path_t dep_path = Path$resolved(Path$from_str(use->path), Path$parent(path));
732 if (is_stale(build_file(path, ".o"), dep_path, false)) {
734 Table$set(to_compile, &path, &staleness, Table$info(&Path$info, &Byte$info));
743 time_t latest_included_modification_time(Path_t path) {
744 static Table_t c_modification_times = EMPTY_TABLE;
745 const TypeInfo_t time_info = {.size = sizeof(time_t), .align = __alignof__(time_t), .tag = OpaqueInfo};
746 time_t *cached_latest = Table$get(c_modification_times, &path, Table$info(&Path$info, &time_info));
747 if (cached_latest) return *cached_latest;
751 if (stat(Path$as_c_string(path), &s) == 0) latest = s.st_mtime;
752 Table$set(&c_modification_times, &path, &latest, Table$info(&Path$info, &time_info));
754 OptionalClosure_t by_line = Path$by_line(path);
755 if (by_line.fn == NULL) return 0;
756 OptionalText_t (*next_line)(void *) = by_line.fn;
757 Path_t parent = Path$parent(path);
758 bool allow_dot_include = Path$has_extension(path, Text("s")) || Path$has_extension(path, Text("S"));
759 for (OptionalText_t line; (line = next_line(by_line.userdata)).tag != TEXT_NONE;) {
760 line = Text$trim(line, Text(" \t"), true, false);
761 if (!Text$starts_with(line, Text("#include"), NULL)
762 && !(allow_dot_include && Text$starts_with(line, Text(".include"), NULL)))
765 // Check for `"` after `#include` or `.include` and some spaces:
766 if (!Text$starts_with(Text$trim(Text$from(line, I(9)), Text(" \t"), true, false), Text("\""), NULL)) continue;
768 List_t chunks = Text$split(line, Text("\""));
769 if (chunks.length < 3) // Should be `#include "foo" ...` -> ["#include ", "foo", "..."]
772 Text_t included = *(Text_t *)(chunks.data + 1 * chunks.stride);
773 Path_t included_path = Path$resolved(Path$from_text(included), parent);
774 time_t included_time = latest_included_modification_time(included_path);
775 if (included_time > latest) {
776 latest = included_time;
777 Table$set(&c_modification_times, &path, &latest, Table$info(&Path$info, &time_info));
783 bool is_stale(Path_t path, Path_t relative_to, bool ignore_missing) {
784 struct stat target_stat;
785 if (stat(Path$as_c_string(path), &target_stat) != 0) {
786 if (ignore_missing) return false;
791 // Any file older than the compiler is stale:
792 if (target_stat.st_mtime < compiler_stat.st_mtime) return true;
795 if (Path$has_extension(relative_to, Text("c")) || Path$has_extension(relative_to, Text("h"))
796 || Path$has_extension(relative_to, Text("s")) || Path$has_extension(relative_to, Text("S"))) {
797 time_t mtime = latest_included_modification_time(relative_to);
798 return target_stat.st_mtime < mtime;
801 struct stat relative_to_stat;
802 if (stat(Path$as_c_string(relative_to), &relative_to_stat) != 0) {
803 if (ignore_missing) return false;
804 print_err("File doesn't exist: ", relative_to);
806 return target_stat.st_mtime < relative_to_stat.st_mtime;
809 bool is_stale_for_any(Path_t path, List_t relative_to, bool ignore_missing) {
810 for (int64_t i = 0; i < (int64_t)relative_to.length; i++) {
811 Path_t r = *(Path_t *)(relative_to.data + i * relative_to.stride);
812 if (is_stale(path, r, ignore_missing)) return true;
817 void transpile_header(env_t *base_env, Path_t path) {
818 Path_t h_filename = build_file(path, ".h");
819 ast_t *ast = parse_file(Path$as_c_string(path), NULL);
820 if (!ast) print_err("Could not parse file: ", path);
822 env_t *module_env = load_module_env(base_env, ast);
824 Text_t h_code = compile_file_header(module_env, Path$resolved(h_filename, Path$from_str(".")), ast);
826 FILE *header = fopen(Path$as_c_string(h_filename), "w");
827 if (!header) print_err("Failed to open header file: ", h_filename);
828 Text$print(header, h_code);
829 if (fclose(header) == -1) print_err("Failed to write header file: ", h_filename);
831 if (!quiet) print("Transpiled header:\t", Path$relative_to(h_filename, Path$current_dir()));
833 if (show_codegen.length > 0) xsystem(show_codegen, " <", h_filename);
836 void transpile_code(env_t *base_env, Path_t path) {
837 Path_t c_filename = build_file(path, ".c");
838 ast_t *ast = parse_file(Path$as_c_string(path), NULL);
839 if (!ast) print_err("Could not parse file: ", path);
841 env_t *module_env = load_module_env(base_env, ast);
843 Text_t c_code = compile_file(module_env, ast);
845 FILE *c_file = fopen(Path$as_c_string(c_filename), "w");
846 if (!c_file) print_err("Failed to write C file: ", c_filename);
848 Text$print(c_file, c_code);
850 binding_t *main_binding = get_binding(module_env, "main");
851 if (main_binding && main_binding->type->tag == FunctionType) {
852 type_t *ret = Match(main_binding->type, FunctionType)->ret;
853 if (ret->tag != VoidType && ret->tag != AbortType)
854 compiler_err(ast->file, ast->start, ast->end, "The main() function in this file has a return type of ",
855 type_to_text(ret), ", but it should not have any return value!");
857 Text$print(c_file, Texts("int parse_and_run$$", main_binding->code, "(int argc, char *argv[]) {\n",
858 module_env->do_source_mapping ? Text("#line 1\n") : EMPTY_TEXT, "tomo_init();\n",
859 namespace_name(module_env, module_env->namespace, Text("$initialize")),
862 compile_cli_arg_call(module_env, ast, main_binding->code, main_binding->type),
867 if (fclose(c_file) == -1) print_err("Failed to output C code to ", c_filename);
869 if (!quiet) print("Transpiled code:\t", Path$relative_to(c_filename, Path$current_dir()));
871 if (show_codegen.length > 0) xsystem(show_codegen, " <", c_filename);
874 void compile_object_file(Path_t path) {
875 Path_t obj_file = build_file(path, ".o");
876 Path_t c_file = build_file(path, ".c");
878 FILE *prog = run_cmd(cc, " ", cflags, " -O", optimization, " -c ", c_file, " -o ", obj_file);
879 if (!prog) print_err("Failed to run C compiler: ", cc);
880 int status = pclose(prog);
881 if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) exit(EXIT_FAILURE);
883 Path$write(build_file(path, ".config"), config_summary, 0644);
885 if (!quiet) print("Compiled object:\t", Path$relative_to(obj_file, Path$current_dir()));
888 Path_t compile_executable(env_t *base_env, Path_t path, Path_t exe_path, List_t object_files, List_t extra_ldlibs) {
889 ast_t *ast = parse_file(Path$as_c_string(path), NULL);
890 if (!ast) print_err("Could not parse file ", path);
891 env_t *env = load_module_env(base_env, ast);
892 binding_t *main_binding = get_binding(env, "main");
893 if (!main_binding || main_binding->type->tag != FunctionType)
894 print_err("No main() function has been defined for ", path, ", so it can't be run!");
896 Path_t manpage_file = build_file(Path$with_extension(path, Text(".1"), true), "");
897 if (clean_build || !Path$is_file(manpage_file, true) || is_stale(manpage_file, path, true)) {
898 Text_t manpage = compile_manpage(Path$base_name(exe_path), ast, Match(main_binding->type, FunctionType)->args);
899 Path$write(manpage_file, manpage, 0644);
900 if (!quiet) print("Wrote manpage:\t", Path$relative_to(manpage_file, Path$current_dir()));
902 if (verbose) whisper("Unchanged: ", manpage_file);
905 if (!clean_build && Path$is_file(exe_path, true) && !is_config_outdated(path)
906 && !is_stale_for_any(exe_path, object_files, false)
907 && !is_stale(exe_path, Path$sibling(path, Text("packages.ini")), true)
908 && !is_stale(exe_path, build_file(path, ":packages.ini"), true)) {
909 if (verbose) whisper("Unchanged: ", exe_path);
913 Text_t program = Texts("extern int parse_and_run$$", main_binding->code,
914 "(int argc, char *argv[]);\n"
915 "__attribute__ ((noinline))\n"
916 "int main(int argc, char *argv[]) {\n"
917 "\treturn parse_and_run$$",
921 Path_t runner_file = build_file(path, ".runner.c");
922 Path$write(runner_file, program, 0644);
924 // .a archive files need to go later in the positional order:
925 List_t archives = EMPTY_LIST;
926 for (int64_t i = 0; i < (int64_t)extra_ldlibs.length;) {
927 Text_t *lib = (Text_t *)(extra_ldlibs.data + i * extra_ldlibs.stride);
928 if (Text$ends_with(*lib, Text(".a"), NULL)) {
929 List$insert(&archives, lib, I(0), sizeof(Text_t));
930 List$remove_at(&extra_ldlibs, I(i + 1), I(1), sizeof(Text_t));
936 FILE *runner = run_cmd(
939 " ", cflags, " -O", optimization,
940 // Linker flags and dynamically linked shared packages:
941 " ", ldflags, " ", ldlibs, " ", list_text(extra_ldlibs), " ",
943 paths_str(object_files),
946 // Statically linked archive files (must come after runner):
947 // Packages are grouped to allow for circular dependencies among
948 // the packages that are used.
949 " ", is_gcc ? Texts("-Wl,--start-group ", list_text(archives), " -Wl,--end-group") : list_text(archives),
950 // Tomo static library:
951 " ", TOMO_PATH, "/lib/libtomo@", TOMO_VERSION, ".a",
955 if (show_codegen.length > 0) {
956 FILE *out = run_cmd(show_codegen);
957 Text$print(out, program);
961 Text$print(runner, program);
962 int status = pclose(runner);
963 if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) exit(EXIT_FAILURE);
965 if (!quiet) print("Compiled executable:\t", Path$relative_to(exe_path, Path$current_dir()));