diff options
| author | Bruce Hill <bruce@bruce-hill.com> | 2025-03-07 15:30:26 -0500 |
|---|---|---|
| committer | Bruce Hill <bruce@bruce-hill.com> | 2025-03-07 15:30:26 -0500 |
| commit | d590b9d386977d342fc3580e473219d5b9e38503 (patch) | |
| tree | fad1948672962d0e09fd21f32ccc8986ab907001 | |
| parent | 17a6313d28b6d179632798eaa24518b1d8166e88 (diff) | |
Fix number precision stuff
| -rw-r--r-- | environment.c | 4 | ||||
| -rw-r--r-- | stdlib/nums.c | 4 |
2 files changed, 4 insertions, 4 deletions
diff --git a/environment.c b/environment.c index aaf6dfe7..d37d19ef 100644 --- a/environment.c +++ b/environment.c @@ -230,7 +230,7 @@ env_t *new_compilation_unit(CORD libname) {"Num", Type(NumType, .bits=TYPE_NBITS64), "Num_t", "Num$info", TypedArray(ns_entry_t, {"near", "Num$near", "func(x,y:Num, ratio=1e-9, min_epsilon=1e-9 -> Bool)"}, {"clamped", "Num$clamped", "func(x,low,high:Num -> Num)"}, - {"format", "Num$format", "func(n:Num, precision=0 -> Text)"}, + {"format", "Num$format", "func(n:Num, precision=16 -> Text)"}, {"scientific", "Num$scientific", "func(n:Num,precision=0 -> Text)"}, {"isinf", "Num$isinf", "func(n:Num -> Bool)"}, {"isfinite", "Num$isfinite", "func(n:Num -> Bool)"}, @@ -259,7 +259,7 @@ env_t *new_compilation_unit(CORD libname) {"Num32", Type(NumType, .bits=TYPE_NBITS32), "Num32_t", "Num32$info", TypedArray(ns_entry_t, {"near", "Num32$near", "func(x,y:Num32, ratio=Num32(1e-9), min_epsilon=Num32(1e-9) -> Bool)"}, {"clamped", "Num32$clamped", "func(x,low,high:Num32 -> Num32)"}, - {"format", "Num32$format", "func(n:Num32, precision=0 -> Text)"}, + {"format", "Num32$format", "func(n:Num32, precision=8 -> Text)"}, {"scientific", "Num32$scientific", "func(n:Num32, precision=0 -> Text)"}, {"isinf", "Num32$isinf", "func(n:Num32 -> Bool)"}, {"isfinite", "Num32$isfinite", "func(n:Num32 -> Bool)"}, diff --git a/stdlib/nums.c b/stdlib/nums.c index b227ccb8..6a645a37 100644 --- a/stdlib/nums.c +++ b/stdlib/nums.c @@ -50,7 +50,7 @@ public CONSTFUNC bool Num$near(double a, double b, double ratio, double absolute } public Text_t Num$format(double f, Int_t precision) { - return Text$format("%.*f", (int)Int64$from_int(precision, false), f); + return Text$format("%.*g", (int)Int64$from_int(precision, false), f); } public Text_t Num$scientific(double f, Int_t precision) { @@ -125,7 +125,7 @@ public CONSTFUNC bool Num32$near(float a, float b, float ratio, float absolute) } public Text_t Num32$format(float f, Int_t precision) { - return Text$format("%.*f", (int)Int64$from_int(precision, false), (double)f); + return Text$format("%.*g", (int)Int64$from_int(precision, false), (double)f); } public Text_t Num32$scientific(float f, Int_t precision) { |