1884 lines
66 KiB
C
1884 lines
66 KiB
C
// Type info and methods for Text datatype, which uses the Boehm "cord" library
|
||
// and libunistr
|
||
|
||
#include <assert.h>
|
||
#include <ctype.h>
|
||
#include <err.h>
|
||
#include <gc.h>
|
||
#include <gmp.h>
|
||
#include <limits.h>
|
||
#include <printf.h>
|
||
#include <readline/history.h>
|
||
#include <readline/readline.h>
|
||
#include <stdbool.h>
|
||
#include <stdint.h>
|
||
#include <stdlib.h>
|
||
#include <sys/param.h>
|
||
|
||
#include <unicase.h>
|
||
#include <unictype.h>
|
||
#include <unigbrk.h>
|
||
#include <uniname.h>
|
||
#include <uninorm.h>
|
||
#include <unistd.h>
|
||
#include <unistdio.h>
|
||
#include <unistr.h>
|
||
|
||
#include "array.h"
|
||
#include "functions.h"
|
||
#include "integers.h"
|
||
#include "text.h"
|
||
#include "types.h"
|
||
|
||
#include "siphash.c"
|
||
|
||
typedef struct {
|
||
int64_t num_codepoints;
|
||
const uint32_t *codepoints;
|
||
const uint8_t *utf8;
|
||
} synthetic_grapheme_t;
|
||
|
||
#define MAX_SYNTHETIC_GRAPHEMES 1024
|
||
static synthetic_grapheme_t synthetic_graphemes[MAX_SYNTHETIC_GRAPHEMES] = {};
|
||
|
||
static int32_t num_synthetic_graphemes = 0;
|
||
|
||
static int32_t get_grapheme(Text_t text, int64_t index);
|
||
|
||
typedef struct {
|
||
int64_t subtext, sum_of_previous_subtexts;
|
||
} iteration_state_t;
|
||
|
||
static int32_t _next_grapheme(Text_t text, iteration_state_t *state, int64_t index);
|
||
|
||
int32_t find_synthetic_grapheme(const uint32_t *codepoints, int64_t len)
|
||
{
|
||
int32_t lo = 0, hi = num_synthetic_graphemes;
|
||
while (lo <= hi) {
|
||
int32_t mid = (lo + hi) / 2;
|
||
int32_t cmp = (synthetic_graphemes[mid].num_codepoints > len) - (synthetic_graphemes[mid].num_codepoints < len);
|
||
if (cmp == 0)
|
||
cmp = memcmp(synthetic_graphemes[mid].codepoints, codepoints, sizeof(uint32_t[len]));
|
||
|
||
if (cmp == 0)
|
||
return mid;
|
||
else if (cmp < 0)
|
||
lo = mid + 1;
|
||
else if (cmp > 0)
|
||
hi = mid - 1;
|
||
}
|
||
return hi;
|
||
}
|
||
|
||
int32_t get_synthetic_grapheme(const uint32_t *codepoints, int64_t len)
|
||
{
|
||
int32_t index = find_synthetic_grapheme(codepoints, len);
|
||
if (index >= 0
|
||
&& index < num_synthetic_graphemes
|
||
&& synthetic_graphemes[index].num_codepoints == len
|
||
&& memcmp(synthetic_graphemes[index].codepoints, codepoints, len) == 0) {
|
||
return -(index+1);
|
||
} else {
|
||
if (index < 0) index = 0;
|
||
|
||
if (num_synthetic_graphemes >= MAX_SYNTHETIC_GRAPHEMES)
|
||
fail("Too many synthetic graphemes!");
|
||
|
||
if (num_synthetic_graphemes > 0 && index != num_synthetic_graphemes) {
|
||
memmove(&synthetic_graphemes[index + 1], &synthetic_graphemes[index],
|
||
sizeof(synthetic_grapheme_t[num_synthetic_graphemes - index]));
|
||
}
|
||
|
||
uint32_t *buf = GC_MALLOC_ATOMIC(sizeof(uint32_t[len]));
|
||
memcpy(buf, codepoints, sizeof(uint32_t[len]));
|
||
synthetic_graphemes[index].codepoints = buf;
|
||
synthetic_graphemes[index].num_codepoints = len;
|
||
|
||
size_t u8_len = 0;
|
||
uint8_t *u8 = u32_to_u8(codepoints, len, NULL, &u8_len);
|
||
uint8_t *gc_u8 = GC_MALLOC_ATOMIC(u8_len + 1);
|
||
memcpy(gc_u8, u8, u8_len);
|
||
gc_u8[u8_len] = '\0';
|
||
synthetic_graphemes[index].utf8 = gc_u8;
|
||
assert(gc_u8);
|
||
free(u8);
|
||
|
||
++num_synthetic_graphemes;
|
||
|
||
return -(index+1);
|
||
}
|
||
}
|
||
|
||
static inline int64_t num_subtexts(Text_t t)
|
||
{
|
||
if (t.tag != TEXT_SUBTEXT) return 1;
|
||
int64_t len = t.length;
|
||
int64_t n = 0;
|
||
while (len > 0) {
|
||
len -= t.subtexts[n].length;
|
||
++n;
|
||
}
|
||
return n;
|
||
}
|
||
|
||
int text_visualize(FILE *stream, Text_t t)
|
||
{
|
||
switch (t.tag) {
|
||
case TEXT_SHORT_ASCII: return fprintf(stream, "<ascii length=%ld>%.*s</ascii>", t.length, t.length, t.short_ascii);
|
||
case TEXT_ASCII: return fprintf(stream, "<ascii length=%ld>%.*s</ascii>", t.length, t.length, t.ascii);
|
||
case TEXT_GRAPHEMES: case TEXT_SHORT_GRAPHEMES: {
|
||
int printed = fprintf(stream, "<graphemes length=%ld>", t.length);
|
||
printed += Text$print(stream, t);
|
||
printed += fprintf(stream, "</graphemes>");
|
||
return printed;
|
||
}
|
||
case TEXT_SUBTEXT: {
|
||
int printed = fprintf(stream, "<text length=%ld>", t.length);
|
||
int64_t to_print = t.length;
|
||
for (int i = 0; to_print > 0; ++i) {
|
||
printed += fprintf(stream, "\n ");
|
||
printed += text_visualize(stream, t.subtexts[i]);
|
||
to_print -= t.subtexts[i].length;
|
||
if (t.subtexts[i].length == 0) break;
|
||
}
|
||
printed += fprintf(stream, "\n</text>");
|
||
return printed;
|
||
}
|
||
default: return 0;
|
||
}
|
||
}
|
||
|
||
public int Text$print(FILE *stream, Text_t t)
|
||
{
|
||
if (t.length == 0) return 0;
|
||
|
||
switch (t.tag) {
|
||
case TEXT_SHORT_ASCII: return fwrite(t.short_ascii, sizeof(char), t.length, stream);
|
||
case TEXT_ASCII: return fwrite(t.ascii, sizeof(char), t.length, stream);
|
||
case TEXT_GRAPHEMES: case TEXT_SHORT_GRAPHEMES: {
|
||
int32_t *graphemes = t.tag == TEXT_SHORT_GRAPHEMES ? t.short_graphemes : t.graphemes;
|
||
int written = 0;
|
||
for (int64_t i = 0; i < t.length; i++) {
|
||
int32_t grapheme = graphemes[i];
|
||
if (grapheme >= 0) {
|
||
uint8_t buf[8];
|
||
size_t len = sizeof(buf);
|
||
uint8_t *u8 = u32_to_u8((uint32_t*)&grapheme, 1, buf, &len);
|
||
written += fwrite(u8, sizeof(char), len, stream);
|
||
if (u8 != buf) free(u8);
|
||
} else {
|
||
const uint8_t *u8 = synthetic_graphemes[-grapheme-1].utf8;
|
||
assert(u8);
|
||
written += fwrite(u8, sizeof(uint8_t), strlen((char*)u8), stream);
|
||
}
|
||
}
|
||
return written;
|
||
}
|
||
case TEXT_SUBTEXT: {
|
||
int written = 0;
|
||
int i = 0;
|
||
for (int64_t to_print = t.length; to_print > 0; to_print -= t.subtexts[i].length, ++i)
|
||
written += Text$print(stream, t.subtexts[i]);
|
||
return written;
|
||
}
|
||
default: return 0;
|
||
}
|
||
}
|
||
|
||
static Text_t concat2(Text_t a, Text_t b)
|
||
{
|
||
if (a.length == 0) return b;
|
||
if (b.length == 0) return a;
|
||
|
||
if (a.tag == TEXT_SUBTEXT && b.tag == TEXT_SUBTEXT) {
|
||
int64_t na = num_subtexts(a);
|
||
int64_t nb = num_subtexts(b);
|
||
Text_t ret = {
|
||
.length=a.length + b.length,
|
||
.tag=TEXT_SUBTEXT,
|
||
.subtexts=GC_MALLOC(sizeof(Text_t[na + nb])),
|
||
};
|
||
memcpy(&ret.subtexts[0], a.subtexts, sizeof(Text_t[na]));
|
||
memcpy(&ret.subtexts[na], b.subtexts, sizeof(Text_t[nb]));
|
||
return ret;
|
||
} else if (a.tag == TEXT_SUBTEXT) {
|
||
int64_t n = num_subtexts(a);
|
||
Text_t ret = {
|
||
.length=a.length + b.length,
|
||
.tag=TEXT_SUBTEXT,
|
||
.subtexts=GC_MALLOC(sizeof(Text_t[n + 1])),
|
||
};
|
||
memcpy(ret.subtexts, a.subtexts, sizeof(Text_t[n]));
|
||
ret.subtexts[n] = b;
|
||
return ret;
|
||
} else if (b.tag == TEXT_SUBTEXT) {
|
||
int64_t n = num_subtexts(b);
|
||
Text_t ret = {
|
||
.length=a.length + b.length,
|
||
.tag=TEXT_SUBTEXT,
|
||
.subtexts=GC_MALLOC(sizeof(Text_t[n + 1])),
|
||
};
|
||
ret.subtexts[0] = a;
|
||
memcpy(&ret.subtexts[1], b.subtexts, sizeof(Text_t[n]));
|
||
return ret;
|
||
} else {
|
||
Text_t ret = {
|
||
.length=a.length + b.length,
|
||
.tag=TEXT_SUBTEXT,
|
||
.subtexts=GC_MALLOC(sizeof(Text_t[2])),
|
||
};
|
||
ret.subtexts[0] = a;
|
||
ret.subtexts[1] = b;
|
||
return ret;
|
||
}
|
||
}
|
||
|
||
public Text_t Text$_concat(int n, Text_t items[n])
|
||
{
|
||
if (n == 0) return (Text_t){.length=0};
|
||
if (n == 1) return items[0];
|
||
if (n == 2) return concat2(items[0], items[1]);
|
||
|
||
int64_t len = 0, subtexts = 0;
|
||
for (int i = 0; i < n; i++) {
|
||
len += items[i].length;
|
||
if (items[i].length > 0)
|
||
subtexts += num_subtexts(items[i]);
|
||
}
|
||
|
||
Text_t ret = {
|
||
.length=len,
|
||
.tag=TEXT_SUBTEXT,
|
||
.subtexts=GC_MALLOC(sizeof(Text_t[len])),
|
||
};
|
||
int64_t sub_i = 0;
|
||
for (int i = 0; i < n; i++) {
|
||
if (items[i].length == 0)
|
||
continue;
|
||
|
||
if (items[i].tag == TEXT_SUBTEXT) {
|
||
for (int64_t j = 0, remainder = items[i].length; remainder > 0; j++) {
|
||
ret.subtexts[sub_i++] = items[i].subtexts[j];
|
||
remainder -= items[i].subtexts[j].length;
|
||
}
|
||
} else {
|
||
ret.subtexts[sub_i++] = items[i];
|
||
}
|
||
}
|
||
return ret;
|
||
}
|
||
|
||
public Text_t Text$slice(Text_t text, Int_t first_int, Int_t last_int)
|
||
{
|
||
int64_t first = Int_to_Int64(first_int, false);
|
||
int64_t last = Int_to_Int64(last_int, false);
|
||
if (first == 0) fail("Invalid index: 0");
|
||
if (last == 0) return (Text_t){.length=0};
|
||
|
||
if (first < 0) first = text.length + first + 1;
|
||
if (last < 0) last = text.length + last + 1;
|
||
|
||
if (last > text.length) last = text.length;
|
||
|
||
if (first > text.length || last < first)
|
||
return (Text_t){.length=0};
|
||
|
||
if (first == 1 && last == text.length)
|
||
return text;
|
||
|
||
switch (text.tag) {
|
||
case TEXT_SHORT_ASCII: {
|
||
Text_t ret = (Text_t) {
|
||
.tag=TEXT_SHORT_ASCII,
|
||
.length=last - first + 1,
|
||
};
|
||
memcpy(ret.short_ascii, text.short_ascii + (first-1), ret.length);
|
||
return ret;
|
||
}
|
||
case TEXT_ASCII: {
|
||
Text_t ret = {
|
||
.tag=TEXT_ASCII,
|
||
.length=last - first + 1,
|
||
.ascii=text.ascii + (first-1),
|
||
};
|
||
return ret;
|
||
}
|
||
case TEXT_SHORT_GRAPHEMES: {
|
||
assert((first == 1 && last == 1) || (first == 2 && last == 2));
|
||
Text_t ret = {
|
||
.tag=TEXT_SHORT_GRAPHEMES,
|
||
.length=1,
|
||
.short_graphemes={text.short_graphemes[first-1]},
|
||
};
|
||
return ret;
|
||
}
|
||
case TEXT_GRAPHEMES: {
|
||
Text_t ret = {
|
||
.tag=TEXT_GRAPHEMES,
|
||
.length=last - first + 1,
|
||
.graphemes=text.graphemes + (first-1),
|
||
};
|
||
return ret;
|
||
}
|
||
case TEXT_SUBTEXT: {
|
||
Text_t *subtexts = text.subtexts;
|
||
while (first > subtexts[0].length) {
|
||
first -= subtexts[0].length;
|
||
last -= subtexts[0].length;
|
||
++subtexts;
|
||
}
|
||
|
||
int64_t needed_len = (last - first) + 1;
|
||
int64_t num_subtexts = 0;
|
||
for (int64_t included = 0; included < needed_len; ) {
|
||
if (included == 0)
|
||
included += subtexts[num_subtexts].length - first + 1;
|
||
else
|
||
included += subtexts[num_subtexts].length;
|
||
num_subtexts += 1;
|
||
}
|
||
if (num_subtexts == 1)
|
||
return Text$slice(subtexts[0], Int64_to_Int(first), Int64_to_Int(last));
|
||
|
||
Text_t ret = {
|
||
.length=needed_len,
|
||
.tag=TEXT_SUBTEXT,
|
||
.subtexts=GC_MALLOC(sizeof(Text_t[num_subtexts])),
|
||
};
|
||
for (int64_t i = 0; i < num_subtexts; i++) {
|
||
ret.subtexts[i] = Text$slice(subtexts[i], Int64_to_Int(first), Int64_to_Int(last));
|
||
first = 1;
|
||
needed_len -= ret.subtexts[i].length;
|
||
last = first + needed_len - 1;
|
||
}
|
||
return ret;
|
||
}
|
||
default: errx(1, "Invalid tag");
|
||
}
|
||
}
|
||
|
||
Text_t text_from_u32(uint32_t *codepoints, int64_t num_codepoints, bool normalize)
|
||
{
|
||
uint32_t norm_buf[128];
|
||
if (normalize) {
|
||
size_t norm_length = sizeof(norm_buf)/sizeof(norm_buf[0]);
|
||
uint32_t *normalized = u32_normalize(UNINORM_NFC, codepoints, num_codepoints, norm_buf, &norm_length);
|
||
codepoints = normalized;
|
||
num_codepoints = norm_length;
|
||
}
|
||
|
||
char breaks[num_codepoints];
|
||
u32_grapheme_breaks(codepoints, num_codepoints, breaks);
|
||
|
||
Text_t ret = {
|
||
.length=0,
|
||
.tag=TEXT_SHORT_GRAPHEMES,
|
||
};
|
||
const uint32_t *src = codepoints;
|
||
int32_t *graphemes = ret.short_graphemes;
|
||
while (src < &codepoints[num_codepoints]) {
|
||
if (ret.tag == TEXT_SHORT_GRAPHEMES && ret.length + 1 > 2) {
|
||
graphemes = GC_MALLOC_ATOMIC(sizeof(int32_t[num_codepoints])); // May be a slight overallocation
|
||
graphemes[0] = ret.short_graphemes[0];
|
||
graphemes[1] = ret.short_graphemes[1];
|
||
ret.tag = TEXT_GRAPHEMES;
|
||
ret.graphemes = graphemes;
|
||
}
|
||
|
||
const uint32_t *next = u32_grapheme_next(src, &codepoints[num_codepoints]);
|
||
if (next == &src[1]) {
|
||
graphemes[ret.length] = (int32_t)*src;
|
||
} else {
|
||
// Synthetic grapheme
|
||
graphemes[ret.length] = get_synthetic_grapheme(src, next-src);
|
||
}
|
||
++ret.length;
|
||
src = next;
|
||
}
|
||
if (normalize && codepoints != norm_buf) free(codepoints);
|
||
return ret;
|
||
}
|
||
|
||
public Text_t Text$from_str(const char *str)
|
||
{
|
||
int64_t ascii_span = 0;
|
||
while (str[ascii_span] && isascii(str[ascii_span]))
|
||
ascii_span++;
|
||
|
||
if (str[ascii_span] == '\0') { // All ASCII
|
||
Text_t ret = {.length=ascii_span};
|
||
if (ascii_span <= 8) {
|
||
ret.tag = TEXT_SHORT_ASCII;
|
||
for (int64_t i = 0; i < ascii_span; i++)
|
||
ret.short_ascii[i] = str[i];
|
||
} else {
|
||
ret.tag = TEXT_ASCII;
|
||
ret.ascii = str;
|
||
}
|
||
return ret;
|
||
} else {
|
||
uint32_t buf[128];
|
||
size_t length = sizeof(buf)/sizeof(buf[0]);
|
||
uint32_t *codepoints = u8_to_u32((uint8_t*)str, ascii_span + strlen(str + ascii_span), buf, &length);
|
||
Text_t ret = text_from_u32(codepoints, length, true);
|
||
if (codepoints != buf) free(codepoints);
|
||
return ret;
|
||
}
|
||
}
|
||
|
||
static void u8_buf_append(Text_t text, char **buf, int64_t *capacity, int64_t *i)
|
||
{
|
||
switch (text.tag) {
|
||
case TEXT_ASCII: case TEXT_SHORT_ASCII: {
|
||
if (*i + text.length > (int64_t)*capacity) {
|
||
*capacity = *i + text.length + 1;
|
||
*buf = GC_REALLOC(*buf, *capacity);
|
||
}
|
||
|
||
const char *bytes = text.tag == TEXT_ASCII ? text.ascii : text.short_ascii;
|
||
memcpy(*buf + *i, bytes, text.length);
|
||
*i += text.length;
|
||
break;
|
||
}
|
||
case TEXT_GRAPHEMES: case TEXT_SHORT_GRAPHEMES: {
|
||
const int32_t *graphemes = text.tag == TEXT_GRAPHEMES ? text.graphemes : text.short_graphemes;
|
||
for (int64_t g = 0; g < text.length; g++) {
|
||
const uint32_t *codepoints = graphemes[g] < 0 ? synthetic_graphemes[-graphemes[g]-1].codepoints : (uint32_t*)&graphemes[g];
|
||
int64_t num_codepoints = graphemes[g] < 0 ? synthetic_graphemes[-graphemes[g]-1].num_codepoints : 1;
|
||
uint8_t u8_buf[64];
|
||
size_t u8_len = sizeof(u8_buf);
|
||
uint8_t *u8 = u32_to_u8(codepoints, num_codepoints, u8_buf, &u8_len);
|
||
|
||
if (*i + (int64_t)u8_len > (int64_t)*capacity) {
|
||
*capacity = *i + u8_len + 1;
|
||
*buf = GC_REALLOC(*buf, *capacity);
|
||
}
|
||
|
||
memcpy(*buf + *i, u8, u8_len);
|
||
*i += u8_len;
|
||
if (u8 != u8_buf) free(u8);
|
||
}
|
||
break;
|
||
}
|
||
case TEXT_SUBTEXT: {
|
||
for (int64_t s = 0, remaining = text.length; remaining > 0; s++) {
|
||
u8_buf_append(text.subtexts[s], buf, capacity, i);
|
||
remaining -= text.subtexts[s].length;
|
||
}
|
||
break;
|
||
}
|
||
default: break;
|
||
}
|
||
}
|
||
|
||
public const char *Text$as_c_string(Text_t text)
|
||
{
|
||
int64_t capacity = text.length + 1;
|
||
char *buf = GC_MALLOC_ATOMIC(capacity);
|
||
int64_t i = 0;
|
||
u8_buf_append(text, &buf, &capacity, &i);
|
||
|
||
if (i + 1 > (int64_t)capacity) {
|
||
capacity = i + 1;
|
||
buf = GC_REALLOC(buf, capacity);
|
||
}
|
||
buf[i] = '\0';
|
||
return buf;
|
||
}
|
||
|
||
public uint64_t Text$hash(Text_t *text)
|
||
{
|
||
if (text->hash != 0) return text->hash;
|
||
siphash sh;
|
||
siphashinit(&sh, sizeof(int32_t[text->length]), (uint64_t*)TOMO_HASH_KEY);
|
||
|
||
union {
|
||
int32_t chunks[2];
|
||
uint64_t whole;
|
||
} tmp;
|
||
switch (text->tag) {
|
||
case TEXT_ASCII: case TEXT_SHORT_ASCII: {
|
||
const char *bytes = text->tag == TEXT_ASCII ? text->ascii : text->short_ascii;
|
||
for (int64_t i = 0; i + 1 < text->length; i++) {
|
||
tmp.chunks[0] = (int32_t)bytes[i];
|
||
tmp.chunks[1] = (int32_t)bytes[i+1];
|
||
siphashadd64bits(&sh, tmp.whole);
|
||
}
|
||
int32_t last = text->length & 0x1 ? (int32_t)bytes[text->length-1] : 0; // Odd number of graphemes
|
||
text->hash = siphashfinish_last_part(&sh, (uint64_t)last);
|
||
break;
|
||
}
|
||
case TEXT_GRAPHEMES: {
|
||
const int32_t *graphemes = text->graphemes;
|
||
for (int64_t i = 0; i + 1 < text->length; i++) {
|
||
tmp.chunks[0] = graphemes[i];
|
||
tmp.chunks[1] = graphemes[i];
|
||
siphashadd64bits(&sh, tmp.whole);
|
||
}
|
||
int32_t last = text->length & 0x1 ? graphemes[text->length-1] : 0; // Odd number of graphemes
|
||
text->hash = siphashfinish_last_part(&sh, (uint64_t)last);
|
||
break;
|
||
}
|
||
case TEXT_SHORT_GRAPHEMES: {
|
||
tmp.chunks[0] = text->short_graphemes[0];
|
||
if (text->length > 1)
|
||
tmp.chunks[1] = text->short_graphemes[1];
|
||
text->hash = siphashfinish_last_part(&sh, (uint64_t)tmp.whole);
|
||
break;
|
||
}
|
||
case TEXT_SUBTEXT: {
|
||
int32_t leftover = 0;
|
||
for (int64_t sub_i = 0, to_hash = text->length; to_hash > 0; ) {
|
||
Text_t subtext = text->subtexts[sub_i];
|
||
if (subtext.tag == TEXT_ASCII || subtext.tag == TEXT_SHORT_ASCII) {
|
||
const char *bytes = subtext.tag == TEXT_ASCII ? subtext.ascii : subtext.short_ascii;
|
||
int64_t grapheme = 0;
|
||
if (leftover) {
|
||
tmp.chunks[0] = leftover;
|
||
tmp.chunks[1] = (int32_t)bytes[0];
|
||
siphashadd64bits(&sh, tmp.whole);
|
||
grapheme += 1;
|
||
}
|
||
for (; grapheme + 1 < subtext.length; grapheme += 2) {
|
||
tmp.chunks[0] = (int32_t)bytes[grapheme];
|
||
tmp.chunks[1] = (int32_t)bytes[grapheme+1];
|
||
siphashadd64bits(&sh, tmp.whole);
|
||
}
|
||
leftover = grapheme < subtext.length ? (int32_t)bytes[grapheme] : 0;
|
||
} else if (subtext.tag == TEXT_SHORT_GRAPHEMES) {
|
||
if (leftover) {
|
||
tmp.chunks[0] = leftover;
|
||
tmp.chunks[1] = subtext.short_graphemes[0];
|
||
siphashadd64bits(&sh, tmp.whole);
|
||
leftover = subtext.length > 1 ? subtext.short_graphemes[1] : 0;
|
||
} else if (subtext.length == 1) {
|
||
leftover = subtext.short_graphemes[0];
|
||
} else {
|
||
tmp.chunks[0] = subtext.short_graphemes[0];
|
||
tmp.chunks[1] = subtext.short_graphemes[1];
|
||
siphashadd64bits(&sh, tmp.whole);
|
||
}
|
||
} else if (subtext.tag == TEXT_GRAPHEMES) {
|
||
int32_t *graphemes = subtext.graphemes;
|
||
int64_t grapheme = 0;
|
||
if (leftover) {
|
||
tmp.chunks[0] = leftover;
|
||
tmp.chunks[1] = graphemes[0];
|
||
siphashadd64bits(&sh, tmp.whole);
|
||
grapheme += 1;
|
||
}
|
||
for (; grapheme + 1 < subtext.length; grapheme += 2) {
|
||
tmp.chunks[0] = graphemes[grapheme];
|
||
tmp.chunks[1] = graphemes[grapheme+1];
|
||
siphashadd64bits(&sh, tmp.whole);
|
||
}
|
||
leftover = grapheme < subtext.length ? graphemes[grapheme] : 0;
|
||
}
|
||
|
||
to_hash -= text->subtexts[sub_i].length;
|
||
|
||
++sub_i;
|
||
}
|
||
|
||
text->hash = siphashfinish_last_part(&sh, leftover);
|
||
break;
|
||
}
|
||
default: errx(1, "Invalid text");
|
||
}
|
||
|
||
if (text->hash == 0)
|
||
text->hash = 1;
|
||
|
||
return text->hash;
|
||
}
|
||
|
||
int32_t _next_grapheme(Text_t text, iteration_state_t *state, int64_t index)
|
||
{
|
||
switch (text.tag) {
|
||
case TEXT_ASCII: return index < text.length ? (int32_t)text.ascii[index] : 0;
|
||
case TEXT_SHORT_ASCII: return index < text.length ? (int32_t)text.short_ascii[index] : 0;
|
||
case TEXT_GRAPHEMES: return index < text.length ? text.graphemes[index] : 0;
|
||
case TEXT_SHORT_GRAPHEMES: return index < text.length ? text.short_graphemes[index] : 0;
|
||
case TEXT_SUBTEXT: {
|
||
iteration_state_t backup_state = {0, 0};
|
||
if (!state) state = &backup_state;
|
||
|
||
if (index < 0 || index >= text.length)
|
||
return 0;
|
||
|
||
while (index < state->sum_of_previous_subtexts && state->subtext > 0) {
|
||
state->sum_of_previous_subtexts -= text.subtexts[state->subtext].length;
|
||
state->subtext -= 1;
|
||
}
|
||
for (;;) {
|
||
if (index < state->sum_of_previous_subtexts + text.subtexts[state->subtext].length)
|
||
return _next_grapheme(text.subtexts[state->subtext], NULL, index - state->sum_of_previous_subtexts);
|
||
state->sum_of_previous_subtexts += text.subtexts[state->subtext].length;
|
||
state->subtext += 1;
|
||
}
|
||
return 0;
|
||
}
|
||
default: errx(1, "Invalid text");
|
||
}
|
||
return 0;
|
||
}
|
||
|
||
int32_t get_grapheme(Text_t text, int64_t index)
|
||
{
|
||
iteration_state_t state = {0, 0};
|
||
return _next_grapheme(text, &state, index);
|
||
}
|
||
|
||
public int32_t Text$compare(const Text_t *a, const Text_t *b)
|
||
{
|
||
int64_t len = MAX(a->length, b->length);
|
||
iteration_state_t a_state = {0, 0}, b_state = {0, 0};
|
||
for (int64_t i = 0; i < len; i++) {
|
||
int32_t ai = _next_grapheme(*a, &a_state, i);
|
||
int32_t bi = _next_grapheme(*b, &b_state, i);
|
||
if (ai == bi) continue;
|
||
int32_t cmp;
|
||
if (ai > 0 && bi > 0) {
|
||
cmp = u32_cmp((uint32_t*)&ai, (uint32_t*)&bi, 1);
|
||
} else if (ai > 0) {
|
||
cmp = u32_cmp2(
|
||
(uint32_t*)&ai, 1,
|
||
synthetic_graphemes[-bi-1].codepoints,
|
||
synthetic_graphemes[-bi-1].num_codepoints);
|
||
} else if (bi > 0) {
|
||
cmp = u32_cmp2(
|
||
synthetic_graphemes[-ai-1].codepoints,
|
||
synthetic_graphemes[-ai-1].num_codepoints,
|
||
(uint32_t*)&bi, 1);
|
||
} else {
|
||
cmp = u32_cmp2(
|
||
synthetic_graphemes[-ai-1].codepoints,
|
||
synthetic_graphemes[-ai-1].num_codepoints,
|
||
synthetic_graphemes[-bi-1].codepoints,
|
||
synthetic_graphemes[-bi-1].num_codepoints);
|
||
}
|
||
if (cmp != 0) return cmp;
|
||
}
|
||
return 0;
|
||
}
|
||
|
||
public bool Text$equal(const Text_t *a, const Text_t *b)
|
||
{
|
||
if (a->length != b->length || (a->hash != 0 && b->hash != 0 && a->hash != b->hash))
|
||
return false;
|
||
int64_t len = a->length;
|
||
iteration_state_t a_state = {0, 0}, b_state = {0, 0};
|
||
for (int64_t i = 0; i < len; i++) {
|
||
int32_t ai = _next_grapheme(*a, &a_state, i);
|
||
int32_t bi = _next_grapheme(*b, &b_state, i);
|
||
if (ai != bi) return false;
|
||
}
|
||
return true;
|
||
}
|
||
|
||
public bool Text$equal_ignoring_case(Text_t a, Text_t b)
|
||
{
|
||
if (a.length != b.length)
|
||
return false;
|
||
int64_t len = a.length;
|
||
iteration_state_t a_state = {0, 0}, b_state = {0, 0};
|
||
const char *language = uc_locale_language();
|
||
for (int64_t i = 0; i < len; i++) {
|
||
int32_t ai = _next_grapheme(a, &a_state, i);
|
||
int32_t bi = _next_grapheme(b, &b_state, i);
|
||
if (ai != bi) {
|
||
const uint32_t *a_codepoints = ai >= 0 ? (uint32_t*)&ai : synthetic_graphemes[-ai-1].codepoints;
|
||
int64_t a_len = ai >= 0 ? 1 : synthetic_graphemes[-ai-1].num_codepoints;
|
||
|
||
const uint32_t *b_codepoints = bi >= 0 ? (uint32_t*)&bi : synthetic_graphemes[-bi-1].codepoints;
|
||
int64_t b_len = bi >= 0 ? 1 : synthetic_graphemes[-bi-1].num_codepoints;
|
||
|
||
int cmp;
|
||
(void)u32_casecmp(a_codepoints, a_len, b_codepoints, b_len, language, UNINORM_NFC, &cmp);
|
||
if (cmp != 0)
|
||
return false;
|
||
}
|
||
}
|
||
return true;
|
||
}
|
||
|
||
public Text_t Text$upper(Text_t text)
|
||
{
|
||
array_t codepoints = Text$utf32_codepoints(text);
|
||
const char *language = uc_locale_language();
|
||
uint32_t buf[128];
|
||
size_t out_len;
|
||
uint32_t *upper = u32_toupper(codepoints.data, codepoints.length, language, UNINORM_NFC, buf, &out_len);
|
||
Text_t ret = text_from_u32(upper, out_len, false);
|
||
if (upper != buf) free(upper);
|
||
return ret;
|
||
}
|
||
|
||
public Text_t Text$lower(Text_t text)
|
||
{
|
||
array_t codepoints = Text$utf32_codepoints(text);
|
||
const char *language = uc_locale_language();
|
||
uint32_t buf[128];
|
||
size_t out_len;
|
||
uint32_t *lower = u32_tolower(codepoints.data, codepoints.length, language, UNINORM_NFC, buf, &out_len);
|
||
Text_t ret = text_from_u32(lower, out_len, false);
|
||
if (lower != buf) free(lower);
|
||
return ret;
|
||
}
|
||
|
||
public Text_t Text$title(Text_t text)
|
||
{
|
||
array_t codepoints = Text$utf32_codepoints(text);
|
||
const char *language = uc_locale_language();
|
||
uint32_t buf[128];
|
||
size_t out_len;
|
||
uint32_t *title = u32_totitle(codepoints.data, codepoints.length, language, UNINORM_NFC, buf, &out_len);
|
||
Text_t ret = text_from_u32(title, out_len, false);
|
||
if (title != buf) free(title);
|
||
return ret;
|
||
}
|
||
|
||
static inline void skip_whitespace(Text_t text, int64_t *i)
|
||
{
|
||
iteration_state_t state = {0, 0};
|
||
while (*i < text.length) {
|
||
int32_t grapheme = _next_grapheme(text, &state, *i);
|
||
if (grapheme > 0 && !uc_is_property_white_space(grapheme))
|
||
return;
|
||
*i += 1;
|
||
}
|
||
}
|
||
|
||
static inline bool match_grapheme(Text_t text, int64_t *i, int32_t grapheme)
|
||
{
|
||
if (*i < text.length && get_grapheme(text, *i) == grapheme) {
|
||
*i += 1;
|
||
return true;
|
||
}
|
||
return false;
|
||
}
|
||
|
||
static inline bool match_str(Text_t text, int64_t *i, const char *str)
|
||
{
|
||
iteration_state_t state = {0, 0};
|
||
int64_t matched = 0;
|
||
while (matched[str]) {
|
||
if (*i + matched >= text.length || _next_grapheme(text, &state, *i + matched) != str[matched])
|
||
return false;
|
||
matched += 1;
|
||
}
|
||
*i += matched;
|
||
return true;
|
||
}
|
||
|
||
static inline bool match_property(Text_t text, int64_t *i, uc_property_t prop)
|
||
{
|
||
if (*i >= text.length) return false;
|
||
int32_t grapheme = get_grapheme(text, *i);
|
||
if (grapheme < 0) // TODO: check every codepoint in the cluster?
|
||
grapheme = synthetic_graphemes[-grapheme-1].codepoints[0];
|
||
|
||
if (uc_is_property(grapheme, prop)) {
|
||
*i += 1;
|
||
return true;
|
||
}
|
||
return false;
|
||
}
|
||
|
||
static int64_t parse_int(Text_t text, int64_t *i)
|
||
{
|
||
iteration_state_t state = {0, 0};
|
||
int64_t value = 0;
|
||
for (;; *i += 1) {
|
||
int32_t grapheme = _next_grapheme(text, &state, *i);
|
||
if (grapheme < 0)
|
||
grapheme = synthetic_graphemes[-grapheme-1].codepoints[0];
|
||
int digit = uc_digit_value(grapheme);
|
||
if (digit < 0) break;
|
||
if (value >= INT64_MAX/10) break;
|
||
value = 10*value + digit;
|
||
}
|
||
return value;
|
||
}
|
||
|
||
const char *get_property_name(Text_t text, int64_t *i)
|
||
{
|
||
skip_whitespace(text, i);
|
||
char *name = GC_MALLOC_ATOMIC(UNINAME_MAX);
|
||
char *dest = name;
|
||
iteration_state_t state = {0, 0};
|
||
while (*i < text.length) {
|
||
int32_t grapheme = _next_grapheme(text, &state, *i);
|
||
if (!(grapheme & ~0xFF) && (isalnum(grapheme) || grapheme == ' ' || grapheme == '_' || grapheme == '-')) {
|
||
*dest = (char)grapheme;
|
||
++dest;
|
||
if (dest >= name + UNINAME_MAX - 1)
|
||
break;
|
||
} else {
|
||
break;
|
||
}
|
||
*i += 1;
|
||
}
|
||
|
||
while (dest > name && dest[-1] == ' ')
|
||
*(dest--) = '\0';
|
||
|
||
if (dest == name) return NULL;
|
||
*dest = '\0';
|
||
return name;
|
||
}
|
||
|
||
#define EAT1(text, state, index, cond) ({\
|
||
int32_t grapheme = _next_grapheme(text, state, index); \
|
||
bool success = (cond); \
|
||
if (success) index += 1; \
|
||
success; })
|
||
|
||
#define EAT2(text, state, index, cond1, cond2) ({\
|
||
int32_t grapheme = _next_grapheme(text, state, index); \
|
||
bool success = (cond1); \
|
||
if (success) { \
|
||
grapheme = _next_grapheme(text, state, index + 1); \
|
||
success = (cond2); \
|
||
if (success) \
|
||
index += 2; \
|
||
} \
|
||
success; })
|
||
|
||
|
||
#define EAT_MANY(text, state, index, cond) ({ int64_t _n = 0; while (EAT1(text, state, index, cond)) { _n += 1; } _n; })
|
||
|
||
int64_t match_email(Text_t text, int64_t text_index)
|
||
{
|
||
// email = local "@" domain
|
||
// local = 1-64 ([a-zA-Z0-9!#$%&‘*+–/=?^_`.{|}~] | non-ascii)
|
||
// domain = dns-label ("." dns-label)*
|
||
// dns-label = 1-63 ([a-zA-Z0-9-] | non-ascii)
|
||
|
||
iteration_state_t state = {0, 0};
|
||
if (text_index > 0) {
|
||
int32_t prev_codepoint = _next_grapheme(text, &state, text_index - 1);
|
||
if (prev_codepoint < 0)
|
||
prev_codepoint = synthetic_graphemes[-prev_codepoint-1].codepoints[0];
|
||
if (uc_is_property_alphabetic(prev_codepoint))
|
||
return -1;
|
||
}
|
||
|
||
int64_t start_index = text_index;
|
||
|
||
// Local part:
|
||
int64_t local_len = 0;
|
||
static const char *allowed_local = "!#$%&‘*+–/=?^_`.{|}~";
|
||
while (EAT1(text, &state, text_index,
|
||
(grapheme & ~0x7F) || isalnum((char)grapheme) || strchr(allowed_local, (char)grapheme))) {
|
||
local_len += 1;
|
||
if (local_len > 64) return -1;
|
||
}
|
||
|
||
if (!EAT1(text, &state, text_index, grapheme == '@'))
|
||
return -1;
|
||
|
||
// Host
|
||
int64_t host_len = 0;
|
||
do {
|
||
int64_t label_len = 0;
|
||
while (EAT1(text, &state, text_index,
|
||
(grapheme & ~0x7F) || isalnum((char)grapheme) || grapheme == '-')) {
|
||
label_len += 1;
|
||
if (label_len > 63) return -1;
|
||
}
|
||
|
||
if (label_len == 0)
|
||
return -1;
|
||
|
||
host_len += label_len;
|
||
if (host_len > 255)
|
||
return -1;
|
||
host_len += 1;
|
||
} while (EAT1(text, &state, text_index, grapheme == '.'));
|
||
|
||
return text_index - start_index;
|
||
}
|
||
|
||
int64_t match_ipv6(Text_t text, int64_t text_index)
|
||
{
|
||
iteration_state_t state = {0, 0};
|
||
if (text_index > 0) {
|
||
int32_t prev_codepoint = _next_grapheme(text, &state, text_index - 1);
|
||
if ((prev_codepoint & ~0x7F) && (isxdigit(prev_codepoint) || prev_codepoint == ':'))
|
||
return -1;
|
||
}
|
||
int64_t start_index = text_index;
|
||
const int NUM_CLUSTERS = 8;
|
||
bool double_colon_used = false;
|
||
for (int cluster = 0; cluster < NUM_CLUSTERS; cluster++) {
|
||
for (int digits = 0; digits < 4; digits++) {
|
||
if (!EAT1(text, &state, text_index, ~(grapheme & ~0x7F) && isxdigit((char)grapheme)))
|
||
break;
|
||
}
|
||
if (EAT1(text, &state, text_index, ~(grapheme & ~0x7F) && isxdigit((char)grapheme)))
|
||
return -1; // Too many digits
|
||
|
||
if (cluster == NUM_CLUSTERS-1) {
|
||
break;
|
||
} else if (!EAT1(text, &state, text_index, grapheme == ':')) {
|
||
if (double_colon_used)
|
||
break;
|
||
return -1;
|
||
}
|
||
|
||
if (EAT1(text, &state, text_index, grapheme == ':')) {
|
||
if (double_colon_used)
|
||
return -1;
|
||
double_colon_used = true;
|
||
}
|
||
}
|
||
return text_index - start_index;
|
||
}
|
||
|
||
static int64_t match_ipv4(Text_t text, int64_t text_index)
|
||
{
|
||
iteration_state_t state = {0, 0};
|
||
if (text_index > 0) {
|
||
int32_t prev_codepoint = _next_grapheme(text, &state, text_index - 1);
|
||
if ((prev_codepoint & ~0x7F) && (isdigit(prev_codepoint) || prev_codepoint == '.'))
|
||
return -1;
|
||
}
|
||
int64_t start_index = text_index;
|
||
|
||
const int NUM_CLUSTERS = 4;
|
||
for (int cluster = 0; cluster < NUM_CLUSTERS; cluster++) {
|
||
for (int digits = 0; digits < 3; digits++) {
|
||
if (!EAT1(text, &state, text_index, ~(grapheme & ~0x7F) && isdigit((char)grapheme))) {
|
||
if (digits == 0) return -1;
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (EAT1(text, &state, text_index, ~(grapheme & ~0x7F) && isdigit((char)grapheme)))
|
||
return -1; // Too many digits
|
||
|
||
if (cluster == NUM_CLUSTERS-1)
|
||
break;
|
||
else if (!EAT1(text, &state, text_index, grapheme == '.'))
|
||
return -1;
|
||
}
|
||
return (text_index - start_index);
|
||
}
|
||
|
||
int64_t match_uri(Text_t text, int64_t text_index)
|
||
{
|
||
// URI = scheme ":" ["//" authority] path ["?" query] ["#" fragment]
|
||
// scheme = [a-zA-Z] [a-zA-Z0-9+.-]
|
||
// authority = [userinfo "@"] host [":" port]
|
||
|
||
iteration_state_t state = {0, 0};
|
||
if (text_index > 0) {
|
||
int32_t prev_codepoint = _next_grapheme(text, &state, text_index - 1);
|
||
if (prev_codepoint < 0)
|
||
prev_codepoint = synthetic_graphemes[-prev_codepoint-1].codepoints[0];
|
||
if (uc_is_property_alphabetic(prev_codepoint))
|
||
return -1;
|
||
}
|
||
|
||
int64_t start_index = text_index;
|
||
|
||
// Scheme:
|
||
if (!EAT1(text, &state, text_index, isalpha(grapheme)))
|
||
return -1;
|
||
|
||
EAT_MANY(text, &state, text_index,
|
||
!(grapheme & ~0x7F) && (isalnum(grapheme) || grapheme == '+' || grapheme == '.' || grapheme == '-'));
|
||
|
||
if (text_index == start_index)
|
||
return -1;
|
||
|
||
if (!match_grapheme(text, &text_index, ':'))
|
||
return -1;
|
||
|
||
// Authority:
|
||
if (match_str(text, &text_index, "//")) {
|
||
int64_t authority_start = text_index;
|
||
// Username or host:
|
||
static const char *forbidden = "#?:@ \t\r\n<>[]{}\\^|\"`/";
|
||
if (EAT_MANY(text, &state, text_index, (grapheme & ~0x7F) || !strchr(forbidden, (char)grapheme)) == 0)
|
||
return -1;
|
||
|
||
if (EAT1(text, &state, text_index, grapheme == '@')) {
|
||
// Found a username, now get a host:
|
||
if (EAT_MANY(text, &state, text_index, (grapheme & ~0x7F) || !strchr(forbidden, (char)grapheme)) == 0)
|
||
return -1;
|
||
} else {
|
||
int64_t ip = authority_start;
|
||
int64_t ipv4_len = match_ipv4(text, ip);
|
||
if (ipv4_len > 0) {
|
||
ip += ipv4_len;
|
||
} else if (match_grapheme(text, &ip, '[')) {
|
||
ip += match_ipv6(text, ip);
|
||
if (ip > authority_start + 1 && match_grapheme(text, &ip, ']'))
|
||
text_index = ip;
|
||
}
|
||
}
|
||
|
||
// Port:
|
||
if (EAT1(text, &state, text_index, grapheme == ':')) {
|
||
if (EAT_MANY(text, &state, text_index, !(grapheme & ~0x7F) && isdigit(grapheme)) == 0)
|
||
return -1;
|
||
}
|
||
if (!EAT1(text, &state, text_index, grapheme == '/'))
|
||
return (text_index - start_index); // No path
|
||
} else {
|
||
// Optional path root:
|
||
EAT1(text, &state, text_index, grapheme == '/');
|
||
}
|
||
|
||
// Path:
|
||
static const char *non_path = " \"#?<>[]{}\\^`|";
|
||
EAT_MANY(text, &state, text_index, (grapheme & ~0x7F) || !strchr(non_path, (char)grapheme));
|
||
|
||
if (EAT1(text, &state, text_index, grapheme == '?')) { // Query
|
||
static const char *non_query = " \"#<>[]{}\\^`|";
|
||
EAT_MANY(text, &state, text_index, (grapheme & ~0x7F) || !strchr(non_query, (char)grapheme));
|
||
}
|
||
|
||
if (EAT1(text, &state, text_index, grapheme == '#')) { // Fragment
|
||
static const char *non_fragment = " \"#<>[]{}\\^`|";
|
||
EAT_MANY(text, &state, text_index, (grapheme & ~0x7F) || !strchr(non_fragment, (char)grapheme));
|
||
}
|
||
return text_index - start_index;
|
||
}
|
||
|
||
int64_t match(Text_t text, Pattern_t pattern, int64_t text_index, int64_t pattern_index)
|
||
{
|
||
if (pattern_index >= pattern.length) return 0;
|
||
int64_t start_index = text_index;
|
||
iteration_state_t pattern_state = {0, 0}, text_state = {0, 0};
|
||
while (pattern_index < pattern.length) {
|
||
int64_t old_pat_index = pattern_index;
|
||
if (EAT2(pattern, &pattern_state, pattern_index,
|
||
uc_is_property(grapheme, UC_PROPERTY_QUOTATION_MARK),
|
||
grapheme == '?')) {
|
||
// Quotations: "?", '?', etc
|
||
int32_t open = _next_grapheme(pattern, &pattern_state, pattern_index-2);
|
||
if (!match_grapheme(text, &text_index, open)) return -1;
|
||
int32_t close = open;
|
||
uc_mirror_char(open, (uint32_t*)&close);
|
||
if (!match_grapheme(pattern, &pattern_index, close))
|
||
fail("Pattern's closing brace is missing: %k", &pattern);
|
||
while (text_index < text.length) {
|
||
int32_t c = _next_grapheme(text, &text_state, text_index);
|
||
if (c == close)
|
||
return (text_index - start_index);
|
||
|
||
if (c == '\\' && text_index < text.length) {
|
||
text_index += 2;
|
||
} else {
|
||
text_index += 1;
|
||
}
|
||
}
|
||
return -1;
|
||
} else if (EAT2(pattern, &pattern_state, pattern_index,
|
||
uc_is_property(grapheme, UC_PROPERTY_PAIRED_PUNCTUATION),
|
||
grapheme == '?')) {
|
||
// Nested punctuation: (?), [?], etc
|
||
int32_t open = _next_grapheme(pattern, &pattern_state, pattern_index-2);
|
||
if (!match_grapheme(text, &text_index, open)) return -1;
|
||
int32_t close = open;
|
||
uc_mirror_char(open, (uint32_t*)&close);
|
||
if (!match_grapheme(pattern, &pattern_index, close))
|
||
fail("Pattern's closing brace is missing: %k", &pattern);
|
||
int64_t depth = 1;
|
||
for (; depth > 0 && text_index < text.length; ++text_index) {
|
||
int32_t c = _next_grapheme(text, &text_state, text_index);
|
||
if (c == open)
|
||
depth += 1;
|
||
else if (c == close)
|
||
depth -= 1;
|
||
}
|
||
if (depth > 0) return -1;
|
||
} else if (EAT1(pattern, &pattern_state, pattern_index,
|
||
grapheme == '{')) { // named patterns {id}, {2-3 hex}, etc.
|
||
skip_whitespace(pattern, &pattern_index);
|
||
int64_t min, max;
|
||
if (uc_is_digit(_next_grapheme(pattern, &pattern_state, pattern_index))) {
|
||
min = parse_int(pattern, &pattern_index);
|
||
skip_whitespace(pattern, &pattern_index);
|
||
if (match_grapheme(pattern, &pattern_index, '+')) {
|
||
max = INT64_MAX;
|
||
} else if (match_grapheme(pattern, &pattern_index, '-')) {
|
||
max = parse_int(pattern, &pattern_index);
|
||
} else {
|
||
max = min;
|
||
}
|
||
} else {
|
||
min = 1, max = INT64_MAX;
|
||
}
|
||
|
||
skip_whitespace(pattern, &pattern_index);
|
||
|
||
bool any = false;
|
||
uc_property_t prop;
|
||
int32_t specific_codepoint = UNINAME_INVALID;
|
||
bool want_to_match = !match_grapheme(pattern, &pattern_index, '!');
|
||
const char *prop_name;
|
||
if (match_str(pattern, &pattern_index, ".."))
|
||
prop_name = "..";
|
||
else
|
||
prop_name = get_property_name(pattern, &pattern_index);
|
||
|
||
if (!prop_name) {
|
||
// Literal character, e.g. {1?}
|
||
specific_codepoint = _next_grapheme(pattern, &pattern_state, pattern_index);
|
||
pattern_index += 1;
|
||
} else if (strlen(prop_name) == 1) {
|
||
// Single letter names: {1+ A}
|
||
specific_codepoint = prop_name[0];
|
||
prop_name = NULL;
|
||
}
|
||
|
||
skip_whitespace(pattern, &pattern_index);
|
||
if (!match_grapheme(pattern, &pattern_index, '}'))
|
||
fail("Missing closing '}' in pattern: %k", &pattern);
|
||
|
||
int64_t before_group = text_index;
|
||
#define FAIL() ({ if (min < 1) { text_index = before_group; continue; } else { return -1; } })
|
||
if (prop_name) {
|
||
switch (tolower(prop_name[0])) {
|
||
case '.':
|
||
if (prop_name[1] == '.') {
|
||
any = true;
|
||
prop = UC_PROPERTY_PRIVATE_USE;
|
||
goto got_prop;
|
||
}
|
||
break;
|
||
case 'd':
|
||
if (strcasecmp(prop_name, "digit") == 0) {
|
||
prop = UC_PROPERTY_DECIMAL_DIGIT;
|
||
goto got_prop;
|
||
}
|
||
break;
|
||
case 'e':
|
||
if (strcasecmp(prop_name, "end") == 0) {
|
||
if (text_index != text.length)
|
||
FAIL();
|
||
continue;
|
||
} else if (prop_name && strcasecmp(prop_name, "email") == 0) {
|
||
int64_t len = match_email(text, text_index);
|
||
if (len < 0)
|
||
FAIL();
|
||
text_index += len;
|
||
continue;
|
||
} else if (prop_name && strcasecmp(prop_name, "emoji") == 0) {
|
||
prop = UC_PROPERTY_EMOJI;
|
||
goto got_prop;
|
||
}
|
||
break;
|
||
case 'i':
|
||
if (prop_name && strcasecmp(prop_name, "id") == 0) {
|
||
if (!EAT1(text, &text_state, text_index,
|
||
uc_is_property(grapheme, UC_PROPERTY_XID_START)))
|
||
FAIL();
|
||
EAT_MANY(text, &text_state, text_index,
|
||
uc_is_property(grapheme, UC_PROPERTY_XID_CONTINUE));
|
||
continue;
|
||
} else if (prop_name && strcasecmp(prop_name, "int") == 0) {
|
||
EAT1(text, &text_state, text_index, grapheme == '-');
|
||
int64_t n = EAT_MANY(text, &text_state, text_index,
|
||
uc_is_property(grapheme, UC_PROPERTY_DECIMAL_DIGIT));
|
||
if (n <= 0)
|
||
FAIL();
|
||
continue;
|
||
} else if (prop_name && strcasecmp(prop_name, "ipv4") == 0) {
|
||
int64_t len = match_ipv4(text, text_index);
|
||
if (len < 0)
|
||
FAIL();
|
||
text_index += len;
|
||
continue;
|
||
} else if (prop_name && strcasecmp(prop_name, "ipv6") == 0) {
|
||
int64_t len = match_ipv6(text, text_index);
|
||
if (len < 0)
|
||
FAIL();
|
||
text_index += len;
|
||
continue;
|
||
} else if (prop_name && strcasecmp(prop_name, "ip") == 0) {
|
||
int64_t len = match_ipv6(text, text_index);
|
||
if (len < 0)
|
||
len = match_ipv4(text, text_index);
|
||
if (len < 0)
|
||
FAIL();
|
||
text_index += len;
|
||
continue;
|
||
}
|
||
break;
|
||
case 'n':
|
||
if (prop_name && strcasecmp(prop_name, "num") == 0) {
|
||
EAT1(text, &text_state, text_index, grapheme == '-');
|
||
int64_t pre_decimal = EAT_MANY(text, &text_state, text_index,
|
||
uc_is_property(grapheme, UC_PROPERTY_DECIMAL_DIGIT));
|
||
bool decimal = (EAT1(text, &text_state, text_index, grapheme == '.') == 1);
|
||
int64_t post_decimal = decimal ? EAT_MANY(text, &text_state, text_index,
|
||
uc_is_property(grapheme, UC_PROPERTY_DECIMAL_DIGIT)) : 0;
|
||
if (pre_decimal == 0 && post_decimal == 0)
|
||
FAIL();
|
||
continue;
|
||
}
|
||
break;
|
||
case 's':
|
||
if (strcasecmp(prop_name, "start") == 0) {
|
||
if (text_index != 0) return -1;
|
||
continue;
|
||
}
|
||
break;
|
||
case 'u':
|
||
if (prop_name && strcasecmp(prop_name, "uri") == 0) {
|
||
int64_t len = match_uri(text, text_index);
|
||
if (len < 0)
|
||
FAIL();
|
||
text_index += len;
|
||
continue;
|
||
} else if (prop_name && strcasecmp(prop_name, "url") == 0) {
|
||
int64_t lookahead = text_index;
|
||
if (!(match_str(text, &lookahead, "https:")
|
||
|| match_str(text, &lookahead, "http:")
|
||
|| match_str(text, &lookahead, "ftp:")
|
||
|| match_str(text, &lookahead, "wss:")
|
||
|| match_str(text, &lookahead, "ws:")))
|
||
FAIL();
|
||
|
||
int64_t len = match_uri(text, text_index);
|
||
if (len < 0)
|
||
FAIL();
|
||
text_index += len;
|
||
continue;
|
||
}
|
||
break;
|
||
}
|
||
|
||
prop = uc_property_byname(prop_name);
|
||
if (!uc_property_is_valid(prop)) {
|
||
specific_codepoint = unicode_name_character(prop_name);
|
||
if (specific_codepoint == UNINAME_INVALID)
|
||
fail("Not a valid property or character name: %s", prop_name);
|
||
}
|
||
}
|
||
got_prop:;
|
||
|
||
if (min == 0 && pattern_index < pattern.length) {
|
||
int64_t match_len = match(text, pattern, text_index, pattern_index);
|
||
if (match_len >= 0)
|
||
return (text_index - start_index) + match_len;
|
||
}
|
||
|
||
for (int64_t count = 0; count < max; ) {
|
||
int32_t grapheme = _next_grapheme(text, &text_state, text_index);
|
||
if (grapheme < 0)
|
||
grapheme = synthetic_graphemes[-grapheme-1].codepoints[0];
|
||
|
||
bool success;
|
||
if (any)
|
||
success = true;
|
||
else if (specific_codepoint != UNINAME_INVALID)
|
||
success = (grapheme == specific_codepoint);
|
||
else
|
||
success = uc_is_property(grapheme, prop);
|
||
|
||
if (success != want_to_match) {
|
||
if (count < min) return -1;
|
||
else break;
|
||
}
|
||
|
||
text_index += 1;
|
||
count += 1;
|
||
|
||
if (count >= min) {
|
||
if (pattern_index < pattern.length) {
|
||
int64_t match_len = match(text, pattern, text_index, pattern_index);
|
||
if (match_len >= 0) {
|
||
return (text_index - start_index) + match_len;
|
||
}
|
||
} else if (text_index >= text.length) {
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
} else {
|
||
// Plain character:
|
||
pattern_index = old_pat_index;
|
||
int32_t pat_grapheme = _next_grapheme(pattern, &pattern_state, pattern_index);
|
||
int32_t text_grapheme = _next_grapheme(text, &text_state, text_index);
|
||
if (pat_grapheme != text_grapheme)
|
||
return -1;
|
||
|
||
pattern_index += 1;
|
||
text_index += 1;
|
||
}
|
||
}
|
||
if (text_index >= text.length && pattern_index < pattern.length)
|
||
return -1;
|
||
return (text_index - start_index);
|
||
}
|
||
|
||
#undef EAT1
|
||
#undef EAT2
|
||
#undef EAT_MANY
|
||
|
||
public Int_t Text$find(Text_t text, Pattern_t pattern, Int_t from_index, int64_t *match_length)
|
||
{
|
||
int64_t first = Int_to_Int64(from_index, false);
|
||
if (first == 0) fail("Invalid index: 0");
|
||
if (first < 0) first = text.length + first + 1;
|
||
if (first > text.length || first < 1)
|
||
return I_small(0);
|
||
|
||
int32_t first_grapheme = get_grapheme(pattern, 0);
|
||
bool find_first = (first_grapheme != '{'
|
||
&& !uc_is_property(first_grapheme, UC_PROPERTY_QUOTATION_MARK)
|
||
&& !uc_is_property(first_grapheme, UC_PROPERTY_PAIRED_PUNCTUATION));
|
||
|
||
iteration_state_t text_state = {0, 0};
|
||
|
||
for (int64_t i = first-1; i < text.length; i++) {
|
||
// Optimization: quickly skip ahead to first char in pattern:
|
||
if (find_first) {
|
||
while (i < text.length && _next_grapheme(text, &text_state, i) != first_grapheme)
|
||
++i;
|
||
}
|
||
|
||
int64_t m = match(text, pattern, i, 0);
|
||
if (m >= 0) {
|
||
if (match_length)
|
||
*match_length = m;
|
||
return I(i+1);
|
||
}
|
||
}
|
||
if (match_length)
|
||
*match_length = -1;
|
||
return I(0);
|
||
}
|
||
|
||
public bool Text$has(Text_t text, Pattern_t pattern)
|
||
{
|
||
return !I_is_zero(Text$find(text, pattern, I_small(1), NULL));
|
||
}
|
||
|
||
public int printf_text_size(const struct printf_info *info, size_t n, int argtypes[n], int sizes[n])
|
||
{
|
||
if (n < 1) return -1;
|
||
(void)info;
|
||
argtypes[0] = PA_POINTER;
|
||
sizes[0] = sizeof(Text_t*);
|
||
return 1;
|
||
}
|
||
|
||
public int printf_text(FILE *stream, const struct printf_info *info, const void *const args[])
|
||
{
|
||
Text_t t = **(Text_t**)args[0];
|
||
if (info->alt)
|
||
return text_visualize(stream, t);
|
||
else
|
||
return Text$print(stream, t);
|
||
}
|
||
|
||
static inline Text_t _quoted(Text_t text, bool colorize, char quote_char)
|
||
{
|
||
// TODO: optimize for ASCII and short strings
|
||
array_t graphemes = {.atomic=1};
|
||
#define add_char(c) Array$insert_value(&graphemes, (uint32_t)c, I_small(0), sizeof(uint32_t))
|
||
#define add_str(s) ({ for (char *_c = s; *_c; ++_c) Array$insert_value(&graphemes, (uint32_t)*_c, I_small(0), sizeof(uint32_t)); })
|
||
if (colorize)
|
||
add_str("\x1b[35m");
|
||
if (quote_char != '"' && quote_char != '\"' && quote_char != '`')
|
||
add_char('$');
|
||
add_char(quote_char);
|
||
|
||
#define add_escaped(str) ({ if (colorize) add_str("\x1b[34;1m"); add_char('\\'); add_str(str); if (colorize) add_str("\x1b[0;35m"); })
|
||
iteration_state_t state = {0, 0};
|
||
for (int64_t i = 0; i < text.length; i++) {
|
||
int32_t g = _next_grapheme(text, &state, i);
|
||
switch (g) {
|
||
case '\a': add_escaped("a"); break;
|
||
case '\b': add_escaped("b"); break;
|
||
case '\x1b': add_escaped("e"); break;
|
||
case '\f': add_escaped("f"); break;
|
||
case '\n': add_escaped("n"); break;
|
||
case '\r': add_escaped("r"); break;
|
||
case '\t': add_escaped("t"); break;
|
||
case '\v': add_escaped("v"); break;
|
||
case '\\': add_escaped("\\"); break;
|
||
case '\x00' ... '\x06': case '\x0E' ... '\x1A':
|
||
case '\x1C' ... '\x1F': case '\x7F' ... '\x7F': {
|
||
if (colorize) add_str("\x1b[34;1m");
|
||
add_char('\\');
|
||
add_char('x');
|
||
char tmp[4];
|
||
sprintf(tmp, "%02X", g);
|
||
add_str(tmp);
|
||
if (colorize)
|
||
add_str("\x1b[0;35m");
|
||
break;
|
||
}
|
||
default: {
|
||
if (g == quote_char)
|
||
add_escaped(((char[2]){quote_char, 0}));
|
||
else
|
||
add_char(g);
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
|
||
add_char(quote_char);
|
||
if (colorize)
|
||
add_str("\x1b[m");
|
||
|
||
return (Text_t){.length=graphemes.length, .tag=TEXT_GRAPHEMES, .graphemes=graphemes.data};
|
||
#undef add_str
|
||
#undef add_char
|
||
#undef add_escaped
|
||
}
|
||
|
||
public Text_t Text$as_text(const void *text, bool colorize, const TypeInfo *info)
|
||
{
|
||
(void)info;
|
||
if (!text) return info && info->TextInfo.lang ? Text$from_str(info->TextInfo.lang) : Text("Text");
|
||
Text_t as_text = _quoted(*(Text_t*)text, colorize, info == &Pattern ? '/' : '"');
|
||
if (info && info->TextInfo.lang && info != &$Text && info != &Pattern)
|
||
as_text = Text$concat(colorize ? Text("\x1b[1m$") : Text("$"), Text$from_str(info->TextInfo.lang), as_text);
|
||
return as_text;
|
||
}
|
||
|
||
public Text_t Text$quoted(Text_t text, bool colorize)
|
||
{
|
||
return _quoted(text, colorize, '"');
|
||
}
|
||
|
||
public array_t Text$find_all(Text_t text, Pattern_t pattern)
|
||
{
|
||
if (pattern.length == 0) // special case
|
||
return (array_t){.length=0};
|
||
|
||
array_t matches = {};
|
||
|
||
Int_t i = I_small(1);
|
||
for (;;) {
|
||
int64_t len;
|
||
Int_t found = Text$find(text, pattern, i, &len);
|
||
if (I_is_zero(found)) break;
|
||
Text_t match = Text$slice(text, found, Int$plus(found, Int64_to_Int(len-1)));
|
||
Array$insert(&matches, &match, I_small(0), sizeof(Text_t));
|
||
i = Int$plus(found, Int64_to_Int(len <= 0 ? 1 : len));
|
||
}
|
||
|
||
return matches;
|
||
}
|
||
|
||
public Text_t Text$replace(Text_t text, Pattern_t pattern, Text_t replacement, Pattern_t placeholder)
|
||
{
|
||
Text_t ret = {.length=0};
|
||
|
||
Int_t i = I_small(1);
|
||
for (;;) {
|
||
int64_t len;
|
||
Int_t found = Text$find(text, pattern, i, &len);
|
||
if (I_is_zero(found)) break;
|
||
|
||
Text_t replacement_text = replacement;
|
||
if (placeholder.length > 0) {
|
||
Text_t matched_text = Text$slice(text, found, Int$plus(found, Int64_to_Int(len-1)));
|
||
replacement_text = Text$replace(replacement, placeholder, matched_text, Text(""));
|
||
}
|
||
|
||
if (Int$compare(&found, &i, &$Text) > 0) {
|
||
Text_t before_slice = Text$slice(text, i, Int$minus(found, I_small(1)));
|
||
ret = Text$concat(ret, before_slice, replacement_text);
|
||
} else {
|
||
ret = concat2(ret, replacement_text);
|
||
}
|
||
i = Int$plus(found, Int64_to_Int(len <= 0 ? 1 : len));
|
||
}
|
||
if (Int_to_Int64(i, false) <= text.length) {
|
||
Text_t last_slice = Text$slice(text, i, Int64_to_Int(text.length));
|
||
ret = concat2(ret, last_slice);
|
||
}
|
||
return ret;
|
||
}
|
||
|
||
public Text_t Text$replace_chain(Text_t text, array_t patterns, array_t replacements, Text_t placeholder)
|
||
{
|
||
if (patterns.length != replacements.length)
|
||
fail("The number of patterns given (%ld) is not the same as the number of replacements (%ld)",
|
||
patterns.length, replacements.length);
|
||
|
||
if (patterns.length == 0) return text;
|
||
|
||
Text_t ret = {.length=0};
|
||
|
||
Pattern_t first_pattern = *(Pattern_t*)(patterns.data);
|
||
int32_t first_grapheme = get_grapheme(first_pattern, 0);
|
||
bool find_first = (first_grapheme != '{'
|
||
&& !uc_is_property(first_grapheme, UC_PROPERTY_QUOTATION_MARK)
|
||
&& !uc_is_property(first_grapheme, UC_PROPERTY_PAIRED_PUNCTUATION));
|
||
|
||
iteration_state_t text_state = {0, 0};
|
||
|
||
int64_t nonmatch_pos = 0;
|
||
for (int64_t pos = 0; pos < text.length; ) {
|
||
// Optimization: quickly skip ahead to first char in pattern:
|
||
if (find_first) {
|
||
while (pos < text.length && _next_grapheme(text, &text_state, pos) != first_grapheme)
|
||
++pos;
|
||
}
|
||
|
||
// Get all match lengths:
|
||
int64_t lengths[patterns.length] = {};
|
||
for (int64_t i = 0, match_pos = pos; i < patterns.length; i++) {
|
||
Pattern_t pattern = *(Pattern_t*)(patterns.data + i*patterns.stride);
|
||
|
||
// If one of the patterns is `?` sandwiched between two pats
|
||
if (i > 0 && i < patterns.length-1 && Text$equal(&pattern, (Pattern_t[1]){Text("?")})) {
|
||
Pattern_t prev_pat = *(Pattern_t*)(patterns.data + (i-1)*patterns.stride);
|
||
int32_t prev_last_grapheme = get_grapheme(prev_pat, prev_pat.length-1);
|
||
if (prev_last_grapheme < 0) goto literal_pat;
|
||
|
||
Pattern_t next_pat = *(Pattern_t*)(patterns.data + (i+1)*patterns.stride);
|
||
int32_t next_first_grapheme = get_grapheme(next_pat, 0);
|
||
if (next_first_grapheme < 0) goto literal_pat;
|
||
|
||
int32_t mirrored = prev_last_grapheme;
|
||
uc_mirror_char(prev_last_grapheme, (uint32_t*)&mirrored);
|
||
|
||
if (next_first_grapheme != mirrored)
|
||
goto literal_pat;
|
||
|
||
if ((uc_is_property_quotation_mark(prev_last_grapheme) && uc_is_property_quotation_mark(next_first_grapheme))
|
||
|| ((uc_is_property_paired_punctuation(prev_last_grapheme)
|
||
&& uc_is_property_paired_punctuation(next_first_grapheme)
|
||
&& uc_is_property_left_of_pair(prev_last_grapheme)))) {
|
||
// $/"/, $/?/, $/"/
|
||
// $/(/, $/?/, $/)/
|
||
|
||
Pattern_t matching_pair_pat = text_from_u32((uint32_t[3]){prev_last_grapheme, '?', next_first_grapheme}, 3, false);
|
||
int64_t enclosing_len = match(text, matching_pair_pat, match_pos-1, 0);
|
||
if (enclosing_len < 0) goto no_match;
|
||
|
||
assert(enclosing_len >= 2);
|
||
lengths[i] = enclosing_len - 2; // Exclude '(' and ')' or whatever delims
|
||
goto found_match;
|
||
}
|
||
}
|
||
|
||
literal_pat:;
|
||
lengths[i] = match(text, pattern, match_pos, 0);
|
||
if (lengths[i] < 0)
|
||
goto no_match;
|
||
|
||
found_match:
|
||
match_pos += lengths[i];
|
||
}
|
||
|
||
// If we skipped over some non-matching text before finding a match, insert it here:
|
||
if (pos > nonmatch_pos) {
|
||
Text_t before_slice = Text$slice(text, Int64_to_Int(nonmatch_pos+1), Int64_to_Int(pos));
|
||
ret = concat2(ret, before_slice);
|
||
}
|
||
|
||
// Concatenate the slices/replacements
|
||
for (int64_t i = 0, replace_pos = pos; i < patterns.length; i++) {
|
||
Text_t replacement = *(Text_t*)(replacements.data + i*replacements.stride);
|
||
if (placeholder.length > 0) {
|
||
Text_t matched_text = Text$slice(text, Int64_to_Int(replace_pos+1), Int64_to_Int(replace_pos + lengths[i]));
|
||
replacement = Text$replace(replacement, placeholder, matched_text, Text(""));
|
||
}
|
||
|
||
ret = concat2(ret, replacement);
|
||
replace_pos += lengths[i];
|
||
}
|
||
|
||
int64_t total_match_len = 0;
|
||
for (int64_t i = 0; i < patterns.length; i++)
|
||
total_match_len += lengths[i];
|
||
|
||
pos += (total_match_len <= 0) ? 1 : total_match_len;
|
||
nonmatch_pos = pos;
|
||
continue;
|
||
|
||
no_match:
|
||
pos += 1;
|
||
continue;
|
||
}
|
||
if (nonmatch_pos <= text.length) {
|
||
Text_t last_slice = Text$slice(text, Int64_to_Int(nonmatch_pos+1), Int64_to_Int(text.length));
|
||
ret = concat2(ret, last_slice);
|
||
}
|
||
return ret;
|
||
}
|
||
|
||
public Text_t Text$replace_all(Text_t text, table_t replacements, Text_t placeholder)
|
||
{
|
||
if (replacements.entries.length == 0) return text;
|
||
|
||
Text_t ret = {.length=0};
|
||
|
||
int64_t nonmatch_pos = 0;
|
||
for (int64_t pos = 0; pos < text.length; ) {
|
||
// Find the first matching pattern at this position:
|
||
for (int64_t i = 0; i < replacements.entries.length; i++) {
|
||
Pattern_t pattern = *(Pattern_t*)(replacements.entries.data + i*replacements.entries.stride);
|
||
int64_t len = match(text, pattern, pos, 0);
|
||
if (len < 0) continue;
|
||
|
||
// If we skipped over some non-matching text before finding a match, insert it here:
|
||
if (pos > nonmatch_pos) {
|
||
Text_t before_slice = Text$slice(text, Int64_to_Int(nonmatch_pos+1), Int64_to_Int(pos));
|
||
ret = concat2(ret, before_slice);
|
||
}
|
||
|
||
// Concatenate the replacement:
|
||
Text_t replacement = *(Text_t*)(replacements.entries.data + i*replacements.entries.stride + sizeof(Text_t));
|
||
if (placeholder.length > 0) {
|
||
Text_t matched_text = Text$slice(text, Int64_to_Int(pos+1), Int64_to_Int(pos + len));
|
||
replacement = Text$replace(replacement, placeholder, matched_text, Text(""));
|
||
}
|
||
ret = concat2(ret, replacement);
|
||
pos += len > 0 ? len : 1;
|
||
nonmatch_pos = pos;
|
||
goto next_pos;
|
||
}
|
||
|
||
pos += 1;
|
||
next_pos:
|
||
continue;
|
||
}
|
||
|
||
if (nonmatch_pos <= text.length) {
|
||
Text_t last_slice = Text$slice(text, Int64_to_Int(nonmatch_pos+1), Int64_to_Int(text.length));
|
||
ret = concat2(ret, last_slice);
|
||
}
|
||
return ret;
|
||
}
|
||
|
||
public array_t Text$split(Text_t text, Pattern_t pattern)
|
||
{
|
||
if (text.length == 0) // special case
|
||
return (array_t){.length=0};
|
||
|
||
if (pattern.length == 0) // special case
|
||
return Text$clusters(text);
|
||
|
||
array_t chunks = {};
|
||
|
||
Int_t i = I_small(1);
|
||
for (;;) {
|
||
int64_t len;
|
||
Int_t found = Text$find(text, pattern, i, &len);
|
||
if (I_is_zero(found)) break;
|
||
Text_t chunk = Text$slice(text, i, Int$minus(found, I_small(1)));
|
||
Array$insert(&chunks, &chunk, I_small(0), sizeof(Text_t));
|
||
i = Int$plus(found, Int64_to_Int(len <= 0 ? 1 : len));
|
||
}
|
||
|
||
Text_t last_chunk = Text$slice(text, i, Int64_to_Int(text.length));
|
||
Array$insert(&chunks, &last_chunk, I_small(0), sizeof(Text_t));
|
||
|
||
return chunks;
|
||
}
|
||
|
||
public Text_t Text$join(Text_t glue, array_t pieces)
|
||
{
|
||
if (pieces.length == 0) return (Text_t){.length=0};
|
||
|
||
Text_t result = *(Text_t*)pieces.data;
|
||
for (int64_t i = 1; i < pieces.length; i++) {
|
||
result = Text$concat(result, glue, *(Text_t*)(pieces.data + i*pieces.stride));
|
||
}
|
||
return result;
|
||
}
|
||
|
||
public Text_t Text$format(const char *fmt, ...)
|
||
{
|
||
va_list args;
|
||
va_start(args, fmt);
|
||
|
||
char buf[9];
|
||
int len = vsnprintf(buf, sizeof(buf), fmt, args);
|
||
Text_t ret;
|
||
if (len <= 8) {
|
||
ret = (Text_t){
|
||
.length=len,
|
||
.tag=TEXT_SHORT_ASCII,
|
||
};
|
||
for (int i = 0; i < len; i++)
|
||
ret.short_ascii[i] = buf[i];
|
||
} else {
|
||
char *str = GC_MALLOC_ATOMIC(len+1);
|
||
vsnprintf(str, len+1, fmt, args);
|
||
ret = Text$from_str(str);
|
||
}
|
||
va_end(args);
|
||
return ret;
|
||
}
|
||
|
||
public array_t Text$clusters(Text_t text)
|
||
{
|
||
array_t clusters = {.atomic=1};
|
||
for (int64_t i = 1; i <= text.length; i++) {
|
||
Text_t cluster = Text$slice(text, Int64_to_Int(i), Int64_to_Int(i));
|
||
Array$insert(&clusters, &cluster, I_small(0), sizeof(Text_t));
|
||
}
|
||
return clusters;
|
||
}
|
||
|
||
public array_t Text$utf32_codepoints(Text_t text)
|
||
{
|
||
array_t codepoints = {.atomic=1};
|
||
iteration_state_t state = {0, 0};
|
||
for (int64_t i = 0; i < text.length; i++) {
|
||
int32_t grapheme = _next_grapheme(text, &state, i);
|
||
if (grapheme < 0) {
|
||
for (int64_t c = 0; c < synthetic_graphemes[-grapheme-1].num_codepoints; c++)
|
||
Array$insert(&codepoints, &synthetic_graphemes[-grapheme-1].codepoints[c], I_small(0), sizeof(uint32_t));
|
||
} else {
|
||
Array$insert(&codepoints, &grapheme, I_small(0), sizeof(uint32_t));
|
||
}
|
||
}
|
||
return codepoints;
|
||
}
|
||
|
||
public array_t Text$utf8_bytes(Text_t text)
|
||
{
|
||
const char *str = Text$as_c_string(text);
|
||
return (array_t){.length=strlen(str), .stride=1, .atomic=1, .data=(void*)str};
|
||
}
|
||
|
||
static inline const char *codepoint_name(uint32_t c)
|
||
{
|
||
char *name = GC_MALLOC_ATOMIC(UNINAME_MAX);
|
||
char *found_name = unicode_character_name(c, name);
|
||
if (found_name) return found_name;
|
||
const uc_block_t *block = uc_block(c);
|
||
assert(block);
|
||
snprintf(name, UNINAME_MAX, "%s-%X", block->name, c);
|
||
return name;
|
||
}
|
||
|
||
public array_t Text$codepoint_names(Text_t text)
|
||
{
|
||
array_t names = {};
|
||
iteration_state_t state = {0, 0};
|
||
for (int64_t i = 0; i < text.length; i++) {
|
||
int32_t grapheme = _next_grapheme(text, &state, i);
|
||
if (grapheme < 0) {
|
||
for (int64_t c = 0; c < synthetic_graphemes[-grapheme-1].num_codepoints; c++) {
|
||
const char *name = codepoint_name(synthetic_graphemes[-grapheme-1].codepoints[c]);
|
||
Text_t name_text = (Text_t){.tag=TEXT_ASCII, .length=strlen(name), .ascii=name};
|
||
Array$insert(&names, &name_text, I_small(0), sizeof(Text_t));
|
||
}
|
||
} else {
|
||
const char *name = codepoint_name(grapheme);
|
||
Text_t name_text = (Text_t){.tag=TEXT_ASCII, .length=strlen(name), .ascii=name};
|
||
Array$insert(&names, &name_text, I_small(0), sizeof(Text_t));
|
||
}
|
||
}
|
||
return names;
|
||
}
|
||
|
||
public Text_t Text$from_codepoints(array_t codepoints)
|
||
{
|
||
if (codepoints.stride != sizeof(int32_t))
|
||
Array$compact(&codepoints, sizeof(int32_t));
|
||
|
||
return text_from_u32(codepoints.data, codepoints.length, true);
|
||
}
|
||
|
||
public Text_t Text$from_codepoint_names(array_t codepoint_names)
|
||
{
|
||
array_t codepoints = {};
|
||
for (int64_t i = 0; i < codepoint_names.length; i++) {
|
||
Text_t *name = ((Text_t*)(codepoint_names.data + i*codepoint_names.stride));
|
||
const char *name_str = Text$as_c_string(*name);
|
||
uint32_t codepoint = unicode_name_character(name_str);
|
||
if (codepoint != UNINAME_INVALID)
|
||
Array$insert(&codepoints, &codepoint, I_small(0), sizeof(uint32_t));
|
||
}
|
||
return Text$from_codepoints(codepoints);
|
||
}
|
||
|
||
public Text_t Text$from_bytes(array_t bytes)
|
||
{
|
||
if (bytes.stride != sizeof(int8_t))
|
||
Array$compact(&bytes, sizeof(int8_t));
|
||
|
||
int8_t nul = 0;
|
||
Array$insert(&bytes, &nul, I_small(0), sizeof(int8_t));
|
||
return Text$from_str(bytes.data);
|
||
}
|
||
|
||
public array_t Text$lines(Text_t text)
|
||
{
|
||
array_t lines = {};
|
||
iteration_state_t state = {0, 0};
|
||
for (int64_t i = 0, line_start = 0; i < text.length; i++) {
|
||
int32_t grapheme = _next_grapheme(text, &state, i);
|
||
if (grapheme == '\r' && _next_grapheme(text, &state, i + 1) == '\n') { // CRLF
|
||
Text_t line = Text$slice(text, Int64_to_Int(line_start+1), Int64_to_Int(i));
|
||
Array$insert(&lines, &line, I_small(0), sizeof(Text_t));
|
||
i += 1; // skip one extra for CR
|
||
line_start = i + 1;
|
||
} else if (grapheme == '\n') { // newline
|
||
Text_t line = Text$slice(text, Int64_to_Int(line_start+1), Int64_to_Int(i));
|
||
Array$insert(&lines, &line, I_small(0), sizeof(Text_t));
|
||
line_start = i + 1;
|
||
} else if (i == text.length-1 && line_start != i) { // last line
|
||
Text_t line = Text$slice(text, Int64_to_Int(line_start+1), Int64_to_Int(i+1));
|
||
Array$insert(&lines, &line, I_small(0), sizeof(Text_t));
|
||
}
|
||
}
|
||
return lines;
|
||
}
|
||
|
||
public const TypeInfo $Text = {
|
||
.size=sizeof(Text_t),
|
||
.align=__alignof__(Text_t),
|
||
.tag=TextInfo,
|
||
.TextInfo={.lang="Text"},
|
||
};
|
||
|
||
public Pattern_t Pattern$escape_text(Text_t text)
|
||
{
|
||
// TODO: optimize for ASCII and short strings
|
||
array_t graphemes = {.atomic=1};
|
||
#define add_char(c) Array$insert_value(&graphemes, (uint32_t)c, I_small(0), sizeof(uint32_t))
|
||
#define add_str(s) ({ for (char *_c = s; *_c; ++_c) Array$insert_value(&graphemes, (uint32_t)*_c, I_small(0), sizeof(uint32_t)); })
|
||
iteration_state_t state = {0, 0};
|
||
for (int64_t i = 0; i < text.length; i++) {
|
||
int32_t g = _next_grapheme(text, &state, i);
|
||
uint32_t g0 = g < 0 ? synthetic_graphemes[-g-1].codepoints[0] : (uint32_t)g;
|
||
|
||
if (g == '{') {
|
||
add_str("{1{}");
|
||
} else if (g0 == '?'
|
||
|| uc_is_property_quotation_mark(g0)
|
||
|| (uc_is_property_paired_punctuation(g0) && uc_is_property_left_of_pair(g0))) {
|
||
add_char('{');
|
||
add_char('1');
|
||
add_char(g);
|
||
add_char('}');
|
||
} else {
|
||
add_char(g);
|
||
}
|
||
}
|
||
return (Text_t){.length=graphemes.length, .tag=TEXT_GRAPHEMES, .graphemes=graphemes.data};
|
||
#undef add_str
|
||
#undef add_char
|
||
#undef add_escaped
|
||
}
|
||
|
||
public const TypeInfo Pattern = {
|
||
.size=sizeof(Text_t),
|
||
.align=__alignof__(Text_t),
|
||
.tag=TextInfo,
|
||
.TextInfo={.lang="Pattern"},
|
||
};
|
||
|
||
// vim: ts=4 sw=0 et cino=L2,l1,(0,W4,m1,\:0
|