better utf8 enum

This commit is contained in:
Tenari
2026-08-13 07:07:24 -05:00
parent 585b13b034
commit 716d165e2d
2 changed files with 13 additions and 21 deletions
+1
View File
@@ -351,6 +351,7 @@ typedef struct StringUTF16Const {
} StringUTF16Const; } StringUTF16Const;
typedef enum Utf8Character { typedef enum Utf8Character {
Utf8CharacterInvalid = 0,
Utf8CharacterAscii, Utf8CharacterAscii,
Utf8CharacterTwoByte, Utf8CharacterTwoByte,
Utf8CharacterThreeByte, Utf8CharacterThreeByte,
+12 -21
View File
@@ -38,7 +38,7 @@ fn Utf8Character utf8CharacterClassify(u8 c) {
return Utf8CharacterFourByte; return Utf8CharacterFourByte;
} else { } else {
assert(false && "Not a valid utf8 starting byte"); assert(false && "Not a valid utf8 starting byte");
return Utf8Character_Count; return Utf8CharacterInvalid;
} }
} }
@@ -55,7 +55,7 @@ fn Utf8Character utf8CharacterClassifyUnsafe(u8 c) {
} else if (c >= 240) { } else if (c >= 240) {
return Utf8CharacterFourByte; return Utf8CharacterFourByte;
} else { } else {
return Utf8Character_Count; return Utf8CharacterInvalid;
} }
} }
@@ -115,29 +115,27 @@ fn bool codepointIsWhitespace(Codepoint c) {
fn Codepoint codepointFromBytes(ptr bytes, u32 offset) { fn Codepoint codepointFromBytes(ptr bytes, u32 offset) {
Codepoint result = {0}; Codepoint result = {0};
result.type = utf8CharacterClassify(bytes[offset]); result.type = utf8CharacterClassify(bytes[offset]);
result.size = result.type;
switch (result.type) { switch (result.type) {
case Utf8CharacterAscii: { case Utf8CharacterAscii: {
result.size = 1;
result.code = bytes[offset]; result.code = bytes[offset];
} break; } break;
case Utf8CharacterTwoByte: { case Utf8CharacterTwoByte: {
result.size = 2;
result.code = ( result.code = (
bytes[offset] << 8 | bytes[offset+1] bytes[offset] << 8 | bytes[offset+1]
); );
} break; } break;
case Utf8CharacterThreeByte: { case Utf8CharacterThreeByte: {
result.size = 3;
result.code = ( result.code = (
bytes[offset] << 16 | bytes[offset+1] << 8 | bytes[offset+2] bytes[offset] << 16 | bytes[offset+1] << 8 | bytes[offset+2]
); );
} break; } break;
case Utf8CharacterFourByte: { case Utf8CharacterFourByte: {
result.size = 4;
result.code = ( result.code = (
bytes[offset] << 24 | bytes[offset+1] << 16 | bytes[offset+2] << 8 | bytes[offset+3] bytes[offset] << 24 | bytes[offset+1] << 16 | bytes[offset+2] << 8 | bytes[offset+3]
); );
} break; } break;
case Utf8CharacterInvalid:
case Utf8Character_Count: { case Utf8Character_Count: {
printf("unabled to classify utf8 codepoint %d\n", bytes[offset]); printf("unabled to classify utf8 codepoint %d\n", bytes[offset]);
assert(false); assert(false);
@@ -148,36 +146,34 @@ fn Codepoint codepointFromBytes(ptr bytes, u32 offset) {
fn Codepoint codepointFromBytesBefore(ptr bytes, u32 offset) { fn Codepoint codepointFromBytesBefore(ptr bytes, u32 offset) {
assert(offset > 0); assert(offset > 0);
Codepoint result = { .type = Utf8Character_Count }; Codepoint result = { .type = Utf8CharacterInvalid };
u32 i = 0; u32 i = 0;
while (result.type == Utf8Character_Count && i <= 4) { while (result.type == Utf8CharacterInvalid && i <= 4) {
i++; i++;
offset -= 1; offset -= 1;
result.type = utf8CharacterClassifyUnsafe(bytes[offset]); result.type = utf8CharacterClassifyUnsafe(bytes[offset]);
} }
result.size = result.type;
switch (result.type) { switch (result.type) {
case Utf8CharacterAscii: { case Utf8CharacterAscii: {
result.size = 1;
result.code = bytes[offset]; result.code = bytes[offset];
} break; } break;
case Utf8CharacterTwoByte: { case Utf8CharacterTwoByte: {
result.size = 2;
result.code = ( result.code = (
bytes[offset] << 8 | bytes[offset+1] bytes[offset] << 8 | bytes[offset+1]
); );
} break; } break;
case Utf8CharacterThreeByte: { case Utf8CharacterThreeByte: {
result.size = 3;
result.code = ( result.code = (
bytes[offset] << 16 | bytes[offset+1] << 8 | bytes[offset+2] bytes[offset] << 16 | bytes[offset+1] << 8 | bytes[offset+2]
); );
} break; } break;
case Utf8CharacterFourByte: { case Utf8CharacterFourByte: {
result.size = 4;
result.code = ( result.code = (
bytes[offset] << 24 | bytes[offset+1] << 16 | bytes[offset+2] << 8 | bytes[offset+3] bytes[offset] << 24 | bytes[offset+1] << 16 | bytes[offset+2] << 8 | bytes[offset+3]
); );
} break; } break;
case Utf8CharacterInvalid:
case Utf8Character_Count: { case Utf8Character_Count: {
printf("unabled to classify utf8 codepoint %d\n", bytes[offset]); printf("unabled to classify utf8 codepoint %d\n", bytes[offset]);
assert(false); assert(false);
@@ -224,6 +220,7 @@ fn void codepointFillBuf(Codepoint cp, ptr buf) {
buf[2] = (cp.code & 0xFF00) >> 8; buf[2] = (cp.code & 0xFF00) >> 8;
buf[3] = cp.code & 0xFF; buf[3] = cp.code & 0xFF;
} break; } break;
case Utf8CharacterInvalid:
case Utf8Character_Count: { case Utf8Character_Count: {
assert(false); assert(false);
} break; } break;
@@ -233,15 +230,9 @@ fn void codepointFillBuf(Codepoint cp, ptr buf) {
fn String stringFromRawCodepoint(Arena* a, u32 c) { fn String stringFromRawCodepoint(Arena* a, u32 c) {
String result = {0}; String result = {0};
Codepoint codepoint = codepointFromRawInt(c); Codepoint codepoint = codepointFromRawInt(c);
switch (codepoint.type) { assert(codepoint.type != Utf8Character_Count);
case Utf8CharacterAscii: result.capacity = 1; break; assert(codepoint.type != Utf8CharacterInvalid);
case Utf8CharacterTwoByte: result.capacity = 2; break; result.capacity = codepoint.size;
case Utf8CharacterThreeByte: result.capacity = 3; break;
case Utf8CharacterFourByte: result.capacity = 4; break;
case Utf8Character_Count: {
assert(false);
} break;
}
result.length = result.capacity; result.length = result.capacity;
result.bytes = arenaAlloc(a, result.length); result.bytes = arenaAlloc(a, result.length);
codepointFillBuf(codepoint, result.bytes); codepointFillBuf(codepoint, result.bytes);