better utf8 enum
This commit is contained in:
@@ -351,6 +351,7 @@ typedef struct StringUTF16Const {
|
|||||||
} StringUTF16Const;
|
} StringUTF16Const;
|
||||||
|
|
||||||
typedef enum Utf8Character {
|
typedef enum Utf8Character {
|
||||||
|
Utf8CharacterInvalid = 0,
|
||||||
Utf8CharacterAscii,
|
Utf8CharacterAscii,
|
||||||
Utf8CharacterTwoByte,
|
Utf8CharacterTwoByte,
|
||||||
Utf8CharacterThreeByte,
|
Utf8CharacterThreeByte,
|
||||||
|
|||||||
@@ -38,7 +38,7 @@ fn Utf8Character utf8CharacterClassify(u8 c) {
|
|||||||
return Utf8CharacterFourByte;
|
return Utf8CharacterFourByte;
|
||||||
} else {
|
} else {
|
||||||
assert(false && "Not a valid utf8 starting byte");
|
assert(false && "Not a valid utf8 starting byte");
|
||||||
return Utf8Character_Count;
|
return Utf8CharacterInvalid;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -55,7 +55,7 @@ fn Utf8Character utf8CharacterClassifyUnsafe(u8 c) {
|
|||||||
} else if (c >= 240) {
|
} else if (c >= 240) {
|
||||||
return Utf8CharacterFourByte;
|
return Utf8CharacterFourByte;
|
||||||
} else {
|
} else {
|
||||||
return Utf8Character_Count;
|
return Utf8CharacterInvalid;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -115,29 +115,27 @@ fn bool codepointIsWhitespace(Codepoint c) {
|
|||||||
fn Codepoint codepointFromBytes(ptr bytes, u32 offset) {
|
fn Codepoint codepointFromBytes(ptr bytes, u32 offset) {
|
||||||
Codepoint result = {0};
|
Codepoint result = {0};
|
||||||
result.type = utf8CharacterClassify(bytes[offset]);
|
result.type = utf8CharacterClassify(bytes[offset]);
|
||||||
|
result.size = result.type;
|
||||||
switch (result.type) {
|
switch (result.type) {
|
||||||
case Utf8CharacterAscii: {
|
case Utf8CharacterAscii: {
|
||||||
result.size = 1;
|
|
||||||
result.code = bytes[offset];
|
result.code = bytes[offset];
|
||||||
} break;
|
} break;
|
||||||
case Utf8CharacterTwoByte: {
|
case Utf8CharacterTwoByte: {
|
||||||
result.size = 2;
|
|
||||||
result.code = (
|
result.code = (
|
||||||
bytes[offset] << 8 | bytes[offset+1]
|
bytes[offset] << 8 | bytes[offset+1]
|
||||||
);
|
);
|
||||||
} break;
|
} break;
|
||||||
case Utf8CharacterThreeByte: {
|
case Utf8CharacterThreeByte: {
|
||||||
result.size = 3;
|
|
||||||
result.code = (
|
result.code = (
|
||||||
bytes[offset] << 16 | bytes[offset+1] << 8 | bytes[offset+2]
|
bytes[offset] << 16 | bytes[offset+1] << 8 | bytes[offset+2]
|
||||||
);
|
);
|
||||||
} break;
|
} break;
|
||||||
case Utf8CharacterFourByte: {
|
case Utf8CharacterFourByte: {
|
||||||
result.size = 4;
|
|
||||||
result.code = (
|
result.code = (
|
||||||
bytes[offset] << 24 | bytes[offset+1] << 16 | bytes[offset+2] << 8 | bytes[offset+3]
|
bytes[offset] << 24 | bytes[offset+1] << 16 | bytes[offset+2] << 8 | bytes[offset+3]
|
||||||
);
|
);
|
||||||
} break;
|
} break;
|
||||||
|
case Utf8CharacterInvalid:
|
||||||
case Utf8Character_Count: {
|
case Utf8Character_Count: {
|
||||||
printf("unabled to classify utf8 codepoint %d\n", bytes[offset]);
|
printf("unabled to classify utf8 codepoint %d\n", bytes[offset]);
|
||||||
assert(false);
|
assert(false);
|
||||||
@@ -148,36 +146,34 @@ fn Codepoint codepointFromBytes(ptr bytes, u32 offset) {
|
|||||||
|
|
||||||
fn Codepoint codepointFromBytesBefore(ptr bytes, u32 offset) {
|
fn Codepoint codepointFromBytesBefore(ptr bytes, u32 offset) {
|
||||||
assert(offset > 0);
|
assert(offset > 0);
|
||||||
Codepoint result = { .type = Utf8Character_Count };
|
Codepoint result = { .type = Utf8CharacterInvalid };
|
||||||
u32 i = 0;
|
u32 i = 0;
|
||||||
while (result.type == Utf8Character_Count && i <= 4) {
|
while (result.type == Utf8CharacterInvalid && i <= 4) {
|
||||||
i++;
|
i++;
|
||||||
offset -= 1;
|
offset -= 1;
|
||||||
result.type = utf8CharacterClassifyUnsafe(bytes[offset]);
|
result.type = utf8CharacterClassifyUnsafe(bytes[offset]);
|
||||||
}
|
}
|
||||||
|
result.size = result.type;
|
||||||
switch (result.type) {
|
switch (result.type) {
|
||||||
case Utf8CharacterAscii: {
|
case Utf8CharacterAscii: {
|
||||||
result.size = 1;
|
|
||||||
result.code = bytes[offset];
|
result.code = bytes[offset];
|
||||||
} break;
|
} break;
|
||||||
case Utf8CharacterTwoByte: {
|
case Utf8CharacterTwoByte: {
|
||||||
result.size = 2;
|
|
||||||
result.code = (
|
result.code = (
|
||||||
bytes[offset] << 8 | bytes[offset+1]
|
bytes[offset] << 8 | bytes[offset+1]
|
||||||
);
|
);
|
||||||
} break;
|
} break;
|
||||||
case Utf8CharacterThreeByte: {
|
case Utf8CharacterThreeByte: {
|
||||||
result.size = 3;
|
|
||||||
result.code = (
|
result.code = (
|
||||||
bytes[offset] << 16 | bytes[offset+1] << 8 | bytes[offset+2]
|
bytes[offset] << 16 | bytes[offset+1] << 8 | bytes[offset+2]
|
||||||
);
|
);
|
||||||
} break;
|
} break;
|
||||||
case Utf8CharacterFourByte: {
|
case Utf8CharacterFourByte: {
|
||||||
result.size = 4;
|
|
||||||
result.code = (
|
result.code = (
|
||||||
bytes[offset] << 24 | bytes[offset+1] << 16 | bytes[offset+2] << 8 | bytes[offset+3]
|
bytes[offset] << 24 | bytes[offset+1] << 16 | bytes[offset+2] << 8 | bytes[offset+3]
|
||||||
);
|
);
|
||||||
} break;
|
} break;
|
||||||
|
case Utf8CharacterInvalid:
|
||||||
case Utf8Character_Count: {
|
case Utf8Character_Count: {
|
||||||
printf("unabled to classify utf8 codepoint %d\n", bytes[offset]);
|
printf("unabled to classify utf8 codepoint %d\n", bytes[offset]);
|
||||||
assert(false);
|
assert(false);
|
||||||
@@ -224,6 +220,7 @@ fn void codepointFillBuf(Codepoint cp, ptr buf) {
|
|||||||
buf[2] = (cp.code & 0xFF00) >> 8;
|
buf[2] = (cp.code & 0xFF00) >> 8;
|
||||||
buf[3] = cp.code & 0xFF;
|
buf[3] = cp.code & 0xFF;
|
||||||
} break;
|
} break;
|
||||||
|
case Utf8CharacterInvalid:
|
||||||
case Utf8Character_Count: {
|
case Utf8Character_Count: {
|
||||||
assert(false);
|
assert(false);
|
||||||
} break;
|
} break;
|
||||||
@@ -233,15 +230,9 @@ fn void codepointFillBuf(Codepoint cp, ptr buf) {
|
|||||||
fn String stringFromRawCodepoint(Arena* a, u32 c) {
|
fn String stringFromRawCodepoint(Arena* a, u32 c) {
|
||||||
String result = {0};
|
String result = {0};
|
||||||
Codepoint codepoint = codepointFromRawInt(c);
|
Codepoint codepoint = codepointFromRawInt(c);
|
||||||
switch (codepoint.type) {
|
assert(codepoint.type != Utf8Character_Count);
|
||||||
case Utf8CharacterAscii: result.capacity = 1; break;
|
assert(codepoint.type != Utf8CharacterInvalid);
|
||||||
case Utf8CharacterTwoByte: result.capacity = 2; break;
|
result.capacity = codepoint.size;
|
||||||
case Utf8CharacterThreeByte: result.capacity = 3; break;
|
|
||||||
case Utf8CharacterFourByte: result.capacity = 4; break;
|
|
||||||
case Utf8Character_Count: {
|
|
||||||
assert(false);
|
|
||||||
} break;
|
|
||||||
}
|
|
||||||
result.length = result.capacity;
|
result.length = result.capacity;
|
||||||
result.bytes = arenaAlloc(a, result.length);
|
result.bytes = arenaAlloc(a, result.length);
|
||||||
codepointFillBuf(codepoint, result.bytes);
|
codepointFillBuf(codepoint, result.bytes);
|
||||||
|
|||||||
Reference in New Issue
Block a user