sdgoij/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-05-11 11:34:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

wpm : portable unicode tolower

Browse Source
This commit is contained in:
Jared Van Bortel
2024-03-25 16:01:58 -04:00
parent e5ddf2fcdd
commit b460e7f5b4
5 changed files with 250 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
unicode.cpp
View File

@@ -271,3 +271,7 @@ uint8_t unicode_utf8_to_byte(const std::string & utf8) {
return map.at(utf8);
}
char32_t unicode_tolower(char32_t cp) {
auto it = unicode_map_lowercase.find(cp);
return it == unicode_map_lowercase.end() ? cp : it->second;
}