Fixed issue with gpt2 regex custom preprocessor

2026-05-14 21:14:10 +00:00 · 2024-04-17 07:40:40 +06:00
parent 753580360b
commit 36d983262e
1 changed files with 1 additions and 0 deletions
--- a/unicode.cpp
+++ b/unicode.cpp
@@ -357,6 +357,7 @@ static std::vector<size_t> unicode_gpt2_regex_preprocess(const std::wstring & wt
                token += utf_char;
            }
        }
+        start += offset;
    }

    return bpe_offsets;