sdgoij/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-05-10 11:04:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

llama : add "virtual sequences"

Browse Source 
ggml-ci
This commit is contained in:
Georgi Gerganov
2025-06-23 16:29:02 +03:00
parent 30b4d4e1b3
commit dfceb012ee
16 changed files with 648 additions and 256 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
examples/parallel/parallel.cpp
View File

@@ -235,7 +235,7 @@ int main(int argc, char ** argv) {
// the max batch size is as large as the context to handle cases where we get very long input prompt from multiple
// users. regardless of the size, the main loop will chunk the batch into a maximum of params.n_batch tokens at a time
llama_batch batch = llama_batch_init(n_ctx, 0, 1);
llama_batch batch = llama_batch_init(n_ctx*n_clients, 0, 1);
int32_t n_total_prompt = 0;
int32_t n_total_gen = 0;
@@ -289,6 +289,7 @@ int main(int argc, char ** argv) {
// all sequences have ended - clear the entire KV cache
for (int i = 1; i <= n_clients; ++i) {
llama_memory_seq_rm(mem, i, -1, -1);
// but keep the system prompt
llama_memory_seq_cp(mem, 0, i, -1, -1);
}