sdgoij/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-05-14 21:14:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

16 cols for Phi-2

Browse Source
This commit is contained in:
Johannes Gäßler
2024-03-30 09:19:19 +01:00
committed by Georgi Gerganov
parent 75aa7b4b18
commit d59ac670bf
1 changed files with 11 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
ggml-cuda/fattn.cu
View File

@@ -579,15 +579,17 @@ void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst
return;
}
int cols_per_block;
if (Q->ne[1] >= 128 && Q->ne[0] <= 128) {
cols_per_block = 64;
} else if (Q->ne[1] >= 64) {
cols_per_block = 32;
} else if (Q->ne[1] >= 32 || Q->ne[0] % 32 != 0) {
cols_per_block = 16;
} else {
cols_per_block = 8;
int cols_per_block = 16;
if (Q->ne[0] % 32 == 0) {
if (Q->ne[1] >= 128 && Q->ne[0] <= 128) {
cols_per_block = 64;
} else if (Q->ne[1] >= 64) {
cols_per_block = 32;
} else if (Q->ne[1] >= 32 || Q->ne[0] % 32 != 0) {
cols_per_block = 16;
} else {
cols_per_block = 8;
}
}
const int frag_m = cols_per_block == 8 ? 32 : 16;
const int nwarps = (Q->ne[0] <= 128 || cols_per_block == 8 ? Q->ne[0] : Q->ne[0]/2) / frag_m;