Support flash attention in --server mode

Mozilla-Ocho · Jun 29, 2024 · 4aea606 · 4aea606
1 parent 7692b85
commit 4aea606
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/llama.cpp/server/server.cpp b/llama.cpp/server/server.cpp
@@ -2278,6 +2278,10 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,
             sparams.read_timeout = std::stoi(argv[i]);
             sparams.write_timeout = std::stoi(argv[i]);
         }
+        else if (arg == "-fa" || arg == "--flash-attn") {
+            params.flash_attn = true;
+            FLAG_flash_attn = true;
+        }
         else if (arg == "-m" || arg == "--model")
         {
             if (++i >= argc)