Introduce /v1/completions endpoint in new server

Mozilla-Ocho · Nov 23, 2024 · 241bf21 · 241bf21
1 parent 0965a4b
commit 241bf21
Show file tree

Hide file tree

Showing 4 changed files with 522 additions and 0 deletions.
diff --git a/llamafile/server/client.cpp b/llamafile/server/client.cpp
@@ -650,6 +650,8 @@ Client::dispatcher()
         return embedding();
     if (p1 == "v1/embeddings")
         return embedding();
+    if (p1 == "v1/completions")
+        return v1_completions();
     if (p1 == "v1/chat/completions")
         return v1_chat_completions();
     if (p1 == "slotz")

diff --git a/llamafile/server/client.h b/llamafile/server/client.h
@@ -44,6 +44,7 @@ struct Slot;
 struct Worker;
 struct TokenizeParams;
 struct EmbeddingParams;
+struct V1CompletionParams;
 struct V1ChatCompletionParams;
 
 struct Client
@@ -109,6 +110,9 @@ struct Client
     bool embedding() __wur;
     bool get_embedding_params(EmbeddingParams*) __wur;
 
+    bool v1_completions() __wur;
+    bool get_v1_completions_params(V1CompletionParams*) __wur;
+
     bool v1_chat_completions() __wur;
     bool get_v1_chat_completions_params(V1ChatCompletionParams*) __wur;
 

diff --git a/llamafile/server/v1_chat_completions.cpp b/llamafile/server/v1_chat_completions.cpp
@@ -557,6 +557,8 @@ Client::v1_chat_completions()
         choice.getObject().erase("delta");
         if (!send_response_chunk(response->content))
             return false;
+        if (!send_response_chunk("data: [DONE]\n\n"))
+            return false;
         return send_response_finish();
     } else {
         Json& usage = response->json["usage"];