Skip to content

Commit

Permalink
Adapt Chinese characters (#327)
Browse files Browse the repository at this point in the history
Signed-off-by: Yue, Wenjiao <wenjiao.yue@intel.com>
  • Loading branch information
WenjiaoYue authored Jun 26, 2024
1 parent 6a3e9db commit 2f47231
Showing 1 changed file with 27 additions and 4 deletions.
31 changes: 27 additions & 4 deletions ChatQnA/docker/ui/svelte/src/routes/+page.svelte
Original file line number Diff line number Diff line change
Expand Up @@ -65,20 +65,43 @@
}
function storeMessages() {
localStorage.setItem(
LOCAL_STORAGE_KEY.STORAGE_CHAT_KEY,
JSON.stringify(chatMessages)
JSON.stringify(chatMessages),
);
}
function decodeEscapedBytes(str: string): string {
const byteArray = str
.split("\\x")
.slice(1)
.map((byte) => parseInt(byte, 16));
return new TextDecoder("utf-8").decode(new Uint8Array(byteArray));
}
function decodeUnicode(str: string): string {
return str.replace(/\\u[\dA-Fa-f]{4}/g, (match) => {
return String.fromCharCode(parseInt(match.replace(/\\u/g, ""), 16));
});
}
const callTextStream = async (query: string, startSendTime: number) => {
const eventSource = await fetchTextStream(query, knowledge_1);
eventSource.addEventListener("message", (e: any) => {
let Msg = e.data;
if (Msg.startsWith("b")) {
const currentMsg = Msg.slice(2, -1);
let currentMsg = Msg.slice(2, -1);
if (/\\x[\dA-Fa-f]{2}/.test(currentMsg)) {
currentMsg = decodeEscapedBytes(currentMsg);
} else if (/\\u[\dA-Fa-f]{4}/.test(currentMsg)) {
currentMsg = decodeUnicode(currentMsg);
}
if (currentMsg !== "</s>") {
currentMsg = currentMsg.replace(/\\n/g, "\n");
}
if (chatMessages[chatMessages.length - 1].role == MessageRole.User) {
chatMessages = [
...chatMessages,
Expand All @@ -99,7 +122,7 @@
loading = false;
let totalTime = parseFloat(
((getCurrentTimeStamp() - startTime) / 1000).toFixed(2)
((getCurrentTimeStamp() - startTime) / 1000).toFixed(2),
);
if (chatMessages.length - 1 !== -1) {
Expand Down

0 comments on commit 2f47231

Please sign in to comment.