From fd51132edf74ba719acf101c21b4b5eb7565b202 Mon Sep 17 00:00:00 2001 From: mic Date: Thu, 3 Oct 2024 21:06:49 +0200 Subject: [PATCH] managing broken chunck. see #147 --- api_webchat/model-worker-ollama.js | 17 ++++-- api_webchat/model-worker-openai.js | 71 +++++++------------------ api_webchat/model-worker-openai_comp.js | 24 +++++++-- 3 files changed, 50 insertions(+), 62 deletions(-) diff --git a/api_webchat/model-worker-ollama.js b/api_webchat/model-worker-ollama.js index def38787..b6a4eca1 100644 --- a/api_webchat/model-worker-ollama.js +++ b/api_webchat/model-worker-ollama.js @@ -63,6 +63,7 @@ self.onmessage = async function(event) { const reader = response.body.getReader(); const decoder = new TextDecoder("utf-8"); + let chunk = ''; try { while (true) { @@ -83,13 +84,19 @@ self.onmessage = async function(event) { break; } // lots of low-level Ollama response parsing stuff - const chunk = decoder.decode(value); + chunk += decoder.decode(value); //console.log(">>>>>>>>>>>>> chunk: " + chunk); const lines = chunk.split("\n"); - const parsedLines = lines - .map((line) => line.replace(/^chunk: /, "").trim()) // Remove the "chunk: " prefix - .filter((line) => line !== "" && line !== "[DONE]") // Remove empty lines and "[DONE]" - .map((line) => JSON.parse(line)); // Parse the JSON string + let parsedLines = []; + chunk = chunk.substring(chunk.lastIndexOf('\n') + 1); + try{ + parsedLines = lines + .map((line) => line.replace(/^chunk: /, "").trim()) // Remove the "chunk: " prefix + .filter((line) => line !== "" && line !== "[DONE]") // Remove empty lines and "[DONE]" + .map((line) => JSON.parse(line)); // Parse the JSON string + }catch(e){ + console.warn("[ThunderAI | model-worker-ollama] broken chunk: " + e); + } for (const parsedLine of parsedLines) { const { message } = parsedLine; diff --git a/api_webchat/model-worker-openai.js b/api_webchat/model-worker-openai.js index de9ba27c..ee0d7955 100644 --- a/api_webchat/model-worker-openai.js +++ b/api_webchat/model-worker-openai.js @@ -22,23 +22,6 @@ import { OpenAI } from '../js/api/openai.js'; -//========================== for testing -// const MOCK_TOKENS = ['Good', ' morning', ' Mr', ' Plop', 'py', ',', 'and', ' I', ' said', '\n', '"', 'Good', ' morn', 'ing', ' Mrs',' Plop', 'py', ,'"', '\n', 'Oh', ' how', ' the', ' win', 'ter', ' even', 'ings', ' must', ' just', ' fly']; -// -// function mockDelay(ms) { -// return new Promise(resolve => setTimeout(resolve, ms)); -// } - -// async function processMockTokens() { -// for (const token of MOCK_TOKENS) { -// await mockDelay(Math.random() * 50 + 50); // Random delay between 100ms and 150ms -// postMessage({ type: 'newToken', payload: { token } }); -// } -// postMessage({ type: 'tokensDone' }); -// } -//========================== for testing - END - - let chatgpt_api_key = null; let chatgpt_model = ''; let openai = null; @@ -55,37 +38,7 @@ self.onmessage = async function(event) { openai = new OpenAI(chatgpt_api_key, chatgpt_model, true); } else if (event.data.type === 'chatMessage') { conversationHistory.push({ role: 'user', content: event.data.message }); - - // ============================== TESTING - // // Simulate sending the message to an HTTP endpoint - // await mockDelay(1000); // Wait for 1 second - - // // Notify that the chat message was sent - // postMessage({ type: 'messageSent' }); - - // // Start processing tokens - // await processMockTokens(); - // return; - // ============================== TESTING - END - - - - // https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo - // 4096 output tokens - // 128,000 input tokens - // const response = await fetch(API_URL, { - // method: "POST", - // headers: { - // "Content-Type": "application/json", - // "Authorization": `Bearer ${openaiApiKey}`, - // }, - // body: JSON.stringify({ - // model: "gpt-4-1106-preview", - // messages: conversationHistory, - // stream: true, - // }), - // }); const response = await openai.fetchResponse(conversationHistory); //4096); postMessage({ type: 'messageSent' }); @@ -106,6 +59,7 @@ self.onmessage = async function(event) { const reader = response.body.getReader(); const decoder = new TextDecoder("utf-8"); + let chunk = ''; while (true) { if (stopStreaming) { @@ -124,12 +78,25 @@ self.onmessage = async function(event) { break; } // lots of low-level OpenAI response parsing stuff - const chunk = decoder.decode(value); + chunk += decoder.decode(value); + // console.log(">>>>>>>>>>>>>> [ThunderAI] chunk: " + JSON.stringify(chunk)); const lines = chunk.split("\n"); - const parsedLines = lines - .map((line) => line.replace(/^data: /, "").trim()) // Remove the "data: " prefix - .filter((line) => line !== "" && line !== "[DONE]") // Remove empty lines and "[DONE]" - .map((line) => JSON.parse(line)); // Parse the JSON string + let parsedLines = []; + chunk = chunk.substring(chunk.lastIndexOf('\n') + 1); + // console.log(">>>>>>>>>>>>>> [ThunderAI] last chunk: " + JSON.stringify(chunk)); + try{ + parsedLines = lines + .map((line) => line.replace(/^data: /, "").trim()) // Remove the "data: " prefix + .filter((line) => line !== "" && line !== "[DONE]") // Remove empty lines and "[DONE]" + // .map((line) => JSON.parse(line)); // Parse the JSON string + .map((line) => { + // console.log(">>>>>>>>>>>>> [ThunderAI] line: " + JSON.stringify(line)); + return JSON.parse(line); + }); + }catch(e){ + // console.log(">>>>>>>>>>>>>> [ThunderAI] last chunk: " + JSON.stringify(chunk)); + console.warn("[ThunderAI | model-worker-openai] broken chunk: " + e); + } for (const parsedLine of parsedLines) { const { choices } = parsedLine; diff --git a/api_webchat/model-worker-openai_comp.js b/api_webchat/model-worker-openai_comp.js index 1fe439b6..5108726d 100644 --- a/api_webchat/model-worker-openai_comp.js +++ b/api_webchat/model-worker-openai_comp.js @@ -60,6 +60,7 @@ self.onmessage = async function(event) { const reader = response.body.getReader(); const decoder = new TextDecoder("utf-8"); + let chunk = ''; while (true) { if (stopStreaming) { @@ -78,12 +79,25 @@ self.onmessage = async function(event) { break; } // lots of low-level OpenAI response parsing stuff - const chunk = decoder.decode(value); + chunk += decoder.decode(value); + // console.log(">>>>>>>>>>>>>> [ThunderAI] chunk: " + JSON.stringify(chunk)); const lines = chunk.split("\n"); - const parsedLines = lines - .map((line) => line.replace(/^data: /, "").trim()) // Remove the "data: " prefix - .filter((line) => line !== "" && line !== "[DONE]") // Remove empty lines and "[DONE]" - .map((line) => JSON.parse(line)); // Parse the JSON string + let parsedLines = []; + chunk = chunk.substring(chunk.lastIndexOf('\n') + 1); + // console.log(">>>>>>>>>>>>>> [ThunderAI] last chunk: " + JSON.stringify(chunk)); + try{ + parsedLines = lines + .map((line) => line.replace(/^data: /, "").trim()) // Remove the "data: " prefix + .filter((line) => line !== "" && line !== "[DONE]") // Remove empty lines and "[DONE]" + // .map((line) => JSON.parse(line)); // Parse the JSON string + .map((line) => { + // console.log(">>>>>>>>>>>>> [ThunderAI] line: " + JSON.stringify(line)); + return JSON.parse(line); + }); + }catch(e){ + // console.error(">>>>>>>>>>>>> [ThunderAI] error: " + e); + console.warn("[ThunderAI | model-worker-openai_comp] broken chunk: " + e); + } for (const parsedLine of parsedLines) { const { choices } = parsedLine;