jason/ThunderAI
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

managing broken chunck. see #147

Browse source
This commit is contained in:
mic 2024-10-03 21:06:49 +02:00
parent f9832a1860
commit fd51132edf
3 changed files with 50 additions and 62 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
api_webchat/model-worker-ollama.js
View file

@ -63,6 +63,7 @@ self.onmessage = async function(event) {
const reader = response.body.getReader();
const decoder = new TextDecoder("utf-8");
let chunk = '';
try {
while (true) {
@ -83,13 +84,19 @@ self.onmessage = async function(event) {
break;
}
// lots of low-level Ollama response parsing stuff
const chunk = decoder.decode(value);
chunk += decoder.decode(value);
//console.log(">>>>>>>>>>>>> chunk: " + chunk);
const lines = chunk.split("\n");
const parsedLines = lines
.map((line) => line.replace(/^chunk: /, "").trim()) // Remove the "chunk: " prefix
.filter((line) => line !== "" && line !== "[DONE]") // Remove empty lines and "[DONE]"
.map((line) => JSON.parse(line)); // Parse the JSON string
let parsedLines = [];
chunk = chunk.substring(chunk.lastIndexOf('\n') + 1);
try{
parsedLines = lines
.map((line) => line.replace(/^chunk: /, "").trim()) // Remove the "chunk: " prefix
.filter((line) => line !== "" && line !== "[DONE]") // Remove empty lines and "[DONE]"
.map((line) => JSON.parse(line)); // Parse the JSON string
}catch(e){
console.warn("[ThunderAI | model-worker-ollama] broken chunk: " + e);
}
for (const parsedLine of parsedLines) {
const { message } = parsedLine;

71
api_webchat/model-worker-openai.js
View file

@ -22,23 +22,6 @@
import { OpenAI } from '../js/api/openai.js';
//========================== for testing
// const MOCK_TOKENS = ['Good', ' morning', ' Mr', ' Plop', 'py', ',', 'and', ' I', ' said', '\n', '"', 'Good', ' morn', 'ing', ' Mrs',' Plop', 'py', ,'"', '\n', 'Oh', ' how', ' the', ' win', 'ter', ' even', 'ings', ' must', ' just', ' fly'];
//
// function mockDelay(ms) {
// return new Promise(resolve => setTimeout(resolve, ms));
// }
// async function processMockTokens() {
// for (const token of MOCK_TOKENS) {
// await mockDelay(Math.random() * 50 + 50); // Random delay between 100ms and 150ms
// postMessage({ type: 'newToken', payload: { token } });
// }
// postMessage({ type: 'tokensDone' });
// }
//========================== for testing - END
let chatgpt_api_key = null;
let chatgpt_model = '';
let openai = null;
@ -55,37 +38,7 @@ self.onmessage = async function(event) {
openai = new OpenAI(chatgpt_api_key, chatgpt_model, true);
} else if (event.data.type === 'chatMessage') {
conversationHistory.push({ role: 'user', content: event.data.message });
// ============================== TESTING
// // Simulate sending the message to an HTTP endpoint
// await mockDelay(1000); // Wait for 1 second
// // Notify that the chat message was sent
// postMessage({ type: 'messageSent' });
// // Start processing tokens
// await processMockTokens();
// return;
// ============================== TESTING - END
// https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo
// 4096 output tokens
// 128,000 input tokens
// const response = await fetch(API_URL, {
// method: "POST",
// headers: {
// "Content-Type": "application/json",
// "Authorization": `Bearer ${openaiApiKey}`,
// },
// body: JSON.stringify({
// model: "gpt-4-1106-preview",
// messages: conversationHistory,
// stream: true,
// }),
// });
const response = await openai.fetchResponse(conversationHistory); //4096);
postMessage({ type: 'messageSent' });
@ -106,6 +59,7 @@ self.onmessage = async function(event) {
const reader = response.body.getReader();
const decoder = new TextDecoder("utf-8");
let chunk = '';
while (true) {
if (stopStreaming) {
@ -124,12 +78,25 @@ self.onmessage = async function(event) {
break;
}
// lots of low-level OpenAI response parsing stuff
const chunk = decoder.decode(value);
chunk += decoder.decode(value);
// console.log(">>>>>>>>>>>>>> [ThunderAI] chunk: " + JSON.stringify(chunk));
const lines = chunk.split("\n");
const parsedLines = lines
.map((line) => line.replace(/^data: /, "").trim()) // Remove the "data: " prefix
.filter((line) => line !== "" && line !== "[DONE]") // Remove empty lines and "[DONE]"
.map((line) => JSON.parse(line)); // Parse the JSON string
let parsedLines = [];
chunk = chunk.substring(chunk.lastIndexOf('\n') + 1);
// console.log(">>>>>>>>>>>>>> [ThunderAI] last chunk: " + JSON.stringify(chunk));
try{
parsedLines = lines
.map((line) => line.replace(/^data: /, "").trim()) // Remove the "data: " prefix
.filter((line) => line !== "" && line !== "[DONE]") // Remove empty lines and "[DONE]"
// .map((line) => JSON.parse(line)); // Parse the JSON string
.map((line) => {
// console.log(">>>>>>>>>>>>> [ThunderAI] line: " + JSON.stringify(line));
return JSON.parse(line);
});
}catch(e){
// console.log(">>>>>>>>>>>>>> [ThunderAI] last chunk: " + JSON.stringify(chunk));
console.warn("[ThunderAI | model-worker-openai] broken chunk: " + e);
}
for (const parsedLine of parsedLines) {
const { choices } = parsedLine;

24
api_webchat/model-worker-openai_comp.js
View file

@ -60,6 +60,7 @@ self.onmessage = async function(event) {
const reader = response.body.getReader();
const decoder = new TextDecoder("utf-8");
let chunk = '';
while (true) {
if (stopStreaming) {
@ -78,12 +79,25 @@ self.onmessage = async function(event) {
break;
}
// lots of low-level OpenAI response parsing stuff
const chunk = decoder.decode(value);
chunk += decoder.decode(value);
// console.log(">>>>>>>>>>>>>> [ThunderAI] chunk: " + JSON.stringify(chunk));
const lines = chunk.split("\n");
const parsedLines = lines
.map((line) => line.replace(/^data: /, "").trim()) // Remove the "data: " prefix
.filter((line) => line !== "" && line !== "[DONE]") // Remove empty lines and "[DONE]"
.map((line) => JSON.parse(line)); // Parse the JSON string
let parsedLines = [];
chunk = chunk.substring(chunk.lastIndexOf('\n') + 1);
// console.log(">>>>>>>>>>>>>> [ThunderAI] last chunk: " + JSON.stringify(chunk));
try{
parsedLines = lines
.map((line) => line.replace(/^data: /, "").trim()) // Remove the "data: " prefix
.filter((line) => line !== "" && line !== "[DONE]") // Remove empty lines and "[DONE]"
// .map((line) => JSON.parse(line)); // Parse the JSON string
.map((line) => {
// console.log(">>>>>>>>>>>>> [ThunderAI] line: " + JSON.stringify(line));
return JSON.parse(line);
});
}catch(e){
// console.error(">>>>>>>>>>>>> [ThunderAI] error: " + e);
console.warn("[ThunderAI | model-worker-openai_comp] broken chunk: " + e);
}
for (const parsedLine of parsedLines) {
const { choices } = parsedLine;