Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2025-12-17 17:00:15 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

⚙️ feat: Adjust Rate of Stream Progress (#3244)

Browse source 
* chore: bump data-provider and add MESSAGES CacheKey

* refactor: avoid saving messages while streaming, save partial text to cache instead

* fix(ci): processChunks

* chore: logging aborted request to debug

* feat: set stream rate for token processing

* chore: specify default stream rate

* fix(ci): Update AppService.js to use optional chaining for endpointLocals assignment

* refactor: abstract the error handler

* feat: streamRate for assistants; refactor: update default rate for token

* refactor: update error handling in assistants/errors.js

* refactor: update error handling in assistants/errors.js
This commit is contained in:
Danny Avila 2024-07-17 10:47:17 -04:00 • committed by GitHub
parent 1c282d1517
commit 5d40d0a37a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
29 changed files with 661 additions and 309 deletions
Download patch file Download diff file Expand all files Collapse all files

18
api/server/services/Files/Audio/streamAudio.js
View file

@ -1,5 +1,6 @@
const WebSocket = require('ws');
const { Message } = require('~/models/Message');
const { CacheKeys } = require('librechat-data-provider');
const { getLogStores } = require('~/cache');
/**
* @param {string[]} voiceIds - Array of voice IDs
@ -104,6 +105,8 @@ function createChunkProcessor(messageId) {
throw new Error('Message ID is required');
}
const messageCache = getLogStores(CacheKeys.MESSAGES);
/**
* @returns {Promise<{ text: string, isFinished: boolean }[] | string>}
*/
@ -116,14 +119,17 @@ function createChunkProcessor(messageId) {
return `No change in message after ${MAX_NO_CHANGE_COUNT} attempts`;
}
const message = await Message.findOne({ messageId }, 'text unfinished').lean();
/** @type { string | { text: string; complete: boolean } } */
const message = await messageCache.get(messageId);
if (!message || !message.text) {
if (!message) {
notFoundCount++;
return [];
}
const { text, unfinished } = message;
const text = typeof message === 'string' ? message : message.text;
const complete = typeof message === 'string' ? false : message.complete;
if (text === processedText) {
noChangeCount++;
}
@ -131,7 +137,7 @@ function createChunkProcessor(messageId) {
const remainingText = text.slice(processedText.length);
const chunks = [];
if (unfinished && remainingText.length >= 20) {
if (!complete && remainingText.length >= 20) {
const separatorIndex = findLastSeparatorIndex(remainingText);
if (separatorIndex !== -1) {
const chunkText = remainingText.slice(0, separatorIndex + 1);
@ -141,7 +147,7 @@ function createChunkProcessor(messageId) {
chunks.push({ text: remainingText, isFinished: false });
processedText = text;
}
} else if (!unfinished && remainingText.trim().length > 0) {
} else if (complete && remainingText.trim().length > 0) {
chunks.push({ text: remainingText.trim(), isFinished: true });
processedText = text;
}