🚀 refactor: Enhance Custom Endpoints, Message Logic, and Payload Handling (#2895)

* chore: use node-fetch for OpenAIClient fetch key for non-crashing usage of AbortController in Bun runtime

* chore: variable order

* fix(useSSE): prevent finalHandler call in abortConversation to update messages/conversation after user navigated away

* chore: params order

* refactor: organize intermediate message logic and ensure correct variables are passed

* fix: Add stt and tts routes before upload limiters, prevent bans

* fix(abortRun): temp fix to delete unfinished messages to avoid message thread parent relationship issues

* refactor: Update AnthropicClient to use node-fetch for fetch key and add proxy support

* fix(gptPlugins): ensure parentMessageId/messageId relationship is maintained

* feat(BaseClient): custom fetch function to analyze/edit payloads just before sending (also prevents abortController crash on Bun runtime)

* feat: `directEndpoint` and `titleMessageRole` custom endpoint options

* chore: Bump version to 0.6.6 in data-provider package.json

api/server/controllers/assistants/chatV2.js

@@ -496,7 +496,7 @@ const chatV2 = async (req, res) => {
         handlers,
         thread_id,
         attachedFileIds,
-        parentMessageId,
+        parentMessageId: userMessageId,
         responseMessage: openai.responseMessage,
         // streamOptions: {
 

api/server/middleware/abortRun.js

@@ -1,6 +1,7 @@
 const { CacheKeys, RunStatus, isUUID } = require('librechat-data-provider');
 const { initializeClient } = require('~/server/services/Endpoints/assistants');
 const { checkMessageGaps, recordUsage } = require('~/server/services/Threads');
+const { deleteMessages } = require('~/models/Message');
 const { getConvo } = require('~/models/Conversation');
 const getLogStores = require('~/cache/getLogStores');
 const { sendMessage } = require('~/server/utils');
@@ -66,13 +67,19 @@ async function abortRun(req, res) {
     logger.error('[abortRun] Error fetching or processing run', error);
   }
 
+  /* TODO: a reconciling strategy between the existing intermediate message would be more optimal than deleting it */
+  await deleteMessages({
+    user: req.user.id,
+    unfinished: true,
+    conversationId,
+  });
   runMessages = await checkMessageGaps({
     openai,
+    run_id,
     endpoint,
     thread_id,
-    run_id,
-    latestMessageId,
     conversationId,
+    latestMessageId,
   });
 
   const finalEvent = {

api/server/routes/ask/gptPlugins.js

@@ -106,7 +106,11 @@ router.post(
     const pluginMap = new Map();
     const onAgentAction = async (action, runId) => {
       pluginMap.set(runId, action.tool);
-      sendIntermediateMessage(res, { plugins });
+      sendIntermediateMessage(res, {
+        plugins,
+        parentMessageId: userMessage.messageId,
+        messageId: responseMessageId,
+      });
     };
 
     const onToolStart = async (tool, input, runId, parentRunId) => {
@@ -124,7 +128,11 @@ router.post(
       }
       const extraTokens = ':::plugin:::\n';
       plugins.push(latestPlugin);
-      sendIntermediateMessage(res, { plugins }, extraTokens);
+      sendIntermediateMessage(
+        res,
+        { plugins, parentMessageId: userMessage.messageId, messageId: responseMessageId },
+        extraTokens,
+      );
     };
 
     const onToolEnd = async (output, runId) => {
@@ -142,7 +150,11 @@ router.post(
 
     const onChainEnd = () => {
       saveMessage({ ...userMessage, user });
-      sendIntermediateMessage(res, { plugins });
+      sendIntermediateMessage(res, {
+        plugins,
+        parentMessageId: userMessage.messageId,
+        messageId: responseMessageId,
+      });
     };
 
     const getAbortData = () => ({

api/server/routes/edit/gptPlugins.js

@@ -110,7 +110,11 @@ router.post(
       if (!start) {
         saveMessage({ ...userMessage, user });
       }
-      sendIntermediateMessage(res, { plugin });
+      sendIntermediateMessage(res, {
+        plugin,
+        parentMessageId: userMessage.messageId,
+        messageId: responseMessageId,
+      });
       // logger.debug('PLUGIN ACTION', formattedAction);
     };
 
@@ -119,7 +123,11 @@ router.post(
       plugin.outputs = steps && steps[0].action ? formatSteps(steps) : 'An error occurred.';
       plugin.loading = false;
       saveMessage({ ...userMessage, user });
-      sendIntermediateMessage(res, { plugin });
+      sendIntermediateMessage(res, {
+        plugin,
+        parentMessageId: userMessage.messageId,
+        messageId: responseMessageId,
+      });
       // logger.debug('CHAIN END', plugin.outputs);
     };
 

api/server/routes/files/index.js

@@ -14,6 +14,10 @@ const initialize = async () => {
   router.use(checkBan);
   router.use(uaParser);
 
+  /* Important: stt/tts routes must be added before the upload limiters */
+  router.use('/stt', stt);
+  router.use('/tts', tts);
+
   const upload = await createMulterInstance();
   const { fileUploadIpLimiter, fileUploadUserLimiter } = createFileLimiters();
   router.post('*', fileUploadIpLimiter, fileUploadUserLimiter);

api/server/services/Endpoints/custom/initializeClient.js

@@ -112,6 +112,8 @@ const initializeClient = async ({ req, res, endpointOption }) => {
     modelDisplayLabel: endpointConfig.modelDisplayLabel,
     titleMethod: endpointConfig.titleMethod ?? 'completion',
     contextStrategy: endpointConfig.summarize ? 'summarize' : null,
+    directEndpoint: endpointConfig.directEndpoint,
+    titleMessageRole: endpointConfig.titleMessageRole,
     endpointTokenConfig,
   };
 

api/server/services/Runs/StreamRunManager.js

@@ -9,9 +9,9 @@ const {
 } = require('librechat-data-provider');
 const { retrieveAndProcessFile } = require('~/server/services/Files/process');
 const { processRequiredActions } = require('~/server/services/ToolService');
+const { saveMessage, updateMessageText } = require('~/models/Message');
 const { createOnProgress, sendMessage } = require('~/server/utils');
 const { processMessages } = require('~/server/services/Threads');
-const { saveMessage } = require('~/models');
 const { logger } = require('~/config');
 
 /**
@@ -68,6 +68,8 @@ class StreamRunManager {
     this.attachedFileIds = fields.attachedFileIds;
     /** @type {undefined | Promise<ChatCompletion>} */
     this.visionPromise = fields.visionPromise;
+    /** @type {boolean} */
+    this.savedInitialMessage = false;
 
     /**
      * @type {Object.<AssistantStreamEvents, (event: AssistantStreamEvent) => Promise<void>>}
@@ -129,6 +131,33 @@ class StreamRunManager {
     sendMessage(this.res, contentData);
   }
 
+  /* <------------------ Misc. Helpers ------------------> */
+  /** Returns the latest intermediate text
+   * @returns {string}
+   */
+  getText() {
+    return this.intermediateText;
+  }
+
+  /** Saves the initial intermediate message
+   * @returns {Promise<void>}
+   */
+  async saveInitialMessage() {
+    return saveMessage({
+      conversationId: this.finalMessage.conversationId,
+      messageId: this.finalMessage.messageId,
+      parentMessageId: this.parentMessageId,
+      model: this.req.body.assistant_id,
+      endpoint: this.req.body.endpoint,
+      isCreatedByUser: false,
+      user: this.req.user.id,
+      text: this.getText(),
+      sender: 'Assistant',
+      unfinished: true,
+      error: false,
+    });
+  }
+
   /* <------------------ Main Event Handlers ------------------> */
 
   /**
@@ -530,23 +559,20 @@ class StreamRunManager {
       const stepKey = message_creation.message_id;
       const index = this.getStepIndex(stepKey);
       this.orderedRunSteps.set(index, message_creation);
-      const getText = () => this.intermediateText;
+
       // Create the Factory Function to stream the message
       const { onProgress: progressCallback } = createOnProgress({
         onProgress: throttle(
           () => {
-            const text = getText();
-            saveMessage({
-              messageId: this.finalMessage.messageId,
-              conversationId: this.finalMessage.conversationId,
-              parentMessageId: this.parentMessageId,
-              model: this.req.body.model,
-              user: this.req.user.id,
-              sender: 'Assistant',
-              unfinished: true,
-              error: false,
-              text,
-            });
+            if (!this.savedInitialMessage) {
+              this.saveInitialMessage();
+              this.savedInitialMessage = true;
+            } else {
+              updateMessageText({
+                messageId: this.finalMessage.messageId,
+                text: this.getText(),
+              });
+            }
           },
           2000,
           { trailing: false },