🤖 Assistants V2 Support: Part 2

🎹 fix: Autocompletion Chrome Bug on Action API Key Input

chore: remove `useOriginNavigate`

chore: set correct OpenAI Storage Source

fix: azure file deletions, instantiate clients by source for deletion

update code interpret files info

feat: deleteResourceFileId

chore: increase poll interval as azure easily rate limits

fix: openai file deletions, TODO: evaluate rejected deletion settled promises to determine which to delete from db records

file source icons

update table file filters

chore: file search info and versioning

fix: retrieval update with necessary tool_resources if specified

fix(useMentions): add optional chaining in case listMap value is undefined

fix: force assistant avatar roundedness

fix: azure assistants, check correct flag

chore: bump data-provider

api/server/controllers/assistants/chatV1.js

@@ -3,11 +3,11 @@ const {
   Constants,
   RunStatus,
   CacheKeys,
-  FileSources,
   ContentTypes,
   EModelEndpoint,
   ViolationTypes,
   ImageVisionTool,
+  checkOpenAIStorage,
   AssistantStreamEvents,
 } = require('librechat-data-provider');
 const {
@@ -361,10 +361,7 @@ const chatV2 = async (req, res) => {
 
       /** @type {MongoFile[]} */
       const attachments = await req.body.endpointOption.attachments;
-      if (
-        attachments &&
-        attachments.every((attachment) => attachment.source === FileSources.openai)
-      ) {
+      if (attachments && attachments.every((attachment) => checkOpenAIStorage(attachment.source))) {
         return;
       }
 
@@ -422,7 +419,7 @@ const chatV2 = async (req, res) => {
 
       if (processedFiles) {
         for (const file of processedFiles) {
-          if (file.source !== FileSources.openai) {
+          if (!checkOpenAIStorage(file.source)) {
             attachedFileIds.delete(file.file_id);
             const index = file_ids.indexOf(file.file_id);
             if (index > -1) {

api/server/controllers/assistants/helpers.js

@@ -1,9 +1,4 @@
-const {
-  EModelEndpoint,
-  FileSources,
-  CacheKeys,
-  defaultAssistantsVersion,
-} = require('librechat-data-provider');
+const { EModelEndpoint, CacheKeys, defaultAssistantsVersion } = require('librechat-data-provider');
 const {
   initializeClient: initAzureClient,
 } = require('~/server/services/Endpoints/azureAssistants');
@@ -121,13 +116,8 @@ const listAssistantsForAzure = async ({ req, res, version, azureConfig = {}, que
   };
 };
 
-async function getOpenAIClient({ req, res, endpointOption, initAppClient }) {
-  let endpoint = req.body.endpoint ?? req.query.endpoint;
-  if (!endpoint && req.baseUrl.includes('files') && req.body.files) {
-    const source = req.body.files[0]?.source;
-    endpoint =
-      source === FileSources.openai ? EModelEndpoint.assistants : EModelEndpoint.azureAssistants;
-  }
+async function getOpenAIClient({ req, res, endpointOption, initAppClient, overrideEndpoint }) {
+  let endpoint = overrideEndpoint ?? req.body.endpoint ?? req.query.endpoint;
   const version = await getCurrentVersion(req, endpoint);
   if (!endpoint) {
     throw new Error(`[${req.baseUrl}] Endpoint is required`);

api/server/controllers/assistants/v2.js

@@ -1,3 +1,4 @@
+const { ToolCallTypes } = require('librechat-data-provider');
 const { validateAndUpdateTool } = require('~/server/services/ActionService');
 const { getOpenAIClient } = require('./helpers');
 const { logger } = require('~/config');
@@ -54,6 +55,7 @@ const createAssistant = async (req, res) => {
 const updateAssistant = async ({ req, openai, assistant_id, updateData }) => {
   const tools = [];
 
+  let hasFileSearch = false;
   for (const tool of updateData.tools ?? []) {
     let actualTool = typeof tool === 'string' ? req.app.locals.availableTools[tool] : tool;
 
@@ -61,6 +63,10 @@ const updateAssistant = async ({ req, openai, assistant_id, updateData }) => {
       continue;
     }
 
+    if (actualTool.type === ToolCallTypes.FILE_SEARCH) {
+      hasFileSearch = true;
+    }
+
     if (!actualTool.function) {
       tools.push(actualTool);
       continue;
@@ -72,6 +78,20 @@ const updateAssistant = async ({ req, openai, assistant_id, updateData }) => {
     }
   }
 
+  if (hasFileSearch && !updateData.tool_resources) {
+    const assistant = await openai.beta.assistants.retrieve(assistant_id);
+    updateData.tool_resources = assistant.tool_resources ?? null;
+  }
+
+  if (hasFileSearch && !updateData.tool_resources?.file_search) {
+    updateData.tool_resources = {
+      ...(updateData.tool_resources ?? {}),
+      file_search: {
+        vector_store_ids: [],
+      },
+    };
+  }
+
   updateData.tools = tools;
 
   if (openai.locals?.azureOptions && updateData.model) {

api/server/routes/files/files.js

@@ -1,6 +1,6 @@
 const fs = require('fs').promises;
 const express = require('express');
-const { isUUID, FileSources } = require('librechat-data-provider');
+const { isUUID, checkOpenAIStorage } = require('librechat-data-provider');
 const {
   filterFile,
   processFileUpload,
@@ -89,7 +89,7 @@ router.get('/download/:userId/:file_id', async (req, res) => {
       return res.status(403).send('Forbidden');
     }
 
-    if (file.source === FileSources.openai && !file.model) {
+    if (checkOpenAIStorage(file.source) && !file.model) {
       logger.warn(`${errorPrefix} has no associated model: ${file_id}`);
       return res.status(400).send('The model used when creating this file is not available');
     }
@@ -110,7 +110,8 @@ router.get('/download/:userId/:file_id', async (req, res) => {
     let passThrough;
     /** @type {ReadableStream | undefined} */
     let fileStream;
-    if (file.source === FileSources.openai) {
+
+    if (checkOpenAIStorage(file.source)) {
       req.body = { model: file.model };
       const { openai } = await initializeClient({ req, res });
       logger.debug(`Downloading file ${file_id} from OpenAI`);

api/server/services/AssistantService.js

@@ -78,7 +78,7 @@ async function createOnTextProgress({
  * @return {Promise<OpenAIAssistantFinish | OpenAIAssistantAction[] | ThreadMessage[] | RequiredActionFunctionToolCall[]>}
  */
 async function getResponse({ openai, run_id, thread_id }) {
-  const run = await waitForRun({ openai, run_id, thread_id, pollIntervalMs: 500 });
+  const run = await waitForRun({ openai, run_id, thread_id, pollIntervalMs: 2000 });
 
   if (run.status === RunStatus.COMPLETED) {
     const messages = await openai.beta.threads.messages.list(thread_id, defaultOrderQuery);

api/server/services/Endpoints/azureAssistants/initializeClient.js

@@ -15,6 +15,44 @@ const OpenAIClient = require('~/app/clients/OpenAIClient');
 const { isUserProvided } = require('~/server/utils');
 const { constructAzureURL } = require('~/utils');
 
+class Files {
+  constructor(client) {
+    this._client = client;
+  }
+  /**
+   * Create an assistant file by attaching a
+   * [File](https://platform.openai.com/docs/api-reference/files) to an
+   * [assistant](https://platform.openai.com/docs/api-reference/assistants).
+   */
+  create(assistantId, body, options) {
+    return this._client.post(`/assistants/${assistantId}/files`, {
+      body,
+      ...options,
+      headers: { 'OpenAI-Beta': 'assistants=v1', ...options?.headers },
+    });
+  }
+
+  /**
+   * Retrieves an AssistantFile.
+   */
+  retrieve(assistantId, fileId, options) {
+    return this._client.get(`/assistants/${assistantId}/files/${fileId}`, {
+      ...options,
+      headers: { 'OpenAI-Beta': 'assistants=v1', ...options?.headers },
+    });
+  }
+
+  /**
+   * Delete an assistant file.
+   */
+  del(assistantId, fileId, options) {
+    return this._client.delete(`/assistants/${assistantId}/files/${fileId}`, {
+      ...options,
+      headers: { 'OpenAI-Beta': 'assistants=v1', ...options?.headers },
+    });
+  }
+}
+
 const initializeClient = async ({ req, res, version, endpointOption, initAppClient = false }) => {
   const { PROXY, OPENAI_ORGANIZATION, AZURE_ASSISTANTS_API_KEY, AZURE_ASSISTANTS_BASE_URL } =
     process.env;
@@ -130,6 +168,8 @@ const initializeClient = async ({ req, res, version, endpointOption, initAppClie
     ...opts,
   });
 
+  openai.beta.assistants.files = new Files(openai);
+
   openai.req = req;
   openai.res = res;
 

api/server/services/Files/Firebase/crud.js

@@ -180,7 +180,15 @@ const deleteFirebaseFile = async (req, file) => {
   if (!fileName.includes(req.user.id)) {
     throw new Error('Invalid file path');
   }
-  await deleteFile('', fileName);
+  try {
+    await deleteFile('', fileName);
+  } catch (error) {
+    logger.error('Error deleting file from Firebase:', error);
+    if (error.code === 'storage/object-not-found') {
+      return;
+    }
+    throw error;
+  }
 };
 
 /**

api/server/services/Files/process.js

@@ -10,21 +10,19 @@ const {
   EModelEndpoint,
   mergeFileConfig,
   hostImageIdSuffix,
+  checkOpenAIStorage,
   hostImageNamePrefix,
   isAssistantsEndpoint,
 } = require('librechat-data-provider');
+const { addResourceFileId, deleteResourceFileId } = require('~/server/controllers/assistants/v2');
 const { convertImage, resizeAndConvert } = require('~/server/services/Files/images');
 const { getOpenAIClient } = require('~/server/controllers/assistants/helpers');
 const { createFile, updateFileUsage, deleteFiles } = require('~/models/File');
-const { addResourceFileId } = require('~/server/controllers/assistants/v2');
 const { LB_QueueAsyncCall } = require('~/server/utils/queue');
 const { getStrategyFunctions } = require('./strategies');
 const { determineFileType } = require('~/server/utils');
 const { logger } = require('~/config');
 
-const checkOpenAIStorage = (source) =>
-  source === FileSources.openai || source === FileSources.azure;
-
 const processFiles = async (files) => {
   const promises = [];
   for (let file of files) {
@@ -39,13 +37,15 @@ const processFiles = async (files) => {
 /**
  * Enqueues the delete operation to the leaky bucket queue if necessary, or adds it directly to promises.
  *
- * @param {Express.Request} req - The express request object.
- * @param {MongoFile} file - The file object to delete.
- * @param {Function} deleteFile - The delete file function.
- * @param {Promise[]} promises - The array of promises to await.
- * @param {OpenAI | undefined} [openai] - If an OpenAI file, the initialized OpenAI client.
+ * @param {object} params - The passed parameters.
+ * @param {Express.Request} params.req - The express request object.
+ * @param {MongoFile} params.file - The file object to delete.
+ * @param {Function} params.deleteFile - The delete file function.
+ * @param {Promise[]} params.promises - The array of promises to await.
+ * @param {string[]} params.resolvedFileIds - The array of promises to await.
+ * @param {OpenAI | undefined} [params.openai] - If an OpenAI file, the initialized OpenAI client.
  */
-function enqueueDeleteOperation(req, file, deleteFile, promises, openai) {
+function enqueueDeleteOperation({ req, file, deleteFile, promises, resolvedFileIds, openai }) {
   if (checkOpenAIStorage(file.source)) {
     // Enqueue to leaky bucket
     promises.push(
@@ -58,6 +58,7 @@ function enqueueDeleteOperation(req, file, deleteFile, promises, openai) {
               logger.error('Error deleting file from OpenAI source', err);
               reject(err);
             } else {
+              resolvedFileIds.push(file.file_id);
               resolve(result);
             }
           },
@@ -67,10 +68,12 @@ function enqueueDeleteOperation(req, file, deleteFile, promises, openai) {
   } else {
     // Add directly to promises
     promises.push(
-      deleteFile(req, file).catch((err) => {
-        logger.error('Error deleting file', err);
-        return Promise.reject(err);
-      }),
+      deleteFile(req, file)
+        .then(() => resolvedFileIds.push(file.file_id))
+        .catch((err) => {
+          logger.error('Error deleting file', err);
+          return Promise.reject(err);
+        }),
     );
   }
 }
@@ -85,35 +88,71 @@ function enqueueDeleteOperation(req, file, deleteFile, promises, openai) {
  * @param {Express.Request} params.req - The express request object.
  * @param {DeleteFilesBody} params.req.body - The request body.
  * @param {string} [params.req.body.assistant_id] - The assistant ID if file uploaded is associated to an assistant.
+ * @param {string} [params.req.body.tool_resource] - The tool resource if assistant file uploaded is associated to a tool resource.
  *
  * @returns {Promise<void>}
  */
 const processDeleteRequest = async ({ req, files }) => {
-  const file_ids = files.map((file) => file.file_id);
-
+  const resolvedFileIds = [];
   const deletionMethods = {};
   const promises = [];
-  promises.push(deleteFiles(file_ids));
 
-  /** @type {OpenAI | undefined} */
-  let openai;
-  if (req.body.assistant_id) {
-    ({ openai } = await getOpenAIClient({ req }));
+  /** @type {Record<string, OpenAI | undefined>} */
+  const client = { [FileSources.openai]: undefined, [FileSources.azure]: undefined };
+  const initializeClients = async () => {
+    const openAIClient = await getOpenAIClient({
+      req,
+      overrideEndpoint: EModelEndpoint.assistants,
+    });
+    client[FileSources.openai] = openAIClient.openai;
+
+    if (!req.app.locals[EModelEndpoint.azureOpenAI]?.assistants) {
+      return;
+    }
+
+    const azureClient = await getOpenAIClient({
+      req,
+      overrideEndpoint: EModelEndpoint.azureAssistants,
+    });
+    client[FileSources.azure] = azureClient.openai;
+  };
+
+  if (req.body.assistant_id !== undefined) {
+    await initializeClients();
   }
 
   for (const file of files) {
     const source = file.source ?? FileSources.local;
 
-    if (checkOpenAIStorage(source) && !openai) {
-      ({ openai } = await getOpenAIClient({ req }));
+    if (checkOpenAIStorage(source) && !client[source]) {
+      await initializeClients();
     }
 
-    if (req.body.assistant_id) {
+    const openai = client[source];
+
+    if (req.body.assistant_id && req.body.tool_resource) {
+      promises.push(
+        deleteResourceFileId({
+          req,
+          openai,
+          file_id: file.file_id,
+          assistant_id: req.body.assistant_id,
+          tool_resource: req.body.tool_resource,
+        }),
+      );
+    } else if (req.body.assistant_id) {
       promises.push(openai.beta.assistants.files.del(req.body.assistant_id, file.file_id));
     }
 
     if (deletionMethods[source]) {
-      enqueueDeleteOperation(req, file, deletionMethods[source], promises, openai);
+      enqueueDeleteOperation({
+        req,
+        file,
+        deleteFile: deletionMethods[source],
+        promises,
+        resolvedFileIds,
+        openai,
+      });
       continue;
     }
 
@@ -123,10 +162,11 @@ const processDeleteRequest = async ({ req, files }) => {
     }
 
     deletionMethods[source] = deleteFile;
-    enqueueDeleteOperation(req, file, deleteFile, promises, openai);
+    enqueueDeleteOperation({ req, file, deleteFile, promises, resolvedFileIds, openai });
   }
 
   await Promise.allSettled(promises);
+  await deleteFiles(resolvedFileIds);
 };
 
 /**
@@ -381,7 +421,10 @@ const processOpenAIFile = async ({
     originalName ? `/${originalName}` : ''
   }`;
   const type = mime.getType(originalName ?? file_id);
-
+  const source =
+    openai.req.body.endpoint === EModelEndpoint.azureAssistants
+      ? FileSources.azure
+      : FileSources.openai;
   const file = {
     ..._file,
     type,
@@ -390,7 +433,7 @@ const processOpenAIFile = async ({
     usage: 1,
     user: userId,
     context: _file.purpose,
-    source: FileSources.openai,
+    source,
     model: openai.req.body.model,
     filename: originalName ?? file_id,
   };
@@ -435,12 +478,14 @@ const processOpenAIImageOutput = async ({ req, buffer, file_id, filename, fileEx
     filename: `${hostImageNamePrefix}${filename}`,
   };
   createFile(file, true);
+  const source =
+    req.body.endpoint === EModelEndpoint.azureAssistants ? FileSources.azure : FileSources.openai;
   createFile(
     {
       ...file,
       file_id,
       filename,
-      source: FileSources.openai,
+      source,
       type: mime.getType(fileExt),
     },
     true,

api/server/services/Runs/handle.js

@@ -55,7 +55,7 @@ async function createRun({ openai, thread_id, body }) {
  * @param {string} params.run_id - The ID of the run to wait for.
  * @param {string} params.thread_id - The ID of the thread associated with the run.
  * @param {RunManager} params.runManager - The RunManager instance to manage run steps.
- * @param {number} [params.pollIntervalMs=750] - The interval for polling the run status; default is 750 milliseconds.
+ * @param {number} [params.pollIntervalMs=2000] - The interval for polling the run status; default is 2000 milliseconds.
  * @param {number} [params.timeout=180000] - The period to wait until timing out polling; default is 3 minutes (in ms).
  * @return {Promise<Run>} A promise that resolves to the last fetched run object.
  */
@@ -64,7 +64,7 @@ async function waitForRun({
   run_id,
   thread_id,
   runManager,
-  pollIntervalMs = 750,
+  pollIntervalMs = 2000,
   timeout = 60000 * 3,
 }) {
   let timeElapsed = 0;
@@ -233,7 +233,7 @@ async function _handleRun({ openai, run_id, thread_id }) {
     run_id,
     thread_id,
     runManager,
-    pollIntervalMs: 750,
+    pollIntervalMs: 2000,
     timeout: 60000,
   });
   const actions = [];