🤖 feat: OpenAI Assistants v2 (initial support) (#2781)

* 🤖 Assistants V2 Support: Part 1

- Separated Azure Assistants to its own endpoint
- File Search / Vector Store integration is incomplete, but can toggle and use storage from playground
- Code Interpreter resource files can be added but not deleted
- GPT-4o is supported
- Many improvements to the Assistants Endpoint overall

data-provider v2 changes

copy existing route as v1

chore: rename new endpoint to reduce comparison operations and add new azure filesource

api: add azureAssistants part 1

force use of version for assistants/assistantsAzure

chore: switch name back to azureAssistants

refactor type version: string | number

Ensure assistants endpoints have version set

fix: isArchived type issue in ConversationListParams

refactor: update assistants mutations/queries with endpoint/version definitions, update Assistants Map structure

chore:  FilePreview component ExtendedFile type assertion

feat: isAssistantsEndpoint helper

chore: remove unused useGenerations

chore(buildTree): type issue

chore(Advanced): type issue (unused component, maybe in future)

first pass for multi-assistant endpoint rewrite

fix(listAssistants): pass params correctly

feat: list separate assistants by endpoint

fix(useTextarea): access assistantMap correctly

fix: assistant endpoint switching, resetting ID

fix: broken during rewrite, selecting assistant mention

fix: set/invalidate assistants endpoint query data correctly

feat: Fix issue with assistant ID not being reset correctly

getOpenAIClient helper function

feat: add toast for assistant deletion

fix: assistants delete right after create issue for azure

fix: assistant patching

refactor: actions to use getOpenAIClient

refactor: consolidate logic into helpers file

fix: issue where conversation data was not initially available

v1 chat support

refactor(spendTokens): only early return if completionTokens isNaN

fix(OpenAIClient): ensure spendTokens has all necessary params

refactor: route/controller logic

fix(assistants/initializeClient): use defaultHeaders field

fix: sanitize default operation id

chore: bump openai package

first pass v2 action service

feat: retroactive domain parsing for actions added via v1

feat: delete db records of actions/assistants on openai assistant deletion

chore: remove vision tools from v2 assistants

feat: v2 upload and delete assistant vision images

WIP first pass, thread attachments

fix: show assistant vision files (save local/firebase copy)

v2 image continue

fix: annotations

fix: refine annotations

show analyze as error if is no longer submitting before progress reaches 1 and show file_search as retrieval tool

fix: abort run, undefined endpoint issue

refactor: consolidate capabilities logic and anticipate versioning

frontend version 2 changes

fix: query selection and filter

add endpoint to unknown filepath

add file ids to resource, deleting in progress

enable/disable file search

remove version log

* 🤖 Assistants V2 Support: Part 2

🎹 fix: Autocompletion Chrome Bug on Action API Key Input

chore: remove `useOriginNavigate`

chore: set correct OpenAI Storage Source

fix: azure file deletions, instantiate clients by source for deletion

update code interpret files info

feat: deleteResourceFileId

chore: increase poll interval as azure easily rate limits

fix: openai file deletions, TODO: evaluate rejected deletion settled promises to determine which to delete from db records

file source icons

update table file filters

chore: file search info and versioning

fix: retrieval update with necessary tool_resources if specified

fix(useMentions): add optional chaining in case listMap value is undefined

fix: force assistant avatar roundedness

fix: azure assistants, check correct flag

chore: bump data-provider

* fix: merge conflict

* ci: fix backend tests due to new updates

* chore: update .env.example

* meilisearch improvements

* localization updates

* chore: update comparisons

* feat: add additional metadata: endpoint, author ID

* chore: azureAssistants ENDPOINTS exclusion warning

api/server/controllers/assistants/v1.js

@@ -0,0 +1,262 @@
+const { FileContext } = require('librechat-data-provider');
+const { getStrategyFunctions } = require('~/server/services/Files/strategies');
+const { deleteAssistantActions } = require('~/server/services/ActionService');
+const { uploadImageBuffer } = require('~/server/services/Files/process');
+const { updateAssistant, getAssistants } = require('~/models/Assistant');
+const { getOpenAIClient, fetchAssistants } = require('./helpers');
+const { deleteFileByFilter } = require('~/models/File');
+const { logger } = require('~/config');
+
+/**
+ * Create an assistant.
+ * @route POST /assistants
+ * @param {AssistantCreateParams} req.body - The assistant creation parameters.
+ * @returns {Assistant} 201 - success response - application/json
+ */
+const createAssistant = async (req, res) => {
+  try {
+    const { openai } = await getOpenAIClient({ req, res });
+
+    const { tools = [], endpoint, ...assistantData } = req.body;
+    assistantData.tools = tools
+      .map((tool) => {
+        if (typeof tool !== 'string') {
+          return tool;
+        }
+
+        return req.app.locals.availableTools[tool];
+      })
+      .filter((tool) => tool);
+
+    let azureModelIdentifier = null;
+    if (openai.locals?.azureOptions) {
+      azureModelIdentifier = assistantData.model;
+      assistantData.model = openai.locals.azureOptions.azureOpenAIApiDeploymentName;
+    }
+
+    assistantData.metadata = {
+      author: req.user.id,
+      endpoint,
+    };
+
+    const assistant = await openai.beta.assistants.create(assistantData);
+    if (azureModelIdentifier) {
+      assistant.model = azureModelIdentifier;
+    }
+    logger.debug('/assistants/', assistant);
+    res.status(201).json(assistant);
+  } catch (error) {
+    logger.error('[/assistants] Error creating assistant', error);
+    res.status(500).json({ error: error.message });
+  }
+};
+
+/**
+ * Retrieves an assistant.
+ * @route GET /assistants/:id
+ * @param {string} req.params.id - Assistant identifier.
+ * @returns {Assistant} 200 - success response - application/json
+ */
+const retrieveAssistant = async (req, res) => {
+  try {
+    /* NOTE: not actually being used right now */
+    const { openai } = await getOpenAIClient({ req, res });
+
+    const assistant_id = req.params.id;
+    const assistant = await openai.beta.assistants.retrieve(assistant_id);
+    res.json(assistant);
+  } catch (error) {
+    logger.error('[/assistants/:id] Error retrieving assistant', error);
+    res.status(500).json({ error: error.message });
+  }
+};
+
+/**
+ * Modifies an assistant.
+ * @route PATCH /assistants/:id
+ * @param {object} req - Express Request
+ * @param {object} req.params - Request params
+ * @param {string} req.params.id - Assistant identifier.
+ * @param {AssistantUpdateParams} req.body - The assistant update parameters.
+ * @returns {Assistant} 200 - success response - application/json
+ */
+const patchAssistant = async (req, res) => {
+  try {
+    const { openai } = await getOpenAIClient({ req, res });
+
+    const assistant_id = req.params.id;
+    const { endpoint: _e, ...updateData } = req.body;
+    updateData.tools = (updateData.tools ?? [])
+      .map((tool) => {
+        if (typeof tool !== 'string') {
+          return tool;
+        }
+
+        return req.app.locals.availableTools[tool];
+      })
+      .filter((tool) => tool);
+
+    if (openai.locals?.azureOptions && updateData.model) {
+      updateData.model = openai.locals.azureOptions.azureOpenAIApiDeploymentName;
+    }
+
+    const updatedAssistant = await openai.beta.assistants.update(assistant_id, updateData);
+    res.json(updatedAssistant);
+  } catch (error) {
+    logger.error('[/assistants/:id] Error updating assistant', error);
+    res.status(500).json({ error: error.message });
+  }
+};
+
+/**
+ * Deletes an assistant.
+ * @route DELETE /assistants/:id
+ * @param {object} req - Express Request
+ * @param {object} req.params - Request params
+ * @param {string} req.params.id - Assistant identifier.
+ * @returns {Assistant} 200 - success response - application/json
+ */
+const deleteAssistant = async (req, res) => {
+  try {
+    const { openai } = await getOpenAIClient({ req, res });
+
+    const assistant_id = req.params.id;
+    const deletionStatus = await openai.beta.assistants.del(assistant_id);
+    if (deletionStatus?.deleted) {
+      await deleteAssistantActions({ req, assistant_id });
+    }
+    res.json(deletionStatus);
+  } catch (error) {
+    logger.error('[/assistants/:id] Error deleting assistant', error);
+    res.status(500).json({ error: 'Error deleting assistant' });
+  }
+};
+
+/**
+ * Returns a list of assistants.
+ * @route GET /assistants
+ * @param {object} req - Express Request
+ * @param {AssistantListParams} req.query - The assistant list parameters for pagination and sorting.
+ * @returns {AssistantListResponse} 200 - success response - application/json
+ */
+const listAssistants = async (req, res) => {
+  try {
+    const body = await fetchAssistants(req, res);
+
+    if (req.app.locals?.[req.query.endpoint]) {
+      /** @type {Partial<TAssistantEndpoint>} */
+      const assistantsConfig = req.app.locals[req.query.endpoint];
+      const { supportedIds, excludedIds } = assistantsConfig;
+      if (supportedIds?.length) {
+        body.data = body.data.filter((assistant) => supportedIds.includes(assistant.id));
+      } else if (excludedIds?.length) {
+        body.data = body.data.filter((assistant) => !excludedIds.includes(assistant.id));
+      }
+    }
+
+    res.json(body);
+  } catch (error) {
+    logger.error('[/assistants] Error listing assistants', error);
+    res.status(500).json({ message: 'Error listing assistants' });
+  }
+};
+
+/**
+ * Returns a list of the user's assistant documents (metadata saved to database).
+ * @route GET /assistants/documents
+ * @returns {AssistantDocument[]} 200 - success response - application/json
+ */
+const getAssistantDocuments = async (req, res) => {
+  try {
+    res.json(await getAssistants({ user: req.user.id }));
+  } catch (error) {
+    logger.error('[/assistants/documents] Error listing assistant documents', error);
+    res.status(500).json({ error: error.message });
+  }
+};
+
+/**
+ * Uploads and updates an avatar for a specific assistant.
+ * @route POST /avatar/:assistant_id
+ * @param {object} req - Express Request
+ * @param {object} req.params - Request params
+ * @param {string} req.params.assistant_id - The ID of the assistant.
+ * @param {Express.Multer.File} req.file - The avatar image file.
+ * @param {object} req.body - Request body
+ * @param {string} [req.body.metadata] - Optional metadata for the assistant's avatar.
+ * @returns {Object} 200 - success response - application/json
+ */
+const uploadAssistantAvatar = async (req, res) => {
+  try {
+    const { assistant_id } = req.params;
+    if (!assistant_id) {
+      return res.status(400).json({ message: 'Assistant ID is required' });
+    }
+
+    let { metadata: _metadata = '{}' } = req.body;
+    const { openai } = await getOpenAIClient({ req, res });
+
+    const image = await uploadImageBuffer({
+      req,
+      context: FileContext.avatar,
+      metadata: {
+        buffer: req.file.buffer,
+      },
+    });
+
+    try {
+      _metadata = JSON.parse(_metadata);
+    } catch (error) {
+      logger.error('[/avatar/:assistant_id] Error parsing metadata', error);
+      _metadata = {};
+    }
+
+    if (_metadata.avatar && _metadata.avatar_source) {
+      const { deleteFile } = getStrategyFunctions(_metadata.avatar_source);
+      try {
+        await deleteFile(req, { filepath: _metadata.avatar });
+        await deleteFileByFilter({ filepath: _metadata.avatar });
+      } catch (error) {
+        logger.error('[/avatar/:assistant_id] Error deleting old avatar', error);
+      }
+    }
+
+    const metadata = {
+      ..._metadata,
+      avatar: image.filepath,
+      avatar_source: req.app.locals.fileStrategy,
+    };
+
+    const promises = [];
+    promises.push(
+      updateAssistant(
+        { assistant_id },
+        {
+          avatar: {
+            filepath: image.filepath,
+            source: req.app.locals.fileStrategy,
+          },
+          user: req.user.id,
+        },
+      ),
+    );
+    promises.push(openai.beta.assistants.update(assistant_id, { metadata }));
+
+    const resolved = await Promise.all(promises);
+    res.status(201).json(resolved[1]);
+  } catch (error) {
+    const message = 'An error occurred while updating the Assistant Avatar';
+    logger.error(message, error);
+    res.status(500).json({ message });
+  }
+};
+
+module.exports = {
+  createAssistant,
+  retrieveAssistant,
+  patchAssistant,
+  deleteAssistant,
+  listAssistants,
+  getAssistantDocuments,
+  uploadAssistantAvatar,
+};