🅰️ feat: Azure OpenAI Assistants API Support (#1992)

* chore: rename dir from `assistant` to plural

* feat: `assistants` field for azure config, spread options in AppService

* refactor: rename constructAzureURL param for azure as `azureOptions`

* chore: bump openai and bun

* chore(loadDefaultModels): change naming of assistant -> assistants

* feat: load azure settings with currect baseURL for assistants' initializeClient

* refactor: add `assistants` flags to groups and model configs, add mapGroupToAzureConfig

* feat(loadConfigEndpoints): initialize assistants endpoint if azure flag `assistants` is enabled

* feat(AppService): determine assistant models on startup, throw Error if none

* refactor(useDeleteAssistantMutation): send model along with assistant id for delete mutations

* feat: support listing and deleting assistants with azure

* feat: add model query to assistant avatar upload

* feat: add azure support for retrieveRun method

* refactor: update OpenAIClient initialization

* chore: update README

* fix(ci): tests passing

* refactor(uploadOpenAIFile): improve logging and use more efficient REST API method

* refactor(useFileHandling): add model to metadata to target Azure region compatible with current model

* chore(files): add azure naming pattern for valid file id recognition

* fix(assistants): initialize openai with first available assistant model if none provided

* refactor(uploadOpenAIFile): add content type for azure, initialize formdata before azure options

* refactor(sleep): move sleep function out of Runs and into `~/server/utils`

* fix(azureOpenAI/assistants): make sure to only overwrite models with assistant models if `assistants` flag is enabled

* refactor(uploadOpenAIFile): revert to old method

* chore(uploadOpenAIFile): use enum for file purpose

* docs: azureOpenAI update guide with more info, examples

* feat: enable/disable assistant capabilities and specify retrieval models

* refactor: optional chain conditional statement in loadConfigModels.js

* docs: add assistants examples

* chore: update librechat.example.yaml

* docs(azure): update note of file upload behavior in Azure OpenAI Assistants

* chore: update docs and add descriptive message about assistant errors

* fix: prevent message submission with invalid assistant or if files loading

* style: update Landing icon & text when assistant is not selected

* chore: bump librechat-data-provider to 0.4.8

* fix(assistants/azure): assign req.body.model for proper azure init to abort runs

api/server/services/Endpoints/assistants/addTitle.js

@@ -0,0 +1,28 @@
+const { CacheKeys } = require('librechat-data-provider');
+const { saveConvo } = require('~/models/Conversation');
+const getLogStores = require('~/cache/getLogStores');
+const { isEnabled } = require('~/server/utils');
+
+const addTitle = async (req, { text, responseText, conversationId, client }) => {
+  const { TITLE_CONVO = 'true' } = process.env ?? {};
+  if (!isEnabled(TITLE_CONVO)) {
+    return;
+  }
+
+  if (client.options.titleConvo === false) {
+    return;
+  }
+
+  const titleCache = getLogStores(CacheKeys.GEN_TITLE);
+  const key = `${req.user.id}-${conversationId}`;
+
+  const title = await client.titleConvo({ text, conversationId, responseText });
+  await titleCache.set(key, title);
+
+  await saveConvo(req.user.id, {
+    conversationId,
+    title,
+  });
+};
+
+module.exports = addTitle;

api/server/services/Endpoints/assistants/buildOptions.js

@@ -0,0 +1,15 @@
+const buildOptions = (endpoint, parsedBody) => {
+  // eslint-disable-next-line no-unused-vars
+  const { promptPrefix, chatGptLabel, resendImages, imageDetail, ...rest } = parsedBody;
+  const endpointOption = {
+    endpoint,
+    promptPrefix,
+    modelOptions: {
+      ...rest,
+    },
+  };
+
+  return endpointOption;
+};
+
+module.exports = buildOptions;

api/server/services/Endpoints/assistants/index.js

@@ -0,0 +1,73 @@
+const addTitle = require('./addTitle');
+const buildOptions = require('./buildOptions');
+const initializeClient = require('./initializeClient');
+
+/**
+ * Asynchronously lists assistants based on provided query parameters.
+ *
+ * Initializes the client with the current request and response objects and lists assistants
+ * according to the query parameters. This function abstracts the logic for non-Azure paths.
+ *
+ * @async
+ * @param {object} params - The parameters object.
+ * @param {object} params.req - The request object, used for initializing the client.
+ * @param {object} params.res - The response object, used for initializing the client.
+ * @param {object} params.query - The query parameters to list assistants (e.g., limit, order).
+ * @returns {Promise<object>} A promise that resolves to the response from the `openai.beta.assistants.list` method call.
+ */
+const listAssistants = async ({ req, res, query }) => {
+  const { openai } = await initializeClient({ req, res });
+  return openai.beta.assistants.list(query);
+};
+
+/**
+ * Asynchronously lists assistants for Azure configured groups.
+ *
+ * Iterates through Azure configured assistant groups, initializes the client with the current request and response objects,
+ * lists assistants based on the provided query parameters, and merges their data alongside the model information into a single array.
+ *
+ * @async
+ * @param {object} params - The parameters object.
+ * @param {object} params.req - The request object, used for initializing the client and manipulating the request body.
+ * @param {object} params.res - The response object, used for initializing the client.
+ * @param {TAzureConfig} params.azureConfig - The Azure configuration object containing assistantGroups and groupMap.
+ * @param {object} params.query - The query parameters to list assistants (e.g., limit, order).
+ * @returns {Promise<AssistantListResponse>} A promise that resolves to an array of assistant data merged with their respective model information.
+ */
+const listAssistantsForAzure = async ({ req, res, azureConfig = {}, query }) => {
+  const promises = [];
+  const models = [];
+
+  const { groupMap, assistantGroups } = azureConfig;
+
+  for (const groupName of assistantGroups) {
+    const group = groupMap[groupName];
+    req.body.model = Object.keys(group?.models)[0];
+    models.push(req.body.model);
+    promises.push(listAssistants({ req, res, query }));
+  }
+
+  const resolvedQueries = await Promise.all(promises);
+  const data = resolvedQueries.flatMap((res, i) =>
+    res.data.map((assistant) => {
+      const model = models[i];
+      return { ...assistant, model } ?? {};
+    }),
+  );
+
+  return {
+    first_id: data[0]?.id,
+    last_id: data[data.length - 1]?.id,
+    object: 'list',
+    has_more: false,
+    data,
+  };
+};
+
+module.exports = {
+  addTitle,
+  buildOptions,
+  initializeClient,
+  listAssistants,
+  listAssistantsForAzure,
+};

api/server/services/Endpoints/assistants/initializeClient.js

@@ -0,0 +1,148 @@
+const OpenAI = require('openai');
+const { HttpsProxyAgent } = require('https-proxy-agent');
+const {
+  EModelEndpoint,
+  resolveHeaders,
+  mapModelToAzureConfig,
+} = require('librechat-data-provider');
+const {
+  getUserKey,
+  getUserKeyExpiry,
+  checkUserKeyExpiry,
+} = require('~/server/services/UserService');
+const OpenAIClient = require('~/app/clients/OpenAIClient');
+const { isUserProvided } = require('~/server/utils');
+const { constructAzureURL } = require('~/utils');
+
+const initializeClient = async ({ req, res, endpointOption, initAppClient = false }) => {
+  const { PROXY, OPENAI_ORGANIZATION, ASSISTANTS_API_KEY, ASSISTANTS_BASE_URL } = process.env;
+
+  const userProvidesKey = isUserProvided(ASSISTANTS_API_KEY);
+  const userProvidesURL = isUserProvided(ASSISTANTS_BASE_URL);
+
+  let userValues = null;
+  if (userProvidesKey || userProvidesURL) {
+    const expiresAt = await getUserKeyExpiry({
+      userId: req.user.id,
+      name: EModelEndpoint.assistants,
+    });
+    checkUserKeyExpiry(
+      expiresAt,
+      'Your Assistants API key has expired. Please provide your API key again.',
+    );
+    userValues = await getUserKey({ userId: req.user.id, name: EModelEndpoint.assistants });
+    try {
+      userValues = JSON.parse(userValues);
+    } catch (e) {
+      throw new Error(
+        'Invalid JSON provided for Assistants API user values. Please provide them again.',
+      );
+    }
+  }
+
+  let apiKey = userProvidesKey ? userValues.apiKey : ASSISTANTS_API_KEY;
+  let baseURL = userProvidesURL ? userValues.baseURL : ASSISTANTS_BASE_URL;
+
+  const opts = {};
+
+  const clientOptions = {
+    reverseProxyUrl: baseURL ?? null,
+    proxy: PROXY ?? null,
+    req,
+    res,
+    ...endpointOption,
+  };
+
+  /** @type {TAzureConfig | undefined} */
+  const azureConfig = req.app.locals[EModelEndpoint.azureOpenAI];
+
+  /** @type {AzureOptions | undefined} */
+  let azureOptions;
+
+  if (azureConfig && azureConfig.assistants) {
+    const { modelGroupMap, groupMap, assistantModels } = azureConfig;
+    const modelName = req.body.model ?? req.query.model ?? assistantModels[0];
+    const {
+      azureOptions: currentOptions,
+      baseURL: azureBaseURL,
+      headers = {},
+      serverless,
+    } = mapModelToAzureConfig({
+      modelName,
+      modelGroupMap,
+      groupMap,
+    });
+
+    azureOptions = currentOptions;
+
+    baseURL = constructAzureURL({
+      baseURL: azureBaseURL ?? 'https://${INSTANCE_NAME}.openai.azure.com/openai',
+      azureOptions,
+    });
+
+    apiKey = azureOptions.azureOpenAIApiKey;
+    opts.defaultQuery = { 'api-version': azureOptions.azureOpenAIApiVersion };
+    opts.defaultHeaders = resolveHeaders({ ...headers, 'api-key': apiKey });
+    opts.model = azureOptions.azureOpenAIApiDeploymentName;
+
+    if (initAppClient) {
+      clientOptions.titleConvo = azureConfig.titleConvo;
+      clientOptions.titleModel = azureConfig.titleModel;
+      clientOptions.titleMethod = azureConfig.titleMethod ?? 'completion';
+
+      const groupName = modelGroupMap[modelName].group;
+      clientOptions.addParams = azureConfig.groupMap[groupName].addParams;
+      clientOptions.dropParams = azureConfig.groupMap[groupName].dropParams;
+      clientOptions.forcePrompt = azureConfig.groupMap[groupName].forcePrompt;
+
+      clientOptions.reverseProxyUrl = baseURL ?? clientOptions.reverseProxyUrl;
+      clientOptions.headers = opts.defaultHeaders;
+      clientOptions.azure = !serverless && azureOptions;
+    }
+  }
+
+  if (!apiKey) {
+    throw new Error('Assistants API key not provided. Please provide it again.');
+  }
+
+  if (baseURL) {
+    opts.baseURL = baseURL;
+  }
+
+  if (PROXY) {
+    opts.httpAgent = new HttpsProxyAgent(PROXY);
+  }
+
+  if (OPENAI_ORGANIZATION) {
+    opts.organization = OPENAI_ORGANIZATION;
+  }
+
+  /** @type {OpenAIClient} */
+  const openai = new OpenAI({
+    apiKey,
+    ...opts,
+  });
+
+  openai.req = req;
+  openai.res = res;
+
+  if (azureOptions) {
+    openai.locals = { ...(openai.locals ?? {}), azureOptions };
+  }
+
+  if (endpointOption && initAppClient) {
+    const client = new OpenAIClient(apiKey, clientOptions);
+    return {
+      client,
+      openai,
+      openAIApiKey: apiKey,
+    };
+  }
+
+  return {
+    openai,
+    openAIApiKey: apiKey,
+  };
+};
+
+module.exports = initializeClient;

api/server/services/Endpoints/assistants/initializeClient.spec.js

@@ -0,0 +1,99 @@
+// const OpenAI = require('openai');
+const { HttpsProxyAgent } = require('https-proxy-agent');
+const { getUserKey, getUserKeyExpiry } = require('~/server/services/UserService');
+const initializeClient = require('./initializeClient');
+// const { OpenAIClient } = require('~/app');
+
+jest.mock('~/server/services/UserService', () => ({
+  getUserKey: jest.fn(),
+  getUserKeyExpiry: jest.fn(),
+  checkUserKeyExpiry: jest.requireActual('~/server/services/UserService').checkUserKeyExpiry,
+}));
+
+const today = new Date();
+const tenDaysFromToday = new Date(today.setDate(today.getDate() + 10));
+const isoString = tenDaysFromToday.toISOString();
+
+describe('initializeClient', () => {
+  // Set up environment variables
+  const originalEnvironment = process.env;
+  const app = {
+    locals: {},
+  };
+
+  beforeEach(() => {
+    jest.resetModules(); // Clears the cache
+    process.env = { ...originalEnvironment }; // Make a copy
+  });
+
+  afterAll(() => {
+    process.env = originalEnvironment; // Restore original env vars
+  });
+
+  test('initializes OpenAI client with default API key and URL', async () => {
+    process.env.ASSISTANTS_API_KEY = 'default-api-key';
+    process.env.ASSISTANTS_BASE_URL = 'https://default.api.url';
+
+    // Assuming 'isUserProvided' to return false for this test case
+    jest.mock('~/server/utils', () => ({
+      isUserProvided: jest.fn().mockReturnValueOnce(false),
+    }));
+
+    const req = { user: { id: 'user123' }, app };
+    const res = {};
+
+    const { openai, openAIApiKey } = await initializeClient({ req, res });
+    expect(openai.apiKey).toBe('default-api-key');
+    expect(openAIApiKey).toBe('default-api-key');
+    expect(openai.baseURL).toBe('https://default.api.url');
+  });
+
+  test('initializes OpenAI client with user-provided API key and URL', async () => {
+    process.env.ASSISTANTS_API_KEY = 'user_provided';
+    process.env.ASSISTANTS_BASE_URL = 'user_provided';
+
+    getUserKey.mockResolvedValue(
+      JSON.stringify({ apiKey: 'user-api-key', baseURL: 'https://user.api.url' }),
+    );
+    getUserKeyExpiry.mockResolvedValue(isoString);
+
+    const req = { user: { id: 'user123' }, app };
+    const res = {};
+
+    const { openai, openAIApiKey } = await initializeClient({ req, res });
+    expect(openAIApiKey).toBe('user-api-key');
+    expect(openai.apiKey).toBe('user-api-key');
+    expect(openai.baseURL).toBe('https://user.api.url');
+  });
+
+  test('throws error for invalid JSON in user-provided values', async () => {
+    process.env.ASSISTANTS_API_KEY = 'user_provided';
+    getUserKey.mockResolvedValue('invalid-json');
+    getUserKeyExpiry.mockResolvedValue(isoString);
+
+    const req = { user: { id: 'user123' } };
+    const res = {};
+
+    await expect(initializeClient({ req, res })).rejects.toThrow(/Invalid JSON/);
+  });
+
+  test('throws error if API key is not provided', async () => {
+    delete process.env.ASSISTANTS_API_KEY; // Simulate missing API key
+
+    const req = { user: { id: 'user123' }, app };
+    const res = {};
+
+    await expect(initializeClient({ req, res })).rejects.toThrow(/Assistants API key not/);
+  });
+
+  test('initializes OpenAI client with proxy configuration', async () => {
+    process.env.ASSISTANTS_API_KEY = 'test-key';
+    process.env.PROXY = 'http://proxy.server';
+
+    const req = { user: { id: 'user123' }, app };
+    const res = {};
+
+    const { openai } = await initializeClient({ req, res });
+    expect(openai.httpAgent).toBeInstanceOf(HttpsProxyAgent);
+  });
+});