🗝️ fix: Resolve User-Provided API Key in Agents API Flow (#12390)

* fix: resolve user-provided API key in Agents API flow

When the Agents API calls initializeCustom, req.body follows the
OpenAI-compatible format (model, messages, stream) and does not
include the `key` field that the regular UI chat flow sends.

Previously, getUserKeyValues was only called when expiresAt
(from req.body.key) was truthy, causing the Agents API to always
fail with NO_USER_KEY for custom endpoints using
apiKey: "user_provided".

This fix decouples the key fetch from the expiry check:
- If expiresAt is present (UI flow): checks expiry AND fetches key
- If expiresAt is absent (Agents API): skips expiry check, still
  fetches key

Fixes #12389

* address review feedback from @danny-avila

- Flatten nested if into two sibling statements (never-nesting style)
- Add inline comment explaining why expiresAt may be absent
- Add negative assertion: checkUserKeyExpiry NOT called in Agents API flow
- Add regression test: expired key still throws EXPIRED_USER_KEY
- Add test for userProvidesURL=true variant in Agents API flow
- Remove unnecessary undefined cast in test params

* fix: CI failure + address remaining review items

- Fix mock leak: use mockImplementationOnce instead of mockImplementation
  to prevent checkUserKeyExpiry throwing impl from leaking into SSRF tests
  (clearAllMocks does not reset implementations)
- Use ErrorTypes.EXPIRED_USER_KEY constant instead of raw string
- Add test: system-defined key/URL should NOT call getUserKeyValues

packages/api/src/endpoints/custom/initialize.spec.ts

@@ -1,4 +1,4 @@
-import { AuthType } from 'librechat-data-provider';
+import { AuthType, ErrorTypes } from 'librechat-data-provider';
 import type { BaseInitializeParams } from '~/types';
 
 const mockValidateEndpointURL = jest.fn();
@@ -68,6 +68,97 @@ function createParams(overrides: {
   };
 }
 
+describe('initializeCustom – Agents API user key resolution', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  it('should fetch user key even when expiresAt is not in request body (Agents API flow)', async () => {
+    const { checkUserKeyExpiry } = jest.requireMock('~/utils');
+    const params = createParams({
+      apiKey: AuthType.USER_PROVIDED,
+      baseURL: 'https://api.example.com/v1',
+      userApiKey: 'sk-user-key',
+    });
+    // Simulate Agents API request body (no `key` field)
+    params.req.body = { model: 'agent_123', messages: [] };
+
+    await initializeCustom(params);
+
+    expect(params.db.getUserKeyValues).toHaveBeenCalledWith({
+      userId: 'user-1',
+      name: 'test-custom',
+    });
+    expect(checkUserKeyExpiry).not.toHaveBeenCalled();
+    expect(mockGetOpenAIConfig).toHaveBeenCalledWith(
+      'sk-user-key',
+      expect.any(Object),
+      'test-custom',
+    );
+  });
+
+  it('should fetch user key for user-provided URL without expiresAt (Agents API flow)', async () => {
+    const { checkUserKeyExpiry } = jest.requireMock('~/utils');
+    const params = createParams({
+      apiKey: 'sk-system-key',
+      baseURL: AuthType.USER_PROVIDED,
+      userBaseURL: 'https://user-api.example.com/v1',
+    });
+    params.req.body = { model: 'agent_123', messages: [] };
+
+    await initializeCustom(params);
+
+    expect(params.db.getUserKeyValues).toHaveBeenCalledWith({
+      userId: 'user-1',
+      name: 'test-custom',
+    });
+    expect(checkUserKeyExpiry).not.toHaveBeenCalled();
+  });
+
+  it('should still check key expiry when expiresAt is provided (UI flow)', async () => {
+    const { checkUserKeyExpiry } = jest.requireMock('~/utils');
+    const params = createParams({
+      apiKey: AuthType.USER_PROVIDED,
+      baseURL: 'https://api.example.com/v1',
+      userApiKey: 'sk-user-key',
+      expiresAt: '2099-01-01',
+    });
+
+    await initializeCustom(params);
+
+    expect(checkUserKeyExpiry).toHaveBeenCalledWith('2099-01-01', 'test-custom');
+    expect(params.db.getUserKeyValues).toHaveBeenCalled();
+  });
+
+  it('should throw EXPIRED_USER_KEY when expiresAt is expired', async () => {
+    const { checkUserKeyExpiry } = jest.requireMock('~/utils');
+    checkUserKeyExpiry.mockImplementationOnce(() => {
+      throw new Error(JSON.stringify({ type: ErrorTypes.EXPIRED_USER_KEY }));
+    });
+
+    const params = createParams({
+      apiKey: AuthType.USER_PROVIDED,
+      baseURL: 'https://api.example.com/v1',
+      userApiKey: 'sk-user-key',
+      expiresAt: '2020-01-01',
+    });
+
+    await expect(initializeCustom(params)).rejects.toThrow(ErrorTypes.EXPIRED_USER_KEY);
+    expect(params.db.getUserKeyValues).not.toHaveBeenCalled();
+  });
+
+  it('should NOT call getUserKeyValues when key and URL are system-defined', async () => {
+    const params = createParams({
+      apiKey: 'sk-system-key',
+      baseURL: 'https://api.provider.com/v1',
+    });
+
+    await initializeCustom(params);
+
+    expect(params.db.getUserKeyValues).not.toHaveBeenCalled();
+  });
+});
+
 describe('initializeCustom – SSRF guard wiring', () => {
   beforeEach(() => {
     jest.clearAllMocks();

packages/api/src/endpoints/custom/initialize.ts

@@ -91,9 +91,15 @@ export async function initializeCustom({
   const userProvidesKey = isUserProvided(CUSTOM_API_KEY);
   const userProvidesURL = isUserProvided(CUSTOM_BASE_URL);
 
-  let userValues = null;
+  // Expiry is only checked when present: the Agents API sends an OpenAI-compatible
+  // request body that does not include `key` (the expiry timestamp), so expiresAt
+  // will be undefined in that flow. The key is still fetched regardless.
   if (expiresAt && (userProvidesKey || userProvidesURL)) {
     checkUserKeyExpiry(expiresAt, endpoint);
+  }
+
+  let userValues = null;
+  if (userProvidesKey || userProvidesURL) {
     userValues = await db.getUserKeyValues({ userId: req.user?.id ?? '', name: endpoint });
   }