🔁 fix: Pass recursionLimit to OpenAI-Compatible Agents API Endpoint (#12510)

* fix: pass recursionLimit to processStream in OpenAI-compatible agents API

The OpenAI-compatible endpoint never passed recursionLimit to LangGraph's
processStream(), silently capping all API-based agent calls at the default
25 steps. Mirror the 3-step cascade already used by the UI path (client.js):
yaml config default → per-agent DB override → max cap.

* refactor: extract resolveRecursionLimit into shared utility

Extract the 3-step recursion limit cascade into a shared
resolveRecursionLimit() function in @librechat/api. Both openai.js and
client.js now call this single source of truth.

Also fixes falsy-guard edge cases where recursion_limit=0 or
maxRecursionLimit=0 would silently misbehave, by using explicit
typeof + positive checks.

Includes unit tests covering all cascade branches and edge cases.

* refactor: use resolveRecursionLimit in openai.js and client.js

Replace duplicated cascade logic in both controllers with the shared
resolveRecursionLimit() utility from @librechat/api.

In openai.js: hoist agentsEConfig to avoid double property walk,
remove displaced comment, add integration test assertions.

In client.js: remove inline cascade that was overriding config
after initial assignment.

* fix: hoist processStream mock for test accessibility

The processStream mock was created inline inside mockResolvedValue,
making it inaccessible via createRun.mock.results (which returns
the Promise, not the resolved value). Hoist it to a module-level
variable so tests can assert on it directly.

* test: improve test isolation and boundary coverage

Use mockReturnValueOnce instead of mockReturnValue to prevent mock
leaking across test boundaries. Add boundary tests for downward
agent override and exact-match maxRecursionLimit.

packages/api/src/agents/config.spec.ts

@@ -0,0 +1,62 @@
+import type { TAgentsEndpoint } from 'librechat-data-provider';
+import { resolveRecursionLimit } from './config';
+
+describe('resolveRecursionLimit', () => {
+  it('returns default 50 when no config or agent provided', () => {
+    expect(resolveRecursionLimit(undefined, undefined)).toBe(50);
+  });
+
+  it('returns default 50 when config has no recursionLimit', () => {
+    expect(resolveRecursionLimit({} as TAgentsEndpoint, {})).toBe(50);
+  });
+
+  it('uses yaml recursionLimit when set', () => {
+    const config = { recursionLimit: 100 } as TAgentsEndpoint;
+    expect(resolveRecursionLimit(config, {})).toBe(100);
+  });
+
+  it('overrides with agent.recursion_limit when set', () => {
+    const config = { recursionLimit: 100 } as TAgentsEndpoint;
+    expect(resolveRecursionLimit(config, { recursion_limit: 200 })).toBe(200);
+  });
+
+  it('caps at maxRecursionLimit', () => {
+    const config = { recursionLimit: 100, maxRecursionLimit: 150 } as TAgentsEndpoint;
+    expect(resolveRecursionLimit(config, { recursion_limit: 200 })).toBe(150);
+  });
+
+  it('caps yaml default at maxRecursionLimit', () => {
+    const config = { recursionLimit: 200, maxRecursionLimit: 100 } as TAgentsEndpoint;
+    expect(resolveRecursionLimit(config, {})).toBe(100);
+  });
+
+  it('ignores agent.recursion_limit of 0', () => {
+    const config = { recursionLimit: 100 } as TAgentsEndpoint;
+    expect(resolveRecursionLimit(config, { recursion_limit: 0 })).toBe(100);
+  });
+
+  it('ignores negative agent.recursion_limit', () => {
+    const config = { recursionLimit: 100 } as TAgentsEndpoint;
+    expect(resolveRecursionLimit(config, { recursion_limit: -5 })).toBe(100);
+  });
+
+  it('ignores maxRecursionLimit of 0', () => {
+    const config = { recursionLimit: 100, maxRecursionLimit: 0 } as TAgentsEndpoint;
+    expect(resolveRecursionLimit(config, { recursion_limit: 200 })).toBe(200);
+  });
+
+  it('does not cap when recursionLimit is within maxRecursionLimit', () => {
+    const config = { recursionLimit: 50, maxRecursionLimit: 200 } as TAgentsEndpoint;
+    expect(resolveRecursionLimit(config, { recursion_limit: 150 })).toBe(150);
+  });
+
+  it('allows agent to override downward below yaml default', () => {
+    const config = { recursionLimit: 100 } as TAgentsEndpoint;
+    expect(resolveRecursionLimit(config, { recursion_limit: 30 })).toBe(30);
+  });
+
+  it('does not cap when agent.recursion_limit equals maxRecursionLimit', () => {
+    const config = { recursionLimit: 50, maxRecursionLimit: 150 } as TAgentsEndpoint;
+    expect(resolveRecursionLimit(config, { recursion_limit: 150 })).toBe(150);
+  });
+});

packages/api/src/agents/config.ts

@@ -0,0 +1,30 @@
+import type { TAgentsEndpoint } from 'librechat-data-provider';
+
+const DEFAULT_RECURSION_LIMIT = 50;
+
+/**
+ * Resolves the effective recursion limit for an agent run via a 3-step cascade:
+ * 1. YAML endpoint config default (falls back to 50)
+ * 2. Per-agent DB override (if set and positive)
+ * 3. Global max cap from YAML (if set and positive)
+ */
+export function resolveRecursionLimit(
+  agentsEConfig: TAgentsEndpoint | undefined,
+  agent: { recursion_limit?: number } | undefined,
+): number {
+  let limit = agentsEConfig?.recursionLimit ?? DEFAULT_RECURSION_LIMIT;
+
+  if (typeof agent?.recursion_limit === 'number' && agent.recursion_limit > 0) {
+    limit = agent.recursion_limit;
+  }
+
+  if (
+    typeof agentsEConfig?.maxRecursionLimit === 'number' &&
+    agentsEConfig.maxRecursionLimit > 0 &&
+    limit > agentsEConfig.maxRecursionLimit
+  ) {
+    limit = agentsEConfig.maxRecursionLimit;
+  }
+
+  return limit;
+}

packages/api/src/agents/index.ts

@@ -1,6 +1,7 @@
 export * from './avatars';
 export * from './chain';
 export * from './client';
+export * from './config';
 export * from './context';
 export * from './edges';
 export * from './handlers';