mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-01-03 09:08:52 +01:00
🛜 refactor: Streamline App Config Usage (#9234)
* WIP: app.locals refactoring
WIP: appConfig
fix: update memory configuration retrieval to use getAppConfig based on user role
fix: update comment for AppConfig interface to clarify purpose
🏷️ refactor: Update tests to use getAppConfig for endpoint configurations
ci: Update AppService tests to initialize app config instead of app.locals
ci: Integrate getAppConfig into remaining tests
refactor: Update multer storage destination to use promise-based getAppConfig and improve error handling in tests
refactor: Rename initializeAppConfig to setAppConfig and update related tests
ci: Mock getAppConfig in various tests to provide default configurations
refactor: Update convertMCPToolsToPlugins to use mcpManager for server configuration and adjust related tests
chore: rename `Config/getAppConfig` -> `Config/app`
fix: streamline OpenAI image tools configuration by removing direct appConfig dependency and using function parameters
chore: correct parameter documentation for imageOutputType in ToolService.js
refactor: remove `getCustomConfig` dependency in config route
refactor: update domain validation to use appConfig for allowed domains
refactor: use appConfig registration property
chore: remove app parameter from AppService invocation
refactor: update AppConfig interface to correct registration and turnstile configurations
refactor: remove getCustomConfig dependency and use getAppConfig in PluginController, multer, and MCP services
refactor: replace getCustomConfig with getAppConfig in STTService, TTSService, and related files
refactor: replace getCustomConfig with getAppConfig in Conversation and Message models, update tempChatRetention functions to use AppConfig type
refactor: update getAppConfig calls in Conversation and Message models to include user role for temporary chat expiration
ci: update related tests
refactor: update getAppConfig call in getCustomConfigSpeech to include user role
fix: update appConfig usage to access allowedDomains from actions instead of registration
refactor: enhance AppConfig to include fileStrategies and update related file strategy logic
refactor: update imports to use normalizeEndpointName from @librechat/api and remove redundant definitions
chore: remove deprecated unused RunManager
refactor: get balance config primarily from appConfig
refactor: remove customConfig dependency for appConfig and streamline loadConfigModels logic
refactor: remove getCustomConfig usage and use app config in file citations
refactor: consolidate endpoint loading logic into loadEndpoints function
refactor: update appConfig access to use endpoints structure across various services
refactor: implement custom endpoints configuration and streamline endpoint loading logic
refactor: update getAppConfig call to include user role parameter
refactor: streamline endpoint configuration and enhance appConfig usage across services
refactor: replace getMCPAuthMap with getUserMCPAuthMap and remove unused getCustomConfig file
refactor: add type annotation for loadedEndpoints in loadEndpoints function
refactor: move /services/Files/images/parse to TS API
chore: add missing FILE_CITATIONS permission to IRole interface
refactor: restructure toolkits to TS API
refactor: separate manifest logic into its own module
refactor: consolidate tool loading logic into a new tools module for startup logic
refactor: move interface config logic to TS API
refactor: migrate checkEmailConfig to TypeScript and update imports
refactor: add FunctionTool interface and availableTools to AppConfig
refactor: decouple caching and DB operations from AppService, make part of consolidated `getAppConfig`
WIP: fix tests
* fix: rebase conflicts
* refactor: remove app.locals references
* refactor: replace getBalanceConfig with getAppConfig in various strategies and middleware
* refactor: replace appConfig?.balance with getBalanceConfig in various controllers and clients
* test: add balance configuration to titleConvo method in AgentClient tests
* chore: remove unused `openai-chat-tokens` package
* chore: remove unused imports in initializeMCPs.js
* refactor: update balance configuration to use getAppConfig instead of getBalanceConfig
* refactor: integrate configMiddleware for centralized configuration handling
* refactor: optimize email domain validation by removing unnecessary async calls
* refactor: simplify multer storage configuration by removing async calls
* refactor: reorder imports for better readability in user.js
* refactor: replace getAppConfig calls with req.config for improved performance
* chore: replace getAppConfig calls with req.config in tests for centralized configuration handling
* chore: remove unused override config
* refactor: add configMiddleware to endpoint route and replace getAppConfig with req.config
* chore: remove customConfig parameter from TTSService constructor
* refactor: pass appConfig from request to processFileCitations for improved configuration handling
* refactor: remove configMiddleware from endpoint route and retrieve appConfig directly in getEndpointsConfig if not in `req.config`
* test: add mockAppConfig to processFileCitations tests for improved configuration handling
* fix: pass req.config to hasCustomUserVars and call without await after synchronous refactor
* fix: type safety in useExportConversation
* refactor: retrieve appConfig using getAppConfig in PluginController and remove configMiddleware from plugins route, to avoid always retrieving when plugins are cached
* chore: change `MongoUser` typedef to `IUser`
* fix: Add `user` and `config` fields to ServerRequest and update JSDoc type annotations from Express.Request to ServerRequest
* fix: remove unused setAppConfig mock from Server configuration tests
This commit is contained in:
Danny Avila
GitHub
parent
e1ad235f17
commit
9a210971f5
210 changed files with 4102 additions and 3465 deletions
|
|
@ -1,5 +1,7 @@
|
|||
const crypto = require('crypto');
|
||||
const fetch = require('node-fetch');
|
||||
const { logger } = require('@librechat/data-schemas');
|
||||
const { getBalanceConfig } = require('@librechat/api');
|
||||
const {
|
||||
supportsBalanceCheck,
|
||||
isAgentsEndpoint,
|
||||
|
|
@ -15,7 +17,6 @@ const { checkBalance } = require('~/models/balanceMethods');
|
|||
const { truncateToolCallOutputs } = require('./prompts');
|
||||
const { getFiles } = require('~/models/File');
|
||||
const TextStream = require('./TextStream');
|
||||
const { logger } = require('~/config');
|
||||
|
||||
class BaseClient {
|
||||
constructor(apiKey, options = {}) {
|
||||
|
|
@ -112,13 +113,15 @@ class BaseClient {
|
|||
* If a correction to the token usage is needed, the method should return an object with the corrected token counts.
|
||||
* Should only be used if `recordCollectedUsage` was not used instead.
|
||||
* @param {string} [model]
|
||||
* @param {AppConfig['balance']} [balance]
|
||||
* @param {number} promptTokens
|
||||
* @param {number} completionTokens
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async recordTokenUsage({ model, promptTokens, completionTokens }) {
|
||||
async recordTokenUsage({ model, balance, promptTokens, completionTokens }) {
|
||||
logger.debug('[BaseClient] `recordTokenUsage` not implemented.', {
|
||||
model,
|
||||
balance,
|
||||
promptTokens,
|
||||
completionTokens,
|
||||
});
|
||||
|
|
@ -571,6 +574,7 @@ class BaseClient {
|
|||
}
|
||||
|
||||
async sendMessage(message, opts = {}) {
|
||||
const appConfig = this.options.req?.config;
|
||||
/** @type {Promise<TMessage>} */
|
||||
let userMessagePromise;
|
||||
const { user, head, isEdited, conversationId, responseMessageId, saveOptions, userMessage } =
|
||||
|
|
@ -657,9 +661,9 @@ class BaseClient {
|
|||
}
|
||||
}
|
||||
|
||||
const balance = this.options.req?.app?.locals?.balance;
|
||||
const balanceConfig = getBalanceConfig(appConfig);
|
||||
if (
|
||||
balance?.enabled &&
|
||||
balanceConfig?.enabled &&
|
||||
supportsBalanceCheck[this.options.endpointType ?? this.options.endpoint]
|
||||
) {
|
||||
await checkBalance({
|
||||
|
|
@ -758,6 +762,7 @@ class BaseClient {
|
|||
usage,
|
||||
promptTokens,
|
||||
completionTokens,
|
||||
balance: balanceConfig,
|
||||
model: responseMessage.model,
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -36,11 +36,11 @@ const { encodeAndFormat } = require('~/server/services/Files/images/encode');
|
|||
const { addSpaceIfNeeded, sleep } = require('~/server/utils');
|
||||
const { spendTokens } = require('~/models/spendTokens');
|
||||
const { handleOpenAIErrors } = require('./tools/util');
|
||||
const { createLLM, RunManager } = require('./llm');
|
||||
const { summaryBuffer } = require('./memory');
|
||||
const { runTitleChain } = require('./chains');
|
||||
const { tokenSplit } = require('./document');
|
||||
const BaseClient = require('./BaseClient');
|
||||
const { createLLM } = require('./llm');
|
||||
const { logger } = require('~/config');
|
||||
|
||||
class OpenAIClient extends BaseClient {
|
||||
|
|
@ -618,10 +618,6 @@ class OpenAIClient extends BaseClient {
|
|||
temperature = 0.2,
|
||||
max_tokens,
|
||||
streaming,
|
||||
context,
|
||||
tokenBuffer,
|
||||
initialMessageCount,
|
||||
conversationId,
|
||||
}) {
|
||||
const modelOptions = {
|
||||
modelName: modelName ?? model,
|
||||
|
|
@ -666,22 +662,12 @@ class OpenAIClient extends BaseClient {
|
|||
configOptions.httpsAgent = new HttpsProxyAgent(this.options.proxy);
|
||||
}
|
||||
|
||||
const { req, res, debug } = this.options;
|
||||
const runManager = new RunManager({ req, res, debug, abortController: this.abortController });
|
||||
this.runManager = runManager;
|
||||
|
||||
const llm = createLLM({
|
||||
modelOptions,
|
||||
configOptions,
|
||||
openAIApiKey: this.apiKey,
|
||||
azure: this.azure,
|
||||
streaming,
|
||||
callbacks: runManager.createCallbacks({
|
||||
context,
|
||||
tokenBuffer,
|
||||
conversationId: this.conversationId ?? conversationId,
|
||||
initialMessageCount,
|
||||
}),
|
||||
});
|
||||
|
||||
return llm;
|
||||
|
|
@ -702,6 +688,7 @@ class OpenAIClient extends BaseClient {
|
|||
* In case of failure, it will return the default title, "New Chat".
|
||||
*/
|
||||
async titleConvo({ text, conversationId, responseText = '' }) {
|
||||
const appConfig = this.options.req?.config;
|
||||
this.conversationId = conversationId;
|
||||
|
||||
if (this.options.attachments) {
|
||||
|
|
@ -730,8 +717,7 @@ class OpenAIClient extends BaseClient {
|
|||
max_tokens: 16,
|
||||
};
|
||||
|
||||
/** @type {TAzureConfig | undefined} */
|
||||
const azureConfig = this.options?.req?.app?.locals?.[EModelEndpoint.azureOpenAI];
|
||||
const azureConfig = appConfig?.endpoints?.[EModelEndpoint.azureOpenAI];
|
||||
|
||||
const resetTitleOptions = !!(
|
||||
(this.azure && azureConfig) ||
|
||||
|
|
@ -1120,6 +1106,7 @@ ${convo}
|
|||
}
|
||||
|
||||
async chatCompletion({ payload, onProgress, abortController = null }) {
|
||||
const appConfig = this.options.req?.config;
|
||||
let error = null;
|
||||
let intermediateReply = [];
|
||||
const errorCallback = (err) => (error = err);
|
||||
|
|
@ -1165,8 +1152,7 @@ ${convo}
|
|||
opts.fetchOptions.agent = new HttpsProxyAgent(this.options.proxy);
|
||||
}
|
||||
|
||||
/** @type {TAzureConfig | undefined} */
|
||||
const azureConfig = this.options?.req?.app?.locals?.[EModelEndpoint.azureOpenAI];
|
||||
const azureConfig = appConfig?.endpoints?.[EModelEndpoint.azureOpenAI];
|
||||
|
||||
if (
|
||||
(this.azure && this.isVisionModel && azureConfig) ||
|
||||
|
|
|
|||
|
|
@ -1,95 +0,0 @@
|
|||
const { promptTokensEstimate } = require('openai-chat-tokens');
|
||||
const { EModelEndpoint, supportsBalanceCheck } = require('librechat-data-provider');
|
||||
const { formatFromLangChain } = require('~/app/clients/prompts');
|
||||
const { getBalanceConfig } = require('~/server/services/Config');
|
||||
const { checkBalance } = require('~/models/balanceMethods');
|
||||
const { logger } = require('~/config');
|
||||
|
||||
const createStartHandler = ({
|
||||
context,
|
||||
conversationId,
|
||||
tokenBuffer = 0,
|
||||
initialMessageCount,
|
||||
manager,
|
||||
}) => {
|
||||
return async (_llm, _messages, runId, parentRunId, extraParams) => {
|
||||
const { invocation_params } = extraParams;
|
||||
const { model, functions, function_call } = invocation_params;
|
||||
const messages = _messages[0].map(formatFromLangChain);
|
||||
|
||||
logger.debug(`[createStartHandler] handleChatModelStart: ${context}`, {
|
||||
model,
|
||||
function_call,
|
||||
});
|
||||
|
||||
if (context !== 'title') {
|
||||
logger.debug(`[createStartHandler] handleChatModelStart: ${context}`, {
|
||||
functions,
|
||||
});
|
||||
}
|
||||
|
||||
const payload = { messages };
|
||||
let prelimPromptTokens = 1;
|
||||
|
||||
if (functions) {
|
||||
payload.functions = functions;
|
||||
prelimPromptTokens += 2;
|
||||
}
|
||||
|
||||
if (function_call) {
|
||||
payload.function_call = function_call;
|
||||
prelimPromptTokens -= 5;
|
||||
}
|
||||
|
||||
prelimPromptTokens += promptTokensEstimate(payload);
|
||||
logger.debug('[createStartHandler]', {
|
||||
prelimPromptTokens,
|
||||
tokenBuffer,
|
||||
});
|
||||
prelimPromptTokens += tokenBuffer;
|
||||
|
||||
try {
|
||||
const balance = await getBalanceConfig();
|
||||
if (balance?.enabled && supportsBalanceCheck[EModelEndpoint.openAI]) {
|
||||
const generations =
|
||||
initialMessageCount && messages.length > initialMessageCount
|
||||
? messages.slice(initialMessageCount)
|
||||
: null;
|
||||
await checkBalance({
|
||||
req: manager.req,
|
||||
res: manager.res,
|
||||
txData: {
|
||||
user: manager.user,
|
||||
tokenType: 'prompt',
|
||||
amount: prelimPromptTokens,
|
||||
debug: manager.debug,
|
||||
generations,
|
||||
model,
|
||||
endpoint: EModelEndpoint.openAI,
|
||||
},
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
logger.error(`[createStartHandler][${context}] checkBalance error`, err);
|
||||
manager.abortController.abort();
|
||||
if (context === 'summary' || context === 'plugins') {
|
||||
manager.addRun(runId, { conversationId, error: err.message });
|
||||
throw new Error(err);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
manager.addRun(runId, {
|
||||
model,
|
||||
messages,
|
||||
functions,
|
||||
function_call,
|
||||
runId,
|
||||
parentRunId,
|
||||
conversationId,
|
||||
prelimPromptTokens,
|
||||
});
|
||||
};
|
||||
};
|
||||
|
||||
module.exports = createStartHandler;
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
const createStartHandler = require('./createStartHandler');
|
||||
|
||||
module.exports = {
|
||||
createStartHandler,
|
||||
};
|
||||
|
|
@ -1,105 +0,0 @@
|
|||
const { createStartHandler } = require('~/app/clients/callbacks');
|
||||
const { spendTokens } = require('~/models/spendTokens');
|
||||
const { logger } = require('~/config');
|
||||
|
||||
class RunManager {
|
||||
constructor(fields) {
|
||||
const { req, res, abortController, debug } = fields;
|
||||
this.abortController = abortController;
|
||||
this.user = req.user.id;
|
||||
this.req = req;
|
||||
this.res = res;
|
||||
this.debug = debug;
|
||||
this.runs = new Map();
|
||||
this.convos = new Map();
|
||||
}
|
||||
|
||||
addRun(runId, runData) {
|
||||
if (!this.runs.has(runId)) {
|
||||
this.runs.set(runId, runData);
|
||||
if (runData.conversationId) {
|
||||
this.convos.set(runData.conversationId, runId);
|
||||
}
|
||||
return runData;
|
||||
} else {
|
||||
const existingData = this.runs.get(runId);
|
||||
const update = { ...existingData, ...runData };
|
||||
this.runs.set(runId, update);
|
||||
if (update.conversationId) {
|
||||
this.convos.set(update.conversationId, runId);
|
||||
}
|
||||
return update;
|
||||
}
|
||||
}
|
||||
|
||||
removeRun(runId) {
|
||||
if (this.runs.has(runId)) {
|
||||
this.runs.delete(runId);
|
||||
} else {
|
||||
logger.error(`[api/app/clients/llm/RunManager] Run with ID ${runId} does not exist.`);
|
||||
}
|
||||
}
|
||||
|
||||
getAllRuns() {
|
||||
return Array.from(this.runs.values());
|
||||
}
|
||||
|
||||
getRunById(runId) {
|
||||
return this.runs.get(runId);
|
||||
}
|
||||
|
||||
getRunByConversationId(conversationId) {
|
||||
const runId = this.convos.get(conversationId);
|
||||
return { run: this.runs.get(runId), runId };
|
||||
}
|
||||
|
||||
createCallbacks(metadata) {
|
||||
return [
|
||||
{
|
||||
handleChatModelStart: createStartHandler({ ...metadata, manager: this }),
|
||||
handleLLMEnd: async (output, runId, _parentRunId) => {
|
||||
const { llmOutput, ..._output } = output;
|
||||
logger.debug(`[RunManager] handleLLMEnd: ${JSON.stringify(metadata)}`, {
|
||||
runId,
|
||||
_parentRunId,
|
||||
llmOutput,
|
||||
});
|
||||
|
||||
if (metadata.context !== 'title') {
|
||||
logger.debug('[RunManager] handleLLMEnd:', {
|
||||
output: _output,
|
||||
});
|
||||
}
|
||||
|
||||
const { tokenUsage } = output.llmOutput;
|
||||
const run = this.getRunById(runId);
|
||||
this.removeRun(runId);
|
||||
|
||||
const txData = {
|
||||
user: this.user,
|
||||
model: run?.model ?? 'gpt-3.5-turbo',
|
||||
...metadata,
|
||||
};
|
||||
|
||||
await spendTokens(txData, tokenUsage);
|
||||
},
|
||||
handleLLMError: async (err) => {
|
||||
logger.error(`[RunManager] handleLLMError: ${JSON.stringify(metadata)}`, err);
|
||||
if (metadata.context === 'title') {
|
||||
return;
|
||||
} else if (metadata.context === 'plugins') {
|
||||
throw new Error(err);
|
||||
}
|
||||
const { conversationId } = metadata;
|
||||
const { run } = this.getRunByConversationId(conversationId);
|
||||
if (run && run.error) {
|
||||
const { error } = run;
|
||||
throw new Error(error);
|
||||
}
|
||||
},
|
||||
},
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = RunManager;
|
||||
|
|
@ -1,9 +1,7 @@
|
|||
const createLLM = require('./createLLM');
|
||||
const RunManager = require('./RunManager');
|
||||
const createCoherePayload = require('./createCoherePayload');
|
||||
|
||||
module.exports = {
|
||||
createLLM,
|
||||
RunManager,
|
||||
createCoherePayload,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -2,6 +2,14 @@ const { Constants } = require('librechat-data-provider');
|
|||
const { initializeFakeClient } = require('./FakeClient');
|
||||
|
||||
jest.mock('~/db/connect');
|
||||
jest.mock('~/server/services/Config', () => ({
|
||||
getAppConfig: jest.fn().mockResolvedValue({
|
||||
// Default app config for tests
|
||||
paths: { uploads: '/tmp' },
|
||||
fileStrategy: 'local',
|
||||
memory: { disabled: false },
|
||||
}),
|
||||
}));
|
||||
jest.mock('~/models', () => ({
|
||||
User: jest.fn(),
|
||||
Key: jest.fn(),
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
const availableTools = require('./manifest.json');
|
||||
const manifest = require('./manifest');
|
||||
|
||||
// Structured Tools
|
||||
const DALLE3 = require('./structured/DALLE3');
|
||||
|
|
@ -13,23 +13,8 @@ const TraversaalSearch = require('./structured/TraversaalSearch');
|
|||
const createOpenAIImageTools = require('./structured/OpenAIImageTools');
|
||||
const TavilySearchResults = require('./structured/TavilySearchResults');
|
||||
|
||||
/** @type {Record<string, TPlugin | undefined>} */
|
||||
const manifestToolMap = {};
|
||||
|
||||
/** @type {Array<TPlugin>} */
|
||||
const toolkits = [];
|
||||
|
||||
availableTools.forEach((tool) => {
|
||||
manifestToolMap[tool.pluginKey] = tool;
|
||||
if (tool.toolkit === true) {
|
||||
toolkits.push(tool);
|
||||
}
|
||||
});
|
||||
|
||||
module.exports = {
|
||||
toolkits,
|
||||
availableTools,
|
||||
manifestToolMap,
|
||||
...manifest,
|
||||
// Structured Tools
|
||||
DALLE3,
|
||||
FluxAPI,
|
||||
|
|
|
|||
20
api/app/clients/tools/manifest.js
Normal file
20
api/app/clients/tools/manifest.js
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
const availableTools = require('./manifest.json');
|
||||
|
||||
/** @type {Record<string, TPlugin | undefined>} */
|
||||
const manifestToolMap = {};
|
||||
|
||||
/** @type {Array<TPlugin>} */
|
||||
const toolkits = [];
|
||||
|
||||
availableTools.forEach((tool) => {
|
||||
manifestToolMap[tool.pluginKey] = tool;
|
||||
if (tool.toolkit === true) {
|
||||
toolkits.push(tool);
|
||||
}
|
||||
});
|
||||
|
||||
module.exports = {
|
||||
toolkits,
|
||||
availableTools,
|
||||
manifestToolMap,
|
||||
};
|
||||
|
|
@ -5,10 +5,10 @@ const fetch = require('node-fetch');
|
|||
const { v4: uuidv4 } = require('uuid');
|
||||
const { ProxyAgent } = require('undici');
|
||||
const { Tool } = require('@langchain/core/tools');
|
||||
const { logger } = require('@librechat/data-schemas');
|
||||
const { getImageBasename } = require('@librechat/api');
|
||||
const { FileContext, ContentTypes } = require('librechat-data-provider');
|
||||
const { getImageBasename } = require('~/server/services/Files/images');
|
||||
const extractBaseURL = require('~/utils/extractBaseURL');
|
||||
const logger = require('~/config/winston');
|
||||
|
||||
const displayMessage =
|
||||
"DALL-E displayed an image. All generated images are already plainly visible, so don't repeat the descriptions in detail. Do not list download links as they are available in the UI already. The user may download the images by clicking on them, but do not mention anything about downloading to the user.";
|
||||
|
|
|
|||
|
|
@ -1,69 +1,16 @@
|
|||
const { z } = require('zod');
|
||||
const axios = require('axios');
|
||||
const { v4 } = require('uuid');
|
||||
const OpenAI = require('openai');
|
||||
const FormData = require('form-data');
|
||||
const { ProxyAgent } = require('undici');
|
||||
const { tool } = require('@langchain/core/tools');
|
||||
const { logAxiosError } = require('@librechat/api');
|
||||
const { logger } = require('@librechat/data-schemas');
|
||||
const { logAxiosError, oaiToolkit } = require('@librechat/api');
|
||||
const { ContentTypes, EImageOutputType } = require('librechat-data-provider');
|
||||
const { getStrategyFunctions } = require('~/server/services/Files/strategies');
|
||||
const { extractBaseURL } = require('~/utils');
|
||||
const extractBaseURL = require('~/utils/extractBaseURL');
|
||||
const { getFiles } = require('~/models/File');
|
||||
|
||||
/** Default descriptions for image generation tool */
|
||||
const DEFAULT_IMAGE_GEN_DESCRIPTION = `
|
||||
Generates high-quality, original images based solely on text, not using any uploaded reference images.
|
||||
|
||||
When to use \`image_gen_oai\`:
|
||||
- To create entirely new images from detailed text descriptions that do NOT reference any image files.
|
||||
|
||||
When NOT to use \`image_gen_oai\`:
|
||||
- If the user has uploaded any images and requests modifications, enhancements, or remixing based on those uploads → use \`image_edit_oai\` instead.
|
||||
|
||||
Generated image IDs will be returned in the response, so you can refer to them in future requests made to \`image_edit_oai\`.
|
||||
`.trim();
|
||||
|
||||
/** Default description for image editing tool */
|
||||
const DEFAULT_IMAGE_EDIT_DESCRIPTION =
|
||||
`Generates high-quality, original images based on text and one or more uploaded/referenced images.
|
||||
|
||||
When to use \`image_edit_oai\`:
|
||||
- The user wants to modify, extend, or remix one **or more** uploaded images, either:
|
||||
- Previously generated, or in the current request (both to be included in the \`image_ids\` array).
|
||||
- Always when the user refers to uploaded images for editing, enhancement, remixing, style transfer, or combining elements.
|
||||
- Any current or existing images are to be used as visual guides.
|
||||
- If there are any files in the current request, they are more likely than not expected as references for image edit requests.
|
||||
|
||||
When NOT to use \`image_edit_oai\`:
|
||||
- Brand-new generations that do not rely on an existing image → use \`image_gen_oai\` instead.
|
||||
|
||||
Both generated and referenced image IDs will be returned in the response, so you can refer to them in future requests made to \`image_edit_oai\`.
|
||||
`.trim();
|
||||
|
||||
/** Default prompt descriptions */
|
||||
const DEFAULT_IMAGE_GEN_PROMPT_DESCRIPTION = `Describe the image you want in detail.
|
||||
Be highly specific—break your idea into layers:
|
||||
(1) main concept and subject,
|
||||
(2) composition and position,
|
||||
(3) lighting and mood,
|
||||
(4) style, medium, or camera details,
|
||||
(5) important features (age, expression, clothing, etc.),
|
||||
(6) background.
|
||||
Use positive, descriptive language and specify what should be included, not what to avoid.
|
||||
List number and characteristics of people/objects, and mention style/technical requirements (e.g., "DSLR photo, 85mm lens, golden hour").
|
||||
Do not reference any uploaded images—use for new image creation from text only.`;
|
||||
|
||||
const DEFAULT_IMAGE_EDIT_PROMPT_DESCRIPTION = `Describe the changes, enhancements, or new ideas to apply to the uploaded image(s).
|
||||
Be highly specific—break your request into layers:
|
||||
(1) main concept or transformation,
|
||||
(2) specific edits/replacements or composition guidance,
|
||||
(3) desired style, mood, or technique,
|
||||
(4) features/items to keep, change, or add (such as objects, people, clothing, lighting, etc.).
|
||||
Use positive, descriptive language and clarify what should be included or changed, not what to avoid.
|
||||
Always base this prompt on the most recently uploaded reference images.`;
|
||||
|
||||
const displayMessage =
|
||||
"The tool displayed an image. All generated images are already plainly visible, so don't repeat the descriptions in detail. Do not list download links as they are available in the UI already. The user may download the images by clicking on them, but do not mention anything about downloading to the user.";
|
||||
|
||||
|
|
@ -91,22 +38,6 @@ function returnValue(value) {
|
|||
return value;
|
||||
}
|
||||
|
||||
const getImageGenDescription = () => {
|
||||
return process.env.IMAGE_GEN_OAI_DESCRIPTION || DEFAULT_IMAGE_GEN_DESCRIPTION;
|
||||
};
|
||||
|
||||
const getImageEditDescription = () => {
|
||||
return process.env.IMAGE_EDIT_OAI_DESCRIPTION || DEFAULT_IMAGE_EDIT_DESCRIPTION;
|
||||
};
|
||||
|
||||
const getImageGenPromptDescription = () => {
|
||||
return process.env.IMAGE_GEN_OAI_PROMPT_DESCRIPTION || DEFAULT_IMAGE_GEN_PROMPT_DESCRIPTION;
|
||||
};
|
||||
|
||||
const getImageEditPromptDescription = () => {
|
||||
return process.env.IMAGE_EDIT_OAI_PROMPT_DESCRIPTION || DEFAULT_IMAGE_EDIT_PROMPT_DESCRIPTION;
|
||||
};
|
||||
|
||||
function createAbortHandler() {
|
||||
return function () {
|
||||
logger.debug('[ImageGenOAI] Image generation aborted');
|
||||
|
|
@ -121,7 +52,9 @@ function createAbortHandler() {
|
|||
* @param {string} fields.IMAGE_GEN_OAI_API_KEY - The OpenAI API key
|
||||
* @param {boolean} [fields.override] - Whether to override the API key check, necessary for app initialization
|
||||
* @param {MongoFile[]} [fields.imageFiles] - The images to be used for editing
|
||||
* @returns {Array} - Array of image tools
|
||||
* @param {string} [fields.imageOutputType] - The image output type configuration
|
||||
* @param {string} [fields.fileStrategy] - The file storage strategy
|
||||
* @returns {Array<ReturnType<tool>>} - Array of image tools
|
||||
*/
|
||||
function createOpenAIImageTools(fields = {}) {
|
||||
/** @type {boolean} Used to initialize the Tool without necessary variables. */
|
||||
|
|
@ -131,8 +64,8 @@ function createOpenAIImageTools(fields = {}) {
|
|||
throw new Error('This tool is only available for agents.');
|
||||
}
|
||||
const { req } = fields;
|
||||
const imageOutputType = req?.app.locals.imageOutputType || EImageOutputType.PNG;
|
||||
const appFileStrategy = req?.app.locals.fileStrategy;
|
||||
const imageOutputType = fields.imageOutputType || EImageOutputType.PNG;
|
||||
const appFileStrategy = fields.fileStrategy;
|
||||
|
||||
const getApiKey = () => {
|
||||
const apiKey = process.env.IMAGE_GEN_OAI_API_KEY ?? '';
|
||||
|
|
@ -285,46 +218,7 @@ Error Message: ${error.message}`);
|
|||
];
|
||||
return [response, { content, file_ids }];
|
||||
},
|
||||
{
|
||||
name: 'image_gen_oai',
|
||||
description: getImageGenDescription(),
|
||||
schema: z.object({
|
||||
prompt: z.string().max(32000).describe(getImageGenPromptDescription()),
|
||||
background: z
|
||||
.enum(['transparent', 'opaque', 'auto'])
|
||||
.optional()
|
||||
.describe(
|
||||
'Sets transparency for the background. Must be one of transparent, opaque or auto (default). When transparent, the output format should be png or webp.',
|
||||
),
|
||||
/*
|
||||
n: z
|
||||
.number()
|
||||
.int()
|
||||
.min(1)
|
||||
.max(10)
|
||||
.optional()
|
||||
.describe('The number of images to generate. Must be between 1 and 10.'),
|
||||
output_compression: z
|
||||
.number()
|
||||
.int()
|
||||
.min(0)
|
||||
.max(100)
|
||||
.optional()
|
||||
.describe('The compression level (0-100%) for webp or jpeg formats. Defaults to 100.'),
|
||||
*/
|
||||
quality: z
|
||||
.enum(['auto', 'high', 'medium', 'low'])
|
||||
.optional()
|
||||
.describe('The quality of the image. One of auto (default), high, medium, or low.'),
|
||||
size: z
|
||||
.enum(['auto', '1024x1024', '1536x1024', '1024x1536'])
|
||||
.optional()
|
||||
.describe(
|
||||
'The size of the generated image. One of 1024x1024, 1536x1024 (landscape), 1024x1536 (portrait), or auto (default).',
|
||||
),
|
||||
}),
|
||||
responseFormat: 'content_and_artifact',
|
||||
},
|
||||
oaiToolkit.image_gen_oai,
|
||||
);
|
||||
|
||||
/**
|
||||
|
|
@ -517,48 +411,7 @@ Error Message: ${error.message || 'Unknown error'}`);
|
|||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'image_edit_oai',
|
||||
description: getImageEditDescription(),
|
||||
schema: z.object({
|
||||
image_ids: z
|
||||
.array(z.string())
|
||||
.min(1)
|
||||
.describe(
|
||||
`
|
||||
IDs (image ID strings) of previously generated or uploaded images that should guide the edit.
|
||||
|
||||
Guidelines:
|
||||
- If the user's request depends on any prior image(s), copy their image IDs into the \`image_ids\` array (in the same order the user refers to them).
|
||||
- Never invent or hallucinate IDs; only use IDs that are still visible in the conversation context.
|
||||
- If no earlier image is relevant, omit the field entirely.
|
||||
`.trim(),
|
||||
),
|
||||
prompt: z.string().max(32000).describe(getImageEditPromptDescription()),
|
||||
/*
|
||||
n: z
|
||||
.number()
|
||||
.int()
|
||||
.min(1)
|
||||
.max(10)
|
||||
.optional()
|
||||
.describe('The number of images to generate. Must be between 1 and 10. Defaults to 1.'),
|
||||
*/
|
||||
quality: z
|
||||
.enum(['auto', 'high', 'medium', 'low'])
|
||||
.optional()
|
||||
.describe(
|
||||
'The quality of the image. One of auto (default), high, medium, or low. High/medium/low only supported for gpt-image-1.',
|
||||
),
|
||||
size: z
|
||||
.enum(['auto', '1024x1024', '1536x1024', '1024x1536', '256x256', '512x512'])
|
||||
.optional()
|
||||
.describe(
|
||||
'The size of the generated images. For gpt-image-1: auto (default), 1024x1024, 1536x1024, 1024x1536. For dall-e-2: 256x256, 512x512, 1024x1024.',
|
||||
),
|
||||
}),
|
||||
responseFormat: 'content_and_artifact',
|
||||
},
|
||||
oaiToolkit.image_edit_oai,
|
||||
);
|
||||
|
||||
return [imageGenTool, imageEditTool];
|
||||
|
|
|
|||
|
|
@ -11,14 +11,14 @@ const paths = require('~/config/paths');
|
|||
const { logger } = require('~/config');
|
||||
|
||||
const displayMessage =
|
||||
'Stable Diffusion displayed an image. All generated images are already plainly visible, so don\'t repeat the descriptions in detail. Do not list download links as they are available in the UI already. The user may download the images by clicking on them, but do not mention anything about downloading to the user.';
|
||||
"Stable Diffusion displayed an image. All generated images are already plainly visible, so don't repeat the descriptions in detail. Do not list download links as they are available in the UI already. The user may download the images by clicking on them, but do not mention anything about downloading to the user.";
|
||||
|
||||
class StableDiffusionAPI extends Tool {
|
||||
constructor(fields) {
|
||||
super();
|
||||
/** @type {string} User ID */
|
||||
this.userId = fields.userId;
|
||||
/** @type {Express.Request | undefined} Express Request object, only provided by ToolService */
|
||||
/** @type {ServerRequest | undefined} Express Request object, only provided by ToolService */
|
||||
this.req = fields.req;
|
||||
/** @type {boolean} Used to initialize the Tool without necessary variables. */
|
||||
this.override = fields.override ?? false;
|
||||
|
|
@ -44,7 +44,7 @@ class StableDiffusionAPI extends Tool {
|
|||
// "negative_prompt":"semi-realistic, cgi, 3d, render, sketch, cartoon, drawing, anime, out of frame, low quality, ugly, mutation, deformed"
|
||||
// - Generate images only once per human query unless explicitly requested by the user`;
|
||||
this.description =
|
||||
'You can generate images using text with \'stable-diffusion\'. This tool is exclusively for visual content.';
|
||||
"You can generate images using text with 'stable-diffusion'. This tool is exclusively for visual content.";
|
||||
this.schema = z.object({
|
||||
prompt: z
|
||||
.string()
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
const { z } = require('zod');
|
||||
const { ytToolkit } = require('@librechat/api');
|
||||
const { tool } = require('@langchain/core/tools');
|
||||
const { youtube } = require('@googleapis/youtube');
|
||||
const { logger } = require('@librechat/data-schemas');
|
||||
const { YoutubeTranscript } = require('youtube-transcript');
|
||||
const { getApiKey } = require('./credentials');
|
||||
const { logger } = require('~/config');
|
||||
|
||||
function extractVideoId(url) {
|
||||
const rawIdRegex = /^[a-zA-Z0-9_-]{11}$/;
|
||||
|
|
@ -29,7 +29,7 @@ function parseTranscript(transcriptResponse) {
|
|||
.map((entry) => entry.text.trim())
|
||||
.filter((text) => text)
|
||||
.join(' ')
|
||||
.replaceAll('&#39;', '\'');
|
||||
.replaceAll('&#39;', "'");
|
||||
}
|
||||
|
||||
function createYouTubeTools(fields = {}) {
|
||||
|
|
@ -42,160 +42,94 @@ function createYouTubeTools(fields = {}) {
|
|||
auth: apiKey,
|
||||
});
|
||||
|
||||
const searchTool = tool(
|
||||
async ({ query, maxResults = 5 }) => {
|
||||
const response = await youtubeClient.search.list({
|
||||
part: 'snippet',
|
||||
q: query,
|
||||
type: 'video',
|
||||
maxResults: maxResults || 5,
|
||||
});
|
||||
const result = response.data.items.map((item) => ({
|
||||
title: item.snippet.title,
|
||||
description: item.snippet.description,
|
||||
url: `https://www.youtube.com/watch?v=${item.id.videoId}`,
|
||||
}));
|
||||
return JSON.stringify(result, null, 2);
|
||||
},
|
||||
{
|
||||
name: 'youtube_search',
|
||||
description: `Search for YouTube videos by keyword or phrase.
|
||||
- Required: query (search terms to find videos)
|
||||
- Optional: maxResults (number of videos to return, 1-50, default: 5)
|
||||
- Returns: List of videos with titles, descriptions, and URLs
|
||||
- Use for: Finding specific videos, exploring content, research
|
||||
Example: query="cooking pasta tutorials" maxResults=3`,
|
||||
schema: z.object({
|
||||
query: z.string().describe('Search query terms'),
|
||||
maxResults: z.number().int().min(1).max(50).optional().describe('Number of results (1-50)'),
|
||||
}),
|
||||
},
|
||||
);
|
||||
const searchTool = tool(async ({ query, maxResults = 5 }) => {
|
||||
const response = await youtubeClient.search.list({
|
||||
part: 'snippet',
|
||||
q: query,
|
||||
type: 'video',
|
||||
maxResults: maxResults || 5,
|
||||
});
|
||||
const result = response.data.items.map((item) => ({
|
||||
title: item.snippet.title,
|
||||
description: item.snippet.description,
|
||||
url: `https://www.youtube.com/watch?v=${item.id.videoId}`,
|
||||
}));
|
||||
return JSON.stringify(result, null, 2);
|
||||
}, ytToolkit.youtube_search);
|
||||
|
||||
const infoTool = tool(
|
||||
async ({ url }) => {
|
||||
const videoId = extractVideoId(url);
|
||||
if (!videoId) {
|
||||
throw new Error('Invalid YouTube URL or video ID');
|
||||
}
|
||||
const infoTool = tool(async ({ url }) => {
|
||||
const videoId = extractVideoId(url);
|
||||
if (!videoId) {
|
||||
throw new Error('Invalid YouTube URL or video ID');
|
||||
}
|
||||
|
||||
const response = await youtubeClient.videos.list({
|
||||
part: 'snippet,statistics',
|
||||
id: videoId,
|
||||
});
|
||||
const response = await youtubeClient.videos.list({
|
||||
part: 'snippet,statistics',
|
||||
id: videoId,
|
||||
});
|
||||
|
||||
if (!response.data.items?.length) {
|
||||
throw new Error('Video not found');
|
||||
}
|
||||
const video = response.data.items[0];
|
||||
if (!response.data.items?.length) {
|
||||
throw new Error('Video not found');
|
||||
}
|
||||
const video = response.data.items[0];
|
||||
|
||||
const result = {
|
||||
title: video.snippet.title,
|
||||
description: video.snippet.description,
|
||||
views: video.statistics.viewCount,
|
||||
likes: video.statistics.likeCount,
|
||||
comments: video.statistics.commentCount,
|
||||
};
|
||||
return JSON.stringify(result, null, 2);
|
||||
},
|
||||
{
|
||||
name: 'youtube_info',
|
||||
description: `Get detailed metadata and statistics for a specific YouTube video.
|
||||
- Required: url (full YouTube URL or video ID)
|
||||
- Returns: Video title, description, view count, like count, comment count
|
||||
- Use for: Getting video metrics and basic metadata
|
||||
- DO NOT USE FOR VIDEO SUMMARIES, USE TRANSCRIPTS FOR COMPREHENSIVE ANALYSIS
|
||||
- Accepts both full URLs and video IDs
|
||||
Example: url="https://youtube.com/watch?v=abc123" or url="abc123"`,
|
||||
schema: z.object({
|
||||
url: z.string().describe('YouTube video URL or ID'),
|
||||
}),
|
||||
},
|
||||
);
|
||||
const result = {
|
||||
title: video.snippet.title,
|
||||
description: video.snippet.description,
|
||||
views: video.statistics.viewCount,
|
||||
likes: video.statistics.likeCount,
|
||||
comments: video.statistics.commentCount,
|
||||
};
|
||||
return JSON.stringify(result, null, 2);
|
||||
}, ytToolkit.youtube_info);
|
||||
|
||||
const commentsTool = tool(
|
||||
async ({ url, maxResults = 10 }) => {
|
||||
const videoId = extractVideoId(url);
|
||||
if (!videoId) {
|
||||
throw new Error('Invalid YouTube URL or video ID');
|
||||
}
|
||||
const commentsTool = tool(async ({ url, maxResults = 10 }) => {
|
||||
const videoId = extractVideoId(url);
|
||||
if (!videoId) {
|
||||
throw new Error('Invalid YouTube URL or video ID');
|
||||
}
|
||||
|
||||
const response = await youtubeClient.commentThreads.list({
|
||||
part: 'snippet',
|
||||
videoId,
|
||||
maxResults: maxResults || 10,
|
||||
});
|
||||
const response = await youtubeClient.commentThreads.list({
|
||||
part: 'snippet',
|
||||
videoId,
|
||||
maxResults: maxResults || 10,
|
||||
});
|
||||
|
||||
const result = response.data.items.map((item) => ({
|
||||
author: item.snippet.topLevelComment.snippet.authorDisplayName,
|
||||
text: item.snippet.topLevelComment.snippet.textDisplay,
|
||||
likes: item.snippet.topLevelComment.snippet.likeCount,
|
||||
}));
|
||||
return JSON.stringify(result, null, 2);
|
||||
},
|
||||
{
|
||||
name: 'youtube_comments',
|
||||
description: `Retrieve top-level comments from a YouTube video.
|
||||
- Required: url (full YouTube URL or video ID)
|
||||
- Optional: maxResults (number of comments, 1-50, default: 10)
|
||||
- Returns: Comment text, author names, like counts
|
||||
- Use for: Sentiment analysis, audience feedback, engagement review
|
||||
Example: url="abc123" maxResults=20`,
|
||||
schema: z.object({
|
||||
url: z.string().describe('YouTube video URL or ID'),
|
||||
maxResults: z
|
||||
.number()
|
||||
.int()
|
||||
.min(1)
|
||||
.max(50)
|
||||
.optional()
|
||||
.describe('Number of comments to retrieve'),
|
||||
}),
|
||||
},
|
||||
);
|
||||
const result = response.data.items.map((item) => ({
|
||||
author: item.snippet.topLevelComment.snippet.authorDisplayName,
|
||||
text: item.snippet.topLevelComment.snippet.textDisplay,
|
||||
likes: item.snippet.topLevelComment.snippet.likeCount,
|
||||
}));
|
||||
return JSON.stringify(result, null, 2);
|
||||
}, ytToolkit.youtube_comments);
|
||||
|
||||
const transcriptTool = tool(
|
||||
async ({ url }) => {
|
||||
const videoId = extractVideoId(url);
|
||||
if (!videoId) {
|
||||
throw new Error('Invalid YouTube URL or video ID');
|
||||
const transcriptTool = tool(async ({ url }) => {
|
||||
const videoId = extractVideoId(url);
|
||||
if (!videoId) {
|
||||
throw new Error('Invalid YouTube URL or video ID');
|
||||
}
|
||||
|
||||
try {
|
||||
try {
|
||||
const transcript = await YoutubeTranscript.fetchTranscript(videoId, { lang: 'en' });
|
||||
return parseTranscript(transcript);
|
||||
} catch (e) {
|
||||
logger.error(e);
|
||||
}
|
||||
|
||||
try {
|
||||
try {
|
||||
const transcript = await YoutubeTranscript.fetchTranscript(videoId, { lang: 'en' });
|
||||
return parseTranscript(transcript);
|
||||
} catch (e) {
|
||||
logger.error(e);
|
||||
}
|
||||
|
||||
try {
|
||||
const transcript = await YoutubeTranscript.fetchTranscript(videoId, { lang: 'de' });
|
||||
return parseTranscript(transcript);
|
||||
} catch (e) {
|
||||
logger.error(e);
|
||||
}
|
||||
|
||||
const transcript = await YoutubeTranscript.fetchTranscript(videoId);
|
||||
const transcript = await YoutubeTranscript.fetchTranscript(videoId, { lang: 'de' });
|
||||
return parseTranscript(transcript);
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to fetch transcript: ${error.message}`);
|
||||
} catch (e) {
|
||||
logger.error(e);
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'youtube_transcript',
|
||||
description: `Fetch and parse the transcript/captions of a YouTube video.
|
||||
- Required: url (full YouTube URL or video ID)
|
||||
- Returns: Full video transcript as plain text
|
||||
- Use for: Content analysis, summarization, translation reference
|
||||
- This is the "Go-to" tool for analyzing actual video content
|
||||
- Attempts to fetch English first, then German, then any available language
|
||||
Example: url="https://youtube.com/watch?v=abc123"`,
|
||||
schema: z.object({
|
||||
url: z.string().describe('YouTube video URL or ID'),
|
||||
}),
|
||||
},
|
||||
);
|
||||
|
||||
const transcript = await YoutubeTranscript.fetchTranscript(videoId);
|
||||
return parseTranscript(transcript);
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to fetch transcript: ${error.message}`);
|
||||
}
|
||||
}, ytToolkit.youtube_transcript);
|
||||
|
||||
return [searchTool, infoTool, commentsTool, transcriptTool];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,43 +1,9 @@
|
|||
const DALLE3 = require('../DALLE3');
|
||||
const { ProxyAgent } = require('undici');
|
||||
|
||||
jest.mock('tiktoken');
|
||||
const processFileURL = jest.fn();
|
||||
|
||||
jest.mock('~/server/services/Files/images', () => ({
|
||||
getImageBasename: jest.fn().mockImplementation((url) => {
|
||||
const parts = url.split('/');
|
||||
const lastPart = parts.pop();
|
||||
const imageExtensionRegex = /\.(jpg|jpeg|png|gif|bmp|tiff|svg)$/i;
|
||||
if (imageExtensionRegex.test(lastPart)) {
|
||||
return lastPart;
|
||||
}
|
||||
return '';
|
||||
}),
|
||||
}));
|
||||
|
||||
jest.mock('fs', () => {
|
||||
return {
|
||||
existsSync: jest.fn(),
|
||||
mkdirSync: jest.fn(),
|
||||
promises: {
|
||||
writeFile: jest.fn(),
|
||||
readFile: jest.fn(),
|
||||
unlink: jest.fn(),
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
jest.mock('path', () => {
|
||||
return {
|
||||
resolve: jest.fn(),
|
||||
join: jest.fn(),
|
||||
relative: jest.fn(),
|
||||
extname: jest.fn().mockImplementation((filename) => {
|
||||
return filename.slice(filename.lastIndexOf('.'));
|
||||
}),
|
||||
};
|
||||
});
|
||||
|
||||
describe('DALLE3 Proxy Configuration', () => {
|
||||
let originalEnv;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,8 @@
|
|||
const OpenAI = require('openai');
|
||||
const { logger } = require('@librechat/data-schemas');
|
||||
const DALLE3 = require('../DALLE3');
|
||||
const logger = require('~/config/winston');
|
||||
|
||||
jest.mock('openai');
|
||||
|
||||
jest.mock('@librechat/data-schemas', () => {
|
||||
return {
|
||||
logger: {
|
||||
|
|
@ -26,25 +25,6 @@ jest.mock('tiktoken', () => {
|
|||
|
||||
const processFileURL = jest.fn();
|
||||
|
||||
jest.mock('~/server/services/Files/images', () => ({
|
||||
getImageBasename: jest.fn().mockImplementation((url) => {
|
||||
// Split the URL by '/'
|
||||
const parts = url.split('/');
|
||||
|
||||
// Get the last part of the URL
|
||||
const lastPart = parts.pop();
|
||||
|
||||
// Check if the last part of the URL matches the image extension regex
|
||||
const imageExtensionRegex = /\.(jpg|jpeg|png|gif|bmp|tiff|svg)$/i;
|
||||
if (imageExtensionRegex.test(lastPart)) {
|
||||
return lastPart;
|
||||
}
|
||||
|
||||
// If the regex test fails, return an empty string
|
||||
return '';
|
||||
}),
|
||||
}));
|
||||
|
||||
const generate = jest.fn();
|
||||
OpenAI.mockImplementation(() => ({
|
||||
images: {
|
||||
|
|
|
|||
|
|
@ -121,18 +121,21 @@ const getAuthFields = (toolKey) => {
|
|||
|
||||
/**
|
||||
*
|
||||
* @param {object} object
|
||||
* @param {string} object.user
|
||||
* @param {object} params
|
||||
* @param {string} params.user
|
||||
* @param {Record<string, Record<string, string>>} [object.userMCPAuthMap]
|
||||
* @param {AbortSignal} [object.signal]
|
||||
* @param {Pick<Agent, 'id' | 'provider' | 'model'>} [object.agent]
|
||||
* @param {string} [object.model]
|
||||
* @param {EModelEndpoint} [object.endpoint]
|
||||
* @param {LoadToolOptions} [object.options]
|
||||
* @param {boolean} [object.useSpecs]
|
||||
* @param {Array<string>} object.tools
|
||||
* @param {boolean} [object.functions]
|
||||
* @param {boolean} [object.returnMap]
|
||||
* @param {Pick<Agent, 'id' | 'provider' | 'model'>} [params.agent]
|
||||
* @param {string} [params.model]
|
||||
* @param {EModelEndpoint} [params.endpoint]
|
||||
* @param {LoadToolOptions} [params.options]
|
||||
* @param {boolean} [params.useSpecs]
|
||||
* @param {Array<string>} params.tools
|
||||
* @param {boolean} [params.functions]
|
||||
* @param {boolean} [params.returnMap]
|
||||
* @param {AppConfig['webSearch']} [params.webSearch]
|
||||
* @param {AppConfig['fileStrategy']} [params.fileStrategy]
|
||||
* @param {AppConfig['imageOutputType']} [params.imageOutputType]
|
||||
* @returns {Promise<{ loadedTools: Tool[], toolContextMap: Object<string, any> } | Record<string,Tool>>}
|
||||
*/
|
||||
const loadTools = async ({
|
||||
|
|
@ -146,6 +149,9 @@ const loadTools = async ({
|
|||
options = {},
|
||||
functions = true,
|
||||
returnMap = false,
|
||||
webSearch,
|
||||
fileStrategy,
|
||||
imageOutputType,
|
||||
}) => {
|
||||
const toolConstructors = {
|
||||
flux: FluxAPI,
|
||||
|
|
@ -204,6 +210,8 @@ const loadTools = async ({
|
|||
...authValues,
|
||||
isAgent: !!agent,
|
||||
req: options.req,
|
||||
imageOutputType,
|
||||
fileStrategy,
|
||||
imageFiles,
|
||||
});
|
||||
},
|
||||
|
|
@ -219,7 +227,7 @@ const loadTools = async ({
|
|||
const imageGenOptions = {
|
||||
isAgent: !!agent,
|
||||
req: options.req,
|
||||
fileStrategy: options.fileStrategy,
|
||||
fileStrategy,
|
||||
processFileURL: options.processFileURL,
|
||||
returnMetadata: options.returnMetadata,
|
||||
uploadImageBuffer: options.uploadImageBuffer,
|
||||
|
|
@ -277,11 +285,10 @@ const loadTools = async ({
|
|||
};
|
||||
continue;
|
||||
} else if (tool === Tools.web_search) {
|
||||
const webSearchConfig = options?.req?.app?.locals?.webSearch;
|
||||
const result = await loadWebSearchAuth({
|
||||
userId: user,
|
||||
loadAuthValues,
|
||||
webSearchConfig,
|
||||
webSearchConfig: webSearch,
|
||||
});
|
||||
const { onSearchResults, onGetHighlights } = options?.[Tools.web_search] ?? {};
|
||||
requestedTools[tool] = async () => {
|
||||
|
|
|
|||
|
|
@ -9,6 +9,27 @@ const mockPluginService = {
|
|||
|
||||
jest.mock('~/server/services/PluginService', () => mockPluginService);
|
||||
|
||||
jest.mock('~/server/services/Config', () => ({
|
||||
getAppConfig: jest.fn().mockResolvedValue({
|
||||
// Default app config for tool tests
|
||||
paths: { uploads: '/tmp' },
|
||||
fileStrategy: 'local',
|
||||
filteredTools: [],
|
||||
includedTools: [],
|
||||
}),
|
||||
getCachedTools: jest.fn().mockResolvedValue({
|
||||
// Default cached tools for tests
|
||||
dalle: {
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'dalle',
|
||||
description: 'DALL-E image generation',
|
||||
parameters: {},
|
||||
},
|
||||
},
|
||||
}),
|
||||
}));
|
||||
|
||||
const { BaseLLM } = require('@langchain/openai');
|
||||
const { Calculator } = require('@langchain/community/tools/calculator');
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue