mirror of
https://github.com/danny-avila/LibreChat.git
synced 2025-12-17 08:50:15 +01:00
🎨 feat: OpenAI Image Tools (GPT-Image-1) (#7079)
* wip: OpenAI Image Generation Tool with customizable options * WIP: First pass OpenAI Image Generation Tool and integrate into existing tools * 🔀 fix: Comment out unused validation for image generation tool parameters * 🔀 refactor: Update primeResources function parameters for better destructuring * feat: Add image_edit resource to EToolResources and update AgentToolResources interface * feat: Enhance file retrieval with tool resource filtering for image editing * refactor: add OpenAI Image Tools for generation and editing, refactor related components, pass current request image attachments as tool resources for editing * refactor: Remove commented-out code and clean up API key retrieval in createOpenAIImageTools function * fix: show message attachments in shared links * fix: Correct parent message retrieval logic for regenerated messages in useChatFunctions * fix: Update primeResources to utilize requestFileSet for image file processing * refactor: Improve description for image generation tool and clarify usage conditions, only provide edit tool if there are images available to edit * chore: Update OpenAI Image Tools icon to use local asset * refactor: Update image generation tool description and logic to prioritize editing tool when files are uploaded * refactor: Enhance image tool descriptions to clarify usage conditions and note potential unavailability of uploaded images * refactor: Update useAttachmentHandler to accept queryClient to update query cache with newly created file * refactor: Add customizable descriptions and prompts for OpenAI image generation and editing tools * chore: Update comments to use JSDoc style for better clarity and consistency * refactor: Rename config variable to clientConfig for clarity and update signal handling in image generation * refactor: Update axios request configuration to include derived signal and baseURL for improved request handling * refactor: Update baseURL environment variable for OpenAI image generation tool configuration * refactor: Enhance axios request configuration with conditional headers and improved clientConfig setup * chore: Update comments for clarity and remove unnecessary lines in OpenAI image tools * refactor: Update description for image generation without files to clarify user instructions * refactor: Simplify target parent message logic for regeneration and resubmission cases * chore: Remove backticks from error messages in image generation and editing functions * refactor: Rename toolResources to toolResourceSet for clarity in file retrieval functions * chore: Remove redundant comments and clean up TODOs in OpenAI image tools * refactor: Rename fileStrategy to appFileStrategy for clarity and improve error handling in image processing * chore: Update react-resizable-panels to version 2.1.8 in package.json and package-lock.json * chore: Ensure required validation for logs and Code of Conduct agreement in bug report template * fix: Update ArtifactPreview to use startupConfig and currentCode from memoized props to prevent unnecessary re-renders * fix: improve robustness of `save & submit` when used from a user-message with existing attachments * fix: add null check for artifact index in CodeEditor to prevent errors, trigger re-render on artifact ID change * fix: standardize default values for artifact properties in Artifact component, avoiding prematurely setting an "empty/default" artifact * fix: reset current artifact ID before setting a new one in ArtifactButton to ensure correct state management * chore: rename `setArtifactId` variable to `setCurrentArtifactId` for consistency * chore: update type annotations in File and S3 CRUD functions for consistency * refactor: improve image handling in OpenAI tools by using image_id references and enhance tool context for image editing * fix: update image_ids schema in image_edit_oai to enforce presence and provide clear guidelines for usage * fix: enhance file fetching logic to ensure user-specific and dimension-validated results * chore: add details on image generation and editing capabilities with various models
This commit is contained in:
Danny Avila
GitHub
parent
0ee1dcc479
commit
c0ebb434a6
30 changed files with 841 additions and 104 deletions
|
|
@ -3,6 +3,7 @@ const {
|
|||
Constants,
|
||||
ErrorTypes,
|
||||
EModelEndpoint,
|
||||
EToolResources,
|
||||
getResponseSender,
|
||||
AgentCapabilities,
|
||||
providerEndpointMap,
|
||||
|
|
@ -41,12 +42,19 @@ const providerConfigMap = {
|
|||
};
|
||||
|
||||
/**
|
||||
* @param {ServerRequest} req
|
||||
* @param {Promise<Array<MongoFile | null>> | undefined} _attachments
|
||||
* @param {AgentToolResources | undefined} _tool_resources
|
||||
* @param {Object} params
|
||||
* @param {ServerRequest} params.req
|
||||
* @param {Promise<Array<MongoFile | null>> | undefined} [params.attachments]
|
||||
* @param {Set<string>} params.requestFileSet
|
||||
* @param {AgentToolResources | undefined} [params.tool_resources]
|
||||
* @returns {Promise<{ attachments: Array<MongoFile | undefined> | undefined, tool_resources: AgentToolResources | undefined }>}
|
||||
*/
|
||||
const primeResources = async (req, _attachments, _tool_resources) => {
|
||||
const primeResources = async ({
|
||||
req,
|
||||
attachments: _attachments,
|
||||
tool_resources: _tool_resources,
|
||||
requestFileSet,
|
||||
}) => {
|
||||
try {
|
||||
/** @type {Array<MongoFile | undefined> | undefined} */
|
||||
let attachments;
|
||||
|
|
@ -54,7 +62,7 @@ const primeResources = async (req, _attachments, _tool_resources) => {
|
|||
const isOCREnabled = (req.app.locals?.[EModelEndpoint.agents]?.capabilities ?? []).includes(
|
||||
AgentCapabilities.ocr,
|
||||
);
|
||||
if (tool_resources.ocr?.file_ids && isOCREnabled) {
|
||||
if (tool_resources[EToolResources.ocr]?.file_ids && isOCREnabled) {
|
||||
const context = await getFiles(
|
||||
{
|
||||
file_id: { $in: tool_resources.ocr.file_ids },
|
||||
|
|
@ -79,17 +87,28 @@ const primeResources = async (req, _attachments, _tool_resources) => {
|
|||
continue;
|
||||
}
|
||||
if (file.metadata?.fileIdentifier) {
|
||||
const execute_code = tool_resources.execute_code ?? {};
|
||||
const execute_code = tool_resources[EToolResources.execute_code] ?? {};
|
||||
if (!execute_code.files) {
|
||||
tool_resources.execute_code = { ...execute_code, files: [] };
|
||||
tool_resources[EToolResources.execute_code] = { ...execute_code, files: [] };
|
||||
}
|
||||
tool_resources.execute_code.files.push(file);
|
||||
tool_resources[EToolResources.execute_code].files.push(file);
|
||||
} else if (file.embedded === true) {
|
||||
const file_search = tool_resources.file_search ?? {};
|
||||
const file_search = tool_resources[EToolResources.file_search] ?? {};
|
||||
if (!file_search.files) {
|
||||
tool_resources.file_search = { ...file_search, files: [] };
|
||||
tool_resources[EToolResources.file_search] = { ...file_search, files: [] };
|
||||
}
|
||||
tool_resources.file_search.files.push(file);
|
||||
tool_resources[EToolResources.file_search].files.push(file);
|
||||
} else if (
|
||||
requestFileSet.has(file.file_id) &&
|
||||
file.type.startsWith('image') &&
|
||||
file.height &&
|
||||
file.width
|
||||
) {
|
||||
const image_edit = tool_resources[EToolResources.image_edit] ?? {};
|
||||
if (!image_edit.files) {
|
||||
tool_resources[EToolResources.image_edit] = { ...image_edit, files: [] };
|
||||
}
|
||||
tool_resources[EToolResources.image_edit].files.push(file);
|
||||
}
|
||||
|
||||
attachments.push(file);
|
||||
|
|
@ -146,7 +165,14 @@ const initializeAgentOptions = async ({
|
|||
(agent.model_parameters?.resendFiles ?? true) === true
|
||||
) {
|
||||
const fileIds = (await getConvoFiles(req.body.conversationId)) ?? [];
|
||||
const toolFiles = await getToolFilesByIds(fileIds);
|
||||
/** @type {Set<EToolResources>} */
|
||||
const toolResourceSet = new Set();
|
||||
for (const tool of agent.tools) {
|
||||
if (EToolResources[tool]) {
|
||||
toolResourceSet.add(EToolResources[tool]);
|
||||
}
|
||||
}
|
||||
const toolFiles = await getToolFilesByIds(fileIds, toolResourceSet);
|
||||
if (requestFiles.length || toolFiles.length) {
|
||||
currentFiles = await processFiles(requestFiles.concat(toolFiles));
|
||||
}
|
||||
|
|
@ -154,11 +180,12 @@ const initializeAgentOptions = async ({
|
|||
currentFiles = await processFiles(requestFiles);
|
||||
}
|
||||
|
||||
const { attachments, tool_resources } = await primeResources(
|
||||
const { attachments, tool_resources } = await primeResources({
|
||||
req,
|
||||
currentFiles,
|
||||
agent.tool_resources,
|
||||
);
|
||||
attachments: currentFiles,
|
||||
tool_resources: agent.tool_resources,
|
||||
requestFileSet: new Set(requestFiles.map((file) => file.file_id)),
|
||||
});
|
||||
|
||||
const provider = agent.provider;
|
||||
const { tools, toolContextMap } = await loadAgentTools({
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue