Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2025-12-17 08:50:15 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 7

🎨 feat: OpenAI Image Tools (GPT-Image-1) (#7079)

Browse source 
* wip: OpenAI Image Generation Tool with customizable options

* WIP: First pass OpenAI Image Generation Tool and integrate into existing tools

* 🔀 fix: Comment out unused validation for image generation tool parameters

* 🔀 refactor: Update primeResources function parameters for better destructuring

* feat: Add image_edit resource to EToolResources and update AgentToolResources interface

* feat: Enhance file retrieval with tool resource filtering for image editing

* refactor: add OpenAI Image Tools for generation and editing, refactor related components, pass current request image attachments as tool resources for editing

* refactor: Remove commented-out code and clean up API key retrieval in createOpenAIImageTools function

* fix: show message attachments in shared links

* fix: Correct parent message retrieval logic for regenerated messages in useChatFunctions

* fix: Update primeResources to utilize requestFileSet for image file processing

* refactor: Improve description for image generation tool and clarify usage conditions, only provide edit tool if there are images available to edit

* chore: Update OpenAI Image Tools icon to use local asset

* refactor: Update image generation tool description and logic to prioritize editing tool when files are uploaded

* refactor: Enhance image tool descriptions to clarify usage conditions and note potential unavailability of uploaded images

* refactor: Update useAttachmentHandler to accept queryClient to update query cache with newly created file

* refactor: Add customizable descriptions and prompts for OpenAI image generation and editing tools

* chore: Update comments to use JSDoc style for better clarity and consistency

* refactor: Rename config variable to clientConfig for clarity and update signal handling in image generation

* refactor: Update axios request configuration to include derived signal and baseURL for improved request handling

* refactor: Update baseURL environment variable for OpenAI image generation tool configuration

* refactor: Enhance axios request configuration with conditional headers and improved clientConfig setup

* chore: Update comments for clarity and remove unnecessary lines in OpenAI image tools

* refactor: Update description for image generation without files to clarify user instructions

* refactor: Simplify target parent message logic for regeneration and resubmission cases

* chore: Remove backticks from error messages in image generation and editing functions

* refactor: Rename toolResources to toolResourceSet for clarity in file retrieval functions

* chore: Remove redundant comments and clean up TODOs in OpenAI image tools

* refactor: Rename fileStrategy to appFileStrategy for clarity and improve error handling in image processing

* chore: Update react-resizable-panels to version 2.1.8 in package.json and package-lock.json

* chore: Ensure required validation for logs and Code of Conduct agreement in bug report template

* fix: Update ArtifactPreview to use startupConfig and currentCode from memoized props to prevent unnecessary re-renders

* fix: improve robustness of `save & submit` when used from a user-message with existing attachments

* fix: add null check for artifact index in CodeEditor to prevent errors, trigger re-render on artifact ID change

* fix: standardize default values for artifact properties in Artifact component, avoiding prematurely setting an "empty/default" artifact

* fix: reset current artifact ID before setting a new one in ArtifactButton to ensure correct state management

* chore: rename `setArtifactId` variable to `setCurrentArtifactId`  for consistency

* chore: update type annotations in File and S3 CRUD functions for consistency

* refactor: improve image handling in OpenAI tools by using image_id references and enhance tool context for image editing

* fix: update image_ids schema in image_edit_oai to enforce presence and provide clear guidelines for usage

* fix: enhance file fetching logic to ensure user-specific and dimension-validated results

* chore: add details on image generation and editing capabilities with various models
This commit is contained in:
Danny Avila 2025-04-26 04:30:58 -04:00 • committed by GitHub
parent 0ee1dcc479
commit c0ebb434a6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
30 changed files with 841 additions and 104 deletions
Download patch file Download diff file Expand all files Collapse all files

38
api/models/File.js
View file

@ -1,4 +1,5 @@
const mongoose = require('mongoose');
const { EToolResources } = require('librechat-data-provider');
const { fileSchema } = require('@librechat/data-schemas');
const { logger } = require('~/config');
@ -8,7 +9,7 @@ const File = mongoose.model('File', fileSchema);
* Finds a file by its file_id with additional query options.
* @param {string} file_id - The unique identifier of the file.
* @param {object} options - Query options for filtering, projection, etc.
* @returns {Promise<IMongoFile>} A promise that resolves to the file document or null.
* @returns {Promise<MongoFile>} A promise that resolves to the file document or null.
*/
const findFileById = async (file_id, options = {}) => {
return await File.findOne({ file_id, ...options }).lean();
@ -20,7 +21,7 @@ const findFileById = async (file_id, options = {}) => {
* @param {Object} [_sortOptions] - Optional sort parameters.
* @param {Object|String} [selectFields={ text: 0 }] - Fields to include/exclude in the query results.
* Default excludes the 'text' field.
* @returns {Promise<Array<IMongoFile>>} A promise that resolves to an array of file documents.
* @returns {Promise<Array<MongoFile>>} A promise that resolves to an array of file documents.
*/
const getFiles = async (filter, _sortOptions, selectFields = { text: 0 }) => {
const sortOptions = { updatedAt: -1, ..._sortOptions };
@ -30,9 +31,10 @@ const getFiles = async (filter, _sortOptions, selectFields = { text: 0 }) => {
/**
* Retrieves tool files (files that are embedded or have a fileIdentifier) from an array of file IDs
* @param {string[]} fileIds - Array of file_id strings to search for
* @returns {Promise<Array<IMongoFile>>} Files that match the criteria
* @param {Set<EToolResources>} toolResourceSet - Optional filter for tool resources
* @returns {Promise<Array<MongoFile>>} Files that match the criteria
*/
const getToolFilesByIds = async (fileIds) => {
const getToolFilesByIds = async (fileIds, toolResourceSet) => {
if (!fileIds || !fileIds.length) {
return [];
}
@ -40,9 +42,19 @@ const getToolFilesByIds = async (fileIds) => {
try {
const filter = {
file_id: { $in: fileIds },
$or: [{ embedded: true }, { 'metadata.fileIdentifier': { $exists: true } }],
};
if (toolResourceSet.size) {
filter.$or = [];
}
if (toolResourceSet.has(EToolResources.file_search)) {
filter.$or.push({ embedded: true });
}
if (toolResourceSet.has(EToolResources.execute_code)) {
filter.$or.push({ 'metadata.fileIdentifier': { $exists: true } });
}
const selectFields = { text: 0 };
const sortOptions = { updatedAt: -1 };
@ -55,9 +67,9 @@ const getToolFilesByIds = async (fileIds) => {
/**
* Creates a new file with a TTL of 1 hour.
* @param {IMongoFile} data - The file data to be created, must contain file_id.
* @param {MongoFile} data - The file data to be created, must contain file_id.
* @param {boolean} disableTTL - Whether to disable the TTL.
* @returns {Promise<IMongoFile>} A promise that resolves to the created file document.
* @returns {Promise<MongoFile>} A promise that resolves to the created file document.
*/
const createFile = async (data, disableTTL) => {
const fileData = {
@ -77,8 +89,8 @@ const createFile = async (data, disableTTL) => {
/**
* Updates a file identified by file_id with new data and removes the TTL.
* @param {IMongoFile} data - The data to update, must contain file_id.
* @returns {Promise<IMongoFile>} A promise that resolves to the updated file document.
* @param {MongoFile} data - The data to update, must contain file_id.
* @returns {Promise<MongoFile>} A promise that resolves to the updated file document.
*/
const updateFile = async (data) => {
const { file_id, ...update } = data;
@ -91,8 +103,8 @@ const updateFile = async (data) => {
/**
* Increments the usage of a file identified by file_id.
* @param {IMongoFile} data - The data to update, must contain file_id and the increment value for usage.
* @returns {Promise<IMongoFile>} A promise that resolves to the updated file document.
* @param {MongoFile} data - The data to update, must contain file_id and the increment value for usage.
* @returns {Promise<MongoFile>} A promise that resolves to the updated file document.
*/
const updateFileUsage = async (data) => {
const { file_id, inc = 1 } = data;
@ -106,7 +118,7 @@ const updateFileUsage = async (data) => {
/**
* Deletes a file identified by file_id.
* @param {string} file_id - The unique identifier of the file to delete.
* @returns {Promise<IMongoFile>} A promise that resolves to the deleted file document or null.
* @returns {Promise<MongoFile>} A promise that resolves to the deleted file document or null.
*/
const deleteFile = async (file_id) => {
return await File.findOneAndDelete({ file_id }).lean();
@ -115,7 +127,7 @@ const deleteFile = async (file_id) => {
/**
* Deletes a file identified by a filter.
* @param {object} filter - The filter criteria to apply.
* @returns {Promise<IMongoFile>} A promise that resolves to the deleted file document or null.
* @returns {Promise<MongoFile>} A promise that resolves to the deleted file document or null.
*/
const deleteFileByFilter = async (filter) => {
return await File.findOneAndDelete(filter).lean();