mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-03-02 06:10:18 +01:00
🚀 Feat: Streamline File Strategies & GPT-4-Vision Settings (#1535)
* chore: fix `endpoint` typescript issues and typo in console info message * feat(api): files GET endpoint and save only file_id references to messages * refactor(client): `useGetFiles` query hook, update file types, optimistic update of filesQuery on file upload * refactor(buildTree): update to use params object and accept fileMap * feat: map files to messages; refactor(ChatView): messages only available after files are fetched * fix: fetch files only when authenticated * feat(api): AppService - rename app.locals.configs to app.locals.paths - load custom config use fileStrategy from yaml config in app.locals * refactor: separate Firebase and Local strategies, call based on config * refactor: modularize file strategies and employ with use of DALL-E * refactor(librechat.yaml): add fileStrategy field * feat: add source to MongoFile schema, as well as BatchFile, and ExtendedFile types * feat: employ file strategies for upload/delete files * refactor(deleteFirebaseFile): add user id validation for firebase file deletion * chore(deleteFirebaseFile): update jsdocs * feat: employ strategies for vision requests * fix(client): handle messages with deleted files * fix(client): ensure `filesToDelete` always saves/sends `file.source` * feat(openAI): configurable `resendImages` and `imageDetail` * refactor(getTokenCountForMessage): recursive process only when array of Objects and only their values (not keys) aside from `image_url` types * feat(OpenAIClient): calculateImageTokenCost * chore: remove comment * refactor(uploadAvatar): employ fileStrategy for avatars, from social logins or user upload * docs: update docs on how to configure fileStrategy * fix(ci): mock winston and winston related modules, update DALLE3.spec.js with changes made * refactor(redis): change terminal message to reflect current development state * fix(DALL-E-2): pass fileStrategy to dall-e
This commit is contained in:
Danny Avila
GitHub
parent
28a6807176
commit
d20970f5c5
81 changed files with 1729 additions and 855 deletions
78
api/server/services/Files/images/avatar.js
Normal file
78
api/server/services/Files/images/avatar.js
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
const sharp = require('sharp');
|
||||
const fs = require('fs').promises;
|
||||
const fetch = require('node-fetch');
|
||||
const User = require('~/models/User');
|
||||
const { getStrategyFunctions } = require('~/server/services/Files/strategies');
|
||||
const { logger } = require('~/config');
|
||||
|
||||
async function convertToWebP(inputBuffer) {
|
||||
return sharp(inputBuffer).resize({ width: 150 }).toFormat('webp').toBuffer();
|
||||
}
|
||||
|
||||
/**
|
||||
* Uploads an avatar image for a user. This function can handle various types of input (URL, Buffer, or File object),
|
||||
* processes the image to a square format, converts it to WebP format, and then uses a specified file strategy for
|
||||
* further processing. It performs validation on the user ID and the input type. The function can throw errors for
|
||||
* invalid input types, fetching issues, or other processing errors.
|
||||
*
|
||||
* @param {Object} params - The parameters object.
|
||||
* @param {string} params.userId - The unique identifier of the user for whom the avatar is being uploaded.
|
||||
* @param {FileSources} params.fileStrategy - The file handling strategy to use, determining how the avatar is processed.
|
||||
* @param {(string|Buffer|File)} params.input - The input representing the avatar image. Can be a URL (string),
|
||||
* a Buffer, or a File object.
|
||||
* @param {string} params.manual - A string flag indicating whether the upload process is manual.
|
||||
*
|
||||
* @returns {Promise<any>}
|
||||
* A promise that resolves to the result of the `processAvatar` function, specific to the chosen file
|
||||
* strategy. Throws an error if any step in the process fails.
|
||||
*
|
||||
* @throws {Error} Throws an error if the user ID is undefined, the input type is invalid, the image fetching fails,
|
||||
* or any other error occurs during the processing.
|
||||
*/
|
||||
async function uploadAvatar({ userId, fileStrategy, input, manual }) {
|
||||
try {
|
||||
if (userId === undefined) {
|
||||
throw new Error('User ID is undefined');
|
||||
}
|
||||
const _id = userId;
|
||||
// TODO: remove direct use of Model, `User`
|
||||
const oldUser = await User.findOne({ _id });
|
||||
|
||||
let imageBuffer;
|
||||
if (typeof input === 'string') {
|
||||
const response = await fetch(input);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch image from URL. Status: ${response.status}`);
|
||||
}
|
||||
imageBuffer = await response.buffer();
|
||||
} else if (input instanceof Buffer) {
|
||||
imageBuffer = input;
|
||||
} else if (typeof input === 'object' && input instanceof File) {
|
||||
const fileContent = await fs.readFile(input.path);
|
||||
imageBuffer = Buffer.from(fileContent);
|
||||
} else {
|
||||
throw new Error('Invalid input type. Expected URL, Buffer, or File.');
|
||||
}
|
||||
|
||||
const { width, height } = await sharp(imageBuffer).metadata();
|
||||
const minSize = Math.min(width, height);
|
||||
const squaredBuffer = await sharp(imageBuffer)
|
||||
.extract({
|
||||
left: Math.floor((width - minSize) / 2),
|
||||
top: Math.floor((height - minSize) / 2),
|
||||
width: minSize,
|
||||
height: minSize,
|
||||
})
|
||||
.toBuffer();
|
||||
|
||||
const webPBuffer = await convertToWebP(squaredBuffer);
|
||||
const { processAvatar } = getStrategyFunctions(fileStrategy);
|
||||
return await processAvatar({ buffer: webPBuffer, User: oldUser, manual });
|
||||
} catch (error) {
|
||||
logger.error('Error uploading the avatar:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = uploadAvatar;
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
const { ref, uploadBytes, getDownloadURL } = require('firebase/storage');
|
||||
const { getFirebaseStorage } = require('~/server/services/Files/Firebase/initialize');
|
||||
const { logger } = require('~/config');
|
||||
|
||||
async function firebaseStrategy(userId, webPBuffer, oldUser, manual) {
|
||||
try {
|
||||
const storage = getFirebaseStorage();
|
||||
if (!storage) {
|
||||
throw new Error('Firebase is not initialized.');
|
||||
}
|
||||
const avatarRef = ref(storage, `images/${userId.toString()}/avatar`);
|
||||
|
||||
await uploadBytes(avatarRef, webPBuffer);
|
||||
const urlFirebase = await getDownloadURL(avatarRef);
|
||||
const isManual = manual === 'true';
|
||||
|
||||
const url = `${urlFirebase}?manual=${isManual}`;
|
||||
if (isManual) {
|
||||
oldUser.avatar = url;
|
||||
await oldUser.save();
|
||||
}
|
||||
return url;
|
||||
} catch (error) {
|
||||
logger.error('Error uploading profile picture:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = firebaseStrategy;
|
||||
|
|
@ -1,32 +0,0 @@
|
|||
const fs = require('fs').promises;
|
||||
const path = require('path');
|
||||
|
||||
async function localStrategy(userId, webPBuffer, oldUser, manual) {
|
||||
const userDir = path.resolve(
|
||||
__dirname,
|
||||
'..',
|
||||
'..',
|
||||
'..',
|
||||
'..',
|
||||
'..',
|
||||
'..',
|
||||
'client',
|
||||
'public',
|
||||
'images',
|
||||
userId,
|
||||
);
|
||||
let avatarPath = path.join(userDir, 'avatar.png');
|
||||
const urlRoute = `/images/${userId}/avatar.png`;
|
||||
await fs.mkdir(userDir, { recursive: true });
|
||||
await fs.writeFile(avatarPath, webPBuffer);
|
||||
const isManual = manual === 'true';
|
||||
let url = `${urlRoute}?manual=${isManual}×tamp=${new Date().getTime()}`;
|
||||
if (isManual) {
|
||||
oldUser.avatar = url;
|
||||
await oldUser.save();
|
||||
}
|
||||
|
||||
return url;
|
||||
}
|
||||
|
||||
module.exports = localStrategy;
|
||||
|
|
@ -1,63 +0,0 @@
|
|||
const sharp = require('sharp');
|
||||
const fetch = require('node-fetch');
|
||||
const fs = require('fs').promises;
|
||||
const User = require('~/models/User');
|
||||
const { getFirebaseStorage } = require('~/server/services/Files/Firebase/initialize');
|
||||
const firebaseStrategy = require('./firebaseStrategy');
|
||||
const localStrategy = require('./localStrategy');
|
||||
const { logger } = require('~/config');
|
||||
|
||||
async function convertToWebP(inputBuffer) {
|
||||
return sharp(inputBuffer).resize({ width: 150 }).toFormat('webp').toBuffer();
|
||||
}
|
||||
|
||||
async function uploadAvatar(userId, input, manual) {
|
||||
try {
|
||||
if (userId === undefined) {
|
||||
throw new Error('User ID is undefined');
|
||||
}
|
||||
const _id = userId;
|
||||
// TODO: remove direct use of Model, `User`
|
||||
const oldUser = await User.findOne({ _id });
|
||||
let imageBuffer;
|
||||
if (typeof input === 'string') {
|
||||
const response = await fetch(input);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch image from URL. Status: ${response.status}`);
|
||||
}
|
||||
imageBuffer = await response.buffer();
|
||||
} else if (input instanceof Buffer) {
|
||||
imageBuffer = input;
|
||||
} else if (typeof input === 'object' && input instanceof File) {
|
||||
const fileContent = await fs.readFile(input.path);
|
||||
imageBuffer = Buffer.from(fileContent);
|
||||
} else {
|
||||
throw new Error('Invalid input type. Expected URL, Buffer, or File.');
|
||||
}
|
||||
const { width, height } = await sharp(imageBuffer).metadata();
|
||||
const minSize = Math.min(width, height);
|
||||
const squaredBuffer = await sharp(imageBuffer)
|
||||
.extract({
|
||||
left: Math.floor((width - minSize) / 2),
|
||||
top: Math.floor((height - minSize) / 2),
|
||||
width: minSize,
|
||||
height: minSize,
|
||||
})
|
||||
.toBuffer();
|
||||
const webPBuffer = await convertToWebP(squaredBuffer);
|
||||
const storage = getFirebaseStorage();
|
||||
if (storage) {
|
||||
const url = await firebaseStrategy(userId, webPBuffer, oldUser, manual);
|
||||
return url;
|
||||
}
|
||||
|
||||
const url = await localStrategy(userId, webPBuffer, oldUser, manual);
|
||||
return url;
|
||||
} catch (error) {
|
||||
logger.error('Error uploading the avatar:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = uploadAvatar;
|
||||
|
|
@ -1,36 +0,0 @@
|
|||
const path = require('path');
|
||||
const sharp = require('sharp');
|
||||
const fs = require('fs');
|
||||
const { resizeImage } = require('./resize');
|
||||
|
||||
async function convertToWebP(req, file, resolution = 'high') {
|
||||
const inputFilePath = file.path;
|
||||
const { buffer: resizedBuffer, width, height } = await resizeImage(inputFilePath, resolution);
|
||||
const extension = path.extname(inputFilePath);
|
||||
|
||||
const { imageOutput } = req.app.locals.config;
|
||||
const userPath = path.join(imageOutput, req.user.id);
|
||||
|
||||
if (!fs.existsSync(userPath)) {
|
||||
fs.mkdirSync(userPath, { recursive: true });
|
||||
}
|
||||
|
||||
const newPath = path.join(userPath, path.basename(inputFilePath));
|
||||
|
||||
if (extension.toLowerCase() === '.webp') {
|
||||
const bytes = Buffer.byteLength(resizedBuffer);
|
||||
await fs.promises.writeFile(newPath, resizedBuffer);
|
||||
const filepath = path.posix.join('/', 'images', req.user.id, path.basename(newPath));
|
||||
return { filepath, bytes, width, height };
|
||||
}
|
||||
|
||||
const outputFilePath = newPath.replace(extension, '.webp');
|
||||
const data = await sharp(resizedBuffer).toFormat('webp').toBuffer();
|
||||
await fs.promises.writeFile(outputFilePath, data);
|
||||
const bytes = Buffer.byteLength(data);
|
||||
const filepath = path.posix.join('/', 'images', req.user.id, path.basename(outputFilePath));
|
||||
await fs.promises.unlink(inputFilePath);
|
||||
return { filepath, bytes, width, height };
|
||||
}
|
||||
|
||||
module.exports = { convertToWebP };
|
||||
|
|
@ -1,45 +1,5 @@
|
|||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { EModelEndpoint } = require('librechat-data-provider');
|
||||
const { updateFile } = require('~/models');
|
||||
|
||||
/**
|
||||
* Encodes an image file to base64.
|
||||
* @param {string} imagePath - The path to the image file.
|
||||
* @returns {Promise<string>} A promise that resolves with the base64 encoded image data.
|
||||
*/
|
||||
function encodeImage(imagePath) {
|
||||
return new Promise((resolve, reject) => {
|
||||
fs.readFile(imagePath, (err, data) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
} else {
|
||||
resolve(data.toString('base64'));
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates the file and encodes the image.
|
||||
* @param {Object} req - The request object.
|
||||
* @param {Object} file - The file object.
|
||||
* @returns {Promise<[MongoFile, string]>} - A promise that resolves to an array of results from updateFile and encodeImage.
|
||||
*/
|
||||
async function updateAndEncode(req, file) {
|
||||
const { publicPath, imageOutput } = req.app.locals.config;
|
||||
const userPath = path.join(imageOutput, req.user.id);
|
||||
|
||||
if (!fs.existsSync(userPath)) {
|
||||
fs.mkdirSync(userPath, { recursive: true });
|
||||
}
|
||||
const filepath = path.join(publicPath, file.filepath);
|
||||
|
||||
const promises = [];
|
||||
promises.push(updateFile({ file_id: file.file_id }));
|
||||
promises.push(encodeImage(filepath));
|
||||
return await Promise.all(promises);
|
||||
}
|
||||
const { EModelEndpoint, FileSources } = require('librechat-data-provider');
|
||||
const { getStrategyFunctions } = require('../strategies');
|
||||
|
||||
/**
|
||||
* Encodes and formats the given files.
|
||||
|
|
@ -50,25 +10,42 @@ async function updateAndEncode(req, file) {
|
|||
*/
|
||||
async function encodeAndFormat(req, files, endpoint) {
|
||||
const promises = [];
|
||||
const encodingMethods = {};
|
||||
|
||||
for (let file of files) {
|
||||
promises.push(updateAndEncode(req, file));
|
||||
const source = file.source ?? FileSources.local;
|
||||
|
||||
if (encodingMethods[source]) {
|
||||
promises.push(encodingMethods[source](req, file));
|
||||
continue;
|
||||
}
|
||||
|
||||
const { prepareImagePayload } = getStrategyFunctions(source);
|
||||
if (!prepareImagePayload) {
|
||||
throw new Error(`Encoding function not implemented for ${source}`);
|
||||
}
|
||||
|
||||
encodingMethods[source] = prepareImagePayload;
|
||||
promises.push(prepareImagePayload(req, file));
|
||||
}
|
||||
|
||||
// TODO: make detail configurable, as of now resizing is done
|
||||
// to prefer "high" but "low" may be used if the image is small enough
|
||||
const detail = req.body.detail ?? 'auto';
|
||||
const encodedImages = await Promise.all(promises);
|
||||
const detail = req.body.imageDetail ?? 'auto';
|
||||
|
||||
/** @type {Array<[MongoFile, string]>} */
|
||||
const formattedImages = await Promise.all(promises);
|
||||
|
||||
const result = {
|
||||
files: [],
|
||||
image_urls: [],
|
||||
};
|
||||
|
||||
for (const [file, base64] of encodedImages) {
|
||||
for (const [file, imageContent] of formattedImages) {
|
||||
const imagePart = {
|
||||
type: 'image_url',
|
||||
image_url: {
|
||||
url: `data:image/webp;base64,${base64}`,
|
||||
url: imageContent.startsWith('http')
|
||||
? imageContent
|
||||
: `data:image/webp;base64,${imageContent}`,
|
||||
detail,
|
||||
},
|
||||
};
|
||||
|
|
@ -81,17 +58,16 @@ async function encodeAndFormat(req, files, endpoint) {
|
|||
|
||||
result.files.push({
|
||||
file_id: file.file_id,
|
||||
filepath: file.filepath,
|
||||
filename: file.filename,
|
||||
type: file.type,
|
||||
height: file.height,
|
||||
width: file.width,
|
||||
// filepath: file.filepath,
|
||||
// filename: file.filename,
|
||||
// type: file.type,
|
||||
// height: file.height,
|
||||
// width: file.width,
|
||||
});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
encodeImage,
|
||||
encodeAndFormat,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,15 +1,13 @@
|
|||
const convert = require('./convert');
|
||||
const avatar = require('./avatar');
|
||||
const encode = require('./encode');
|
||||
const parse = require('./parse');
|
||||
const resize = require('./resize');
|
||||
const validate = require('./validate');
|
||||
const uploadAvatar = require('./avatar/uploadAvatar');
|
||||
|
||||
module.exports = {
|
||||
...convert,
|
||||
...encode,
|
||||
...parse,
|
||||
...resize,
|
||||
...validate,
|
||||
uploadAvatar,
|
||||
avatar,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
const URL = require('url').URL;
|
||||
const path = require('path');
|
||||
|
||||
const imageExtensionRegex = /\.(jpg|jpeg|png|gif|bmp|tiff|svg)$/i;
|
||||
const imageExtensionRegex = /\.(jpg|jpeg|png|gif|bmp|tiff|svg|webp)$/i;
|
||||
|
||||
/**
|
||||
* Extracts the image basename from a given URL.
|
||||
|
|
@ -22,6 +22,24 @@ function getImageBasename(urlString) {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts the basename of a file from a given URL.
|
||||
*
|
||||
* @param {string} urlString - The URL string from which the file basename is to be extracted.
|
||||
* @returns {string} The basename of the file from the URL.
|
||||
* Returns an empty string if the URL parsing fails.
|
||||
*/
|
||||
function getFileBasename(urlString) {
|
||||
try {
|
||||
const url = new URL(urlString);
|
||||
return path.basename(url.pathname);
|
||||
} catch (error) {
|
||||
// If URL parsing fails, return an empty string
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
getImageBasename,
|
||||
getFileBasename,
|
||||
};
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue