mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-01-24 03:06:12 +01:00
🚀 Feat: Streamline File Strategies & GPT-4-Vision Settings (#1535)
* chore: fix `endpoint` typescript issues and typo in console info message * feat(api): files GET endpoint and save only file_id references to messages * refactor(client): `useGetFiles` query hook, update file types, optimistic update of filesQuery on file upload * refactor(buildTree): update to use params object and accept fileMap * feat: map files to messages; refactor(ChatView): messages only available after files are fetched * fix: fetch files only when authenticated * feat(api): AppService - rename app.locals.configs to app.locals.paths - load custom config use fileStrategy from yaml config in app.locals * refactor: separate Firebase and Local strategies, call based on config * refactor: modularize file strategies and employ with use of DALL-E * refactor(librechat.yaml): add fileStrategy field * feat: add source to MongoFile schema, as well as BatchFile, and ExtendedFile types * feat: employ file strategies for upload/delete files * refactor(deleteFirebaseFile): add user id validation for firebase file deletion * chore(deleteFirebaseFile): update jsdocs * feat: employ strategies for vision requests * fix(client): handle messages with deleted files * fix(client): ensure `filesToDelete` always saves/sends `file.source` * feat(openAI): configurable `resendImages` and `imageDetail` * refactor(getTokenCountForMessage): recursive process only when array of Objects and only their values (not keys) aside from `image_url` types * feat(OpenAIClient): calculateImageTokenCost * chore: remove comment * refactor(uploadAvatar): employ fileStrategy for avatars, from social logins or user upload * docs: update docs on how to configure fileStrategy * fix(ci): mock winston and winston related modules, update DALLE3.spec.js with changes made * refactor(redis): change terminal message to reflect current development state * fix(DALL-E-2): pass fileStrategy to dall-e
This commit is contained in:
Danny Avila
GitHub
parent
28a6807176
commit
d20970f5c5
81 changed files with 1729 additions and 855 deletions
|
|
@ -1,20 +1,13 @@
|
|||
// From https://platform.openai.com/docs/guides/images/usage?context=node
|
||||
// To use this tool, you must pass in a configured OpenAIApi object.
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { z } = require('zod');
|
||||
const OpenAI = require('openai');
|
||||
const { v4: uuidv4 } = require('uuid');
|
||||
const { Tool } = require('langchain/tools');
|
||||
const { HttpsProxyAgent } = require('https-proxy-agent');
|
||||
const {
|
||||
saveImageToFirebaseStorage,
|
||||
getFirebaseStorageImageUrl,
|
||||
getFirebaseStorage,
|
||||
} = require('~/server/services/Files/Firebase');
|
||||
const { getImageBasename } = require('~/server/services/Files/images');
|
||||
const { processFileURL } = require('~/server/services/Files/process');
|
||||
const extractBaseURL = require('~/utils/extractBaseURL');
|
||||
const saveImageFromUrl = require('../saveImageFromUrl');
|
||||
const { logger } = require('~/config');
|
||||
|
||||
const { DALLE3_SYSTEM_PROMPT, DALLE_REVERSE_PROXY, PROXY } = process.env;
|
||||
|
|
@ -23,6 +16,7 @@ class DALLE3 extends Tool {
|
|||
super();
|
||||
|
||||
this.userId = fields.userId;
|
||||
this.fileStrategy = fields.fileStrategy;
|
||||
let apiKey = fields.DALLE_API_KEY || this.getApiKey();
|
||||
const config = { apiKey };
|
||||
if (DALLE_REVERSE_PROXY) {
|
||||
|
|
@ -91,12 +85,8 @@ class DALLE3 extends Tool {
|
|||
.trim();
|
||||
}
|
||||
|
||||
getMarkdownImageUrl(imageName) {
|
||||
const imageUrl = path
|
||||
.join(this.relativeImageUrl, imageName)
|
||||
.replace(/\\/g, '/')
|
||||
.replace('public/', '');
|
||||
return ``;
|
||||
wrapInMarkdown(imageUrl) {
|
||||
return ``;
|
||||
}
|
||||
|
||||
async _call(data) {
|
||||
|
|
@ -143,43 +133,19 @@ Error Message: ${error.message}`;
|
|||
});
|
||||
}
|
||||
|
||||
this.outputPath = path.resolve(
|
||||
__dirname,
|
||||
'..',
|
||||
'..',
|
||||
'..',
|
||||
'..',
|
||||
'..',
|
||||
'client',
|
||||
'public',
|
||||
'images',
|
||||
this.userId,
|
||||
);
|
||||
const appRoot = path.resolve(__dirname, '..', '..', '..', '..', '..', 'client');
|
||||
this.relativeImageUrl = path.relative(appRoot, this.outputPath);
|
||||
try {
|
||||
const result = await processFileURL({
|
||||
fileStrategy: this.fileStrategy,
|
||||
userId: this.userId,
|
||||
URL: theImageUrl,
|
||||
fileName: imageName,
|
||||
basePath: 'images',
|
||||
});
|
||||
|
||||
// Check if directory exists, if not create it
|
||||
if (!fs.existsSync(this.outputPath)) {
|
||||
fs.mkdirSync(this.outputPath, { recursive: true });
|
||||
}
|
||||
const storage = getFirebaseStorage();
|
||||
if (storage) {
|
||||
try {
|
||||
await saveImageToFirebaseStorage(this.userId, theImageUrl, imageName);
|
||||
this.result = await getFirebaseStorageImageUrl(`${this.userId}/${imageName}`);
|
||||
logger.debug('[DALL-E-3] result: ' + this.result);
|
||||
} catch (error) {
|
||||
logger.error('Error while saving the image to Firebase Storage:', error);
|
||||
this.result = `Failed to save the image to Firebase Storage. ${error.message}`;
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
await saveImageFromUrl(theImageUrl, this.outputPath, imageName);
|
||||
this.result = this.getMarkdownImageUrl(imageName);
|
||||
} catch (error) {
|
||||
logger.error('Error while saving the image locally:', error);
|
||||
this.result = `Failed to save the image locally. ${error.message}`;
|
||||
}
|
||||
this.result = this.wrapInMarkdown(result);
|
||||
} catch (error) {
|
||||
logger.error('Error while saving the image:', error);
|
||||
this.result = `Failed to save the image locally. ${error.message}`;
|
||||
}
|
||||
|
||||
return this.result;
|
||||
|
|
|
|||
|
|
@ -1,20 +1,13 @@
|
|||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const OpenAI = require('openai');
|
||||
const DALLE3 = require('../DALLE3');
|
||||
const {
|
||||
getFirebaseStorage,
|
||||
saveImageToFirebaseStorage,
|
||||
} = require('~/server/services/Files/Firebase');
|
||||
const saveImageFromUrl = require('../../saveImageFromUrl');
|
||||
const { processFileURL } = require('~/server/services/Files/process');
|
||||
|
||||
const { logger } = require('~/config');
|
||||
|
||||
jest.mock('openai');
|
||||
|
||||
jest.mock('~/server/services/Files/Firebase', () => ({
|
||||
getFirebaseStorage: jest.fn(),
|
||||
saveImageToFirebaseStorage: jest.fn(),
|
||||
getFirebaseStorageImageUrl: jest.fn(),
|
||||
jest.mock('~/server/services/Files/process', () => ({
|
||||
processFileURL: jest.fn(),
|
||||
}));
|
||||
|
||||
jest.mock('~/server/services/Files/images', () => ({
|
||||
|
|
@ -50,10 +43,6 @@ jest.mock('fs', () => {
|
|||
};
|
||||
});
|
||||
|
||||
jest.mock('../../saveImageFromUrl', () => {
|
||||
return jest.fn();
|
||||
});
|
||||
|
||||
jest.mock('path', () => {
|
||||
return {
|
||||
resolve: jest.fn(),
|
||||
|
|
@ -99,10 +88,8 @@ describe('DALLE3', () => {
|
|||
|
||||
it('should generate markdown image URL correctly', () => {
|
||||
const imageName = 'test.png';
|
||||
path.join.mockReturnValue('images/test.png');
|
||||
path.relative.mockReturnValue('images/test.png');
|
||||
const markdownImage = dalle.getMarkdownImageUrl(imageName);
|
||||
expect(markdownImage).toBe('');
|
||||
const markdownImage = dalle.wrapInMarkdown(imageName);
|
||||
expect(markdownImage).toBe('');
|
||||
});
|
||||
|
||||
it('should call OpenAI API with correct parameters', async () => {
|
||||
|
|
@ -122,11 +109,7 @@ describe('DALLE3', () => {
|
|||
};
|
||||
|
||||
generate.mockResolvedValue(mockResponse);
|
||||
saveImageFromUrl.mockResolvedValue(true);
|
||||
fs.existsSync.mockReturnValue(true);
|
||||
path.resolve.mockReturnValue('/fakepath/images');
|
||||
path.join.mockReturnValue('/fakepath/images/img-test.png');
|
||||
path.relative.mockReturnValue('images/img-test.png');
|
||||
processFileURL.mockResolvedValue('http://example.com/img-test.png');
|
||||
|
||||
const result = await dalle._call(mockData);
|
||||
|
||||
|
|
@ -138,6 +121,7 @@ describe('DALLE3', () => {
|
|||
prompt: mockData.prompt,
|
||||
n: 1,
|
||||
});
|
||||
|
||||
expect(result).toContain('![generated image]');
|
||||
});
|
||||
|
||||
|
|
@ -184,23 +168,6 @@ describe('DALLE3', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('should create the directory if it does not exist', async () => {
|
||||
const mockData = {
|
||||
prompt: 'A test prompt',
|
||||
};
|
||||
const mockResponse = {
|
||||
data: [
|
||||
{
|
||||
url: 'http://example.com/img-test.png',
|
||||
},
|
||||
],
|
||||
};
|
||||
generate.mockResolvedValue(mockResponse);
|
||||
fs.existsSync.mockReturnValue(false); // Simulate directory does not exist
|
||||
await dalle._call(mockData);
|
||||
expect(fs.mkdirSync).toHaveBeenCalledWith(expect.any(String), { recursive: true });
|
||||
});
|
||||
|
||||
it('should log an error and return the image URL if there is an error saving the image', async () => {
|
||||
const mockData = {
|
||||
prompt: 'A test prompt',
|
||||
|
|
@ -214,31 +181,12 @@ describe('DALLE3', () => {
|
|||
};
|
||||
const error = new Error('Error while saving the image');
|
||||
generate.mockResolvedValue(mockResponse);
|
||||
saveImageFromUrl.mockRejectedValue(error);
|
||||
processFileURL.mockRejectedValue(error);
|
||||
const result = await dalle._call(mockData);
|
||||
expect(logger.error).toHaveBeenCalledWith('Error while saving the image locally:', error);
|
||||
expect(logger.error).toHaveBeenCalledWith('Error while saving the image:', error);
|
||||
expect(result).toBe('Failed to save the image locally. Error while saving the image');
|
||||
});
|
||||
|
||||
it('should save image to Firebase Storage if Firebase is initialized', async () => {
|
||||
const mockData = {
|
||||
prompt: 'A test prompt',
|
||||
};
|
||||
const mockImageUrl = 'http://example.com/img-test.png';
|
||||
const mockResponse = { data: [{ url: mockImageUrl }] };
|
||||
generate.mockResolvedValue(mockResponse);
|
||||
getFirebaseStorage.mockReturnValue({}); // Simulate Firebase being initialized
|
||||
|
||||
await dalle._call(mockData);
|
||||
|
||||
expect(getFirebaseStorage).toHaveBeenCalled();
|
||||
expect(saveImageToFirebaseStorage).toHaveBeenCalledWith(
|
||||
undefined,
|
||||
mockImageUrl,
|
||||
expect.any(String),
|
||||
);
|
||||
});
|
||||
|
||||
it('should handle error when saving image to Firebase Storage fails', async () => {
|
||||
const mockData = {
|
||||
prompt: 'A test prompt',
|
||||
|
|
@ -247,17 +195,11 @@ describe('DALLE3', () => {
|
|||
const mockResponse = { data: [{ url: mockImageUrl }] };
|
||||
const error = new Error('Error while saving to Firebase');
|
||||
generate.mockResolvedValue(mockResponse);
|
||||
getFirebaseStorage.mockReturnValue({}); // Simulate Firebase being initialized
|
||||
saveImageToFirebaseStorage.mockRejectedValue(error);
|
||||
processFileURL.mockRejectedValue(error);
|
||||
|
||||
const result = await dalle._call(mockData);
|
||||
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
'Error while saving the image to Firebase Storage:',
|
||||
error,
|
||||
);
|
||||
expect(result).toBe(
|
||||
'Failed to save the image to Firebase Storage. Error while saving to Firebase',
|
||||
);
|
||||
expect(logger.error).toHaveBeenCalledWith('Error while saving the image:', error);
|
||||
expect(result).toContain('Failed to save the image');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue