diff --git a/api/app/clients/ChatGPTClient.js b/api/app/clients/ChatGPTClient.js index 0a7f6fc7d8..104e9e5ac3 100644 --- a/api/app/clients/ChatGPTClient.js +++ b/api/app/clients/ChatGPTClient.js @@ -1,19 +1,21 @@ const Keyv = require('keyv'); const crypto = require('crypto'); +const { CohereClient } = require('cohere-ai'); +const { fetchEventSource } = require('@waylaidwanderer/fetch-event-source'); +const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken'); const { + ImageDetail, EModelEndpoint, resolveHeaders, CohereConstants, mapModelToAzureConfig, } = require('librechat-data-provider'); -const { CohereClient } = require('cohere-ai'); -const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken'); -const { fetchEventSource } = require('@waylaidwanderer/fetch-event-source'); +const { extractBaseURL, constructAzureURL, genAzureChatCompletion } = require('~/utils'); +const { createContextHandlers } = require('./prompts'); const { createCoherePayload } = require('./llm'); const { Agent, ProxyAgent } = require('undici'); const BaseClient = require('./BaseClient'); const { logger } = require('~/config'); -const { extractBaseURL, constructAzureURL, genAzureChatCompletion } = require('~/utils'); const CHATGPT_MODEL = 'gpt-3.5-turbo'; const tokenizersCache = {}; @@ -612,21 +614,66 @@ ${botMessage.message} async buildPrompt(messages, { isChatGptModel = false, promptPrefix = null }) { promptPrefix = (promptPrefix || this.options.promptPrefix || '').trim(); + + // Handle attachments and create augmentedPrompt + if (this.options.attachments) { + const attachments = await this.options.attachments; + const lastMessage = messages[messages.length - 1]; + + if (this.message_file_map) { + this.message_file_map[lastMessage.messageId] = attachments; + } else { + this.message_file_map = { + [lastMessage.messageId]: attachments, + }; + } + + const files = await this.addImageURLs(lastMessage, attachments); + this.options.attachments = files; + + this.contextHandlers = createContextHandlers(this.options.req, lastMessage.text); + } + + if (this.message_file_map) { + this.contextHandlers = createContextHandlers( + this.options.req, + messages[messages.length - 1].text, + ); + } + + // Calculate image token cost and process embedded files + messages.forEach((message, i) => { + if (this.message_file_map && this.message_file_map[message.messageId]) { + const attachments = this.message_file_map[message.messageId]; + for (const file of attachments) { + if (file.embedded) { + this.contextHandlers?.processFile(file); + continue; + } + + messages[i].tokenCount = + (messages[i].tokenCount || 0) + + this.calculateImageTokenCost({ + width: file.width, + height: file.height, + detail: this.options.imageDetail ?? ImageDetail.auto, + }); + } + } + }); + + if (this.contextHandlers) { + this.augmentedPrompt = await this.contextHandlers.createContext(); + promptPrefix = this.augmentedPrompt + promptPrefix; + } + if (promptPrefix) { // If the prompt prefix doesn't end with the end token, add it. if (!promptPrefix.endsWith(`${this.endToken}`)) { promptPrefix = `${promptPrefix.trim()}${this.endToken}\n\n`; } promptPrefix = `${this.startToken}Instructions:\n${promptPrefix}`; - } else { - const currentDateString = new Date().toLocaleDateString('en-us', { - year: 'numeric', - month: 'long', - day: 'numeric', - }); - promptPrefix = `${this.startToken}Instructions:\nYou are ChatGPT, a large language model trained by OpenAI. Respond conversationally.\nCurrent date: ${currentDateString}${this.endToken}\n\n`; } - const promptSuffix = `${this.startToken}${this.chatGptLabel}:\n`; // Prompt ChatGPT to respond. const instructionsPayload = { @@ -714,10 +761,6 @@ ${botMessage.message} this.maxResponseTokens, ); - if (this.options.debug) { - console.debug(`Prompt : ${prompt}`); - } - if (isChatGptModel) { return { prompt: [instructionsPayload, messagePayload], context }; } diff --git a/api/app/clients/specs/OpenAIClient.test.js b/api/app/clients/specs/OpenAIClient.test.js index 413150b7e3..0725efd9d8 100644 --- a/api/app/clients/specs/OpenAIClient.test.js +++ b/api/app/clients/specs/OpenAIClient.test.js @@ -611,15 +611,7 @@ describe('OpenAIClient', () => { expect(getCompletion).toHaveBeenCalled(); expect(getCompletion.mock.calls.length).toBe(1); - const currentDateString = new Date().toLocaleDateString('en-us', { - year: 'numeric', - month: 'long', - day: 'numeric', - }); - - expect(getCompletion.mock.calls[0][0]).toBe( - `||>Instructions:\nYou are ChatGPT, a large language model trained by OpenAI. Respond conversationally.\nCurrent date: ${currentDateString}\n\n||>User:\nHi mom!\n||>Assistant:\n`, - ); + expect(getCompletion.mock.calls[0][0]).toBe('||>User:\nHi mom!\n||>Assistant:\n'); expect(fetchEventSource).toHaveBeenCalled(); expect(fetchEventSource.mock.calls.length).toBe(1);