diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/app/clients/AnthropicClient.js b/api/app/clients/AnthropicClient.js index a3fba29d5c..18f926c804 100644 --- a/api/app/clients/AnthropicClient.js +++ b/api/app/clients/AnthropicClient.js @@ -27,6 +27,7 @@ const { const { getModelMaxTokens, getModelMaxOutputTokens, matchModelName } = require('~/utils'); const { spendTokens, spendStructuredTokens } = require('~/models/spendTokens'); const { encodeAndFormat } = require('~/server/services/Files/images/encode'); +const { encodeAndFormatDocuments } = require('~/server/services/Files/documents'); const { sleep } = require('~/server/utils'); const BaseClient = require('./BaseClient'); const { logger } = require('~/config'); @@ -312,6 +313,33 @@ class AnthropicClient extends BaseClient { return files; } + async addDocuments(message, attachments) { + // Only process documents + const documentResult = await encodeAndFormatDocuments( + this.options.req, + attachments, + EModelEndpoint.anthropic, + ); + + message.documents = + documentResult.documents && documentResult.documents.length + ? documentResult.documents + : undefined; + + return documentResult.files; + } + + async processAttachments(message, attachments) { + // Process both images and documents + const [imageFiles, documentFiles] = await Promise.all([ + this.addImageURLs(message, attachments), + this.addDocuments(message, attachments), + ]); + + // Combine files from both processors + return [...imageFiles, ...documentFiles]; + } + /** * @param {object} params * @param {number} params.promptTokens @@ -382,7 +410,7 @@ class AnthropicClient extends BaseClient { }; } - const files = await this.addImageURLs(latestMessage, attachments); + const files = await this.processAttachments(latestMessage, attachments); this.options.attachments = files; } @@ -941,7 +969,7 @@ class AnthropicClient extends BaseClient { const content = ` ${convo} - + Please generate a title for this conversation.`; const titleMessage = { role: 'user', content }; diff --git a/api/app/clients/BaseClient.js b/api/app/clients/BaseClient.js index 19f7073fe2..03a557c4d7 100644 --- a/api/app/clients/BaseClient.js +++ b/api/app/clients/BaseClient.js @@ -1233,7 +1233,7 @@ class BaseClient { {}, ); - await this.addImageURLs(message, files, this.visionMode); + await this.processAttachments(message, files, this.visionMode); this.message_file_map[message.messageId] = files; return message; diff --git a/api/app/clients/GoogleClient.js b/api/app/clients/GoogleClient.js index 2ec23a0a06..989654c7e3 100644 --- a/api/app/clients/GoogleClient.js +++ b/api/app/clients/GoogleClient.js @@ -268,7 +268,7 @@ class GoogleClient extends BaseClient { const formattedMessages = []; const attachments = await this.options.attachments; const latestMessage = { ...messages[messages.length - 1] }; - const files = await this.addImageURLs(latestMessage, attachments, VisionModes.generative); + const files = await this.processAttachments(latestMessage, attachments, VisionModes.generative); this.options.attachments = files; messages[messages.length - 1] = latestMessage; @@ -312,6 +312,20 @@ class GoogleClient extends BaseClient { return files; } + // eslint-disable-next-line no-unused-vars + async addDocuments(message, attachments) { + // GoogleClient doesn't support document processing yet + // Return empty results for consistency + return []; + } + + async processAttachments(message, attachments, mode = '') { + // For GoogleClient, only process images + const imageFiles = await this.addImageURLs(message, attachments, mode); + const documentFiles = await this.addDocuments(message, attachments); + return [...imageFiles, ...documentFiles]; + } + /** * Builds the augmented prompt for attachments * TODO: Add File API Support @@ -345,7 +359,7 @@ class GoogleClient extends BaseClient { const { prompt } = await this.buildMessagesPrompt(messages, parentMessageId); - const files = await this.addImageURLs(latestMessage, attachments); + const files = await this.processAttachments(latestMessage, attachments); this.options.attachments = files; diff --git a/api/app/clients/OpenAIClient.js b/api/app/clients/OpenAIClient.js index 700fca6a7b..99df601878 100644 --- a/api/app/clients/OpenAIClient.js +++ b/api/app/clients/OpenAIClient.js @@ -372,6 +372,19 @@ class OpenAIClient extends BaseClient { return files; } + async addDocuments(message, attachments) { + // OpenAI doesn't support native document processing yet + // Return empty results for consistency + return []; + } + + async processAttachments(message, attachments) { + // For OpenAI, only process images + const imageFiles = await this.addImageURLs(message, attachments); + const documentFiles = await this.addDocuments(message, attachments); + return [...imageFiles, ...documentFiles]; + } + async buildMessages(messages, parentMessageId, { promptPrefix = null }, opts) { let orderedMessages = this.constructor.getMessagesForConversation({ messages, @@ -400,7 +413,7 @@ class OpenAIClient extends BaseClient { }; } - const files = await this.addImageURLs( + const files = await this.processAttachments( orderedMessages[orderedMessages.length - 1], attachments, ); diff --git a/api/app/clients/memory/example.js b/api/app/clients/memory/example.js index 49bf9dde08..7e59bc2a18 100644 --- a/api/app/clients/memory/example.js +++ b/api/app/clients/memory/example.js @@ -22,17 +22,17 @@ '\n' + 'Celestia had the power to grant one wish to anyone who dared to find her abode. Ethan, captivated by the tales he had read and yearning for something greater, approached the cottage with trepidation. When he shared his desire to embark on a grand adventure, Celestia smiled warmly and agreed to grant his wish.\n' + '\n' + - 'With a wave of her wand and a sprinkle of stardust, Celestia bestowed upon Ethan a magical necklace. This necklace, adorned with a rare gemstone called the Eye of Imagination, had the power to turn dreams and imagination into reality. From that moment forward, Ethan\'s every thought and idea became manifest.\n' + + "With a wave of her wand and a sprinkle of stardust, Celestia bestowed upon Ethan a magical necklace. This necklace, adorned with a rare gemstone called the Eye of Imagination, had the power to turn dreams and imagination into reality. From that moment forward, Ethan's every thought and idea became manifest.\n" + '\n' + 'Energized by this newfound power, Ethan continued his journey, encountering mythical creatures, solving riddles, and overcoming treacherous obstacles along the way. With the Eye of Imagination, he brought life to ancient statues, unlocked hidden doors, and even tamed fiery dragons.\n' + '\n' + 'As days turned into weeks and weeks into months, Ethan became wiser and more in tune with the world around him. He learned that true adventure was not merely about seeking thrills and conquering the unknown, but also about fostering compassion, friendship, and a deep appreciation for the beauty of the ordinary.\n' + '\n' + - 'Eventually, Ethan\'s journey led him back to his village. With the Eye of Imagination, he transformed the village into a place of wonders and endless possibilities. Fields blossomed into vibrant gardens, simple tools turned into intricate works of art, and the villagers felt a renewed sense of hope and inspiration.\n' + + "Eventually, Ethan's journey led him back to his village. With the Eye of Imagination, he transformed the village into a place of wonders and endless possibilities. Fields blossomed into vibrant gardens, simple tools turned into intricate works of art, and the villagers felt a renewed sense of hope and inspiration.\n" + '\n' + - 'Ethan, now known as the Village Magician, realized that the true magic lied within everyone\'s hearts. He taught the villagers to embrace their creativity, to dream big, and to never underestimate the power of imagination. And so, the village flourished, becoming a beacon of wonder and creativity for all to see.\n' + + "Ethan, now known as the Village Magician, realized that the true magic lied within everyone's hearts. He taught the villagers to embrace their creativity, to dream big, and to never underestimate the power of imagination. And so, the village flourished, becoming a beacon of wonder and creativity for all to see.\n" + '\n' + - 'In the years that followed, Ethan\'s adventures continued, though mostly within the confines of his beloved village. But he never forgot the thrill of that first grand adventure. And every now and then, when looking up at the starry night sky, he would allow his mind to wander, knowing that the greatest adventures were still waiting to be discovered.', + "In the years that followed, Ethan's adventures continued, though mostly within the confines of his beloved village. But he never forgot the thrill of that first grand adventure. And every now and then, when looking up at the starry night sky, he would allow his mind to wander, knowing that the greatest adventures were still waiting to be discovered.", }, { role: 'user', @@ -41,30 +41,30 @@ '\n' + 'Once there was a young lad by the name of Ethan, raised in a little hamlet nestled betwixt the verdant knolls, who possessed an irrepressible yearning for knowledge, a thirst unquenchable and a spirit teeming with curiosity. As the golden sun bathed the bucolic land in its effulgent light, he would tread through the village, his ears attuned to the tales spun by the townsfolk, his eyes absorbing the tapestry woven by the world surrounding him.\n' + '\n' + - 'One radiant day, whilst exploring the periphery of the settlement, Ethan chanced upon a timeworn tome, ensconced amidst the roots of an ancient oak, cloaked in the shroud of neglect. The dust gathered upon it spoke of time\'s relentless march. A book of fairy tales – garnished with vivid descriptions of mystical woods, fantastical beasts, and ventures daring beyond the ordinary humdrum existence. Intrigued and beguiled, Ethan pried open the weathered pages and succumbed to their beckoning whispers.\n' + + "One radiant day, whilst exploring the periphery of the settlement, Ethan chanced upon a timeworn tome, ensconced amidst the roots of an ancient oak, cloaked in the shroud of neglect. The dust gathered upon it spoke of time's relentless march. A book of fairy tales – garnished with vivid descriptions of mystical woods, fantastical beasts, and ventures daring beyond the ordinary humdrum existence. Intrigued and beguiled, Ethan pried open the weathered pages and succumbed to their beckoning whispers.\n" + '\n' + - 'In each tale, he was transported to a realm of enchantment and wonderment, inexorably tugging at the strings of his yearning for peripatetic exploration. Inspired by the narratives he had devoured, Ethan resolved to bid adieu to kinfolk and embark upon a sojourn, with dreams of procuring a firsthand glimpse into the domain of mystique that lay beyond the village\'s circumscribed boundary.\n' + + "In each tale, he was transported to a realm of enchantment and wonderment, inexorably tugging at the strings of his yearning for peripatetic exploration. Inspired by the narratives he had devoured, Ethan resolved to bid adieu to kinfolk and embark upon a sojourn, with dreams of procuring a firsthand glimpse into the domain of mystique that lay beyond the village's circumscribed boundary.\n" + '\n' + 'Thus, he bade tearful farewells, girding himself for a path that guided him to a dense and captivating woodland, whispered of as a sanctuary to mythical beings and clandestine troves of treasures. As Ethan plunged deeper into the heart of the arboreal labyrinth, he felt a palpable surge of electricity, as though the sylvan sentinels whispered enigmatic secrets that only the perceptive ear could discern.\n' + '\n' + - 'It wasn\'t long before his path intertwined with that of a capricious sprite christened Sparkle, bearing an impish grin and eyes sparkling with mischief. Sparkle played the role of Virgil to Ethan\'s Dante, guiding him through the intricate tapestry of arboreal scions, issuing warnings of perils concealed and spinning tales of ancient entities that called this very bosky enclave home.\n' + + "It wasn't long before his path intertwined with that of a capricious sprite christened Sparkle, bearing an impish grin and eyes sparkling with mischief. Sparkle played the role of Virgil to Ethan's Dante, guiding him through the intricate tapestry of arboreal scions, issuing warnings of perils concealed and spinning tales of ancient entities that called this very bosky enclave home.\n" + '\n' + 'Together, they stumbled upon a luminous lake, its shimmering waters imbued with a celestial light. At the center lay a diminutive island, upon which reposed a cottage fashioned from tender petals and verdant leaves. It belonged to an ancient sorceress of considerable wisdom, Celestia by name.\n' + '\n' + - 'Celestia, with her power to bestow a single wish on any intrepid soul who happened upon her abode, met Ethan\'s desire with a congenial nod, his fervor for a grand expedition not lost on her penetrating gaze. In response, she bequeathed unto him a necklace of magical manufacture – adorned with the rare gemstone known as the Eye of Imagination – whose very essence transformed dreams into vivid reality. From that moment forward, not a single cogitation nor nebulous fanciful notion of Ethan\'s ever lacked physicality.\n' + + "Celestia, with her power to bestow a single wish on any intrepid soul who happened upon her abode, met Ethan's desire with a congenial nod, his fervor for a grand expedition not lost on her penetrating gaze. In response, she bequeathed unto him a necklace of magical manufacture – adorned with the rare gemstone known as the Eye of Imagination – whose very essence transformed dreams into vivid reality. From that moment forward, not a single cogitation nor nebulous fanciful notion of Ethan's ever lacked physicality.\n" + '\n' + 'Energized by this newfound potency, Ethan continued his sojourn, encountering mythical creatures, unraveling cerebral enigmas, and braving perils aplenty along the winding roads of destiny. Armed with the Eye of Imagination, he brought forth life from immobile statuary, unlocked forbidding portals, and even tamed the ferocious beasts of yore – their fiery breath reduced to a whisper.\n' + '\n' + - 'As the weeks metamorphosed into months, Ethan grew wiser and more attuned to the ebb and flow of the world enveloping him. He gleaned that true adventure isn\'t solely confined to sating a thirst for adrenaline and conquering the unknown; indeed, it resides in fostering compassion, fostering amicable bonds, and cherishing the beauty entwined within the quotidian veld.\n' + + "As the weeks metamorphosed into months, Ethan grew wiser and more attuned to the ebb and flow of the world enveloping him. He gleaned that true adventure isn't solely confined to sating a thirst for adrenaline and conquering the unknown; indeed, it resides in fostering compassion, fostering amicable bonds, and cherishing the beauty entwined within the quotidian veld.\n" + '\n' + - 'Eventually, Ethan\'s quest drew him homeward, back to his village. Buoying the Eye of Imagination\'s ethereal power, he imbued the hitherto unremarkable settlement with the patina of infinite possibilities. The bounteous fields bloomed into kaleidoscopic gardens, simple instruments transmuting into intricate masterpieces, and the villagers themselves clasped within their hearts a renewed ardor, a conflagration of hope and inspiration.\n' + + "Eventually, Ethan's quest drew him homeward, back to his village. Buoying the Eye of Imagination's ethereal power, he imbued the hitherto unremarkable settlement with the patina of infinite possibilities. The bounteous fields bloomed into kaleidoscopic gardens, simple instruments transmuting into intricate masterpieces, and the villagers themselves clasped within their hearts a renewed ardor, a conflagration of hope and inspiration.\n" + '\n' + - 'Behold Ethan, at present hailed as the Village Magician – a cognomen befitting his sorcery wielded within the confines of the community he adored. His exploits may have become tethered to a geographical locale, but the flame kindled by that premier sojourn never flickered. Occasionally, beneath the veiled twilight canopy, when peering skyward toward the astral canvas bedecked with countless luminescent diamonds, he\'d allow his mind to wander, cognizant of boundless adventures yet to take flight, forever awaiting his indomitable spirit.', + "Behold Ethan, at present hailed as the Village Magician – a cognomen befitting his sorcery wielded within the confines of the community he adored. His exploits may have become tethered to a geographical locale, but the flame kindled by that premier sojourn never flickered. Occasionally, beneath the veiled twilight canopy, when peering skyward toward the astral canvas bedecked with countless luminescent diamonds, he'd allow his mind to wander, cognizant of boundless adventures yet to take flight, forever awaiting his indomitable spirit.", }, { role: 'assistant', content: - 'Beautifully rewritten! It captures the essence of Cormac McCarthy\'s style, with its descriptive language and evocative imagery. The story retains the spirit of adventure and the power of imagination seen in the original. Well done! If you have any more requests or if there\'s anything else I can assist you with, feel free to let me know.', + "Beautifully rewritten! It captures the essence of Cormac McCarthy's style, with its descriptive language and evocative imagery. The story retains the spirit of adventure and the power of imagination seen in the original. Well done! If you have any more requests or if there's anything else I can assist you with, feel free to let me know.", }, { role: 'user', @@ -73,19 +73,19 @@ { role: 'assistant', content: - 'Certainly! Here\'s a short story reflecting Cormac McCarthy\'s style:\n' + + "Certainly! Here's a short story reflecting Cormac McCarthy's style:\n" + '\n' + 'In a desolate wasteland scarred by the sins of mankind, a lone traveler ventured forth, his name forgotten by time, etched upon the shadows of memory. He navigated the barren plains, where the sun beat down, merciless and unrelenting, scorching the parched earth beneath his weary boots.\n' + '\n' + 'Desolation was his only companion, the forsaken landscape a testament to the darkness that had befallen this forsaken realm. The wind whispered secrets long lost, carrying with it the echoes of a civilization long since undone, swallowed by the abyss.\n' + '\n' + - 'Amongst the shattered remnants of fallen cities, he sought solace, a glimmer of hope amidst the desolation. Rusted remnants of steel and concrete stood as sentinels of mankind\'s hubris, their jagged edges a chilling reminder of the fragility of all that was once deemed invincible.\n' + + "Amongst the shattered remnants of fallen cities, he sought solace, a glimmer of hope amidst the desolation. Rusted remnants of steel and concrete stood as sentinels of mankind's hubris, their jagged edges a chilling reminder of the fragility of all that was once deemed invincible.\n" + '\n' + 'As twilight descended, casting a pallid hue upon the lifeless land, he happened upon a dilapidated house, its walls crumbling under the weight of time. Drawn to the desolation, he pushed open the creaking door, its rusty hinges screaming in protest.\n' + '\n' + 'Inside, a world frozen in time awaited him. Dust particles danced through the air, illuminated by slivers of light that pierced the broken windows. Here, the past lay trapped, a melancholic tableau of lives lost and dreams shattered.\n' + '\n' + - 'He wandered through the empty rooms, bare walls whispering stories untold. In the corner, a dilapidated piano stood, its ivory keys long since abandoned by a musician\'s touch. He approached it cautiously, a faint glimmer of anticipation igniting in his weather-worn eyes.\n' + + "He wandered through the empty rooms, bare walls whispering stories untold. In the corner, a dilapidated piano stood, its ivory keys long since abandoned by a musician's touch. He approached it cautiously, a faint glimmer of anticipation igniting in his weather-worn eyes.\n" + '\n' + 'Determined, he carefully brushed away the dust and debris that cloaked the instrument. With trembling hands, he pressed a key, and a discordant note filled the air, mingling with the sighs of forgotten melodies. The sound echoed through the silence, bringing with it the remnants of forgotten symphonies.\n' + '\n' + @@ -103,13 +103,13 @@ { role: 'assistant', content: - 'Thank you! I\'m glad you enjoyed it. If you have any more writing requests or if there\'s anything else I can assist you with, feel free to let me know.', + "Thank you! I'm glad you enjoyed it. If you have any more writing requests or if there's anything else I can assist you with, feel free to let me know.", }, { role: 'user', content: 'you are very helpful' }, { role: 'assistant', content: - 'Thank you for your kind words! I\'m here to assist you in any way I can. If you have any more questions, need further assistance, or just want to chat, feel free to reach out.', + "Thank you for your kind words! I'm here to assist you in any way I can. If you have any more questions, need further assistance, or just want to chat, feel free to reach out.", }, { role: 'user', content: 'no you man' }, ]; diff --git a/api/app/clients/memory/summaryBuffer.demo.js b/api/app/clients/memory/summaryBuffer.demo.js index fc575c3032..b88e6b411c 100644 --- a/api/app/clients/memory/summaryBuffer.demo.js +++ b/api/app/clients/memory/summaryBuffer.demo.js @@ -9,7 +9,7 @@ const chatPromptMemory = new ConversationSummaryBufferMemory({ }); (async () => { - await chatPromptMemory.saveContext({ input: 'hi my name\'s Danny' }, { output: 'whats up' }); + await chatPromptMemory.saveContext({ input: "hi my name's Danny" }, { output: 'whats up' }); await chatPromptMemory.saveContext({ input: 'not much you' }, { output: 'not much' }); await chatPromptMemory.saveContext( { input: 'are you excited for the olympics?' }, diff --git a/api/app/clients/output_parsers/addImages.spec.js b/api/app/clients/output_parsers/addImages.spec.js index 7c5a04137e..8e4e74428a 100644 --- a/api/app/clients/output_parsers/addImages.spec.js +++ b/api/app/clients/output_parsers/addImages.spec.js @@ -74,7 +74,7 @@ describe('addImages', () => { it('should append correctly from a real scenario', () => { responseMessage.text = - 'Here is the generated image based on your request. It depicts a surreal landscape filled with floating musical notes. The style is impressionistic, with vibrant sunset hues dominating the scene. At the center, there\'s a silhouette of a grand piano, adding a dreamy emotion to the overall image. This could serve as a unique and creative music album cover. Would you like to make any changes or generate another image?'; + "Here is the generated image based on your request. It depicts a surreal landscape filled with floating musical notes. The style is impressionistic, with vibrant sunset hues dominating the scene. At the center, there's a silhouette of a grand piano, adding a dreamy emotion to the overall image. This could serve as a unique and creative music album cover. Would you like to make any changes or generate another image?"; const originalText = responseMessage.text; const imageMarkdown = '![generated image](/images/img-RnVWaYo2Yg4x3e0isICiMuf5.png)'; intermediateSteps.push({ observation: imageMarkdown }); diff --git a/api/app/clients/output_parsers/handleOutputs.js b/api/app/clients/output_parsers/handleOutputs.js index b25eaaad80..fc04d73a16 100644 --- a/api/app/clients/output_parsers/handleOutputs.js +++ b/api/app/clients/output_parsers/handleOutputs.js @@ -65,14 +65,14 @@ function buildPromptPrefix({ result, message, functionsAgent }) { const preliminaryAnswer = result.output?.length > 0 ? `Preliminary Answer: "${result.output.trim()}"` : ''; const prefix = preliminaryAnswer - ? 'review and improve the answer you generated using plugins in response to the User Message below. The user hasn\'t seen your answer or thoughts yet.' + ? "review and improve the answer you generated using plugins in response to the User Message below. The user hasn't seen your answer or thoughts yet." : 'respond to the User Message below based on your preliminary thoughts & actions.'; return `As a helpful AI Assistant, ${prefix}${errorMessage}\n${internalActions} ${preliminaryAnswer} Reply conversationally to the User based on your ${ - preliminaryAnswer ? 'preliminary answer, ' : '' -}internal actions, thoughts, and observations, making improvements wherever possible, but do not modify URLs. + preliminaryAnswer ? 'preliminary answer, ' : '' + }internal actions, thoughts, and observations, making improvements wherever possible, but do not modify URLs. ${ preliminaryAnswer ? '' diff --git a/api/app/clients/prompts/addCacheControl.spec.js b/api/app/clients/prompts/addCacheControl.spec.js index c46ffd95e3..7bf43aabcc 100644 --- a/api/app/clients/prompts/addCacheControl.spec.js +++ b/api/app/clients/prompts/addCacheControl.spec.js @@ -6,7 +6,7 @@ describe('addCacheControl', () => { { role: 'user', content: [{ type: 'text', text: 'Hello' }] }, { role: 'assistant', content: [{ type: 'text', text: 'Hi there' }] }, { role: 'user', content: [{ type: 'text', text: 'How are you?' }] }, - { role: 'assistant', content: [{ type: 'text', text: 'I\'m doing well, thanks!' }] }, + { role: 'assistant', content: [{ type: 'text', text: "I'm doing well, thanks!" }] }, { role: 'user', content: [{ type: 'text', text: 'Great!' }] }, ]; @@ -22,7 +22,7 @@ describe('addCacheControl', () => { { role: 'user', content: 'Hello' }, { role: 'assistant', content: 'Hi there' }, { role: 'user', content: 'How are you?' }, - { role: 'assistant', content: 'I\'m doing well, thanks!' }, + { role: 'assistant', content: "I'm doing well, thanks!" }, { role: 'user', content: 'Great!' }, ]; @@ -140,7 +140,7 @@ describe('addCacheControl', () => { { role: 'user', content: 'Hello' }, { role: 'assistant', content: 'Hi there' }, { role: 'user', content: [{ type: 'text', text: 'How are you?' }] }, - { role: 'assistant', content: 'I\'m doing well, thanks!' }, + { role: 'assistant', content: "I'm doing well, thanks!" }, { role: 'user', content: 'Great!' }, ]; @@ -160,7 +160,7 @@ describe('addCacheControl', () => { }, ]); expect(result[1].content).toBe('Hi there'); - expect(result[3].content).toBe('I\'m doing well, thanks!'); + expect(result[3].content).toBe("I'm doing well, thanks!"); }); test('should handle edge case with multiple content types', () => { diff --git a/api/app/clients/prompts/formatAgentMessages.spec.js b/api/app/clients/prompts/formatAgentMessages.spec.js index 360fa00a34..1aee3edf71 100644 --- a/api/app/clients/prompts/formatAgentMessages.spec.js +++ b/api/app/clients/prompts/formatAgentMessages.spec.js @@ -130,7 +130,7 @@ describe('formatAgentMessages', () => { content: [ { type: ContentTypes.TEXT, - [ContentTypes.TEXT]: 'I\'ll search for that information.', + [ContentTypes.TEXT]: "I'll search for that information.", tool_call_ids: ['search_1'], }, { @@ -144,7 +144,7 @@ describe('formatAgentMessages', () => { }, { type: ContentTypes.TEXT, - [ContentTypes.TEXT]: 'Now, I\'ll convert the temperature.', + [ContentTypes.TEXT]: "Now, I'll convert the temperature.", tool_call_ids: ['convert_1'], }, { @@ -156,7 +156,7 @@ describe('formatAgentMessages', () => { output: '23.89°C', }, }, - { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'Here\'s your answer.' }, + { type: ContentTypes.TEXT, [ContentTypes.TEXT]: "Here's your answer." }, ], }, ]; @@ -171,7 +171,7 @@ describe('formatAgentMessages', () => { expect(result[4]).toBeInstanceOf(AIMessage); // Check first AIMessage - expect(result[0].content).toBe('I\'ll search for that information.'); + expect(result[0].content).toBe("I'll search for that information."); expect(result[0].tool_calls).toHaveLength(1); expect(result[0].tool_calls[0]).toEqual({ id: 'search_1', @@ -187,7 +187,7 @@ describe('formatAgentMessages', () => { ); // Check second AIMessage - expect(result[2].content).toBe('Now, I\'ll convert the temperature.'); + expect(result[2].content).toBe("Now, I'll convert the temperature."); expect(result[2].tool_calls).toHaveLength(1); expect(result[2].tool_calls[0]).toEqual({ id: 'convert_1', @@ -202,7 +202,7 @@ describe('formatAgentMessages', () => { // Check final AIMessage expect(result[4].content).toStrictEqual([ - { [ContentTypes.TEXT]: 'Here\'s your answer.', type: ContentTypes.TEXT }, + { [ContentTypes.TEXT]: "Here's your answer.", type: ContentTypes.TEXT }, ]); }); @@ -217,7 +217,7 @@ describe('formatAgentMessages', () => { role: 'assistant', content: [{ type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'How can I help you?' }], }, - { role: 'user', content: 'What\'s the weather?' }, + { role: 'user', content: "What's the weather?" }, { role: 'assistant', content: [ @@ -240,7 +240,7 @@ describe('formatAgentMessages', () => { { role: 'assistant', content: [ - { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'Here\'s the weather information.' }, + { type: ContentTypes.TEXT, [ContentTypes.TEXT]: "Here's the weather information." }, ], }, ]; @@ -265,12 +265,12 @@ describe('formatAgentMessages', () => { { [ContentTypes.TEXT]: 'How can I help you?', type: ContentTypes.TEXT }, ]); expect(result[2].content).toStrictEqual([ - { [ContentTypes.TEXT]: 'What\'s the weather?', type: ContentTypes.TEXT }, + { [ContentTypes.TEXT]: "What's the weather?", type: ContentTypes.TEXT }, ]); expect(result[3].content).toBe('Let me check that for you.'); expect(result[4].content).toBe('Sunny, 75°F'); expect(result[5].content).toStrictEqual([ - { [ContentTypes.TEXT]: 'Here\'s the weather information.', type: ContentTypes.TEXT }, + { [ContentTypes.TEXT]: "Here's the weather information.", type: ContentTypes.TEXT }, ]); // Check that there are no consecutive AIMessages diff --git a/api/app/clients/prompts/formatMessages.js b/api/app/clients/prompts/formatMessages.js index 376178b6be..95600dd0b8 100644 --- a/api/app/clients/prompts/formatMessages.js +++ b/api/app/clients/prompts/formatMessages.js @@ -3,24 +3,61 @@ const { EModelEndpoint, ContentTypes } = require('librechat-data-provider'); const { HumanMessage, AIMessage, SystemMessage } = require('@langchain/core/messages'); /** - * Formats a message to OpenAI Vision API payload format. + * Formats a message with document attachments for specific endpoints. * * @param {Object} params - The parameters for formatting. * @param {Object} params.message - The message object to format. - * @param {string} [params.message.role] - The role of the message sender (must be 'user'). - * @param {string} [params.message.content] - The text content of the message. + * @param {Array} [params.documents] - The document attachments for the message. * @param {EModelEndpoint} [params.endpoint] - Identifier for specific endpoint handling + * @returns {(Object)} - The formatted message. + */ +const formatDocumentMessage = ({ message, documents, endpoint }) => { + const contentParts = []; + + // Add documents first (for Anthropic PDFs) + if (documents && documents.length > 0) { + contentParts.push(...documents); + } + + // Add text content + contentParts.push({ type: ContentTypes.TEXT, text: message.content }); + + if (endpoint === EModelEndpoint.anthropic) { + message.content = contentParts; + return message; + } + + // For other endpoints, might need different handling + message.content = contentParts; + return message; +}; + +/** + * Formats a message with vision capabilities (image_urls) for specific endpoints. + * + * @param {Object} params - The parameters for formatting. + * @param {Object} params.message - The message object to format. * @param {Array} [params.image_urls] - The image_urls to attach to the message. + * @param {EModelEndpoint} [params.endpoint] - Identifier for specific endpoint handling * @returns {(Object)} - The formatted message. */ const formatVisionMessage = ({ message, image_urls, endpoint }) => { + const contentParts = []; + + // Add images + if (image_urls && image_urls.length > 0) { + contentParts.push(...image_urls); + } + + // Add text content + contentParts.push({ type: ContentTypes.TEXT, text: message.content }); + if (endpoint === EModelEndpoint.anthropic) { - message.content = [...image_urls, { type: ContentTypes.TEXT, text: message.content }]; + message.content = contentParts; return message; } message.content = [{ type: ContentTypes.TEXT, text: message.content }, ...image_urls]; - return message; }; @@ -58,7 +95,18 @@ const formatMessage = ({ message, userName, assistantName, endpoint, langChain = content, }; - const { image_urls } = message; + const { image_urls, documents } = message; + + // Handle documents + if (Array.isArray(documents) && documents.length > 0 && role === 'user') { + return formatDocumentMessage({ + message: formattedMessage, + documents: message.documents, + endpoint, + }); + } + + // Handle images if (Array.isArray(image_urls) && image_urls.length > 0 && role === 'user') { return formatVisionMessage({ message: formattedMessage, @@ -146,7 +194,21 @@ const formatAgentMessages = (payload) => { message.content = [{ type: ContentTypes.TEXT, [ContentTypes.TEXT]: message.content }]; } if (message.role !== 'assistant') { - messages.push(formatMessage({ message, langChain: true })); + // Check if message has documents and preserve array structure + const hasDocuments = + Array.isArray(message.content) && + message.content.some((part) => part && part.type === 'document'); + + if (hasDocuments && message.role === 'user') { + // For user messages with documents, create HumanMessage directly with array content + messages.push(new HumanMessage({ content: message.content })); + } else if (hasDocuments && message.role === 'system') { + // For system messages with documents, create SystemMessage directly with array content + messages.push(new SystemMessage({ content: message.content })); + } else { + // Use regular formatting for messages without documents + messages.push(formatMessage({ message, langChain: true })); + } continue; } @@ -239,6 +301,8 @@ const formatAgentMessages = (payload) => { module.exports = { formatMessage, + formatDocumentMessage, + formatVisionMessage, formatFromLangChain, formatAgentMessages, formatLangChainMessages, diff --git a/api/app/clients/prompts/instructions.js b/api/app/clients/prompts/instructions.js index c630711771..b97b1bfecc 100644 --- a/api/app/clients/prompts/instructions.js +++ b/api/app/clients/prompts/instructions.js @@ -1,8 +1,8 @@ module.exports = { instructions: - 'Remember, all your responses MUST be in the format described. Do not respond unless it\'s in the format described, using the structure of Action, Action Input, etc.', + "Remember, all your responses MUST be in the format described. Do not respond unless it's in the format described, using the structure of Action, Action Input, etc.", errorInstructions: - '\nYou encountered an error in attempting a response. The user is not aware of the error so you shouldn\'t mention it.\nReview the actions taken carefully in case there is a partial or complete answer within them.\nError Message:', + "\nYou encountered an error in attempting a response. The user is not aware of the error so you shouldn't mention it.\nReview the actions taken carefully in case there is a partial or complete answer within them.\nError Message:", imageInstructions: 'You must include the exact image paths from above, formatted in Markdown syntax: ![alt-text](URL)', completionInstructions: diff --git a/api/app/clients/prompts/shadcn-docs/generate.js b/api/app/clients/prompts/shadcn-docs/generate.js index 6cb56f1077..6a4cf8c7e3 100644 --- a/api/app/clients/prompts/shadcn-docs/generate.js +++ b/api/app/clients/prompts/shadcn-docs/generate.js @@ -18,17 +18,17 @@ function generateShadcnPrompt(options) { Here are the components that are available, along with how to import them, and how to use them: ${Object.values(components) - .map((component) => { - if (useXML) { - return dedent` + .map((component) => { + if (useXML) { + return dedent` ${component.componentName} ${component.importDocs} ${component.usageDocs} `; - } else { - return dedent` + } else { + return dedent` # ${component.componentName} ## Import Instructions @@ -37,9 +37,9 @@ function generateShadcnPrompt(options) { ## Usage Instructions ${component.usageDocs} `; - } - }) - .join('\n\n')} + } + }) + .join('\n\n')} `; return systemPrompt; diff --git a/api/app/clients/tools/structured/AzureAISearch.js b/api/app/clients/tools/structured/AzureAISearch.js index e25da94426..7432d0722e 100644 --- a/api/app/clients/tools/structured/AzureAISearch.js +++ b/api/app/clients/tools/structured/AzureAISearch.js @@ -18,7 +18,7 @@ class AzureAISearch extends Tool { super(); this.name = 'azure-ai-search'; this.description = - 'Use the \'azure-ai-search\' tool to retrieve search results relevant to your input'; + "Use the 'azure-ai-search' tool to retrieve search results relevant to your input"; /* Used to initialize the Tool without necessary variables. */ this.override = fields.override ?? false; diff --git a/api/app/clients/tools/structured/FluxAPI.js b/api/app/clients/tools/structured/FluxAPI.js index 80f9772200..59902fbdd2 100644 --- a/api/app/clients/tools/structured/FluxAPI.js +++ b/api/app/clients/tools/structured/FluxAPI.js @@ -8,7 +8,7 @@ const { FileContext, ContentTypes } = require('librechat-data-provider'); const { logger } = require('~/config'); const displayMessage = - 'Flux displayed an image. All generated images are already plainly visible, so don\'t repeat the descriptions in detail. Do not list download links as they are available in the UI already. The user may download the images by clicking on them, but do not mention anything about downloading to the user.'; + "Flux displayed an image. All generated images are already plainly visible, so don't repeat the descriptions in detail. Do not list download links as they are available in the UI already. The user may download the images by clicking on them, but do not mention anything about downloading to the user."; /** * FluxAPI - A tool for generating high-quality images from text prompts using the Flux API. diff --git a/api/app/clients/tools/structured/OpenWeather.js b/api/app/clients/tools/structured/OpenWeather.js index b84225101c..f92fe522ce 100644 --- a/api/app/clients/tools/structured/OpenWeather.js +++ b/api/app/clients/tools/structured/OpenWeather.js @@ -232,7 +232,7 @@ class OpenWeather extends Tool { if (['current_forecast', 'timestamp', 'daily_aggregation', 'overview'].includes(action)) { if (typeof finalLat !== 'number' || typeof finalLon !== 'number') { - return 'Error: lat and lon are required and must be numbers for this action (or specify \'city\').'; + return "Error: lat and lon are required and must be numbers for this action (or specify 'city')."; } } @@ -243,7 +243,7 @@ class OpenWeather extends Tool { let dt; if (action === 'timestamp') { if (!date) { - return 'Error: For timestamp action, a \'date\' in YYYY-MM-DD format is required.'; + return "Error: For timestamp action, a 'date' in YYYY-MM-DD format is required."; } dt = this.convertDateToUnix(date); } diff --git a/api/app/clients/tools/structured/StableDiffusion.js b/api/app/clients/tools/structured/StableDiffusion.js index 25a9e0abd3..9e67066de0 100644 --- a/api/app/clients/tools/structured/StableDiffusion.js +++ b/api/app/clients/tools/structured/StableDiffusion.js @@ -11,7 +11,7 @@ const paths = require('~/config/paths'); const { logger } = require('~/config'); const displayMessage = - 'Stable Diffusion displayed an image. All generated images are already plainly visible, so don\'t repeat the descriptions in detail. Do not list download links as they are available in the UI already. The user may download the images by clicking on them, but do not mention anything about downloading to the user.'; + "Stable Diffusion displayed an image. All generated images are already plainly visible, so don't repeat the descriptions in detail. Do not list download links as they are available in the UI already. The user may download the images by clicking on them, but do not mention anything about downloading to the user."; class StableDiffusionAPI extends Tool { constructor(fields) { @@ -44,7 +44,7 @@ class StableDiffusionAPI extends Tool { // "negative_prompt":"semi-realistic, cgi, 3d, render, sketch, cartoon, drawing, anime, out of frame, low quality, ugly, mutation, deformed" // - Generate images only once per human query unless explicitly requested by the user`; this.description = - 'You can generate images using text with \'stable-diffusion\'. This tool is exclusively for visual content.'; + "You can generate images using text with 'stable-diffusion'. This tool is exclusively for visual content."; this.schema = z.object({ prompt: z .string() diff --git a/api/app/clients/tools/structured/TraversaalSearch.js b/api/app/clients/tools/structured/TraversaalSearch.js index e8ceeda134..0e6b7fc1c9 100644 --- a/api/app/clients/tools/structured/TraversaalSearch.js +++ b/api/app/clients/tools/structured/TraversaalSearch.js @@ -21,7 +21,7 @@ class TraversaalSearch extends Tool { query: z .string() .describe( - 'A properly written sentence to be interpreted by an AI to search the web according to the user\'s request.', + "A properly written sentence to be interpreted by an AI to search the web according to the user's request.", ), }); @@ -38,7 +38,6 @@ class TraversaalSearch extends Tool { return apiKey; } - // eslint-disable-next-line no-unused-vars async _call({ query }, _runManager) { const body = { query: [query], diff --git a/api/app/clients/tools/structured/YouTube.js b/api/app/clients/tools/structured/YouTube.js index aa19fc211f..d9d3f76733 100644 --- a/api/app/clients/tools/structured/YouTube.js +++ b/api/app/clients/tools/structured/YouTube.js @@ -29,7 +29,7 @@ function parseTranscript(transcriptResponse) { .map((entry) => entry.text.trim()) .filter((text) => text) .join(' ') - .replaceAll('&#39;', '\''); + .replaceAll('&#39;', "'"); } function createYouTubeTools(fields = {}) { diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.js index 803475e2e3..63b1cdbc5f 100644 --- a/api/server/controllers/agents/client.js +++ b/api/server/controllers/agents/client.js @@ -226,6 +226,28 @@ class AgentClient extends BaseClient { return files; } + async addDocuments(message, attachments) { + const documentResult = + await require('~/server/services/Files/documents').encodeAndFormatDocuments( + this.options.req, + attachments, + this.options.agent.provider, + ); + message.documents = + documentResult.documents && documentResult.documents.length + ? documentResult.documents + : undefined; + return documentResult.files; + } + + async processAttachments(message, attachments) { + const [imageFiles, documentFiles] = await Promise.all([ + this.addImageURLs(message, attachments), + this.addDocuments(message, attachments), + ]); + return [...imageFiles, ...documentFiles]; + } + async buildMessages( messages, parentMessageId, @@ -259,7 +281,7 @@ class AgentClient extends BaseClient { }; } - const files = await this.addImageURLs( + const files = await this.processAttachments( orderedMessages[orderedMessages.length - 1], attachments, ); @@ -282,6 +304,23 @@ class AgentClient extends BaseClient { assistantName: this.options?.modelLabel, }); + if ( + message.documents && + message.documents.length > 0 && + message.role === 'user' && + this.options.agent.provider === EModelEndpoint.anthropic + ) { + const contentParts = []; + contentParts.push(...message.documents); + if (message.image_urls && message.image_urls.length > 0) { + contentParts.push(...message.image_urls); + } + const textContent = + typeof formattedMessage.content === 'string' ? formattedMessage.content : ''; + contentParts.push({ type: 'text', text: textContent }); + formattedMessage.content = contentParts; + } + if (message.ocr && i !== orderedMessages.length - 1) { if (typeof formattedMessage.content === 'string') { formattedMessage.content = message.ocr + '\n' + formattedMessage.content; @@ -777,6 +816,51 @@ class AgentClient extends BaseClient { }; const toolSet = new Set((this.options.agent.tools ?? []).map((tool) => tool && tool.name)); + + if ( + this.options.agent.provider === EModelEndpoint.anthropic && + payload && + Array.isArray(payload) + ) { + let userMessageWithDocs = null; + + if (this.userMessage?.documents) { + userMessageWithDocs = this.userMessage; + } else if (this.currentMessages?.length > 0) { + const lastMessage = this.currentMessages[this.currentMessages.length - 1]; + if (lastMessage.documents?.length > 0) { + userMessageWithDocs = lastMessage; + } + } else if (this.messages?.length > 0) { + const lastMessage = this.messages[this.messages.length - 1]; + if (lastMessage.documents?.length > 0) { + userMessageWithDocs = lastMessage; + } + } + + if (userMessageWithDocs) { + for (const payloadMessage of payload) { + if ( + payloadMessage.role === 'user' && + userMessageWithDocs.text === payloadMessage.content + ) { + if (typeof payloadMessage.content === 'string') { + payloadMessage.content = [ + ...userMessageWithDocs.documents, + { type: 'text', text: payloadMessage.content }, + ]; + } else if (Array.isArray(payloadMessage.content)) { + payloadMessage.content = [ + ...userMessageWithDocs.documents, + ...payloadMessage.content, + ]; + } + break; + } + } + } + } + let { messages: initialMessages, indexTokenCountMap } = formatAgentMessages( payload, this.indexTokenCountMap, diff --git a/api/server/routes/__tests__/config.spec.js b/api/server/routes/__tests__/config.spec.js index 054e4726f0..7d7d3ea13a 100644 --- a/api/server/routes/__tests__/config.spec.js +++ b/api/server/routes/__tests__/config.spec.js @@ -43,7 +43,6 @@ afterEach(() => { //TODO: This works/passes locally but http request tests fail with 404 in CI. Need to figure out why. -// eslint-disable-next-line jest/no-disabled-tests describe.skip('GET /', () => { it('should return 200 and the correct body', async () => { process.env.APP_TITLE = 'Test Title'; diff --git a/api/server/services/Endpoints/azureAssistants/build.js b/api/server/services/Endpoints/azureAssistants/build.js index 54a32e4d3c..53b1dbeb68 100644 --- a/api/server/services/Endpoints/azureAssistants/build.js +++ b/api/server/services/Endpoints/azureAssistants/build.js @@ -3,7 +3,6 @@ const generateArtifactsPrompt = require('~/app/clients/prompts/artifacts'); const { getAssistant } = require('~/models/Assistant'); const buildOptions = async (endpoint, parsedBody) => { - const { promptPrefix, assistant_id, iconURL, greeting, spec, artifacts, ...modelOptions } = parsedBody; const endpointOption = removeNullishValues({ diff --git a/api/server/services/Endpoints/openAI/initialize.spec.js b/api/server/services/Endpoints/openAI/initialize.spec.js index 16563e4b26..067486b431 100644 --- a/api/server/services/Endpoints/openAI/initialize.spec.js +++ b/api/server/services/Endpoints/openAI/initialize.spec.js @@ -112,11 +112,11 @@ describe('initializeClient', () => { test('should initialize client with Azure credentials when endpoint is azureOpenAI', async () => { process.env.AZURE_API_KEY = 'test-azure-api-key'; (process.env.AZURE_OPENAI_API_INSTANCE_NAME = 'some-value'), - (process.env.AZURE_OPENAI_API_DEPLOYMENT_NAME = 'some-value'), - (process.env.AZURE_OPENAI_API_VERSION = 'some-value'), - (process.env.AZURE_OPENAI_API_COMPLETIONS_DEPLOYMENT_NAME = 'some-value'), - (process.env.AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME = 'some-value'), - (process.env.OPENAI_API_KEY = 'test-openai-api-key'); + (process.env.AZURE_OPENAI_API_DEPLOYMENT_NAME = 'some-value'), + (process.env.AZURE_OPENAI_API_VERSION = 'some-value'), + (process.env.AZURE_OPENAI_API_COMPLETIONS_DEPLOYMENT_NAME = 'some-value'), + (process.env.AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME = 'some-value'), + (process.env.OPENAI_API_KEY = 'test-openai-api-key'); process.env.DEBUG_OPENAI = 'false'; process.env.OPENAI_SUMMARIZE = 'false'; diff --git a/api/server/services/Files/documents/encode.js b/api/server/services/Files/documents/encode.js new file mode 100644 index 0000000000..6781ae4a69 --- /dev/null +++ b/api/server/services/Files/documents/encode.js @@ -0,0 +1,166 @@ +const { EModelEndpoint } = require('librechat-data-provider'); +const { getStrategyFunctions } = require('~/server/services/Files/strategies'); +const { validateAnthropicPdf } = require('../validation/pdfValidator'); + +/** + * Converts a readable stream to a buffer. + * + * @param {NodeJS.ReadableStream} stream - The readable stream to convert. + * @returns {Promise} - Promise resolving to the buffer. + */ +async function streamToBuffer(stream) { + return new Promise((resolve, reject) => { + const chunks = []; + + stream.on('data', (chunk) => { + chunks.push(chunk); + }); + + stream.on('end', () => { + try { + const buffer = Buffer.concat(chunks); + chunks.length = 0; // Clear the array + resolve(buffer); + } catch (err) { + reject(err); + } + }); + + stream.on('error', (error) => { + chunks.length = 0; + reject(error); + }); + }).finally(() => { + // Clean up the stream if required + if (stream.destroy && typeof stream.destroy === 'function') { + stream.destroy(); + } + }); +} + +/** + * Processes and encodes document files for various endpoints + * + * @param {Express.Request} req - Express request object + * @param {MongoFile[]} files - Array of file objects to process + * @param {string} endpoint - The endpoint identifier (e.g., EModelEndpoint.anthropic) + * @returns {Promise<{documents: MessageContentDocument[], files: MongoFile[]}>} + */ +async function encodeAndFormatDocuments(req, files, endpoint) { + const promises = []; + /** @type {Record, 'prepareDocumentPayload' | 'getDownloadStream'>>} */ + const encodingMethods = {}; + /** @type {{ documents: MessageContentDocument[]; files: MongoFile[] }} */ + const result = { + documents: [], + files: [], + }; + + if (!files || !files.length) { + return result; + } + + // Filter for document files only + const documentFiles = files.filter( + (file) => file.type === 'application/pdf' || file.type?.startsWith('application/'), // Future: support for other document types + ); + + if (!documentFiles.length) { + return result; + } + + for (let file of documentFiles) { + /** @type {FileSources} */ + const source = file.source ?? 'local'; + + // Only process PDFs for Anthropic for now + if (file.type !== 'application/pdf' || endpoint !== EModelEndpoint.anthropic) { + continue; + } + + if (!encodingMethods[source]) { + encodingMethods[source] = getStrategyFunctions(source); + } + + // Prepare file metadata + const fileMetadata = { + file_id: file.file_id || file._id, + temp_file_id: file.temp_file_id, + filepath: file.filepath, + source: file.source, + filename: file.filename, + type: file.type, + }; + + promises.push([file, fileMetadata]); + } + + const results = await Promise.allSettled( + promises.map(async ([file, fileMetadata]) => { + if (!file || !fileMetadata) { + return { file: null, content: null, metadata: fileMetadata }; + } + + try { + const source = file.source ?? 'local'; + const { getDownloadStream } = encodingMethods[source]; + + const stream = await getDownloadStream(req, file.filepath); + const buffer = await streamToBuffer(stream); + const documentContent = buffer.toString('base64'); + + return { + file, + content: documentContent, + metadata: fileMetadata, + }; + } catch (error) { + console.error(`Error processing document ${file.filename}:`, error); + return { file, content: null, metadata: fileMetadata }; + } + }), + ); + + for (const settledResult of results) { + if (settledResult.status === 'rejected') { + console.error('Document processing failed:', settledResult.reason); + continue; + } + + const { file, content, metadata } = settledResult.value; + + if (!content || !file) { + if (metadata) { + result.files.push(metadata); + } + continue; + } + + if (file.type === 'application/pdf' && endpoint === EModelEndpoint.anthropic) { + const pdfBuffer = Buffer.from(content, 'base64'); + const validation = await validateAnthropicPdf(pdfBuffer, pdfBuffer.length); + + if (!validation.isValid) { + throw new Error(`PDF validation failed: ${validation.error}`); + } + + const documentPart = { + type: 'document', + source: { + type: 'base64', + media_type: 'application/pdf', + data: content, + }, + }; + + result.documents.push(documentPart); + result.files.push(metadata); + } + } + + return result; +} + +module.exports = { + encodeAndFormatDocuments, +}; diff --git a/api/server/services/Files/documents/index.js b/api/server/services/Files/documents/index.js new file mode 100644 index 0000000000..6082509ba9 --- /dev/null +++ b/api/server/services/Files/documents/index.js @@ -0,0 +1,5 @@ +const { encodeAndFormatDocuments } = require('./encode'); + +module.exports = { + encodeAndFormatDocuments, +}; diff --git a/api/server/services/Files/process.js b/api/server/services/Files/process.js index 38ccdafdd7..f0ffc17bce 100644 --- a/api/server/services/Files/process.js +++ b/api/server/services/Files/process.js @@ -391,7 +391,17 @@ const processFileUpload = async ({ req, res, metadata }) => { const isAssistantUpload = isAssistantsEndpoint(metadata.endpoint); const assistantSource = metadata.endpoint === EModelEndpoint.azureAssistants ? FileSources.azure : FileSources.openai; - const source = isAssistantUpload ? assistantSource : FileSources.vectordb; + + // Use local storage for Anthropic native PDF support, vectordb for others + const isAnthropicUpload = metadata.endpoint === EModelEndpoint.anthropic; + let source; + if (isAssistantUpload) { + source = assistantSource; + } else if (isAnthropicUpload) { + source = FileSources.local; + } else { + source = FileSources.vectordb; + } const { handleFileUpload } = getStrategyFunctions(source); const { file_id, temp_file_id } = metadata; diff --git a/api/server/services/Files/validation/pdfValidator.js b/api/server/services/Files/validation/pdfValidator.js new file mode 100644 index 0000000000..7dfc2a4aac --- /dev/null +++ b/api/server/services/Files/validation/pdfValidator.js @@ -0,0 +1,77 @@ +const { logger } = require('~/config'); +const { anthropicPdfSizeLimit } = require('librechat-data-provider'); + +/** + * Validates if a PDF meets Anthropic's requirements + * @param {Buffer} pdfBuffer - The PDF file as a buffer + * @param {number} fileSize - The file size in bytes + * @returns {Promise<{isValid: boolean, error?: string}>} + */ +async function validateAnthropicPdf(pdfBuffer, fileSize) { + try { + // Check file size (32MB limit) + if (fileSize > anthropicPdfSizeLimit) { + return { + isValid: false, + error: `PDF file size (${Math.round(fileSize / (1024 * 1024))}MB) exceeds Anthropic's 32MB limit`, + }; + } + + // Basic PDF header validation + if (!pdfBuffer || pdfBuffer.length < 5) { + return { + isValid: false, + error: 'Invalid PDF file: too small or corrupted', + }; + } + + // Check PDF magic bytes + const pdfHeader = pdfBuffer.subarray(0, 5).toString(); + if (!pdfHeader.startsWith('%PDF-')) { + return { + isValid: false, + error: 'Invalid PDF file: missing PDF header', + }; + } + + // Check for password protection/encryption + const pdfContent = pdfBuffer.toString('binary'); + if ( + pdfContent.includes('/Encrypt ') || + pdfContent.includes('/U (') || + pdfContent.includes('/O (') + ) { + return { + isValid: false, + error: 'PDF is password-protected or encrypted. Anthropic requires unencrypted PDFs.', + }; + } + + // Estimate page count (this is a rough estimation) + const pageMatches = pdfContent.match(/\/Type[\s]*\/Page[^s]/g); + const estimatedPages = pageMatches ? pageMatches.length : 1; + + if (estimatedPages > 100) { + return { + isValid: false, + error: `PDF has approximately ${estimatedPages} pages, exceeding Anthropic's 100-page limit`, + }; + } + + logger.debug( + `PDF validation passed: ${Math.round(fileSize / 1024)}KB, ~${estimatedPages} pages`, + ); + + return { isValid: true }; + } catch (error) { + logger.error('PDF validation error:', error); + return { + isValid: false, + error: 'Failed to validate PDF file', + }; + } +} + +module.exports = { + validateAnthropicPdf, +}; diff --git a/api/server/services/Threads/processMessages.spec.js b/api/server/services/Threads/processMessages.spec.js index 673b96bf7c..a89f2b9f5b 100644 --- a/api/server/services/Threads/processMessages.spec.js +++ b/api/server/services/Threads/processMessages.spec.js @@ -255,7 +255,7 @@ describe('processMessages', () => { type: 'text', text: { value: - 'The text you have uploaded is from the book "Harry Potter and the Philosopher\'s Stone" by J.K. Rowling. It follows the story of a young boy named Harry Potter who discovers that he is a wizard on his eleventh birthday. Here are some key points of the narrative:\n\n1. **Discovery and Invitation to Hogwarts**: Harry learns that he is a wizard and receives an invitation to attend Hogwarts School of Witchcraft and Wizardry【11:2†source】【11:4†source】.\n\n2. **Shopping for Supplies**: Hagrid takes Harry to Diagon Alley to buy his school supplies, including his wand from Ollivander\'s【11:9†source】【11:14†source】.\n\n3. **Introduction to Hogwarts**: Harry is introduced to Hogwarts, the magical school where he will learn about magic and discover more about his own background【11:12†source】【11:18†source】.\n\n4. **Meeting Friends and Enemies**: At Hogwarts, Harry makes friends like Ron Weasley and Hermione Granger, and enemies like Draco Malfoy【11:16†source】.\n\n5. **Uncovering the Mystery**: Harry, along with Ron and Hermione, uncovers the mystery of the Philosopher\'s Stone and its connection to the dark wizard Voldemort【11:1†source】【11:10†source】【11:7†source】.\n\nThese points highlight Harry\'s initial experiences in the magical world and set the stage for his adventures at Hogwarts.', + "The text you have uploaded is from the book \"Harry Potter and the Philosopher's Stone\" by J.K. Rowling. It follows the story of a young boy named Harry Potter who discovers that he is a wizard on his eleventh birthday. Here are some key points of the narrative:\n\n1. **Discovery and Invitation to Hogwarts**: Harry learns that he is a wizard and receives an invitation to attend Hogwarts School of Witchcraft and Wizardry【11:2†source】【11:4†source】.\n\n2. **Shopping for Supplies**: Hagrid takes Harry to Diagon Alley to buy his school supplies, including his wand from Ollivander's【11:9†source】【11:14†source】.\n\n3. **Introduction to Hogwarts**: Harry is introduced to Hogwarts, the magical school where he will learn about magic and discover more about his own background【11:12†source】【11:18†source】.\n\n4. **Meeting Friends and Enemies**: At Hogwarts, Harry makes friends like Ron Weasley and Hermione Granger, and enemies like Draco Malfoy【11:16†source】.\n\n5. **Uncovering the Mystery**: Harry, along with Ron and Hermione, uncovers the mystery of the Philosopher's Stone and its connection to the dark wizard Voldemort【11:1†source】【11:10†source】【11:7†source】.\n\nThese points highlight Harry's initial experiences in the magical world and set the stage for his adventures at Hogwarts.", annotations: [ { type: 'file_citation', @@ -424,7 +424,7 @@ These points highlight Harry's initial experiences in the magical world and set type: 'text', text: { value: - 'The text you have uploaded is from the book "Harry Potter and the Philosopher\'s Stone" by J.K. Rowling. It follows the story of a young boy named Harry Potter who discovers that he is a wizard on his eleventh birthday. Here are some key points of the narrative:\n\n1. **Discovery and Invitation to Hogwarts**: Harry learns that he is a wizard and receives an invitation to attend Hogwarts School of Witchcraft and Wizardry【11:2†source】【11:4†source】.\n\n2. **Shopping for Supplies**: Hagrid takes Harry to Diagon Alley to buy his school supplies, including his wand from Ollivander\'s【11:9†source】【11:14†source】.\n\n3. **Introduction to Hogwarts**: Harry is introduced to Hogwarts, the magical school where he will learn about magic and discover more about his own background【11:12†source】【11:18†source】.\n\n4. **Meeting Friends and Enemies**: At Hogwarts, Harry makes friends like Ron Weasley and Hermione Granger, and enemies like Draco Malfoy【11:16†source】.\n\n5. **Uncovering the Mystery**: Harry, along with Ron and Hermione, uncovers the mystery of the Philosopher\'s Stone and its connection to the dark wizard Voldemort【11:1†source】【11:10†source】【11:7†source】.\n\nThese points highlight Harry\'s initial experiences in the magical world and set the stage for his adventures at Hogwarts.', + "The text you have uploaded is from the book \"Harry Potter and the Philosopher's Stone\" by J.K. Rowling. It follows the story of a young boy named Harry Potter who discovers that he is a wizard on his eleventh birthday. Here are some key points of the narrative:\n\n1. **Discovery and Invitation to Hogwarts**: Harry learns that he is a wizard and receives an invitation to attend Hogwarts School of Witchcraft and Wizardry【11:2†source】【11:4†source】.\n\n2. **Shopping for Supplies**: Hagrid takes Harry to Diagon Alley to buy his school supplies, including his wand from Ollivander's【11:9†source】【11:14†source】.\n\n3. **Introduction to Hogwarts**: Harry is introduced to Hogwarts, the magical school where he will learn about magic and discover more about his own background【11:12†source】【11:18†source】.\n\n4. **Meeting Friends and Enemies**: At Hogwarts, Harry makes friends like Ron Weasley and Hermione Granger, and enemies like Draco Malfoy【11:16†source】.\n\n5. **Uncovering the Mystery**: Harry, along with Ron and Hermione, uncovers the mystery of the Philosopher's Stone and its connection to the dark wizard Voldemort【11:1†source】【11:10†source】【11:7†source】.\n\nThese points highlight Harry's initial experiences in the magical world and set the stage for his adventures at Hogwarts.", annotations: [ { type: 'file_citation', @@ -582,7 +582,7 @@ These points highlight Harry's initial experiences in the magical world and set type: 'text', text: { value: - 'This is a test ^1^ with pre-existing citation-like text. Here\'s a real citation【11:2†source】.', + "This is a test ^1^ with pre-existing citation-like text. Here's a real citation【11:2†source】.", annotations: [ { type: 'file_citation', @@ -610,7 +610,7 @@ These points highlight Harry's initial experiences in the magical world and set }); const expectedText = - 'This is a test ^1^ with pre-existing citation-like text. Here\'s a real citation^1^.\n\n^1.^ test.txt'; + "This is a test ^1^ with pre-existing citation-like text. Here's a real citation^1^.\n\n^1.^ test.txt"; expect(result.text).toBe(expectedText); expect(result.edited).toBe(true); diff --git a/api/strategies/validators.spec.js b/api/strategies/validators.spec.js index 312f06923d..06340fc14c 100644 --- a/api/strategies/validators.spec.js +++ b/api/strategies/validators.spec.js @@ -258,7 +258,7 @@ describe('Zod Schemas', () => { email: 'john@example.com', password: 'password123', confirm_password: 'password123', - extraField: 'I shouldn\'t be here', + extraField: "I shouldn't be here", }); expect(result.success).toBe(true); }); @@ -407,7 +407,7 @@ describe('Zod Schemas', () => { 'john{doe}', // Contains `{` and `}` 'j', // Only one character 'a'.repeat(81), // More than 80 characters - '\' OR \'1\'=\'1\'; --', // SQL Injection + "' OR '1'='1'; --", // SQL Injection '{$ne: null}', // MongoDB Injection '', // Basic XSS '">', // XSS breaking out of an attribute diff --git a/client/src/@types/i18next.d.ts b/client/src/@types/i18next.d.ts index 2d50f5a3cd..82f1ce1a3d 100644 --- a/client/src/@types/i18next.d.ts +++ b/client/src/@types/i18next.d.ts @@ -1,9 +1,9 @@ import { defaultNS, resources } from '~/locales/i18n'; declare module 'i18next' { - interface CustomTypeOptions { - defaultNS: typeof defaultNS; - resources: typeof resources.en; - strictKeyChecks: true - } -} \ No newline at end of file + interface CustomTypeOptions { + defaultNS: typeof defaultNS; + resources: typeof resources.en; + strictKeyChecks: true; + } +} diff --git a/client/src/components/Auth/__tests__/Registration.spec.tsx b/client/src/components/Auth/__tests__/Registration.spec.tsx index 72a21b63b6..a06f8c2ffb 100644 --- a/client/src/components/Auth/__tests__/Registration.spec.tsx +++ b/client/src/components/Auth/__tests__/Registration.spec.tsx @@ -156,7 +156,6 @@ test('renders registration form', () => { ); }); -// eslint-disable-next-line jest/no-commented-out-tests // test('calls registerUser.mutate on registration', async () => { // const mutate = jest.fn(); // const { getByTestId, getByRole, history } = setup({ diff --git a/client/src/components/Chat/Input/Files/AttachFileChat.tsx b/client/src/components/Chat/Input/Files/AttachFileChat.tsx index 6bdecca22a..69b7d6b7bf 100644 --- a/client/src/components/Chat/Input/Files/AttachFileChat.tsx +++ b/client/src/components/Chat/Input/Files/AttachFileChat.tsx @@ -36,6 +36,7 @@ function AttachFileChat({ disableInputs }: { disableInputs: boolean }) { disabled={disableInputs} conversationId={conversationId} endpointFileConfig={endpointFileConfig} + endpoint={endpoint} /> ); } diff --git a/client/src/components/Chat/Input/Files/AttachFileMenu.tsx b/client/src/components/Chat/Input/Files/AttachFileMenu.tsx index 5e42dd063a..66063e9651 100644 --- a/client/src/components/Chat/Input/Files/AttachFileMenu.tsx +++ b/client/src/components/Chat/Input/Files/AttachFileMenu.tsx @@ -1,7 +1,7 @@ import React, { useRef, useState, useMemo } from 'react'; import * as Ariakit from '@ariakit/react'; import { useSetRecoilState } from 'recoil'; -import { FileSearch, ImageUpIcon, TerminalSquareIcon, FileType2Icon } from 'lucide-react'; +import { FileSearch, ImageUpIcon, TerminalSquareIcon, FileType2Icon, FileText } from 'lucide-react'; import { FileUpload, TooltipAnchor, DropdownPopup, AttachmentIcon } from '@librechat/client'; import { EToolResources, EModelEndpoint, defaultAgentCapabilities } from 'librechat-data-provider'; import type { EndpointFileConfig } from 'librechat-data-provider'; @@ -13,9 +13,15 @@ interface AttachFileMenuProps { conversationId: string; disabled?: boolean | null; endpointFileConfig?: EndpointFileConfig; + endpoint?: string | null; } -const AttachFileMenu = ({ disabled, conversationId, endpointFileConfig }: AttachFileMenuProps) => { +const AttachFileMenu = ({ + disabled, + conversationId, + endpointFileConfig, + endpoint, +}: AttachFileMenuProps) => { const localize = useLocalize(); const isUploadDisabled = disabled ?? false; const inputRef = useRef(null); @@ -23,7 +29,7 @@ const AttachFileMenu = ({ disabled, conversationId, endpointFileConfig }: Attach const setEphemeralAgent = useSetRecoilState(ephemeralAgentByConvoId(conversationId)); const [toolResource, setToolResource] = useState(); const { handleFileChange } = useFileHandling({ - overrideEndpoint: EModelEndpoint.agents, + overrideEndpoint: endpoint === EModelEndpoint.anthropic ? undefined : EModelEndpoint.agents, overrideEndpointFileConfig: endpointFileConfig, }); @@ -34,12 +40,18 @@ const AttachFileMenu = ({ disabled, conversationId, endpointFileConfig }: Attach * */ const capabilities = useAgentCapabilities(agentsConfig?.capabilities ?? defaultAgentCapabilities); - const handleUploadClick = (isImage?: boolean) => { + const handleUploadClick = (fileType?: 'image' | 'document') => { if (!inputRef.current) { return; } inputRef.current.value = ''; - inputRef.current.accept = isImage === true ? 'image/*' : ''; + if (fileType === 'image') { + inputRef.current.accept = 'image/*'; + } else if (fileType === 'document') { + inputRef.current.accept = '.pdf,application/pdf'; + } else { + inputRef.current.accept = ''; + } inputRef.current.click(); inputRef.current.accept = ''; }; @@ -50,13 +62,26 @@ const AttachFileMenu = ({ disabled, conversationId, endpointFileConfig }: Attach label: localize('com_ui_upload_image_input'), onClick: () => { setToolResource(undefined); - handleUploadClick(true); + handleUploadClick('image'); }, icon: , }, ]; - if (capabilities.ocrEnabled) { + // Add document upload option for Anthropic endpoints + if (endpoint === EModelEndpoint.anthropic) { + items.push({ + label: 'Upload Document', + onClick: () => { + setToolResource(undefined); + handleUploadClick('document'); + }, + icon: , + }); + } + + // Hide OCR and File Search for Anthropic endpoints (native PDF support makes these irrelevant) + if (capabilities.ocrEnabled && endpoint !== EModelEndpoint.anthropic) { items.push({ label: localize('com_ui_upload_ocr_text'), onClick: () => { @@ -67,7 +92,7 @@ const AttachFileMenu = ({ disabled, conversationId, endpointFileConfig }: Attach }); } - if (capabilities.fileSearchEnabled) { + if (capabilities.fileSearchEnabled && endpoint !== EModelEndpoint.anthropic) { items.push({ label: localize('com_ui_upload_file_search'), onClick: () => { @@ -95,7 +120,7 @@ const AttachFileMenu = ({ disabled, conversationId, endpointFileConfig }: Attach } return items; - }, [capabilities, localize, setToolResource, setEphemeralAgent]); + }, [capabilities, localize, setToolResource, setEphemeralAgent, endpoint]); const menuTrigger = ( +
= ({ >
-
+
{icon != null ? icon : null}
{title} @@ -72,7 +72,7 @@ const MenuItem: FC = ({ viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg" - className="icon-md block " + className="icon-md block" >
) : ( -
+
)} diff --git a/client/src/components/Messages/MessageContent.tsx b/client/src/components/Messages/MessageContent.tsx index e472b1e8dd..265db302c2 100644 --- a/client/src/components/Messages/MessageContent.tsx +++ b/client/src/components/Messages/MessageContent.tsx @@ -1,7 +1,7 @@ import React from 'react'; import { useMessageProcess } from '~/hooks'; import type { TMessageProps } from '~/common'; -// eslint-disable-next-line import/no-cycle + import MultiMessage from '~/components/Chat/Messages/MultiMessage'; import ContentRender from './ContentRender'; @@ -64,7 +64,7 @@ export default function MessageContent(props: TMessageProps) {
) : ( -
+
)} diff --git a/client/src/components/Plugins/Store/PluginStoreDialog.tsx b/client/src/components/Plugins/Store/PluginStoreDialog.tsx index b54fdada34..54c4b91863 100644 --- a/client/src/components/Plugins/Store/PluginStoreDialog.tsx +++ b/client/src/components/Plugins/Store/PluginStoreDialog.tsx @@ -187,8 +187,7 @@ function PluginStoreDialog({ isOpen, setIsOpen }: TPluginStoreDialogProps) { value={searchValue} onChange={handleSearch} placeholder={localize('com_nav_plugin_search')} - className=" - text-token-text-primary flex rounded-md border border-border-heavy bg-surface-tertiary py-2 pl-10 pr-2" + className="text-token-text-primary flex rounded-md border border-border-heavy bg-surface-tertiary py-2 pl-10 pr-2" />
diff --git a/client/src/components/Plugins/Store/PluginTooltip.tsx b/client/src/components/Plugins/Store/PluginTooltip.tsx index 2f60d1e11f..0fe44aaeb8 100644 --- a/client/src/components/Plugins/Store/PluginTooltip.tsx +++ b/client/src/components/Plugins/Store/PluginTooltip.tsx @@ -9,7 +9,7 @@ type TPluginTooltipProps = { function PluginTooltip({ content, position }: TPluginTooltipProps) { return ( - +
diff --git a/client/src/components/Share/MultiMessage.tsx b/client/src/components/Share/MultiMessage.tsx index 349f2d55a8..24ec2a5dd3 100644 --- a/client/src/components/Share/MultiMessage.tsx +++ b/client/src/components/Share/MultiMessage.tsx @@ -2,7 +2,7 @@ import { useEffect } from 'react'; import { useRecoilState } from 'recoil'; import type { TMessage } from 'librechat-data-provider'; import type { TMessageProps } from '~/common'; -// eslint-disable-next-line import/no-cycle + import Message from './Message'; import store from '~/store'; diff --git a/client/src/components/SidePanel/Builder/ActionsAuth.tsx b/client/src/components/SidePanel/Builder/ActionsAuth.tsx index 15ff89626e..84b24bce6d 100644 --- a/client/src/components/SidePanel/Builder/ActionsAuth.tsx +++ b/client/src/components/SidePanel/Builder/ActionsAuth.tsx @@ -33,9 +33,7 @@ export default function ActionsAuth({ disableOAuth }: { disableOAuth?: boolean }
-
- {localize(getAuthLocalizationKey(type))} -
+
{localize(getAuthLocalizationKey(type))}