✂️ fix: Remove Image Payloads from Memory Processing (#8770)

api/server/controllers/agents/client.js

@@ -512,6 +512,39 @@ class AgentClient extends BaseClient {
     return withoutKeys;
   }
 
+  /**
+   * Filters out image URLs from message content
+   * @param {BaseMessage} message - The message to filter
+   * @returns {BaseMessage} - A new message with image URLs removed
+   */
+  filterImageUrls(message) {
+    if (!message.content || typeof message.content === 'string') {
+      return message;
+    }
+
+    if (Array.isArray(message.content)) {
+      const filteredContent = message.content.filter(
+        (part) => part.type !== ContentTypes.IMAGE_URL,
+      );
+
+      if (filteredContent.length === 1 && filteredContent[0].type === ContentTypes.TEXT) {
+        const MessageClass = message.constructor;
+        return new MessageClass({
+          content: filteredContent[0].text,
+          additional_kwargs: message.additional_kwargs,
+        });
+      }
+
+      const MessageClass = message.constructor;
+      return new MessageClass({
+        content: filteredContent,
+        additional_kwargs: message.additional_kwargs,
+      });
+    }
+
+    return message;
+  }
+
   /**
    * @param {BaseMessage[]} messages
    * @returns {Promise<void | (TAttachment | null)[]>}
@@ -540,7 +573,8 @@ class AgentClient extends BaseClient {
         }
       }
 
-      const bufferString = getBufferString(messagesToProcess);
+      const filteredMessages = messagesToProcess.map((msg) => this.filterImageUrls(msg));
+      const bufferString = getBufferString(filteredMessages);
       const bufferMessage = new HumanMessage(`# Current Chat:\n\n${bufferString}`);
       return await this.processMemory([bufferMessage]);
     } catch (error) {

api/server/controllers/agents/client.test.js

@@ -727,4 +727,231 @@ describe('AgentClient - titleConvo', () => {
       });
     });
   });
+
+  describe('runMemory method', () => {
+    let client;
+    let mockReq;
+    let mockRes;
+    let mockAgent;
+    let mockOptions;
+    let mockProcessMemory;
+
+    beforeEach(() => {
+      jest.clearAllMocks();
+
+      mockAgent = {
+        id: 'agent-123',
+        endpoint: EModelEndpoint.openAI,
+        provider: EModelEndpoint.openAI,
+        model_parameters: {
+          model: 'gpt-4',
+        },
+      };
+
+      mockReq = {
+        app: {
+          locals: {
+            memory: {
+              messageWindowSize: 3,
+            },
+          },
+        },
+        user: {
+          id: 'user-123',
+          personalization: {
+            memories: true,
+          },
+        },
+      };
+
+      mockRes = {};
+
+      mockOptions = {
+        req: mockReq,
+        res: mockRes,
+        agent: mockAgent,
+      };
+
+      mockProcessMemory = jest.fn().mockResolvedValue([]);
+
+      client = new AgentClient(mockOptions);
+      client.processMemory = mockProcessMemory;
+      client.conversationId = 'convo-123';
+      client.responseMessageId = 'response-123';
+    });
+
+    it('should filter out image URLs from message content', async () => {
+      const { HumanMessage, AIMessage } = require('@langchain/core/messages');
+      const messages = [
+        new HumanMessage({
+          content: [
+            {
+              type: 'text',
+              text: 'What is in this image?',
+            },
+            {
+              type: 'image_url',
+              image_url: {
+                url: 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==',
+                detail: 'auto',
+              },
+            },
+          ],
+        }),
+        new AIMessage('I can see a small red pixel in the image.'),
+        new HumanMessage({
+          content: [
+            {
+              type: 'text',
+              text: 'What about this one?',
+            },
+            {
+              type: 'image_url',
+              image_url: {
+                url: 'data:image/jpeg;base64,/9j/4AAQSkZJRgABAQEAYABgAAD/',
+                detail: 'high',
+              },
+            },
+          ],
+        }),
+      ];
+
+      await client.runMemory(messages);
+
+      expect(mockProcessMemory).toHaveBeenCalledTimes(1);
+      const processedMessage = mockProcessMemory.mock.calls[0][0][0];
+
+      // Verify the buffer message was created
+      expect(processedMessage.constructor.name).toBe('HumanMessage');
+      expect(processedMessage.content).toContain('# Current Chat:');
+
+      // Verify that image URLs are not in the buffer string
+      expect(processedMessage.content).not.toContain('image_url');
+      expect(processedMessage.content).not.toContain('data:image');
+      expect(processedMessage.content).not.toContain('base64');
+
+      // Verify text content is preserved
+      expect(processedMessage.content).toContain('What is in this image?');
+      expect(processedMessage.content).toContain('I can see a small red pixel in the image.');
+      expect(processedMessage.content).toContain('What about this one?');
+    });
+
+    it('should handle messages with only text content', async () => {
+      const { HumanMessage, AIMessage } = require('@langchain/core/messages');
+      const messages = [
+        new HumanMessage('Hello, how are you?'),
+        new AIMessage('I am doing well, thank you!'),
+        new HumanMessage('That is great to hear.'),
+      ];
+
+      await client.runMemory(messages);
+
+      expect(mockProcessMemory).toHaveBeenCalledTimes(1);
+      const processedMessage = mockProcessMemory.mock.calls[0][0][0];
+
+      expect(processedMessage.content).toContain('Hello, how are you?');
+      expect(processedMessage.content).toContain('I am doing well, thank you!');
+      expect(processedMessage.content).toContain('That is great to hear.');
+    });
+
+    it('should handle mixed content types correctly', async () => {
+      const { HumanMessage } = require('@langchain/core/messages');
+      const { ContentTypes } = require('librechat-data-provider');
+
+      const messages = [
+        new HumanMessage({
+          content: [
+            {
+              type: 'text',
+              text: 'Here is some text',
+            },
+            {
+              type: ContentTypes.IMAGE_URL,
+              image_url: {
+                url: 'https://example.com/image.png',
+              },
+            },
+            {
+              type: 'text',
+              text: ' and more text',
+            },
+          ],
+        }),
+      ];
+
+      await client.runMemory(messages);
+
+      expect(mockProcessMemory).toHaveBeenCalledTimes(1);
+      const processedMessage = mockProcessMemory.mock.calls[0][0][0];
+
+      // Should contain text parts but not image URLs
+      expect(processedMessage.content).toContain('Here is some text');
+      expect(processedMessage.content).toContain('and more text');
+      expect(processedMessage.content).not.toContain('example.com/image.png');
+      expect(processedMessage.content).not.toContain('IMAGE_URL');
+    });
+
+    it('should preserve original messages without mutation', async () => {
+      const { HumanMessage } = require('@langchain/core/messages');
+      const originalContent = [
+        {
+          type: 'text',
+          text: 'Original text',
+        },
+        {
+          type: 'image_url',
+          image_url: {
+            url: 'data:image/png;base64,ABC123',
+          },
+        },
+      ];
+
+      const messages = [
+        new HumanMessage({
+          content: [...originalContent],
+        }),
+      ];
+
+      await client.runMemory(messages);
+
+      // Verify original message wasn't mutated
+      expect(messages[0].content).toHaveLength(2);
+      expect(messages[0].content[1].type).toBe('image_url');
+      expect(messages[0].content[1].image_url.url).toBe('data:image/png;base64,ABC123');
+    });
+
+    it('should handle message window size correctly', async () => {
+      const { HumanMessage, AIMessage } = require('@langchain/core/messages');
+      const messages = [
+        new HumanMessage('Message 1'),
+        new AIMessage('Response 1'),
+        new HumanMessage('Message 2'),
+        new AIMessage('Response 2'),
+        new HumanMessage('Message 3'),
+        new AIMessage('Response 3'),
+      ];
+
+      // Window size is set to 3 in mockReq
+      await client.runMemory(messages);
+
+      expect(mockProcessMemory).toHaveBeenCalledTimes(1);
+      const processedMessage = mockProcessMemory.mock.calls[0][0][0];
+
+      // Should only include last 3 messages due to window size
+      expect(processedMessage.content).toContain('Message 3');
+      expect(processedMessage.content).toContain('Response 3');
+      expect(processedMessage.content).not.toContain('Message 1');
+      expect(processedMessage.content).not.toContain('Response 1');
+    });
+
+    it('should return early if processMemory is not set', async () => {
+      const { HumanMessage } = require('@langchain/core/messages');
+      client.processMemory = null;
+
+      const result = await client.runMemory([new HumanMessage('Test')]);
+
+      expect(result).toBeUndefined();
+      expect(mockProcessMemory).not.toHaveBeenCalled();
+    });
+  });
 });