🗂️ feat: Send Attachments Directly to Provider (Google) (#9100)

* feat: add validation for google PDFs and add google endpoint as a document supporting endpoint

* feat: add proper pdf formatting for google endpoints (requires PR #14 in agents)

* feat: add multimodal support for google endpoint attachments

* feat: add audio file svg

* fix: refactor attachments logic so multi-attachment messages work properly

* feat: add video file svg

* fix: allows for followup questions of uploaded multimodal attachments

* fix: remove incorrect final message filtering that was breaking Attachment component rendering

api/server/services/Files/Audio/encode.js

@@ -0,0 +1,111 @@
+const { EModelEndpoint, isDocumentSupportedEndpoint } = require('librechat-data-provider');
+const { getStrategyFunctions } = require('~/server/services/Files/strategies');
+const { validateAudio } = require('@librechat/api');
+const { streamToBuffer } = require('~/server/services/Files/Documents/encode');
+
+/**
+ * Encodes and formats audio files for different endpoints
+ * @param {Express.Request} req - The request object
+ * @param {Array<MongoFile>} files - Array of audio files
+ * @param {EModelEndpoint} endpoint - The endpoint to format for
+ * @returns {Promise<{ audios: Array, files: Array<MongoFile> }>}
+ */
+async function encodeAndFormatAudios(req, files, endpoint) {
+  const promises = [];
+  const encodingMethods = {};
+  /** @type {{ audios: any[]; files: MongoFile[] }} */
+  const result = {
+    audios: [],
+    files: [],
+  };
+
+  for (const file of files) {
+    if (!file || !file.filepath) {
+      continue;
+    }
+
+    const source = file.source ?? 'local';
+    if (!encodingMethods[source]) {
+      encodingMethods[source] = getStrategyFunctions(source);
+    }
+
+    const fileMetadata = {
+      file_id: file.file_id || file._id,
+      temp_file_id: file.temp_file_id,
+      filepath: file.filepath,
+      source: file.source,
+      filename: file.filename,
+      type: file.type,
+    };
+
+    promises.push([file, fileMetadata]);
+  }
+
+  const results = await Promise.allSettled(
+    promises.map(async ([file, fileMetadata]) => {
+      if (!file || !fileMetadata) {
+        return { file: null, content: null, metadata: fileMetadata };
+      }
+
+      try {
+        const source = file.source ?? 'local';
+        const { getDownloadStream } = encodingMethods[source];
+
+        const stream = await getDownloadStream(req, file.filepath);
+        const buffer = await streamToBuffer(stream);
+        const audioContent = buffer.toString('base64');
+
+        return {
+          file,
+          content: audioContent,
+          metadata: fileMetadata,
+        };
+      } catch (error) {
+        console.error(`Error processing audio ${file.filename}:`, error);
+        return { file, content: null, metadata: fileMetadata };
+      }
+    }),
+  );
+
+  for (const settledResult of results) {
+    if (settledResult.status === 'rejected') {
+      console.error('Audio processing failed:', settledResult.reason);
+      continue;
+    }
+
+    const { file, content, metadata } = settledResult.value;
+
+    if (!content || !file) {
+      if (metadata) {
+        result.files.push(metadata);
+      }
+      continue;
+    }
+
+    if (file.type.startsWith('audio/') && isDocumentSupportedEndpoint(endpoint)) {
+      const audioBuffer = Buffer.from(content, 'base64');
+
+      const validation = await validateAudio(audioBuffer, audioBuffer.length, endpoint);
+      if (!validation.isValid) {
+        throw new Error(`Audio validation failed: ${validation.error}`);
+      }
+
+      if (endpoint === EModelEndpoint.google) {
+        const audioPart = {
+          type: 'audio',
+          mimeType: file.type,
+          data: content,
+        };
+        result.audios.push(audioPart);
+      }
+
+      result.files.push(metadata);
+    }
+  }
+
+  return result;
+}
+
+module.exports = {
+  encodeAndFormatAudios,
+};

api/server/services/Files/Video/encode.js

@@ -0,0 +1,111 @@
+const { EModelEndpoint, isDocumentSupportedEndpoint } = require('librechat-data-provider');
+const { getStrategyFunctions } = require('~/server/services/Files/strategies');
+const { validateVideo } = require('@librechat/api');
+const { streamToBuffer } = require('~/server/services/Files/Documents/encode');
+
+/**
+ * Encodes and formats video files for different endpoints
+ * @param {Express.Request} req - The request object
+ * @param {Array<MongoFile>} files - Array of video files
+ * @param {EModelEndpoint} endpoint - The endpoint to format for
+ * @returns {Promise<{ videos: Array, files: Array<MongoFile> }>}
+ */
+async function encodeAndFormatVideos(req, files, endpoint) {
+  const promises = [];
+  const encodingMethods = {};
+  /** @type {{ videos: any[]; files: MongoFile[] }} */
+  const result = {
+    videos: [],
+    files: [],
+  };
+
+  for (const file of files) {
+    if (!file || !file.filepath) {
+      continue;
+    }
+
+    const source = file.source ?? 'local';
+    if (!encodingMethods[source]) {
+      encodingMethods[source] = getStrategyFunctions(source);
+    }
+
+    const fileMetadata = {
+      file_id: file.file_id || file._id,
+      temp_file_id: file.temp_file_id,
+      filepath: file.filepath,
+      source: file.source,
+      filename: file.filename,
+      type: file.type,
+    };
+
+    promises.push([file, fileMetadata]);
+  }
+
+  const results = await Promise.allSettled(
+    promises.map(async ([file, fileMetadata]) => {
+      if (!file || !fileMetadata) {
+        return { file: null, content: null, metadata: fileMetadata };
+      }
+
+      try {
+        const source = file.source ?? 'local';
+        const { getDownloadStream } = encodingMethods[source];
+
+        const stream = await getDownloadStream(req, file.filepath);
+        const buffer = await streamToBuffer(stream);
+        const videoContent = buffer.toString('base64');
+
+        return {
+          file,
+          content: videoContent,
+          metadata: fileMetadata,
+        };
+      } catch (error) {
+        console.error(`Error processing video ${file.filename}:`, error);
+        return { file, content: null, metadata: fileMetadata };
+      }
+    }),
+  );
+
+  for (const settledResult of results) {
+    if (settledResult.status === 'rejected') {
+      console.error('Video processing failed:', settledResult.reason);
+      continue;
+    }
+
+    const { file, content, metadata } = settledResult.value;
+
+    if (!content || !file) {
+      if (metadata) {
+        result.files.push(metadata);
+      }
+      continue;
+    }
+
+    if (file.type.startsWith('video/') && isDocumentSupportedEndpoint(endpoint)) {
+      const videoBuffer = Buffer.from(content, 'base64');
+
+      const validation = await validateVideo(videoBuffer, videoBuffer.length, endpoint);
+      if (!validation.isValid) {
+        throw new Error(`Video validation failed: ${validation.error}`);
+      }
+
+      if (endpoint === EModelEndpoint.google) {
+        const videoPart = {
+          type: 'video',
+          mimeType: file.type,
+          data: content,
+        };
+        result.videos.push(videoPart);
+      }
+
+      result.files.push(metadata);
+    }
+  }
+
+  return result;
+}
+
+module.exports = {
+  encodeAndFormatVideos,
+};

api/server/services/Files/documents/encode.js

@@ -159,6 +159,13 @@ async function encodeAndFormatDocuments(req, files, endpoint) {
           file_data: `data:application/pdf;base64,${content}`,
         };
         result.documents.push(documentPart);
+      } else if (endpoint === EModelEndpoint.google) {
+        const documentPart = {
+          type: 'document',
+          mimeType: 'application/pdf',
+          data: content,
+        };
+        result.documents.push(documentPart);
       }
 
       result.files.push(metadata);
@@ -170,4 +177,5 @@ async function encodeAndFormatDocuments(req, files, endpoint) {
 
 module.exports = {
   encodeAndFormatDocuments,
+  streamToBuffer,
 };

api/server/services/Files/documents/index.js

@@ -1,5 +1,6 @@
-const { encodeAndFormatDocuments } = require('./encode');
+const { encodeAndFormatDocuments, streamToBuffer } = require('./encode');
 
 module.exports = {
   encodeAndFormatDocuments,
+  streamToBuffer,
 };