🗂️ feat: Send Attachments Directly to Provider (Google) (#9100)

* feat: add validation for google PDFs and add google endpoint as a document supporting endpoint

* feat: add proper pdf formatting for google endpoints (requires PR #14 in agents)

* feat: add multimodal support for google endpoint attachments

* feat: add audio file svg

* fix: refactor attachments logic so multi-attachment messages work properly

* feat: add video file svg

* fix: allows for followup questions of uploaded multimodal attachments

* fix: remove incorrect final message filtering that was breaking Attachment component rendering

packages/api/src/files/validation.ts

@@ -5,6 +5,16 @@ export interface PDFValidationResult {
   error?: string;
 }
 
+export interface VideoValidationResult {
+  isValid: boolean;
+  error?: string;
+}
+
+export interface AudioValidationResult {
+  isValid: boolean;
+  error?: string;
+}
+
 export async function validatePdf(
   pdfBuffer: Buffer,
   fileSize: number,
@@ -18,6 +28,10 @@ export async function validatePdf(
     return validateOpenAIPdf(fileSize);
   }
 
+  if (endpoint === EModelEndpoint.google) {
+    return validateGooglePdf(fileSize);
+  }
+
   return { isValid: true };
 }
 
@@ -96,3 +110,76 @@ async function validateOpenAIPdf(fileSize: number): Promise<PDFValidationResult>
 
   return { isValid: true };
 }
+
+async function validateGooglePdf(fileSize: number): Promise<PDFValidationResult> {
+  if (fileSize > 20 * 1024 * 1024) {
+    return {
+      isValid: false,
+      error: "PDF file size exceeds Google's 20MB limit",
+    };
+  }
+
+  return { isValid: true };
+}
+
+/**
+ * Validates video files for different endpoints
+ * @param videoBuffer - The video file as a buffer
+ * @param fileSize - The file size in bytes
+ * @param endpoint - The endpoint to validate for
+ * @returns Promise that resolves to validation result
+ */
+export async function validateVideo(
+  videoBuffer: Buffer,
+  fileSize: number,
+  endpoint: EModelEndpoint,
+): Promise<VideoValidationResult> {
+  if (endpoint === EModelEndpoint.google) {
+    if (fileSize > 20 * 1024 * 1024) {
+      return {
+        isValid: false,
+        error: `Video file size (${Math.round(fileSize / (1024 * 1024))}MB) exceeds Google's 20MB limit`,
+      };
+    }
+  }
+
+  if (!videoBuffer || videoBuffer.length < 10) {
+    return {
+      isValid: false,
+      error: 'Invalid video file: too small or corrupted',
+    };
+  }
+
+  return { isValid: true };
+}
+
+/**
+ * Validates audio files for different endpoints
+ * @param audioBuffer - The audio file as a buffer
+ * @param fileSize - The file size in bytes
+ * @param endpoint - The endpoint to validate for
+ * @returns Promise that resolves to validation result
+ */
+export async function validateAudio(
+  audioBuffer: Buffer,
+  fileSize: number,
+  endpoint: EModelEndpoint,
+): Promise<AudioValidationResult> {
+  if (endpoint === EModelEndpoint.google) {
+    if (fileSize > 20 * 1024 * 1024) {
+      return {
+        isValid: false,
+        error: `Audio file size (${Math.round(fileSize / (1024 * 1024))}MB) exceeds Google's 20MB limit`,
+      };
+    }
+  }
+
+  if (!audioBuffer || audioBuffer.length < 10) {
+    return {
+      isValid: false,
+      error: 'Invalid audio file: too small or corrupted',
+    };
+  }
+
+  return { isValid: true };
+}

packages/client/src/svgs/AudioPaths.tsx

@@ -0,0 +1,41 @@
+export default function AudioPaths() {
+  return (
+    <>
+      <path
+        d="M8 15v6"
+        stroke="white"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M13 8v20"
+        stroke="white"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M18 10v16"
+        stroke="white"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M23 6v24"
+        stroke="white"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M28 12v12"
+        stroke="white"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+    </>
+  );
+}

packages/client/src/svgs/VideoPaths.tsx

@@ -0,0 +1,10 @@
+export default function VideoPaths() {
+  return (
+    <>
+      {/* Video container - rounded rectangle (not filled) */}
+      <rect x="8" y="10" width="20" height="16" rx="3" stroke="white" strokeWidth="2" fill="none" />
+      {/* Play button - centered and pointing right */}
+      <path d="M22 18l-6 4v-8L22 18z" fill="white" />
+    </>
+  );
+}

packages/client/src/svgs/index.ts

@@ -65,9 +65,11 @@ export { default as PersonalizationIcon } from './PersonalizationIcon';
 export { default as MCPIcon } from './MCPIcon';
 export { default as VectorIcon } from './VectorIcon';
 export { default as SquirclePlusIcon } from './SquirclePlusIcon';
+export { default as AudioPaths } from './AudioPaths';
 export { default as CodePaths } from './CodePaths';
 export { default as FileIcon } from './FileIcon';
 export { default as FilePaths } from './FilePaths';
 export { default as SheetPaths } from './SheetPaths';
 export { default as TextPaths } from './TextPaths';
+export { default as VideoPaths } from './VideoPaths';
 export { default as SharePointIcon } from './SharePointIcon';

packages/data-provider/src/file-config.ts

@@ -57,6 +57,27 @@ export const fullMimeTypesList = [
   'application/zip',
   'image/svg',
   'image/svg+xml',
+  // Video formats
+  'video/mp4',
+  'video/avi',
+  'video/mov',
+  'video/wmv',
+  'video/flv',
+  'video/webm',
+  'video/mkv',
+  'video/m4v',
+  'video/3gp',
+  'video/ogv',
+  // Audio formats
+  'audio/mp3',
+  'audio/wav',
+  'audio/ogg',
+  'audio/m4a',
+  'audio/aac',
+  'audio/flac',
+  'audio/wma',
+  'audio/opus',
+  'audio/mpeg',
   ...excelFileTypes,
 ];
 
@@ -123,7 +144,9 @@ export const applicationMimeTypes =
 export const imageMimeTypes = /^image\/(jpeg|gif|png|webp|heic|heif)$/;
 
 export const audioMimeTypes =
-  /^audio\/(mp3|mpeg|mpeg3|wav|wave|x-wav|ogg|vorbis|mp4|x-m4a|flac|x-flac|webm)$/;
+  /^audio\/(mp3|mpeg|mpeg3|wav|wave|x-wav|ogg|vorbis|mp4|x-m4a|flac|x-flac|webm|aac|wma|opus)$/;
+
+export const videoMimeTypes = /^video\/(mp4|avi|mov|wmv|flv|webm|mkv|m4v|3gp|ogv)$/;
 
 export const defaultOCRMimeTypes = [
   imageMimeTypes,
@@ -142,8 +165,9 @@ export const supportedMimeTypes = [
   excelMimeTypes,
   applicationMimeTypes,
   imageMimeTypes,
+  videoMimeTypes,
   audioMimeTypes,
-  /** Supported by LC Code Interpreter PAI */
+  /** Supported by LC Code Interpreter API */
   /^image\/(svg|svg\+xml)$/,
 ];
 

packages/data-provider/src/schemas.ts

@@ -38,6 +38,7 @@ export const documentSupportedEndpoints = new Set<EModelEndpoint>([
   EModelEndpoint.anthropic,
   EModelEndpoint.openAI,
   EModelEndpoint.azureOpenAI,
+  EModelEndpoint.google,
 ]);
 
 export const isDocumentSupportedEndpoint = (endpoint: EModelEndpoint): boolean => {