🖼️ feat: Add support for HEIC image format (#7914)

* feat: Add HEIC image format support with client-side conversion - Add HEIC/HEIF mime types to supported image formats - Install heic-to library for client-side HEIC to JPEG conversion - Create heicConverter utility with detection and conversion functions - Integrate HEIC processing into file upload flow - Add error handling and localization for HEIC conversion failures - Maintain backward compatibility with existing image formats - Resolves #5570

* feat: Add UI feedback during HEIC conversion - Show file thumbnail

* Addressing eslint errors

* Addressing the vite bundler issue

client/src/hooks/Files/useFileHandling.ts

@@ -1,24 +1,25 @@
-import { v4 } from 'uuid';
-import debounce from 'lodash/debounce';
 import { useQueryClient } from '@tanstack/react-query';
-import React, { useState, useEffect, useCallback, useRef, useMemo } from 'react';
+import type { TEndpointsConfig, TError } from 'librechat-data-provider';
 import {
-  QueryKeys,
-  EModelEndpoint,
-  mergeFileConfig,
-  isAgentsEndpoint,
-  isAssistantsEndpoint,
   defaultAssistantsVersion,
   fileConfig as defaultFileConfig,
+  EModelEndpoint,
+  isAgentsEndpoint,
+  isAssistantsEndpoint,
+  mergeFileConfig,
+  QueryKeys,
 } from 'librechat-data-provider';
-import type { TEndpointsConfig, TError } from 'librechat-data-provider';
+import debounce from 'lodash/debounce';
+import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
+import { v4 } from 'uuid';
 import type { ExtendedFile, FileSetter } from '~/common';
-import { useUploadFileMutation, useGetFileConfig } from '~/data-provider';
+import { useGetFileConfig, useUploadFileMutation } from '~/data-provider';
 import useLocalize, { TranslationKeys } from '~/hooks/useLocalize';
-import { useDelayedUploadToast } from './useDelayedUploadToast';
-import { useToastContext } from '~/Providers/ToastContext';
 import { useChatContext } from '~/Providers/ChatContext';
+import { useToastContext } from '~/Providers/ToastContext';
 import { logger, validateFiles } from '~/utils';
+import { processFileForUpload } from '~/utils/heicConverter';
+import { useDelayedUploadToast } from './useDelayedUploadToast';
 import useUpdateFiles from './useUpdateFiles';
 
 type UseFileHandling = {
@@ -262,41 +263,110 @@ const useFileHandling = (params?: UseFileHandling) => {
     for (const originalFile of fileList) {
       const file_id = v4();
       try {
-        const preview = URL.createObjectURL(originalFile);
-        const extendedFile: ExtendedFile = {
+        // Create initial preview with original file
+        const initialPreview = URL.createObjectURL(originalFile);
+
+        // Create initial ExtendedFile to show immediately
+        const initialExtendedFile: ExtendedFile = {
           file_id,
           file: originalFile,
           type: originalFile.type,
-          preview,
-          progress: 0.2,
+          preview: initialPreview,
+          progress: 0.1, // Show as processing
           size: originalFile.size,
         };
 
         if (_toolResource != null && _toolResource !== '') {
-          extendedFile.tool_resource = _toolResource;
+          initialExtendedFile.tool_resource = _toolResource;
         }
 
-        const isImage = originalFile.type.split('/')[0] === 'image';
-        const tool_resource =
-          extendedFile.tool_resource ?? params?.additionalMetadata?.tool_resource;
-        if (isAgentsEndpoint(endpoint) && !isImage && tool_resource == null) {
-          /** Note: this needs to be removed when we can support files to providers */
-          setError('com_error_files_unsupported_capability');
-          continue;
+        // Add file immediately to show in UI
+        addFile(initialExtendedFile);
+
+        // Check if HEIC conversion is needed and show toast
+        const isHEIC =
+          originalFile.type === 'image/heic' ||
+          originalFile.type === 'image/heif' ||
+          originalFile.name.toLowerCase().match(/\.(heic|heif)$/);
+
+        if (isHEIC) {
+          showToast({
+            message: localize('com_info_heic_converting'),
+            status: 'info',
+            duration: 3000,
+          });
         }
 
-        addFile(extendedFile);
+        // Process file for HEIC conversion if needed
+        const processedFile = await processFileForUpload(
+          originalFile,
+          0.9,
+          (conversionProgress) => {
+            // Update progress during HEIC conversion (0.1 to 0.5 range for conversion)
+            const adjustedProgress = 0.1 + conversionProgress * 0.4;
+            replaceFile({
+              ...initialExtendedFile,
+              progress: adjustedProgress,
+            });
+          },
+        );
 
-        if (isImage) {
-          loadImage(extendedFile, preview);
-          continue;
+        // If file was converted, update with new file and preview
+        if (processedFile !== originalFile) {
+          URL.revokeObjectURL(initialPreview); // Clean up original preview
+          const newPreview = URL.createObjectURL(processedFile);
+
+          const updatedExtendedFile: ExtendedFile = {
+            ...initialExtendedFile,
+            file: processedFile,
+            type: processedFile.type,
+            preview: newPreview,
+            progress: 0.5, // Conversion complete, ready for upload
+            size: processedFile.size,
+          };
+
+          replaceFile(updatedExtendedFile);
+
+          const isImage = processedFile.type.split('/')[0] === 'image';
+          if (isImage) {
+            loadImage(updatedExtendedFile, newPreview);
+            continue;
+          }
+
+          await startUpload(updatedExtendedFile);
+        } else {
+          // File wasn't converted, proceed with original
+          const isImage = originalFile.type.split('/')[0] === 'image';
+          const tool_resource =
+            initialExtendedFile.tool_resource ?? params?.additionalMetadata?.tool_resource;
+          if (isAgentsEndpoint(endpoint) && !isImage && tool_resource == null) {
+            /** Note: this needs to be removed when we can support files to providers */
+            setError('com_error_files_unsupported_capability');
+            continue;
+          }
+
+          // Update progress to show ready for upload
+          const readyExtendedFile = {
+            ...initialExtendedFile,
+            progress: 0.2,
+          };
+          replaceFile(readyExtendedFile);
+
+          if (isImage) {
+            loadImage(readyExtendedFile, initialPreview);
+            continue;
+          }
+
+          await startUpload(readyExtendedFile);
         }
-
-        await startUpload(extendedFile);
       } catch (error) {
         deleteFileById(file_id);
         console.log('file handling error', error);
-        setError('com_error_files_process');
+        if (error instanceof Error && error.message.includes('HEIC')) {
+          setError('com_error_heic_conversion');
+        } else {
+          setError('com_error_files_process');
+        }
       }
     }
   };

client/src/locales/en/translation.json

@@ -276,6 +276,8 @@
   "com_error_files_upload": "An error occurred while uploading the file.",
   "com_error_files_upload_canceled": "The file upload request was canceled. Note: the file upload may still be processing and will need to be manually deleted.",
   "com_error_files_validation": "An error occurred while validating the file.",
+  "com_error_heic_conversion": "Failed to convert HEIC image to JPEG. Please try converting the image manually or use a different format.",
+  "com_info_heic_converting": "Converting HEIC image to JPEG...",
   "com_error_input_length": "The latest message token count is too long, exceeding the token limit, or your token limit parameters are misconfigured, adversely affecting the context window. More info: {{0}}. Please shorten your message, adjust the max context size from the conversation parameters, or fork the conversation to continue.",
   "com_error_invalid_agent_provider": "The \"{{0}}\" provider is not available for use with Agents. Please go to your agent's settings and select a currently available provider.",
   "com_error_invalid_user_key": "Invalid key provided. Please provide a valid key and try again.",

client/src/utils/heicConverter.ts

@@ -0,0 +1,79 @@
+import { heicTo, isHeic } from 'heic-to';
+
+/**
+ * Check if a file is in HEIC format
+ * @param file - The file to check
+ * @returns Promise<boolean> - True if the file is HEIC
+ */
+export const isHEICFile = async (file: File): Promise<boolean> => {
+  try {
+    return await isHeic(file);
+  } catch (error) {
+    console.warn('Error checking if file is HEIC:', error);
+    // Fallback to mime type check
+    return file.type === 'image/heic' || file.type === 'image/heif';
+  }
+};
+
+/**
+ * Convert HEIC file to JPEG
+ * @param file - The HEIC file to convert
+ * @param quality - JPEG quality (0-1), default is 0.9
+ * @param onProgress - Optional callback to track conversion progress
+ * @returns Promise<File> - The converted JPEG file
+ */
+export const convertHEICToJPEG = async (
+  file: File,
+  quality: number = 0.9,
+  onProgress?: (progress: number) => void,
+): Promise<File> => {
+  try {
+    // Report conversion start
+    onProgress?.(0.3);
+
+    const convertedBlob = await heicTo({
+      blob: file,
+      type: 'image/jpeg',
+      quality,
+    });
+
+    // Report conversion completion
+    onProgress?.(0.8);
+
+    // Create a new File object with the converted blob
+    const convertedFile = new File([convertedBlob], file.name.replace(/\.(heic|heif)$/i, '.jpg'), {
+      type: 'image/jpeg',
+      lastModified: file.lastModified,
+    });
+
+    // Report file creation completion
+    onProgress?.(1.0);
+
+    return convertedFile;
+  } catch (error) {
+    console.error('Error converting HEIC to JPEG:', error);
+    throw new Error('Failed to convert HEIC image to JPEG');
+  }
+};
+
+/**
+ * Process a file, converting it from HEIC to JPEG if necessary
+ * @param file - The file to process
+ * @param quality - JPEG quality for conversion (0-1), default is 0.9
+ * @param onProgress - Optional callback to track conversion progress
+ * @returns Promise<File> - The processed file (converted if it was HEIC, original otherwise)
+ */
+export const processFileForUpload = async (
+  file: File,
+  quality: number = 0.9,
+  onProgress?: (progress: number) => void,
+): Promise<File> => {
+  const isHEIC = await isHEICFile(file);
+
+  if (isHEIC) {
+    console.log('HEIC file detected, converting to JPEG...');
+    return convertHEICToJPEG(file, quality, onProgress);
+  }
+
+  return file;
+};