🖼️ feat: Add support for HEIC image format (#7914)

* feat: Add HEIC image format support with client-side conversion - Add HEIC/HEIF mime types to supported image formats - Install heic-to library for client-side HEIC to JPEG conversion - Create heicConverter utility with detection and conversion functions - Integrate HEIC processing into file upload flow - Add error handling and localization for HEIC conversion failures - Maintain backward compatibility with existing image formats - Resolves #5570 * feat: Add UI feedback during HEIC conversion - Show file thumbnail * Addressing eslint errors * Addressing the vite bundler issue
2025-09-21 21:50:49 +02:00 · 2025-06-18 06:42:15 +05:30 · 2025-06-18 06:42:15 +05:30 · 3c9357580e
commit 3c9357580e
parent 10c0d7d474
7 changed files with 204 additions and 38 deletions
--- a/client/package.json
+++ b/client/package.json
@ -65,6 +65,7 @@
    "export-from-json": "^1.7.2",
    "filenamify": "^6.0.0",
    "framer-motion": "^11.5.4",
+    "heic-to": "^1.1.14",
    "html-to-image": "^1.11.11",
    "i18next": "^24.2.2",
    "i18next-browser-languagedetector": "^8.0.3",
--- a/client/src/hooks/Files/useFileHandling.ts
+++ b/client/src/hooks/Files/useFileHandling.ts
@ -1,24 +1,25 @@
-import { v4 } from 'uuid';
-import debounce from 'lodash/debounce';
 import { useQueryClient } from '@tanstack/react-query';
-import React, { useState, useEffect, useCallback, useRef, useMemo } from 'react';
+import type { TEndpointsConfig, TError } from 'librechat-data-provider';
 import {
-  QueryKeys,
-  EModelEndpoint,
-  mergeFileConfig,
-  isAgentsEndpoint,
-  isAssistantsEndpoint,
  defaultAssistantsVersion,
  fileConfig as defaultFileConfig,
+  EModelEndpoint,
+  isAgentsEndpoint,
+  isAssistantsEndpoint,
+  mergeFileConfig,
+  QueryKeys,
 } from 'librechat-data-provider';
-import type { TEndpointsConfig, TError } from 'librechat-data-provider';
+import debounce from 'lodash/debounce';
+import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
+import { v4 } from 'uuid';
 import type { ExtendedFile, FileSetter } from '~/common';
-import { useUploadFileMutation, useGetFileConfig } from '~/data-provider';
+import { useGetFileConfig, useUploadFileMutation } from '~/data-provider';
 import useLocalize, { TranslationKeys } from '~/hooks/useLocalize';
-import { useDelayedUploadToast } from './useDelayedUploadToast';
-import { useToastContext } from '~/Providers/ToastContext';
 import { useChatContext } from '~/Providers/ChatContext';
+import { useToastContext } from '~/Providers/ToastContext';
 import { logger, validateFiles } from '~/utils';
+import { processFileForUpload } from '~/utils/heicConverter';
+import { useDelayedUploadToast } from './useDelayedUploadToast';
 import useUpdateFiles from './useUpdateFiles';

 type UseFileHandling = {
@ -262,41 +263,110 @@ const useFileHandling = (params?: UseFileHandling) => {
    for (const originalFile of fileList) {
      const file_id = v4();
      try {
-        const preview = URL.createObjectURL(originalFile);
-        const extendedFile: ExtendedFile = {
+        // Create initial preview with original file
+        const initialPreview = URL.createObjectURL(originalFile);
+
+        // Create initial ExtendedFile to show immediately
+        const initialExtendedFile: ExtendedFile = {
          file_id,
          file: originalFile,
          type: originalFile.type,
-          preview,
-          progress: 0.2,
+          preview: initialPreview,
+          progress: 0.1, // Show as processing
          size: originalFile.size,
        };

        if (_toolResource != null && _toolResource !== '') {
-          extendedFile.tool_resource = _toolResource;
+          initialExtendedFile.tool_resource = _toolResource;
        }

-        const isImage = originalFile.type.split('/')[0] === 'image';
-        const tool_resource =
-          extendedFile.tool_resource ?? params?.additionalMetadata?.tool_resource;
-        if (isAgentsEndpoint(endpoint) && !isImage && tool_resource == null) {
-          /** Note: this needs to be removed when we can support files to providers */
-          setError('com_error_files_unsupported_capability');
-          continue;
+        // Add file immediately to show in UI
+        addFile(initialExtendedFile);
+
+        // Check if HEIC conversion is needed and show toast
+        const isHEIC =
+          originalFile.type === 'image/heic' ||
+          originalFile.type === 'image/heif' ||
+          originalFile.name.toLowerCase().match(/\.(heic|heif)$/);
+
+        if (isHEIC) {
+          showToast({
+            message: localize('com_info_heic_converting'),
+            status: 'info',
+            duration: 3000,
+          });
        }

-        addFile(extendedFile);
+        // Process file for HEIC conversion if needed
+        const processedFile = await processFileForUpload(
+          originalFile,
+          0.9,
+          (conversionProgress) => {
+            // Update progress during HEIC conversion (0.1 to 0.5 range for conversion)
+            const adjustedProgress = 0.1 + conversionProgress * 0.4;
+            replaceFile({
+              ...initialExtendedFile,
+              progress: adjustedProgress,
+            });
+          },
+        );

-        if (isImage) {
-          loadImage(extendedFile, preview);
-          continue;
+        // If file was converted, update with new file and preview
+        if (processedFile !== originalFile) {
+          URL.revokeObjectURL(initialPreview); // Clean up original preview
+          const newPreview = URL.createObjectURL(processedFile);
+
+          const updatedExtendedFile: ExtendedFile = {
+            ...initialExtendedFile,
+            file: processedFile,
+            type: processedFile.type,
+            preview: newPreview,
+            progress: 0.5, // Conversion complete, ready for upload
+            size: processedFile.size,
+          };
+
+          replaceFile(updatedExtendedFile);
+
+          const isImage = processedFile.type.split('/')[0] === 'image';
+          if (isImage) {
+            loadImage(updatedExtendedFile, newPreview);
+            continue;
+          }
+
+          await startUpload(updatedExtendedFile);
+        } else {
+          // File wasn't converted, proceed with original
+          const isImage = originalFile.type.split('/')[0] === 'image';
+          const tool_resource =
+            initialExtendedFile.tool_resource ?? params?.additionalMetadata?.tool_resource;
+          if (isAgentsEndpoint(endpoint) && !isImage && tool_resource == null) {
+            /** Note: this needs to be removed when we can support files to providers */
+            setError('com_error_files_unsupported_capability');
+            continue;
+          }
+
+          // Update progress to show ready for upload
+          const readyExtendedFile = {
+            ...initialExtendedFile,
+            progress: 0.2,
+          };
+          replaceFile(readyExtendedFile);
+
+          if (isImage) {
+            loadImage(readyExtendedFile, initialPreview);
+            continue;
+          }
+
+          await startUpload(readyExtendedFile);
        }
-
-        await startUpload(extendedFile);
      } catch (error) {
        deleteFileById(file_id);
        console.log('file handling error', error);
-        setError('com_error_files_process');
+        if (error instanceof Error && error.message.includes('HEIC')) {
+          setError('com_error_heic_conversion');
+        } else {
+          setError('com_error_files_process');
+        }
      }
    }
  };
--- a/client/src/locales/en/translation.json
+++ b/client/src/locales/en/translation.json
@ -276,6 +276,8 @@
  "com_error_files_upload": "An error occurred while uploading the file.",
  "com_error_files_upload_canceled": "The file upload request was canceled. Note: the file upload may still be processing and will need to be manually deleted.",
  "com_error_files_validation": "An error occurred while validating the file.",
+  "com_error_heic_conversion": "Failed to convert HEIC image to JPEG. Please try converting the image manually or use a different format.",
+  "com_info_heic_converting": "Converting HEIC image to JPEG...",
  "com_error_input_length": "The latest message token count is too long, exceeding the token limit, or your token limit parameters are misconfigured, adversely affecting the context window. More info: {{0}}. Please shorten your message, adjust the max context size from the conversation parameters, or fork the conversation to continue.",
  "com_error_invalid_agent_provider": "The \"{{0}}\" provider is not available for use with Agents. Please go to your agent's settings and select a currently available provider.",
  "com_error_invalid_user_key": "Invalid key provided. Please provide a valid key and try again.",
--- a/client/src/utils/heicConverter.ts
+++ b/client/src/utils/heicConverter.ts
@ -0,0 +1,79 @@
+import { heicTo, isHeic } from 'heic-to';
+
+/**
+ * Check if a file is in HEIC format
+ * @param file - The file to check
+ * @returns Promise<boolean> - True if the file is HEIC
+ */
+export const isHEICFile = async (file: File): Promise<boolean> => {
+  try {
+    return await isHeic(file);
+  } catch (error) {
+    console.warn('Error checking if file is HEIC:', error);
+    // Fallback to mime type check
+    return file.type === 'image/heic' || file.type === 'image/heif';
+  }
+};
+
+/**
+ * Convert HEIC file to JPEG
+ * @param file - The HEIC file to convert
+ * @param quality - JPEG quality (0-1), default is 0.9
+ * @param onProgress - Optional callback to track conversion progress
+ * @returns Promise<File> - The converted JPEG file
+ */
+export const convertHEICToJPEG = async (
+  file: File,
+  quality: number = 0.9,
+  onProgress?: (progress: number) => void,
+): Promise<File> => {
+  try {
+    // Report conversion start
+    onProgress?.(0.3);
+
+    const convertedBlob = await heicTo({
+      blob: file,
+      type: 'image/jpeg',
+      quality,
+    });
+
+    // Report conversion completion
+    onProgress?.(0.8);
+
+    // Create a new File object with the converted blob
+    const convertedFile = new File([convertedBlob], file.name.replace(/\.(heic|heif)$/i, '.jpg'), {
+      type: 'image/jpeg',
+      lastModified: file.lastModified,
+    });
+
+    // Report file creation completion
+    onProgress?.(1.0);
+
+    return convertedFile;
+  } catch (error) {
+    console.error('Error converting HEIC to JPEG:', error);
+    throw new Error('Failed to convert HEIC image to JPEG');
+  }
+};
+
+/**
+ * Process a file, converting it from HEIC to JPEG if necessary
+ * @param file - The file to process
+ * @param quality - JPEG quality for conversion (0-1), default is 0.9
+ * @param onProgress - Optional callback to track conversion progress
+ * @returns Promise<File> - The processed file (converted if it was HEIC, original otherwise)
+ */
+export const processFileForUpload = async (
+  file: File,
+  quality: number = 0.9,
+  onProgress?: (progress: number) => void,
+): Promise<File> => {
+  const isHEIC = await isHEICFile(file);
+
+  if (isHEIC) {
+    console.log('HEIC file detected, converting to JPEG...');
+    return convertHEICToJPEG(file, quality, onProgress);
+  }
+
+  return file;
+};
--- a/client/vite.config.ts
+++ b/client/vite.config.ts
@ -1,10 +1,10 @@
-import path from 'path';
-import { defineConfig } from 'vite';
 import react from '@vitejs/plugin-react';
-import { VitePWA } from 'vite-plugin-pwa';
+import path from 'path';
+import type { Plugin } from 'vite';
+import { defineConfig } from 'vite';
 import { compression } from 'vite-plugin-compression2';
 import { nodePolyfills } from 'vite-plugin-node-polyfills';
-import type { Plugin } from 'vite';
+import { VitePWA } from 'vite-plugin-pwa';

 // https://vitejs.dev/config/
 export default defineConfig(({ command }) => ({
@ -169,6 +169,9 @@ export default defineConfig(({ command }) => ({
            if (id.includes('react-select') || id.includes('downshift')) {
              return 'advanced-inputs';
            }
+            if (id.includes('heic-to')) {
+              return 'heic-converter';
+            }

            // Existing chunks
            if (id.includes('@radix-ui')) {
--- a/package-lock.json
+++ b/package-lock.json
@ -2464,6 +2464,7 @@
        "export-from-json": "^1.7.2",
        "filenamify": "^6.0.0",
        "framer-motion": "^11.5.4",
+        "heic-to": "^1.1.14",
        "html-to-image": "^1.11.11",
        "i18next": "^24.2.2",
        "i18next-browser-languagedetector": "^8.0.3",
@ -32074,6 +32075,12 @@
        "url": "https://opencollective.com/unified"
      }
    },
+    "node_modules/heic-to": {
+      "version": "1.1.14",
+      "resolved": "https://registry.npmjs.org/heic-to/-/heic-to-1.1.14.tgz",
+      "integrity": "sha512-CxJE27BF6JcQvrL1giK478iSZr7EJNTnAN2Th1rAJiN1BSMYZxDLm4PL/p/ha3aSqVHvCo+YNk++5tIj0JVxLQ==",
+      "license": "LGPL-3.0"
+    },
    "node_modules/highlight.js": {
      "version": "11.8.0",
      "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-11.8.0.tgz",
--- a/packages/data-provider/src/file-config.ts
+++ b/packages/data-provider/src/file-config.ts
@ -1,6 +1,6 @@
 import { z } from 'zod';
 import { EModelEndpoint } from './schemas';
-import type { FileConfig, EndpointFileConfig } from './types/files';
+import type { EndpointFileConfig, FileConfig } from './types/files';

 export const supportsFiles = {
  [EModelEndpoint.openAI]: true,
@ -49,6 +49,8 @@ export const fullMimeTypesList = [
  'text/javascript',
  'image/gif',
  'image/png',
+  'image/heic',
+  'image/heif',
  'application/x-tar',
  'application/typescript',
  'application/xml',
@ -80,6 +82,8 @@ export const codeInterpreterMimeTypesList = [
  'text/javascript',
  'image/gif',
  'image/png',
+  'image/heic',
+  'image/heif',
  'application/x-tar',
  'application/typescript',
  'application/xml',
@ -105,7 +109,7 @@ export const retrievalMimeTypesList = [
  'text/plain',
 ];

-export const imageExtRegex = /\.(jpg|jpeg|png|gif|webp)$/i;
+export const imageExtRegex = /\.(jpg|jpeg|png|gif|webp|heic|heif)$/i;

 export const excelMimeTypes =
  /^application\/(vnd\.ms-excel|msexcel|x-msexcel|x-ms-excel|x-excel|x-dos_ms_excel|xls|x-xls|vnd\.openxmlformats-officedocument\.spreadsheetml\.sheet)$/;
@ -116,7 +120,7 @@ export const textMimeTypes =
 export const applicationMimeTypes =
  /^(application\/(epub\+zip|csv|json|pdf|x-tar|typescript|vnd\.openxmlformats-officedocument\.(wordprocessingml\.document|presentationml\.presentation|spreadsheetml\.sheet)|xml|zip))$/;

-export const imageMimeTypes = /^image\/(jpeg|gif|png|webp)$/;
+export const imageMimeTypes = /^image\/(jpeg|gif|png|webp|heic|heif)$/;

 export const supportedMimeTypes = [
  textMimeTypes,