diff --git a/client/package.json b/client/package.json index 7cb983d21..5218ebdc6 100644 --- a/client/package.json +++ b/client/package.json @@ -65,6 +65,7 @@ "export-from-json": "^1.7.2", "filenamify": "^6.0.0", "framer-motion": "^11.5.4", + "heic-to": "^1.1.14", "html-to-image": "^1.11.11", "i18next": "^24.2.2", "i18next-browser-languagedetector": "^8.0.3", diff --git a/client/src/hooks/Files/useFileHandling.ts b/client/src/hooks/Files/useFileHandling.ts index 69a99fef3..9e03f2933 100644 --- a/client/src/hooks/Files/useFileHandling.ts +++ b/client/src/hooks/Files/useFileHandling.ts @@ -1,24 +1,25 @@ -import { v4 } from 'uuid'; -import debounce from 'lodash/debounce'; import { useQueryClient } from '@tanstack/react-query'; -import React, { useState, useEffect, useCallback, useRef, useMemo } from 'react'; +import type { TEndpointsConfig, TError } from 'librechat-data-provider'; import { - QueryKeys, - EModelEndpoint, - mergeFileConfig, - isAgentsEndpoint, - isAssistantsEndpoint, defaultAssistantsVersion, fileConfig as defaultFileConfig, + EModelEndpoint, + isAgentsEndpoint, + isAssistantsEndpoint, + mergeFileConfig, + QueryKeys, } from 'librechat-data-provider'; -import type { TEndpointsConfig, TError } from 'librechat-data-provider'; +import debounce from 'lodash/debounce'; +import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { v4 } from 'uuid'; import type { ExtendedFile, FileSetter } from '~/common'; -import { useUploadFileMutation, useGetFileConfig } from '~/data-provider'; +import { useGetFileConfig, useUploadFileMutation } from '~/data-provider'; import useLocalize, { TranslationKeys } from '~/hooks/useLocalize'; -import { useDelayedUploadToast } from './useDelayedUploadToast'; -import { useToastContext } from '~/Providers/ToastContext'; import { useChatContext } from '~/Providers/ChatContext'; +import { useToastContext } from '~/Providers/ToastContext'; import { logger, validateFiles } from '~/utils'; +import { processFileForUpload } from '~/utils/heicConverter'; +import { useDelayedUploadToast } from './useDelayedUploadToast'; import useUpdateFiles from './useUpdateFiles'; type UseFileHandling = { @@ -262,41 +263,110 @@ const useFileHandling = (params?: UseFileHandling) => { for (const originalFile of fileList) { const file_id = v4(); try { - const preview = URL.createObjectURL(originalFile); - const extendedFile: ExtendedFile = { + // Create initial preview with original file + const initialPreview = URL.createObjectURL(originalFile); + + // Create initial ExtendedFile to show immediately + const initialExtendedFile: ExtendedFile = { file_id, file: originalFile, type: originalFile.type, - preview, - progress: 0.2, + preview: initialPreview, + progress: 0.1, // Show as processing size: originalFile.size, }; if (_toolResource != null && _toolResource !== '') { - extendedFile.tool_resource = _toolResource; + initialExtendedFile.tool_resource = _toolResource; } - const isImage = originalFile.type.split('/')[0] === 'image'; - const tool_resource = - extendedFile.tool_resource ?? params?.additionalMetadata?.tool_resource; - if (isAgentsEndpoint(endpoint) && !isImage && tool_resource == null) { - /** Note: this needs to be removed when we can support files to providers */ - setError('com_error_files_unsupported_capability'); - continue; + // Add file immediately to show in UI + addFile(initialExtendedFile); + + // Check if HEIC conversion is needed and show toast + const isHEIC = + originalFile.type === 'image/heic' || + originalFile.type === 'image/heif' || + originalFile.name.toLowerCase().match(/\.(heic|heif)$/); + + if (isHEIC) { + showToast({ + message: localize('com_info_heic_converting'), + status: 'info', + duration: 3000, + }); } - addFile(extendedFile); + // Process file for HEIC conversion if needed + const processedFile = await processFileForUpload( + originalFile, + 0.9, + (conversionProgress) => { + // Update progress during HEIC conversion (0.1 to 0.5 range for conversion) + const adjustedProgress = 0.1 + conversionProgress * 0.4; + replaceFile({ + ...initialExtendedFile, + progress: adjustedProgress, + }); + }, + ); - if (isImage) { - loadImage(extendedFile, preview); - continue; + // If file was converted, update with new file and preview + if (processedFile !== originalFile) { + URL.revokeObjectURL(initialPreview); // Clean up original preview + const newPreview = URL.createObjectURL(processedFile); + + const updatedExtendedFile: ExtendedFile = { + ...initialExtendedFile, + file: processedFile, + type: processedFile.type, + preview: newPreview, + progress: 0.5, // Conversion complete, ready for upload + size: processedFile.size, + }; + + replaceFile(updatedExtendedFile); + + const isImage = processedFile.type.split('/')[0] === 'image'; + if (isImage) { + loadImage(updatedExtendedFile, newPreview); + continue; + } + + await startUpload(updatedExtendedFile); + } else { + // File wasn't converted, proceed with original + const isImage = originalFile.type.split('/')[0] === 'image'; + const tool_resource = + initialExtendedFile.tool_resource ?? params?.additionalMetadata?.tool_resource; + if (isAgentsEndpoint(endpoint) && !isImage && tool_resource == null) { + /** Note: this needs to be removed when we can support files to providers */ + setError('com_error_files_unsupported_capability'); + continue; + } + + // Update progress to show ready for upload + const readyExtendedFile = { + ...initialExtendedFile, + progress: 0.2, + }; + replaceFile(readyExtendedFile); + + if (isImage) { + loadImage(readyExtendedFile, initialPreview); + continue; + } + + await startUpload(readyExtendedFile); } - - await startUpload(extendedFile); } catch (error) { deleteFileById(file_id); console.log('file handling error', error); - setError('com_error_files_process'); + if (error instanceof Error && error.message.includes('HEIC')) { + setError('com_error_heic_conversion'); + } else { + setError('com_error_files_process'); + } } } }; diff --git a/client/src/locales/en/translation.json b/client/src/locales/en/translation.json index d7171a032..c314329ed 100644 --- a/client/src/locales/en/translation.json +++ b/client/src/locales/en/translation.json @@ -276,6 +276,8 @@ "com_error_files_upload": "An error occurred while uploading the file.", "com_error_files_upload_canceled": "The file upload request was canceled. Note: the file upload may still be processing and will need to be manually deleted.", "com_error_files_validation": "An error occurred while validating the file.", + "com_error_heic_conversion": "Failed to convert HEIC image to JPEG. Please try converting the image manually or use a different format.", + "com_info_heic_converting": "Converting HEIC image to JPEG...", "com_error_input_length": "The latest message token count is too long, exceeding the token limit, or your token limit parameters are misconfigured, adversely affecting the context window. More info: {{0}}. Please shorten your message, adjust the max context size from the conversation parameters, or fork the conversation to continue.", "com_error_invalid_agent_provider": "The \"{{0}}\" provider is not available for use with Agents. Please go to your agent's settings and select a currently available provider.", "com_error_invalid_user_key": "Invalid key provided. Please provide a valid key and try again.", diff --git a/client/src/utils/heicConverter.ts b/client/src/utils/heicConverter.ts new file mode 100644 index 000000000..a14e09a05 --- /dev/null +++ b/client/src/utils/heicConverter.ts @@ -0,0 +1,79 @@ +import { heicTo, isHeic } from 'heic-to'; + +/** + * Check if a file is in HEIC format + * @param file - The file to check + * @returns Promise - True if the file is HEIC + */ +export const isHEICFile = async (file: File): Promise => { + try { + return await isHeic(file); + } catch (error) { + console.warn('Error checking if file is HEIC:', error); + // Fallback to mime type check + return file.type === 'image/heic' || file.type === 'image/heif'; + } +}; + +/** + * Convert HEIC file to JPEG + * @param file - The HEIC file to convert + * @param quality - JPEG quality (0-1), default is 0.9 + * @param onProgress - Optional callback to track conversion progress + * @returns Promise - The converted JPEG file + */ +export const convertHEICToJPEG = async ( + file: File, + quality: number = 0.9, + onProgress?: (progress: number) => void, +): Promise => { + try { + // Report conversion start + onProgress?.(0.3); + + const convertedBlob = await heicTo({ + blob: file, + type: 'image/jpeg', + quality, + }); + + // Report conversion completion + onProgress?.(0.8); + + // Create a new File object with the converted blob + const convertedFile = new File([convertedBlob], file.name.replace(/\.(heic|heif)$/i, '.jpg'), { + type: 'image/jpeg', + lastModified: file.lastModified, + }); + + // Report file creation completion + onProgress?.(1.0); + + return convertedFile; + } catch (error) { + console.error('Error converting HEIC to JPEG:', error); + throw new Error('Failed to convert HEIC image to JPEG'); + } +}; + +/** + * Process a file, converting it from HEIC to JPEG if necessary + * @param file - The file to process + * @param quality - JPEG quality for conversion (0-1), default is 0.9 + * @param onProgress - Optional callback to track conversion progress + * @returns Promise - The processed file (converted if it was HEIC, original otherwise) + */ +export const processFileForUpload = async ( + file: File, + quality: number = 0.9, + onProgress?: (progress: number) => void, +): Promise => { + const isHEIC = await isHEICFile(file); + + if (isHEIC) { + console.log('HEIC file detected, converting to JPEG...'); + return convertHEICToJPEG(file, quality, onProgress); + } + + return file; +}; diff --git a/client/vite.config.ts b/client/vite.config.ts index 98451b6c0..4ce4fc3b8 100644 --- a/client/vite.config.ts +++ b/client/vite.config.ts @@ -1,10 +1,10 @@ -import path from 'path'; -import { defineConfig } from 'vite'; import react from '@vitejs/plugin-react'; -import { VitePWA } from 'vite-plugin-pwa'; +import path from 'path'; +import type { Plugin } from 'vite'; +import { defineConfig } from 'vite'; import { compression } from 'vite-plugin-compression2'; import { nodePolyfills } from 'vite-plugin-node-polyfills'; -import type { Plugin } from 'vite'; +import { VitePWA } from 'vite-plugin-pwa'; // https://vitejs.dev/config/ export default defineConfig(({ command }) => ({ @@ -169,6 +169,9 @@ export default defineConfig(({ command }) => ({ if (id.includes('react-select') || id.includes('downshift')) { return 'advanced-inputs'; } + if (id.includes('heic-to')) { + return 'heic-converter'; + } // Existing chunks if (id.includes('@radix-ui')) { diff --git a/package-lock.json b/package-lock.json index ed1ef7486..d9dc4cc2e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2464,6 +2464,7 @@ "export-from-json": "^1.7.2", "filenamify": "^6.0.0", "framer-motion": "^11.5.4", + "heic-to": "^1.1.14", "html-to-image": "^1.11.11", "i18next": "^24.2.2", "i18next-browser-languagedetector": "^8.0.3", @@ -32074,6 +32075,12 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/heic-to": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/heic-to/-/heic-to-1.1.14.tgz", + "integrity": "sha512-CxJE27BF6JcQvrL1giK478iSZr7EJNTnAN2Th1rAJiN1BSMYZxDLm4PL/p/ha3aSqVHvCo+YNk++5tIj0JVxLQ==", + "license": "LGPL-3.0" + }, "node_modules/highlight.js": { "version": "11.8.0", "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-11.8.0.tgz", diff --git a/packages/data-provider/src/file-config.ts b/packages/data-provider/src/file-config.ts index 050fb2541..80521b4c4 100644 --- a/packages/data-provider/src/file-config.ts +++ b/packages/data-provider/src/file-config.ts @@ -1,6 +1,6 @@ import { z } from 'zod'; import { EModelEndpoint } from './schemas'; -import type { FileConfig, EndpointFileConfig } from './types/files'; +import type { EndpointFileConfig, FileConfig } from './types/files'; export const supportsFiles = { [EModelEndpoint.openAI]: true, @@ -49,6 +49,8 @@ export const fullMimeTypesList = [ 'text/javascript', 'image/gif', 'image/png', + 'image/heic', + 'image/heif', 'application/x-tar', 'application/typescript', 'application/xml', @@ -80,6 +82,8 @@ export const codeInterpreterMimeTypesList = [ 'text/javascript', 'image/gif', 'image/png', + 'image/heic', + 'image/heif', 'application/x-tar', 'application/typescript', 'application/xml', @@ -105,7 +109,7 @@ export const retrievalMimeTypesList = [ 'text/plain', ]; -export const imageExtRegex = /\.(jpg|jpeg|png|gif|webp)$/i; +export const imageExtRegex = /\.(jpg|jpeg|png|gif|webp|heic|heif)$/i; export const excelMimeTypes = /^application\/(vnd\.ms-excel|msexcel|x-msexcel|x-ms-excel|x-excel|x-dos_ms_excel|xls|x-xls|vnd\.openxmlformats-officedocument\.spreadsheetml\.sheet)$/; @@ -116,7 +120,7 @@ export const textMimeTypes = export const applicationMimeTypes = /^(application\/(epub\+zip|csv|json|pdf|x-tar|typescript|vnd\.openxmlformats-officedocument\.(wordprocessingml\.document|presentationml\.presentation|spreadsheetml\.sheet)|xml|zip))$/; -export const imageMimeTypes = /^image\/(jpeg|gif|png|webp)$/; +export const imageMimeTypes = /^image\/(jpeg|gif|png|webp|heic|heif)$/; export const supportedMimeTypes = [ textMimeTypes,