🗂️ feat: Better Persistence for Code Execution Files Between Sessions (#11362)

* refactor: process code output files for re-use (WIP)

* feat: file attachment handling with additional metadata for downloads

* refactor: Update directory path logic for local file saving based on basePath

* refactor: file attachment handling to support TFile type and improve data merging logic

* feat: thread filtering of code-generated files

- Introduced parentMessageId parameter in addedConvo and initialize functions to enhance thread management.
- Updated related methods to utilize parentMessageId for retrieving messages and filtering code-generated files by conversation threads.
- Enhanced type definitions to include parentMessageId in relevant interfaces for better clarity and usage.

* chore: imports/params ordering

* feat: update file model to use messageId for filtering and processing

- Changed references from 'message' to 'messageId' in file-related methods for consistency.
- Added messageId field to the file schema and updated related types.
- Enhanced file processing logic to accommodate the new messageId structure.

* feat: enhance file retrieval methods to support user-uploaded execute_code files

- Added a new method `getUserCodeFiles` to retrieve user-uploaded execute_code files, excluding code-generated files.
- Updated existing file retrieval methods to improve filtering logic and handle edge cases.
- Enhanced thread data extraction to collect both message IDs and file IDs efficiently.
- Integrated `getUserCodeFiles` into relevant endpoints for better file management in conversations.

* chore: update @librechat/agents package version to 3.0.78 in package-lock.json and related package.json files

* refactor: file processing and retrieval logic

- Added a fallback mechanism for download URLs when files exceed size limits or cannot be processed locally.
- Implemented a deduplication strategy for code-generated files based on conversationId and filename to optimize storage.
- Updated file retrieval methods to ensure proper filtering by messageIds, preventing orphaned files from being included.
- Introduced comprehensive tests for new thread data extraction functionality, covering edge cases and performance considerations.

* fix: improve file retrieval tests and handling of optional properties

- Updated tests to safely access optional properties using non-null assertions.
- Modified test descriptions for clarity regarding the exclusion of execute_code files.
- Ensured that the retrieval logic correctly reflects the expected outcomes for file queries.

* test: add comprehensive unit tests for processCodeOutput functionality

- Introduced a new test suite for the processCodeOutput function, covering various scenarios including file retrieval, creation, and processing for both image and non-image files.
- Implemented mocks for dependencies such as axios, logger, and file models to isolate tests and ensure reliable outcomes.
- Validated behavior for existing files, new file creation, and error handling, including size limits and fallback mechanisms.
- Enhanced test coverage for metadata handling and usage increment logic, ensuring robust verification of file processing outcomes.

* test: enhance file size limit enforcement in processCodeOutput tests

- Introduced a configurable file size limit for tests to improve flexibility and coverage.
- Mocked the `librechat-data-provider` to allow dynamic adjustment of file size limits during tests.
- Updated the file size limit enforcement test to validate behavior when files exceed specified limits, ensuring proper fallback to download URLs.
- Reset file size limit after tests to maintain isolation for subsequent test cases.

client/src/components/Chat/Messages/Content/Parts/Attachment.tsx

@@ -8,9 +8,13 @@ import { cn } from '~/utils';
 
 const FileAttachment = memo(({ attachment }: { attachment: Partial<TAttachment> }) => {
   const [isVisible, setIsVisible] = useState(false);
+  const file = attachment as TFile & TAttachmentMetadata;
   const { handleDownload } = useAttachmentLink({
     href: attachment.filepath ?? '',
     filename: attachment.filename ?? '',
+    file_id: file.file_id,
+    user: file.user,
+    source: file.source,
   });
   const extension = attachment.filename?.split('.').pop();
 

client/src/components/Chat/Messages/Content/Parts/LogContent.tsx

@@ -65,6 +65,7 @@ const LogContent: React.FC<LogContentProps> = ({ output = '', renderImages, atta
       return `${filename} ${localize('com_download_expired')}`;
     }
 
+    const fileData = file as TFile & TAttachmentMetadata;
     const filepath = file.filepath || '';
 
     // const expirationText = expiresAt
@@ -72,7 +73,13 @@ const LogContent: React.FC<LogContentProps> = ({ output = '', renderImages, atta
     //   : ` ${localize('com_click_to_download')}`;
 
     return (
-      <LogLink href={filepath} filename={filename}>
+      <LogLink
+        href={filepath}
+        filename={filename}
+        file_id={fileData.file_id}
+        user={fileData.user}
+        source={fileData.source}
+      >
         {'- '}
         {filename} {localize('com_click_to_download')}
       </LogLink>

client/src/components/Chat/Messages/Content/Parts/LogLink.tsx

@@ -1,21 +1,56 @@
 import React from 'react';
+import { FileSources } from 'librechat-data-provider';
 import { useToastContext } from '@librechat/client';
-import { useCodeOutputDownload } from '~/data-provider';
+import { useCodeOutputDownload, useFileDownload } from '~/data-provider';
 
 interface LogLinkProps {
   href: string;
   filename: string;
+  file_id?: string;
+  user?: string;
+  source?: string;
   children: React.ReactNode;
 }
 
-export const useAttachmentLink = ({ href, filename }: Pick<LogLinkProps, 'href' | 'filename'>) => {
+interface AttachmentLinkOptions {
+  href: string;
+  filename: string;
+  file_id?: string;
+  user?: string;
+  source?: string;
+}
+
+/**
+ * Determines if a file is stored locally (not an external API URL).
+ * Files with these sources are stored on the LibreChat server and should
+ * use the /api/files/download endpoint instead of direct URL access.
+ */
+const isLocallyStoredSource = (source?: string): boolean => {
+  if (!source) {
+    return false;
+  }
+  return [FileSources.local, FileSources.firebase, FileSources.s3, FileSources.azure_blob].includes(
+    source as FileSources,
+  );
+};
+
+export const useAttachmentLink = ({
+  href,
+  filename,
+  file_id,
+  user,
+  source,
+}: AttachmentLinkOptions) => {
   const { showToast } = useToastContext();
-  const { refetch: downloadFile } = useCodeOutputDownload(href);
+
+  const useLocalDownload = isLocallyStoredSource(source) && !!file_id && !!user;
+  const { refetch: downloadFromApi } = useFileDownload(user, file_id);
+  const { refetch: downloadFromUrl } = useCodeOutputDownload(href);
 
   const handleDownload = async (event: React.MouseEvent<HTMLAnchorElement | HTMLButtonElement>) => {
     event.preventDefault();
     try {
-      const stream = await downloadFile();
+      const stream = useLocalDownload ? await downloadFromApi() : await downloadFromUrl();
       if (stream.data == null || stream.data === '') {
         console.error('Error downloading file: No data found');
         showToast({
@@ -39,8 +74,8 @@ export const useAttachmentLink = ({ href, filename }: Pick<LogLinkProps, 'href'
   return { handleDownload };
 };
 
-const LogLink: React.FC<LogLinkProps> = ({ href, filename, children }) => {
-  const { handleDownload } = useAttachmentLink({ href, filename });
+const LogLink: React.FC<LogLinkProps> = ({ href, filename, file_id, user, source, children }) => {
+  const { handleDownload } = useAttachmentLink({ href, filename, file_id, user, source });
   return (
     <a
       href={href}

client/src/hooks/SSE/useAttachmentHandler.ts

@@ -1,7 +1,12 @@
 import { useSetRecoilState } from 'recoil';
 import type { QueryClient } from '@tanstack/react-query';
 import { QueryKeys, Tools } from 'librechat-data-provider';
-import type { TAttachment, EventSubmission, MemoriesResponse } from 'librechat-data-provider';
+import type {
+  MemoriesResponse,
+  EventSubmission,
+  TAttachment,
+  TFile,
+} from 'librechat-data-provider';
 import { handleMemoryArtifact } from '~/utils/memory';
 import store from '~/store';
 
@@ -11,9 +16,24 @@ export default function useAttachmentHandler(queryClient?: QueryClient) {
   return ({ data }: { data: TAttachment; submission: EventSubmission }) => {
     const { messageId } = data;
 
-    if (queryClient && data?.filepath && !data.filepath.includes('/api/files')) {
-      queryClient.setQueryData([QueryKeys.files], (oldData: TAttachment[] | undefined) => {
-        return [data, ...(oldData || [])];
+    const fileData = data as TFile;
+    if (
+      queryClient &&
+      fileData?.file_id &&
+      fileData?.filepath &&
+      !fileData.filepath.includes('/api/files')
+    ) {
+      queryClient.setQueryData([QueryKeys.files], (oldData: TFile[] | undefined) => {
+        if (!oldData) {
+          return [fileData];
+        }
+        const existingIndex = oldData.findIndex((file) => file.file_id === fileData.file_id);
+        if (existingIndex > -1) {
+          const updated = [...oldData];
+          updated[existingIndex] = { ...oldData[existingIndex], ...fileData };
+          return updated;
+        }
+        return [fileData, ...oldData];
       });
     }