🚀 Feat: Streamline File Strategies & GPT-4-Vision Settings (#1535)

* chore: fix `endpoint` typescript issues and typo in console info message

* feat(api): files GET endpoint and save only file_id references to messages

* refactor(client): `useGetFiles` query hook, update file types, optimistic update of filesQuery on file upload

* refactor(buildTree): update to use params object and accept fileMap

* feat: map files to messages; refactor(ChatView): messages only available after files are fetched

* fix: fetch files only when authenticated

* feat(api): AppService
- rename app.locals.configs to app.locals.paths
- load custom config use fileStrategy from yaml config in app.locals

* refactor: separate Firebase and Local strategies, call based on config

* refactor: modularize file strategies and employ with use of DALL-E

* refactor(librechat.yaml): add fileStrategy field

* feat: add source to MongoFile schema, as well as BatchFile, and ExtendedFile types

* feat: employ file strategies for upload/delete files

* refactor(deleteFirebaseFile): add user id validation for firebase file deletion

* chore(deleteFirebaseFile): update jsdocs

* feat: employ strategies for vision requests

* fix(client): handle messages with deleted files

* fix(client): ensure `filesToDelete` always saves/sends `file.source`

* feat(openAI): configurable `resendImages` and `imageDetail`

* refactor(getTokenCountForMessage): recursive process only when array of Objects and only their values (not keys) aside from `image_url` types

* feat(OpenAIClient): calculateImageTokenCost

* chore: remove comment

* refactor(uploadAvatar): employ fileStrategy for avatars, from social logins or user upload

* docs: update docs on how to configure fileStrategy

* fix(ci): mock winston and winston related modules, update DALLE3.spec.js with changes made

* refactor(redis): change terminal message to reflect current development state

* fix(DALL-E-2): pass fileStrategy to dall-e

client/src/components/Chat/ChatView.tsx

@@ -6,11 +6,12 @@ import { useChatHelpers, useSSE } from '~/hooks';
 // import GenerationButtons from './Input/GenerationButtons';
 import MessagesView from './Messages/MessagesView';
 // import OptionsBar from './Input/OptionsBar';
+import { useGetFiles } from '~/data-provider';
+import { buildTree, mapFiles } from '~/utils';
 import { Spinner } from '~/components/svg';
 import { ChatContext } from '~/Providers';
 import Presentation from './Presentation';
 import ChatForm from './Input/ChatForm';
-import { buildTree } from '~/utils';
 import Landing from './Landing';
 import Header from './Header';
 import Footer from './Footer';
@@ -21,11 +22,16 @@ function ChatView({ index = 0 }: { index?: number }) {
   const submissionAtIndex = useRecoilValue(store.submissionByIndex(0));
   useSSE(submissionAtIndex);
 
+  const { data: fileMap } = useGetFiles({
+    select: mapFiles,
+  });
+
   const { data: messagesTree = null, isLoading } = useGetMessagesByConvoId(conversationId ?? '', {
     select: (data) => {
-      const dataTree = buildTree(data, false);
+      const dataTree = buildTree({ messages: data, fileMap });
       return dataTree?.length === 0 ? null : dataTree ?? null;
     },
+    enabled: !!fileMap,
   });
 
   const chatHelpers = useChatHelpers(index, conversationId);

client/src/components/Chat/Input/Files/Images.tsx

@@ -1,5 +1,6 @@
 import debounce from 'lodash/debounce';
 import { useState, useEffect, useCallback } from 'react';
+import { FileSources } from 'librechat-data-provider';
 import type { BatchFile } from 'librechat-data-provider';
 import { useDeleteFilesMutation } from '~/data-provider';
 import { useSetFilesToDelete } from '~/hooks';
@@ -70,13 +71,20 @@ export default function Images({
   }
 
   const deleteFile = (_file: ExtendedFile) => {
-    const { file_id, progress, temp_file_id = '', filepath = '' } = _file;
+    const {
+      file_id,
+      progress,
+      temp_file_id = '',
+      filepath = '',
+      source = FileSources.local,
+    } = _file;
     if (progress < 1) {
       return;
     }
-    const file = {
+    const file: BatchFile = {
       file_id,
       filepath,
+      source,
     };
 
     setFiles((currentFiles) => {

client/src/components/Chat/Messages/Content/MessageContent.tsx

@@ -30,7 +30,7 @@ const ErrorMessage = ({ text }: TText) => {
 // Display Message Component
 const DisplayMessage = ({ text, isCreatedByUser, message, showCursor }: TDisplayProps) => {
   const imageFiles = message?.files
-    ? message.files.filter((file) => file.type.startsWith('image/'))
+    ? message.files.filter((file) => file.type && file.type.startsWith('image/'))
     : null;
   return (
     <Container>

client/src/components/Chat/Presentation.tsx

@@ -1,4 +1,5 @@
 import { useEffect } from 'react';
+import { FileSources } from 'librechat-data-provider';
 import type { ExtendedFile } from '~/common';
 import { useDragHelpers, useSetFilesToDelete } from '~/hooks';
 import DragDropOverlay from './Input/Files/DragDropOverlay';
@@ -25,6 +26,7 @@ export default function Presentation({ children }: { children: React.ReactNode }
       .map((file) => ({
         file_id: file.file_id,
         filepath: file.filepath as string,
+        source: file.source as FileSources,
       }));
 
     if (files.length === 0) {

client/src/components/Endpoints/PopoverButtons.tsx

@@ -18,7 +18,7 @@ export default function PopoverButtons({
   buttonClass,
   iconClass = '',
 }: {
-  endpoint: EModelEndpoint;
+  endpoint: EModelEndpoint | string;
   buttonClass?: string;
   iconClass?: string;
 }) {
@@ -32,7 +32,9 @@ export default function PopoverButtons({
   const buttons: { [key: string]: TPopoverButton[] } = {
     google: [
       {
-        label: (showExamples ? localize('com_endpoint_hide') : localize('com_endpoint_show')) + localize('com_endpoint_examples'),
+        label:
+          (showExamples ? localize('com_endpoint_hide') : localize('com_endpoint_show')) +
+          localize('com_endpoint_examples'),
         buttonClass: isCodeChat ? 'disabled' : '',
         handler: triggerExamples,
         icon: <MessagesSquared className={cn('mr-1 w-[14px]', iconClass)} />,
@@ -40,7 +42,10 @@ export default function PopoverButtons({
     ],
     gptPlugins: [
       {
-        label: localize('com_endpoint_show_what_settings', showAgentSettings ? localize('com_endpoint_completion') : localize('com_endpoint_agent')),
+        label: localize(
+          'com_endpoint_show_what_settings',
+          showAgentSettings ? localize('com_endpoint_completion') : localize('com_endpoint_agent'),
+        ),
         buttonClass: '',
         handler: () => setShowAgentSettings((prev) => !prev),
         icon: <GPTIcon className={cn('mr-1 w-[14px]', iconClass)} size={24} />,

client/src/components/Endpoints/Settings/OpenAI.tsx

@@ -1,11 +1,13 @@
 import TextareaAutosize from 'react-textarea-autosize';
+import { ImageDetail, imageDetailNumeric, imageDetailValue } from 'librechat-data-provider';
 import {
-  SelectDropDown,
   Input,
   Label,
+  Switch,
   Slider,
-  InputNumber,
   HoverCard,
+  InputNumber,
+  SelectDropDown,
   HoverCardTrigger,
 } from '~/components/ui';
 import { cn, defaultTextProps, optionText, removeFocusOutlines } from '~/utils/';
@@ -20,6 +22,8 @@ export default function Settings({ conversation, setOption, models, readonly }:
     return null;
   }
   const {
+    endpoint,
+    endpointType,
     model,
     chatGptLabel,
     promptPrefix,
@@ -27,6 +31,8 @@ export default function Settings({ conversation, setOption, models, readonly }:
     top_p: topP,
     frequency_penalty: freqP,
     presence_penalty: presP,
+    resendImages,
+    imageDetail,
   } = conversation;
   const setModel = setOption('model');
   const setChatGptLabel = setOption('chatGptLabel');
@@ -35,6 +41,10 @@ export default function Settings({ conversation, setOption, models, readonly }:
   const setTopP = setOption('top_p');
   const setFreqP = setOption('frequency_penalty');
   const setPresP = setOption('presence_penalty');
+  const setResendImages = setOption('resendImages');
+  const setImageDetail = setOption('imageDetail');
+
+  const optionEndpoint = endpointType ?? endpoint;
 
   return (
     <div className="grid grid-cols-5 gap-6">
@@ -126,7 +136,7 @@ export default function Settings({ conversation, setOption, models, readonly }:
               className="flex h-4 w-full"
             />
           </HoverCardTrigger>
-          <OptionHover endpoint={conversation?.endpoint ?? ''} type="temp" side={ESide.Left} />
+          <OptionHover endpoint={optionEndpoint ?? ''} type="temp" side={ESide.Left} />
         </HoverCard>
         <HoverCard openDelay={300}>
           <HoverCardTrigger className="grid w-full items-center gap-2">
@@ -164,7 +174,7 @@ export default function Settings({ conversation, setOption, models, readonly }:
               className="flex h-4 w-full"
             />
           </HoverCardTrigger>
-          <OptionHover endpoint={conversation?.endpoint ?? ''} type="topp" side={ESide.Left} />
+          <OptionHover endpoint={optionEndpoint ?? ''} type="topp" side={ESide.Left} />
         </HoverCard>
 
         <HoverCard openDelay={300}>
@@ -203,7 +213,7 @@ export default function Settings({ conversation, setOption, models, readonly }:
               className="flex h-4 w-full"
             />
           </HoverCardTrigger>
-          <OptionHover endpoint={conversation?.endpoint ?? ''} type="freq" side={ESide.Left} />
+          <OptionHover endpoint={optionEndpoint ?? ''} type="freq" side={ESide.Left} />
         </HoverCard>
 
         <HoverCard openDelay={300}>
@@ -242,8 +252,66 @@ export default function Settings({ conversation, setOption, models, readonly }:
               className="flex h-4 w-full"
             />
           </HoverCardTrigger>
-          <OptionHover endpoint={conversation?.endpoint ?? ''} type="pres" side={ESide.Left} />
+          <OptionHover endpoint={optionEndpoint ?? ''} type="pres" side={ESide.Left} />
         </HoverCard>
+        <div className="w-full">
+          <div className="mb-2 flex w-full justify-between gap-2">
+            <label
+              htmlFor="resend-images"
+              className="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70 dark:text-gray-50"
+            >
+              <small>{localize('com_endpoint_plug_resend_images')}</small>
+            </label>
+            <label
+              htmlFor="image-detail-value"
+              className="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70 dark:text-gray-50"
+            >
+              <small>Image Detail</small>
+            </label>
+            <Input
+              id="image-detail-value"
+              disabled={true}
+              value={imageDetail ?? ImageDetail.auto}
+              className={cn(
+                defaultTextProps,
+                optionText,
+                'flex rounded-md bg-transparent py-2 text-xs focus:outline-none focus:ring-2 focus:ring-slate-400 focus:ring-offset-2 dark:border-slate-700',
+                'pointer-events-none max-h-5 w-12 border-0 group-hover/temp:border-gray-200',
+              )}
+            />
+          </div>
+          <div className="flex w-full justify-between gap-2">
+            <HoverCard openDelay={500}>
+              <HoverCardTrigger>
+                <Switch
+                  id="resend-images"
+                  checked={resendImages ?? false}
+                  onCheckedChange={(checked: boolean) => setResendImages(checked)}
+                  disabled={readonly}
+                  className="flex"
+                />
+                <OptionHover endpoint={optionEndpoint ?? ''} type="resend" side={ESide.Bottom} />
+              </HoverCardTrigger>
+            </HoverCard>
+            <HoverCard openDelay={500}>
+              <HoverCardTrigger className="flex w-[52%] md:w-[125px]">
+                <Slider
+                  id="image-detail-slider"
+                  disabled={readonly}
+                  value={[
+                    imageDetailNumeric[imageDetail ?? ''] ?? imageDetailNumeric[ImageDetail.auto],
+                  ]}
+                  onValueChange={(value) => setImageDetail(imageDetailValue[value[0]])}
+                  doubleClickHandler={() => setImageDetail(ImageDetail.auto)}
+                  max={2}
+                  min={0}
+                  step={1}
+                />
+                <OptionHover endpoint={optionEndpoint ?? ''} type="detail" side={ESide.Bottom} />
+              </HoverCardTrigger>
+            </HoverCard>
+          </div>
+        </div>
       </div>
     </div>
   );

client/src/components/Endpoints/Settings/OptionHover.tsx

@@ -15,6 +15,8 @@ const openAI = {
   topp: 'com_endpoint_openai_topp',
   freq: 'com_endpoint_openai_freq',
   pres: 'com_endpoint_openai_pres',
+  resend: 'com_endpoint_openai_resend',
+  detail: 'com_endpoint_openai_detail',
 };
 
 const types = {
@@ -47,7 +49,7 @@ function OptionHover({ endpoint, type, side }: TOptionHoverProps) {
   }
   return (
     <HoverCardPortal>
-      <HoverCardContent side={side} className="w-80 ">
+      <HoverCardContent side={side} className="z-[80]  w-80">
         <div className="space-y-2">
           <p className="text-sm text-gray-600 dark:text-gray-300">{localize(text)}</p>
         </div>

client/src/components/Nav/ExportConversation/ExportModal.jsx

@@ -24,7 +24,7 @@ export default function ExportModal({ open, onOpenChange, conversation }) {
 
   const { data: messagesTree = null } = useGetMessagesByConvoId(conversation.conversationId ?? '', {
     select: (data) => {
-      const dataTree = buildTree(data, false);
+      const dataTree = buildTree({ messages: data });
       return dataTree?.length === 0 ? null : dataTree ?? null;
     },
   });