🎨 feat: OpenAI Image Tools (GPT-Image-1) (#7079)

* wip: OpenAI Image Generation Tool with customizable options

* WIP: First pass OpenAI Image Generation Tool and integrate into existing tools

* 🔀 fix: Comment out unused validation for image generation tool parameters

* 🔀 refactor: Update primeResources function parameters for better destructuring

* feat: Add image_edit resource to EToolResources and update AgentToolResources interface

* feat: Enhance file retrieval with tool resource filtering for image editing

* refactor: add OpenAI Image Tools for generation and editing, refactor related components, pass current request image attachments as tool resources for editing

* refactor: Remove commented-out code and clean up API key retrieval in createOpenAIImageTools function

* fix: show message attachments in shared links

* fix: Correct parent message retrieval logic for regenerated messages in useChatFunctions

* fix: Update primeResources to utilize requestFileSet for image file processing

* refactor: Improve description for image generation tool and clarify usage conditions, only provide edit tool if there are images available to edit

* chore: Update OpenAI Image Tools icon to use local asset

* refactor: Update image generation tool description and logic to prioritize editing tool when files are uploaded

* refactor: Enhance image tool descriptions to clarify usage conditions and note potential unavailability of uploaded images

* refactor: Update useAttachmentHandler to accept queryClient to update query cache with newly created file

* refactor: Add customizable descriptions and prompts for OpenAI image generation and editing tools

* chore: Update comments to use JSDoc style for better clarity and consistency

* refactor: Rename config variable to clientConfig for clarity and update signal handling in image generation

* refactor: Update axios request configuration to include derived signal and baseURL for improved request handling

* refactor: Update baseURL environment variable for OpenAI image generation tool configuration

* refactor: Enhance axios request configuration with conditional headers and improved clientConfig setup

* chore: Update comments for clarity and remove unnecessary lines in OpenAI image tools

* refactor: Update description for image generation without files to clarify user instructions

* refactor: Simplify target parent message logic for regeneration and resubmission cases

* chore: Remove backticks from error messages in image generation and editing functions

* refactor: Rename toolResources to toolResourceSet for clarity in file retrieval functions

* chore: Remove redundant comments and clean up TODOs in OpenAI image tools

* refactor: Rename fileStrategy to appFileStrategy for clarity and improve error handling in image processing

* chore: Update react-resizable-panels to version 2.1.8 in package.json and package-lock.json

* chore: Ensure required validation for logs and Code of Conduct agreement in bug report template

* fix: Update ArtifactPreview to use startupConfig and currentCode from memoized props to prevent unnecessary re-renders

* fix: improve robustness of `save & submit` when used from a user-message with existing attachments

* fix: add null check for artifact index in CodeEditor to prevent errors, trigger re-render on artifact ID change

* fix: standardize default values for artifact properties in Artifact component, avoiding prematurely setting an "empty/default" artifact

* fix: reset current artifact ID before setting a new one in ArtifactButton to ensure correct state management

* chore: rename `setArtifactId` variable to `setCurrentArtifactId`  for consistency

* chore: update type annotations in File and S3 CRUD functions for consistency

* refactor: improve image handling in OpenAI tools by using image_id references and enhance tool context for image editing

* fix: update image_ids schema in image_edit_oai to enforce presence and provide clear guidelines for usage

* fix: enhance file fetching logic to ensure user-specific and dimension-validated results

* chore: add details on image generation and editing capabilities with various models

client/package.json

@@ -86,7 +86,7 @@
     "react-i18next": "^15.4.0",
     "react-lazy-load-image-component": "^1.6.0",
     "react-markdown": "^9.0.1",
-    "react-resizable-panels": "^2.1.7",
+    "react-resizable-panels": "^2.1.8",
     "react-router-dom": "^6.11.2",
     "react-speech-recognition": "^3.10.0",
     "react-textarea-autosize": "^8.4.0",

client/src/common/types.ts

@@ -306,11 +306,14 @@ export type TAskProps = {
 export type TOptions = {
   editedMessageId?: string | null;
   editedText?: string | null;
-  isResubmission?: boolean;
   isRegenerate?: boolean;
   isContinued?: boolean;
   isEdited?: boolean;
   overrideMessages?: t.TMessage[];
+  /** This value is only true when the user submits a message with "Save & Submit" for a user-created message */
+  isResubmission?: boolean;
+  /** Currently only utilized when `isResubmission === true`, uses that message's currently attached files */
+  overrideFiles?: t.TMessage['files'];
 };
 
 export type TAskFunction = (props: TAskProps, options?: TOptions) => void;

client/src/components/Artifacts/Artifact.tsx

@@ -34,6 +34,10 @@ export const artifactPlugin: Pluggable = () => {
   };
 };
 
+const defaultTitle = 'untitled';
+const defaultType = 'unknown';
+const defaultIdentifier = 'lc-no-identifier';
+
 export function Artifact({
   node,
   ...props
@@ -58,15 +62,18 @@ export function Artifact({
     const content = extractContent(props.children);
     logger.log('artifacts', 'updateArtifact: content.length', content.length);
 
-    const title = props.title ?? 'Untitled Artifact';
-    const type = props.type ?? 'unknown';
-    const identifier = props.identifier ?? 'no-identifier';
+    const title = props.title ?? defaultTitle;
+    const type = props.type ?? defaultType;
+    const identifier = props.identifier ?? defaultIdentifier;
     const artifactKey = `${identifier}_${type}_${title}_${messageId}`
       .replace(/\s+/g, '_')
       .toLowerCase();
 
     throttledUpdateRef.current(() => {
       const now = Date.now();
+      if (artifactKey === `${defaultIdentifier}_${defaultType}_${defaultTitle}_${messageId}`) {
+        return;
+      }
 
       const currentArtifact: Artifact = {
         id: artifactKey,

client/src/components/Artifacts/ArtifactButton.tsx

@@ -1,4 +1,4 @@
-import { useSetRecoilState } from 'recoil';
+import { useSetRecoilState, useResetRecoilState } from 'recoil';
 import type { Artifact } from '~/common';
 import FilePreview from '~/components/Chat/Input/Files/FilePreview';
 import { useLocalize } from '~/hooks';
@@ -8,7 +8,8 @@ import store from '~/store';
 const ArtifactButton = ({ artifact }: { artifact: Artifact | null }) => {
   const localize = useLocalize();
   const setVisible = useSetRecoilState(store.artifactsVisible);
-  const setArtifactId = useSetRecoilState(store.currentArtifactId);
+  const setCurrentArtifactId = useSetRecoilState(store.currentArtifactId);
+  const resetCurrentArtifactId = useResetRecoilState(store.currentArtifactId);
   if (artifact === null || artifact === undefined) {
     return null;
   }
@@ -19,12 +20,15 @@ const ArtifactButton = ({ artifact }: { artifact: Artifact | null }) => {
       <button
         type="button"
         onClick={() => {
-          setArtifactId(artifact.id);
+          resetCurrentArtifactId();
           setVisible(true);
+          setTimeout(() => {
+            setCurrentArtifactId(artifact.id);
+          }, 15);
         }}
         className="relative overflow-hidden rounded-xl border border-border-medium transition-all duration-300 hover:border-border-xheavy hover:shadow-lg"
       >
-        <div className="w-fit bg-surface-tertiary p-2 ">
+        <div className="w-fit bg-surface-tertiary p-2">
           <div className="flex flex-row items-center gap-2">
             <FilePreview fileType={fileType} className="relative" />
             <div className="overflow-hidden text-left">

client/src/components/Artifacts/ArtifactCodeEditor.tsx

@@ -66,6 +66,9 @@ const CodeEditor = ({
     if (isMutating) {
       return;
     }
+    if (artifact.index == null) {
+      return;
+    }
 
     const currentCode = (sandpack.files['/' + fileKey] as SandpackBundlerFile | undefined)?.code;
 

client/src/components/Artifacts/ArtifactPreview.tsx

@@ -5,27 +5,27 @@ import {
   SandpackProviderProps,
 } from '@codesandbox/sandpack-react/unstyled';
 import type { SandpackPreviewRef } from '@codesandbox/sandpack-react/unstyled';
+import type { TStartupConfig } from 'librechat-data-provider';
 import type { ArtifactFiles } from '~/common';
 import { sharedFiles, sharedOptions } from '~/utils/artifacts';
-import { useGetStartupConfig } from '~/data-provider';
-import { useEditorContext } from '~/Providers';
 
 export const ArtifactPreview = memo(function ({
   files,
   fileKey,
-  previewRef,
-  sharedProps,
   template,
+  sharedProps,
+  previewRef,
+  currentCode,
+  startupConfig,
 }: {
   files: ArtifactFiles;
   fileKey: string;
   template: SandpackProviderProps['template'];
   sharedProps: Partial<SandpackProviderProps>;
   previewRef: React.MutableRefObject<SandpackPreviewRef>;
+  currentCode?: string;
+  startupConfig?: TStartupConfig;
 }) {
-  const { currentCode } = useEditorContext();
-  const { data: config } = useGetStartupConfig();
-
   const artifactFiles = useMemo(() => {
     if (Object.keys(files).length === 0) {
       return files;
@@ -43,18 +43,16 @@ export const ArtifactPreview = memo(function ({
   }, [currentCode, files, fileKey]);
 
   const options: typeof sharedOptions = useMemo(() => {
-    if (!config) {
+    if (!startupConfig) {
       return sharedOptions;
     }
     const _options: typeof sharedOptions = {
       ...sharedOptions,
-      bundlerURL: template === 'static' ? config.staticBundlerURL : config.bundlerURL,
+      bundlerURL: template === 'static' ? startupConfig.staticBundlerURL : startupConfig.bundlerURL,
     };
 
     return _options;
-  }, [config, template]);
-
-  console.log(options);
+  }, [startupConfig, template]);
 
   if (Object.keys(artifactFiles).length === 0) {
     return null;

client/src/components/Artifacts/ArtifactTabs.tsx

@@ -1,11 +1,13 @@
-import { useRef } from 'react';
+import { useRef, useEffect } from 'react';
 import * as Tabs from '@radix-ui/react-tabs';
 import type { SandpackPreviewRef, CodeEditorRef } from '@codesandbox/sandpack-react';
 import type { Artifact } from '~/common';
 import useArtifactProps from '~/hooks/Artifacts/useArtifactProps';
 import { useAutoScroll } from '~/hooks/Artifacts/useAutoScroll';
 import { ArtifactCodeEditor } from './ArtifactCodeEditor';
+import { useGetStartupConfig } from '~/data-provider';
 import { ArtifactPreview } from './ArtifactPreview';
+import { useEditorContext } from '~/Providers';
 import { cn } from '~/utils';
 
 export default function ArtifactTabs({
@@ -21,6 +23,16 @@ export default function ArtifactTabs({
   editorRef: React.MutableRefObject<CodeEditorRef>;
   previewRef: React.MutableRefObject<SandpackPreviewRef>;
 }) {
+  const { currentCode, setCurrentCode } = useEditorContext();
+  const { data: startupConfig } = useGetStartupConfig();
+  const lastIdRef = useRef<string | null>(null);
+  useEffect(() => {
+    if (artifact.id !== lastIdRef.current) {
+      setCurrentCode(undefined);
+    }
+    lastIdRef.current = artifact.id;
+  }, [setCurrentCode, artifact.id]);
+
   const content = artifact.content ?? '';
   const contentRef = useRef<HTMLDivElement>(null);
   useAutoScroll({ ref: contentRef, content, isSubmitting });
@@ -53,6 +65,8 @@ export default function ArtifactTabs({
           template={template}
           previewRef={previewRef}
           sharedProps={sharedProps}
+          currentCode={currentCode}
+          startupConfig={startupConfig}
         />
       </Tabs.Content>
     </>

client/src/components/Chat/Messages/Content/EditMessage.tsx

@@ -61,6 +61,7 @@ const EditMessage = ({
         },
         {
           isResubmission: true,
+          overrideFiles: message.files,
         },
       );
 

client/src/components/Chat/Messages/Content/SearchContent.tsx

@@ -1,16 +1,23 @@
-import { Suspense } from 'react';
+import { Suspense, useMemo } from 'react';
 import { useRecoilValue } from 'recoil';
-import type { TMessage, TMessageContentParts } from 'librechat-data-provider';
+import { ContentTypes } from 'librechat-data-provider';
+import type { Agents, TMessage, TMessageContentParts } from 'librechat-data-provider';
 import { UnfinishedMessage } from './MessageContent';
 import { DelayedRender } from '~/components/ui';
 import MarkdownLite from './MarkdownLite';
-import { cn } from '~/utils';
+import { cn, mapAttachments } from '~/utils';
 import store from '~/store';
 import Part from './Part';
 
 const SearchContent = ({ message }: { message: TMessage }) => {
   const enableUserMsgMarkdown = useRecoilValue(store.enableUserMsgMarkdown);
   const { messageId } = message;
+  const messageAttachmentsMap = useRecoilValue(store.messageAttachmentsMap);
+  const attachmentMap = useMemo(
+    () => mapAttachments(message?.attachments ?? messageAttachmentsMap[messageId] ?? []),
+    [message?.attachments, messageAttachmentsMap, messageId],
+  );
+
   if (Array.isArray(message.content) && message.content.length > 0) {
     return (
       <>
@@ -20,13 +27,17 @@ const SearchContent = ({ message }: { message: TMessage }) => {
             if (!part) {
               return null;
             }
+
+            const toolCallId =
+              (part?.[ContentTypes.TOOL_CALL] as Agents.ToolCall | undefined)?.id ?? '';
+            const attachments = attachmentMap[toolCallId];
             return (
               <Part
                 key={`display-${messageId}-${idx}`}
                 showCursor={false}
                 isSubmitting={false}
                 isCreatedByUser={message.isCreatedByUser}
-                messageId={message.messageId}
+                attachments={attachments}
                 part={part}
               />
             );

client/src/hooks/Chat/useChatFunctions.ts

@@ -95,6 +95,7 @@ export default function useChatFunctions({
       isContinued = false,
       isEdited = false,
       overrideMessages,
+      overrideFiles,
     } = {},
   ) => {
     setShowStopButton(false);
@@ -147,11 +148,17 @@ export default function useChatFunctions({
       conversationId = null;
     }
 
-    const parentMessage = currentMessages.find(
-      (msg) => msg.messageId === latestMessage?.parentMessageId,
+    const targetParentMessageId = isRegenerate ? messageId : latestMessage?.parentMessageId;
+    /**
+     * If the user regenerated or resubmitted the message, the current parent is technically
+     * the latest user message, which is passed into `ask`; otherwise, we can rely on the
+     * latestMessage to find the parent.
+     */
+    const targetParentMessage = currentMessages.find(
+      (msg) => msg.messageId === targetParentMessageId,
     );
 
-    let thread_id = parentMessage?.thread_id ?? latestMessage?.thread_id;
+    let thread_id = targetParentMessage?.thread_id ?? latestMessage?.thread_id;
     if (thread_id == null) {
       thread_id = currentMessages.find((message) => message.thread_id)?.thread_id;
     }
@@ -159,7 +166,7 @@ export default function useChatFunctions({
     const endpointsConfig = queryClient.getQueryData<TEndpointsConfig>([QueryKeys.endpoints]);
     const endpointType = getEndpointField(endpointsConfig, endpoint, 'type');
 
-    // set the endpoint option
+    /** This becomes part of the `endpointOption` */
     const convo = parseCompactConvo({
       endpoint: endpoint as EndpointSchemaKey,
       endpointType: endpointType as EndpointSchemaKey,
@@ -201,10 +208,14 @@ export default function useChatFunctions({
       error: false,
     };
 
+    const submissionFiles = overrideFiles ?? targetParentMessage?.files;
     const reuseFiles =
-      (isRegenerate || isResubmission) && parentMessage?.files && parentMessage.files.length > 0;
+      (isRegenerate || (overrideFiles != null && overrideFiles.length)) &&
+      submissionFiles &&
+      submissionFiles.length > 0;
+
     if (setFiles && reuseFiles === true) {
-      currentMsg.files = parentMessage.files;
+      currentMsg.files = [...submissionFiles];
       setFiles(new Map());
       setFilesToDelete({});
     } else if (setFiles && files && files.size > 0) {
@@ -219,7 +230,6 @@ export default function useChatFunctions({
       setFilesToDelete({});
     }
 
-    // construct the placeholder response message
     const generation = editedText ?? latestMessage?.text ?? '';
     const responseText = isEditOrContinue ? generation : '';
 

client/src/hooks/SSE/useAttachmentHandler.ts

@@ -1,13 +1,21 @@
 import { useSetRecoilState } from 'recoil';
+import { QueryKeys } from 'librechat-data-provider';
+import type { QueryClient } from '@tanstack/react-query';
 import type { TAttachment, EventSubmission } from 'librechat-data-provider';
 import store from '~/store';
 
-export default function useAttachmentHandler() {
+export default function useAttachmentHandler(queryClient?: QueryClient) {
   const setAttachmentsMap = useSetRecoilState(store.messageAttachmentsMap);
 
   return ({ data }: { data: TAttachment; submission: EventSubmission }) => {
     const { messageId } = data;
 
+    if (queryClient) {
+      queryClient.setQueryData([QueryKeys.files], (oldData: TAttachment[] | undefined) => {
+        return [data, ...(oldData || [])];
+      });
+    }
+
     setAttachmentsMap((prevMap) => {
       const messageAttachments =
         (prevMap as Record<string, TAttachment[] | undefined>)[messageId] || [];

client/src/hooks/SSE/useEventHandlers.ts

@@ -185,7 +185,7 @@ export default function useEventHandlers({
     setIsSubmitting,
     lastAnnouncementTimeRef,
   });
-  const attachmentHandler = useAttachmentHandler();
+  const attachmentHandler = useAttachmentHandler(queryClient);
 
   const messageHandler = useCallback(
     (data: string | undefined, submission: EventSubmission) => {