🚀 Feat: Streamline File Strategies & GPT-4-Vision Settings (#1535)

* chore: fix `endpoint` typescript issues and typo in console info message

* feat(api): files GET endpoint and save only file_id references to messages

* refactor(client): `useGetFiles` query hook, update file types, optimistic update of filesQuery on file upload

* refactor(buildTree): update to use params object and accept fileMap

* feat: map files to messages; refactor(ChatView): messages only available after files are fetched

* fix: fetch files only when authenticated

* feat(api): AppService
- rename app.locals.configs to app.locals.paths
- load custom config use fileStrategy from yaml config in app.locals

* refactor: separate Firebase and Local strategies, call based on config

* refactor: modularize file strategies and employ with use of DALL-E

* refactor(librechat.yaml): add fileStrategy field

* feat: add source to MongoFile schema, as well as BatchFile, and ExtendedFile types

* feat: employ file strategies for upload/delete files

* refactor(deleteFirebaseFile): add user id validation for firebase file deletion

* chore(deleteFirebaseFile): update jsdocs

* feat: employ strategies for vision requests

* fix(client): handle messages with deleted files

* fix(client): ensure `filesToDelete` always saves/sends `file.source`

* feat(openAI): configurable `resendImages` and `imageDetail`

* refactor(getTokenCountForMessage): recursive process only when array of Objects and only their values (not keys) aside from `image_url` types

* feat(OpenAIClient): calculateImageTokenCost

* chore: remove comment

* refactor(uploadAvatar): employ fileStrategy for avatars, from social logins or user upload

* docs: update docs on how to configure fileStrategy

* fix(ci): mock winston and winston related modules, update DALLE3.spec.js with changes made

* refactor(redis): change terminal message to reflect current development state

* fix(DALL-E-2): pass fileStrategy to dall-e
This commit is contained in:
Danny Avila 2024-01-11 11:37:54 -05:00 committed by GitHub
parent 28a6807176
commit d20970f5c5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
81 changed files with 1729 additions and 855 deletions

View file

@ -1,3 +1,4 @@
import { FileSources } from 'librechat-data-provider';
import type {
TConversation,
TMessage,
@ -230,6 +231,7 @@ export interface ExtendedFile {
size: number;
preview: string;
progress: number;
source?: FileSources;
}
export type ContextType = { navVisible: boolean; setNavVisible: (visible: boolean) => void };

View file

@ -6,11 +6,12 @@ import { useChatHelpers, useSSE } from '~/hooks';
// import GenerationButtons from './Input/GenerationButtons';
import MessagesView from './Messages/MessagesView';
// import OptionsBar from './Input/OptionsBar';
import { useGetFiles } from '~/data-provider';
import { buildTree, mapFiles } from '~/utils';
import { Spinner } from '~/components/svg';
import { ChatContext } from '~/Providers';
import Presentation from './Presentation';
import ChatForm from './Input/ChatForm';
import { buildTree } from '~/utils';
import Landing from './Landing';
import Header from './Header';
import Footer from './Footer';
@ -21,11 +22,16 @@ function ChatView({ index = 0 }: { index?: number }) {
const submissionAtIndex = useRecoilValue(store.submissionByIndex(0));
useSSE(submissionAtIndex);
const { data: fileMap } = useGetFiles({
select: mapFiles,
});
const { data: messagesTree = null, isLoading } = useGetMessagesByConvoId(conversationId ?? '', {
select: (data) => {
const dataTree = buildTree(data, false);
const dataTree = buildTree({ messages: data, fileMap });
return dataTree?.length === 0 ? null : dataTree ?? null;
},
enabled: !!fileMap,
});
const chatHelpers = useChatHelpers(index, conversationId);

View file

@ -1,5 +1,6 @@
import debounce from 'lodash/debounce';
import { useState, useEffect, useCallback } from 'react';
import { FileSources } from 'librechat-data-provider';
import type { BatchFile } from 'librechat-data-provider';
import { useDeleteFilesMutation } from '~/data-provider';
import { useSetFilesToDelete } from '~/hooks';
@ -70,13 +71,20 @@ export default function Images({
}
const deleteFile = (_file: ExtendedFile) => {
const { file_id, progress, temp_file_id = '', filepath = '' } = _file;
const {
file_id,
progress,
temp_file_id = '',
filepath = '',
source = FileSources.local,
} = _file;
if (progress < 1) {
return;
}
const file = {
const file: BatchFile = {
file_id,
filepath,
source,
};
setFiles((currentFiles) => {

View file

@ -30,7 +30,7 @@ const ErrorMessage = ({ text }: TText) => {
// Display Message Component
const DisplayMessage = ({ text, isCreatedByUser, message, showCursor }: TDisplayProps) => {
const imageFiles = message?.files
? message.files.filter((file) => file.type.startsWith('image/'))
? message.files.filter((file) => file.type && file.type.startsWith('image/'))
: null;
return (
<Container>

View file

@ -1,4 +1,5 @@
import { useEffect } from 'react';
import { FileSources } from 'librechat-data-provider';
import type { ExtendedFile } from '~/common';
import { useDragHelpers, useSetFilesToDelete } from '~/hooks';
import DragDropOverlay from './Input/Files/DragDropOverlay';
@ -25,6 +26,7 @@ export default function Presentation({ children }: { children: React.ReactNode }
.map((file) => ({
file_id: file.file_id,
filepath: file.filepath as string,
source: file.source as FileSources,
}));
if (files.length === 0) {

View file

@ -18,7 +18,7 @@ export default function PopoverButtons({
buttonClass,
iconClass = '',
}: {
endpoint: EModelEndpoint;
endpoint: EModelEndpoint | string;
buttonClass?: string;
iconClass?: string;
}) {
@ -32,7 +32,9 @@ export default function PopoverButtons({
const buttons: { [key: string]: TPopoverButton[] } = {
google: [
{
label: (showExamples ? localize('com_endpoint_hide') : localize('com_endpoint_show')) + localize('com_endpoint_examples'),
label:
(showExamples ? localize('com_endpoint_hide') : localize('com_endpoint_show')) +
localize('com_endpoint_examples'),
buttonClass: isCodeChat ? 'disabled' : '',
handler: triggerExamples,
icon: <MessagesSquared className={cn('mr-1 w-[14px]', iconClass)} />,
@ -40,7 +42,10 @@ export default function PopoverButtons({
],
gptPlugins: [
{
label: localize('com_endpoint_show_what_settings', showAgentSettings ? localize('com_endpoint_completion') : localize('com_endpoint_agent')),
label: localize(
'com_endpoint_show_what_settings',
showAgentSettings ? localize('com_endpoint_completion') : localize('com_endpoint_agent'),
),
buttonClass: '',
handler: () => setShowAgentSettings((prev) => !prev),
icon: <GPTIcon className={cn('mr-1 w-[14px]', iconClass)} size={24} />,

View file

@ -1,11 +1,13 @@
import TextareaAutosize from 'react-textarea-autosize';
import { ImageDetail, imageDetailNumeric, imageDetailValue } from 'librechat-data-provider';
import {
SelectDropDown,
Input,
Label,
Switch,
Slider,
InputNumber,
HoverCard,
InputNumber,
SelectDropDown,
HoverCardTrigger,
} from '~/components/ui';
import { cn, defaultTextProps, optionText, removeFocusOutlines } from '~/utils/';
@ -20,6 +22,8 @@ export default function Settings({ conversation, setOption, models, readonly }:
return null;
}
const {
endpoint,
endpointType,
model,
chatGptLabel,
promptPrefix,
@ -27,6 +31,8 @@ export default function Settings({ conversation, setOption, models, readonly }:
top_p: topP,
frequency_penalty: freqP,
presence_penalty: presP,
resendImages,
imageDetail,
} = conversation;
const setModel = setOption('model');
const setChatGptLabel = setOption('chatGptLabel');
@ -35,6 +41,10 @@ export default function Settings({ conversation, setOption, models, readonly }:
const setTopP = setOption('top_p');
const setFreqP = setOption('frequency_penalty');
const setPresP = setOption('presence_penalty');
const setResendImages = setOption('resendImages');
const setImageDetail = setOption('imageDetail');
const optionEndpoint = endpointType ?? endpoint;
return (
<div className="grid grid-cols-5 gap-6">
@ -126,7 +136,7 @@ export default function Settings({ conversation, setOption, models, readonly }:
className="flex h-4 w-full"
/>
</HoverCardTrigger>
<OptionHover endpoint={conversation?.endpoint ?? ''} type="temp" side={ESide.Left} />
<OptionHover endpoint={optionEndpoint ?? ''} type="temp" side={ESide.Left} />
</HoverCard>
<HoverCard openDelay={300}>
<HoverCardTrigger className="grid w-full items-center gap-2">
@ -164,7 +174,7 @@ export default function Settings({ conversation, setOption, models, readonly }:
className="flex h-4 w-full"
/>
</HoverCardTrigger>
<OptionHover endpoint={conversation?.endpoint ?? ''} type="topp" side={ESide.Left} />
<OptionHover endpoint={optionEndpoint ?? ''} type="topp" side={ESide.Left} />
</HoverCard>
<HoverCard openDelay={300}>
@ -203,7 +213,7 @@ export default function Settings({ conversation, setOption, models, readonly }:
className="flex h-4 w-full"
/>
</HoverCardTrigger>
<OptionHover endpoint={conversation?.endpoint ?? ''} type="freq" side={ESide.Left} />
<OptionHover endpoint={optionEndpoint ?? ''} type="freq" side={ESide.Left} />
</HoverCard>
<HoverCard openDelay={300}>
@ -242,8 +252,66 @@ export default function Settings({ conversation, setOption, models, readonly }:
className="flex h-4 w-full"
/>
</HoverCardTrigger>
<OptionHover endpoint={conversation?.endpoint ?? ''} type="pres" side={ESide.Left} />
<OptionHover endpoint={optionEndpoint ?? ''} type="pres" side={ESide.Left} />
</HoverCard>
<div className="w-full">
<div className="mb-2 flex w-full justify-between gap-2">
<label
htmlFor="resend-images"
className="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70 dark:text-gray-50"
>
<small>{localize('com_endpoint_plug_resend_images')}</small>
</label>
<label
htmlFor="image-detail-value"
className="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70 dark:text-gray-50"
>
<small>Image Detail</small>
</label>
<Input
id="image-detail-value"
disabled={true}
value={imageDetail ?? ImageDetail.auto}
className={cn(
defaultTextProps,
optionText,
'flex rounded-md bg-transparent py-2 text-xs focus:outline-none focus:ring-2 focus:ring-slate-400 focus:ring-offset-2 dark:border-slate-700',
'pointer-events-none max-h-5 w-12 border-0 group-hover/temp:border-gray-200',
)}
/>
</div>
<div className="flex w-full justify-between gap-2">
<HoverCard openDelay={500}>
<HoverCardTrigger>
<Switch
id="resend-images"
checked={resendImages ?? false}
onCheckedChange={(checked: boolean) => setResendImages(checked)}
disabled={readonly}
className="flex"
/>
<OptionHover endpoint={optionEndpoint ?? ''} type="resend" side={ESide.Bottom} />
</HoverCardTrigger>
</HoverCard>
<HoverCard openDelay={500}>
<HoverCardTrigger className="flex w-[52%] md:w-[125px]">
<Slider
id="image-detail-slider"
disabled={readonly}
value={[
imageDetailNumeric[imageDetail ?? ''] ?? imageDetailNumeric[ImageDetail.auto],
]}
onValueChange={(value) => setImageDetail(imageDetailValue[value[0]])}
doubleClickHandler={() => setImageDetail(ImageDetail.auto)}
max={2}
min={0}
step={1}
/>
<OptionHover endpoint={optionEndpoint ?? ''} type="detail" side={ESide.Bottom} />
</HoverCardTrigger>
</HoverCard>
</div>
</div>
</div>
</div>
);

View file

@ -15,6 +15,8 @@ const openAI = {
topp: 'com_endpoint_openai_topp',
freq: 'com_endpoint_openai_freq',
pres: 'com_endpoint_openai_pres',
resend: 'com_endpoint_openai_resend',
detail: 'com_endpoint_openai_detail',
};
const types = {
@ -47,7 +49,7 @@ function OptionHover({ endpoint, type, side }: TOptionHoverProps) {
}
return (
<HoverCardPortal>
<HoverCardContent side={side} className="w-80 ">
<HoverCardContent side={side} className="z-[80] w-80">
<div className="space-y-2">
<p className="text-sm text-gray-600 dark:text-gray-300">{localize(text)}</p>
</div>

View file

@ -24,7 +24,7 @@ export default function ExportModal({ open, onOpenChange, conversation }) {
const { data: messagesTree = null } = useGetMessagesByConvoId(conversation.conversationId ?? '', {
select: (data) => {
const dataTree = buildTree(data, false);
const dataTree = buildTree({ messages: data });
return dataTree?.length === 0 ? null : dataTree ?? null;
},
});

View file

@ -1,7 +1,7 @@
import { useMutation, useQueryClient } from '@tanstack/react-query';
import type { UseMutationResult } from '@tanstack/react-query';
import type {
FileUploadResponse,
TFileUpload,
UploadMutationOptions,
FileUploadBody,
DeleteFilesResponse,
@ -23,7 +23,7 @@ import store from '~/store';
export const useUploadImageMutation = (
options?: UploadMutationOptions,
): UseMutationResult<
FileUploadResponse, // response data
TFileUpload, // response data
unknown, // error
FileUploadBody, // request
unknown // context

View file

@ -1,6 +1,18 @@
import { UseQueryOptions, useQuery, QueryObserverResult } from '@tanstack/react-query';
import { QueryKeys, dataService } from 'librechat-data-provider';
import type { TPreset } from 'librechat-data-provider';
import { UseQueryOptions, useQuery, QueryObserverResult } from '@tanstack/react-query';
import type { TPreset, TFile } from 'librechat-data-provider';
export const useGetFiles = <TData = TFile[] | boolean>(
config?: UseQueryOptions<TFile[], unknown, TData>,
): QueryObserverResult<TData, unknown> => {
return useQuery<TFile[], unknown, TData>([QueryKeys.files], () => dataService.getFiles(), {
refetchOnWindowFocus: false,
refetchOnReconnect: false,
refetchOnMount: false,
...config,
});
};
export const useGetPresetsQuery = (
config?: UseQueryOptions<TPreset[]>,
): QueryObserverResult<TPreset[], unknown> => {

View file

@ -1,6 +1,9 @@
import { v4 } from 'uuid';
import debounce from 'lodash/debounce';
import { QueryKeys } from 'librechat-data-provider';
import { useQueryClient } from '@tanstack/react-query';
import { useState, useEffect, useCallback } from 'react';
import type { TFile } from 'librechat-data-provider';
import type { ExtendedFile } from '~/common';
import { useToastContext } from '~/Providers/ToastContext';
import { useChatContext } from '~/Providers/ChatContext';
@ -16,6 +19,7 @@ const totalSizeLimit = maxSize * 1024 * 1024; // 25 MB
const supportedTypes = ['image/jpeg', 'image/jpg', 'image/png', 'image/webp'];
const useFileHandling = () => {
const queryClient = useQueryClient();
const { showToast } = useToastContext();
const [errors, setErrors] = useState<string[]>([]);
const setError = (error: string) => setErrors((prevErrors) => [...prevErrors, error]);
@ -116,6 +120,9 @@ const useFileHandling = () => {
filepath: data.filepath,
});
const _files = queryClient.getQueryData<TFile[]>([QueryKeys.files]) ?? [];
queryClient.setQueryData([QueryKeys.files], [..._files, data]);
setTimeout(() => {
updateFileById(data.temp_file_id, {
progress: 1,
@ -126,6 +133,7 @@ const useFileHandling = () => {
height: data.height,
width: data.width,
filename: data.filename,
source: data.source,
});
}, 300);
},

View file

@ -1,4 +1,5 @@
import { useCallback } from 'react';
import { FileSources } from 'librechat-data-provider';
import { useGetEndpointsQuery } from 'librechat-data-provider/react-query';
import {
useSetRecoilState,
@ -122,10 +123,11 @@ const useNewConvo = (index = 0) => {
if (conversation.conversationId === 'new' && !modelsData) {
const filesToDelete = Array.from(files.values())
.filter((file) => file.filepath)
.filter((file) => file.filepath && file.source)
.map((file) => ({
file_id: file.file_id,
filepath: file.filepath as string,
source: file.source as FileSources, // Ensure that the source is of type FileSources
}));
setFiles(new Map());

View file

@ -162,6 +162,10 @@ export default {
'Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model\'s likelihood to repeat the same line verbatim.',
com_endpoint_openai_pres:
'Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model\'s likelihood to talk about new topics.',
com_endpoint_openai_resend:
'Resend all previously attached images. Note: this can significantly increase token cost and you may experience errors with many image attachments.',
com_endpoint_openai_detail:
'The resolution for Vision requests. "Low" is cheaper and faster, "High" is more detailed and expensive, and "Auto" will automatically choose between the two based on the image resolution.',
com_endpoint_openai_custom_name_placeholder: 'Set a custom name for ChatGPT',
com_endpoint_openai_prompt_prefix_placeholder:
'Set custom instructions to include in System Message. Default: none',
@ -177,6 +181,7 @@ export default {
com_endpoint_frequency_penalty: 'Frequency Penalty',
com_endpoint_presence_penalty: 'Presence Penalty',
com_endpoint_plug_use_functions: 'Use Functions',
com_endpoint_plug_resend_images: 'Resend Images',
com_endpoint_plug_skip_completion: 'Skip Completion',
com_endpoint_disabled_with_tools: 'disabled with tools',
com_endpoint_disabled_with_tools_placeholder: 'Disabled with Tools Selected',

View file

@ -6,7 +6,7 @@ import {
useGetModelsQuery,
useGetEndpointsQuery,
} from 'librechat-data-provider/react-query';
import { TPreset } from 'librechat-data-provider';
import type { TPreset } from 'librechat-data-provider';
import { useNewConvo, useConfigOverride } from '~/hooks';
import ChatView from '~/components/Chat/ChatView';
import useAuthRedirect from './useAuthRedirect';
@ -15,6 +15,7 @@ import store from '~/store';
export default function ChatRoute() {
const index = 0;
useConfigOverride();
const { conversationId } = useParams();
const { conversation } = store.useCreateConversationAtom(index);

View file

@ -1,11 +1,12 @@
/* eslint-disable react-hooks/exhaustive-deps */
import { useEffect, useState } from 'react';
import { useRecoilValue, useSetRecoilState } from 'recoil';
import { Outlet, useLocation } from 'react-router-dom';
import { useRecoilValue, useSetRecoilState } from 'recoil';
import { useGetModelsQuery, useGetSearchEnabledQuery } from 'librechat-data-provider/react-query';
import type { ContextType } from '~/common';
import { useAuthContext, useServerStream, useConversation } from '~/hooks';
import { Nav, MobileNav } from '~/components/Nav';
import { useGetFiles } from '~/data-provider';
import store from '~/store';
export default function Root() {
@ -24,6 +25,7 @@ export default function Root() {
const setIsSearchEnabled = useSetRecoilState(store.isSearchEnabled);
const setModelsConfig = useSetRecoilState(store.modelsConfig);
useGetFiles({ enabled: isAuthenticated });
const searchEnabledQuery = useGetSearchEnabledQuery({ enabled: isAuthenticated });
const modelsQuery = useGetModelsQuery({ enabled: isAuthenticated && modelsQueryEnabled });

View file

@ -18,7 +18,7 @@ const messages = atom<TMessagesAtom>({
const messagesTree = selector({
key: 'messagesTree',
get: ({ get }) => {
return buildTree(get(messages), false);
return buildTree({ messages: get(messages) });
},
});

View file

@ -20,7 +20,7 @@ const searchResultMessages = atom<TMessage[] | null>({
const searchResultMessagesTree = selector({
key: 'searchResultMessagesTree',
get: ({ get }) => {
return buildTree(get(searchResultMessages), true);
return buildTree({ messages: get(searchResultMessages), groupAll: true });
},
});

View file

@ -1,11 +1,19 @@
import { TMessage } from 'librechat-data-provider';
import { TFile, TMessage } from 'librechat-data-provider';
const even =
'w-full border-b border-black/10 dark:border-gray-900/50 text-gray-800 bg-white dark:text-gray-100 group dark:bg-gray-800 hover:bg-gray-100/25 hover:text-gray-700 dark:hover:bg-gray-900 dark:hover:text-gray-200';
const odd =
'w-full border-b border-black/10 bg-gray-50 dark:border-gray-900/50 text-gray-800 dark:text-gray-100 group bg-gray-100 dark:bg-gray-1000 hover:bg-gray-100/40 hover:text-gray-700 dark:hover:bg-[#3b3d49] dark:hover:text-gray-200';
export default function buildTree(messages: TMessage[] | null, groupAll = false) {
export default function buildTree({
messages,
fileMap,
groupAll = false,
}: {
messages: TMessage[] | null;
fileMap?: Record<string, TFile>;
groupAll?: boolean;
}) {
if (messages === null) {
return null;
}
@ -21,6 +29,12 @@ export default function buildTree(messages: TMessage[] | null, groupAll = false)
messages.forEach((message) => {
messageMap[message.messageId] = { ...message, children: [] };
if (message.files && fileMap) {
messageMap[message.messageId].files = message.files.map(
(file) => fileMap[file.file_id] ?? file,
);
}
const parentMessage = messageMap[message.parentMessageId ?? ''];
if (parentMessage) {
parentMessage.children.push(messageMap[message.messageId]);

12
client/src/utils/files.ts Normal file
View file

@ -0,0 +1,12 @@
import type { TFile } from 'librechat-data-provider';
/** Maps Files by `file_id` for quick lookup */
export function mapFiles(files: TFile[]) {
const fileMap = {} as Record<string, TFile>;
for (const file of files) {
fileMap[file.file_id] = file;
}
return fileMap;
}

View file

@ -1,4 +1,5 @@
export * from './json';
export * from './files';
export * from './presets';
export * from './languages';
export * from './endpoints';

View file

@ -12,6 +12,8 @@ export const getPresetIcon = (preset: TPreset, Icon) => {
});
};
type TEndpoints = Array<string | EModelEndpoint>;
export const getPresetTitle = (preset: TPreset) => {
const {
endpoint,
@ -26,12 +28,16 @@ export const getPresetTitle = (preset: TPreset) => {
let modelInfo = model || '';
let label = '';
if (
endpoint &&
[EModelEndpoint.azureOpenAI, EModelEndpoint.openAI, EModelEndpoint.custom].includes(endpoint)
) {
const usesChatGPTLabel: TEndpoints = [
EModelEndpoint.azureOpenAI,
EModelEndpoint.openAI,
EModelEndpoint.custom,
];
const usesModelLabel: TEndpoints = [EModelEndpoint.google, EModelEndpoint.anthropic];
if (endpoint && usesChatGPTLabel.includes(endpoint)) {
label = chatGptLabel || '';
} else if (endpoint && [EModelEndpoint.google, EModelEndpoint.anthropic].includes(endpoint)) {
} else if (endpoint && usesModelLabel.includes(endpoint)) {
label = modelLabel || '';
} else if (endpoint === EModelEndpoint.bingAI) {
modelInfo = jailbreak ? 'Sydney' : modelInfo;