Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2025-09-22 06:00:56 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

🔀 feat: Save & Submit Message Content Parts (#8171)

Browse source 
* 🐛 fix: Enhance provider validation and error handling in getProviderConfig function

* WIP: edit text part

* refactor: Allow updating of both TEXT and THINK content types in message updates

* WIP: first pass, save & submit

* chore: remove legacy generation user message field

* feat: merge edited content

* fix: update placeholder and description for bedrock setting

* fix: remove unsupported warning message for AI resubmission
This commit is contained in:
Danny Avila 2025-07-01 15:43:10 -04:00 • committed by GitHub
parent a648ad3d13
commit 434289fe92
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 240 additions and 84 deletions
Download patch file Download diff file Expand all files Collapse all files

85
api/app/clients/BaseClient.js
View file

@ -13,7 +13,6 @@ const {
const { getMessages, saveMessage, updateMessage, saveConvo, getConvo } = require('~/models');
const { checkBalance } = require('~/models/balanceMethods');
const { truncateToolCallOutputs } = require('./prompts');
const { addSpaceIfNeeded } = require('~/server/utils');
const { getFiles } = require('~/models/File');
const TextStream = require('./TextStream');
const { logger } = require('~/config');
@ -572,7 +571,7 @@ class BaseClient {
});
}
const { generation = '' } = opts;
const { editedContent } = opts;
// It's not necessary to push to currentMessages
// depending on subclass implementation of handling messages
@ -587,11 +586,21 @@ class BaseClient {
isCreatedByUser: false,
model: this.modelOptions?.model ?? this.model,
sender: this.sender,
text: generation,
};
this.currentMessages.push(userMessage, latestMessage);
} else {
latestMessage.text = generation;
} else if (editedContent != null) {
// Handle editedContent for content parts
if (editedContent && latestMessage.content && Array.isArray(latestMessage.content)) {
const { index, text, type } = editedContent;
if (index >= 0 && index < latestMessage.content.length) {
const contentPart = latestMessage.content[index];
if (type === ContentTypes.THINK && contentPart.type === ContentTypes.THINK) {
contentPart[ContentTypes.THINK] = text;
} else if (type === ContentTypes.TEXT && contentPart.type === ContentTypes.TEXT) {
contentPart[ContentTypes.TEXT] = text;
}
}
}
}
this.continued = true;
} else {
@ -672,16 +681,32 @@ class BaseClient {
};
if (typeof completion === 'string') {
responseMessage.text = addSpaceIfNeeded(generation) + completion;
responseMessage.text = completion;
} else if (
Array.isArray(completion) &&
(this.clientName === EModelEndpoint.agents ||
isParamEndpoint(this.options.endpoint, this.options.endpointType))
) {
responseMessage.text = '';
responseMessage.content = completion;
if (!opts.editedContent || this.currentMessages.length === 0) {
responseMessage.content = completion;
} else {
const latestMessage = this.currentMessages[this.currentMessages.length - 1];
if (!latestMessage?.content) {
responseMessage.content = completion;
} else {
const existingContent = [...latestMessage.content];
const { type: editedType } = opts.editedContent;
responseMessage.content = this.mergeEditedContent(
existingContent,
completion,
editedType,
);
}
}
} else if (Array.isArray(completion)) {
responseMessage.text = addSpaceIfNeeded(generation) + completion.join('');
responseMessage.text = completion.join('');
}
if (
@ -1095,6 +1120,50 @@ class BaseClient {
return numTokens;
}
/**
* Merges completion content with existing content when editing TEXT or THINK types
* @param {Array} existingContent - The existing content array
* @param {Array} newCompletion - The new completion content
* @param {string} editedType - The type of content being edited
* @returns {Array} The merged content array
*/
mergeEditedContent(existingContent, newCompletion, editedType) {
if (!newCompletion.length) {
return existingContent.concat(newCompletion);
}
if (editedType !== ContentTypes.TEXT && editedType !== ContentTypes.THINK) {
return existingContent.concat(newCompletion);
}
const lastIndex = existingContent.length - 1;
const lastExisting = existingContent[lastIndex];
const firstNew = newCompletion[0];
if (lastExisting?.type !== firstNew?.type || firstNew?.type !== editedType) {
return existingContent.concat(newCompletion);
}
const mergedContent = [...existingContent];
if (editedType === ContentTypes.TEXT) {
mergedContent[lastIndex] = {
...mergedContent[lastIndex],
[ContentTypes.TEXT]:
(mergedContent[lastIndex][ContentTypes.TEXT] || '') + (firstNew[ContentTypes.TEXT] || ''),
};
} else {
mergedContent[lastIndex] = {
...mergedContent[lastIndex],
[ContentTypes.THINK]:
(mergedContent[lastIndex][ContentTypes.THINK] || '') +
(firstNew[ContentTypes.THINK] || ''),
};
}
// Add remaining completion items
return mergedContent.concat(newCompletion.slice(1));
}
async sendPayload(payload, opts = {}) {
if (opts && typeof opts === 'object') {
this.setOptions(opts);

11
api/server/controllers/agents/request.js
View file

@ -14,8 +14,11 @@ const AgentController = async (req, res, next, initializeClient, addTitle) => {
text,
endpointOption,
conversationId,
isContinued = false,
editedContent = null,
parentMessageId = null,
overrideParentMessageId = null,
responseMessageId: editedResponseMessageId = null,
} = req.body;
let sender;
@ -67,7 +70,7 @@ const AgentController = async (req, res, next, initializeClient, addTitle) => {
handler();
}
} catch (e) {
// Ignore cleanup errors
logger.error('[AgentController] Error in cleanup handler', e);
}
}
}
@ -155,7 +158,7 @@ const AgentController = async (req, res, next, initializeClient, addTitle) => {
try {
res.removeListener('close', closeHandler);
} catch (e) {
// Ignore
logger.error('[AgentController] Error removing close listener', e);
}
});
@ -163,10 +166,14 @@ const AgentController = async (req, res, next, initializeClient, addTitle) => {
user: userId,
onStart,
getReqData,
isContinued,
editedContent,
conversationId,
parentMessageId,
abortController,
overrideParentMessageId,
isEdited: !!editedContent,
responseMessageId: editedResponseMessageId,
progressOptions: {
res,
},

7
api/server/routes/messages.js
View file

@ -235,12 +235,13 @@ router.put('/:conversationId/:messageId', validateMessageReq, async (req, res) =
return res.status(400).json({ error: 'Content part not found' });
}
if (updatedContent[index].type !== ContentTypes.TEXT) {
const currentPartType = updatedContent[index].type;
if (currentPartType !== ContentTypes.TEXT && currentPartType !== ContentTypes.THINK) {
return res.status(400).json({ error: 'Cannot update non-text content' });
}
const oldText = updatedContent[index].text;
updatedContent[index] = { type: ContentTypes.TEXT, text };
const oldText = updatedContent[index][currentPartType];
updatedContent[index] = { type: currentPartType, [currentPartType]: text };
let tokenCount = message.tokenCount;
if (tokenCount !== undefined) {

17
api/server/services/Endpoints/index.js
View file

@ -7,6 +7,16 @@ const initCustom = require('~/server/services/Endpoints/custom/initialize');
const initGoogle = require('~/server/services/Endpoints/google/initialize');
const { getCustomEndpointConfig } = require('~/server/services/Config');
/** Check if the provider is a known custom provider
* @param {string | undefined} [provider] - The provider string
* @returns {boolean} - True if the provider is a known custom provider, false otherwise
*/
function isKnownCustomProvider(provider) {
return [Providers.XAI, Providers.OLLAMA, Providers.DEEPSEEK, Providers.OPENROUTER].includes(
provider || '',
);
}
const providerConfigMap = {
[Providers.XAI]: initCustom,
[Providers.OLLAMA]: initCustom,
@ -46,6 +56,13 @@ async function getProviderConfig(provider) {
overrideProvider = Providers.OPENAI;
}
if (isKnownCustomProvider(overrideProvider)) {
customEndpointConfig = await getCustomEndpointConfig(provider);
if (!customEndpointConfig) {
throw new Error(`Provider ${provider} not supported`);
}
}
return {
getOptions,
overrideProvider,

5
client/src/common/types.ts
View file

@ -336,6 +336,11 @@ export type TAskProps = {
export type TOptions = {
editedMessageId?: string | null;
editedText?: string | null;
editedContent?: {
index: number;
text: string;
type: 'text' | 'think';
};
isRegenerate?: boolean;
isContinued?: boolean;
isEdited?: boolean;

13
client/src/components/Chat/Messages/Content/ContentParts.tsx
View file

@ -81,14 +81,23 @@ const ContentParts = memo(
return (
<>
{content.map((part, idx) => {
if (part?.type !== ContentTypes.TEXT || typeof part.text !== 'string') {
if (!part) {
return null;
}
const isTextPart =
part?.type === ContentTypes.TEXT ||
typeof (part as unknown as Agents.MessageContentText)?.text !== 'string';
const isThinkPart =
part?.type === ContentTypes.THINK ||
typeof (part as unknown as Agents.ReasoningDeltaUpdate)?.think !== 'string';
if (!isTextPart && !isThinkPart) {
return null;
}
return (
<EditTextPart
index={idx}
text={part.text}
part={part as Agents.MessageContentText | Agents.ReasoningDeltaUpdate}
messageId={messageId}
isSubmitting={isSubmitting}
enterEdit={enterEdit}

39
client/src/components/Chat/Messages/Content/Parts/EditTextPart.tsx
View file

@ -1,8 +1,9 @@
import { useRef, useEffect, useCallback, useMemo } from 'react';
import { useForm } from 'react-hook-form';
import { ContentTypes } from 'librechat-data-provider';
import { useRecoilState, useRecoilValue } from 'recoil';
import { useRef, useEffect, useCallback, useMemo } from 'react';
import { useUpdateMessageContentMutation } from 'librechat-data-provider/react-query';
import type { Agents } from 'librechat-data-provider';
import type { TEditProps } from '~/common';
import Container from '~/components/Chat/Messages/Content/Container';
import { useChatContext, useAddedChatContext } from '~/Providers';
@ -12,18 +13,19 @@ import { useLocalize } from '~/hooks';
import store from '~/store';
const EditTextPart = ({
text,
part,
index,
messageId,
isSubmitting,
enterEdit,
}: Omit<TEditProps, 'message' | 'ask'> & {
}: Omit<TEditProps, 'message' | 'ask' | 'text'> & {
index: number;
messageId: string;
part: Agents.MessageContentText | Agents.ReasoningDeltaUpdate;
}) => {
const localize = useLocalize();
const { addedIndex } = useAddedChatContext();
const { getMessages, setMessages, conversation } = useChatContext();
const { ask, getMessages, setMessages, conversation } = useChatContext();
const [latestMultiMessage, setLatestMultiMessage] = useRecoilState(
store.latestMessageFamily(addedIndex),
);
@ -34,15 +36,16 @@ const EditTextPart = ({
[getMessages, messageId],
);
const chatDirection = useRecoilValue(store.chatDirection);
const textAreaRef = useRef<HTMLTextAreaElement | null>(null);
const updateMessageContentMutation = useUpdateMessageContentMutation(conversationId ?? '');
const chatDirection = useRecoilValue(store.chatDirection).toLowerCase();
const isRTL = chatDirection === 'rtl';
const isRTL = chatDirection?.toLowerCase() === 'rtl';
const { register, handleSubmit, setValue } = useForm({
defaultValues: {
text: text ?? '',
text: (ContentTypes.THINK in part ? part.think : part.text) || '',
},
});
@ -55,15 +58,7 @@ const EditTextPart = ({
}
}, []);
/*
const resubmitMessage = () => {
showToast({
status: 'warning',
message: localize('com_warning_resubmit_unsupported'),
});
// const resubmitMessage = (data: { text: string }) => {
// Not supported by AWS Bedrock
const resubmitMessage = (data: { text: string }) => {
const messages = getMessages();
const parentMessage = messages?.find((msg) => msg.messageId === message?.parentMessageId);
@ -73,17 +68,19 @@ const EditTextPart = ({
ask(
{ ...parentMessage },
{
editedText: data.text,
editedContent: {
index,
text: data.text,
type: part.type,
},
editedMessageId: messageId,
isRegenerate: true,
isEdited: true,
},
);
setSiblingIdx((siblingIdx ?? 0) - 1);
enterEdit(true);
};
*/
const updateMessage = (data: { text: string }) => {
const messages = getMessages();
@ -167,13 +164,13 @@ const EditTextPart = ({
/>
</div>
<div className="mt-2 flex w-full justify-center text-center">
{/* <button
<button
className="btn btn-primary relative mr-2"
disabled={isSubmitting}
onClick={handleSubmit(resubmitMessage)}
>
{localize('com_ui_save_submit')}
</button> */}
</button>
<button
className="btn btn-secondary relative mr-2"
disabled={isSubmitting}

68
client/src/hooks/Chat/useChatFunctions.ts
View file

@ -6,6 +6,7 @@ import {
QueryKeys,
ContentTypes,
EModelEndpoint,
isAgentsEndpoint,
parseCompactConvo,
replaceSpecialVars,
isAssistantsEndpoint,
@ -36,15 +37,6 @@ const logChatRequest = (request: Record<string, unknown>) => {
logger.log('=====================================');
};
const usesContentStream = (endpoint: EModelEndpoint | undefined, endpointType?: string) => {
if (endpointType === EModelEndpoint.custom) {
return true;
}
if (endpoint === EModelEndpoint.openAI || endpoint === EModelEndpoint.azureOpenAI) {
return true;
}
};
export default function useChatFunctions({
index = 0,
files,
@ -93,7 +85,7 @@ export default function useChatFunctions({
messageId = null,
},
{
editedText = null,
editedContent = null,
editedMessageId = null,
isResubmission = false,
isRegenerate = false,
@ -245,14 +237,11 @@ export default function useChatFunctions({
setFilesToDelete({});
}
const generation = editedText ?? latestMessage?.text ?? '';
const responseText = isEditOrContinue ? generation : '';
const responseMessageId =
editedMessageId ?? (latestMessage?.messageId ? latestMessage?.messageId + '_' : null) ?? null;
const initialResponse: TMessage = {
sender: responseSender,
text: responseText,
text: '',
endpoint: endpoint ?? '',
parentMessageId: isRegenerate ? messageId : intermediateId,
messageId: responseMessageId ?? `${isRegenerate ? messageId : intermediateId}_`,
@ -272,34 +261,37 @@ export default function useChatFunctions({
{
type: ContentTypes.TEXT,
[ContentTypes.TEXT]: {
value: responseText,
value: '',
},
},
];
} else if (endpoint === EModelEndpoint.agents) {
initialResponse.model = conversation?.agent_id ?? '';
} else if (endpoint != null) {
initialResponse.model = isAgentsEndpoint(endpoint)
? (conversation?.agent_id ?? '')
: (conversation?.model ?? '');
initialResponse.text = '';
initialResponse.content = [
{
type: ContentTypes.TEXT,
[ContentTypes.TEXT]: {
value: responseText,
if (editedContent && latestMessage?.content) {
initialResponse.content = cloneDeep(latestMessage.content);
const { index, text, type } = editedContent;
if (initialResponse.content && index >= 0 && index < initialResponse.content.length) {
const contentPart = initialResponse.content[index];
if (type === ContentTypes.THINK && contentPart.type === ContentTypes.THINK) {
contentPart[ContentTypes.THINK] = text;
} else if (type === ContentTypes.TEXT && contentPart.type === ContentTypes.TEXT) {
contentPart[ContentTypes.TEXT] = text;
}
}
} else {
initialResponse.content = [
{
type: ContentTypes.TEXT,
[ContentTypes.TEXT]: {
value: '',
},
},
},
];
setShowStopButton(true);
} else if (usesContentStream(endpoint, endpointType)) {
initialResponse.text = '';
initialResponse.content = [
{
type: ContentTypes.TEXT,
[ContentTypes.TEXT]: {
value: responseText,
},
},
];
setShowStopButton(true);
} else {
];
}
setShowStopButton(true);
}
@ -316,7 +308,6 @@ export default function useChatFunctions({
endpointOption,
userMessage: {
...currentMsg,
generation,
responseMessageId,
overrideParentMessageId: isRegenerate ? messageId : null,
},
@ -328,6 +319,7 @@ export default function useChatFunctions({
initialResponse,
isTemporary,
ephemeralAgent,
editedContent,
};
if (isRegenerate) {

59
client/src/hooks/SSE/useStepHandler.ts
View file

@ -55,6 +55,26 @@ export default function useStepHandler({
const messageMap = useRef(new Map<string, TMessage>());
const stepMap = useRef(new Map<string, Agents.RunStep>());
const calculateContentIndex = (
baseIndex: number,
initialContent: TMessageContentParts[],
incomingContentType: string,
existingContent?: TMessageContentParts[],
): number => {
/** Only apply -1 adjustment for TEXT or THINK types when they match existing content */
if (
initialContent.length > 0 &&
(incomingContentType === ContentTypes.TEXT || incomingContentType === ContentTypes.THINK)
) {
const targetIndex = baseIndex + initialContent.length - 1;
const existingType = existingContent?.[targetIndex]?.type;
if (existingType === incomingContentType) {
return targetIndex;
}
}
return baseIndex + initialContent.length;
};
const updateContent = (
message: TMessage,
index: number,
@ -170,6 +190,11 @@ export default function useStepHandler({
lastAnnouncementTimeRef.current = currentTime;
}
let initialContent: TMessageContentParts[] = [];
if (submission?.editedContent != null) {
initialContent = submission?.initialResponse?.content ?? initialContent;
}
if (event === 'on_run_step') {
const runStep = data as Agents.RunStep;
const responseMessageId = runStep.runId ?? '';
@ -189,7 +214,7 @@ export default function useStepHandler({
parentMessageId: userMessage.messageId,
conversationId: userMessage.conversationId,
messageId: responseMessageId,
content: [],
content: initialContent,
};
messageMap.current.set(responseMessageId, response);
@ -214,7 +239,9 @@ export default function useStepHandler({
},
};
updatedResponse = updateContent(updatedResponse, runStep.index, contentPart);
/** Tool calls don't need index adjustment */
const currentIndex = runStep.index + initialContent.length;
updatedResponse = updateContent(updatedResponse, currentIndex, contentPart);
});
messageMap.current.set(responseMessageId, updatedResponse);
@ -234,7 +261,9 @@ export default function useStepHandler({
const response = messageMap.current.get(responseMessageId);
if (response) {
const updatedResponse = updateContent(response, agent_update.index, data);
// Agent updates don't need index adjustment
const currentIndex = agent_update.index + initialContent.length;
const updatedResponse = updateContent(response, currentIndex, data);
messageMap.current.set(responseMessageId, updatedResponse);
const currentMessages = getMessages() || [];
setMessages([...currentMessages.slice(0, -1), updatedResponse]);
@ -255,7 +284,13 @@ export default function useStepHandler({
? messageDelta.delta.content[0]
: messageDelta.delta.content;
const updatedResponse = updateContent(response, runStep.index, contentPart);
const currentIndex = calculateContentIndex(
runStep.index,
initialContent,
contentPart.type || '',
response.content,
);
const updatedResponse = updateContent(response, currentIndex, contentPart);
messageMap.current.set(responseMessageId, updatedResponse);
const currentMessages = getMessages() || [];
@ -277,7 +312,13 @@ export default function useStepHandler({
? reasoningDelta.delta.content[0]
: reasoningDelta.delta.content;
const updatedResponse = updateContent(response, runStep.index, contentPart);
const currentIndex = calculateContentIndex(
runStep.index,
initialContent,
contentPart.type || '',
response.content,
);
const updatedResponse = updateContent(response, currentIndex, contentPart);
messageMap.current.set(responseMessageId, updatedResponse);
const currentMessages = getMessages() || [];
@ -318,7 +359,9 @@ export default function useStepHandler({
contentPart.tool_call.expires_at = runStepDelta.delta.expires_at;
}
updatedResponse = updateContent(updatedResponse, runStep.index, contentPart);
/** Tool calls don't need index adjustment */
const currentIndex = runStep.index + initialContent.length;
updatedResponse = updateContent(updatedResponse, currentIndex, contentPart);
});
messageMap.current.set(responseMessageId, updatedResponse);
@ -350,7 +393,9 @@ export default function useStepHandler({
tool_call: result.tool_call,
};
updatedResponse = updateContent(updatedResponse, runStep.index, contentPart, true);
/** Tool calls don't need index adjustment */
const currentIndex = runStep.index + initialContent.length;
updatedResponse = updateContent(updatedResponse, currentIndex, contentPart, true);
messageMap.current.set(responseMessageId, updatedResponse);
const updatedMessages = messages.map((msg) =>

3
client/src/locales/en/translation.json
View file

@ -1067,6 +1067,5 @@
"com_ui_x_selected": "{{0}} selected",
"com_ui_yes": "Yes",
"com_ui_zoom": "Zoom",
"com_user_message": "You",
"com_warning_resubmit_unsupported": "Resubmitting the AI message is not supported for this endpoint."
"com_user_message": "You"
}

2
packages/data-provider/src/createPayload.ts
View file

@ -11,6 +11,7 @@ export default function createPayload(submission: t.TSubmission) {
isContinued,
isTemporary,
ephemeralAgent,
editedContent,
} = submission;
const { conversationId } = s.tConvoUpdateSchema.parse(conversation);
const { endpoint: _e, endpointType } = endpointOption as {
@ -34,6 +35,7 @@ export default function createPayload(submission: t.TSubmission) {
isContinued: !!(isEdited && isContinued),
conversationId,
isTemporary,
editedContent,
};
return { server, payload };

4
packages/data-provider/src/parameterSettings.ts
View file

@ -401,7 +401,9 @@ const bedrock: Record<string, SettingDefinition> = {
labelCode: true,
type: 'number',
component: 'input',
placeholder: 'com_endpoint_anthropic_maxoutputtokens',
description: 'com_endpoint_anthropic_maxoutputtokens',
descriptionCode: true,
placeholder: 'com_nav_theme_system',
placeholderCode: true,
optionType: 'model',
columnSpan: 2,

1
packages/data-provider/src/schemas.ts
View file

@ -503,6 +503,7 @@ export const tMessageSchema = z.object({
title: z.string().nullable().or(z.literal('New Chat')).default('New Chat'),
sender: z.string().optional(),
text: z.string(),
/** @deprecated */
generation: z.string().nullable().optional(),
isCreatedByUser: z.boolean(),
error: z.boolean().optional(),

10
packages/data-provider/src/types.ts
View file

@ -109,6 +109,11 @@ export type TPayload = Partial<TMessage> &
messages?: TMessages;
isTemporary: boolean;
ephemeralAgent?: TEphemeralAgent | null;
editedContent?: {
index: number;
text: string;
type: 'text' | 'think';
} | null;
};
export type TSubmission = {
@ -127,6 +132,11 @@ export type TSubmission = {
endpointOption: TEndpointOption;
clientTimestamp?: string;
ephemeralAgent?: TEphemeralAgent | null;
editedContent?: {
index: number;
text: string;
type: 'text' | 'think';
} | null;
};
export type EventSubmission = Omit<TSubmission, 'initialResponse'> & { initialResponse: TMessage };