mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-04-07 00:15:23 +02:00
feat: extend data retention to files, tool calls, and shared links
Add expiredAt field and TTL indexes to file, toolCall, and share schemas. Set expiredAt on tool calls, shared links, and file uploads when retentionMode is "all" or chat is temporary. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
30109e90b0
commit
48973752d3
13 changed files with 101 additions and 17 deletions
|
|
@ -1,12 +1,13 @@
|
|||
const { nanoid } = require('nanoid');
|
||||
const { EnvVar } = require('@librechat/agents');
|
||||
const { logger } = require('@librechat/data-schemas');
|
||||
const { checkAccess, loadWebSearchAuth } = require('@librechat/api');
|
||||
const { checkAccess, loadWebSearchAuth, createTempChatExpirationDate } = require('@librechat/api');
|
||||
const {
|
||||
Tools,
|
||||
AuthType,
|
||||
Permissions,
|
||||
ToolCallTypes,
|
||||
RetentionMode,
|
||||
PermissionTypes,
|
||||
} = require('librechat-data-provider');
|
||||
const { getRoleByName, createToolCall, getToolCallsByConvo, getMessage } = require('~/models');
|
||||
|
|
@ -181,6 +182,15 @@ const callTool = async (req, res) => {
|
|||
user: req.user.id,
|
||||
};
|
||||
|
||||
if (req?.body?.isTemporary || appConfig?.interfaceConfig?.retentionMode === RetentionMode.ALL) {
|
||||
try {
|
||||
toolCallData.expiredAt = createTempChatExpirationDate(appConfig?.interfaceConfig);
|
||||
} catch (err) {
|
||||
logger.error('Error creating tool call expiration date:', err);
|
||||
toolCallData.expiredAt = null;
|
||||
}
|
||||
}
|
||||
|
||||
if (!artifact || !artifact.files || toolId !== Tools.execute_code) {
|
||||
createToolCall(toolCallData).catch((error) => {
|
||||
logger.error(`Error creating tool call: ${error.message}`);
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
const express = require('express');
|
||||
const { isEnabled } = require('@librechat/api');
|
||||
const { isEnabled, createTempChatExpirationDate } = require('@librechat/api');
|
||||
const { logger } = require('@librechat/data-schemas');
|
||||
const { RetentionMode } = require('librechat-data-provider');
|
||||
const {
|
||||
getSharedMessages,
|
||||
createSharedLink,
|
||||
|
|
@ -98,7 +99,20 @@ router.get('/link/:conversationId', requireJwtAuth, async (req, res) => {
|
|||
router.post('/:conversationId', requireJwtAuth, async (req, res) => {
|
||||
try {
|
||||
const { targetMessageId } = req.body;
|
||||
const created = await createSharedLink(req.user.id, req.params.conversationId, targetMessageId);
|
||||
let expiredAt;
|
||||
if (req?.config?.interfaceConfig?.retentionMode === RetentionMode.ALL) {
|
||||
try {
|
||||
expiredAt = createTempChatExpirationDate(req.config?.interfaceConfig);
|
||||
} catch (err) {
|
||||
logger.error('Error creating shared link expiration date:', err);
|
||||
}
|
||||
}
|
||||
const created = await createSharedLink(
|
||||
req.user.id,
|
||||
req.params.conversationId,
|
||||
targetMessageId,
|
||||
expiredAt,
|
||||
);
|
||||
if (created) {
|
||||
res.status(200).json(created);
|
||||
} else {
|
||||
|
|
@ -112,7 +126,15 @@ router.post('/:conversationId', requireJwtAuth, async (req, res) => {
|
|||
|
||||
router.patch('/:shareId', requireJwtAuth, async (req, res) => {
|
||||
try {
|
||||
const updatedShare = await updateSharedLink(req.user.id, req.params.shareId);
|
||||
let expiredAt;
|
||||
if (req?.config?.interfaceConfig?.retentionMode === RetentionMode.ALL) {
|
||||
try {
|
||||
expiredAt = createTempChatExpirationDate(req.config?.interfaceConfig);
|
||||
} catch (err) {
|
||||
logger.error('Error creating shared link expiration date:', err);
|
||||
}
|
||||
}
|
||||
const updatedShare = await updateSharedLink(req.user.id, req.params.shareId, expiredAt);
|
||||
if (updatedShare) {
|
||||
res.status(200).json(updatedShare);
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ const { filterFilesByAgentAccess } = require('~/server/services/Files/permission
|
|||
const { createFile, getFiles, updateFile, claimCodeFile } = require('~/models');
|
||||
const { getStrategyFunctions } = require('~/server/services/Files/strategies');
|
||||
const { convertImage } = require('~/server/services/Files/images/convert');
|
||||
const { getRetentionExpiry } = require('~/server/services/Files/process');
|
||||
const { determineFileType } = require('~/server/utils');
|
||||
|
||||
const axios = createAxiosInstance();
|
||||
|
|
@ -182,6 +183,7 @@ const processCodeOutput = async ({
|
|||
source: appConfig.fileStrategy,
|
||||
context: FileContext.execute_code,
|
||||
metadata: { fileIdentifier },
|
||||
...getRetentionExpiry(req),
|
||||
};
|
||||
await createFile(file, true);
|
||||
return Object.assign(file, { messageId, toolCallId });
|
||||
|
|
@ -241,6 +243,7 @@ const processCodeOutput = async ({
|
|||
context: FileContext.execute_code,
|
||||
usage: isUpdate ? (claimed.usage ?? 0) + 1 : 1,
|
||||
createdAt: isUpdate ? claimed.createdAt : formattedDate,
|
||||
...getRetentionExpiry(req),
|
||||
};
|
||||
|
||||
await createFile(file, true);
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ const {
|
|||
FileContext,
|
||||
FileSources,
|
||||
imageExtRegex,
|
||||
RetentionMode,
|
||||
EModelEndpoint,
|
||||
EToolResources,
|
||||
mergeFileConfig,
|
||||
|
|
@ -20,7 +21,12 @@ const {
|
|||
} = require('librechat-data-provider');
|
||||
const { EnvVar } = require('@librechat/agents');
|
||||
const { logger } = require('@librechat/data-schemas');
|
||||
const { sanitizeFilename, parseText, processAudioFile } = require('@librechat/api');
|
||||
const {
|
||||
sanitizeFilename,
|
||||
parseText,
|
||||
processAudioFile,
|
||||
createTempChatExpirationDate,
|
||||
} = require('@librechat/api');
|
||||
const {
|
||||
convertImage,
|
||||
resizeAndConvert,
|
||||
|
|
@ -37,6 +43,23 @@ const { determineFileType } = require('~/server/utils');
|
|||
const { STTService } = require('./Audio/STTService');
|
||||
const db = require('~/models');
|
||||
|
||||
/**
|
||||
* Returns `{ expiredAt }` when the request indicates data retention applies, otherwise `{}`.
|
||||
* Spread into file data objects before calling createFile.
|
||||
* @param {ServerRequest} req
|
||||
* @returns {{ expiredAt?: Date }}
|
||||
*/
|
||||
function getRetentionExpiry(req) {
|
||||
if (req?.body?.isTemporary || req?.config?.interfaceConfig?.retentionMode === RetentionMode.ALL) {
|
||||
try {
|
||||
return { expiredAt: createTempChatExpirationDate(req.config?.interfaceConfig) };
|
||||
} catch (_err) {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a modular file upload wrapper that ensures filename sanitization
|
||||
* across all storage strategies. This prevents storage-specific implementations
|
||||
|
|
@ -307,6 +330,7 @@ const processImageFile = async ({ req, res, metadata, returnFile = false }) => {
|
|||
context: FileContext.message_attachment,
|
||||
source,
|
||||
type: `image/${appConfig.imageOutputType}`,
|
||||
...getRetentionExpiry(req),
|
||||
width,
|
||||
height,
|
||||
},
|
||||
|
|
@ -359,6 +383,7 @@ const uploadImageBuffer = async ({ req, context, metadata = {}, resize = true })
|
|||
source,
|
||||
type,
|
||||
width,
|
||||
...getRetentionExpiry(req),
|
||||
height,
|
||||
},
|
||||
true,
|
||||
|
|
@ -445,6 +470,7 @@ const processFileUpload = async ({ req, res, metadata }) => {
|
|||
context: isAssistantUpload ? FileContext.assistants : FileContext.message_attachment,
|
||||
model: isAssistantUpload ? req.body.model : undefined,
|
||||
type: file.mimetype,
|
||||
...getRetentionExpiry(req),
|
||||
embedded,
|
||||
source,
|
||||
height,
|
||||
|
|
@ -541,6 +567,7 @@ const processAgentFileUpload = async ({ req, res, metadata }) => {
|
|||
filename: file.originalname,
|
||||
model: messageAttachment ? undefined : req.body.model,
|
||||
context: messageAttachment ? FileContext.message_attachment : FileContext.agents,
|
||||
...getRetentionExpiry(req),
|
||||
});
|
||||
|
||||
if (!messageAttachment && tool_resource) {
|
||||
|
|
@ -717,6 +744,7 @@ const processAgentFileUpload = async ({ req, res, metadata }) => {
|
|||
source,
|
||||
height,
|
||||
width,
|
||||
...getRetentionExpiry(req),
|
||||
});
|
||||
|
||||
const result = await db.createFile(fileInfo, true);
|
||||
|
|
@ -762,6 +790,7 @@ const processOpenAIFile = async ({
|
|||
source,
|
||||
model: openai.req.body.model,
|
||||
filename: originalName ?? file_id,
|
||||
...getRetentionExpiry(openai.req),
|
||||
};
|
||||
|
||||
if (saveFile) {
|
||||
|
|
@ -805,6 +834,7 @@ const processOpenAIImageOutput = async ({ req, buffer, file_id, filename, fileEx
|
|||
context: FileContext.assistants_output,
|
||||
file_id,
|
||||
filename,
|
||||
...getRetentionExpiry(req),
|
||||
};
|
||||
db.createFile(file, true);
|
||||
return file;
|
||||
|
|
@ -961,6 +991,7 @@ async function saveBase64Image(
|
|||
user: req.user.id,
|
||||
bytes: image.bytes,
|
||||
width: image.width,
|
||||
...getRetentionExpiry(req),
|
||||
height: image.height,
|
||||
},
|
||||
true,
|
||||
|
|
@ -1047,6 +1078,7 @@ function filterFile({ req, image, isAvatar }) {
|
|||
|
||||
module.exports = {
|
||||
filterFile,
|
||||
getRetentionExpiry,
|
||||
processFileURL,
|
||||
saveBase64Image,
|
||||
processImageFile,
|
||||
|
|
|
|||
|
|
@ -61,9 +61,9 @@ const BookmarkMenu: FC = () => {
|
|||
|
||||
const isActiveConvo = Boolean(
|
||||
conversation &&
|
||||
conversationId &&
|
||||
conversationId !== Constants.NEW_CONVO &&
|
||||
conversationId !== 'search',
|
||||
conversationId &&
|
||||
conversationId !== Constants.NEW_CONVO &&
|
||||
conversationId !== 'search',
|
||||
);
|
||||
|
||||
const handleSubmit = useCallback(
|
||||
|
|
|
|||
|
|
@ -180,10 +180,7 @@ export function createConversationMethods(
|
|||
update.isTemporary = false;
|
||||
}
|
||||
|
||||
if (
|
||||
isTemporary ||
|
||||
interfaceConfig?.retentionMode === RetentionMode.ALL
|
||||
) {
|
||||
if (isTemporary || interfaceConfig?.retentionMode === RetentionMode.ALL) {
|
||||
try {
|
||||
update.expiredAt = createTempChatExpirationDate(interfaceConfig);
|
||||
} catch (err) {
|
||||
|
|
|
|||
|
|
@ -92,10 +92,7 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa
|
|||
messageId: params.newMessageId || params.messageId,
|
||||
};
|
||||
|
||||
if (
|
||||
isTemporary ||
|
||||
interfaceConfig?.retentionMode === RetentionMode.ALL
|
||||
) {
|
||||
if (isTemporary || interfaceConfig?.retentionMode === RetentionMode.ALL) {
|
||||
try {
|
||||
update.expiredAt = createTempChatExpirationDate(interfaceConfig);
|
||||
} catch (err) {
|
||||
|
|
|
|||
|
|
@ -345,6 +345,7 @@ export function createShareMethods(mongoose: typeof import('mongoose')) {
|
|||
user: string,
|
||||
conversationId: string,
|
||||
targetMessageId?: string,
|
||||
expiredAt?: Date,
|
||||
): Promise<t.CreateShareResult> {
|
||||
if (!user || !conversationId) {
|
||||
throw new ShareServiceError('Missing required parameters', 'INVALID_PARAMS');
|
||||
|
|
@ -408,6 +409,7 @@ export function createShareMethods(mongoose: typeof import('mongoose')) {
|
|||
title,
|
||||
user,
|
||||
...(targetMessageId && { targetMessageId }),
|
||||
...(expiredAt && { expiredAt }),
|
||||
});
|
||||
|
||||
return { shareId, conversationId };
|
||||
|
|
@ -460,7 +462,11 @@ export function createShareMethods(mongoose: typeof import('mongoose')) {
|
|||
/**
|
||||
* Update a shared link with new messages
|
||||
*/
|
||||
async function updateSharedLink(user: string, shareId: string): Promise<t.UpdateShareResult> {
|
||||
async function updateSharedLink(
|
||||
user: string,
|
||||
shareId: string,
|
||||
expiredAt?: Date,
|
||||
): Promise<t.UpdateShareResult> {
|
||||
if (!user || !shareId) {
|
||||
throw new ShareServiceError('Missing required parameters', 'INVALID_PARAMS');
|
||||
}
|
||||
|
|
@ -485,6 +491,7 @@ export function createShareMethods(mongoose: typeof import('mongoose')) {
|
|||
messages: updatedMessages,
|
||||
user,
|
||||
shareId: newShareId,
|
||||
...(expiredAt && { expiredAt }),
|
||||
};
|
||||
|
||||
const updatedShare = (await SharedLink.findOneAndUpdate({ shareId, user }, update, {
|
||||
|
|
|
|||
|
|
@ -82,12 +82,16 @@ const file: Schema<IMongoFile> = new Schema(
|
|||
type: String,
|
||||
index: true,
|
||||
},
|
||||
expiredAt: {
|
||||
type: Date,
|
||||
},
|
||||
},
|
||||
{
|
||||
timestamps: true,
|
||||
},
|
||||
);
|
||||
|
||||
file.index({ expiredAt: 1 }, { expireAfterSeconds: 0 });
|
||||
file.index({ createdAt: 1, updatedAt: 1 });
|
||||
file.index(
|
||||
{ filename: 1, conversationId: 1, context: 1, tenantId: 1 },
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ export interface ISharedLink extends Document {
|
|||
shareId?: string;
|
||||
targetMessageId?: string;
|
||||
isPublic: boolean;
|
||||
expiredAt?: Date;
|
||||
createdAt?: Date;
|
||||
updatedAt?: Date;
|
||||
tenantId?: string;
|
||||
|
|
@ -45,10 +46,14 @@ const shareSchema: Schema<ISharedLink> = new Schema(
|
|||
type: String,
|
||||
index: true,
|
||||
},
|
||||
expiredAt: {
|
||||
type: Date,
|
||||
},
|
||||
},
|
||||
{ timestamps: true },
|
||||
);
|
||||
|
||||
shareSchema.index({ expiredAt: 1 }, { expireAfterSeconds: 0 });
|
||||
shareSchema.index({ conversationId: 1, user: 1, targetMessageId: 1, tenantId: 1 });
|
||||
|
||||
export default shareSchema;
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ export interface IToolCallData extends Document {
|
|||
attachments?: TAttachment[];
|
||||
blockIndex?: number;
|
||||
partIndex?: number;
|
||||
expiredAt?: Date;
|
||||
createdAt?: Date;
|
||||
updatedAt?: Date;
|
||||
tenantId?: string;
|
||||
|
|
@ -50,10 +51,14 @@ const toolCallSchema: Schema<IToolCallData> = new Schema(
|
|||
type: String,
|
||||
index: true,
|
||||
},
|
||||
expiredAt: {
|
||||
type: Date,
|
||||
},
|
||||
},
|
||||
{ timestamps: true },
|
||||
);
|
||||
|
||||
toolCallSchema.index({ expiredAt: 1 }, { expireAfterSeconds: 0 });
|
||||
toolCallSchema.index({ messageId: 1, user: 1, tenantId: 1 });
|
||||
toolCallSchema.index({ conversationId: 1, user: 1, tenantId: 1 });
|
||||
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ export interface IMongoFile extends Omit<Document, 'model'> {
|
|||
fileIdentifier?: string;
|
||||
};
|
||||
expiresAt?: Date;
|
||||
expiredAt?: Date;
|
||||
createdAt?: Date;
|
||||
updatedAt?: Date;
|
||||
tenantId?: string;
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ export interface ISharedLink {
|
|||
shareId?: string;
|
||||
targetMessageId?: string;
|
||||
isPublic: boolean;
|
||||
expiredAt?: Date;
|
||||
createdAt?: Date;
|
||||
updatedAt?: Date;
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue