Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2026-03-06 00:00:18 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 9

Merge branch 'dev' into feat/prompt-enhancement

Browse source
This commit is contained in:
Marco Beretta 2025-06-23 14:48:46 +02:00 • committed by GitHub
commit 3d261a969d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
365 changed files with 23826 additions and 8790 deletions
Download patch file Download diff file Expand all files Collapse all files

114
packages/data-schemas/README.md
View file

@ -1,114 +0,0 @@
# `@librechat/data-schemas`
Mongoose schemas and models for LibreChat. This package provides a comprehensive collection of Mongoose schemas used across the LibreChat project, enabling robust data modeling and validation for various entities such as actions, agents, messages, users, and more.
## Features
- **Modular Schemas:** Includes schemas for actions, agents, assistants, balance, banners, categories, conversation tags, conversations, files, keys, messages, plugin authentication, presets, projects, prompts, prompt groups, roles, sessions, shared links, tokens, tool calls, transactions, and users.
- **TypeScript Support:** Provides TypeScript definitions for type-safe development.
- **Ready for Mongoose Integration:** Easily integrate with Mongoose to create models and interact with your MongoDB database.
- **Flexible & Extensible:** Designed to support the evolving needs of LibreChat while being adaptable to other projects.
## Installation
Install the package via npm or yarn:
```bash
npm install @librechat/data-schemas
```
Or with yarn:
```bash
yarn add @librechat/data-schemas
```
## Usage
After installation, you can import and use the schemas in your project. For example, to create a Mongoose model for a user:
```js
import mongoose from 'mongoose';
import { userSchema } from '@librechat/data-schemas';
const UserModel = mongoose.model('User', userSchema);
// Now you can use UserModel to create, read, update, and delete user documents.
```
You can also import other schemas as needed:
```js
import { actionSchema, agentSchema, messageSchema } from '@librechat/data-schemas';
```
Each schema is designed to integrate seamlessly with Mongoose and provides indexes, timestamps, and validations tailored for LibreChat’s use cases.
## Development
This package uses Rollup and TypeScript for building and bundling.
### Available Scripts
- **Build:**
Cleans the `dist` directory and builds the package.
```bash
npm run build
```
- **Build Watch:**
Rebuilds automatically on file changes.
```bash
npm run build:watch
```
- **Test:**
Runs tests with coverage in watch mode.
```bash
npm run test
```
- **Test (CI):**
Runs tests with coverage for CI environments.
```bash
npm run test:ci
```
- **Verify:**
Runs tests in CI mode to verify code integrity.
```bash
npm run verify
```
- **Clean:**
Removes the `dist` directory.
```bash
npm run clean
```
For those using Bun, equivalent scripts are available:
- **Bun Clean:** `bun run b:clean`
- **Bun Build:** `bun run b:build`
## Repository & Issues
The source code is maintained on GitHub.
- **Repository:** [LibreChat Repository](https://github.com/danny-avila/LibreChat.git)
- **Issues & Bug Reports:** [LibreChat Issues](https://github.com/danny-avila/LibreChat/issues)
## License
This project is licensed under the [MIT License](LICENSE).
## Contributing
Contributions to improve and expand the data schemas are welcome. If you have suggestions, improvements, or bug fixes, please open an issue or submit a pull request on the [GitHub repository](https://github.com/danny-avila/LibreChat/issues).
For more detailed documentation on each schema and model, please refer to the source code or visit the [LibreChat website](https://librechat.ai).

3
packages/data-schemas/jest.config.mjs
View file

@ -5,6 +5,7 @@ export default {
testResultsProcessor: 'jest-junit',
moduleNameMapper: {
'^@src/(.*)$': '<rootDir>/src/$1',
'^~/(.*)$': '<rootDir>/src/$1',
},
// coverageThreshold: {
// global: {
@ -16,4 +17,4 @@ export default {
// },
restoreMocks: true,
testTimeout: 15000,
};
};

10
packages/data-schemas/package.json
View file

@ -1,6 +1,6 @@
{
"name": "@librechat/data-schemas",
"version": "0.0.7",
"version": "0.0.10",
"description": "Mongoose schemas and models for LibreChat",
"type": "module",
"main": "dist/index.cjs",
@ -51,6 +51,7 @@
"@types/traverse": "^0.6.37",
"jest": "^29.5.0",
"jest-junit": "^16.0.0",
"mongodb-memory-server": "^10.1.4",
"rimraf": "^5.0.1",
"rollup": "^4.22.4",
"rollup-plugin-generate-package-json": "^3.2.0",
@ -60,13 +61,14 @@
"typescript": "^5.0.4"
},
"peerDependencies": {
"keyv": "^5.3.2",
"mongoose": "^8.12.1",
"librechat-data-provider": "*",
"jsonwebtoken": "^9.0.2",
"keyv": "^5.3.2",
"klona": "^2.0.6",
"librechat-data-provider": "*",
"lodash": "^4.17.21",
"meilisearch": "^0.38.0",
"mongoose": "^8.12.1",
"nanoid": "^3.3.7",
"traverse": "^0.6.11",
"winston": "^3.17.0",
"winston-daily-rotate-file": "^5.0.0"

2
packages/data-schemas/src/config/meiliLogger.ts
View file

@ -2,7 +2,7 @@ import path from 'path';
import winston from 'winston';
import 'winston-daily-rotate-file';
const logDir = path.join(__dirname, '..', 'logs');
const logDir = path.join(__dirname, '..', '..', '..', 'api', 'logs');
const { NODE_ENV, DEBUG_LOGGING = 'false' } = process.env;

5
packages/data-schemas/src/config/winston.ts
View file

@ -3,10 +3,8 @@ import winston from 'winston';
import 'winston-daily-rotate-file';
import { redactFormat, redactMessage, debugTraverse, jsonTruncateFormat } from './parsers';
// Define log directory
const logDir = path.join(__dirname, '..', 'logs');
const logDir = path.join(__dirname, '..', '..', '..', 'api', 'logs');
// Type-safe environment variables
const { NODE_ENV, DEBUG_LOGGING, CONSOLE_JSON, DEBUG_CONSOLE } = process.env;
const useConsoleJson = typeof CONSOLE_JSON === 'string' && CONSOLE_JSON.toLowerCase() === 'true';
@ -15,7 +13,6 @@ const useDebugConsole = typeof DEBUG_CONSOLE === 'string' && DEBUG_CONSOLE.toLow
const useDebugLogging = typeof DEBUG_LOGGING === 'string' && DEBUG_LOGGING.toLowerCase() === 'true';
// Define custom log levels
const levels: winston.config.AbstractConfigSetLevels = {
error: 0,
warn: 1,

1
packages/data-schemas/src/index.ts
View file

@ -3,5 +3,6 @@ export * from './schema';
export { createModels } from './models';
export { createMethods } from './methods';
export type * from './types';
export type * from './methods';
export { default as logger } from './config/winston';
export { default as meiliLogger } from './config/meiliLogger';

16
packages/data-schemas/src/methods/index.ts
View file

@ -2,6 +2,10 @@ import { createUserMethods, type UserMethods } from './user';
import { createSessionMethods, type SessionMethods } from './session';
import { createTokenMethods, type TokenMethods } from './token';
import { createRoleMethods, type RoleMethods } from './role';
/* Memories */
import { createMemoryMethods, type MemoryMethods } from './memory';
import { createShareMethods, type ShareMethods } from './share';
import { createPluginAuthMethods, type PluginAuthMethods } from './pluginAuth';
/**
* Creates all database methods for all collections
@ -12,7 +16,17 @@ export function createMethods(mongoose: typeof import('mongoose')) {
...createSessionMethods(mongoose),
...createTokenMethods(mongoose),
...createRoleMethods(mongoose),
...createMemoryMethods(mongoose),
...createShareMethods(mongoose),
...createPluginAuthMethods(mongoose),
};
}
export type AllMethods = UserMethods & SessionMethods & TokenMethods & RoleMethods;
export type { MemoryMethods, ShareMethods, TokenMethods, PluginAuthMethods };
export type AllMethods = UserMethods &
SessionMethods &
TokenMethods &
RoleMethods &
MemoryMethods &
ShareMethods &
PluginAuthMethods;

168
packages/data-schemas/src/methods/memory.ts Normal file
View file

@ -0,0 +1,168 @@
import { Types } from 'mongoose';
import logger from '~/config/winston';
import type * as t from '~/types';
/**
* Formats a date in YYYY-MM-DD format
*/
const formatDate = (date: Date): string => {
return date.toISOString().split('T')[0];
};
// Factory function that takes mongoose instance and returns the methods
export function createMemoryMethods(mongoose: typeof import('mongoose')) {
const MemoryEntry = mongoose.models.MemoryEntry;
/**
* Creates a new memory entry for a user
* Throws an error if a memory with the same key already exists
*/
async function createMemory({
userId,
key,
value,
tokenCount = 0,
}: t.SetMemoryParams): Promise<t.MemoryResult> {
try {
if (key?.toLowerCase() === 'nothing') {
return { ok: false };
}
const existingMemory = await MemoryEntry.findOne({ userId, key });
if (existingMemory) {
throw new Error('Memory with this key already exists');
}
await MemoryEntry.create({
userId,
key,
value,
tokenCount,
updated_at: new Date(),
});
return { ok: true };
} catch (error) {
throw new Error(
`Failed to create memory: ${error instanceof Error ? error.message : 'Unknown error'}`,
);
}
}
/**
* Sets or updates a memory entry for a user
*/
async function setMemory({
userId,
key,
value,
tokenCount = 0,
}: t.SetMemoryParams): Promise<t.MemoryResult> {
try {
if (key?.toLowerCase() === 'nothing') {
return { ok: false };
}
await MemoryEntry.findOneAndUpdate(
{ userId, key },
{
value,
tokenCount,
updated_at: new Date(),
},
{
upsert: true,
new: true,
},
);
return { ok: true };
} catch (error) {
throw new Error(
`Failed to set memory: ${error instanceof Error ? error.message : 'Unknown error'}`,
);
}
}
/**
* Deletes a specific memory entry for a user
*/
async function deleteMemory({ userId, key }: t.DeleteMemoryParams): Promise<t.MemoryResult> {
try {
const result = await MemoryEntry.findOneAndDelete({ userId, key });
return { ok: !!result };
} catch (error) {
throw new Error(
`Failed to delete memory: ${error instanceof Error ? error.message : 'Unknown error'}`,
);
}
}
/**
* Gets all memory entries for a user
*/
async function getAllUserMemories(
userId: string | Types.ObjectId,
): Promise<t.IMemoryEntryLean[]> {
try {
return (await MemoryEntry.find({ userId }).lean()) as t.IMemoryEntryLean[];
} catch (error) {
throw new Error(
`Failed to get all memories: ${error instanceof Error ? error.message : 'Unknown error'}`,
);
}
}
/**
* Gets and formats all memories for a user in two different formats
*/
async function getFormattedMemories({
userId,
}: t.GetFormattedMemoriesParams): Promise<t.FormattedMemoriesResult> {
try {
const memories = await getAllUserMemories(userId);
if (!memories || memories.length === 0) {
return { withKeys: '', withoutKeys: '', totalTokens: 0 };
}
const sortedMemories = memories.sort(
(a, b) => new Date(a.updated_at!).getTime() - new Date(b.updated_at!).getTime(),
);
const totalTokens = sortedMemories.reduce((sum, memory) => {
return sum + (memory.tokenCount || 0);
}, 0);
const withKeys = sortedMemories
.map((memory, index) => {
const date = formatDate(new Date(memory.updated_at!));
const tokenInfo = memory.tokenCount ? ` [${memory.tokenCount} tokens]` : '';
return `${index + 1}. [${date}]. ["key": "${memory.key}"]${tokenInfo}. ["value": "${memory.value}"]`;
})
.join('\n\n');
const withoutKeys = sortedMemories
.map((memory, index) => {
const date = formatDate(new Date(memory.updated_at!));
return `${index + 1}. [${date}]. ${memory.value}`;
})
.join('\n\n');
return { withKeys, withoutKeys, totalTokens };
} catch (error) {
logger.error('Failed to get formatted memories:', error);
return { withKeys: '', withoutKeys: '', totalTokens: 0 };
}
}
return {
setMemory,
createMemory,
deleteMemory,
getAllUserMemories,
getFormattedMemories,
};
}
export type MemoryMethods = ReturnType<typeof createMemoryMethods>;

140
packages/data-schemas/src/methods/pluginAuth.ts Normal file
View file

@ -0,0 +1,140 @@
import type { DeleteResult, Model } from 'mongoose';
import type { IPluginAuth } from '~/schema/pluginAuth';
import type {
FindPluginAuthsByKeysParams,
UpdatePluginAuthParams,
DeletePluginAuthParams,
FindPluginAuthParams,
} from '~/types';
// Factory function that takes mongoose instance and returns the methods
export function createPluginAuthMethods(mongoose: typeof import('mongoose')) {
const PluginAuth: Model<IPluginAuth> = mongoose.models.PluginAuth;
/**
* Finds a single plugin auth entry by userId and authField
*/
async function findOnePluginAuth({
userId,
authField,
}: FindPluginAuthParams): Promise<IPluginAuth | null> {
try {
return await PluginAuth.findOne({ userId, authField }).lean();
} catch (error) {
throw new Error(
`Failed to find plugin auth: ${error instanceof Error ? error.message : 'Unknown error'}`,
);
}
}
/**
* Finds multiple plugin auth entries by userId and pluginKeys
*/
async function findPluginAuthsByKeys({
userId,
pluginKeys,
}: FindPluginAuthsByKeysParams): Promise<IPluginAuth[]> {
try {
if (!pluginKeys || pluginKeys.length === 0) {
return [];
}
return await PluginAuth.find({
userId,
pluginKey: { $in: pluginKeys },
}).lean();
} catch (error) {
throw new Error(
`Failed to find plugin auths: ${error instanceof Error ? error.message : 'Unknown error'}`,
);
}
}
/**
* Updates or creates a plugin auth entry
*/
async function updatePluginAuth({
userId,
authField,
pluginKey,
value,
}: UpdatePluginAuthParams): Promise<IPluginAuth> {
try {
const existingAuth = await PluginAuth.findOne({ userId, pluginKey, authField }).lean();
if (existingAuth) {
return await PluginAuth.findOneAndUpdate(
{ userId, pluginKey, authField },
{ $set: { value } },
{ new: true, upsert: true },
).lean();
} else {
const newPluginAuth = await new PluginAuth({
userId,
authField,
value,
pluginKey,
});
await newPluginAuth.save();
return newPluginAuth.toObject();
}
} catch (error) {
throw new Error(
`Failed to update plugin auth: ${error instanceof Error ? error.message : 'Unknown error'}`,
);
}
}
/**
* Deletes plugin auth entries based on provided parameters
*/
async function deletePluginAuth({
userId,
authField,
pluginKey,
all = false,
}: DeletePluginAuthParams): Promise<DeleteResult> {
try {
if (all) {
const filter: DeletePluginAuthParams = { userId };
if (pluginKey) {
filter.pluginKey = pluginKey;
}
return await PluginAuth.deleteMany(filter);
}
if (!authField) {
throw new Error('authField is required when all is false');
}
return await PluginAuth.deleteOne({ userId, authField });
} catch (error) {
throw new Error(
`Failed to delete plugin auth: ${error instanceof Error ? error.message : 'Unknown error'}`,
);
}
}
/**
* Deletes all plugin auth entries for a user
*/
async function deleteAllUserPluginAuths(userId: string): Promise<DeleteResult> {
try {
return await PluginAuth.deleteMany({ userId });
} catch (error) {
throw new Error(
`Failed to delete all user plugin auths: ${error instanceof Error ? error.message : 'Unknown error'}`,
);
}
}
return {
findOnePluginAuth,
findPluginAuthsByKeys,
updatePluginAuth,
deletePluginAuth,
deleteAllUserPluginAuths,
};
}
export type PluginAuthMethods = ReturnType<typeof createPluginAuthMethods>;

4
packages/data-schemas/src/methods/session.ts
View file

@ -13,7 +13,9 @@ export class SessionError extends Error {
}
const { REFRESH_TOKEN_EXPIRY } = process.env ?? {};
const expires = eval(REFRESH_TOKEN_EXPIRY ?? '0') ?? 1000 * 60 * 60 * 24 * 7; // 7 days default
const expires = REFRESH_TOKEN_EXPIRY
? eval(REFRESH_TOKEN_EXPIRY)
: 1000 * 60 * 60 * 24 * 7; // 7 days default
// Factory function that takes mongoose instance and returns the methods
export function createSessionMethods(mongoose: typeof import('mongoose')) {

1043
packages/data-schemas/src/methods/share.test.ts Normal file
View file

File diff suppressed because it is too large Load diff

442
packages/data-schemas/src/methods/share.ts Normal file
View file

@ -0,0 +1,442 @@
import { nanoid } from 'nanoid';
import { Constants } from 'librechat-data-provider';
import type { FilterQuery, Model } from 'mongoose';
import type { SchemaWithMeiliMethods } from '~/models/plugins/mongoMeili';
import type * as t from '~/types';
import logger from '~/config/winston';
class ShareServiceError extends Error {
code: string;
constructor(message: string, code: string) {
super(message);
this.name = 'ShareServiceError';
this.code = code;
}
}
function memoizedAnonymizeId(prefix: string) {
const memo = new Map<string, string>();
return (id: string) => {
if (!memo.has(id)) {
memo.set(id, `${prefix}_${nanoid()}`);
}
return memo.get(id) as string;
};
}
const anonymizeConvoId = memoizedAnonymizeId('convo');
const anonymizeAssistantId = memoizedAnonymizeId('a');
const anonymizeMessageId = (id: string) =>
id === Constants.NO_PARENT ? id : memoizedAnonymizeId('msg')(id);
function anonymizeConvo(conversation: Partial<t.IConversation> & Partial<t.ISharedLink>) {
if (!conversation) {
return null;
}
const newConvo = { ...conversation };
if (newConvo.assistant_id) {
newConvo.assistant_id = anonymizeAssistantId(newConvo.assistant_id);
}
return newConvo;
}
function anonymizeMessages(messages: t.IMessage[], newConvoId: string): t.IMessage[] {
if (!Array.isArray(messages)) {
return [];
}
const idMap = new Map<string, string>();
return messages.map((message) => {
const newMessageId = anonymizeMessageId(message.messageId);
idMap.set(message.messageId, newMessageId);
type MessageAttachment = {
messageId?: string;
conversationId?: string;
[key: string]: unknown;
};
const anonymizedAttachments = (message.attachments as MessageAttachment[])?.map(
(attachment) => {
return {
...attachment,
messageId: newMessageId,
conversationId: newConvoId,
};
},
);
return {
...message,
messageId: newMessageId,
parentMessageId:
idMap.get(message.parentMessageId || '') ||
anonymizeMessageId(message.parentMessageId || ''),
conversationId: newConvoId,
model: message.model?.startsWith('asst_')
? anonymizeAssistantId(message.model)
: message.model,
attachments: anonymizedAttachments,
} as t.IMessage;
});
}
/** Factory function that takes mongoose instance and returns the methods */
export function createShareMethods(mongoose: typeof import('mongoose')) {
/**
* Get shared messages for a public share link
*/
async function getSharedMessages(shareId: string): Promise<t.SharedMessagesResult | null> {
try {
const SharedLink = mongoose.models.SharedLink as Model<t.ISharedLink>;
const share = (await SharedLink.findOne({ shareId, isPublic: true })
.populate({
path: 'messages',
select: '-_id -__v -user',
})
.select('-_id -__v -user')
.lean()) as (t.ISharedLink & { messages: t.IMessage[] }) | null;
if (!share?.conversationId || !share.isPublic) {
return null;
}
const newConvoId = anonymizeConvoId(share.conversationId);
const result: t.SharedMessagesResult = {
shareId: share.shareId || shareId,
title: share.title,
isPublic: share.isPublic,
createdAt: share.createdAt,
updatedAt: share.updatedAt,
conversationId: newConvoId,
messages: anonymizeMessages(share.messages, newConvoId),
};
return result;
} catch (error) {
logger.error('[getSharedMessages] Error getting share link', {
error: error instanceof Error ? error.message : 'Unknown error',
shareId,
});
throw new ShareServiceError('Error getting share link', 'SHARE_FETCH_ERROR');
}
}
/**
* Get shared links for a specific user with pagination and search
*/
async function getSharedLinks(
user: string,
pageParam?: Date,
pageSize: number = 10,
isPublic: boolean = true,
sortBy: string = 'createdAt',
sortDirection: string = 'desc',
search?: string,
): Promise<t.SharedLinksResult> {
try {
const SharedLink = mongoose.models.SharedLink as Model<t.ISharedLink>;
const Conversation = mongoose.models.Conversation as SchemaWithMeiliMethods;
const query: FilterQuery<t.ISharedLink> = { user, isPublic };
if (pageParam) {
if (sortDirection === 'desc') {
query[sortBy] = { $lt: pageParam };
} else {
query[sortBy] = { $gt: pageParam };
}
}
if (search && search.trim()) {
try {
const searchResults = await Conversation.meiliSearch(search);
if (!searchResults?.hits?.length) {
return {
links: [],
nextCursor: undefined,
hasNextPage: false,
};
}
const conversationIds = searchResults.hits.map((hit) => hit.conversationId);
query['conversationId'] = { $in: conversationIds };
} catch (searchError) {
logger.error('[getSharedLinks] Meilisearch error', {
error: searchError instanceof Error ? searchError.message : 'Unknown error',
user,
});
return {
links: [],
nextCursor: undefined,
hasNextPage: false,
};
}
}
const sort: Record<string, 1 | -1> = {};
sort[sortBy] = sortDirection === 'desc' ? -1 : 1;
const sharedLinks = await SharedLink.find(query)
.sort(sort)
.limit(pageSize + 1)
.select('-__v -user')
.lean();
const hasNextPage = sharedLinks.length > pageSize;
const links = sharedLinks.slice(0, pageSize);
const nextCursor = hasNextPage
? (links[links.length - 1][sortBy as keyof t.ISharedLink] as Date)
: undefined;
return {
links: links.map((link) => ({
shareId: link.shareId || '',
title: link?.title || 'Untitled',
isPublic: link.isPublic,
createdAt: link.createdAt || new Date(),
conversationId: link.conversationId,
})),
nextCursor,
hasNextPage,
};
} catch (error) {
logger.error('[getSharedLinks] Error getting shares', {
error: error instanceof Error ? error.message : 'Unknown error',
user,
});
throw new ShareServiceError('Error getting shares', 'SHARES_FETCH_ERROR');
}
}
/**
* Delete all shared links for a user
*/
async function deleteAllSharedLinks(user: string): Promise<t.DeleteAllSharesResult> {
try {
const SharedLink = mongoose.models.SharedLink as Model<t.ISharedLink>;
const result = await SharedLink.deleteMany({ user });
return {
message: 'All shared links deleted successfully',
deletedCount: result.deletedCount,
};
} catch (error) {
logger.error('[deleteAllSharedLinks] Error deleting shared links', {
error: error instanceof Error ? error.message : 'Unknown error',
user,
});
throw new ShareServiceError('Error deleting shared links', 'BULK_DELETE_ERROR');
}
}
/**
* Create a new shared link for a conversation
*/
async function createSharedLink(
user: string,
conversationId: string,
): Promise<t.CreateShareResult> {
if (!user || !conversationId) {
throw new ShareServiceError('Missing required parameters', 'INVALID_PARAMS');
}
try {
const Message = mongoose.models.Message as SchemaWithMeiliMethods;
const SharedLink = mongoose.models.SharedLink as Model<t.ISharedLink>;
const Conversation = mongoose.models.Conversation as SchemaWithMeiliMethods;
const [existingShare, conversationMessages] = await Promise.all([
SharedLink.findOne({ conversationId, user, isPublic: true })
.select('-_id -__v -user')
.lean() as Promise<t.ISharedLink | null>,
Message.find({ conversationId, user }).sort({ createdAt: 1 }).lean(),
]);
if (existingShare && existingShare.isPublic) {
logger.error('[createSharedLink] Share already exists', {
user,
conversationId,
});
throw new ShareServiceError('Share already exists', 'SHARE_EXISTS');
} else if (existingShare) {
await SharedLink.deleteOne({ conversationId, user });
}
const conversation = (await Conversation.findOne({ conversationId, user }).lean()) as {
title?: string;
} | null;
// Check if user owns the conversation
if (!conversation) {
throw new ShareServiceError(
'Conversation not found or access denied',
'CONVERSATION_NOT_FOUND',
);
}
// Check if there are any messages to share
if (!conversationMessages || conversationMessages.length === 0) {
throw new ShareServiceError('No messages to share', 'NO_MESSAGES');
}
const title = conversation.title || 'Untitled';
const shareId = nanoid();
await SharedLink.create({
shareId,
conversationId,
messages: conversationMessages,
title,
user,
});
return { shareId, conversationId };
} catch (error) {
if (error instanceof ShareServiceError) {
throw error;
}
logger.error('[createSharedLink] Error creating shared link', {
error: error instanceof Error ? error.message : 'Unknown error',
user,
conversationId,
});
throw new ShareServiceError('Error creating shared link', 'SHARE_CREATE_ERROR');
}
}
/**
* Get a shared link for a conversation
*/
async function getSharedLink(
user: string,
conversationId: string,
): Promise<t.GetShareLinkResult> {
if (!user || !conversationId) {
throw new ShareServiceError('Missing required parameters', 'INVALID_PARAMS');
}
try {
const SharedLink = mongoose.models.SharedLink as Model<t.ISharedLink>;
const share = (await SharedLink.findOne({ conversationId, user, isPublic: true })
.select('shareId -_id')
.lean()) as { shareId?: string } | null;
if (!share) {
return { shareId: null, success: false };
}
return { shareId: share.shareId || null, success: true };
} catch (error) {
logger.error('[getSharedLink] Error getting shared link', {
error: error instanceof Error ? error.message : 'Unknown error',
user,
conversationId,
});
throw new ShareServiceError('Error getting shared link', 'SHARE_FETCH_ERROR');
}
}
/**
* Update a shared link with new messages
*/
async function updateSharedLink(user: string, shareId: string): Promise<t.UpdateShareResult> {
if (!user || !shareId) {
throw new ShareServiceError('Missing required parameters', 'INVALID_PARAMS');
}
try {
const SharedLink = mongoose.models.SharedLink as Model<t.ISharedLink>;
const Message = mongoose.models.Message as SchemaWithMeiliMethods;
const share = (await SharedLink.findOne({ shareId, user })
.select('-_id -__v -user')
.lean()) as t.ISharedLink | null;
if (!share) {
throw new ShareServiceError('Share not found', 'SHARE_NOT_FOUND');
}
const updatedMessages = await Message.find({ conversationId: share.conversationId, user })
.sort({ createdAt: 1 })
.lean();
const newShareId = nanoid();
const update = {
messages: updatedMessages,
user,
shareId: newShareId,
};
const updatedShare = (await SharedLink.findOneAndUpdate({ shareId, user }, update, {
new: true,
upsert: false,
runValidators: true,
}).lean()) as t.ISharedLink | null;
if (!updatedShare) {
throw new ShareServiceError('Share update failed', 'SHARE_UPDATE_ERROR');
}
anonymizeConvo(updatedShare);
return { shareId: newShareId, conversationId: updatedShare.conversationId };
} catch (error) {
logger.error('[updateSharedLink] Error updating shared link', {
error: error instanceof Error ? error.message : 'Unknown error',
user,
shareId,
});
throw new ShareServiceError(
error instanceof ShareServiceError ? error.message : 'Error updating shared link',
error instanceof ShareServiceError ? error.code : 'SHARE_UPDATE_ERROR',
);
}
}
/**
* Delete a shared link
*/
async function deleteSharedLink(
user: string,
shareId: string,
): Promise<t.DeleteShareResult | null> {
if (!user || !shareId) {
throw new ShareServiceError('Missing required parameters', 'INVALID_PARAMS');
}
try {
const SharedLink = mongoose.models.SharedLink as Model<t.ISharedLink>;
const result = await SharedLink.findOneAndDelete({ shareId, user }).lean();
if (!result) {
return null;
}
return {
success: true,
shareId,
message: 'Share deleted successfully',
};
} catch (error) {
logger.error('[deleteSharedLink] Error deleting shared link', {
error: error instanceof Error ? error.message : 'Unknown error',
user,
shareId,
});
throw new ShareServiceError('Error deleting shared link', 'SHARE_DELETE_ERROR');
}
}
// Return all methods
return {
getSharedLink,
getSharedLinks,
createSharedLink,
updateSharedLink,
deleteSharedLink,
getSharedMessages,
deleteAllSharedLinks,
};
}
export type ShareMethods = ReturnType<typeof createShareMethods>;

163
packages/data-schemas/src/methods/user.test.ts Normal file
View file

@ -0,0 +1,163 @@
import mongoose from 'mongoose';
import { createUserMethods } from './user';
import { signPayload } from '~/crypto';
import type { IUser } from '~/types';
jest.mock('~/crypto', () => ({
signPayload: jest.fn(),
}));
describe('User Methods', () => {
const mockSignPayload = signPayload as jest.MockedFunction<typeof signPayload>;
let userMethods: ReturnType<typeof createUserMethods>;
beforeEach(() => {
jest.clearAllMocks();
userMethods = createUserMethods(mongoose);
});
describe('generateToken', () => {
const mockUser = {
_id: 'user123',
username: 'testuser',
provider: 'local',
email: 'test@example.com',
name: 'Test User',
avatar: '',
role: 'user',
emailVerified: false,
createdAt: new Date(),
updatedAt: new Date(),
} as IUser;
afterEach(() => {
delete process.env.SESSION_EXPIRY;
delete process.env.JWT_SECRET;
});
it('should default to 15 minutes when SESSION_EXPIRY is not set', async () => {
process.env.JWT_SECRET = 'test-secret';
mockSignPayload.mockResolvedValue('mocked-token');
await userMethods.generateToken(mockUser);
expect(mockSignPayload).toHaveBeenCalledWith({
payload: {
id: mockUser._id,
username: mockUser.username,
provider: mockUser.provider,
email: mockUser.email,
},
secret: 'test-secret',
expirationTime: 900, // 15 minutes in seconds
});
});
it('should default to 15 minutes when SESSION_EXPIRY is empty string', async () => {
process.env.SESSION_EXPIRY = '';
process.env.JWT_SECRET = 'test-secret';
mockSignPayload.mockResolvedValue('mocked-token');
await userMethods.generateToken(mockUser);
expect(mockSignPayload).toHaveBeenCalledWith({
payload: {
id: mockUser._id,
username: mockUser.username,
provider: mockUser.provider,
email: mockUser.email,
},
secret: 'test-secret',
expirationTime: 900, // 15 minutes in seconds
});
});
it('should use custom expiry when SESSION_EXPIRY is set to a valid expression', async () => {
process.env.SESSION_EXPIRY = '1000 * 60 * 30'; // 30 minutes
process.env.JWT_SECRET = 'test-secret';
mockSignPayload.mockResolvedValue('mocked-token');
await userMethods.generateToken(mockUser);
expect(mockSignPayload).toHaveBeenCalledWith({
payload: {
id: mockUser._id,
username: mockUser.username,
provider: mockUser.provider,
email: mockUser.email,
},
secret: 'test-secret',
expirationTime: 1800, // 30 minutes in seconds
});
});
it('should default to 15 minutes when SESSION_EXPIRY evaluates to falsy value', async () => {
process.env.SESSION_EXPIRY = '0'; // This will evaluate to 0, which is falsy
process.env.JWT_SECRET = 'test-secret';
mockSignPayload.mockResolvedValue('mocked-token');
await userMethods.generateToken(mockUser);
expect(mockSignPayload).toHaveBeenCalledWith({
payload: {
id: mockUser._id,
username: mockUser.username,
provider: mockUser.provider,
email: mockUser.email,
},
secret: 'test-secret',
expirationTime: 900, // 15 minutes in seconds
});
});
it('should throw error when no user is provided', async () => {
process.env.JWT_SECRET = 'test-secret';
await expect(userMethods.generateToken(null as unknown as IUser)).rejects.toThrow(
'No user provided',
);
});
it('should return the token from signPayload', async () => {
process.env.SESSION_EXPIRY = '1000 * 60 * 60'; // 1 hour
process.env.JWT_SECRET = 'test-secret';
const expectedToken = 'generated-jwt-token';
mockSignPayload.mockResolvedValue(expectedToken);
const token = await userMethods.generateToken(mockUser);
expect(token).toBe(expectedToken);
});
it('should handle invalid SESSION_EXPIRY expressions gracefully', async () => {
process.env.SESSION_EXPIRY = 'invalid expression';
process.env.JWT_SECRET = 'test-secret';
mockSignPayload.mockResolvedValue('mocked-token');
// Mock console.warn to verify it's called
const consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation();
await userMethods.generateToken(mockUser);
// Should use default value when eval fails
expect(mockSignPayload).toHaveBeenCalledWith({
payload: {
id: mockUser._id,
username: mockUser.username,
provider: mockUser.provider,
email: mockUser.email,
},
secret: 'test-secret',
expirationTime: 900, // 15 minutes in seconds (default)
});
// Verify warning was logged
expect(consoleWarnSpy).toHaveBeenCalledWith(
'Invalid SESSION_EXPIRY expression, using default:',
expect.any(SyntaxError),
);
consoleWarnSpy.mockRestore();
});
});
});

43
packages/data-schemas/src/methods/user.ts
View file

@ -145,7 +145,18 @@ export function createUserMethods(mongoose: typeof import('mongoose')) {
throw new Error('No user provided');
}
const expires = eval(process.env.SESSION_EXPIRY ?? '0') ?? 1000 * 60 * 15;
let expires = 1000 * 60 * 15;
if (process.env.SESSION_EXPIRY !== undefined && process.env.SESSION_EXPIRY !== '') {
try {
const evaluated = eval(process.env.SESSION_EXPIRY);
if (evaluated) {
expires = evaluated;
}
} catch (error) {
console.warn('Invalid SESSION_EXPIRY expression, using default:', error);
}
}
return await signPayload({
payload: {
@ -159,6 +170,35 @@ export function createUserMethods(mongoose: typeof import('mongoose')) {
});
}
/**
* Update a user's personalization memories setting.
* Handles the edge case where the personalization object doesn't exist.
*/
async function toggleUserMemories(
userId: string,
memoriesEnabled: boolean,
): Promise<IUser | null> {
const User = mongoose.models.User;
// First, ensure the personalization object exists
const user = await User.findById(userId);
if (!user) {
return null;
}
// Use $set to update the nested field, which will create the personalization object if it doesn't exist
const updateOperation = {
$set: {
'personalization.memories': memoriesEnabled,
},
};
return (await User.findByIdAndUpdate(userId, updateOperation, {
new: true,
runValidators: true,
}).lean()) as IUser | null;
}
// Return all methods
return {
findUser,
@ -168,6 +208,7 @@ export function createUserMethods(mongoose: typeof import('mongoose')) {
getUserById,
deleteUserById,
generateToken,
toggleUserMemories,
};
}

11
packages/data-schemas/src/models/convo.ts
View file

@ -1,10 +1,21 @@
import type * as t from '~/types';
import mongoMeili from '~/models/plugins/mongoMeili';
import convoSchema from '~/schema/convo';
/**
* Creates or returns the Conversation model using the provided mongoose instance and schema
*/
export function createConversationModel(mongoose: typeof import('mongoose')) {
if (process.env.MEILI_HOST && process.env.MEILI_MASTER_KEY) {
convoSchema.plugin(mongoMeili, {
mongoose,
host: process.env.MEILI_HOST,
apiKey: process.env.MEILI_MASTER_KEY,
/** Note: Will get created automatically if it doesn't exist already */
indexName: 'convos',
primaryKey: 'conversationId',
});
}
return (
mongoose.models.Conversation || mongoose.model<t.IConversation>('Conversation', convoSchema)
);

2
packages/data-schemas/src/models/index.ts
View file

@ -20,6 +20,7 @@ import { createPromptGroupModel } from './promptGroup';
import { createConversationTagModel } from './conversationTag';
import { createSharedLinkModel } from './sharedLink';
import { createToolCallModel } from './toolCall';
import { createMemoryModel } from './memory';
/**
* Creates all database models for all collections
@ -48,5 +49,6 @@ export function createModels(mongoose: typeof import('mongoose')) {
ConversationTag: createConversationTagModel(mongoose),
SharedLink: createSharedLinkModel(mongoose),
ToolCall: createToolCallModel(mongoose),
MemoryEntry: createMemoryModel(mongoose),
};
}

6
packages/data-schemas/src/models/memory.ts Normal file
View file

@ -0,0 +1,6 @@
import memorySchema from '~/schema/memory';
import type { IMemoryEntry } from '~/types/memory';
export function createMemoryModel(mongoose: typeof import('mongoose')) {
return mongoose.models.MemoryEntry || mongoose.model<IMemoryEntry>('MemoryEntry', memorySchema);
}

13
packages/data-schemas/src/models/message.ts
View file

@ -1,9 +1,20 @@
import messageSchema from '~/schema/message';
import type * as t from '~/types';
import mongoMeili from '~/models/plugins/mongoMeili';
import messageSchema from '~/schema/message';
/**
* Creates or returns the Message model using the provided mongoose instance and schema
*/
export function createMessageModel(mongoose: typeof import('mongoose')) {
if (process.env.MEILI_HOST && process.env.MEILI_MASTER_KEY) {
messageSchema.plugin(mongoMeili, {
mongoose,
host: process.env.MEILI_HOST,
apiKey: process.env.MEILI_MASTER_KEY,
indexName: 'messages',
primaryKey: 'messageId',
});
}
return mongoose.models.Message || mongoose.model<t.IMessage>('Message', messageSchema);
}

551
packages/data-schemas/src/models/plugins/mongoMeili.ts
View file

@ -1,6 +1,16 @@
import _ from 'lodash';
import { MeiliSearch, Index } from 'meilisearch';
import mongoose, { Schema, Document, Model, Query } from 'mongoose';
import { MeiliSearch } from 'meilisearch';
import type { SearchResponse, Index } from 'meilisearch';
import type {
CallbackWithoutResultAndOptionalError,
FilterQuery,
Document,
Schema,
Query,
Types,
Model,
} from 'mongoose';
import type { IConversation, IMessage } from '~/types';
import logger from '~/config/meiliLogger';
interface MongoMeiliOptions {
@ -8,6 +18,9 @@ interface MongoMeiliOptions {
apiKey: string;
indexName: string;
primaryKey: string;
mongoose: typeof import('mongoose');
syncBatchSize?: number;
syncDelayMs?: number;
}
interface MeiliIndexable {
@ -20,28 +33,51 @@ interface ContentItem {
text?: string;
}
interface DocumentWithMeiliIndex extends Document {
_meiliIndex?: boolean;
preprocessObjectForIndex?: () => Record<string, unknown>;
addObjectToMeili?: () => Promise<void>;
updateObjectToMeili?: () => Promise<void>;
deleteObjectFromMeili?: () => Promise<void>;
postSaveHook?: () => void;
postUpdateHook?: () => void;
postRemoveHook?: () => void;
conversationId?: string;
content?: ContentItem[];
messageId?: string;
unfinished?: boolean;
messages?: unknown[];
title?: string;
toJSON(): Record<string, unknown>;
interface SyncProgress {
lastSyncedId?: string;
totalProcessed: number;
totalDocuments: number;
isComplete: boolean;
}
interface SchemaWithMeiliMethods extends Model<DocumentWithMeiliIndex> {
syncWithMeili(): Promise<void>;
interface _DocumentWithMeiliIndex extends Document {
_meiliIndex?: boolean;
preprocessObjectForIndex?: () => Record<string, unknown>;
addObjectToMeili?: (next: CallbackWithoutResultAndOptionalError) => Promise<void>;
updateObjectToMeili?: (next: CallbackWithoutResultAndOptionalError) => Promise<void>;
deleteObjectFromMeili?: (next: CallbackWithoutResultAndOptionalError) => Promise<void>;
postSaveHook?: (next: CallbackWithoutResultAndOptionalError) => void;
postUpdateHook?: (next: CallbackWithoutResultAndOptionalError) => void;
postRemoveHook?: (next: CallbackWithoutResultAndOptionalError) => void;
}
export type DocumentWithMeiliIndex = _DocumentWithMeiliIndex & IConversation & Partial<IMessage>;
export interface SchemaWithMeiliMethods extends Model<DocumentWithMeiliIndex> {
syncWithMeili(options?: { resumeFromId?: string }): Promise<void>;
getSyncProgress(): Promise<SyncProgress>;
processSyncBatch(
index: Index<MeiliIndexable>,
documents: Array<Record<string, unknown>>,
updateOps: Array<{
updateOne: {
filter: Record<string, unknown>;
update: { $set: { _meiliIndex: boolean } };
};
}>,
): Promise<void>;
cleanupMeiliIndex(
index: Index<MeiliIndexable>,
primaryKey: string,
batchSize: number,
delayMs: number,
): Promise<void>;
setMeiliIndexSettings(settings: Record<string, unknown>): Promise<unknown>;
meiliSearch(q: string, params: Record<string, unknown>, populate: boolean): Promise<unknown>;
meiliSearch(
q: string,
params?: Record<string, unknown>,
populate?: boolean,
): Promise<SearchResponse<MeiliIndexable, Record<string, unknown>>>;
}
// Environment flags
@ -56,6 +92,14 @@ const searchEnabled = process.env.SEARCH != null && process.env.SEARCH.toLowerCa
const meiliEnabled =
process.env.MEILI_HOST != null && process.env.MEILI_MASTER_KEY != null && searchEnabled;
/**
* Get sync configuration from environment variables
*/
const getSyncConfig = () => ({
batchSize: parseInt(process.env.MEILI_SYNC_BATCH_SIZE || '100', 10),
delayMs: parseInt(process.env.MEILI_SYNC_DELAY_MS || '100', 10),
});
/**
* Local implementation of parseTextParts to avoid dependency on librechat-data-provider
* Extracts text content from an array of content items
@ -91,6 +135,26 @@ const validateOptions = (options: Partial<MongoMeiliOptions>): void => {
});
};
/**
* Helper function to process documents in batches with rate limiting
*/
const processBatch = async <T>(
items: T[],
batchSize: number,
delayMs: number,
processor: (batch: T[]) => Promise<void>,
): Promise<void> => {
for (let i = 0; i < items.length; i += batchSize) {
const batch = items.slice(i, i + batchSize);
await processor(batch);
// Add delay between batches to prevent overwhelming resources
if (i + batchSize < items.length && delayMs > 0) {
await new Promise((resolve) => setTimeout(resolve, delayMs));
}
}
};
/**
* Factory function to create a MeiliMongooseModel class which extends a Mongoose model.
* This class contains static and instance methods to synchronize and manage the MeiliSearch index
@ -99,127 +163,213 @@ const validateOptions = (options: Partial<MongoMeiliOptions>): void => {
* @param config - Configuration object.
* @param config.index - The MeiliSearch index object.
* @param config.attributesToIndex - List of attributes to index.
* @param config.syncOptions - Sync configuration options.
* @returns A class definition that will be loaded into the Mongoose schema.
*/
const createMeiliMongooseModel = ({
index,
attributesToIndex,
syncOptions,
}: {
index: Index<MeiliIndexable>;
attributesToIndex: string[];
syncOptions: { batchSize: number; delayMs: number };
}) => {
const primaryKey = attributesToIndex[0];
const syncConfig = { ...getSyncConfig(), ...syncOptions };
class MeiliMongooseModel {
/**
* Synchronizes the data between the MongoDB collection and the MeiliSearch index.
*
* The synchronization process involves:
* 1. Fetching all documents from the MongoDB collection and MeiliSearch index.
* 2. Comparing documents from both sources.
* 3. Deleting documents from MeiliSearch that no longer exist in MongoDB.
* 4. Adding documents to MeiliSearch that exist in MongoDB but not in the index.
* 5. Updating documents in MeiliSearch if key fields (such as `text` or `title`) differ.
* 6. Updating the `_meiliIndex` field in MongoDB to indicate the indexing status.
*
* Note: The function processes documents in batches because MeiliSearch's
* `index.getDocuments` requires an exact limit and `index.addDocuments` does not handle
* partial failures in a batch.
*
* @returns {Promise<void>} Resolves when the synchronization is complete.
* Get the current sync progress
*/
static async syncWithMeili(this: SchemaWithMeiliMethods): Promise<void> {
static async getSyncProgress(this: SchemaWithMeiliMethods): Promise<SyncProgress> {
const totalDocuments = await this.countDocuments();
const indexedDocuments = await this.countDocuments({ _meiliIndex: true });
return {
totalProcessed: indexedDocuments,
totalDocuments,
isComplete: indexedDocuments === totalDocuments,
};
}
/**
* Synchronizes the data between the MongoDB collection and the MeiliSearch index.
* Now uses streaming and batching to reduce memory usage.
*/
static async syncWithMeili(
this: SchemaWithMeiliMethods,
options?: { resumeFromId?: string },
): Promise<void> {
try {
let moreDocuments = true;
const mongoDocuments = await this.find().lean();
const startTime = Date.now();
const { batchSize, delayMs } = syncConfig;
logger.info(
`[syncWithMeili] Starting sync for ${primaryKey === 'messageId' ? 'messages' : 'conversations'} with batch size ${batchSize}`,
);
// Build query with resume capability
const query: FilterQuery<unknown> = {};
if (options?.resumeFromId) {
query._id = { $gt: options.resumeFromId };
}
// Get total count for progress tracking
const totalCount = await this.countDocuments(query);
let processedCount = 0;
// First, handle documents that need to be removed from Meili
await this.cleanupMeiliIndex(index, primaryKey, batchSize, delayMs);
// Process MongoDB documents in batches using cursor
const cursor = this.find(query)
.select(attributesToIndex.join(' ') + ' _meiliIndex')
.sort({ _id: 1 })
.batchSize(batchSize)
.cursor();
const format = (doc: Record<string, unknown>) =>
_.omitBy(_.pick(doc, attributesToIndex), (v, k) => k.startsWith('$'));
const mongoMap = new Map(
mongoDocuments.map((doc) => {
const typedDoc = doc as Record<string, unknown>;
return [typedDoc[primaryKey], format(typedDoc)];
}),
);
const indexMap = new Map<unknown, Record<string, unknown>>();
let offset = 0;
const batchSize = 1000;
while (moreDocuments) {
const batch = await index.getDocuments({ limit: batchSize, offset });
if (batch.results.length === 0) {
moreDocuments = false;
}
for (const doc of batch.results) {
indexMap.set(doc[primaryKey], format(doc));
}
offset += batchSize;
}
logger.debug('[syncWithMeili]', { indexMap: indexMap.size, mongoMap: mongoMap.size });
const updateOps: Array<{
let documentBatch: Array<Record<string, unknown>> = [];
let updateOps: Array<{
updateOne: {
filter: Record<string, unknown>;
update: { $set: { _meiliIndex: boolean } };
};
}> = [];
// Process documents present in the MeiliSearch index
for (const [id, doc] of indexMap) {
const update: Record<string, unknown> = {};
update[primaryKey] = id;
if (mongoMap.has(id)) {
const mongoDoc = mongoMap.get(id);
if (
(doc.text && doc.text !== mongoDoc?.text) ||
(doc.title && doc.title !== mongoDoc?.title)
) {
logger.debug(
`[syncWithMeili] ${id} had document discrepancy in ${
doc.text ? 'text' : 'title'
} field`,
);
updateOps.push({
updateOne: { filter: update, update: { $set: { _meiliIndex: true } } },
});
await index.addDocuments([doc]);
// Process documents in streaming fashion
for await (const doc of cursor) {
const typedDoc = doc.toObject() as unknown as Record<string, unknown>;
const formatted = format(typedDoc);
// Check if document needs indexing
if (!typedDoc._meiliIndex) {
documentBatch.push(formatted);
updateOps.push({
updateOne: {
filter: { _id: typedDoc._id },
update: { $set: { _meiliIndex: true } },
},
});
}
processedCount++;
// Process batch when it reaches the configured size
if (documentBatch.length >= batchSize) {
await this.processSyncBatch(index, documentBatch, updateOps);
documentBatch = [];
updateOps = [];
// Log progress
const progress = Math.round((processedCount / totalCount) * 100);
logger.info(`[syncWithMeili] Progress: ${progress}% (${processedCount}/${totalCount})`);
// Add delay to prevent overwhelming resources
if (delayMs > 0) {
await new Promise((resolve) => setTimeout(resolve, delayMs));
}
} else {
await index.deleteDocument(id as string);
updateOps.push({
updateOne: { filter: update, update: { $set: { _meiliIndex: false } } },
});
}
}
// Process documents present in MongoDB
for (const [id, doc] of mongoMap) {
const update: Record<string, unknown> = {};
update[primaryKey] = id;
if (!indexMap.has(id)) {
await index.addDocuments([doc]);
updateOps.push({
updateOne: { filter: update, update: { $set: { _meiliIndex: true } } },
});
} else if (doc._meiliIndex === false) {
updateOps.push({
updateOne: { filter: update, update: { $set: { _meiliIndex: true } } },
});
}
// Process remaining documents
if (documentBatch.length > 0) {
await this.processSyncBatch(index, documentBatch, updateOps);
}
const duration = Date.now() - startTime;
logger.info(
`[syncWithMeili] Completed sync for ${primaryKey === 'messageId' ? 'messages' : 'conversations'} in ${duration}ms`,
);
} catch (error) {
logger.error('[syncWithMeili] Error during sync:', error);
throw error;
}
}
/**
* Process a batch of documents for syncing
*/
static async processSyncBatch(
this: SchemaWithMeiliMethods,
index: Index<MeiliIndexable>,
documents: Array<Record<string, unknown>>,
updateOps: Array<{
updateOne: {
filter: Record<string, unknown>;
update: { $set: { _meiliIndex: boolean } };
};
}>,
): Promise<void> {
if (documents.length === 0) {
return;
}
try {
// Add documents to MeiliSearch
await index.addDocuments(documents);
// Update MongoDB to mark documents as indexed
if (updateOps.length > 0) {
await this.collection.bulkWrite(updateOps);
logger.debug(
`[syncWithMeili] Finished indexing ${
primaryKey === 'messageId' ? 'messages' : 'conversations'
}`,
);
}
} catch (error) {
logger.error('[syncWithMeili] Error adding document to Meili', error);
logger.error('[processSyncBatch] Error processing batch:', error);
// Don't throw - allow sync to continue with other documents
}
}
/**
* Clean up documents in MeiliSearch that no longer exist in MongoDB
*/
static async cleanupMeiliIndex(
this: SchemaWithMeiliMethods,
index: Index<MeiliIndexable>,
primaryKey: string,
batchSize: number,
delayMs: number,
): Promise<void> {
try {
let offset = 0;
let moreDocuments = true;
while (moreDocuments) {
const batch = await index.getDocuments({ limit: batchSize, offset });
if (batch.results.length === 0) {
moreDocuments = false;
break;
}
const meiliIds = batch.results.map((doc) => doc[primaryKey]);
const query: Record<string, unknown> = {};
query[primaryKey] = { $in: meiliIds };
// Find which documents exist in MongoDB
const existingDocs = await this.find(query).select(primaryKey).lean();
const existingIds = new Set(
existingDocs.map((doc: Record<string, unknown>) => doc[primaryKey]),
);
// Delete documents that don't exist in MongoDB
const toDelete = meiliIds.filter((id) => !existingIds.has(id));
if (toDelete.length > 0) {
await Promise.all(toDelete.map((id) => index.deleteDocument(id as string)));
logger.debug(`[cleanupMeiliIndex] Deleted ${toDelete.length} orphaned documents`);
}
offset += batchSize;
// Add delay between batches
if (delayMs > 0) {
await new Promise((resolve) => setTimeout(resolve, delayMs));
}
}
} catch (error) {
logger.error('[cleanupMeiliIndex] Error during cleanup:', error);
}
}
@ -238,7 +388,7 @@ const createMeiliMongooseModel = ({
q: string,
params: Record<string, unknown>,
populate: boolean,
): Promise<unknown> {
): Promise<SearchResponse<MeiliIndexable, Record<string, unknown>>> {
const data = await index.search(q, params);
if (populate) {
@ -303,30 +453,61 @@ const createMeiliMongooseModel = ({
}
/**
* Adds the current document to the MeiliSearch index
* Adds the current document to the MeiliSearch index with retry logic
*/
async addObjectToMeili(this: DocumentWithMeiliIndex): Promise<void> {
async addObjectToMeili(
this: DocumentWithMeiliIndex,
next: CallbackWithoutResultAndOptionalError,
): Promise<void> {
const object = this.preprocessObjectForIndex!();
try {
await index.addDocuments([object]);
} catch (error) {
logger.error('[addObjectToMeili] Error adding document to Meili', error);
const maxRetries = 3;
let retryCount = 0;
while (retryCount < maxRetries) {
try {
await index.addDocuments([object]);
break;
} catch (error) {
retryCount++;
if (retryCount >= maxRetries) {
logger.error('[addObjectToMeili] Error adding document to Meili after retries:', error);
return next();
}
// Exponential backoff
await new Promise((resolve) => setTimeout(resolve, Math.pow(2, retryCount) * 1000));
}
}
await this.collection.updateMany(
{ _id: this._id as mongoose.Types.ObjectId },
{ $set: { _meiliIndex: true } },
);
try {
await this.collection.updateMany(
{ _id: this._id as Types.ObjectId },
{ $set: { _meiliIndex: true } },
);
} catch (error) {
logger.error('[addObjectToMeili] Error updating _meiliIndex field:', error);
return next();
}
next();
}
/**
* Updates the current document in the MeiliSearch index
*/
async updateObjectToMeili(this: DocumentWithMeiliIndex): Promise<void> {
const object = _.omitBy(_.pick(this.toJSON(), attributesToIndex), (v, k) =>
k.startsWith('$'),
);
await index.updateDocuments([object]);
async updateObjectToMeili(
this: DocumentWithMeiliIndex,
next: CallbackWithoutResultAndOptionalError,
): Promise<void> {
try {
const object = _.omitBy(_.pick(this.toJSON(), attributesToIndex), (v, k) =>
k.startsWith('$'),
);
await index.updateDocuments([object]);
next();
} catch (error) {
logger.error('[updateObjectToMeili] Error updating document in Meili:', error);
return next();
}
}
/**
@ -334,8 +515,17 @@ const createMeiliMongooseModel = ({
*
* @returns {Promise<void>}
*/
async deleteObjectFromMeili(this: DocumentWithMeiliIndex): Promise<void> {
await index.deleteDocument(this._id as string);
async deleteObjectFromMeili(
this: DocumentWithMeiliIndex,
next: CallbackWithoutResultAndOptionalError,
): Promise<void> {
try {
await index.deleteDocument(this._id as string);
next();
} catch (error) {
logger.error('[deleteObjectFromMeili] Error deleting document from Meili:', error);
return next();
}
}
/**
@ -344,11 +534,11 @@ const createMeiliMongooseModel = ({
* If the document is already indexed (i.e. `_meiliIndex` is true), it updates it;
* otherwise, it adds the document to the index.
*/
postSaveHook(this: DocumentWithMeiliIndex): void {
postSaveHook(this: DocumentWithMeiliIndex, next: CallbackWithoutResultAndOptionalError): void {
if (this._meiliIndex) {
this.updateObjectToMeili!();
this.updateObjectToMeili!(next);
} else {
this.addObjectToMeili!();
this.addObjectToMeili!(next);
}
}
@ -358,9 +548,14 @@ const createMeiliMongooseModel = ({
* This hook is triggered after a document update, ensuring that changes are
* propagated to the MeiliSearch index if the document is indexed.
*/
postUpdateHook(this: DocumentWithMeiliIndex): void {
postUpdateHook(
this: DocumentWithMeiliIndex,
next: CallbackWithoutResultAndOptionalError,
): void {
if (this._meiliIndex) {
this.updateObjectToMeili!();
this.updateObjectToMeili!(next);
} else {
next();
}
}
@ -370,9 +565,14 @@ const createMeiliMongooseModel = ({
* This hook is triggered after a document is removed, ensuring that the document
* is also removed from the MeiliSearch index if it was previously indexed.
*/
postRemoveHook(this: DocumentWithMeiliIndex): void {
postRemoveHook(
this: DocumentWithMeiliIndex,
next: CallbackWithoutResultAndOptionalError,
): void {
if (this._meiliIndex) {
this.deleteObjectFromMeili!();
this.deleteObjectFromMeili!(next);
} else {
next();
}
}
}
@ -396,8 +596,11 @@ const createMeiliMongooseModel = ({
* @param options.apiKey - The MeiliSearch API key.
* @param options.indexName - The name of the MeiliSearch index.
* @param options.primaryKey - The primary key field for indexing.
* @param options.syncBatchSize - Batch size for sync operations.
* @param options.syncDelayMs - Delay between batches in milliseconds.
*/
export default function mongoMeili(schema: Schema, options: MongoMeiliOptions): void {
const mongoose = options.mongoose;
validateOptions(options);
// Add _meiliIndex field to the schema to track if a document has been indexed in MeiliSearch.
@ -411,11 +614,38 @@ export default function mongoMeili(schema: Schema, options: MongoMeiliOptions):
});
const { host, apiKey, indexName, primaryKey } = options;
const syncOptions = {
batchSize: options.syncBatchSize || getSyncConfig().batchSize,
delayMs: options.syncDelayMs || getSyncConfig().delayMs,
};
const client = new MeiliSearch({ host, apiKey });
client.createIndex(indexName, { primaryKey });
/** Create index only if it doesn't exist */
const index = client.index<MeiliIndexable>(indexName);
// Check if index exists and create if needed
(async () => {
try {
await index.getRawInfo();
logger.debug(`[mongoMeili] Index ${indexName} already exists`);
} catch (error) {
const errorCode = (error as { code?: string })?.code;
if (errorCode === 'index_not_found') {
try {
logger.info(`[mongoMeili] Creating new index: ${indexName}`);
await client.createIndex(indexName, { primaryKey });
logger.info(`[mongoMeili] Successfully created index: ${indexName}`);
} catch (createError) {
// Index might have been created by another instance
logger.debug(`[mongoMeili] Index ${indexName} may already exist:`, createError);
}
} else {
logger.error(`[mongoMeili] Error checking index ${indexName}:`, error);
}
}
})();
// Collect attributes from the schema that should be indexed
const attributesToIndex: string[] = [
...Object.entries(schema.obj).reduce<string[]>((results, [key, value]) => {
@ -424,19 +654,19 @@ export default function mongoMeili(schema: Schema, options: MongoMeiliOptions):
}, []),
];
schema.loadClass(createMeiliMongooseModel({ index, attributesToIndex }));
schema.loadClass(createMeiliMongooseModel({ index, attributesToIndex, syncOptions }));
// Register Mongoose hooks
schema.post('save', function (doc: DocumentWithMeiliIndex) {
doc.postSaveHook?.();
schema.post('save', function (doc: DocumentWithMeiliIndex, next) {
doc.postSaveHook?.(next);
});
schema.post('updateOne', function (doc: DocumentWithMeiliIndex) {
doc.postUpdateHook?.();
schema.post('updateOne', function (doc: DocumentWithMeiliIndex, next) {
doc.postUpdateHook?.(next);
});
schema.post('deleteOne', function (doc: DocumentWithMeiliIndex) {
doc.postRemoveHook?.();
schema.post('deleteOne', function (doc: DocumentWithMeiliIndex, next) {
doc.postRemoveHook?.(next);
});
// Pre-deleteMany hook: remove corresponding documents from MeiliSearch when multiple documents are deleted.
@ -447,29 +677,40 @@ export default function mongoMeili(schema: Schema, options: MongoMeiliOptions):
try {
const conditions = (this as Query<unknown, unknown>).getQuery();
const { batchSize, delayMs } = syncOptions;
if (Object.prototype.hasOwnProperty.call(schema.obj, 'messages')) {
const convoIndex = client.index('convos');
const deletedConvos = await mongoose
.model('Conversation')
.find(conditions as mongoose.FilterQuery<unknown>)
.find(conditions as FilterQuery<unknown>)
.select('conversationId')
.lean();
const promises = deletedConvos.map((convo: Record<string, unknown>) =>
convoIndex.deleteDocument(convo.conversationId as string),
);
await Promise.all(promises);
// Process deletions in batches
await processBatch(deletedConvos, batchSize, delayMs, async (batch) => {
const promises = batch.map((convo: Record<string, unknown>) =>
convoIndex.deleteDocument(convo.conversationId as string),
);
await Promise.all(promises);
});
}
if (Object.prototype.hasOwnProperty.call(schema.obj, 'messageId')) {
const messageIndex = client.index('messages');
const deletedMessages = await mongoose
.model('Message')
.find(conditions as mongoose.FilterQuery<unknown>)
.find(conditions as FilterQuery<unknown>)
.select('messageId')
.lean();
const promises = deletedMessages.map((message: Record<string, unknown>) =>
messageIndex.deleteDocument(message.messageId as string),
);
await Promise.all(promises);
// Process deletions in batches
await processBatch(deletedMessages, batchSize, delayMs, async (batch) => {
const promises = batch.map((message: Record<string, unknown>) =>
messageIndex.deleteDocument(message.messageId as string),
);
await Promise.all(promises);
});
}
return next();
} catch (error) {
@ -484,13 +725,13 @@ export default function mongoMeili(schema: Schema, options: MongoMeiliOptions):
});
// Post-findOneAndUpdate hook
schema.post('findOneAndUpdate', async function (doc: DocumentWithMeiliIndex) {
schema.post('findOneAndUpdate', async function (doc: DocumentWithMeiliIndex, next) {
if (!meiliEnabled) {
return;
return next();
}
if (doc.unfinished) {
return;
return next();
}
let meiliDoc: Record<string, unknown> | undefined;
@ -507,9 +748,9 @@ export default function mongoMeili(schema: Schema, options: MongoMeiliOptions):
}
if (meiliDoc && meiliDoc.title === doc.title) {
return;
return next();
}
doc.postSaveHook?.();
doc.postSaveHook?.(next);
});
}

11
packages/data-schemas/src/schema/convo.ts
View file

@ -1,5 +1,4 @@
import { Schema } from 'mongoose';
import mongoMeili from '~/models/plugins/mongoMeili';
import { conversationPreset } from './defaults';
import { IConversation } from '~/types';
@ -48,14 +47,4 @@ convoSchema.index({ expiredAt: 1 }, { expireAfterSeconds: 0 });
convoSchema.index({ createdAt: 1, updatedAt: 1 });
convoSchema.index({ conversationId: 1, user: 1 }, { unique: true });
if (process.env.MEILI_HOST && process.env.MEILI_MASTER_KEY) {
convoSchema.plugin(mongoMeili, {
host: process.env.MEILI_HOST,
apiKey: process.env.MEILI_MASTER_KEY,
/** Note: Will get created automatically if it doesn't exist already */
indexName: 'convos',
primaryKey: 'conversationId',
});
}
export default convoSchema;

1
packages/data-schemas/src/schema/index.ts
View file

@ -21,3 +21,4 @@ export { default as tokenSchema } from './token';
export { default as toolCallSchema } from './toolCall';
export { default as transactionSchema } from './transaction';
export { default as userSchema } from './user';
export { default as memorySchema } from './memory';

33
packages/data-schemas/src/schema/memory.ts Normal file
View file

@ -0,0 +1,33 @@
import { Schema } from 'mongoose';
import type { IMemoryEntry } from '~/types/memory';
const MemoryEntrySchema: Schema<IMemoryEntry> = new Schema({
userId: {
type: Schema.Types.ObjectId,
ref: 'User',
index: true,
required: true,
},
key: {
type: String,
required: true,
validate: {
validator: (v: string) => /^[a-z_]+$/.test(v),
message: 'Key must only contain lowercase letters and underscores',
},
},
value: {
type: String,
required: true,
},
tokenCount: {
type: Number,
default: 0,
},
updated_at: {
type: Date,
default: Date.now,
},
});
export default MemoryEntrySchema;

10
packages/data-schemas/src/schema/message.ts
View file

@ -1,6 +1,5 @@
import mongoose, { Schema } from 'mongoose';
import type { IMessage } from '~/types/message';
import mongoMeili from '~/models/plugins/mongoMeili';
const messageSchema: Schema<IMessage> = new Schema(
{
@ -166,13 +165,4 @@ messageSchema.index({ expiredAt: 1 }, { expireAfterSeconds: 0 });
messageSchema.index({ createdAt: 1 });
messageSchema.index({ messageId: 1, user: 1 }, { unique: true });
if (process.env.MEILI_HOST && process.env.MEILI_MASTER_KEY) {
messageSchema.plugin(mongoMeili, {
host: process.env.MEILI_HOST,
apiKey: process.env.MEILI_MASTER_KEY,
indexName: 'messages',
primaryKey: 'messageId',
});
}
export default messageSchema;

12
packages/data-schemas/src/schema/pluginAuth.ts
View file

@ -1,13 +1,5 @@
import { Schema, Document } from 'mongoose';
export interface IPluginAuth extends Document {
authField: string;
value: string;
userId: string;
pluginKey?: string;
createdAt?: Date;
updatedAt?: Date;
}
import { Schema } from 'mongoose';
import type { IPluginAuth } from '~/types';
const pluginAuthSchema: Schema<IPluginAuth> = new Schema(
{

6
packages/data-schemas/src/schema/promptGroup.ts
View file

@ -63,11 +63,11 @@ const promptGroupSchema = new Schema<IPromptGroupDocument>(
type: String,
index: true,
validate: {
validator: function (v: unknown): boolean {
validator: function (v: string | undefined | null): boolean {
return v === undefined || v === null || v === '' || /^[a-z0-9-]+$/.test(v);
},
message: (props: unknown) =>
`${props.value} is not a valid command. Only lowercase alphanumeric characters and hyphens are allowed.`,
message: (props: { value?: string } | undefined) =>
`${props?.value ?? 'Value'} is not a valid command. Only lowercase alphanumeric characters and hyphens are allowed.`,
},
maxlength: [
Constants.COMMANDS_MAX_LENGTH as number,

13
packages/data-schemas/src/schema/role.ts
View file

@ -13,6 +13,13 @@ const rolePermissionsSchema = new Schema(
[Permissions.USE]: { type: Boolean, default: true },
[Permissions.CREATE]: { type: Boolean, default: true },
},
[PermissionTypes.MEMORIES]: {
[Permissions.USE]: { type: Boolean, default: true },
[Permissions.CREATE]: { type: Boolean, default: true },
[Permissions.UPDATE]: { type: Boolean, default: true },
[Permissions.READ]: { type: Boolean, default: true },
[Permissions.OPT_OUT]: { type: Boolean, default: true },
},
[PermissionTypes.AGENTS]: {
[Permissions.SHARED_GLOBAL]: { type: Boolean, default: false },
[Permissions.USE]: { type: Boolean, default: true },
@ -45,6 +52,12 @@ const roleSchema: Schema<IRole> = new Schema({
[Permissions.USE]: true,
[Permissions.CREATE]: true,
},
[PermissionTypes.MEMORIES]: {
[Permissions.USE]: true,
[Permissions.CREATE]: true,
[Permissions.UPDATE]: true,
[Permissions.READ]: true,
},
[PermissionTypes.AGENTS]: {
[Permissions.SHARED_GLOBAL]: false,
[Permissions.USE]: true,

8
packages/data-schemas/src/schema/user.ts
View file

@ -150,6 +150,14 @@ const userSchema = new Schema<IUser>(
],
default: [],
_id: false,
personalization: {
type: {
memories: {
type: Boolean,
default: true,
},
},
default: {},
},
},
{ timestamps: true },

7
packages/data-schemas/src/types/index.ts
View file

@ -1,3 +1,6 @@
import type { Types } from 'mongoose';
export type ObjectId = Types.ObjectId;
export * from './user';
export * from './token';
export * from './convo';
@ -10,3 +13,7 @@ export * from './role';
export * from './action';
export * from './assistant';
export * from './file';
export * from './share';
export * from './pluginAuth';
/* Memories */
export * from './memory';

48
packages/data-schemas/src/types/memory.ts Normal file
View file

@ -0,0 +1,48 @@
import type { Types, Document } from 'mongoose';
// Base memory interfaces
export interface IMemoryEntry extends Document {
userId: Types.ObjectId;
key: string;
value: string;
tokenCount?: number;
updated_at?: Date;
}
export interface IMemoryEntryLean {
_id: Types.ObjectId;
userId: Types.ObjectId;
key: string;
value: string;
tokenCount?: number;
updated_at?: Date;
__v?: number;
}
// Method parameter interfaces
export interface SetMemoryParams {
userId: string | Types.ObjectId;
key: string;
value: string;
tokenCount?: number;
}
export interface DeleteMemoryParams {
userId: string | Types.ObjectId;
key: string;
}
export interface GetFormattedMemoriesParams {
userId: string | Types.ObjectId;
}
// Result interfaces
export interface MemoryResult {
ok: boolean;
}
export interface FormattedMemoriesResult {
withKeys: string;
withoutKeys: string;
totalTokens?: number;
}

40
packages/data-schemas/src/types/pluginAuth.ts Normal file
View file

@ -0,0 +1,40 @@
import type { Document } from 'mongoose';
export interface IPluginAuth extends Document {
authField: string;
value: string;
userId: string;
pluginKey?: string;
createdAt?: Date;
updatedAt?: Date;
}
export interface PluginAuthQuery {
userId: string;
authField?: string;
pluginKey?: string;
}
export interface FindPluginAuthParams {
userId: string;
authField: string;
}
export interface FindPluginAuthsByKeysParams {
userId: string;
pluginKeys: string[];
}
export interface UpdatePluginAuthParams {
userId: string;
authField: string;
pluginKey: string;
value: string;
}
export interface DeletePluginAuthParams {
userId: string;
authField?: string;
pluginKey?: string;
all?: boolean;
}

6
packages/data-schemas/src/types/role.ts
View file

@ -12,6 +12,12 @@ export interface IRole extends Document {
[Permissions.USE]?: boolean;
[Permissions.CREATE]?: boolean;
};
[PermissionTypes.MEMORIES]?: {
[Permissions.USE]?: boolean;
[Permissions.CREATE]?: boolean;
[Permissions.UPDATE]?: boolean;
[Permissions.READ]?: boolean;
};
[PermissionTypes.AGENTS]?: {
[Permissions.SHARED_GLOBAL]?: boolean;
[Permissions.USE]?: boolean;

66
packages/data-schemas/src/types/share.ts Normal file
View file

@ -0,0 +1,66 @@
import type { Types } from 'mongoose';
import type { IMessage } from './message';
export interface ISharedLink {
_id?: Types.ObjectId;
conversationId: string;
title?: string;
user?: string;
messages?: Types.ObjectId[];
shareId?: string;
isPublic: boolean;
createdAt?: Date;
updatedAt?: Date;
}
export interface ShareServiceError extends Error {
code: string;
}
export interface SharedLinksResult {
links: Array<{
shareId: string;
title: string;
isPublic: boolean;
createdAt: Date;
conversationId: string;
}>;
nextCursor?: Date;
hasNextPage: boolean;
}
export interface SharedMessagesResult {
conversationId: string;
messages: Array<IMessage>;
shareId: string;
title?: string;
isPublic: boolean;
createdAt?: Date;
updatedAt?: Date;
}
export interface CreateShareResult {
shareId: string;
conversationId: string;
}
export interface UpdateShareResult {
shareId: string;
conversationId: string;
}
export interface DeleteShareResult {
success: boolean;
shareId: string;
message: string;
}
export interface GetShareLinkResult {
shareId: string | null;
success: boolean;
}
export interface DeleteAllSharesResult {
message: string;
deletedCount: number;
}

3
packages/data-schemas/src/types/user.ts
View file

@ -35,6 +35,9 @@ export interface IUser extends Document {
promptGroupId: Types.ObjectId;
order: number;
}>;
personalization?: {
memories?: boolean;
};
createdAt?: Date;
updatedAt?: Date;
}