mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-02-21 01:44:09 +01:00
🐘 feat: FerretDB Compatibility (#11769)
* feat: replace unsupported MongoDB aggregation operators for FerretDB compatibility Replace $lookup, $unwind, $sample, $replaceRoot, and $addFields aggregation stages which are unsupported on FerretDB v2.x (postgres-documentdb backend). - Prompt.js: Replace $lookup/$unwind/$project pipelines with find().select().lean() + attachProductionPrompts() batch helper. Replace $group/$replaceRoot/$sample in getRandomPromptGroups with distinct() + Fisher-Yates shuffle. - Agent/Prompt migration scripts: Replace $lookup anti-join pattern with distinct() + $nin two-step queries for finding un-migrated resources. All replacement patterns verified against FerretDB v2.7.0. Co-authored-by: Cursor <cursoragent@cursor.com> * fix: use $pullAll for simple array removals, fix memberIds type mismatches Replace $pull with $pullAll for exact-value scalar array removals. Both operators work on MongoDB and FerretDB, but $pullAll is more explicit for exact matching (no condition expressions). Fix critical type mismatch bugs where ObjectId values were used against String[] memberIds arrays in Group queries: - config/delete-user.js: use string uid instead of ObjectId user._id - e2e/setup/cleanupUser.ts: convert userId.toString() before query Harden PermissionService.bulkUpdateResourcePermissions abort handling to prevent crash when abortTransaction is called after commitTransaction. All changes verified against FerretDB v2.7.0 and MongoDB Memory Server. Co-authored-by: Cursor <cursoragent@cursor.com> * fix: harden transaction support probe for FerretDB compatibility Commit the transaction before aborting in supportsTransactions probe, and wrap abortTransaction in try-catch to prevent crashes when abort is called after a successful commit (observed behavior on FerretDB). Co-authored-by: Cursor <cursoragent@cursor.com> * feat: add FerretDB compatibility test suite, retry utilities, and CI config Add comprehensive FerretDB integration test suite covering: - $pullAll scalar array operations - $pull with subdocument conditions - $lookup replacement (find + manual join) - $sample replacement (distinct + Fisher-Yates) - $bit and $bitsAllSet operations - Migration anti-join pattern - Multi-tenancy (useDb, scaling, write amplification) - Sharding proof-of-concept - Production operations (backup/restore, schema migration, deadlock retry) Add production retryWithBackoff utility for deadlock recovery during concurrent index creation on FerretDB/DocumentDB backends. Add UserController.spec.js tests for deleteUserController (runs in CI). Configure jest and eslint to isolate FerretDB tests from CI pipelines: - packages/data-schemas/jest.config.mjs: ignore misc/ directory - eslint.config.mjs: ignore packages/data-schemas/misc/ Include Docker Compose config for local FerretDB v2.7 + postgres-documentdb, dedicated jest/tsconfig for the test files, and multi-tenancy findings doc. Co-authored-by: Cursor <cursoragent@cursor.com> * style: brace formatting in aclEntry.ts modifyPermissionBits Co-authored-by: Cursor <cursoragent@cursor.com> * refactor: reorganize retry utilities and update imports - Moved retryWithBackoff utility to a new file `retry.ts` for better structure. - Updated imports in `orgOperations.ferretdb.spec.ts` to reflect the new location of retry utilities. - Removed old import statement for retryWithBackoff from index.ts to streamline exports. * test: add $pullAll coverage for ConversationTag and PermissionService Add integration tests for deleteConversationTag verifying $pullAll removes tags from conversations correctly, and for syncUserEntraGroupMemberships verifying $pullAll removes user from non-matching Entra groups while preserving local group membership. --------- Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Danny Avila
parent
c3da148fa0
commit
591e59ec5b
35 changed files with 4727 additions and 347 deletions
|
|
@ -107,7 +107,7 @@ async function gracefulExit(code = 0) {
|
|||
await Promise.all(tasks);
|
||||
|
||||
// 6) Remove user from all groups
|
||||
await Group.updateMany({ memberIds: user._id }, { $pull: { memberIds: user._id } });
|
||||
await Group.updateMany({ memberIds: uid }, { $pullAll: { memberIds: [uid] } });
|
||||
|
||||
// 7) Finally delete the user document itself
|
||||
await User.deleteOne({ _id: uid });
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ const connect = require('./connect');
|
|||
const { grantPermission } = require('~/server/services/PermissionService');
|
||||
const { getProjectByName } = require('~/models/Project');
|
||||
const { findRoleByIdentifier } = require('~/models');
|
||||
const { Agent } = require('~/db/models');
|
||||
const { Agent, AclEntry } = require('~/db/models');
|
||||
|
||||
async function migrateAgentPermissionsEnhanced({ dryRun = true, batchSize = 100 } = {}) {
|
||||
await connect();
|
||||
|
|
@ -39,48 +39,17 @@ async function migrateAgentPermissionsEnhanced({ dryRun = true, batchSize = 100
|
|||
|
||||
logger.info(`Found ${globalAgentIds.size} agents in global project`);
|
||||
|
||||
// Find agents without ACL entries using DocumentDB-compatible approach
|
||||
const agentsToMigrate = await Agent.aggregate([
|
||||
{
|
||||
$lookup: {
|
||||
from: 'aclentries',
|
||||
localField: '_id',
|
||||
foreignField: 'resourceId',
|
||||
as: 'aclEntries',
|
||||
},
|
||||
},
|
||||
{
|
||||
$addFields: {
|
||||
userAclEntries: {
|
||||
$filter: {
|
||||
input: '$aclEntries',
|
||||
as: 'aclEntry',
|
||||
cond: {
|
||||
$and: [
|
||||
{ $eq: ['$$aclEntry.resourceType', ResourceType.AGENT] },
|
||||
{ $eq: ['$$aclEntry.principalType', PrincipalType.USER] },
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
$match: {
|
||||
author: { $exists: true, $ne: null },
|
||||
userAclEntries: { $size: 0 },
|
||||
},
|
||||
},
|
||||
{
|
||||
$project: {
|
||||
_id: 1,
|
||||
id: 1,
|
||||
name: 1,
|
||||
author: 1,
|
||||
isCollaborative: 1,
|
||||
},
|
||||
},
|
||||
]);
|
||||
const migratedAgentIds = await AclEntry.distinct('resourceId', {
|
||||
resourceType: ResourceType.AGENT,
|
||||
principalType: PrincipalType.USER,
|
||||
});
|
||||
|
||||
const agentsToMigrate = await Agent.find({
|
||||
_id: { $nin: migratedAgentIds },
|
||||
author: { $exists: true, $ne: null },
|
||||
})
|
||||
.select('_id id name author isCollaborative')
|
||||
.lean();
|
||||
|
||||
const categories = {
|
||||
globalEditAccess: [], // Global project + collaborative -> Public EDIT
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ const connect = require('./connect');
|
|||
const { grantPermission } = require('~/server/services/PermissionService');
|
||||
const { getProjectByName } = require('~/models/Project');
|
||||
const { findRoleByIdentifier } = require('~/models');
|
||||
const { PromptGroup } = require('~/db/models');
|
||||
const { PromptGroup, AclEntry } = require('~/db/models');
|
||||
|
||||
async function migrateToPromptGroupPermissions({ dryRun = true, batchSize = 100 } = {}) {
|
||||
await connect();
|
||||
|
|
@ -41,48 +41,17 @@ async function migrateToPromptGroupPermissions({ dryRun = true, batchSize = 100
|
|||
|
||||
logger.info(`Found ${globalPromptGroupIds.size} prompt groups in global project`);
|
||||
|
||||
// Find promptGroups without ACL entries
|
||||
const promptGroupsToMigrate = await PromptGroup.aggregate([
|
||||
{
|
||||
$lookup: {
|
||||
from: 'aclentries',
|
||||
localField: '_id',
|
||||
foreignField: 'resourceId',
|
||||
as: 'aclEntries',
|
||||
},
|
||||
},
|
||||
{
|
||||
$addFields: {
|
||||
promptGroupAclEntries: {
|
||||
$filter: {
|
||||
input: '$aclEntries',
|
||||
as: 'aclEntry',
|
||||
cond: {
|
||||
$and: [
|
||||
{ $eq: ['$$aclEntry.resourceType', ResourceType.PROMPTGROUP] },
|
||||
{ $eq: ['$$aclEntry.principalType', PrincipalType.USER] },
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
$match: {
|
||||
author: { $exists: true, $ne: null },
|
||||
promptGroupAclEntries: { $size: 0 },
|
||||
},
|
||||
},
|
||||
{
|
||||
$project: {
|
||||
_id: 1,
|
||||
name: 1,
|
||||
author: 1,
|
||||
authorName: 1,
|
||||
category: 1,
|
||||
},
|
||||
},
|
||||
]);
|
||||
const migratedGroupIds = await AclEntry.distinct('resourceId', {
|
||||
resourceType: ResourceType.PROMPTGROUP,
|
||||
principalType: PrincipalType.USER,
|
||||
});
|
||||
|
||||
const promptGroupsToMigrate = await PromptGroup.find({
|
||||
_id: { $nin: migratedGroupIds },
|
||||
author: { $exists: true, $ne: null },
|
||||
})
|
||||
.select('_id name author authorName category')
|
||||
.lean();
|
||||
|
||||
const categories = {
|
||||
globalViewAccess: [], // PromptGroup in global project -> Public VIEW
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue