🐘 feat: FerretDB Compatibility (#11769)

* feat: replace unsupported MongoDB aggregation operators for FerretDB compatibility

Replace $lookup, $unwind, $sample, $replaceRoot, and $addFields aggregation
stages which are unsupported on FerretDB v2.x (postgres-documentdb backend).

- Prompt.js: Replace $lookup/$unwind/$project pipelines with find().select().lean()
  + attachProductionPrompts() batch helper. Replace $group/$replaceRoot/$sample
  in getRandomPromptGroups with distinct() + Fisher-Yates shuffle.
- Agent/Prompt migration scripts: Replace $lookup anti-join pattern with
  distinct() + $nin two-step queries for finding un-migrated resources.

All replacement patterns verified against FerretDB v2.7.0.

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix: use $pullAll for simple array removals, fix memberIds type mismatches

Replace $pull with $pullAll for exact-value scalar array removals. Both
operators work on MongoDB and FerretDB, but $pullAll is more explicit for
exact matching (no condition expressions).

Fix critical type mismatch bugs where ObjectId values were used against
String[] memberIds arrays in Group queries:
- config/delete-user.js: use string uid instead of ObjectId user._id
- e2e/setup/cleanupUser.ts: convert userId.toString() before query

Harden PermissionService.bulkUpdateResourcePermissions abort handling to
prevent crash when abortTransaction is called after commitTransaction.

All changes verified against FerretDB v2.7.0 and MongoDB Memory Server.

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix: harden transaction support probe for FerretDB compatibility

Commit the transaction before aborting in supportsTransactions probe, and
wrap abortTransaction in try-catch to prevent crashes when abort is called
after a successful commit (observed behavior on FerretDB).

Co-authored-by: Cursor <cursoragent@cursor.com>

* feat: add FerretDB compatibility test suite, retry utilities, and CI config

Add comprehensive FerretDB integration test suite covering:
- $pullAll scalar array operations
- $pull with subdocument conditions
- $lookup replacement (find + manual join)
- $sample replacement (distinct + Fisher-Yates)
- $bit and $bitsAllSet operations
- Migration anti-join pattern
- Multi-tenancy (useDb, scaling, write amplification)
- Sharding proof-of-concept
- Production operations (backup/restore, schema migration, deadlock retry)

Add production retryWithBackoff utility for deadlock recovery during
concurrent index creation on FerretDB/DocumentDB backends.

Add UserController.spec.js tests for deleteUserController (runs in CI).

Configure jest and eslint to isolate FerretDB tests from CI pipelines:
- packages/data-schemas/jest.config.mjs: ignore misc/ directory
- eslint.config.mjs: ignore packages/data-schemas/misc/

Include Docker Compose config for local FerretDB v2.7 + postgres-documentdb,
dedicated jest/tsconfig for the test files, and multi-tenancy findings doc.

Co-authored-by: Cursor <cursoragent@cursor.com>

* style: brace formatting in aclEntry.ts modifyPermissionBits

Co-authored-by: Cursor <cursoragent@cursor.com>

* refactor: reorganize retry utilities and update imports

- Moved retryWithBackoff utility to a new file `retry.ts` for better structure.
- Updated imports in `orgOperations.ferretdb.spec.ts` to reflect the new location of retry utilities.
- Removed old import statement for retryWithBackoff from index.ts to streamline exports.

* test: add $pullAll coverage for ConversationTag and PermissionService

Add integration tests for deleteConversationTag verifying $pullAll
removes tags from conversations correctly, and for
syncUserEntraGroupMemberships verifying $pullAll removes user from
non-matching Entra groups while preserving local group membership.

---------

Co-authored-by: Cursor <cursoragent@cursor.com>

packages/api/src/agents/migration.ts

@@ -24,7 +24,7 @@ export interface MigrationCheckParams {
 }
 
 interface AgentMigrationData {
-  _id: string;
+  _id: unknown;
   id: string;
   name: string;
   author: string;
@@ -81,48 +81,18 @@ export async function checkAgentPermissionsMigration({
     const globalProject = await methods.getProjectByName(GLOBAL_PROJECT_NAME, ['agentIds']);
     const globalAgentIds = new Set(globalProject?.agentIds || []);
 
-    // Find agents without ACL entries (no batching for efficiency on startup)
-    const agentsToMigrate: AgentMigrationData[] = await AgentModel.aggregate([
-      {
-        $lookup: {
-          from: 'aclentries',
-          localField: '_id',
-          foreignField: 'resourceId',
-          as: 'aclEntries',
-        },
-      },
-      {
-        $addFields: {
-          userAclEntries: {
-            $filter: {
-              input: '$aclEntries',
-              as: 'aclEntry',
-              cond: {
-                $and: [
-                  { $eq: ['$$aclEntry.resourceType', ResourceType.AGENT] },
-                  { $eq: ['$$aclEntry.principalType', PrincipalType.USER] },
-                ],
-              },
-            },
-          },
-        },
-      },
-      {
-        $match: {
-          author: { $exists: true, $ne: null },
-          userAclEntries: { $size: 0 },
-        },
-      },
-      {
-        $project: {
-          _id: 1,
-          id: 1,
-          name: 1,
-          author: 1,
-          isCollaborative: 1,
-        },
-      },
-    ]);
+    const AclEntry = mongoose.model('AclEntry');
+    const migratedAgentIds = await AclEntry.distinct('resourceId', {
+      resourceType: ResourceType.AGENT,
+      principalType: PrincipalType.USER,
+    });
+
+    const agentsToMigrate = (await AgentModel.find({
+      _id: { $nin: migratedAgentIds },
+      author: { $exists: true, $ne: null },
+    })
+      .select('_id id name author isCollaborative')
+      .lean()) as unknown as AgentMigrationData[];
 
     const categories: {
       globalEditAccess: AgentMigrationData[];

packages/api/src/prompts/migration.ts

@@ -24,7 +24,7 @@ export interface PromptMigrationCheckParams {
 }
 
 interface PromptGroupMigrationData {
-  _id: string;
+  _id: { toString(): string };
   name: string;
   author: string;
   authorName?: string;
@@ -81,48 +81,18 @@ export async function checkPromptPermissionsMigration({
       (globalProject?.promptGroupIds || []).map((id) => id.toString()),
     );
 
-    // Find promptGroups without ACL entries (no batching for efficiency on startup)
-    const promptGroupsToMigrate: PromptGroupMigrationData[] = await PromptGroupModel.aggregate([
-      {
-        $lookup: {
-          from: 'aclentries',
-          localField: '_id',
-          foreignField: 'resourceId',
-          as: 'aclEntries',
-        },
-      },
-      {
-        $addFields: {
-          promptGroupAclEntries: {
-            $filter: {
-              input: '$aclEntries',
-              as: 'aclEntry',
-              cond: {
-                $and: [
-                  { $eq: ['$$aclEntry.resourceType', ResourceType.PROMPTGROUP] },
-                  { $eq: ['$$aclEntry.principalType', PrincipalType.USER] },
-                ],
-              },
-            },
-          },
-        },
-      },
-      {
-        $match: {
-          author: { $exists: true, $ne: null },
-          promptGroupAclEntries: { $size: 0 },
-        },
-      },
-      {
-        $project: {
-          _id: 1,
-          name: 1,
-          author: 1,
-          authorName: 1,
-          category: 1,
-        },
-      },
-    ]);
+    const AclEntry = mongoose.model('AclEntry');
+    const migratedGroupIds = await AclEntry.distinct('resourceId', {
+      resourceType: ResourceType.PROMPTGROUP,
+      principalType: PrincipalType.USER,
+    });
+
+    const promptGroupsToMigrate = (await PromptGroupModel.find({
+      _id: { $nin: migratedGroupIds },
+      author: { $exists: true, $ne: null },
+    })
+      .select('_id name author authorName category')
+      .lean()) as unknown as PromptGroupMigrationData[];
 
     const categories: {
       globalViewAccess: PromptGroupMigrationData[];