🐘 feat: FerretDB Compatibility (#11769)

* feat: replace unsupported MongoDB aggregation operators for FerretDB compatibility

Replace $lookup, $unwind, $sample, $replaceRoot, and $addFields aggregation
stages which are unsupported on FerretDB v2.x (postgres-documentdb backend).

- Prompt.js: Replace $lookup/$unwind/$project pipelines with find().select().lean()
  + attachProductionPrompts() batch helper. Replace $group/$replaceRoot/$sample
  in getRandomPromptGroups with distinct() + Fisher-Yates shuffle.
- Agent/Prompt migration scripts: Replace $lookup anti-join pattern with
  distinct() + $nin two-step queries for finding un-migrated resources.

All replacement patterns verified against FerretDB v2.7.0.

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix: use $pullAll for simple array removals, fix memberIds type mismatches

Replace $pull with $pullAll for exact-value scalar array removals. Both
operators work on MongoDB and FerretDB, but $pullAll is more explicit for
exact matching (no condition expressions).

Fix critical type mismatch bugs where ObjectId values were used against
String[] memberIds arrays in Group queries:
- config/delete-user.js: use string uid instead of ObjectId user._id
- e2e/setup/cleanupUser.ts: convert userId.toString() before query

Harden PermissionService.bulkUpdateResourcePermissions abort handling to
prevent crash when abortTransaction is called after commitTransaction.

All changes verified against FerretDB v2.7.0 and MongoDB Memory Server.

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix: harden transaction support probe for FerretDB compatibility

Commit the transaction before aborting in supportsTransactions probe, and
wrap abortTransaction in try-catch to prevent crashes when abort is called
after a successful commit (observed behavior on FerretDB).

Co-authored-by: Cursor <cursoragent@cursor.com>

* feat: add FerretDB compatibility test suite, retry utilities, and CI config

Add comprehensive FerretDB integration test suite covering:
- $pullAll scalar array operations
- $pull with subdocument conditions
- $lookup replacement (find + manual join)
- $sample replacement (distinct + Fisher-Yates)
- $bit and $bitsAllSet operations
- Migration anti-join pattern
- Multi-tenancy (useDb, scaling, write amplification)
- Sharding proof-of-concept
- Production operations (backup/restore, schema migration, deadlock retry)

Add production retryWithBackoff utility for deadlock recovery during
concurrent index creation on FerretDB/DocumentDB backends.

Add UserController.spec.js tests for deleteUserController (runs in CI).

Configure jest and eslint to isolate FerretDB tests from CI pipelines:
- packages/data-schemas/jest.config.mjs: ignore misc/ directory
- eslint.config.mjs: ignore packages/data-schemas/misc/

Include Docker Compose config for local FerretDB v2.7 + postgres-documentdb,
dedicated jest/tsconfig for the test files, and multi-tenancy findings doc.

Co-authored-by: Cursor <cursoragent@cursor.com>

* style: brace formatting in aclEntry.ts modifyPermissionBits

Co-authored-by: Cursor <cursoragent@cursor.com>

* refactor: reorganize retry utilities and update imports

- Moved retryWithBackoff utility to a new file `retry.ts` for better structure.
- Updated imports in `orgOperations.ferretdb.spec.ts` to reflect the new location of retry utilities.
- Removed old import statement for retryWithBackoff from index.ts to streamline exports.

* test: add $pullAll coverage for ConversationTag and PermissionService

Add integration tests for deleteConversationTag verifying $pullAll
removes tags from conversations correctly, and for
syncUserEntraGroupMemberships verifying $pullAll removes user from
non-matching Entra groups while preserving local group membership.

---------

Co-authored-by: Cursor <cursoragent@cursor.com>

api/models/ConversationTag.spec.js

@@ -0,0 +1,114 @@
+const mongoose = require('mongoose');
+const { MongoMemoryServer } = require('mongodb-memory-server');
+const { ConversationTag, Conversation } = require('~/db/models');
+const { deleteConversationTag } = require('./ConversationTag');
+
+let mongoServer;
+
+beforeAll(async () => {
+  mongoServer = await MongoMemoryServer.create();
+  await mongoose.connect(mongoServer.getUri());
+});
+
+afterAll(async () => {
+  await mongoose.disconnect();
+  await mongoServer.stop();
+});
+
+afterEach(async () => {
+  await ConversationTag.deleteMany({});
+  await Conversation.deleteMany({});
+});
+
+describe('ConversationTag model - $pullAll operations', () => {
+  const userId = new mongoose.Types.ObjectId().toString();
+
+  describe('deleteConversationTag', () => {
+    it('should remove the tag from all conversations that have it', async () => {
+      await ConversationTag.create({ tag: 'work', user: userId, position: 1 });
+
+      await Conversation.create([
+        { conversationId: 'conv1', user: userId, endpoint: 'openAI', tags: ['work', 'important'] },
+        { conversationId: 'conv2', user: userId, endpoint: 'openAI', tags: ['work'] },
+        { conversationId: 'conv3', user: userId, endpoint: 'openAI', tags: ['personal'] },
+      ]);
+
+      await deleteConversationTag(userId, 'work');
+
+      const convos = await Conversation.find({ user: userId }).sort({ conversationId: 1 }).lean();
+      expect(convos[0].tags).toEqual(['important']);
+      expect(convos[1].tags).toEqual([]);
+      expect(convos[2].tags).toEqual(['personal']);
+    });
+
+    it('should delete the tag document itself', async () => {
+      await ConversationTag.create({ tag: 'temp', user: userId, position: 1 });
+
+      const result = await deleteConversationTag(userId, 'temp');
+
+      expect(result).toBeDefined();
+      expect(result.tag).toBe('temp');
+
+      const remaining = await ConversationTag.find({ user: userId }).lean();
+      expect(remaining).toHaveLength(0);
+    });
+
+    it('should return null when the tag does not exist', async () => {
+      const result = await deleteConversationTag(userId, 'nonexistent');
+      expect(result).toBeNull();
+    });
+
+    it('should adjust positions of tags after the deleted one', async () => {
+      await ConversationTag.create([
+        { tag: 'first', user: userId, position: 1 },
+        { tag: 'second', user: userId, position: 2 },
+        { tag: 'third', user: userId, position: 3 },
+      ]);
+
+      await deleteConversationTag(userId, 'first');
+
+      const tags = await ConversationTag.find({ user: userId }).sort({ position: 1 }).lean();
+      expect(tags).toHaveLength(2);
+      expect(tags[0].tag).toBe('second');
+      expect(tags[0].position).toBe(1);
+      expect(tags[1].tag).toBe('third');
+      expect(tags[1].position).toBe(2);
+    });
+
+    it('should not affect conversations of other users', async () => {
+      const otherUser = new mongoose.Types.ObjectId().toString();
+
+      await ConversationTag.create({ tag: 'shared-name', user: userId, position: 1 });
+      await ConversationTag.create({ tag: 'shared-name', user: otherUser, position: 1 });
+
+      await Conversation.create([
+        { conversationId: 'mine', user: userId, endpoint: 'openAI', tags: ['shared-name'] },
+        { conversationId: 'theirs', user: otherUser, endpoint: 'openAI', tags: ['shared-name'] },
+      ]);
+
+      await deleteConversationTag(userId, 'shared-name');
+
+      const myConvo = await Conversation.findOne({ conversationId: 'mine' }).lean();
+      const theirConvo = await Conversation.findOne({ conversationId: 'theirs' }).lean();
+
+      expect(myConvo.tags).toEqual([]);
+      expect(theirConvo.tags).toEqual(['shared-name']);
+    });
+
+    it('should handle duplicate tags in conversations correctly', async () => {
+      await ConversationTag.create({ tag: 'dup', user: userId, position: 1 });
+
+      const conv = await Conversation.create({
+        conversationId: 'conv-dup',
+        user: userId,
+        endpoint: 'openAI',
+        tags: ['dup', 'other', 'dup'],
+      });
+
+      await deleteConversationTag(userId, 'dup');
+
+      const updated = await Conversation.findById(conv._id).lean();
+      expect(updated.tags).toEqual(['other']);
+    });
+  });
+});