LibreChat/packages/api/src/prompts/migration.ts

203 lines
6.3 KiB
TypeScript
Raw Normal View History

import { logger } from '@librechat/data-schemas';
import { AccessRoleIds, ResourceType, PrincipalType, Constants } from 'librechat-data-provider';
import { ensureRequiredCollectionsExist } from '../db/utils';
import type { AccessRoleMethods, IPromptGroupDocument } from '@librechat/data-schemas';
import type { Model, Mongoose } from 'mongoose';
const { GLOBAL_PROJECT_NAME } = Constants;
export interface PromptMigrationCheckDbMethods {
findRoleByIdentifier: AccessRoleMethods['findRoleByIdentifier'];
getProjectByName: (
projectName: string,
fieldsToSelect?: string[] | null,
) => Promise<{
promptGroupIds?: string[];
[key: string]: unknown;
} | null>;
}
export interface PromptMigrationCheckParams {
mongoose: Mongoose;
methods: PromptMigrationCheckDbMethods;
PromptGroupModel: Model<IPromptGroupDocument>;
}
interface PromptGroupMigrationData {
🐘 feat: FerretDB Compatibility (#11769) * feat: replace unsupported MongoDB aggregation operators for FerretDB compatibility Replace $lookup, $unwind, $sample, $replaceRoot, and $addFields aggregation stages which are unsupported on FerretDB v2.x (postgres-documentdb backend). - Prompt.js: Replace $lookup/$unwind/$project pipelines with find().select().lean() + attachProductionPrompts() batch helper. Replace $group/$replaceRoot/$sample in getRandomPromptGroups with distinct() + Fisher-Yates shuffle. - Agent/Prompt migration scripts: Replace $lookup anti-join pattern with distinct() + $nin two-step queries for finding un-migrated resources. All replacement patterns verified against FerretDB v2.7.0. Co-authored-by: Cursor <cursoragent@cursor.com> * fix: use $pullAll for simple array removals, fix memberIds type mismatches Replace $pull with $pullAll for exact-value scalar array removals. Both operators work on MongoDB and FerretDB, but $pullAll is more explicit for exact matching (no condition expressions). Fix critical type mismatch bugs where ObjectId values were used against String[] memberIds arrays in Group queries: - config/delete-user.js: use string uid instead of ObjectId user._id - e2e/setup/cleanupUser.ts: convert userId.toString() before query Harden PermissionService.bulkUpdateResourcePermissions abort handling to prevent crash when abortTransaction is called after commitTransaction. All changes verified against FerretDB v2.7.0 and MongoDB Memory Server. Co-authored-by: Cursor <cursoragent@cursor.com> * fix: harden transaction support probe for FerretDB compatibility Commit the transaction before aborting in supportsTransactions probe, and wrap abortTransaction in try-catch to prevent crashes when abort is called after a successful commit (observed behavior on FerretDB). Co-authored-by: Cursor <cursoragent@cursor.com> * feat: add FerretDB compatibility test suite, retry utilities, and CI config Add comprehensive FerretDB integration test suite covering: - $pullAll scalar array operations - $pull with subdocument conditions - $lookup replacement (find + manual join) - $sample replacement (distinct + Fisher-Yates) - $bit and $bitsAllSet operations - Migration anti-join pattern - Multi-tenancy (useDb, scaling, write amplification) - Sharding proof-of-concept - Production operations (backup/restore, schema migration, deadlock retry) Add production retryWithBackoff utility for deadlock recovery during concurrent index creation on FerretDB/DocumentDB backends. Add UserController.spec.js tests for deleteUserController (runs in CI). Configure jest and eslint to isolate FerretDB tests from CI pipelines: - packages/data-schemas/jest.config.mjs: ignore misc/ directory - eslint.config.mjs: ignore packages/data-schemas/misc/ Include Docker Compose config for local FerretDB v2.7 + postgres-documentdb, dedicated jest/tsconfig for the test files, and multi-tenancy findings doc. Co-authored-by: Cursor <cursoragent@cursor.com> * style: brace formatting in aclEntry.ts modifyPermissionBits Co-authored-by: Cursor <cursoragent@cursor.com> * refactor: reorganize retry utilities and update imports - Moved retryWithBackoff utility to a new file `retry.ts` for better structure. - Updated imports in `orgOperations.ferretdb.spec.ts` to reflect the new location of retry utilities. - Removed old import statement for retryWithBackoff from index.ts to streamline exports. * test: add $pullAll coverage for ConversationTag and PermissionService Add integration tests for deleteConversationTag verifying $pullAll removes tags from conversations correctly, and for syncUserEntraGroupMemberships verifying $pullAll removes user from non-matching Entra groups while preserving local group membership. --------- Co-authored-by: Cursor <cursoragent@cursor.com>
2026-02-13 02:14:34 -05:00
_id: { toString(): string };
name: string;
author: string;
authorName?: string;
category?: string;
}
export interface PromptMigrationCheckResult {
totalToMigrate: number;
globalViewAccess: number;
privateGroups: number;
details?: {
globalViewAccess: Array<{ name: string; _id: string; category: string }>;
privateGroups: Array<{ name: string; _id: string; category: string }>;
};
}
/**
* Check if prompt groups need to be migrated to the new permission system
* This performs a dry-run check similar to the migration script
*/
export async function checkPromptPermissionsMigration({
methods,
mongoose,
PromptGroupModel,
}: PromptMigrationCheckParams): Promise<PromptMigrationCheckResult> {
logger.debug('Checking if prompt permissions migration is needed');
try {
/** Native MongoDB database instance */
const db = mongoose.connection.db;
if (db) {
await ensureRequiredCollectionsExist(db);
}
// Verify required roles exist
const ownerRole = await methods.findRoleByIdentifier(AccessRoleIds.PROMPTGROUP_OWNER);
const viewerRole = await methods.findRoleByIdentifier(AccessRoleIds.PROMPTGROUP_VIEWER);
const editorRole = await methods.findRoleByIdentifier(AccessRoleIds.PROMPTGROUP_EDITOR);
if (!ownerRole || !viewerRole || !editorRole) {
logger.warn(
'Required promptGroup roles not found. Permission system may not be fully initialized.',
);
return {
totalToMigrate: 0,
globalViewAccess: 0,
privateGroups: 0,
};
}
/** Global project prompt group IDs */
const globalProject = await methods.getProjectByName(GLOBAL_PROJECT_NAME, ['promptGroupIds']);
const globalPromptGroupIds = new Set(
(globalProject?.promptGroupIds || []).map((id) => id.toString()),
);
🐘 feat: FerretDB Compatibility (#11769) * feat: replace unsupported MongoDB aggregation operators for FerretDB compatibility Replace $lookup, $unwind, $sample, $replaceRoot, and $addFields aggregation stages which are unsupported on FerretDB v2.x (postgres-documentdb backend). - Prompt.js: Replace $lookup/$unwind/$project pipelines with find().select().lean() + attachProductionPrompts() batch helper. Replace $group/$replaceRoot/$sample in getRandomPromptGroups with distinct() + Fisher-Yates shuffle. - Agent/Prompt migration scripts: Replace $lookup anti-join pattern with distinct() + $nin two-step queries for finding un-migrated resources. All replacement patterns verified against FerretDB v2.7.0. Co-authored-by: Cursor <cursoragent@cursor.com> * fix: use $pullAll for simple array removals, fix memberIds type mismatches Replace $pull with $pullAll for exact-value scalar array removals. Both operators work on MongoDB and FerretDB, but $pullAll is more explicit for exact matching (no condition expressions). Fix critical type mismatch bugs where ObjectId values were used against String[] memberIds arrays in Group queries: - config/delete-user.js: use string uid instead of ObjectId user._id - e2e/setup/cleanupUser.ts: convert userId.toString() before query Harden PermissionService.bulkUpdateResourcePermissions abort handling to prevent crash when abortTransaction is called after commitTransaction. All changes verified against FerretDB v2.7.0 and MongoDB Memory Server. Co-authored-by: Cursor <cursoragent@cursor.com> * fix: harden transaction support probe for FerretDB compatibility Commit the transaction before aborting in supportsTransactions probe, and wrap abortTransaction in try-catch to prevent crashes when abort is called after a successful commit (observed behavior on FerretDB). Co-authored-by: Cursor <cursoragent@cursor.com> * feat: add FerretDB compatibility test suite, retry utilities, and CI config Add comprehensive FerretDB integration test suite covering: - $pullAll scalar array operations - $pull with subdocument conditions - $lookup replacement (find + manual join) - $sample replacement (distinct + Fisher-Yates) - $bit and $bitsAllSet operations - Migration anti-join pattern - Multi-tenancy (useDb, scaling, write amplification) - Sharding proof-of-concept - Production operations (backup/restore, schema migration, deadlock retry) Add production retryWithBackoff utility for deadlock recovery during concurrent index creation on FerretDB/DocumentDB backends. Add UserController.spec.js tests for deleteUserController (runs in CI). Configure jest and eslint to isolate FerretDB tests from CI pipelines: - packages/data-schemas/jest.config.mjs: ignore misc/ directory - eslint.config.mjs: ignore packages/data-schemas/misc/ Include Docker Compose config for local FerretDB v2.7 + postgres-documentdb, dedicated jest/tsconfig for the test files, and multi-tenancy findings doc. Co-authored-by: Cursor <cursoragent@cursor.com> * style: brace formatting in aclEntry.ts modifyPermissionBits Co-authored-by: Cursor <cursoragent@cursor.com> * refactor: reorganize retry utilities and update imports - Moved retryWithBackoff utility to a new file `retry.ts` for better structure. - Updated imports in `orgOperations.ferretdb.spec.ts` to reflect the new location of retry utilities. - Removed old import statement for retryWithBackoff from index.ts to streamline exports. * test: add $pullAll coverage for ConversationTag and PermissionService Add integration tests for deleteConversationTag verifying $pullAll removes tags from conversations correctly, and for syncUserEntraGroupMemberships verifying $pullAll removes user from non-matching Entra groups while preserving local group membership. --------- Co-authored-by: Cursor <cursoragent@cursor.com>
2026-02-13 02:14:34 -05:00
const AclEntry = mongoose.model('AclEntry');
const migratedGroupIds = await AclEntry.distinct('resourceId', {
resourceType: ResourceType.PROMPTGROUP,
principalType: PrincipalType.USER,
});
const promptGroupsToMigrate = (await PromptGroupModel.find({
_id: { $nin: migratedGroupIds },
author: { $exists: true, $ne: null },
})
.select('_id name author authorName category')
.lean()) as unknown as PromptGroupMigrationData[];
const categories: {
globalViewAccess: PromptGroupMigrationData[];
privateGroups: PromptGroupMigrationData[];
} = {
globalViewAccess: [],
privateGroups: [],
};
promptGroupsToMigrate.forEach((group) => {
const isGlobalGroup = globalPromptGroupIds.has(group._id.toString());
if (isGlobalGroup) {
categories.globalViewAccess.push(group);
} else {
categories.privateGroups.push(group);
}
});
const result: PromptMigrationCheckResult = {
totalToMigrate: promptGroupsToMigrate.length,
globalViewAccess: categories.globalViewAccess.length,
privateGroups: categories.privateGroups.length,
};
// Add details for debugging
if (promptGroupsToMigrate.length > 0) {
result.details = {
globalViewAccess: categories.globalViewAccess.map((g) => ({
name: g.name,
_id: g._id.toString(),
category: g.category || 'uncategorized',
})),
privateGroups: categories.privateGroups.map((g) => ({
name: g.name,
_id: g._id.toString(),
category: g.category || 'uncategorized',
})),
};
}
logger.debug('Prompt migration check completed', {
totalToMigrate: result.totalToMigrate,
globalViewAccess: result.globalViewAccess,
privateGroups: result.privateGroups,
});
return result;
} catch (error) {
logger.error('Failed to check prompt permissions migration', error);
// Return zero counts on error to avoid blocking startup
return {
totalToMigrate: 0,
globalViewAccess: 0,
privateGroups: 0,
};
}
}
/**
* Log migration warning to console if prompt groups need migration
*/
export function logPromptMigrationWarning(result: PromptMigrationCheckResult): void {
if (result.totalToMigrate === 0) {
return;
}
// Create a visible warning box
const border = '='.repeat(80);
const warning = [
'',
border,
' IMPORTANT: PROMPT PERMISSIONS MIGRATION REQUIRED',
border,
'',
` Total prompt groups to migrate: ${result.totalToMigrate}`,
` - Global View Access: ${result.globalViewAccess} prompt groups`,
` - Private Prompt Groups: ${result.privateGroups} prompt groups`,
'',
' The new prompt sharing system requires migrating existing prompt groups.',
' Please run the following command to migrate your prompts:',
'',
' npm run migrate:prompt-permissions',
'',
' For a dry run (preview) of what will be migrated:',
'',
' npm run migrate:prompt-permissions:dry-run',
'',
' This migration will:',
' 1. Grant owner permissions to prompt authors',
' 2. Set public view permissions for prompts in the global project',
' 3. Keep private prompts accessible only to their authors',
'',
border,
'',
];
// Use console methods directly for visibility
console.log('\n' + warning.join('\n') + '\n');
// Also log with logger for consistency
logger.warn('Prompt permissions migration required', {
totalToMigrate: result.totalToMigrate,
globalViewAccess: result.globalViewAccess,
privateGroups: result.privateGroups,
});
}