mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-02-26 04:14:10 +01:00
📦 refactor: Consolidate DB models, encapsulating Mongoose usage in data-schemas (#11830)
* chore: move database model methods to /packages/data-schemas * chore: add TypeScript ESLint rule to warn on unused variables * refactor: model imports to streamline access - Consolidated model imports across various files to improve code organization and reduce redundancy. - Updated imports for models such as Assistant, Message, Conversation, and others to a unified import path. - Adjusted middleware and service files to reflect the new import structure, ensuring functionality remains intact. - Enhanced test files to align with the new import paths, maintaining test coverage and integrity. * chore: migrate database models to packages/data-schemas and refactor all direct Mongoose Model usage outside of data-schemas * test: update agent model mocks in unit tests - Added `getAgent` mock to `client.test.js` to enhance test coverage for agent-related functionality. - Removed redundant `getAgent` and `getAgents` mocks from `openai.spec.js` and `responses.unit.spec.js` to streamline test setup and reduce duplication. - Ensured consistency in agent mock implementations across test files. * fix: update types in data-schemas * refactor: enhance type definitions in transaction and spending methods - Updated type definitions in `checkBalance.ts` to use specific request and response types. - Refined `spendTokens.ts` to utilize a new `SpendTxData` interface for better clarity and type safety. - Improved transaction handling in `transaction.ts` by introducing `TransactionResult` and `TxData` interfaces, ensuring consistent data structures across methods. - Adjusted unit tests in `transaction.spec.ts` to accommodate new type definitions and enhance robustness. * refactor: streamline model imports and enhance code organization - Consolidated model imports across various controllers and services to a unified import path, improving code clarity and reducing redundancy. - Updated multiple files to reflect the new import structure, ensuring all functionalities remain intact. - Enhanced overall code organization by removing duplicate import statements and optimizing the usage of model methods. * feat: implement loadAddedAgent and refactor agent loading logic - Introduced `loadAddedAgent` function to handle loading agents from added conversations, supporting multi-convo parallel execution. - Created a new `load.ts` file to encapsulate agent loading functionalities, including `loadEphemeralAgent` and `loadAgent`. - Updated the `index.ts` file to export the new `load` module instead of the deprecated `loadAgent`. - Enhanced type definitions and improved error handling in the agent loading process. - Adjusted unit tests to reflect changes in the agent loading structure and ensure comprehensive coverage. * refactor: enhance balance handling with new update interface - Introduced `IBalanceUpdate` interface to streamline balance update operations across the codebase. - Updated `upsertBalanceFields` method signatures in `balance.ts`, `transaction.ts`, and related tests to utilize the new interface for improved type safety. - Adjusted type imports in `balance.spec.ts` to include `IBalanceUpdate`, ensuring consistency in balance management functionalities. - Enhanced overall code clarity and maintainability by refining type definitions related to balance operations. * feat: add unit tests for loadAgent functionality and enhance agent loading logic - Introduced comprehensive unit tests for the `loadAgent` function, covering various scenarios including null and empty agent IDs, loading of ephemeral agents, and permission checks. - Enhanced the `initializeClient` function by moving `getConvoFiles` to the correct position in the database method exports, ensuring proper functionality. - Improved test coverage for agent loading, including handling of non-existent agents and user permissions. * chore: reorder memory method exports for consistency - Moved `deleteAllUserMemories` to the correct position in the exported memory methods, ensuring a consistent and logical order of method exports in `memory.ts`.
This commit is contained in:
Danny Avila
parent
10e5c5f677
commit
7a62c42675
182 changed files with 8596 additions and 8153 deletions
|
|
@ -1,7 +1,22 @@
|
|||
import { roleDefaults, SystemRoles } from 'librechat-data-provider';
|
||||
import {
|
||||
CacheKeys,
|
||||
SystemRoles,
|
||||
roleDefaults,
|
||||
permissionsSchema,
|
||||
removeNullishValues,
|
||||
} from 'librechat-data-provider';
|
||||
import type { IRole } from '~/types';
|
||||
import logger from '~/config/winston';
|
||||
|
||||
// Factory function that takes mongoose instance and returns the methods
|
||||
export function createRoleMethods(mongoose: typeof import('mongoose')) {
|
||||
export interface RoleDeps {
|
||||
/** Returns a cache store for the given key. Injected from getLogStores. */
|
||||
getCache?: (key: string) => {
|
||||
get: (k: string) => Promise<unknown>;
|
||||
set: (k: string, v: unknown) => Promise<void>;
|
||||
};
|
||||
}
|
||||
|
||||
export function createRoleMethods(mongoose: typeof import('mongoose'), deps: RoleDeps = {}) {
|
||||
/**
|
||||
* Initialize default roles in the system.
|
||||
* Creates the default roles (ADMIN, USER) if they don't exist in the database.
|
||||
|
|
@ -30,18 +45,310 @@ export function createRoleMethods(mongoose: typeof import('mongoose')) {
|
|||
}
|
||||
|
||||
/**
|
||||
* List all roles in the system (for testing purposes)
|
||||
* Returns an array of all roles with their names and permissions
|
||||
* List all roles in the system.
|
||||
*/
|
||||
async function listRoles() {
|
||||
const Role = mongoose.models.Role;
|
||||
return await Role.find({}).select('name permissions').lean();
|
||||
}
|
||||
|
||||
// Return all methods you want to expose
|
||||
/**
|
||||
* Retrieve a role by name and convert the found role document to a plain object.
|
||||
* If the role with the given name doesn't exist and the name is a system defined role,
|
||||
* create it and return the lean version.
|
||||
*/
|
||||
async function getRoleByName(roleName: string, fieldsToSelect: string | string[] | null = null) {
|
||||
const cache = deps.getCache?.(CacheKeys.ROLES);
|
||||
try {
|
||||
if (cache) {
|
||||
const cachedRole = await cache.get(roleName);
|
||||
if (cachedRole) {
|
||||
return cachedRole as IRole;
|
||||
}
|
||||
}
|
||||
const Role = mongoose.models.Role;
|
||||
let query = Role.findOne({ name: roleName });
|
||||
if (fieldsToSelect) {
|
||||
query = query.select(fieldsToSelect);
|
||||
}
|
||||
const role = await query.lean().exec();
|
||||
|
||||
if (!role && SystemRoles[roleName as keyof typeof SystemRoles]) {
|
||||
const newRole = await new Role(roleDefaults[roleName as keyof typeof roleDefaults]).save();
|
||||
if (cache) {
|
||||
await cache.set(roleName, newRole);
|
||||
}
|
||||
return newRole.toObject() as IRole;
|
||||
}
|
||||
if (cache) {
|
||||
await cache.set(roleName, role);
|
||||
}
|
||||
return role as unknown as IRole;
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to retrieve or create role: ${(error as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update role values by name.
|
||||
*/
|
||||
async function updateRoleByName(roleName: string, updates: Partial<IRole>) {
|
||||
const cache = deps.getCache?.(CacheKeys.ROLES);
|
||||
try {
|
||||
const Role = mongoose.models.Role;
|
||||
const role = await Role.findOneAndUpdate(
|
||||
{ name: roleName },
|
||||
{ $set: updates },
|
||||
{ new: true, lean: true },
|
||||
)
|
||||
.select('-__v')
|
||||
.lean()
|
||||
.exec();
|
||||
if (cache) {
|
||||
await cache.set(roleName, role);
|
||||
}
|
||||
return role as unknown as IRole;
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to update role: ${(error as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates access permissions for a specific role and multiple permission types.
|
||||
*/
|
||||
async function updateAccessPermissions(
|
||||
roleName: string,
|
||||
permissionsUpdate: Record<string, Record<string, boolean>>,
|
||||
roleData?: IRole,
|
||||
) {
|
||||
const updates: Record<string, Record<string, boolean>> = {};
|
||||
for (const [permissionType, permissions] of Object.entries(permissionsUpdate)) {
|
||||
if (
|
||||
permissionsSchema.shape &&
|
||||
permissionsSchema.shape[permissionType as keyof typeof permissionsSchema.shape]
|
||||
) {
|
||||
updates[permissionType] = removeNullishValues(permissions) as Record<string, boolean>;
|
||||
}
|
||||
}
|
||||
if (!Object.keys(updates).length) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const role = roleData ?? (await getRoleByName(roleName));
|
||||
if (!role) {
|
||||
return;
|
||||
}
|
||||
|
||||
const currentPermissions =
|
||||
((role as unknown as Record<string, unknown>).permissions as Record<
|
||||
string,
|
||||
Record<string, boolean>
|
||||
>) || {};
|
||||
const updatedPermissions: Record<string, Record<string, boolean>> = { ...currentPermissions };
|
||||
let hasChanges = false;
|
||||
|
||||
const unsetFields: Record<string, number> = {};
|
||||
const permissionTypes = Object.keys(permissionsSchema.shape || {});
|
||||
for (const permType of permissionTypes) {
|
||||
if (
|
||||
(role as unknown as Record<string, unknown>)[permType] &&
|
||||
typeof (role as unknown as Record<string, unknown>)[permType] === 'object'
|
||||
) {
|
||||
logger.info(
|
||||
`Migrating '${roleName}' role from old schema: found '${permType}' at top level`,
|
||||
);
|
||||
|
||||
updatedPermissions[permType] = {
|
||||
...updatedPermissions[permType],
|
||||
...((role as unknown as Record<string, unknown>)[permType] as Record<string, boolean>),
|
||||
};
|
||||
|
||||
unsetFields[permType] = 1;
|
||||
hasChanges = true;
|
||||
}
|
||||
}
|
||||
|
||||
// Migrate legacy SHARED_GLOBAL → SHARE for PROMPTS and AGENTS.
|
||||
// SHARED_GLOBAL was removed in favour of SHARE in PR #11283. If the DB still has
|
||||
// SHARED_GLOBAL but not SHARE, inherit the value so sharing intent is preserved.
|
||||
const legacySharedGlobalTypes = ['PROMPTS', 'AGENTS'];
|
||||
for (const legacyPermType of legacySharedGlobalTypes) {
|
||||
const existingTypePerms = currentPermissions[legacyPermType];
|
||||
if (
|
||||
existingTypePerms &&
|
||||
'SHARED_GLOBAL' in existingTypePerms &&
|
||||
!('SHARE' in existingTypePerms) &&
|
||||
updates[legacyPermType] &&
|
||||
// Don't override an explicit SHARE value the caller already provided
|
||||
!('SHARE' in updates[legacyPermType])
|
||||
) {
|
||||
const inheritedValue = existingTypePerms['SHARED_GLOBAL'];
|
||||
updates[legacyPermType]['SHARE'] = inheritedValue;
|
||||
logger.info(
|
||||
`Migrating '${roleName}' role ${legacyPermType}.SHARED_GLOBAL=${inheritedValue} → SHARE`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
for (const [permissionType, permissions] of Object.entries(updates)) {
|
||||
const currentTypePermissions = currentPermissions[permissionType] || {};
|
||||
updatedPermissions[permissionType] = { ...currentTypePermissions };
|
||||
|
||||
for (const [permission, value] of Object.entries(permissions)) {
|
||||
if (currentTypePermissions[permission] !== value) {
|
||||
updatedPermissions[permissionType][permission] = value;
|
||||
hasChanges = true;
|
||||
logger.info(
|
||||
`Updating '${roleName}' role permission '${permissionType}' '${permission}' from ${currentTypePermissions[permission]} to: ${value}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up orphaned SHARED_GLOBAL fields left in DB after the schema rename.
|
||||
// Since we $set the full permissions object, deleting from updatedPermissions
|
||||
// is sufficient to remove the field from MongoDB.
|
||||
for (const legacyPermType of legacySharedGlobalTypes) {
|
||||
const existingTypePerms = currentPermissions[legacyPermType];
|
||||
if (existingTypePerms && 'SHARED_GLOBAL' in existingTypePerms) {
|
||||
if (!updates[legacyPermType]) {
|
||||
// permType wasn't in the update payload so the migration block above didn't run.
|
||||
// Create a writable copy and handle the SHARED_GLOBAL → SHARE inheritance here
|
||||
// to avoid removing SHARED_GLOBAL without writing SHARE (data loss).
|
||||
updatedPermissions[legacyPermType] = { ...existingTypePerms };
|
||||
if (!('SHARE' in existingTypePerms)) {
|
||||
updatedPermissions[legacyPermType]['SHARE'] = existingTypePerms['SHARED_GLOBAL'];
|
||||
logger.info(
|
||||
`Migrating '${roleName}' role ${legacyPermType}.SHARED_GLOBAL=${existingTypePerms['SHARED_GLOBAL']} → SHARE`,
|
||||
);
|
||||
}
|
||||
}
|
||||
delete updatedPermissions[legacyPermType]['SHARED_GLOBAL'];
|
||||
hasChanges = true;
|
||||
logger.info(
|
||||
`Removed legacy SHARED_GLOBAL field from '${roleName}' role ${legacyPermType} permissions`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (hasChanges) {
|
||||
const Role = mongoose.models.Role;
|
||||
const updateObj = { permissions: updatedPermissions };
|
||||
|
||||
if (Object.keys(unsetFields).length > 0) {
|
||||
logger.info(
|
||||
`Unsetting old schema fields for '${roleName}' role: ${Object.keys(unsetFields).join(', ')}`,
|
||||
);
|
||||
|
||||
try {
|
||||
await Role.updateOne(
|
||||
{ name: roleName },
|
||||
{
|
||||
$set: updateObj,
|
||||
$unset: unsetFields,
|
||||
},
|
||||
);
|
||||
|
||||
const cache = deps.getCache?.(CacheKeys.ROLES);
|
||||
const updatedRole = await Role.findOne({ name: roleName }).select('-__v').lean().exec();
|
||||
if (cache) {
|
||||
await cache.set(roleName, updatedRole);
|
||||
}
|
||||
|
||||
logger.info(`Updated role '${roleName}' and removed old schema fields`);
|
||||
} catch (updateError) {
|
||||
logger.error(`Error during role migration update: ${(updateError as Error).message}`);
|
||||
throw updateError;
|
||||
}
|
||||
} else {
|
||||
await updateRoleByName(roleName, updateObj as unknown as Partial<IRole>);
|
||||
}
|
||||
|
||||
logger.info(`Updated '${roleName}' role permissions`);
|
||||
} else {
|
||||
logger.info(`No changes needed for '${roleName}' role permissions`);
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(`Failed to update ${roleName} role permissions:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Migrates roles from old schema to new schema structure.
|
||||
*/
|
||||
async function migrateRoleSchema(roleName?: string): Promise<number> {
|
||||
try {
|
||||
const Role = mongoose.models.Role;
|
||||
let roles;
|
||||
if (roleName) {
|
||||
const role = await Role.findOne({ name: roleName });
|
||||
roles = role ? [role] : [];
|
||||
} else {
|
||||
roles = await Role.find({});
|
||||
}
|
||||
|
||||
logger.info(`Migrating ${roles.length} roles to new schema structure`);
|
||||
let migratedCount = 0;
|
||||
|
||||
for (const role of roles) {
|
||||
const permissionTypes = Object.keys(permissionsSchema.shape || {});
|
||||
const unsetFields: Record<string, number> = {};
|
||||
let hasOldSchema = false;
|
||||
|
||||
for (const permType of permissionTypes) {
|
||||
if (role[permType] && typeof role[permType] === 'object') {
|
||||
hasOldSchema = true;
|
||||
role.permissions = role.permissions || {};
|
||||
role.permissions[permType] = {
|
||||
...role.permissions[permType],
|
||||
...role[permType],
|
||||
};
|
||||
unsetFields[permType] = 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (hasOldSchema) {
|
||||
try {
|
||||
logger.info(`Migrating role '${role.name}' from old schema structure`);
|
||||
|
||||
await Role.updateOne(
|
||||
{ _id: role._id },
|
||||
{
|
||||
$set: { permissions: role.permissions },
|
||||
$unset: unsetFields,
|
||||
},
|
||||
);
|
||||
|
||||
const cache = deps.getCache?.(CacheKeys.ROLES);
|
||||
if (cache) {
|
||||
const updatedRole = await Role.findById(role._id).lean().exec();
|
||||
await cache.set(role.name, updatedRole);
|
||||
}
|
||||
|
||||
migratedCount++;
|
||||
logger.info(`Migrated role '${role.name}'`);
|
||||
} catch (error) {
|
||||
logger.error(`Failed to migrate role '${role.name}': ${(error as Error).message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logger.info(`Migration complete: ${migratedCount} roles migrated`);
|
||||
return migratedCount;
|
||||
} catch (error) {
|
||||
logger.error(`Role schema migration failed: ${(error as Error).message}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
listRoles,
|
||||
initializeRoles,
|
||||
getRoleByName,
|
||||
updateRoleByName,
|
||||
updateAccessPermissions,
|
||||
migrateRoleSchema,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue