🏗️ refactor: Extract DB layers to data-schemas for shared use (#7650)

* refactor: move model definitions and database-related methods to packages/data-schemas

* ci: update tests due to new DB structure

fix: disable mocking `librechat-data-provider`

feat: Add schema exports to data-schemas package

- Introduced a new schema module that exports various schemas including action, agent, and user schemas.
- Updated index.ts to include the new schema exports for better modularity and organization.

ci: fix appleStrategy tests

fix: Agent.spec.js

ci: refactor handleTools tests to use MongoMemoryServer for in-memory database

fix: getLogStores imports

ci: update banViolation tests to use MongoMemoryServer and improve session mocking

test: refactor samlStrategy tests to improve mock configurations and user handling

ci: fix crypto mock in handleText tests for improved accuracy

ci: refactor spendTokens tests to improve model imports and setup

ci: refactor Message model tests to use MongoMemoryServer and improve database interactions

* refactor: streamline IMessage interface and move feedback properties to types/message.ts

* refactor: use exported initializeRoles from `data-schemas`, remove api workspace version (this serves as an example of future migrations that still need to happen)

* refactor: update model imports to use destructuring from `~/db/models` for consistency and clarity

* refactor: remove unused mongoose imports from model files for cleaner code

* refactor: remove unused mongoose imports from Share, Prompt, and Transaction model files for cleaner code

* refactor: remove unused import in Transaction model for cleaner code

* ci: update deploy workflow to reference new Docker Dev Branch Images Build and add new workflow for building Docker images on dev branch

* chore: cleanup imports

api/server/controllers/AuthController.js

@@ -1,6 +1,7 @@
-const openIdClient = require('openid-client');
 const cookies = require('cookie');
 const jwt = require('jsonwebtoken');
+const openIdClient = require('openid-client');
+const { logger } = require('@librechat/data-schemas');
 const {
   registerUser,
   resetPassword,
@@ -8,9 +9,8 @@ const {
   requestPasswordReset,
   setOpenIDAuthTokens,
 } = require('~/server/services/AuthService');
-const { findSession, getUserById, deleteAllUserSessions, findUser } = require('~/models');
+const { findUser, getUserById, deleteAllUserSessions, findSession } = require('~/models');
 const { getOpenIdConfig } = require('~/strategies');
-const { logger } = require('~/config');
 const { isEnabled } = require('~/server/utils');
 
 const registrationController = async (req, res) => {
@@ -96,7 +96,10 @@ const refreshController = async (req, res) => {
     }
 
     // Find the session with the hashed refresh token
-    const session = await findSession({ userId: userId, refreshToken: refreshToken });
+    const session = await findSession({
+      userId: userId,
+      refreshToken: refreshToken,
+    });
 
     if (session && session.expiration > new Date()) {
       const token = await setAuthTokens(userId, res, session._id);

api/server/controllers/Balance.js

@@ -1,4 +1,4 @@
-const Balance = require('~/models/Balance');
+const { Balance } = require('~/db/models');
 
 async function balanceController(req, res) {
   const balanceData = await Balance.findOne(

api/server/controllers/TwoFactorController.js

@@ -1,12 +1,12 @@
+const { logger } = require('@librechat/data-schemas');
 const {
+  verifyTOTP,
+  getTOTPSecret,
+  verifyBackupCode,
   generateTOTPSecret,
   generateBackupCodes,
-  verifyTOTP,
-  verifyBackupCode,
-  getTOTPSecret,
 } = require('~/server/services/twoFactorService');
-const { updateUser, getUserById } = require('~/models');
-const { logger } = require('~/config');
+const { getUserById, updateUser } = require('~/models');
 const { encryptV3 } = require('~/server/utils/crypto');
 
 const safeAppTitle = (process.env.APP_TITLE || 'LibreChat').replace(/\s+/g, '');

api/server/controllers/UserController.js

@@ -1,12 +1,11 @@
 const {
   Tools,
-  Constants,
   FileSources,
   webSearchKeys,
   extractWebSearchEnvVars,
 } = require('librechat-data-provider');
+const { logger } = require('@librechat/data-schemas');
 const {
-  Balance,
   getFiles,
   updateUser,
   deleteFiles,
@@ -16,16 +15,14 @@ const {
   deleteUserById,
   deleteAllUserSessions,
 } = require('~/models');
-const User = require('~/models/User');
 const { updateUserPluginAuth, deleteUserPluginAuth } = require('~/server/services/PluginService');
 const { updateUserPluginsService, deleteUserKey } = require('~/server/services/UserService');
 const { verifyEmail, resendVerificationEmail } = require('~/server/services/AuthService');
 const { needsRefresh, getNewS3URL } = require('~/server/services/Files/S3/crud');
 const { processDeleteRequest } = require('~/server/services/Files/process');
+const { Transaction, Balance, User } = require('~/db/models');
 const { deleteAllSharedLinks } = require('~/models/Share');
 const { deleteToolCalls } = require('~/models/ToolCall');
-const { Transaction } = require('~/models/Transaction');
-const { logger } = require('~/config');
 
 const getUserController = async (req, res) => {
   /** @type {MongoUser} */

api/server/controllers/auth/TwoFactorAuthController.js

@@ -1,12 +1,12 @@
 const jwt = require('jsonwebtoken');
+const { logger } = require('@librechat/data-schemas');
 const {
   verifyTOTP,
-  verifyBackupCode,
   getTOTPSecret,
+  verifyBackupCode,
 } = require('~/server/services/twoFactorService');
 const { setAuthTokens } = require('~/server/services/AuthService');
-const { getUserById } = require('~/models/userMethods');
-const { logger } = require('~/config');
+const { getUserById } = require('~/models');
 
 /**
  * Verifies the 2FA code during login using a temporary token.

api/server/index.js

@@ -9,8 +9,9 @@ const passport = require('passport');
 const mongoSanitize = require('express-mongo-sanitize');
 const fs = require('fs');
 const cookieParser = require('cookie-parser');
+const { connectDb, indexSync } = require('~/db');
+
 const { jwtLogin, passportLogin } = require('~/strategies');
-const { connectDb, indexSync } = require('~/lib/db');
 const { isEnabled } = require('~/server/utils');
 const { ldapLogin } = require('~/strategies');
 const { logger } = require('~/config');
@@ -36,6 +37,7 @@ const startServer = async () => {
     axios.defaults.headers.common['Accept-Encoding'] = 'gzip';
   }
   await connectDb();
+
   logger.info('Connected to MongoDB');
   await indexSync();
 

api/server/middleware/checkBan.js

@@ -1,12 +1,12 @@
 const { Keyv } = require('keyv');
 const uap = require('ua-parser-js');
+const { logger } = require('@librechat/data-schemas');
 const { ViolationTypes } = require('librechat-data-provider');
 const { isEnabled, removePorts } = require('~/server/utils');
 const keyvMongo = require('~/cache/keyvMongo');
 const denyRequest = require('./denyRequest');
 const { getLogStores } = require('~/cache');
 const { findUser } = require('~/models');
-const { logger } = require('~/config');
 
 const banCache = new Keyv({ store: keyvMongo, namespace: ViolationTypes.BAN, ttl: 0 });
 const message = 'Your account has been temporarily banned due to violations of our service.';

api/server/middleware/checkInviteUser.js

@@ -1,5 +1,5 @@
 const { getInvite } = require('~/models/inviteUser');
-const { deleteTokens } = require('~/models/Token');
+const { deleteTokens } = require('~/models');
 
 async function checkInviteUser(req, res, next) {
   const token = req.body.token;

api/server/middleware/setBalanceConfig.js

@@ -1,6 +1,6 @@
+const { logger } = require('@librechat/data-schemas');
 const { getBalanceConfig } = require('~/server/services/Config');
-const Balance = require('~/models/Balance');
-const { logger } = require('~/config');
+const { Balance } = require('~/db/models');
 
 /**
  * Middleware to synchronize user balance settings with current balance configuration.

api/server/routes/messages.js

@@ -1,4 +1,5 @@
 const express = require('express');
+const { logger } = require('@librechat/data-schemas');
 const { ContentTypes } = require('librechat-data-provider');
 const {
   saveConvo,
@@ -13,8 +14,7 @@ const { requireJwtAuth, validateMessageReq } = require('~/server/middleware');
 const { cleanUpPrimaryKeyValue } = require('~/lib/utils/misc');
 const { getConvosQueried } = require('~/models/Conversation');
 const { countTokens } = require('~/server/utils');
-const { Message } = require('~/models/Message');
-const { logger } = require('~/config');
+const { Message } = require('~/db/models');
 
 const router = express.Router();
 router.use(requireJwtAuth);
@@ -40,7 +40,11 @@ router.get('/', async (req, res) => {
     const sortOrder = sortDirection === 'asc' ? 1 : -1;
 
     if (conversationId && messageId) {
-      const message = await Message.findOne({ conversationId, messageId, user: user }).lean();
+      const message = await Message.findOne({
+        conversationId,
+        messageId,
+        user: user,
+      }).lean();
       response = { messages: message ? [message] : [], nextCursor: null };
     } else if (conversationId) {
       const filter = { conversationId, user: user };

api/server/services/ActionService.js

@@ -17,9 +17,9 @@ const { logger, getFlowStateManager, sendEvent } = require('~/config');
 const { encryptV2, decryptV2 } = require('~/server/utils/crypto');
 const { getActions, deleteActions } = require('~/models/Action');
 const { deleteAssistant } = require('~/models/Assistant');
-const { findToken } = require('~/models/Token');
 const { logAxiosError } = require('~/utils');
 const { getLogStores } = require('~/cache');
+const { findToken } = require('~/models');
 
 const JWT_SECRET = process.env.JWT_SECRET;
 const toolNameRegex = /^[a-zA-Z0-9_-]+$/;

api/server/services/AppService.interface.spec.js

@@ -1,5 +1,7 @@
-jest.mock('~/models/Role', () => ({
+jest.mock('~/models', () => ({
   initializeRoles: jest.fn(),
+}));
+jest.mock('~/models/Role', () => ({
   updateAccessPermissions: jest.fn(),
   getRoleByName: jest.fn(),
   updateRoleByName: jest.fn(),

api/server/services/AppService.js

@@ -25,8 +25,8 @@ const { processModelSpecs } = require('./start/modelSpecs');
 const { initializeS3 } = require('./Files/S3/initialize');
 const { loadAndFormatTools } = require('./ToolService');
 const { agentsConfigSetup } = require('./start/agents');
-const { initializeRoles } = require('~/models/Role');
 const { isEnabled } = require('~/server/utils');
+const { initializeRoles } = require('~/models');
 const { getMCPManager } = require('~/config');
 const paths = require('~/config/paths');
 

api/server/services/AppService.spec.js

@@ -24,8 +24,10 @@ jest.mock('./Config/loadCustomConfig', () => {
 jest.mock('./Files/Firebase/initialize', () => ({
   initializeFirebase: jest.fn(),
 }));
-jest.mock('~/models/Role', () => ({
+jest.mock('~/models', () => ({
   initializeRoles: jest.fn(),
+}));
+jest.mock('~/models/Role', () => ({
   updateAccessPermissions: jest.fn(),
 }));
 jest.mock('./ToolService', () => ({

api/server/services/AuthService.js

@@ -3,24 +3,23 @@ const { webcrypto } = require('node:crypto');
 const { SystemRoles, errorsToString } = require('librechat-data-provider');
 const {
   findUser,
-  countUsers,
   createUser,
   updateUser,
-  getUserById,
-  generateToken,
-  deleteUserById,
-} = require('~/models/userMethods');
-const {
-  createToken,
   findToken,
-  deleteTokens,
+  countUsers,
+  getUserById,
   findSession,
+  createToken,
+  deleteTokens,
   deleteSession,
   createSession,
+  generateToken,
+  deleteUserById,
   generateRefreshToken,
 } = require('~/models');
 const { isEnabled, checkEmailConfig, sendEmail } = require('~/server/utils');
 const { isEmailDomainAllowed } = require('~/server/services/domains');
+const { getBalanceConfig } = require('~/server/services/Config');
 const { registerSchema } = require('~/strategies/validators');
 const { logger } = require('~/config');
 
@@ -146,6 +145,7 @@ const verifyEmail = async (req) => {
   }
 
   const updatedUser = await updateUser(emailVerificationData.userId, { emailVerified: true });
+
   if (!updatedUser) {
     logger.warn(`[verifyEmail] [User update failed] [Email: ${decodedEmail}]`);
     return new Error('Failed to update user verification status');
@@ -155,6 +155,7 @@ const verifyEmail = async (req) => {
   logger.info(`[verifyEmail] Email verification successful [Email: ${decodedEmail}]`);
   return { message: 'Email verification was successful', status: 'success' };
 };
+
 /**
  * Register a new user.
  * @param {MongoUser} user <email, password, name, username>
@@ -216,7 +217,9 @@ const registerUser = async (user, additionalData = {}) => {
 
     const emailEnabled = checkEmailConfig();
     const disableTTL = isEnabled(process.env.ALLOW_UNVERIFIED_EMAIL_LOGIN);
-    const newUser = await createUser(newUserData, disableTTL, true);
+    const balanceConfig = await getBalanceConfig();
+
+    const newUser = await createUser(newUserData, balanceConfig, disableTTL, true);
     newUserId = newUser._id;
     if (emailEnabled && !newUser.emailVerified) {
       await sendVerificationEmail({
@@ -389,6 +392,7 @@ const setAuthTokens = async (userId, res, sessionId = null) => {
     throw error;
   }
 };
+
 /**
  * @function setOpenIDAuthTokens
  * Set OpenID Authentication Tokens

api/server/services/Files/Azure/images.js

@@ -1,10 +1,9 @@
 const fs = require('fs');
 const path = require('path');
 const sharp = require('sharp');
+const { logger } = require('@librechat/data-schemas');
 const { resizeImageBuffer } = require('../images/resize');
-const { updateUser } = require('~/models/userMethods');
-const { updateFile } = require('~/models/File');
-const { logger } = require('~/config');
+const { updateUser, updateFile } = require('~/models');
 const { saveBufferToAzure } = require('./crud');
 
 /**

api/server/services/Files/Firebase/images.js

@@ -1,11 +1,10 @@
 const fs = require('fs');
 const path = require('path');
 const sharp = require('sharp');
+const { logger } = require('@librechat/data-schemas');
 const { resizeImageBuffer } = require('../images/resize');
-const { updateUser } = require('~/models/userMethods');
+const { updateUser, updateFile } = require('~/models');
 const { saveBufferToFirebase } = require('./crud');
-const { updateFile } = require('~/models/File');
-const { logger } = require('~/config');
 
 /**
  * Converts an image file to the target format. The function first resizes the image based on the specified

api/server/services/Files/Local/images.js

@@ -2,8 +2,7 @@ const fs = require('fs');
 const path = require('path');
 const sharp = require('sharp');
 const { resizeImageBuffer } = require('../images/resize');
-const { updateUser } = require('~/models/userMethods');
-const { updateFile } = require('~/models/File');
+const { updateUser, updateFile } = require('~/models');
 
 /**
  * Converts an image file to the target format. The function first resizes the image based on the specified

api/server/services/Files/S3/images.js

@@ -1,11 +1,10 @@
 const fs = require('fs');
 const path = require('path');
 const sharp = require('sharp');
+const { logger } = require('@librechat/data-schemas');
 const { resizeImageBuffer } = require('../images/resize');
-const { updateUser } = require('~/models/userMethods');
+const { updateUser, updateFile } = require('~/models');
 const { saveBufferToS3 } = require('./crud');
-const { updateFile } = require('~/models/File');
-const { logger } = require('~/config');
 
 const defaultBasePath = 'images';
 

api/server/services/PluginService.js

@@ -1,5 +1,5 @@
-const PluginAuth = require('~/models/schema/pluginAuthSchema');
-const { encrypt, decrypt } = require('~/server/utils/');
+const { encrypt, decrypt } = require('~/server/utils/crypto');
+const { PluginAuth } = require('~/db/models');
 const { logger } = require('~/config');
 
 /**

api/server/services/UserService.js

@@ -1,7 +1,8 @@
+const { logger } = require('@librechat/data-schemas');
 const { ErrorTypes } = require('librechat-data-provider');
-const { encrypt, decrypt } = require('~/server/utils');
-const { updateUser, Key } = require('~/models');
-const { logger } = require('~/config');
+const { encrypt, decrypt } = require('~/server/utils/crypto');
+const { updateUser } = require('~/models');
+const { Key } = require('~/db/models');
 
 /**
  * Updates the plugins for a user based on the action specified (install/uninstall).

api/server/services/signPayload.js

@@ -1,26 +0,0 @@
-const jwt = require('jsonwebtoken');
-
-/**
- * Signs a given payload using either the `jose` library (for Bun runtime) or `jsonwebtoken`.
- *
- * @async
- * @function
- * @param {Object} options - The options for signing the payload.
- * @param {Object} options.payload - The payload to be signed.
- * @param {string} options.secret - The secret key used for signing.
- * @param {number} options.expirationTime - The expiration time in seconds.
- * @returns {Promise<string>} Returns a promise that resolves to the signed JWT.
- * @throws {Error} Throws an error if there's an issue during signing.
- *
- * @example
- * const signedPayload = await signPayload({
- *   payload: { userId: 123 },
- *   secret: 'my-secret-key',
- *   expirationTime: 3600
- * });
- */
-async function signPayload({ payload, secret, expirationTime }) {
-  return jwt.sign(payload, secret, { expiresIn: expirationTime });
-}
-
-module.exports = signPayload;

api/server/services/twoFactorService.js

@@ -1,6 +1,6 @@
 const { webcrypto } = require('node:crypto');
-const { decryptV3, decryptV2 } = require('../utils/crypto');
-const { hashBackupCode } = require('~/server/utils/crypto');
+const { hashBackupCode, decryptV3, decryptV2 } = require('~/server/utils/crypto');
+const { updateUser } = require('~/models');
 
 // Base32 alphabet for TOTP secret encoding.
 const BASE32_ALPHABET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567';
@@ -172,7 +172,6 @@ const verifyBackupCode = async ({ user, backupCode }) => {
         : codeObj,
     );
     // Update the user record with the marked backup code.
-    const { updateUser } = require('~/models');
     await updateUser(user._id, { backupCodes: updatedBackupCodes });
     return true;
   }

api/server/utils/crypto.js

@@ -106,12 +106,6 @@ function decryptV3(encryptedValue) {
   return decrypted.toString('utf8');
 }
 
-async function hashToken(str) {
-  const data = new TextEncoder().encode(str);
-  const hashBuffer = await webcrypto.subtle.digest('SHA-256', data);
-  return Buffer.from(hashBuffer).toString('hex');
-}
-
 async function getRandomValues(length) {
   if (!Number.isInteger(length) || length <= 0) {
     throw new Error('Length must be a positive integer');
@@ -141,7 +135,6 @@ module.exports = {
   decryptV2,
   encryptV3,
   decryptV3,
-  hashToken,
   hashBackupCode,
   getRandomValues,
 };

api/server/utils/handleText.spec.js

@@ -50,9 +50,13 @@ describe('isEnabled', () => {
   });
 });
 
-jest.mock('crypto', () => ({
-  randomBytes: jest.fn().mockReturnValue(Buffer.from('abc123', 'hex')),
-}));
+jest.mock('crypto', () => {
+  const actualModule = jest.requireActual('crypto');
+  return {
+    ...actualModule,
+    randomBytes: jest.fn().mockReturnValue(Buffer.from('abc123', 'hex')),
+  };
+});
 
 describe('sanitizeFilename', () => {
   test('removes directory components (1/2)', () => {