🏗️ refactor: Extract DB layers to data-schemas for shared use (#7650)

* refactor: move model definitions and database-related methods to packages/data-schemas

* ci: update tests due to new DB structure

fix: disable mocking `librechat-data-provider`

feat: Add schema exports to data-schemas package

- Introduced a new schema module that exports various schemas including action, agent, and user schemas.
- Updated index.ts to include the new schema exports for better modularity and organization.

ci: fix appleStrategy tests

fix: Agent.spec.js

ci: refactor handleTools tests to use MongoMemoryServer for in-memory database

fix: getLogStores imports

ci: update banViolation tests to use MongoMemoryServer and improve session mocking

test: refactor samlStrategy tests to improve mock configurations and user handling

ci: fix crypto mock in handleText tests for improved accuracy

ci: refactor spendTokens tests to improve model imports and setup

ci: refactor Message model tests to use MongoMemoryServer and improve database interactions

* refactor: streamline IMessage interface and move feedback properties to types/message.ts

* refactor: use exported initializeRoles from `data-schemas`, remove api workspace version (this serves as an example of future migrations that still need to happen)

* refactor: update model imports to use destructuring from `~/db/models` for consistency and clarity

* refactor: remove unused mongoose imports from model files for cleaner code

* refactor: remove unused mongoose imports from Share, Prompt, and Transaction model files for cleaner code

* refactor: remove unused import in Transaction model for cleaner code

* ci: update deploy workflow to reference new Docker Dev Branch Images Build and add new workflow for building Docker images on dev branch

* chore: cleanup imports

packages/data-schemas/src/schema/action.ts

@@ -1,31 +1,5 @@
-import mongoose, { Schema, Document } from 'mongoose';
-
-export interface IAction extends Document {
-  user: mongoose.Types.ObjectId;
-  action_id: string;
-  type: string;
-  settings?: unknown;
-  agent_id?: string;
-  assistant_id?: string;
-  metadata: {
-    api_key?: string;
-    auth: {
-      authorization_type?: string;
-      custom_auth_header?: string;
-      type: 'service_http' | 'oauth' | 'none';
-      authorization_content_type?: string;
-      authorization_url?: string;
-      client_url?: string;
-      scope?: string;
-      token_exchange_method: 'default_post' | 'basic_auth_header' | null;
-    };
-    domain: string;
-    privacy_policy_url?: string;
-    raw_spec?: string;
-    oauth_client_id?: string;
-    oauth_client_secret?: string;
-  };
-}
+import mongoose, { Schema } from 'mongoose';
+import type { IAction } from '~/types';
 
 // Define the Auth sub-schema with type-safety.
 const AuthSchema = new Schema(

packages/data-schemas/src/schema/agent.ts

@@ -1,33 +1,5 @@
-import { Schema, Document, Types } from 'mongoose';
-export interface IAgent extends Omit<Document, 'model'> {
-  id: string;
-  name?: string;
-  description?: string;
-  instructions?: string;
-  avatar?: {
-    filepath: string;
-    source: string;
-  };
-  provider: string;
-  model: string;
-  model_parameters?: Record<string, unknown>;
-  artifacts?: string;
-  access_level?: number;
-  recursion_limit?: number;
-  tools?: string[];
-  tool_kwargs?: Array<unknown>;
-  actions?: string[];
-  author: Types.ObjectId;
-  authorName?: string;
-  hide_sequential_outputs?: boolean;
-  end_after_tools?: boolean;
-  agent_ids?: string[];
-  isCollaborative?: boolean;
-  conversation_starters?: string[];
-  tool_resources?: unknown;
-  projectIds?: Types.ObjectId[];
-  versions?: Omit<IAgent, 'versions'>[];
-}
+import { Schema } from 'mongoose';
+import type { IAgent } from '~/types';
 
 const agentSchema = new Schema<IAgent>(
   {

packages/data-schemas/src/schema/assistant.ts

@@ -1,18 +1,5 @@
-import { Schema, Document, Types } from 'mongoose';
-
-export interface IAssistant extends Document {
-  user: Types.ObjectId;
-  assistant_id: string;
-  avatar?: {
-    filepath: string;
-    source: string;
-  };
-  conversation_starters?: string[];
-  access_level?: number;
-  file_ids?: string[];
-  actions?: string[];
-  append_current_datetime?: boolean;
-}
+import { Schema } from 'mongoose';
+import type { IAssistant } from '~/types';
 
 const assistantSchema = new Schema<IAssistant>(
   {

packages/data-schemas/src/schema/balance.ts

@@ -1,17 +1,7 @@
-import { Schema, Document, Types } from 'mongoose';
+import { Schema } from 'mongoose';
+import type * as t from '~/types';
 
-export interface IBalance extends Document {
-  user: Types.ObjectId;
-  tokenCredits: number;
-  // Automatic refill settings
-  autoRefillEnabled: boolean;
-  refillIntervalValue: number;
-  refillIntervalUnit: 'seconds' | 'minutes' | 'hours' | 'days' | 'weeks' | 'months';
-  lastRefill: Date;
-  refillAmount: number;
-}
-
-const balanceSchema = new Schema<IBalance>({
+const balanceSchema = new Schema<t.IBalance>({
   user: {
     type: Schema.Types.ObjectId,
     ref: 'User',

packages/data-schemas/src/schema/convo.ts

@@ -1,57 +1,7 @@
-import mongoose, { Schema, Document, Types } from 'mongoose';
+import { Schema } from 'mongoose';
+import mongoMeili from '~/models/plugins/mongoMeili';
 import { conversationPreset } from './defaults';
-
-// @ts-ignore
-export interface IConversation extends Document {
-  conversationId: string;
-  title?: string;
-  user?: string;
-  messages?: Types.ObjectId[];
-  agentOptions?: unknown;
-  // Fields provided by conversationPreset (adjust types as needed)
-  endpoint?: string;
-  endpointType?: string;
-  model?: string;
-  region?: string;
-  chatGptLabel?: string;
-  examples?: unknown[];
-  modelLabel?: string;
-  promptPrefix?: string;
-  temperature?: number;
-  top_p?: number;
-  topP?: number;
-  topK?: number;
-  maxOutputTokens?: number;
-  maxTokens?: number;
-  presence_penalty?: number;
-  frequency_penalty?: number;
-  file_ids?: string[];
-  resendImages?: boolean;
-  promptCache?: boolean;
-  thinking?: boolean;
-  thinkingBudget?: number;
-  system?: string;
-  resendFiles?: boolean;
-  imageDetail?: string;
-  agent_id?: string;
-  assistant_id?: string;
-  instructions?: string;
-  stop?: string[];
-  isArchived?: boolean;
-  iconURL?: string;
-  greeting?: string;
-  spec?: string;
-  tags?: string[];
-  tools?: string[];
-  maxContextTokens?: number;
-  max_tokens?: number;
-  reasoning_effort?: string;
-  // Additional fields
-  files?: string[];
-  expiredAt?: Date;
-  createdAt?: Date;
-  updatedAt?: Date;
-}
+import { IConversation } from '~/types';
 
 const convoSchema: Schema<IConversation> = new Schema(
   {
@@ -71,9 +21,9 @@ const convoSchema: Schema<IConversation> = new Schema(
       type: String,
       index: true,
     },
-    messages: [{ type: mongoose.Schema.Types.ObjectId, ref: 'Message' }],
+    messages: [{ type: Schema.Types.ObjectId, ref: 'Message' }],
     agentOptions: {
-      type: mongoose.Schema.Types.Mixed,
+      type: Schema.Types.Mixed,
     },
     ...conversationPreset,
     agent_id: {
@@ -98,4 +48,14 @@ convoSchema.index({ expiredAt: 1 }, { expireAfterSeconds: 0 });
 convoSchema.index({ createdAt: 1, updatedAt: 1 });
 convoSchema.index({ conversationId: 1, user: 1 }, { unique: true });
 
+if (process.env.MEILI_HOST && process.env.MEILI_MASTER_KEY) {
+  convoSchema.plugin(mongoMeili, {
+    host: process.env.MEILI_HOST,
+    apiKey: process.env.MEILI_MASTER_KEY,
+    /** Note: Will get created automatically if it doesn't exist already */
+    indexName: 'convos',
+    primaryKey: 'conversationId',
+  });
+}
+
 export default convoSchema;

packages/data-schemas/src/schema/file.ts

@@ -1,32 +1,6 @@
-import mongoose, { Schema, Document, Types } from 'mongoose';
+import mongoose, { Schema } from 'mongoose';
 import { FileSources } from 'librechat-data-provider';
-
-// @ts-ignore
-export interface IMongoFile extends Document {
-  user: Types.ObjectId;
-  conversationId?: string;
-  file_id: string;
-  temp_file_id?: string;
-  bytes: number;
-  text?: string;
-  filename: string;
-  filepath: string;
-  object: 'file';
-  embedded?: boolean;
-  type: string;
-  context?: string;
-  usage: number;
-  source: string;
-  model?: string;
-  width?: number;
-  height?: number;
-  metadata?: {
-    fileIdentifier?: string;
-  };
-  expiresAt?: Date;
-  createdAt?: Date;
-  updatedAt?: Date;
-}
+import type { IMongoFile } from '~/types';
 
 const file: Schema<IMongoFile> = new Schema(
   {

packages/data-schemas/src/schema/index.ts

@@ -0,0 +1,23 @@
+export { default as actionSchema } from './action';
+export { default as agentSchema } from './agent';
+export { default as assistantSchema } from './assistant';
+export { default as balanceSchema } from './balance';
+export { default as bannerSchema } from './banner';
+export { default as categoriesSchema } from './categories';
+export { default as conversationTagSchema } from './conversationTag';
+export { default as convoSchema } from './convo';
+export { default as fileSchema } from './file';
+export { default as keySchema } from './key';
+export { default as messageSchema } from './message';
+export { default as pluginAuthSchema } from './pluginAuth';
+export { default as presetSchema } from './preset';
+export { default as projectSchema } from './project';
+export { default as promptSchema } from './prompt';
+export { default as promptGroupSchema } from './promptGroup';
+export { default as roleSchema } from './role';
+export { default as sessionSchema } from './session';
+export { default as shareSchema } from './share';
+export { default as tokenSchema } from './token';
+export { default as toolCallSchema } from './toolCall';
+export { default as transactionSchema } from './transaction';
+export { default as userSchema } from './user';

packages/data-schemas/src/schema/message.ts

@@ -1,47 +1,6 @@
-import mongoose, { Schema, Document } from 'mongoose';
-import { TFeedbackRating, TFeedbackTag } from 'librechat-data-provider';
-
-// @ts-ignore
-export interface IMessage extends Document {
-  messageId: string;
-  conversationId: string;
-  user: string;
-  model?: string;
-  endpoint?: string;
-  conversationSignature?: string;
-  clientId?: string;
-  invocationId?: number;
-  parentMessageId?: string;
-  tokenCount?: number;
-  summaryTokenCount?: number;
-  sender?: string;
-  text?: string;
-  summary?: string;
-  isCreatedByUser: boolean;
-  unfinished?: boolean;
-  error?: boolean;
-  finish_reason?: string;
-  feedback?: {
-    rating: TFeedbackRating;
-    tag: TFeedbackTag | undefined;
-    text?: string;
-  };
-  _meiliIndex?: boolean;
-  files?: unknown[];
-  plugin?: {
-    latest?: string;
-    inputs?: unknown[];
-    outputs?: string;
-  };
-  plugins?: unknown[];
-  content?: unknown[];
-  thread_id?: string;
-  iconURL?: string;
-  attachments?: unknown[];
-  expiredAt?: Date;
-  createdAt?: Date;
-  updatedAt?: Date;
-}
+import mongoose, { Schema } from 'mongoose';
+import type { IMessage } from '~/types/message';
+import mongoMeili from '~/models/plugins/mongoMeili';
 
 const messageSchema: Schema<IMessage> = new Schema(
   {
@@ -207,4 +166,13 @@ messageSchema.index({ expiredAt: 1 }, { expireAfterSeconds: 0 });
 messageSchema.index({ createdAt: 1 });
 messageSchema.index({ messageId: 1, user: 1 }, { unique: true });
 
+if (process.env.MEILI_HOST && process.env.MEILI_MASTER_KEY) {
+  messageSchema.plugin(mongoMeili, {
+    host: process.env.MEILI_HOST,
+    apiKey: process.env.MEILI_MASTER_KEY,
+    indexName: 'messages',
+    primaryKey: 'messageId',
+  });
+}
+
 export default messageSchema;

packages/data-schemas/src/schema/role.ts

@@ -1,36 +1,6 @@
-import { Schema, Document } from 'mongoose';
+import { Schema } from 'mongoose';
 import { PermissionTypes, Permissions } from 'librechat-data-provider';
-
-export interface IRole extends Document {
-  name: string;
-  permissions: {
-    [PermissionTypes.BOOKMARKS]?: {
-      [Permissions.USE]?: boolean;
-    };
-    [PermissionTypes.PROMPTS]?: {
-      [Permissions.SHARED_GLOBAL]?: boolean;
-      [Permissions.USE]?: boolean;
-      [Permissions.CREATE]?: boolean;
-    };
-    [PermissionTypes.AGENTS]?: {
-      [Permissions.SHARED_GLOBAL]?: boolean;
-      [Permissions.USE]?: boolean;
-      [Permissions.CREATE]?: boolean;
-    };
-    [PermissionTypes.MULTI_CONVO]?: {
-      [Permissions.USE]?: boolean;
-    };
-    [PermissionTypes.TEMPORARY_CHAT]?: {
-      [Permissions.USE]?: boolean;
-    };
-    [PermissionTypes.RUN_CODE]?: {
-      [Permissions.USE]?: boolean;
-    };
-    [PermissionTypes.WEB_SEARCH]?: {
-      [Permissions.USE]?: boolean;
-    };
-  };
-}
+import type { IRole } from '~/types';
 
 // Create a sub-schema for permissions. Notice we disable _id for this subdocument.
 const rolePermissionsSchema = new Schema(

packages/data-schemas/src/schema/session.ts

@@ -1,10 +1,5 @@
-import mongoose, { Schema, Document, Types } from 'mongoose';
-
-export interface ISession extends Document {
-  refreshTokenHash: string;
-  expiration: Date;
-  user: Types.ObjectId;
-}
+import mongoose, { Schema } from 'mongoose';
+import { ISession } from '~/types';
 
 const sessionSchema: Schema<ISession> = new Schema({
   refreshTokenHash: {

packages/data-schemas/src/schema/token.ts

@@ -1,15 +1,5 @@
-import { Schema, Document, Types } from 'mongoose';
-
-export interface IToken extends Document {
-  userId: Types.ObjectId;
-  email?: string;
-  type?: string;
-  identifier?: string;
-  token: string;
-  createdAt: Date;
-  expiresAt: Date;
-  metadata?: Map<string, unknown>;
-}
+import { Schema } from 'mongoose';
+import { IToken } from '~/types';
 
 const tokenSchema: Schema<IToken> = new Schema({
   userId: {

packages/data-schemas/src/schema/user.ts

@@ -1,39 +1,6 @@
-import { Schema, Document } from 'mongoose';
+import { Schema } from 'mongoose';
 import { SystemRoles } from 'librechat-data-provider';
-
-export interface IUser extends Document {
-  name?: string;
-  username?: string;
-  email: string;
-  emailVerified: boolean;
-  password?: string;
-  avatar?: string;
-  provider: string;
-  role?: string;
-  googleId?: string;
-  facebookId?: string;
-  openidId?: string;
-  samlId?: string;
-  ldapId?: string;
-  githubId?: string;
-  discordId?: string;
-  appleId?: string;
-  plugins?: unknown[];
-  twoFactorEnabled?: boolean;
-  totpSecret?: string;
-  backupCodes?: Array<{
-    codeHash: string;
-    used: boolean;
-    usedAt?: Date | null;
-  }>;
-  refreshToken?: Array<{
-    refreshToken: string;
-  }>;
-  expiresAt?: Date;
-  termsAccepted?: boolean;
-  createdAt?: Date;
-  updatedAt?: Date;
-}
+import { IUser } from '~/types';
 
 // Session sub-schema
 const SessionSchema = new Schema(
@@ -56,7 +23,7 @@ const BackupCodeSchema = new Schema(
   { _id: false },
 );
 
-const User = new Schema<IUser>(
+const userSchema = new Schema<IUser>(
   {
     name: {
       type: String,
@@ -166,4 +133,4 @@ const User = new Schema<IUser>(
   { timestamps: true },
 );
 
-export default User;
+export default userSchema;