🏗️ refactor: Extract DB layers to data-schemas for shared use (#7650)

* refactor: move model definitions and database-related methods to packages/data-schemas

* ci: update tests due to new DB structure

fix: disable mocking `librechat-data-provider`

feat: Add schema exports to data-schemas package

- Introduced a new schema module that exports various schemas including action, agent, and user schemas.
- Updated index.ts to include the new schema exports for better modularity and organization.

ci: fix appleStrategy tests

fix: Agent.spec.js

ci: refactor handleTools tests to use MongoMemoryServer for in-memory database

fix: getLogStores imports

ci: update banViolation tests to use MongoMemoryServer and improve session mocking

test: refactor samlStrategy tests to improve mock configurations and user handling

ci: fix crypto mock in handleText tests for improved accuracy

ci: refactor spendTokens tests to improve model imports and setup

ci: refactor Message model tests to use MongoMemoryServer and improve database interactions

* refactor: streamline IMessage interface and move feedback properties to types/message.ts

* refactor: use exported initializeRoles from `data-schemas`, remove api workspace version (this serves as an example of future migrations that still need to happen)

* refactor: update model imports to use destructuring from `~/db/models` for consistency and clarity

* refactor: remove unused mongoose imports from model files for cleaner code

* refactor: remove unused mongoose imports from Share, Prompt, and Transaction model files for cleaner code

* refactor: remove unused import in Transaction model for cleaner code

* ci: update deploy workflow to reference new Docker Dev Branch Images Build and add new workflow for building Docker images on dev branch

* chore: cleanup imports

packages/data-schemas/src/config/meiliLogger.ts

@@ -0,0 +1,75 @@
+import path from 'path';
+import winston from 'winston';
+import 'winston-daily-rotate-file';
+
+const logDir = path.join(__dirname, '..', 'logs');
+
+const { NODE_ENV, DEBUG_LOGGING = 'false' } = process.env;
+
+const useDebugLogging =
+  (typeof DEBUG_LOGGING === 'string' && DEBUG_LOGGING.toLowerCase() === 'true') ||
+  DEBUG_LOGGING === 'true';
+
+const levels: winston.config.AbstractConfigSetLevels = {
+  error: 0,
+  warn: 1,
+  info: 2,
+  http: 3,
+  verbose: 4,
+  debug: 5,
+  activity: 6,
+  silly: 7,
+};
+
+winston.addColors({
+  info: 'green',
+  warn: 'italic yellow',
+  error: 'red',
+  debug: 'blue',
+});
+
+const level = (): string => {
+  const env = NODE_ENV || 'development';
+  const isDevelopment = env === 'development';
+  return isDevelopment ? 'debug' : 'warn';
+};
+
+const fileFormat = winston.format.combine(
+  winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
+  winston.format.errors({ stack: true }),
+  winston.format.splat(),
+);
+
+const logLevel = useDebugLogging ? 'debug' : 'error';
+const transports: winston.transport[] = [
+  new winston.transports.DailyRotateFile({
+    level: logLevel,
+    filename: `${logDir}/meiliSync-%DATE%.log`,
+    datePattern: 'YYYY-MM-DD',
+    zippedArchive: true,
+    maxSize: '20m',
+    maxFiles: '14d',
+    format: fileFormat,
+  }),
+];
+
+const consoleFormat = winston.format.combine(
+  winston.format.colorize({ all: true }),
+  winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
+  winston.format.printf((info) => `${info.timestamp} ${info.level}: ${info.message}`),
+);
+
+transports.push(
+  new winston.transports.Console({
+    level: 'info',
+    format: consoleFormat,
+  }),
+);
+
+const logger = winston.createLogger({
+  level: level(),
+  levels,
+  transports,
+});
+
+export default logger;

packages/data-schemas/src/config/parsers.ts

@@ -0,0 +1,241 @@
+import { klona } from 'klona';
+import winston from 'winston';
+import traverse from 'traverse';
+
+const SPLAT_SYMBOL = Symbol.for('splat');
+const MESSAGE_SYMBOL = Symbol.for('message');
+const CONSOLE_JSON_STRING_LENGTH: number =
+  parseInt(process.env.CONSOLE_JSON_STRING_LENGTH || '', 10) || 255;
+
+const sensitiveKeys: RegExp[] = [
+  /^(sk-)[^\s]+/, // OpenAI API key pattern
+  /(Bearer )[^\s]+/, // Header: Bearer token pattern
+  /(api-key:? )[^\s]+/, // Header: API key pattern
+  /(key=)[^\s]+/, // URL query param: sensitive key pattern (Google)
+];
+
+/**
+ * Determines if a given value string is sensitive and returns matching regex patterns.
+ *
+ * @param valueStr - The value string to check.
+ * @returns An array of regex patterns that match the value string.
+ */
+function getMatchingSensitivePatterns(valueStr: string): RegExp[] {
+  if (valueStr) {
+    // Filter and return all regex patterns that match the value string
+    return sensitiveKeys.filter((regex) => regex.test(valueStr));
+  }
+  return [];
+}
+
+/**
+ * Redacts sensitive information from a console message and trims it to a specified length if provided.
+ * @param str - The console message to be redacted.
+ * @param trimLength - The optional length at which to trim the redacted message.
+ * @returns The redacted and optionally trimmed console message.
+ */
+function redactMessage(str: string, trimLength?: number): string {
+  if (!str) {
+    return '';
+  }
+
+  const patterns = getMatchingSensitivePatterns(str);
+  patterns.forEach((pattern) => {
+    str = str.replace(pattern, '$1[REDACTED]');
+  });
+
+  if (trimLength !== undefined && str.length > trimLength) {
+    return `${str.substring(0, trimLength)}...`;
+  }
+
+  return str;
+}
+
+/**
+ * Redacts sensitive information from log messages if the log level is 'error'.
+ * Note: Intentionally mutates the object.
+ * @param info - The log information object.
+ * @returns The modified log information object.
+ */
+const redactFormat = winston.format((info: winston.Logform.TransformableInfo) => {
+  if (info.level === 'error') {
+    // Type guard to ensure message is a string
+    if (typeof info.message === 'string') {
+      info.message = redactMessage(info.message);
+    }
+
+    // Handle MESSAGE_SYMBOL with type safety
+    const symbolValue = (info as Record<string | symbol, unknown>)[MESSAGE_SYMBOL];
+    if (typeof symbolValue === 'string') {
+      (info as Record<string | symbol, unknown>)[MESSAGE_SYMBOL] = redactMessage(symbolValue);
+    }
+  }
+  return info;
+});
+
+/**
+ * Truncates long strings, especially base64 image data, within log messages.
+ *
+ * @param value - The value to be inspected and potentially truncated.
+ * @param length - The length at which to truncate the value. Default: 100.
+ * @returns The truncated or original value.
+ */
+const truncateLongStrings = (value: unknown, length = 100): unknown => {
+  if (typeof value === 'string') {
+    return value.length > length ? value.substring(0, length) + '... [truncated]' : value;
+  }
+
+  return value;
+};
+
+/**
+ * An array mapping function that truncates long strings (objects converted to JSON strings).
+ * @param item - The item to be condensed.
+ * @returns The condensed item.
+ */
+const condenseArray = (item: unknown): string | unknown => {
+  if (typeof item === 'string') {
+    return truncateLongStrings(JSON.stringify(item));
+  } else if (typeof item === 'object') {
+    return truncateLongStrings(JSON.stringify(item));
+  }
+  return item;
+};
+
+/**
+ * Formats log messages for debugging purposes.
+ * - Truncates long strings within log messages.
+ * - Condenses arrays by truncating long strings and objects as strings within array items.
+ * - Redacts sensitive information from log messages if the log level is 'error'.
+ * - Converts log information object to a formatted string.
+ *
+ * @param options - The options for formatting log messages.
+ * @returns The formatted log message.
+ */
+const debugTraverse = winston.format.printf(
+  ({ level, message, timestamp, ...metadata }: Record<string, unknown>) => {
+    if (!message) {
+      return `${timestamp} ${level}`;
+    }
+
+    // Type-safe version of the CJS logic: !message?.trim || typeof message !== 'string'
+    if (typeof message !== 'string' || !message.trim) {
+      return `${timestamp} ${level}: ${JSON.stringify(message)}`;
+    }
+
+    let msg = `${timestamp} ${level}: ${truncateLongStrings(message.trim(), 150)}`;
+
+    try {
+      if (level !== 'debug') {
+        return msg;
+      }
+
+      if (!metadata) {
+        return msg;
+      }
+
+      // Type-safe access to SPLAT_SYMBOL using bracket notation
+      const metadataRecord = metadata as Record<string | symbol, unknown>;
+      const splatArray = metadataRecord[SPLAT_SYMBOL];
+      const debugValue = Array.isArray(splatArray) ? splatArray[0] : undefined;
+
+      if (!debugValue) {
+        return msg;
+      }
+
+      if (debugValue && Array.isArray(debugValue)) {
+        msg += `\n${JSON.stringify(debugValue.map(condenseArray))}`;
+        return msg;
+      }
+
+      if (typeof debugValue !== 'object') {
+        return (msg += ` ${debugValue}`);
+      }
+
+      msg += '\n{';
+
+      const copy = klona(metadata);
+
+      traverse(copy).forEach(function (this: traverse.TraverseContext, value: unknown) {
+        if (typeof this?.key === 'symbol') {
+          return;
+        }
+
+        let _parentKey = '';
+        const parent = this.parent;
+
+        if (typeof parent?.key !== 'symbol' && parent?.key) {
+          _parentKey = parent.key;
+        }
+
+        const parentKey = `${parent && parent.notRoot ? _parentKey + '.' : ''}`;
+        const tabs = `${parent && parent.notRoot ? '    ' : '  '}`;
+        const currentKey = this?.key ?? 'unknown';
+
+        if (this.isLeaf && typeof value === 'string') {
+          const truncatedText = truncateLongStrings(value);
+          msg += `\n${tabs}${parentKey}${currentKey}: ${JSON.stringify(truncatedText)},`;
+        } else if (this.notLeaf && Array.isArray(value) && value.length > 0) {
+          const currentMessage = `\n${tabs}// ${value.length} ${currentKey.replace(/s$/, '')}(s)`;
+          this.update(currentMessage, true);
+          msg += currentMessage;
+          const stringifiedArray = value.map(condenseArray);
+          msg += `\n${tabs}${parentKey}${currentKey}: [${stringifiedArray}],`;
+        } else if (this.isLeaf && typeof value === 'function') {
+          msg += `\n${tabs}${parentKey}${currentKey}: function,`;
+        } else if (this.isLeaf) {
+          msg += `\n${tabs}${parentKey}${currentKey}: ${value},`;
+        }
+      });
+
+      msg += '\n}';
+      return msg;
+    } catch (e: unknown) {
+      const errorMessage = e instanceof Error ? e.message : 'Unknown error';
+      return (msg += `\n[LOGGER PARSING ERROR] ${errorMessage}`);
+    }
+  },
+);
+
+/**
+ * Truncates long string values in JSON log objects.
+ * Prevents outputting extremely long values (e.g., base64, blobs).
+ */
+const jsonTruncateFormat = winston.format((info: winston.Logform.TransformableInfo) => {
+  const truncateLongStrings = (str: string, maxLength: number): string =>
+    str.length > maxLength ? str.substring(0, maxLength) + '...' : str;
+
+  const seen = new WeakSet<object>();
+
+  const truncateObject = (obj: unknown): unknown => {
+    if (typeof obj !== 'object' || obj === null) {
+      return obj;
+    }
+
+    // Handle circular references - now with proper object type
+    if (seen.has(obj)) {
+      return '[Circular]';
+    }
+    seen.add(obj);
+
+    if (Array.isArray(obj)) {
+      return obj.map((item) => truncateObject(item));
+    }
+
+    // We know this is an object at this point
+    const objectRecord = obj as Record<string, unknown>;
+    const newObj: Record<string, unknown> = {};
+    Object.entries(objectRecord).forEach(([key, value]) => {
+      if (typeof value === 'string') {
+        newObj[key] = truncateLongStrings(value, CONSOLE_JSON_STRING_LENGTH);
+      } else {
+        newObj[key] = truncateObject(value);
+      }
+    });
+    return newObj;
+  };
+
+  return truncateObject(info) as winston.Logform.TransformableInfo;
+});
+
+export { redactFormat, redactMessage, debugTraverse, jsonTruncateFormat };

packages/data-schemas/src/config/winston.ts

@@ -0,0 +1,123 @@
+import path from 'path';
+import winston from 'winston';
+import 'winston-daily-rotate-file';
+import { redactFormat, redactMessage, debugTraverse, jsonTruncateFormat } from './parsers';
+
+// Define log directory
+const logDir = path.join(__dirname, '..', 'logs');
+
+// Type-safe environment variables
+const { NODE_ENV, DEBUG_LOGGING, CONSOLE_JSON, DEBUG_CONSOLE } = process.env;
+
+const useConsoleJson = typeof CONSOLE_JSON === 'string' && CONSOLE_JSON.toLowerCase() === 'true';
+
+const useDebugConsole = typeof DEBUG_CONSOLE === 'string' && DEBUG_CONSOLE.toLowerCase() === 'true';
+
+const useDebugLogging = typeof DEBUG_LOGGING === 'string' && DEBUG_LOGGING.toLowerCase() === 'true';
+
+// Define custom log levels
+const levels: winston.config.AbstractConfigSetLevels = {
+  error: 0,
+  warn: 1,
+  info: 2,
+  http: 3,
+  verbose: 4,
+  debug: 5,
+  activity: 6,
+  silly: 7,
+};
+
+winston.addColors({
+  info: 'green',
+  warn: 'italic yellow',
+  error: 'red',
+  debug: 'blue',
+});
+
+const level = (): string => {
+  const env = NODE_ENV || 'development';
+  return env === 'development' ? 'debug' : 'warn';
+};
+
+const fileFormat = winston.format.combine(
+  redactFormat(),
+  winston.format.timestamp({ format: () => new Date().toISOString() }),
+  winston.format.errors({ stack: true }),
+  winston.format.splat(),
+);
+
+const transports: winston.transport[] = [
+  new winston.transports.DailyRotateFile({
+    level: 'error',
+    filename: `${logDir}/error-%DATE%.log`,
+    datePattern: 'YYYY-MM-DD',
+    zippedArchive: true,
+    maxSize: '20m',
+    maxFiles: '14d',
+    format: fileFormat,
+  }),
+];
+
+if (useDebugLogging) {
+  transports.push(
+    new winston.transports.DailyRotateFile({
+      level: 'debug',
+      filename: `${logDir}/debug-%DATE%.log`,
+      datePattern: 'YYYY-MM-DD',
+      zippedArchive: true,
+      maxSize: '20m',
+      maxFiles: '14d',
+      format: winston.format.combine(fileFormat, debugTraverse),
+    }),
+  );
+}
+
+const consoleFormat = winston.format.combine(
+  redactFormat(),
+  winston.format.colorize({ all: true }),
+  winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
+  winston.format.printf((info) => {
+    const message = `${info.timestamp} ${info.level}: ${info.message}`;
+    return info.level.includes('error') ? redactMessage(message) : message;
+  }),
+);
+
+let consoleLogLevel: string = 'info';
+if (useDebugConsole) {
+  consoleLogLevel = 'debug';
+}
+
+// Add console transport
+if (useDebugConsole) {
+  transports.push(
+    new winston.transports.Console({
+      level: consoleLogLevel,
+      format: useConsoleJson
+        ? winston.format.combine(fileFormat, jsonTruncateFormat(), winston.format.json())
+        : winston.format.combine(fileFormat, debugTraverse),
+    }),
+  );
+} else if (useConsoleJson) {
+  transports.push(
+    new winston.transports.Console({
+      level: consoleLogLevel,
+      format: winston.format.combine(fileFormat, jsonTruncateFormat(), winston.format.json()),
+    }),
+  );
+} else {
+  transports.push(
+    new winston.transports.Console({
+      level: consoleLogLevel,
+      format: consoleFormat,
+    }),
+  );
+}
+
+// Create logger
+const logger = winston.createLogger({
+  level: level(),
+  levels,
+  transports,
+});
+
+export default logger;