📦 refactor: Consolidate DB models, encapsulating Mongoose usage in data-schemas (#11830)

* chore: move database model methods to /packages/data-schemas

* chore: add TypeScript ESLint rule to warn on unused variables

* refactor: model imports to streamline access

- Consolidated model imports across various files to improve code organization and reduce redundancy.
- Updated imports for models such as Assistant, Message, Conversation, and others to a unified import path.
- Adjusted middleware and service files to reflect the new import structure, ensuring functionality remains intact.
- Enhanced test files to align with the new import paths, maintaining test coverage and integrity.

* chore: migrate database models to packages/data-schemas and refactor all direct Mongoose Model usage outside of data-schemas

* test: update agent model mocks in unit tests

- Added `getAgent` mock to `client.test.js` to enhance test coverage for agent-related functionality.
- Removed redundant `getAgent` and `getAgents` mocks from `openai.spec.js` and `responses.unit.spec.js` to streamline test setup and reduce duplication.
- Ensured consistency in agent mock implementations across test files.

* fix: update types in data-schemas

* refactor: enhance type definitions in transaction and spending methods

- Updated type definitions in `checkBalance.ts` to use specific request and response types.
- Refined `spendTokens.ts` to utilize a new `SpendTxData` interface for better clarity and type safety.
- Improved transaction handling in `transaction.ts` by introducing `TransactionResult` and `TxData` interfaces, ensuring consistent data structures across methods.
- Adjusted unit tests in `transaction.spec.ts` to accommodate new type definitions and enhance robustness.

* refactor: streamline model imports and enhance code organization

- Consolidated model imports across various controllers and services to a unified import path, improving code clarity and reducing redundancy.
- Updated multiple files to reflect the new import structure, ensuring all functionalities remain intact.
- Enhanced overall code organization by removing duplicate import statements and optimizing the usage of model methods.

* feat: implement loadAddedAgent and refactor agent loading logic

- Introduced `loadAddedAgent` function to handle loading agents from added conversations, supporting multi-convo parallel execution.
- Created a new `load.ts` file to encapsulate agent loading functionalities, including `loadEphemeralAgent` and `loadAgent`.
- Updated the `index.ts` file to export the new `load` module instead of the deprecated `loadAgent`.
- Enhanced type definitions and improved error handling in the agent loading process.
- Adjusted unit tests to reflect changes in the agent loading structure and ensure comprehensive coverage.

* refactor: enhance balance handling with new update interface

- Introduced `IBalanceUpdate` interface to streamline balance update operations across the codebase.
- Updated `upsertBalanceFields` method signatures in `balance.ts`, `transaction.ts`, and related tests to utilize the new interface for improved type safety.
- Adjusted type imports in `balance.spec.ts` to include `IBalanceUpdate`, ensuring consistency in balance management functionalities.
- Enhanced overall code clarity and maintainability by refining type definitions related to balance operations.

* feat: add unit tests for loadAgent functionality and enhance agent loading logic

- Introduced comprehensive unit tests for the `loadAgent` function, covering various scenarios including null and empty agent IDs, loading of ephemeral agents, and permission checks.
- Enhanced the `initializeClient` function by moving `getConvoFiles` to the correct position in the database method exports, ensuring proper functionality.
- Improved test coverage for agent loading, including handling of non-existent agents and user permissions.

* chore: reorder memory method exports for consistency

- Moved `deleteAllUserMemories` to the correct position in the exported memory methods, ensuring a consistent and logical order of method exports in `memory.ts`.

packages/data-schemas/src/utils/index.ts

@@ -1 +1,3 @@
+export * from './string';
+export * from './tempChatRetention';
 export * from './transactions';

packages/data-schemas/src/utils/string.ts

@@ -0,0 +1,6 @@
+/**
+ * Escapes special regex characters in a string.
+ */
+export function escapeRegExp(str: string): string {
+  return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}

packages/data-schemas/src/utils/tempChatRetention.spec.ts

@@ -0,0 +1,137 @@
+import type { AppConfig } from '~/types';
+import {
+  createTempChatExpirationDate,
+  getTempChatRetentionHours,
+  DEFAULT_RETENTION_HOURS,
+  MIN_RETENTION_HOURS,
+  MAX_RETENTION_HOURS,
+} from './tempChatRetention';
+
+describe('tempChatRetention', () => {
+  const originalEnv = process.env;
+
+  beforeEach(() => {
+    jest.resetModules();
+    process.env = { ...originalEnv };
+    delete process.env.TEMP_CHAT_RETENTION_HOURS;
+  });
+
+  afterAll(() => {
+    process.env = originalEnv;
+  });
+
+  describe('getTempChatRetentionHours', () => {
+    it('should return default retention hours when no config or env var is set', () => {
+      const result = getTempChatRetentionHours();
+      expect(result).toBe(DEFAULT_RETENTION_HOURS);
+    });
+
+    it('should use environment variable when set', () => {
+      process.env.TEMP_CHAT_RETENTION_HOURS = '48';
+      const result = getTempChatRetentionHours();
+      expect(result).toBe(48);
+    });
+
+    it('should use config value when set', () => {
+      const config: Partial<AppConfig> = {
+        interfaceConfig: {
+          temporaryChatRetention: 12,
+        },
+      };
+      const result = getTempChatRetentionHours(config?.interfaceConfig);
+      expect(result).toBe(12);
+    });
+
+    it('should prioritize config over environment variable', () => {
+      process.env.TEMP_CHAT_RETENTION_HOURS = '48';
+      const config: Partial<AppConfig> = {
+        interfaceConfig: {
+          temporaryChatRetention: 12,
+        },
+      };
+      const result = getTempChatRetentionHours(config?.interfaceConfig);
+      expect(result).toBe(12);
+    });
+
+    it('should enforce minimum retention period', () => {
+      const config: Partial<AppConfig> = {
+        interfaceConfig: {
+          temporaryChatRetention: 0,
+        },
+      };
+      const result = getTempChatRetentionHours(config?.interfaceConfig);
+      expect(result).toBe(MIN_RETENTION_HOURS);
+    });
+
+    it('should enforce maximum retention period', () => {
+      const config: Partial<AppConfig> = {
+        interfaceConfig: {
+          temporaryChatRetention: 10000,
+        },
+      };
+      const result = getTempChatRetentionHours(config?.interfaceConfig);
+      expect(result).toBe(MAX_RETENTION_HOURS);
+    });
+
+    it('should handle invalid environment variable', () => {
+      process.env.TEMP_CHAT_RETENTION_HOURS = 'invalid';
+      const result = getTempChatRetentionHours();
+      expect(result).toBe(DEFAULT_RETENTION_HOURS);
+    });
+
+    it('should handle invalid config value', () => {
+      const config: Partial<AppConfig> = {
+        interfaceConfig: {
+          temporaryChatRetention: 'invalid' as unknown as number,
+        },
+      };
+      const result = getTempChatRetentionHours(config?.interfaceConfig);
+      expect(result).toBe(DEFAULT_RETENTION_HOURS);
+    });
+  });
+
+  describe('createTempChatExpirationDate', () => {
+    it('should create expiration date with default retention period', () => {
+      const beforeCall = Date.now();
+      const result = createTempChatExpirationDate();
+      const afterCall = Date.now();
+
+      const expectedMin = beforeCall + DEFAULT_RETENTION_HOURS * 60 * 60 * 1000;
+      const expectedMax = afterCall + DEFAULT_RETENTION_HOURS * 60 * 60 * 1000;
+
+      // Result should be between expectedMin and expectedMax
+      expect(result.getTime()).toBeGreaterThanOrEqual(expectedMin);
+      expect(result.getTime()).toBeLessThanOrEqual(expectedMax);
+    });
+
+    it('should create expiration date with custom retention period', () => {
+      const config: Partial<AppConfig> = {
+        interfaceConfig: {
+          temporaryChatRetention: 12,
+        },
+      };
+
+      const beforeCall = Date.now();
+      const result = createTempChatExpirationDate(config?.interfaceConfig);
+      const afterCall = Date.now();
+
+      const expectedMin = beforeCall + 12 * 60 * 60 * 1000;
+      const expectedMax = afterCall + 12 * 60 * 60 * 1000;
+
+      // Result should be between expectedMin and expectedMax
+      expect(result.getTime()).toBeGreaterThanOrEqual(expectedMin);
+      expect(result.getTime()).toBeLessThanOrEqual(expectedMax);
+    });
+
+    it('should return a Date object', () => {
+      const result = createTempChatExpirationDate();
+      expect(result).toBeInstanceOf(Date);
+    });
+
+    it('should return a future date', () => {
+      const now = new Date();
+      const result = createTempChatExpirationDate();
+      expect(result.getTime()).toBeGreaterThan(now.getTime());
+    });
+  });
+});

packages/data-schemas/src/utils/tempChatRetention.ts

@@ -0,0 +1,77 @@
+import logger from '~/config/winston';
+import type { AppConfig } from '~/types';
+
+/**
+ * Default retention period for temporary chats in hours
+ */
+export const DEFAULT_RETENTION_HOURS = 24 * 30; // 30 days
+
+/**
+ * Minimum allowed retention period in hours
+ */
+export const MIN_RETENTION_HOURS = 1;
+
+/**
+ * Maximum allowed retention period in hours (1 year = 8760 hours)
+ */
+export const MAX_RETENTION_HOURS = 8760;
+
+/**
+ * Gets the temporary chat retention period from environment variables or config
+ * @param interfaceConfig - The custom configuration object
+ * @returns The retention period in hours
+ */
+export function getTempChatRetentionHours(
+  interfaceConfig?: AppConfig['interfaceConfig'] | null,
+): number {
+  let retentionHours = DEFAULT_RETENTION_HOURS;
+
+  // Check environment variable first
+  if (process.env.TEMP_CHAT_RETENTION_HOURS) {
+    const envValue = parseInt(process.env.TEMP_CHAT_RETENTION_HOURS, 10);
+    if (!isNaN(envValue)) {
+      retentionHours = envValue;
+    } else {
+      logger.warn(
+        `Invalid TEMP_CHAT_RETENTION_HOURS environment variable: ${process.env.TEMP_CHAT_RETENTION_HOURS}. Using default: ${DEFAULT_RETENTION_HOURS} hours.`,
+      );
+    }
+  }
+
+  // Check config file (takes precedence over environment variable)
+  if (interfaceConfig?.temporaryChatRetention !== undefined) {
+    const configValue = interfaceConfig.temporaryChatRetention;
+    if (typeof configValue === 'number' && !isNaN(configValue)) {
+      retentionHours = configValue;
+    } else {
+      logger.warn(
+        `Invalid temporaryChatRetention in config: ${configValue}. Using ${retentionHours} hours.`,
+      );
+    }
+  }
+
+  // Validate the retention period
+  if (retentionHours < MIN_RETENTION_HOURS) {
+    logger.warn(
+      `Temporary chat retention period ${retentionHours} is below minimum ${MIN_RETENTION_HOURS} hours. Using minimum value.`,
+    );
+    retentionHours = MIN_RETENTION_HOURS;
+  } else if (retentionHours > MAX_RETENTION_HOURS) {
+    logger.warn(
+      `Temporary chat retention period ${retentionHours} exceeds maximum ${MAX_RETENTION_HOURS} hours. Using maximum value.`,
+    );
+    retentionHours = MAX_RETENTION_HOURS;
+  }
+
+  return retentionHours;
+}
+
+/**
+ * Creates an expiration date for temporary chats
+ * @param interfaceConfig - The custom configuration object
+ * @returns The expiration date
+ */
+export function createTempChatExpirationDate(interfaceConfig?: AppConfig['interfaceConfig']): Date {
+  const retentionHours = getTempChatRetentionHours(interfaceConfig);
+  return new Date(Date.now() + retentionHours * 60 * 60 * 1000);
+}