mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-02-12 04:24:24 +01:00
🧵 refactor: Migrate Endpoint Initialization to TypeScript (#10794)
* refactor: move endpoint initialization methods to typescript * refactor: move agent init to packages/api - Introduced `initialize.ts` for agent initialization, including file processing and tool loading. - Updated `resources.ts` to allow optional appConfig parameter. - Enhanced endpoint configuration handling in various initialization files to support model parameters. - Added new artifacts and prompts for React component generation. - Refactored existing code to improve type safety and maintainability. * refactor: streamline endpoint initialization and enhance type safety - Updated initialization functions across various endpoints to use a consistent request structure, replacing `unknown` types with `ServerResponse`. - Simplified request handling by directly extracting keys from the request body. - Improved type safety by ensuring user IDs are safely accessed with optional chaining. - Removed unnecessary parameters and streamlined model options handling for better clarity and maintainability. * refactor: moved ModelService and extractBaseURL to packages/api - Added comprehensive tests for the models fetching functionality, covering scenarios for OpenAI, Anthropic, Google, and Ollama models. - Updated existing endpoint index to include the new models module. - Enhanced utility functions for URL extraction and model data processing. - Improved type safety and error handling across the models fetching logic. * refactor: consolidate utility functions and remove unused files - Merged `deriveBaseURL` and `extractBaseURL` into the `@librechat/api` module for better organization. - Removed redundant utility files and their associated tests to streamline the codebase. - Updated imports across various client files to utilize the new consolidated functions. - Enhanced overall maintainability by reducing the number of utility modules. * refactor: replace ModelService references with direct imports from @librechat/api and remove ModelService file * refactor: move encrypt/decrypt methods and key db methods to data-schemas, use `getProviderConfig` from `@librechat/api` * chore: remove unused 'res' from options in AgentClient * refactor: file model imports and methods - Updated imports in various controllers and services to use the unified file model from '~/models' instead of '~/models/File'. - Consolidated file-related methods into a new file methods module in the data-schemas package. - Added comprehensive tests for file methods including creation, retrieval, updating, and deletion. - Enhanced the initializeAgent function to accept dependency injection for file-related methods. - Improved error handling and logging in file methods. * refactor: streamline database method references in agent initialization * refactor: enhance file method tests and update type references to IMongoFile * refactor: consolidate database method imports in agent client and initialization * chore: remove redundant import of initializeAgent from @librechat/api * refactor: move checkUserKeyExpiry utility to @librechat/api and update references across endpoints * refactor: move updateUserPlugins logic to user.ts and simplify UserController * refactor: update imports for user key management and remove UserService * refactor: remove unused Anthropics and Bedrock endpoint files and clean up imports * refactor: consolidate and update encryption imports across various files to use @librechat/data-schemas * chore: update file model mock to use unified import from '~/models' * chore: import order * refactor: remove migrated to TS agent.js file and its associated logic from the endpoints * chore: add reusable function to extract imports from source code in unused-packages workflow * chore: enhance unused-packages workflow to include @librechat/api dependencies and improve dependency extraction * chore: improve dependency extraction in unused-packages workflow with enhanced error handling and debugging output * chore: add detailed debugging output to unused-packages workflow for better visibility into unused dependencies and exclusion lists * chore: refine subpath handling in unused-packages workflow to correctly process scoped and non-scoped package imports * chore: clean up unused debug output in unused-packages workflow and reorganize type imports in initialize.ts
This commit is contained in:
Danny Avila
parent
1a11b64266
commit
04a4a2aa44
103 changed files with 4135 additions and 2647 deletions
272
packages/data-schemas/src/methods/file.ts
Normal file
272
packages/data-schemas/src/methods/file.ts
Normal file
|
|
@ -0,0 +1,272 @@
|
|||
import logger from '../config/winston';
|
||||
import { EToolResources, FileContext } from 'librechat-data-provider';
|
||||
import type { FilterQuery, SortOrder, Model } from 'mongoose';
|
||||
import type { IMongoFile } from '~/types/file';
|
||||
|
||||
/** Factory function that takes mongoose instance and returns the file methods */
|
||||
export function createFileMethods(mongoose: typeof import('mongoose')) {
|
||||
/**
|
||||
* Finds a file by its file_id with additional query options.
|
||||
* @param file_id - The unique identifier of the file
|
||||
* @param options - Query options for filtering, projection, etc.
|
||||
* @returns A promise that resolves to the file document or null
|
||||
*/
|
||||
async function findFileById(
|
||||
file_id: string,
|
||||
options: Record<string, unknown> = {},
|
||||
): Promise<IMongoFile | null> {
|
||||
const File = mongoose.models.File as Model<IMongoFile>;
|
||||
return File.findOne({ file_id, ...options }).lean();
|
||||
}
|
||||
|
||||
/** Select fields for query projection - 0 to exclude, 1 to include */
|
||||
type SelectProjection = Record<string, 0 | 1>;
|
||||
|
||||
/**
|
||||
* Retrieves files matching a given filter, sorted by the most recently updated.
|
||||
* @param filter - The filter criteria to apply
|
||||
* @param _sortOptions - Optional sort parameters
|
||||
* @param selectFields - Fields to include/exclude in the query results. Default excludes the 'text' field
|
||||
* @param options - Additional query options (userId, agentId for ACL)
|
||||
* @returns A promise that resolves to an array of file documents
|
||||
*/
|
||||
async function getFiles(
|
||||
filter: FilterQuery<IMongoFile>,
|
||||
_sortOptions?: Record<string, SortOrder> | null,
|
||||
selectFields?: SelectProjection | string | null,
|
||||
): Promise<IMongoFile[] | null> {
|
||||
const File = mongoose.models.File as Model<IMongoFile>;
|
||||
const sortOptions = { updatedAt: -1 as SortOrder, ..._sortOptions };
|
||||
const query = File.find(filter);
|
||||
if (selectFields != null) {
|
||||
query.select(selectFields);
|
||||
} else {
|
||||
query.select({ text: 0 });
|
||||
}
|
||||
return await query.sort(sortOptions).lean();
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves tool files (files that are embedded or have a fileIdentifier) from an array of file IDs
|
||||
* @param fileIds - Array of file_id strings to search for
|
||||
* @param toolResourceSet - Optional filter for tool resources
|
||||
* @returns Files that match the criteria
|
||||
*/
|
||||
async function getToolFilesByIds(
|
||||
fileIds: string[],
|
||||
toolResourceSet?: Set<EToolResources>,
|
||||
): Promise<IMongoFile[]> {
|
||||
if (!fileIds || !fileIds.length || !toolResourceSet?.size) {
|
||||
return [];
|
||||
}
|
||||
|
||||
try {
|
||||
const filter: FilterQuery<IMongoFile> = {
|
||||
file_id: { $in: fileIds },
|
||||
$or: [],
|
||||
};
|
||||
|
||||
if (toolResourceSet.has(EToolResources.context)) {
|
||||
filter.$or?.push({ text: { $exists: true, $ne: null }, context: FileContext.agents });
|
||||
}
|
||||
if (toolResourceSet.has(EToolResources.file_search)) {
|
||||
filter.$or?.push({ embedded: true });
|
||||
}
|
||||
if (toolResourceSet.has(EToolResources.execute_code)) {
|
||||
filter.$or?.push({ 'metadata.fileIdentifier': { $exists: true } });
|
||||
}
|
||||
|
||||
const selectFields: SelectProjection = { text: 0 };
|
||||
const sortOptions = { updatedAt: -1 as SortOrder };
|
||||
|
||||
const results = await getFiles(filter, sortOptions, selectFields);
|
||||
return results ?? [];
|
||||
} catch (error) {
|
||||
logger.error('[getToolFilesByIds] Error retrieving tool files:', error);
|
||||
throw new Error('Error retrieving tool files');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new file with a TTL of 1 hour.
|
||||
* @param data - The file data to be created, must contain file_id
|
||||
* @param disableTTL - Whether to disable the TTL
|
||||
* @returns A promise that resolves to the created file document
|
||||
*/
|
||||
async function createFile(
|
||||
data: Partial<IMongoFile>,
|
||||
disableTTL?: boolean,
|
||||
): Promise<IMongoFile | null> {
|
||||
const File = mongoose.models.File as Model<IMongoFile>;
|
||||
const fileData: Partial<IMongoFile> = {
|
||||
...data,
|
||||
expiresAt: new Date(Date.now() + 3600 * 1000),
|
||||
};
|
||||
|
||||
if (disableTTL) {
|
||||
delete fileData.expiresAt;
|
||||
}
|
||||
|
||||
return File.findOneAndUpdate({ file_id: data.file_id }, fileData, {
|
||||
new: true,
|
||||
upsert: true,
|
||||
}).lean();
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates a file identified by file_id with new data and removes the TTL.
|
||||
* @param data - The data to update, must contain file_id
|
||||
* @returns A promise that resolves to the updated file document
|
||||
*/
|
||||
async function updateFile(
|
||||
data: Partial<IMongoFile> & { file_id: string },
|
||||
): Promise<IMongoFile | null> {
|
||||
const File = mongoose.models.File as Model<IMongoFile>;
|
||||
const { file_id, ...update } = data;
|
||||
const updateOperation = {
|
||||
$set: update,
|
||||
$unset: { expiresAt: '' },
|
||||
};
|
||||
return File.findOneAndUpdate({ file_id }, updateOperation, {
|
||||
new: true,
|
||||
}).lean();
|
||||
}
|
||||
|
||||
/**
|
||||
* Increments the usage of a file identified by file_id.
|
||||
* @param data - The data to update, must contain file_id and the increment value for usage
|
||||
* @returns A promise that resolves to the updated file document
|
||||
*/
|
||||
async function updateFileUsage(data: {
|
||||
file_id: string;
|
||||
inc?: number;
|
||||
}): Promise<IMongoFile | null> {
|
||||
const File = mongoose.models.File as Model<IMongoFile>;
|
||||
const { file_id, inc = 1 } = data;
|
||||
const updateOperation = {
|
||||
$inc: { usage: inc },
|
||||
$unset: { expiresAt: '', temp_file_id: '' },
|
||||
};
|
||||
return File.findOneAndUpdate({ file_id }, updateOperation, {
|
||||
new: true,
|
||||
}).lean();
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes a file identified by file_id.
|
||||
* @param file_id - The unique identifier of the file to delete
|
||||
* @returns A promise that resolves to the deleted file document or null
|
||||
*/
|
||||
async function deleteFile(file_id: string): Promise<IMongoFile | null> {
|
||||
const File = mongoose.models.File as Model<IMongoFile>;
|
||||
return File.findOneAndDelete({ file_id }).lean();
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes a file identified by a filter.
|
||||
* @param filter - The filter criteria to apply
|
||||
* @returns A promise that resolves to the deleted file document or null
|
||||
*/
|
||||
async function deleteFileByFilter(filter: FilterQuery<IMongoFile>): Promise<IMongoFile | null> {
|
||||
const File = mongoose.models.File as Model<IMongoFile>;
|
||||
return File.findOneAndDelete(filter).lean();
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes multiple files identified by an array of file_ids.
|
||||
* @param file_ids - The unique identifiers of the files to delete
|
||||
* @param user - Optional user ID to filter by
|
||||
* @returns A promise that resolves to the result of the deletion operation
|
||||
*/
|
||||
async function deleteFiles(
|
||||
file_ids: string[],
|
||||
user?: string,
|
||||
): Promise<{ deletedCount?: number }> {
|
||||
const File = mongoose.models.File as Model<IMongoFile>;
|
||||
let deleteQuery: FilterQuery<IMongoFile> = { file_id: { $in: file_ids } };
|
||||
if (user) {
|
||||
deleteQuery = { user: user };
|
||||
}
|
||||
return File.deleteMany(deleteQuery);
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch updates files with new signed URLs in MongoDB
|
||||
* @param updates - Array of updates in the format { file_id, filepath }
|
||||
*/
|
||||
async function batchUpdateFiles(
|
||||
updates: Array<{ file_id: string; filepath: string }>,
|
||||
): Promise<void> {
|
||||
if (!updates || updates.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const File = mongoose.models.File as Model<IMongoFile>;
|
||||
const bulkOperations = updates.map((update) => ({
|
||||
updateOne: {
|
||||
filter: { file_id: update.file_id },
|
||||
update: { $set: { filepath: update.filepath } },
|
||||
},
|
||||
}));
|
||||
|
||||
const result = await File.bulkWrite(bulkOperations);
|
||||
logger.info(`Updated ${result.modifiedCount} files with new S3 URLs`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates usage tracking for multiple files.
|
||||
* Processes files and optional fileIds, updating their usage count in the database.
|
||||
*
|
||||
* @param files - Array of file objects to process
|
||||
* @param fileIds - Optional array of file IDs to process
|
||||
* @returns Array of updated file documents (with null results filtered out)
|
||||
*/
|
||||
async function updateFilesUsage(
|
||||
files: Array<{ file_id: string }>,
|
||||
fileIds?: string[],
|
||||
): Promise<IMongoFile[]> {
|
||||
const promises: Promise<IMongoFile | null>[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
for (const file of files) {
|
||||
const { file_id } = file;
|
||||
if (seen.has(file_id)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(file_id);
|
||||
promises.push(updateFileUsage({ file_id }));
|
||||
}
|
||||
|
||||
if (!fileIds) {
|
||||
const results = await Promise.all(promises);
|
||||
return results.filter((result): result is IMongoFile => result != null);
|
||||
}
|
||||
|
||||
for (const file_id of fileIds) {
|
||||
if (seen.has(file_id)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(file_id);
|
||||
promises.push(updateFileUsage({ file_id }));
|
||||
}
|
||||
|
||||
const results = await Promise.all(promises);
|
||||
return results.filter((result): result is IMongoFile => result != null);
|
||||
}
|
||||
|
||||
return {
|
||||
findFileById,
|
||||
getFiles,
|
||||
getToolFilesByIds,
|
||||
createFile,
|
||||
updateFile,
|
||||
updateFileUsage,
|
||||
deleteFile,
|
||||
deleteFiles,
|
||||
deleteFileByFilter,
|
||||
batchUpdateFiles,
|
||||
updateFilesUsage,
|
||||
};
|
||||
}
|
||||
|
||||
export type FileMethods = ReturnType<typeof createFileMethods>;
|
||||
Loading…
Add table
Add a link
Reference in a new issue