🚀 Feat: Streamline File Strategies & GPT-4-Vision Settings (#1535)

* chore: fix `endpoint` typescript issues and typo in console info message

* feat(api): files GET endpoint and save only file_id references to messages

* refactor(client): `useGetFiles` query hook, update file types, optimistic update of filesQuery on file upload

* refactor(buildTree): update to use params object and accept fileMap

* feat: map files to messages; refactor(ChatView): messages only available after files are fetched

* fix: fetch files only when authenticated

* feat(api): AppService
- rename app.locals.configs to app.locals.paths
- load custom config use fileStrategy from yaml config in app.locals

* refactor: separate Firebase and Local strategies, call based on config

* refactor: modularize file strategies and employ with use of DALL-E

* refactor(librechat.yaml): add fileStrategy field

* feat: add source to MongoFile schema, as well as BatchFile, and ExtendedFile types

* feat: employ file strategies for upload/delete files

* refactor(deleteFirebaseFile): add user id validation for firebase file deletion

* chore(deleteFirebaseFile): update jsdocs

* feat: employ strategies for vision requests

* fix(client): handle messages with deleted files

* fix(client): ensure `filesToDelete` always saves/sends `file.source`

* feat(openAI): configurable `resendImages` and `imageDetail`

* refactor(getTokenCountForMessage): recursive process only when array of Objects and only their values (not keys) aside from `image_url` types

* feat(OpenAIClient): calculateImageTokenCost

* chore: remove comment

* refactor(uploadAvatar): employ fileStrategy for avatars, from social logins or user upload

* docs: update docs on how to configure fileStrategy

* fix(ci): mock winston and winston related modules, update DALLE3.spec.js with changes made

* refactor(redis): change terminal message to reflect current development state

* fix(DALL-E-2): pass fileStrategy to dall-e

packages/data-provider/package.json

@@ -1,6 +1,6 @@
 {
   "name": "librechat-data-provider",
-  "version": "0.3.5",
+  "version": "0.3.6",
   "description": "data services for librechat apps",
   "main": "dist/index.js",
   "module": "dist/index.es.js",

packages/data-provider/src/config.ts

@@ -1,5 +1,8 @@
 import { z } from 'zod';
 import { EModelEndpoint, eModelEndpointSchema } from './schemas';
+import { FileSources } from './types/files';
+
+export const fileSourceSchema = z.nativeEnum(FileSources);
 
 export const endpointSchema = z.object({
   name: z.string().refine((value) => !eModelEndpointSchema.safeParse(value).success, {
@@ -25,6 +28,7 @@ export const endpointSchema = z.object({
 export const configSchema = z.object({
   version: z.string(),
   cache: z.boolean(),
+  fileStrategy: fileSourceSchema.optional(),
   endpoints: z
     .object({
       custom: z.array(endpointSchema.partial()),
@@ -184,3 +188,27 @@ export enum AuthKeys {
    */
   GOOGLE_API_KEY = 'GOOGLE_API_KEY',
 }
+
+/**
+ * Enum for Image Detail Cost.
+ *
+ * **Low Res Fixed Cost:** `85`
+ *
+ * **High Res Calculation:**
+ *
+ * Number of `512px` Tiles * `170` + `85` (Additional Cost)
+ */
+export enum ImageDetailCost {
+  /**
+   * Low resolution is a fixed value.
+   */
+  LOW = 85,
+  /**
+   * High resolution Cost Per Tile
+   */
+  HIGH = 170,
+  /**
+   * Additional Cost added to High Resolution Total Cost
+   */
+  ADDITIONAL = 85,
+}

packages/data-provider/src/data-service.ts

@@ -193,7 +193,11 @@ export const listAssistants = (
 
 /* Files */
 
-export const uploadImage = (data: FormData): Promise<f.FileUploadResponse> => {
+export const getFiles = (): Promise<f.TFile[]> => {
+  return request.get(endpoints.files());
+};
+
+export const uploadImage = (data: FormData): Promise<f.TFileUpload> => {
   return request.postMultiPart(endpoints.images(), data);
 };
 

packages/data-provider/src/keys.ts

@@ -16,6 +16,7 @@ export enum QueryKeys {
   assistants = 'assistants',
   assistant = 'assistant',
   endpointsConfigOverride = 'endpointsConfigOverride',
+  files = 'files',
 }
 
 export enum MutationKeys {

packages/data-provider/src/schemas.ts

@@ -52,6 +52,26 @@ export const eModelEndpointSchema = z.nativeEnum(EModelEndpoint);
 
 export const extendedModelEndpointSchema = z.union([eModelEndpointSchema, z.string()]);
 
+export enum ImageDetail {
+  low = 'low',
+  auto = 'auto',
+  high = 'high',
+}
+
+export const imageDetailNumeric = {
+  [ImageDetail.low]: 0,
+  [ImageDetail.auto]: 1,
+  [ImageDetail.high]: 2,
+};
+
+export const imageDetailValue = {
+  0: ImageDetail.low,
+  1: ImageDetail.auto,
+  2: ImageDetail.high,
+};
+
+export const eImageDetailSchema = z.nativeEnum(ImageDetail);
+
 export const tPluginAuthConfigSchema = z.object({
   authField: z.string(),
   label: z.string(),
@@ -140,8 +160,8 @@ export type TMessage = z.input<typeof tMessageSchema> & {
   plugin?: TResPlugin | null;
   plugins?: TResPlugin[];
   files?: {
-    type: string;
     file_id: string;
+    type?: string;
     filename?: string;
     preview?: string;
     filepath?: string;
@@ -184,6 +204,9 @@ export const tConversationSchema = z.object({
   toneStyle: z.string().nullable().optional(),
   maxOutputTokens: z.number().optional(),
   agentOptions: tAgentOptionsSchema.nullable().optional(),
+  /* vision */
+  resendImages: z.boolean().optional(),
+  imageDetail: eImageDetailSchema.optional(),
   /* assistant */
   assistant_id: z.string().optional(),
   thread_id: z.string().optional(),
@@ -234,6 +257,8 @@ export const openAISchema = tConversationSchema
     top_p: true,
     presence_penalty: true,
     frequency_penalty: true,
+    resendImages: true,
+    imageDetail: true,
   })
   .transform((obj) => ({
     ...obj,
@@ -244,6 +269,8 @@ export const openAISchema = tConversationSchema
     top_p: obj.top_p ?? 1,
     presence_penalty: obj.presence_penalty ?? 0,
     frequency_penalty: obj.frequency_penalty ?? 0,
+    resendImages: obj.resendImages ?? false,
+    imageDetail: obj.imageDetail ?? ImageDetail.auto,
   }))
   .catch(() => ({
     model: 'gpt-3.5-turbo',
@@ -253,6 +280,8 @@ export const openAISchema = tConversationSchema
     top_p: 1,
     presence_penalty: 0,
     frequency_penalty: 0,
+    resendImages: false,
+    imageDetail: ImageDetail.auto,
   }));
 
 export const googleSchema = tConversationSchema
@@ -455,6 +484,8 @@ export const compactOpenAISchema = tConversationSchema
     top_p: true,
     presence_penalty: true,
     frequency_penalty: true,
+    resendImages: true,
+    imageDetail: true,
   })
   .transform((obj: Partial<TConversation>) => {
     const newObj: Partial<TConversation> = { ...obj };
@@ -473,6 +504,12 @@ export const compactOpenAISchema = tConversationSchema
     if (newObj.frequency_penalty === 0) {
       delete newObj.frequency_penalty;
     }
+    if (newObj.resendImages !== true) {
+      delete newObj.resendImages;
+    }
+    if (newObj.imageDetail === ImageDetail.auto) {
+      delete newObj.imageDetail;
+    }
 
     return removeNullishValues(newObj);
   })

packages/data-provider/src/types/files.ts

@@ -1,13 +1,25 @@
-export type FileUploadResponse = {
+export enum FileSources {
+  local = 'local',
+  firebase = 'firebase',
+  openai = 'openai',
+  s3 = 's3',
+}
+
+export type TFile = {
   message: string;
   file_id: string;
-  temp_file_id: string;
   filepath: string;
   filename: string;
   type: string;
   size: number;
-  height: number;
-  width: number;
+  temp_file_id?: string;
+  source?: FileSources;
+  height?: number;
+  width?: number;
+};
+
+export type TFileUpload = TFile & {
+  temp_file_id: string;
 };
 
 export type AvatarUploadResponse = {
@@ -20,7 +32,7 @@ export type FileUploadBody = {
 };
 
 export type UploadMutationOptions = {
-  onSuccess?: (data: FileUploadResponse, variables: FileUploadBody, context?: unknown) => void;
+  onSuccess?: (data: TFileUpload, variables: FileUploadBody, context?: unknown) => void;
   onMutate?: (variables: FileUploadBody) => void | Promise<unknown>;
   onError?: (error: unknown, variables: FileUploadBody, context?: unknown) => void;
 };
@@ -39,6 +51,7 @@ export type DeleteFilesResponse = {
 export type BatchFile = {
   file_id: string;
   filepath: string;
+  source: FileSources;
 };
 
 export type DeleteFilesBody = {