🚀 Feat: Streamline File Strategies & GPT-4-Vision Settings (#1535)

* chore: fix `endpoint` typescript issues and typo in console info message

* feat(api): files GET endpoint and save only file_id references to messages

* refactor(client): `useGetFiles` query hook, update file types, optimistic update of filesQuery on file upload

* refactor(buildTree): update to use params object and accept fileMap

* feat: map files to messages; refactor(ChatView): messages only available after files are fetched

* fix: fetch files only when authenticated

* feat(api): AppService
- rename app.locals.configs to app.locals.paths
- load custom config use fileStrategy from yaml config in app.locals

* refactor: separate Firebase and Local strategies, call based on config

* refactor: modularize file strategies and employ with use of DALL-E

* refactor(librechat.yaml): add fileStrategy field

* feat: add source to MongoFile schema, as well as BatchFile, and ExtendedFile types

* feat: employ file strategies for upload/delete files

* refactor(deleteFirebaseFile): add user id validation for firebase file deletion

* chore(deleteFirebaseFile): update jsdocs

* feat: employ strategies for vision requests

* fix(client): handle messages with deleted files

* fix(client): ensure `filesToDelete` always saves/sends `file.source`

* feat(openAI): configurable `resendImages` and `imageDetail`

* refactor(getTokenCountForMessage): recursive process only when array of Objects and only their values (not keys) aside from `image_url` types

* feat(OpenAIClient): calculateImageTokenCost

* chore: remove comment

* refactor(uploadAvatar): employ fileStrategy for avatars, from social logins or user upload

* docs: update docs on how to configure fileStrategy

* fix(ci): mock winston and winston related modules, update DALLE3.spec.js with changes made

* refactor(redis): change terminal message to reflect current development state

* fix(DALL-E-2): pass fileStrategy to dall-e

api/server/services/Files/images/encode.js

@@ -1,45 +1,5 @@
-const fs = require('fs');
-const path = require('path');
-const { EModelEndpoint } = require('librechat-data-provider');
-const { updateFile } = require('~/models');
-
-/**
- * Encodes an image file to base64.
- * @param {string} imagePath - The path to the image file.
- * @returns {Promise<string>} A promise that resolves with the base64 encoded image data.
- */
-function encodeImage(imagePath) {
-  return new Promise((resolve, reject) => {
-    fs.readFile(imagePath, (err, data) => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve(data.toString('base64'));
-      }
-    });
-  });
-}
-
-/**
- * Updates the file and encodes the image.
- * @param {Object} req - The request object.
- * @param {Object} file - The file object.
- * @returns {Promise<[MongoFile, string]>}  - A promise that resolves to an array of results from updateFile and encodeImage.
- */
-async function updateAndEncode(req, file) {
-  const { publicPath, imageOutput } = req.app.locals.config;
-  const userPath = path.join(imageOutput, req.user.id);
-
-  if (!fs.existsSync(userPath)) {
-    fs.mkdirSync(userPath, { recursive: true });
-  }
-  const filepath = path.join(publicPath, file.filepath);
-
-  const promises = [];
-  promises.push(updateFile({ file_id: file.file_id }));
-  promises.push(encodeImage(filepath));
-  return await Promise.all(promises);
-}
+const { EModelEndpoint, FileSources } = require('librechat-data-provider');
+const { getStrategyFunctions } = require('../strategies');
 
 /**
  * Encodes and formats the given files.
@@ -50,25 +10,42 @@ async function updateAndEncode(req, file) {
  */
 async function encodeAndFormat(req, files, endpoint) {
   const promises = [];
+  const encodingMethods = {};
+
   for (let file of files) {
-    promises.push(updateAndEncode(req, file));
+    const source = file.source ?? FileSources.local;
+
+    if (encodingMethods[source]) {
+      promises.push(encodingMethods[source](req, file));
+      continue;
+    }
+
+    const { prepareImagePayload } = getStrategyFunctions(source);
+    if (!prepareImagePayload) {
+      throw new Error(`Encoding function not implemented for ${source}`);
+    }
+
+    encodingMethods[source] = prepareImagePayload;
+    promises.push(prepareImagePayload(req, file));
   }
 
-  // TODO: make detail configurable, as of now resizing is done
-  // to prefer "high" but "low" may be used if the image is small enough
-  const detail = req.body.detail ?? 'auto';
-  const encodedImages = await Promise.all(promises);
+  const detail = req.body.imageDetail ?? 'auto';
+
+  /** @type {Array<[MongoFile, string]>} */
+  const formattedImages = await Promise.all(promises);
 
   const result = {
     files: [],
     image_urls: [],
   };
 
-  for (const [file, base64] of encodedImages) {
+  for (const [file, imageContent] of formattedImages) {
     const imagePart = {
       type: 'image_url',
       image_url: {
-        url: `data:image/webp;base64,${base64}`,
+        url: imageContent.startsWith('http')
+          ? imageContent
+          : `data:image/webp;base64,${imageContent}`,
         detail,
       },
     };
@@ -81,17 +58,16 @@ async function encodeAndFormat(req, files, endpoint) {
 
     result.files.push({
       file_id: file.file_id,
-      filepath: file.filepath,
-      filename: file.filename,
-      type: file.type,
-      height: file.height,
-      width: file.width,
+      // filepath: file.filepath,
+      // filename: file.filename,
+      // type: file.type,
+      // height: file.height,
+      // width: file.width,
     });
   }
   return result;
 }
 
 module.exports = {
-  encodeImage,
   encodeAndFormat,
 };