🚀 Feat: Streamline File Strategies & GPT-4-Vision Settings (#1535)

* chore: fix `endpoint` typescript issues and typo in console info message

* feat(api): files GET endpoint and save only file_id references to messages

* refactor(client): `useGetFiles` query hook, update file types, optimistic update of filesQuery on file upload

* refactor(buildTree): update to use params object and accept fileMap

* feat: map files to messages; refactor(ChatView): messages only available after files are fetched

* fix: fetch files only when authenticated

* feat(api): AppService
- rename app.locals.configs to app.locals.paths
- load custom config use fileStrategy from yaml config in app.locals

* refactor: separate Firebase and Local strategies, call based on config

* refactor: modularize file strategies and employ with use of DALL-E

* refactor(librechat.yaml): add fileStrategy field

* feat: add source to MongoFile schema, as well as BatchFile, and ExtendedFile types

* feat: employ file strategies for upload/delete files

* refactor(deleteFirebaseFile): add user id validation for firebase file deletion

* chore(deleteFirebaseFile): update jsdocs

* feat: employ strategies for vision requests

* fix(client): handle messages with deleted files

* fix(client): ensure `filesToDelete` always saves/sends `file.source`

* feat(openAI): configurable `resendImages` and `imageDetail`

* refactor(getTokenCountForMessage): recursive process only when array of Objects and only their values (not keys) aside from `image_url` types

* feat(OpenAIClient): calculateImageTokenCost

* chore: remove comment

* refactor(uploadAvatar): employ fileStrategy for avatars, from social logins or user upload

* docs: update docs on how to configure fileStrategy

* fix(ci): mock winston and winston related modules, update DALLE3.spec.js with changes made

* refactor(redis): change terminal message to reflect current development state

* fix(DALL-E-2): pass fileStrategy to dall-e

api/strategies/discordStrategy.js

@@ -1,12 +1,14 @@
 const { Strategy: DiscordStrategy } = require('passport-discord');
+const { createNewUser, handleExistingUser } = require('./process');
 const { logger } = require('~/config');
 const User = require('~/models/User');
-const { useFirebase, uploadAvatar } = require('~/server/services/Files/images');
 
 const discordLogin = async (accessToken, refreshToken, profile, cb) => {
   try {
     const email = profile.email;
     const discordId = profile.id;
+
+    // TODO: remove direct access of User model
     const oldUser = await User.findOne({ email });
     const ALLOW_SOCIAL_REGISTRATION =
       process.env.ALLOW_SOCIAL_REGISTRATION?.toLowerCase() === 'true';
@@ -21,12 +23,20 @@ const discordLogin = async (accessToken, refreshToken, profile, cb) => {
     }
 
     if (oldUser) {
-      await handleExistingUser(oldUser, avatarUrl, useFirebase);
+      await handleExistingUser(oldUser, avatarUrl);
       return cb(null, oldUser);
     }
 
     if (ALLOW_SOCIAL_REGISTRATION) {
-      const newUser = await createNewUser(profile, discordId, email, avatarUrl, useFirebase);
+      const newUser = await createNewUser({
+        email,
+        avatarUrl,
+        provider: 'discord',
+        providerKey: 'discordId',
+        providerId: discordId,
+        username: profile.username,
+        name: profile.global_name,
+      });
       return cb(null, newUser);
     }
   } catch (err) {
@@ -35,38 +45,6 @@ const discordLogin = async (accessToken, refreshToken, profile, cb) => {
   }
 };
 
-const handleExistingUser = async (oldUser, avatarUrl, useFirebase) => {
-  if (!useFirebase && !oldUser.avatar.includes('?manual=true')) {
-    oldUser.avatar = avatarUrl;
-    await oldUser.save();
-  } else if (useFirebase && !oldUser.avatar.includes('?manual=true')) {
-    const userId = oldUser._id;
-    const newavatarUrl = await uploadAvatar(userId, avatarUrl);
-    oldUser.avatar = newavatarUrl;
-    await oldUser.save();
-  }
-};
-
-const createNewUser = async (profile, discordId, email, avatarUrl, useFirebase) => {
-  const newUser = await new User({
-    provider: 'discord',
-    discordId,
-    username: profile.username,
-    email,
-    name: profile.global_name,
-    avatar: avatarUrl,
-  }).save();
-
-  if (useFirebase) {
-    const userId = newUser._id;
-    const newavatarUrl = await uploadAvatar(userId, avatarUrl);
-    newUser.avatar = newavatarUrl;
-    await newUser.save();
-  }
-
-  return newUser;
-};
-
 module.exports = () =>
   new DiscordStrategy(
     {

api/strategies/facebookStrategy.js

@@ -1,7 +1,7 @@
 const FacebookStrategy = require('passport-facebook').Strategy;
+const { createNewUser, handleExistingUser } = require('./process');
 const { logger } = require('~/config');
 const User = require('~/models/User');
-const { useFirebase, uploadAvatar } = require('~/server/services/Files/images');
 
 const facebookLogin = async (accessToken, refreshToken, profile, cb) => {
   try {
@@ -13,12 +13,20 @@ const facebookLogin = async (accessToken, refreshToken, profile, cb) => {
     const avatarUrl = profile.photos[0]?.value;
 
     if (oldUser) {
-      await handleExistingUser(oldUser, avatarUrl, useFirebase);
+      await handleExistingUser(oldUser, avatarUrl);
       return cb(null, oldUser);
     }
 
     if (ALLOW_SOCIAL_REGISTRATION) {
-      const newUser = await createNewUser(profile, facebookId, email, avatarUrl, useFirebase);
+      const newUser = await createNewUser({
+        email,
+        avatarUrl,
+        provider: 'facebook',
+        providerKey: 'facebookId',
+        providerId: facebookId,
+        username: profile.displayName,
+        name: profile.name?.givenName + ' ' + profile.name?.familyName,
+      });
       return cb(null, newUser);
     }
   } catch (err) {
@@ -27,38 +35,6 @@ const facebookLogin = async (accessToken, refreshToken, profile, cb) => {
   }
 };
 
-const handleExistingUser = async (oldUser, avatarUrl, useFirebase) => {
-  if (!useFirebase && !oldUser.avatar.includes('?manual=true')) {
-    oldUser.avatar = avatarUrl;
-    await oldUser.save();
-  } else if (useFirebase && !oldUser.avatar.includes('?manual=true')) {
-    const userId = oldUser._id;
-    const newavatarUrl = await uploadAvatar(userId, avatarUrl);
-    oldUser.avatar = newavatarUrl;
-    await oldUser.save();
-  }
-};
-
-const createNewUser = async (profile, facebookId, email, avatarUrl, useFirebase) => {
-  const newUser = await new User({
-    provider: 'facebook',
-    facebookId,
-    username: profile.displayName,
-    email,
-    name: profile.name?.givenName + ' ' + profile.name?.familyName,
-    avatar: avatarUrl,
-  }).save();
-
-  if (useFirebase) {
-    const userId = newUser._id;
-    const newavatarUrl = await uploadAvatar(userId, avatarUrl);
-    newUser.avatar = newavatarUrl;
-    await newUser.save();
-  }
-
-  return newUser;
-};
-
 module.exports = () =>
   new FacebookStrategy(
     {

api/strategies/githubStrategy.js

@@ -1,7 +1,7 @@
 const { Strategy: GitHubStrategy } = require('passport-github2');
+const { createNewUser, handleExistingUser } = require('./process');
 const { logger } = require('~/config');
 const User = require('~/models/User');
-const { useFirebase, uploadAvatar } = require('~/server/services/Files/images');
 
 const githubLogin = async (accessToken, refreshToken, profile, cb) => {
   try {
@@ -13,12 +13,21 @@ const githubLogin = async (accessToken, refreshToken, profile, cb) => {
     const avatarUrl = profile.photos[0].value;
 
     if (oldUser) {
-      await handleExistingUser(oldUser, avatarUrl, useFirebase);
+      await handleExistingUser(oldUser, avatarUrl);
       return cb(null, oldUser);
     }
 
     if (ALLOW_SOCIAL_REGISTRATION) {
-      const newUser = await createNewUser(profile, githubId, email, avatarUrl, useFirebase);
+      const newUser = await createNewUser({
+        email,
+        avatarUrl,
+        provider: 'github',
+        providerKey: 'githubId',
+        providerId: githubId,
+        username: profile.username,
+        name: profile.displayName,
+        emailVerified: profile.emails[0].verified,
+      });
       return cb(null, newUser);
     }
   } catch (err) {
@@ -27,39 +36,6 @@ const githubLogin = async (accessToken, refreshToken, profile, cb) => {
   }
 };
 
-const handleExistingUser = async (oldUser, avatarUrl, useFirebase) => {
-  if (!useFirebase && !oldUser.avatar.includes('?manual=true')) {
-    oldUser.avatar = avatarUrl;
-    await oldUser.save();
-  } else if (useFirebase && !oldUser.avatar.includes('?manual=true')) {
-    const userId = oldUser._id;
-    const avatarURL = await uploadAvatar(userId, avatarUrl);
-    oldUser.avatar = avatarURL;
-    await oldUser.save();
-  }
-};
-
-const createNewUser = async (profile, githubId, email, avatarUrl, useFirebase) => {
-  const newUser = await new User({
-    provider: 'github',
-    githubId,
-    username: profile.username,
-    email,
-    emailVerified: profile.emails[0].verified,
-    name: profile.displayName,
-    avatar: avatarUrl,
-  }).save();
-
-  if (useFirebase) {
-    const userId = newUser._id;
-    const avatarURL = await uploadAvatar(userId, avatarUrl);
-    newUser.avatar = avatarURL;
-    await newUser.save();
-  }
-
-  return newUser;
-};
-
 module.exports = () =>
   new GitHubStrategy(
     {

api/strategies/googleStrategy.js

@@ -1,7 +1,7 @@
 const { Strategy: GoogleStrategy } = require('passport-google-oauth20');
+const { createNewUser, handleExistingUser } = require('./process');
 const { logger } = require('~/config');
 const User = require('~/models/User');
-const { useFirebase, uploadAvatar } = require('~/server/services/Files/images');
 
 const googleLogin = async (accessToken, refreshToken, profile, cb) => {
   try {
@@ -13,12 +13,21 @@ const googleLogin = async (accessToken, refreshToken, profile, cb) => {
     const avatarUrl = profile.photos[0].value;
 
     if (oldUser) {
-      await handleExistingUser(oldUser, avatarUrl, useFirebase);
+      await handleExistingUser(oldUser, avatarUrl);
       return cb(null, oldUser);
     }
 
     if (ALLOW_SOCIAL_REGISTRATION) {
-      const newUser = await createNewUser(profile, googleId, email, avatarUrl, useFirebase);
+      const newUser = await createNewUser({
+        email,
+        avatarUrl,
+        provider: 'google',
+        providerKey: 'googleId',
+        providerId: googleId,
+        username: profile.name.givenName,
+        name: `${profile.name.givenName} ${profile.name.familyName}`,
+        emailVerified: profile.emails[0].verified,
+      });
       return cb(null, newUser);
     }
   } catch (err) {
@@ -27,39 +36,6 @@ const googleLogin = async (accessToken, refreshToken, profile, cb) => {
   }
 };
 
-const handleExistingUser = async (oldUser, avatarUrl, useFirebase) => {
-  if ((!useFirebase && !oldUser.avatar.includes('?manual=true')) || oldUser.avatar === null) {
-    oldUser.avatar = avatarUrl;
-    await oldUser.save();
-  } else if (useFirebase && !oldUser.avatar.includes('?manual=true')) {
-    const userId = oldUser._id;
-    const avatarURL = await uploadAvatar(userId, avatarUrl);
-    oldUser.avatar = avatarURL;
-    await oldUser.save();
-  }
-};
-
-const createNewUser = async (profile, googleId, email, avatarUrl, useFirebase) => {
-  const newUser = await new User({
-    provider: 'google',
-    googleId,
-    username: profile.name.givenName,
-    email,
-    emailVerified: profile.emails[0].verified,
-    name: `${profile.name.givenName} ${profile.name.familyName}`,
-    avatar: avatarUrl,
-  }).save();
-
-  if (useFirebase) {
-    const userId = newUser._id;
-    const avatarURL = await uploadAvatar(userId, avatarUrl);
-    newUser.avatar = avatarURL;
-    await newUser.save();
-  }
-
-  return newUser;
-};
-
 module.exports = () =>
   new GoogleStrategy(
     {

api/strategies/process.js

@@ -0,0 +1,92 @@
+const { FileSources } = require('librechat-data-provider');
+const uploadAvatar = require('~/server/services/Files/images/avatar');
+const User = require('~/models/User');
+
+/**
+ * Updates the avatar URL of an existing user. If the user's avatar URL does not include the query parameter
+ * '?manual=true', it updates the user's avatar with the provided URL. For local file storage, it directly updates
+ * the avatar URL, while for other storage types, it processes the avatar URL using the specified file strategy.
+ *
+ * @param {User} oldUser - The existing user object that needs to be updated. Expected to have an 'avatar' property.
+ * @param {string} avatarUrl - The new avatar URL to be set for the user.
+ *
+ * @returns {Promise<void>}
+ *          The function updates the user's avatar and saves the user object. It does not return any value.
+ *
+ * @throws {Error} Throws an error if there's an issue saving the updated user object.
+ */
+const handleExistingUser = async (oldUser, avatarUrl) => {
+  const fileStrategy = process.env.CDN_PROVIDER;
+  const isLocal = fileStrategy === FileSources.local;
+
+  if (isLocal && !oldUser.avatar.includes('?manual=true')) {
+    oldUser.avatar = avatarUrl;
+    await oldUser.save();
+  } else if (!isLocal && !oldUser.avatar.includes('?manual=true')) {
+    const userId = oldUser._id;
+    const newavatarUrl = await uploadAvatar({ userId, input: avatarUrl, fileStrategy });
+    oldUser.avatar = newavatarUrl;
+    await oldUser.save();
+  }
+};
+
+/**
+ * Creates a new user with the provided user details. If the file strategy is not local, the avatar URL is
+ * processed using the specified file strategy. The new user is saved to the database with the processed or
+ * original avatar URL.
+ *
+ * @param {Object} params - The parameters object for user creation.
+ * @param {string} params.email - The email of the new user.
+ * @param {string} params.avatarUrl - The avatar URL of the new user.
+ * @param {string} params.provider - The provider of the user's account.
+ * @param {string} params.providerKey - The key to identify the provider in the user model.
+ * @param {string} params.providerId - The provider-specific ID of the user.
+ * @param {string} params.username - The username of the new user.
+ * @param {string} params.name - The name of the new user.
+ * @param {boolean} [params.emailVerified=false] - Optional. Indicates whether the user's email is verified. Defaults to false.
+ *
+ * @returns {Promise<User>}
+ *          A promise that resolves to the newly created user object.
+ *
+ * @throws {Error} Throws an error if there's an issue creating or saving the new user object.
+ */
+const createNewUser = async ({
+  email,
+  avatarUrl,
+  provider,
+  providerKey,
+  providerId,
+  username,
+  name,
+  emailVerified,
+}) => {
+  const update = {
+    email,
+    avatar: avatarUrl,
+    provider,
+    [providerKey]: providerId,
+    username,
+    name,
+    emailVerified,
+  };
+
+  // TODO: remove direct access of User model
+  const newUser = await new User(update).save();
+
+  const fileStrategy = process.env.CDN_PROVIDER;
+  const isLocal = fileStrategy === FileSources.local;
+
+  if (!isLocal) {
+    const userId = newUser._id;
+    const newavatarUrl = await uploadAvatar({ userId, input: avatarUrl, fileStrategy });
+    newUser.avatar = newavatarUrl;
+    await newUser.save();
+  }
+
+  return newUser;
+};
+
+module.exports = {
+  handleExistingUser,
+  createNewUser,
+};