feat: Vision Support + New UI (#1203)

* feat: add timer duration to showToast, show toast for preset selection

* refactor: replace old /chat/ route with /c/. e2e tests will fail here

* refactor: move typedefs to root of /api/ and add a few to assistant types in TS

* refactor: reorganize data-provider imports, fix dependency cycle, strategize new plan to separate react dependent packages

* feat: add dataService for uploading images

* feat(data-provider): add mutation keys

* feat: file resizing and upload

* WIP: initial API image handling

* fix: catch JSON.parse of localStorage tools

* chore: experimental: use module-alias for absolute imports

* refactor: change temp_file_id strategy

* fix: updating files state by using Map and defining react query callbacks in a way that keeps them during component unmount, initial delete handling

* feat: properly handle file deletion

* refactor: unexpose complete filepath and resize from server for higher fidelity

* fix: make sure resized height, width is saved, catch bad requests

* refactor: use absolute imports

* fix: prevent setOptions from being called more than once for OpenAIClient, made note to fix for PluginsClient

* refactor: import supportsFiles and models vars from schemas

* fix: correctly replace temp file id

* refactor(BaseClient): use absolute imports, pass message 'opts' to buildMessages method, count tokens for nested objects/arrays

* feat: add validateVisionModel to determine if model has vision capabilities

* chore(checkBalance): update jsdoc

* feat: formatVisionMessage: change message content format dependent on role and image_urls passed

* refactor: add usage to File schema, make create and updateFile, correctly set and remove TTL

* feat: working vision support
TODO: file size, type, amount validations, making sure they are styled right, and making sure you can add images from the clipboard/dragging

* feat: clipboard support for uploading images

* feat: handle files on drop to screen, refactor top level view code to Presentation component so the useDragHelpers hook  has ChatContext

* fix(Images): replace uploaded images in place

* feat: add filepath validation to protect sensitive files

* fix: ensure correct file_ids are push and not the Map key values

* fix(ToastContext): type issue

* feat: add basic file validation

* fix(useDragHelpers): correct context issue with `files` dependency

* refactor: consolidate setErrors logic to setError

* feat: add dialog Image overlay on image click

* fix: close endpoints menu on click

* chore: set detail to auto, make note for configuration

* fix: react warning (button desc. of button)

* refactor: optimize filepath handling, pass file_ids to images for easier re-use

* refactor: optimize image file handling, allow re-using files in regen, pass more file metadata in messages

* feat: lazy loading images including use of upload preview

* fix: SetKeyDialog closing, stopPropagation on Dialog content click

* style(EndpointMenuItem): tighten up the style, fix dark theme showing in lightmode, make menu more ux friendly

* style: change maxheight of all settings textareas to 138px from 300px

* style: better styling for textarea and enclosing buttons

* refactor(PresetItems): swap back edit and delete icons

* feat: make textarea placeholder dynamic to endpoint

* style: show user hover buttons only on hover when message is streaming

* fix: ordered list not going past 9, fix css

* feat: add User/AI labels; style: hide loading spinner

* feat: add back custom footer, change original footer text

* feat: dynamic landing icons based on endpoint

* chore: comment out assistants route

* fix: autoScroll to newest on /c/ view

* fix: Export Conversation on new UI

* style: match message style of official more closely

* ci: fix api jest unit tests, comment out e2e tests for now as they will fail until addressed

* feat: more file validation and use blob in preview field, not filepath, to fix temp deletion

* feat: filefilter for multer

* feat: better AI labels based on custom name, model, and endpoint instead of  `ChatGPT`

api/models/File.js

@@ -0,0 +1,96 @@
+const mongoose = require('mongoose');
+const fileSchema = require('./schema/fileSchema');
+
+const File = mongoose.model('File', fileSchema);
+
+/**
+ * Finds a file by its file_id with additional query options.
+ * @param {string} file_id - The unique identifier of the file.
+ * @param {object} options - Query options for filtering, projection, etc.
+ * @returns {Promise<MongoFile>} A promise that resolves to the file document or null.
+ */
+const findFileById = async (file_id, options = {}) => {
+  return await File.findOne({ file_id, ...options }).lean();
+};
+
+/**
+ * Retrieves files matching a given filter.
+ * @param {Object} filter - The filter criteria to apply.
+ * @returns {Promise<Array<MongoFile>>} A promise that resolves to an array of file documents.
+ */
+const getFiles = async (filter) => {
+  return await File.find(filter).lean();
+};
+
+/**
+ * Creates a new file with a TTL of 1 hour.
+ * @param {Object} data - The file data to be created, must contain file_id.
+ * @returns {Promise<MongoFile>} A promise that resolves to the created file document.
+ */
+const createFile = async (data) => {
+  const fileData = {
+    ...data,
+    expiresAt: new Date(Date.now() + 3600 * 1000),
+  };
+  return await File.findOneAndUpdate({ file_id: data.file_id }, fileData, {
+    new: true,
+    upsert: true,
+  }).lean();
+};
+
+/**
+ * Updates a file identified by file_id with new data and removes the TTL.
+ * @param {Object} data - The data to update, must contain file_id.
+ * @returns {Promise<MongoFile>} A promise that resolves to the updated file document.
+ */
+const updateFile = async (data) => {
+  const { file_id, ...update } = data;
+  const updateOperation = {
+    $set: update,
+    $unset: { expiresAt: '' }, // Remove the expiresAt field to prevent TTL
+  };
+  return await File.findOneAndUpdate({ file_id }, updateOperation, { new: true }).lean();
+};
+
+/**
+ * Increments the usage of a file identified by file_id.
+ * @param {Object} data - The data to update, must contain file_id and the increment value for usage.
+ * @returns {Promise<MongoFile>} A promise that resolves to the updated file document.
+ */
+const updateFileUsage = async (data) => {
+  const { file_id, inc = 1 } = data;
+  const updateOperation = {
+    $inc: { usage: inc },
+    $unset: { expiresAt: '' },
+  };
+  return await File.findOneAndUpdate({ file_id }, updateOperation, { new: true }).lean();
+};
+
+/**
+ * Deletes a file identified by file_id.
+ * @param {string} file_id - The unique identifier of the file to delete.
+ * @returns {Promise<MongoFile>} A promise that resolves to the deleted file document or null.
+ */
+const deleteFile = async (file_id) => {
+  return await File.findOneAndDelete({ file_id }).lean();
+};
+
+/**
+ * Deletes multiple files identified by an array of file_ids.
+ * @param {Array<string>} file_ids - The unique identifiers of the files to delete.
+ * @returns {Promise<Object>} A promise that resolves to the result of the deletion operation.
+ */
+const deleteFiles = async (file_ids) => {
+  return await File.deleteMany({ file_id: { $in: file_ids } });
+};
+
+module.exports = {
+  File,
+  findFileById,
+  getFiles,
+  createFile,
+  updateFile,
+  updateFileUsage,
+  deleteFile,
+  deleteFiles,
+};

api/models/Message.js

@@ -18,6 +18,7 @@ module.exports = {
     error,
     unfinished,
     cancelled,
+    files,
     isEdited = false,
     finish_reason = null,
     tokenCount = null,
@@ -30,29 +31,31 @@ module.exports = {
       if (!validConvoId.success) {
         return;
       }
+
+      const update = {
+        user,
+        messageId: newMessageId || messageId,
+        conversationId,
+        parentMessageId,
+        sender,
+        text,
+        isCreatedByUser,
+        isEdited,
+        finish_reason,
+        error,
+        unfinished,
+        cancelled,
+        tokenCount,
+        plugin,
+        plugins,
+        model,
+      };
+
+      if (files) {
+        update.files = files;
+      }
       // may also need to update the conversation here
-      await Message.findOneAndUpdate(
-        { messageId },
-        {
-          user,
-          messageId: newMessageId || messageId,
-          conversationId,
-          parentMessageId,
-          sender,
-          text,
-          isCreatedByUser,
-          isEdited,
-          finish_reason,
-          error,
-          unfinished,
-          cancelled,
-          tokenCount,
-          plugin,
-          plugins,
-          model,
-        },
-        { upsert: true, new: true },
-      );
+      await Message.findOneAndUpdate({ messageId }, update, { upsert: true, new: true });
 
       return {
         messageId,

api/models/checkBalance.js

@@ -7,8 +7,8 @@ const { logViolation } = require('../cache');
  * @async
  * @function
  * @param {Object} params - The function parameters.
- * @param {Object} params.req - The Express request object.
- * @param {Object} params.res - The Express response object.
+ * @param {Express.Request} params.req - The Express request object.
+ * @param {Express.Response} params.res - The Express response object.
  * @param {Object} params.txData - The transaction data.
  * @param {string} params.txData.user - The user ID or identifier.
  * @param {('prompt' | 'completion')} params.txData.tokenType - The type of token.

api/models/index.js

@@ -7,6 +7,15 @@ const {
 } = require('./Message');
 const { getConvoTitle, getConvo, saveConvo, deleteConvos } = require('./Conversation');
 const { getPreset, getPresets, savePreset, deletePresets } = require('./Preset');
+const {
+  findFileById,
+  createFile,
+  updateFile,
+  deleteFile,
+  deleteFiles,
+  getFiles,
+  updateFileUsage,
+} = require('./File');
 const Key = require('./Key');
 const User = require('./User');
 const Session = require('./Session');
@@ -35,4 +44,12 @@ module.exports = {
   getPresets,
   savePreset,
   deletePresets,
+
+  findFileById,
+  createFile,
+  updateFile,
+  deleteFile,
+  deleteFiles,
+  getFiles,
+  updateFileUsage,
 };

api/models/schema/fileSchema.js

@@ -0,0 +1,79 @@
+const mongoose = require('mongoose');
+
+/**
+ * @typedef {Object} MongoFile
+ * @property {mongoose.Schema.Types.ObjectId} user - User ID
+ * @property {string} [conversationId] - Optional conversation ID
+ * @property {string} file_id - File identifier
+ * @property {string} [temp_file_id] - Temporary File identifier
+ * @property {number} bytes - Size of the file in bytes
+ * @property {string} filename - Name of the file
+ * @property {string} filepath - Location of the file
+ * @property {'file'} object - Type of object, always 'file'
+ * @property {string} type - Type of file
+ * @property {number} usage - Number of uses of the file
+ * @property {number} [width] - Optional width of the file
+ * @property {number} [height] - Optional height of the file
+ * @property {Date} [expiresAt] - Optional height of the file
+ */
+const fileSchema = mongoose.Schema(
+  {
+    user: {
+      type: mongoose.Schema.Types.ObjectId,
+      ref: 'User',
+      index: true,
+      required: true,
+    },
+    conversationId: {
+      type: String,
+      ref: 'Conversation',
+      index: true,
+    },
+    file_id: {
+      type: String,
+      // required: true,
+      index: true,
+    },
+    temp_file_id: {
+      type: String,
+      // required: true,
+    },
+    bytes: {
+      type: Number,
+      required: true,
+    },
+    usage: {
+      type: Number,
+      required: true,
+      default: 0,
+    },
+    filename: {
+      type: String,
+      required: true,
+    },
+    filepath: {
+      type: String,
+      required: true,
+    },
+    object: {
+      type: String,
+      required: true,
+      default: 'file',
+    },
+    type: {
+      type: String,
+      required: true,
+    },
+    width: Number,
+    height: Number,
+    expiresAt: {
+      type: Date,
+      expires: 3600,
+    },
+  },
+  {
+    timestamps: true,
+  },
+);
+
+module.exports = fileSchema;

api/models/schema/messageSchema.js

@@ -85,6 +85,7 @@ const messageSchema = mongoose.Schema(
       select: false,
       default: false,
     },
+    files: [{ type: mongoose.Schema.Types.Mixed }],
     plugin: {
       latest: {
         type: String,