feat: Vision Support + New UI (#1203)

* feat: add timer duration to showToast, show toast for preset selection

* refactor: replace old /chat/ route with /c/. e2e tests will fail here

* refactor: move typedefs to root of /api/ and add a few to assistant types in TS

* refactor: reorganize data-provider imports, fix dependency cycle, strategize new plan to separate react dependent packages

* feat: add dataService for uploading images

* feat(data-provider): add mutation keys

* feat: file resizing and upload

* WIP: initial API image handling

* fix: catch JSON.parse of localStorage tools

* chore: experimental: use module-alias for absolute imports

* refactor: change temp_file_id strategy

* fix: updating files state by using Map and defining react query callbacks in a way that keeps them during component unmount, initial delete handling

* feat: properly handle file deletion

* refactor: unexpose complete filepath and resize from server for higher fidelity

* fix: make sure resized height, width is saved, catch bad requests

* refactor: use absolute imports

* fix: prevent setOptions from being called more than once for OpenAIClient, made note to fix for PluginsClient

* refactor: import supportsFiles and models vars from schemas

* fix: correctly replace temp file id

* refactor(BaseClient): use absolute imports, pass message 'opts' to buildMessages method, count tokens for nested objects/arrays

* feat: add validateVisionModel to determine if model has vision capabilities

* chore(checkBalance): update jsdoc

* feat: formatVisionMessage: change message content format dependent on role and image_urls passed

* refactor: add usage to File schema, make create and updateFile, correctly set and remove TTL

* feat: working vision support
TODO: file size, type, amount validations, making sure they are styled right, and making sure you can add images from the clipboard/dragging

* feat: clipboard support for uploading images

* feat: handle files on drop to screen, refactor top level view code to Presentation component so the useDragHelpers hook  has ChatContext

* fix(Images): replace uploaded images in place

* feat: add filepath validation to protect sensitive files

* fix: ensure correct file_ids are push and not the Map key values

* fix(ToastContext): type issue

* feat: add basic file validation

* fix(useDragHelpers): correct context issue with `files` dependency

* refactor: consolidate setErrors logic to setError

* feat: add dialog Image overlay on image click

* fix: close endpoints menu on click

* chore: set detail to auto, make note for configuration

* fix: react warning (button desc. of button)

* refactor: optimize filepath handling, pass file_ids to images for easier re-use

* refactor: optimize image file handling, allow re-using files in regen, pass more file metadata in messages

* feat: lazy loading images including use of upload preview

* fix: SetKeyDialog closing, stopPropagation on Dialog content click

* style(EndpointMenuItem): tighten up the style, fix dark theme showing in lightmode, make menu more ux friendly

* style: change maxheight of all settings textareas to 138px from 300px

* style: better styling for textarea and enclosing buttons

* refactor(PresetItems): swap back edit and delete icons

* feat: make textarea placeholder dynamic to endpoint

* style: show user hover buttons only on hover when message is streaming

* fix: ordered list not going past 9, fix css

* feat: add User/AI labels; style: hide loading spinner

* feat: add back custom footer, change original footer text

* feat: dynamic landing icons based on endpoint

* chore: comment out assistants route

* fix: autoScroll to newest on /c/ view

* fix: Export Conversation on new UI

* style: match message style of official more closely

* ci: fix api jest unit tests, comment out e2e tests for now as they will fail until addressed

* feat: more file validation and use blob in preview field, not filepath, to fix temp deletion

* feat: filefilter for multer

* feat: better AI labels based on custom name, model, and endpoint instead of  `ChatGPT`

api/server/services/AssistantService.js

@@ -1,21 +1,5 @@
 const RunManager = require('./Runs/RunMananger');
 
-/**
- * @typedef {import('openai').OpenAI} OpenAI
- * @typedef {import('openai').OpenAI.Beta.Threads.ThreadMessage} ThreadMessage
- * @typedef {import('openai').OpenAI.Beta.Threads.RequiredActionFunctionToolCall} RequiredActionFunctionToolCall
- * @typedef {import('./Runs/RunManager').RunManager} RunManager
- */
-
-/**
- * @typedef {Object} Thread
- * @property {string} id - The identifier of the thread.
- * @property {string} object - The object type, always 'thread'.
- * @property {number} created_at - The Unix timestamp (in seconds) for when the thread was created.
- * @property {Object} [metadata] - Optional metadata associated with the thread.
- * @property {Message[]} [messages] - An array of messages associated with the thread.
- */
-
 /**
  * @typedef {Object} Message
  * @property {string} id - The identifier of the message.
@@ -247,27 +231,6 @@ async function waitForRun({ openai, run_id, thread_id, runManager, pollIntervalM
   return run;
 }
 
-/**
- * @typedef {Object} AgentAction
- * @property {string} tool - The name of the tool used.
- * @property {string} toolInput - The input provided to the tool.
- * @property {string} log - A log or message associated with the action.
- */
-
-/**
- * @typedef {Object} AgentFinish
- * @property {Record<string, any>} returnValues - The return values of the agent's execution.
- * @property {string} log - A log or message associated with the finish.
- */
-
-/**
- * @typedef {AgentFinish & { run_id: string; thread_id: string; }} OpenAIAssistantFinish
- */
-
-/**
- * @typedef {AgentAction & { toolCallId: string; run_id: string; thread_id: string; }} OpenAIAssistantAction
- */
-
 /**
  * Retrieves the response from an OpenAI run.
  *

api/server/services/Files/images/convert.js

@@ -0,0 +1,17 @@
+const path = require('path');
+const sharp = require('sharp');
+const fs = require('fs').promises;
+const { resizeImage } = require('./resize');
+
+async function convertToWebP(inputFilePath, resolution = 'high') {
+  const { buffer: resizedBuffer, width, height } = await resizeImage(inputFilePath, resolution);
+  const outputFilePath = inputFilePath.replace(/\.[^/.]+$/, '') + '.webp';
+  const data = await sharp(resizedBuffer).toFormat('webp').toBuffer();
+  await fs.writeFile(outputFilePath, data);
+  const bytes = Buffer.byteLength(data);
+  const filepath = path.posix.join('/', 'images', 'temp', path.basename(outputFilePath));
+  await fs.unlink(inputFilePath);
+  return { filepath, bytes, width, height };
+}
+
+module.exports = { convertToWebP };

api/server/services/Files/images/encode.js

@@ -0,0 +1,80 @@
+const fs = require('fs');
+const path = require('path');
+const { updateFile } = require('~/models');
+
+function encodeImage(imagePath) {
+  return new Promise((resolve, reject) => {
+    fs.readFile(imagePath, (err, data) => {
+      if (err) {
+        reject(err);
+      } else {
+        resolve(data.toString('base64'));
+      }
+    });
+  });
+}
+
+async function encodeAndMove(req, file) {
+  const { publicPath, imageOutput } = req.app.locals.config;
+  const userPath = path.join(imageOutput, req.user.id);
+
+  if (!fs.existsSync(userPath)) {
+    fs.mkdirSync(userPath, { recursive: true });
+  }
+  const filepath = path.join(publicPath, file.filepath);
+
+  if (!filepath.includes('temp')) {
+    const base64 = await encodeImage(filepath);
+    return [file, base64];
+  }
+
+  const newPath = path.join(userPath, path.basename(file.filepath));
+  await fs.promises.rename(filepath, newPath);
+  const newFilePath = path.posix.join('/', 'images', req.user.id, path.basename(file.filepath));
+  const promises = [];
+  promises.push(updateFile({ file_id: file.file_id, filepath: newFilePath }));
+  promises.push(encodeImage(newPath));
+  return await Promise.all(promises);
+}
+
+async function encodeAndFormat(req, files) {
+  const promises = [];
+  for (let file of files) {
+    promises.push(encodeAndMove(req, file));
+  }
+
+  // TODO: make detail configurable, as of now resizing is done
+  // to prefer "high" but "low" may be used if the image is small enough
+  const detail = req.body.detail ?? 'auto';
+  const encodedImages = await Promise.all(promises);
+
+  const result = {
+    files: [],
+    image_urls: [],
+  };
+
+  for (const [file, base64] of encodedImages) {
+    result.image_urls.push({
+      type: 'image_url',
+      image_url: {
+        url: `data:image/webp;base64,${base64}`,
+        detail,
+      },
+    });
+
+    result.files.push({
+      file_id: file.file_id,
+      filepath: file.filepath,
+      filename: file.filename,
+      type: file.type,
+      height: file.height,
+      width: file.width,
+    });
+  }
+  return result;
+}
+
+module.exports = {
+  encodeImage,
+  encodeAndFormat,
+};

api/server/services/Files/images/index.js

@@ -0,0 +1,11 @@
+const convert = require('./convert');
+const encode = require('./encode');
+const resize = require('./resize');
+const validate = require('./validate');
+
+module.exports = {
+  ...convert,
+  ...encode,
+  ...resize,
+  ...validate,
+};

api/server/services/Files/images/resize.js

@@ -0,0 +1,52 @@
+const sharp = require('sharp');
+
+async function resizeImage(inputFilePath, resolution) {
+  const maxLowRes = 512;
+  const maxShortSideHighRes = 768;
+  const maxLongSideHighRes = 2000;
+
+  let newWidth, newHeight;
+  let resizeOptions = { fit: 'inside', withoutEnlargement: true };
+
+  if (resolution === 'low') {
+    resizeOptions.width = maxLowRes;
+    resizeOptions.height = maxLowRes;
+  } else if (resolution === 'high') {
+    const metadata = await sharp(inputFilePath).metadata();
+    const isWidthShorter = metadata.width < metadata.height;
+
+    if (isWidthShorter) {
+      // Width is the shorter side
+      newWidth = Math.min(metadata.width, maxShortSideHighRes);
+      // Calculate new height to maintain aspect ratio
+      newHeight = Math.round((metadata.height / metadata.width) * newWidth);
+      // Ensure the long side does not exceed the maximum allowed
+      if (newHeight > maxLongSideHighRes) {
+        newHeight = maxLongSideHighRes;
+        newWidth = Math.round((metadata.width / metadata.height) * newHeight);
+      }
+    } else {
+      // Height is the shorter side
+      newHeight = Math.min(metadata.height, maxShortSideHighRes);
+      // Calculate new width to maintain aspect ratio
+      newWidth = Math.round((metadata.width / metadata.height) * newHeight);
+      // Ensure the long side does not exceed the maximum allowed
+      if (newWidth > maxLongSideHighRes) {
+        newWidth = maxLongSideHighRes;
+        newHeight = Math.round((metadata.height / metadata.width) * newWidth);
+      }
+    }
+
+    resizeOptions.width = newWidth;
+    resizeOptions.height = newHeight;
+  } else {
+    throw new Error('Invalid resolution parameter');
+  }
+
+  const resizedBuffer = await sharp(inputFilePath).resize(resizeOptions).toBuffer();
+
+  const resizedMetadata = await sharp(resizedBuffer).metadata();
+  return { buffer: resizedBuffer, width: resizedMetadata.width, height: resizedMetadata.height };
+}
+
+module.exports = { resizeImage };

api/server/services/Files/images/validate.js

@@ -0,0 +1,13 @@
+const { visionModels } = require('~/server/routes/endpoints/schemas');
+
+function validateVisionModel(model) {
+  if (!model) {
+    return false;
+  }
+
+  return visionModels.some((visionModel) => model.includes(visionModel));
+}
+
+module.exports = {
+  validateVisionModel,
+};

api/server/services/Files/index.js

@@ -0,0 +1,9 @@
+const localStrategy = require('./localStrategy');
+const process = require('./process');
+const save = require('./save');
+
+module.exports = {
+  ...save,
+  ...process,
+  localStrategy,
+};

api/server/services/Files/localStrategy.js

@@ -0,0 +1,34 @@
+const { createFile } = require('~/models');
+const { convertToWebP } = require('./images/convert');
+
+/**
+ * Applies the local strategy for image uploads.
+ * Saves file metadata to the database with an expiry TTL.
+ * Files must be deleted from the server filesystem manually.
+ *
+ * @param {Object} params - The parameters object.
+ * @param {Express.Response} params.res - The Express response object.
+ * @param {Express.Multer.File} params.file - The uploaded file.
+ * @param {ImageMetadata} params.metadata - Additional metadata for the file.
+ * @returns {Promise<void>}
+ */
+const localStrategy = async ({ res, file, metadata }) => {
+  const { file_id, temp_file_id } = metadata;
+  const { filepath, bytes, width, height } = await convertToWebP(file.path);
+  const result = await createFile(
+    {
+      file_id,
+      temp_file_id,
+      bytes,
+      filepath,
+      filename: file.originalname,
+      type: 'image/webp',
+      width,
+      height,
+    },
+    true,
+  );
+  res.status(200).json({ message: 'File uploaded and processed successfully', ...result });
+};
+
+module.exports = localStrategy;

api/server/services/Files/process.js

@@ -0,0 +1,29 @@
+const { updateFileUsage } = require('~/models');
+
+// const mapImageUrls = (files, detail) => {
+//   return files
+//     .filter((file) => file.type.includes('image'))
+//     .map((file) => ({
+//       type: 'image_url',
+//       image_url: {
+//         /* Temporarily set to path to encode later */
+//         url: file.filepath,
+//         detail,
+//       },
+//     }));
+// };
+
+const processFiles = async (files) => {
+  const promises = [];
+  for (let file of files) {
+    const { file_id } = file;
+    promises.push(updateFileUsage({ file_id }));
+  }
+
+  // TODO: calculate token cost when image is first uploaded
+  return await Promise.all(promises);
+};
+
+module.exports = {
+  processFiles,
+};

api/server/services/Files/save.js

@@ -0,0 +1,47 @@
+const fs = require('fs');
+const path = require('path');
+
+/**
+ * Saves a file to a specified output path with a new filename.
+ *
+ * @param {Express.Multer.File} file - The file object to be saved. Should contain properties like 'originalname' and 'path'.
+ * @param {string} outputPath - The path where the file should be saved.
+ * @param {string} outputFilename - The new filename for the saved file (without extension).
+ * @returns {Promise<string>} The full path of the saved file.
+ * @throws Will throw an error if the file saving process fails.
+ */
+async function saveFile(file, outputPath, outputFilename) {
+  try {
+    if (!fs.existsSync(outputPath)) {
+      fs.mkdirSync(outputPath, { recursive: true });
+    }
+
+    const fileExtension = path.extname(file.originalname);
+    const filenameWithExt = outputFilename + fileExtension;
+    const outputFilePath = path.join(outputPath, filenameWithExt);
+    fs.copyFileSync(file.path, outputFilePath);
+    fs.unlinkSync(file.path);
+
+    return outputFilePath;
+  } catch (error) {
+    console.error('Error while saving the file:', error);
+    throw error;
+  }
+}
+
+/**
+ * Saves an uploaded image file to a specified directory based on the user's ID and a filename.
+ *
+ * @param {Express.Request} req - The Express request object, containing the user's information and app configuration.
+ * @param {Express.Multer.File} file - The uploaded file object.
+ * @param {string} filename - The new filename to assign to the saved image (without extension).
+ * @returns {Promise<void>}
+ * @throws Will throw an error if the image saving process fails.
+ */
+const saveLocalImage = async (req, file, filename) => {
+  const imagePath = req.app.locals.config.imageOutput;
+  const outputPath = path.join(imagePath, req.user.id ?? '');
+  await saveFile(file, outputPath, filename);
+};
+
+module.exports = { saveFile, saveLocalImage };