feat: Vision Support + New UI (#1203)

* feat: add timer duration to showToast, show toast for preset selection

* refactor: replace old /chat/ route with /c/. e2e tests will fail here

* refactor: move typedefs to root of /api/ and add a few to assistant types in TS

* refactor: reorganize data-provider imports, fix dependency cycle, strategize new plan to separate react dependent packages

* feat: add dataService for uploading images

* feat(data-provider): add mutation keys

* feat: file resizing and upload

* WIP: initial API image handling

* fix: catch JSON.parse of localStorage tools

* chore: experimental: use module-alias for absolute imports

* refactor: change temp_file_id strategy

* fix: updating files state by using Map and defining react query callbacks in a way that keeps them during component unmount, initial delete handling

* feat: properly handle file deletion

* refactor: unexpose complete filepath and resize from server for higher fidelity

* fix: make sure resized height, width is saved, catch bad requests

* refactor: use absolute imports

* fix: prevent setOptions from being called more than once for OpenAIClient, made note to fix for PluginsClient

* refactor: import supportsFiles and models vars from schemas

* fix: correctly replace temp file id

* refactor(BaseClient): use absolute imports, pass message 'opts' to buildMessages method, count tokens for nested objects/arrays

* feat: add validateVisionModel to determine if model has vision capabilities

* chore(checkBalance): update jsdoc

* feat: formatVisionMessage: change message content format dependent on role and image_urls passed

* refactor: add usage to File schema, make create and updateFile, correctly set and remove TTL

* feat: working vision support
TODO: file size, type, amount validations, making sure they are styled right, and making sure you can add images from the clipboard/dragging

* feat: clipboard support for uploading images

* feat: handle files on drop to screen, refactor top level view code to Presentation component so the useDragHelpers hook  has ChatContext

* fix(Images): replace uploaded images in place

* feat: add filepath validation to protect sensitive files

* fix: ensure correct file_ids are push and not the Map key values

* fix(ToastContext): type issue

* feat: add basic file validation

* fix(useDragHelpers): correct context issue with `files` dependency

* refactor: consolidate setErrors logic to setError

* feat: add dialog Image overlay on image click

* fix: close endpoints menu on click

* chore: set detail to auto, make note for configuration

* fix: react warning (button desc. of button)

* refactor: optimize filepath handling, pass file_ids to images for easier re-use

* refactor: optimize image file handling, allow re-using files in regen, pass more file metadata in messages

* feat: lazy loading images including use of upload preview

* fix: SetKeyDialog closing, stopPropagation on Dialog content click

* style(EndpointMenuItem): tighten up the style, fix dark theme showing in lightmode, make menu more ux friendly

* style: change maxheight of all settings textareas to 138px from 300px

* style: better styling for textarea and enclosing buttons

* refactor(PresetItems): swap back edit and delete icons

* feat: make textarea placeholder dynamic to endpoint

* style: show user hover buttons only on hover when message is streaming

* fix: ordered list not going past 9, fix css

* feat: add User/AI labels; style: hide loading spinner

* feat: add back custom footer, change original footer text

* feat: dynamic landing icons based on endpoint

* chore: comment out assistants route

* fix: autoScroll to newest on /c/ view

* fix: Export Conversation on new UI

* style: match message style of official more closely

* ci: fix api jest unit tests, comment out e2e tests for now as they will fail until addressed

* feat: more file validation and use blob in preview field, not filepath, to fix temp deletion

* feat: filefilter for multer

* feat: better AI labels based on custom name, model, and endpoint instead of  `ChatGPT`

api/server/routes/files/files.js

@@ -0,0 +1,58 @@
+const { z } = require('zod');
+const fs = require('fs').promises;
+const express = require('express');
+const { deleteFiles } = require('~/models');
+const path = require('path');
+
+const router = express.Router();
+
+const isUUID = z.string().uuid();
+
+const isValidPath = (base, subfolder, filepath) => {
+  const normalizedBase = path.resolve(base, subfolder, 'temp');
+  const normalizedFilepath = path.resolve(filepath);
+  return normalizedFilepath.startsWith(normalizedBase);
+};
+
+const deleteFile = async (req, file) => {
+  const { publicPath } = req.app.locals.config;
+  const parts = file.filepath.split(path.sep);
+  const subfolder = parts[1];
+  const filepath = path.join(publicPath, file.filepath);
+
+  if (!isValidPath(publicPath, subfolder, filepath)) {
+    throw new Error('Invalid file path');
+  }
+
+  await fs.unlink(filepath);
+};
+
+router.delete('/', async (req, res) => {
+  try {
+    const { files: _files } = req.body;
+    const files = _files.filter((file) => {
+      if (!file.file_id) {
+        return false;
+      }
+      if (!file.filepath) {
+        return false;
+      }
+      return isUUID.safeParse(file.file_id).success;
+    });
+
+    const file_ids = files.map((file) => file.file_id);
+    const promises = [];
+    promises.push(await deleteFiles(file_ids));
+    for (const file of files) {
+      promises.push(deleteFile(req, file));
+    }
+
+    await Promise.all(promises);
+    res.status(200).json({ message: 'Files deleted successfully' });
+  } catch (error) {
+    console.error('Error deleting files:', error);
+    res.status(400).json({ message: 'Error in request', error: error.message });
+  }
+});
+
+module.exports = router;

api/server/routes/files/images.js

@@ -0,0 +1,58 @@
+const { z } = require('zod');
+const fs = require('fs').promises;
+const express = require('express');
+const upload = require('./multer');
+const { localStrategy } = require('~/server/services/Files');
+
+const router = express.Router();
+
+router.post('/', upload.single('file'), async (req, res) => {
+  const file = req.file;
+  const metadata = req.body;
+  // TODO: add file size/type validation
+
+  const uuidSchema = z.string().uuid();
+
+  try {
+    if (!file) {
+      throw new Error('No file provided');
+    }
+
+    if (!metadata.file_id) {
+      throw new Error('No file_id provided');
+    }
+
+    if (!metadata.width) {
+      throw new Error('No width provided');
+    }
+
+    if (!metadata.height) {
+      throw new Error('No height provided');
+    }
+    /* parse to validate api call */
+    uuidSchema.parse(metadata.file_id);
+    metadata.temp_file_id = metadata.file_id;
+    metadata.file_id = req.file_id;
+    await localStrategy({ res, file, metadata });
+  } catch (error) {
+    console.error('Error processing file:', error);
+    try {
+      await fs.unlink(file.path);
+    } catch (error) {
+      console.error('Error deleting file:', error);
+    }
+    res.status(500).json({ message: 'Error processing file' });
+  }
+
+  // do this if strategy is not local
+  // finally {
+  //   try {
+  //     // await fs.unlink(file.path);
+  //   } catch (error) {
+  //     console.error('Error deleting file:', error);
+
+  //   }
+  // }
+});
+
+module.exports = router;

api/server/routes/files/index.js

@@ -0,0 +1,22 @@
+const express = require('express');
+const router = express.Router();
+const {
+  uaParser,
+  checkBan,
+  requireJwtAuth,
+  // concurrentLimiter,
+  // messageIpLimiter,
+  // messageUserLimiter,
+} = require('../../middleware');
+
+const files = require('./files');
+const images = require('./images');
+
+router.use(requireJwtAuth);
+router.use(checkBan);
+router.use(uaParser);
+
+router.use('/', files);
+router.use('/images', images);
+
+module.exports = router;

api/server/routes/files/multer.js

@@ -0,0 +1,41 @@
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+const multer = require('multer');
+
+const supportedTypes = ['image/jpeg', 'image/jpg', 'image/png', 'image/webp'];
+const sizeLimit = 20 * 1024 * 1024; // 20 MB
+
+const storage = multer.diskStorage({
+  destination: function (req, file, cb) {
+    const outputPath = path.join(req.app.locals.config.imageOutput, 'temp');
+    if (!fs.existsSync(outputPath)) {
+      fs.mkdirSync(outputPath, { recursive: true });
+    }
+    cb(null, outputPath);
+  },
+  filename: function (req, file, cb) {
+    req.file_id = crypto.randomUUID();
+    const fileExt = path.extname(file.originalname);
+    cb(null, `img-${req.file_id}${fileExt}`);
+  },
+});
+
+const fileFilter = (req, file, cb) => {
+  if (!supportedTypes.includes(file.mimetype)) {
+    return cb(
+      new Error('Unsupported file type. Only JPEG, JPG, PNG, and WEBP files are allowed.'),
+      false,
+    );
+  }
+
+  if (file.size > sizeLimit) {
+    return cb(new Error(`File size exceeds ${sizeLimit / 1024 / 1024} MB.`), false);
+  }
+
+  cb(null, true);
+};
+
+const upload = multer({ storage, fileFilter });
+
+module.exports = upload;