feat: Vision Support + New UI (#1203)

* feat: add timer duration to showToast, show toast for preset selection

* refactor: replace old /chat/ route with /c/. e2e tests will fail here

* refactor: move typedefs to root of /api/ and add a few to assistant types in TS

* refactor: reorganize data-provider imports, fix dependency cycle, strategize new plan to separate react dependent packages

* feat: add dataService for uploading images

* feat(data-provider): add mutation keys

* feat: file resizing and upload

* WIP: initial API image handling

* fix: catch JSON.parse of localStorage tools

* chore: experimental: use module-alias for absolute imports

* refactor: change temp_file_id strategy

* fix: updating files state by using Map and defining react query callbacks in a way that keeps them during component unmount, initial delete handling

* feat: properly handle file deletion

* refactor: unexpose complete filepath and resize from server for higher fidelity

* fix: make sure resized height, width is saved, catch bad requests

* refactor: use absolute imports

* fix: prevent setOptions from being called more than once for OpenAIClient, made note to fix for PluginsClient

* refactor: import supportsFiles and models vars from schemas

* fix: correctly replace temp file id

* refactor(BaseClient): use absolute imports, pass message 'opts' to buildMessages method, count tokens for nested objects/arrays

* feat: add validateVisionModel to determine if model has vision capabilities

* chore(checkBalance): update jsdoc

* feat: formatVisionMessage: change message content format dependent on role and image_urls passed

* refactor: add usage to File schema, make create and updateFile, correctly set and remove TTL

* feat: working vision support
TODO: file size, type, amount validations, making sure they are styled right, and making sure you can add images from the clipboard/dragging

* feat: clipboard support for uploading images

* feat: handle files on drop to screen, refactor top level view code to Presentation component so the useDragHelpers hook  has ChatContext

* fix(Images): replace uploaded images in place

* feat: add filepath validation to protect sensitive files

* fix: ensure correct file_ids are push and not the Map key values

* fix(ToastContext): type issue

* feat: add basic file validation

* fix(useDragHelpers): correct context issue with `files` dependency

* refactor: consolidate setErrors logic to setError

* feat: add dialog Image overlay on image click

* fix: close endpoints menu on click

* chore: set detail to auto, make note for configuration

* fix: react warning (button desc. of button)

* refactor: optimize filepath handling, pass file_ids to images for easier re-use

* refactor: optimize image file handling, allow re-using files in regen, pass more file metadata in messages

* feat: lazy loading images including use of upload preview

* fix: SetKeyDialog closing, stopPropagation on Dialog content click

* style(EndpointMenuItem): tighten up the style, fix dark theme showing in lightmode, make menu more ux friendly

* style: change maxheight of all settings textareas to 138px from 300px

* style: better styling for textarea and enclosing buttons

* refactor(PresetItems): swap back edit and delete icons

* feat: make textarea placeholder dynamic to endpoint

* style: show user hover buttons only on hover when message is streaming

* fix: ordered list not going past 9, fix css

* feat: add User/AI labels; style: hide loading spinner

* feat: add back custom footer, change original footer text

* feat: dynamic landing icons based on endpoint

* chore: comment out assistants route

* fix: autoScroll to newest on /c/ view

* fix: Export Conversation on new UI

* style: match message style of official more closely

* ci: fix api jest unit tests, comment out e2e tests for now as they will fail until addressed

* feat: more file validation and use blob in preview field, not filepath, to fix temp deletion

* feat: filefilter for multer

* feat: better AI labels based on custom name, model, and endpoint instead of  `ChatGPT`

api/server/routes/ask/openAI.js

@@ -1,9 +1,9 @@
 const express = require('express');
 const router = express.Router();
-const { getResponseSender } = require('../endpoints/schemas');
-const { sendMessage, createOnProgress } = require('../../utils');
-const { addTitle, initializeClient } = require('../endpoints/openAI');
-const { saveMessage, getConvoTitle, getConvo } = require('../../../models');
+const { sendMessage, createOnProgress } = require('~/server/utils');
+const { saveMessage, getConvoTitle, getConvo } = require('~/models');
+const { getResponseSender } = require('~/server/routes/endpoints/schemas');
+const { addTitle, initializeClient } = require('~/server/routes/endpoints/openAI');
 const {
   handleAbort,
   createAbortController,
@@ -11,7 +11,7 @@ const {
   setHeaders,
   validateEndpoint,
   buildEndpointOption,
-} = require('../../middleware');
+} = require('~/server/middleware');
 
 router.post('/abort', handleAbort());
 
@@ -93,8 +93,7 @@ router.post('/', validateEndpoint, buildEndpointOption, setHeaders, async (req,
 
   try {
     const { client } = await initializeClient({ req, res, endpointOption });
-
-    let response = await client.sendMessage(text, {
+    const messageOptions = {
       user,
       parentMessageId,
       conversationId,
@@ -108,7 +107,9 @@ router.post('/', validateEndpoint, buildEndpointOption, setHeaders, async (req,
         text,
         parentMessageId: overrideParentMessageId || userMessageId,
       }),
-    });
+    };
+
+    let response = await client.sendMessage(text, messageOptions);
 
     if (overrideParentMessageId) {
       response.parentMessageId = overrideParentMessageId;
@@ -118,7 +119,10 @@ router.post('/', validateEndpoint, buildEndpointOption, setHeaders, async (req,
       response = { ...response, ...metadata };
     }
 
-    await saveMessage({ ...response, user });
+    if (client.options.attachments) {
+      userMessage.files = client.options.attachments;
+      delete userMessage.image_urls;
+    }
 
     sendMessage(res, {
       title: await getConvoTitle(user, conversationId),
@@ -129,6 +133,9 @@ router.post('/', validateEndpoint, buildEndpointOption, setHeaders, async (req,
     });
     res.end();
 
+    await saveMessage({ ...response, user });
+    await saveMessage(userMessage);
+
     if (parentMessageId === '00000000-0000-0000-0000-000000000000' && newConvo) {
       addTitle(req, {
         text,

api/server/routes/endpoints/openAI/initializeClient.js

@@ -1,7 +1,7 @@
-const { OpenAIClient } = require('../../../../app');
-const { isEnabled } = require('../../../utils');
-const { getAzureCredentials } = require('../../../../utils');
-const { getUserKey, checkUserKeyExpiry } = require('../../../services/UserService');
+const { OpenAIClient } = require('~/app');
+const { isEnabled } = require('~/server/utils');
+const { getAzureCredentials } = require('~/utils');
+const { getUserKey, checkUserKeyExpiry } = require('~/server/services/UserService');
 
 const initializeClient = async ({ req, res, endpointOption }) => {
   const {

api/server/routes/endpoints/schemas.js

@@ -11,6 +11,41 @@ const EModelEndpoint = {
   assistant: 'assistant',
 };
 
+const alternateName = {
+  [EModelEndpoint.openAI]: 'OpenAI',
+  [EModelEndpoint.assistant]: 'Assistants',
+  [EModelEndpoint.azureOpenAI]: 'Azure OpenAI',
+  [EModelEndpoint.bingAI]: 'Bing',
+  [EModelEndpoint.chatGPTBrowser]: 'ChatGPT',
+  [EModelEndpoint.gptPlugins]: 'Plugins',
+  [EModelEndpoint.google]: 'PaLM',
+  [EModelEndpoint.anthropic]: 'Anthropic',
+};
+
+const supportsFiles = {
+  [EModelEndpoint.openAI]: true,
+  [EModelEndpoint.assistant]: true,
+};
+
+const openAIModels = [
+  'gpt-3.5-turbo-16k-0613',
+  'gpt-3.5-turbo-16k',
+  'gpt-4-1106-preview',
+  'gpt-3.5-turbo',
+  'gpt-3.5-turbo-1106',
+  'gpt-4-vision-preview',
+  'gpt-4',
+  'gpt-3.5-turbo-instruct-0914',
+  'gpt-3.5-turbo-0613',
+  'gpt-3.5-turbo-0301',
+  'gpt-3.5-turbo-instruct',
+  'gpt-4-0613',
+  'text-davinci-003',
+  'gpt-4-0314',
+];
+
+const visionModels = ['gpt-4-vision', 'llava-13b'];
+
 const eModelEndpointSchema = z.nativeEnum(EModelEndpoint);
 
 const tPluginAuthConfigSchema = z.object({
@@ -321,7 +356,7 @@ const parseConvo = (endpoint, conversation, possibleValues) => {
 };
 
 const getResponseSender = (endpointOption) => {
-  const { endpoint, chatGptLabel, modelLabel, jailbreak } = endpointOption;
+  const { model, endpoint, chatGptLabel, modelLabel, jailbreak } = endpointOption;
 
   if (
     [
@@ -331,7 +366,14 @@ const getResponseSender = (endpointOption) => {
       EModelEndpoint.chatGPTBrowser,
     ].includes(endpoint)
   ) {
-    return chatGptLabel ?? 'ChatGPT';
+    if (chatGptLabel) {
+      return chatGptLabel;
+    } else if (model && model.includes('gpt-3')) {
+      return 'GPT-3.5';
+    } else if (model && model.includes('gpt-4')) {
+      return 'GPT-4';
+    }
+    return alternateName[endpoint] ?? 'ChatGPT';
   }
 
   if (endpoint === EModelEndpoint.bingAI) {
@@ -353,4 +395,8 @@ module.exports = {
   parseConvo,
   getResponseSender,
   EModelEndpoint,
+  supportsFiles,
+  openAIModels,
+  visionModels,
+  alternateName,
 };

api/server/routes/files/files.js

@@ -0,0 +1,58 @@
+const { z } = require('zod');
+const fs = require('fs').promises;
+const express = require('express');
+const { deleteFiles } = require('~/models');
+const path = require('path');
+
+const router = express.Router();
+
+const isUUID = z.string().uuid();
+
+const isValidPath = (base, subfolder, filepath) => {
+  const normalizedBase = path.resolve(base, subfolder, 'temp');
+  const normalizedFilepath = path.resolve(filepath);
+  return normalizedFilepath.startsWith(normalizedBase);
+};
+
+const deleteFile = async (req, file) => {
+  const { publicPath } = req.app.locals.config;
+  const parts = file.filepath.split(path.sep);
+  const subfolder = parts[1];
+  const filepath = path.join(publicPath, file.filepath);
+
+  if (!isValidPath(publicPath, subfolder, filepath)) {
+    throw new Error('Invalid file path');
+  }
+
+  await fs.unlink(filepath);
+};
+
+router.delete('/', async (req, res) => {
+  try {
+    const { files: _files } = req.body;
+    const files = _files.filter((file) => {
+      if (!file.file_id) {
+        return false;
+      }
+      if (!file.filepath) {
+        return false;
+      }
+      return isUUID.safeParse(file.file_id).success;
+    });
+
+    const file_ids = files.map((file) => file.file_id);
+    const promises = [];
+    promises.push(await deleteFiles(file_ids));
+    for (const file of files) {
+      promises.push(deleteFile(req, file));
+    }
+
+    await Promise.all(promises);
+    res.status(200).json({ message: 'Files deleted successfully' });
+  } catch (error) {
+    console.error('Error deleting files:', error);
+    res.status(400).json({ message: 'Error in request', error: error.message });
+  }
+});
+
+module.exports = router;

api/server/routes/files/images.js

@@ -0,0 +1,58 @@
+const { z } = require('zod');
+const fs = require('fs').promises;
+const express = require('express');
+const upload = require('./multer');
+const { localStrategy } = require('~/server/services/Files');
+
+const router = express.Router();
+
+router.post('/', upload.single('file'), async (req, res) => {
+  const file = req.file;
+  const metadata = req.body;
+  // TODO: add file size/type validation
+
+  const uuidSchema = z.string().uuid();
+
+  try {
+    if (!file) {
+      throw new Error('No file provided');
+    }
+
+    if (!metadata.file_id) {
+      throw new Error('No file_id provided');
+    }
+
+    if (!metadata.width) {
+      throw new Error('No width provided');
+    }
+
+    if (!metadata.height) {
+      throw new Error('No height provided');
+    }
+    /* parse to validate api call */
+    uuidSchema.parse(metadata.file_id);
+    metadata.temp_file_id = metadata.file_id;
+    metadata.file_id = req.file_id;
+    await localStrategy({ res, file, metadata });
+  } catch (error) {
+    console.error('Error processing file:', error);
+    try {
+      await fs.unlink(file.path);
+    } catch (error) {
+      console.error('Error deleting file:', error);
+    }
+    res.status(500).json({ message: 'Error processing file' });
+  }
+
+  // do this if strategy is not local
+  // finally {
+  //   try {
+  //     // await fs.unlink(file.path);
+  //   } catch (error) {
+  //     console.error('Error deleting file:', error);
+
+  //   }
+  // }
+});
+
+module.exports = router;

api/server/routes/files/index.js

@@ -0,0 +1,22 @@
+const express = require('express');
+const router = express.Router();
+const {
+  uaParser,
+  checkBan,
+  requireJwtAuth,
+  // concurrentLimiter,
+  // messageIpLimiter,
+  // messageUserLimiter,
+} = require('../../middleware');
+
+const files = require('./files');
+const images = require('./images');
+
+router.use(requireJwtAuth);
+router.use(checkBan);
+router.use(uaParser);
+
+router.use('/', files);
+router.use('/images', images);
+
+module.exports = router;

api/server/routes/files/multer.js

@@ -0,0 +1,41 @@
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+const multer = require('multer');
+
+const supportedTypes = ['image/jpeg', 'image/jpg', 'image/png', 'image/webp'];
+const sizeLimit = 20 * 1024 * 1024; // 20 MB
+
+const storage = multer.diskStorage({
+  destination: function (req, file, cb) {
+    const outputPath = path.join(req.app.locals.config.imageOutput, 'temp');
+    if (!fs.existsSync(outputPath)) {
+      fs.mkdirSync(outputPath, { recursive: true });
+    }
+    cb(null, outputPath);
+  },
+  filename: function (req, file, cb) {
+    req.file_id = crypto.randomUUID();
+    const fileExt = path.extname(file.originalname);
+    cb(null, `img-${req.file_id}${fileExt}`);
+  },
+});
+
+const fileFilter = (req, file, cb) => {
+  if (!supportedTypes.includes(file.mimetype)) {
+    return cb(
+      new Error('Unsupported file type. Only JPEG, JPG, PNG, and WEBP files are allowed.'),
+      false,
+    );
+  }
+
+  if (file.size > sizeLimit) {
+    return cb(new Error(`File size exceeds ${sizeLimit / 1024 / 1024} MB.`), false);
+  }
+
+  cb(null, true);
+};
+
+const upload = multer({ storage, fileFilter });
+
+module.exports = upload;

api/server/routes/index.js

@@ -16,6 +16,7 @@ const plugins = require('./plugins');
 const user = require('./user');
 const config = require('./config');
 const assistants = require('./assistants');
+const files = require('./files');
 
 module.exports = {
   search,
@@ -36,4 +37,5 @@ module.exports = {
   plugins,
   config,
   assistants,
+  files,
 };