feat: Vision Support + New UI (#1203)

* feat: add timer duration to showToast, show toast for preset selection

* refactor: replace old /chat/ route with /c/. e2e tests will fail here

* refactor: move typedefs to root of /api/ and add a few to assistant types in TS

* refactor: reorganize data-provider imports, fix dependency cycle, strategize new plan to separate react dependent packages

* feat: add dataService for uploading images

* feat(data-provider): add mutation keys

* feat: file resizing and upload

* WIP: initial API image handling

* fix: catch JSON.parse of localStorage tools

* chore: experimental: use module-alias for absolute imports

* refactor: change temp_file_id strategy

* fix: updating files state by using Map and defining react query callbacks in a way that keeps them during component unmount, initial delete handling

* feat: properly handle file deletion

* refactor: unexpose complete filepath and resize from server for higher fidelity

* fix: make sure resized height, width is saved, catch bad requests

* refactor: use absolute imports

* fix: prevent setOptions from being called more than once for OpenAIClient, made note to fix for PluginsClient

* refactor: import supportsFiles and models vars from schemas

* fix: correctly replace temp file id

* refactor(BaseClient): use absolute imports, pass message 'opts' to buildMessages method, count tokens for nested objects/arrays

* feat: add validateVisionModel to determine if model has vision capabilities

* chore(checkBalance): update jsdoc

* feat: formatVisionMessage: change message content format dependent on role and image_urls passed

* refactor: add usage to File schema, make create and updateFile, correctly set and remove TTL

* feat: working vision support
TODO: file size, type, amount validations, making sure they are styled right, and making sure you can add images from the clipboard/dragging

* feat: clipboard support for uploading images

* feat: handle files on drop to screen, refactor top level view code to Presentation component so the useDragHelpers hook  has ChatContext

* fix(Images): replace uploaded images in place

* feat: add filepath validation to protect sensitive files

* fix: ensure correct file_ids are push and not the Map key values

* fix(ToastContext): type issue

* feat: add basic file validation

* fix(useDragHelpers): correct context issue with `files` dependency

* refactor: consolidate setErrors logic to setError

* feat: add dialog Image overlay on image click

* fix: close endpoints menu on click

* chore: set detail to auto, make note for configuration

* fix: react warning (button desc. of button)

* refactor: optimize filepath handling, pass file_ids to images for easier re-use

* refactor: optimize image file handling, allow re-using files in regen, pass more file metadata in messages

* feat: lazy loading images including use of upload preview

* fix: SetKeyDialog closing, stopPropagation on Dialog content click

* style(EndpointMenuItem): tighten up the style, fix dark theme showing in lightmode, make menu more ux friendly

* style: change maxheight of all settings textareas to 138px from 300px

* style: better styling for textarea and enclosing buttons

* refactor(PresetItems): swap back edit and delete icons

* feat: make textarea placeholder dynamic to endpoint

* style: show user hover buttons only on hover when message is streaming

* fix: ordered list not going past 9, fix css

* feat: add User/AI labels; style: hide loading spinner

* feat: add back custom footer, change original footer text

* feat: dynamic landing icons based on endpoint

* chore: comment out assistants route

* fix: autoScroll to newest on /c/ view

* fix: Export Conversation on new UI

* style: match message style of official more closely

* ci: fix api jest unit tests, comment out e2e tests for now as they will fail until addressed

* feat: more file validation and use blob in preview field, not filepath, to fix temp deletion

* feat: filefilter for multer

* feat: better AI labels based on custom name, model, and endpoint instead of  `ChatGPT`

packages/data-provider/src/api-endpoints.ts

@@ -63,3 +63,7 @@ export const plugins = () => '/api/plugins';
 export const config = () => '/api/config';
 
 export const assistants = (id?: string) => `/api/assistants${id ? `/${id}` : ''}`;
+
+export const files = () => '/api/files';
+
+export const images = () => `${files()}/images`;

packages/data-provider/src/assistants.ts

@@ -5,9 +5,9 @@ import type {
   QueryObserverResult,
   UseInfiniteQueryOptions,
 } from '@tanstack/react-query';
-import * as t from './types';
+import * as t from './types/assistants';
 import * as dataService from './data-service';
-import { QueryKeys } from './query-keys';
+import { QueryKeys } from './keys';
 
 /**
  * Hook for listing all assistants, with optional parameters provided for pagination and sorting

packages/data-provider/src/data-service.ts

@@ -1,7 +1,7 @@
+import * as f from './types/files';
+import * as a from './types/assistants';
 import * as t from './types';
 import * as s from './schemas';
-/* TODO: fix dependency cycle */
-// eslint-disable-next-line import/no-cycle
 import request from './request';
 import * as endpoints from './api-endpoints';
 
@@ -128,8 +128,6 @@ export const register = (payload: t.TRegisterUser) => {
   return request.post(endpoints.register(), payload);
 };
 
-export const refreshToken = (retry?: boolean) => request.post(endpoints.refreshToken(retry));
-
 export const userKeyQuery = (name: string): Promise<t.TCheckUserKeyResponse> =>
   request.get(endpoints.userKeyQuery(name));
 
@@ -159,18 +157,20 @@ export const getStartupConfig = (): Promise<t.TStartupConfig> => {
   return request.get(endpoints.config());
 };
 
-export const createAssistant = (data: t.AssistantCreateParams): Promise<t.Assistant> => {
+/* Assistants */
+
+export const createAssistant = (data: a.AssistantCreateParams): Promise<a.Assistant> => {
   return request.post(endpoints.assistants(), data);
 };
 
-export const getAssistantById = (assistant_id: string): Promise<t.Assistant> => {
+export const getAssistantById = (assistant_id: string): Promise<a.Assistant> => {
   return request.get(endpoints.assistants(assistant_id));
 };
 
 export const updateAssistant = (
   assistant_id: string,
-  data: t.AssistantUpdateParams,
-): Promise<t.Assistant> => {
+  data: a.AssistantUpdateParams,
+): Promise<a.Assistant> => {
   return request.patch(endpoints.assistants(assistant_id), data);
 };
 
@@ -179,7 +179,18 @@ export const deleteAssistant = (assistant_id: string): Promise<void> => {
 };
 
 export const listAssistants = (
-  params?: t.AssistantListParams,
-): Promise<t.AssistantListResponse> => {
+  params?: a.AssistantListParams,
+): Promise<a.AssistantListResponse> => {
   return request.get(endpoints.assistants(), { params });
 };
+
+/* Files */
+
+export const uploadImage = (data: FormData): Promise<f.FileUploadResponse> => {
+  return request.postMultiPart(endpoints.images(), data);
+};
+
+export const deleteFiles = async (files: f.BatchFile[]): Promise<f.DeleteFilesResponse> =>
+  request.deleteWithOptions(endpoints.files(), {
+    data: { files },
+  });

packages/data-provider/src/index.ts

@@ -1,9 +1,19 @@
-export * from './data-service';
-export { default as request } from './request';
+/* types/schemas/schema helpers */
 export * from './types';
-export * from './assistants';
-export * from './query-keys';
+export * from './types/assistants';
+export * from './types/files';
+/*
+ * react query
+ * TODO: move to client, or move schemas/types to their own package
+ */
 export * from './react-query-service';
+export * from './keys';
+export * from './assistants';
+/* api call helpers */
 export * from './headers-helpers';
+export { default as request } from './request';
+import * as dataService from './data-service';
+export { dataService };
+/* general helpers */
 export * from './sse';
 export { default as createPayload } from './createPayload';

packages/data-provider/src/keys.ts

@@ -16,3 +16,8 @@ export enum QueryKeys {
   assistants = 'assistants',
   assistant = 'assistant',
 }
+
+export enum MutationKeys {
+  imageUpload = 'imageUpload',
+  fileDelete = 'fileDelete',
+}

packages/data-provider/src/react-query-service.ts

@@ -9,7 +9,8 @@ import {
 import * as t from './types';
 import * as s from './schemas';
 import * as dataService from './data-service';
-import { QueryKeys } from './query-keys';
+import request from './request';
+import { QueryKeys } from './keys';
 
 export const useAbortRequestWithMessage = (): UseMutationResult<
   void,
@@ -405,7 +406,7 @@ export const useRefreshTokenMutation = (): UseMutationResult<
   unknown
 > => {
   const queryClient = useQueryClient();
-  return useMutation(() => dataService.refreshToken(), {
+  return useMutation(() => request.refreshToken(), {
     onMutate: () => {
       queryClient.invalidateQueries([QueryKeys.models]);
     },

packages/data-provider/src/request.ts

@@ -1,13 +1,57 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 import axios, { AxiosRequestConfig, AxiosError } from 'axios';
-/* TODO: fix dependency cycle */
-// eslint-disable-next-line import/no-cycle
-import { refreshToken } from './data-service';
 import { setTokenHeader } from './headers-helpers';
+import * as endpoints from './api-endpoints';
+
+async function _get<T>(url: string, options?: AxiosRequestConfig): Promise<T> {
+  const response = await axios.get(url, { ...options });
+  return response.data;
+}
+
+async function _post(url: string, data?: any) {
+  const response = await axios.post(url, JSON.stringify(data), {
+    headers: { 'Content-Type': 'application/json' },
+  });
+  return response.data;
+}
+
+async function _postMultiPart(url: string, formData: FormData, options?: AxiosRequestConfig) {
+  const response = await axios.post(url, formData, {
+    ...options,
+    headers: { 'Content-Type': 'multipart/form-data' },
+  });
+  return response.data;
+}
+
+async function _put(url: string, data?: any) {
+  const response = await axios.put(url, JSON.stringify(data), {
+    headers: { 'Content-Type': 'application/json' },
+  });
+  return response.data;
+}
+
+async function _delete<T>(url: string): Promise<T> {
+  const response = await axios.delete(url);
+  return response.data;
+}
+
+async function _deleteWithOptions<T>(url: string, options?: AxiosRequestConfig): Promise<T> {
+  const response = await axios.delete(url, { ...options });
+  return response.data;
+}
+
+async function _patch(url: string, data?: any) {
+  const response = await axios.patch(url, JSON.stringify(data), {
+    headers: { 'Content-Type': 'application/json' },
+  });
+  return response.data;
+}
 
 let isRefreshing = false;
 let failedQueue: { resolve: (value?: any) => void; reject: (reason?: any) => void }[] = [];
 
+const refreshToken = (retry?: boolean) => _post(endpoints.refreshToken(retry));
+
 const processQueue = (error: AxiosError | null, token: string | null = null) => {
   failedQueue.forEach((prom) => {
     if (error) {
@@ -68,50 +112,6 @@ axios.interceptors.response.use(
   },
 );
 
-async function _get<T>(url: string, options?: AxiosRequestConfig): Promise<T> {
-  const response = await axios.get(url, { ...options });
-  return response.data;
-}
-
-async function _post(url: string, data?: any) {
-  const response = await axios.post(url, JSON.stringify(data), {
-    headers: { 'Content-Type': 'application/json' },
-  });
-  return response.data;
-}
-
-async function _postMultiPart(url: string, formData: FormData, options?: AxiosRequestConfig) {
-  const response = await axios.post(url, formData, {
-    ...options,
-    headers: { 'Content-Type': 'multipart/form-data' },
-  });
-  return response.data;
-}
-
-async function _put(url: string, data?: any) {
-  const response = await axios.put(url, JSON.stringify(data), {
-    headers: { 'Content-Type': 'application/json' },
-  });
-  return response.data;
-}
-
-async function _delete<T>(url: string): Promise<T> {
-  const response = await axios.delete(url);
-  return response.data;
-}
-
-async function _deleteWithOptions<T>(url: string, options?: AxiosRequestConfig): Promise<T> {
-  const response = await axios.delete(url, { ...options });
-  return response.data;
-}
-
-async function _patch(url: string, data?: any) {
-  const response = await axios.patch(url, JSON.stringify(data), {
-    headers: { 'Content-Type': 'application/json' },
-  });
-  return response.data;
-}
-
 export default {
   get: _get,
   post: _post,
@@ -120,4 +120,5 @@ export default {
   delete: _delete,
   deleteWithOptions: _deleteWithOptions,
   patch: _patch,
+  refreshToken,
 };

packages/data-provider/src/schemas.ts

@@ -11,6 +11,17 @@ export enum EModelEndpoint {
   assistant = 'assistant',
 }
 
+export const alternateName = {
+  [EModelEndpoint.openAI]: 'OpenAI',
+  [EModelEndpoint.assistant]: 'Assistants',
+  [EModelEndpoint.azureOpenAI]: 'Azure OpenAI',
+  [EModelEndpoint.bingAI]: 'Bing',
+  [EModelEndpoint.chatGPTBrowser]: 'ChatGPT',
+  [EModelEndpoint.gptPlugins]: 'Plugins',
+  [EModelEndpoint.google]: 'PaLM',
+  [EModelEndpoint.anthropic]: 'Anthropic',
+};
+
 export const EndpointURLs: { [key in EModelEndpoint]: string } = {
   [EModelEndpoint.azureOpenAI]: '/api/ask/azureOpenAI',
   [EModelEndpoint.openAI]: '/api/ask/openAI',
@@ -29,6 +40,28 @@ export const modularEndpoints = new Set<EModelEndpoint | string>([
   EModelEndpoint.openAI,
 ]);
 
+export const supportsFiles = {
+  [EModelEndpoint.openAI]: true,
+  [EModelEndpoint.assistant]: true,
+};
+
+export const openAIModels = [
+  'gpt-3.5-turbo-16k-0613',
+  'gpt-3.5-turbo-16k',
+  'gpt-4-1106-preview',
+  'gpt-3.5-turbo',
+  'gpt-3.5-turbo-1106',
+  'gpt-4-vision-preview',
+  'gpt-4',
+  'gpt-3.5-turbo-instruct-0914',
+  'gpt-3.5-turbo-0613',
+  'gpt-3.5-turbo-0301',
+  'gpt-3.5-turbo-instruct',
+  'gpt-4-0613',
+  'text-davinci-003',
+  'gpt-4-0314',
+];
+
 export const eModelEndpointSchema = z.nativeEnum(EModelEndpoint);
 
 export const tPluginAuthConfigSchema = z.object({
@@ -118,6 +151,15 @@ export type TMessage = z.input<typeof tMessageSchema> & {
   children?: TMessage[];
   plugin?: TResPlugin | null;
   plugins?: TResPlugin[];
+  files?: {
+    type: string;
+    file_id: string;
+    filename?: string;
+    preview?: string;
+    filepath?: string;
+    height?: number;
+    width?: number;
+  }[];
 };
 
 export const tConversationSchema = z.object({
@@ -455,7 +497,7 @@ export type TEndpointOption = {
 };
 
 export const getResponseSender = (endpointOption: TEndpointOption): string => {
-  const { endpoint, chatGptLabel, modelLabel, jailbreak } = endpointOption;
+  const { model, endpoint, chatGptLabel, modelLabel, jailbreak } = endpointOption;
 
   if (
     [
@@ -465,7 +507,14 @@ export const getResponseSender = (endpointOption: TEndpointOption): string => {
       EModelEndpoint.chatGPTBrowser,
     ].includes(endpoint)
   ) {
-    return chatGptLabel ?? 'ChatGPT';
+    if (chatGptLabel) {
+      return chatGptLabel;
+    } else if (model && model.includes('gpt-3')) {
+      return 'GPT-3.5';
+    } else if (model && model.includes('gpt-4')) {
+      return 'GPT-4';
+    }
+    return alternateName[endpoint] ?? 'ChatGPT';
   }
 
   if (endpoint === EModelEndpoint.bingAI) {

packages/data-provider/src/sse.js

@@ -4,7 +4,7 @@
  * All rights reserved.
  */
 
-import { refreshToken } from './data-service';
+import request from './request';
 import { setTokenHeader } from './headers-helpers';
 
 var SSE = function (url, options) {
@@ -113,7 +113,7 @@ var SSE = function (url, options) {
     if (this.xhr.status === 401 && !this._retry) {
       this._retry = true;
       try {
-        const refreshResponse = await refreshToken();
+        const refreshResponse = await request.refreshToken();
         this.headers = {
           'Content-Type': 'application/json',
           Authorization: `Bearer ${refreshResponse.token}`,

packages/data-provider/src/types.ts

@@ -2,8 +2,6 @@ import OpenAI from 'openai';
 import type { UseMutationResult } from '@tanstack/react-query';
 import type { TResPlugin, TMessage, TConversation, TEndpointOption } from './schemas';
 
-export * from './types/assistants';
-
 export type TOpenAIMessage = OpenAI.Chat.ChatCompletionMessageParam;
 export type TOpenAIFunction = OpenAI.Chat.ChatCompletionCreateParams.Function;
 export type TOpenAIFunctionCall = OpenAI.Chat.ChatCompletionCreateParams.FunctionCallOption;

packages/data-provider/src/types/assistants.ts

@@ -59,3 +59,14 @@ export type AssistantListResponse = {
   last_id: string;
   has_more: boolean;
 };
+
+export type File = {
+  file_id: string;
+  id?: string;
+  temp_file_id?: string;
+  bytes: number;
+  created_at: number;
+  filename: string;
+  object: string;
+  purpose: 'fine-tune' | 'fine-tune-results' | 'assistants' | 'assistants_output';
+};

packages/data-provider/src/types/files.ts

@@ -0,0 +1,42 @@
+export type FileUploadResponse = {
+  message: string;
+  file_id: string;
+  temp_file_id: string;
+  filepath: string;
+  filename: string;
+  type: string;
+  size: number;
+  height: number;
+  width: number;
+};
+
+export type FileUploadBody = {
+  formData: FormData;
+  file_id: string;
+};
+
+export type UploadMutationOptions = {
+  onSuccess?: (data: FileUploadResponse, variables: FileUploadBody, context?: unknown) => void;
+  onMutate?: (variables: FileUploadBody) => void | Promise<unknown>;
+  onError?: (error: unknown, variables: FileUploadBody, context?: unknown) => void;
+};
+
+export type DeleteFilesResponse = {
+  message: string;
+  result: Record<string, unknown>;
+};
+
+export type BatchFile = {
+  file_id: string;
+  filepath: string;
+};
+
+export type DeleteFilesBody = {
+  files: BatchFile[];
+};
+
+export type DeleteMutationOptions = {
+  onSuccess?: (data: DeleteFilesResponse, variables: DeleteFilesBody, context?: unknown) => void;
+  onMutate?: (variables: DeleteFilesBody) => void | Promise<unknown>;
+  onError?: (error: unknown, variables: DeleteFilesBody, context?: unknown) => void;
+};