Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2025-12-17 00:40:14 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

👓 feat: Vision Support for Assistants (#2195)

Browse source 
* refactor(assistants/chat): use promises to speed up initialization, initialize shared variables, include `attachedFileIds` to streamRunManager

* chore: additional typedefs

* fix(OpenAIClient): handle edge case where attachments promise is resolved

* feat: createVisionPrompt

* feat: Vision Support for Assistants
This commit is contained in:
Danny Avila 2024-03-24 23:43:00 -04:00 • committed by GitHub
parent 1f0fb497f8
commit 798e8763d0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 376 additions and 100 deletions
Download patch file Download diff file Expand all files Collapse all files

20
packages/data-provider/src/schemas.ts
View file

@ -1,5 +1,6 @@
import { z } from 'zod';
import type { TMessageContentParts } from './types/assistants';
import { Tools } from './types/assistants';
import type { TMessageContentParts, FunctionTool, FunctionToolCall } from './types/assistants';
import type { TFile } from './types/files';
export const isUUID = z.string().uuid();
@ -25,9 +26,26 @@ export const defaultAssistantFormValues = {
model: '',
functions: [],
code_interpreter: false,
image_vision: false,
retrieval: false,
};
export const ImageVisionTool: FunctionTool = {
type: Tools.function,
[Tools.function]: {
name: 'image_vision',
description: 'Get detailed text descriptions for all current image attachments.',
parameters: {
type: 'object',
properties: {},
required: [],
},
},
};
export const isImageVisionTool = (tool: FunctionTool | FunctionToolCall) =>
tool.type === 'function' && tool.function?.name === ImageVisionTool?.function?.name;
export const endpointSettings = {
[EModelEndpoint.google]: {
model: {