Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2025-12-18 09:20:15 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

👓 feat: Vision Support for Assistants (#2195)

Browse source 
* refactor(assistants/chat): use promises to speed up initialization, initialize shared variables, include `attachedFileIds` to streamRunManager

* chore: additional typedefs

* fix(OpenAIClient): handle edge case where attachments promise is resolved

* feat: createVisionPrompt

* feat: Vision Support for Assistants
This commit is contained in:
Danny Avila 2024-03-24 23:43:00 -04:00 • committed by GitHub
parent 1f0fb497f8
commit 798e8763d0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 376 additions and 100 deletions
Download patch file Download diff file Expand all files Collapse all files

43
client/src/components/SidePanel/Builder/AssistantPanel.tsx
View file

@ -8,6 +8,7 @@ import {
Capabilities,
EModelEndpoint,
actionDelimiter,
ImageVisionTool,
defaultAssistantFormValues,
} from 'librechat-data-provider';
import type { AssistantForm, AssistantPanelProps } from '~/common';
@ -82,6 +83,10 @@ export default function AssistantPanel({
() => assistants?.capabilities?.includes(Capabilities.code_interpreter),
[assistants],
);
const imageVisionEnabled = useMemo(
() => assistants?.capabilities?.includes(Capabilities.image_vision),
[assistants],
);
useEffect(() => {
if (model && !retrievalModels.has(model)) {
@ -157,6 +162,9 @@ export default function AssistantPanel({
if (data.retrieval) {
tools.push({ type: Tools.retrieval });
}
if (data.image_vision) {
tools.push(ImageVisionTool);
}
const {
name,
@ -374,6 +382,37 @@ export default function AssistantPanel({
</label>
</div>
)}
{imageVisionEnabled && (
<div className="flex items-center">
<Controller
name={Capabilities.image_vision}
control={control}
render={({ field }) => (
<Checkbox
{...field}
checked={field.value}
onCheckedChange={field.onChange}
className="relative float-left mr-2 inline-flex h-4 w-4 cursor-pointer"
value={field?.value?.toString()}
/>
)}
/>
<label
className="form-check-label text-token-text-primary w-full cursor-pointer"
htmlFor={Capabilities.image_vision}
onClick={() =>
setValue(Capabilities.image_vision, !getValues(Capabilities.image_vision), {
shouldDirty: true,
})
}
>
<div className="flex items-center">
{localize('com_assistants_image_vision')}
<QuestionMark />
</div>
</label>
</div>
)}
{retrievalEnabled && (
<div className="flex items-center">
<Controller
@ -417,9 +456,9 @@ export default function AssistantPanel({
${actionsEnabled ? localize('com_assistants_actions') : ''}`}
</label>
<div className="space-y-1">
{functions.map((func) => (
{functions.map((func, i) => (
<AssistantTool
key={func}
key={`${func}-${i}-${assistant_id}`}
tool={func}
allTools={allTools}
assistant_id={assistant_id}