From 4d89adfc57793827719cf3ce3b6c7649605bf851 Mon Sep 17 00:00:00 2001
From: Danny Avila <110412045+danny-avila@users.noreply.github.com>
Date: Tue, 12 Sep 2023 11:41:15 -0400
Subject: [PATCH] =?UTF-8?q?fix(Anthropic):=20Correct=20Payload=20&=20Incre?=
 =?UTF-8?q?ase=20Default=20Token=20Size=20=F0=9F=94=A7=20(#933)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* fix: don't pass unnecessary fields to anthropic payload

* fix: increase maxOutputTokens range

* chore: remove debugging mode
---
 api/app/clients/AnthropicClient.js                     | 9 +++++----
 api/server/routes/ask/anthropic.js                     | 2 +-
 api/server/routes/endpoints/schemas.js                 | 4 ++--
 client/src/components/Endpoints/Settings/Anthropic.tsx | 8 ++++----
 packages/data-provider/src/schemas.ts                  | 4 ++--
 5 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/api/app/clients/AnthropicClient.js b/api/app/clients/AnthropicClient.js
index ebec514040..304208e49d 100644
--- a/api/app/clients/AnthropicClient.js
+++ b/api/app/clients/AnthropicClient.js
@@ -268,13 +268,14 @@ class AnthropicClient extends BaseClient {
     };
 
     let text = '';
+    const { model, stream, maxOutputTokens, ...rest } = this.modelOptions;
     const requestOptions = {
       prompt: payload,
-      model: this.modelOptions.model,
-      stream: this.modelOptions.stream || true,
-      max_tokens_to_sample: this.modelOptions.maxOutputTokens || 1500,
+      model,
+      stream: stream || true,
+      max_tokens_to_sample: maxOutputTokens || 1500,
       metadata,
-      ...modelOptions,
+      ...rest,
     };
     if (this.options.debug) {
       console.log('AnthropicClient: requestOptions');
diff --git a/api/server/routes/ask/anthropic.js b/api/server/routes/ask/anthropic.js
index 3517e928b8..637fc090aa 100644
--- a/api/server/routes/ask/anthropic.js
+++ b/api/server/routes/ask/anthropic.js
@@ -91,7 +91,7 @@ router.post(
 
       let response = await client.sendMessage(text, {
         getIds,
-        debug: false,
+        // debug: true,
         user: req.user.id,
         conversationId,
         parentMessageId,
diff --git a/api/server/routes/endpoints/schemas.js b/api/server/routes/endpoints/schemas.js
index ba36abe64c..7c948f2959 100644
--- a/api/server/routes/endpoints/schemas.js
+++ b/api/server/routes/endpoints/schemas.js
@@ -191,7 +191,7 @@ const anthropicSchema = tConversationSchema
     modelLabel: obj.modelLabel ?? null,
     promptPrefix: obj.promptPrefix ?? null,
     temperature: obj.temperature ?? 1,
-    maxOutputTokens: obj.maxOutputTokens ?? 1024,
+    maxOutputTokens: obj.maxOutputTokens ?? 4000,
     topP: obj.topP ?? 0.7,
     topK: obj.topK ?? 5,
   }))
@@ -200,7 +200,7 @@ const anthropicSchema = tConversationSchema
     modelLabel: null,
     promptPrefix: null,
     temperature: 1,
-    maxOutputTokens: 1024,
+    maxOutputTokens: 4000,
     topP: 0.7,
     topK: 5,
   }));
diff --git a/client/src/components/Endpoints/Settings/Anthropic.tsx b/client/src/components/Endpoints/Settings/Anthropic.tsx
index 156c610368..05255ea22b 100644
--- a/client/src/components/Endpoints/Settings/Anthropic.tsx
+++ b/client/src/components/Endpoints/Settings/Anthropic.tsx
@@ -203,14 +203,14 @@ export default function Settings({ conversation, setOption, models, readonly }:
             <div className="flex justify-between">
               {localize('com_endpoint_max_output_tokens')}{' '}
               <small className="opacity-40">
-                ({localize('com_endpoint_default_with_num', '1024')})
+                ({localize('com_endpoint_default_with_num', '4000')})
               </small>
               <InputNumber
                 id="max-tokens-int"
                 disabled={readonly}
                 value={maxOutputTokens}
                 onChange={(value) => setMaxOutputTokens(Number(value))}
-                max={1024}
+                max={4000}
                 min={1}
                 step={1}
                 controls={false}
@@ -225,10 +225,10 @@ export default function Settings({ conversation, setOption, models, readonly }:
             </div>
             <Slider
               disabled={readonly}
-              value={[maxOutputTokens ?? 1024]}
+              value={[maxOutputTokens ?? 4000]}
               onValueChange={(value) => setMaxOutputTokens(value[0])}
               doubleClickHandler={() => setMaxOutputTokens(0)}
-              max={1024}
+              max={4000}
               min={1}
               step={1}
               className="flex h-4 w-full"
diff --git a/packages/data-provider/src/schemas.ts b/packages/data-provider/src/schemas.ts
index 6f21e58860..e703389f25 100644
--- a/packages/data-provider/src/schemas.ts
+++ b/packages/data-provider/src/schemas.ts
@@ -267,7 +267,7 @@ export const anthropicSchema = tConversationSchema
     modelLabel: obj.modelLabel ?? null,
     promptPrefix: obj.promptPrefix ?? null,
     temperature: obj.temperature ?? 1,
-    maxOutputTokens: obj.maxOutputTokens ?? 1024,
+    maxOutputTokens: obj.maxOutputTokens ?? 4000,
     topP: obj.topP ?? 0.7,
     topK: obj.topK ?? 5,
   }))
@@ -276,7 +276,7 @@ export const anthropicSchema = tConversationSchema
     modelLabel: null,
     promptPrefix: null,
     temperature: 1,
-    maxOutputTokens: 1024,
+    maxOutputTokens: 4000,
     topP: 0.7,
     topK: 5,
   }));