🪄 feat: Agent Artifacts (#5804)

* refactor: remove artifacts toggle

* refactor: allow hiding side panel while allowing artifacts view

* chore: rename SidePanelGroup to SidePanel for clarity

* Revert "refactor: remove artifacts toggle"

This reverts commit f884c2cfcd.

* feat: add artifacts capability to agent configuration

* refactor: conditionally set artifacts mode based on endpoint type

* feat: Artifacts Capability for Agents

* refactor: enhance getStreamText method to handle intermediate replies and add `stream_options` for openai/azure

* feat: localize progress text and improve UX in CodeAnalyze and ExecuteCode components for expanding analysis

api/app/clients/OpenAIClient.js

@@ -1066,9 +1066,14 @@ ${convo}
     });
   }
 
-  getStreamText() {
+  /**
+   *
+   * @param {string[]} [intermediateReply]
+   * @returns {string}
+   */
+  getStreamText(intermediateReply) {
     if (!this.streamHandler) {
-      return '';
+      return intermediateReply?.join('') ?? '';
     }
 
     let thinkMatch;
@@ -1088,7 +1093,10 @@ ${convo}
       }
     }
 
-    const reasoningTokens = reasoningText.length > 0 ? `:::thinking\n${reasoningText}\n:::\n` : '';
+    const reasoningTokens =
+      reasoningText.length > 0
+        ? `:::thinking\n${reasoningText.replace('<think>', '').replace('</think>', '').trim()}\n:::\n`
+        : '';
 
     return `${reasoningTokens}${this.streamHandler.tokens.join('')}`;
   }
@@ -1327,11 +1335,19 @@ ${convo}
         streamPromise = new Promise((resolve) => {
           streamResolve = resolve;
         });
+        /** @type {OpenAI.OpenAI.CompletionCreateParamsStreaming} */
+        const params = {
+          ...modelOptions,
+          stream: true,
+        };
+        if (
+          this.options.endpoint === EModelEndpoint.openAI ||
+          this.options.endpoint === EModelEndpoint.azureOpenAI
+        ) {
+          params.stream_options = { include_usage: true };
+        }
         const stream = await openai.beta.chat.completions
-          .stream({
-            ...modelOptions,
-            stream: true,
-          })
+          .stream(params)
           .on('abort', () => {
             /* Do nothing here */
           })
@@ -1471,7 +1487,7 @@ ${convo}
         err?.message?.includes('abort') ||
         (err instanceof OpenAI.APIError && err?.message?.includes('abort'))
       ) {
-        return intermediateReply.join('');
+        return this.getStreamText(intermediateReply);
       }
       if (
         err?.message?.includes(
@@ -1489,7 +1505,7 @@ ${convo}
         if (this.streamHandler && this.streamHandler.reasoningTokens.length) {
           return this.getStreamText();
         } else if (intermediateReply.length > 0) {
-          return intermediateReply.join('');
+          return this.getStreamText(intermediateReply);
         } else {
           throw err;
         }
@@ -1497,7 +1513,7 @@ ${convo}
         if (this.streamHandler && this.streamHandler.reasoningTokens.length) {
           return this.getStreamText();
         } else if (intermediateReply.length > 0) {
-          return intermediateReply.join('');
+          return this.getStreamText(intermediateReply);
         } else {
           throw err;
         }

api/models/schema/agent.js

@@ -35,6 +35,9 @@ const agentSchema = mongoose.Schema(
     model_parameters: {
       type: Object,
     },
+    artifacts: {
+      type: String,
+    },
     access_level: {
       type: Number,
     },

api/server/services/Endpoints/agents/initialize.js

@@ -13,6 +13,7 @@ const getBedrockOptions = require('~/server/services/Endpoints/bedrock/options')
 const initOpenAI = require('~/server/services/Endpoints/openAI/initialize');
 const initCustom = require('~/server/services/Endpoints/custom/initialize');
 const initGoogle = require('~/server/services/Endpoints/google/initialize');
+const generateArtifactsPrompt = require('~/app/clients/prompts/artifacts');
 const { getCustomEndpointConfig } = require('~/server/services/Config');
 const { loadAgentTools } = require('~/server/services/ToolService');
 const AgentClient = require('~/server/controllers/agents/client');
@@ -72,6 +73,16 @@ const primeResources = async (_attachments, _tool_resources) => {
   }
 };
 
+/**
+ * @param {object} params
+ * @param {ServerRequest} params.req
+ * @param {ServerResponse} params.res
+ * @param {Agent} params.agent
+ * @param {object} [params.endpointOption]
+ * @param {AgentToolResources} [params.tool_resources]
+ * @param {boolean} [params.isInitialAgent]
+ * @returns {Promise<Agent>}
+ */
 const initializeAgentOptions = async ({
   req,
   res,
@@ -132,6 +143,13 @@ const initializeAgentOptions = async ({
     agent.model_parameters.model = agent.model;
   }
 
+  if (typeof agent.artifacts === 'string' && agent.artifacts !== '') {
+    agent.additional_instructions = generateArtifactsPrompt({
+      endpoint: agent.provider,
+      artifacts: agent.artifacts,
+    });
+  }
+
   const tokensModel =
     agent.provider === EModelEndpoint.azureOpenAI ? agent.model : agent.model_parameters.model;
 

api/server/utils/handleText.js

@@ -200,6 +200,7 @@ function generateConfig(key, baseURL, endpoint) {
     config.capabilities = [
       AgentCapabilities.execute_code,
       AgentCapabilities.file_search,
+      AgentCapabilities.artifacts,
       AgentCapabilities.actions,
       AgentCapabilities.tools,
     ];