refactor(plugins): Improve OpenAPI handling, Show Multiple Plugins, & Other Improvements (#845)

* feat(PluginsClient.js): add conversationId to options object in the constructor feat(PluginsClient.js): add support for Code Interpreter plugin feat(PluginsClient.js): add support for Code Interpreter plugin in the availableTools manifest feat(CodeInterpreter.js): add CodeInterpreterTools module feat(CodeInterpreter.js): add RunCommand class feat(CodeInterpreter.js): add ReadFile class feat(CodeInterpreter.js): add WriteFile class feat(handleTools.js): add support for loading Code Interpreter plugin * chore(api): update langchain dependency to version 0.0.123 * fix(CodeInterpreter.js): add support for extracting environment from code fix(WriteFile.js): add support for extracting environment from data fix(extractionChain.js): add utility functions for creating extraction chain from Zod schema fix(handleTools.js): refactor getOpenAIKey function to handle user-provided API key fix(handleTools.js): pass model and openAIApiKey to CodeInterpreter constructor * fix(tools): rename CodeInterpreterTools to E2BTools fix(tools): rename code_interpreter pluginKey to e2b_code_interpreter * chore(PluginsClient.js): comment out unused import and function findMessageContent feat(PluginsClient.js): add support for CodeSherpa plugin feat(PluginsClient.js): add CodeSherpaTools to available tools feat(PluginsClient.js): update manifest.json to include CodeSherpa plugin feat(CodeSherpaTools.js): create RunCode and RunCommand classes for CodeSherpa plugin feat(E2BTools.js): Add E2BTools module for extracting environment from code and running commands, reading and writing files fix(codesherpa.js): Remove codesherpa module as it is no longer needed feat(handleTools.js): add support for CodeSherpaTools in loadTools function feat(loadToolSuite.js): create loadToolSuite utility function to load a suite of tools * feat(PluginsClient.js): add support for CodeSherpa v2 plugin feat(PluginsClient.js): add CodeSherpa v1 plugin to available tools feat(PluginsClient.js): add CodeSherpa v2 plugin to available tools feat(PluginsClient.js): update manifest.json for CodeSherpa v1 plugin feat(PluginsClient.js): update manifest.json for CodeSherpa v2 plugin feat(CodeSherpa.js): implement CodeSherpa plugin for interactive code and shell command execution feat(CodeSherpaTools.js): implement RunCode and RunCommand plugins for CodeSherpa v1 feat(CodeSherpaTools.js): update RunCode and RunCommand plugins for CodeSherpa v2 fix(handleTools.js): add CodeSherpa import statement fix(handleTools.js): change pluginKey from 'codesherpa' to 'codesherpa_tools' fix(handleTools.js): remove model and openAIApiKey from options object in e2b_code_interpreter tool fix(handleTools.js): remove openAIApiKey from options object in codesherpa_tools tool fix(loadToolSuite.js): remove model and openAIApiKey parameters from loadToolSuite function * feat(initializeFunctionsAgent.js): add prefix to agentArgs in initializeFunctionsAgent function The prefix is added to the agentArgs in the initializeFunctionsAgent function. This prefix is used to provide instructions to the agent when it receives any instructions from a webpage, plugin, or other tool. The agent will notify the user immediately and ask them if they wish to carry out or ignore the instructions. * feat(PluginsClient.js): add ChatTool to the list of tools if it meets the conditions feat(tools/index.js): import and export ChatTool feat(ChatTool.js): create ChatTool class with necessary properties and methods * fix(initializeFunctionsAgent.js): update PREFIX message to include sharing all output from the tool fix(E2BTools.js): update descriptions for RunCommand, ReadFile, and WriteFile plugins to provide more clarity and context * chore: rebuild package-lock after rebase * chore: remove deleted file from rebase * wip: refactor plugin message handling to mirror chat.openai.com, handle incoming stream for plugin use * wip: new plugin handling * wip: show multiple plugins handling * feat(plugins): save new plugins array * chore: bump langchain * feat(experimental): support streaming in between plugins * refactor(PluginsClient): factor out helper methods to avoid bloating the class, refactor(gptPlugins): use agent action for mapping the name of action * fix(handleTools): fix tests by adding condition to return original toolFunctions map * refactor(MessageContent): Allow the last index to be last in case it has text (may change with streaming) * feat(Plugins): add handleParsingErrors, useful when LLM does not invoke function params * chore: edit out experimental codesherpa integration * refactor(OpenAPIPlugin): rework tool to be 'function-first', as the spec functions are explicitly passed to agent model * refactor(initializeFunctionsAgent): improve error handling and system message * refactor(CodeSherpa, Wolfram): optimize token usage by delegating bulk of instructions to system message * style(Plugins): match official style with input/outputs * chore: remove unnecessary console logs used for testing * fix(abortMiddleware): render markdown when message is aborted * feat(plugins): add BrowserOp * refactor(OpenAPIPlugin): improve prompt handling * fix(useGenerations): hide edit button when message is submitting/streaming * refactor(loadSpecs): optimize OpenAPI spec loading by only loading requested specs instead of all of them * fix(loadSpecs): will retain original behavior when no tools are passed to the function * fix(MessageContent): ensure cursor only shows up for last message and last display index fix(Message): show legacy plugin and pass isLast to Content * chore: remove console.logs * docs: update docs based on breaking changes and new features refactor(structured/SD): use description_for_model for detailed prompting * docs(azure): make plugins section more clear * refactor(structured/SD): change default payload to SD-WebUI to prefer realism and config for SDXL * refactor(structured/SD): further improve system message prompt * docs: update breaking changes after rebase * refactor(MessageContent): factor out EditMessage, types, Container to separate files, rename Content -> Markdown * fix(CodeInterpreter): linting errors * chore: reduce browser console logs from message streams * chore: re-enable debug logs for plugins/langchain to help with user troubleshooting * chore(manifest.json): add [Experimental] tag to CodeInterpreter plugins, which are not intended as the end-all be-all implementation of this feature for Librechat
2025-12-30 07:08:50 +01:00 · 2023-08-28 12:03:08 -04:00 · 2023-08-28 12:03:08 -04:00 · d3e7627046
commit d3e7627046
parent 66b8580487
51 changed files with 2829 additions and 1577 deletions
--- a/api/app/clients/tools/structured/ChatTool.js
+++ b/api/app/clients/tools/structured/ChatTool.js
@ -0,0 +1,23 @@
+const { StructuredTool } = require('langchain/tools');
+const { z } = require('zod');
+
+// proof of concept
+class ChatTool extends StructuredTool {
+  constructor({ onAgentAction }) {
+    super();
+    this.handleAction = onAgentAction;
+    this.name = 'talk_to_user';
+    this.description =
+      'Use this to chat with the user between your use of other tools/plugins/APIs. You should explain your motive and thought process in a conversational manner, while also analyzing the output of tools/plugins, almost as a self-reflection step to communicate if you\'ve arrived at the correct answer or used the tools/plugins effectively.';
+    this.schema = z.object({
+      message: z.string().describe('Message to the user.'),
+      // next_step: z.string().optional().describe('The next step to take.'),
+    });
+  }
+
+  async _call({ message }) {
+    return `Message to user: ${message}`;
+  }
+}
+
+module.exports = ChatTool;
--- a/api/app/clients/tools/structured/CodeSherpa.js
+++ b/api/app/clients/tools/structured/CodeSherpa.js
@ -0,0 +1,165 @@
+const { StructuredTool } = require('langchain/tools');
+const axios = require('axios');
+const { z } = require('zod');
+
+const headers = {
+  'Content-Type': 'application/json',
+};
+
+function getServerURL() {
+  const url = process.env.CODESHERPA_SERVER_URL || '';
+  if (!url) {
+    throw new Error('Missing CODESHERPA_SERVER_URL environment variable.');
+  }
+  return url;
+}
+
+class RunCode extends StructuredTool {
+  constructor() {
+    super();
+    this.name = 'RunCode';
+    this.description =
+      'Use this plugin to run code with the following parameters\ncode: your code\nlanguage: either Python, Rust, or C++.';
+    this.headers = headers;
+    this.schema = z.object({
+      code: z.string().describe('The code to be executed in the REPL-like environment.'),
+      language: z.string().describe('The programming language of the code to be executed.'),
+    });
+  }
+
+  async _call({ code, language = 'python' }) {
+    // console.log('<--------------- Running Code --------------->', { code, language });
+    const response = await axios({
+      url: `${this.url}/repl`,
+      method: 'post',
+      headers: this.headers,
+      data: { code, language },
+    });
+    // console.log('<--------------- Sucessfully ran Code --------------->', response.data);
+    return response.data.result;
+  }
+}
+
+class RunCommand extends StructuredTool {
+  constructor() {
+    super();
+    this.name = 'RunCommand';
+    this.description =
+      'Runs the provided terminal command and returns the output or error message.';
+    this.headers = headers;
+    this.schema = z.object({
+      command: z.string().describe('The terminal command to be executed.'),
+    });
+  }
+
+  async _call({ command }) {
+    const response = await axios({
+      url: `${this.url}/command`,
+      method: 'post',
+      headers: this.headers,
+      data: {
+        command,
+      },
+    });
+    return response.data.result;
+  }
+}
+
+class CodeSherpa extends StructuredTool {
+  constructor(fields) {
+    super();
+    this.name = 'CodeSherpa';
+    this.url = fields.CODESHERPA_SERVER_URL || getServerURL();
+    //     this.description = `A plugin for interactive code execution, and shell command execution.
+
+    // Run code: provide "code" and "language"
+    // - Execute Python code interactively for general programming, tasks, data analysis, visualizations, and more.
+    // - Pre-installed packages: matplotlib, seaborn, pandas, numpy, scipy, openpyxl. If you need to install additional packages, use the \`pip install\` command.
+    // - When a user asks for visualization, save the plot to \`static/images/\` directory, and embed it in the response using \`http://localhost:3333/static/images/\` URL.
+    // - Always save all media files created to \`static/images/\` directory, and embed them in responses using \`http://localhost:3333/static/images/\` URL.
+
+    // Run command: provide "command" only
+    // - Run terminal commands and interact with the filesystem, run scripts, and more.
+    // - Install python packages using \`pip install\` command.
+    // - Always embed media files created or uploaded using \`http://localhost:3333/static/images/\` URL in responses.
+    // - Access user-uploaded files in \`static/uploads/\` directory using \`http://localhost:3333/static/uploads/\` URL.`;
+    this.description = `This plugin allows interactive code and shell command execution. 
+
+    To run code, supply "code" and "language". Python has pre-installed packages: matplotlib, seaborn, pandas, numpy, scipy, openpyxl. Additional ones can be installed via pip.
+    
+    To run commands, provide "command" only. This allows interaction with the filesystem, script execution, and package installation using pip. Created or uploaded media files are embedded in responses using a specific URL.`;
+    this.schema = z.object({
+      code: z
+        .string()
+        .optional()
+        .describe(
+          `The code to be executed in the REPL-like environment. You must save all media files created to \`${this.url}/static/images/\` and embed them in responses with markdown`,
+        ),
+      language: z
+        .string()
+        .optional()
+        .describe(
+          'The programming language of the code to be executed, you must also include code.',
+        ),
+      command: z
+        .string()
+        .optional()
+        .describe(
+          'The terminal command to be executed. Only provide this if you want to run a command instead of code.',
+        ),
+    });
+
+    this.RunCode = new RunCode({ url: this.url });
+    this.RunCommand = new RunCommand({ url: this.url });
+    this.runCode = this.RunCode._call.bind(this);
+    this.runCommand = this.RunCommand._call.bind(this);
+  }
+
+  async _call({ code, language, command }) {
+    if (code?.length > 0) {
+      return await this.runCode({ code, language });
+    } else if (command) {
+      return await this.runCommand({ command });
+    } else {
+      return 'Invalid parameters provided.';
+    }
+  }
+}
+
+/* TODO: support file upload */
+// class UploadFile extends StructuredTool {
+//   constructor(fields) {
+//     super();
+//     this.name = 'UploadFile';
+//     this.url = fields.CODESHERPA_SERVER_URL || getServerURL();
+//     this.description = 'Endpoint to upload a file.';
+//     this.headers = headers;
+//     this.schema = z.object({
+//       file: z.string().describe('The file to be uploaded.'),
+//     });
+//   }
+
+//   async _call(data) {
+//     const formData = new FormData();
+//     formData.append('file', fs.createReadStream(data.file));
+
+//     const response = await axios({
+//       url: `${this.url}/upload`,
+//       method: 'post',
+//       headers: {
+//         ...this.headers,
+//         'Content-Type': `multipart/form-data; boundary=${formData._boundary}`,
+//       },
+//       data: formData,
+//     });
+//     return response.data;
+//   }
+// }
+
+// module.exports = [
+//   RunCode,
+//   RunCommand,
+//   // UploadFile
+// ];
+
+module.exports = CodeSherpa;
--- a/api/app/clients/tools/structured/CodeSherpaTools.js
+++ b/api/app/clients/tools/structured/CodeSherpaTools.js
@ -0,0 +1,121 @@
+const { StructuredTool } = require('langchain/tools');
+const axios = require('axios');
+const { z } = require('zod');
+
+function getServerURL() {
+  const url = process.env.CODESHERPA_SERVER_URL || '';
+  if (!url) {
+    throw new Error('Missing CODESHERPA_SERVER_URL environment variable.');
+  }
+  return url;
+}
+
+const headers = {
+  'Content-Type': 'application/json',
+};
+
+class RunCode extends StructuredTool {
+  constructor(fields) {
+    super();
+    this.name = 'RunCode';
+    this.url = fields.CODESHERPA_SERVER_URL || getServerURL();
+    this.description_for_model = `// A plugin for interactive code execution
+// Guidelines:
+// Always provide code and language as such: {{"code": "print('Hello World!')", "language": "python"}}
+// Execute Python code interactively for general programming, tasks, data analysis, visualizations, and more.
+// Pre-installed packages: matplotlib, seaborn, pandas, numpy, scipy, openpyxl.If you need to install additional packages, use the \`pip install\` command.
+// When a user asks for visualization, save the plot to \`static/images/\` directory, and embed it in the response using \`${this.url}/static/images/\` URL.
+// Always save alls media files created to \`static/images/\` directory, and embed them in responses using \`${this.url}/static/images/\` URL.
+// Always embed media files created or uploaded using \`${this.url}/static/images/\` URL in responses.
+// Access user-uploaded files in\`static/uploads/\` directory using \`${this.url}/static/uploads/\` URL.
+// Remember to save any plots/images created, so you can embed it in the response, to \`static/images/\` directory, and embed them as instructed before.`;
+    this.description =
+      'This plugin allows interactive code execution. Follow the guidelines to get the best results.';
+    this.headers = headers;
+    this.schema = z.object({
+      code: z.string().optional().describe('The code to be executed in the REPL-like environment.'),
+      language: z
+        .string()
+        .optional()
+        .describe('The programming language of the code to be executed.'),
+    });
+  }
+
+  async _call({ code, language = 'python' }) {
+    // console.log('<--------------- Running Code --------------->', { code, language });
+    const response = await axios({
+      url: `${this.url}/repl`,
+      method: 'post',
+      headers: this.headers,
+      data: { code, language },
+    });
+    // console.log('<--------------- Sucessfully ran Code --------------->', response.data);
+    return response.data.result;
+  }
+}
+
+class RunCommand extends StructuredTool {
+  constructor(fields) {
+    super();
+    this.name = 'RunCommand';
+    this.url = fields.CODESHERPA_SERVER_URL || getServerURL();
+    this.description_for_model = `// Run terminal commands and interact with the filesystem, run scripts, and more.
+// Guidelines:
+// Always provide command as such: {{"command": "ls -l"}}
+// Install python packages using \`pip install\` command.
+// Always embed media files created or uploaded using \`${this.url}/static/images/\` URL in responses.
+// Access user-uploaded files in\`static/uploads/\` directory using \`${this.url}/static/uploads/\` URL.`;
+    this.description =
+      'A plugin for interactive shell command execution. Follow the guidelines to get the best results.';
+    this.headers = headers;
+    this.schema = z.object({
+      command: z.string().describe('The terminal command to be executed.'),
+    });
+  }
+
+  async _call(data) {
+    const response = await axios({
+      url: `${this.url}/command`,
+      method: 'post',
+      headers: this.headers,
+      data,
+    });
+    return response.data.result;
+  }
+}
+
+/* TODO: support file upload */
+// class UploadFile extends StructuredTool {
+//   constructor(fields) {
+//     super();
+//     this.name = 'UploadFile';
+//     this.url = fields.CODESHERPA_SERVER_URL || getServerURL();
+//     this.description = 'Endpoint to upload a file.';
+//     this.headers = headers;
+//     this.schema = z.object({
+//       file: z.string().describe('The file to be uploaded.'),
+//     });
+//   }
+
+//   async _call(data) {
+//     const formData = new FormData();
+//     formData.append('file', fs.createReadStream(data.file));
+
+//     const response = await axios({
+//       url: `${this.url}/upload`,
+//       method: 'post',
+//       headers: {
+//         ...this.headers,
+//         'Content-Type': `multipart/form-data; boundary=${formData._boundary}`,
+//       },
+//       data: formData,
+//     });
+//     return response.data;
+//   }
+// }
+
+module.exports = [
+  RunCode,
+  RunCommand,
+  // UploadFile
+];
--- a/api/app/clients/tools/structured/E2BTools.js
+++ b/api/app/clients/tools/structured/E2BTools.js
@ -0,0 +1,154 @@
+const { StructuredTool } = require('langchain/tools');
+const { PromptTemplate } = require('langchain/prompts');
+const { createExtractionChainFromZod } = require('./extractionChain');
+// const { ChatOpenAI } = require('langchain/chat_models/openai');
+const axios = require('axios');
+const { z } = require('zod');
+
+const envs = ['Nodejs', 'Go', 'Bash', 'Rust', 'Python3', 'PHP', 'Java', 'Perl', 'DotNET'];
+const env = z.enum(envs);
+
+const template = `Extract the correct environment for the following code.
+
+It must be one of these values: ${envs.join(', ')}.
+
+Code:
+{input}
+`;
+
+const prompt = PromptTemplate.fromTemplate(template);
+
+// const schema = {
+//   type: 'object',
+//   properties: {
+//     env: { type: 'string' },
+//   },
+//   required: ['env'],
+// };
+
+const zodSchema = z.object({
+  env: z.string(),
+});
+
+async function extractEnvFromCode(code, model) {
+  // const chatModel = new ChatOpenAI({ openAIApiKey, modelName: 'gpt-4-0613', temperature: 0 });
+  const chain = createExtractionChainFromZod(zodSchema, model, { prompt, verbose: true });
+  const result = await chain.run(code);
+  console.log('<--------------- extractEnvFromCode --------------->');
+  console.log(result);
+  return result.env;
+}
+
+function getServerURL() {
+  const url = process.env.E2B_SERVER_URL || '';
+  if (!url) {
+    throw new Error('Missing E2B_SERVER_URL environment variable.');
+  }
+  return url;
+}
+
+const headers = {
+  'Content-Type': 'application/json',
+  'openai-conversation-id': 'some-uuid',
+};
+
+class RunCommand extends StructuredTool {
+  constructor(fields) {
+    super();
+    this.name = 'RunCommand';
+    this.url = fields.E2B_SERVER_URL || getServerURL();
+    this.description =
+      'This plugin allows interactive code execution by allowing terminal commands to be ran in the requested environment. To be used in tandem with WriteFile and ReadFile for Code interpretation and execution.';
+    this.headers = headers;
+    this.headers['openai-conversation-id'] = fields.conversationId;
+    this.schema = z.object({
+      command: z.string().describe('Terminal command to run, appropriate to the environment'),
+      workDir: z.string().describe('Working directory to run the command in'),
+      env: env.describe('Environment to run the command in'),
+    });
+  }
+
+  async _call(data) {
+    console.log(`<--------------- Running ${data} --------------->`);
+    const response = await axios({
+      url: `${this.url}/commands`,
+      method: 'post',
+      headers: this.headers,
+      data,
+    });
+    return JSON.stringify(response.data);
+  }
+}
+
+class ReadFile extends StructuredTool {
+  constructor(fields) {
+    super();
+    this.name = 'ReadFile';
+    this.url = fields.E2B_SERVER_URL || getServerURL();
+    this.description =
+      'This plugin allows reading a file from requested environment. To be used in tandem with WriteFile and RunCommand for Code interpretation and execution.';
+    this.headers = headers;
+    this.headers['openai-conversation-id'] = fields.conversationId;
+    this.schema = z.object({
+      path: z.string().describe('Path of the file to read'),
+      env: env.describe('Environment to read the file from'),
+    });
+  }
+
+  async _call(data) {
+    console.log(`<--------------- Reading ${data} --------------->`);
+    const response = await axios.get(`${this.url}/files`, { params: data, headers: this.headers });
+    return response.data;
+  }
+}
+
+class WriteFile extends StructuredTool {
+  constructor(fields) {
+    super();
+    this.name = 'WriteFile';
+    this.url = fields.E2B_SERVER_URL || getServerURL();
+    this.model = fields.model;
+    this.description =
+      'This plugin allows interactive code execution by first writing to a file in the requested environment. To be used in tandem with ReadFile and RunCommand for Code interpretation and execution.';
+    this.headers = headers;
+    this.headers['openai-conversation-id'] = fields.conversationId;
+    this.schema = z.object({
+      path: z.string().describe('Path to write the file to'),
+      content: z.string().describe('Content to write in the file. Usually code.'),
+      env: env.describe('Environment to write the file to'),
+    });
+  }
+
+  async _call(data) {
+    let { env, path, content } = data;
+    console.log(`<--------------- environment ${env} typeof ${typeof env}--------------->`);
+    if (env && !envs.includes(env)) {
+      console.log(`<--------------- Invalid environment ${env} --------------->`);
+      env = await extractEnvFromCode(content, this.model);
+    } else if (!env) {
+      console.log('<--------------- Undefined environment --------------->');
+      env = await extractEnvFromCode(content, this.model);
+    }
+
+    const payload = {
+      params: {
+        path,
+        env,
+      },
+      data: {
+        content,
+      },
+    };
+    console.log('Writing to file', JSON.stringify(payload));
+
+    await axios({
+      url: `${this.url}/files`,
+      method: 'put',
+      headers: this.headers,
+      ...payload,
+    });
+    return `Successfully written to ${path} in ${env}`;
+  }
+}
+
+module.exports = [RunCommand, ReadFile, WriteFile];
--- a/api/app/clients/tools/structured/StableDiffusion.js
+++ b/api/app/clients/tools/structured/StableDiffusion.js
@ -11,14 +11,18 @@ class StableDiffusionAPI extends StructuredTool {
    super();
    this.name = 'stable-diffusion';
    this.url = fields.SD_WEBUI_URL || this.getServerURL();
-    this.description = `You can generate images with 'stable-diffusion'. This tool is exclusively for visual content.
-Guidelines:
- Visually describe the moods, details, structures, styles, and/or proportions of the image. Remember, the focus is on visual attributes.
- Craft your input by "showing" and not "telling" the imagery. Think in terms of what you'd want to see in a photograph or a painting.
- Here's an example for generating a realistic portrait photo of a man:
-"prompt":"photo of a man in black clothes, half body, high detailed skin, coastline, overcast weather, wind, waves, 8k uhd, dslr, soft lighting, high quality, film grain, Fujifilm XT3"
-"negative_prompt":"semi-realistic, cgi, 3d, render, sketch, cartoon, drawing, anime, out of frame, low quality, ugly, mutation, deformed"
- Generate images only once per human query unless explicitly requested by the user`;
+    this.description_for_model = `// Generate images and visuals using text.
+// Guidelines:
+// - ALWAYS use {{"prompt": "7+ detailed keywords", "negative_prompt": "7+ detailed keywords"}} structure for queries.
+// - ALWAYS include the markdown url in your final response to show the user: ![caption](/images/id.png)
+// - Visually describe the moods, details, structures, styles, and/or proportions of the image. Remember, the focus is on visual attributes.
+// - Craft your input by "showing" and not "telling" the imagery. Think in terms of what you'd want to see in a photograph or a painting.
+// - Here's an example for generating a realistic portrait photo of a man:
+// "prompt":"photo of a man in black clothes, half body, high detailed skin, coastline, overcast weather, wind, waves, 8k uhd, dslr, soft lighting, high quality, film grain, Fujifilm XT3"
+// "negative_prompt":"semi-realistic, cgi, 3d, render, sketch, cartoon, drawing, anime, out of frame, low quality, ugly, mutation, deformed"
+// - Generate images only once per human query unless explicitly requested by the user`;
+    this.description =
+      'You can generate images using text with \'stable-diffusion\'. This tool is exclusively for visual content.';
    this.schema = z.object({
      prompt: z
        .string()
@ -59,7 +63,11 @@ Guidelines:
    const payload = {
      prompt,
      negative_prompt,
-      steps: 20,
+      sampler_index: 'DPM++ 2M Karras',
+      cfg_scale: 4.5,
+      steps: 22,
+      width: 1024,
+      height: 1024,
    };
    const response = await axios.post(`${url}/sdapi/v1/txt2img`, payload);
    const image = response.data.images[0];
--- a/api/app/clients/tools/structured/Wolfram.js
+++ b/api/app/clients/tools/structured/Wolfram.js
@ -8,19 +8,37 @@ class WolframAlphaAPI extends StructuredTool {
    super();
    this.name = 'wolfram';
    this.apiKey = fields.WOLFRAM_APP_ID || this.getAppId();
-    this.description = `WolframAlpha offers computation, math, curated knowledge, and real-time data. It handles natural language queries and performs complex calculations. 
-Guidelines include:
- Use English for queries and inform users if information isn't from Wolfram.
- Use "6*10^14" for exponent notation and single-line strings for input.
- Use Markdown for formulas and simplify queries to keywords.
- Use single-letter variable names and named physical constants.
- Include a space between compound units and consider equations without units when solving.
- Make separate calls for each property and choose relevant 'Assumptions' if results aren't relevant.
- The tool also performs data analysis, plotting, and information retrieval.`;
+    this.description_for_model = `// Access dynamic computation and curated data from WolframAlpha and Wolfram Cloud.
+    // General guidelines:
+    // - Use only getWolframAlphaResults or getWolframCloudResults endpoints.
+    // - Prefer getWolframAlphaResults unless Wolfram Language code should be evaluated.
+    // - Use getWolframAlphaResults for natural-language queries in English; translate non-English queries before sending, then respond in the original language.
+    // - Use getWolframCloudResults for problems solvable with Wolfram Language code.
+    // - Suggest only Wolfram Language for external computation.
+    // - Inform users if information is not from Wolfram endpoints.
+    // - Display image URLs with Image Markdown syntax: ![caption](https://imageURL/.../MSPStoreType=image/png&s=18). You must prefix the caption brackets with "!".
+    // - ALWAYS use this exponent notation: \`6*10^14\`, NEVER \`6e14\`.
+    // - ALWAYS use {{"input": query}} structure for queries to Wolfram endpoints; \`query\` must ONLY be a single-line string.
+    // - ALWAYS use proper Markdown formatting for all math, scientific, and chemical formulas, symbols, etc.:  '$$\n[expression]\n$$' for standalone cases and '\( [expression] \)' when inline.
+    // - Format inline Wolfram Language code with Markdown code formatting.
+    // - Never mention your knowledge cutoff date; Wolfram may return more recent data. getWolframAlphaResults guidelines:
+    // - Understands natural language queries about entities in chemistry, physics, geography, history, art, astronomy, and more.
+    // - Performs mathematical calculations, date and unit conversions, formula solving, etc.
+    // - Convert inputs to simplified keyword queries whenever possible (e.g. convert "how many people live in France" to "France population").
+    // - Use ONLY single-letter variable names, with or without integer subscript (e.g., n, n1, n_1).
+    // - Use named physical constants (e.g., 'speed of light') without numerical substitution.
+    // - Include a space between compound units (e.g., "Ω m" for "ohm*meter").
+    // - To solve for a variable in an equation with units, consider solving a corresponding equation without units; exclude counting units (e.g., books), include genuine units (e.g., kg).
+    // - If data for multiple properties is needed, make separate calls for each property.
+    // - If a Wolfram Alpha result is not relevant to the query:
+    // -- If Wolfram provides multiple 'Assumptions' for a query, choose the more relevant one(s) without explaining the initial result. If you are unsure, ask the user to choose.
+    // -- Re-send the exact same 'input' with NO modifications, and add the 'assumption' parameter, formatted as a list, with the relevant values.
+    // -- ONLY simplify or rephrase the initial query if a more relevant 'Assumption' or other input suggestions are not provided.
+    // -- Do not explain each step unless user input is needed. Proceed directly to making a better API call based on the available assumptions.`;
+    this.description = `WolframAlpha offers computation, math, curated knowledge, and real-time data. It handles natural language queries and performs complex calculations.
+    Follow the guidelines to get the best results.`;
    this.schema = z.object({
-      nl_query: z
-        .string()
-        .describe('Natural language query to WolframAlpha following the guidelines'),
+      input: z.string().describe('Natural language query to WolframAlpha following the guidelines'),
    });
  }

@ -54,8 +72,8 @@ Guidelines include:

  async _call(data) {
    try {
-      const { nl_query } = data;
-      const url = this.createWolframAlphaURL(nl_query);
+      const { input } = data;
+      const url = this.createWolframAlphaURL(input);
      const response = await this.fetchRawText(url);
      return response;
    } catch (error) {
--- a/api/app/clients/tools/structured/extractionChain.js
+++ b/api/app/clients/tools/structured/extractionChain.js
@ -0,0 +1,52 @@
+const { zodToJsonSchema } = require('zod-to-json-schema');
+const { PromptTemplate } = require('langchain/prompts');
+const { JsonKeyOutputFunctionsParser } = require('langchain/output_parsers');
+const { LLMChain } = require('langchain/chains');
+function getExtractionFunctions(schema) {
+  return [
+    {
+      name: 'information_extraction',
+      description: 'Extracts the relevant information from the passage.',
+      parameters: {
+        type: 'object',
+        properties: {
+          info: {
+            type: 'array',
+            items: {
+              type: schema.type,
+              properties: schema.properties,
+              required: schema.required,
+            },
+          },
+        },
+        required: ['info'],
+      },
+    },
+  ];
+}
+const _EXTRACTION_TEMPLATE = `Extract and save the relevant entities mentioned in the following passage together with their properties.
+
+Passage:
+{input}
+`;
+function createExtractionChain(schema, llm, options = {}) {
+  const { prompt = PromptTemplate.fromTemplate(_EXTRACTION_TEMPLATE), ...rest } = options;
+  const functions = getExtractionFunctions(schema);
+  const outputParser = new JsonKeyOutputFunctionsParser({ attrName: 'info' });
+  return new LLMChain({
+    llm,
+    prompt,
+    llmKwargs: { functions },
+    outputParser,
+    tags: ['openai_functions', 'extraction'],
+    ...rest,
+  });
+}
+function createExtractionChainFromZod(schema, llm) {
+  return createExtractionChain(zodToJsonSchema(schema), llm);
+}
+
+module.exports = {
+  createExtractionChain,
+  createExtractionChainFromZod,
+};