🎨 fix: Optimize StableDiffusion API Tool and Fix for Assistants Usage (#2253)

* chore: update docs

* fix(StableDiffusion): optimize API responses and file handling, return expected metadata for Assistants endpoint

api/app/clients/tools/structured/DALLE3.js

@@ -12,14 +12,15 @@ const { logger } = require('~/config');
 class DALLE3 extends Tool {
   constructor(fields = {}) {
     super();
-    /* Used to initialize the Tool without necessary variables. */
+    /** @type {boolean} Used to initialize the Tool without necessary variables. */
     this.override = fields.override ?? false;
-    /* Necessary for output to contain all image metadata. */
+    /** @type {boolean} Necessary for output to contain all image metadata. */
     this.returnMetadata = fields.returnMetadata ?? false;
 
     this.userId = fields.userId;
     this.fileStrategy = fields.fileStrategy;
     if (fields.processFileURL) {
+      /** @type {processFileURL} Necessary for output to contain all image metadata. */
       this.processFileURL = fields.processFileURL.bind(this);
     }
 
@@ -165,13 +166,7 @@ Error Message: ${error.message}`;
       });
 
       if (this.returnMetadata) {
-        this.result = {
-          file_id: result.file_id,
-          filename: result.filename,
-          filepath: result.filepath,
-          height: result.height,
-          width: result.width,
-        };
+        this.result = result;
       } else {
         this.result = this.wrapInMarkdown(result.filepath);
       }

api/app/clients/tools/structured/StableDiffusion.js

@@ -4,14 +4,27 @@ const { z } = require('zod');
 const path = require('path');
 const axios = require('axios');
 const sharp = require('sharp');
+const { v4: uuidv4 } = require('uuid');
 const { StructuredTool } = require('langchain/tools');
+const { FileContext } = require('librechat-data-provider');
+const paths = require('~/config/paths');
 const { logger } = require('~/config');
 
 class StableDiffusionAPI extends StructuredTool {
   constructor(fields) {
     super();
-    /* Used to initialize the Tool without necessary variables. */
+    /** @type {string} User ID */
+    this.userId = fields.userId;
+    /** @type {Express.Request | undefined} Express Request object, only provided by ToolService */
+    this.req = fields.req;
+    /** @type {boolean} Used to initialize the Tool without necessary variables. */
     this.override = fields.override ?? false;
+    /** @type {boolean} Necessary for output to contain all image metadata. */
+    this.returnMetadata = fields.returnMetadata ?? false;
+    if (fields.uploadImageBuffer) {
+      /** @type {uploadImageBuffer} Necessary for output to contain all image metadata. */
+      this.uploadImageBuffer = fields.uploadImageBuffer.bind(this);
+    }
 
     this.name = 'stable-diffusion';
     this.url = fields.SD_WEBUI_URL || this.getServerURL();
@@ -47,7 +60,7 @@ class StableDiffusionAPI extends StructuredTool {
 
   getMarkdownImageUrl(imageName) {
     const imageUrl = path
-      .join(this.relativeImageUrl, imageName)
+      .join(this.relativePath, this.userId, imageName)
       .replace(/\\/g, '/')
       .replace('public/', '');
     return `![generated image](/${imageUrl})`;
@@ -73,46 +86,67 @@ class StableDiffusionAPI extends StructuredTool {
       width: 1024,
       height: 1024,
     };
-    const response = await axios.post(`${url}/sdapi/v1/txt2img`, payload);
-    const image = response.data.images[0];
-    const pngPayload = { image: `data:image/png;base64,${image}` };
-    const response2 = await axios.post(`${url}/sdapi/v1/png-info`, pngPayload);
-    const info = response2.data.info;
+    const generationResponse = await axios.post(`${url}/sdapi/v1/txt2img`, payload);
+    const image = generationResponse.data.images[0];
 
-    // Generate unique name
-    const imageName = `${Date.now()}.png`;
-    this.outputPath = path.resolve(
-      __dirname,
-      '..',
-      '..',
-      '..',
-      '..',
-      '..',
-      'client',
-      'public',
-      'images',
-    );
-    const appRoot = path.resolve(__dirname, '..', '..', '..', '..', '..', 'client');
-    this.relativeImageUrl = path.relative(appRoot, this.outputPath);
+    /** @type {{ height: number, width: number, seed: number, infotexts: string[] }} */
+    let info = {};
+    try {
+      info = JSON.parse(generationResponse.data.info);
+    } catch (error) {
+      logger.error('[StableDiffusion] Error while getting image metadata:', error);
+    }
 
-    // Check if directory exists, if not create it
-    if (!fs.existsSync(this.outputPath)) {
-      fs.mkdirSync(this.outputPath, { recursive: true });
+    const file_id = uuidv4();
+    const imageName = `${file_id}.png`;
+    const { imageOutput: imageOutputPath, clientPath } = paths;
+    const filepath = path.join(imageOutputPath, this.userId, imageName);
+    this.relativePath = path.relative(clientPath, imageOutputPath);
+
+    if (!fs.existsSync(imageOutputPath)) {
+      fs.mkdirSync(imageOutputPath, { recursive: true });
     }
 
     try {
       const buffer = Buffer.from(image.split(',', 1)[0], 'base64');
+      if (this.returnMetadata && this.uploadImageBuffer && this.req) {
+        const file = await this.uploadImageBuffer({
+          req: this.req,
+          context: FileContext.image_generation,
+          resize: false,
+          metadata: {
+            buffer,
+            height: info.height,
+            width: info.width,
+            bytes: Buffer.byteLength(buffer),
+            filename: imageName,
+            type: 'image/png',
+            file_id,
+          },
+        });
+
+        const generationInfo = info.infotexts[0].split('\n').pop();
+        return {
+          ...file,
+          prompt,
+          metadata: {
+            negative_prompt,
+            seed: info.seed,
+            info: generationInfo,
+          },
+        };
+      }
+
       await sharp(buffer)
         .withMetadata({
           iptcpng: {
-            parameters: info,
+            parameters: info.infotexts[0],
           },
         })
-        .toFile(this.outputPath + '/' + imageName);
+        .toFile(filepath);
       this.result = this.getMarkdownImageUrl(imageName);
     } catch (error) {
       logger.error('[StableDiffusion] Error while saving the image:', error);
-      // this.result = theImageUrl;
     }
 
     return this.result;

api/app/clients/tools/util/handleTools.js

@@ -237,9 +237,11 @@ const loadTools = async ({
   }
 
   const imageGenOptions = {
+    req: options.req,
     fileStrategy: options.fileStrategy,
     processFileURL: options.processFileURL,
     returnMetadata: options.returnMetadata,
+    uploadImageBuffer: options.uploadImageBuffer,
   };
 
   const toolOptions = {

api/config/paths.js

@@ -1,7 +1,9 @@
 const path = require('path');
 
 module.exports = {
+  root: path.resolve(__dirname, '..', '..'),
   uploads: path.resolve(__dirname, '..', '..', 'uploads'),
+  clientPath: path.resolve(__dirname, '..', '..', 'client'),
   dist: path.resolve(__dirname, '..', '..', 'client', 'dist'),
   publicPath: path.resolve(__dirname, '..', '..', 'client', 'public'),
   imageOutput: path.resolve(__dirname, '..', '..', 'client', 'public', 'images'),

api/server/services/Files/images/resize.js

@@ -62,14 +62,14 @@ async function resizeImageBuffer(inputBuffer, resolution, endpoint) {
 }
 
 /**
- * Resizes an image buffer to webp format as well as reduces 150 px width.
+ * Resizes an image buffer to webp format as well as reduces by specified or default 150 px width.
  *
  * @param {Buffer} inputBuffer - The buffer of the image to be resized.
  * @returns {Promise<{ buffer: Buffer, width: number, height: number, bytes: number }>} An object containing the resized image buffer, its size and dimensions.
  * @throws Will throw an error if the resolution parameter is invalid.
  */
-async function resizeAndConvert(inputBuffer) {
-  const resizedBuffer = await sharp(inputBuffer).resize({ width: 150 }).toFormat('webp').toBuffer();
+async function resizeAndConvert(inputBuffer, width = 150) {
+  const resizedBuffer = await sharp(inputBuffer).resize({ width }).toFormat('webp').toBuffer();
   const resizedMetadata = await sharp(resizedBuffer).metadata();
   return {
     buffer: resizedBuffer,

api/server/services/Files/process.js

@@ -223,26 +223,32 @@ const processImageFile = async ({ req, res, file, metadata }) => {
  * @param {Object} params - The parameters object.
  * @param {Express.Request} params.req - The Express request object.
  * @param {FileContext} params.context - The context of the file (e.g., 'avatar', 'image_generation', etc.)
+ * @param {boolean} [params.resize=true] - Whether to resize and convert the image to WebP. Default is `true`.
+ * @param {{ buffer: Buffer, width: number, height: number, bytes: number, filename: string, type: string, file_id: string }} [params.metadata] - Required metadata for the file if resize is false.
  * @returns {Promise<{ filepath: string, filename: string, source: string, type: 'image/webp'}>}
  */
-const uploadImageBuffer = async ({ req, context }) => {
+const uploadImageBuffer = async ({ req, context, metadata = {}, resize = true }) => {
   const source = req.app.locals.fileStrategy;
   const { saveBuffer } = getStrategyFunctions(source);
-  const { buffer, width, height, bytes } = await resizeAndConvert(req.file.buffer);
-  const file_id = v4();
-  const fileName = `img-${file_id}.webp`;
+  let { buffer, width, height, bytes, filename, file_id, type } = metadata;
+  if (resize) {
+    file_id = v4();
+    type = 'image/webp';
+    ({ buffer, width, height, bytes } = await resizeAndConvert(req.file.buffer));
+    filename = path.basename(req.file.originalname, path.extname(req.file.originalname)) + '.webp';
+  }
 
-  const filepath = await saveBuffer({ userId: req.user.id, fileName, buffer });
+  const filepath = await saveBuffer({ userId: req.user.id, fileName: filename, buffer });
   return await createFile(
     {
       user: req.user.id,
       file_id,
       bytes,
       filepath,
-      filename: req.file.originalname,
+      filename,
       context,
       source,
-      type: 'image/webp',
+      type,
       width,
       height,
     },

api/server/services/ToolService.js

@@ -12,8 +12,8 @@ const {
   openapiToFunction,
   validateAndParseOpenAPISpec,
 } = require('librechat-data-provider');
+const { processFileURL, uploadImageBuffer } = require('~/server/services/Files/process');
 const { loadActionSets, createActionTool, domainParser } = require('./ActionService');
-const { processFileURL } = require('~/server/services/Files/process');
 const { recordUsage } = require('~/server/services/Threads');
 const { loadTools } = require('~/app/clients/tools/util');
 const { redactMessage } = require('~/config/parsers');
@@ -147,7 +147,7 @@ const processVisionRequest = async (client, currentAction) => {
 
 /**
  * Processes return required actions from run.
- * @param {OpenAIClient} client - OpenAI or StreamRunManager Client.
+ * @param {OpenAIClient | StreamRunManager} client - OpenAI (legacy) or StreamRunManager Client.
  * @param {RequiredAction[]} requiredActions - The required actions to submit outputs for.
  * @returns {Promise<ToolOutputs>} The outputs of the tools.
  */
@@ -164,6 +164,8 @@ async function processRequiredActions(client, requiredActions) {
     functions: true,
     options: {
       processFileURL,
+      req: client.req,
+      uploadImageBuffer,
       openAIApiKey: client.apiKey,
       fileStrategy: client.req.app.locals.fileStrategy,
       returnMetadata: true,

api/typedefs.js

@@ -386,6 +386,18 @@
  * @memberof typedefs
  */
 
+/**
+ * @exports uploadImageBuffer
+ * @typedef {import('~/server/services/Files/process').uploadImageBuffer} uploadImageBuffer
+ * @memberof typedefs
+ */
+
+/**
+ * @exports processFileURL
+ * @typedef {import('~/server/services/Files/process').processFileURL} processFileURL
+ * @memberof typedefs
+ */
+
 /**
  * @exports AssistantCreateParams
  * @typedef {import('librechat-data-provider').AssistantCreateParams} AssistantCreateParams

docs/install/installation/docker_compose_install.md

@@ -51,7 +51,12 @@ Once you have completed all the setup, you can start the LibreChat application b
 That's it! If you need more detailed information on configuring your compose file, see my notes below.
 
 ## Updating LibreChat
-The following commands will fetch the latest code of LibreChat and build a new docker image.
+
+As of v0.7.0+, Docker installations transitioned from building images locally to using prebuilt images [hosted on Github Container registry](https://github.com/danny-avila?tab=packages&repo_name=LibreChat).
+
+You can still build the image locally, as shown in the commented commands below. More info on building the image locally in the [Docker Compose Override Section](../configuration/docker_override.md).
+
+The following commands will fetch the latest LibreChat project changes, including any necessary changes to the docker compose files, as well as the latest prebuilt images.
 
 ```bash
 # Stop the running container(s)
@@ -63,13 +68,16 @@ git pull
 # Pull the latest LibreChat image (default setup)
 docker compose pull
 
+# If building the LibreChat image Locally, build without cache (legacy setup)
+# docker compose build --no-cache
+
 # Start LibreChat
 docker compose up
 ```
 
-If you're having issues running the above commands, you can try a comprehensive approach:
+If you're having issues running the above commands, you can try a comprehensive approach instead:
 
-Prefix commands with `sudo` according to your environment permissions.
+Note: you may need to prefix commands with `sudo` according to your environment permissions.
 
 ```bash
 # Stop the container (if running)