feat: DALL-E-3 support 🎨 (#1147)

* feat: DALL-E-3 support

* fix(ci): lock-in openai dependency for types used in data-provider

api/app/clients/tools/index.js

@@ -2,6 +2,7 @@ const GoogleSearchAPI = require('./GoogleSearch');
 const HttpRequestTool = require('./HttpRequestTool');
 const AIPluginTool = require('./AIPluginTool');
 const OpenAICreateImage = require('./DALL-E');
+const DALLE3 = require('./structured/DALLE3');
 const StructuredSD = require('./structured/StableDiffusion');
 const StableDiffusionAPI = require('./StableDiffusion');
 const WolframAlphaAPI = require('./Wolfram');
@@ -23,6 +24,7 @@ module.exports = {
   HttpRequestTool,
   AIPluginTool,
   OpenAICreateImage,
+  DALLE3,
   StableDiffusionAPI,
   StructuredSD,
   WolframAlphaAPI,

api/app/clients/tools/manifest.json

@@ -95,6 +95,19 @@
       }
     ]
   },
+  {
+    "name": "DALL-E-3",
+    "pluginKey": "dalle",
+    "description": "[DALL-E-3] Create realistic images and art from a description in natural language",
+    "icon": "https://i.imgur.com/u2TzXzH.png",
+    "authConfig": [
+      {
+        "authField": "DALLE_API_KEY",
+        "label": "OpenAI API Key",
+        "description": "You can use DALL-E with your API Key from OpenAI."
+      }
+    ]
+  },
   {
     "name": "Calculator",
     "pluginKey": "calculator",

api/app/clients/tools/structured/DALLE3.js

@@ -0,0 +1,138 @@
+// From https://platform.openai.com/docs/guides/images/usage?context=node
+// To use this tool, you must pass in a configured OpenAIApi object.
+const fs = require('fs');
+const { z } = require('zod');
+const OpenAI = require('openai');
+const { Tool } = require('langchain/tools');
+const saveImageFromUrl = require('../saveImageFromUrl');
+const path = require('path');
+
+class DALLE3 extends Tool {
+  constructor(fields = {}) {
+    super();
+
+    let apiKey = fields.DALLE_API_KEY || this.getApiKey();
+    let config = { apiKey };
+    this.openai = new OpenAI(config);
+    this.name = 'dalle';
+    this.description = `Use DALLE to create images from text descriptions:
+.
+    - Ensure prompts are detailed and specify the image type and size.
+    - Describe human features explicitly to promote diversity without bias.`;
+    this.description_for_model = `// Whenever a description of an image is given, generate prompts (following these rules), and use dalle to create the image. If the user does not ask for a specific number of images, default to creating 2 prompts to send to dalle that are written to be as diverse as possible. All prompts sent to dalle must abide by the following policies:
+    // 1. Prompts must be in English. Translate to English if needed.
+    // 2. Only one image can be created per function call.
+    // 3. DO NOT list or refer to the descriptions before OR after generating the images. They should ONLY ever be written out ONCE, in the \`"prompts"\` field of the request. You do not need to ask for permission to generate, just do it!
+    // 4. Always mention the image type (photo, oil painting, watercolor painting, illustration, cartoon, drawing, vector, render, etc.) at the beginning of the caption. Unless the captions suggests otherwise, make one of the images a photo.
+    // 5. Diversify depictions of ALL images with people to always include always DESCENT and GENDER for EACH person using direct terms. Adjust only human descriptions.
+    // - EXPLICITLY specify these attributes, not abstractly reference them. The attributes should be specified in a minimal way and should directly describe their physical form.
+    // - Your choices should be grounded in reality. For example, all of a given OCCUPATION should not be the same gender or race. Additionally, focus on creating diverse, inclusive, and exploratory scenes via the properties you choose during rewrites.  Make choices that may be insightful or unique sometimes.
+    // - Use "various" or "diverse" ONLY IF the description refers to groups of more than 3 people. Do not change the number of people requested in the original description.
+    // - Don't alter memes, fictional character origins, or unseen people. Maintain the original prompt's intent and prioritize quality.
+    // - Do not create any imagery that would be offensive.
+    // - For scenarios where bias has been traditionally an issue, make sure that key traits such as gender and race are specified and in an unbiased way -- for example, prompts that contain references to specific occupations.
+    // The prompt must intricately describe every part of the image in concrete, objective detail. THINK about what the end goal of the description is, and extrapolate that to what would make satisfying images.
+    // All descriptions sent to dalle should be a paragraph of text that is extremely descriptive and detailed. Each should be more than 3 sentences long.`;
+    this.schema = z.object({
+      prompt: z
+        .string()
+        .max(4000)
+        .describe(
+          'A text description of the desired image, following the rules, up to 4000 characters.',
+        ),
+      quality: z
+        .enum(['hd', 'standard'])
+        .describe('The quality of the generated image. Only `hd` and `standard` are supported.'),
+      size: z
+        .enum(['1024x1024', '1792x1024', '1024x1792'])
+        .describe(
+          'The size of the requested image. Use 1024x1024 (square) as the default, 1792x1024 if the user requests a wide image, and 1024x1792 for full-body portraits. Always include this parameter in the request.',
+        ),
+    });
+  }
+
+  getApiKey() {
+    const apiKey = process.env.DALLE_API_KEY || '';
+    if (!apiKey) {
+      throw new Error('Missing DALLE_API_KEY environment variable.');
+    }
+    return apiKey;
+  }
+
+  replaceUnwantedChars(inputString) {
+    return inputString
+      .replace(/\r\n|\r|\n/g, ' ')
+      .replace('"', '')
+      .trim();
+  }
+
+  getMarkdownImageUrl(imageName) {
+    const imageUrl = path
+      .join(this.relativeImageUrl, imageName)
+      .replace(/\\/g, '/')
+      .replace('public/', '');
+    return `![generated image](/${imageUrl})`;
+  }
+
+  async _call(data) {
+    const { prompt, quality = 'standard', size = '1024x1024' } = data;
+    if (!prompt) {
+      throw new Error('Missing required field: prompt');
+    }
+    const resp = await this.openai.images.generate({
+      model: 'dall-e-3',
+      quality,
+      size,
+      prompt: this.replaceUnwantedChars(prompt),
+      n: 1,
+    });
+
+    const theImageUrl = resp.data[0].url;
+
+    if (!theImageUrl) {
+      throw new Error('No image URL returned from OpenAI API.');
+    }
+
+    const regex = /img-[\w\d]+.png/;
+    const match = theImageUrl.match(regex);
+    let imageName = '1.png';
+
+    if (match) {
+      imageName = match[0];
+      console.log(imageName); // Output: img-lgCf7ppcbhqQrz6a5ear6FOb.png
+    } else {
+      console.log('No image name found in the string.');
+    }
+
+    this.outputPath = path.resolve(
+      __dirname,
+      '..',
+      '..',
+      '..',
+      '..',
+      '..',
+      'client',
+      'public',
+      'images',
+    );
+    const appRoot = path.resolve(__dirname, '..', '..', '..', '..', '..', 'client');
+    this.relativeImageUrl = path.relative(appRoot, this.outputPath);
+
+    // Check if directory exists, if not create it
+    if (!fs.existsSync(this.outputPath)) {
+      fs.mkdirSync(this.outputPath, { recursive: true });
+    }
+
+    try {
+      await saveImageFromUrl(theImageUrl, this.outputPath, imageName);
+      this.result = this.getMarkdownImageUrl(imageName);
+    } catch (error) {
+      console.error('Error while saving the image:', error);
+      this.result = theImageUrl;
+    }
+
+    return this.result;
+  }
+}
+
+module.exports = DALLE3;

api/app/clients/tools/util/handleTools.js

@@ -15,6 +15,7 @@ const {
   HttpRequestTool,
   OpenAICreateImage,
   StableDiffusionAPI,
+  DALLE3,
   StructuredSD,
   AzureCognitiveSearch,
   StructuredACS,
@@ -176,6 +177,7 @@ const loadTools = async ({
   const requestedTools = {};
 
   if (functions) {
+    toolConstructors.dalle = DALLE3;
     toolConstructors.codesherpa = CodeSherpa;
   }
 

api/package.json

@@ -52,7 +52,7 @@
     "mongoose": "^7.1.1",
     "nodejs-gpt": "^1.37.4",
     "nodemailer": "^6.9.4",
-    "openai": "^4.11.1",
+    "openai": "^4.16.1",
     "openai-chat-tokens": "^0.2.8",
     "openid-client": "^5.4.2",
     "passport": "^0.6.0",

package-lock.json

@@ -73,7 +73,7 @@
         "mongoose": "^7.1.1",
         "nodejs-gpt": "^1.37.4",
         "nodemailer": "^6.9.4",
-        "openai": "^4.11.1",
+        "openai": "^4.16.1",
         "openai-chat-tokens": "^0.2.8",
         "openid-client": "^5.4.2",
         "passport": "^0.6.0",
@@ -17937,9 +17937,9 @@
       }
     },
     "node_modules/openai": {
-      "version": "4.11.1",
-      "resolved": "https://registry.npmjs.org/openai/-/openai-4.11.1.tgz",
-      "integrity": "sha512-GU0HQWbejXuVAQlDjxIE8pohqnjptFDIm32aPlNT1H9ucMz1VJJD0DaTJRQsagNaJ97awWjjVLEG7zCM6sm4SA==",
+      "version": "4.16.1",
+      "resolved": "https://registry.npmjs.org/openai/-/openai-4.16.1.tgz",
+      "integrity": "sha512-Gr+uqUN1ICSk6VhrX64E+zL7skjI1TgPr/XUN+ZQuNLLOvx15+XZulx/lSW4wFEAQzgjBDlMBbBeikguGIjiMg==",
       "dependencies": {
         "@types/node": "^18.11.18",
         "@types/node-fetch": "^2.6.4",
@@ -17948,7 +17948,8 @@
         "digest-fetch": "^1.3.0",
         "form-data-encoder": "1.7.2",
         "formdata-node": "^4.3.2",
-        "node-fetch": "^2.6.7"
+        "node-fetch": "^2.6.7",
+        "web-streams-polyfill": "^3.2.1"
       },
       "bin": {
         "openai": "bin/cli"
@@ -17967,6 +17968,14 @@
       "resolved": "https://registry.npmjs.org/@types/node/-/node-18.18.3.tgz",
       "integrity": "sha512-0OVfGupTl3NBFr8+iXpfZ8NR7jfFO+P1Q+IO/q0wbo02wYkP5gy36phojeYWpLQ6WAMjl+VfmqUk2YbUfp0irA=="
     },
+    "node_modules/openai/node_modules/web-streams-polyfill": {
+      "version": "3.2.1",
+      "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz",
+      "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==",
+      "engines": {
+        "node": ">= 8"
+      }
+    },
     "node_modules/openapi-types": {
       "version": "12.1.3",
       "resolved": "https://registry.npmjs.org/openapi-types/-/openapi-types-12.1.3.tgz",
@@ -22832,6 +22841,11 @@
         "node": ">=14.0"
       }
     },
+    "node_modules/undici-types": {
+      "version": "5.26.5",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
+      "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA=="
+    },
     "node_modules/unicode-canonical-property-names-ecmascript": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-2.0.0.tgz",
@@ -24151,6 +24165,32 @@
         "url": "https://github.com/sponsors/isaacs"
       }
     },
+    "packages/data-provider/node_modules/openai": {
+      "version": "4.11.1",
+      "resolved": "https://registry.npmjs.org/openai/-/openai-4.11.1.tgz",
+      "integrity": "sha512-GU0HQWbejXuVAQlDjxIE8pohqnjptFDIm32aPlNT1H9ucMz1VJJD0DaTJRQsagNaJ97awWjjVLEG7zCM6sm4SA==",
+      "dependencies": {
+        "@types/node": "^18.11.18",
+        "@types/node-fetch": "^2.6.4",
+        "abort-controller": "^3.0.0",
+        "agentkeepalive": "^4.2.1",
+        "digest-fetch": "^1.3.0",
+        "form-data-encoder": "1.7.2",
+        "formdata-node": "^4.3.2",
+        "node-fetch": "^2.6.7"
+      },
+      "bin": {
+        "openai": "bin/cli"
+      }
+    },
+    "packages/data-provider/node_modules/openai/node_modules/@types/node": {
+      "version": "18.18.8",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-18.18.8.tgz",
+      "integrity": "sha512-OLGBaaK5V3VRBS1bAkMVP2/W9B+H8meUfl866OrMNQqt7wDgdpWPp5o6gmIc9pB+lIQHSq4ZL8ypeH1vPxcPaQ==",
+      "dependencies": {
+        "undici-types": "~5.26.4"
+      }
+    },
     "packages/data-provider/node_modules/rimraf": {
       "version": "5.0.1",
       "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.1.tgz",

packages/data-provider/package.json

@@ -28,7 +28,7 @@
   "dependencies": {
     "@tanstack/react-query": "^4.28.0",
     "axios": "^1.3.4",
-    "openai": "^4.11.1",
+    "openai": "4.11.1",
     "zod": "^3.22.4"
   },
   "devDependencies": {