Merge branch 'main' into feature/entra-id-azure-integration

api/app/clients/BaseClient.js

@@ -3,6 +3,7 @@ const fetch = require('node-fetch');
 const { logger } = require('@librechat/data-schemas');
 const {
   getBalanceConfig,
+  extractFileContext,
   encodeAndFormatAudios,
   encodeAndFormatVideos,
   encodeAndFormatDocuments,
@@ -10,6 +11,7 @@ const {
 const {
   Constants,
   ErrorTypes,
+  FileSources,
   ContentTypes,
   excludedKeys,
   EModelEndpoint,
@@ -21,6 +23,7 @@ const { getMessages, saveMessage, updateMessage, saveConvo, getConvo } = require
 const { getStrategyFunctions } = require('~/server/services/Files/strategies');
 const { checkBalance } = require('~/models/balanceMethods');
 const { truncateToolCallOutputs } = require('./prompts');
+const countTokens = require('~/server/utils/countTokens');
 const { getFiles } = require('~/models/File');
 const TextStream = require('./TextStream');
 
@@ -1245,27 +1248,62 @@ class BaseClient {
     return audioResult.files;
   }
 
+  /**
+   * Extracts text context from attachments and sets it on the message.
+   * This handles text that was already extracted from files (OCR, transcriptions, document text, etc.)
+   * @param {TMessage} message - The message to add context to
+   * @param {MongoFile[]} attachments - Array of file attachments
+   * @returns {Promise<void>}
+   */
+  async addFileContextToMessage(message, attachments) {
+    const fileContext = await extractFileContext({
+      attachments,
+      req: this.options?.req,
+      tokenCountFn: (text) => countTokens(text),
+    });
+
+    if (fileContext) {
+      message.fileContext = fileContext;
+    }
+  }
+
   async processAttachments(message, attachments) {
     const categorizedAttachments = {
       images: [],
-      documents: [],
       videos: [],
       audios: [],
+      documents: [],
     };
 
+    const allFiles = [];
+
     for (const file of attachments) {
+      /** @type {FileSources} */
+      const source = file.source ?? FileSources.local;
+      if (source === FileSources.text) {
+        allFiles.push(file);
+        continue;
+      }
+      if (file.embedded === true || file.metadata?.fileIdentifier != null) {
+        allFiles.push(file);
+        continue;
+      }
+
       if (file.type.startsWith('image/')) {
         categorizedAttachments.images.push(file);
       } else if (file.type === 'application/pdf') {
         categorizedAttachments.documents.push(file);
+        allFiles.push(file);
       } else if (file.type.startsWith('video/')) {
         categorizedAttachments.videos.push(file);
+        allFiles.push(file);
       } else if (file.type.startsWith('audio/')) {
         categorizedAttachments.audios.push(file);
+        allFiles.push(file);
       }
     }
 
-    const [imageFiles, documentFiles, videoFiles, audioFiles] = await Promise.all([
+    const [imageFiles] = await Promise.all([
       categorizedAttachments.images.length > 0
         ? this.addImageURLs(message, categorizedAttachments.images)
         : Promise.resolve([]),
@@ -1280,7 +1318,8 @@ class BaseClient {
         : Promise.resolve([]),
     ]);
 
-    const allFiles = [...imageFiles, ...documentFiles, ...videoFiles, ...audioFiles];
+    allFiles.push(...imageFiles);
+
     const seenFileIds = new Set();
     const uniqueFiles = [];
 
@@ -1345,6 +1384,7 @@ class BaseClient {
         {},
       );
 
+      await this.addFileContextToMessage(message, files);
       await this.processAttachments(message, files);
 
       this.message_file_map[message.messageId] = files;

api/app/clients/prompts/artifacts.js

@@ -3,6 +3,7 @@ const { EModelEndpoint, ArtifactModes } = require('librechat-data-provider');
 const { generateShadcnPrompt } = require('~/app/clients/prompts/shadcn-docs/generate');
 const { components } = require('~/app/clients/prompts/shadcn-docs/components');
 
+/** @deprecated */
 // eslint-disable-next-line no-unused-vars
 const artifactsPromptV1 = dedent`The assistant can create and reference artifacts during conversations.
   
@@ -115,6 +116,7 @@ Here are some examples of correct usage of artifacts:
     </assistant_response>
   </example>
 </examples>`;
+
 const artifactsPrompt = dedent`The assistant can create and reference artifacts during conversations.
   
 Artifacts are for substantial, self-contained content that users might modify or reuse, displayed in a separate UI window for clarity.
@@ -165,6 +167,10 @@ Artifacts are for substantial, self-contained content that users might modify or
     - SVG: "image/svg+xml"
       - The user interface will render the Scalable Vector Graphics (SVG) image within the artifact tags.
       - The assistant should specify the viewbox of the SVG rather than defining a width/height
+    - Markdown: "text/markdown" or "text/md"
+      - The user interface will render Markdown content placed within the artifact tags.
+      - Supports standard Markdown syntax including headers, lists, links, images, code blocks, tables, and more.
+      - Both "text/markdown" and "text/md" are accepted as valid MIME types for Markdown content.
     - Mermaid Diagrams: "application/vnd.mermaid"
       - The user interface will render Mermaid diagrams placed within the artifact tags.
     - React Components: "application/vnd.react"
@@ -366,6 +372,10 @@ Artifacts are for substantial, self-contained content that users might modify or
     - SVG: "image/svg+xml"
       - The user interface will render the Scalable Vector Graphics (SVG) image within the artifact tags.
       - The assistant should specify the viewbox of the SVG rather than defining a width/height
+    - Markdown: "text/markdown" or "text/md"
+      - The user interface will render Markdown content placed within the artifact tags.
+      - Supports standard Markdown syntax including headers, lists, links, images, code blocks, tables, and more.
+      - Both "text/markdown" and "text/md" are accepted as valid MIME types for Markdown content.
     - Mermaid Diagrams: "application/vnd.mermaid"
       - The user interface will render Mermaid diagrams placed within the artifact tags.
     - React Components: "application/vnd.react"