From 9d71b5834585904ad288234ebb65955734fefb96 Mon Sep 17 00:00:00 2001
From: Daniel Avila <messagedaniel@protonmail.com>
Date: Fri, 10 Mar 2023 22:21:15 -0500
Subject: [PATCH 1/2] fix: handles unsupported hjs languages in client response

---
 api/app/detectCode.js  |  44 +++---
 api/utils/languages.js | 318 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 339 insertions(+), 23 deletions(-)
 create mode 100644 api/utils/languages.js

diff --git a/api/app/detectCode.js b/api/app/detectCode.js
index 5b7d213a67..34de71e6f6 100644
--- a/api/app/detectCode.js
+++ b/api/app/detectCode.js
@@ -1,20 +1,33 @@
 const { ModelOperations } = require('@vscode/vscode-languagedetection');
+const languages = require('../utils/languages.js');
 const codeRegex = /(```[\s\S]*?```)/g;
-const languageMatch = /```(\w+)/;
+// const languageMatch = /```(\w+)/;
+const replaceRegex = /```\w+/g;
 
-const detectCode = async (text) => {
+const detectCode = async (input) => {
   try {
+    let text = input;
     if (!text.match(codeRegex)) {
-      // console.log('disqualified for non-code match')
       return text;
     }
 
-    if (text.match(languageMatch)) {
-      // console.log('disqualified for language match')
+    const langMatches = text.match(replaceRegex);
+
+    if (langMatches?.length > 0) {
+      langMatches.forEach(match => {
+        let lang = match.split('```')[1].trim();
+
+        if (languages.has(lang)) {
+          return;
+        }
+
+        console.log('[detectCode.js] replacing', match, 'with', '```shell');
+        text = text.replace(match, '```shell');
+      });
+
       return text;
     }
 
-    // console.log('qualified for code match');
     const modelOperations = new ModelOperations();
     const regexSplit = (await import('./regexSplit.mjs')).default;
     const parts = regexSplit(text, codeRegex);
@@ -22,10 +35,9 @@ const detectCode = async (text) => {
     const output = parts.map(async (part) => {
       if (part.match(codeRegex)) {
         const code = part.slice(3, -3);
-        const language = await modelOperations.runModel(code);
-        return part.replace(/^```/, `\`\`\`${language[0].languageId}`);
+        let lang = (await modelOperations.runModel(code))[0].languageId;
+        return part.replace(/^```/, `\`\`\`${languages.has(lang) ? lang : 'shell'}`);
       } else {
-        // return i > 0 ? '\n' + part : part;
         return part;
       }
     });
@@ -37,18 +49,4 @@ const detectCode = async (text) => {
   }
 };
 
-// const example3 = {
-//   text: "By default, the function generates an 8-character password with uppercase and lowercase letters and digits, but no special characters.\n\nTo use this function, simply call it with the desired arguments. For example:\n\n```\n>>> generate_password()\n'wE5pUxV7'\n>>> generate_password(length=12, special_chars=True)\n'M4v&^gJ*8#qH'\n>>> generate_password(uppercase=False, digits=False)\n'zajyprxr'\n``` \n\nNote that the randomness is used to select characters from the available character sets, but the resulting password is always deterministic given the same inputs. This makes the function useful for generating secure passwords that meet specific requirements."
-// };
-
-// const example4 = {
-//   text: 'here\'s a cool function:\n```\nimport random\nimport string\n\ndef generate_password(length=8, uppercase=True, lowercase=True, digits=True, special_chars=False):\n    """Generate a random password with specified requirements.\n\n    Args:\n        length (int): The length of the password. Default is 8.\n        uppercase (bool): Whether to include uppercase letters. Default is True.\n        lowercase (bool): Whether to include lowercase letters. Default is True.\n        digits (bool): Whether to include digits. Default is True.\n        special_chars (bool): Whether to include special characters. Default is False.\n\n    Returns:\n        str: A random password with the specified requirements.\n    """\n    # Define character sets to use in password generation\n    chars = ""\n    if uppercase:\n        chars += string.ascii_uppercase\n    if lowercase:\n        chars += string.ascii_lowercase\n    if digits:\n        chars += string.digits\n    if special_chars:\n        chars += string.punctuation\n\n    # Generate the password\n    password = "".join(random.choice(chars) for _ in range(length))\n    return password\n```\n\nThis function takes several arguments'
-// };
-
-// write an immediately invoked function to test this
-// (async () => {
-//   const result = await detectCode(example3.text);
-//   console.log(result);
-// })();
-
 module.exports = detectCode;
diff --git a/api/utils/languages.js b/api/utils/languages.js
new file mode 100644
index 0000000000..443a81a753
--- /dev/null
+++ b/api/utils/languages.js
@@ -0,0 +1,318 @@
+const languages = new Set([
+ 'adoc',
+ 'apacheconf',
+ 'arm',
+ 'as',
+ 'asc',
+ 'atom',
+ 'bat',
+ 'bf',
+ 'bind',
+ 'c++',
+ 'capnp',
+ 'cc',
+ 'clj',
+ 'cls',
+ 'cmake.in',
+ 'cmd',
+ 'coffee',
+ 'console',
+ 'cr',
+ 'craftcms',
+ 'crm',
+ 'cs',
+ 'cson',
+ 'cts',
+ 'cxx',
+ 'dfm',
+ 'docker',
+ 'dst',
+ 'erl',
+ 'f90',
+ 'f95',
+ 'fs',
+ 'gawk',
+ 'gemspec',
+ 'gms',
+ 'golang',
+ 'gololang',
+ 'gss',
+ 'gyp',
+ 'h',
+ 'h++',
+ 'hbs',
+ 'hh',
+ 'hpp',
+ 'hs',
+ 'html',
+ 'html.handlebars',
+ 'html.hbs',
+ 'https',
+ 'hx',
+ 'hxx',
+ 'hylang',
+ 'i7',
+ 'iced',
+ 'ino',
+ 'instances',
+ 'irb',
+ 'jinja',
+ 'js',
+ 'jsp',
+ 'jsx',
+ 'julia-repl',
+ 'kdb',
+ 'kt',
+ 'lassoscript',
+ 'ls',
+ 'ls',
+ 'mak',
+ 'make',
+ 'mawk',
+ 'md',
+ 'mipsasm',
+ 'mk',
+ 'mkd',
+ 'mkdown',
+ 'ml',
+ 'ml',
+ 'mm',
+ 'mma',
+ 'moon',
+ 'mts',
+ 'nawk',
+ 'nc',
+ 'nginxconf',
+ 'nimrod',
+ 'objc',
+ 'obj-c',
+ 'obj-c++',
+ 'objective-c++',
+ 'osascript',
+ 'pas',
+ 'pascal',
+ 'patch',
+ 'pcmk',
+ 'pf.conf',
+ 'pl',
+ 'plist',
+ 'pm',
+ 'podspec',
+ 'postgres',
+ 'postgresql',
+ 'pp',
+ 'ps',
+ 'ps1',
+ 'py',
+ 'pycon',
+ 'rb',
+ 're',
+ 'rs',
+ 'rss',
+ 'sas',
+ 'scad',
+ 'sci',
+ 'sh',
+ 'st',
+ 'stanfuncs',
+ 'step',
+ 'stp',
+ 'styl',
+ 'svg',
+ 'tao',
+ 'text',
+ 'thor',
+ 'tk',
+ 'toml',
+ 'ts',
+ 'tsx',
+ 'txt',
+ 'v',
+ 'vb',
+ 'vbs',
+ 'wl',
+ 'x++',
+ 'xhtml',
+ 'xjb',
+ 'xls',
+ 'xlsx',
+ 'xpath',
+ 'xq',
+ 'xsd',
+ 'xsl',
+ 'yaml',
+ 'zep',
+ 'zone',
+ 'zsh',
+ '1c',
+ 'abnf',
+ 'accesslog',
+ 'actionscript',
+ 'ada',
+ 'angelscript',
+ 'apache',
+ 'applescript',
+ 'arcade',
+ 'arduino',
+ 'armasm',
+ 'asciidoc',
+ 'aspectj',
+ 'autohotkey',
+ 'autoit',
+ 'avrasm',
+ 'awk',
+ 'axapta',
+ 'bash',
+ 'basic',
+ 'bnf',
+ 'brainfuck',
+ 'c',
+ 'cal',
+ 'capnproto',
+ 'clojure',
+ 'cmake',
+ 'coffeescript',
+ 'coq',
+ 'cos',
+ 'cpp',
+ 'crmsh',
+ 'crystal',
+ 'csharp',
+ 'csp',
+ 'css',
+ 'd',
+ 'dart',
+ 'diff',
+ 'django',
+ 'dns',
+ 'dockerfile',
+ 'dos',
+ 'dpr',
+ 'dsconfig',
+ 'dts',
+ 'dust',
+ 'ebnf',
+ 'elixir',
+ 'elm',
+ 'erlang',
+ 'excel',
+ 'fix',
+ 'fortran',
+ 'fsharp',
+ 'gams',
+ 'gauss',
+ 'gcode',
+ 'gherkin',
+ 'glsl',
+ 'go',
+ 'golo',
+ 'gradle',
+ 'graph',
+ 'graphql',
+ 'groovy',
+ 'haml',
+ 'handlebars',
+ 'haskell',
+ 'haxe',
+ 'http',
+ 'hy',
+ 'inform7',
+ 'ini',
+ 'irpf90',
+ 'java',
+ 'javascript',
+ 'json',
+ 'julia',
+ 'k',
+ 'kotlin',
+ 'lasso',
+ 'ldif',
+ 'leaf',
+ 'less',
+ 'lisp',
+ 'livecodeserver',
+ 'livescript',
+ 'lua',
+ 'makefile',
+ 'markdown',
+ 'mathematica',
+ 'matlab',
+ 'maxima',
+ 'mel',
+ 'mercury',
+ 'mips',
+ 'mizar',
+ 'mojolicious',
+ 'monkey',
+ 'moonscript',
+ 'n1ql',
+ 'nginx',
+ 'nim',
+ 'nix',
+ 'nsis',
+ 'objectivec',
+ 'ocaml',
+ 'openscad',
+ 'oxygene',
+ 'p21',
+ 'parser3',
+ 'perl',
+ 'pf',
+ 'pgsql',
+ 'php',
+ 'plaintext',
+ 'pony',
+ 'powershell',
+ 'processing',
+ 'profile',
+ 'prolog',
+ 'properties',
+ 'protobuf',
+ 'puppet',
+ 'python',
+ 'python-repl',
+ 'qml',
+ 'r',
+ 'reasonml',
+ 'rib',
+ 'rsl',
+ 'ruby',
+ 'ruleslanguage',
+ 'rust',
+ 'SAS',
+ 'scala' ,
+ 'scheme',
+ 'scilab',
+ 'scss',
+ 'shell',
+ 'smali',
+ 'smalltalk',
+ 'sml',
+ 'sql',
+ 'stan',
+ 'stata',
+ 'stylus',
+ 'subunit',
+ 'swift',
+ 'tap',
+ 'tcl',
+ 'tex',
+ 'thrift',
+ 'tp',
+ 'twig',
+ 'typescript',
+ 'vala',
+ 'vbnet',
+ 'vbscript',
+ 'verilog',
+ 'vhdl',
+ 'vim',
+ 'x86asm',
+ 'xl',
+ 'xml',
+ 'xquery',
+ 'yml',
+ 'zephir',
+]);
+
+module.exports = languages;
\ No newline at end of file

From 09659be0023cdcb539ea26f1e8bbdd3f6dffc6ac Mon Sep 17 00:00:00 2001
From: Daniel Avila <messagedaniel@protonmail.com>
Date: Sat, 11 Mar 2023 10:28:46 -0500
Subject: [PATCH 2/2] fix: unsupported lang handler in client

---
 client/src/components/Messages/Highlight.jsx |   8 +-
 client/src/utils/languages.js                | 318 +++++++++++++++++++
 2 files changed, 323 insertions(+), 3 deletions(-)
 create mode 100644 client/src/utils/languages.js

diff --git a/client/src/components/Messages/Highlight.jsx b/client/src/components/Messages/Highlight.jsx
index 81a9a9e582..e17228df10 100644
--- a/client/src/components/Messages/Highlight.jsx
+++ b/client/src/components/Messages/Highlight.jsx
@@ -1,16 +1,18 @@
 import React, { useState, useEffect } from 'react';
 import hljs from 'highlight.js';
+import languages from '~/utils/languages';
 
 export default function Highlight({language, code}) {
   const [highlightedCode, setHighlightedCode] = useState(code);
+  const lang = languages.has(language) ? language : 'shell';
 
   useEffect(() => {
-    setHighlightedCode(hljs.highlight(code, { language }).value);
-  }, [code, language]);
+    setHighlightedCode(hljs.highlight(code, { language: lang }).value);
+  }, [code, lang]);
 
   return (
     <pre>
-      <code className={`language-${language}`} dangerouslySetInnerHTML={{__html: highlightedCode}}/>
+      <code className={`language-${lang}`} dangerouslySetInnerHTML={{__html: highlightedCode}}/>
     </pre>
   );
 }
\ No newline at end of file
diff --git a/client/src/utils/languages.js b/client/src/utils/languages.js
new file mode 100644
index 0000000000..443a81a753
--- /dev/null
+++ b/client/src/utils/languages.js
@@ -0,0 +1,318 @@
+const languages = new Set([
+ 'adoc',
+ 'apacheconf',
+ 'arm',
+ 'as',
+ 'asc',
+ 'atom',
+ 'bat',
+ 'bf',
+ 'bind',
+ 'c++',
+ 'capnp',
+ 'cc',
+ 'clj',
+ 'cls',
+ 'cmake.in',
+ 'cmd',
+ 'coffee',
+ 'console',
+ 'cr',
+ 'craftcms',
+ 'crm',
+ 'cs',
+ 'cson',
+ 'cts',
+ 'cxx',
+ 'dfm',
+ 'docker',
+ 'dst',
+ 'erl',
+ 'f90',
+ 'f95',
+ 'fs',
+ 'gawk',
+ 'gemspec',
+ 'gms',
+ 'golang',
+ 'gololang',
+ 'gss',
+ 'gyp',
+ 'h',
+ 'h++',
+ 'hbs',
+ 'hh',
+ 'hpp',
+ 'hs',
+ 'html',
+ 'html.handlebars',
+ 'html.hbs',
+ 'https',
+ 'hx',
+ 'hxx',
+ 'hylang',
+ 'i7',
+ 'iced',
+ 'ino',
+ 'instances',
+ 'irb',
+ 'jinja',
+ 'js',
+ 'jsp',
+ 'jsx',
+ 'julia-repl',
+ 'kdb',
+ 'kt',
+ 'lassoscript',
+ 'ls',
+ 'ls',
+ 'mak',
+ 'make',
+ 'mawk',
+ 'md',
+ 'mipsasm',
+ 'mk',
+ 'mkd',
+ 'mkdown',
+ 'ml',
+ 'ml',
+ 'mm',
+ 'mma',
+ 'moon',
+ 'mts',
+ 'nawk',
+ 'nc',
+ 'nginxconf',
+ 'nimrod',
+ 'objc',
+ 'obj-c',
+ 'obj-c++',
+ 'objective-c++',
+ 'osascript',
+ 'pas',
+ 'pascal',
+ 'patch',
+ 'pcmk',
+ 'pf.conf',
+ 'pl',
+ 'plist',
+ 'pm',
+ 'podspec',
+ 'postgres',
+ 'postgresql',
+ 'pp',
+ 'ps',
+ 'ps1',
+ 'py',
+ 'pycon',
+ 'rb',
+ 're',
+ 'rs',
+ 'rss',
+ 'sas',
+ 'scad',
+ 'sci',
+ 'sh',
+ 'st',
+ 'stanfuncs',
+ 'step',
+ 'stp',
+ 'styl',
+ 'svg',
+ 'tao',
+ 'text',
+ 'thor',
+ 'tk',
+ 'toml',
+ 'ts',
+ 'tsx',
+ 'txt',
+ 'v',
+ 'vb',
+ 'vbs',
+ 'wl',
+ 'x++',
+ 'xhtml',
+ 'xjb',
+ 'xls',
+ 'xlsx',
+ 'xpath',
+ 'xq',
+ 'xsd',
+ 'xsl',
+ 'yaml',
+ 'zep',
+ 'zone',
+ 'zsh',
+ '1c',
+ 'abnf',
+ 'accesslog',
+ 'actionscript',
+ 'ada',
+ 'angelscript',
+ 'apache',
+ 'applescript',
+ 'arcade',
+ 'arduino',
+ 'armasm',
+ 'asciidoc',
+ 'aspectj',
+ 'autohotkey',
+ 'autoit',
+ 'avrasm',
+ 'awk',
+ 'axapta',
+ 'bash',
+ 'basic',
+ 'bnf',
+ 'brainfuck',
+ 'c',
+ 'cal',
+ 'capnproto',
+ 'clojure',
+ 'cmake',
+ 'coffeescript',
+ 'coq',
+ 'cos',
+ 'cpp',
+ 'crmsh',
+ 'crystal',
+ 'csharp',
+ 'csp',
+ 'css',
+ 'd',
+ 'dart',
+ 'diff',
+ 'django',
+ 'dns',
+ 'dockerfile',
+ 'dos',
+ 'dpr',
+ 'dsconfig',
+ 'dts',
+ 'dust',
+ 'ebnf',
+ 'elixir',
+ 'elm',
+ 'erlang',
+ 'excel',
+ 'fix',
+ 'fortran',
+ 'fsharp',
+ 'gams',
+ 'gauss',
+ 'gcode',
+ 'gherkin',
+ 'glsl',
+ 'go',
+ 'golo',
+ 'gradle',
+ 'graph',
+ 'graphql',
+ 'groovy',
+ 'haml',
+ 'handlebars',
+ 'haskell',
+ 'haxe',
+ 'http',
+ 'hy',
+ 'inform7',
+ 'ini',
+ 'irpf90',
+ 'java',
+ 'javascript',
+ 'json',
+ 'julia',
+ 'k',
+ 'kotlin',
+ 'lasso',
+ 'ldif',
+ 'leaf',
+ 'less',
+ 'lisp',
+ 'livecodeserver',
+ 'livescript',
+ 'lua',
+ 'makefile',
+ 'markdown',
+ 'mathematica',
+ 'matlab',
+ 'maxima',
+ 'mel',
+ 'mercury',
+ 'mips',
+ 'mizar',
+ 'mojolicious',
+ 'monkey',
+ 'moonscript',
+ 'n1ql',
+ 'nginx',
+ 'nim',
+ 'nix',
+ 'nsis',
+ 'objectivec',
+ 'ocaml',
+ 'openscad',
+ 'oxygene',
+ 'p21',
+ 'parser3',
+ 'perl',
+ 'pf',
+ 'pgsql',
+ 'php',
+ 'plaintext',
+ 'pony',
+ 'powershell',
+ 'processing',
+ 'profile',
+ 'prolog',
+ 'properties',
+ 'protobuf',
+ 'puppet',
+ 'python',
+ 'python-repl',
+ 'qml',
+ 'r',
+ 'reasonml',
+ 'rib',
+ 'rsl',
+ 'ruby',
+ 'ruleslanguage',
+ 'rust',
+ 'SAS',
+ 'scala' ,
+ 'scheme',
+ 'scilab',
+ 'scss',
+ 'shell',
+ 'smali',
+ 'smalltalk',
+ 'sml',
+ 'sql',
+ 'stan',
+ 'stata',
+ 'stylus',
+ 'subunit',
+ 'swift',
+ 'tap',
+ 'tcl',
+ 'tex',
+ 'thrift',
+ 'tp',
+ 'twig',
+ 'typescript',
+ 'vala',
+ 'vbnet',
+ 'vbscript',
+ 'verilog',
+ 'vhdl',
+ 'vim',
+ 'x86asm',
+ 'xl',
+ 'xml',
+ 'xquery',
+ 'yml',
+ 'zephir',
+]);
+
+module.exports = languages;
\ No newline at end of file