mirror of
https://github.com/danny-avila/LibreChat.git
synced 2025-09-22 06:00:56 +02:00
🧹 chore(/config/): add tsconfig.json & linting (#2680)
This commit is contained in:
parent
bcdddaed72
commit
c83d9d61d4
10 changed files with 187 additions and 141 deletions
|
@ -132,6 +132,13 @@ module.exports = {
|
|||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
files: './config/translations/**/*.ts',
|
||||
parser: '@typescript-eslint/parser',
|
||||
parserOptions: {
|
||||
project: './config/translations/tsconfig.json',
|
||||
},
|
||||
},
|
||||
{
|
||||
files: ['./packages/data-provider/specs/**/*.ts'],
|
||||
parserOptions: {
|
||||
|
|
|
@ -1,10 +1,6 @@
|
|||
import Anthropic from '@anthropic-ai/sdk';
|
||||
import type * as a from '@anthropic-ai/sdk';
|
||||
import {
|
||||
parseParamFromPrompt,
|
||||
genTranslationPrompt,
|
||||
} from '../../api/app/clients/prompts/titlePrompts';
|
||||
|
||||
import { parseParamFromPrompt, genTranslationPrompt } from '~/app/clients/prompts/titlePrompts';
|
||||
|
||||
/**
|
||||
* Get the initialized Anthropic client.
|
||||
|
|
|
@ -6,21 +6,28 @@ async function main(baseFilePath: string, languagesDir: string) {
|
|||
const { default: baseLanguage } = await import(path.resolve(baseFilePath));
|
||||
const files = fs.readdirSync(languagesDir);
|
||||
|
||||
for (let file of files) {
|
||||
for (const file of files) {
|
||||
const ext = path.extname(file);
|
||||
if (ext !== '.ts' && ext !== '.tsx') continue; // Only process TypeScript files
|
||||
if (ext !== '.ts' && ext !== '.tsx') {
|
||||
continue;
|
||||
}
|
||||
|
||||
const filePath = path.resolve(languagesDir, file);
|
||||
if (filePath === baseFilePath) continue; // Skip the base language file
|
||||
if (filePath === baseFilePath) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const { default: otherLanguage } = await import(filePath);
|
||||
let comparisons = {};
|
||||
const comparisons = {};
|
||||
|
||||
for (let key in otherLanguage) {
|
||||
if (otherLanguage.hasOwnProperty(key) && baseLanguage.hasOwnProperty(key)) {
|
||||
for (const key in otherLanguage) {
|
||||
if (
|
||||
Object.prototype.hasOwnProperty.call(otherLanguage, key) &&
|
||||
Object.prototype.hasOwnProperty.call(baseLanguage, key)
|
||||
) {
|
||||
comparisons[key] = {
|
||||
english: baseLanguage[key],
|
||||
translated: otherLanguage[key]
|
||||
translated: otherLanguage[key],
|
||||
};
|
||||
}
|
||||
}
|
||||
|
@ -28,7 +35,11 @@ async function main(baseFilePath: string, languagesDir: string) {
|
|||
let fileContent = fs.readFileSync(filePath, 'utf8');
|
||||
const comparisonsObjRegex = /export const comparisons = {[\s\S]*?};/gm;
|
||||
const hasComparisons = comparisonsObjRegex.test(fileContent);
|
||||
const comparisonsExport = `\nexport const comparisons = ${JSON.stringify(comparisons, null, 2)};\n`;
|
||||
const comparisonsExport = `\nexport const comparisons = ${JSON.stringify(
|
||||
comparisons,
|
||||
null,
|
||||
2,
|
||||
)};\n`;
|
||||
|
||||
if (hasComparisons) {
|
||||
fileContent = fileContent.replace(comparisonsObjRegex, comparisonsExport);
|
||||
|
@ -36,7 +47,7 @@ async function main(baseFilePath: string, languagesDir: string) {
|
|||
fileContent = fileContent.trim() + comparisonsExport;
|
||||
}
|
||||
|
||||
fs.writeFileSync(filePath, fileContent); // Write updated content back to file
|
||||
fs.writeFileSync(filePath, fileContent);
|
||||
}
|
||||
|
||||
// Execute ESLint with the --fix option on the entire directory
|
||||
|
@ -58,14 +69,12 @@ const baseFilePath = path.resolve(languagesDir, 'Eng.ts');
|
|||
|
||||
main(baseFilePath, languagesDir).catch(console.error);
|
||||
|
||||
|
||||
// const prompt = `
|
||||
|
||||
// Write a prompt that is mindful of the nuances in the language with respect to its English counterpart, which serves as the baseline for translations. Here are the comparisons between the language translations and their English counterparts:
|
||||
|
||||
// ${comparisons}
|
||||
|
||||
|
||||
// Please consider the above comparisons to enhance understanding and guide improvements in translations. Provide insights or suggestions that could help refine the translation process, focusing on cultural and contextual relevance.
|
||||
|
||||
// Please craft a prompt that can be used to better inform future translations to this language. Write this prompt in the translated language, with all its nuances detected, not in the English.
|
||||
|
|
|
@ -2,15 +2,15 @@ import dotenv from 'dotenv';
|
|||
dotenv.config({
|
||||
path: './',
|
||||
});
|
||||
import { OpenAIEmbeddings } from "@langchain/openai";
|
||||
import { HNSWLib } from "@langchain/community/vectorstores/hnswlib";
|
||||
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
|
||||
import * as fs from "fs";
|
||||
import * as path from "path";
|
||||
import { OpenAIEmbeddings } from '@langchain/openai';
|
||||
import { HNSWLib } from '@langchain/community/vectorstores/hnswlib';
|
||||
import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
export const storeEmbeddings = async (modulePath: string) => {
|
||||
try {
|
||||
const text = fs.readFileSync(modulePath, "utf8");
|
||||
const text = fs.readFileSync(modulePath, 'utf8');
|
||||
const textSplitter = new RecursiveCharacterTextSplitter({ chunkSize: 600 });
|
||||
const docs = await textSplitter.createDocuments([text]);
|
||||
const vectorStore = await HNSWLib.fromDocuments(docs, new OpenAIEmbeddings());
|
||||
|
@ -26,10 +26,10 @@ export const storeEmbeddings = async (modulePath: string) => {
|
|||
|
||||
await vectorStore.save(directory);
|
||||
} catch (error) {
|
||||
console.error(`Error storing embeddings`);
|
||||
console.error('Error storing embeddings');
|
||||
console.error(error);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
export const loadEmbeddings = async (modulePath: string) => {
|
||||
try {
|
||||
|
@ -37,7 +37,7 @@ export const loadEmbeddings = async (modulePath: string) => {
|
|||
const loadedVectorStore = await HNSWLib.load(directory, new OpenAIEmbeddings());
|
||||
return loadedVectorStore;
|
||||
} catch (error) {
|
||||
console.error(`Error loading embeddings`);
|
||||
console.error('Error loading embeddings');
|
||||
console.error(error);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
|
|
@ -5,11 +5,13 @@ const baseDirPath = './client/src/localization/languages';
|
|||
const promptsDirPath = './client/src/localization/prompts/instructions';
|
||||
|
||||
async function ensureDirectoryExists(directory: string) {
|
||||
return fs.promises.access(directory).catch(() => fs.promises.mkdir(directory, { recursive: true }));
|
||||
return fs.promises
|
||||
.access(directory)
|
||||
.catch(() => fs.promises.mkdir(directory, { recursive: true }));
|
||||
}
|
||||
|
||||
// Helper function to generate Markdown from an object, recursively if needed
|
||||
function generateMarkdownFromObject(obj: any, depth: number = 0): string {
|
||||
function generateMarkdownFromObject(obj: object, depth = 0): string {
|
||||
if (typeof obj !== 'object' || obj === null) {
|
||||
return String(obj);
|
||||
}
|
||||
|
@ -61,7 +63,8 @@ async function createPromptsForTranslations() {
|
|||
const files = await fs.promises.readdir(baseDirPath);
|
||||
|
||||
for (const file of files) {
|
||||
if (!file.includes('Eng.ts')) { // Ensure English or base file is excluded
|
||||
if (!file.includes('Eng.ts')) {
|
||||
// Ensure English or base file is excluded
|
||||
const filePath = path.join(baseDirPath, file);
|
||||
const promptContent = await generatePromptForFile(filePath, file);
|
||||
const outputFilePath = path.join(promptsDirPath, `${path.basename(file, '.ts')}.md`);
|
||||
|
|
|
@ -8,7 +8,7 @@ async function readKeysFromFile(filePath: string): Promise<string[]> {
|
|||
}
|
||||
|
||||
async function compareKeys(baseKeys: string[], keysFromOtherFile: string[]): Promise<string[]> {
|
||||
const missingKeys = baseKeys.filter(key => !keysFromOtherFile.includes(key));
|
||||
const missingKeys = baseKeys.filter((key) => !keysFromOtherFile.includes(key));
|
||||
return missingKeys;
|
||||
}
|
||||
|
||||
|
@ -18,10 +18,14 @@ async function main(baseFilePath: string, languagesDir: string) {
|
|||
const files = fs.readdirSync(languagesDir);
|
||||
for (const file of files) {
|
||||
const ext = path.extname(file);
|
||||
if (ext !== '.ts' && ext !== '.tsx') continue; // Ensure it's a TypeScript file
|
||||
if (ext !== '.ts' && ext !== '.tsx') {
|
||||
continue;
|
||||
} // Ensure it's a TypeScript file
|
||||
|
||||
const compareFilePath = path.resolve(languagesDir, file);
|
||||
if (compareFilePath === baseFilePath) continue; // Skip the base file
|
||||
if (compareFilePath === baseFilePath) {
|
||||
continue;
|
||||
} // Skip the base file
|
||||
|
||||
try {
|
||||
const keysFromOtherFile = await readKeysFromFile(compareFilePath);
|
||||
|
|
|
@ -16,11 +16,11 @@ export default async function main(baseFilePath: string, compareFilePath: string
|
|||
const compareModule = await import(compareFilePath);
|
||||
const compareKeys = Object.keys(compareModule.default);
|
||||
|
||||
const missingKeys = baseKeys.filter(key => !compareKeys.includes(key));
|
||||
const missingKeys = baseKeys.filter((key) => !compareKeys.includes(key));
|
||||
if (missingKeys.length > 0) {
|
||||
const keyTranslations = {};
|
||||
for (const key of missingKeys) {
|
||||
const baselineTranslation = baseModule.default[key] || "No baseline translation available";
|
||||
const baselineTranslation = baseModule.default[key] || 'No baseline translation available';
|
||||
const result = await processMissingKey({
|
||||
key,
|
||||
baselineTranslation,
|
||||
|
@ -31,7 +31,10 @@ export default async function main(baseFilePath: string, compareFilePath: string
|
|||
}
|
||||
|
||||
const outputDir = path.dirname(compareFilePath);
|
||||
const outputFileName = `${path.basename(compareFilePath, path.extname(compareFilePath))}_missing_keys.json`;
|
||||
const outputFileName = `${path.basename(
|
||||
compareFilePath,
|
||||
path.extname(compareFilePath),
|
||||
)}_missing_keys.json`;
|
||||
const outputFilePath = path.join(outputDir, outputFileName);
|
||||
fs.writeFileSync(outputFilePath, JSON.stringify(keyTranslations, null, 2));
|
||||
}
|
||||
|
|
|
@ -20,7 +20,7 @@ export async function processLanguageModule(moduleName: string, modulePath: stri
|
|||
await storeEmbeddings(modulePath);
|
||||
vectorStoreMap[moduleName] = await loadEmbeddings(modulePath);
|
||||
const baseKeys = Object.keys((await import(modulePath)).default);
|
||||
console.log(`Keys in module: ${moduleName}:`, baseKeys.length)
|
||||
console.log(`Keys in module: ${moduleName}:`, baseKeys.length);
|
||||
missingKeyMap[moduleName] = 0;
|
||||
return prompt;
|
||||
}
|
||||
|
@ -31,7 +31,10 @@ export async function processMissingKey({
|
|||
moduleName,
|
||||
translationPrompt,
|
||||
}: {
|
||||
key: string, baselineTranslation: string, moduleName: string, translationPrompt: string
|
||||
key: string;
|
||||
baselineTranslation: string;
|
||||
moduleName: string;
|
||||
translationPrompt: string;
|
||||
}) {
|
||||
missingKeyMap[moduleName]++;
|
||||
const vectorStore = vectorStoreMap[moduleName];
|
||||
|
@ -42,6 +45,6 @@ export async function processMissingKey({
|
|||
translationPrompt,
|
||||
context,
|
||||
});
|
||||
console.log(`"${key}": "${translation}",\n`)
|
||||
console.log(`"${key}": "${translation}",\n`);
|
||||
return translation;
|
||||
}
|
||||
|
|
|
@ -6,10 +6,14 @@ async function scanDirectory(baseFilePath: string, languagesDir: string) {
|
|||
const files = fs.readdirSync(languagesDir);
|
||||
for (const file of files) {
|
||||
const ext = path.extname(file);
|
||||
if (ext !== '.ts' && ext !== '.tsx') continue;
|
||||
if (ext !== '.ts' && ext !== '.tsx') {
|
||||
continue;
|
||||
}
|
||||
|
||||
const compareFilePath = path.resolve(languagesDir, file);
|
||||
if (compareFilePath === baseFilePath) continue;
|
||||
if (compareFilePath === baseFilePath) {
|
||||
continue;
|
||||
}
|
||||
|
||||
await main(baseFilePath, compareFilePath);
|
||||
}
|
||||
|
|
17
config/translations/tsconfig.json
Normal file
17
config/translations/tsconfig.json
Normal file
|
@ -0,0 +1,17 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"noImplicitAny": false,
|
||||
"baseUrl": ".",
|
||||
"outDir": "./dist",
|
||||
"strict": true,
|
||||
"moduleResolution": "node",
|
||||
"esModuleInterop": true,
|
||||
"target": "es6",
|
||||
"module": "esnext",
|
||||
"lib": ["dom", "es6"],
|
||||
"paths": {
|
||||
"~/*": ["../../api/*"]
|
||||
}
|
||||
},
|
||||
"include": ["*.ts"]
|
||||
}
|
Loading…
Add table
Add a link
Reference in a new issue