🌏 i18n: Added Missing Localizations (Ar, De, Es, Fr, It, Jp, Ko, Ru, Zh) (#4540)

* chore: remove comparisons

* feat: use prompt caching for translations

* chore: wip translation readme

* i18n: korean translations

* refactor: use promises for faster translation processing

* refactor: update translation model to 'claude-3-5-sonnet-20241022'

* refactor: optimize sleep duration for translation processing

* i18n: add missing keys

* refactor: standardize languages in their own respective languages

* Refactor translation instructions in README.md

config/translations/README.md

@@ -0,0 +1,17 @@
+## Translations
+
+### Requirements:
+- All dependencies installed, run `npm ci` in the root directory.
+- bun: https://bun.sh/
+- `ANTHROPIC_API_KEY` in project root directory `.env` file.
+
+### ⚠️ Warning:
+
+This script can be expensive, several dollars worth, even with prompt caching. It can also be slow if has not been ran in a while, with translations contributed.
+
+### Instructions:
+
+1. Main script: Run `bun config/translations/scan.ts` from the root directory.
+2. Observe translations being generated in all supported languages.
+  - Supported languages are localizations with general translation prompts:
+      - These prompts are directly found in `client/src/localization/prompts`.

config/translations/anthropic.ts

@@ -10,6 +10,9 @@ export function getClient() {
   /** @type {Anthropic.default.RequestOptions} */
   const options = {
     apiKey: process.env.ANTHROPIC_API_KEY,
+    defaultHeaders: {
+      'anthropic-beta': 'prompt-caching-2024-07-31',
+    },
   };
 
   return new Anthropic(options);
@@ -29,12 +32,12 @@ export function getClient() {
  */
 export async function translateKeyPhrase({ key, baselineTranslation, translationPrompt, context }) {
   let translation: string | undefined;
-  const model = 'claude-3-sonnet-20240229';
+  const model = 'claude-3-5-sonnet-20241022';
   const prompt = genTranslationPrompt(translationPrompt);
   const system = prompt;
 
   const translateCompletion = async () => {
-    const content = `Current key: \`${key}\`
+    const text = `Current key: \`${key}\`
 
     Baseline translation: ${baselineTranslation}
     
@@ -46,7 +49,17 @@ export async function translateKeyPhrase({ key, baselineTranslation, translation
     
     <invoke>\n<tool_name>submit_translation</tool_name>\n<parameters>\n<translation>Your Translation Here</translation>\n</parameters>\n</invoke>`;
 
-    const message: a.Anthropic.MessageParam = { role: 'user', content };
+    const message: a.Anthropic.MessageParam = {
+      role: 'user',
+      content: [
+        {
+          type: 'text',
+          text,
+          /* @ts-ignore */
+          cache_control: { type: 'ephemeral' },
+        },
+      ],
+    };
     const requestOptions: a.Anthropic.MessageCreateParamsNonStreaming = {
       model,
       temperature: 0.3,

config/translations/main.ts

@@ -1,6 +1,7 @@
 import fs from 'fs';
 import path from 'path';
 import { processLanguageModule, processMissingKey } from './process';
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
 
 export default async function main(baseFilePath: string, compareFilePath: string) {
   const prompt = await processLanguageModule(path.basename(compareFilePath), compareFilePath);
@@ -17,19 +18,30 @@ export default async function main(baseFilePath: string, compareFilePath: string
   const compareKeys = Object.keys(compareModule.default);
 
   const missingKeys = baseKeys.filter((key) => !compareKeys.includes(key));
+  const promises: Array<Promise<void>> = [];
   if (missingKeys.length > 0) {
     const keyTranslations = {};
     for (const key of missingKeys) {
       const baselineTranslation = baseModule.default[key] || 'No baseline translation available';
-      const result = await processMissingKey({
-        key,
-        baselineTranslation,
-        translationPrompt: prompt,
-        moduleName: path.basename(compareFilePath),
-      });
-      keyTranslations[key] = result;
+      const handleMissingKey = async () => {
+        try {
+          const result = await processMissingKey({
+            key,
+            baselineTranslation,
+            translationPrompt: prompt,
+            moduleName: path.basename(compareFilePath),
+          });
+          keyTranslations[key] = result;
+        } catch (e) {
+          console.error(`Error processing key: ${key}`, e);
+        }
+      };
+
+      promises.push(handleMissingKey());
+      await sleep(700);
     }
 
+    await Promise.all(promises);
     const outputDir = path.dirname(compareFilePath);
     const outputFileName = `${path.basename(
       compareFilePath,