ci: add e2e workflow, optimize client code for testing (#771)

* refactor(e2e): fix tests with latest changes, convert to TS, use test Ids

* chore(EndpointMenu.jsx): add data-testid attribute to new-conversation-menu button

* refactor(EndpointItem): add data-testid attr., convert to TS

* refactor(e2e): remove unnecessary awaits and convert to TS

* chore(playwright.config.local.ts): add absolute path to server index.js file
chore(playwright.config.local.ts): add dotenv configuration
chore(playwright.config.local.ts): change webServer command to use absolute path
chore(playwright.config.local.ts): add NODE_ENV and process.env to webServer env
chore(playwright.config.local.ts): remove unused import
chore(login.spec.js): delete login.spec.js file

* chore(.gitignore): add 'my.secrets' to the list of ignored files
fix(Registration.tsx): add 'data-testid' attribute to the error message div
fix(Registration.spec.tsx): comment out test case that calls 'registerUser.mutate'

* chore(ConvoIcon.tsx): add data-testid attribute to svg element
chore(messages.spec.ts): refactor conversation navigation logic

* chore(playwright.config.ts): add support for absolute path to server index.js file
feat(playwright.config.ts): add support for dotenv configuration
feat(playwright.config.ts): set NODE_ENV to 'production' in webServer environment variables

* chore(workflows): comment out push event and specify paths for pull_request event in backend-review.yml
chore(workflows): comment out push event and specify paths for pull_request event in frontend-review.yml

* chore(install.js): add check to skip install script in CI environment

* chore: complete playwright workflow

* chore(Landing.tsx): add data-testid attribute to landing title element
chore(authenticate.ts): update selector to wait for landing title element by test id instead of text content

* chore(playwright.yml): add step to upload screenshot artifact on failure
fix(authenticate.ts): capture screenshot before waiting for landing title and increase timeout due to GH Actions load time

* chore(playwright.yml): rename artifact name from 'screenshot' to 'login-screenshot'
feat(LoginForm.tsx): add data-testid attribute to login button
fix(authenticate.ts): change screenshot name to 'login-screenshot.png' and conditionally take screenshot only in CI environment

* chore(playwright.yml): add CI environment variable and set it to true

* chore(playwright.yml): update Playwright installation command
chore(playwright.config.ts): update storageState path to use process.cwd()

* fix(playwright.yml): update node version to 18 in setup-node action
fix(playwright.yml): update actions/cache to v3 in Cache Node.js modules step
fix(playwright.yml): update actions/cache to v3 in Cache Playwright installations step
fix(authenticate.ts): change login button click to press 'Enter' on password input

* chore(playwright.yml): update E2E_USER_EMAIL and E2E_USER_PASSWORD values for testing purposes
chore(authenticate.ts): add console.dir to log user object for debugging

* chore(playwright.yml): add step to upload storageState artifact

The storageState artifact is now uploaded as part of the workflow. This artifact contains the state of the storage used during the end-to-end tests. It will be retained for 2 days.

* chore(playwright.yml): comment out upload screenshot step
chore(playwright.config.ts): change NODE_ENV to development
chore(authenticate.ts): comment out screenshot related code

* chore(playwright.config.ts): add SESSION_EXPIRY environment variable with value 86400000

* chore(playwright.yml): update environment variables in Playwright workflow
fix(General.tsx): add data-testid attributes to clear conversations buttons
test(messages.spec.ts): add setup and teardown steps for clearing conversations before and after tests

* fix(messages.spec.ts): fix clearing conversations before and after message tests
feat(messages.spec.ts): add beforeEach and afterEach hooks to create and close new page for each test

* chore: remove storageStage upload artifact

e2e/playwright.config.local.ts

@@ -1,5 +1,9 @@
 import { PlaywrightTestConfig } from '@playwright/test';
 import mainConfig from './playwright.config';
+import path from 'path';
+const absolutePath = path.resolve(process.cwd(), 'api/server/index.js');
+import dotenv from 'dotenv';
+dotenv.config();
 
 const config: PlaywrightTestConfig = {
   ...mainConfig,
@@ -7,7 +11,11 @@ const config: PlaywrightTestConfig = {
   globalSetup: require.resolve('./setup/global-setup.local'),
   webServer: {
     ...mainConfig.webServer,
-    command: 'node ../api/server/index.js',
+    command: `node ${absolutePath}`,
+    env: {
+      NODE_ENV: 'production',
+      ...process.env,
+    },
   },
   fullyParallel: false, // if you are on Windows, keep this as `false`. On a Mac, `true` could make tests faster (maybe on some Windows too, just try)
   // workers: 1,

e2e/playwright.config.ts

@@ -1,5 +1,8 @@
 import { defineConfig, devices } from '@playwright/test';
 import path from 'path';
+const absolutePath = path.resolve(process.cwd(), 'api/server/index.js');
+import dotenv from 'dotenv';
+dotenv.config();
 
 export default defineConfig({
   globalSetup: require.resolve('./setup/global-setup'),
@@ -16,7 +19,7 @@ export default defineConfig({
   /* Opt out of parallel tests on CI. */
   workers: process.env.CI ? 1 : undefined,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
-  // reporter: [['html', { outputFolder: 'playwright-report' }]],
+  reporter: [['html', { outputFolder: 'playwright-report' }]],
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
   use: {
     baseURL: 'http://localhost:3080',
@@ -24,7 +27,7 @@ export default defineConfig({
     trace: 'retain-on-failure',
     ignoreHTTPSErrors: true,
     headless: true,
-    storageState: path.resolve('./e2e/storageState.json'),
+    storageState: path.resolve(process.cwd(), 'e2e/storageState.json'),
     screenshot: 'only-on-failure',
   },
   expect: {
@@ -49,10 +52,17 @@ export default defineConfig({
 
   /* Run your local dev server before starting the tests */
   webServer: {
-    command: 'node ../api/server/index.js',
+    command: `node ${absolutePath}`,
     port: 3080,
+    stdout: 'pipe',
+    ignoreHTTPSErrors: true,
     // url: 'http://localhost:3080',
     timeout: 30_000,
     reuseExistingServer: true,
+    env: {
+      ...process.env,
+      NODE_ENV: 'development',
+      SESSION_EXPIRY: '86400000',
+    },
   },
 });

e2e/setup/authenticate.ts

@@ -1,17 +1,20 @@
 import { Page, FullConfig, chromium } from '@playwright/test';
+import dotenv from 'dotenv';
+dotenv.config();
 
 type User = { username: string; password: string };
 
 async function login(page: Page, user: User) {
   await page.locator('input[name="email"]').fill(user.username);
   await page.locator('input[name="password"]').fill(user.password);
-  await page.locator('button[type="submit"]').click();
+  await page.locator('input[name="password"]').press('Enter');
 }
 
 async function authenticate(config: FullConfig, user: User) {
   console.log('🤖: global setup has been started');
   const { baseURL, storageState } = config.projects[0].use;
   console.log('🤖: using baseURL', baseURL);
+  console.dir(user, { depth: null });
   const browser = await chromium.launch();
   const page = await browser.newPage();
   console.log('🤖: 🗝  authenticating user:', user.username);
@@ -21,7 +24,12 @@ async function authenticate(config: FullConfig, user: User) {
   }
   await page.goto(baseURL);
   await login(page, user);
-  await page.locator('h1:has-text("LibreChat")').waitFor();
+  // const loginPromise = page.getByTestId('landing-title').waitFor({ timeout: 25000 }); // due to GH Actions load time
+  // if (process.env.NODE_ENV === 'ci') {
+  //   await page.screenshot({ path: 'login-screenshot.png' });
+  // }
+  // await loginPromise;
+  await page.waitForURL(`${baseURL}/chat/new`);
   console.log('🤖: ✔️  user successfully authenticated');
   // Set localStorage before navigating to the page
   await page.context().addInitScript(() => {

e2e/specs/landing.spec.ts

@@ -5,7 +5,7 @@ test.describe('Landing suite', () => {
   test('Landing title', async ({ page }) => {
     await page.goto('http://localhost:3080/');
     const pageTitle = await page.textContent('#landing-title');
-    expect(pageTitle.length).toBeGreaterThan(0);
+    expect(pageTitle?.length).toBeGreaterThan(0);
   });
 
   test('Create Conversation', async ({ page }) => {
@@ -25,7 +25,7 @@ test.describe('Landing suite', () => {
     await page.waitForSelector('nav > div');
     await page.waitForSelector('nav > div > div > svg', { state: 'detached' });
 
-    let beforeAdding = (await getItems()).length;
+    const beforeAdding = (await getItems()).length;
 
     const input = await page.locator('form').getByRole('textbox');
     await input.click();
@@ -36,7 +36,7 @@ test.describe('Landing suite', () => {
 
     // Wait for the message to be sent
     await page.waitForTimeout(3500);
-    let afterAdding = (await getItems()).length;
+    const afterAdding = (await getItems()).length;
 
     expect(afterAdding).toBeGreaterThanOrEqual(beforeAdding);
   });

e2e/specs/login.spec.js

@@ -1,9 +0,0 @@
-// import { expect, test } from '@playwright/test';
-
-// test('landing page', async ({ page }) => {
-//   await page.goto('http://localhost:3080/');
-//   const pageTitle = await page.$eval('h1', pageTitle => pageTitle.textContent);
-//   console.log('pageTitle', pageTitle);
-//   expect(pageTitle.length).toBeGreaterThan(0);
-//   expect(pageTitle).toEqual('Welcome back');
-// });

e2e/specs/messages.spec.ts

@@ -1,13 +1,46 @@
 import { expect, test } from '@playwright/test';
+import type { Response, Page } from '@playwright/test';
 
 const basePath = 'http://localhost:3080/chat/';
 const initialUrl = `${basePath}new`;
 const endpoints = ['google', 'openAI', 'azureOpenAI', 'bingAI', 'chatGPTBrowser', 'gptPlugins'];
-function isUUID(uuid) {
-  let regex = /^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$/;
+
+function isUUID(uuid: string) {
+  const regex = /^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$/;
   return regex.test(uuid);
 }
 
+async function clearConvos(page: Page) {
+  await page.goto(initialUrl);
+  await page.getByRole('button', { name: 'test' }).click();
+  await page.getByText('Settings').click();
+  await page.getByTestId('clear-convos-initial').click();
+  await page.getByTestId('clear-convos-confirm').click();
+  await page.waitForSelector('[data-testid="convo-icon"]', { state: 'detached' });
+  await page.getByRole('button', { name: 'Close' }).click();
+}
+
+test.beforeAll(async ({ browser }) => {
+  console.log('🤖: clearing conversations before message tests.');
+  const page = await browser.newPage();
+  await clearConvos(page);
+});
+
+test.afterAll(async ({ browser }) => {
+  console.log('🤖: clearing conversations after message tests.');
+  const page = await browser.newPage();
+  await clearConvos(page);
+});
+
+test.beforeEach(async ({ browser, page }) => {
+  page = await browser.newPage();
+  await page.goto(initialUrl);
+});
+
+test.afterEach(async ({ page }) => {
+  await page.close();
+});
+
 test.describe('Messaging suite', () => {
   test('textbox should be focused after receiving message & test expected navigation', async ({
     page,
@@ -15,8 +48,6 @@ test.describe('Messaging suite', () => {
     test.setTimeout(120000);
     const message = 'hi';
     const endpoint = endpoints[1];
-    const initialUrl = 'http://localhost:3080/chat/new';
-
     await page.goto(initialUrl);
     await page.locator('#new-conversation-menu').click();
     await page.locator(`#${endpoint}`).click();
@@ -24,13 +55,13 @@ test.describe('Messaging suite', () => {
     await page.locator('form').getByRole('textbox').fill(message);
 
     const responsePromise = [
-      page.waitForResponse(async (response) => {
+      page.waitForResponse(async (response: Response) => {
         return response.url().includes(`/api/ask/${endpoint}`) && response.status() === 200;
       }),
       page.locator('form').getByRole('textbox').press('Enter'),
     ];
 
-    const [response] = await Promise.all(responsePromise);
+    const [response] = (await Promise.all(responsePromise)) as [Response];
     const responseBody = await response.body();
     const messageSuccess = responseBody.includes('"final":true');
     expect(messageSuccess).toBe(true);
@@ -45,20 +76,22 @@ test.describe('Messaging suite', () => {
     expect(currentUrl).toBe(initialUrl);
 
     //cleanup the conversation
-    await page.getByRole('navigation').getByRole('button').nth(1).click();
+    await page.getByText('New chat', { exact: true }).click();
     expect(page.url()).toBe(initialUrl);
-    await page.getByTestId('convo-item').nth(1).click();
+
+    // Click on the first conversation
+    await page.getByTestId('convo-icon').first().click({ timeout: 5000 });
     const finalUrl = page.url();
-    const conversationId = finalUrl.split(basePath).pop();
+    const conversationId = finalUrl.split(basePath).pop() ?? '';
     expect(isUUID(conversationId)).toBeTruthy();
   });
 
   // in this spec as we are testing post-message navigation, we are not testing the message response
   test('Page navigations', async ({ page }) => {
     await page.goto(initialUrl);
-    await page.getByTestId('convo-item').nth(1).click();
+    await page.getByTestId('convo-icon').first().click({ timeout: 5000 });
     const currentUrl = page.url();
-    const conversationId = currentUrl.split(basePath).pop();
+    const conversationId = currentUrl.split(basePath).pop() ?? '';
     expect(isUUID(conversationId)).toBeTruthy();
     await page.getByText('New chat', { exact: true }).click();
     expect(page.url()).toBe(initialUrl);

e2e/specs/nav.spec.ts

@@ -18,7 +18,7 @@ test.describe('Navigation suite', () => {
     expect(modal).toBeTruthy();
 
     const modalTitle = await page.getByRole('heading', { name: 'Settings' }).textContent();
-    expect(modalTitle.length).toBeGreaterThan(0);
+    expect(modalTitle?.length).toBeGreaterThan(0);
     expect(modalTitle).toEqual('Settings');
 
     const modalTabList = await page.getByRole('tablist', { name: 'Settings' }).isVisible();
@@ -30,10 +30,10 @@ test.describe('Navigation suite', () => {
     const modalClearConvos = await page.getByRole('button', { name: 'Clear' }).isVisible();
     expect(modalClearConvos).toBeTruthy();
 
-    const modalTheme = await page.getByRole('combobox');
+    const modalTheme = page.getByRole('combobox').first();
     expect(modalTheme.isVisible()).toBeTruthy();
 
-    async function changeMode(theme) {
+    async function changeMode(theme: string) {
       // change the value to 'dark' and 'light' and see if the theme changes
       await modalTheme.selectOption({ label: theme });
       await page.waitForTimeout(1000);

e2e/specs/popup.spec.ts

@@ -6,7 +6,7 @@ test.describe('Endpoints Presets suite', () => {
     await page.getByRole('button', { name: 'New Topic' }).click();
 
     // includes the icon + endpoint names in obj property
-    const endpointItem = await page.getByRole('menuitemradio', { name: 'ChatGPT OpenAI' });
+    const endpointItem = page.getByRole('menuitemradio', { name: 'ChatGPT OpenAI' });
     await endpointItem.click();
 
     await page.getByRole('button', { name: 'New Topic' }).click();

e2e/specs/settings.spec.ts

@@ -3,16 +3,42 @@ import { expect, test } from '@playwright/test';
 test.describe('Settings suite', () => {
   test('Last Bing settings', async ({ page }) => {
     await page.goto('http://localhost:3080/');
-    const newTopicButton = await page.getByRole('button', { name: 'New Topic' });
+    await page.evaluate(() =>
+      window.localStorage.setItem(
+        'lastConversationSetup',
+        JSON.stringify({
+          conversationId: 'new',
+          title: 'New Chat',
+          endpoint: 'bingAI',
+          createdAt: '',
+          updatedAt: '',
+          jailbreak: false,
+          context: null,
+          systemMessage: null,
+          toneStyle: 'creative',
+          jailbreakConversationId: null,
+          conversationSignature: null,
+          clientId: null,
+          invocationId: 1,
+        }),
+      ),
+    );
+    await page.goto('http://localhost:3080/');
+
+    const initialLocalStorage = await page.evaluate(() => window.localStorage);
+    const lastConvoSetup = JSON.parse(initialLocalStorage.lastConversationSetup);
+    expect(lastConvoSetup.endpoint).toEqual('bingAI');
+
+    const newTopicButton = page.getByTestId('new-conversation-menu');
     await newTopicButton.click();
 
     // includes the icon + endpoint names in obj property
-    const endpointItem = await page.getByRole('menuitemradio', { name: 'BingAI Bing' });
+    const endpointItem = page.getByTestId('endpoint-item-bingAI');
     await endpointItem.click();
 
     await page.getByTestId('text-input').click();
-    const button1 = await page.getByRole('button', { name: 'Mode: BingAI' });
-    const button2 = await page.getByRole('button', { name: 'Mode: Sydney' });
+    const button1 = page.getByRole('button', { name: 'Mode: BingAI' });
+    const button2 = page.getByRole('button', { name: 'Mode: Sydney' });
 
     try {
       await button1.click({ timeout: 100 });
@@ -43,7 +69,7 @@ test.describe('Settings suite', () => {
     const { jailbreak, toneStyle } = lastBingSettings;
     expect(jailbreak).toBeTruthy();
     expect(toneStyle).toEqual('balanced');
-    const button = await page.getByRole('button', { name: 'Mode: Sydney' });
+    const button = page.getByRole('button', { name: 'Mode: Sydney' });
     expect(button.count()).toBeTruthy();
   });
 });