ℹ️ refactor: Remove use of Agenda for Conversation Imports (#3024)

* chore: remove agenda and npm audit fix

* refactor: import conversations without agenda

* chore: update package-lock.json and data-provider version to 0.6.7

* fix: import conversations

* chore: client npm audit fix

api/server/utils/import/importConversations.js

@@ -1,18 +1,14 @@
 const fs = require('fs').promises;
-const jobScheduler = require('~/server/utils/jobScheduler');
 const { getImporter } = require('./importers');
 const { indexSync } = require('~/lib/db');
 const { logger } = require('~/config');
 
-const IMPORT_CONVERSATION_JOB_NAME = 'import conversation';
-
 /**
  * Job definition for importing a conversation.
- * @param {import('agenda').Job} job - The job object.
- * @param {Function} done - The done function.
+ * @param {{ filepath, requestUserId }} job - The job object.
  */
-const importConversationJob = async (job, done) => {
-  const { filepath, requestUserId } = job.attrs.data;
+const importConversations = async (job) => {
+  const { filepath, requestUserId } = job;
   try {
     logger.debug(`user: ${requestUserId} | Importing conversation(s) from file...`);
     const fileData = await fs.readFile(filepath, 'utf8');
@@ -22,10 +18,8 @@ const importConversationJob = async (job, done) => {
     // Sync Meilisearch index
     await indexSync();
     logger.debug(`user: ${requestUserId} | Finished importing conversations`);
-    done();
   } catch (error) {
     logger.error(`user: ${requestUserId} | Failed to import conversation: `, error);
-    done(error);
   } finally {
     try {
       await fs.unlink(filepath);
@@ -35,7 +29,4 @@ const importConversationJob = async (job, done) => {
   }
 };
 
-// Call the jobScheduler.define function at startup
-jobScheduler.define(IMPORT_CONVERSATION_JOB_NAME, importConversationJob);
-
-module.exports = { IMPORT_CONVERSATION_JOB_NAME };
+module.exports = importConversations;

api/server/utils/import/index.js

@@ -1,5 +1,7 @@
 const importers = require('./importers');
+const importConversations = require('./importConversations');
 
 module.exports = {
   ...importers,
+  importConversations,
 };

api/server/utils/jobScheduler.js

@@ -1,99 +0,0 @@
-const Agenda = require('agenda');
-const { logger } = require('~/config');
-const mongodb = require('mongodb');
-
-/**
- * Class for scheduling and running jobs.
- * The workflow is as follows: start the job scheduler, define a job, and then schedule the job using defined job name.
- */
-class JobScheduler {
-  constructor() {
-    this.agenda = new Agenda({ db: { address: process.env.MONGO_URI } });
-  }
-
-  /**
-   * Starts the job scheduler.
-   */
-  async start() {
-    try {
-      logger.info('Starting Agenda...');
-      await this.agenda.start();
-      logger.info('Agenda successfully started and connected to MongoDB.');
-    } catch (error) {
-      logger.error('Failed to start Agenda:', error);
-    }
-  }
-
-  /**
-   * Schedules a job to start immediately.
-   * @param {string} jobName - The name of the job to schedule.
-   * @param {string} filepath - The filepath to pass to the job.
-   * @param {string} userId - The ID of the user requesting the job.
-   * @returns {Promise<{ id: string }>} - A promise that resolves with the ID of the scheduled job.
-   * @throws {Error} - If the job fails to schedule.
-   */
-  async now(jobName, filepath, userId) {
-    try {
-      const job = await this.agenda.now(jobName, { filepath, requestUserId: userId });
-      logger.debug(`Job '${job.attrs.name}' scheduled successfully.`);
-      return { id: job.attrs._id.toString() };
-    } catch (error) {
-      throw new Error(`Failed to schedule job '${jobName}': ${error}`);
-    }
-  }
-
-  /**
-   * Gets the status of a job.
-   * @param {string} jobId - The ID of the job to get the status of.
-   * @returns {Promise<{ id: string, userId: string, name: string, failReason: string, status: string } | null>} - A promise that resolves with the job status or null if the job is not found.
-   * @throws {Error} - If multiple jobs are found.
-   */
-  async getJobStatus(jobId) {
-    const job = await this.agenda.jobs({ _id: new mongodb.ObjectId(jobId) });
-    if (!job || job.length === 0) {
-      return null;
-    }
-
-    if (job.length > 1) {
-      // This should never happen
-      throw new Error('Multiple jobs found.');
-    }
-
-    const jobDetails = {
-      id: job[0]._id,
-      userId: job[0].attrs.data.requestUserId,
-      name: job[0].attrs.name,
-      failReason: job[0].attrs.failReason,
-      status: !job[0].attrs.lastRunAt
-        ? 'scheduled'
-        : job[0].attrs.failedAt
-          ? 'failed'
-          : job[0].attrs.lastFinishedAt
-            ? 'completed'
-            : 'running',
-    };
-
-    return jobDetails;
-  }
-
-  /**
-   * Defines a new job.
-   * @param {string} name - The name of the job.
-   * @param {Function} jobFunction - The function to run when the job is executed.
-   */
-  define(name, jobFunction) {
-    this.agenda.define(name, async (job, done) => {
-      try {
-        await jobFunction(job, done);
-      } catch (error) {
-        logger.error(`Failed to run job '${name}': ${error}`);
-        done(error);
-      }
-    });
-  }
-}
-
-const jobScheduler = new JobScheduler();
-jobScheduler.start();
-
-module.exports = jobScheduler;