Merge pull request #4 from catalyzt-tech/RPGF-90-Migrate-Agora-Cache-To-Worker

RPGF-90 Migrate agora cache to worker

Author: wit03

README.md

@@ -11,14 +11,15 @@ Worker dynamically imports and executes TypeScript files from the `src/scripts`
     - It should export a `CRON_TIMER` variable that will be used to set the cronjob timer
         - CRON_TIMER is the cronjob timer for the module, it can be undefined if the module is not a cronjob
         - Visit https://github.com/node-cron/node-cron to learn the format of cronjob
+        - <a href="https://cloud.google.com/scheduler/docs/configuring/cron-job-schedules">https://cloud.google.com/scheduler/docs/configuring/cron-job-schedules</a>
 
 ## Api Route
 - Get /api/ will list all the static file that can be served
 - Get /api/(any-file-name) will serve the file
 - Get /api/(any-folder-name) will list all the file in the folder
 - Get /api/(any-folder-name)/(any-file-name) will serve the file
 - Get /api/modules will list all the modules that can we run
-- Post /api/modules/run/(any-module-name) will run the module 
+- Post /backoffice/modules/run/(any-module-name) will run the module 
     - it required post body with a password
 
 
@@ -35,4 +36,3 @@ Worker dynamically imports and executes TypeScript files from the `src/scripts`
     	pnpm run build
 	    node dist/src/index.js
     ```
-

data/agora-cache/agora-cache.json

@@ -3,9 +3,12 @@ const MODULE_NAME = "MODULE_NAME"
 const DATA_DIR = [ 'data', 'any-folder-name']
 
 // CRON_TIMER is the cron job timer for the module, it can be undefined if the module is not a cron job
+// Which evaluates to 'At 0 seconds, 0 minutes every 1st hour'.
+// const CRON_TIMER = "0 0 */1 * * *"
 // visit https://github.com/node-cron/node-cron to get the cron job timer
 const CRON_TIMER:string | undefined = undefined
 
+
 async function Run() {
 
     console.log(`${MODULE_NAME} is starting . . .`);

data/retropgf5-live-data/retropgf5-live-data.json

@@ -5,7 +5,7 @@ import { FilePathAndUrl } from "../list-all-files-github-type";
 import { chunkArray, convertSpecialChar, headerGithub } from "../../utils/utils";
 import { Savefile } from "../save-file/save-file";
 
-
+// SaveFileMdxGithub: use to save a file that scrape from github
 export async function SaveFileMdxGithub(docs: FilePathAndUrl[], requiredPath: string[] = [], baseRefUrl:string) {
 
     try {
@@ -31,7 +31,7 @@ export async function SaveFileMdxGithub(docs: FilePathAndUrl[], requiredPath: st
                         headers: headers
                     });
 
-                    let content:string = "";
+                    let content:string = ""
 
                     // decode encrypt content from github
                     const decodedContent = Buffer.from(response.data.content, 'base64').toString('utf-8');

docs/example.ts

@@ -24,6 +24,6 @@ export async function Savefile(data: any, requiredPath: string[] = [], fileName:
         console.log(`File successfully written to ${filePath}`);
     } catch (error) {
         console.error("An error occurred during the save file process:", error);
-        throw error; // Re-throw the error for proper handling upstream
+        throw new Error("An error occurred during the save file process");
     }
 }

lib/op-github/save-file-mdx-from-github.ts

@@ -60,7 +60,7 @@ async function processScripts() {
                         const dataDir = module.DATA_DIR.join("").replace("data","");
 
                         // get the api path
-                        const apiPath = "/api/modules/run/" + file;
+                        const apiPath = "/backoffice/modules/run/" + file;
 
                         // add to the res object
                         fileNameAndApiPath[file] = apiPath;

lib/save-file/save-file.ts

@@ -0,0 +1,10 @@
+export interface Response {
+    meta: Meta;
+    data: any;
+}
+
+export interface Meta {
+    has_next:       boolean;
+    total_returned: number;
+    next_offset:    number;
+}

src/index.ts

@@ -0,0 +1,61 @@
+import axios, { AxiosError, AxiosResponse } from 'axios'
+import { Response } from './agora-api-type'
+
+// CallApiAgora: fetch a project base on limit and offset
+
+const API_URL = 'https://vote.optimism.io/api/v1/retrofunding/rounds/5/projects'
+
+async function CallApiAgora(limit: number, offset: number, authorizeKey: string): Promise<Response> {
+  const url = `${API_URL}?limit=${limit}&offset=${offset}`
+  try {
+    const response: AxiosResponse<Response> = await axios.get(url, {
+      maxBodyLength: Infinity,
+      headers: { 
+        'Authorization': `Bearer ${authorizeKey}`
+      }
+    })
+
+    return response.data
+  } catch (error) {
+    if (axios.isAxiosError(error)) {
+      const axiosError = error as AxiosError
+      if (axiosError.response) {
+        console.error(`API error: ${axiosError.response.status} - ${axiosError.response.statusText}`)
+        throw new Error(`API error: ${axiosError.response.status} - ${axiosError.response.statusText}`)
+      } else {
+        console.error('Network error:', axiosError.message)
+        throw new Error(`Network error: ${axiosError.message}`)
+      }
+    }
+    console.error('Unexpected error:', error)
+    throw error
+  }
+}
+
+// GetAllProjects: fetch all the exist project until the has_next is false
+export async function GetAllProjects(authorizeKey: string): Promise<Response[]> {
+  let conditionBreak = false
+  let limit = 100
+  let offset = 0
+
+  let datas: Response[] = []
+  
+  while (!conditionBreak) {
+    try {
+      const data = await CallApiAgora(limit, offset, authorizeKey)
+      if (data){
+        datas = datas.concat(data)
+        if (data.meta.has_next) {
+          offset += 100
+        } else {
+          conditionBreak = true
+        }
+      }
+    } catch (error) {
+      console.error('Error fetching projects:', error)
+      throw error
+    }
+  }
+
+  return datas
+}

src/scripts/agora-cache/_helper/agora-api-type.d.ts

@@ -0,0 +1,32 @@
+import { Savefile } from "../../../lib/save-file/save-file"
+import { GetAllProjects } from "./_helper/agora-api"
+
+const DATA_DIR = ["data", "agora-cache"]
+const FILENAME = "agora-cache.json"
+
+// Schedule a task to run every 30 minutes
+const CRON_TIMER: string | undefined = "*/30 * * * *"
+
+async function Run() {
+  console.log("Agora cache starting is starting . . .")
+  try {
+
+    const agoraKey = process.env.AGORA_KEY
+    if (!agoraKey){
+      throw new Error("Can't find agora key")
+    }
+
+    let projects = await GetAllProjects(agoraKey)
+    if (!projects || projects.length === 0) {
+      throw new Error("No projects data found")
+    }
+    await Savefile(JSON.stringify(projects), DATA_DIR, FILENAME)
+    
+  } catch (error) {
+    console.error("An error occurred during the Agora cache starting process:", error)
+  } finally {
+    console.log("Agora cache starting process finished.")
+  }
+}
+
+export { Run, DATA_DIR, CRON_TIMER }