ci: optimize crowdin

Author: sheepzh

script/crowdin/client.ts

@@ -19,14 +19,12 @@ const MAIN_BRANCH_NAME = 'main'
  */
 class PaginationIterator<T> {
     private offset = 0
-    private limit = 25
+    private limit = 500
     private isEnd = false
     private buf: T[] = []
     private cursor = 0
-    private query: (pagination: Pagination) => Promise<ResponseList<T>>
 
-    constructor(query: (pagination: Pagination) => Promise<ResponseList<T>>) {
-        this.query = query
+    constructor(private query: (pagination: Pagination) => Promise<ResponseList<T>>) {
     }
 
     reset(): void {
@@ -169,17 +167,6 @@ export class CrowdinClient {
         return response.data
     }
 
-    async restoreFile(storage: UploadStorageModel.Storage, existFile: SourceFilesModel.File): Promise<SourceFilesModel.File> {
-        const response = await this.crowdin.sourceFilesApi.updateOrRestoreFile(PROJECT_ID, existFile.id, { storageId: storage.id })
-        return response.data
-    }
-
-    getFileByName(param: NameKey): Promise<SourceFilesModel.File | undefined> {
-        return new PaginationIterator(
-            p => this.crowdin.sourceFilesApi.listProjectFiles(PROJECT_ID, { ...p, branchId: param.branchId })
-        ).findFirst(t => t.name === param.name)
-    }
-
     getDirByName(param: NameKey): Promise<SourceFilesModel.Directory | undefined> {
         return new PaginationIterator(
             p => this.crowdin.sourceFilesApi.listProjectDirectories(PROJECT_ID, { ...p, branchId: param.branchId })
@@ -200,6 +187,12 @@ export class CrowdinClient {
         ).findAll()
     }
 
+    async downloadSourceFile(fileId: number): Promise<Record<string, unknown>> {
+        const res = await this.crowdin.sourceFilesApi.downloadFile(PROJECT_ID, fileId)
+        const response = await fetch(res.data.url)
+        return await response.json() as Record<string, unknown>
+    }
+
     listStringsByFile(fileId: number): Promise<SourceStringsModel.String[]> {
         return new PaginationIterator(
             p => this.crowdin.sourceStringsApi.listProjectStrings(PROJECT_ID, { ...p, fileId: fileId })
@@ -280,11 +273,24 @@ export class CrowdinClient {
             skipUntranslatedStrings: true,
         })
         const buildId = buildRes.data.id
+        const maxRetries = 120
+        let retryCount = 0
         while (true) {
-            // Wait finished
-            const res = await this.crowdin.translationsApi.downloadTranslations(PROJECT_ID, buildId)
-            return res.data.url
+            if (retryCount >= maxRetries) {
+                throw new Error(`Build timed out after ${maxRetries} retries: buildId=${buildId}`)
+            }
+            const statusRes = await this.crowdin.translationsApi.checkBuildStatus(PROJECT_ID, buildId)
+            const { status, progress } = statusRes.data
+            console.log(`Build status: ${status}, progress: ${progress}%`)
+            if (status === 'finished') break
+            if (status === 'canceled' || status === 'failed') {
+                throw new Error(`Build ${status}: buildId=${buildId}`)
+            }
+            retryCount++
+            await new Promise(resolve => setTimeout(resolve, 1000))
         }
+        const res = await this.crowdin.translationsApi.downloadTranslations(PROJECT_ID, buildId)
+        return res.data.url
     }
 }
 

script/crowdin/common.ts

@@ -92,7 +92,7 @@ export const RSC_FILE_SUFFIX = "-resource.json"
  * @param dir the directory of messages
  * @returns
  */
-export async function readAllMessages(dir: Dir): Promise<Record<string, Messages<any>>> {
+export async function readAllMessages(dir: Dir): Promise<Record<string, Messages<Record<string, unknown>>>> {
     const dirPath = path.join(MSG_BASE, dir)
 
     const files = fs.readdirSync(dirPath)
@@ -101,7 +101,7 @@ export async function readAllMessages(dir: Dir): Promise<Record<string, Messages
         if (!file.endsWith(RSC_FILE_SUFFIX)) {
             return
         }
-        const message = (await import(`@i18n/message/${dir}/${file}`))?.default as Messages<any>
+        const message = (await import(`@i18n/message/${dir}/${file}`))?.default as Messages<Record<string, unknown>>
         const name = file.replace(RSC_FILE_SUFFIX, '')
         message && (result[name] = message)
         return
@@ -110,10 +110,10 @@ export async function readAllMessages(dir: Dir): Promise<Record<string, Messages
 }
 
 /**
- * Merge crowdin message into locale resource json files
+ * Merge crowdin message into locale resource JSON files
  *
  * @param dir dir
- * @param filename the name of json file
+ * @param filename the name of JSON file
  * @param messages crowdin messages
  */
 export async function mergeMessage(
@@ -123,7 +123,7 @@ export async function mergeMessage(
 ): Promise<void> {
     const dirPath = path.join(MSG_BASE, dir)
     const filePath = path.join(dirPath, filename)
-    const existMessages = (await import(`@i18n/message/${dir}/${filename}`))?.default as Messages<any>
+    const existMessages = (await import(`@i18n/message/${dir}/${filename}`))?.default as Messages<Record<string, unknown>>
     if (!existMessages) {
         logError(`Failed to find local code: dir=${dir}, filename=${filename}`)
         return
@@ -155,22 +155,22 @@ export async function mergeMessage(
 }
 
 function logError(msg: string) {
-    console.error(`[CROWDIN-ERROR] ${msg}`)
+    console.error(`\x1b[31m[CROWDIN-ERROR]\x1b[0m ${msg}`)
 }
 
 function checkPlaceholder(translated: string, source: string) {
     const allSourcePlaceholders =
-        Array.from(source.matchAll(/\{(.*?)\}/g))
+        Array.from(source.matchAll(/\{(.*?)}/g))
             .map(matched => matched[1])
             .sort()
     const allTranslatedPlaceholders =
-        Array.from(translated.matchAll(/\{(.*?)\}/g))
+        Array.from(translated.matchAll(/\{(.*?)}/g))
             .map(matched => matched[1])
             .sort()
     if (allSourcePlaceholders.length != allTranslatedPlaceholders.length) {
         return false
     }
-    for (let i = 0; i++; i < allSourcePlaceholders.length) {
+    for (let i = 0; i < allSourcePlaceholders.length; i++) {
         if (allSourcePlaceholders[i] !== allTranslatedPlaceholders[i]) {
             return false
         }
@@ -220,3 +220,5 @@ export async function checkMainBranch(client: CrowdinClient): Promise<SourceFile
     const branch = await client.getMainBranch()
     return branch ?? exitWith("Main branch is null")
 }
+
+

script/crowdin/download.ts

@@ -0,0 +1,49 @@
+import decompress from "decompress"
+import { existsSync, readFileSync } from "fs"
+import { rm, writeFile } from "fs/promises"
+import { join } from "path"
+import { type Dir, type ItemSet, transMsg } from "./common"
+
+const TEMP_FILE_NAME = join(process.cwd(), ".crowdin-temp.zip")
+const TEMP_DIR = join(process.cwd(), ".crowdin-temp")
+
+export async function downloadProjectZip(url: string): Promise<string> {
+    const res = await fetch(url)
+    const blob = await res.blob()
+    const buffer = Buffer.from(await blob.arrayBuffer())
+    await writeFile(TEMP_FILE_NAME, buffer)
+    console.log("Downloaded project zip file")
+    if (existsSync(TEMP_DIR)) {
+        await rm(TEMP_DIR, { recursive: true, force: true })
+    }
+    await decompress(TEMP_FILE_NAME, TEMP_DIR)
+    console.log("Decompressed zip file")
+    return TEMP_DIR
+}
+
+export async function clearTempFile() {
+    await rm(TEMP_FILE_NAME, { force: true })
+    await rm(TEMP_DIR, { recursive: true, force: true })
+}
+
+/**
+ * Read a single file from the unzipped Crowdin project translation.
+ *
+ * @param tmpDir temp directory containing unzipped translations
+ * @param langDir language subdirectory (e.g. 'zh-CN' or 'en')
+ * @param dir message directory
+ * @param crowdinFileName file name with .json extension
+ */
+export function readCrowdinZipFile(tmpDir: string, langDir: string, dir: Dir, crowdinFileName: string): ItemSet {
+    const filePath = join(tmpDir, langDir, dir, crowdinFileName)
+    if (!existsSync(filePath)) {
+        return {}
+    }
+    try {
+        const json = readFileSync(filePath).toString()
+        return transMsg(JSON.parse(json))
+    } catch (error) {
+        console.warn(`Failed to parse crowdin zip file: ${filePath}`, error)
+        return {}
+    }
+}

script/crowdin/export-translation.ts

@@ -1,26 +1,17 @@
-import decompress from "decompress"
-import { existsSync, readdirSync, readFileSync, rm, writeFile } from "fs"
+import { readdirSync, readFileSync } from "fs"
 import { join } from "path"
 import { getClientFromEnv } from "./client"
 import {
-    ALL_DIRS, ALL_TRANS_LOCALES,
-    checkMainBranch,
-    crowdinLangOf,
-    type Dir,
-    type ItemSet,
-    mergeMessage,
-    RSC_FILE_SUFFIX,
+    ALL_DIRS, ALL_TRANS_LOCALES, checkMainBranch, crowdinLangOf, type Dir, type ItemSet, mergeMessage, RSC_FILE_SUFFIX,
     transMsg
 } from "./common"
+import { clearTempFile, downloadProjectZip } from "./download"
 
-const TEMP_FILE_NAME = join(process.cwd(), ".crowdin-temp.zip")
-const TEMP_DIR = join(process.cwd(), ".crowdin-temp")
-
-async function processDir(dir: Dir): Promise<void> {
+async function processDir(tmpDir: string, dir: Dir): Promise<void> {
     const fileSets: Record<string, Partial<Record<tt4b.Locale, ItemSet>>> = {}
     for (const locale of ALL_TRANS_LOCALES) {
         const crowdinLang = crowdinLangOf(locale)
-        const dirPath = join(TEMP_DIR, crowdinLang, dir)
+        const dirPath = join(tmpDir, crowdinLang, dir)
         const files = readdirSync(dirPath)
         for (const fileName of files) {
             const json = readFileSync(join(dirPath, fileName)).toString()
@@ -34,44 +25,18 @@ async function processDir(dir: Dir): Promise<void> {
     }
 }
 
-async function downloadProjectZip(url: string): Promise<void> {
-    const res = await fetch(url)
-    const blob = await res.blob()
-    const buffer = Buffer.from(await blob.arrayBuffer())
-    await new Promise(resolve => writeFile(TEMP_FILE_NAME, buffer, resolve))
-}
-
-async function compressProjectZip(): Promise<void> {
-    if (existsSync(TEMP_DIR)) {
-        await new Promise(resolve => rm(TEMP_DIR, { recursive: true }, resolve))
-    }
-    await decompress(TEMP_FILE_NAME, TEMP_DIR)
-}
-
-async function clearTempFile() {
-    await new Promise(resolve => rm(TEMP_FILE_NAME, resolve))
-    await new Promise(resolve => rm(TEMP_DIR, { recursive: true }, resolve))
-}
-
 async function main() {
     const client = getClientFromEnv()
     const branch = await checkMainBranch(client)
     const zipUrl = await client.buildProjectTranslation(branch.id)
     console.log("Built project translations")
     console.log(zipUrl)
-    await downloadProjectZip(zipUrl)
-    console.log("Downloaded project zip file")
-    try {
-        await compressProjectZip()
-        console.log("Compressed zip file")
-        for (const dir of ALL_DIRS) {
-            await processDir(dir)
-            console.log("Processed dir: " + dir)
-        }
-    } finally {
-        clearTempFile()
-        console.log("Cleaned temp files")
+    const tmpDir = await downloadProjectZip(zipUrl)
+
+    for (const dir of ALL_DIRS) {
+        await processDir(tmpDir, dir)
+        console.log("Processed dir: " + dir)
     }
 }
 
-main()
+main().finally(clearTempFile)

script/crowdin/sync-source.ts

@@ -1,15 +1,20 @@
-import { type SourceFilesModel, type SourceStringsModel } from "@crowdin/crowdin-api-client"
+import type { SourceFilesModel, SourceStringsModel } from "@crowdin/crowdin-api-client"
 import { toMap } from "@util/array"
 import { type CrowdinClient, getClientFromEnv, type NameKey } from "./client"
-import {
-    ALL_DIRS,
-    Dir,
-    isIgnored,
-    ItemSet,
-    readAllMessages,
-    SOURCE_LOCALE,
-    transMsg
-} from "./common"
+import { ALL_DIRS, type Dir, isIgnored, type ItemSet, readAllMessages, SOURCE_LOCALE, transMsg } from "./common"
+
+/**
+ * Check if local source content is the same as Crowdin source content
+ */
+function isSourceUnchanged(localContent: ItemSet, crowdinContent: ItemSet): boolean {
+    const localKeys = Object.keys(localContent).filter(k => !!localContent[k])
+    const crowdinKeys = Object.keys(crowdinContent)
+    if (localKeys.length !== crowdinKeys.length) return false
+    for (const key of localKeys) {
+        if (localContent[key] !== crowdinContent[key]) return false
+    }
+    return true
+}
 
 async function initBranch(client: CrowdinClient): Promise<SourceFilesModel.Branch> {
     const branch = await client.getOrCreateMainBranch()
@@ -85,6 +90,14 @@ async function processByDir(client: CrowdinClient, dir: Dir, branch: SourceFiles
             const storage = await client.createStorage(crowdinFilename, fileContent)
             existFile = await client.createFile(directory.id, storage, crowdinFilename)
             console.log(`Created new file: dir=${dir}, fileName=${crowdinFilename}, id=${existFile.id}`)
+        } else {
+            // Download source file from Crowdin and compare
+            const crowdinJson = await client.downloadSourceFile(existFile.id)
+            const crowdinContent = transMsg(crowdinJson)
+            if (isSourceUnchanged(fileContent, crowdinContent)) {
+                console.log(`No source diff for ${dir}/${crowdinFilename}, skipped`)
+                continue
+            }
         }
         // Process by strings
         await processStrings(client, existFile, fileContent)