Perf: improve memory GC

Author: SukkaW

Build/build-reject-domainset.ts

@@ -90,8 +90,14 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as
       hostsDownloads.map(task => task(childSpan).then(appendArrayToRejectOutput)),
       hostsExtraDownloads.map(task => task(childSpan).then(appendArrayToRejectExtraOutput)),
 
-      domainListsDownloads.map(task => task(childSpan).then(appendArrayToRejectOutput)),
-      domainListsExtraDownloads.map(task => task(childSpan).then(appendArrayToRejectExtraOutput)),
+      domainListsDownloads.map((task, index) => task(
+        childSpan,
+        DOMAIN_LISTS[index][2] ? rejectOutput.addDomainSuffix.bind(rejectOutput) : rejectOutput.addDomain.bind(rejectOutput)
+      )),
+      domainListsExtraDownloads.map((task, index) => task(
+        childSpan,
+        DOMAIN_LISTS_EXTRA[index][2] ? rejectExtraOutput.addDomainSuffix.bind(rejectExtraOutput) : rejectExtraOutput.addDomain.bind(rejectExtraOutput)
+      )),
 
       adguardFiltersDownloads.map(
         task => task(childSpan).then(({ whiteDomains, whiteDomainSuffixes, blackDomains, blackDomainSuffixes }) => {

Build/lib/get-phishing-domains.ts

@@ -7,10 +7,9 @@ import type { Span } from '../trace';
 import { appendArrayInPlaceCurried } from './append-array-in-place';
 import { PHISHING_DOMAIN_LISTS_EXTRA, PHISHING_HOSTS_EXTRA } from '../constants/reject-data-source';
 
-const downloads = [
-  ...PHISHING_DOMAIN_LISTS_EXTRA.map(entry => processDomainListsWithPreload(...entry)),
-  ...PHISHING_HOSTS_EXTRA.map(entry => processHostsWithPreload(...entry))
-];
+const domainListsDownloads = PHISHING_DOMAIN_LISTS_EXTRA.map(entry => processDomainListsWithPreload(...entry));
+
+const downloads = PHISHING_HOSTS_EXTRA.map(entry => processHostsWithPreload(...entry));
 
 const pool = new Worktank({
   name: 'process-phishing-domains',
@@ -187,6 +186,11 @@ export function getPhishingDomains(parentSpan: Span) {
       const domainGroups = await Promise.all(downloads.map(task => task(curSpan)));
       domainGroups.forEach(appendArrayInPlaceCurried(domainArr));
 
+      await Promise.all(domainListsDownloads.map((task, index) => task(
+        curSpan,
+        PHISHING_DOMAIN_LISTS_EXTRA[index][2] ? (domain: string) => domainArr.push('.' + domain) : domainArr.push.bind(domainArr)
+      )));
+
       return domainArr;
     });
 

Build/lib/parse-filter/domainlists.ts

@@ -1,42 +1,26 @@
-import { fastNormalizeDomain, fastNormalizeDomainWithoutWww } from '../normalize-domain';
+import { fastNormalizeDomainWithoutWww } from '../normalize-domain';
 import { onBlackFound } from './shared';
 import { fetchAssets } from '../fetch-assets';
 import type { Span } from '../../trace';
 
-function domainListLineCb(line: string, set: string[], meta: string, normalizeDomain = fastNormalizeDomain) {
-  const domain = normalizeDomain(line);
-  if (!domain) return;
-
-  onBlackFound(domain, meta);
-
-  set.push(domain);
-}
-
-function domainListLineCbIncludeAllSubdomain(line: string, set: string[], meta: string, normalizeDomain = fastNormalizeDomain) {
-  const domain = normalizeDomain(line);
-  if (!domain) return;
-
-  onBlackFound(domain, meta);
-
-  set.push('.' + domain);
-}
 export function processDomainListsWithPreload(
   domainListsUrl: string, mirrors: string[] | null,
-  includeAllSubDomain = false
+  _includeAllSubDomain = false
 ) {
   const downloadPromise = fetchAssets(domainListsUrl, mirrors, true);
-  const lineCb = includeAllSubDomain ? domainListLineCbIncludeAllSubdomain : domainListLineCb;
 
-  return (span: Span) => span.traceChildAsync(`process domainlist: ${domainListsUrl}`, async (span) => {
+  return (span: Span, onDomain: (domain: string) => void) => span.traceChildAsync(`process domainlist: ${domainListsUrl}`, async (span) => {
     const filterRules = await span.traceChildPromise('download', downloadPromise);
-    const domainSets: string[] = [];
 
     span.traceChildSync('parse domain list', () => {
       for (let i = 0, len = filterRules.length; i < len; i++) {
-        lineCb(filterRules[i], domainSets, domainListsUrl, fastNormalizeDomainWithoutWww);
+        const domain = fastNormalizeDomainWithoutWww(filterRules[i]);
+        if (!domain) return;
+
+        onBlackFound(domain, domainListsUrl);
+
+        onDomain(domain);
       }
     });
-
-    return domainSets;
   });
 }