deploy: 0dc0e7e02969f3e6b714e016eab853a289dad7a7

Build/_build-phishing-domainset.js

@@ -0,0 +1,50 @@
+const psl = require('psl');
+const { processFilterRules } = require('./lib/parse-filter.js');
+
+(async () => {
+  const domainSet = Array.from(
+    (
+      await processFilterRules('https://curbengh.github.io/phishing-filter/phishing-filter-agh.txt')
+    ).black
+  );
+  const domainCountMap = {};
+
+  for (let i = 0, len = domainSet.length; i < len; i++) {
+    const line = domainSet[i];
+    // starts with #
+    if (line.charCodeAt(0) === 35) {
+      continue;
+    }
+    if (line.trim().length === 0) {
+      continue;
+    }
+
+    const domain = line.charCodeAt(0) === 46 ? line.slice(1) : line;
+    const parsed = psl.parse(domain);
+
+    if (parsed.input === parsed.tld) {
+      continue;
+    }
+
+    if (line.length > 25) {
+      domainCountMap[parsed.domain] ||= 0;
+      domainCountMap[parsed.domain] += 1;
+    }
+  }
+
+  Object.entries(domainCountMap).forEach(([domain, count]) => {
+    if (
+      count > 10
+      && (
+        domain.endsWith('.xyz')
+        || domain.endsWith('.top')
+        || domain.endsWith('.icu')
+        || domain.endsWith('.win')
+        || domain.endsWith('.shop')
+        || domain.endsWith('.cyou')
+      )
+    ) {
+      console.log('.'+ domain);
+    }
+  });
+})();

Build/build-reject-domainset.js

@@ -202,6 +202,27 @@ const threads = isCI ? cpuCount : cpuCount / 2;
     });
   });
 
+  // Read Special Phishing Suffix list
+  await fsPromises.readFile(pathResolve(__dirname, '../List/domainset/reject_phishing.conf'), { encoding: 'utf-8' }).then(data => {
+    data.split('\n').forEach(line => {
+      const trimmed = line.trim();
+      if (
+        line.startsWith('#')
+        || line.startsWith(' ')
+        || line.startsWith('\r')
+        || line.startsWith('\n')
+        || trimmed === ''
+      ) {
+        return;
+      }
+
+      /* if (domainSets.has(line) || domainSets.has(`.${line}`)) {
+        console.warn(`|${line}| is already in the list!`);
+      } */
+      domainSuffixSet.add(trimmed);
+    });
+  });
+
   console.log(`Import ${domainKeywordsSet.size} black keywords and ${domainSuffixSet.size} black suffixes!`);
 
   previousSize = domainSets.size;

Build/lib/parse-filter.js

@@ -107,14 +107,20 @@ async function processFilterRules (filterRulesUrl, fallbackUrls) {
   /** @type Set<string> */
   const blacklistDomainSets = new Set();
 
-  /** @type string[] */
-  const filterRules = (
-    await Promise.any(
-      [filterRulesUrl, ...(fallbackUrls || [])].map(
-        async url => (await fetchWithRetry(url)).text()
+  let filterRules;
+  try {
+    /** @type string[] */
+    filterRules = (
+      await Promise.any(
+        [filterRulesUrl, ...(fallbackUrls || [])].map(
+          async url => (await fetchWithRetry(url)).text()
+        )
       )
-    )
-  ).split('\n').map(line => line.trim());
+    ).split('\n').map(line => line.trim());
+  } catch (e) {
+    console.log('Download Rule for [' + filterRulesUrl + '] failed');
+    throw e;
+  }
 
   filterRules.forEach(line => {
     const lineStartsWithDoubleVerticalBar = line.startsWith('||');