mirror of
https://gitlab.com/SukkaW/ruleset.skk.moe.git
synced 2026-01-04 11:30:26 +00:00
deploy: 00d9bf80239d3f2c8583e76a897872af858be575
This commit is contained in:
SukkaBot
@@ -2,10 +2,10 @@ const { default: got } = require('got-cjs');
|
||||
const { promises: fsPromises } = require('fs');
|
||||
const { resolve: pathResolve } = require('path');
|
||||
const { cpus } = require('os');
|
||||
const { isIP } = require('net');
|
||||
|
||||
const threads = Math.max(cpus().length, 12);
|
||||
|
||||
const rIPv4 = /((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/;
|
||||
const rDomain = /^(((?!\-))(xn\-\-)?[a-z0-9\-_]{0,61}[a-z0-9]{1,1}\.)*(xn\-\-)?([a-z0-9\-]{1,61}|[a-z0-9\-]{1,30})\.[a-z]{2,}$/m
|
||||
|
||||
const Piscina = require('piscina');
|
||||
@@ -83,53 +83,23 @@ async function processFilterRules(filterRulesUrl) {
|
||||
}
|
||||
|
||||
/** @type Set<string> */
|
||||
const whitelistDomainSets = new Set([
|
||||
'localhost',
|
||||
'broadcasthost',
|
||||
'ip6-loopback',
|
||||
'ip6-localnet',
|
||||
'ip6-mcastprefix',
|
||||
'ip6-allnodes',
|
||||
'ip6-allrouters',
|
||||
'ip6-allhosts',
|
||||
'mcastprefix',
|
||||
'analytics.google.com',
|
||||
'msa.cdn.mediaset.net', // Added manually using DOMAIN-KEYWORDS
|
||||
'cloud.answerhub.com',
|
||||
'ae01.alicdn.com',
|
||||
'whoami.akamai.net',
|
||||
'whoami.ds.akahelp.net',
|
||||
'pxlk9.net.', // This one is malformed from EasyList, which I will manually add instead
|
||||
'instant.page', // No, it doesn't violate anyone's privacy. I will whitelist it
|
||||
'piwik.pro',
|
||||
'mixpanel.com',
|
||||
'heapanalytics.com',
|
||||
'dataunlocker.com',
|
||||
'segment.com',
|
||||
'segment.io',
|
||||
'segmentify.com'
|
||||
]);
|
||||
const whitelistDomainSets = new Set();
|
||||
/** @type Set<string> */
|
||||
const blacklistDomainSets = new Set();
|
||||
|
||||
/** @type Set<string> */
|
||||
const blackIPSets = new Set();
|
||||
|
||||
/** @type string[] */
|
||||
const filterRules = (await got(filterRulesUrl).text()).split('\n');
|
||||
const filterRules = (await got(filterRulesUrl).text()).split('\n').map(line => line.trim());
|
||||
|
||||
filterRules.forEach(line => {
|
||||
if (
|
||||
line.includes('#')
|
||||
line === ''
|
||||
|| line.includes('#')
|
||||
|| line.includes('!')
|
||||
|| line.startsWith(' ')
|
||||
|| line.startsWith('\r')
|
||||
|| line.startsWith('\n')
|
||||
|| line.includes('*')
|
||||
|| line.includes('/')
|
||||
|| line.includes('$')
|
||||
|| line.includes('$') && !line.startsWith('@@')
|
||||
|| line.trim() === ''
|
||||
|| rIPv4.test(line)
|
||||
|| isIP(line) !== 0
|
||||
) {
|
||||
return;
|
||||
}
|
||||
@@ -138,9 +108,17 @@ async function processFilterRules(filterRulesUrl) {
|
||||
&& (
|
||||
line.endsWith('^')
|
||||
|| line.endsWith('^|')
|
||||
|| line.endsWith('^$badfilter')
|
||||
|| line.endsWith('^$1p')
|
||||
)
|
||||
) {
|
||||
const domain = `${line.replaceAll('@@||', '').replaceAll('^|', '').replaceAll('^', '')}`.trim();
|
||||
const domain = line
|
||||
.replaceAll('@@||', '')
|
||||
.replaceAll('^$badfilter', '')
|
||||
.replaceAll('^$1p', '')
|
||||
.replaceAll('^|', '')
|
||||
.replaceAll('^', '')
|
||||
.trim();
|
||||
if (rDomain.test(domain)) {
|
||||
whitelistDomainSets.add(domain);
|
||||
}
|
||||
@@ -198,9 +176,9 @@ async function processFilterRules(filterRulesUrl) {
|
||||
if (
|
||||
line.startsWith('#')
|
||||
|| line.startsWith(' ')
|
||||
|| line === '' || line === ' '
|
||||
|| line.startsWith('\r')
|
||||
|| line.startsWith('\n')
|
||||
|| line.trim() === ''
|
||||
) {
|
||||
return;
|
||||
}
|
||||
@@ -217,23 +195,74 @@ async function processFilterRules(filterRulesUrl) {
|
||||
|
||||
// Parse from AdGuard Filters
|
||||
/** @type Set<string> */
|
||||
const filterRuleWhitelistDomainSets = new Set();
|
||||
const filterRuleWhitelistDomainSets = new Set([
|
||||
'localhost',
|
||||
'broadcasthost',
|
||||
'ip6-loopback',
|
||||
'ip6-localnet',
|
||||
'ip6-mcastprefix',
|
||||
'ip6-allnodes',
|
||||
'ip6-allrouters',
|
||||
'ip6-allhosts',
|
||||
'mcastprefix',
|
||||
'analytics.google.com',
|
||||
'msa.cdn.mediaset.net', // Added manually using DOMAIN-KEYWORDS
|
||||
'cloud.answerhub.com',
|
||||
'ae01.alicdn.com',
|
||||
'whoami.akamai.net',
|
||||
'whoami.ds.akahelp.net',
|
||||
'pxlk9.net.', // This one is malformed from EasyList, which I will manually add instead
|
||||
'instant.page', // No, it doesn't violate anyone's privacy. I will whitelist it
|
||||
'piwik.pro',
|
||||
'mixpanel.com',
|
||||
'heapanalytics.com',
|
||||
'dataunlocker.com',
|
||||
'segment.com',
|
||||
'segment.io',
|
||||
'segmentify.com'
|
||||
]);
|
||||
|
||||
(await Promise.all([
|
||||
processFilterRules('https://easylist.to/easylist/easylist.txt'),
|
||||
processFilterRules('https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt'),
|
||||
processFilterRules('https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_11_Mobile/filter.txt'),
|
||||
processFilterRules('https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_3_Spyware/filter.txt'),
|
||||
processFilterRules('https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_2_English/filter.txt'),
|
||||
processFilterRules('https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_224_Chinese/filter.txt'),
|
||||
processFilterRules('https://filters.adtidy.org/extension/ublock/filters/224.txt'),
|
||||
processFilterRules('https://easylist.to/easylist/easyprivacy.txt'),
|
||||
processFilterRules('https://raw.githubusercontent.com/DandelionSprout/adfilt/master/GameConsoleAdblockList.txt'),
|
||||
processFilterRules('https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/SmartTV-AGH.txt'),
|
||||
processFilterRules('https://curben.gitlab.io/malware-filter/urlhaus-filter-agh-online.txt'),
|
||||
processFilterRules('https://curben.gitlab.io/malware-filter/pup-filter-agh.txt'),
|
||||
processFilterRules('https://curben.gitlab.io/malware-filter/phishing-filter-agh.txt'),
|
||||
processFilterRules('https://curben.gitlab.io/malware-filter/pup-filter-agh.txt')
|
||||
])).forEach(({ white, black }) => {
|
||||
// Easy List
|
||||
'https://easylist.to/easylist/easylist.txt',
|
||||
// AdGuard DNS Filter
|
||||
'https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt',
|
||||
// uBlock Origin Filter List
|
||||
'https://ublockorigin.github.io/uAssetsCDN/filters/filters.txt',
|
||||
'https://ublockorigin.github.io/uAssetsCDN/filters/filters-2020.txt',
|
||||
'https://ublockorigin.github.io/uAssetsCDN/filters/filters-2021.txt',
|
||||
'https://ublockorigin.github.io/uAssetsCDN/filters/filters-2022.txt',
|
||||
// uBlock Origin Badware Risk List
|
||||
'https://ublockorigin.github.io/uAssets/filters/badware.txt',
|
||||
// uBlock Origin Privacy List
|
||||
'https://ublockorigin.github.io/uAssets/filters/privacy.txt',
|
||||
// uBlock Origin Resource Abuse
|
||||
'https://ublockorigin.github.io/uAssets/filters/resource-abuse.txt',
|
||||
// uBlock Origin Unbreak
|
||||
'https://ublockorigin.github.io/uAssets/filters/unbreak.txt',
|
||||
// AdGuard Base Filter
|
||||
'https://filters.adtidy.org/extension/ublock/filters/2_without_easylist.txt',
|
||||
// AdGuard Mobile AD
|
||||
'https://filters.adtidy.org/extension/ublock/filters/11.txt',
|
||||
// AdGuard Tracking Protection
|
||||
'https://filters.adtidy.org/extension/ublock/filters/3.txt',
|
||||
// AdGuard Japanese filter
|
||||
'https://filters.adtidy.org/extension/ublock/filters/7.txt',
|
||||
// AdGuard Chinese filter (EasyList China + AdGuard Chinese filter)
|
||||
'https://filters.adtidy.org/extension/ublock/filters/224.txt',
|
||||
// Easy Privacy
|
||||
'https://easylist.to/easylist/easyprivacy.txt',
|
||||
// Curben's Malware Online UrlHaus
|
||||
'https://curben.gitlab.io/malware-filter/urlhaus-filter-agh-online.txt',
|
||||
// Curben's Phishing Online Filter
|
||||
'https://curben.gitlab.io/malware-filter/phishing-filter-agh.txt',
|
||||
// Curben's PUP List
|
||||
'https://curben.gitlab.io/malware-filter/pup-filter-agh.txt',
|
||||
// GameConsoleAdblockList
|
||||
'https://raw.githubusercontent.com/DandelionSprout/adfilt/master/GameConsoleAdblockList.txt',
|
||||
// PiHoleBlocklist
|
||||
'https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/SmartTV-AGH.txt',
|
||||
].map(processFilterRules))).forEach(({ white, black }) => {
|
||||
white.forEach(i => filterRuleWhitelistDomainSets.add(i));
|
||||
black.forEach(i => domainSets.add(i));
|
||||
});
|
||||
@@ -256,7 +285,7 @@ async function processFilterRules(filterRulesUrl) {
|
||||
});
|
||||
});
|
||||
|
||||
console.log(`Import ${domainKeywordsSet.size} black keywords!`);
|
||||
console.log(`Import ${domainKeywordsSet.size} black keywords and ${domainSuffixSet.size} black suffixes!`);
|
||||
|
||||
const beforeDeduping = domainSets.size;
|
||||
// Dedupe domainSets
|
||||
@@ -267,8 +296,14 @@ async function processFilterRules(filterRulesUrl) {
|
||||
});
|
||||
|
||||
(await Promise.all([
|
||||
piscina.run({ keywords: domainKeywordsSet, suffixes: domainSuffixSet, input: domainSets }, { name: 'dedupeKeywords' }),
|
||||
piscina.run({ whiteList: filterRuleWhitelistDomainSets, input: domainSets }, { name: 'whitelisted' })
|
||||
piscina.run(
|
||||
{ keywords: domainKeywordsSet, suffixes: domainSuffixSet, input: domainSets },
|
||||
{ name: 'dedupeKeywords' }
|
||||
),
|
||||
piscina.run(
|
||||
{ whiteList: filterRuleWhitelistDomainSets, input: domainSets },
|
||||
{ name: 'whitelisted' }
|
||||
)
|
||||
])).forEach(set => {
|
||||
set.forEach(i => domainSets.delete(i));
|
||||
});
|
||||
@@ -276,13 +311,18 @@ async function processFilterRules(filterRulesUrl) {
|
||||
const originalFullSet = new Set([...domainSets]);
|
||||
|
||||
(await Promise.all(
|
||||
Array.from(domainSets).reduce((result, element, index) => {
|
||||
Array.from(domainSets)
|
||||
.reduce((result, element, index) => {
|
||||
const chunk = index % threads;
|
||||
result[chunk] ??= [];
|
||||
|
||||
result[chunk].push(element);
|
||||
return result;
|
||||
}, []).map(chunk => piscina.run({ input: chunk, fullSet: originalFullSet }, { name: 'dedupe' }))
|
||||
}, [])
|
||||
.map(chunk => piscina.run(
|
||||
{ input: chunk, fullSet: originalFullSet },
|
||||
{ name: 'dedupe' }
|
||||
))
|
||||
)).forEach(set => {
|
||||
set.forEach(i => domainSets.delete(i));
|
||||
});
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -629,7 +629,6 @@ analytics.slashdotmedia.com
|
||||
.eviltracker.net
|
||||
.getexceptional.com
|
||||
.jumptap.com
|
||||
.localytics.com
|
||||
.mobile-collector.newrelic.com
|
||||
.playtomic.com
|
||||
.stathat.com
|
||||
@@ -1527,7 +1526,6 @@ gemini.yahoo.com
|
||||
ysm.yahoo.com
|
||||
yads.c.yimg.jp
|
||||
yads.yahoo.co.jp
|
||||
.flurry.com
|
||||
.flurry.cachefly.net
|
||||
|
||||
# >> Disqus
|
||||
@@ -1628,9 +1626,9 @@ stnetsdk.vivo.com.cn
|
||||
.stsdk.vivo.com.cn
|
||||
|
||||
# >> Gionee
|
||||
0.0.0.0 ads.gionee.com
|
||||
0.0.0.0 pdl.gionee.com
|
||||
0.0.0.0 adres.myaora.net
|
||||
ads.gionee.com
|
||||
pdl.gionee.com
|
||||
adres.myaora.net
|
||||
|
||||
# >> 勾正数据
|
||||
.gz-data.com
|
||||
|
||||
Reference in New Issue
Block a user