deploy: 39f3dacf6e0a1459f4f46ccbd1677148a85ab07c

This commit is contained in:
SukkaBot
2022-08-30 18:31:52 +00:00
parent 50f8a9860e
commit 5ce683b9c2
4 changed files with 862 additions and 50 deletions

View File

@@ -43,39 +43,3 @@ exports.dedupe = ({ chunk }) => {
return outputToBeRemoved;
};
exports.whitelisted = ({ whiteList }) => {
const outputToBeRemoved = new Set();
for (const domain of workerData) {
for (const white of whiteList) {
if (domain.includes(white) || white.includes(domain)) {
outputToBeRemoved.add(domain);
break;
}
}
}
return outputToBeRemoved;
};
exports.dedupeKeywords = ({ keywords, suffixes }) => {
const outputToBeRemoved = new Set();
for (const domain of workerData) {
for (const keyword of keywords) {
if (domain.includes(keyword) || keyword.includes(domain)) {
outputToBeRemoved.add(domain);
break;
}
}
for (const suffix of suffixes) {
if (domain.endsWith(suffix)) {
outputToBeRemoved.add(domain);
break;
}
}
}
return outputToBeRemoved;
}