deploy: c2f9456e32cfcaeaa359c4b761d27a7648ef4570

This commit is contained in:
SukkaBot
2022-07-04 16:08:10 +00:00
parent 0788462b22
commit 811e3eaa8f
4 changed files with 333 additions and 15 deletions

View File

@@ -3,14 +3,15 @@ const { workerData } = require('piscina');
exports.dedupe = ({ chunk }) => {
const outputToBeRemoved = new Set();
for (const domainFromInput of chunk) {
for (let i = 0, l = chunk.length; i < l; i++) {
const domainFromInput = chunk[i];
for (const domainFromFullSet of workerData) {
if (domainFromFullSet === domainFromInput) continue;
if (domainFromFullSet.charAt(0) !== '.') continue;
if (
`.${domainFromInput}` === domainFromFullSet
|| domainFromInput.endsWith(domainFromFullSet)
// `.${domainFromInput}` === domainFromFullSet
domainFromInput.endsWith(domainFromFullSet)
) {
outputToBeRemoved.add(domainFromInput);
break;