Import script

This commit is contained in:
shinigami-eyes 2018-09-16 12:20:54 +02:00
parent 458ba9de27
commit 920c7d1208
1 changed files with 16 additions and 0 deletions

16
import-data.js Normal file
View File

@ -0,0 +1,16 @@
var bloomfilter = require("./extension/bloomfilter.js");
var fs = require('fs');
function importFile(txt, name) {
var b = new bloomfilter.BloomFilter(/*bits*/ 359440, 17);
for (var line of fs.readFileSync(txt).toString().split('\n')) {
line = line.trim();
if (line) b.add(line);
}
var bucketsAsBytes = new Uint8Array(b.buckets.buffer);
fs.writeFileSync("extension/data/" + name + ".dat", new Buffer(bucketsAsBytes));
}
importFile("C:\\ML\\facebook\\transphobic.txt", "transphobic");
importFile("C:\\ML\\facebook\\t-friendly.txt", "t-friendly");