Skip to content

Instantly share code, notes, and snippets.

@costa92

costa92/ipip-free-database-to-csv.js

Created March 14, 2017 02:12
Show Gist options
  • Save costa92/cea0d24c2d73c14e61611721aa13afe0 to your computer and use it in GitHub Desktop.
Save costa92/cea0d24c2d73c14e61611721aa13afe0 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
ipip-free-database-to-csv.js
"use strict";
/**
* @Description 将 http://www.ipip.net/download.html 里的免费版 IP 地址数据库导出为 CSV（逗号分隔）格式的纯文本文件，可用于导入到其他数据库（如 sqlite3、postgresql 等）中
* @Author Colin Cheng<zbinlin@outlook.com>
*/
const fs = require("fs");
const DAT = "./17monipdb.dat";
const fd = fs.openSync(DAT, "r");
const offsetBuffer = Buffer.alloc(4);
fs.readSync(fd, offsetBuffer, 0, 4, 0);
const offset = offsetBuffer.readUInt32BE(0);
const startOffset = 4;
const maxCompLength = offset - 1024 - 4;
const startIndex = 1 * 4 + startOffset;
const startBuffer = Buffer.alloc(4);
fs.readSync(fd, startBuffer, 0, 4, startIndex);
const start = startBuffer.readUInt32LE() * 8 + 1024;
function toInt(str) {
return str.split(".").reverse().reduce((num, part, idx) => {
num |= part << (idx * 8);
return num;
}, 0);
}
function toStr(num) {
return [num >>> 24 & 0xFF, num >>> 16 & 0xFF, num >>> 8 & 0xFF, num & 0xFF].join(".");
}
let startIp = toInt("1.0.0.0");
const ary = [];
for (let i = start; i < maxCompLength; i += 8) {
const buf = Buffer.alloc(8);
fs.readSync(fd, buf, 0, 8, i + startOffset);
const endIp = buf.readUInt32BE();
const addrOffset = Buffer.from([...buf.slice(4, 7), 0x00]).readUInt32LE();
const addrLength = buf[7];
const addr = Buffer.alloc(addrLength);
fs.readSync(fd, addr, 0, addrLength, offset + addrOffset - 1024);
ary.push([
toStr(startIp),
toStr(endIp),
addr.toString("utf8").split("\t"),
]);
startIp = endIp + 1;
}
fs.closeSync(fd);
fs.writeFileSync("out.csv", ary.join("\n"));
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment