Created
January 5, 2013 11:40
-
-
Save hippietrail/4461158 to your computer and use it in GitHub Desktop.
My attempt at the minimal code to read text files one line at a time in node.js
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
var fs = require('fs'), | |
StringDecoder = require('string_decoder').StringDecoder, | |
util = require('util'); | |
function lineByLine(fd) { | |
var blob = ''; | |
var blobStart = 0; | |
var blobEnd = 0; | |
var decoder = new StringDecoder('utf8'); | |
var CHUNK_SIZE = 16384; | |
var chunk = new Buffer(CHUNK_SIZE); | |
var eolPos = -1; | |
var lastChunk = false; | |
var moreLines = true; | |
var readMore = true; | |
// each line | |
while (moreLines) { | |
readMore = true; | |
// append more chunks from the file onto the end of our blob of text until we have an EOL or EOF | |
while (readMore) { | |
// do we have a whole line? (with LF) | |
eolPos = blob.indexOf('\n', blobStart); | |
if (eolPos !== -1) { | |
blobEnd = eolPos; | |
readMore = false; | |
// do we have the last line? (no LF) | |
} else if (lastChunk) { | |
blobEnd = blob.length; | |
readMore = false; | |
// otherwise read more | |
} else { | |
var bytesRead = fs.readSync(fd, chunk, 0, CHUNK_SIZE, null); | |
lastChunk = bytesRead !== CHUNK_SIZE; | |
// TODO blob grows until memory is exhausted! | |
blob += decoder.write(chunk.slice(0, bytesRead)); | |
} | |
} | |
if (blobStart < blob.length) { | |
processLine(blob.substring(blobStart, blobEnd + 1)); | |
blobStart = blobEnd + 1; | |
if (blobStart >= CHUNK_SIZE) { | |
// blobStart is in characters, CHUNK_SIZE is in octets | |
var freeable = blobStart / CHUNK_SIZE; | |
blob = blob.substring(CHUNK_SIZE); | |
blobStart -= CHUNK_SIZE; | |
blobEnd -= CHUNK_SIZE; | |
} | |
} else { | |
moreLines = false; | |
} | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment