Created
December 8, 2013 04:06
-
-
Save wwwy3y3/7853228 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// population of all cities over 1 million people using the cities1000 dataset from http://geonames.org | |
var fs = require('fs'); | |
var cities = fs.createReadStream(__dirname + '/cities1000.txt'); | |
var split = require('split'); | |
var through = require('through'); | |
var fields = require('cities1000').fields; | |
var popIndex = fields.indexOf('population'); | |
var sum = 0; | |
cities.pipe(split()).pipe(through(write, end)) | |
function write (line) { | |
if (line === '') return; | |
var row = line.split('\t'); | |
var n = Number(row[popIndex]); | |
if (n >= 1000 * 1000) { | |
sum += n; | |
process.stdout.write(sum + ' \r'); | |
} | |
} | |
function end () { | |
console.log('\nTOTAL: ' + sum); | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// san francisco city lot streaming json parser | |
var JSONStream = require('JSONStream'); | |
var fs = require('fs'); | |
var parser = JSONStream.parse([ 'features', true ]); | |
parser.on('data', function (row) { | |
console.log( | |
row.properties.TO_ST, | |
row.properties.FROM_ST, | |
row.properties.STREET, | |
row.geometry.coordinates | |
); | |
}); | |
fs.createReadStream('citylots.json').pipe(parser); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// scrape the memory alpha Data page for chronology data | |
var request = require('request'); | |
var trumpet = require('trumpet'); | |
var concat = require('concat-stream'); | |
var tr = trumpet(); | |
var data = {}; | |
var latest; | |
tr.selectAll('dt a', function (dt) { | |
dt.createReadStream().pipe(concat(function (body) { | |
latest = body.toString('utf8'); | |
})); | |
}); | |
tr.selectAll('dd', function (dd) { | |
dd.createReadStream().pipe(concat(function (body) { | |
data[latest] = body.toString('utf8').trim(); | |
})); | |
}); | |
tr.on('end', function () { | |
console.log(JSON.stringify(data, null, 2)); | |
}); | |
request('http://en.memory-alpha.org/wiki/Data').pipe(tr); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment