Skip to content

Instantly share code, notes, and snippets.

@davetroy
Created August 29, 2012 19:32
Show Gist options
  • Save davetroy/3517643 to your computer and use it in GitHub Desktop.
Save davetroy/3517643 to your computer and use it in GitHub Desktop.
Capture tweets for a given lat/lon bounding box
#!/usr/bin/env ruby
$: << File.dirname(__FILE__)
require 'rubygems'
require 'eventmachine'
require 'em-http'
require 'yajl'
require 'yajl/json_gem'
class Streamer
def initialize
Signal.trap("INT") { EM.stop }
Signal.trap("TERM") { EM.stop }
EventMachine.run {
Hose.new
}
puts "shutting down!"
end
end
class Hose
PARSER = Yajl::Parser.new
CREDS = ["TWITTER_USERNAME", "TWITTER_PASSWORD"]
DATADIR = "/home/twittervision/baltimore"
def initialize
@http = EventMachine::HttpRequest.new('https://stream.twitter.com/1/statuses/filter.json',
:connection_timeout => 0, :inactivity_timeout => 0).post(
:head => {'authorization' => CREDS},
:body => {:locations => '-76.75,39.18,-76.43,39.40'}
)
PARSER.on_parse_complete = proc { |h| process(h) }
@http.stream { |json| PARSER << json }
@http.errback { |e| restart }
@http.callback { |c| restart }
end
def process(s)
if s['geo']
lat = s['geo']['coordinates'][0]
lon = s['geo']['coordinates'][1]
elsif s['coordinates']
lat = s['coordinates']['coordinates'][1]
lon = s['coordinates']['coordinates'][0]
elsif s['place']
c = s['place']['bounding_box']['coordinates'][0]
lat = (c[0][1] + c[2][1])/2.0
lon = (c[0][0] + c[1][0])/2.0
end
s['lat'] = lat
s['lon'] = lon
return unless lat && lon && (lat>=39.18 && lon>=-76.75) && (lat<=39.40 && lon<=76.43)
ll = "#{lat},#{lon}"
http2 = EventMachine::HttpRequest.new("http://maps.googleapis.com/maps/api/geocode/json?latlng=#{ll}\&sensor=false").get
http2.callback { display(s, http2.response) }
end
def datestring
Time.now.strftime("%Y%m%d")
end
def display(s,c)
begin
addr = JSON.parse(c)
s['formatted_address'] = addr['results'].first['formatted_address']
rescue
s['formatted_address'] = "[#{s['lat']}, #{s['lon']}]"
end
ts = Time.now
s['timestamp'] = ts.to_i
File.open("#{DATADIR}/#{datestring}.txt", "a") do |f|
f.write "#{s['user']['screen_name']}: #{s['text']}\n#{s['formatted_address']} (#{ts.strftime('%H:%M:%S')})\n\n"
end
File.open("#{DATADIR}/#{datestring}.json", "a") do |f|
f.write s.to_json + "\n"
end
end
def restart
puts "restarting"
Hose.new
end
end
Streamer.new
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment