Created
August 15, 2014 15:40
-
-
Save Mackaber/61fe339d3da5dc02e669 to your computer and use it in GitHub Desktop.
Http bot example
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Gets the first 10 results of the query without using google API | |
# And then prints them as a JSON | |
# Usage google.rb [your search] | |
require "net/https" | |
require "uri" | |
require "nokogiri" | |
require "json" | |
q = URI::encode(ARGV[0]) | |
uri = URI.parse("https://google.com") | |
http = Net::HTTP.new(uri.host, uri.port) | |
http.use_ssl = true | |
http.verify_mode = OpenSSL::SSL::VERIFY_NONE | |
path = "/search?output=search&sclient=psy-ab&q=#{q}" | |
headers = { | |
'accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', | |
# 'accept-encoding' => 'gzip,deflate,sdch', | |
'accept-language' => 'es-ES,es;q=0.8,en;q=0.6', | |
'cookie' => '', | |
'referer' => '', | |
'user-agent' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36', | |
'x-client-data' => '' | |
} | |
response = http.get path, headers | |
#Follow the redirect | |
if response.code == "301" || response.code == "302" | |
response = http.get response['Location'] | |
end | |
document = Nokogiri::HTML(response.body) | |
results_list = [] | |
# Force Encoding! force_encoding("ISO-8859-1").encode("UTF-8") | |
document.css('li.g').each do |result| | |
results_list.push({ | |
:title => result.css('h3.r').css('a').text.force_encoding("ISO-8859-1").encode("UTF-8"), | |
:link => result.css('h3.r').css('a').first["href"].force_encoding("ISO-8859-1").encode("UTF-8"), | |
:content => result.css('span.st').text.force_encoding("ISO-8859-1").encode("UTF-8") | |
}) | |
end | |
puts results_list.to_json |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment