Created
September 22, 2017 05:25
-
-
Save ijharulislam/a922d92123c27582e3c046bac3d3f385 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import requests | |
import socket | |
import socks | |
import csv | |
import sqlite3 | |
import datetime | |
from scrapy.http import HtmlResponse | |
socks.set_default_proxy(socks.SOCKS5, "proxy.torguard.io", 1090, username="", password="") | |
socket.socket = socks.socksocket | |
data = [] | |
# Test Proxy | |
proxy = requests.get("https://icanhazip.com") | |
print("Current IP:", proxy.content) | |
# Get the content | |
r = requests.get("https://coinmarketcap.com/gainers-losers/#gainers-1h") | |
response = HtmlResponse(url="My URL", body=r.text, encoding='utf-8') | |
coins = response.xpath('//*[@id="gainers-1h"]/div/table/tbody//tr') | |
for coin in coins: | |
output = {} | |
output["Name"] = coin.xpath('td[2]/a/text()').extract_first() | |
output["Symbol"] = coin.xpath("td[3]/text()").extract_first() | |
output["Volume"] = coin.xpath('td[4]/a/text()').extract_first() | |
output["Pirce"] = coin.xpath('td[5]/a/text()').extract_first() | |
output["1h"] = coin.xpath("td[6]/text()").extract_first() | |
print(output) | |
data.append(output) | |
# Writing CSV | |
with open('CoinInfo.csv', 'w') as f: | |
dict_writer = csv.DictWriter(f, data[0].keys()) | |
dict_writer.writeheader() | |
dict_writer.writerows(data) | |
def insert_into_db(): | |
now = '{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()) | |
conn = sqlite3.connect('coininfo.db') | |
c = conn.cursor() | |
c.execute('''CREATE TABLE IF NOT EXISTS coininfo | |
(name text, symbol text, volume text, price text, h1 text, created_on text)''') | |
for d in data: | |
c.execute("""INSERT INTO coininfo VALUES (?,?,?,?,?,?)""", (d["Name"], d["Symbol"], d["Volume"], d["Pirce"], d["1h"], now,)) | |
conn.commit() | |
# Test Data Insertion | |
for row in c.execute('SELECT * FROM coininfo ORDER BY volume'): | |
print ("ROW:", row) | |
conn.close() | |
insert_into_db() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment