Last active
February 8, 2022 03:10
-
-
Save wlensinas/af799bcd204c8e7e7e2ed4e2b4557993 to your computer and use it in GitHub Desktop.
This is a simple scraper for get a list of crypto currency prices
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import requests | |
import concurrent.futures | |
from bs4 import BeautifulSoup | |
from pprint import pprint | |
from re import sub | |
sites = [ | |
('Bitcoin', 'https://coinmarketcap.com/currencies/bitcoin/'), | |
('Ethereum', 'https://coinmarketcap.com/currencies/ethereum/'), | |
('BNB', 'https://coinmarketcap.com/currencies/bnb/'), | |
('Cardano', 'https://coinmarketcap.com/currencies/cardano/'), | |
('XRP', 'https://coinmarketcap.com/currencies/xrp/'), | |
('Solana', 'https://coinmarketcap.com/currencies/solana/'), | |
('Terra (Luna)', 'https://coinmarketcap.com/currencies/terra-luna/'), | |
] | |
def sort_crypto_price(crypto_data): | |
return sorted(crypto_data, key= lambda k:k['price'], reverse=True) | |
def get_request(crypto_url, crypto_name): | |
res = requests.get(crypto_url) | |
return (crypto_name, res.text) | |
def get_data(crypto_sites): | |
with concurrent.futures.ThreadPoolExecutor() as executor: | |
futures = [] | |
cripto_data = [] | |
for cs in crypto_sites: | |
futures.append(executor.submit(get_request, crypto_url=cs[1], crypto_name=cs[0])) | |
for future in concurrent.futures.as_completed(futures): | |
cripto_data.append(future.result()) | |
return cripto_data | |
def convert_str_to_decimal(price: str) -> float: | |
return float(sub(r'[^\d.]', '', price)) | |
def get_crypto_data(crypto_data): | |
values = [] | |
CRYPTO_NAME = 0 | |
TEXT_DATA = 1 | |
for cd in crypto_data: | |
soup = BeautifulSoup(cd[TEXT_DATA], 'html.parser') | |
price = soup.select('.priceValue') | |
values.append( | |
{'cripto': cd[CRYPTO_NAME], | |
'price': convert_str_to_decimal(price[0].find('span').getText())}) | |
return values | |
data_sites = get_data(sites) | |
sorted_data = sort_crypto_price(get_crypto_data(data_sites)) | |
pprint(sorted_data) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
beautifulsoup4==4.10.0 | |
certifi==2021.10.8 | |
charset-normalizer==2.0.11 | |
idna==3.3 | |
requests==2.27.1 | |
soupsieve==2.3.1 | |
urllib3==1.26.8 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Steps to use this code:
pip install -r requirements.txt
python3 mini_crypto_scraper.py
Output:
If you have a suggest, please write below :)