Skip to content

Instantly share code, notes, and snippets.

@sgraaf

sgraaf/rotatingproxysession.py

Created October 14, 2020 16:46
Show Gist options
  • Save sgraaf/422e07eea04891157dff3cf56d00eb27 to your computer and use it in GitHub Desktop.
Save sgraaf/422e07eea04891157dff3cf56d00eb27 to your computer and use it in GitHub Desktop.
Download ZIP
A requests Session that rotates (free) proxies for GET-requests.
Raw
rotatingproxysession.py
from itertools import cycle
from lxml import html
from requests import Response, Session
class RotatingProxySession(Session):
def __init__(self) -> None:
super().__init__()
r = super().get('https://free-proxy-list.net/')
tree = html.fromstring(r.content)
ip_addresses = tree.xpath('//tbody/tr/td[7][contains(text(),"yes")]/../td[1]/text()')
ports = tree.xpath('//tbody/tr/td[7][contains(text(),"yes")]/../td[2]/text()')
assert len(ip_addresses) == len(ports)
self.proxies_cycle = cycle([f'{ip_address}:{port}' for ip_address, port in zip(ip_addresses, ports)])
def get(self, url, **kwargs) -> Response:
kwargs.pop('proxies', None)
proxy = next(self.proxies_cycle)
return super().get(url, proxies={'http': proxy, 'https': proxy}, **kwargs)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment