Skip to content

Instantly share code, notes, and snippets.

@mpolatcan

mpolatcan/shell_oil_turkey_scraper.py

Last active July 11, 2020 19:37
Show Gist options
  • Save mpolatcan/0e0ce0fbc5eab6ee7e0fe870805e23df to your computer and use it in GitHub Desktop.
Save mpolatcan/0e0ce0fbc5eab6ee7e0fe870805e23df to your computer and use it in GitHub Desktop.
Download ZIP
Shell Oil Turkey oil prices data scraping
Raw
shell_oil_turkey_scraper.py
# Written by Mutlu Polatcan
# 09.07.2020
# =============================================
import requests
import re
from datetime import datetime, timedelta
class ShellOilTurkeyScraper:
URL = "https://www.turkiyeshell.com/pompatest/History.aspx"
KEY_VIEW_STATE = "__VIEWSTATE"
KEY_VIEW_STATE_GENERATOR = "__VIEWSTATEGENERATOR"
KEY_CALLBACK_STATE = "cb_all$grdPrices$CallbackState"
KEY_CALLBACK_ID = "__CALLBACKID"
KEY_CALLBACK_PARAM = "__CALLBACKPARAM"
KEY_EVENT_VALIDATION = "__EVENTVALIDATION"
VIEW_STATE = (
"/wEPDwUJMTk4NDEzNDY5D2QWAgIDD2QWAmYPZBYCZg9kFgJmD2QWAmYPZBYOAgEPPCsABgEADxYCHgVWYWx1ZQYACMe3AR7Y"
"iGRkAgMPFCsABg8WAh8ABgCIQba4ItiIZGRkPCsACgEIPCsABgEAFgIeB01heERhdGUGAIhBtrgi2IhkZGQCBQ8UKwAGDxYE"
"HwAFCFNlw6dpbml6Hg9EYXRhU291cmNlQm91bmRnZGRkPCsACQEIFCsABBYEHhJFbmFibGVDYWxsYmFja01vZGVoHidFbmFi"
"bGVTeW5jaHJvbml6YXRpb25PblBlcmZvcm1DYWxsYmFjayBoZA8WAh4KSXNTYXZlZEFsbGcPFCsAURQrAAEWCB4EVGV4dAUF"
"QURBTkEfAAUDMDAxHghJbWFnZVVybGUeDlJ1bnRpbWVDcmVhdGVkZxQrAAEWCB8GBQhBRElZQU1BTh8ABQMwMDIfB2UfCGcU"
"KwABFggfBgUFQUZZT04fAAUDMDAzHwdlHwhnFCsAARYIHwYFBEFHUkkfAAUDMDA0HwdlHwhnFCsAARYIHwYFBkFNQVNZQR8A"
"BQMwMDUfB2UfCGcUKwABFggfBgUGQU5LQVJBHwAFAzAwNh8HZR8IZxQrAAEWCB8GBQdBTlRBTFlBHwAFAzAwNx8HZR8IZxQr"
"AAEWCB8GBQZBUlRWSU4fAAUDMDA4HwdlHwhnFCsAARYIHwYFBUFZRElOHwAFAzAwOR8HZR8IZxQrAAEWCB8GBQlCQUxJS0VT"
"SVIfAAUDMDEwHwdlHwhnFCsAARYIHwYFB0JJTEVDSUsfAAUDMDExHwdlHwhnFCsAARYIHwYFBkJJTkdPTB8ABQMwMTIfB2Uf"
"CGcUKwABFggfBgUGQklUTElTHwAFAzAxMx8HZR8IZxQrAAEWCB8GBQRCT0xVHwAFAzAxNB8HZR8IZxQrAAEWCB8GBQZCVVJE"
"VVIfAAUDMDE1HwdlHwhnFCsAARYIHwYFBUJVUlNBHwAFAzAxNh8HZR8IZxQrAAEWCB8GBQlDQU5BS0tBTEUfAAUDMDE3Hwdl"
"HwhnFCsAARYIHwYFB0NBTktJUkkfAAUDMDE4HwdlHwhnFCsAARYIHwYFBUNPUlVNHwAFAzAxOR8HZR8IZxQrAAEWCB8GBQdE"
"RU5JWkxJHwAFAzAyMB8HZR8IZxQrAAEWCB8GBQpESVlBUkJBS0lSHwAFAzAyMR8HZR8IZxQrAAEWCB8GBQZFRElSTkUfAAUD"
"MDIyHwdlHwhnFCsAARYIHwYFBkVMQVpJRx8ABQMwMjMfB2UfCGcUKwABFggfBgUIRVJaSU5DQU4fAAUDMDI0HwdlHwhnFCsA"
"ARYIHwYFB0VSWlVSVU0fAAUDMDI1HwdlHwhnFCsAARYIHwYFCUVTS0lTRUhJUh8ABQMwMjYfB2UfCGcUKwABFggfBgUJR0Fa"
"SUFOVEVQHwAFAzAyNx8HZR8IZxQrAAEWCB8GBQdHSVJFU1VOHwAFAzAyOB8HZR8IZxQrAAEWCB8GBQlHVU1VU0hBTkUfAAUD"
"MDI5HwdlHwhnFCsAARYIHwYFB0hBS0tBUkkfAAUDMDMwHwdlHwhnFCsAARYIHwYFBUhBVEFZHwAFAzAzMR8HZR8IZxQrAAEW"
"CB8GBQdJU1BBUlRBHwAFAzAzMh8HZR8IZxQrAAEWCB8GBQZNRVJTSU4fAAUDMDMzHwdlHwhnFCsAARYIHwYFCElTVEFOQlVM"
"HwAFAzAzNB8HZR8IZxQrAAEWCB8GBQVJWk1JUh8ABQMwMzUfB2UfCGcUKwABFggfBgUES0FSUx8ABQMwMzYfB2UfCGcUKwAB"
"FggfBgUJS0FTVEFNT05VHwAFAzAzNx8HZR8IZxQrAAEWCB8GBQdLQVlTRVJJHwAFAzAzOB8HZR8IZxQrAAEWCB8GBQpLSVJL"
"TEFSRUxJHwAFAzAzOR8HZR8IZxQrAAEWCB8GBQhLSVJTRUhJUh8ABQMwNDAfB2UfCGcUKwABFggfBgUHS09DQUVMSR8ABQMw"
"NDEfB2UfCGcUKwABFggfBgUFS09OWUEfAAUDMDQyHwdlHwhnFCsAARYIHwYFB0tVVEFIWUEfAAUDMDQzHwdlHwhnFCsAARYI"
"HwYFB01BTEFUWUEfAAUDMDQ0HwdlHwhnFCsAARYIHwYFBk1BTklTQR8ABQMwNDUfB2UfCGcUKwABFggfBgUHSy5NQVJBUx8A"
"BQMwNDYfB2UfCGcUKwABFggfBgUGTUFSRElOHwAFAzA0Nx8HZR8IZxQrAAEWCB8GBQVNVUdMQR8ABQMwNDgfB2UfCGcUKwAB"
"FggfBgUDTVVTHwAFAzA0OR8HZR8IZxQrAAEWCB8GBQhORVZTRUhJUh8ABQMwNTAfB2UfCGcUKwABFggfBgUFTklHREUfAAUD"
"MDUxHwdlHwhnFCsAARYIHwYFBE9SRFUfAAUDMDUyHwdlHwhnFCsAARYIHwYFBFJJWkUfAAUDMDUzHwdlHwhnFCsAARYIHwYF"
"B1NBS0FSWUEfAAUDMDU0HwdlHwhnFCsAARYIHwYFBlNBTVNVTh8ABQMwNTUfB2UfCGcUKwABFggfBgUFU0lJUlQfAAUDMDU2"
"HwdlHwhnFCsAARYIHwYFBVNJTk9QHwAFAzA1Nx8HZR8IZxQrAAEWCB8GBQVTSVZBUx8ABQMwNTgfB2UfCGcUKwABFggfBgUI"
"VEVLSVJEQUcfAAUDMDU5HwdlHwhnFCsAARYIHwYFBVRPS0FUHwAFAzA2MB8HZR8IZxQrAAEWCB8GBQdUUkFCWk9OHwAFAzA2"
"MR8HZR8IZxQrAAEWCB8GBQdUVU5DRUxJHwAFAzA2Mh8HZR8IZxQrAAEWCB8GBQlTQU5MSVVSRkEfAAUDMDYzHwdlHwhnFCsA"
"ARYIHwYFBFVTQUsfAAUDMDY0HwdlHwhnFCsAARYIHwYFA1ZBTh8ABQMwNjUfB2UfCGcUKwABFggfBgUGWU9aR0FUHwAFAzA2"
"Nh8HZR8IZxQrAAEWCB8GBQlaT05HVUxEQUsfAAUDMDY3HwdlHwhnFCsAARYIHwYFB0FLU0FSQVkfAAUDMDY4HwdlHwhnFCsA"
"ARYIHwYFB0JBWUJVUlQfAAUDMDY5HwdlHwhnFCsAARYIHwYFB0tBUkFNQU4fAAUDMDcwHwdlHwhnFCsAARYIHwYFCUtJUklL"
"S0FMRR8ABQMwNzEfB2UfCGcUKwABFggfBgUGQkFUTUFOHwAFAzA3Mh8HZR8IZxQrAAEWCB8GBQZTSVJOQUsfAAUDMDczHwdl"
"HwhnFCsAARYIHwYFBkJBUlRJTh8ABQMwNzQfB2UfCGcUKwABFggfBgUHQVJEQUhBTh8ABQMwNzUfB2UfCGcUKwABFggfBgUF"
"SUdESVIfAAUDMDc2HwdlHwhnFCsAARYIHwYFBllBTE9WQR8ABQMwNzcfB2UfCGcUKwABFggfBgUHS0FSQUJVSx8ABQMwNzgf"
"B2UfCGcUKwABFggfBgUFS0lMSVMfAAUDMDc5HwdlHwhnFCsAARYIHwYFCE9TTUFOSVlFHwAFAzA4MB8HZR8IZxQrAAEWCB8G"
"BQVEVVpDRR8ABQMwODEfB2UfCGdkZGRkZAIHDxQrAAYPFgIfAAUIU2XDp2luaXpkZGQ8KwAJAQg8KwAEAQAWBB8DaB8EaGRk"
"ZAINDzwrABgCAA8WBB8CZx4VQ2xpZW50VmlzaWJsZUludGVybmFsaGQVPCsABgEFFCsAAmRkZAIPDzwrAAYBAA8WAh8JaGRk"
"AhEPPCsABgEADxYCHwloZGQYAQUeX19Db250cm9sc1JlcXVpcmVQb3N0QmFja0tleV9fFgoFE2NiX2FsbCRkZV9iZGF0ZSRE"
"REQFGWNiX2FsbCRkZV9iZGF0ZSREREQkQyRGTlAFE2NiX2FsbCRkZV9lZGF0ZSREREQFGWNiX2FsbCRkZV9lZGF0ZSREREQk"
"QyRGTlAFFmNiX2FsbCRjYl9wcm92aW5jZSREREQFFGNiX2FsbCRjYl9jb3VudHkkREREBRJjYl9hbGwkQVNQeEJ1dHRvbjEF"
"EGNiX2FsbCRncmRQcmljZXMFEGNiX2FsbCRidG5fcHJpbnQFEWNiX2FsbCRidG5fZXhwb3J0YzhGOh1a7pEBBQhgHhyGOdiu"
"Se9LYQmWZmXmFD0sa0g="
)
VIEW_STATE_GENERATOR = "2280C5C4"
CALLBACK_STATE = (
"BwIHAgIERGF0YQbcBwAAAAAHAAAABwAAAAAAAAAHAAAAAA0AAAANZmNyZWF0aW9uZGF0ZQVUYXJpa"
"AgAAAPEsGwDxLBsBwAABsSwbMOnZQbEsGzDp2UHAAAIZnVlbDAwMDFXSy5CZW56aW4mbmJzcDs5NS"
"ZuYnNwO09rdGFuJm5ic3A7PC9icj4oVEwvTHQpJm5ic3A7Jm5ic3A7PC9icj5TaGVsbCZuYnNwO0Z"
"1ZWxzYXZlJm5ic3A7BQAACGZ1ZWwwMDM3SksuQmVuemluJm5ic3A7OTUmbmJzcDtPa3RhbjwvYnI+"
"KFRML0x0KSZuYnNwOzwvYnI+U2hlbGwmbmJzcDtWLVBvd2VyJm5ic3A7BQAACGZ1ZWwwMDM5SE1vd"
"G9yaW4mbmJzcDs8L2JyPihUTC9MdCkmbmJzcDs8L2JyPlNoZWxsJm5ic3A7RnVlbHNhdmUmbmJzcD"
"s8L2JyPkRpZXNlbAUAAAhmdWVsMDAzNEhNb3RvcmluJm5ic3A7PC9icj4oVEwvTHQpJm5ic3A7PC9"
"icj5TaGVsbCZuYnNwO1YtUG93ZXImbmJzcDtEaWVzZWwmbmJzcDsFAAAIZnVlbDAwMzgxVlAmbmJz"
"cDtEaWVzZWw8L2JyPitHVEw8L2JyPihNb3RvcmluKTwvYnI+KFRML0x0KQUAAAhmdWVsMDA3MRlHY"
"Xo8L2JyPllhZ8SxPC9icj4oVEwvTHQpBQAACGZ1ZWwwMDcyEkthbHlhazwvYnI+KFRML0tnKQUAAA"
"hmdWVsMDA3MzVZw7xrc2VrPC9icj5Lw7xrw7xydGzDvDwvYnI+RnVlbCZuYnNwO09pbDwvYnI+KFR"
"ML0tnKQUAAAhmdWVsMDA3NRlGdWVsJm5ic3A7T2lsPC9icj4oVEwvS2cpBQAACGZ1ZWwwMDA1OE90"
"b2dheiZuYnNwOzwvYnI+KFRML0x0KTwvYnI+U2hlbGwmbmJzcDtBdXRvZ2FzJm5ic3A7TFBHBQAAA"
"AAAAAcABwAHAAcABv//CAIEAEAU3Roe2AgHAgVBREFOQQcCBkNFWUhBTgUDHhkAAAAAAAAAAAAAAA"
"ADAAUDRhkAAAAAAAAAAAAAAAADAAUDPhcAAAAAAAAAAAAAAAADAAUDZhcAAAAAAAAAAAAAAAADAAw"
"FAwwSAAAAAAAAAAAAAAAAAwAFA4gOAAAAAAAAAAAAAAAAAwAFA6AKAAAAAAAAAAAAAAAAAwAFA6IN"
"AAAAAAAAAAAAAAAAAwAMBwAHAAb//wgCBAAAfgfkHtgIBwIFQURBTkEHAgZDRVlIQU4FAx4ZAAAAA"
"AAAAAAAAAAAAwAFA0YZAAAAAAAAAAAAAAAAAwAFAz4XAAAAAAAAAAAAAAAAAwAFA2YXAAAAAAAAAA"
"AAAAAAAwAMBQMMEgAAAAAAAAAAAAAAAAMABQOIDgAAAAAAAAAAAAAAAAMABQOgCgAAAAAAAAAAAAA"
"AAAMABQOiDQAAAAAAAAAAAAAAAAMADAcABwAG//8IAgQAwOcxrR/YCAcCBUFEQU5BBwIGQ0VZSEFO"
"BQMeGQAAAAAAAAAAAAAAAAMABQNGGQAAAAAAAAAAAAAAAAMABQM+FwAAAAAAAAAAAAAAAAMABQNmF"
"wAAAAAAAAAAAAAAAAMADAUDDBIAAAAAAAAAAAAAAAADAAUDiA4AAAAAAAAAAAAAAAADAAUDoAoAAA"
"AAAAAAAAAAAAADAAUDog0AAAAAAAAAAAAAAAADAAwHAAcABv//CAIEAIBRXHYg2AgHAgVBREFOQQc"
"CBkNFWUhBTgUDHhkAAAAAAAAAAAAAAAADAAUDRhkAAAAAAAAAAAAAAAADAAUDPhcAAAAAAAAAAAAA"
"AAADAAUDZhcAAAAAAAAAAAAAAAADAAwFAwwSAAAAAAAAAAAAAAAAAwAFA4gOAAAAAAAAAAAAAAAAA"
"wAFA6AKAAAAAAAAAAAAAAAAAwAFA6INAAAAAAAAAAAAAAAAAwAMBwAHAAb//wgCBABAu4Y/IdgIBw"
"IFQURBTkEHAgZDRVlIQU4FAx4ZAAAAAAAAAAAAAAAAAwAFA0YZAAAAAAAAAAAAAAAAAwAFAz4XAAA"
"AAAAAAAAAAAAAAwAFA2YXAAAAAAAAAAAAAAAAAwAMBQMMEgAAAAAAAAAAAAAAAAMABQOIDgAAAAAA"
"AAAAAAAAAAMABQOgCgAAAAAAAAAAAAAAAAMABQOiDQAAAAAAAAAAAAAAAAMADAcABwAG//8IAgQAA"
"CWxCCLYCAcCBUFEQU5BBwIGQ0VZSEFOBQMeGQAAAAAAAAAAAAAAAAMABQNGGQAAAAAAAAAAAAAAAA"
"MABQOiFwAAAAAAAAAAAAAAAAMABQPKFwAAAAAAAAAAAAAAAAMADAUDZhIAAAAAAAAAAAAAAAADAAU"
"DKA8AAAAAAAAAAAAAAAADAAUDGAsAAAAAAAAAAAAAAAADAAUDOA4AAAAAAAAAAAAAAAADAAwHAAcA"
"Bv//CAIEAMCO29Ei2AgHAgVBREFOQQcCBkNFWUhBTgUDghkAAAAAAAAAAAAAAAADAAUDqhkAAAAAA"
"AAAAAAAAAADAAUDohcAAAAAAAAAAAAAAAADAAUDyhcAAAAAAAAAAAAAAAADAAwFA2YSAAAAAAAAAA"
"AAAAAAAwAFAygPAAAAAAAAAAAAAAAAAwAFAxgLAAAAAAAAAAAAAAAAAwAFAzgOAAAAAAAAAAAAAAA"
"AAwAMAgVTdGF0ZQdgBw0HAAIBBwECAQcCAgEHAwIBBwQCAQcFAgEHBgIBBwcCAQcIAgEHCQIBBwoC"
"AQcLAgEHDAIBBwAHAAcABwACAAUAAACACQIACQIAAgADBwQCAAcAAgEHBwcAAgEHAAcA"
)
CALLBACK_ID = "cb_all"
CALLBACK_PARAM = (
'c0:{{"Action":"OnFilter","Params":{{"bdate":"{begin_date}T21:00:00.000Z","edate":"{end_date}T21:00:00.000Z",'
'"county_code":{county},"province_code":{province}}}}}'
)
EVENT_VALIDATION = (
"/wEdAAeP5rF9s+S3U0xlf4oy0sQGwRlsNRn8/m7V4pjRumy3oNa/Nj1jEAsx1220kosRSDiDo1kbtLLybUhLaI3FVSR"
"IXFuT4BFOc2QbmbHldF+lNeJjSx1SRh59KOmvCw5ASfRvm7bqwTDNIBtyHGFuUI1CeasdjsddPo+EKO0H7jqQtHQMlY"
"MYN7YXKgJa4O4w9Rg="
)
PROVINCE_AND_COUNTIES = {
"ADANA": ("MERKEZ,ALADAG,CEYHAN,FEKE,IMAMOGLU,KARAISALI,KARATAS,KOZAN,POZANTI,SAIMBEYLI,TUFANBEYLI,YUMURTALIK,"
"SEYHAN,YUREGIR,SARICAM,TARSUS").split(","),
"ADIYAMAN": "MERKEZ,BESNI,CELIKHAN,GERGER,GOLBASI,KAHTA,SAMSAT,SINCIK,TUT".split(","),
"AFYON": ("MERKEZ,BASMAKCI,BAYAT,BOLVADIN,CAY,DAZKIRI,DINAR,EMIRDAG,IHSANIYE,ISCEHISAR,SANDIKLI,SINCANLI,SUHUT,"
"SULTANDAGI,COBANLAR,EVCILER,HOCALAR,KIZILOREN,SINANPASA,AKOREN").split(","),
"AGRI": "MERKEZ,DIYADIN,DOGUBEYAZIT,ELESKIRT,HAMUR,PATNOS,TASLICAY,TUTAK".split(","),
"AMASYA": "MERKEZ,GOYNUCEK,GUMUSHACIKOY,MERZIFON,SULUOVA,TASOVA,HAMAMOZU".split(","),
"ANKARA": ("MERKEZ,AYAS,BALA,BEYPAZARI,CAMLIDERE,CUBUK,ELMADAG,GUDUL,HAYMANA,KALECIK,KAZAN,KIZILCAHAMAM,"
"NALLIHAN,POLATLI,S.KOCHISAR,AKYURT,EVREN,ALTINDAG,CANKAYA,ETIMESGUT,GOLBASI,KECIOREN,MAMAK,"
"SINCAN,YENIMAHALLE,PURSAKLAR").split(","),
"ANTALYA": ("MERKEZ,AKSEKI,ALANYA,ELMALI,FINIKE,GAZIPASA,GUNDOGMUS,KALE,KAS,KORKUTELI,KUMLUCA,MANAVGAT,SERIK,"
"IBRADI,KEMER,KONYAALTI,MURATPASA,KEPEZ,AKSU,DOSEMEALTI").split(","),
"ARTVIN": "MERKEZ,ARDANUC,ARHAVI,BORCKA,CAKMAKKAYA,HOPA,MURGUL,SAVSAT,YUSUFELI".split(","),
"AYDIN": ("MERKEZ,BOZDOGAN,BUHARKENT,CINE,GERMENCIK,INCIRLIOVA,KARACASU,KOCARLI,KUSADASI,KUYUCAK,NAZILLI,SOKE,"
"SULTANHISAR,YENIPAZAR,KARPUZLU,KOSK,YENIHISAR,DIDIM,DAVUTLAR").split(","),
"BALIKESIR": ("MERKEZ,AYVALIK,BALYA,BANDIRMA,BIGADIC,BURHANIYE,DURSUNBEY,EDREMIT,ERDEK,GONEN,HAVRAN,IVRINDI,"
"KEPSUT,MANYAS,MARMARAADASI,SAVASTEPE,SINDIRGI,SUSURLUK,GOMEC,ALTINOLUK,KARESI").split(","),
"BILECIK": "MERKEZ,BOZUYUK,GOLPAZARI,OSMANELI,PAZARYERI,SOGUT,YENIPAZAR,INHISAR".split(","),
"BINGOL": "MERKEZ,ADAKLI,GENC,KARLIOVA,KIGI,SOLHAN,YAYLADERE,YEDISU".split(","),
"BITLIS": "MERKEZ,ADILCEVAZ,AHLAT,GUROYMAK,HIZAN,MUTKI,TATVAN".split(","),
"BOLU": "MERKEZ,GEREDE,GOYNUK,KIBRISCIK,MENGEN,MUDURNU,SEBEN,DORTDIVAN,YENICAGA".split(","),
"BURDUR": "MERKEZ,AGLASUN,BUCAK,GOLHISAR,KARAMANLI,KEMER,TEFENNI,YESILOVA,ALTINYAYLA,CAVDIR,CELTIKCI".split(","),
"BURSA": ("MERKEZ,BUYUKORHAN,GEMLIK,HARMANCIK,INEGOL,IZNIK,KARACABEY,KELEZ,M.KEMALPASA,MUDANYA,NILUFER,"
"ORHANELI,ORHANGAZI,YENISEHIR,YILDIRIM,GURSU,KESTEL,OSMANGAZI").split(","),
"CANAKKALE": "MERKEZ,AYVACIK,BAYRAMIC,BIGA,BOZCAADA,CAN,ECEABAT,EZINE,GELIBOLU,GOKCEADA,LAPSEKI,YENICE".split(","),
"CANKIRI": ("MERKEZ,A.KARACALAR,CERKES,ELDIVAN,ILGAZ,KIZILIRMAK,KURSUNLU,ORTA,SABANOZU,YAPRAKLI,BAYRAMOREN,"
"KORGUN").split(","),
"CORUM": ("MERKEZ,ALACA,BAYAT,BOGAZKALE,ISKILIP,KARGI,MECITOZU,ORTAKOY,OSMANCIK,SUNGURLU,UGURLUDAG,DODURGA,"
"OGUZLAR,LACIN").split(","),
"DENIZLI": ("MERKEZEFENDI,MERKEZ,ACIPAYAM,BABADAG,BEKILLI,BULDAN,CAL,CAMELI,CARDAK,CIVRIL,GUNEY,HONAZ,KALE,"
"PAMUKKALE,SARAYKOY,SERINHISAR,TAVAS,BAKLAN,BEYAGAC,BOZKURT,AKKOY").split(","),
"DIYARBAKIR": ("MERKEZ,BISMIL,CERMIK,CINAR,CUNGUS,DICLE,EGIL,ERGANI,HANI,HAZRO,KULP,LICE,SILVAN,KOCAKOY,"
"KAYAPINAR").split(","),
"EDIRNE": "MERKEZ,ENES,HAVSA,IPSALA,KESAN,LALAPASA,MERIC,SULOGLU,UZUNKOPRU".split(","),
"ELAZIG": "MERKEZ,AGIN,ARICAK,BASKIL,KARAKOCAN,KEBAN,KOVANCILAR,MADEN,PALU,SIVRICE,ALACAKAYA".split(","),
"ERZINCAN": "MERKEZ,CAYIRLI,ILIC,KEMAH,KEMALIYE,REFAHIYE,TERCAN,UZUMLU,OTLUKBELI".split(","),
"ERZURUM": ("MERKEZ,ASKALE,CAT,HINIS,HORASAN,ISPIR,KARACOBAN,KARAYAZI,NARMAN,OLTU,OLUR,PASINLER,SENKAYA,TEKMAN,"
"TORTUM,UZUNDERE,ILICA,KOPRUKOY,PAZARYOLU,PALANDOKEN,YAKUTIYE").split(","),
"ESKISEHIR": ("MERKEZ,ALPU,BEYLIKOVA,CIFTELER,INONU,KIRKA,MAHMUDIYE,MIHALICCIK,SARICAKAYA,SEYITGAZI,SIVRIHISAR,"
"GUNYUZU,HAN,MIHALGAZI,ODUNPAZARI,TEPEBASI").split(","),
"GAZIANTEP": ("MERKEZ,ARABAN,ISLAHIYE,NIZIP,OGUZELI,SAHINBEY,YAVUZELI,KARGAMIS,NURDAGI,BASPINAR,"
"SEHITKAMIL").split(","),
"GIRESUN": ("MERKEZ,ALUCRA,BULANCAK,DERELI,ESPIYE,EYNESIL,GORELE,KESAP,PIRAZIZ,S.KARAHISAR,TIREBOLU,YAGLIDERE,"
"CANAKCI,GUCE,CAMOLUK,DOGANKENT").split(","),
"GUMUSHANE": "MERKEZ,KELKIT,KOSE,SIRAN,TORUL,KURTUN".split(","),
"HAKKARI": "MERKEZ,CUKURCA,ESENDERE,SEMDINLI,YUKSEKOVA".split(","),
"HATAY": ("MERKEZ,ALTINOZU,DORTYOL,ERZIN,HASSA,ISKENDERUN,KIRIKHAN,REYHANLI,SAMANDAG,YAYLADAGI,BELEN,KUMLU,"
"ANTAKYA").split(","),
"ISPARTA": ("MERKEZ,AKSU,ATABEY,EGIRDIR,GELENDOST,KECIBORLU,SARKIKARAAGAC,SENIRKENT,SUTCULER,ULUBORLU,YALVAC,"
"GONEN,YENISARBADEMLI").split(","),
"MERSIN": ("MERKEZ,ANAMUR,AYDINCIK,ERDEMLI,GULNAR,MUT,SILIFKE,TARSUS,YENICE,CAMLIYAYLA,BOZYAZI,AKDENIZ,"
"YENISEHIR,TOROSLAR,MEZITLI").split(","),
"ISTANBUL": ("ISTANBUL_ANA,ADALAR,CATALCA,SILE,SILIVRI,ISTANBUL_AVR,TUZLA,AVCILAR,BAGCILAR,BAHCELIEVLER,"
"BAKIRKOY,BAYRAMPASA,BESIKTAS,BEYKOZ,BEYOGLU,EMINONU,ESENLER,EYUP,FATIH,GAZIOSMANPASA,GUNGOREN,"
"KADIKOY,KAGITHANE,KARTAL,KUCUKCEKMECE,MALTEPE,PENDIK,SARIYER,SISLI,UMRANIYE,USKUDAR,ZEYTINBURNU,"
"BUYUKCEKMECE,SULTANBEYLI,ESENYURT,SANCAKTEPE,ATASEHIR,ARNAVUTKOY,CEKMEKOY,BASAKSEHIR,"
"BEYLIKDUZU").split(","),
"IZMIR": ("MERKEZ,ALIAGA,BAYINDIR,BERGAMA,BEYDAG,CESME,DIKILI,FOCA,KARABURUN,KEMALPASA,KINIK,KIRAZ,MENDERES,"
"MENEMEN,ODEMIS,SEFERIHISAR,SELCUK,TIRE,TORBALI,URLA,KARABAGLAR,YAGHANELER,BALCOVA,BORNOVA,BUCA,"
"CIGLI,GAZIEMIR,GUZELBAHCE,KARSIYAKA,KONAK,NARLIDERE").split(","),
"KARS": "MERKEZ,AKYAKA,ARPACAY,DIGOR,KAGIZMAN,SARIKAMIS,SELIM,SUSUZ".split(","),
"KASTAMONU": ("MERKEZ,ABANA,ARAC,AZDAVAY,BOZKURT,CATALZEYTIN,CIDE,DADAY,DEVREKANI,IHSANGAZI,INEBOLU,KURE,"
"PINARBASI,SENPAZAR,TASKOPRU,TOSYA,AGLI,DOGANYURT,HANONU,SEYDILER").split(","),
"KAYSERI": ("MERKEZ,AKKISLA,BUNYAN,DEVELI,FELAHIYE,INCESU,PINARBASI,SARIOGLAN,SARIZ,TALAS,TOMARZA,YAHYALI,"
"YESILHISAR,OZVATAN,HACILAR,KOCASINAN,MELIKGAZI").split(","),
"KIRKLARELI": "MERKEZ,BABAESKI,DEMIRKOY,KOFCAZ,LULEBURGAZ,PEHLIVANKOY,PINARHISAR,VIZE".split(","),
"KIRSEHIR": "MERKEZ,AKPINAR,CICEKDAGI,KAMAN,MUCUR,AKCAKENT,BOZTEPE".split(","),
"KOCAELI": ("MERKEZ,GEBZE,GOLCUK,HEREKE,KANDIRA,KARAMURSEL,KORFEZ,KOSEKOY,DARICA,IZMIT,BASISKELE,DERINCE,"
"CAYIROVA,KARTEPE").split(","),
"KONYA": ("MERKEZ,AKOREN,AKSEHIR,ALTINEKIN,BEYSEHIR,BOZKIR,CIHANBEYLI,CUMRA,DEREBUCAK,DOGANHISAR,HADIM,HUYUK,"
"ILGIN,KADINHANI,KARAPINAR,KONYAEREGLI,KULU,SARAYONU,SEYDISEHIR,TASKENT,YUNAK,AHIRLI,CELTIK,DERBENT,"
"EMIRGAZI,GUNEYSINIR,HALKAPINAR,TUZLUKCU,YALIHUYUK,KARATAY,MERAM,SELCUKLU,SEREFLIKOCHISAR").split(","),
"KUTAHYA": ("MERKEZ,ALTINTAS,ASLANAPA,DOMANIC,DUMLUPINAR,EMET,GEDIZ,HISARCIK,SAPHANE,SIMAV,TAVSANLI,"
"CAVDARHISAR,PAZARLAR").split(","),
"MALATYA": ("MERKEZ,AKCADAG,ARAPKIR,ARGUVAN,BATTALGAZI,DARENDE,DOGANSEHIR,HEKIMHAN,POTURGE,YESILYURT,DOGANYIL,"
"KALE,KULUNCAK,YAZIHAN").split(","),
"MANISA": ("MERKEZ,AHMETLI,AKHISAR,ALASEHIR,DEMIRCI,GOLMARMARA,GORDES,KIRKAGAC,KULA,SALIHLI,SARIGOL,SARUHANLI,"
"SELENDI,SOMA,TURGUTLU,KOPRUBASI").split(","),
"K.MARAS": ("MERKEZ,AFSIN,ANDIRIN,CAGLAYANCERIT,ELBISTAN,GOKSUN,PAZARCIK,TURKOGLU,EKINOZU,NURHAK").split(","),
"MARDIN": "MERKEZ,DARGECIT,DERIK,KIZILTEPE,MAZIDAGI,MIDYAT,NUSAYBIN,OMERLI,SAVUR,YESILLI".split(","),
"MUGLA": "MERKEZ,BODRUM,DALAMAN,DATCA,FETHIYE,KOYCEGIZ,MARMARIS,MILAS,ORTACA,ULA,YATAGAN,KAVAKLIDERE".split(","),
"MUS": "MERKEZ,BULANIK,HASKOY,MALAZGIRT,VARTO,KORKUT".split(","),
"NEVSEHIR": "MERKEZ,ACIGOL,AVANOS,DERINKUYU,GULSEHIR,HACIBEKTAS,KOZAKLI,URGUP,KAYMAKLI".split(","),
"NIGDE": "MERKEZ,BOR,CAMARDI,ULUKISLA,ALTUNHISAR,KEMERHISAR,CIFTLIK".split(","),
"ORDU": ("MERKEZ,AKKUS,AYBASTI,FATSA,GOLKOY,GULYALI,GURGENTEPE,KORGAN,KUMRU,MESUDIYE,PERSEMBE,ULUBEY,UNYE,"
"CAMAS,CATALPINAR,CAYBASI,IKIZCE,KABADUZ,KABATAS").split(","),
"RIZE": ("MERKEZ,ARDESEN,CAMLIHEMSIN,CAYELI,FINDIKLI,GUNEYSU,IKIZDERE,KALKANDERE,PAZAR,DEREPAZARI,HEMSIN,"
"IYIDERE").split(","),
"SAKARYA": ("SERDIVAN,MERKEZ,AKYAZI,ARIFIYE,GEYVE,HENDEK,KARASU,KAYNARCA,KOCAALI,PAMUKOVA,SAPANCA,TARAKLI,"
"FERIZLI,SOGUTLU,KARAPURCEK,ADAPAZARI,ERENLER").split(","),
"SAMSUN": ("MERKEZ,ALACAM,ASARCIK,BAFRA,CARSAMBA,HAVZA,KAVAK,LADIK,ONDOKUZMAYIS,SALIPAZARI,TEKKEKOY,TERME,"
"VEZIRKOPRU,AYVACIK,YAKAKENT,ILKADIM,ATAKUM").split(","),
"SIIRT": "MERKEZ,BAYKAN,ERUH,KURTALAN,PERVARI,SIRVAN,AYDINLAR".split(","),
"SINOP": "MERKEZ,AYANCIK,BOYABAT,DURAGAN,ERFELEK,GERZE,TURKELI,DIKMEN,SARAYDUZU".split(","),
"SIVAS": ("MERKEZ,ALTINYAYLA,DIVRIGI,GEMEREK,GURUN,HAFIK,IMRANLI,KANGAL,KOYULHISAR,SARKISLA,SUSEHRI,YILDIZELI,"
"ZARA,AKINCILAR,DOGANSAR,GULOVA,ULAS").split(","),
"TEKIRDAG": "MERKEZ,CERKEZKOY,CORLU,HAYRABOLU,MALKARA,MARMARAEREGLI,MURATLI,SARAY,SARKOY".split(","),
"TOKAT": "MERKEZ,ALMUS,ARTOVA,ERBAA,NIKSAR,PAZAR,RESADIYE,TURHAL,YESILYURT,ZILE,BASCIFTLIK,SULUSARAY".split(","),
"TRABZON": ("MERKEZ,AKCAABAT,ARAKLI,ARSIN,BESIKDUZU,CAYKARA,MACKA,OF,SALIPAZARI,SURMENE,TONYA,VAKFIKEBIR,"
"YOMRA,CARSIBASI,DERNEKPAZARI,DUZKOY,HAYRAT,KOPRUBASI").split(","),
"TUNCELI": "MERKEZ,CEMISKEZEK,HOZAT,MAZGIRT,NAZIMIYE,OVACIK,PERTEK,PULUMUR".split(","),
"SANLIURFA": "MERKEZ,AKCAKALE,BIRECIK,BOZOVA,CEYLANPINAR,HALFETI,HARRAN,HILVAN,SIVEREK,SURUC,VIRANSEHIR".split(","),
"USAK": "MERKEZ,BANAZ,ESME,KARAHALLI,SIVASLI,ULUBEY".split(","),
"VAN": "MERKEZ,BAHCESARAY,BASKALE,CALDIRAN,CATAK,ERCIS,GEVAS,GURPINAR,MURADIYE,OZALP,EDREMIT,SARAY,IPEKYOLU".split(","),
"YOZGAT": ("MERKEZ,AKDAGMADENI,BOGAZLIYAN,CAYIRALAN,CEKEREK,SARIKAYA,SEFAATLI,SORGUN,YERKOY,AYDINCIK,CANDIR,"
"KADISEHRI,SARAYKENT,YENIFAKILI").split(","),
"ZONGULDAK": "MERKEZ,ALAPLI,CAYCUMA,DEVREK,KARAD.EREGLI,GOKCEBEY".split(","),
"AKSARAY": "MERKEZ,GUZELYURT,ORTAKOY,SARIYAGSI,ESKIL,GULAGAC".split(","),
"BAYBURT": "MERKEZ,AYDINTEPE,DEMIROZU".split(","),
"KARAMAN": "MERKEZ,AYRANCI,ERMENEK,K.KARABEKIR,BASYAYLA,SARIVELILER".split(","),
"KIRIKKALE": "MERKEZ,DELICE,KESKIN,SULAKYURT,BAHSILI,BALISEYH,CELEBI,KARAKECILI,YAHSIHAN".split(","),
"BATMAN": "MERKEZ,GERCUS,BESIRI,KOZLUK,SASON,HASANKEYF".split(","),
"SIRNAK": 'MERKEZ,BEYTULSEBAB,ULUDERE,CIZRE,IDIL,SILOPI,GUCLUKONAK'.split(","),
"BARTIN": "MERKEZ,AMASRA,KURUCASILE,ULUS".split(","),
"ARDAHAN": "MERKEZ,CILDIR,DAMAL,GOLE,HANAK,POSOF".split(","),
"IGDIR": "MERKEZ,ARALIK,KARAKOYUNLU,TUZLUCA".split(","),
"YALOVA": "MERKEZ,ALTINOVA,ARMUTLU,CINARCIK,CIFTLIKKOY,TERMAL".split(","),
"KARABUK": "MERKEZ,EFLANI,ESKIPAZAR,OVACIK,SAFRANBOLU,YENICE,GEREDE".split(","),
"KILIS": "MERKEZ,ELBEYLI,MUSABEYLI,POLATELI".split(","),
"OSMANIYE": "MERKEZ,BAHCE,DUZICI,TOPRAKKALE,SUMBAS,HASANBEYLI,KADIRLI".split(","),
"DUZCE": "MERKEZ,AKCAKOCA,CUMAYERI,CILIMLI,GOLYAKA,GUMUSOVA,KAYNASLI,YIGILCA".split(",")
}
PROVINCE_AND_COUNTY_CODES = {
province: {
"id": "{}{}".format("".join(["0" for x in range(3 - len(str(province_id)))]), str(province_id)),
"counties": {
county: "{}{}{}{}".format("".join(["0" for y in range(3 - len(str(province_id)))]), str(province_id),
"".join(["0" for z in range(3 - len(str(county_id)))]), str(county_id))
for county_id, county in enumerate(counties, start=1)
}
}
for province_id, (province, counties) in enumerate(PROVINCE_AND_COUNTIES.items(), start=1)
}
@staticmethod
def make_request(begin_date, end_date, province, county=None):
begin_date = (datetime.strptime(begin_date, "%Y-%m-%d") - timedelta(days=1)).date()
end_date = (datetime.strptime(end_date, "%Y-%m-%d") - timedelta(days=1)).date()
province = province.upper()
if county:
county = county.upper()
response = requests.post(
url=ShellOilTurkeyScraper.URL,
headers={"Content-type": "application/x-www-form-urlencoded; charset=UTF-8"},
data={
ShellOilTurkeyScraper.KEY_VIEW_STATE: ShellOilTurkeyScraper.VIEW_STATE,
ShellOilTurkeyScraper.KEY_VIEW_STATE_GENERATOR: ShellOilTurkeyScraper.VIEW_STATE_GENERATOR,
ShellOilTurkeyScraper.KEY_CALLBACK_STATE: ShellOilTurkeyScraper.CALLBACK_STATE,
ShellOilTurkeyScraper.KEY_CALLBACK_ID: ShellOilTurkeyScraper.CALLBACK_ID,
ShellOilTurkeyScraper.KEY_CALLBACK_PARAM: ShellOilTurkeyScraper.CALLBACK_PARAM.format(
begin_date=begin_date,
end_date=end_date,
province="\"{}\"".format(ShellOilTurkeyScraper.PROVINCE_AND_COUNTY_CODES[province]["id"]),
county="\"{}\"".format(ShellOilTurkeyScraper.PROVINCE_AND_COUNTY_CODES[province]["counties"][county])
if county else "null"
),
ShellOilTurkeyScraper.KEY_EVENT_VALIDATION: ShellOilTurkeyScraper.EVENT_VALIDATION
}
).content.decode("utf-8")
columns = [
" ".join(column.replace("&nbsp", " ").replace(";", "").replace("</br>", " ").split())
for column in re.findall("<td style=\"color:#333333;.*?\">(.*?)</td>", response)
]
data = re.findall(r"<td class=\"ClassTxt1 dxgv\" .*?>(.*?)</td>", response)
if len(data) > 0:
data = [
data[batch_id * len(columns):(batch_id + 1) * len(columns)]
for batch_id in range(int(len(data) / len(columns)))
]
return columns, data
@staticmethod
def json(begin_date, end_date, province, county=None):
columns, data = ShellOilTurkeyScraper.make_request(begin_date, end_date, province, county)
return [dict(zip(columns, row)) for row in data]
@staticmethod
def csv(begin_date, end_date, province, county=None):
columns, data = ShellOilTurkeyScraper.make_request(begin_date, end_date, province, county)
return "\n".join([",".join(columns)] + [",".join(row) for row in data])
if __name__ == "__main__":
print(
ShellOilTurkeyScraper.csv(
begin_date="2020-07-04",
end_date="2020-07-11",
province="istanbul",
county="umraniye"
)
)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment