Skip to content

Instantly share code, notes, and snippets.

Show Gist options
  • Save DerekHawkins/b9641b8205e21de070477ce6d2b4bb65 to your computer and use it in GitHub Desktop.
Save DerekHawkins/b9641b8205e21de070477ce6d2b4bb65 to your computer and use it in GitHub Desktop.
core_web_vitals_analysis_example.py
import pandas as pd
import requests
import urllib
import time
import re
# Data Visualization
from plotly import tools
import chart_studio
chart_studio.tools.set_credentials_file(username='Your Plotly Username', api_key='Your Plotly API Key')
import plotly.graph_objects as go
import chart_studio.plotly as py
#Load PageSpeed API Credentials
page_speed_key = "Your PageSpeed API Key"
check = "captchaResult"
### Load SEMRush API Credentials
api_key = 'Your SEMRush API Key'
rush_service_url = 'https://api.semrush.com'
service_url = 'https://www.googleapis.com/pagespeedonline/v5/runPagespeed\/'
def speed_test(url, device, page_speed_key):
path = '(\\\/)'
params = {
"?url": url,
'strategy': device,
'key': page_speed_key,
}
data = urllib.parse.urlencode(params, doseq=True)
main_call = urllib.parse.urljoin(service_url, data)
main_call = re.sub(path, '', main_call)
main_call = main_call.replace(r'%3F', r'?')
return main_call
def pull_ranking_urls(phrase):
params = {
"?type": "phrase_organic",
'key': api_key,
'phrase': phrase,
'database': 'us',
'display_limit': '10'
}
data = urllib.parse.urlencode(params, doseq=True)
main_call = urllib.parse.urljoin(rush_service_url, data)
main_call = main_call.replace(r'%3F', r'?')
return main_call
def parse_response(call_data):
results = []
data = call_data.decode('unicode_escape')
lines = data.split('\r\n')
lines = list(filter(bool, lines))
columns = lines[0].split(';')
for line in lines[1:]:
result = {}
for i, datum in enumerate(line.split(';')):
result[columns[i]] = datum.strip('"\n\r\t')
results.append(result)
return results
rankings = pd.DataFrame(parse_response(requests.get(pull_ranking_urls(phrase='best summer shoes')).content))
concat_frame = []
for test_url in rankings['Url']:
call = requests.get(speed_test(url=test_url, device='mobile', key=key))
response = call.json()
if check in response:
pass
else:
x = "Error Found with the following URL: %s" % (test_url), ", remove or revise"
frame = []
for i in response['loadingExperience']['metrics']:
rating = ['FAST', 'AVERAGE', 'POOR']
data = {
'Category': i,
'Ranking': response['loadingExperience']['metrics'][i]['category'],
'Pencentile':response['loadingExperience']['metrics'][i]['percentile']
}
for label, num in zip(rating, response['loadingExperience']['metrics'][i]['distributions']):
data[label] = round(num['proportion'], 2)
frame.append(data)
df = pd.DataFrame(frame)
df['Url'] = test_url
concat_frame.append(df)
final = pd.concat(concat_frame)
plot = final[final['Category'].str.contains('LARGEST_CONTENTFUL_PAINT_MS')].reset_index(drop=True)
plot = plot[['Url', 'FAST', 'AVERAGE', 'POOR']]
fig = go.Figure()
fig.add_trace(go.Bar(
y=plot['Url'],
x=plot['FAST'],
name='FAST',
orientation='h',
marker=dict(
color='rgba(2, 217, 70, 0.6)')))
fig.add_trace(go.Bar(
y=plot['Url'],
x=plot['AVERAGE'],
name='AVERAGE',
orientation='h',
marker=dict(
color='rgba(235, 208, 5, 0.6)')
)
)
fig.add_trace(go.Bar(
y=plot['Url'],
x=plot['POOR'],
name='POOR',
orientation='h',
marker=dict(
color='rgba(242, 27, 12, 0.6)')
)
)
fig.update_layout(
title={
'text': 'Core Web Vitals: Top URLs for "Best Summer Shoes"',
'y':0.9,
'x':0.5,
'xanchor': 'center',
'yanchor': 'top'},
barmode='stack',
width=1200,
height=600,
)
py.plot(fig, filename='core_web_vitals')
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment