-
-
Save Benny-Git/0a73a280116fc61f8e0a21e9d6e62d8d to your computer and use it in GitHub Desktop.
Scrape your ecowater data. https://gnulnx.net/2020/02/18/ecowater-api-scraping/
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/python3 | |
import requests | |
import json | |
import re | |
import paho.mqtt.client as mqtt | |
import paho.mqtt.publish as publish | |
# Regex to match the hidden input on the initial log in page | |
request_validation_re = re.compile(r'<input name="__RequestVerificationToken" type="hidden" value="([^"]*)" />') | |
# The serial number of your ecowater device | |
dsn = { "dsn": 'serialnumber' } | |
# The initial form data | |
payload = { | |
"Email" : "username", | |
"Password" : "password", | |
"Remember" : 'false' | |
} | |
# The headers needed for the JSON request | |
headers = { | |
'Accept': '*/*', | |
#'Accept-Encoding': 'gzip, deflate, br', | |
'Accept-Language' : 'en-US,en;q=0.5', | |
'X-Requested-With': 'XMLHttpRequest', | |
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', | |
'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:73.0) Gecko/20100101 Firefox/73.0' | |
} | |
# MQTT server details | |
mqtt_host = '127.0.0.1' | |
mqtt_port = 1883 | |
mqtt_user = 'username' | |
mqtt_pass = 'password' | |
def querySite(): | |
with requests.Session() as s: | |
# Initial GET request | |
try: | |
g = s.get('https://wifi.ecowater.com/Site/Login') | |
except requests.exceptions.RequestException as e: | |
print(f"Problem connecting to ecowater.com: {e}") | |
raise | |
# Grab the token from the hidden input | |
tokens = request_validation_re.findall(g.text) | |
payload['__RequestVerificationToken'] = tokens[0] | |
# Log in to the site | |
try: | |
login = s.post('https://wifi.ecowater.com/Site/Login', data=payload) | |
except requests.exceptions.RequestException as e: | |
print(f"Problem logging in to ecowater.com: {e}") | |
raise | |
# Add the correct Referer header | |
headers['Referer'] = login.url + '/' + dsn['dsn'] | |
# Query the JSON endpoint for the data that we actually want | |
try: | |
data = s.post('https://wifi.ecowater.com/Dashboard/UpdateFrequentData', data=dsn, headers=headers) | |
except requests.exceptions.RequestException as e: | |
print(f"Problem getting JSON from ecowater.com: {e}") | |
raise | |
if data.status_code != 200: | |
print("Status code from ecowater.com was not 200") | |
raise ValueError("Status code from ecowater.com was not 200") | |
parseData(data.text) | |
def parseData(text): | |
# Load the data in to json | |
j = json.loads(text) | |
# Ensure at least one of the returned values makes sense (sanity check) | |
# All values were "0" once or twice. | |
assert j['water_flow'] >= 0, "Water flow does not appear to be a real number." | |
# Placeholder for each message | |
messages = [] | |
# Extract the next recharge date, into something usable. | |
# False if we match 'Not Scheduled', else True | |
# Only sets this topic if we have the initial regex match. | |
nextRecharge_re = "device-info-nextRecharge'\)\.html\('(?P<nextRecharge>.*)'" | |
nextRecharge_result = re.search(nextRecharge_re, j['recharge']) | |
if nextRecharge_result: | |
msg = { | |
'topic': 'ecowater/rechargeTomorrow', | |
'retain': True, | |
'payload': False if nextRecharge_result.group('nextRecharge') == 'Not Scheduled' else True | |
} | |
messages.append(msg) | |
# Delete the elements that we don't want | |
del j['out_of_salt'] | |
del j['out_of_salt_days'] | |
del j['water_units'] | |
del j['time'] | |
del j['recharge'] | |
# Format each piece of json data in to a mqtt 'message' | |
for d in j: | |
msg = { | |
'topic': 'ecowater/' + d, | |
'retain': True, | |
'payload': j[d] | |
} | |
messages.append(msg) | |
publishMessages(messages) | |
def publishMessages(messages): | |
# Publish the message, consisting of all of the json data | |
if len(messages) > 0: | |
publish.multiple(messages, hostname=mqtt_host, port=mqtt_port, auth={'username': mqtt_user, 'password': mqtt_pass}) | |
else: | |
print('Messages is empty - nothing to publish.') | |
if __name__ == '__main__': | |
querySite() |
Hi,
I have exactly the same error since 4 days.
The content of the result of
data = s.post('https://wifi.ecowater.com/Dashboard/UpdateFrequentData', data=dsn, headers=headers)
is unreadable. Any help would be grateful.
Thanks,
Nico
Hi Nico,
you have to comment off the line 26. ( 'Accept-Encoding': 'gzip, deflate, br',)
then it works again...
regards
Oliver
Hi Oliver,
Thanks for your quick answer ! It works again...
Have a nice day.
Nico
Thanks for the updates. The URL and line 26 need to be changed indeed.
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Hi,
it seems, that its not working any more. I just copied your code and tried it, but got a few errors.
First i think, that the url has changed to wifi.ecowater instead of www.wifi.ecowater.
But i also got following error with the json:
Traceback (most recent call last):
File "/home/pi/ecowater_test.py", line 133, in
querySite()
File "/home/pi/ecowater_test.py", line 77, in querySite
parseData(data.text)
File "/home/pi/ecowater_test.py", line 82, in parseData
j = json.loads(text)
File "/usr/local/lib/python3.9/json/init.py", line 346, in loads
return _default_decoder.decode(s)
File "/usr/local/lib/python3.9/json/decoder.py", line 340, in decode
raise JSONDecodeError("Extra data", s, end)
json.decoder.JSONDecodeError: Extra data: line 1 column 2 (char 1)
Could you please check, if your code is running and what's happened, that it's not working anymore?
Thanks
Oliver