Last active
March 26, 2018 23:40
-
-
Save leodc/a3545aa2d08802af4e7f763c19056075 to your computer and use it in GitHub Desktop.
script para generar compilados de los datos abiertos de conagua: https://datos.gob.mx/busca/dataset/precipitacion-actual-y-acumulada-por-estacion
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/python3 | |
| #coding=utf-8 | |
| import json | |
| import time | |
| from xmljson import parker, Parker | |
| from xml.etree.ElementTree import fromstring | |
| from suds.client import Client | |
| from datetime import timedelta, date | |
| # Y / M / D | |
| start_date = date(2003, 1, 1) | |
| end_date = date(2008, 10, 19) | |
| # end_date = date(2018, 2, 26) | |
| delta = timedelta(days=1) | |
| client = Client(url="https://correo1.conagua.gob.mx/google/Google.asmx?WSDL") | |
| # get geographic info | |
| print("Obteniendo información geografica de las estaciones") | |
| locationDict = {} | |
| for location in json.loads(client.service.CatalogoEstaciones()): | |
| locationDict[ location["Estacion"] ] = { | |
| "nombre": location["Nombre"], | |
| "latitud": location["Latitud"], | |
| "longitud": location["Longitud"] | |
| } | |
| print("-- Done") | |
| readMode = "w+" | |
| with open("conagua_compilado_test.csv", mode=readMode, encoding="UTF-8") as compilado: | |
| # print headers | |
| headers = ["estacion", "fecha", "precipitacion", "temperatura_obs", "latitud", "longitud", "nombre"] | |
| if readMode == "w+": | |
| compilado.write(",".join(headers) + "\n") | |
| d = end_date | |
| while d >= start_date: | |
| try: | |
| search_date = d.strftime("%Y/%m/%d") | |
| print("searching ..." + search_date) | |
| # parsing temperatura values | |
| temperaturaDiariaResponse = client.service.TemperaturaDiariaGrupo(search_date) | |
| temperaturaJSONResponse = parker.data(fromstring(temperaturaDiariaResponse), preserve_root=True) | |
| temperaturaDict = {} | |
| for grupoEstacion in temperaturaJSONResponse["TemperaturaDiaria"]["GrupoEstacion"]: | |
| temperaturaDict[ grupoEstacion["Estacion"] ] = grupoEstacion["tobs"] | |
| # get precipitacion and print to file | |
| precipitacionDiariaResponse = client.service.PrecipitacionDiariaGrupo(search_date) | |
| for medicion in json.loads(precipitacionDiariaResponse): | |
| estacion = medicion["Estacion"] | |
| line = "{estacion},{fecha},{precipitacion},{temperatura},{latitud},{longitud},\"{nombre}\"\n" | |
| temperatura = temperaturaDict[estacion] if estacion in temperaturaDict else "" | |
| latitud = locationDict[estacion]["latitud"] if locationDict[estacion]["latitud"] is not None or locationDict[estacion]["latitud"] is not None else "" | |
| longitud = locationDict[estacion]["longitud"] if locationDict[estacion]["longitud"] is not None or locationDict[estacion]["longitud"] is not None else "" | |
| nombre = locationDict[estacion]["nombre"] if locationDict[estacion]["nombre"] is not None or locationDict[estacion]["nombre"] is not None else "" | |
| compilado.write(line.format(estacion=estacion, fecha=search_date, precipitacion=medicion["Prec"], temperatura=temperatura, latitud=latitud, longitud=longitud, nombre=nombre )) | |
| d -= delta | |
| except: | |
| print("Error obteniendo los datos, reintentando...") | |
| finally: | |
| time.sleep(.100) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment