Created
September 5, 2024 23:19
-
-
Save vncsna/f1beef72b146089500d95c68af9942fc to your computer and use it in GitHub Desktop.
Save aws logs to postgresql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import subprocess | |
| import psycopg2 | |
| from psycopg2 import sql | |
| conn = psycopg2.connect(dsn="postgresql://localhost:5432/vncsna") | |
| cursor = conn.cursor() | |
| cursor.execute( | |
| """ | |
| CREATE TABLE IF NOT EXISTS cloudwatch_logs ( | |
| timestamp TIMESTAMP, | |
| log_group TEXT, | |
| log_entry TEXT | |
| ) | |
| """ | |
| ) | |
| conn.commit() | |
| def process_log_entry(timestamp, log_group, log_entry): | |
| cursor.execute( | |
| "INSERT INTO cloudwatch_logs (timestamp, log_group, log_entry) VALUES (%s, %s, %s)", | |
| (timestamp, log_group, log_entry) | |
| ) | |
| conn.commit() | |
| def stream_logs_to_postgres(log_group): | |
| process = subprocess.Popen( | |
| ['aws', 'logs', 'tail', log_group, '--follow', '--output', 'json'], | |
| stdout=subprocess.PIPE, | |
| stderr=subprocess.PIPE, | |
| universal_newlines=True | |
| ) | |
| for line in process.stdout: | |
| try: | |
| timestamp, log_group, log_entry = line.strip().split(maxsplit=2) | |
| process_log_entry(timestamp, log_group, log_entry) | |
| except Exception as e: | |
| print(f"Error processing log entry: {e}") | |
| continue | |
| if __name__ == "__main__": | |
| stream_logs_to_postgres("<log-group-name>") | |
| conn.close() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment