Last active
January 9, 2023 22:17
-
-
Save hermes-pimentel/996a50993bb351136c0a83ed21152ac5 to your computer and use it in GitHub Desktop.
msk-servicediscovery-http.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import json | |
| import boto3 | |
| from functools import reduce | |
| import logging | |
| logger = logging.getLogger() | |
| logger.setLevel(logging.INFO) | |
| client = boto3.client('kafka') | |
| def list_brokers(cluster_arn): | |
| """ | |
| Gets the list of brokers for a cluster. | |
| """ | |
| logger.info("Listing Brokers and generating targets") | |
| brokers = client.list_nodes(ClusterArn=cluster_arn) | |
| broker_list = list(map(lambda x: f"{x['BrokerNodeInfo']['Endpoints'][0]}:11001", brokers['NodeInfoList'])) | |
| broker_list += list(map(lambda x: f"{x['BrokerNodeInfo']['Endpoints'][0]}:11002", brokers['NodeInfoList'])) | |
| return broker_list | |
| def generate_targets(): | |
| """ | |
| Generates a list of targets for monitoring. | |
| """ | |
| # Gets the list of provisioned clusters using the SDK pagination | |
| logger.info("Getting MSK Cluster List with Prometheus JMX Enabled") | |
| clusters = client.get_paginator('list_clusters_v2').paginate(ClusterTypeFilter='PROVISIONED') | |
| # Filters the list of clusters to include only those that are enabled for monitoring | |
| enabled_clusters = reduce( | |
| lambda x, y: x + y, | |
| [ | |
| cluster_page['ClusterInfoList'] | |
| for cluster_page in clusters | |
| if all( | |
| cluster_info['State'] not in ['DELETING', 'CREATING'] | |
| and cluster_info['Provisioned']['OpenMonitoring']['Prometheus']['JmxExporter']['EnabledInBroker'] == True | |
| for cluster_info in cluster_page['ClusterInfoList'] | |
| ) | |
| ] | |
| ) | |
| json_doc = [ | |
| { | |
| 'labels': { | |
| 'ClusterArn': cluster_info['ClusterArn'], | |
| 'ClusterName': cluster_info['ClusterName'], | |
| 'job': f"msk-{cluster_info['ClusterName']}" | |
| }, | |
| 'targets': list_brokers(cluster_info['ClusterArn']) | |
| } | |
| for cluster_info in enabled_clusters | |
| ] | |
| return json_doc | |
| def lambda_handler(event, context): | |
| logger.info("Starting Lambda Function") | |
| try: | |
| return generate_targets() | |
| except Exception as e: | |
| logger.exception("Error...") | |
| raise e |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment