EMQX e4.4.17 --> Prometheus --> CloudWatch

Tested on a stock Ubuntu 22.04 AMI.

EMQX

sudo apt update
sudo apt install ./emqx-ee-4.4.17-otp24.3.4.2-1-ubuntu22.04-amd64.deb
sudo systemctl restart emqx
curl -f "127.0.0.1:8081/api/v4/emqx_prometheus?type=prometheus"

Prometheus

cat > prometheus.yaml << EOF
global:
  scrape_interval: 5s
  scrape_timeout: 1s
scrape_configs:
- job_name: emqx
  sample_limit: 10000
  metrics_path: "/api/v4/emqx_prometheus"
  params:
    type: ["prometheus"]
  honor_labels: true
  static_configs:
    - targets:
      - "127.0.0.1:8081"
EOF
sudo mkdir /etc/prometheus
sudo mv prometheus.yaml /etc/prometheus
wget https://github.com/prometheus/prometheus/releases/download/v2.37.7/prometheus-2.37.7.linux-amd64.tar.gz
tar xfz prometheus-2.37.7.linux-amd64.tar.gz
sudo mv prometheus-2.37.7.linux-amd64/prometheus /usr/local/bin/
sudo useradd --no-create-home --shell /bin/false prometheus
sudo mkdir /var/lib/prometheus
sudo chown prometheus:prometheus /var/lib/prometheus
cat > prometheus.service << EOF
[Unit]
Description=Prometheus
After=network-online.target
Requires=local-fs.target
After=local-fs.target

[Service]
Type=simple
User=prometheus
Group=prometheus
ExecReload=/bin/kill -HUP $MAINPID
ExecStart=/usr/local/bin/prometheus --storage.tsdb.path=/var/lib/prometheus --config.file=/etc/prometheus/prometheus.yaml
SyslogIdentifier=prometheus
Restart=always

[Install]
WantedBy=multi-user.target
EOF
sudo mv prometheus.service /lib/systemd/system/
sudo systemctl daemon-reload
sudo systemctl restart prometheus
sudo apt install jq
curl http://localhost:9090/api/v1/targets/metadata | jq .

CloudWatch

wget https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb
sudo apt install ./amazon-cloudwatch-agent.deb
cat > cwagent-config.json <<EOF
{
   "agent": {
      "region": "eu-west-1"
   },
   "logs":{
      "metrics_collected":{
         "prometheus":{
            "cluster_name":"prometheus-cluster",
            "log_group_name":"Prometheus",
            "prometheus_config_path":"/etc/prometheus/prometheus.yaml",
            "emf_processor":{
               "metric_declaration_dedup":true,
               "metric_namespace":"CWAgent-Prometheus",
               "metric_unit":{
                  "emqx_messages_qos0_received": "Count",
                  "emqx_session_created": "Count",
                  "erlang_vm_statistics_bytes_received_total": "Count"
               },
               "metric_declaration":[
                  {
                     "source_labels":["job"],
                     "label_matcher":"emqx",
                     "dimensions": [["instance"]],
                     "metric_selectors":[
                        "^emqx_messages_qos0_received$",
                        "^emqx_session_created$",
                        "^erlang_vm_statistics_bytes_received_total$"
                     ]
                  }
               ]
            }
         }
      }
   }
}
EOF
sudo mv cwagent-config.json /opt/aws/amazon-cloudwatch-agent/var/
sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:/opt/aws/amazon-cloudwatch-agent/var/cwagent-config.json