Skip to content

Instantly share code, notes, and snippets.

@rickhull
Last active September 2, 2024 16:17
Show Gist options
  • Save rickhull/895b0cb38fdd537c1078a858cf15d63e to your computer and use it in GitHub Desktop.
Save rickhull/895b0cb38fdd537c1078a858cf15d63e to your computer and use it in GitHub Desktop.
Prometheus, Grafana, Loki, Promtail on NixOS
# MONITORING: services run on loopback interface
# nginx reverse proxy exposes services to network
# - grafana:3010
# - prometheus:3020
# - loki:3030
# - promtail:3031
# prometheus: port 3020 (8020)
#
services.prometheus = {
port = 3020;
enable = true;
exporters = {
node = {
port = 3021;
enabledCollectors = [ "systemd" ];
enable = true;
};
};
# ingest the published nodes
scrapeConfigs = [{
job_name = "nodes";
static_configs = [{
targets = [
"127.0.0.1:${toString config.services.prometheus.exporters.node.port}"
];
}];
}];
};
# loki: port 3030 (8030)
#
services.loki = {
enable = true;
configuration = {
server.http_listen_port = 3030;
auth_enabled = false;
ingester = {
lifecycler = {
address = "127.0.0.1";
ring = {
kvstore = {
store = "inmemory";
};
replication_factor = 1;
};
};
chunk_idle_period = "1h";
max_chunk_age = "1h";
chunk_target_size = 999999;
chunk_retain_period = "30s";
max_transfer_retries = 0;
};
schema_config = {
configs = [{
from = "2022-06-06";
store = "boltdb-shipper";
object_store = "filesystem";
schema = "v11";
index = {
prefix = "index_";
period = "24h";
};
}];
};
storage_config = {
boltdb_shipper = {
active_index_directory = "/var/lib/loki/boltdb-shipper-active";
cache_location = "/var/lib/loki/boltdb-shipper-cache";
cache_ttl = "24h";
shared_store = "filesystem";
};
filesystem = {
directory = "/var/lib/loki/chunks";
};
};
limits_config = {
reject_old_samples = true;
reject_old_samples_max_age = "168h";
};
chunk_store_config = {
max_look_back_period = "0s";
};
table_manager = {
retention_deletes_enabled = false;
retention_period = "0s";
};
compactor = {
working_directory = "/var/lib/loki";
shared_store = "filesystem";
compactor_ring = {
kvstore = {
store = "inmemory";
};
};
};
};
# user, group, dataDir, extraFlags, (configFile)
};
# promtail: port 3031 (8031)
#
services.promtail = {
enable = true;
configuration = {
server = {
http_listen_port = 3031;
grpc_listen_port = 0;
};
positions = {
filename = "/tmp/positions.yaml";
};
clients = [{
url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
}];
scrape_configs = [{
job_name = "journal";
journal = {
max_age = "12h";
labels = {
job = "systemd-journal";
host = "pihole";
};
};
relabel_configs = [{
source_labels = [ "__journal__systemd_unit" ];
target_label = "unit";
}];
}];
};
# extraFlags
};
# grafana: port 3010 (8010)
#
services.grafana = {
port = 3010;
# WARNING: this should match nginx setup!
# prevents "Request origin is not authorized"
rootUrl = "http://192.168.1.10:8010"; # helps with nginx / ws / live
protocol = "http";
addr = "127.0.0.1";
analytics.reporting.enable = false;
enable = true;
provision = {
enable = true;
datasources = [
{
name = "Prometheus";
type = "prometheus";
access = "proxy";
url = "http://127.0.0.1:${toString config.services.prometheus.port}";
}
{
name = "Loki";
type = "loki";
access = "proxy";
url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}";
}
];
};
};
# nginx reverse proxy
services.nginx = {
enable = true;
recommendedProxySettings = true;
recommendedOptimisation = true;
recommendedGzipSettings = true;
# recommendedTlsSettings = true;
upstreams = {
"grafana" = {
servers = {
"127.0.0.1:${toString config.services.grafana.port}" = {};
};
};
"prometheus" = {
servers = {
"127.0.0.1:${toString config.services.prometheus.port}" = {};
};
};
"loki" = {
servers = {
"127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}" = {};
};
};
"promtail" = {
servers = {
"127.0.0.1:${toString config.services.promtail.configuration.server.http_listen_port}" = {};
};
};
};
virtualHosts.grafana = {
locations."/" = {
proxyPass = "http://grafana";
proxyWebsockets = true;
};
listen = [{
addr = "192.168.1.10";
port = 8010;
}];
};
virtualHosts.prometheus = {
locations."/".proxyPass = "http://prometheus";
listen = [{
addr = "192.168.1.10";
port = 8020;
}];
};
# confirm with http://192.168.1.10:8030/loki/api/v1/status/buildinfo
# (or) /config /metrics /ready
virtualHosts.loki = {
locations."/".proxyPass = "http://loki";
listen = [{
addr = "192.168.1.10";
port = 8030;
}];
};
virtualHosts.promtail = {
locations."/".proxyPass = "http://promtail";
listen = [{
addr = "192.168.1.10";
port = 8031;
}];
};
};
@IncredibleLaser
Copy link

Thanks for the configuration. I was in the process of doing this piece by piece and this helped a lot.

Small remark though, the current latest schema for loki is v12. So if you're setting this up from scratch, you might want to change it. Personally, I have no clue about the difference between the versions. You can have multiple schemas active depending on the date the data is associated with: https://grafana.com/docs/loki/latest/operations/storage/schema/

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment