diff --git a/clusters/unraid-lab/apps/etcd-metrics-network-policy/application.yaml b/clusters/unraid-lab/apps/etcd-metrics-network-policy/application.yaml new file mode 100644 index 0000000..e1b5aa2 --- /dev/null +++ b/clusters/unraid-lab/apps/etcd-metrics-network-policy/application.yaml @@ -0,0 +1,22 @@ +# The Talos etcd metrics listener is unauthenticated. Only the Grafana metrics +# collector may initiate connections to control-plane port 2381. +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: etcd-metrics-network-policy + namespace: argocd +spec: + project: default + source: + repoURL: https://github.com/syscode-labs/syscode-homelab-gitops-apps.git + targetRevision: HEAD + path: clusters/unraid-lab/apps/etcd-metrics-network-policy/manifests + destination: + server: https://kubernetes.default.svc + namespace: monitoring + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true diff --git a/clusters/unraid-lab/apps/etcd-metrics-network-policy/manifests/policy.yaml b/clusters/unraid-lab/apps/etcd-metrics-network-policy/manifests/policy.yaml new file mode 100644 index 0000000..189ae0e --- /dev/null +++ b/clusters/unraid-lab/apps/etcd-metrics-network-policy/manifests/policy.yaml @@ -0,0 +1,23 @@ +apiVersion: cilium.io/v2 +kind: CiliumClusterwideNetworkPolicy +metadata: + name: deny-etcd-metrics-outside-grafana +spec: + description: Deny unauthenticated etcd metrics access from all but Grafana Alloy metrics. + endpointSelector: + matchExpressions: + - key: io.cilium.k8s.policy.serviceaccount + operator: NotIn + values: + - grafana-k8s-monitoring-alloy-metrics + # A targeted deny rule must not turn every selected workload into default-deny. + enableDefaultDeny: + egress: false + egressDeny: + - toEntities: + - host + - remote-node + toPorts: + - ports: + - port: "2381" + protocol: TCP diff --git a/omni/scripts/apply-etcd-metrics.sh b/omni/scripts/apply-etcd-metrics.sh new file mode 100755 index 0000000..1353bf2 --- /dev/null +++ b/omni/scripts/apply-etcd-metrics.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Expose unauthenticated etcd metrics only to the cluster's control planes. +# The companion Cilium policy restricts Pod-to-node traffic to Grafana Alloy. +set -euo pipefail + +CLUSTER="unraid-lab" +PATCH_ID="520-cluster-${CLUSTER}-etcd-metrics" + +temporary="$(mktemp -d)" +trap 'rm -rf "$temporary"' EXIT + +cat >"$temporary/patch.yaml" <<'YAML' +cluster: + etcd: + extraArgs: + listen-metrics-urls: http://0.0.0.0:2381 +--- +apiVersion: v1alpha1 +kind: NetworkRuleConfig +name: etcd-metrics +portSelector: + ports: + - 2381 + protocol: tcp +ingress: + - subnet: 192.168.122.109/32 + - subnet: 192.168.122.190/32 + - subnet: 192.168.122.214/32 +YAML + +python3 - "$temporary/patch.yaml" "$temporary/configpatch.yaml" "$CLUSTER" "$PATCH_ID" <<'PY' +import sys +from pathlib import Path + +import yaml + + +class Literal(str): + pass + + +def represent_literal(dumper, data): + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|") + + +yaml.SafeDumper.add_representer(Literal, represent_literal) + +patch_path, configpatch_path, cluster, patch_id = map(Path, sys.argv[1:]) +patch = patch_path.read_text() +list(yaml.safe_load_all(patch)) +configpatch = { + "metadata": { + "namespace": "default", + "type": "ConfigPatches.omni.sidero.dev", + "id": str(patch_id), + "labels": {"omni.sidero.dev/cluster": str(cluster)}, + }, + "spec": {"data": Literal(patch)}, +} +configpatch_path.write_text(yaml.safe_dump(configpatch, sort_keys=False)) +print("Talos etcd metrics patch: valid") +PY + +if [[ "${1:-}" == "--apply" ]]; then + omnictl apply -f "$temporary/configpatch.yaml" + printf 'Etcd metrics ConfigPatch applied. Reboot control-plane nodes one at a time before enabling the Grafana scrape.\n' +else + printf 'Etcd metrics ConfigPatch rendered and validated; re-run with --apply to push to Omni.\n' +fi diff --git a/values/base/grafana-k8s-monitoring.yaml b/values/base/grafana-k8s-monitoring.yaml index 8736630..4eda317 100644 --- a/values/base/grafana-k8s-monitoring.yaml +++ b/values/base/grafana-k8s-monitoring.yaml @@ -61,6 +61,25 @@ annotationAutodiscovery: collectors: alloy-metrics: presets: [clustered, statefulset] + # Talos does not publish etcd as a Kubernetes Pod, so Grafana's etcd + # integration cannot discover it. Scrape the three control-plane listeners + # directly; access is limited by the accompanying Talos and Cilium policies. + extraConfig: | + discovery.static "talos_etcd" { + targets = [ + { "__address__" = "192.168.122.109:2381", "instance" = "unraid-lab-control-planes-gwvq7k", "job" = "integrations/etcd" }, + { "__address__" = "192.168.122.190:2381", "instance" = "unraid-lab-control-planes-hqf6vh", "job" = "integrations/etcd" }, + { "__address__" = "192.168.122.214:2381", "instance" = "unraid-lab-control-planes-htvzs7", "job" = "integrations/etcd" }, + ] + } + + prometheus.scrape "talos_etcd" { + targets = discovery.static.talos_etcd.targets + scrape_interval = "60s" + scrape_timeout = "10s" + scrape_protocols = ["OpenMetricsText1.0.0", "OpenMetricsText0.0.1", "PrometheusText0.0.4"] + forward_to = [prometheus.remote_write.grafana_cloud_metrics.receiver] + } alloy-singleton: presets: [singleton] telemetryServices: