fix(monitoring): isolate Talos etcd metrics

clusters/unraid-lab/apps/etcd-metrics-network-policy/application.yaml

@@ -0,0 +1,22 @@
+# The Talos etcd metrics listener is unauthenticated. Only the Grafana metrics
+# collector may initiate connections to control-plane port 2381.
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: etcd-metrics-network-policy
+  namespace: argocd
+spec:
+  project: default
+  source:
+    repoURL: https://github.com/syscode-labs/syscode-homelab-gitops-apps.git
+    targetRevision: HEAD
+    path: clusters/unraid-lab/apps/etcd-metrics-network-policy/manifests
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: monitoring
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true

clusters/unraid-lab/apps/etcd-metrics-network-policy/manifests/policy.yaml

@@ -0,0 +1,23 @@
+apiVersion: cilium.io/v2
+kind: CiliumClusterwideNetworkPolicy
+metadata:
+  name: deny-etcd-metrics-outside-grafana
+spec:
+  description: Deny unauthenticated etcd metrics access from all but Grafana Alloy metrics.
+  endpointSelector:
+    matchExpressions:
+      - key: io.cilium.k8s.policy.serviceaccount
+        operator: NotIn
+        values:
+          - grafana-k8s-monitoring-alloy-metrics
+  # A targeted deny rule must not turn every selected workload into default-deny.
+  enableDefaultDeny:
+    egress: false
+  egressDeny:
+    - toEntities:
+        - host
+        - remote-node
+      toPorts:
+        - ports:
+            - port: "2381"
+          protocol: TCP

omni/scripts/apply-etcd-metrics.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+# Expose unauthenticated etcd metrics only to the cluster's control planes.
+# The companion Cilium policy restricts Pod-to-node traffic to Grafana Alloy.
+set -euo pipefail
+
+CLUSTER="unraid-lab"
+PATCH_ID="520-cluster-${CLUSTER}-etcd-metrics"
+
+temporary="$(mktemp -d)"
+trap 'rm -rf "$temporary"' EXIT
+
+cat >"$temporary/patch.yaml" <<'YAML'
+cluster:
+  etcd:
+    extraArgs:
+      listen-metrics-urls: http://0.0.0.0:2381
+---
+apiVersion: v1alpha1
+kind: NetworkRuleConfig
+name: etcd-metrics
+portSelector:
+  ports:
+    - 2381
+  protocol: tcp
+ingress:
+  - subnet: 192.168.122.109/32
+  - subnet: 192.168.122.190/32
+  - subnet: 192.168.122.214/32
+YAML
+
+python3 - "$temporary/patch.yaml" "$temporary/configpatch.yaml" "$CLUSTER" "$PATCH_ID" <<'PY'
+import sys
+from pathlib import Path
+
+import yaml
+
+
+class Literal(str):
+    pass
+
+
+def represent_literal(dumper, data):
+    return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|")
+
+
+yaml.SafeDumper.add_representer(Literal, represent_literal)
+
+patch_path, configpatch_path, cluster, patch_id = map(Path, sys.argv[1:])
+patch = patch_path.read_text()
+list(yaml.safe_load_all(patch))
+configpatch = {
+    "metadata": {
+        "namespace": "default",
+        "type": "ConfigPatches.omni.sidero.dev",
+        "id": str(patch_id),
+        "labels": {"omni.sidero.dev/cluster": str(cluster)},
+    },
+    "spec": {"data": Literal(patch)},
+}
+configpatch_path.write_text(yaml.safe_dump(configpatch, sort_keys=False))
+print("Talos etcd metrics patch: valid")
+PY
+
+if [[ "${1:-}" == "--apply" ]]; then
+  omnictl apply -f "$temporary/configpatch.yaml"
+  printf 'Etcd metrics ConfigPatch applied. Reboot control-plane nodes one at a time before enabling the Grafana scrape.\n'
+else
+  printf 'Etcd metrics ConfigPatch rendered and validated; re-run with --apply to push to Omni.\n'
+fi

values/base/grafana-k8s-monitoring.yaml

@@ -61,6 +61,25 @@ annotationAutodiscovery:
 collectors:
   alloy-metrics:
     presets: [clustered, statefulset]
+    # Talos does not publish etcd as a Kubernetes Pod, so Grafana's etcd
+    # integration cannot discover it. Scrape the three control-plane listeners
+    # directly; access is limited by the accompanying Talos and Cilium policies.
+    extraConfig: |
+      discovery.static "talos_etcd" {
+        targets = [
+          { "__address__" = "192.168.122.109:2381", "instance" = "unraid-lab-control-planes-gwvq7k", "job" = "integrations/etcd" },
+          { "__address__" = "192.168.122.190:2381", "instance" = "unraid-lab-control-planes-hqf6vh", "job" = "integrations/etcd" },
+          { "__address__" = "192.168.122.214:2381", "instance" = "unraid-lab-control-planes-htvzs7", "job" = "integrations/etcd" },
+        ]
+      }
+
+      prometheus.scrape "talos_etcd" {
+        targets = discovery.static.talos_etcd.targets
+        scrape_interval = "60s"
+        scrape_timeout = "10s"
+        scrape_protocols = ["OpenMetricsText1.0.0", "OpenMetricsText0.0.1", "PrometheusText0.0.4"]
+        forward_to = [prometheus.remote_write.grafana_cloud_metrics.receiver]
+      }
   alloy-singleton:
     presets: [singleton]
 telemetryServices: