rebase

pgdb01-cirrus/templates/alert-email.yaml

@@ -0,0 +1,34 @@
+apiVersion: monitoring.coreos.com/v1alpha1
+kind: AlertmanagerConfig
+metadata:
+  name: gdex-app-team
+  namespace: rda
+  labels:
+    alertmanagerConfig: gdex
+    namespace: rda
+spec:
+  route:
+    receiver: gdex-app-team
+    groupBy:
+      - alertname
+    groupWait: 10s
+    groupInterval: 1m
+    repeatInterval: 60m 
+    matchers:
+      - name: namespace
+        value: rda
+        matchType: "="
+    routes:
+      - receiver: "null"
+        matchers:
+          - name: alertname
+            value: InfoInhibitor
+            matchType: "="
+
+  receivers:
+    - name: gdex-app-team
+      emailConfigs:
+        - to: decs-info@ucar.edu
+          from: alertmanager@k8s.ucar.edu
+          smarthost: vdir.ucar.edu:25  
+    - name: "null"

pgdb01-cirrus/templates/alert-rule.yaml

@@ -0,0 +1,48 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: gdex-pg-replication-alerts
+  namespace: rda
+  labels:
+    team: gdex-app-team
+    release: kube-prometheus-stack
+spec:
+  groups:
+  - name: pg.replication
+    interval: 60s
+    rules:
+    - alert: PGReplicationLagHigh
+      expr: |
+        cnpg_pg_replication_lag{namespace="rda"} > 100
+      for: 15m
+      labels:
+        severity: warning
+        team: gdex-app-team
+        namespace: rda
+      annotations:
+        summary: "PostgresDB replication lag high on {{`{{ $labels.pod }}`}}"
+        description: "Replication lag is {{`{{ $value }}`}} WAL segments behind on {{`{{ $labels.pod }}`}} in cluster {{`{{ $labels.cluster }}`}}"
+
+    - alert: PGReplicationBroken
+      expr: |
+        cnpg_pg_replication_streaming_replicas{namespace="rda"} == 0
+      for: 5m
+      labels:
+        severity: critical
+        team: gdex-app-team
+        namespace: rda
+      annotations:
+        summary: "PostgresDB cluster replication broken for {{`{{ $labels.cluster }}`}}"
+        description: "Cluster {{`{{ $labels.cluster }}`}} has no streaming replicas. Replication may be broken."
+
+    - alert: PGClusterNotHealthy
+      expr: |
+        cnpg_collector_up{namespace="rda"} == 0
+      for: 5m
+      labels:
+        severity: critical
+        team: gdex-app-team
+        namespace: rda
+      annotations:
+        summary: "PostgresDB cluster {{`{{ $labels.cluster }}`}} is not healthy"
+        description: "The Postgres exporter for cluster {{`{{ $labels.cluster }}`}} is down, indicating cluster health issues"

pgdb01-cirrus/templates/backups_external_secret.yaml

@@ -0,0 +1,25 @@
+{{- if .Values.db.backups.s3.enabled }}
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: backup-s3-creds
+  namespace: {{ .Release.Namespace }}
+spec:
+  data:
+  - remoteRef:
+      key: {{ .Values.db.backups.s3.secretPath }}
+      property: access_key
+    secretKey: access_key
+  - remoteRef:
+      key: {{ .Values.db.backups.s3.secretPath }}
+      property: secret_key
+    secretKey: secret_key
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: SecretStore
+    name: rda-ro
+  target:
+    creationPolicy: Owner
+    deletionPolicy: Retain
+    name: {{ .Values.db.backups.s3.secretName }}
+{{- end -}} 

pgdb01-cirrus/templates/postgres_cluster.yaml

@@ -10,6 +10,34 @@ spec:
   instances: {{ .Values.db.instances }}
   storage:
     size: {{ .Values.db.size }}
+  resources:
+    limits:
+      cpu: {{ .Values.db.resource.limits.cpu | quote }}
+      memory: {{ .Values.db.resource.limits.memory }}
+
+  {{- if .Values.db.backups }}
+  backup:     
+    {{- if .Values.db.backups.volumeSnapshot }}
+    volumeSnapshot:
+      className: {{ .Values.db.backups.volumeSnapshot.snapshotClassName }}
+    {{- end }}
+    {{- if .Values.db.backups.s3.enabled }}
+    barmanObjectStore:
+      wal:
+        compression: bzip2
+      data:
+        compression: bzip2
+      destinationPath: {{ .Values.db.backups.s3.destinationPath }}
+      endpointURL: {{ .Values.db.backups.s3.endpointURL }}
+      s3Credentials:
+          accessKeyId:
+            name: {{ .Values.db.backups.s3.secretName }}
+            key: access_key
+          secretAccessKey:
+            name: {{ .Values.db.backups.s3.secretName }}
+            key: secret_key
+    {{- end }}
+  {{- end }}
 
   # Add TLS certificates for encrypted communication
   certificates:
@@ -22,19 +50,14 @@ spec:
   # Configure postgres superuser from su_external_secret
   superuserSecret:
     name: "{{ .Values.db.name }}-superuser"
-
-  # Allow outside hosts to connect to the database
+
   postgresql:
     parameters:
       # Connection settings
-      listen_addresses: "*"
-      port: "5432"
       max_connections: "500"
 
       # SSL Configuration
-      ssl: "on"
       ssl_ciphers: "HIGH:!aNULL"
-      ssl_prefer_server_ciphers: "on"
       ssl_min_protocol_version: "TLSv1.3"
 
       # Memory settings
@@ -55,23 +78,16 @@ spec:
       min_wal_size: "1GB"
 
       # Replication settings
-      max_wal_senders: "3"
-      max_replication_slots: "3"
-      wal_keep_size: "256"
-      max_slot_wal_keep_size: "-1"
-      hot_standby: "on"
+      max_wal_senders: "6"
+      max_replication_slots: "6"
+      wal_keep_size: "1024MB"
+      max_slot_wal_keep_size: "100GB"
       max_standby_archive_delay: "-1"
       max_standby_streaming_delay: "-1"
 
       # Logging settings
-      log_destination: "stderr"
       logging_collector: "on"
-      log_directory: "log"
-      log_filename: "postgresql-%Y-%m-%d_%H%M%S.log"
-      log_file_mode: "0644"
       log_rotation_age: "0"
-      log_rotation_size: "1GB"
-      log_truncate_on_rotation: "off"
       log_min_duration_statement: "120000"
       log_line_prefix: "%t %a [%p] "
       log_timezone: "America/Denver"
@@ -87,7 +103,7 @@ spec:
 
       # Lock management
       max_locks_per_transaction: "1024"
-
+      
     pg_hba:
       # Local connections with md5 authentication
       - local all root md5
@@ -107,4 +123,7 @@ spec:
       - host replication all 127.0.0.1/32 md5
 
       # Remote replication
-      - host replication all 128.117.0.0/16 trust
+      - host replication all 128.117.0.0/16 trust
+
+      # Remote replication from Kubernetes pod network
+      - host replication all 10.0.0.0/16 trust

pgdb01-cirrus/templates/su_external_secret.yaml

@@ -1,16 +1,15 @@
-apiVersion: external-secrets.io/v1beta1
+apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
   name: {{ .Values.db.name }}-superuser-esos
   namespace: {{ .Release.Namespace }}
 spec:
   refreshInterval: 1h
   secretStoreRef:
-    name: user-ro
+    name: rda-ro
     kind: SecretStore
   target:
     name: {{ .Values.db.name }}-superuser
-    type: kubernetes.io/basic-auth
   data:
     - secretKey: username
       remoteRef:

pgdb01-cirrus/values.yaml

@@ -1,9 +1,18 @@
 db:
   name: pgdb01
   group: pgdb01
-  instances: 3
-  size: 5000Gi
+  instances: 2
+  size: 8000Gi
   superUser:
     usernameKey: username
     passwordKey: password
-    secretPath: gdex/pgdb01
+    secretPath: gdex/pgdb01
+  resource:
+    limits:
+      cpu: '16'
+      memory: 128Gi
+
+  backups:
+    enabled: false
+    s3:
+      enabled: false

pgdb02-cirrus/templates/alert-email.yaml

@@ -0,0 +1,34 @@
+apiVersion: monitoring.coreos.com/v1alpha1
+kind: AlertmanagerConfig
+metadata:
+  name: gdex-app-team
+  namespace: rda
+  labels:
+    alertmanagerConfig: gdex
+    namespace: rda
+spec:
+  route:
+    receiver: gdex-app-team
+    groupBy:
+      - alertname
+    groupWait: 10s
+    groupInterval: 1m
+    repeatInterval: 60m 
+    matchers:
+      - name: namespace
+        value: rda
+        matchType: "="
+    routes:
+      - receiver: "null"
+        matchers:
+          - name: alertname
+            value: InfoInhibitor
+            matchType: "="
+
+  receivers:
+    - name: gdex-app-team
+      emailConfigs:
+        - to: decs-info@ucar.edu
+          from: alertmanager@k8s.ucar.edu
+          smarthost: vdir.ucar.edu:25
+    - name: "null"

pgdb02-cirrus/templates/alert-rule.yaml

@@ -0,0 +1,48 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: gdex-pg-replication-alerts
+  namespace: rda
+  labels:
+    team: gdex-app-team
+    release: kube-prometheus-stack
+spec:
+  groups:
+  - name: pg.replication
+    interval: 60s
+    rules:
+    - alert: PGReplicationLagHigh
+      expr: |
+        cnpg_pg_replication_lag{namespace="rda"} > 100
+      for: 15m
+      labels:
+        severity: warning
+        team: gdex-app-team
+        namespace: rda
+      annotations:
+        summary: "PostgresDB replication lag high on {{`{{ $labels.pod }}`}}"
+        description: "Replication lag is {{`{{ $value }}`}} WAL segments behind on {{`{{ $labels.pod }}`}} in cluster {{`{{ $labels.cluster }}`}}"
+
+    - alert: PGReplicationBroken
+      expr: |
+        cnpg_pg_replication_streaming_replicas{namespace="rda", cluster="pgdb03"} == 0
+      for: 5m
+      labels:
+        severity: critical
+        team: gdex-app-team
+        namespace: rda
+      annotations:
+        summary: "PostgresDB cluster replication broken for {{`{{ $labels.cluster }}`}}"
+        description: "Cluster {{`{{ $labels.cluster }}`}} has no streaming replicas. Replication may be broken."
+
+    - alert: PGClusterNotHealthy
+      expr: |
+        cnpg_collector_up{namespace="rda"} == 0
+      for: 5m
+      labels:
+        severity: critical
+        team: gdex-app-team
+        namespace: rda
+      annotations:
+        summary: "PostgresDB cluster {{`{{ $labels.cluster }}`}} is not healthy"
+        description: "The Postgres exporter for cluster {{`{{ $labels.cluster }}`}} is down, indicating cluster health issues"

pgdb02-cirrus/templates/backup_compressor.yaml

@@ -0,0 +1,40 @@
+{{- if .Values.db.backups.s3.enabled }}
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: {{ .Values.db.name }}-s3-backup-compressor
+spec:
+  schedule: "0 12 * * *"
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          restartPolicy: OnFailure
+          containers:
+          - name: compressor
+            image: hub.k8s.ucar.edu/khrpcek/backup-compressor:kmh
+            command: ["/multi_compress.sh"]
+            imagePullPolicy: Always
+            env:
+            - name: BASEDIR
+              value: "{{ .Values.db.backups.s3.destinationPath }}/{{ .Values.db.name }}/base/"
+            - name: AWS_ENDPOINT_URL
+              value: "{{ .Values.db.backups.s3.endpointURL }}"
+            - name: AWS_ACCESS_KEY_ID
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.db.backups.s3.secretName }}
+                  key: access_key
+            - name: AWS_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.db.backups.s3.secretName }}
+                  key: secret_key
+            resources:
+              requests:
+                memory: 12Gi
+                cpu: 30
+              limits:
+                memory: 16Gi
+                cpu: 32
+{{- end }}