From 14fde64f742b68ba703d93822003b35f24f007c2 Mon Sep 17 00:00:00 2001
From: Yann SEGET <yann.seget@actemium.ch>
Date: Mon, 10 Aug 2026 08:17:45 +0200
Subject: [PATCH 1/2] fix(hosting): disable ClickHouse system-log telemetry
 tables and apply profile settings via users.d

ClickHouse's system log tables ship unbounded (no TTL); on the recommended
webapp machine size their background merges eventually exceed the memory cap
and are retried forever, pinning the CPU and failing the webapp's own inserts.
Port the dev stack's disable list (PR #3565) to hosting/docker, keep
query_log/error_log with a config-level TTL, and move the profile settings to
users.d where they actually take effect.

fixes #4343

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 .../hosting-clickhouse-system-logs.md         |  6 +++
 hosting/docker/clickhouse/override.xml        | 48 +++++++++++++++----
 hosting/docker/clickhouse/users-override.xml  | 21 ++++++++
 hosting/docker/webapp/docker-compose.yml      |  1 +
 4 files changed, 67 insertions(+), 9 deletions(-)
 create mode 100644 .server-changes/hosting-clickhouse-system-logs.md
 create mode 100644 hosting/docker/clickhouse/users-override.xml

diff --git a/.server-changes/hosting-clickhouse-system-logs.md b/.server-changes/hosting-clickhouse-system-logs.md
new file mode 100644
index 0000000000..efd4166ae5
--- /dev/null
+++ b/.server-changes/hosting-clickhouse-system-logs.md
@@ -0,0 +1,6 @@
+---
+area: hosting
+type: fix
+---
+
+Self-hosted ClickHouse no longer accumulates unbounded system-log telemetry (metric_log, text_log, asynchronous_metric_log, ...) that eventually pins the CPU in a background merge-retry loop on the recommended machine size; query_log and error_log are kept with a TTL, and the low-memory profile settings now actually apply (moved from config.d to users.d).
diff --git a/hosting/docker/clickhouse/override.xml b/hosting/docker/clickhouse/override.xml
index 41897c2984..f9088054a7 100644
--- a/hosting/docker/clickhouse/override.xml
+++ b/hosting/docker/clickhouse/override.xml
@@ -9,12 +9,42 @@
     <!-- Official recommendations for systems with <16GB RAM -->
     <mark_cache_size>524288000</mark_cache_size> <!-- 500MB -->
     <concurrent_threads_soft_limit_num>1</concurrent_threads_soft_limit_num>
-    <profiles>
-        <default>
-        <max_block_size>8192</max_block_size>
-        <max_download_threads>1</max_download_threads>
-        <input_format_parallel_parsing>0</input_format_parallel_parsing>
-        <output_format_parallel_formatting>0</output_format_parallel_formatting>
-        </default>
-    </profiles>
-</clickhouse>
\ No newline at end of file
+
+    <!-- ClickHouse's own telemetry tables have no TTL and grow without bound; on the
+         recommended webapp machine size their background merges eventually stop fitting
+         in memory and are retried forever (there is no backoff for failed merges),
+         pinning the CPU and ultimately failing the webapp's own inserts (#4343).
+         The ClickHouse low-RAM guide recommends disabling exactly these tables:
+         https://clickhouse.com/docs/operations/tips
+         Same list as the dev stack (docker/config/clickhouse-disable-system-logs.xml),
+         extended with the newer log tables. -->
+    <metric_log remove="1"/>
+    <asynchronous_metric_log remove="1"/>
+    <part_log remove="1"/>
+    <trace_log remove="1"/>
+    <query_thread_log remove="1"/>
+    <session_log remove="1"/>
+    <text_log remove="1"/>
+    <zookeeper_log remove="1"/>
+    <processors_profile_log remove="1"/>
+    <asynchronous_insert_log remove="1"/>
+    <backup_log remove="1"/>
+    <latency_log remove="1"/>
+    <query_metric_log remove="1"/>
+    <opentelemetry_span_log remove="1"/>
+    <query_views_log remove="1"/>
+
+    <!-- Keep query_log and error_log (small and useful for debugging), but bounded.
+         A config-level TTL survives log-table recreation, unlike ALTER ... MODIFY TTL.
+         Note: ClickHouse renames the existing table to *_log_<N> when this changes;
+         drop those leftovers to reclaim disk. -->
+    <query_log>
+        <ttl>event_date + INTERVAL 7 DAY DELETE</ttl>
+    </query_log>
+    <error_log>
+        <ttl>event_date + INTERVAL 30 DAY DELETE</ttl>
+    </error_log>
+
+    <!-- Profile settings do NOT live here: config.d is silently ignored for them.
+         See users-override.xml, mounted under users.d. -->
+</clickhouse>
diff --git a/hosting/docker/clickhouse/users-override.xml b/hosting/docker/clickhouse/users-override.xml
new file mode 100644
index 0000000000..a3964125b8
--- /dev/null
+++ b/hosting/docker/clickhouse/users-override.xml
@@ -0,0 +1,21 @@
+<clickhouse>
+    <profiles>
+        <default>
+            <!-- Low-memory settings for the recommended webapp machine size. These were
+                 previously in config.d/override.xml where profile settings are silently
+                 ignored — they only take effect from the users config tree (#4343). -->
+            <max_block_size>8192</max_block_size>
+            <max_download_threads>1</max_download_threads>
+            <input_format_parallel_parsing>0</input_format_parallel_parsing>
+            <output_format_parallel_formatting>0</output_format_parallel_formatting>
+            <!-- The memory/query profilers sample stacks into system.trace_log
+                 (a Memory/MemoryPeak sample every 4 MB allocated, by default) — the main
+                 firehose that fills it. trace_log is disabled in override.xml; turn the
+                 samplers off too so they don't burn CPU producing discarded rows. -->
+            <memory_profiler_step>0</memory_profiler_step>
+            <memory_profiler_sample_probability>0</memory_profiler_sample_probability>
+            <query_profiler_real_time_period_ns>0</query_profiler_real_time_period_ns>
+            <query_profiler_cpu_time_period_ns>0</query_profiler_cpu_time_period_ns>
+        </default>
+    </profiles>
+</clickhouse>
diff --git a/hosting/docker/webapp/docker-compose.yml b/hosting/docker/webapp/docker-compose.yml
index a8bc1167b7..b893c17eca 100644
--- a/hosting/docker/webapp/docker-compose.yml
+++ b/hosting/docker/webapp/docker-compose.yml
@@ -177,6 +177,7 @@ services:
       - clickhouse:/var/lib/clickhouse
       - ../clickhouse/data-paths.xml:/etc/clickhouse-server/config.d/data-paths.xml:ro
       - ../clickhouse/override.xml:/etc/clickhouse-server/config.d/override.xml:ro
+      - ../clickhouse/users-override.xml:/etc/clickhouse-server/users.d/override.xml:ro
     networks:
       - webapp
     healthcheck:

From 75f4f5946e36e3e6b3933187e1873f8c3b869e2b Mon Sep 17 00:00:00 2001
From: Yann SEGET <yann.seget@actemium.ch>
Date: Mon, 10 Aug 2026 08:48:38 +0200
Subject: [PATCH 2/2] chore: reword .server-changes entry as a user-facing
 release note

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 .server-changes/hosting-clickhouse-system-logs.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.server-changes/hosting-clickhouse-system-logs.md b/.server-changes/hosting-clickhouse-system-logs.md
index efd4166ae5..dc6724232f 100644
--- a/.server-changes/hosting-clickhouse-system-logs.md
+++ b/.server-changes/hosting-clickhouse-system-logs.md
@@ -3,4 +3,4 @@ area: hosting
 type: fix
 ---
 
-Self-hosted ClickHouse no longer accumulates unbounded system-log telemetry (metric_log, text_log, asynchronous_metric_log, ...) that eventually pins the CPU in a background merge-retry loop on the recommended machine size; query_log and error_log are kept with a TTL, and the low-memory profile settings now actually apply (moved from config.d to users.d).
+Self-hosted ClickHouse now disables its unbounded internal telemetry tables and keeps query/error logs on a bounded retention, fixing runaway CPU and memory usage on the recommended machine size. Existing self-hosted deployments must drop the previously written disabled log tables separately to reclaim disk.