From 9b90a90b89ba24060e83fedf0551040a6cac0918 Mon Sep 17 00:00:00 2001 From: Priyesh Karatha Date: Tue, 19 May 2026 09:28:22 +0530 Subject: [PATCH 1/2] HDDS-15312. Improve VolumeInfoMetrics to include MinFreeSpace and Non-Ozone used space --- .../common/volume/VolumeInfoMetrics.java | 18 +++++++++++++----- .../common/volume/TestVolumeInfoMetrics.java | 9 +++++++-- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfoMetrics.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfoMetrics.java index 0cb0c9d56a98..32fdaf49e757 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfoMetrics.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfoMetrics.java @@ -51,14 +51,20 @@ public class VolumeInfoMetrics implements MetricsSource { Interns.info("OzoneUsed", "Ozone used space"); private static final MetricsInfo RESERVED = Interns.info("Reserved", "Reserved Space"); - private static final MetricsInfo TOTAL_CAPACITY = - Interns.info("TotalCapacity", "Ozone capacity + reserved space"); private static final MetricsInfo FS_CAPACITY = Interns.info("FilesystemCapacity", "Filesystem capacity as reported by the local filesystem"); private static final MetricsInfo FS_AVAILABLE = Interns.info("FilesystemAvailable", "Filesystem available space as reported by the local filesystem"); private static final MetricsInfo FS_USED = Interns.info("FilesystemUsed", "Filesystem used space (FilesystemCapacity - FilesystemAvailable)"); + private static final MetricsInfo MIN_FREE_SPACE = + Interns.info("MinFreeSpace", + "Minimum free space threshold (soft limit) reported to SCM, " + + "derived from hdds.datanode.volume.min.free.space.percent / hdds.datanode.volume.min.free.space"); + private static final MetricsInfo NON_OZONE_USED = + Interns.info("NonOzoneUsed", + "Space on the filesystem consumed by non-Ozone workloads " + + "(FilesystemUsed - OzoneUsed)"); private final MetricsRegistry registry; private final String metricsSourceName; @@ -238,15 +244,17 @@ public void getMetrics(MetricsCollector collector, boolean all) { SpaceUsageSource.Fixed fsUsage = volumeUsage.realUsage(); SpaceUsageSource usage = volumeUsage.getCurrentUsage(fsUsage); long reserved = volumeUsage.getReservedInBytes(); + long ozoneCapacity = usage.getCapacity(); builder - .addGauge(CAPACITY, usage.getCapacity()) + .addGauge(CAPACITY, ozoneCapacity) .addGauge(AVAILABLE, usage.getAvailable()) .addGauge(USED, usage.getUsedSpace()) .addGauge(RESERVED, reserved) - .addGauge(TOTAL_CAPACITY, usage.getCapacity() + reserved) .addGauge(FS_CAPACITY, fsUsage.getCapacity()) .addGauge(FS_AVAILABLE, fsUsage.getAvailable()) - .addGauge(FS_USED, fsUsage.getUsedSpace()); + .addGauge(FS_USED, fsUsage.getUsedSpace()) + .addGauge(MIN_FREE_SPACE, volume.getReportedFreeSpaceToSpare(ozoneCapacity)) + .addGauge(NON_OZONE_USED, VolumeUsage.getOtherUsed(fsUsage)); } } } diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeInfoMetrics.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeInfoMetrics.java index 7dc96458fdb7..22ad5a256cb3 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeInfoMetrics.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeInfoMetrics.java @@ -19,6 +19,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -43,6 +44,7 @@ void testVolumeInfoMetricsExposeOzoneAndFilesystemGauges() { when(volume.getType()).thenReturn(HddsVolume.VolumeType.DATA_VOLUME); when(volume.getCommittedBytes()).thenReturn(10L); when(volume.getContainers()).thenReturn(3L); + when(volume.getReportedFreeSpaceToSpare(anyLong())).thenReturn(20L); VolumeUsage volumeUsage = mock(VolumeUsage.class); when(volume.getVolumeUsage()).thenReturn(volumeUsage); @@ -56,7 +58,7 @@ void testVolumeInfoMetricsExposeOzoneAndFilesystemGauges() { when(volumeUsage.getReservedInBytes()).thenReturn(50L); // Raw filesystem stats - when(volumeUsage.realUsage()).thenReturn(new SpaceUsageSource.Fixed(2000L, 1500L, 500L)); + when(volumeUsage.realUsage()).thenReturn(new SpaceUsageSource.Fixed(2000L, 1100L, 500L)); VolumeInfoMetrics metrics = new VolumeInfoMetrics("test-vol-1", volume); try { @@ -72,8 +74,11 @@ void testVolumeInfoMetricsExposeOzoneAndFilesystemGauges() { assertThat(findMetric(all, "OzoneUsed")).isEqualTo(100L); assertThat(findMetric(all, "FilesystemCapacity")).isEqualTo(2000L); - assertThat(findMetric(all, "FilesystemAvailable")).isEqualTo(1500L); + assertThat(findMetric(all, "FilesystemAvailable")).isEqualTo(1100L); assertThat(findMetric(all, "FilesystemUsed")).isEqualTo(500L); + + assertThat(findMetric(all, "MinFreeSpace")).isEqualTo(20L); + assertThat(findMetric(all, "NonOzoneUsed")).isEqualTo(400L); } finally { metrics.unregister(); } From 1570a3ece27dac1c7101bab7f29ab92594603b9d Mon Sep 17 00:00:00 2001 From: Priyesh Karatha Date: Fri, 22 May 2026 14:38:31 +0530 Subject: [PATCH 2/2] making fsUsed to difference between capacity and available --- .../hadoop/ozone/container/common/volume/VolumeInfoMetrics.java | 2 +- .../ozone/container/common/volume/TestVolumeInfoMetrics.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfoMetrics.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfoMetrics.java index 32fdaf49e757..c6e17d5cfd58 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfoMetrics.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfoMetrics.java @@ -252,7 +252,7 @@ public void getMetrics(MetricsCollector collector, boolean all) { .addGauge(RESERVED, reserved) .addGauge(FS_CAPACITY, fsUsage.getCapacity()) .addGauge(FS_AVAILABLE, fsUsage.getAvailable()) - .addGauge(FS_USED, fsUsage.getUsedSpace()) + .addGauge(FS_USED, fsUsage.getCapacity() - fsUsage.getAvailable()) .addGauge(MIN_FREE_SPACE, volume.getReportedFreeSpaceToSpare(ozoneCapacity)) .addGauge(NON_OZONE_USED, VolumeUsage.getOtherUsed(fsUsage)); } diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeInfoMetrics.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeInfoMetrics.java index 22ad5a256cb3..beab82b17aa6 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeInfoMetrics.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeInfoMetrics.java @@ -75,7 +75,7 @@ void testVolumeInfoMetricsExposeOzoneAndFilesystemGauges() { assertThat(findMetric(all, "FilesystemCapacity")).isEqualTo(2000L); assertThat(findMetric(all, "FilesystemAvailable")).isEqualTo(1100L); - assertThat(findMetric(all, "FilesystemUsed")).isEqualTo(500L); + assertThat(findMetric(all, "FilesystemUsed")).isEqualTo(900L); // FilesystemCapacity - FilesystemAvailable assertThat(findMetric(all, "MinFreeSpace")).isEqualTo(20L); assertThat(findMetric(all, "NonOzoneUsed")).isEqualTo(400L);