branch-4.1: [fix](fe) Keep table version monotonic after truncate #66255

fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java

@@ -3814,8 +3814,10 @@ public void truncateTable(String dbName, String tableName, PartitionNamesInfo pa
 
             //replace
             Map<Long, RecyclePartitionParam> recyclePartitionParamMap  =  new HashMap<>();
+            long version = Config.isNotCloudMode() ? olapTable.getNextVersion() : 0L;
+            long versionTimeMs = Config.isNotCloudMode() ? System.currentTimeMillis() : 0L;
             oldPartitions = truncateTableInternal(olapTable, newPartitions,
-                    truncateEntireTable, recyclePartitionParamMap, forceDrop);
+                    truncateEntireTable, recyclePartitionParamMap, forceDrop, version, versionTimeMs);
             if (truncateEntireTable) {
                 Env.getCurrentEnv().getAnalysisManager().removeTableStats(olapTable.getId());
             } else {
@@ -3827,7 +3829,7 @@ public void truncateTable(String dbName, String tableName, PartitionNamesInfo pa
             TruncateTableInfo info =
                     new TruncateTableInfo(db.getId(), db.getFullName(), olapTable.getId(), olapTable.getName(),
                     newPartitions, truncateEntireTable,
-                            rawTruncateSql, oldPartitions, forceDrop, updateRecords);
+                            rawTruncateSql, oldPartitions, forceDrop, updateRecords, version, versionTimeMs);
             Env.getCurrentEnv().getEditLog().logTruncateTable(info);
         } catch (DdlException e) {
             failedCleanCallback.run();
@@ -3843,7 +3845,8 @@ public void truncateTable(String dbName, String tableName, PartitionNamesInfo pa
     }
 
     private List<Partition> truncateTableInternal(OlapTable olapTable, List<Partition> newPartitions,
-            boolean isEntireTable, Map<Long, RecyclePartitionParam> recyclePartitionParamMap, boolean isforceDrop) {
+            boolean isEntireTable, Map<Long, RecyclePartitionParam> recyclePartitionParamMap, boolean isforceDrop,
+            long version, long versionTimeMs) {
         // use new partitions to replace the old ones.
         List<Partition> oldPartitions = Lists.newArrayList();
         for (Partition newPartition : newPartitions) {
@@ -3873,9 +3876,13 @@ private List<Partition> truncateTableInternal(OlapTable olapTable, List<Partitio
             olapTable.dropPartitionForTruncate(olapTable.getDatabase().getId(), isforceDrop, pair.getValue());
         }
 
-        // Reset table-level visibleVersion to TABLE_INIT_VERSION so it stays consistent
-        // with the newly created partitions (which also start at PARTITION_INIT_VERSION).
-        olapTable.resetVisibleVersion();
+        if (Config.isNotCloudMode() && version > 0) {
+            // Persisted values make the version transition deterministic during journal replay.
+            olapTable.updateVisibleVersionAndTime(version, versionTimeMs);
+        } else {
+            // Preserve legacy replay and Cloud's local cache invalidation behavior.
+            olapTable.resetVisibleVersion();
+        }
 
         return oldPartitions;
     }
@@ -3889,7 +3896,8 @@ public void replayTruncateTable(TruncateTableInfo info) throws MetaNotFoundExcep
         try {
             Map<Long, RecyclePartitionParam> recyclePartitionParamMap =  new HashMap<>();
             truncateTableInternal(olapTable, info.getPartitions(), info.isEntireTable(),
-                                    recyclePartitionParamMap, isForceDrop);
+                                    recyclePartitionParamMap, isForceDrop,
+                                    info.getVersion(), info.getVersionTimeMs());
 
             // add tablet to inverted index
             TabletInvertedIndex invertedIndex = Env.getCurrentInvertedIndex();

fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java

@@ -60,6 +60,10 @@ public class TruncateTableInfo implements Writable {
     private Map<Long, Long> updateRecords;
     @SerializedName(value = "ut")
     private long updateTime;
+    @SerializedName(value = "version")
+    private long version;
+    @SerializedName(value = "versionTime")
+    private long versionTimeMs;
 
     public TruncateTableInfo() {
 
@@ -68,7 +72,7 @@ public TruncateTableInfo() {
     // for internal table
     public TruncateTableInfo(long dbId, String db, long tblId, String table, List<Partition> partitions,
             boolean isEntireTable, String rawSql, List<Partition> oldPartitions, boolean force,
-            Map<Long, Long> updateRecords) {
+            Map<Long, Long> updateRecords, long version, long versionTimeMs) {
         this.dbId = dbId;
         this.db = db;
         this.tblId = tblId;
@@ -81,6 +85,8 @@ public TruncateTableInfo(long dbId, String db, long tblId, String table, List<Pa
         }
         this.force = force;
         this.updateRecords = updateRecords;
+        this.version = version;
+        this.versionTimeMs = versionTimeMs;
     }
 
     // for external table
@@ -144,6 +150,14 @@ public long getUpdateTime() {
         return updateTime;
     }
 
+    public long getVersion() {
+        return version;
+    }
+
+    public long getVersionTimeMs() {
+        return versionTimeMs;
+    }
+
     public static TruncateTableInfo read(DataInput in) throws IOException {
         String json = Text.readString(in);
         return GsonUtils.GSON.fromJson(json, TruncateTableInfo.class);

fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/TruncateTableCommandTest.java

@@ -173,11 +173,15 @@ public void testTruncateTable() throws Exception {
         checkShowTabletResultNum("internal.testcommand.tblcommand", "p20210903", 4);
         checkShowTabletResultNum("internal.testcommand.tblcommand", "p20210904", 5);
 
+        OlapTable table = Env.getCurrentInternalCatalog().getDbNullable("testcommand")
+                .getOlapTableOrDdlException("tblcommand");
+        long visibleVersion = table.getVisibleVersion();
         String truncateStr = "truncate table internal.testcommand.tblcommand;";
         NereidsParser nereidsParser = new NereidsParser();
         LogicalPlan plan = nereidsParser.parseSingle(truncateStr);
         Assertions.assertTrue(plan instanceof TruncateTableCommand);
         Env.getCurrentEnv().truncateTable((TruncateTableCommand) plan);
+        Assertions.assertEquals(visibleVersion + 1, table.getVisibleVersion());
 
         checkShowTabletResultNum("internal.testcommand.tblcommand", "p20210901", 2);
         checkShowTabletResultNum("internal.testcommand.tblcommand", "p20210902", 3);
@@ -188,6 +192,7 @@ public void testTruncateTable() throws Exception {
         plan = nereidsParser.parseSingle(truncateStr);
         Assertions.assertTrue(plan instanceof TruncateTableCommand);
         Env.getCurrentEnv().truncateTable((TruncateTableCommand) plan);
+        Assertions.assertEquals(visibleVersion + 2, table.getVisibleVersion());
 
         checkShowTabletResultNum("internal.testcommand.tblcommand", "p20210901", 2);
         checkShowTabletResultNum("internal.testcommand.tblcommand", "p20210902", 3);

fe/fe-core/src/test/java/org/apache/doris/persist/TruncateTableInfoTest.java

@@ -0,0 +1,46 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.persist;
+
+import org.apache.doris.persist.gson.GsonUtils;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+class TruncateTableInfoTest {
+    @Test
+    void testVersionSerialization() {
+        TruncateTableInfo info = new TruncateTableInfo(1L, "db", 2L, "tbl", Lists.newArrayList(),
+                true, "TRUNCATE TABLE tbl", Lists.newArrayList(), true, Maps.newHashMap(), 10L, 20L);
+
+        TruncateTableInfo deserialized = GsonUtils.GSON.fromJson(info.toJson(), TruncateTableInfo.class);
+
+        Assertions.assertEquals(10L, deserialized.getVersion());
+        Assertions.assertEquals(20L, deserialized.getVersionTimeMs());
+    }
+
+    @Test
+    void testLegacyVersionDefaults() {
+        TruncateTableInfo deserialized = GsonUtils.GSON.fromJson("{}", TruncateTableInfo.class);
+
+        Assertions.assertEquals(0L, deserialized.getVersion());
+        Assertions.assertEquals(0L, deserialized.getVersionTimeMs());
+    }
+}

regression-test/data/mtmv_p0/test_truncate_table_mtmv.out

@@ -8,5 +8,9 @@
 2	2
 3	3
 
+-- !non_pct_truncate --
+2	2
+3	3
+
 -- !truncate_table --
 

regression-test/suites/mtmv_p0/test_truncate_table_mtmv.groovy

@@ -23,6 +23,7 @@ suite("test_truncate_table_mtmv","mtmv") {
     String mvName = "${suiteName}_mv"
     sql """drop table if exists `${tableName}`"""
     sql """drop materialized view if exists ${mvName};"""
+    sql """drop table if exists test_truncate_table_mtmv_dim"""
 
     sql """
         CREATE TABLE ${tableName}
@@ -42,6 +43,17 @@ suite("test_truncate_table_mtmv","mtmv") {
             "replication_num" = "1"
         );
         """
+    sql """
+        CREATE TABLE test_truncate_table_mtmv_dim (
+            k2 TINYINT NOT NULL
+        )
+        UNIQUE KEY(k2)
+        DISTRIBUTED BY HASH(k2) BUCKETS 1
+        PROPERTIES (
+            'replication_num' = '1',
+            'enable_unique_key_merge_on_write' = 'true'
+        )
+        """
     sql """
         CREATE MATERIALIZED VIEW ${mvName}
         BUILD DEFERRED REFRESH AUTO ON MANUAL
@@ -51,12 +63,15 @@ suite("test_truncate_table_mtmv","mtmv") {
         'replication_num' = '1'
         )
         AS
-        SELECT * from ${tableName};
+        SELECT fact.k2, fact.k3
+        FROM ${tableName} fact
+        INNER JOIN test_truncate_table_mtmv_dim dim ON fact.k2 = dim.k2;
         """
 
     sql """
         insert into ${tableName} values(1,1),(2,2),(3,3);
         """
+    sql """insert into test_truncate_table_mtmv_dim values(1),(2),(3)"""
      sql """
         REFRESH MATERIALIZED VIEW ${mvName} AUTO
         """
@@ -73,6 +88,19 @@ suite("test_truncate_table_mtmv","mtmv") {
     waitingMTMVTaskFinishedByMvName(mvName)
     order_qt_truncate_partition "SELECT * FROM ${mvName}"
 
+    // Save a non-PCT table snapshot at version 3 while the MV does not contain k2=2.
+    sql """delete from test_truncate_table_mtmv_dim where k2=2"""
+    sql """REFRESH MATERIALIZED VIEW ${mvName} COMPLETE"""
+    waitingMTMVTaskFinishedByMvName(mvName)
+
+    // Without monotonic table versions, reset to 1 plus two inserts collides with version 3.
+    sql """truncate table test_truncate_table_mtmv_dim"""
+    sql """insert into test_truncate_table_mtmv_dim values(1),(2)"""
+    sql """insert into test_truncate_table_mtmv_dim values(3)"""
+    sql """REFRESH MATERIALIZED VIEW ${mvName} AUTO"""
+    waitingMTMVTaskFinishedByMvName(mvName)
+    order_qt_non_pct_truncate "SELECT * FROM ${mvName}"
+
     // truncate table
     sql """
         truncate table ${tableName};

regression-test/suites/nereids_rules_p0/rewrite_simple_agg_to_constant/truncate_version_reset.groovy

@@ -16,25 +16,8 @@
 // under the License.
 
 /**
- * Regression test for: TRUNCATE TABLE must reset TableAttributes.visibleVersion.
- *
- * Bug: before the fix, truncateTableInternal() replaced partition data but did
- * not call olapTable.resetVisibleVersion().  As a result:
- *   - Partition.visibleVersion was reset to PARTITION_INIT_VERSION (1).
- *   - TableAttributes.visibleVersion kept its old, higher value.
- *   - TableAttributes.visibleVersionTime was never updated.
- *
- * Consequence for RewriteSimpleAggToConstantRule / SimpleAggCacheMgr:
- *   The cache entry was keyed by versionTime.  Because versionTime did not
- *   change at truncate time, the *caller* saw the same versionTime as the
- *   stale cached entry → cache HIT → the rule returned the pre-truncate
- *   count/min/max instead of the correct post-truncate values.
- *
- * The fix adds olapTable.resetVisibleVersion() inside truncateTableInternal(),
- * which bumps both visibleVersion (back to TABLE_INIT_VERSION = 1) and
- * visibleVersionTime (to System.currentTimeMillis()).  The new versionTime
- * differs from the cached entry's versionTime → cache MISS → the rule
- * correctly falls back to BE execution and returns the right result.
+ * TRUNCATE changes visible rows, so it must advance the table version metadata.
+ * SimpleAggCacheMgr uses that metadata to invalidate cached aggregate values.
  */
 suite("truncate_version_reset") {
     sql "SET enable_nereids_planner=true"
@@ -86,23 +69,14 @@ suite("truncate_version_reset") {
     // Confirm the cached count is correct before truncate.
     order_qt_count_before_truncate "SELECT count(*) FROM tbl;"
 
-    // -----------------------------------------------------------------------
-    // TRUNCATE the table.
-    // After the fix, resetVisibleVersion() is called inside
-    // truncateTableInternal(), which updates visibleVersionTime.
-    // The cache entry's versionTime no longer matches → cache is invalidated.
-    // -----------------------------------------------------------------------
+    // Truncate must invalidate the cached aggregate value.
     sql "TRUNCATE TABLE tbl;"
 
     // count(*) must return 0.
     // Without the fix, the stale cache entry (count = 5) would be returned.
     order_qt_count_after_truncate "SELECT count(*) FROM tbl;"
 
-    // -----------------------------------------------------------------------
     // Insert new rows after truncate, then verify count(*) reflects them.
-    // This also validates that the version counter is correctly reset so
-    // subsequent transactions start from the right next-version.
-    // -----------------------------------------------------------------------
     sql "INSERT INTO tbl VALUES (10, 100), (20, 200);"
 
     // After insert the count must be 2.