flatkv migration testings test

.github/workflows/integration-test.yml

@@ -114,35 +114,70 @@ jobs:
             # rotation, etc.) should be appended here rather than added as
             # new matrix rows, so the CI surface stays one row per concern.
             #
-            # Step ordering matters:
-            #   1-2  Deploy EVM fixture and capture baseline RPC reads while
-            #        the cluster is still in memiavl-only mode.
-            #   3    Run the offline FlatKV import on every validator and
-            #        restart the cluster with flatkv enabled (dual_write,
-            #        lattice-hash off to preserve AppHash trajectory across
-            #        the import boundary).
-            #   4-5  Re-run the same fixture probe and physical-layout check
-            #        against the post-import cluster; must match step 2.
-            #   6    SIGKILL one validator, restart, and assert 4-node
-            #        FlatKV digest equality. This exercises WAL recovery on
-            #        the post-import flatkv state, which mirrors the real
-            #        production scenario (chains reach flatkv via migration,
-            #        not via genesis flag).
-            #   7-9  Destructive disaster-recovery scenarios. They run last
+            # The cluster boots with GIGA_STORAGE=true, so FlatKV is enabled
+            # from genesis (sc-write-mode = test_only_dual_write) and the
+            # FlatKV LtHash is folded into every block's AppHash from block 1.
+            # No mid-life migration is exercised here; an offline
+            # memiavl -> FlatKV import would change the AppHash trajectory
+            # at the import boundary and break tendermint replay.
+            #
+            # Step ordering:
+            #   1-3  Deploy an EVM fixture, smoke-check EVM RPC reads, and
+            #        verify the fixture round-tripped through dual_write into
+            #        physical FlatKV.
+            #   4    SIGKILL one validator, restart, and assert 4-node FlatKV
+            #        digest equality. Exercises WAL recovery on a live FlatKV
+            #        cluster.
+            #   5-7  Destructive disaster-recovery scenarios. They run last
             #        because they wipe or damage one validator's local state.
             name: "FlatKV Integration",
+            env: "GIGA_STORAGE=true",
             scripts: [
               "docker exec sei-node-0 integration_test/contracts/deploy_flatkv_evm_fixture.sh",
               "python3 integration_test/scripts/runner.py integration_test/seidb/flatkv_evm_test.yaml",
-              "./integration_test/contracts/import_flatkv_evm_cluster.sh",
-              "python3 integration_test/scripts/runner.py integration_test/seidb/flatkv_evm_test.yaml",
               "docker exec sei-node-0 integration_test/contracts/verify_flatkv_evm_store.sh",
               "./integration_test/contracts/verify_flatkv_crash_recovery.sh",
               "./integration_test/contracts/verify_flatkv_statesync_crash_recovery.sh",
               "./integration_test/contracts/verify_flatkv_total_loss_recovery.sh",
               "./integration_test/contracts/verify_flatkv_partial_loss_fails_loudly.sh",
             ],
           },
+          {
+            # 0 -> 1 (MigrateEVM) cluster migration coverage.
+            #
+            # The cluster boots with GIGA_MIGRATE_FROM_MEMIAVL=true so
+            # every validator starts in sc-write-mode = memiavl_only,
+            # i.e. v0 (FlatKV not yet allocated). This is the inverse of
+            # the FlatKV Integration row above (which boots in
+            # test_only_dual_write) and is what makes the cutover
+            # script's pre-flip check meaningful: if the matrix env ever
+            # silently lands the cluster in any mode other than
+            # memiavl_only, the script will fail loudly at the pre-flip
+            # grep instead of "succeeding" with a no-op migration.
+            #
+            # Steps:
+            #   1  Deposit an EVM fixture while in v0 so the migration
+            #      has real account+code+storage to drain (otherwise
+            #      "migration" trivially completes against an empty
+            #      tree and the test exercises nothing).
+            #   2  Coordinated stop -> sed sc-write-mode -> restart on
+            #      all 4 validators, then poll seidb migrate-evm-status
+            #      until every validator reports completion. Cross-
+            #      validator FlatKV digest agreement is asserted at a
+            #      shared post-migration height; any non-determinism in
+            #      the batch copier would surface here as a digest
+            #      mismatch.
+            #   3  Re-run the fixture round-trip check against the
+            #      now-FlatKV-backed EVM state, confirming pre-migration
+            #      data survives the cutover intact (read transparency).
+            name: "FlatKV EVM Migration 0->1",
+            env: "GIGA_MIGRATE_FROM_MEMIAVL=true",
+            scripts: [
+              "docker exec sei-node-0 integration_test/contracts/deploy_flatkv_evm_fixture.sh",
+              "./integration_test/contracts/verify_flatkv_evm_migration_0_to_1.sh",
+              "docker exec sei-node-0 integration_test/contracts/verify_flatkv_evm_store.sh",
+            ],
+          },
           {
             name: "EVM Module",
             env: "GIGA_STORAGE=true",

Makefile

@@ -295,7 +295,8 @@ CLUSTER_ENV_VARS = DOCKER_PLATFORM=$(DOCKER_PLATFORM) USERID=$(shell id -u) GROU
 	GIGA_OCC=$(GIGA_OCC) \
 	RECEIPT_BACKEND=$(RECEIPT_BACKEND) \
 	AUTOBAHN=$(AUTOBAHN) \
-	GIGA_STORAGE=$(GIGA_STORAGE)
+	GIGA_STORAGE=$(GIGA_STORAGE) \
+	GIGA_MIGRATE_FROM_MEMIAVL=$(GIGA_MIGRATE_FROM_MEMIAVL)
 
 # Run a 4-node docker containers
 docker-cluster-start: docker-cluster-stop build-docker-node
@@ -321,7 +322,7 @@ docker-cluster-start-skipbuild: docker-cluster-stop build-docker-node
 		else \
 			DETACH_FLAG=""; \
 		fi; \
-		DOCKER_PLATFORM=$(DOCKER_PLATFORM) USERID=$(shell id -u) GROUPID=$(shell id -g) GOCACHE=$(shell go env GOCACHE) NUM_ACCOUNTS=10 SKIP_BUILD=true docker compose up $$DETACH_FLAG
+		$(CLUSTER_ENV_VARS) SKIP_BUILD=true docker compose up $$DETACH_FLAG
 .PHONY: localnet-start
 
 # Stop 4-node docker containers

app/seidb.go

@@ -28,8 +28,14 @@ const (
 	FlagSCHistoricalProofRateLimit   = "state-commit.sc-historical-proof-rate-limit"
 	FlagSCHistoricalProofBurst       = "state-commit.sc-historical-proof-burst"
 	FlagSCWriteMode                  = "state-commit.sc-write-mode"
-	FlagSCReadMode                   = "state-commit.sc-read-mode"
-	FlagSCEnableLatticeHash          = "state-commit.sc-enable-lattice-hash"
+	// Per-block batch size used by the MigrationManager when sc-write-mode
+	// is one of the in-flight modes (migrate_evm, migrate_bank,
+	// migrate_all_but_bank). Optional: when unset in app.toml the field
+	// stays at DefaultStateCommitConfig().KeysToMigratePerBlock (= 1024),
+	// which is appropriate for production drains. Lowering it spreads the
+	// migration across more blocks, which is useful for tests that need to
+	// exercise the resume / hybrid-read path mid-flight.
+	FlagSCKeysToMigratePerBlock = "state-commit.sc-keys-to-migrate-per-block"
 
 	// SS Store configs
 	FlagSSEnable            = "state-store.ss-enable"
@@ -111,15 +117,6 @@ func parseSCConfigs(appOpts servertypes.AppOptions) config.StateCommitConfig {
 		}
 		scConfig.WriteMode = parsedWM
 	}
-	if rm := cast.ToString(appOpts.Get(FlagSCReadMode)); rm != "" {
-		parsedRM, err := config.ParseReadMode(rm)
-		if err != nil {
-			panic(fmt.Sprintf("invalid EVM SS read mode %q: %s", rm, err))
-		}
-		scConfig.ReadMode = parsedRM
-	}
-
-	scConfig.EnableLatticeHash = cast.ToBool(appOpts.Get(FlagSCEnableLatticeHash))
 
 	if v := appOpts.Get(FlagSCHistoricalProofMaxInFlight); v != nil {
 		scConfig.HistoricalProofMaxInFlight = cast.ToInt(v)
@@ -130,6 +127,16 @@ func parseSCConfigs(appOpts servertypes.AppOptions) config.StateCommitConfig {
 	if v := appOpts.Get(FlagSCHistoricalProofBurst); v != nil {
 		scConfig.HistoricalProofBurst = cast.ToInt(v)
 	}
+	// Guard with v != nil so that an absent app.toml entry preserves the
+	// default of 1024 instead of clobbering it to 0, which would fail
+	// StateCommitConfig.Validate ("keys-to-migrate-per-block must be > 0")
+	// and bring the node down at startup the first time write-mode is
+	// flipped to a migration mode.
+	if v := appOpts.Get(FlagSCKeysToMigratePerBlock); v != nil {
+		if n := cast.ToInt(v); n > 0 {
+			scConfig.KeysToMigratePerBlock = n
+		}
+	}
 
 	return scConfig
 }

app/seidb_test.go

@@ -37,8 +37,6 @@ func (t TestSeiDBAppOpts) Get(s string) interface{} {
 		return defaultSCConfig.MemIAVLConfig.SnapshotPrefetchThreshold
 	case FlagSCSnapshotWriteRateMBps:
 		return defaultSCConfig.MemIAVLConfig.SnapshotWriteRateMBps
-	case FlagSCEnableLatticeHash:
-		return defaultSCConfig.EnableLatticeHash
 	case FlagSSEnable:
 		return defaultSSConfig.Enable
 	case FlagSSBackend:

docker/docker-compose.yml

@@ -21,6 +21,7 @@ services:
       - RECEIPT_BACKEND
       - AUTOBAHN
       - GIGA_STORAGE
+      - GIGA_MIGRATE_FROM_MEMIAVL
     volumes:
       - "${PROJECT_HOME}:/sei-protocol/sei-chain:Z"
       - "${PROJECT_HOME}/../sei-tendermint:/sei-protocol/sei-tendermint:Z"
@@ -54,6 +55,7 @@ services:
       - RECEIPT_BACKEND
       - AUTOBAHN
       - GIGA_STORAGE
+      - GIGA_MIGRATE_FROM_MEMIAVL
     volumes:
       - "${PROJECT_HOME}:/sei-protocol/sei-chain:Z"
       - "${PROJECT_HOME}/../sei-tendermint:/sei-protocol/sei-tendermint:Z"
@@ -83,6 +85,7 @@ services:
       - RECEIPT_BACKEND
       - AUTOBAHN
       - GIGA_STORAGE
+      - GIGA_MIGRATE_FROM_MEMIAVL
     ports:
       - "26662-26664:26656-26658"
       - "9094-9095:9090-9091"
@@ -116,6 +119,7 @@ services:
       - RECEIPT_BACKEND
       - AUTOBAHN
       - GIGA_STORAGE
+      - GIGA_MIGRATE_FROM_MEMIAVL
     ports:
       - "26665-26667:26656-26658"
       - "9096-9097:9090-9091"

docker/localnode/config/app.toml

@@ -236,6 +236,13 @@ sc-snapshot-writer-limit = 2
 # CacheSize defines the size of the LRU cache for each store on top of the tree, default to 100000.
 sc-cache-size = 1000
 
+# KeysToMigratePerBlock controls how many EVM keys the in-flight migration
+# (sc-write-mode = migrate_evm / migrate_bank / migrate_all_but_bank) drains
+# from memiavl into flatkv per block. Default 1024 is appropriate for
+# production drains; tests lower it to spread the migration across more
+# blocks and exercise the resume / hybrid-read path.
+sc-keys-to-migrate-per-block = 1024
+
 [state-store]
 
 # Enable defines if the state-store should be enabled for historical queries.

docker/localnode/scripts/step4_config_override.sh

@@ -5,6 +5,13 @@ GIGA_EXECUTOR=${GIGA_EXECUTOR:-false}
 GIGA_OCC=${GIGA_OCC:-false}
 AUTOBAHN=${AUTOBAHN:-false}
 GIGA_STORAGE=${GIGA_STORAGE:-false}
+# GIGA_MIGRATE_FROM_MEMIAVL=true boots the cluster in v0 (memiavl_only):
+# memiavl is the sole SC backend, FlatKV is not allocated. This is the
+# starting point for the 0->1 (MigrateEVM) cluster test, which drives a
+# real workload in this mode and then performs a coordinated stop/flip/
+# restart into migrate_evm. Mutually exclusive with GIGA_STORAGE=true;
+# the script picks the more specific override if both are set.
+GIGA_MIGRATE_FROM_MEMIAVL=${GIGA_MIGRATE_FROM_MEMIAVL:-false}
 
 APP_CONFIG_FILE="build/generated/node_$NODE_ID/app.toml"
 TENDERMINT_CONFIG_FILE="build/generated/node_$NODE_ID/config.toml"
@@ -23,37 +30,47 @@ sed -i.bak -e "s|^snapshot-directory *=.*|snapshot-directory = \"./build/generat
 # Enable slow mode
 sed -i.bak -e 's/slow = .*/slow = true/' ~/.sei/config/app.toml
 
+# Boot the cluster in v0 (memiavl_only) for the 0->1 migration test.
+# Doing this here keeps the override surface narrow: the test runner
+# only has to set one env var to ship a v0-shaped config, and the
+# follow-up flip script just rewrites sc-write-mode in place during the
+# coordinated stop.
+if [ "$GIGA_MIGRATE_FROM_MEMIAVL" = "true" ]; then
+  echo "Booting node $NODE_ID in memiavl_only mode (0->1 migration starting point)..."
+  if grep -q '^sc-write-mode[[:space:]]*=' ~/.sei/config/app.toml; then
+    sed -i 's/^sc-write-mode[[:space:]]*=.*/sc-write-mode = "memiavl_only"/' ~/.sei/config/app.toml
+  else
+    sed -i '/^\[state-store\]/i sc-write-mode = "memiavl_only"' ~/.sei/config/app.toml
+  fi
+  # The EVM SS split is irrelevant in this mode (flatkv is not allocated),
+  # but explicitly disabling it keeps app.toml self-describing in case an
+  # operator inspects it post-flip.
+  sed -i 's/^evm-ss-split[[:space:]]*=.*/evm-ss-split = false/' ~/.sei/config/app.toml
+fi
+
 # Enable Giga Storage: FlatKV SC dual-write + EVM SS split.
 # When GIGA_STORAGE=true we also default the receipt backend to parquet; callers
 # can still override this by setting RECEIPT_BACKEND explicitly.
 # Set GIGA_STORAGE=false to disable.
-if [ "$GIGA_STORAGE" = "true" ]; then
+# GIGA_MIGRATE_FROM_MEMIAVL takes precedence: if both are set, the memiavl-only
+# block above ran first and the test runner is responsible for the cutover.
+if [ "$GIGA_STORAGE" = "true" ] && [ "$GIGA_MIGRATE_FROM_MEMIAVL" != "true" ]; then
   RECEIPT_BACKEND=${RECEIPT_BACKEND:-parquet}
   echo "Enabling Giga Storage for node $NODE_ID..."
 
-  # --- SC layer: dual_write + split_read + lattice hash ---
-  # SC must use dual_write (not split_write) because block execution reads
-  # EVM data from the memiavl tree via GetChildStoreByName. With split_write,
-  # EVM data only goes to FlatKV and the memiavl tree becomes stale.
-  # dual_write keeps memiavl up-to-date for reads while also populating FlatKV.
-  if grep -q "sc-write-mode" ~/.sei/config/app.toml; then
-    sed -i 's/sc-write-mode = .*/sc-write-mode = "dual_write"/' ~/.sei/config/app.toml
-  else
-    sed -i '/^\[state-store\]/i sc-write-mode = "dual_write"' ~/.sei/config/app.toml
-  fi
-  if grep -q "sc-read-mode" ~/.sei/config/app.toml; then
-    sed -i 's/sc-read-mode = .*/sc-read-mode = "split_read"/' ~/.sei/config/app.toml
-  else
-    sed -i '/^\[state-store\]/i sc-read-mode = "split_read"' ~/.sei/config/app.toml
-  fi
-  if grep -q "sc-enable-lattice-hash" ~/.sei/config/app.toml; then
-    sed -i 's/sc-enable-lattice-hash = .*/sc-enable-lattice-hash = true/' ~/.sei/config/app.toml
+  # --- SC layer: test_only_dual_write ---
+  # SC must use test_only_dual_write because block execution reads EVM data
+  # from the memiavl tree via GetChildStoreByName. dual-write keeps memiavl
+  # up-to-date for reads while also populating FlatKV. This mode is for test
+  # clusters only — never deploy to testnet/mainnet.
+  if grep -q '^sc-write-mode[[:space:]]*=' ~/.sei/config/app.toml; then
+    sed -i 's/^sc-write-mode[[:space:]]*=.*/sc-write-mode = "test_only_dual_write"/' ~/.sei/config/app.toml
   else
-    sed -i '/^\[state-store\]/i sc-enable-lattice-hash = true' ~/.sei/config/app.toml
+    sed -i '/^\[state-store\]/i sc-write-mode = "test_only_dual_write"' ~/.sei/config/app.toml
   fi
 
   # --- SS layer: enable EVM split ---
-  sed -i 's/evm-ss-split = .*/evm-ss-split = true/' ~/.sei/config/app.toml
+  sed -i 's/^evm-ss-split[[:space:]]*=.*/evm-ss-split = true/' ~/.sei/config/app.toml
 fi
 
 # Enable Giga Executor (evmone-based) if requested

docker/rpcnode/scripts/step1_configure_init.sh

@@ -24,21 +24,11 @@ if [ "$GIGA_STORAGE" = "true" ]; then
   RECEIPT_BACKEND=${RECEIPT_BACKEND:-parquet}
   echo "Enabling Giga Storage for RPC node..."
 
-  # SC layer: must match validators (dual_write + split_read + lattice hash)
+  # SC layer: must match validators (test_only_dual_write)
   if grep -q "sc-write-mode" ~/.sei/config/app.toml; then
-    sed -i 's/sc-write-mode = .*/sc-write-mode = "dual_write"/' ~/.sei/config/app.toml
+    sed -i 's/sc-write-mode = .*/sc-write-mode = "test_only_dual_write"/' ~/.sei/config/app.toml
   else
-    sed -i '/^\[state-store\]/i sc-write-mode = "dual_write"' ~/.sei/config/app.toml
-  fi
-  if grep -q "sc-read-mode" ~/.sei/config/app.toml; then
-    sed -i 's/sc-read-mode = .*/sc-read-mode = "split_read"/' ~/.sei/config/app.toml
-  else
-    sed -i '/^\[state-store\]/i sc-read-mode = "split_read"' ~/.sei/config/app.toml
-  fi
-  if grep -q "sc-enable-lattice-hash" ~/.sei/config/app.toml; then
-    sed -i 's/sc-enable-lattice-hash = .*/sc-enable-lattice-hash = true/' ~/.sei/config/app.toml
-  else
-    sed -i '/^\[state-store\]/i sc-enable-lattice-hash = true' ~/.sei/config/app.toml
+    sed -i '/^\[state-store\]/i sc-write-mode = "test_only_dual_write"' ~/.sei/config/app.toml
   fi
 
   # SS layer: enable EVM split