feat(loadtest): QoL changes (#892)

* feat(loadtest): parallel nonce fetching, gas units, stop on insufficient funds

* fix: logging

* chore: fix logs

* fix: log fields

* fix: rename flag

* feat: `--private-txs` flag

* fix: make gen

* fix: validate private txs modes

* feat: add --rpc-headers flag to loadtest command

* fix: lint

Author: minhd-vu

cmd/dbbench/dbbench.go

@@ -390,7 +390,7 @@ benchLoop:
 				_, err := db.Get(tmpKey)
 				keyLock.Unlock()
 				if err != nil {
-					log.Error().Str("key", hex.EncodeToString(tmpKey)).Err(err).Msg("db random read error")
+					log.Error().Str("key", hex.EncodeToString(tmpKey)).Err(err).Msg("DB random read error")
 				}
 				wg.Done()
 				<-pool

cmd/fixnoncegap/fixnoncegap.go

@@ -112,7 +112,7 @@ func fixNonceGap(cmd *cobra.Command, args []string) error {
 	var maxNonce uint64
 	if inputFixNonceGapArgs.maxNonce != 0 {
 		maxNonce = inputFixNonceGapArgs.maxNonce
-		log.Info().Uint64("maxNonce", maxNonce).Msg("maxNonce loaded from --max-nonce flag")
+		log.Info().Uint64("maxNonce", maxNonce).Msg("Loaded max nonce from --max-nonce flag")
 	} else {
 		log.Info().Msg("--max-nonce flag not set")
 		maxNonce, err = getMaxNonce(addr)
@@ -124,7 +124,7 @@ func fixNonceGap(cmd *cobra.Command, args []string) error {
 
 	// check if there is a nonce gap
 	if maxNonce == 0 || currentNonce >= maxNonce {
-		log.Info().Stringer("addr", addr).Msg("There is no nonce gap.")
+		log.Info().Stringer("addr", addr).Msg("There is no nonce gap")
 		return nil
 	}
 	log.Info().
@@ -170,15 +170,15 @@ func fixNonceGap(cmd *cobra.Command, args []string) error {
 				log.Error().Err(err).Msg("Unable to sign tx")
 				return err
 			}
-			log.Info().Stringer("hash", signedTx.Hash()).Msgf("sending tx with nonce %d", txTemplate.Nonce)
+			log.Info().Stringer("hash", signedTx.Hash()).Msgf("Sending transaction with nonce %d", txTemplate.Nonce)
 
 			err = rpcClient.SendTransaction(cmd.Context(), signedTx)
 			if err != nil {
 				if strings.Contains(err.Error(), "nonce too low") {
-					log.Info().Stringer("hash", signedTx.Hash()).Msgf("another tx with nonce %d was mined while trying to increase the fee, skipping it", txTemplate.Nonce)
+					log.Info().Stringer("hash", signedTx.Hash()).Msgf("Another transaction with nonce %d was mined while trying to increase the fee, skipping", txTemplate.Nonce)
 					break out
 				} else if strings.Contains(err.Error(), "already known") {
-					log.Info().Stringer("hash", signedTx.Hash()).Msgf("same tx with nonce %d already exists, skipping it", txTemplate.Nonce)
+					log.Info().Stringer("hash", signedTx.Hash()).Msgf("Same transaction with nonce %d already exists, skipping", txTemplate.Nonce)
 					break out
 				} else if strings.Contains(err.Error(), "replacement transaction underpriced") ||
 					strings.Contains(err.Error(), "INTERNAL_ERROR: could not replace existing tx") {
@@ -195,11 +195,11 @@ func fixNonceGap(cmd *cobra.Command, args []string) error {
 							txTemplateCopy.GasPrice = new(big.Int).Add(txTemplateCopy.GasPrice, big.NewInt(1))
 						}
 						tx = types.NewTx(&txTemplateCopy)
-						log.Info().Stringer("hash", signedTx.Hash()).Msgf("tx with nonce %d is underpriced, increasing fee. From %d To %d", txTemplate.Nonce, oldGasPrice, txTemplateCopy.GasPrice)
+						log.Info().Stringer("hash", signedTx.Hash()).Msgf("Transaction with nonce %d is underpriced, increasing fee from %d to %d", txTemplate.Nonce, oldGasPrice, txTemplateCopy.GasPrice)
 						time.Sleep(time.Second)
 						continue
 					} else {
-						log.Info().Stringer("hash", signedTx.Hash()).Msgf("another tx with nonce %d already exists, skipping it", txTemplate.Nonce)
+						log.Info().Stringer("hash", signedTx.Hash()).Msgf("Another transaction with nonce %d already exists, skipping", txTemplate.Nonce)
 						break out
 					}
 				}
@@ -214,7 +214,7 @@ func fixNonceGap(cmd *cobra.Command, args []string) error {
 	}
 
 	if lastTx != nil {
-		log.Info().Stringer("hash", lastTx.Hash()).Msg("waiting for the last tx to get mined")
+		log.Info().Stringer("hash", lastTx.Hash()).Msg("Waiting for the last transaction to get mined")
 		err := WaitMineTransaction(cmd.Context(), rpcClient, lastTx, 600)
 		if err != nil {
 			log.Error().Err(err).Msg("Unable to wait for last tx to get mined")
@@ -261,10 +261,10 @@ func WaitMineTransaction(ctx context.Context, client *ethclient.Client, tx *type
 				continue
 			}
 			if r.Status != 0 {
-				log.Info().Stringer("hash", r.TxHash).Msg("transaction successful")
+				log.Info().Stringer("hash", r.TxHash).Msg("Transaction successful")
 				return nil
 			} else if r.Status == 0 {
-				log.Error().Stringer("hash", r.TxHash).Msg("transaction failed")
+				log.Error().Stringer("hash", r.TxHash).Msg("Transaction failed")
 				return nil
 			}
 			time.Sleep(1 * time.Second)
@@ -273,22 +273,22 @@ func WaitMineTransaction(ctx context.Context, client *ethclient.Client, tx *type
 }
 
 func getMaxNonce(addr common.Address) (uint64, error) {
-	log.Info().Msg("getting max nonce from txpool_content")
+	log.Info().Msg("Getting max nonce from txpool_content")
 	maxNonce, err := getMaxNonceFromTxPool(addr)
 	if err == nil {
-		log.Info().Uint64("maxNonce", maxNonce).Msg("maxNonce loaded from txpool_content")
+		log.Info().Uint64("maxNonce", maxNonce).Msg("Loaded max nonce from txpool_content")
 		return maxNonce, nil
 	}
-	log.Warn().Err(err).Msg("unable to get max nonce from txpool_content")
+	log.Warn().Err(err).Msg("Unable to get max nonce from txpool_content")
 
-	log.Info().Msg("getting max nonce from pending nonce")
+	log.Info().Msg("Getting max nonce from pending nonce")
 	// if the error is not about txpool_content, falls back to PendingNonceAt
 	maxNonce, err = rpcClient.PendingNonceAt(context.Background(), addr)
 	if err != nil {
 		return 0, err
 	}
 
-	log.Info().Uint64("maxNonce", maxNonce).Msg("maxNonce loaded from pending nonce")
+	log.Info().Uint64("maxNonce", maxNonce).Msg("Loaded max nonce from pending nonce")
 	return maxNonce, nil
 }
 
@@ -318,7 +318,7 @@ func getMaxNonceFromTxPool(addr common.Address) (uint64, error) {
 			nonceInt, ok := new(big.Int).SetString(nonce, 10)
 			if !ok {
 				err = fmt.Errorf("invalid nonce found: %s", nonce)
-				log.Warn().Err(err).Msg("unable to get txpool_content")
+				log.Warn().Err(err).Msg("Unable to get txpool_content")
 				return 0, err
 			}
 

cmd/foldtrace/foldtrace.go

@@ -70,7 +70,7 @@ var FoldTraceCmd = &cobra.Command{
 
 			if isCall(op.OP) {
 				if len(op.Stack) < 6 {
-					log.Warn().Int("stackLength", len(op.Stack)).Msg("detected a call with a stack that's too short")
+					log.Warn().Int("stackLength", len(op.Stack)).Msg("Detected a call with a stack that's too short")
 				} else {
 					currentLabel = op.OP + " to " + op.Stack[len(op.Stack)-2]
 				}
@@ -95,7 +95,7 @@ var FoldTraceCmd = &cobra.Command{
 				folded[currentMetricPath] += metricValue
 			}
 
-			log.Trace().Strs("context", contexts).Uint64("depth", currentDepth).Str("currentLabel", currentLabel).Uint64("pc", op.PC).Msg("trace operation")
+			log.Trace().Strs("context", contexts).Uint64("depth", currentDepth).Str("currentLabel", currentLabel).Uint64("pc", op.PC).Msg("Trace operation")
 
 		}
 		for context, metric := range folded {
@@ -125,7 +125,7 @@ func getActualUsedGas(index int, td *TraceData) uint64 {
 
 		return op.Gas - td.StructLogs[i].Gas
 	}
-	log.Warn().Uint64("pc", op.PC).Uint64("depth", op.Depth).Msg("unable to look ahead for gas use")
+	log.Warn().Uint64("pc", op.PC).Uint64("depth", op.Depth).Msg("Unable to look ahead for gas use")
 	return op.GasCost
 }
 

cmd/fund/fund.go

@@ -82,7 +82,7 @@ func runFunding(ctx context.Context) error {
 	}
 
 	// Fund wallets.
-	log.Debug().Msg("checking if multicall3 is supported")
+	log.Debug().Msg("Checking if multicall3 is supported")
 	var multicall3Addr *common.Address
 	if len(params.Multicall3Address) > 0 {
 		addr := common.HexToAddress(params.Multicall3Address)
@@ -100,7 +100,7 @@ func runFunding(ctx context.Context) error {
 			Msg("multicall3 is supported and will be used to fund wallets")
 		err = fundWalletsWithMulticall3(ctx, c, tops, addresses, multicall3Addr)
 	} else {
-		log.Info().Msg("multicall3 is not supported, will use funder contract to fund wallets")
+		log.Info().Msg("Multicall3 is not supported, will use funder contract to fund wallets")
 		err = fundWalletsWithFunder(ctx, c, tops, privateKey, addresses, privateKeys)
 	}
 	if err != nil {
@@ -378,7 +378,7 @@ func fundWalletsWithMulticall3(ctx context.Context, c *ethclient.Client, tops *b
 	if params.AccountsPerFundingTx > 0 && params.AccountsPerFundingTx < accsToFundPerTx {
 		accsToFundPerTx = params.AccountsPerFundingTx
 	}
-	log.Debug().Uint64("accsToFundPerTx", accsToFundPerTx).Msg("multicall3 max accounts to fund per tx")
+	log.Debug().Uint64("accsToFundPerTx", accsToFundPerTx).Msg("Multicall3 max accounts to fund per tx")
 	chSize := (uint64(len(wallets)) / accsToFundPerTx) + 1
 
 	var txsCh chan *types.Transaction
@@ -412,7 +412,7 @@ func fundWalletsWithMulticall3(ctx context.Context, c *ethclient.Client, tops *b
 				if rl != nil {
 					iErr = rl.Wait(ctx)
 					if iErr != nil {
-						log.Error().Err(iErr).Msg("rate limiter wait failed before funding accounts with multicall3")
+						log.Error().Err(iErr).Msg("Rate limiter wait failed before funding accounts with multicall3")
 						return
 					}
 				}
@@ -434,7 +434,7 @@ func fundWalletsWithMulticall3(ctx context.Context, c *ethclient.Client, tops *b
 				}
 				if iErr != nil {
 					errCh <- iErr
-					log.Error().Err(iErr).Msg("failed to fund accounts with multicall3")
+					log.Error().Err(iErr).Msg("Failed to fund accounts with multicall3")
 					return
 				}
 				log.Info().
@@ -465,7 +465,7 @@ func fundWalletsWithMulticall3(ctx context.Context, c *ethclient.Client, tops *b
 		return combinedErrors
 	}
 
-	log.Info().Msg("all funding transactions sent, waiting for confirmation...")
+	log.Info().Msg("All funding transactions sent, waiting for confirmation")
 
 	// ensure the txs to fund sending accounts using multicall3 were mined successfully
 	for tx := range txsCh {
@@ -478,7 +478,7 @@ func fundWalletsWithMulticall3(ctx context.Context, c *ethclient.Client, tops *b
 
 		r, err := util.WaitReceipt(ctx, c, tx.Hash())
 		if err != nil {
-			log.Error().Err(err).Msg("failed to wait for transaction to fund accounts with multicall3")
+			log.Error().Err(err).Msg("Failed to wait for transaction to fund accounts with multicall3")
 			return err
 		}
 		if r == nil || r.Status != types.ReceiptStatusSuccessful {

cmd/hash/hash.go

@@ -193,7 +193,7 @@ func (p *poseidongoldWrapper) Sum(b []byte) []byte {
 	var err error
 	for {
 		n, _ := p.b.Read(buf)
-		log.Info().Bytes("current-buffer", buf).Msg("summing")
+		log.Info().Bytes("current-buffer", buf).Msg("Summing")
 		if n > 64 {
 			panic("What?? that shouldn't have happened")
 		}
@@ -205,7 +205,7 @@ func (p *poseidongoldWrapper) Sum(b []byte) []byte {
 			os.Exit(1)
 		}
 		if n < 64 {
-			log.Info().Int("n", n).Msg("done")
+			log.Info().Int("n", n).Msg("Done")
 			break
 		}
 

cmd/loadtest/cmd.go

@@ -36,15 +36,14 @@ var uniswapCfg = &config.UniswapV3Config{}
 
 // gasManagerCfg holds gas manager configuration.
 var gasManagerCfg = &config.GasManagerConfig{}
-var gasManagerEnabled bool
 
 // LoadtestCmd represents the loadtest command.
 var LoadtestCmd = &cobra.Command{
 	Use:   "loadtest",
 	Short: "Run a generic load test against an Eth/EVM style JSON-RPC endpoint.",
 	Long:  loadtestUsage,
 	Args:  cobra.NoArgs,
-	PreRunE: func(cmd *cobra.Command, args []string) (err error) {
+	PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) {
 		cfg.RPCURL, err = flag.GetRPCURL(cmd)
 		if err != nil {
 			return err
@@ -56,20 +55,24 @@ var LoadtestCmd = &cobra.Command{
 		zerolog.DurationFieldUnit = time.Second
 		zerolog.DurationFieldInteger = true
 
-		if gasManagerEnabled {
+		cfg.Headers, err = config.ParseRPCHeaders(cfg.RPCHeaders)
+		if err != nil {
+			return err
+		}
+
+		if gasManagerCfg.Enabled {
 			if err = gasManagerCfg.Validate(); err != nil {
 				return err
 			}
+			cfg.GasManager = gasManagerCfg
 		}
 
+		return nil
+	},
+	PreRunE: func(cmd *cobra.Command, args []string) error {
 		return cfg.Validate()
 	},
 	RunE: func(cmd *cobra.Command, args []string) error {
-		// Attach gas manager config only when explicitly enabled.
-		cfg.GasManager = nil
-		if gasManagerEnabled {
-			cfg.GasManager = gasManagerCfg
-		}
 		return loadtest.Run(cmd.Context(), cfg)
 	},
 }
@@ -87,11 +90,6 @@ var uniswapv3Cmd = &cobra.Command{
 		// Override mode to uniswapv3 and attach configs.
 		cfg.Modes = []string{"v3"}
 		cfg.UniswapV3 = uniswapCfg
-		cfg.GasManager = nil
-		if gasManagerEnabled {
-			cfg.GasManager = gasManagerCfg
-		}
-
 		return loadtest.Run(cmd.Context(), cfg)
 	},
 }
@@ -115,6 +113,7 @@ func initPersistentFlags() {
 	pf.BoolVar(&cfg.EthCallOnly, "eth-call-only", false, "call contracts without sending transactions (incompatible with adaptive rate limiting and summarization)")
 	pf.BoolVar(&cfg.EthCallOnlyLatestBlock, "eth-call-only-latest", false, "execute on latest block instead of original block in call-only mode with recall")
 	pf.BoolVar(&cfg.OutputRawTxOnly, "output-raw-tx-only", false, "output raw signed transaction hex without sending (works with most modes except RPC and UniswapV3)")
+	pf.BoolVar(&cfg.PrivateTxs, "private-txs", false, "send transactions via eth_sendRawTransactionPrivate")
 	pf.Uint64Var(&cfg.EthAmountInWei, "eth-amount-in-wei", 0, "amount of ether in wei to send per transaction")
 	pf.Float64Var(&cfg.RateLimit, "rate-limit", 4, "requests per second limit (use negative value to remove limit)")
 	pf.BoolVar(&cfg.AdaptiveRateLimit, "adaptive-rate-limit", false, "enable AIMD-style congestion control to automatically adjust request rate")
@@ -125,9 +124,9 @@ func initPersistentFlags() {
 	pf.Float64Var(&cfg.GasPriceMultiplier, "gas-price-multiplier", 1, "a multiplier to increase or decrease the gas price")
 	pf.Int64Var(&cfg.Seed, "seed", 123456, "a seed for generating random values and addresses")
 	pf.Uint64Var(&cfg.ForceGasLimit, "gas-limit", 0, "manually specify gas limit (useful to avoid eth_estimateGas or when auto-computation fails)")
-	pf.Uint64Var(&cfg.ForceGasPrice, "gas-price", 0, "manually specify gas price (useful when auto-detection fails)")
+	pf.Var(&flag.GasValue{Val: &cfg.ForceGasPrice}, "gas-price", "gas price with unit support (e.g., \"100gwei\", \"1000000000\")")
 	pf.Uint64Var(&cfg.StartNonce, "nonce", 0, "use this flag to manually set the starting nonce")
-	pf.Uint64Var(&cfg.ForcePriorityGasPrice, "priority-gas-price", 0, "gas tip price for EIP-1559 transactions")
+	pf.Var(&flag.GasValue{Val: &cfg.ForcePriorityGasPrice}, "priority-gas-price", "gas tip for EIP-1559 with unit support (e.g., \"2gwei\")")
 	pf.BoolVar(&cfg.ShouldProduceSummary, "summarize", false, "produce execution summary after load test (can take a long time for large tests)")
 	pf.Uint64Var(&cfg.BatchSize, "batch-size", 999, "batch size for receipt fetching (default: 999)")
 	pf.StringVar(&cfg.SummaryOutputMode, "output-mode", "text", "format mode for summary output (json | text)")
@@ -136,13 +135,15 @@ func initPersistentFlags() {
 	pf.BoolVar(&cfg.FireAndForget, "send-only", false, "alias for --fire-and-forget")
 	pf.BoolVar(&cfg.CheckForPreconf, "check-preconf", false, "check for preconf status after sending tx")
 	pf.StringVar(&cfg.PreconfStatsFile, "preconf-stats-file", "", "path for preconf stats JSON output, updated every 2 seconds")
+	pf.BoolVar(&cfg.StopOnInsufficientFunds, "stop-on-insufficient-funds", false, "stop sending from account when it encounters insufficient funds error")
+	pf.StringVar(&cfg.RPCHeaders, "rpc-headers", "", "custom HTTP headers for RPC requests (format: \"key1:value1,key2:value2\")")
 
 	initGasManagerFlags()
 }
 
 func initGasManagerFlags() {
 	pf := LoadtestCmd.PersistentFlags()
-	pf.BoolVar(&gasManagerEnabled, "gas-manager-enabled", false, "enable block-based gas manager (oscillation wave + gas budget vault)")
+	pf.BoolVar(&gasManagerCfg.Enabled, "gas-manager-enabled", false, "enable block-based gas manager (oscillation wave + gas budget vault)")
 
 	// Oscillation wave
 	pf.StringVar(&gasManagerCfg.OscillationWave, "gas-manager-oscillation-wave", "flat", "type of oscillation wave (flat | sine | square | triangle | sawtooth)")
@@ -167,6 +168,7 @@ func initFlags() {
 	f.StringVar(&cfg.SendingAccountsFile, "sending-accounts-file", "", "file with sending account private keys, one per line (avoids pool queue and preserves accounts across runs)")
 	f.StringVar(&cfg.DumpSendingAccountsFile, "dump-sending-accounts-file", "", "file path to dump generated private keys when using --sending-accounts-count")
 	f.Uint64Var(&cfg.AccountsPerFundingTx, "accounts-per-funding-tx", 400, "number of accounts to fund per multicall3 transaction")
+	f.BoolVar(&cfg.SequentialNonceFetch, "sequential-nonce-fetch", false, "fetch nonces sequentially instead of in parallel (use if hitting rate limits)")
 	f.Uint64Var(&cfg.MaxBaseFeeWei, "max-base-fee-wei", 0, "maximum base fee in wei (pause sending new transactions when exceeded, useful during network congestion)")
 	f.StringSliceVarP(&cfg.Modes, "mode", "m", []string{"t"}, `testing mode (can specify multiple like "d,t"):
 2, erc20 - send ERC20 tokens

cmd/monitor/monitor.go

@@ -452,7 +452,7 @@ func (ms *monitorStatus) processBatchesConcurrently(ctx context.Context, rpc *et
 				return nil
 			}
 			if err := backoff.Retry(retryable, b); err != nil {
-				log.Error().Err(err).Msg("unable to retry")
+				log.Error().Err(err).Msg("Unable to retry")
 				errorsMutex.Lock()
 				errs = append(errs, err)
 				errorsMutex.Unlock()

cmd/monitor/ui/ui.go

@@ -460,7 +460,7 @@ func waitForReceipt(ctx context.Context, rpcClient *ethrpc.Client, txHash string
 		}
 
 		if result.TransactionHash == "" {
-			log.Info().Msg("Receipt not found, waiting more...")
+			log.Info().Msg("Receipt not found, waiting more")
 			time.Sleep(2 * time.Second)
 			continue
 		}

cmd/monitorv2/monitorv2.go

@@ -55,7 +55,7 @@ var MonitorV2Cmd = &cobra.Command{
 			go func() {
 				log.Info().Str("addr", pprofAddr).Msg("Starting pprof server")
 				if err := http.ListenAndServe(pprofAddr, nil); err != nil {
-					log.Error().Err(err).Msg("pprof server failed")
+					log.Error().Err(err).Msg("The pprof server failed")
 				}
 			}()
 		}