add triedb busy error

arbitrum/apibackend.go

@@ -532,7 +532,7 @@ func (a *APIBackend) NewMatcherBackend() filtermaps.MatcherBackend {
 	return a.b.filterMaps.NewMatcherBackend()
 }
 
-func StateAndHeaderFromHeader(ctx context.Context, chainDb ethdb.Database, bc *core.BlockChain, maxRecreateStateDepth int64, header *types.Header, err error, archiveClientsManager *archiveFallbackClientsManager) (*state.StateDB, *types.Header, error) {
+func StateAndHeaderFromHeader(ctx context.Context, chainDb ethdb.Database, bc *core.BlockChain, maxRecreateStateDepth int64, errorWhenTriedbBusy bool, header *types.Header, err error, archiveClientsManager *archiveFallbackClientsManager) (*state.StateDB, *types.Header, error) {
 	if err != nil {
 		return nil, header, err
 	}
@@ -545,6 +545,9 @@ func StateAndHeaderFromHeader(ctx context.Context, chainDb ethdb.Database, bc *c
 	if archiveClientsManager != nil && header.Number.Uint64() <= archiveClientsManager.lastAvailableBlock() {
 		return nil, header, &types.ErrUseArchiveFallback{BlockNum: header.Number.Uint64()}
 	}
+	if errorWhenTriedbBusy && bc.StateCache().TrieDB().IsBusyCommitting() {
+		return nil, header, errors.New("please retry, triedb is busy committing state")
+	}
 	stateFor := func(db state.Database, snapshots *snapshot.Tree) func(header *types.Header) (*state.StateDB, StateReleaseFunc, error) {
 		return func(header *types.Header) (*state.StateDB, StateReleaseFunc, error) {
 			if header.Root != (common.Hash{}) {
@@ -618,7 +621,7 @@ func StateAndHeaderFromHeader(ctx context.Context, chainDb ethdb.Database, bc *c
 
 func (a *APIBackend) StateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error) {
 	header, err := a.HeaderByNumber(ctx, number)
-	return StateAndHeaderFromHeader(ctx, a.ChainDb(), a.b.arb.BlockChain(), a.b.config.MaxRecreateStateDepth, header, err, a.archiveClientsManager)
+	return StateAndHeaderFromHeader(ctx, a.ChainDb(), a.b.arb.BlockChain(), a.b.config.MaxRecreateStateDepth, a.b.config.ErrorWhenTriedbBusy, header, err, a.archiveClientsManager)
 }
 
 func (a *APIBackend) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error) {
@@ -629,7 +632,7 @@ func (a *APIBackend) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOr
 	if ishash && header != nil && header.Number.Cmp(bc.CurrentBlock().Number) > 0 && bc.GetCanonicalHash(header.Number.Uint64()) != hash {
 		return nil, nil, errors.New("requested block ahead of current block and the hash is not currently canonical")
 	}
-	return StateAndHeaderFromHeader(ctx, a.ChainDb(), a.b.arb.BlockChain(), a.b.config.MaxRecreateStateDepth, header, err, a.archiveClientsManager)
+	return StateAndHeaderFromHeader(ctx, a.ChainDb(), a.b.arb.BlockChain(), a.b.config.MaxRecreateStateDepth, a.b.config.ErrorWhenTriedbBusy, header, err, a.archiveClientsManager)
 }
 
 func (a *APIBackend) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, checkLive bool, preferDisk bool) (statedb *state.StateDB, release tracers.StateReleaseFunc, err error) {

arbitrum/config.go

@@ -44,6 +44,7 @@ type Config struct {
 	ClassicRedirect        string        `koanf:"classic-redirect"`
 	ClassicRedirectTimeout time.Duration `koanf:"classic-redirect-timeout"`
 	MaxRecreateStateDepth  int64         `koanf:"max-recreate-state-depth"`
+	ErrorWhenTriedbBusy    bool          `koanf:"error-when-triedb-busy"`
 
 	AllowMethod []string `koanf:"allow-method"`
 
@@ -89,6 +90,7 @@ func ConfigAddOptions(prefix string, f *flag.FlagSet) {
 	f.Int(prefix+".filter-log-cache-size", DefaultConfig.FilterLogCacheSize, "log filter system maximum number of cached blocks")
 	f.Duration(prefix+".filter-timeout", DefaultConfig.FilterTimeout, "log filter system maximum time filters stay active")
 	f.Int64(prefix+".max-recreate-state-depth", DefaultConfig.MaxRecreateStateDepth, "maximum depth for recreating state, measured in l2 gas (0=don't recreate state, -1=infinite, -2=use default value for archive or non-archive node (whichever is configured))")
+	f.Bool(prefix+".error-when-triedb-busy", DefaultConfig.ErrorWhenTriedbBusy, "if enabled, rpc calls requiring state access will return an error when triedb is busy committing state; otherwise the request will wait and might timeout")
 	f.StringSlice(prefix+".allow-method", DefaultConfig.AllowMethod, "list of whitelisted rpc methods")
 	arbDebug := DefaultConfig.ArbDebug
 	f.Uint64(prefix+".arbdebug.block-range-bound", arbDebug.BlockRangeBound, "bounds the number of blocks arbdebug calls may return")
@@ -114,6 +116,7 @@ var DefaultConfig = Config{
 	FeeHistoryMaxBlockCount: 1024,
 	ClassicRedirect:         "",
 	MaxRecreateStateDepth:   UninitializedMaxRecreateStateDepth, // default value should be set for depending on node type (archive / non-archive)
+	ErrorWhenTriedbBusy:     false,
 	AllowMethod:             []string{},
 	ArbDebug: ArbDebugConfig{
 		BlockRangeBound:   256,

triedb/database.go

@@ -332,3 +332,13 @@ func (db *Database) IsVerkle() bool {
 func (db *Database) Disk() ethdb.Database {
 	return db.disk
 }
+
+// HashScheme: check if we are busy committing and holding main db lock for a longer time
+// PathScheme: always false (not necessary)
+func (db *Database) IsBusyCommitting() bool {
+	hdb, ok := db.backend.(*hashdb.Database)
+	if !ok {
+		return false
+	}
+	return hdb.IsBusyCommitting()
+}

triedb/hashdb/database.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"reflect"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/VictoriaMetrics/fastcache"
@@ -78,6 +79,9 @@ var Defaults = &Config{
 // the disk database. The aim is to accumulate trie writes in-memory and only
 // periodically flush a couple tries to disk, garbage collecting the remainder.
 type Database struct {
+	// Arbitrum:
+	committing atomic.Bool
+
 	diskdb  ethdb.Database              // Persistent storage for matured trie nodes
 	cleans  *fastcache.Cache            // GC friendly memory cache of clean node RLPs
 	dirties map[common.Hash]*cachedNode // Data and references relationships of dirty trie nodes
@@ -410,6 +414,9 @@ func (db *Database) Commit(node common.Hash, report bool) error {
 	db.lock.Lock()
 	defer db.lock.Unlock()
 
+	db.committing.Store(true)
+	defer db.committing.Store(false)
+
 	// Create a database batch to flush persistent data out. It is important that
 	// outside code doesn't see an inconsistent state (referenced data removed from
 	// memory cache during commit but not yet in persistent storage). This is ensured
@@ -644,3 +651,9 @@ func (reader *reader) Node(owner common.Hash, path []byte, hash common.Hash) ([]
 func (db *Database) StateReader(root common.Hash) (database.StateReader, error) {
 	return nil, errors.New("not implemented")
 }
+
+// returns true if hashdb is performing commit in this moment
+// thread safe - can be called concurrently, but does not guarantee that the returned result remains true after the call
+func (db *Database) IsBusyCommitting() bool {
+	return db.committing.Load()
+}