Added pebbledb support to go-quai

This branch was produced by squashing all the commits below

Cherry picked initial pebble db commit - ed51b8c5d -

ethdb/pebble: fix nil callbacks (#26650)

ethdb/pebble: Fix `MemTableStopWritesThreshold` (#26692)

MemTableStopWritesThreshold was set to the max size of all memtables before blocking writing but should be set to the max number of memtables. This is documented [here](https://github.com/cockroachdb/pebble/blob/master/options.go#L738-L742).

ethdb/pebble: fix range compaction (#26771)

* ethdb/pebble: fix range compaction

* ethdb/pebble: add comment

ethdb/pebble: fix max memorytable size (#26776)

core/rawdb, ethdb/pebble: disable pebble on openbsd (#26801)

ethdb/pebble: use atomic type (#27014)

ethdb/pebble: prevent shutdown-panic (#27238)

One difference between pebble and leveldb is that the latter returns error when performing Get on a closed database, the former does a panic. This may be triggered during shutdown (see #27237)

This PR changes the pebble driver so we check that the db is not closed already, for several operations. It also adds tests to the db test-suite, so the previously implicit assumption of "not panic:ing at ops on closed database" is covered by tests.

ethdb/pebble: fix NewBatchWithSize to set db (#27350)

ethdb/pebble: fsync for batch writes (#27522)

This is likely the culprit behind several data corruption issues, e.g. where data has been
written to the freezer, but the deletion from pebble does not go through due to process
crash.

ethdb/pebble: use sync mode for pebble writes (#27615)

Makefile

@@ -42,6 +42,7 @@ BASE_CMD = nice -n -20 ./build/bin/go-quai --$(NETWORK) --syncmode $(SYNCMODE) -
 BASE_CMD += --http --http.vhosts=* --http.addr $(HTTP_ADDR) --http.api $(HTTP_API)
 BASE_CMD += --ws --ws.addr $(WS_ADDR) --ws.api $(WS_API)
 BASE_CMD += --slices $(SLICES)
+BASE_CMD += --db.engine $(DB_ENGINE)
 ifeq ($(ENABLE_ARCHIVE),true)
 	BASE_CMD += --gcmode archive
 endif

cmd/go-quai/genesis_test.go

@@ -17,9 +17,11 @@
 package main
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strconv"
 	"testing"
 )
 
@@ -68,6 +70,7 @@ var customGenesisTests = []struct {
 // Tests that initializing Quai with a custom genesis block and chain definitions
 // work properly.
 func TestCustomGenesis(t *testing.T) {
+	t.Parallel()
 	for i, tt := range customGenesisTests {
 		// Create a temporary data directory to use and inspect later
 		datadir := tmpdir(t)
@@ -89,3 +92,101 @@ func TestCustomGenesis(t *testing.T) {
 		quai.ExpectExit()
 	}
 }
+
+// TestCustomBackend that the backend selection and detection (leveldb vs pebble) works properly.
+func TestCustomBackend(t *testing.T) {
+	t.Parallel()
+	// Test pebble, but only on 64-bit platforms
+	if strconv.IntSize != 64 {
+		t.Skip("Custom backends are only available on 64-bit platform")
+	}
+	genesis := `{
+		"alloc"      : {},
+		"coinbase"   : "0x0000000000000000000000000000000000000000",
+			"difficulty" : "0x20000",
+			"extraData"  : "",
+			"gasLimit"   : "0x2fefd8",
+			"nonce"      : "0x0000000000001338",
+			"mixhash"    : "0x0000000000000000000000000000000000000000000000000000000000000000",
+			"parentHash" : "0x0000000000000000000000000000000000000000000000000000000000000000",
+			"timestamp"  : "0x00",
+			"config"     : {}
+	}`
+	type backendTest struct {
+		initArgs   []string
+		initExpect string
+		execArgs   []string
+		execExpect string
+	}
+	testfunc := func(t *testing.T, tt backendTest) error {
+		// Create a temporary data directory to use and inspect later
+		datadir := t.TempDir()
+
+		// Initialize the data directory with the custom genesis block
+		json := filepath.Join(datadir, "genesis.json")
+		if err := os.WriteFile(json, []byte(genesis), 0600); err != nil {
+			return fmt.Errorf("failed to write genesis file: %v", err)
+		}
+		{ // Init
+			args := append(tt.initArgs, "--datadir", datadir, "init", json)
+			geth := runGeth(t, args...)
+			geth.ExpectRegexp(tt.initExpect)
+			geth.ExpectExit()
+		}
+		{ // Exec + query
+			args := append(tt.execArgs, "--networkid", "1337", "--syncmode=full", "--cache", "16",
+				"--datadir", datadir, "--maxpeers", "0", "--port", "0", "--authrpc.port", "0",
+				"--nodiscover", "--nat", "none", "--ipcdisable",
+				"--exec", "eth.getBlock(0).nonce", "console")
+			geth := runGeth(t, args...)
+			geth.ExpectRegexp(tt.execExpect)
+			geth.ExpectExit()
+		}
+		return nil
+	}
+	for i, tt := range []backendTest{
+		{ // When not specified, it should default to leveldb
+			execArgs:   []string{"--db.engine", "leveldb"},
+			execExpect: "0x0000000000001338",
+		},
+		{ // Explicit leveldb
+			initArgs:   []string{"--db.engine", "leveldb"},
+			execArgs:   []string{"--db.engine", "leveldb"},
+			execExpect: "0x0000000000001338",
+		},
+		{ // Explicit leveldb first, then autodiscover
+			initArgs:   []string{"--db.engine", "leveldb"},
+			execExpect: "0x0000000000001338",
+		},
+		{ // Explicit pebble
+			initArgs:   []string{"--db.engine", "pebble"},
+			execArgs:   []string{"--db.engine", "pebble"},
+			execExpect: "0x0000000000001338",
+		},
+		{ // Explicit pebble, then auto-discover
+			initArgs:   []string{"--db.engine", "pebble"},
+			execExpect: "0x0000000000001338",
+		},
+		{ // Can't start pebble on top of leveldb
+			initArgs:   []string{"--db.engine", "leveldb"},
+			execArgs:   []string{"--db.engine", "pebble"},
+			execExpect: `Fatal: Failed to register the Ethereum service: db.engine choice was pebble but found pre-existing leveldb database in specified data directory`,
+		},
+		{ // Can't start leveldb on top of pebble
+			initArgs:   []string{"--db.engine", "pebble"},
+			execArgs:   []string{"--db.engine", "leveldb"},
+			execExpect: `Fatal: Failed to register the Ethereum service: db.engine choice was leveldb but found pre-existing pebble database in specified data directory`,
+		},
+		{ // Reject invalid backend choice
+			initArgs:   []string{"--db.engine", "mssql"},
+			initExpect: `Fatal: Invalid choice for db.engine 'mssql', allowed 'leveldb' or 'pebble'`,
+			// Since the init fails, this will return the (default) mainnet genesis
+			// block nonce
+			execExpect: `0x0000000000000042`,
+		},
+	} {
+		if err := testfunc(t, tt); err != nil {
+			t.Fatalf("test %d-leveldb: %v", i, err)
+		}
+	}
+}

cmd/go-quai/main.go

@@ -68,6 +68,7 @@ var (
 		utils.ConsensusEngineFlag,
 		utils.DNSDiscoveryFlag,
 		utils.DataDirFlag,
+		utils.DBEngineFlag,
 		utils.DeveloperFlag,
 		utils.DeveloperPeriodFlag,
 		utils.DiscoveryV5Flag,

cmd/utils/flags.go

@@ -117,6 +117,11 @@ var (
 		Name:  "datadir.minfreedisk",
 		Usage: "Minimum free disk space in MB, once reached triggers auto shut down (default = --cache.gc converted to MB, 0 = disabled)",
 	}
+	DBEngineFlag = &cli.StringFlag{
+		Name:  "db.engine",
+		Usage: "Backing database implementation to use ('leveldb' or 'pebble')",
+		Value: "pebble",
+	}
 	KeyStoreDirFlag = DirectoryFlag{
 		Name:  "keystore",
 		Usage: "Directory for the keystore (default = inside the datadir)",
@@ -653,6 +658,20 @@ var (
 	}
 )
 
+var (
+	// DatabasePathFlags is the flag group of all database path flags.
+	DatabasePathFlags = []cli.Flag{
+		DataDirFlag,
+		AncientFlag,
+	}
+)
+
+func init() {
+	if rawdb.PebbleEnabled {
+		DatabasePathFlags = append(DatabasePathFlags, DBEngineFlag)
+	}
+}
+
 // MakeDataDir retrieves the currently requested data directory, terminating
 // if none (or the empty string) is specified. If the node is starting a testnet,
 // then a subdirectory of the specified datadir will be used.
@@ -1115,6 +1134,14 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) {
 	if ctx.GlobalIsSet(InsecureUnlockAllowedFlag.Name) {
 		cfg.InsecureUnlockAllowed = ctx.GlobalBool(InsecureUnlockAllowedFlag.Name)
 	}
+	if ctx.IsSet(DBEngineFlag.Name) {
+		dbEngine := ctx.String(DBEngineFlag.Name)
+		if dbEngine != "leveldb" && dbEngine != "pebble" {
+			Fatalf("Invalid choice for db.engine '%s', allowed 'leveldb' or 'pebble'", dbEngine)
+		}
+		log.Info(fmt.Sprintf("Using %s as db engine", dbEngine))
+		cfg.DBEngine = dbEngine
+	}
 }
 
 func setDataDir(ctx *cli.Context, cfg *node.Config) {

core/rawdb/database.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"path/filepath"
 	"sync/atomic"
 	"time"
 
@@ -229,17 +230,84 @@ func NewLevelDBDatabase(file string, cache int, handles int, namespace string, r
 	if err != nil {
 		return nil, err
 	}
+	log.Info("Using LevelDB as the backing database")
 	return NewDatabase(db), nil
 }
 
-// NewLevelDBDatabaseWithFreezer creates a persistent key-value database with a
-// freezer moving immutable chain segments into cold storage.
-func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, freezer string, namespace string, readonly bool) (ethdb.Database, error) {
-	kvdb, err := leveldb.New(file, cache, handles, namespace, readonly)
+const (
+	dbPebble  = "pebble"
+	dbLeveldb = "leveldb"
+)
+
+// hasPreexistingDb checks the given data directory whether a database is already
+// instantiated at that location, and if so, returns the type of database (or the
+// empty string).
+func hasPreexistingDb(path string) string {
+	if _, err := os.Stat(filepath.Join(path, "CURRENT")); err != nil {
+		return "" // No pre-existing db
+	}
+	if matches, err := filepath.Glob(filepath.Join(path, "OPTIONS*")); len(matches) > 0 || err != nil {
+		if err != nil {
+			panic(err) // only possible if the pattern is malformed
+		}
+		return dbPebble
+	}
+	return dbLeveldb
+}
+
+// OpenOptions contains the options to apply when opening a database.
+// OBS: If AncientsDirectory is empty, it indicates that no freezer is to be used.
+type OpenOptions struct {
+	Type              string // "leveldb" | "pebble"
+	Directory         string // the datadir
+	AncientsDirectory string // the ancients-dir
+	Namespace         string // the namespace for database relevant metrics
+	Cache             int    // the capacity(in megabytes) of the data caching
+	Handles           int    // number of files to be open simultaneously
+	ReadOnly          bool
+}
+
+// openKeyValueDatabase opens a disk-based key-value database, e.g. leveldb or pebble.
+//
+//	                      type == null          type != null
+//	                   +----------------------------------------
+//	db is non-existent |  leveldb default  |  specified type
+//	db is existent     |  from db          |  specified type (if compatible)
+func openKeyValueDatabase(o OpenOptions) (ethdb.Database, error) {
+	existingDb := hasPreexistingDb(o.Directory)
+	if len(existingDb) != 0 && len(o.Type) != 0 && o.Type != existingDb {
+		return nil, fmt.Errorf("db.engine choice was %v but found pre-existing %v database in specified data directory", o.Type, existingDb)
+	}
+	if o.Type == dbPebble || existingDb == dbPebble {
+		if PebbleEnabled {
+			log.Info("Using pebble as the backing database")
+			return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly)
+		} else {
+			return nil, errors.New("db.engine 'pebble' not supported on this platform")
+		}
+	}
+	if len(o.Type) != 0 && o.Type != dbLeveldb {
+		return nil, fmt.Errorf("unknown db.engine %v", o.Type)
+	}
+	log.Info("Using leveldb as the backing database")
+	// Use leveldb, either as default (no explicit choice), or pre-existing, or chosen explicitly
+	return NewLevelDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly)
+}
+
+// Open opens both a disk-based key-value database such as leveldb or pebble, but also
+// integrates it with a freezer database -- if the AncientDir option has been
+// set on the provided OpenOptions.
+// The passed o.AncientDir indicates the path of root ancient directory where
+// the chain freezer can be opened.
+func Open(o OpenOptions) (ethdb.Database, error) {
+	kvdb, err := openKeyValueDatabase(o)
 	if err != nil {
 		return nil, err
 	}
-	frdb, err := NewDatabaseWithFreezer(kvdb, freezer, namespace, readonly)
+	if len(o.AncientsDirectory) == 0 {
+		return kvdb, nil
+	}
+	frdb, err := NewDatabaseWithFreezer(kvdb, o.AncientsDirectory, o.Namespace, o.ReadOnly)
 	if err != nil {
 		kvdb.Close()
 		return nil, err

core/rawdb/databases_64bit.go

@@ -0,0 +1,37 @@
+// Copyright 2023 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>
+
+//go:build (arm64 || amd64) && !openbsd
+
+package rawdb
+
+import (
+	"github.com/dominant-strategies/go-quai/ethdb"
+	"github.com/dominant-strategies/go-quai/ethdb/pebble"
+)
+
+// Pebble is unsuported on 32bit architecture
+const PebbleEnabled = true
+
+// NewPebbleDBDatabase creates a persistent key-value database without a freezer
+// moving immutable chain segments into cold storage.
+func NewPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) {
+	db, err := pebble.New(file, cache, handles, namespace, readonly)
+	if err != nil {
+		return nil, err
+	}
+	return NewDatabase(db), nil
+}

core/rawdb/databases_non64bit.go

@@ -0,0 +1,34 @@
+// Copyright 2023 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+//go:build !((arm64 || amd64) && !openbsd)
+
+package rawdb
+
+import (
+	"errors"
+
+	"github.com/dominant-strategies/go-quai/ethdb"
+)
+
+// Pebble is unsuported on 32bit architecture
+const PebbleEnabled = false
+
+// NewPebbleDBDatabase creates a persistent key-value database without a freezer
+// moving immutable chain segments into cold storage.
+func NewPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) {
+	return nil, errors.New("pebble is not supported on this platform")
+}