added split (sync and async) db connections pool

CHANGELOG.md

@@ -4,11 +4,17 @@
 
 - Added support for SMTP `LOGIN` auth for Microsoft/Outlook and other providers that dont't support the `PLAIN` auth method ([#1217](https://github.com/pocketbase/pocketbase/discussions/1217#discussioncomment-4387970)).
 
-- Reduced memory consumption (~20% improvement).
+- Reduced memory consumption (you can expect ~20% less allocated memory).
+
+- Added support for split (async and sync) DB connections pool increasing even further the concurrent throughput.
 
 - Improved record references delete performance.
 
-- Removed the unnecessary parenthesis in the generated filter SQL query, reducing the "parse stack overflow" errors.
+- Removed the unnecessary parenthesis in the generated filter SQL query, reducing the "_parse stack overflow_" errors.
+
+- Fixed `~` expressions backslash literal escaping ([#1231](https://github.com/pocketbase/pocketbase/discussions/1231)).
+
+- Changed `core.NewBaseApp(dir, encryptionEnv, isDebug)` to `NewBaseApp(config *BaseAppConfig)` which allows to further configure the app instance.
 
 - Removed `rest.UploadedFile` struct (see below `filesystem.File`).
 
@@ -27,9 +33,7 @@
   forms.RecordUpsert.RemoveFiles(key, filenames...)     // marks the filenames for deletion
   ```
 
-- Fixed `LIKE` expressions backslash escaping ([#1231](https://github.com/pocketbase/pocketbase/discussions/1231)).
-
-- Trigger the `password` validators in any of the others password change fields is set.
+- Trigger the `password` validators if any of the others password change fields is set.
 
 
 ## v0.9.2

core/app.go

@@ -16,6 +16,11 @@ import (
 
 // App defines the main PocketBase app interface.
 type App interface {
+	// Deprecated:
+	// This method may get removed in the near future.
+	// It is recommended to access the logs db instance from app.Dao().DB() or
+	// if you want more flexibility - app.Dao().AsyncDB() and app.Dao().SyncDB().
+	//
 	// DB returns the default app database instance.
 	DB() *dbx.DB
 
@@ -26,6 +31,11 @@ type App interface {
 	// trying to access the request logs table will result in error.
 	Dao() *daos.Dao
 
+	// Deprecated:
+	// This method may get removed in the near future.
+	// It is recommended to access the logs db instance from app.LogsDao().DB() or
+	// if you want more flexibility - app.LogsDao().AsyncDB() and app.LogsDao().SyncDB().
+	//
 	// LogsDB returns the app logs database instance.
 	LogsDB() *dbx.DB
 

core/base.go

@@ -26,16 +26,18 @@ var _ App = (*BaseApp)(nil)
 // BaseApp implements core.App and defines the base PocketBase app structure.
 type BaseApp struct {
 	// configurable parameters
-	isDebug       bool
-	dataDir       string
-	encryptionEnv string
+	isDebug          bool
+	dataDir          string
+	encryptionEnv    string
+	dataMaxOpenConns int
+	dataMaxIdleConns int
+	logsMaxOpenConns int
+	logsMaxIdleConns int
 
 	// internals
 	cache               *store.Store[any]
 	settings            *settings.Settings
-	db                  *dbx.DB
 	dao                 *daos.Dao
-	logsDB              *dbx.DB
 	logsDao             *daos.Dao
 	subscriptionsBroker *subscriptions.Broker
 
@@ -132,15 +134,30 @@ type BaseApp struct {
 	onCollectionsAfterImportRequest  *hook.Hook[*CollectionsImportEvent]
 }
 
+// BaseAppConfig defines a BaseApp configuration option
+type BaseAppConfig struct {
+	DataDir          string
+	EncryptionEnv    string
+	IsDebug          bool
+	DataMaxOpenConns int // default to 600
+	DataMaxIdleConns int // default 20
+	LogsMaxOpenConns int // default to 500
+	LogsMaxIdleConns int // default to 10
+}
+
 // NewBaseApp creates and returns a new BaseApp instance
 // configured with the provided arguments.
 //
 // To initialize the app, you need to call `app.Bootstrap()`.
-func NewBaseApp(dataDir string, encryptionEnv string, isDebug bool) *BaseApp {
+func NewBaseApp(config *BaseAppConfig) *BaseApp {
 	app := &BaseApp{
-		dataDir:             dataDir,
-		isDebug:             isDebug,
-		encryptionEnv:       encryptionEnv,
+		dataDir:             config.DataDir,
+		isDebug:             config.IsDebug,
+		encryptionEnv:       config.EncryptionEnv,
+		dataMaxOpenConns:    config.DataMaxOpenConns,
+		dataMaxIdleConns:    config.DataMaxIdleConns,
+		logsMaxOpenConns:    config.LogsMaxOpenConns,
+		logsMaxIdleConns:    config.LogsMaxIdleConns,
 		cache:               store.New[any](nil),
 		settings:            settings.New(),
 		subscriptionsBroker: subscriptions.NewBroker(),
@@ -283,14 +300,20 @@ func (app *BaseApp) Bootstrap() error {
 // ResetBootstrapState takes care for releasing initialized app resources
 // (eg. closing db connections).
 func (app *BaseApp) ResetBootstrapState() error {
-	if app.db != nil {
-		if err := app.db.Close(); err != nil {
+	if app.Dao() != nil {
+		if err := app.Dao().AsyncDB().(*dbx.DB).Close(); err != nil {
+			return err
+		}
+		if err := app.Dao().SyncDB().(*dbx.DB).Close(); err != nil {
 			return err
 		}
 	}
 
-	if app.logsDB != nil {
-		if err := app.logsDB.Close(); err != nil {
+	if app.LogsDao() != nil {
+		if err := app.LogsDao().AsyncDB().(*dbx.DB).Close(); err != nil {
+			return err
+		}
+		if err := app.LogsDao().SyncDB().(*dbx.DB).Close(); err != nil {
 			return err
 		}
 	}
@@ -302,19 +325,47 @@ func (app *BaseApp) ResetBootstrapState() error {
 	return nil
 }
 
+// Deprecated:
+// This method may get removed in the near future.
+// It is recommended to access the db instance from app.Dao().DB() or
+// if you want more flexibility - app.Dao().AsyncDB() and app.Dao().SyncDB().
+//
 // DB returns the default app database instance.
 func (app *BaseApp) DB() *dbx.DB {
-	return app.db
+	if app.Dao() == nil {
+		return nil
+	}
+
+	db, ok := app.Dao().DB().(*dbx.DB)
+	if !ok {
+		return nil
+	}
+
+	return db
 }
 
 // Dao returns the default app Dao instance.
 func (app *BaseApp) Dao() *daos.Dao {
 	return app.dao
 }
 
+// Deprecated:
+// This method may get removed in the near future.
+// It is recommended to access the logs db instance from app.LogsDao().DB() or
+// if you want more flexibility - app.LogsDao().AsyncDB() and app.LogsDao().SyncDB().
+//
 // LogsDB returns the app logs database instance.
 func (app *BaseApp) LogsDB() *dbx.DB {
-	return app.logsDB
+	if app.LogsDao() == nil {
+		return nil
+	}
+
+	db, ok := app.LogsDao().DB().(*dbx.DB)
+	if !ok {
+		return nil
+	}
+
+	return db
 }
 
 // LogsDao returns the app logs Dao instance.
@@ -751,41 +802,81 @@ func (app *BaseApp) OnCollectionsAfterImportRequest() *hook.Hook[*CollectionsImp
 // -------------------------------------------------------------------
 
 func (app *BaseApp) initLogsDB() error {
-	var connectErr error
-	app.logsDB, connectErr = connectDB(filepath.Join(app.DataDir(), "logs.db"))
-	if connectErr != nil {
-		return connectErr
+	maxOpenConns := 500
+	maxIdleConns := 10
+	if app.logsMaxOpenConns > 0 {
+		maxOpenConns = app.logsMaxOpenConns
+	}
+	if app.logsMaxIdleConns > 0 {
+		maxIdleConns = app.logsMaxIdleConns
 	}
 
-	app.logsDao = daos.New(app.logsDB)
+	asyncDB, err := connectDB(filepath.Join(app.DataDir(), "logs.db"))
+	if err != nil {
+		return err
+	}
+	asyncDB.DB().SetMaxOpenConns(maxOpenConns)
+	asyncDB.DB().SetMaxIdleConns(maxIdleConns)
+	asyncDB.DB().SetConnMaxIdleTime(5 * time.Minute)
+
+	syncDB, err := connectDB(filepath.Join(app.DataDir(), "logs.db"))
+	if err != nil {
+		return err
+	}
+	syncDB.DB().SetMaxOpenConns(1)
+	syncDB.DB().SetMaxIdleConns(1)
+	syncDB.DB().SetConnMaxIdleTime(5 * time.Minute)
+
+	app.logsDao = daos.NewMultiDB(asyncDB, syncDB)
 
 	return nil
 }
 
 func (app *BaseApp) initDataDB() error {
-	var connectErr error
-	app.db, connectErr = connectDB(filepath.Join(app.DataDir(), "data.db"))
-	if connectErr != nil {
-		return connectErr
+	maxOpenConns := 600
+	maxIdleConns := 20
+	if app.dataMaxOpenConns > 0 {
+		maxOpenConns = app.dataMaxOpenConns
+	}
+	if app.dataMaxIdleConns > 0 {
+		maxIdleConns = app.dataMaxIdleConns
+	}
+
+	asyncDB, err := connectDB(filepath.Join(app.DataDir(), "data.db"))
+	if err != nil {
+		return err
+	}
+	asyncDB.DB().SetMaxOpenConns(maxOpenConns)
+	asyncDB.DB().SetMaxIdleConns(maxIdleConns)
+	asyncDB.DB().SetConnMaxIdleTime(5 * time.Minute)
+
+	syncDB, err := connectDB(filepath.Join(app.DataDir(), "data.db"))
+	if err != nil {
+		return err
 	}
+	syncDB.DB().SetMaxOpenConns(1)
+	syncDB.DB().SetMaxIdleConns(1)
+	syncDB.DB().SetConnMaxIdleTime(5 * time.Minute)
 
 	if app.IsDebug() {
-		app.db.QueryLogFunc = func(ctx context.Context, t time.Duration, sql string, rows *sql.Rows, err error) {
+		syncDB.QueryLogFunc = func(ctx context.Context, t time.Duration, sql string, rows *sql.Rows, err error) {
 			color.HiBlack("[%.2fms] %v\n", float64(t.Milliseconds()), sql)
 		}
+		asyncDB.QueryLogFunc = syncDB.QueryLogFunc
 
-		app.db.ExecLogFunc = func(ctx context.Context, t time.Duration, sql string, result sql.Result, err error) {
+		syncDB.ExecLogFunc = func(ctx context.Context, t time.Duration, sql string, result sql.Result, err error) {
 			color.HiBlack("[%.2fms] %v\n", float64(t.Milliseconds()), sql)
 		}
+		asyncDB.ExecLogFunc = syncDB.ExecLogFunc
 	}
 
-	app.dao = app.createDaoWithHooks(app.db)
+	app.dao = app.createDaoWithHooks(asyncDB, syncDB)
 
 	return nil
 }
 
-func (app *BaseApp) createDaoWithHooks(db dbx.Builder) *daos.Dao {
-	dao := daos.New(db)
+func (app *BaseApp) createDaoWithHooks(asyncDB, syncDB dbx.Builder) *daos.Dao {
+	dao := daos.NewMultiDB(asyncDB, syncDB)
 
 	dao.BeforeCreateFunc = func(eventDao *daos.Dao, m models.Model) error {
 		return app.OnModelBeforeCreate().Trigger(&ModelEvent{eventDao, m})

core/base_test.go

@@ -11,7 +11,11 @@ func TestNewBaseApp(t *testing.T) {
 	const testDataDir = "./pb_base_app_test_data_dir/"
 	defer os.RemoveAll(testDataDir)
 
-	app := NewBaseApp(testDataDir, "test_env", true)
+	app := NewBaseApp(&BaseAppConfig{
+		DataDir:       testDataDir,
+		EncryptionEnv: "test_env",
+		IsDebug:       true,
+	})
 
 	if app.dataDir != testDataDir {
 		t.Fatalf("expected dataDir %q, got %q", testDataDir, app.dataDir)
@@ -42,7 +46,11 @@ func TestBaseAppBootstrap(t *testing.T) {
 	const testDataDir = "./pb_base_app_test_data_dir/"
 	defer os.RemoveAll(testDataDir)
 
-	app := NewBaseApp(testDataDir, "pb_test_env", false)
+	app := NewBaseApp(&BaseAppConfig{
+		DataDir:       testDataDir,
+		EncryptionEnv: "pb_test_env",
+		IsDebug:       false,
+	})
 	defer app.ResetBootstrapState()
 
 	// bootstrap
@@ -112,29 +120,33 @@ func TestBaseAppGetters(t *testing.T) {
 	const testDataDir = "./pb_base_app_test_data_dir/"
 	defer os.RemoveAll(testDataDir)
 
-	app := NewBaseApp(testDataDir, "pb_test_env", false)
+	app := NewBaseApp(&BaseAppConfig{
+		DataDir:       testDataDir,
+		EncryptionEnv: "pb_test_env",
+		IsDebug:       false,
+	})
 	defer app.ResetBootstrapState()
 
 	if err := app.Bootstrap(); err != nil {
 		t.Fatal(err)
 	}
 
-	if app.db != app.DB() {
-		t.Fatalf("Expected app.DB %v, got %v", app.DB(), app.db)
-	}
-
 	if app.dao != app.Dao() {
 		t.Fatalf("Expected app.Dao %v, got %v", app.Dao(), app.dao)
 	}
 
-	if app.logsDB != app.LogsDB() {
-		t.Fatalf("Expected app.LogsDB %v, got %v", app.LogsDB(), app.logsDB)
+	if app.dao.AsyncDB() != app.DB() {
+		t.Fatalf("Expected app.DB %v, got %v", app.DB(), app.dao.AsyncDB())
 	}
 
 	if app.logsDao != app.LogsDao() {
 		t.Fatalf("Expected app.LogsDao %v, got %v", app.LogsDao(), app.logsDao)
 	}
 
+	if app.logsDao.AsyncDB() != app.LogsDB() {
+		t.Fatalf("Expected app.LogsDB %v, got %v", app.LogsDB(), app.logsDao.AsyncDB())
+	}
+
 	if app.dataDir != app.DataDir() {
 		t.Fatalf("Expected app.DataDir %v, got %v", app.DataDir(), app.dataDir)
 	}
@@ -400,7 +412,11 @@ func TestBaseAppNewMailClient(t *testing.T) {
 	const testDataDir = "./pb_base_app_test_data_dir/"
 	defer os.RemoveAll(testDataDir)
 
-	app := NewBaseApp(testDataDir, "pb_test_env", false)
+	app := NewBaseApp(&BaseAppConfig{
+		DataDir:       testDataDir,
+		EncryptionEnv: "pb_test_env",
+		IsDebug:       false,
+	})
 
 	client1 := app.NewMailClient()
 	if val, ok := client1.(*mailer.Sendmail); !ok {
@@ -419,7 +435,11 @@ func TestBaseAppNewFilesystem(t *testing.T) {
 	const testDataDir = "./pb_base_app_test_data_dir/"
 	defer os.RemoveAll(testDataDir)
 
-	app := NewBaseApp(testDataDir, "pb_test_env", false)
+	app := NewBaseApp(&BaseAppConfig{
+		DataDir:       testDataDir,
+		EncryptionEnv: "pb_test_env",
+		IsDebug:       false,
+	})
 
 	// local
 	local, localErr := app.NewFilesystem()

core/db.go

@@ -0,0 +1,20 @@
+package core
+
+import (
+	"github.com/pocketbase/dbx"
+)
+
+func initPragmas(db *dbx.DB) error {
+	// note: the busy_timeout pragma must be first because
+	// the connection needs to be set to block on busy before WAL mode
+	// is set in case it hasn't been already set by another connection
+	_, err := db.NewQuery(`
+		PRAGMA busy_timeout       = 10000;
+		PRAGMA journal_mode       = WAL;
+		PRAGMA journal_size_limit = 100000000;
+		PRAGMA synchronous        = NORMAL;
+		PRAGMA foreign_keys       = TRUE;
+	`).Execute()
+
+	return err
+}