added search skipTotal support

CHANGELOG.md

@@ -93,6 +93,21 @@
 
 - Added support for wrapped API errors (_in case Go 1.20+ is used with multiple wrapped errors, `apis.ApiError` takes precedence_).
 
+- Changes to the List/Search APIs
+
+  - Increased the max allowed `?perPage` limit to 1000.
+
+  - Reverted the default `COUNT` column to `id` as there are some common situations where it can negatively impact the query performance.
+    Additionally, from this version we also set `PRAGMA temp_store = MEMORY` so that also helps with the temp B-TREE creation when `id` is used.
+    _There are still scenarios where `COUNT` queries with `rowid` executes faster, but the majority of the time when nested relations lookups are used it seems to have the opposite effect (at least based on the benchmarks dataset)._
+
+  - The count and regular select statements also now executes concurrently, meaning that we no longer perform normalization over the `page` parameter and in case the user
+    request a page that doesn't exist (eg. `?page=99999999`) we'll return empty `items` array.
+
+  - (@todo docs) Added new query parameter `?skipTotal=1` to skip the `COUNT` query performed with the list/search actions ([#2965](https://github.com/pocketbase/pocketbase/discussions/2965)).
+    If `?skipTotal=1` is set, the response fields `totalItems` and `totalPages` will have `-1` value (this is to avoid having different JSON responses and to differentiate from the zero default).
+    With the latest JS SDK 0.16+ and Dart SDK v0.11+ versions `skipTotal=1` is set by default for the `getFirstListItem()` and `getFullList()` requests.
+
 
 ## v0.16.10
 

apis/record_crud.go

@@ -68,11 +68,6 @@ func (api *recordApi) list(c echo.Context) error {
 	searchProvider := search.NewProvider(fieldsResolver).
 		Query(api.app.Dao().RecordQuery(collection))
 
-	// views don't have "rowid" so we fallback to "id"
-	if collection.IsView() {
-		searchProvider.CountCol("id")
-	}
-
 	if requestInfo.Admin == nil && collection.ListRule != nil {
 		searchProvider.AddFilter(search.FilterData(*collection.ListRule))
 	}

core/db_cgo.go

@@ -28,6 +28,7 @@ func init() {
 					PRAGMA journal_size_limit = 200000000;
 					PRAGMA synchronous        = NORMAL;
 					PRAGMA foreign_keys       = ON;
+					PRAGMA temp_store         = MEMORY;
 				`, nil)
 
 				return err

core/db_nocgo.go

@@ -11,7 +11,7 @@ func connectDB(dbPath string) (*dbx.DB, error) {
 	// Note: the busy_timeout pragma must be first because
 	// the connection needs to be set to block on busy before WAL mode
 	// is set in case it hasn't been already set by another connection.
-	pragmas := "?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=journal_size_limit(200000000)&_pragma=synchronous(NORMAL)&_pragma=foreign_keys(ON)"
+	pragmas := "?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=journal_size_limit(200000000)&_pragma=synchronous(NORMAL)&_pragma=foreign_keys(ON)&_pragma=temp_store(MEMORY)"
 
 	db, err := dbx.Open("sqlite", dbPath+pragmas)
 	if err != nil {

tools/search/provider.go

@@ -7,20 +7,22 @@ import (
 	"strconv"
 
 	"github.com/pocketbase/dbx"
+	"golang.org/x/sync/errgroup"
 )
 
 // DefaultPerPage specifies the default returned search result items.
 const DefaultPerPage int = 30
 
 // MaxPerPage specifies the maximum allowed search result items returned in a single page.
-const MaxPerPage int = 500
+const MaxPerPage int = 1000
 
 // url search query params
 const (
-	PageQueryParam    string = "page"
-	PerPageQueryParam string = "perPage"
-	SortQueryParam    string = "sort"
-	FilterQueryParam  string = "filter"
+	PageQueryParam      string = "page"
+	PerPageQueryParam   string = "perPage"
+	SortQueryParam      string = "sort"
+	FilterQueryParam    string = "filter"
+	SkipTotalQueryParam string = "skipTotal"
 )
 
 // Result defines the returned search result structure.
@@ -36,6 +38,7 @@ type Result struct {
 type Provider struct {
 	fieldResolver FieldResolver
 	query         *dbx.SelectQuery
+	skipTotal     bool
 	countCol      string
 	page          int
 	perPage       int
@@ -57,7 +60,7 @@ type Provider struct {
 func NewProvider(fieldResolver FieldResolver) *Provider {
 	return &Provider{
 		fieldResolver: fieldResolver,
-		countCol:      "_rowid_",
+		countCol:      "id",
 		page:          1,
 		perPage:       DefaultPerPage,
 		sort:          []SortField{},
@@ -71,8 +74,16 @@ func (s *Provider) Query(query *dbx.SelectQuery) *Provider {
 	return s
 }
 
-// CountCol allows changing the default column (_rowid_) that is used
+// SkipTotal changes the `skipTotal` field of the current search provider.
+func (s *Provider) SkipTotal(skipTotal bool) *Provider {
+	s.skipTotal = skipTotal
+	return s
+}
+
+// CountCol allows changing the default column (id) that is used
 // to generated the COUNT SQL query statement.
+//
+// This field is ignored if skipTotal is true.
 func (s *Provider) CountCol(name string) *Provider {
 	s.countCol = name
 	return s
@@ -132,30 +143,38 @@ func (s *Provider) Parse(urlQuery string) error {
 		return err
 	}
 
-	if rawPage := params.Get(PageQueryParam); rawPage != "" {
-		page, err := strconv.Atoi(rawPage)
+	if raw := params.Get(SkipTotalQueryParam); raw != "" {
+		v, err := strconv.ParseBool(raw)
 		if err != nil {
 			return err
 		}
-		s.Page(page)
+		s.SkipTotal(v)
 	}
 
-	if rawPerPage := params.Get(PerPageQueryParam); rawPerPage != "" {
-		perPage, err := strconv.Atoi(rawPerPage)
+	if raw := params.Get(PageQueryParam); raw != "" {
+		v, err := strconv.Atoi(raw)
 		if err != nil {
 			return err
 		}
-		s.PerPage(perPage)
+		s.Page(v)
 	}
 
-	if rawSort := params.Get(SortQueryParam); rawSort != "" {
-		for _, sortField := range ParseSortFromString(rawSort) {
+	if raw := params.Get(PerPageQueryParam); raw != "" {
+		v, err := strconv.Atoi(raw)
+		if err != nil {
+			return err
+		}
+		s.PerPage(v)
+	}
+
+	if raw := params.Get(SortQueryParam); raw != "" {
+		for _, sortField := range ParseSortFromString(raw) {
 			s.AddSort(sortField)
 		}
 	}
 
-	if rawFilter := params.Get(FilterQueryParam); rawFilter != "" {
-		s.AddFilter(FilterData(rawFilter))
+	if raw := params.Get(FilterQueryParam); raw != "" {
+		s.AddFilter(FilterData(raw))
 	}
 
 	return nil
@@ -165,10 +184,10 @@ func (s *Provider) Parse(urlQuery string) error {
 // the provided `items` slice with the found models.
 func (s *Provider) Exec(items any) (*Result, error) {
 	if s.query == nil {
-		return nil, errors.New("Query is not set.")
+		return nil, errors.New("query is not set")
 	}
 
-	// clone provider's query
+	// shallow clone the provider's query
 	modelsQuery := *s.query
 
 	// build filters
@@ -198,18 +217,9 @@ func (s *Provider) Exec(items any) (*Result, error) {
 		return nil, err
 	}
 
-	queryInfo := modelsQuery.Info()
-
-	// count
-	var totalCount int64
-	var baseTable string
-	if len(queryInfo.From) > 0 {
-		baseTable = queryInfo.From[0]
-	}
-	clone := modelsQuery
-	countQuery := clone.Distinct(false).Select("COUNT(DISTINCT [[" + baseTable + "." + s.countCol + "]])").OrderBy()
-	if err := countQuery.Row(&totalCount); err != nil {
-		return nil, err
+	// normalize page
+	if s.page <= 0 {
+		s.page = 1
 	}
 
 	// normalize perPage
@@ -219,31 +229,65 @@ func (s *Provider) Exec(items any) (*Result, error) {
 		s.perPage = MaxPerPage
 	}
 
-	totalPages := int(math.Ceil(float64(totalCount) / float64(s.perPage)))
+	// negative value to differentiate from the zero default
+	totalCount := -1
+	totalPages := -1
+
+	// prepare a count query from the base one
+	countQuery := modelsQuery // shallow clone
+	countExec := func() error {
+		queryInfo := countQuery.Info()
+		countCol := s.countCol
+		if len(queryInfo.From) > 0 {
+			countCol = queryInfo.From[0] + "." + countCol
+		}
 
-	// normalize page according to the total count
-	if s.page <= 0 || totalCount == 0 {
-		s.page = 1
-	} else if s.page > totalPages {
-		s.page = totalPages
+		// note: countQuery is shallow cloned and slice/map in-place modifications should be avoided
+		err := countQuery.Distinct(false).
+			Select("COUNT(DISTINCT [[" + countCol + "]])").
+			OrderBy( /* reset */ ).
+			Row(&totalCount)
+		if err != nil {
+			return err
+		}
+
+		totalPages = int(math.Ceil(float64(totalCount) / float64(s.perPage)))
+
+		return nil
 	}
 
-	// apply pagination
-	modelsQuery.Limit(int64(s.perPage))
-	modelsQuery.Offset(int64(s.perPage * (s.page - 1)))
+	// apply pagination to the original query and fetch the models
+	modelsExec := func() error {
+		modelsQuery.Limit(int64(s.perPage))
+		modelsQuery.Offset(int64(s.perPage * (s.page - 1)))
 
-	// fetch models
-	if err := modelsQuery.All(items); err != nil {
-		return nil, err
+		return modelsQuery.All(items)
+	}
+
+	if !s.skipTotal {
+		// execute the 2 queries concurrently
+		errg := new(errgroup.Group)
+		errg.SetLimit(2)
+		errg.Go(countExec)
+		errg.Go(modelsExec)
+		if err := errg.Wait(); err != nil {
+			return nil, err
+		}
+	} else {
+		if err := modelsExec(); err != nil {
+			return nil, err
+		}
 	}
 
-	return &Result{
+	result := &Result{
 		Page:       s.page,
 		PerPage:    s.perPage,
-		TotalItems: int(totalCount),
+		TotalItems: totalCount,
 		TotalPages: totalPages,
 		Items:      items,
-	}, nil
+	}
+
+	return result, nil
 }
 
 // ParseAndExec is a short convenient method to trigger both