added back relation filter reference support

CHANGELOG.md

@@ -1,10 +1,16 @@
 ## (WIP) v0.22.0
 
-- Admin UI improvements:
-  - Sync collection changes across multiple opened tabs.
-  - Fixed vertical image popup preview scrolling.
-  - Added options to export a subset of collections.
-  - Added option to import a subset of collections without deleting the others ([#3403](https://github.com/pocketbase/pocketbase/issues/3403)).
+- Added support for back/indirect relation `filter`/`sort` (single and multiple).
+  The syntax to reference back relation fields is `yourCollection_via_yourRelField.*`.
+  ⚠️ To avoid excessive joins, the nested relations resolver is now limited to max 6 level depth (similar to `expand`).
+  _Note that in the future there will be also more advanced and granular options to specify a subset of the fields that are filterable/sortable._
+  @todo add Admin UI autocomplete
+  @todo update "Working with relations" docs
+
+- Added support for multiple back/indirect relation `expand` and updated the keys to use the `_via_` reference syntax (`yourCollection_via_yourRelField`).
+  _To minimize the breaking changes, the old parenthesis reference syntax (`yourCollection(yourRelField)`) will still continue to work but it is soft-deprecated and there will be a console log reminding you to change it to the new one._
+
+- ⚠️ Collections and fields are no longer allowed to have `_via_` in their name to avoid collisions with the back/indirect relation reference syntax.
 
 - Added `jsvm.Config.OnInit` optional config function to allow registering custom Go bindings to the JSVM.
 
@@ -18,9 +24,15 @@
   oauth2
   ```
 
-- Upgraded to `aws-sdk-go-v2` and added special handling for GCS to workaround the previous [headers signature issue](https://github.com/pocketbase/pocketbase/issues/2231) that we had with v2.
+- Upgraded to `aws-sdk-go-v2` and added special handling for GCS to workaround the previous [GCS headers signature issue](https://github.com/pocketbase/pocketbase/issues/2231) that we had with v2.
   _This should also fix the SVG/JSON zero response when using Cloudflare R2 ([#4287](https://github.com/pocketbase/pocketbase/issues/4287#issuecomment-1925168142), [#2068](https://github.com/pocketbase/pocketbase/discussions/2068), [#2952](https://github.com/pocketbase/pocketbase/discussions/2952))._
-  _If you are using S3, please verify that you have a green check in the Admin UI for your S3 configuration (I've tested the new version with GCS, MinIO, Cloudflare R2 and Wasabi)._
+  _If you are using S3 for uploaded files or backups, please verify that you have a green check in the Admin UI for your S3 configuration (I've tested the new version with GCS, MinIO, Cloudflare R2 and Wasabi)._
+
+- Admin UI improvements:
+  - Sync collection changes across multiple opened tabs.
+  - Fixed vertical image popup preview scrolling.
+  - Added options to export a subset of collections.
+  - Added option to import a subset of collections without deleting the others ([#3403](https://github.com/pocketbase/pocketbase/issues/3403)).
 
 - Other minor improvements (updated the `ghupdate` plugin to use the configured executable name when printing to the console, fixed the error reporting of `admin update/delete` commands, etc.).
 

apis/collection_test.go

@@ -893,7 +893,8 @@ func TestCollectionUpdate(t *testing.T) {
 					{"type":"text","name":"password"},
 					{"type":"text","name":"passwordConfirm"},
 					{"type":"text","name":"oldPassword"}
-				]
+				],
+				"indexes": []
 			}`),
 			RequestHeaders: map[string]string{
 				"Authorization": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6InN5d2JoZWNuaDQ2cmhtMCIsInR5cGUiOiJhZG1pbiIsImV4cCI6MjIwODk4NTI2MX0.M1m--VOqGyv0d23eeUc0r9xE8ZzHaYVmVFw1VZW6gT8",

daos/collection_test.go

@@ -153,9 +153,11 @@ func TestFindCollectionReferences(t *testing.T) {
 		"rel_one_no_cascade",
 		"rel_one_no_cascade_required",
 		"rel_one_cascade",
+		"rel_one_unique",
 		"rel_many_no_cascade",
 		"rel_many_no_cascade_required",
 		"rel_many_cascade",
+		"rel_many_unique",
 	}
 
 	for col, fields := range result {
@@ -756,7 +758,7 @@ func TestImportCollections(t *testing.T) {
 					"demo1":      15,
 					"demo2":      2,
 					"demo3":      2,
-					"demo4":      11,
+					"demo4":      13,
 					"demo5":      6,
 					"new_import": 1,
 				}
@@ -774,37 +776,38 @@ func TestImportCollections(t *testing.T) {
 		},
 	}
 
-	for _, scenario := range scenarios {
-		testApp, _ := tests.NewTestApp()
-		defer testApp.Cleanup()
+	for _, s := range scenarios {
+		t.Run(s.name, func(t *testing.T) {
+			testApp, _ := tests.NewTestApp()
+			defer testApp.Cleanup()
 
-		importedCollections := []*models.Collection{}
+			importedCollections := []*models.Collection{}
 
-		// load data
-		loadErr := json.Unmarshal([]byte(scenario.jsonData), &importedCollections)
-		if loadErr != nil {
-			t.Fatalf("[%s] Failed to load  data: %v", scenario.name, loadErr)
-			continue
-		}
+			// load data
+			loadErr := json.Unmarshal([]byte(s.jsonData), &importedCollections)
+			if loadErr != nil {
+				t.Fatalf("Failed to load  data: %v", loadErr)
+			}
 
-		err := testApp.Dao().ImportCollections(importedCollections, scenario.deleteMissing, scenario.beforeRecordsSync)
+			err := testApp.Dao().ImportCollections(importedCollections, s.deleteMissing, s.beforeRecordsSync)
 
-		hasErr := err != nil
-		if hasErr != scenario.expectError {
-			t.Errorf("[%s] Expected hasErr to be %v, got %v (%v)", scenario.name, scenario.expectError, hasErr, err)
-		}
+			hasErr := err != nil
+			if hasErr != s.expectError {
+				t.Fatalf("Expected hasErr to be %v, got %v (%v)", s.expectError, hasErr, err)
+			}
 
-		// check collections count
-		collections := []*models.Collection{}
-		if err := testApp.Dao().CollectionQuery().All(&collections); err != nil {
-			t.Fatal(err)
-		}
-		if len(collections) != scenario.expectCollectionsCount {
-			t.Errorf("[%s] Expected %d collections, got %d", scenario.name, scenario.expectCollectionsCount, len(collections))
-		}
+			// check collections count
+			collections := []*models.Collection{}
+			if err := testApp.Dao().CollectionQuery().All(&collections); err != nil {
+				t.Fatal(err)
+			}
+			if len(collections) != s.expectCollectionsCount {
+				t.Fatalf("Expected %d collections, got %d", s.expectCollectionsCount, len(collections))
+			}
 
-		if scenario.afterTestFunc != nil {
-			scenario.afterTestFunc(testApp, collections)
-		}
+			if s.afterTestFunc != nil {
+				s.afterTestFunc(testApp, collections)
+			}
+		})
 	}
 }

daos/record.go

@@ -198,8 +198,6 @@ func (dao *Dao) FindRecordsByIds(
 	return records, nil
 }
 
-// @todo consider to depricate as it may be easier to just use dao.RecordQuery()
-//
 // FindRecordsByExpr finds all records by the specified db expression.
 //
 // Returns all collection records if no expressions are provided.

daos/record_expand.go

@@ -1,21 +1,24 @@
 package daos
 
 import (
+	"errors"
 	"fmt"
+	"log"
 	"regexp"
 	"strings"
 
 	"github.com/pocketbase/dbx"
 	"github.com/pocketbase/pocketbase/models"
 	"github.com/pocketbase/pocketbase/models/schema"
 	"github.com/pocketbase/pocketbase/tools/dbutils"
-	"github.com/pocketbase/pocketbase/tools/inflector"
 	"github.com/pocketbase/pocketbase/tools/list"
 	"github.com/pocketbase/pocketbase/tools/security"
 	"github.com/pocketbase/pocketbase/tools/types"
 )
 
 // MaxExpandDepth specifies the max allowed nested expand depth path.
+//
+// @todo Consider eventually reusing resolvers.maxNestedRels
 const MaxExpandDepth = 6
 
 // ExpandFetchFunc defines the function that is used to fetch the expanded relation records.
@@ -51,13 +54,15 @@ func (dao *Dao) ExpandRecords(records []*models.Record, expands []string, optFet
 	return failed
 }
 
-var indirectExpandRegex = regexp.MustCompile(`^(\w+)\((\w+)\)$`)
+// Deprecated
+var indirectExpandRegexOld = regexp.MustCompile(`^(\w+)\((\w+)\)$`)
+
+var indirectExpandRegex = regexp.MustCompile(`^(\w+)_via_(\w+)$`)
 
 // notes:
 // - if fetchFunc is nil, dao.FindRecordsByIds will be used
 // - all records are expected to be from the same collection
 // - if MaxExpandDepth is reached, the function returns nil ignoring the remaining expand path
-// - indirect expands are supported only with single relation fields
 func (dao *Dao) expandRecords(records []*models.Record, expandPath string, fetchFunc ExpandFetchFunc, recursionLevel int) error {
 	if fetchFunc == nil {
 		// load a default fetchFunc
@@ -77,7 +82,22 @@ func (dao *Dao) expandRecords(records []*models.Record, expandPath string, fetch
 	var relCollection *models.Collection
 
 	parts := strings.SplitN(expandPath, ".", 2)
-	matches := indirectExpandRegex.FindStringSubmatch(parts[0])
+	var matches []string
+
+	// @todo remove the old syntax support
+	if strings.Contains(parts[0], "(") {
+		matches = indirectExpandRegexOld.FindStringSubmatch(parts[0])
+		if len(matches) == 3 {
+			log.Printf(
+				"%s expand format is deprecated and will be removed in the future. Consider replacing it with %s_via_%s.\n",
+				matches[0],
+				matches[1],
+				matches[2],
+			)
+		}
+	} else {
+		matches = indirectExpandRegex.FindStringSubmatch(parts[0])
+	}
 
 	if len(matches) == 3 {
 		indirectRel, _ := dao.FindCollectionByNameOrId(matches[1])
@@ -95,47 +115,47 @@ func (dao *Dao) expandRecords(records []*models.Record, expandPath string, fetch
 		if indirectRelFieldOptions == nil || indirectRelFieldOptions.CollectionId != mainCollection.Id {
 			return fmt.Errorf("Invalid indirect relation field path %q.", parts[0])
 		}
-		if indirectRelFieldOptions.IsMultiple() {
-			// for now don't allow multi-relation indirect fields expand
-			// due to eventual poor query performance with large data sets.
-			return fmt.Errorf("Multi-relation fields cannot be indirectly expanded in %q.", parts[0])
-		}
-
-		recordIds := make([]any, len(records))
-		for i, record := range records {
-			recordIds[i] = record.Id
-		}
 
-		// @todo after the index optimizations consider allowing
-		// indirect expand for multi-relation fields
-		indirectRecords, err := dao.FindRecordsByExpr(
-			indirectRel.Id,
-			dbx.In(inflector.Columnify(matches[2]), recordIds...),
-		)
-		if err != nil {
-			return err
-		}
-		mappedIndirectRecordIds := make(map[string][]string, len(indirectRecords))
-		for _, indirectRecord := range indirectRecords {
-			recId := indirectRecord.GetString(matches[2])
-			if recId != "" {
-				mappedIndirectRecordIds[recId] = append(mappedIndirectRecordIds[recId], indirectRecord.Id)
+		// add the related id(s) as a dynamic relation field value to
+		// allow further expand checks at later stage in a more unified manner
+		prepErr := func() error {
+			q := dao.DB().Select("id").From(indirectRel.Name)
+
+			if indirectRelFieldOptions.IsMultiple() {
+				q.AndWhere(dbx.Exists(dbx.NewExp(fmt.Sprintf(
+					"SELECT 1 FROM %s je WHERE je.value = {:id}",
+					dbutils.JsonEach(indirectRelField.Name),
+				))))
+			} else {
+				q.AndWhere(dbx.NewExp("[[" + indirectRelField.Name + "]] = {:id}"))
 			}
-		}
 
-		// add the indirect relation ids as a new relation field value
-		for _, record := range records {
-			relIds, ok := mappedIndirectRecordIds[record.Id]
-			if ok && len(relIds) > 0 {
-				record.Set(parts[0], relIds)
+			pq := q.Build().Prepare()
+
+			for _, record := range records {
+				var relIds []string
+
+				err := pq.Bind(dbx.Params{"id": record.Id}).Column(&relIds)
+				if err != nil {
+					return errors.Join(err, pq.Close())
+				}
+
+				if len(relIds) > 0 {
+					record.Set(parts[0], relIds)
+				}
 			}
+
+			return pq.Close()
+		}()
+		if prepErr != nil {
+			return prepErr
 		}
 
 		relFieldOptions = &schema.RelationOptions{
 			MaxSelect:    nil,
 			CollectionId: indirectRel.Id,
 		}
-		if isRelFieldUnique(indirectRel, indirectRelField.Name) {
+		if dbutils.HasSingleColumnUniqueIndex(indirectRelField.Name, indirectRel.Indexes) {
 			relFieldOptions.MaxSelect = types.Pointer(1)
 		}
 		// indirect relation