logs refactoring

CHANGELOG.md

@@ -1,3 +1,35 @@
+## v0.20.0-rc4
+
+- Bumped the minimum required Go version to 1.21.0 in order to integrate with the builtin `slog` package.
+
+- removed _requests table in favor of _logs
+
+- Renamed:
+  ```
+  Dao.RequestQuery(...)      -> Dao.LogQuery(...)
+  Dao.FindRequestById(...)   -> Dao.FindLogById(...)
+  Dao.RequestsStats(...)     -> Dao.LogsStats(...)
+  Dao.RequestsStats(...)     -> Dao.LogsStats(...)
+  Dao.DeleteOldRequests(...) -> Dao.DeleteOldLogs(...)
+  Dao.SaveRequest(...)       -> Dao.SaveLog(...)
+  ```
+
+- removed app.IsDebug() and the `--debug` flag
+
+- (@todo docs) Implemented `slog.Logger` via `app.Logger()`.
+  Logs db writes are debounced and batched. DB write happens on
+    - 3sec after the last debounced log write
+    - when the batch threshold, currently 200, is reached (this is primarily to prevent the memory to grow unrestricted)
+    - right before app termination to attempt saving the current logs queue
+  Several minor log improvements:
+    - Log the requests execution times.
+    - Added option to toggle IP request logging.
+    - Added option to specify a minimum log level.
+    - Added option to export individual or bulk selected logs as json.
+
+- Soft-deprecated and renamed `app.Cache()` with `app.Store()`.
+
+
 ## v0.20.0-rc3
 
 - Synced with the recent fixes in v0.19.4.

apis/admin.go

@@ -1,7 +1,6 @@
 package apis
 
 import (
-	"log"
 	"net/http"
 
 	"github.com/labstack/echo/v5"
@@ -129,9 +128,8 @@ func (api *adminApi) requestPasswordReset(c echo.Context) error {
 			return api.app.OnAdminBeforeRequestPasswordResetRequest().Trigger(event, func(e *core.AdminRequestPasswordResetEvent) error {
 				// run in background because we don't need to show the result to the client
 				routine.FireAndForget(func() {
-					if err := next(e.Admin); err != nil && api.app.IsDebug() {
-						// @todo replace after logs generalization
-						log.Println(err)
+					if err := next(e.Admin); err != nil {
+						api.app.Logger().Error("Failed to send admin password reset request.", "error", err)
 					}
 				})
 

apis/backup.go

@@ -2,7 +2,6 @@ package apis
 
 import (
 	"context"
-	"log"
 	"net/http"
 	"path/filepath"
 	"time"
@@ -69,7 +68,7 @@ func (api *backupApi) list(c echo.Context) error {
 }
 
 func (api *backupApi) create(c echo.Context) error {
-	if api.app.Cache().Has(core.CacheKeyActiveBackup) {
+	if api.app.Store().Has(core.StoreKeyActiveBackup) {
 		return NewBadRequestError("Try again later - another backup/restore process has already been started", nil)
 	}
 
@@ -152,7 +151,7 @@ func (api *backupApi) download(c echo.Context) error {
 }
 
 func (api *backupApi) restore(c echo.Context) error {
-	if api.app.Cache().Has(core.CacheKeyActiveBackup) {
+	if api.app.Store().Has(core.StoreKeyActiveBackup) {
 		return NewBadRequestError("Try again later - another backup/restore process has already been started.", nil)
 	}
 
@@ -181,8 +180,8 @@ func (api *backupApi) restore(c echo.Context) error {
 		// give some optimistic time to write the response
 		time.Sleep(1 * time.Second)
 
-		if err := api.app.RestoreBackup(ctx, key); err != nil && api.app.IsDebug() {
-			log.Println(err)
+		if err := api.app.RestoreBackup(ctx, key); err != nil {
+			api.app.Logger().Error("Failed to restore backup", "key", key, "error", err.Error())
 		}
 	}()
 
@@ -203,7 +202,7 @@ func (api *backupApi) delete(c echo.Context) error {
 
 	key := c.PathParam("key")
 
-	if key != "" && cast.ToString(api.app.Cache().Get(core.CacheKeyActiveBackup)) == key {
+	if key != "" && cast.ToString(api.app.Store().Get(core.StoreKeyActiveBackup)) == key {
 		return NewBadRequestError("The backup is currently being used and cannot be deleted.", nil)
 	}
 

apis/backup_test.go

@@ -116,7 +116,7 @@ func TestBackupsCreate(t *testing.T) {
 				"Authorization": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6InN5d2JoZWNuaDQ2cmhtMCIsInR5cGUiOiJhZG1pbiIsImV4cCI6MjIwODk4NTI2MX0.M1m--VOqGyv0d23eeUc0r9xE8ZzHaYVmVFw1VZW6gT8",
 			},
 			BeforeTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
-				app.Cache().Set(core.CacheKeyActiveBackup, "")
+				app.Store().Set(core.StoreKeyActiveBackup, "")
 			},
 			AfterTestFunc: func(t *testing.T, app *tests.TestApp, res *http.Response) {
 				ensureNoBackups(t, app)
@@ -562,7 +562,7 @@ func TestBackupsDelete(t *testing.T) {
 				}
 
 				// mock active backup with the same name to delete
-				app.Cache().Set(core.CacheKeyActiveBackup, "test1.zip")
+				app.Store().Set(core.StoreKeyActiveBackup, "test1.zip")
 			},
 			AfterTestFunc: func(t *testing.T, app *tests.TestApp, res *http.Response) {
 				noTestBackupFilesChanges(t, app)
@@ -583,7 +583,7 @@ func TestBackupsDelete(t *testing.T) {
 				}
 
 				// mock active backup with different name
-				app.Cache().Set(core.CacheKeyActiveBackup, "new.zip")
+				app.Store().Set(core.StoreKeyActiveBackup, "new.zip")
 			},
 			AfterTestFunc: func(t *testing.T, app *tests.TestApp, res *http.Response) {
 				files, err := getBackupFiles(app)
@@ -700,7 +700,7 @@ func TestBackupsRestore(t *testing.T) {
 					t.Fatal(err)
 				}
 
-				app.Cache().Set(core.CacheKeyActiveBackup, "")
+				app.Store().Set(core.StoreKeyActiveBackup, "")
 			},
 			ExpectedStatus:  400,
 			ExpectedContent: []string{`"data":{}`},

apis/base.go

@@ -6,11 +6,12 @@ import (
 	"errors"
 	"fmt"
 	"io/fs"
-	"log"
+	"log/slog"
 	"net/http"
 	"net/url"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/labstack/echo/v5"
 	"github.com/labstack/echo/v5/middleware"
@@ -26,7 +27,7 @@ const trailedAdminPath = "/_/"
 // system and app specific routes and middlewares.
 func InitApi(app core.App) (*echo.Echo, error) {
 	e := echo.New()
-	e.Debug = app.IsDebug()
+	e.Debug = false
 	e.JSONSerializer = &rest.Serializer{
 		FieldsParam: fieldsQueryParam,
 	}
@@ -49,51 +50,48 @@ func InitApi(app core.App) (*echo.Echo, error) {
 	e.Pre(LoadAuthContext(app))
 	e.Use(middleware.Recover())
 	e.Use(middleware.Secure())
+	e.Use(func(next echo.HandlerFunc) echo.HandlerFunc {
+		return func(c echo.Context) error {
+			c.Set(ContextExecStartKey, time.Now())
+
+			return next(c)
+		}
+	})
 
 	// custom error handler
 	e.HTTPErrorHandler = func(c echo.Context, err error) {
 		if err == nil {
 			return // no error
 		}
 
-		if c.Response().Committed {
-			if app.IsDebug() {
-				log.Println("HTTPErrorHandler response was already committed:", err)
-			}
-			return
-		}
-
 		var apiErr *ApiError
 
 		if errors.As(err, &apiErr) {
-			if app.IsDebug() && apiErr.RawData() != nil {
-				log.Println(apiErr.RawData())
-			}
+			// already an api error...
 		} else if v := new(echo.HTTPError); errors.As(err, &v) {
-			if v.Internal != nil && app.IsDebug() {
-				log.Println(v.Internal)
-			}
 			msg := fmt.Sprintf("%v", v.Message)
 			apiErr = NewApiError(v.Code, msg, v)
 		} else {
-			if app.IsDebug() {
-				log.Println(err)
-			}
-
 			if errors.Is(err, sql.ErrNoRows) {
 				apiErr = NewNotFoundError("", err)
 			} else {
 				apiErr = NewBadRequestError("", err)
 			}
 		}
 
+		logRequest(app, c, apiErr)
+
+		if c.Response().Committed {
+			return // already commited
+		}
+
 		event := new(core.ApiErrorEvent)
 		event.HttpContext = c
 		event.Error = apiErr
 
 		// send error response
 		hookErr := app.OnBeforeApiError().Trigger(event, func(e *core.ApiErrorEvent) error {
-			if c.Response().Committed {
+			if e.HttpContext.Response().Committed {
 				return nil
 			}
 
@@ -106,12 +104,11 @@ func InitApi(app core.App) (*echo.Echo, error) {
 		})
 
 		if hookErr == nil {
-			if err := app.OnAfterApiError().Trigger(event); err != nil && app.IsDebug() {
-				log.Println(hookErr)
+			if err := app.OnAfterApiError().Trigger(event); err != nil {
+				app.Logger().Debug("OnAfterApiError failure", slog.String("error", hookErr.Error()))
 			}
-		} else if app.IsDebug() {
-			// truly rare case; eg. client already disconnected
-			log.Println(hookErr)
+		} else {
+			app.Logger().Debug("OnBeforeApiError error (truly rare case, eg. client already disconnected)", slog.String("error", hookErr.Error()))
 		}
 	}
 
@@ -215,7 +212,7 @@ func updateHasAdminsCache(app core.App) error {
 		return err
 	}
 
-	app.Cache().Set(hasAdminsCacheKey, total > 0)
+	app.Store().Set(hasAdminsCacheKey, total > 0)
 
 	return nil
 }
@@ -240,14 +237,14 @@ func installerRedirect(app core.App) echo.MiddlewareFunc {
 				return next(c)
 			}
 
-			hasAdmins := cast.ToBool(app.Cache().Get(hasAdminsCacheKey))
+			hasAdmins := cast.ToBool(app.Store().Get(hasAdminsCacheKey))
 
 			if !hasAdmins {
 				// update the cache to make sure that the admin wasn't created by another process
 				if err := updateHasAdminsCache(app); err != nil {
 					return err
 				}
-				hasAdmins = cast.ToBool(app.Cache().Get(hasAdminsCacheKey))
+				hasAdmins = cast.ToBool(app.Store().Get(hasAdminsCacheKey))
 			}
 
 			_, hasInstallerParam := c.Request().URL.Query()["installer"]

apis/collection_test.go

@@ -1141,7 +1141,7 @@ func TestCollectionsImport(t *testing.T) {
 			},
 			ExpectedEvents: map[string]int{
 				"OnCollectionsBeforeImportRequest": 1,
-				"OnModelBeforeDelete":              4,
+				"OnModelBeforeDelete":              3,
 			},
 			AfterTestFunc: func(t *testing.T, app *tests.TestApp, res *http.Response) {
 				collections := []*models.Collection{}

apis/health.go

@@ -32,7 +32,7 @@ func (api *healthApi) healthCheck(c echo.Context) error {
 	resp := new(healthCheckResponse)
 	resp.Code = http.StatusOK
 	resp.Message = "API is healthy."
-	resp.Data.CanBackup = !api.app.Cache().Has(core.CacheKeyActiveBackup)
+	resp.Data.CanBackup = !api.app.Store().Has(core.StoreKeyActiveBackup)
 
 	return c.JSON(http.StatusOK, resp)
 }

apis/logs.go

@@ -15,27 +15,27 @@ func bindLogsApi(app core.App, rg *echo.Group) {
 	api := logsApi{app: app}
 
 	subGroup := rg.Group("/logs", RequireAdminAuth())
-	subGroup.GET("/requests", api.requestsList)
-	subGroup.GET("/requests/stats", api.requestsStats)
-	subGroup.GET("/requests/:id", api.requestView)
+	subGroup.GET("", api.list)
+	subGroup.GET("/stats", api.stats)
+	subGroup.GET("/:id", api.view)
 }
 
 type logsApi struct {
 	app core.App
 }
 
-var requestFilterFields = []string{
+var logFilterFields = []string{
 	"rowid", "id", "created", "updated",
-	"url", "method", "status", "auth",
-	"remoteIp", "userIp", "referer", "userAgent",
+	"level", "message", "data",
+	`^data\.[\w\.\:]*\w+$`,
 }
 
-func (api *logsApi) requestsList(c echo.Context) error {
-	fieldResolver := search.NewSimpleFieldResolver(requestFilterFields...)
+func (api *logsApi) list(c echo.Context) error {
+	fieldResolver := search.NewSimpleFieldResolver(logFilterFields...)
 
 	result, err := search.NewProvider(fieldResolver).
-		Query(api.app.LogsDao().RequestQuery()).
-		ParseAndExec(c.QueryParams().Encode(), &[]*models.Request{})
+		Query(api.app.LogsDao().LogQuery()).
+		ParseAndExec(c.QueryParams().Encode(), &[]*models.Log{})
 
 	if err != nil {
 		return NewBadRequestError("", err)
@@ -44,8 +44,8 @@ func (api *logsApi) requestsList(c echo.Context) error {
 	return c.JSON(http.StatusOK, result)
 }
 
-func (api *logsApi) requestsStats(c echo.Context) error {
-	fieldResolver := search.NewSimpleFieldResolver(requestFilterFields...)
+func (api *logsApi) stats(c echo.Context) error {
+	fieldResolver := search.NewSimpleFieldResolver(logFilterFields...)
 
 	filter := c.QueryParam(search.FilterQueryParam)
 
@@ -58,24 +58,24 @@ func (api *logsApi) requestsStats(c echo.Context) error {
 		}
 	}
 
-	stats, err := api.app.LogsDao().RequestsStats(expr)
+	stats, err := api.app.LogsDao().LogsStats(expr)
 	if err != nil {
-		return NewBadRequestError("Failed to generate requests stats.", err)
+		return NewBadRequestError("Failed to generate logs stats.", err)
 	}
 
 	return c.JSON(http.StatusOK, stats)
 }
 
-func (api *logsApi) requestView(c echo.Context) error {
+func (api *logsApi) view(c echo.Context) error {
 	id := c.PathParam("id")
 	if id == "" {
 		return NewNotFoundError("", nil)
 	}
 
-	request, err := api.app.LogsDao().FindRequestById(id)
-	if err != nil || request == nil {
+	log, err := api.app.LogsDao().FindLogById(id)
+	if err != nil || log == nil {
 		return NewNotFoundError("", err)
 	}
 
-	return c.JSON(http.StatusOK, request)
+	return c.JSON(http.StatusOK, log)
 }