[Playground] Move modify saved snippets functionality to to cloudfunc…

…tions (apache#26026)

* Move priviledged Datastore operations to Cloudfunctions

* Terraform changes to deploy cloud functions

* Remove setting for snippets retention period as it's now hardcoded in a cloud function

* service account access review

* Datastore namespace added

* Update .gitignore

* Terraform fixes to permissions and namespaces

* Move PutSnippet() into a separate CloudFunction

* Remove unused role

* Do not apply data migrations which are already applied

* Move DB migration to external command

* Hook up DB migration script to GKE deployment task

* Remove trailing whitespace

* Apply migrations in ToB integration tests

* CloudFunctions review

* remove unused code

* Fix EOL on EOF

* Update Readme

* 'Cloud Functions Developer' role added to requirements

* Fix issue with namespace value being ignored

* Paths for archive_file set relative to path.root

* Pass namespace into cloudfunctions

* change arcive

* fix name

* fix module

* Cloud Datastore User role added

Required to run DB migration tool

* Variable renamed

---------

Co-authored-by: Sergey Makarkin <[email protected]>
Co-authored-by: Sergey Makarkin <[email protected]>
Co-authored-by: Rouslan <[email protected]>
Co-authored-by: rshamunov <[email protected]>

.gitignore

@@ -136,6 +136,7 @@ playground/frontend/playground_components_dev/pubspec.lock
 **/*.tfstate.*
 **/*.hcl
 **/*.tfvars
+playground/cloudfunction.zip
 
 # Ignore Katas auto-generated files
 **/*-remote-info.yaml

learning/tour-of-beam/backend/docker-compose.yml

@@ -42,6 +42,7 @@ services:
       - CACHE_TYPE=local
       - SDK_CONFIG=/opt/playground/backend/sdks-emulator.yaml
       - PROTOCOL_TYPE=TCP
+      - APPLY_MIGRATIONS=True
     ports:
       - "8000:8080"
     depends_on:

playground/backend/README.md

@@ -129,7 +129,6 @@ These properties are stored in `backend/properties.yaml` file:
 - `max_snippet_size` - is the file content size limit. Since 1 character occupies 1 byte of memory, and 1 MB is approximately equal to 1000000 bytes, then maximum size of the snippet is 1000000.
 - `id_length` - is the length of the identifier that is used to store data in the cloud datastore. It's appropriate length to save storage size in the cloud datastore and provide good randomnicity.
 - `removing_unused_snippets_cron` - is the cron expression for the scheduled task to remove unused snippets.
-- `removing_unused_snippets_days` - is the number of days after which a snippet becomes unused.
 
 ## Running the server app via Docker
 

playground/backend/build.gradle.kts

@@ -72,7 +72,7 @@ task("removeUnusedSnippet") {
     doLast {
       exec {
          executable("go")
-         args("run", "cmd/remove_unused_snippets.go", "cleanup",
+         args("run", "cmd/remove_unused_snippets/remove_unused_snippets.go", "cleanup",
           "-day_diff", System.getProperty("dayDiff"), "-project_id", System.getProperty("projectId"),
           "-namespace", System.getProperty("namespace"))
       }
@@ -83,7 +83,7 @@ task("removeSnippet") {
     doLast {
       exec {
          executable("go")
-         args("run", "cmd/remove_unused_snippets.go", "remove",
+         args("run", "cmd/remove_unused_snippets/remove_unused_snippets.go", "remove",
           "-snippet_id", System.getProperty("snippetId"), "-project_id", System.getProperty("projectId"),
           "-namespace", System.getProperty("namespace"))
       }

playground/backend/cmd/migration_tool/migration_tool.go

@@ -0,0 +1,56 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"beam.apache.org/playground/backend/internal/constants"
+	"beam.apache.org/playground/backend/internal/db/datastore"
+	"beam.apache.org/playground/backend/internal/db/mapper"
+	"beam.apache.org/playground/backend/internal/db/schema"
+	"beam.apache.org/playground/backend/internal/logger"
+	"context"
+	"flag"
+	"fmt"
+	"os"
+)
+
+func main() {
+	projectId := flag.String("project-id", "", "GCP project id")
+	sdkConfigPath := flag.String("sdk-config", "", "Path to the sdk config file")
+	namespace := flag.String("namespace", constants.Namespace, "Datastore namespace")
+
+	flag.Parse()
+
+	ctx := context.WithValue(context.Background(), constants.DatastoreNamespaceKey, *namespace)
+
+	cwd, err := os.Getwd()
+	if err != nil {
+		fmt.Printf("Couldn't get the current working directory, err: %s \n", err.Error())
+		os.Exit(1)
+	}
+	logger.SetupLogger(context.Background(), cwd, *projectId)
+
+	migratedDb, err := datastore.New(ctx, mapper.NewPrecompiledObjectMapper(), nil, *projectId)
+	if err != nil {
+		logger.Fatalf("Couldn't create DB client instance, err: %s \n", err.Error())
+		os.Exit(1)
+	}
+
+	if err := migratedDb.ApplyMigrations(ctx, schema.Migrations, *sdkConfigPath); err != nil {
+		logger.Fatalf("Couldn't apply migrations, err: %s \n", err.Error())
+		os.Exit(1)
+	}
+}

playground/backend/cmd/remove_unused_snippets.go → playground/backend/cmd/remove_unused_snippets/remove_unused_snippets.go

@@ -30,7 +30,7 @@ import (
 
 func createDatastoreClient(ctx context.Context, projectId string) (*datastore.Datastore, error) {
 	pcMapper := mapper.NewPrecompiledObjectMapper()
-	db, err := datastore.New(ctx, pcMapper, projectId)
+	db, err := datastore.New(ctx, pcMapper, nil, projectId)
 	if err != nil {
 		logger.Errorf("Couldn't create the database client, err: %s \n", err.Error())
 		return nil, err
@@ -49,7 +49,8 @@ func cleanup(dayDiff int, projectId, namespace string) error {
 		return err
 	}
 
-	err = db.DeleteUnusedSnippets(ctx, int32(dayDiff))
+	retentionPeriod := time.Duration(dayDiff) * 24 * time.Hour
+	err = db.DeleteUnusedSnippets(ctx, retentionPeriod)
 	if err != nil {
 		logger.Errorf("Couldn't delete unused code snippets, err: %s \n", err.Error())
 		return err

playground/backend/cmd/server/controller_test.go

@@ -15,6 +15,7 @@
 package main
 
 import (
+	"beam.apache.org/playground/backend/internal/db/schema"
 	"context"
 	"fmt"
 	"io/fs"
@@ -42,8 +43,6 @@ import (
 	datastoreDb "beam.apache.org/playground/backend/internal/db/datastore"
 	"beam.apache.org/playground/backend/internal/db/entity"
 	"beam.apache.org/playground/backend/internal/db/mapper"
-	"beam.apache.org/playground/backend/internal/db/schema"
-	"beam.apache.org/playground/backend/internal/db/schema/migration"
 	"beam.apache.org/playground/backend/internal/environment"
 	"beam.apache.org/playground/backend/internal/logger"
 	"beam.apache.org/playground/backend/internal/tests/test_cleaner"
@@ -116,7 +115,9 @@ func setupServer(sdk pb.Sdk) *grpc.Server {
 	if err = os.Setenv("APP_WORK_DIR", path); err != nil {
 		panic(err)
 	}
-	if err = os.Setenv("SDK_CONFIG", "../../../sdks-emulator.yaml"); err != nil {
+
+	sdkConfigPath := "../../../sdks-emulator.yaml"
+	if err = os.Setenv("SDK_CONFIG", sdkConfigPath); err != nil {
 		panic(err)
 	}
 	if err = os.Setenv("PROPERTY_PATH", "../../."); err != nil {
@@ -145,17 +146,17 @@ func setupServer(sdk pb.Sdk) *grpc.Server {
 		panic(err)
 	}
 
-	// setup initial data
-	versions := []schema.Version{
-		new(migration.InitialStructure),
-		new(migration.AddingComplexityProperty),
+	err = dbEmulator.ApplyMigrations(ctx, schema.Migrations, sdkConfigPath)
+	if err != nil {
+		panic(err)
 	}
-	dbSchema := schema.New(ctx, dbEmulator, appEnv, props, versions)
-	actualSchemaVersion, err := dbSchema.InitiateData()
+
+	migrationVersion, err := dbEmulator.GetCurrentDbMigrationVersion(ctx)
 	if err != nil {
 		panic(err)
 	}
-	appEnv.SetSchemaVersion(actualSchemaVersion)
+
+	appEnv.SetSchemaVersion(migrationVersion)
 
 	// download test data to the Datastore Emulator
 	test_data.DownloadCatalogsWithMockData(ctx)

playground/backend/cmd/server/server.go

@@ -16,12 +16,12 @@
 package main
 
 import (
+	"beam.apache.org/playground/backend/internal/external_functions"
 	"context"
 	"fmt"
-	"os"
-
 	"github.com/improbable-eng/grpc-web/go/grpcweb"
 	"google.golang.org/grpc"
+	"os"
 
 	pb "beam.apache.org/playground/backend/internal/api/v1"
 	"beam.apache.org/playground/backend/internal/cache"
@@ -32,8 +32,6 @@ import (
 	"beam.apache.org/playground/backend/internal/db/datastore"
 	"beam.apache.org/playground/backend/internal/db/entity"
 	"beam.apache.org/playground/backend/internal/db/mapper"
-	"beam.apache.org/playground/backend/internal/db/schema"
-	"beam.apache.org/playground/backend/internal/db/schema/migration"
 	"beam.apache.org/playground/backend/internal/environment"
 	"beam.apache.org/playground/backend/internal/logger"
 	"beam.apache.org/playground/backend/internal/tasks"
@@ -67,22 +65,27 @@ func runServer() error {
 	// Examples catalog should be retrieved and saved to cache only if the server doesn't suppose to run code, i.e. SDK is unspecified
 	// Database setup only if the server doesn't suppose to run code, i.e. SDK is unspecified
 	if envService.BeamSdkEnvs.ApacheBeamSdk == pb.Sdk_SDK_UNSPECIFIED {
+		externalFunctions := external_functions.NewExternalFunctionsComponent(envService.ApplicationEnvs)
+
 		props, err = environment.NewProperties(envService.ApplicationEnvs.PropertyPath())
 		if err != nil {
 			return err
 		}
 
-		dbClient, err = datastore.New(ctx, mapper.NewPrecompiledObjectMapper(), envService.ApplicationEnvs.GoogleProjectId())
+		dbClient, err = datastore.New(ctx, mapper.NewPrecompiledObjectMapper(), externalFunctions, envService.ApplicationEnvs.GoogleProjectId())
 		if err != nil {
 			return err
 		}
 
 		downloadCatalogsToDatastoreEmulator(ctx)
 
-		if err = setupDBStructure(ctx, dbClient, &envService.ApplicationEnvs, props); err != nil {
+		migrationVersion, err := dbClient.GetCurrentDbMigrationVersion(ctx)
+		if err != nil {
 			return err
 		}
 
+		envService.ApplicationEnvs.SetSchemaVersion(migrationVersion)
+
 		sdks, err := setupSdkCatalog(ctx, cacheService, dbClient)
 		if err != nil {
 			return err
@@ -97,7 +100,7 @@ func runServer() error {
 
 		// Since only router server has the scheduled task, the task creation is here
 		scheduledTasks := tasks.New(ctx)
-		if err = scheduledTasks.StartRemovingExtraSnippets(props.RemovingUnusedSnptsCron, props.RemovingUnusedSnptsDays, dbClient); err != nil {
+		if err = scheduledTasks.StartRemovingExtraSnippets(props.RemovingUnusedSnptsCron, externalFunctions); err != nil {
 			return err
 		}
 	}
@@ -224,26 +227,6 @@ func setupExamplesCatalogFromDatastore(ctx context.Context, cacheService cache.C
 	return nil
 }
 
-// setupDBStructure initializes the data structure
-func setupDBStructure(ctx context.Context, db db.Database, appEnv *environment.ApplicationEnvs, props *environment.Properties) error {
-	versions := []schema.Version{
-		new(migration.InitialStructure),
-		new(migration.AddingComplexityProperty),
-	}
-	dbSchema := schema.New(ctx, db, appEnv, props, versions)
-	actualSchemaVersion, err := dbSchema.InitiateData()
-	if err != nil {
-		return err
-	}
-	if actualSchemaVersion == "" {
-		errMsg := "schema version must not be empty"
-		logger.Error(errMsg)
-		return fmt.Errorf(errMsg)
-	}
-	appEnv.SetSchemaVersion(actualSchemaVersion)
-	return nil
-}
-
 func main() {
 	err := runServer()
 	if err != nil {

playground/backend/containers/router/Dockerfile

@@ -39,6 +39,10 @@ RUN go mod tidy -v &&\
     cd cmd/server &&\
     go build -ldflags="-X main.BuildCommitHash=$GIT_COMMIT -X main.BuildCommitTimestamp=$GIT_TIMESTAMP" -o /go/bin/server_go_backend
 
+# Build migration tool
+RUN cd cmd/migration_tool &&\
+    go build -o /go/bin/migration_tool
+
 # Null image
 FROM debian:stable-20221114-slim
 
@@ -61,6 +65,7 @@ ENV PROPERTY_PATH=/opt/playground/backend/
 
 # Copy build result
 COPY --from=build /go/bin/server_go_backend /opt/playground/backend/
+COPY --from=build /go/bin/migration_tool /opt/playground/backend/
 COPY --from=build /go/src/playground/backend/configs /opt/playground/backend/configs/
 
 COPY sdks.yaml /opt/playground/backend/sdks.yaml

playground/backend/containers/router/docker-compose.local.yml

@@ -35,6 +35,7 @@ services:
       CACHE_TYPE: remote
       CACHE_ADDRESS: redis:6379
       SDK_CONFIG: /opt/playground/backend/sdks-emulator.yaml
+      APPLY_MIGRATIONS: "True"
     ports:
       - "8080:8080"
     depends_on:

playground/backend/containers/router/entrypoint.sh

@@ -14,4 +14,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Check if APPLY_MIGRATIONS is set and apply mgirations in such case
+if [ -n "$APPLY_MIGRATIONS" ]; then
+    echo "Applying migrations"
+    # If SDK_CONFIG is not set, set it to default value
+    if [ -z "$SDK_CONFIG" ]; then
+        SDK_CONFIG=/opt/playground/backend/sdks.yaml
+    fi
+
+    if [ -n "$DATASTORE_NAMESPACE" ]; then
+        /opt/playground/backend/migration_tool -project-id $GOOGLE_CLOUD_PROJECT -sdk-config $SDK_CONFIG -namespace $DATASTORE_NAMESPACE
+    else
+        /opt/playground/backend/migration_tool -project-id $GOOGLE_CLOUD_PROJECT -sdk-config $SDK_CONFIG
+    fi
+fi
+
 /opt/playground/backend/server_go_backend