[Playground] update snippet by persistence_key (apache#24056)

* proto * AIO * -frontend * testify * +license * -build * index * skipKey * Update playground/backend/internal/db/datastore/datastore_db.go Co-authored-by: Danny McCormick <[email protected]> Co-authored-by: Danny McCormick <[email protected]>
rshamunov · Nov 10, 2022 · 36b0c8f · 36b0c8f
1 parent df553d1
commit 36b0c8f
Show file tree

Hide file tree

Showing 13 changed files with 501 additions and 518 deletions.
diff --git a/playground/api/v1/api.proto b/playground/api/v1/api.proto
@@ -273,6 +273,7 @@ message SaveSnippetRequest {
   // The pipeline options as they would be passed to the program (e.g. "--option1 value1 --option2 value2")
   string pipeline_options = 3;
   Complexity complexity = 4;
+  string persistence_key = 5;
 }
 
 // SaveSnippetResponse contains information of the generated ID.

diff --git a/playground/backend/cmd/server/controller.go b/playground/backend/cmd/server/controller.go
@@ -408,23 +408,23 @@ func (controller *playgroundController) GetDefaultPrecompiledObject(ctx context.
 }
 
 // SaveSnippet returns the generated ID
-func (controller *playgroundController) SaveSnippet(ctx context.Context, info *pb.SaveSnippetRequest) (*pb.SaveSnippetResponse, error) {
-	if info.Sdk == pb.Sdk_SDK_UNSPECIFIED {
-		logger.Errorf("SaveSnippet(): unimplemented sdk: %s\n", info.Sdk)
-		return nil, errors.InvalidArgumentError(errorTitleSaveSnippet, "Sdk is not implemented yet: %s", info.Sdk.String())
+func (controller *playgroundController) SaveSnippet(ctx context.Context, req *pb.SaveSnippetRequest) (*pb.SaveSnippetResponse, error) {
+	if req.Sdk == pb.Sdk_SDK_UNSPECIFIED {
+		logger.Errorf("SaveSnippet(): unimplemented sdk: %s\n", req.Sdk)
+		return nil, errors.InvalidArgumentError(errorTitleSaveSnippet, "Sdk is not implemented yet: %s", req.Sdk.String())
 	}
 	if controller.db == nil {
 		logger.Error("SaveSnippet(): the runner is trying to save the snippet")
 		return nil, errors.InvalidArgumentError(errorTitleSaveSnippet, "The runner doesn't support snippets")
 	}
-	if info.Files == nil || len(info.Files) == 0 {
+	if req.Files == nil || len(req.Files) == 0 {
 		logger.Error("SaveSnippet(): files are empty")
 		return nil, errors.InvalidArgumentError(errorTitleSaveSnippet, "Snippet must have files")
 	}
 
-	snippet := controller.entityMapper.ToSnippet(info)
+	snippet := controller.entityMapper.ToSnippet(req)
 
-	for _, file := range info.Files {
+	for _, file := range req.Files {
 		if file.Content == "" {
 			logger.Error("SaveSnippet(): entity is empty")
 			return nil, errors.InvalidArgumentError(errorTitleSaveSnippet, "Snippet must have some content")
@@ -434,7 +434,7 @@ func (controller *playgroundController) SaveSnippet(ctx context.Context, info *p
 			logger.Errorf("SaveSnippet(): entity is too large. Max entity size: %d symbols", maxSnippetSize)
 			return nil, errors.InvalidArgumentError(errorTitleSaveSnippet, "Snippet size is more than %d symbols", maxSnippetSize)
 		}
-		fileEntity, err := controller.entityMapper.ToFileEntity(info, file)
+		fileEntity, err := controller.entityMapper.ToFileEntity(req, file)
 		if err != nil {
 			logger.Errorf("SaveSnippet(): file has wrong properties, err: %s", err.Error())
 			return nil, errors.InvalidArgumentError(errorTitleSaveSnippet, "File content is invalid")

diff --git a/playground/backend/cmd/server/controller_test.go b/playground/backend/cmd/server/controller_test.go
@@ -28,6 +28,7 @@ import (
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/stretchr/testify/assert"
 	"go.uber.org/goleak"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
@@ -45,6 +46,7 @@ import (
 	"beam.apache.org/playground/backend/internal/db/schema"
 	"beam.apache.org/playground/backend/internal/db/schema/migration"
 	"beam.apache.org/playground/backend/internal/environment"
+	"beam.apache.org/playground/backend/internal/logger"
 	"beam.apache.org/playground/backend/internal/tests/test_cleaner"
 	"beam.apache.org/playground/backend/internal/tests/test_data"
 	"beam.apache.org/playground/backend/internal/tests/test_utils"
@@ -95,6 +97,8 @@ func setup() *grpc.Server {
 		panic(err)
 	}
 
+	logger.SetupLogger(ctx, "local", "some_google_project_id")
+
 	// setup cache
 	cacheService = local.New(ctx)
 
@@ -965,6 +969,70 @@ func TestPlaygroundController_GetSnippet(t *testing.T) {
 	}
 }
 
+func makeSaveSnippetRequest() *pb.SaveSnippetRequest {
+	return &pb.SaveSnippetRequest{
+		Files: []*pb.SnippetFile{
+			{Name: "main.py", Content: "import sys; sys.exit(0)", IsMain: true},
+		},
+		Sdk:             pb.Sdk_SDK_PYTHON,
+		PipelineOptions: "some pipe opts",
+		PersistenceKey:  "persistent_key_1",
+	}
+}
+
+func TestPlaygroundController_SaveSnippetPersistent(t *testing.T) {
+	defer goleak.VerifyNone(t, opt)
+	client, closeFunc := getPlaygroundServiceClient(ctx, t)
+	defer closeFunc()
+	snip := makeSaveSnippetRequest()
+
+	t.Log("SaveSnippet, insert 1st version")
+	resp1, err := client.SaveSnippet(ctx, snip)
+	if err != nil {
+		t.Fatalf("1st SaveSnippet failed: %v", err)
+	}
+
+	t.Log("GetSnippet by 1st snippet_id")
+	content, err := client.GetSnippet(ctx, &pb.GetSnippetRequest{Id: resp1.Id})
+	if err != nil {
+		t.Fatalf("1st GetSnippet() error = %v", err)
+	}
+	assert.Equal(t, content.Files[0].Content, snip.Files[0].Content)
+	assert.Equal(t, content.PipelineOptions, snip.PipelineOptions)
+	assert.Equal(t, content.Sdk, snip.Sdk)
+
+	t.Log("PutSnippet: insert 2nd version")
+	snip.Files[0].Content = "some new content"
+	snip.PipelineOptions = "new pipeline opts"
+	resp2, err := client.SaveSnippet(ctx, snip)
+	if err != nil {
+		t.Fatalf("2nd SaveSnippet failed: %v", err)
+	}
+
+	if resp2.Id == resp1.Id {
+		t.Error("snippet_id is the same")
+	}
+
+	t.Log("GetSnippet 1st version: not found")
+	_, err = client.GetSnippet(ctx, &pb.GetSnippetRequest{Id: resp1.Id})
+	if err == nil {
+		t.Fatal("1st snippet not deleted")
+	}
+
+	t.Log("GetSnippet 2nd version")
+	content, err = client.GetSnippet(ctx, &pb.GetSnippetRequest{Id: resp2.Id})
+	if err != nil {
+		t.Fatalf("get 2nd snippet: %v", err)
+	}
+	assert.Equal(t, content.Files[0].Content, snip.Files[0].Content)
+	assert.Equal(t, content.PipelineOptions, snip.PipelineOptions)
+	assert.Equal(t, content.Sdk, snip.Sdk)
+
+	t.Log("cleanup 2nd version only")
+	test_cleaner.CleanFiles(ctx, t, resp2.Id, 1)
+	test_cleaner.CleanSnippet(ctx, t, resp2.Id)
+}
+
 func TestPlaygroundController_GetPrecompiledObjects(t *testing.T) {
 	defer goleak.VerifyNone(t, opt)
 	client, closeFunc := getPlaygroundServiceClient(ctx, t)

diff --git a/playground/backend/go.mod b/playground/backend/go.mod
@@ -18,7 +18,7 @@ module beam.apache.org/playground/backend
 go 1.18
 
 require (
-	cloud.google.com/go/datastore v1.6.0
+	cloud.google.com/go/datastore v1.9.0
 	cloud.google.com/go/logging v1.4.2
 	github.com/go-redis/redis/v8 v8.11.4
 	github.com/go-redis/redismock/v8 v8.0.6
@@ -27,45 +27,48 @@ require (
 	github.com/procyon-projects/chrono v1.1.0
 	github.com/rs/cors v1.8.0
 	github.com/spf13/viper v1.12.0
+	github.com/stretchr/testify v1.7.1
 	go.uber.org/goleak v1.1.12
-	google.golang.org/grpc v1.47.0
-	google.golang.org/protobuf v1.28.0
+	google.golang.org/grpc v1.50.1
+	google.golang.org/protobuf v1.28.1
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
-	cloud.google.com/go v0.102.1 // indirect
-	cloud.google.com/go/compute v1.7.0 // indirect
+	cloud.google.com/go v0.104.0 // indirect
+	cloud.google.com/go/compute v1.10.0 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/fsnotify/fsnotify v1.5.4 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/google/go-cmp v0.5.8 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect
-	github.com/googleapis/gax-go/v2 v2.4.0 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.6.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/klauspost/compress v1.13.1 // indirect
 	github.com/magiconair/properties v1.8.6 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/spf13/afero v1.8.2 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/subosito/gotenv v1.3.0 // indirect
 	go.opencensus.io v0.23.0 // indirect
-	golang.org/x/net v0.0.0-20220617184016-355a448f1bc9 // indirect
-	golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb // indirect
-	golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect
-	golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c // indirect
+	golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458 // indirect
+	golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1 // indirect
+	golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0 // indirect
+	golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect
 	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
-	google.golang.org/api v0.84.0 // indirect
+	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
+	google.golang.org/api v0.99.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad // indirect
+	google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/ini.v1 v1.66.4 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect