Skip to content

Commit

Permalink
[Playground] notebook url backend API (apache#24425)
Browse files Browse the repository at this point in the history 
* proto

* WIP

* generateProto

* fmt

* NIT
  • Loading branch information
@eantyshev
eantyshev authored Dec 15, 2022
1 parent aa6cc16 commit f16d5d5
Show file tree
Hide file tree
Showing 11 changed files with 422 additions and 352 deletions.
6 changes: 5 additions & 1 deletion playground/api/v1/api.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ message PrecompiledObject{
string description = 3;
PrecompiledObjectType type = 4;
string pipeline_options = 5;
// Link to the example in the Beam repository
// TODO mark reserved after #24402 update sequence is done
string link = 6;
bool multifile = 7;
int32 context_line = 8;
Expand All @@ -189,6 +189,10 @@ message PrecompiledObject{
Complexity complexity = 11;
repeated string tags = 12;
repeated Dataset datasets = 13;

// Link to the example in the Beam repository
string url_vcs = 14;
string url_notebook = 15;
}

// Categories represent the array of messages with sdk and categories at this sdk
Expand Down
4 changes: 4 additions & 0 deletions playground/backend/cmd/server/controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1119,6 +1119,8 @@ func TestPlaygroundController_GetPrecompiledObject(t *testing.T) {
ContextLine: 10,
PipelineOptions: "MOCK_P_OPTS",
Link: "MOCK_PATH",
UrlVcs: "MOCK_URL_VCS",
UrlNotebook: "MOCK_URL_NOTEBOOK",
Description: "MOCK_DESCR",
DefaultExample: true,
Complexity: pb.Complexity_COMPLEXITY_MEDIUM,
Expand All @@ -1144,6 +1146,8 @@ func TestPlaygroundController_GetPrecompiledObject(t *testing.T) {
ContextLine: 10,
PipelineOptions: "MOCK_P_OPTS",
Link: "MOCK_PATH",
UrlVcs: "MOCK_URL_VCS",
UrlNotebook: "MOCK_URL_NOTEBOOK",
Description: "MOCK_DESCR",
DefaultExample: false,
Complexity: pb.Complexity_COMPLEXITY_MEDIUM,
Expand Down
551 changes: 286 additions & 265 deletions playground/backend/internal/api/v1/api.pb.go
View file

Large diffs are not rendered by default.

18 changes: 10 additions & 8 deletions playground/backend/internal/components/cache_component_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -291,14 +291,16 @@ func getCatalog() []*pb.Categories {

func saveExample(name, sdk string) {
_, _ = datastoreDb.Client.Put(ctx, utils.GetExampleKey(ctx, sdk, name), &entity.ExampleEntity{
Name: name,
Sdk: utils.GetSdkKey(ctx, sdk),
Descr: "MOCK_DESCR",
Cats: []string{"MOCK_CATEGORY"},
Path: "MOCK_PATH",
Type: pb.PrecompiledObjectType_PRECOMPILED_OBJECT_TYPE_EXAMPLE.String(),
Origin: constants.ExampleOrigin,
SchVer: utils.GetSchemaVerKey(ctx, "MOCK_VERSION"),
Name: name,
Sdk: utils.GetSdkKey(ctx, sdk),
Descr: "MOCK_DESCR",
Cats: []string{"MOCK_CATEGORY"},
Path: "MOCK_PATH",
UrlVCS: "MOCK_URL_VCS",
UrlNotebook: "MOCK_URL_NOTEBOOK",
Type: pb.PrecompiledObjectType_PRECOMPILED_OBJECT_TYPE_EXAMPLE.String(),
Origin: constants.ExampleOrigin,
SchVer: utils.GetSchemaVerKey(ctx, "MOCK_VERSION"),
})
}

Expand Down
2 changes: 2 additions & 0 deletions playground/backend/internal/db/dto/precompiled_object.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ type ObjectInfo struct {
Categories []string
PipelineOptions string
Link string
UrlVCS string
UrlNotebook string
Multifile bool
ContextLine int32
DefaultExample bool
Expand Down
20 changes: 11 additions & 9 deletions playground/backend/internal/db/entity/example.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,17 @@ package entity
import "cloud.google.com/go/datastore"

type ExampleEntity struct {
Name string `datastore:"name"`
Sdk *datastore.Key `datastore:"sdk"`
Descr string `datastore:"descr"`
Tags []string `datastore:"tags"`
Cats []string `datastore:"cats"`
Path string `datastore:"path"`
Type string `datastore:"type"`
Origin string `datastore:"origin"`
SchVer *datastore.Key `datastore:"schVer"`
Name string `datastore:"name"`
Sdk *datastore.Key `datastore:"sdk"`
Descr string `datastore:"descr"`
Tags []string `datastore:"tags"`
Cats []string `datastore:"cats"`
Path string `datastore:"path"` // TODO remove after #24402 update sequence is done
Type string `datastore:"type"`
Origin string `datastore:"origin"`
SchVer *datastore.Key `datastore:"schVer"`
UrlVCS string `datastore:"urlVCS"`
UrlNotebook string `datastore:"urlNotebook"`
}

type PrecompiledObjectEntity struct {
Expand Down
8 changes: 8 additions & 0 deletions playground/backend/internal/db/mapper/precompiled_object_mapper.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ func (pom *PrecompiledObjectMapper) ToObjectInfo(exampleDTO *dto.ExampleDTO) *dt
Categories: exampleDTO.Example.Cats,
PipelineOptions: exampleDTO.Snippet.PipeOpts,
Link: exampleDTO.Example.Path,
UrlVCS: exampleDTO.Example.UrlVCS,
UrlNotebook: exampleDTO.Example.UrlNotebook,
Multifile: exampleDTO.HasMultiFiles(),
ContextLine: exampleDTO.GetContextLine(),
DefaultExample: exampleDTO.IsDefault(),
Expand Down Expand Up @@ -98,6 +100,8 @@ func (pom *PrecompiledObjectMapper) ToDefaultPrecompiledObjects(defaultExamplesD
Type: pb.PrecompiledObjectType(pb.PrecompiledObjectType_value[example.Type]),
PipelineOptions: defaultExamplesDTO.Snippets[exampleIndx].PipeOpts,
Link: example.Path,
UrlVcs: example.UrlVCS,
UrlNotebook: example.UrlNotebook,
Multifile: false,
ContextLine: defaultExamplesDTO.Files[exampleIndx].CntxLine,
DefaultExample: true,
Expand All @@ -117,6 +121,8 @@ func (pom *PrecompiledObjectMapper) ToPrecompiledObj(exampleDTO *dto.ExampleDTO)
Type: exampleDTO.GetType(),
PipelineOptions: exampleDTO.Snippet.PipeOpts,
Link: exampleDTO.Example.Path,
UrlVcs: exampleDTO.Example.UrlVCS,
UrlNotebook: exampleDTO.Example.UrlNotebook,
Multifile: exampleDTO.HasMultiFiles(),
ContextLine: exampleDTO.GetContextLine(),
DefaultExample: exampleDTO.IsDefault(),
Expand Down Expand Up @@ -204,6 +210,8 @@ func putPrecompiledObjectsToCategory(categoryName string, precompiledObjects *dt
Type: object.Type,
PipelineOptions: object.PipelineOptions,
Link: object.Link,
UrlVcs: object.UrlVCS,
UrlNotebook: object.UrlNotebook,
Multifile: object.Multifile,
ContextLine: object.ContextLine,
DefaultExample: object.DefaultExample,
Expand Down
113 changes: 54 additions & 59 deletions playground/backend/internal/db/mapper/precompiled_object_mapper_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,66 +87,61 @@ func TestPrecompiledObjectMapper_ToPrecompiledObj(t *testing.T) {

func TestPrecompiledObjectMapper_ToDefaultPrecompiledObjects(t *testing.T) {
actualResult := pcObjMapper.ToDefaultPrecompiledObjects(getDefaultExamplesDTO())
javaPCObj, ok := actualResult[pb.Sdk_SDK_JAVA]
if !ok ||
javaPCObj.DefaultExample != true ||
javaPCObj.Name != "1_MOCK_DEFAULT_EXAMPLE" ||
javaPCObj.Multifile != false ||
javaPCObj.Type.String() != "PRECOMPILED_OBJECT_TYPE_EXAMPLE" ||
javaPCObj.ContextLine != 32 ||
javaPCObj.Link != "MOCK_PATH" ||
javaPCObj.Description != "MOCK_DESCR" ||
javaPCObj.PipelineOptions != "MOCK_OPTIONS" ||
javaPCObj.CloudPath != "SDK_JAVA/PRECOMPILED_OBJECT_TYPE_EXAMPLE/1_MOCK_DEFAULT_EXAMPLE" ||
javaPCObj.Sdk != pb.Sdk_SDK_JAVA {
t.Error("ToDefaultPrecompiledObjects() unexpected result for SDK_JAVA")
}
goPCObj, ok := actualResult[pb.Sdk_SDK_GO]
if !ok ||
goPCObj.DefaultExample != true ||
goPCObj.Name != "2_MOCK_DEFAULT_EXAMPLE" ||
goPCObj.Multifile != false ||
goPCObj.Type.String() != "PRECOMPILED_OBJECT_TYPE_EXAMPLE" ||
goPCObj.ContextLine != 32 ||
goPCObj.Link != "MOCK_PATH" ||
goPCObj.Description != "MOCK_DESCR" ||
goPCObj.PipelineOptions != "MOCK_OPTIONS" ||
goPCObj.CloudPath != "SDK_GO/PRECOMPILED_OBJECT_TYPE_EXAMPLE/2_MOCK_DEFAULT_EXAMPLE" ||
goPCObj.Sdk != pb.Sdk_SDK_GO {
t.Error("ToDefaultPrecompiledObjects() unexpected result for SDK_GO")
}
scioPCObj, ok := actualResult[pb.Sdk_SDK_SCIO]
if !ok ||
scioPCObj.DefaultExample != true ||
scioPCObj.Name != "4_MOCK_DEFAULT_EXAMPLE" ||
scioPCObj.Multifile != false ||
scioPCObj.Type.String() != "PRECOMPILED_OBJECT_TYPE_EXAMPLE" ||
scioPCObj.ContextLine != 32 ||
scioPCObj.Link != "MOCK_PATH" ||
scioPCObj.Description != "MOCK_DESCR" ||
scioPCObj.PipelineOptions != "MOCK_OPTIONS" ||
scioPCObj.CloudPath != "SDK_SCIO/PRECOMPILED_OBJECT_TYPE_EXAMPLE/4_MOCK_DEFAULT_EXAMPLE" ||
scioPCObj.Sdk != pb.Sdk_SDK_SCIO {
t.Error("ToDefaultPrecompiledObjects() unexpected result for SDK_SCIO")
}
pythonPCObj, ok := actualResult[pb.Sdk_SDK_PYTHON]
if !ok ||
pythonPCObj.DefaultExample != true ||
pythonPCObj.Name != "3_MOCK_DEFAULT_EXAMPLE" ||
pythonPCObj.Multifile != false ||
pythonPCObj.Type.String() != "PRECOMPILED_OBJECT_TYPE_EXAMPLE" ||
pythonPCObj.ContextLine != 32 ||
pythonPCObj.Link != "MOCK_PATH" ||
pythonPCObj.Description != "MOCK_DESCR" ||
pythonPCObj.PipelineOptions != "MOCK_OPTIONS" ||
pythonPCObj.CloudPath != "SDK_PYTHON/PRECOMPILED_OBJECT_TYPE_EXAMPLE/3_MOCK_DEFAULT_EXAMPLE" ||
pythonPCObj.Sdk != pb.Sdk_SDK_PYTHON {
t.Error("ToDefaultPrecompiledObjects() unexpected result for SDK_PYTHON")
}
_, ok = actualResult[pb.Sdk_SDK_UNSPECIFIED]
if ok {
t.Error("ToDefaultPrecompiledObjects() unexpected result for SDK_UNSPECIFIED")
expected := map[pb.Sdk]*pb.PrecompiledObject{
pb.Sdk_SDK_JAVA: {
DefaultExample: true,
Complexity: pb.Complexity_COMPLEXITY_MEDIUM,
Name: "1_MOCK_DEFAULT_EXAMPLE",
Multifile: false,
Type: pb.PrecompiledObjectType_PRECOMPILED_OBJECT_TYPE_EXAMPLE,
ContextLine: 32,
Link: "MOCK_PATH",
Description: "MOCK_DESCR",
PipelineOptions: "MOCK_OPTIONS",
CloudPath: "SDK_JAVA/PRECOMPILED_OBJECT_TYPE_EXAMPLE/1_MOCK_DEFAULT_EXAMPLE",
Sdk: pb.Sdk_SDK_JAVA,
},
pb.Sdk_SDK_GO: {
DefaultExample: true,
Complexity: pb.Complexity_COMPLEXITY_MEDIUM,
Name: "2_MOCK_DEFAULT_EXAMPLE",
Multifile: false,
Type: pb.PrecompiledObjectType_PRECOMPILED_OBJECT_TYPE_EXAMPLE,
ContextLine: 32,
Link: "MOCK_PATH",
Description: "MOCK_DESCR",
PipelineOptions: "MOCK_OPTIONS",
CloudPath: "SDK_GO/PRECOMPILED_OBJECT_TYPE_EXAMPLE/2_MOCK_DEFAULT_EXAMPLE",
Sdk: pb.Sdk_SDK_GO,
},
pb.Sdk_SDK_PYTHON: {
DefaultExample: true,
Complexity: pb.Complexity_COMPLEXITY_MEDIUM,
Name: "3_MOCK_DEFAULT_EXAMPLE",
Multifile: false,
Type: pb.PrecompiledObjectType_PRECOMPILED_OBJECT_TYPE_EXAMPLE,
ContextLine: 32,
Link: "MOCK_PATH",
Description: "MOCK_DESCR",
PipelineOptions: "MOCK_OPTIONS",
CloudPath: "SDK_PYTHON/PRECOMPILED_OBJECT_TYPE_EXAMPLE/3_MOCK_DEFAULT_EXAMPLE",
Sdk: pb.Sdk_SDK_PYTHON,
},
pb.Sdk_SDK_SCIO: {
DefaultExample: true,
Complexity: pb.Complexity_COMPLEXITY_MEDIUM,
Name: "4_MOCK_DEFAULT_EXAMPLE",
Multifile: false,
Type: pb.PrecompiledObjectType_PRECOMPILED_OBJECT_TYPE_EXAMPLE,
ContextLine: 32,
Link: "MOCK_PATH",
Description: "MOCK_DESCR",
PipelineOptions: "MOCK_OPTIONS",
CloudPath: "SDK_SCIO/PRECOMPILED_OBJECT_TYPE_EXAMPLE/4_MOCK_DEFAULT_EXAMPLE",
Sdk: pb.Sdk_SDK_SCIO,
},
}
assert.Equal(t, expected, actualResult)
}

func TestPrecompiledObjectMapper_ToArrayCategories(t *testing.T) {
Expand Down
20 changes: 11 additions & 9 deletions playground/backend/internal/tests/test_data/test_data.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,15 +120,17 @@ func createExampleEntities(ctx context.Context) ([]*datastore.Key, []*entity.Exa

func createExampleEntity(ctx context.Context, name, sdk string) *entity.ExampleEntity {
return &entity.ExampleEntity{
Name: name,
Sdk: utils.GetSdkKey(ctx, sdk),
Descr: "MOCK_DESCR",
Tags: []string{"MOCK_TAG_1", "MOCK_TAG_2", "MOCK_TAG_3"},
Cats: []string{"MOCK_CAT_1", "MOCK_CAT_2", "MOCK_CAT_3"},
Path: "MOCK_PATH",
Type: pb.PrecompiledObjectType_PRECOMPILED_OBJECT_TYPE_EXAMPLE.String(),
Origin: constants.ExampleOrigin,
SchVer: utils.GetSchemaVerKey(ctx, "MOCK_VERSION"),
Name: name,
Sdk: utils.GetSdkKey(ctx, sdk),
Descr: "MOCK_DESCR",
Tags: []string{"MOCK_TAG_1", "MOCK_TAG_2", "MOCK_TAG_3"},
Cats: []string{"MOCK_CAT_1", "MOCK_CAT_2", "MOCK_CAT_3"},
Path: "MOCK_PATH",
UrlVCS: "MOCK_URL_VCS",
UrlNotebook: "MOCK_URL_NOTEBOOK",
Type: pb.PrecompiledObjectType_PRECOMPILED_OBJECT_TYPE_EXAMPLE.String(),
Origin: constants.ExampleOrigin,
SchVer: utils.GetSchemaVerKey(ctx, "MOCK_VERSION"),
}
}

Expand Down
28 changes: 28 additions & 0 deletions playground/frontend/playground_components/lib/src/api/v1/api.pb.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1055,6 +1055,8 @@ class PrecompiledObject extends $pb.GeneratedMessage {
..e<Complexity>(11, const $core.bool.fromEnvironment('protobuf.omit_field_names') ? '' : 'complexity', $pb.PbFieldType.OE, defaultOrMaker: Complexity.COMPLEXITY_UNSPECIFIED, valueOf: Complexity.valueOf, enumValues: Complexity.values)
..pPS(12, const $core.bool.fromEnvironment('protobuf.omit_field_names') ? '' : 'tags')
..pc<Dataset>(13, const $core.bool.fromEnvironment('protobuf.omit_field_names') ? '' : 'datasets', $pb.PbFieldType.PM, subBuilder: Dataset.create)
..aOS(14, const $core.bool.fromEnvironment('protobuf.omit_field_names') ? '' : 'urlVcs')
..aOS(15, const $core.bool.fromEnvironment('protobuf.omit_field_names') ? '' : 'urlNotebook')
..hasRequiredFields = false
;

Expand All @@ -1073,6 +1075,8 @@ class PrecompiledObject extends $pb.GeneratedMessage {
Complexity? complexity,
$core.Iterable<$core.String>? tags,
$core.Iterable<Dataset>? datasets,
$core.String? urlVcs,
$core.String? urlNotebook,
}) {
final _result = create();
if (cloudPath != null) {
Expand Down Expand Up @@ -1114,6 +1118,12 @@ class PrecompiledObject extends $pb.GeneratedMessage {
if (datasets != null) {
_result.datasets.addAll(datasets);
}
if (urlVcs != null) {
_result.urlVcs = urlVcs;
}
if (urlNotebook != null) {
_result.urlNotebook = urlNotebook;
}
return _result;
}
factory PrecompiledObject.fromBuffer($core.List<$core.int> i, [$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) => create()..mergeFromBuffer(i, r);
Expand Down Expand Up @@ -1241,6 +1251,24 @@ class PrecompiledObject extends $pb.GeneratedMessage {

@$pb.TagNumber(13)
$core.List<Dataset> get datasets => $_getList(12);

@$pb.TagNumber(14)
$core.String get urlVcs => $_getSZ(13);
@$pb.TagNumber(14)
set urlVcs($core.String v) { $_setString(13, v); }
@$pb.TagNumber(14)
$core.bool hasUrlVcs() => $_has(13);
@$pb.TagNumber(14)
void clearUrlVcs() => clearField(14);

@$pb.TagNumber(15)
$core.String get urlNotebook => $_getSZ(14);
@$pb.TagNumber(15)
set urlNotebook($core.String v) { $_setString(14, v); }
@$pb.TagNumber(15)
$core.bool hasUrlNotebook() => $_has(14);
@$pb.TagNumber(15)
void clearUrlNotebook() => clearField(15);
}

class Categories_Category extends $pb.GeneratedMessage {
Expand Down
4 changes: 3 additions & 1 deletion playground/frontend/playground_components/lib/src/api/v1/api.pbjson.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -321,11 +321,13 @@ const PrecompiledObject$json = const {
const {'1': 'complexity', '3': 11, '4': 1, '5': 14, '6': '.api.v1.Complexity', '10': 'complexity'},
const {'1': 'tags', '3': 12, '4': 3, '5': 9, '10': 'tags'},
const {'1': 'datasets', '3': 13, '4': 3, '5': 11, '6': '.api.v1.Dataset', '10': 'datasets'},
const {'1': 'url_vcs', '3': 14, '4': 1, '5': 9, '10': 'urlVcs'},
const {'1': 'url_notebook', '3': 15, '4': 1, '5': 9, '10': 'urlNotebook'},
],
};

/// Descriptor for `PrecompiledObject`. Decode as a `google.protobuf.DescriptorProto`.
final $typed_data.Uint8List precompiledObjectDescriptor = $convert.base64Decode('ChFQcmVjb21waWxlZE9iamVjdBIdCgpjbG91ZF9wYXRoGAEgASgJUgljbG91ZFBhdGgSEgoEbmFtZRgCIAEoCVIEbmFtZRIgCgtkZXNjcmlwdGlvbhgDIAEoCVILZGVzY3JpcHRpb24SMQoEdHlwZRgEIAEoDjIdLmFwaS52MS5QcmVjb21waWxlZE9iamVjdFR5cGVSBHR5cGUSKQoQcGlwZWxpbmVfb3B0aW9ucxgFIAEoCVIPcGlwZWxpbmVPcHRpb25zEhIKBGxpbmsYBiABKAlSBGxpbmsSHAoJbXVsdGlmaWxlGAcgASgIUgltdWx0aWZpbGUSIQoMY29udGV4dF9saW5lGAggASgFUgtjb250ZXh0TGluZRInCg9kZWZhdWx0X2V4YW1wbGUYCSABKAhSDmRlZmF1bHRFeGFtcGxlEh0KA3NkaxgKIAEoDjILLmFwaS52MS5TZGtSA3NkaxIyCgpjb21wbGV4aXR5GAsgASgOMhIuYXBpLnYxLkNvbXBsZXhpdHlSCmNvbXBsZXhpdHkSEgoEdGFncxgMIAMoCVIEdGFncxIrCghkYXRhc2V0cxgNIAMoCzIPLmFwaS52MS5EYXRhc2V0UghkYXRhc2V0cw==');
final $typed_data.Uint8List precompiledObjectDescriptor = $convert.base64Decode('ChFQcmVjb21waWxlZE9iamVjdBIdCgpjbG91ZF9wYXRoGAEgASgJUgljbG91ZFBhdGgSEgoEbmFtZRgCIAEoCVIEbmFtZRIgCgtkZXNjcmlwdGlvbhgDIAEoCVILZGVzY3JpcHRpb24SMQoEdHlwZRgEIAEoDjIdLmFwaS52MS5QcmVjb21waWxlZE9iamVjdFR5cGVSBHR5cGUSKQoQcGlwZWxpbmVfb3B0aW9ucxgFIAEoCVIPcGlwZWxpbmVPcHRpb25zEhIKBGxpbmsYBiABKAlSBGxpbmsSHAoJbXVsdGlmaWxlGAcgASgIUgltdWx0aWZpbGUSIQoMY29udGV4dF9saW5lGAggASgFUgtjb250ZXh0TGluZRInCg9kZWZhdWx0X2V4YW1wbGUYCSABKAhSDmRlZmF1bHRFeGFtcGxlEh0KA3NkaxgKIAEoDjILLmFwaS52MS5TZGtSA3NkaxIyCgpjb21wbGV4aXR5GAsgASgOMhIuYXBpLnYxLkNvbXBsZXhpdHlSCmNvbXBsZXhpdHkSEgoEdGFncxgMIAMoCVIEdGFncxIrCghkYXRhc2V0cxgNIAMoCzIPLmFwaS52MS5EYXRhc2V0UghkYXRhc2V0cxIXCgd1cmxfdmNzGA4gASgJUgZ1cmxWY3MSIQoMdXJsX25vdGVib29rGA8gASgJUgt1cmxOb3RlYm9vaw==');
@$core.Deprecated('Use categoriesDescriptor instead')
const Categories$json = const {
'1': 'Categories',
Expand Down

0 comments on commit f16d5d5

Please sign in to comment.