Skip to content

Commit 9f9f7be

Browse files
vaikasvaikas
authored
Introduce key/value store interface + cm backed version of it. (knative#1173)
* simple configstore for saving state in configmaps * introduce interface, tests * address pr feedback * return interface -> add Init to interface
1 parent 0840da9 commit 9f9f7be

File tree

3 files changed

+354
-0
lines changed

3 files changed

+354
-0
lines changed

‎kvstore/kvstore.go

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
/*
2+
Copyright 2020 The Knative Authors
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
https://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package kvstore
18+
19+
import (
20+
"context"
21+
)
22+
23+
type Interface interface {
24+
// Init loads the configstore from the backing store if it exists, or
25+
// if it does not, will create an empty one.
26+
Init(ctx context.Context) error
27+
// Load loads the configstore from the backing store
28+
Load(ctx context.Context) error
29+
// Save saves the configstore to the backing store
30+
Save(ctx context.Context) error
31+
// Get gets the key from the KVStore into the provided value
32+
Get(ctx context.Context, key string, value interface{}) error
33+
// Set sets the key into the KVStore from the provided value
34+
Set(ctx context.Context, key string, value interface{}) error
35+
}

‎kvstore/kvstore_cm.go

+123
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
/*
2+
Copyright 2020 The Knative Authors
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
https://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
// Simple abstraction for storing state on a k8s ConfigMap. Very very simple
18+
// and uses a single entry in the ConfigMap.data for storing serialized
19+
// JSON of the generic data that Load/Save uses. Handy for things like sources
20+
// that need to persist some state (checkpointing for example).
21+
package kvstore
22+
23+
import (
24+
"context"
25+
"encoding/json"
26+
"fmt"
27+
28+
corev1 "k8s.io/api/core/v1"
29+
apierrors "k8s.io/apimachinery/pkg/api/errors"
30+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
31+
v1 "k8s.io/client-go/kubernetes/typed/core/v1"
32+
"knative.dev/pkg/logging"
33+
)
34+
35+
type configMapKVStore struct {
36+
cmClient v1.ConfigMapInterface
37+
name string
38+
namespace string
39+
data map[string]string
40+
}
41+
42+
var (
43+
_ Interface = (*configMapKVStore)(nil)
44+
)
45+
46+
func NewConfigMapKVStore(ctx context.Context, name string, namespace string, clientset v1.CoreV1Interface) Interface {
47+
48+
return &configMapKVStore{name: name, namespace: namespace, cmClient: clientset.ConfigMaps(namespace)}
49+
}
50+
51+
// Init initializes configMapKVStore either by loading or creating an empty one.
52+
func (cs *configMapKVStore) Init(ctx context.Context) error {
53+
l := logging.FromContext(ctx)
54+
l.Info("Initializing configMapKVStore...")
55+
56+
err := cs.Load(ctx)
57+
if apierrors.IsNotFound(err) {
58+
l.Info("No config found, creating empty")
59+
return cs.createConfigMap()
60+
}
61+
return err
62+
}
63+
64+
// Load fetches the ConfigMap from k8s and unmarshals the data found
65+
// in the configdatakey type as specified by value.
66+
func (cs *configMapKVStore) Load(ctx context.Context) error {
67+
cm, err := cs.cmClient.Get(cs.name, metav1.GetOptions{})
68+
if err != nil {
69+
return err
70+
}
71+
cs.data = cm.Data
72+
return nil
73+
}
74+
75+
// Save takes the value given in, and marshals it into a string
76+
// and saves it into the k8s ConfigMap under the configdatakey.
77+
func (cs *configMapKVStore) Save(ctx context.Context) error {
78+
cm, err := cs.cmClient.Get(cs.name, metav1.GetOptions{})
79+
if err != nil {
80+
return err
81+
}
82+
cm.Data = cs.data
83+
_, err = cs.cmClient.Update(cm)
84+
return err
85+
}
86+
87+
// Get retrieves and unmarshals the value from the map.
88+
func (cs *configMapKVStore) Get(ctx context.Context, key string, value interface{}) error {
89+
v, ok := cs.data[key]
90+
if !ok {
91+
return fmt.Errorf("key %s does not exist", key)
92+
}
93+
err := json.Unmarshal([]byte(v), value)
94+
if err != nil {
95+
return fmt.Errorf("Failed to Unmarshal %q: %v", v, err)
96+
}
97+
return nil
98+
}
99+
100+
// Set marshals and sets the value given under specified key.
101+
func (cs *configMapKVStore) Set(ctx context.Context, key string, value interface{}) error {
102+
bytes, err := json.Marshal(value)
103+
if err != nil {
104+
return fmt.Errorf("Failed to Marshal: %v", err)
105+
}
106+
cs.data[key] = string(bytes)
107+
return nil
108+
}
109+
110+
func (cs *configMapKVStore) createConfigMap() error {
111+
cm := &corev1.ConfigMap{
112+
TypeMeta: metav1.TypeMeta{
113+
APIVersion: "v1",
114+
Kind: "ConfigMap",
115+
},
116+
ObjectMeta: metav1.ObjectMeta{
117+
Name: cs.name,
118+
Namespace: cs.namespace,
119+
},
120+
}
121+
_, err := cs.cmClient.Create(cm)
122+
return err
123+
}

‎kvstore/kvstore_cm_test.go

+196
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
/*
2+
Copyright 2020 The Knative Authors
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package kvstore
18+
19+
import (
20+
"context"
21+
"encoding/json"
22+
"reflect"
23+
"testing"
24+
25+
corev1 "k8s.io/api/core/v1"
26+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
27+
"k8s.io/apimachinery/pkg/runtime"
28+
"k8s.io/client-go/kubernetes/fake"
29+
v1 "k8s.io/client-go/kubernetes/typed/core/v1"
30+
clientgotesting "k8s.io/client-go/testing"
31+
)
32+
33+
const (
34+
namespace = "mynamespace"
35+
name = "mycm"
36+
)
37+
38+
type testStruct struct {
39+
LastThingProcessed string
40+
Stuff []string
41+
}
42+
43+
type testClient struct {
44+
created *corev1.ConfigMap
45+
updated *corev1.ConfigMap
46+
clientset v1.CoreV1Interface
47+
}
48+
49+
func NewTestClient(objects ...runtime.Object) *testClient {
50+
tc := testClient{}
51+
cs := fake.NewSimpleClientset(objects...)
52+
cs.PrependReactor("create", "*", func(action clientgotesting.Action) (handled bool, ret runtime.Object, err error) {
53+
createAction := action.(clientgotesting.CreateAction)
54+
tc.created = createAction.GetObject().(*corev1.ConfigMap)
55+
return true, tc.created, nil
56+
})
57+
cs.PrependReactor("update", "*", func(action clientgotesting.Action) (handled bool, ret runtime.Object, err error) {
58+
updateAction := action.(clientgotesting.UpdateAction)
59+
tc.updated = updateAction.GetObject().(*corev1.ConfigMap)
60+
return true, tc.updated, nil
61+
})
62+
tc.clientset = cs.CoreV1()
63+
return &tc
64+
}
65+
66+
func TestInitCreates(t *testing.T) {
67+
tc := NewTestClient()
68+
cs := NewConfigMapKVStore(context.Background(), name, namespace, tc.clientset)
69+
err := cs.Init(context.Background())
70+
if err != nil {
71+
t.Errorf("Failed to Init ConfigStore: %v", err)
72+
}
73+
if tc.created == nil {
74+
t.Errorf("ConfigMap not created")
75+
}
76+
if len(tc.created.Data) != 0 {
77+
t.Errorf("ConfigMap data is not empty")
78+
}
79+
if tc.updated != nil {
80+
t.Errorf("ConfigMap updated")
81+
}
82+
}
83+
84+
func TestLoadNonexisting(t *testing.T) {
85+
tc := NewTestClient()
86+
if NewConfigMapKVStore(context.Background(), name, namespace, tc.clientset).Load(context.Background()) == nil {
87+
t.Error("non-existent store load didn't fail")
88+
}
89+
}
90+
91+
func TestInitLoads(t *testing.T) {
92+
tc := NewTestClient([]runtime.Object{configMap(map[string]string{"foo": marshal(t, "bar")})}...)
93+
cs := NewConfigMapKVStore(context.Background(), name, namespace, tc.clientset)
94+
err := cs.Init(context.Background())
95+
if err != nil {
96+
t.Errorf("Failed to Init ConfigStore: %v", err)
97+
}
98+
if tc.created != nil {
99+
t.Errorf("ConfigMap created")
100+
}
101+
if tc.updated != nil {
102+
t.Errorf("ConfigMap updated")
103+
}
104+
var ret string
105+
err = cs.Get(context.Background(), "foo", &ret)
106+
if err != nil {
107+
t.Errorf("failed to return string: %v", err)
108+
}
109+
if ret != "bar" {
110+
t.Errorf("got back unexpected value, wanted %q got %q", "bar", ret)
111+
}
112+
if cs.Get(context.Background(), "not there", &ret) == nil {
113+
t.Error("non-existent key didn't error")
114+
}
115+
}
116+
117+
func TestLoadSaveUpdate(t *testing.T) {
118+
tc := NewTestClient([]runtime.Object{configMap(map[string]string{"foo": marshal(t, "bar")})}...)
119+
cs := NewConfigMapKVStore(context.Background(), name, namespace, tc.clientset)
120+
err := cs.Init(context.Background())
121+
if err != nil {
122+
t.Errorf("Failed to Init ConfigStore: %v", err)
123+
}
124+
cs.Set(context.Background(), "jimmy", "otherbar")
125+
cs.Save(context.Background())
126+
if tc.updated == nil {
127+
t.Errorf("ConfigMap Not updated")
128+
}
129+
var ret string
130+
err = cs.Get(context.Background(), "jimmy", &ret)
131+
if err != nil {
132+
t.Errorf("failed to return string: %v", err)
133+
}
134+
if err != nil {
135+
t.Errorf("failed to return string: %v", err)
136+
}
137+
if ret != "otherbar" {
138+
t.Errorf("got back unexpected value, wanted %q got %q", "bar", ret)
139+
}
140+
}
141+
142+
func TestLoadSaveUpdateComplex(t *testing.T) {
143+
ts := testStruct{
144+
LastThingProcessed: "somethingie",
145+
Stuff: []string{"first", "second", "third"},
146+
}
147+
148+
tc := NewTestClient([]runtime.Object{configMap(map[string]string{"foo": marshal(t, &ts)})}...)
149+
cs := NewConfigMapKVStore(context.Background(), name, namespace, tc.clientset)
150+
err := cs.Init(context.Background())
151+
if err != nil {
152+
t.Errorf("Failed to Init ConfigStore: %v", err)
153+
}
154+
ts2 := testStruct{
155+
LastThingProcessed: "otherthingie",
156+
Stuff: []string{"fourth", "fifth", "sixth"},
157+
}
158+
cs.Set(context.Background(), "jimmy", &ts2)
159+
cs.Save(context.Background())
160+
if tc.updated == nil {
161+
t.Errorf("ConfigMap Not updated")
162+
}
163+
var ret testStruct
164+
err = cs.Get(context.Background(), "jimmy", &ret)
165+
if err != nil {
166+
t.Errorf("failed to return string: %v", err)
167+
}
168+
if err != nil {
169+
t.Errorf("failed to return string: %v", err)
170+
}
171+
if !reflect.DeepEqual(ret, ts2) {
172+
t.Errorf("got back unexpected value, wanted %+v got %+v", ts2, ret)
173+
}
174+
}
175+
176+
func marshal(t *testing.T, value interface{}) string {
177+
bytes, err := json.Marshal(value)
178+
if err != nil {
179+
t.Fatalf("Failed to Marshal %q: %v", value, err)
180+
}
181+
return string(bytes)
182+
}
183+
184+
func configMap(data map[string]string) *corev1.ConfigMap {
185+
return &corev1.ConfigMap{
186+
TypeMeta: metav1.TypeMeta{
187+
APIVersion: "v1",
188+
Kind: "ConfigMap",
189+
},
190+
ObjectMeta: metav1.ObjectMeta{
191+
Name: name,
192+
Namespace: namespace,
193+
},
194+
Data: data,
195+
}
196+
}

0 commit comments

Comments
 (0)