Skip to content

Commit

Permalink
Apply garbage collection for tree (yorkie-team#566)
Browse files Browse the repository at this point in the history
* Apply garbage collection for tree

* Correct issue that miscalculates treePos by path

* Correct issue when node.Parent is nil
---------

Co-authored-by: Youngteac Hong <[email protected]>
  • Loading branch information
JOOHOJANG and hackerwins authored Jun 29, 2023
1 parent 23b0062 commit db99457
Show file tree
Hide file tree
Showing 14 changed files with 318 additions and 54 deletions.
6 changes: 3 additions & 3 deletions pkg/document/change/context.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,9 +78,9 @@ func (c *Context) RegisterRemovedElementPair(parent crdt.Container, deleted crdt
c.root.RegisterRemovedElementPair(parent, deleted)
}

// RegisterTextElementWithGarbage register the given text element with garbage to hash table.
func (c *Context) RegisterTextElementWithGarbage(textType crdt.TextElement) {
c.root.RegisterTextElementWithGarbage(textType)
// RegisterElementHasRemovedNodes register the given text element with garbage to hash table.
func (c *Context) RegisterElementHasRemovedNodes(element crdt.GCElement) {
c.root.RegisterElementHasRemovedNodes(element)
}

// LastTimeTicket returns the last time ticket issued by this context.
Expand Down
6 changes: 3 additions & 3 deletions pkg/document/crdt/element.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,11 @@ type Container interface {
DeleteByCreatedAt(createdAt *time.Ticket, deletedAt *time.Ticket) Element
}

// TextElement represents Text.
type TextElement interface {
// GCElement represents Element which has GC.
type GCElement interface {
Element
removedNodesLen() int
purgeTextNodesWithGarbage(ticket *time.Ticket) int
purgeRemovedNodesBefore(ticket *time.Ticket) int
}

// Element represents JSON element.
Expand Down
4 changes: 2 additions & 2 deletions pkg/document/crdt/rga_tree_split.go
Original file line number Diff line number Diff line change
Expand Up @@ -631,8 +631,8 @@ func (s *RGATreeSplit[V]) removedNodesLen() int {
return len(s.removedNodeMap)
}

// purgeTextNodesWithGarbage physically purges nodes that have been removed.
func (s *RGATreeSplit[V]) purgeTextNodesWithGarbage(ticket *time.Ticket) int {
// purgeRemovedNodesBefore physically purges nodes that have been removed.
func (s *RGATreeSplit[V]) purgeRemovedNodesBefore(ticket *time.Ticket) int {
count := 0
for _, node := range s.removedNodeMap {
if node.removedAt != nil && ticket.Compare(node.removedAt) >= 0 {
Expand Down
24 changes: 12 additions & 12 deletions pkg/document/crdt/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,15 +39,15 @@ type Root struct {
object *Object
elementMapByCreatedAt map[string]Element
removedElementPairMapByCreatedAt map[string]ElementPair
textElementWithGarbageMapByCreatedAt map[string]TextElement
elementHasRemovedNodesSetByCreatedAt map[string]GCElement
}

// NewRoot creates a new instance of Root.
func NewRoot(root *Object) *Root {
r := &Root{
elementMapByCreatedAt: make(map[string]Element),
removedElementPairMapByCreatedAt: make(map[string]ElementPair),
textElementWithGarbageMapByCreatedAt: make(map[string]TextElement),
elementHasRemovedNodesSetByCreatedAt: make(map[string]GCElement),
}

r.object = root
Expand Down Expand Up @@ -95,9 +95,9 @@ func (r *Root) RegisterRemovedElementPair(parent Container, elem Element) {
}
}

// RegisterTextElementWithGarbage register the given text element with garbage to hash table.
func (r *Root) RegisterTextElementWithGarbage(textType TextElement) {
r.textElementWithGarbageMapByCreatedAt[textType.CreatedAt().Key()] = textType
// RegisterElementHasRemovedNodes register the given element with garbage to hash table.
func (r *Root) RegisterElementHasRemovedNodes(element GCElement) {
r.elementHasRemovedNodesSetByCreatedAt[element.CreatedAt().Key()] = element
}

// DeepCopy copies itself deeply.
Expand All @@ -120,12 +120,12 @@ func (r *Root) GarbageCollect(ticket *time.Ticket) int {
}
}

for _, text := range r.textElementWithGarbageMapByCreatedAt {
purgedTextNodes := text.purgeTextNodesWithGarbage(ticket)
if purgedTextNodes > 0 {
delete(r.textElementWithGarbageMapByCreatedAt, text.CreatedAt().Key())
for _, node := range r.elementHasRemovedNodesSetByCreatedAt {
purgedNodes := node.purgeRemovedNodesBefore(ticket)
if purgedNodes > 0 {
delete(r.elementHasRemovedNodesSetByCreatedAt, node.CreatedAt().Key())
}
count += purgedTextNodes
count += purgedNodes
}

return count
Expand Down Expand Up @@ -157,8 +157,8 @@ func (r *Root) GarbageLen() int {
}
}

for _, text := range r.textElementWithGarbageMapByCreatedAt {
count += text.removedNodesLen()
for _, element := range r.elementHasRemovedNodesSetByCreatedAt {
count += element.removedNodesLen()
}

return count
Expand Down
20 changes: 10 additions & 10 deletions pkg/document/crdt/root_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,9 @@ import (
"github.com/yorkie-team/yorkie/test/helper"
)

func registerTextElementWithGarbage(fromPos, toPos *crdt.RGATreeSplitNodePos, root *crdt.Root, text crdt.TextElement) {
func registerElementHasRemovedNodes(fromPos, toPos *crdt.RGATreeSplitNodePos, root *crdt.Root, text crdt.GCElement) {
if !fromPos.Equal(toPos) {
root.RegisterTextElementWithGarbage(text)
root.RegisterElementHasRemovedNodes(text)
}
}

Expand Down Expand Up @@ -61,28 +61,28 @@ func TestRoot(t *testing.T) {
fromPos, toPos, _ := text.CreateRange(0, 0)
_, _, err := text.Edit(fromPos, toPos, nil, "Hello World", nil, ctx.IssueTimeTicket())
assert.NoError(t, err)
registerTextElementWithGarbage(fromPos, toPos, root, text)
registerElementHasRemovedNodes(fromPos, toPos, root, text)
assert.Equal(t, "Hello World", text.String())
assert.Equal(t, 0, root.GarbageLen())

fromPos, toPos, _ = text.CreateRange(5, 10)
_, _, err = text.Edit(fromPos, toPos, nil, "Yorkie", nil, ctx.IssueTimeTicket())
assert.NoError(t, err)
registerTextElementWithGarbage(fromPos, toPos, root, text)
registerElementHasRemovedNodes(fromPos, toPos, root, text)
assert.Equal(t, "HelloYorkied", text.String())
assert.Equal(t, 1, root.GarbageLen())

fromPos, toPos, _ = text.CreateRange(0, 5)
_, _, err = text.Edit(fromPos, toPos, nil, "", nil, ctx.IssueTimeTicket())
assert.NoError(t, err)
registerTextElementWithGarbage(fromPos, toPos, root, text)
registerElementHasRemovedNodes(fromPos, toPos, root, text)
assert.Equal(t, "Yorkied", text.String())
assert.Equal(t, 2, root.GarbageLen())

fromPos, toPos, _ = text.CreateRange(6, 7)
_, _, err = text.Edit(fromPos, toPos, nil, "", nil, ctx.IssueTimeTicket())
assert.NoError(t, err)
registerTextElementWithGarbage(fromPos, toPos, root, text)
registerElementHasRemovedNodes(fromPos, toPos, root, text)
assert.Equal(t, "Yorkie", text.String())
assert.Equal(t, 3, root.GarbageLen())

Expand Down Expand Up @@ -121,7 +121,7 @@ func TestRoot(t *testing.T) {
fromPos, toPos, _ := text.CreateRange(tc.from, tc.to)
_, _, err := text.Edit(fromPos, toPos, nil, tc.content, nil, ctx.IssueTimeTicket())
assert.NoError(t, err)
registerTextElementWithGarbage(fromPos, toPos, root, text)
registerElementHasRemovedNodes(fromPos, toPos, root, text)
assert.Equal(t, tc.want, text.String())
assert.Equal(t, tc.garbage, root.GarbageLen())
}
Expand All @@ -138,21 +138,21 @@ func TestRoot(t *testing.T) {
fromPos, toPos, _ := text.CreateRange(0, 0)
_, _, err := text.Edit(fromPos, toPos, nil, "Hello World", nil, ctx.IssueTimeTicket())
assert.NoError(t, err)
registerTextElementWithGarbage(fromPos, toPos, root, text)
registerElementHasRemovedNodes(fromPos, toPos, root, text)
assert.Equal(t, `[{"val":"Hello World"}]`, text.Marshal())
assert.Equal(t, 0, root.GarbageLen())

fromPos, toPos, _ = text.CreateRange(6, 11)
_, _, err = text.Edit(fromPos, toPos, nil, "Yorkie", nil, ctx.IssueTimeTicket())
assert.NoError(t, err)
registerTextElementWithGarbage(fromPos, toPos, root, text)
registerElementHasRemovedNodes(fromPos, toPos, root, text)
assert.Equal(t, `[{"val":"Hello "},{"val":"Yorkie"}]`, text.Marshal())
assert.Equal(t, 1, root.GarbageLen())

fromPos, toPos, _ = text.CreateRange(0, 6)
_, _, err = text.Edit(fromPos, toPos, nil, "", nil, ctx.IssueTimeTicket())
assert.NoError(t, err)
registerTextElementWithGarbage(fromPos, toPos, root, text)
registerElementHasRemovedNodes(fromPos, toPos, root, text)
assert.Equal(t, `[{"val":"Yorkie"}]`, text.Marshal())
assert.Equal(t, 2, root.GarbageLen())

Expand Down
6 changes: 3 additions & 3 deletions pkg/document/crdt/text.go
Original file line number Diff line number Diff line change
Expand Up @@ -309,7 +309,7 @@ func (t *Text) removedNodesLen() int {
return t.rgaTreeSplit.removedNodesLen()
}

// purgeTextNodesWithGarbage physically purges nodes that have been removed.
func (t *Text) purgeTextNodesWithGarbage(ticket *time.Ticket) int {
return t.rgaTreeSplit.purgeTextNodesWithGarbage(ticket)
// purgeRemovedNodesBefore physically purges nodes that have been removed.
func (t *Text) purgeRemovedNodesBefore(ticket *time.Ticket) int {
return t.rgaTreeSplit.purgeRemovedNodesBefore(ticket)
}
81 changes: 73 additions & 8 deletions pkg/document/crdt/tree.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package crdt
import (
"errors"
"fmt"
"strconv"
"strings"
"unicode/utf16"

Expand Down Expand Up @@ -248,9 +249,10 @@ func (n *TreeNode) DeepCopy() *TreeNode {
// Tree represents the tree of CRDT. It has doubly linked list structure and
// index tree structure.
type Tree struct {
DummyHead *TreeNode
IndexTree *index.Tree[*TreeNode]
NodeMapByPos *llrb.Tree[*TreePos, *TreeNode]
DummyHead *TreeNode
IndexTree *index.Tree[*TreeNode]
NodeMapByPos *llrb.Tree[*TreePos, *TreeNode]
removedNodeMap map[string]*TreeNode

createdAt *time.Ticket
movedAt *time.Ticket
Expand All @@ -260,10 +262,11 @@ type Tree struct {
// NewTree creates a new instance of Tree.
func NewTree(root *TreeNode, createdAt *time.Ticket) *Tree {
tree := &Tree{
DummyHead: NewTreeNode(DummyTreePos, DummyHeadType, nil),
IndexTree: index.NewTree[*TreeNode](root.IndexTreeNode),
NodeMapByPos: llrb.NewTree[*TreePos, *TreeNode](),
createdAt: createdAt,
DummyHead: NewTreeNode(DummyTreePos, DummyHeadType, nil),
IndexTree: index.NewTree[*TreeNode](root.IndexTreeNode),
NodeMapByPos: llrb.NewTree[*TreePos, *TreeNode](),
removedNodeMap: make(map[string]*TreeNode),
createdAt: createdAt,
}

previous := tree.DummyHead
Expand All @@ -282,6 +285,64 @@ func (t *Tree) Marshal() string {
return builder.String()
}

// removedNodesLen returns the length of removed nodes.
func (t *Tree) removedNodesLen() int {
return len(t.removedNodeMap)
}

// purgeRemovedNodesBefore physically purges nodes that have been removed.
func (t *Tree) purgeRemovedNodesBefore(ticket *time.Ticket) int {
count := 0
nodesToBeRemoved := make(map[*TreeNode]bool)

for _, node := range t.removedNodeMap {
if node.RemovedAt != nil && ticket.Compare(node.RemovedAt) >= 0 {
count++
nodesToBeRemoved[node] = true
}
}

index.TraverseAll(t.IndexTree, func(node *index.Node[*TreeNode], depth int) {
_, ok := nodesToBeRemoved[node.Value]

if ok {
parent := node.Parent

if parent == nil {
count--
delete(nodesToBeRemoved, node.Value)

return
}

parent.RemoveChild(node)
}
})

for node := range nodesToBeRemoved {
t.NodeMapByPos.Remove(node.Pos)
t.Purge(node)
delete(t.removedNodeMap, node.Pos.CreatedAt.StructureAsString()+":"+strconv.Itoa(node.Pos.Offset))
}

return count
}

// Purge physically purges the given node.
func (t *Tree) Purge(node *TreeNode) {
if node.Prev != nil {
node.Prev.Next = node.Next
}

if node.Next != nil {
node.Next.Prev = node.Prev
}

node.Prev = nil
node.Next = nil
node.InsPrev = nil
}

// marshal returns the JSON encoding of this Tree.
func marshal(builder *strings.Builder, node *TreeNode) {
if node.IsText() {
Expand Down Expand Up @@ -411,6 +472,10 @@ func (t *Tree) Edit(from, to *TreePos, content *TreeNode, editedAt *time.Ticket)
isRangeOnSameBranch := toPos.Node.IsAncestorOf(fromPos.Node)
for _, node := range toBeRemoveds {
node.remove(editedAt)

if node.IsRemoved() {
t.removedNodeMap[node.Pos.CreatedAt.StructureAsString()+":"+strconv.Itoa(node.Pos.Offset)] = node
}
}

// move the alive children of the removed block node
Expand All @@ -433,7 +498,7 @@ func (t *Tree) Edit(from, to *TreePos, content *TreeNode, editedAt *time.Ticket)
}
}
} else {
if fromPos.Node.Parent.Value.IsRemoved() {
if fromPos.Node.Parent != nil && fromPos.Node.Parent.Value.IsRemoved() {
toPos.Node.Parent.Prepend(fromPos.Node.Parent.Children()...)
}
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/document/json/text.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ func (p *Text) Edit(from, to int, content string, attributes ...map[string]strin
ticket,
))
if !fromPos.Equal(toPos) {
p.context.RegisterTextElementWithGarbage(p)
p.context.RegisterElementHasRemovedNodes(p)
}

return p
Expand Down
8 changes: 8 additions & 0 deletions pkg/document/json/tree.go
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,10 @@ func (t *Tree) Edit(fromIdx, toIdx int, content *TreeNode) bool {
ticket,
))

if fromPos.CreatedAt.Compare(toPos.CreatedAt) != 0 || fromPos.Offset != toPos.Offset {
t.context.RegisterElementHasRemovedNodes(t.Tree)
}

return true
}

Expand Down Expand Up @@ -147,6 +151,10 @@ func (t *Tree) EditByPath(fromPath []int, toPath []int, content *TreeNode) bool
ticket,
))

if fromPos.CreatedAt.Compare(toPos.CreatedAt) != 0 || fromPos.Offset != toPos.Offset {
t.context.RegisterElementHasRemovedNodes(t.Tree)
}

return true
}

Expand Down
2 changes: 1 addition & 1 deletion pkg/document/operations/edit.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ func (e *Edit) Execute(root *crdt.Root) error {
return err
}
if !e.from.Equal(e.to) {
root.RegisterTextElementWithGarbage(obj)
root.RegisterElementHasRemovedNodes(obj)
}
default:
return ErrNotApplicableDataType
Expand Down
4 changes: 4 additions & 0 deletions pkg/document/operations/tree_edit.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,10 @@ func (e *TreeEdit) Execute(root *crdt.Root) error {
content = e.Content().DeepCopy()
}
obj.Edit(e.from, e.to, content, e.executedAt)

if e.from.CreatedAt.Compare(e.to.CreatedAt) != 0 || e.from.Offset != e.to.Offset {
root.RegisterElementHasRemovedNodes(obj)
}
default:
return ErrNotApplicableDataType
}
Expand Down
Loading

0 comments on commit db99457

Please sign in to comment.