forked from daangn/gorean
-
Notifications
You must be signed in to change notification settings - Fork 0
/
edgengram_test.go
115 lines (105 loc) · 3.15 KB
/
edgengram_test.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
package gorean
import (
"testing"
"github.com/stretchr/testify/assert"
)
var (
dummyQuestionGenerateEdgeNGramTokensKorean = "까투리왕"
dummyAnswerGenerateEdgeNGramTokensKorean = []string{
"ㄱ",
"ㄲ",
"까",
"깥",
"까ㅌ",
"까투",
"까툴",
"까투ㄹ",
"까투리",
"까투링",
"까투리ㅇ",
"까투리오",
"까투리와",
"까투리왕",
}
dummyQuestionGenerateEdgeNGramTokensKoreanWithWhitespace = "마따 말기"
dummyAnswerGenerateEdgeNGramTokensKoreanWithWhitespace = []string{
"ㅁ",
"마",
"마ㄷ",
"맏",
"마ㄸ",
"마따",
"마따 ",
"마따 ㅁ",
"마따 마",
"마따 말",
"마따 맑",
"마따 말ㄱ",
"마따 말기",
}
dummyQuestionGenerateEdgeNGramTokensKoreanWithEnglish = "님부스brush JCM 2000우효"
dummyAnswerGenerateEdgeNGramTokensKoreanWithEnglish = []string{
"ㄴ",
"니",
"님",
"님ㅂ",
"님부",
"님붓",
"님부ㅅ",
"님부스",
"님부스b",
"님부스br",
"님부스bru",
"님부스brus",
"님부스brush",
"님부스brush ",
"님부스brush J",
"님부스brush JC",
"님부스brush JCM",
"님부스brush JCM ",
"님부스brush JCM 2",
"님부스brush JCM 20",
"님부스brush JCM 200",
"님부스brush JCM 2000",
"님부스brush JCM 2000ㅇ",
"님부스brush JCM 2000우",
"님부스brush JCM 2000웋",
"님부스brush JCM 2000우ㅎ",
"님부스brush JCM 2000우효",
}
dummyQuestionGenerateEdgeNGramTokensSameJaumAtJongsungChosungCase1 = "립밤"
dummyAnswerGenerateEdgeNGramTokensSameJaumAtJongsungChosungCase1 = []string{
"ㄹ",
"리",
"립",
"립ㅂ",
"립바",
"립밤",
}
dummyQuestionGenerateEdgeNGramTokensSameJaumAtJongsungChosungCase2 = "낮잠"
dummyAnswerGenerateEdgeNGramTokensSameJaumAtJongsungChosungCase2 = []string{
"ㄴ",
"나",
"낮",
"낮ㅈ",
"낮자",
"낮잠",
}
)
func Test_GenerateForwardNGramTokens(t *testing.T) {
tokensKorean, err := GenerateEdgeNGramTokens(dummyQuestionGenerateEdgeNGramTokensKorean)
assert.Empty(t, err)
assert.Equal(t, dummyAnswerGenerateEdgeNGramTokensKorean, tokensKorean, "")
tokensKoreanWithWhitespace, err := GenerateEdgeNGramTokens(dummyQuestionGenerateEdgeNGramTokensKoreanWithWhitespace)
assert.Empty(t, err)
assert.Equal(t, dummyAnswerGenerateEdgeNGramTokensKoreanWithWhitespace, tokensKoreanWithWhitespace, "equal")
tokensKoreanWithEnglish, err := GenerateEdgeNGramTokens(dummyQuestionGenerateEdgeNGramTokensKoreanWithEnglish)
assert.Empty(t, err)
assert.Equal(t, dummyAnswerGenerateEdgeNGramTokensKoreanWithEnglish, tokensKoreanWithEnglish, "equal")
tokensSameJaumAtJongsungChosungCase1, err := GenerateEdgeNGramTokens(dummyQuestionGenerateEdgeNGramTokensSameJaumAtJongsungChosungCase1)
assert.Empty(t, err)
assert.Equal(t, dummyAnswerGenerateEdgeNGramTokensSameJaumAtJongsungChosungCase1, tokensSameJaumAtJongsungChosungCase1, "equal")
tokensSameJaumAtJongsungChosungCase2, err := GenerateEdgeNGramTokens(dummyQuestionGenerateEdgeNGramTokensSameJaumAtJongsungChosungCase2)
assert.Empty(t, err)
assert.Equal(t, dummyAnswerGenerateEdgeNGramTokensSameJaumAtJongsungChosungCase2, tokensSameJaumAtJongsungChosungCase2, "equal")
}