Skip to content

Commit 39162a4

Browse files
corgy-wcorgy-w
authoredAug 28, 2024
[Improve][Connector-V2] Fake supports column configuration (apache#7503)
* [Improve][Connector-V2] Fake supports column configuration * [Improve][Connector-V2] optimized code
1 parent 696f294 commit 39162a4

File tree

6 files changed

+253
-69
lines changed

6 files changed

+253
-69
lines changed
 

‎seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/schema/ReadonlyConfigParser.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,8 @@ public List<Column> parse(ReadonlyConfig schemaConfig) {
9595
String value = entry.getValue();
9696
SeaTunnelDataType<?> dataType =
9797
SeaTunnelDataTypeConvertorUtil.deserializeSeaTunnelDataType(key, value);
98-
PhysicalColumn column = PhysicalColumn.of(key, dataType, 0, true, null, null);
98+
PhysicalColumn column =
99+
PhysicalColumn.of(key, dataType, null, null, true, null, null);
99100
columns.add(column);
100101
}
101102
return columns;

‎seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeDataGenerator.java

+40-32
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@
1818
package org.apache.seatunnel.connectors.seatunnel.fake.source;
1919

2020
import org.apache.seatunnel.api.table.catalog.CatalogTable;
21+
import org.apache.seatunnel.api.table.catalog.Column;
2122
import org.apache.seatunnel.api.table.type.ArrayType;
22-
import org.apache.seatunnel.api.table.type.DecimalType;
2323
import org.apache.seatunnel.api.table.type.MapType;
2424
import org.apache.seatunnel.api.table.type.RowKind;
2525
import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
@@ -34,9 +34,11 @@
3434

3535
import java.io.IOException;
3636
import java.lang.reflect.Array;
37+
import java.math.BigDecimal;
3738
import java.util.ArrayList;
3839
import java.util.HashMap;
3940
import java.util.List;
41+
import java.util.function.Function;
4042

4143
public class FakeDataGenerator {
4244
private final CatalogTable catalogTable;
@@ -71,12 +73,11 @@ private SeaTunnelRow convertRow(FakeConfig.RowData rowData) {
7173
}
7274

7375
private SeaTunnelRow randomRow() {
74-
SeaTunnelRowType rowType = catalogTable.getSeaTunnelRowType();
75-
String[] fieldNames = rowType.getFieldNames();
76-
SeaTunnelDataType<?>[] fieldTypes = rowType.getFieldTypes();
77-
List<Object> randomRow = new ArrayList<>(fieldNames.length);
78-
for (SeaTunnelDataType<?> fieldType : fieldTypes) {
79-
randomRow.add(randomColumnValue(fieldType));
76+
// Generate random data according to the data type and data colum of the table
77+
List<Column> physicalColumns = catalogTable.getTableSchema().getColumns();
78+
List<Object> randomRow = new ArrayList<>(physicalColumns.size());
79+
for (Column column : physicalColumns) {
80+
randomRow.add(randomColumnValue(column));
8081
}
8182
SeaTunnelRow seaTunnelRow = new SeaTunnelRow(randomRow.toArray());
8283
seaTunnelRow.setTableId(tableId);
@@ -103,15 +104,16 @@ public List<SeaTunnelRow> generateFakedRows(int rowNum) {
103104
}
104105

105106
@SuppressWarnings("magicnumber")
106-
private Object randomColumnValue(SeaTunnelDataType<?> fieldType) {
107+
private Object randomColumnValue(Column column) {
108+
SeaTunnelDataType<?> fieldType = column.getDataType();
107109
switch (fieldType.getSqlType()) {
108110
case ARRAY:
109111
ArrayType<?, ?> arrayType = (ArrayType<?, ?>) fieldType;
110112
SeaTunnelDataType<?> elementType = arrayType.getElementType();
111113
int length = fakeConfig.getArraySize();
112114
Object array = Array.newInstance(elementType.getTypeClass(), length);
113115
for (int i = 0; i < length; i++) {
114-
Object value = randomColumnValue(elementType);
116+
Object value = randomColumnValue(column.copy(elementType));
115117
Array.set(array, i, value);
116118
}
117119
return array;
@@ -122,64 +124,70 @@ private Object randomColumnValue(SeaTunnelDataType<?> fieldType) {
122124
HashMap<Object, Object> objectMap = new HashMap<>();
123125
int mapSize = fakeConfig.getMapSize();
124126
for (int i = 0; i < mapSize; i++) {
125-
Object key = randomColumnValue(keyType);
126-
Object value = randomColumnValue(valueType);
127+
Object key = randomColumnValue(column.copy(keyType));
128+
Object value = randomColumnValue(column.copy(valueType));
127129
objectMap.put(key, value);
128130
}
129131
return objectMap;
130132
case STRING:
131-
return fakeDataRandomUtils.randomString();
133+
return value(column, String::toString, fakeDataRandomUtils::randomString);
132134
case BOOLEAN:
133-
return fakeDataRandomUtils.randomBoolean();
135+
return value(column, Boolean::parseBoolean, fakeDataRandomUtils::randomBoolean);
134136
case TINYINT:
135-
return fakeDataRandomUtils.randomTinyint();
137+
return value(column, Byte::parseByte, fakeDataRandomUtils::randomTinyint);
136138
case SMALLINT:
137-
return fakeDataRandomUtils.randomSmallint();
139+
return value(column, Short::parseShort, fakeDataRandomUtils::randomSmallint);
138140
case INT:
139-
return fakeDataRandomUtils.randomInt();
141+
return value(column, Integer::parseInt, fakeDataRandomUtils::randomInt);
140142
case BIGINT:
141-
return fakeDataRandomUtils.randomBigint();
143+
return value(column, Long::parseLong, fakeDataRandomUtils::randomBigint);
142144
case FLOAT:
143-
return fakeDataRandomUtils.randomFloat();
145+
return value(column, Float::parseFloat, fakeDataRandomUtils::randomFloat);
144146
case DOUBLE:
145-
return fakeDataRandomUtils.randomDouble();
147+
return value(column, Double::parseDouble, fakeDataRandomUtils::randomDouble);
146148
case DECIMAL:
147-
DecimalType decimalType = (DecimalType) fieldType;
148-
return fakeDataRandomUtils.randomBigDecimal(
149-
decimalType.getPrecision(), decimalType.getScale());
149+
return value(column, BigDecimal::new, fakeDataRandomUtils::randomBigDecimal);
150150
case NULL:
151151
return null;
152152
case BYTES:
153-
return fakeDataRandomUtils.randomBytes();
153+
return value(column, String::getBytes, fakeDataRandomUtils::randomBytes);
154154
case DATE:
155-
return fakeDataRandomUtils.randomLocalDate();
155+
return value(column, String::toString, fakeDataRandomUtils::randomLocalDate);
156156
case TIME:
157-
return fakeDataRandomUtils.randomLocalTime();
157+
return value(column, String::toString, fakeDataRandomUtils::randomLocalTime);
158158
case TIMESTAMP:
159-
return fakeDataRandomUtils.randomLocalDateTime();
159+
return value(column, String::toString, fakeDataRandomUtils::randomLocalDateTime);
160160
case ROW:
161161
SeaTunnelDataType<?>[] fieldTypes = ((SeaTunnelRowType) fieldType).getFieldTypes();
162162
Object[] objects = new Object[fieldTypes.length];
163163
for (int i = 0; i < fieldTypes.length; i++) {
164-
Object object = randomColumnValue(fieldTypes[i]);
164+
Object object = randomColumnValue(column.copy(fieldTypes[i]));
165165
objects[i] = object;
166166
}
167167
return new SeaTunnelRow(objects);
168168
case BINARY_VECTOR:
169-
return fakeDataRandomUtils.randomBinaryVector();
169+
return fakeDataRandomUtils.randomBinaryVector(column);
170170
case FLOAT_VECTOR:
171-
return fakeDataRandomUtils.randomFloatVector();
171+
return fakeDataRandomUtils.randomFloatVector(column);
172172
case FLOAT16_VECTOR:
173-
return fakeDataRandomUtils.randomFloat16Vector();
173+
return fakeDataRandomUtils.randomFloat16Vector(column);
174174
case BFLOAT16_VECTOR:
175-
return fakeDataRandomUtils.randomBFloat16Vector();
175+
return fakeDataRandomUtils.randomBFloat16Vector(column);
176176
case SPARSE_FLOAT_VECTOR:
177-
return fakeDataRandomUtils.randomSparseFloatVector();
177+
return fakeDataRandomUtils.randomSparseFloatVector(column);
178178
default:
179179
// never got in there
180180
throw new FakeConnectorException(
181181
CommonErrorCodeDeprecated.UNSUPPORTED_DATA_TYPE,
182182
"SeaTunnel Fake source connector not support this data type");
183183
}
184184
}
185+
186+
private static <T> T value(
187+
Column column, Function<String, T> convert, Function<Column, T> generate) {
188+
if (column.getDefaultValue() != null) {
189+
return convert.apply(column.getDefaultValue().toString());
190+
}
191+
return generate.apply(column);
192+
}
185193
}

‎seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/utils/FakeDataRandomUtils.java

+58-35
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717

1818
package org.apache.seatunnel.connectors.seatunnel.fake.utils;
1919

20+
import org.apache.seatunnel.api.table.catalog.Column;
21+
import org.apache.seatunnel.api.table.type.DecimalType;
2022
import org.apache.seatunnel.common.utils.BufferUtils;
2123
import org.apache.seatunnel.connectors.seatunnel.fake.config.FakeConfig;
2224

@@ -25,6 +27,7 @@
2527
import org.apache.commons.lang3.RandomUtils;
2628

2729
import java.math.BigDecimal;
30+
import java.math.RoundingMode;
2831
import java.nio.ByteBuffer;
2932
import java.time.LocalDate;
3033
import java.time.LocalDateTime;
@@ -45,38 +48,42 @@ private static <T> T randomFromList(List<T> list) {
4548
return list.get(index);
4649
}
4750

48-
public Boolean randomBoolean() {
51+
public Boolean randomBoolean(Column column) {
4952
return RandomUtils.nextInt(0, 2) == 1;
5053
}
5154

52-
public BigDecimal randomBigDecimal(int precision, int scale) {
55+
public BigDecimal randomBigDecimal(Column column) {
56+
DecimalType dataType = (DecimalType) column.getDataType();
5357
return new BigDecimal(
54-
RandomStringUtils.randomNumeric(precision - scale)
58+
RandomStringUtils.randomNumeric(dataType.getPrecision() - dataType.getScale())
5559
+ "."
56-
+ RandomStringUtils.randomNumeric(scale));
60+
+ RandomStringUtils.randomNumeric(dataType.getScale()));
5761
}
5862

59-
public byte[] randomBytes() {
63+
public byte[] randomBytes(Column column) {
6064
return RandomStringUtils.randomAlphabetic(fakeConfig.getBytesLength()).getBytes();
6165
}
6266

63-
public String randomString() {
67+
public String randomString(Column column) {
6468
List<String> stringTemplate = fakeConfig.getStringTemplate();
6569
if (!CollectionUtils.isEmpty(stringTemplate)) {
6670
return randomFromList(stringTemplate);
6771
}
68-
return RandomStringUtils.randomAlphabetic(fakeConfig.getStringLength());
72+
return RandomStringUtils.randomAlphabetic(
73+
column.getColumnLength() != null
74+
? column.getColumnLength().intValue()
75+
: fakeConfig.getStringLength());
6976
}
7077

71-
public Byte randomTinyint() {
78+
public Byte randomTinyint(Column column) {
7279
List<Integer> tinyintTemplate = fakeConfig.getTinyintTemplate();
7380
if (!CollectionUtils.isEmpty(tinyintTemplate)) {
7481
return randomFromList(tinyintTemplate).byteValue();
7582
}
7683
return (byte) RandomUtils.nextInt(fakeConfig.getTinyintMin(), fakeConfig.getTinyintMax());
7784
}
7885

79-
public Short randomSmallint() {
86+
public Short randomSmallint(Column column) {
8087
List<Integer> smallintTemplate = fakeConfig.getSmallintTemplate();
8188
if (!CollectionUtils.isEmpty(smallintTemplate)) {
8289
return randomFromList(smallintTemplate).shortValue();
@@ -85,48 +92,55 @@ public Short randomSmallint() {
8592
RandomUtils.nextInt(fakeConfig.getSmallintMin(), fakeConfig.getSmallintMax());
8693
}
8794

88-
public Integer randomInt() {
95+
public Integer randomInt(Column column) {
8996
List<Integer> intTemplate = fakeConfig.getIntTemplate();
9097
if (!CollectionUtils.isEmpty(intTemplate)) {
9198
return randomFromList(intTemplate);
9299
}
93100
return RandomUtils.nextInt(fakeConfig.getIntMin(), fakeConfig.getIntMax());
94101
}
95102

96-
public Long randomBigint() {
103+
public Long randomBigint(Column column) {
97104
List<Long> bigTemplate = fakeConfig.getBigTemplate();
98105
if (!CollectionUtils.isEmpty(bigTemplate)) {
99106
return randomFromList(bigTemplate);
100107
}
101108
return RandomUtils.nextLong(fakeConfig.getBigintMin(), fakeConfig.getBigintMax());
102109
}
103110

104-
public Float randomFloat() {
111+
public Float randomFloat(Column column) {
105112
List<Double> floatTemplate = fakeConfig.getFloatTemplate();
106113
if (!CollectionUtils.isEmpty(floatTemplate)) {
107114
return randomFromList(floatTemplate).floatValue();
108115
}
109-
return RandomUtils.nextFloat(
110-
(float) fakeConfig.getFloatMin(), (float) fakeConfig.getFloatMax());
116+
float v =
117+
RandomUtils.nextFloat(
118+
(float) fakeConfig.getFloatMin(), (float) fakeConfig.getFloatMax());
119+
return column.getScale() == null
120+
? v
121+
: new BigDecimal(v).setScale(column.getScale(), RoundingMode.HALF_UP).floatValue();
111122
}
112123

113-
public Double randomDouble() {
124+
public Double randomDouble(Column column) {
114125
List<Double> doubleTemplate = fakeConfig.getDoubleTemplate();
115126
if (!CollectionUtils.isEmpty(doubleTemplate)) {
116127
return randomFromList(doubleTemplate);
117128
}
118-
return RandomUtils.nextDouble(fakeConfig.getDoubleMin(), fakeConfig.getDoubleMax());
129+
double v = RandomUtils.nextDouble(fakeConfig.getDoubleMin(), fakeConfig.getDoubleMax());
130+
return column.getScale() == null
131+
? v
132+
: new BigDecimal(v).setScale(column.getScale(), RoundingMode.HALF_UP).floatValue();
119133
}
120134

121-
public LocalDate randomLocalDate() {
122-
return randomLocalDateTime().toLocalDate();
135+
public LocalDate randomLocalDate(Column column) {
136+
return randomLocalDateTime(column).toLocalDate();
123137
}
124138

125-
public LocalTime randomLocalTime() {
126-
return randomLocalDateTime().toLocalTime();
139+
public LocalTime randomLocalTime(Column column) {
140+
return randomLocalDateTime(column).toLocalTime();
127141
}
128142

129-
public LocalDateTime randomLocalDateTime() {
143+
public LocalDateTime randomLocalDateTime(Column column) {
130144
int year;
131145
int month;
132146
int day;
@@ -172,25 +186,32 @@ public LocalDateTime randomLocalDateTime() {
172186
return LocalDateTime.of(year, month, day, hour, minute, second);
173187
}
174188

175-
public ByteBuffer randomBinaryVector() {
176-
int byteCount = fakeConfig.getBinaryVectorDimension() / 8;
189+
public ByteBuffer randomBinaryVector(Column column) {
190+
int byteCount =
191+
(column.getScale() != null)
192+
? column.getScale() / 8
193+
: fakeConfig.getBinaryVectorDimension() / 8;
177194
// binary vector doesn't care endian since each byte is independent
178195
return ByteBuffer.wrap(RandomUtils.nextBytes(byteCount));
179196
}
180197

181-
public ByteBuffer randomFloatVector() {
182-
Float[] floatVector = new Float[fakeConfig.getVectorDimension()];
183-
for (int i = 0; i < fakeConfig.getVectorDimension(); i++) {
198+
public ByteBuffer randomFloatVector(Column column) {
199+
int count =
200+
(column.getScale() != null) ? column.getScale() : fakeConfig.getVectorDimension();
201+
Float[] floatVector = new Float[count];
202+
for (int i = 0; i < count; i++) {
184203
floatVector[i] =
185204
RandomUtils.nextFloat(
186205
fakeConfig.getVectorFloatMin(), fakeConfig.getVectorFloatMax());
187206
}
188207
return BufferUtils.toByteBuffer(floatVector);
189208
}
190209

191-
public ByteBuffer randomFloat16Vector() {
192-
Short[] float16Vector = new Short[fakeConfig.getVectorDimension()];
193-
for (int i = 0; i < fakeConfig.getVectorDimension(); i++) {
210+
public ByteBuffer randomFloat16Vector(Column column) {
211+
int count =
212+
(column.getScale() != null) ? column.getScale() : fakeConfig.getVectorDimension();
213+
Short[] float16Vector = new Short[count];
214+
for (int i = 0; i < count; i++) {
194215
float value =
195216
RandomUtils.nextFloat(
196217
fakeConfig.getVectorFloatMin(), fakeConfig.getVectorFloatMax());
@@ -199,9 +220,11 @@ public ByteBuffer randomFloat16Vector() {
199220
return BufferUtils.toByteBuffer(float16Vector);
200221
}
201222

202-
public ByteBuffer randomBFloat16Vector() {
203-
Short[] bfloat16Vector = new Short[fakeConfig.getVectorDimension()];
204-
for (int i = 0; i < fakeConfig.getVectorDimension(); i++) {
223+
public ByteBuffer randomBFloat16Vector(Column column) {
224+
int count =
225+
(column.getScale() != null) ? column.getScale() : fakeConfig.getVectorDimension();
226+
Short[] bfloat16Vector = new Short[count];
227+
for (int i = 0; i < count; i++) {
205228
float value =
206229
RandomUtils.nextFloat(
207230
fakeConfig.getVectorFloatMin(), fakeConfig.getVectorFloatMax());
@@ -210,10 +233,10 @@ public ByteBuffer randomBFloat16Vector() {
210233
return BufferUtils.toByteBuffer(bfloat16Vector);
211234
}
212235

213-
public Map<Integer, Float> randomSparseFloatVector() {
236+
public Map<Integer, Float> randomSparseFloatVector(Column column) {
214237
Map<Integer, Float> sparseVector = new HashMap<>();
215-
216-
Integer nonZeroElements = fakeConfig.getVectorDimension();
238+
int nonZeroElements =
239+
(column.getScale() != null) ? column.getScale() : fakeConfig.getVectorDimension();
217240
while (nonZeroElements > 0) {
218241
Integer index = RandomUtils.nextInt();
219242
Float value =

‎seatunnel-connectors-v2/connector-fake/src/test/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeDataGeneratorTest.java

+55
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.apache.seatunnel.api.configuration.ReadonlyConfig;
2424
import org.apache.seatunnel.api.table.catalog.CatalogTableUtil;
2525
import org.apache.seatunnel.api.table.catalog.TablePath;
26+
import org.apache.seatunnel.api.table.type.BasicType;
2627
import org.apache.seatunnel.api.table.type.RowKind;
2728
import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
2829
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
@@ -38,6 +39,7 @@
3839
import java.io.FileNotFoundException;
3940
import java.net.URISyntaxException;
4041
import java.net.URL;
42+
import java.nio.ByteBuffer;
4143
import java.nio.file.Paths;
4244
import java.util.Arrays;
4345
import java.util.List;
@@ -141,6 +143,59 @@ public void testVectorParse(String conf) throws FileNotFoundException, URISyntax
141143
Assertions.assertNotNull(seaTunnelRows);
142144
}
143145

146+
@ParameterizedTest
147+
@ValueSource(strings = {"fake-data.column.conf"})
148+
public void testColumnDataParse(String conf) throws FileNotFoundException, URISyntaxException {
149+
ReadonlyConfig testConfig = getTestConfigFile(conf);
150+
FakeConfig fakeConfig = FakeConfig.buildWithConfig(testConfig);
151+
FakeDataGenerator fakeDataGenerator = new FakeDataGenerator(fakeConfig);
152+
List<SeaTunnelRow> seaTunnelRows =
153+
fakeDataGenerator.generateFakedRows(fakeConfig.getRowNum());
154+
seaTunnelRows.forEach(
155+
seaTunnelRow -> {
156+
Assertions.assertEquals(
157+
seaTunnelRow.getField(0).toString(), "Andersen's Fairy Tales");
158+
Assertions.assertEquals(seaTunnelRow.getField(1).toString().length(), 100);
159+
Assertions.assertEquals(seaTunnelRow.getField(2).toString(), "10.1");
160+
Assertions.assertNotNull(seaTunnelRow.getField(3).toString());
161+
Assertions.assertNotNull(seaTunnelRow.getField(4).toString());
162+
// VectorType.VECTOR_FLOAT_TYPE
163+
Assertions.assertEquals(
164+
8, ((ByteBuffer) seaTunnelRow.getField(5)).capacity() / 4);
165+
// VectorType.VECTOR_BINARY_TYPE
166+
Assertions.assertEquals(
167+
16, ((ByteBuffer) seaTunnelRow.getField(6)).capacity() * 8);
168+
// VectorType.VECTOR_FLOAT16_TYPE
169+
Assertions.assertEquals(
170+
8, ((ByteBuffer) seaTunnelRow.getField(7)).capacity() / 2);
171+
// VectorType.VECTOR_BFLOAT16_TYPE
172+
Assertions.assertEquals(
173+
8, ((ByteBuffer) seaTunnelRow.getField(8)).capacity() / 2);
174+
// VectorType.VECTOR_SPARSE_FLOAT_TYPE
175+
Assertions.assertEquals(8, ((Map) seaTunnelRow.getField(9)).size());
176+
Assertions.assertEquals(
177+
268,
178+
seaTunnelRow.getBytesSize(
179+
new SeaTunnelRowType(
180+
new String[] {
181+
"field1", "field2", "field3", "field4", "field5",
182+
"field6", "field7", "field8", "field9", "field10"
183+
},
184+
new SeaTunnelDataType<?>[] {
185+
BasicType.STRING_TYPE,
186+
BasicType.STRING_TYPE,
187+
BasicType.FLOAT_TYPE,
188+
BasicType.FLOAT_TYPE,
189+
BasicType.DOUBLE_TYPE,
190+
VectorType.VECTOR_FLOAT_TYPE,
191+
VectorType.VECTOR_BINARY_TYPE,
192+
VectorType.VECTOR_FLOAT16_TYPE,
193+
VectorType.VECTOR_BFLOAT16_TYPE,
194+
VectorType.VECTOR_SPARSE_FLOAT_TYPE
195+
})));
196+
});
197+
}
198+
144199
private ReadonlyConfig getTestConfigFile(String configFile)
145200
throws FileNotFoundException, URISyntaxException {
146201
if (!configFile.startsWith("/")) {

‎seatunnel-connectors-v2/connector-fake/src/test/resources/complex.schema.conf

+1-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ FakeSource {
2323
string.length = 10
2424
schema = {
2525
fields {
26-
c_map = "map<string, array<int>>"
26+
c_map = "map<string, map<string, string>>"
2727
c_array = "array<int>"
2828
c_string = string
2929
c_boolean = boolean

‎seatunnel-connectors-v2/connector-fake/src/test/resources/fake-data.column.conf

+97
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
18+
FakeSource {
19+
row.num = 5
20+
vector.float.max=1
21+
vector.float.min=0
22+
float.max = 2
23+
float.min = 0
24+
double.max = 4
25+
double.min = 2
26+
27+
# low weight
28+
string.length = 4
29+
vector.dimension= 4
30+
binary.vector.dimension=8
31+
# end
32+
33+
schema = {
34+
columns = [
35+
{
36+
name = book_name
37+
type = string
38+
defaultValue = "Andersen's Fairy Tales"
39+
comment = "book name"
40+
},
41+
{
42+
name = book_reader_testimonials
43+
type = string
44+
columnLength = 100
45+
comment = "book reader testimonials"
46+
},
47+
{
48+
name = book_price
49+
type = float
50+
defaultValue = 10.1
51+
comment = "book price"
52+
},
53+
{
54+
name = book_percentage_popularity
55+
type = float
56+
columnScale = 4
57+
comment = "book percentage popularity"
58+
},
59+
{
60+
name = book_distribution_law
61+
type = double
62+
columnScale = 2
63+
comment = "book distribution law"
64+
},
65+
{
66+
name = book_intro_1
67+
type = float_vector
68+
columnScale =8
69+
comment = "vector"
70+
},
71+
{
72+
name = book_intro_2
73+
type = binary_vector
74+
columnScale = 16
75+
comment = "vector"
76+
},
77+
{
78+
name = book_intro_3
79+
type = float16_vector
80+
columnScale =8
81+
comment = "vector"
82+
},
83+
{
84+
name = book_intro_4
85+
type = bfloat16_vector
86+
columnScale =8
87+
comment = "vector"
88+
},
89+
{
90+
name = book_intro_5
91+
type = sparse_float_vector
92+
columnScale =8
93+
comment = "vector"
94+
}
95+
]
96+
}
97+
}

0 commit comments

Comments
 (0)
Please sign in to comment.