diff --git a/docs/dev/table/connectors/datagen.md b/docs/dev/table/connectors/datagen.md new file mode 100644 index 0000000000000..f1e336bb1711e --- /dev/null +++ b/docs/dev/table/connectors/datagen.md @@ -0,0 +1,152 @@ +--- +title: "DataGen SQL Connector" +nav-title: DataGen +nav-parent_id: sql-connectors +nav-pos: 4 +--- + + +Scan Source: Bounded +Scan Source: UnBounded + +* This will be replaced by the TOC +{:toc} + +The Datagen connector allows for reading by data generation rules. + +The Datagen connector can work with [Computed Column syntax]({% link dev/table/sql/create.md %}#create-table). +This allows you to generate records flexibly. + +The Datagen connector is built-in. + +Attention Complex types are not supported: Array, Map, Row. Please construct these types by computed column. + +How to create a Datagen table +---------------- + +The boundedness of table: when the generation of field data in the table is completed, the reading +is finished. So the boundedness of the table depends on the boundedness of fields. + +For each field, there are two ways to generate data: + +- Random generator is the default generator, you can specify random max and min values. For char/varchar/string, the length can be specified. It is a unbounded generator. +- Sequence generator, you can specify sequence start and end values. It is a bounded generator, when the sequence number reaches the end value, the reading ends. + +
+
+{% highlight sql %} +CREATE TABLE datagen ( + f_sequence INT, + f_random INT, + f_random_str STRING, + ts AS localtimestamp, + WATERMARK FOR ts AS ts +) WITH ( + 'connector' = 'datagen', + + -- optional options -- + + 'rows-per-second'='5', + + 'fields.f_sequence.kind'='sequence', + 'fields.f_sequence.start'='1', + 'fields.f_sequence.end'='1000', + + 'fields.f_random.min'='1', + 'fields.f_random.max'='1000', + + 'fields.f_random_str.length'='10' +) +{% endhighlight %} +
+
+ +Connector Options +---------------- + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
OptionRequiredDefaultTypeDescription
connector
required(none)StringSpecify what connector to use, here should be 'datagen'.
rows-per-second
optional10000LongRows per second to control the emit rate.
fields.#.kind
optionalrandomStringGenerator of this '#' field. Can be 'sequence' or 'random'.
fields.#.min
optional(Minimum value of type)(Type of field)Minimum value of random generator, work for number types.
fields.#.max
optional(Maximum value of type)(Type of field)Maximum value of random generator, work for number types.
fields.#.length
optional100IntegerLength for string generating of random generator, work for char/varchar/string.
fields.#.start
optional(none)(Type of field)Start value of sequence generator.
fields.#.end
optional(none)(Type of field)End value of sequence generator.
diff --git a/docs/dev/table/connectors/datagen.zh.md b/docs/dev/table/connectors/datagen.zh.md new file mode 100644 index 0000000000000..b11558d9002df --- /dev/null +++ b/docs/dev/table/connectors/datagen.zh.md @@ -0,0 +1,152 @@ +--- +title: "DataGen SQL Connector" +nav-title: DataGen +nav-parent_id: sql-connectors +nav-pos: 4 +--- + + +Scan Source: Bounded +Scan Source: UnBounded + +* This will be replaced by the TOC +{:toc} + +The Datagen connector allows for reading by data generation rules. + +The Datagen connector can work with [Computed Column syntax]({% link dev/table/sql/create.zh.md %}#create-table). +This allows you to generate records flexibly. + +The Datagen connector is built-in. + +Attention Complex types are not supported: Array, Map, Row. Please construct these types by computed column. + +How to create a Datagen table +---------------- + +The boundedness of table: when the generation of field data in the table is completed, the reading +is finished. So the boundedness of the table depends on the boundedness of fields. + +For each field, there are two ways to generate data: + +- Random generator is the default generator, you can specify random max and min values. For char/varchar/string, the length can be specified. It is a unbounded generator. +- Sequence generator, you can specify sequence start and end values. It is a bounded generator, when the sequence number reaches the end value, the reading ends. + +
+
+{% highlight sql %} +CREATE TABLE datagen ( + f_sequence INT, + f_random INT, + f_random_str STRING, + ts AS localtimestamp, + WATERMARK FOR ts AS ts +) WITH ( + 'connector' = 'datagen', + + -- optional options -- + + 'rows-per-second'='5', + + 'fields.f_sequence.kind'='sequence', + 'fields.f_sequence.start'='1', + 'fields.f_sequence.end'='1000', + + 'fields.f_random.min'='1', + 'fields.f_random.max'='1000', + + 'fields.f_random_str.length'='10' +) +{% endhighlight %} +
+
+ +Connector Options +---------------- + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
OptionRequiredDefaultTypeDescription
connector
required(none)StringSpecify what connector to use, here should be 'datagen'.
rows-per-second
optional10000LongRows per second to control the emit rate.
fields.#.kind
optionalrandomStringGenerator of this '#' field. Can be 'sequence' or 'random'.
fields.#.min
optional(Minimum value of type)(Type of field)Minimum value of random generator, work for number types.
fields.#.max
optional(Maximum value of type)(Type of field)Maximum value of random generator, work for number types.
fields.#.length
optional100IntegerLength for string generating of random generator, work for char/varchar/string.
fields.#.start
optional(none)(Type of field)Start value of sequence generator.
fields.#.end
optional(none)(Type of field)End value of sequence generator.