Add InfluxDB reader plugin (wgzhao#105)

Add InfluxDB reader plugin wgzhao#96
tobe369 · Jan 16, 2021 · 5a50187 · 5a50187
1 parent 698ff76
commit 5a50187
Show file tree

Hide file tree

Showing 16 changed files with 560 additions and 3 deletions.
diff --git a/README.md b/README.md
@@ -39,6 +39,7 @@
 3. hbase20xreader
 4. jsonfilereader
 5. kudureader
+6. influxdbreader
 
 #### writer plugin
 
@@ -81,6 +82,7 @@
 | HBase 2.x   | 支持 | 支持 | hbase20xsqlreader/hbase20xsqlwriter| 通过[Phoenix](https://phoenix.apache.org)操作HBase |
 | HDFS        | 支持 | 支持 | hdfsreader/hdfswriter | HDFS 2.x 以上版本 |
 | Greenplum   | 支持 | 支持 | postgresqlreader/greenplumwriter | |
+| InfluxDB    | 支持 | 不支持 | influxdbreader | 仅支持1.x版本，2.0及以上暂不支持 | 
 | json        | 支持 | 不支持 | jsonfilereader | |
 | kudu        | 支持 | 支持 | kudureader/kuduwriter | 通过原生接口，计划更新Impala连接 | 
 | MongoDB     | 支持 | 支持 | mongodbreader/mongodbwriter | |

diff --git a/core/src/main/job/influxdb2pg.json b/core/src/main/job/influxdb2pg.json
@@ -0,0 +1,49 @@
+{
+    "job": {
+        "content": [
+            {
+                "reader": {
+                    "name": "influxdbreader",
+                    "parameter": {
+                        "column": [
+                            "*"
+                        ],
+                        "connection": [
+                            {
+                                "endpoint": "http://localhost:8086",
+                                "database": "NOAA_water_database",
+                                "table": "h2o_feet"
+                            }
+                        ],
+                        "username": "influx",
+                        "password": "influx123"
+                    }
+                },
+                "writer": {
+                    "name": "postgresqlwriter",
+                    "parameter": {
+                        "username": "wgzhao",
+                        "password": "wgzhao",
+                        "column": [
+                            "*"
+                        ],
+                        "connection": [
+                            {
+                                "table": [
+                                    "influx_tbl"
+                                ],
+                                "jdbcUrl": "jdbc:postgresql://localhost:5432/wgzhao"
+                            }
+                        ]
+                    }
+                }
+            }
+        ],
+        "setting": {
+            "speed": {
+                "bytes": -1,
+                "channel": 1
+            }
+        }
+    }
+}
diff --git a/core/src/main/job/influxdb2stream.json b/core/src/main/job/influxdb2stream.json
@@ -0,0 +1,37 @@
+{
+  "job": {
+    "content": [
+      {
+        "reader": {
+          "name": "influxdbreader",
+          "parameter": {
+            "column": [
+              "*"
+            ],
+            "connection": [
+              {
+                "endpoint": "http://localhost:8086",
+                "database": "NOAA_water_database",
+                "table": "h2o_feet"
+              }
+            ],
+            "username": "influx",
+            "password": "influx123"
+          }
+        },
+        "writer": {
+          "name": "streamwriter",
+          "parameter": {
+            "print": "true"
+          }
+        }
+      }
+    ],
+    "setting": {
+      "speed": {
+        "bytes": -1,
+        "channel": 1
+      }
+    }
+  }
+}
diff --git a/docs/src/main/sphinx/reader/influxdbreader.md b/docs/src/main/sphinx/reader/influxdbreader.md
@@ -0,0 +1,95 @@
+# InfluxDBReader
+
+InfluxDBReader 插件实现了从 [InfluxDB](https://www.influxdata.com) 读取数据。底层实现上，是通过调用 InfluQL 语言查询表，然后获得返回数据。
+
+## 示例
+
+以下示例用来演示该插件如何从指定表(即指标)上读取数据并输出到终端
+
+### 创建需要的库表和数据
+
+通过以下命令来创建需要读取的表以及数据
+
+```bash
+# create database
+influx --execute "CREATE DATABASE NOAA_water_database"
+# download sample data
+curl https://s3.amazonaws.com/noaa.water-database/NOAA_data.txt -o NOAA_data.txt
+# import data via influx-cli
+influx -import -path=NOAA_data.txt -precision=s -database=NOAA_water_database
+```
+
+### 创建 job 文件
+
+创建 `job/influxdb2stream.json` 文件，内容如下：
+
+```json
+{
+  "job": {
+    "content": [
+      {
+        "reader": {
+          "name": "influxdbreader",
+          "parameter": {
+            "column": [
+              "*"
+            ],
+            "connection": [
+              {
+                "endpoint": "http://localhost:8086",
+                "database": "NOAA_water_database",
+                "table": "h2o_feet",
+                "where": "1=1"
+              }
+            ],
+            "username": "influx",
+            "password": "influx123"
+          }
+        },
+        "writer": {
+          "name": "streamwriter",
+          "parameter": {
+            "print": "true"
+          }
+        }
+      }
+    ],
+    "setting": {
+      "speed": {
+        "bytes": -1,
+        "channel": 1
+      }
+    }
+  }
+}
+```
+
+### 运行
+
+执行下面的命令进行数据采集
+
+```bash
+bin/datax.py job/influxdb2stream.json
+```
+
+##  参数说明
+
+| 配置项          | 是否必须 |  数据类型   |默认值 |         描述   |
+| :-------------- | :------: | ------ |-------|-------------- |
+| endpoint         |    是   | string | | 无     | InfluxDB 连接串 ｜
+| username        |    是    | string | 无     | 数据源的用户名 |
+| password        |    否    | string | 无     | 数据源指定用户名的密码 |
+| database        |  是      | string |  无      | 数据源指定的数据库  |
+| table           |    是    | string |无     | 所选取的需要同步的表名,使用JSON数据格式，当配置为多张表时，用户自己需保证多张表是同一表结构 |
+| column          |    是    | list  | 无     |  所配置的表中需要同步的列名集合，详细描述[rdbmreader](rdbmsreader.md) ｜
+| where           |    否    | 无     | 针对表的筛选条件 |
+| querySql        |    否    | 无     | 使用自定义的SQL而不是指定表来获取数据，当配置了这一项之后，DataX系统就会忽略 `table`，`column`这些配置项 |
+
+##  类型转换
+
+当前实现是将所有字段当作字符串处理
+
+
+## 限制
+
+1. 当前插件仅支持 1.x 版本，2.0 及以上并不支持
diff --git a/influxdbreader/package.xml b/influxdbreader/package.xml
@@ -0,0 +1,35 @@
+<assembly
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+        xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-component-1.1.2.xsd">
+    <id>release</id>
+    <formats>
+        <format>dir</format>
+    </formats>
+    <includeBaseDirectory>false</includeBaseDirectory>
+    <fileSets>
+        <fileSet>
+            <directory>src/main/resources</directory>
+            <includes>
+                <include>plugin.json</include>
+                <include>plugin_job_template.json</include>
+            </includes>
+            <outputDirectory>plugin/reader/influxdbreader</outputDirectory>
+        </fileSet>
+        <fileSet>
+            <directory>target/</directory>
+            <includes>
+                <include>influxdbreader-${project.version}.jar</include>
+            </includes>
+            <outputDirectory>plugin/reader/influxdbreader</outputDirectory>
+        </fileSet>
+    </fileSets>
+
+    <dependencySets>
+        <dependencySet>
+            <useProjectArtifact>false</useProjectArtifact>
+            <outputDirectory>plugin/reader/influxdbreader/libs</outputDirectory>
+            <scope>runtime</scope>
+        </dependencySet>
+    </dependencySets>
+</assembly>
diff --git a/influxdbreader/pom.xml b/influxdbreader/pom.xml
@@ -0,0 +1,64 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>com.alibaba.datax</groupId>
+        <artifactId>datax-all</artifactId>
+        <version>3.1.8-SNAPSHOT</version>
+    </parent>
+    <artifactId>influxdbreader</artifactId>
+    <name>influxdbreader</name>
+    <packaging>jar</packaging>
+
+    <properties>
+        <influxdbClient.version>2.21</influxdbClient.version>
+    </properties>
+    <dependencies>
+        <dependency>
+            <groupId>com.alibaba.datax</groupId>
+            <artifactId>datax-common</artifactId>
+            <version>${project.version}</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>slf4j-log4j12</artifactId>
+                    <groupId>org.slf4j</groupId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
+        <dependency>
+            <groupId>com.alibaba.datax</groupId>
+            <artifactId>plugin-rdbms-util</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.influxdb</groupId>
+            <artifactId>influxdb-java</artifactId>
+            <version>${influxdbClient.version}</version>
+        </dependency>
+
+    </dependencies>
+    <build>
+        <plugins>
+            <plugin>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <configuration>
+                    <descriptors>
+                        <descriptor>package.xml</descriptor>
+                    </descriptors>
+                    <finalName>datax</finalName>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>release</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>