Elastic connector (apache#2546)

### Motivation

Added a sink connector that writes JSON documents into ElasticSearch

### Modifications

Added new pulsar-io module and associated integration tests

### Result

An ElasticSearch sink connector will be available for use.

pulsar-io/elastic-search/pom.xml

@@ -0,0 +1,91 @@
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.pulsar</groupId>
+    <artifactId>pulsar-io</artifactId>
+    <version>2.2.0-incubating-SNAPSHOT</version>
+  </parent>
+  <artifactId>pulsar-io-elastic-search</artifactId>
+  <name>Pulsar IO :: ElasticSearch</name>
+
+  <repositories>
+    <repository>
+        <id>jcenter</id>
+        <url>https://jcenter.bintray.com/</url>
+    </repository>
+  </repositories>
+
+  <dependencies>
+
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>pulsar-io-core</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-databind</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>com.fasterxml.jackson.dataformat</groupId>
+      <artifactId>jackson-dataformat-yaml</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-lang3</artifactId>
+      <version>3.4</version>
+    </dependency>
+
+  	<dependency>
+  		<groupId>org.elasticsearch.client</groupId>
+  		<artifactId>elasticsearch-rest-high-level-client</artifactId>
+  		<version>6.3.2</version>
+  	</dependency>
+
+  	<dependency>
+        <groupId>net.andreinc.mockneat</groupId>
+        <artifactId>mockneat</artifactId>
+        <version>0.2.2</version>
+        <scope>test</scope>
+    </dependency>
+
+  	<dependency>
+  		<groupId>com.google.code.gson</groupId>
+  		<artifactId>gson</artifactId>
+  		<scope>test</scope>
+  	</dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.nifi</groupId>
+        <artifactId>nifi-nar-maven-plugin</artifactId>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>

pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchAbstractSink.java

@@ -0,0 +1,150 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.io.elasticsearch;
+
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Map;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.http.HttpHost;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.pulsar.functions.api.Record;
+import org.apache.pulsar.io.core.KeyValue;
+import org.apache.pulsar.io.core.Sink;
+import org.apache.pulsar.io.core.SinkContext;
+import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
+import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.client.Requests;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestClientBuilder;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
+
+/**
+ * The base abstract class for ElasticSearch sinks.
+ * Users need to implement extractKeyValue function to use this sink.
+ * This class assumes that the input will be JSON documents
+ */
+public abstract class ElasticSearchAbstractSink<K, V> implements Sink<byte[]> {
+
+    protected static final String DOCUMENT = "doc";
+
+    private URL url;
+    private RestHighLevelClient client;
+    private CredentialsProvider credentialsProvider;
+    private ElasticSearchConfig elasticSearchConfig;
+
+    @Override
+    public void open(Map<String, Object> config, SinkContext sinkContext) throws Exception {
+        elasticSearchConfig = ElasticSearchConfig.load(config);
+        elasticSearchConfig.validate();
+        createIndexIfNeeded();
+    }
+
+    @Override
+    public void close() throws Exception {
+        client.close();
+    }
+
+    @Override
+    public void write(Record<byte[]> record) {
+        KeyValue<K, V> keyValue = extractKeyValue(record);
+        IndexRequest indexRequest = Requests.indexRequest(elasticSearchConfig.getIndexName());
+        indexRequest.type(DOCUMENT);
+        indexRequest.source(keyValue.getValue(), XContentType.JSON);
+
+        try {
+        IndexResponse indexResponse = getClient().index(indexRequest);
+            if (indexResponse.getResult().equals(DocWriteResponse.Result.CREATED)) {
+                record.ack();
+            } else {
+                record.fail();
+            }
+        } catch (final IOException ex) {
+            record.fail();
+        }
+    }
+
+    public abstract KeyValue<K, V> extractKeyValue(Record<byte[]> record);
+
+    private void createIndexIfNeeded() throws IOException {
+        GetIndexRequest request = new GetIndexRequest();
+        request.indices(elasticSearchConfig.getIndexName());
+        boolean exists = getClient().indices().exists(request);
+
+        if (!exists) {
+            CreateIndexRequest cireq = new CreateIndexRequest(elasticSearchConfig.getIndexName());
+
+            cireq.settings(Settings.builder()
+               .put("index.number_of_shards", elasticSearchConfig.getIndexNumberOfShards())
+               .put("index.number_of_replicas", elasticSearchConfig.getIndexNumberOfReplicas()));
+
+            CreateIndexResponse ciresp = getClient().indices().create(cireq);
+            if (!ciresp.isAcknowledged() || !ciresp.isShardsAcknowledged()) {
+                throw new RuntimeException("Unable to create index.");
+            }
+        }
+    }
+
+    private URL getUrl() throws MalformedURLException {
+        if (url == null) {
+            url = new URL(elasticSearchConfig.getElasticSearchUrl());
+        }
+        return url;
+    }
+
+    private CredentialsProvider getCredentialsProvider() {
+
+        if (StringUtils.isEmpty(elasticSearchConfig.getUsername())
+            || StringUtils.isEmpty(elasticSearchConfig.getPassword())) {
+            return null;
+        }
+
+        credentialsProvider = new BasicCredentialsProvider();
+        credentialsProvider.setCredentials(AuthScope.ANY,
+                new UsernamePasswordCredentials(elasticSearchConfig.getUsername(),
+                        elasticSearchConfig.getPassword()));
+        return credentialsProvider;
+    }
+
+    private RestHighLevelClient getClient() throws MalformedURLException {
+        if (client == null) {
+          CredentialsProvider cp = getCredentialsProvider();
+          RestClientBuilder builder = RestClient.builder(new HttpHost(getUrl().getHost(),
+                  getUrl().getPort(), getUrl().getProtocol()));
+
+          if (cp != null) {
+              builder.setHttpClientConfigCallback(httpClientBuilder ->
+              httpClientBuilder.setDefaultCredentialsProvider(cp));
+          }
+          client = new RestHighLevelClient(builder);
+        }
+        return client;
+    }
+}

pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchConfig.java

@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.io.elasticsearch;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import java.io.File;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.Map;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.ToString;
+import lombok.experimental.Accessors;
+import org.apache.commons.lang3.StringUtils;
+
+/**
+ * Configuration class for the ElasticSearch Sink Connector.
+ */
+@Data
+@Setter
+@Getter
+@EqualsAndHashCode
+@ToString
+@Accessors(chain = true)
+public class ElasticSearchConfig implements Serializable {
+
+    private static final long serialVersionUID = 1L;
+
+    private String elasticSearchUrl;
+
+    private String indexName;
+
+    private int indexNumberOfShards = 1;
+
+    private int indexNumberOfReplicas = 1;
+
+    private String username;
+
+    private String password;
+
+    public static ElasticSearchConfig load(String yamlFile) throws IOException {
+        ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
+        return mapper.readValue(new File(yamlFile), ElasticSearchConfig.class);
+    }
+
+    public static ElasticSearchConfig load(Map<String, Object> map) throws IOException {
+        ObjectMapper mapper = new ObjectMapper();
+        return mapper.readValue(new ObjectMapper().writeValueAsString(map), ElasticSearchConfig.class);
+    }
+
+    public void validate() {
+        if (StringUtils.isEmpty(elasticSearchUrl) || StringUtils.isEmpty(indexName)) {
+            throw new IllegalArgumentException("Required property not set.");
+        }
+
+        if ((StringUtils.isNotEmpty(username) && StringUtils.isEmpty(password))
+           || (StringUtils.isEmpty(username) && StringUtils.isNotEmpty(password))) {
+            throw new IllegalArgumentException("Values for both Username & password are required.");
+        }
+
+        if (indexNumberOfShards < 1) {
+            throw new IllegalArgumentException("indexNumberOfShards must be a positive integer");
+        }
+
+        if (indexNumberOfReplicas < 1) {
+            throw new IllegalArgumentException("indexNumberOfReplicas must be a positive integer");
+        }
+    }
+}

pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchStringSink.java

@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.io.elasticsearch;
+
+import org.apache.pulsar.functions.api.Record;
+import org.apache.pulsar.io.core.KeyValue;
+
+/**
+ * Concrete ElasticSearch sink.
+ * This class assumes that the input will be JSON documents
+ */
+public class ElasticSearchStringSink extends ElasticSearchAbstractSink<String, String> {
+
+    @Override
+    public KeyValue<String, String> extractKeyValue(Record<byte[]> record) {
+        String key = record.getKey().orElseGet(() -> new String(record.getValue()));
+        return new KeyValue<>(key, new String(record.getValue()));
+    }
+}

pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/package-info.java

@@ -0,0 +1,19 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.io.elasticsearch;